import { createHash } from 'node:crypto'; import { execFileSync } from 'node:child_process'; import { mkdir, readFile, writeFile, readdir, open } from 'node:fs/promises'; import { fileURLToPath } from 'node:url'; import path from 'node:path'; import { parseArgs } from 'node:util'; import { DESIGN_VERSION, BUILDS, buildPlan, makePayload, observe, summarize } from './design.mjs'; import { FOLLOWUP_SUITES, buildFollowupPlan } from './followup.mjs'; import { economicTools, nationPlans, lifecycleProfiles, strategyPlans } from './economy.mjs'; const root = fileURLToPath(new URL('../../', import.meta.url)); const { values } = parseArgs({ options: { suite: { type: 'string', default: 'units' }, samples: { type: 'string', default: '64' }, seed: { type: 'string', default: 'balance-2026-09-v1' }, out: { type: 'string' }, match: { type: 'string' }, plan: { type: 'boolean', default: false }, }, }); const samples = Number(values.samples); if (!Number.isSafeInteger(samples) || samples < 2 || samples > 10000) throw new Error('--samples must be 2..10000'); if ( !['units', 'items', 'traits', 'nations', 'families', 'tiers', 'interactions', ...FOLLOWUP_SUITES].includes( values.suite ) ) throw new Error('Unknown suite'); const out = path.resolve(root, values.out ?? `test-results/balance-lab/${values.suite}`); // Build before importing; a previous branch's dist must never become measurement evidence. for (const pkg of ['common', 'logic']) { execFileSync('pnpm', ['--filter', `@sammo-ts/${pkg}`, 'build'], { cwd: root, stdio: ['ignore', 'ignore', 'pipe'] }); } const logic = await import('../../packages/logic/dist/index.js'); const common = await import('../../packages/common/dist/index.js'); const unitSet = logic.parseUnitSetDefinition( JSON.parse(await readFile(path.join(root, 'resources/unitset/unitset_che.json'), 'utf8')) ); const allItems = await logic.loadItemModules([...logic.ITEM_KEYS]); const war = await logic.loadWarTraitModules([...logic.WAR_TRAIT_KEYS]); const nations = await logic.loadNationTraitModules([...logic.NATION_TRAIT_KEYS]); const { CommandResolver } = await import('../../packages/logic/dist/src/actions/turn/general/che_징병.js'); const { CommandResolver: VolunteerResolver } = await import('../../packages/logic/dist/src/actions/turn/nation/che_의병모집.js'); const economics = economicTools(logic, CommandResolver, allItems, war, nations); const spec = (entry) => ({ key: entry.key, name: entry.name, info: entry.info, slot: entry.slot, unique: entry.unique, cost: entry.cost, selection: entry.selection, hooks: Object.keys(entry).filter( (key) => typeof entry[key] === 'function' && !['getName', 'getInfo'].includes(key) ), events: Object.keys(entry.eventHandlers ?? {}), }); const catalog = { scope: 'CHE unitset + all registered unique modules; scenario acquisition eligibility is NOT asserted', units: unitSet.crewTypes.filter((unit) => unit.armType > 0), items: allItems.filter((item) => item.unique).map(spec), war: war.map(spec), nations: nations.map(spec), }; let cells = FOLLOWUP_SUITES.includes(values.suite) ? buildFollowupPlan(catalog, values.suite) : buildPlan(catalog, values.suite); if (values.match) cells = cells.filter((cell) => `${cell.key}/${cell.build}/${cell.mode}/${cell.role}/${cell.budget ?? ''}`.includes(values.match) ); if (!cells.length) throw new Error('No experiment cells selected'); async function hashTree(directories) { const hash = createHash('sha256'); const walk = async (relative) => { for (const entry of (await readdir(path.join(root, relative), { withFileTypes: true })).sort((a, b) => a.name.localeCompare(b.name) )) { const child = path.join(relative, entry.name); if (entry.isDirectory()) await walk(child); else if (entry.isFile()) hash.update(child).update(await readFile(path.join(root, child))); } }; for (const dir of directories) await walk(dir); return hash.digest('hex'); } const sourceHash = await hashTree([ 'packages/common/src', 'packages/logic/src', 'resources/unitset', 'tools/balance-lab', ]); const manifest = { design: DESIGN_VERSION, suite: values.suite, seed: values.seed, samples, commit: execFileSync('git', ['rev-parse', 'HEAD'], { cwd: root, encoding: 'utf8' }).trim(), dirty: execFileSync('git', ['status', '--porcelain'], { cwd: root, encoding: 'utf8' }).trim(), sourceHash, node: process.version, builds: BUILDS, cells: cells.length, battles: cells.reduce((sum, cell) => sum + samples * (cell.factorialPatches ? 4 : cell.paired ? 2 : 1), 0), protocol: { seedPairing: 'same seed within cell and treatment/control; RNG consumption may diverge', inference: 'per-cell descriptive normal CI; correlated cells and multiple tests prevent global significance claims', economics: 'equal-gold uses base crew cost; recruited uses production CommandResolver costs/population/train/capacity; neither is lifetime cost', outcome: 'fraction enemy troops lost minus fraction own troops lost; siege uses wall damage separately', environment: 'synthetic CHE defaults tech 3000/year 200; cell overrides are recorded in plan; no production data or mutation', }, completed: false, }; await mkdir(out, { recursive: true }); // Never silently truncate a previous run (including a partially completed run). const claim = await open(path.join(out, 'manifest.json'), 'wx'); await claim.writeFile(JSON.stringify(manifest, null, 2)); await claim.close(); await writeFile(path.join(out, 'catalog.json'), JSON.stringify(catalog, null, 2)); await writeFile(path.join(out, 'plan.json'), JSON.stringify(cells, null, 2)); if (values.plan) { console.log(JSON.stringify({ out, ...manifest })); process.exit(0); } // Nation coefficients are observed through the actual module hooks. Fail on newly context-dependent // hooks rather than silently inventing a partial domain context for them. const noContext = new Proxy( {}, { get: (_, key) => { throw new Error(`Nation probe needs context.${String(key)}`); }, } ); const nationProfiles = nations.map((module) => { const domestic = {}; for (const action of ['농업', '상업', '기술', '치안', '민심', '인구', '수비', '성벽', '계략', '징병', '모병']) { domestic[action] = Object.fromEntries( ['score', 'cost', 'success'].map((kind) => { const base = kind === 'success' ? 0.5 : 100; return [kind, module.onCalcDomestic?.(noContext, action, kind, base) ?? base]; }) ); } return { key: module.key, domestic, income: Object.fromEntries( ['gold', 'rice', 'pop'].map((kind) => [kind, module.onCalcNationalIncome?.(noContext, kind, 100) ?? 100]) ), populationLoss: module.onCalcNationalIncome?.(noContext, 'pop', -100) ?? -100, strategy: Object.fromEntries( ['의병모집', '급습', '수몰'].map((action) => [ action, Object.fromEntries( ['delay', 'globalDelay'].map((kind) => [ kind, [12, 24, 36].map((base) => ({ base, actual: module.onCalcStrategic?.(noContext, action, kind, base) ?? base, })), ]) ), ]) ), }; }); await writeFile(path.join(out, 'nation-profiles.json'), JSON.stringify(nationProfiles, null, 2)); await writeFile(path.join(out, 'nation-plans.json'), JSON.stringify(nationPlans(nationProfiles), null, 2)); const probePayload = makePayload( unitSet, { unit: 1100, opponent: 1100, build: 'martial', role: 'attack', mode: 'field' }, 'economic-probe' ); const economicProfiles = [...allItems.filter((item) => item.unique), ...war].map((module) => economics.profile(probePayload.attackerGeneral, probePayload.attackerNation, probePayload.time, module) ); await writeFile(path.join(out, 'economic-profiles.json'), JSON.stringify(economicProfiles, null, 2)); await writeFile( path.join(out, 'lifecycle-profiles.json'), JSON.stringify( lifecycleProfiles( logic, common, probePayload, [...allItems.filter((item) => item.unique), ...war], values.seed ), null, 2 ) ); await writeFile( path.join(out, 'strategy-plans.json'), JSON.stringify(strategyPlans(VolunteerResolver, probePayload, nations, allItems), null, 2) ); const raw = await open(path.join(out, 'samples.jsonl'), 'wx'); const summary = []; const started = Date.now(); try { for (const cell of cells) { const rows = []; for (let index = 0; index < samples; index++) { // Omit candidate key and role: common random numbers across candidate and role strata. const seed = JSON.stringify([ values.seed, cell.build, cell.opponent, cell.mode, cell.budget ?? 'equal-crew', cell.condition ?? 'ready', index, ]); const run = (treatment, patch) => { const payload = makePayload(unitSet, patch ? { ...cell, generalPatch: patch } : cell, seed, treatment); const recruited = cell.budget === 'recruited' ? economics.recruit(payload, cell) : null; let result; logic.processBattleSimJob(payload, { onBattleResolved: (battle) => { result = observe(battle, payload, cell.role); }, }); if (!result) throw new Error('Battle did not execute'); if (recruited) for (const [key, value] of Object.entries(recruited)) result[`recruit_${key}`] = value; return result; }; const treatment = run(true); const control = cell.paired ? run(false) : null; const delta = control ? Object.fromEntries( Object.entries(treatment).map(([key, value]) => [ key, value === null ? null : value - control[key], ]) ) : null; const singles = cell.factorialPatches?.map((patch) => run(true, patch)); const synergy = singles ? Object.fromEntries( Object.entries(treatment).map(([key, value]) => [ key, value === null ? null : value - singles[0][key] - singles[1][key] + control[key], ]) ) : null; const row = { cell: cell.id, index, seed, treatment, control, delta, singles, synergy }; rows.push(row); await raw.write(JSON.stringify(row) + '\n'); } const aggregate = (field) => Object.fromEntries( Object.keys(rows[0][field]).map((metric) => { const values = rows.map((row) => row[field][metric]); return [metric, values[0] === null ? null : summarize(values)]; }) ); summary.push({ ...cell, treatment: aggregate('treatment'), control: cell.paired ? aggregate('control') : null, delta: cell.paired ? aggregate('delta') : null, synergy: cell.factorialPatches ? aggregate('synergy') : null, }); if (summary.length % 25 === 0) console.error( `${values.suite}: ${summary.length}/${cells.length} cells, ${Math.round((Date.now() - started) / 1000)}s` ); } } finally { await raw.close(); } const finalHash = await hashTree([ 'packages/common/src', 'packages/logic/src', 'resources/unitset', 'tools/balance-lab', ]); if (finalHash !== sourceHash) throw new Error('Source changed during run; result is not complete'); await writeFile(path.join(out, 'summary.json'), JSON.stringify(summary, null, 2)); const coverage = [...catalog.items, ...catalog.war].map((entry) => { const evaluated = summary.filter((row) => row.key === entry.key); const changed = evaluated.some( (row) => row.delta && Object.values(row.delta).some((metric) => metric && Math.abs(metric.mean) > 1e-10) ); return { key: entry.key, cells: evaluated.length, hooks: entry.hooks, events: entry.events, battleObservation: !evaluated.length ? 'not-scheduled' : changed ? 'difference-observed' : 'no-difference-in-tested-conditions', economicProfile: true, limitation: 'No observed difference is not a balance pass or proof of worthlessness', }; }); await writeFile(path.join(out, 'coverage.json'), JSON.stringify(coverage, null, 2)); manifest.completed = true; manifest.elapsedSeconds = (Date.now() - started) / 1000; await writeFile(path.join(out, 'manifest.json'), JSON.stringify(manifest, null, 2)); console.log( JSON.stringify({ out, cells: cells.length, battles: manifest.battles, elapsedSeconds: manifest.elapsedSeconds }) );