313 lines
13 KiB
JavaScript
313 lines
13 KiB
JavaScript
import { createHash } from 'node:crypto';
|
|
import { execFileSync } from 'node:child_process';
|
|
import { mkdir, readFile, writeFile, readdir, open } from 'node:fs/promises';
|
|
import { fileURLToPath } from 'node:url';
|
|
import path from 'node:path';
|
|
import { parseArgs } from 'node:util';
|
|
import { DESIGN_VERSION, BUILDS, buildPlan, makePayload, observe, summarize } from './design.mjs';
|
|
import { FOLLOWUP_SUITES, buildFollowupPlan } from './followup.mjs';
|
|
import { economicTools, nationPlans, lifecycleProfiles, strategyPlans } from './economy.mjs';
|
|
|
|
const root = fileURLToPath(new URL('../../', import.meta.url));
|
|
const { values } = parseArgs({
|
|
options: {
|
|
suite: { type: 'string', default: 'units' },
|
|
samples: { type: 'string', default: '64' },
|
|
seed: { type: 'string', default: 'balance-2026-09-v1' },
|
|
out: { type: 'string' },
|
|
match: { type: 'string' },
|
|
plan: { type: 'boolean', default: false },
|
|
},
|
|
});
|
|
const samples = Number(values.samples);
|
|
if (!Number.isSafeInteger(samples) || samples < 2 || samples > 10000) throw new Error('--samples must be 2..10000');
|
|
if (
|
|
!['units', 'items', 'traits', 'nations', 'families', 'tiers', 'interactions', ...FOLLOWUP_SUITES].includes(
|
|
values.suite
|
|
)
|
|
)
|
|
throw new Error('Unknown suite');
|
|
const out = path.resolve(root, values.out ?? `test-results/balance-lab/${values.suite}`);
|
|
// Build before importing; a previous branch's dist must never become measurement evidence.
|
|
for (const pkg of ['common', 'logic']) {
|
|
execFileSync('pnpm', ['--filter', `@sammo-ts/${pkg}`, 'build'], { cwd: root, stdio: ['ignore', 'ignore', 'pipe'] });
|
|
}
|
|
const logic = await import('../../packages/logic/dist/index.js');
|
|
const common = await import('../../packages/common/dist/index.js');
|
|
const unitSet = logic.parseUnitSetDefinition(
|
|
JSON.parse(await readFile(path.join(root, 'resources/unitset/unitset_che.json'), 'utf8'))
|
|
);
|
|
const allItems = await logic.loadItemModules([...logic.ITEM_KEYS]);
|
|
const war = await logic.loadWarTraitModules([...logic.WAR_TRAIT_KEYS]);
|
|
const nations = await logic.loadNationTraitModules([...logic.NATION_TRAIT_KEYS]);
|
|
const { CommandResolver } = await import('../../packages/logic/dist/src/actions/turn/general/che_징병.js');
|
|
const { CommandResolver: VolunteerResolver } =
|
|
await import('../../packages/logic/dist/src/actions/turn/nation/che_의병모집.js');
|
|
const economics = economicTools(logic, CommandResolver, allItems, war, nations);
|
|
const spec = (entry) => ({
|
|
key: entry.key,
|
|
name: entry.name,
|
|
info: entry.info,
|
|
slot: entry.slot,
|
|
unique: entry.unique,
|
|
cost: entry.cost,
|
|
selection: entry.selection,
|
|
hooks: Object.keys(entry).filter(
|
|
(key) => typeof entry[key] === 'function' && !['getName', 'getInfo'].includes(key)
|
|
),
|
|
events: Object.keys(entry.eventHandlers ?? {}),
|
|
});
|
|
const catalog = {
|
|
scope: 'CHE unitset + all registered unique modules; scenario acquisition eligibility is NOT asserted',
|
|
units: unitSet.crewTypes.filter((unit) => unit.armType > 0),
|
|
items: allItems.filter((item) => item.unique).map(spec),
|
|
war: war.map(spec),
|
|
nations: nations.map(spec),
|
|
};
|
|
let cells = FOLLOWUP_SUITES.includes(values.suite)
|
|
? buildFollowupPlan(catalog, values.suite)
|
|
: buildPlan(catalog, values.suite);
|
|
if (values.match)
|
|
cells = cells.filter((cell) =>
|
|
`${cell.key}/${cell.build}/${cell.mode}/${cell.role}/${cell.budget ?? ''}`.includes(values.match)
|
|
);
|
|
if (!cells.length) throw new Error('No experiment cells selected');
|
|
async function hashTree(directories) {
|
|
const hash = createHash('sha256');
|
|
const walk = async (relative) => {
|
|
for (const entry of (await readdir(path.join(root, relative), { withFileTypes: true })).sort((a, b) =>
|
|
a.name.localeCompare(b.name)
|
|
)) {
|
|
const child = path.join(relative, entry.name);
|
|
if (entry.isDirectory()) await walk(child);
|
|
else if (entry.isFile()) hash.update(child).update(await readFile(path.join(root, child)));
|
|
}
|
|
};
|
|
for (const dir of directories) await walk(dir);
|
|
return hash.digest('hex');
|
|
}
|
|
const sourceHash = await hashTree([
|
|
'packages/common/src',
|
|
'packages/logic/src',
|
|
'resources/unitset',
|
|
'tools/balance-lab',
|
|
]);
|
|
const manifest = {
|
|
design: DESIGN_VERSION,
|
|
suite: values.suite,
|
|
seed: values.seed,
|
|
samples,
|
|
commit: execFileSync('git', ['rev-parse', 'HEAD'], { cwd: root, encoding: 'utf8' }).trim(),
|
|
dirty: execFileSync('git', ['status', '--porcelain'], { cwd: root, encoding: 'utf8' }).trim(),
|
|
sourceHash,
|
|
node: process.version,
|
|
builds: BUILDS,
|
|
cells: cells.length,
|
|
battles: cells.reduce((sum, cell) => sum + samples * (cell.factorialPatches ? 4 : cell.paired ? 2 : 1), 0),
|
|
protocol: {
|
|
seedPairing: 'same seed within cell and treatment/control; RNG consumption may diverge',
|
|
inference:
|
|
'per-cell descriptive normal CI; correlated cells and multiple tests prevent global significance claims',
|
|
economics:
|
|
'equal-gold uses base crew cost; recruited uses production CommandResolver costs/population/train/capacity; neither is lifetime cost',
|
|
outcome: 'fraction enemy troops lost minus fraction own troops lost; siege uses wall damage separately',
|
|
environment:
|
|
'synthetic CHE defaults tech 3000/year 200; cell overrides are recorded in plan; no production data or mutation',
|
|
},
|
|
completed: false,
|
|
};
|
|
await mkdir(out, { recursive: true });
|
|
// Never silently truncate a previous run (including a partially completed run).
|
|
const claim = await open(path.join(out, 'manifest.json'), 'wx');
|
|
await claim.writeFile(JSON.stringify(manifest, null, 2));
|
|
await claim.close();
|
|
await writeFile(path.join(out, 'catalog.json'), JSON.stringify(catalog, null, 2));
|
|
await writeFile(path.join(out, 'plan.json'), JSON.stringify(cells, null, 2));
|
|
if (values.plan) {
|
|
console.log(JSON.stringify({ out, ...manifest }));
|
|
process.exit(0);
|
|
}
|
|
|
|
// Nation coefficients are observed through the actual module hooks. Fail on newly context-dependent
|
|
// hooks rather than silently inventing a partial domain context for them.
|
|
const noContext = new Proxy(
|
|
{},
|
|
{
|
|
get: (_, key) => {
|
|
throw new Error(`Nation probe needs context.${String(key)}`);
|
|
},
|
|
}
|
|
);
|
|
const nationProfiles = nations.map((module) => {
|
|
const domestic = {};
|
|
for (const action of ['농업', '상업', '기술', '치안', '민심', '인구', '수비', '성벽', '계략', '징병', '모병']) {
|
|
domestic[action] = Object.fromEntries(
|
|
['score', 'cost', 'success'].map((kind) => {
|
|
const base = kind === 'success' ? 0.5 : 100;
|
|
return [kind, module.onCalcDomestic?.(noContext, action, kind, base) ?? base];
|
|
})
|
|
);
|
|
}
|
|
return {
|
|
key: module.key,
|
|
domestic,
|
|
income: Object.fromEntries(
|
|
['gold', 'rice', 'pop'].map((kind) => [kind, module.onCalcNationalIncome?.(noContext, kind, 100) ?? 100])
|
|
),
|
|
populationLoss: module.onCalcNationalIncome?.(noContext, 'pop', -100) ?? -100,
|
|
strategy: Object.fromEntries(
|
|
['의병모집', '급습', '수몰'].map((action) => [
|
|
action,
|
|
Object.fromEntries(
|
|
['delay', 'globalDelay'].map((kind) => [
|
|
kind,
|
|
[12, 24, 36].map((base) => ({
|
|
base,
|
|
actual: module.onCalcStrategic?.(noContext, action, kind, base) ?? base,
|
|
})),
|
|
])
|
|
),
|
|
])
|
|
),
|
|
};
|
|
});
|
|
await writeFile(path.join(out, 'nation-profiles.json'), JSON.stringify(nationProfiles, null, 2));
|
|
await writeFile(path.join(out, 'nation-plans.json'), JSON.stringify(nationPlans(nationProfiles), null, 2));
|
|
const probePayload = makePayload(
|
|
unitSet,
|
|
{ unit: 1100, opponent: 1100, build: 'martial', role: 'attack', mode: 'field' },
|
|
'economic-probe'
|
|
);
|
|
const economicProfiles = [...allItems.filter((item) => item.unique), ...war].map((module) =>
|
|
economics.profile(probePayload.attackerGeneral, probePayload.attackerNation, probePayload.time, module)
|
|
);
|
|
await writeFile(path.join(out, 'economic-profiles.json'), JSON.stringify(economicProfiles, null, 2));
|
|
await writeFile(
|
|
path.join(out, 'lifecycle-profiles.json'),
|
|
JSON.stringify(
|
|
lifecycleProfiles(
|
|
logic,
|
|
common,
|
|
probePayload,
|
|
[...allItems.filter((item) => item.unique), ...war],
|
|
values.seed
|
|
),
|
|
null,
|
|
2
|
|
)
|
|
);
|
|
await writeFile(
|
|
path.join(out, 'strategy-plans.json'),
|
|
JSON.stringify(strategyPlans(VolunteerResolver, probePayload, nations, allItems), null, 2)
|
|
);
|
|
|
|
const raw = await open(path.join(out, 'samples.jsonl'), 'wx');
|
|
const summary = [];
|
|
const started = Date.now();
|
|
try {
|
|
for (const cell of cells) {
|
|
const rows = [];
|
|
for (let index = 0; index < samples; index++) {
|
|
// Omit candidate key and role: common random numbers across candidate and role strata.
|
|
const seed = JSON.stringify([
|
|
values.seed,
|
|
cell.build,
|
|
cell.opponent,
|
|
cell.mode,
|
|
cell.budget ?? 'equal-crew',
|
|
cell.condition ?? 'ready',
|
|
index,
|
|
]);
|
|
const run = (treatment, patch) => {
|
|
const payload = makePayload(unitSet, patch ? { ...cell, generalPatch: patch } : cell, seed, treatment);
|
|
const recruited = cell.budget === 'recruited' ? economics.recruit(payload, cell) : null;
|
|
let result;
|
|
logic.processBattleSimJob(payload, {
|
|
onBattleResolved: (battle) => {
|
|
result = observe(battle, payload, cell.role);
|
|
},
|
|
});
|
|
if (!result) throw new Error('Battle did not execute');
|
|
if (recruited) for (const [key, value] of Object.entries(recruited)) result[`recruit_${key}`] = value;
|
|
return result;
|
|
};
|
|
const treatment = run(true);
|
|
const control = cell.paired ? run(false) : null;
|
|
const delta = control
|
|
? Object.fromEntries(
|
|
Object.entries(treatment).map(([key, value]) => [
|
|
key,
|
|
value === null ? null : value - control[key],
|
|
])
|
|
)
|
|
: null;
|
|
const singles = cell.factorialPatches?.map((patch) => run(true, patch));
|
|
const synergy = singles
|
|
? Object.fromEntries(
|
|
Object.entries(treatment).map(([key, value]) => [
|
|
key,
|
|
value === null ? null : value - singles[0][key] - singles[1][key] + control[key],
|
|
])
|
|
)
|
|
: null;
|
|
const row = { cell: cell.id, index, seed, treatment, control, delta, singles, synergy };
|
|
rows.push(row);
|
|
await raw.write(JSON.stringify(row) + '\n');
|
|
}
|
|
const aggregate = (field) =>
|
|
Object.fromEntries(
|
|
Object.keys(rows[0][field]).map((metric) => {
|
|
const values = rows.map((row) => row[field][metric]);
|
|
return [metric, values[0] === null ? null : summarize(values)];
|
|
})
|
|
);
|
|
summary.push({
|
|
...cell,
|
|
treatment: aggregate('treatment'),
|
|
control: cell.paired ? aggregate('control') : null,
|
|
delta: cell.paired ? aggregate('delta') : null,
|
|
synergy: cell.factorialPatches ? aggregate('synergy') : null,
|
|
});
|
|
if (summary.length % 25 === 0)
|
|
console.error(
|
|
`${values.suite}: ${summary.length}/${cells.length} cells, ${Math.round((Date.now() - started) / 1000)}s`
|
|
);
|
|
}
|
|
} finally {
|
|
await raw.close();
|
|
}
|
|
const finalHash = await hashTree([
|
|
'packages/common/src',
|
|
'packages/logic/src',
|
|
'resources/unitset',
|
|
'tools/balance-lab',
|
|
]);
|
|
if (finalHash !== sourceHash) throw new Error('Source changed during run; result is not complete');
|
|
await writeFile(path.join(out, 'summary.json'), JSON.stringify(summary, null, 2));
|
|
const coverage = [...catalog.items, ...catalog.war].map((entry) => {
|
|
const evaluated = summary.filter((row) => row.key === entry.key);
|
|
const changed = evaluated.some(
|
|
(row) => row.delta && Object.values(row.delta).some((metric) => metric && Math.abs(metric.mean) > 1e-10)
|
|
);
|
|
return {
|
|
key: entry.key,
|
|
cells: evaluated.length,
|
|
hooks: entry.hooks,
|
|
events: entry.events,
|
|
battleObservation: !evaluated.length
|
|
? 'not-scheduled'
|
|
: changed
|
|
? 'difference-observed'
|
|
: 'no-difference-in-tested-conditions',
|
|
economicProfile: true,
|
|
limitation: 'No observed difference is not a balance pass or proof of worthlessness',
|
|
};
|
|
});
|
|
await writeFile(path.join(out, 'coverage.json'), JSON.stringify(coverage, null, 2));
|
|
manifest.completed = true;
|
|
manifest.elapsedSeconds = (Date.now() - started) / 1000;
|
|
await writeFile(path.join(out, 'manifest.json'), JSON.stringify(manifest, null, 2));
|
|
console.log(
|
|
JSON.stringify({ out, cells: cells.length, battles: manifest.battles, elapsedSeconds: manifest.elapsedSeconds })
|
|
);
|