From 4b00c1052d40cf581e6d137df19a00416bec75bc Mon Sep 17 00:00:00 2001 From: hided62 Date: Tue, 29 Sep 2026 10:55:43 +0000 Subject: [PATCH] =?UTF-8?q?=EB=82=B4=EB=B6=80=20=EB=B0=B8=EB=9F=B0?= =?UTF-8?q?=EC=8A=A4=20=EB=B9=84=EA=B5=90=20CLI=EC=99=80=20=EC=A1=B0?= =?UTF-8?q?=EA=B1=B4=EB=B3=84=20=EB=8C=80=EC=A1=B0=20=EC=8B=A4=ED=97=98?= =?UTF-8?q?=EC=9D=84=20=EC=B6=94=EA=B0=80=ED=95=9C=EB=8B=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- package.json | 3 + tools/balance-lab/README.md | 107 +++++++++ tools/balance-lab/analyze.mjs | 114 ++++++++++ tools/balance-lab/design.mjs | 366 ++++++++++++++++++++++++++++++ tools/balance-lab/design.test.mjs | 108 +++++++++ tools/balance-lab/engine.test.mjs | 45 ++++ tools/balance-lab/run.mjs | 247 ++++++++++++++++++++ 7 files changed, 990 insertions(+) create mode 100644 tools/balance-lab/README.md create mode 100644 tools/balance-lab/analyze.mjs create mode 100644 tools/balance-lab/design.mjs create mode 100644 tools/balance-lab/design.test.mjs create mode 100644 tools/balance-lab/engine.test.mjs create mode 100644 tools/balance-lab/run.mjs diff --git a/package.json b/package.json index b51d9562..2187fdcb 100644 --- a/package.json +++ b/package.json @@ -35,6 +35,9 @@ "check:legacy:scenario": "SAMMO_REQUIRE_REF_SOURCE=1 pnpm --filter @sammo-ts/game-engine test monthlyCatalogCoverage.test.ts scenarioLoader.test.ts scenarioComposition.test.ts", "check:architecture": "node tools/check-package-boundaries.mjs && node tools/check-game-clock-participants.mjs", "check:typescript-toolchain": "node tools/check-typescript-toolchain.mjs", + "balance:run": "node tools/balance-lab/run.mjs", + "balance:analyze": "node tools/balance-lab/analyze.mjs", + "test:balance": "pnpm --filter @sammo-ts/common build && pnpm --filter @sammo-ts/logic build && node --test tools/balance-lab/*.test.mjs", "test:architecture": "node --test tools/check-package-boundaries.test.mjs", "test:image-sync": "node --test tools/sync-image-repository.test.mjs", "test:e2e:frontend-legacy": "playwright test --config tools/frontend-legacy-parity/playwright.config.mjs --tsconfig tools/frontend-legacy-parity/tsconfig.json", diff --git a/tools/balance-lab/README.md b/tools/balance-lab/README.md new file mode 100644 index 00000000..88828d8b --- /dev/null +++ b/tools/balance-lab/README.md @@ -0,0 +1,107 @@ +# 내부 밸런스 실험실 + +공개 API/화면과 연결하지 않는 Node CLI다. 실제 `processBattleSimJob` → +`resolveWarBattle`의 관찰 callback으로 측정하며 DB/Redis/운영 seed를 사용하지 않는다. +게임 수치나 알고리즘은 바꾸지 않는다. Ref 동일성 test를 대체하지 않는다. + +```sh +fnm use 24 +node --test tools/balance-lab/*.test.mjs +node tools/balance-lab/run.mjs --suite units --samples 64 --out test-results/balance-lab/units-64 +node tools/balance-lab/run.mjs --suite items --samples 32 --out test-results/balance-lab/items-32 +node tools/balance-lab/run.mjs --suite traits --samples 64 --out test-results/balance-lab/traits-64 +node tools/balance-lab/run.mjs --suite nations --samples 32 --out test-results/balance-lab/nations-32 +``` + +`pnpm` 실행 파일이 PATH에 있어야 한다. 최초 환경에서 필요하면 `corepack enable`. +CLI는 common/logic을 먼저 build하여 오래된 dist를 측정하지 않는다. 출력 경로는 +매 실행 새 경로를 지정한다. 기존 manifest가 있으면 중단하여 이전 증거를 보존한다. +`--plan`은 build/catalog/설계만 출력한다. `--match 1100/martial`처럼 key/build/mode/role/budget +부분 문자열로 후속 정밀 실험을 좁힐 수 있다. `--seed`는 실험용 문자열이다. +표본수는 cell별 독립 seed 수이며 공수와 treatment/control은 같은 seed를 사용한다. + +## 비교 계약 + +- 병종군: 기본 5종과 모든 병종의 상대 행렬. 보→궁→기→보의 유리 방향을 + 공격/수비 각각 확인한다. 같은 seed의 양 역할은 독립 표본 두 개로 세지 않는다. +- 병종: CHE 33종, 통무/통지/통솔/무지 4개 총합 210 배분, 기본 5종 상대, + 공수 역할과 성벽 단독 전투. 기술 요구량과 지역 제한은 catalog에 보존한다. +- 유니크: 등록된 모든 unique 모듈을 슬롯에 장착하고 같은 슬롯이 빈 대조군과 + 짝지어 차이를 낸다. 모든 등록 모듈의 기계적 효과 조사이며 특정 시나리오의 + 획득 가능성이나 희소성·경매가격을 뜻하지 않는다. +- 전투특기: 20종을 무특기와 비교한다. 기본 보궁기/귀병/충차에서 각각 관측한다. +- 국가성향: 중립 대조 전투 외에 실제 모듈의 내정 score/cost/success, 세입, + 인구 증가·손실과 전략 delay/globalDelay를 probe한다. 전투에서 차이가 없다고 + 성향이 없다고 판정하지 않는다. + +`equal-crew`는 7천 명 **상한**이며 원 통솔×100을 넘지 않는다. `equal-gold`는 +중립 징병의 병종 기본 금 계수로 비용을 맞추되 같은 통솔 상한을 적용한다. +실제 징병에 필요한 군량·민심·주민·명령 턴, 장비/특기의 징병 할인은 이 정규화에 +포함되지 않는다. `unit.rice`는 전투 군량 계수이며 징병 군량 단가로 합산하지 않는다. +원 능력치 총합은 같아도 역할별로 불리한 배분이 존재한다. 배분을 합친 한 줄 순위를 +제품 밸런스 결론으로 쓰지 않는다. + +야전 주지표 `exchange`는 `상대 병력 손실률 - 자기 병력 손실률`이다. 성벽 피해는 +포함하지 않고 공성에서는 `null`이다. 생존·전멸·성벽 피해·군량·부상·phase를 같이 +보며 phase 종료를 임의 승리로 바꾸지 않는다. 무피해 승리의 무한 교환비도 만들지 않는다. + +`samples.jsonl`은 cell/index/seed, treatment/control/차이를 보존한다. `summary.json`은 +cell별 평균, 표준편차, 정규근사 95% 구간과 10/50/90 분위수를 제공한다. 낮은 n, +확률 0/1 근처와 희귀 발동에서는 정규구간이 정확하지 않다. 수천 cell의 구간을 +다중검정 보정 없는 확정 유의성으로 쓰지 않는다. 같은 seed여도 효과가 RNG 소비 +경로를 바꿀 수 있으며 개별 전투의 RNG tape가 동일하다는 뜻은 아니다. + +## 판정 절차 + +1. 모든 후보를 저표본 탐색하고 결과를 보기 전에 실용 차이 기준을 정한다. + 초기 검토 기준은 야전 손실률 차이 0.10, 동일 역할 공성 피해 +25%다. + 이는 밸런스 허용치 확정이 아니라 추가 조사 후보를 정하는 기준이다. +2. 우위 후보를 별도 seed 256회 이상으로 재검사한다. 상성 역전 여부, 공수/배분/ + 병력 비용 조건에서의 일관성, 하위 10% 성과와 불리한 상대를 확인한다. +3. 3등급끼리 직접 교전, 동급 내 Pareto 지배, 지역/획득 제약, 특기×유니크 + 상호작용, 소모·연속 교전, 초중후반 기술/훈련/숙련 변화로 검증한다. +4. 게시판·과거 로그는 가설과 상황 분포의 근거로 쓴다. 버전·스탯·장비·턴 앞뒤 + 상태가 없는 로그만으로 원인 효과나 메타 점유율을 계산하지 않는다. +5. 국가성향은 수입·내정 턴·전략 빈도·군사 역할을 다목적으로 비교한다. + 나라 크기/유저 활성도/외교의 효과를 성향 효과로 오인하지 않는다. + +## 증거와 현재 제한 + +manifest는 Git HEAD/dirty 상태, common/logic source·병종 resource·실험 도구 SHA-256, +Node, 설계 버전, seed, cell/전투 수와 완료 여부를 기록한다. 실행 도중 source 변경이 +있으면 완료하지 않는다. `completed: false`인 디렉터리는 완성 결과가 아니다. +raw sample/빌드 산출물은 Git에 넣지 않고 상위 보고서에 경로·해시와 집계를 남긴다. + +현재 v2는 단일/연속 상대 전투와 성향 hook의 비교 기반이다. 획득 pool·경제 수명주기·스노우볼, +능력치 증가로 늘어난 실제 징병 상한, 실제 시즌 메타 비중은 아직 측정하지 않는다. +위 판정 절차의 후속 검증을 끝내기 전에는 전체 밸런스 평가 완료로 보고하지 않는다. + +## 추가 비교와 해석 출력 + +```sh +node tools/balance-lab/run.mjs --suite families --samples 256 --out test-results/balance-lab/families-256 +node tools/balance-lab/run.mjs --suite tiers --samples 128 --out test-results/balance-lab/tiers-128 +node tools/balance-lab/run.mjs --suite interactions --samples 256 --out test-results/balance-lab/interactions-256 +node tools/balance-lab/analyze.mjs test-results/balance-lab/families-256 +``` + +`families`는 보궁기에 통무, 귀병에 통지, 차병에 통솔 배분을 적용하고 서로 다른 +배분의 상대와 싸운다. 원 통솔×100까지 병력을 채우는 `capacity` 조건도 포함한다. +`tiers`는 요구 기술 3000인 모든 병종의 직접 대진을 같은 방식으로 계산한다. +따라서 단순히 약한 기본병을 잘 잡는 후보만 고르는 문제를 보완한다. + +`interactions`는 공성/반계/격노/집중/환술/척사/위압/징병의 10개 특기·아이템 +조합에서 `둘 다 - 특기만 - 아이템만 + 둘 다 없음`을 같은 seed로 측정한다. +이는 선택한 지표에서의 통계적 상호작용이지 trigger 중첩 버그의 판정은 아니다. +준비 완료와 병력 절반·훈사70 상태, 야전/수비장수 3명/공성을 분리한다. + +공성에는 성방·성벽 1000과 5000의 두 조건을 둔다. 5000은 상한에 가려지는 +효과를 보기 위한 내구도 스트레스 조건이며 현재 운영 도시의 대표값이라는 +의미가 아니다. `wallDamage`는 도시 전투 HP 피해이며 HP는 성방×10이다. +원 `city.wall` 필드 감소량과 같은 단위가 아니다. 성벽을 모두 제거해 피해량이 +같더라도 사상자·phase·군량 차이를 함께 확인해야 한다. + +`screen.md`와 `screen.json`은 조건별 행렬과 후보를 정리한다. cell 평균들의 +최솟값/최댓값은 확률 분포의 분위수나 신뢰구간이 아니다. 원 표본의 분위수와 +구간은 `summary.json`에 따로 있다. 손실률 평균만으로 전쟁 승리나 국가 생존을 +예측하지 않는다. 도구 수정 후에는 새 출력 경로에 재실행한다. diff --git a/tools/balance-lab/analyze.mjs b/tools/balance-lab/analyze.mjs new file mode 100644 index 00000000..6ad31b98 --- /dev/null +++ b/tools/balance-lab/analyze.mjs @@ -0,0 +1,114 @@ +import { readFile, writeFile } from 'node:fs/promises'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const root = fileURLToPath(new URL('../../', import.meta.url)); +const directory = path.resolve(root, process.argv[2] ?? 'test-results/balance-lab/units'); +const read = async (file) => JSON.parse(await readFile(path.join(directory, file), 'utf8')); +const manifest = await read('manifest.json'); +if (!manifest.completed) throw new Error('Cannot analyze an incomplete experiment'); +const rows = await read('summary.json'); +const catalog = await read('catalog.json'); +const names = new Map([ + ...catalog.units.map((entry) => [String(entry.id), entry.name]), + ...[...catalog.items, ...catalog.war, ...catalog.nations].map((entry) => [entry.key, entry.name]), +]); +const name = (key) => names.get(key) ?? key; +const mean = (numbers) => numbers.reduce((sum, x) => sum + x, 0) / numbers.length; +const percent = (number) => (number * 100).toFixed(1); +const lines = [ + `# ${manifest.suite}: ${manifest.design}`, + '', + `- Source: ${manifest.commit}, SHA-256 ${manifest.sourceHash}`, + `- ${manifest.cells} cells × ${manifest.samples} seeds; ${manifest.battles} battles.`, + '- Synthetic scenario screen, not live win rate. Cell intervals are descriptive; no multiplicity correction.', + '- Exchange = enemy loss fraction minus own loss fraction; numbers below are percentage points.', + '- Opponents and attack/defence cells have equal weight. These are designed scenario means, not independent samples.', + '', +]; +const groups = new Map(); +for (const row of rows.filter((entry) => entry.mode === 'field')) { + const key = JSON.stringify([row.key, row.build, row.budget ?? 'equal-crew']); + if (!groups.has(key)) groups.set(key, []); + groups.get(key).push(row); +} +const ranking = [...groups.values()] + .map((group) => { + const paired = Boolean(group[0].delta); + const scores = group.map((row) => (paired ? row.delta : row.treatment).exchange.mean); + return { + key: group[0].key, + build: group[0].build, + budget: group[0].budget ?? 'equal-crew', + effect: paired ? 'vs-empty-control' : 'absolute-exchange', + cells: group.length, + mean: mean(scores), + worstCell: Math.min(...scores), + bestCell: Math.max(...scores), + aboveScreen: scores.filter((score) => score >= 0.1).length / scores.length, + belowScreen: scores.filter((score) => score <= -0.1).length / scores.length, + }; + }) + .sort((a, b) => b.mean - a.mean); +lines.push( + '## Field screen', + '', + '| Candidate | Build | Budget | Mean | Worst cell | Best cell | Cells >=10 pp |', + '| --- | --- | --- | ---: | ---: | ---: | ---: |' +); +for (const row of ranking) + lines.push( + `| ${name(row.key)} | ${row.build} | ${row.budget} | ${percent(row.mean)} | ${percent(row.worstCell)} | ${percent(row.bestCell)} | ${percent(row.aboveScreen)}% |` + ); +if (['units', 'families', 'tiers'].includes(manifest.suite)) { + lines.push('', '## Matchup matrices (attack/defence mean)', ''); + const strata = [...new Set(rows.filter((row) => row.mode === 'field').map((row) => `${row.build}/${row.budget}`))]; + for (const stratum of strata) { + const subset = rows.filter((row) => row.mode === 'field' && `${row.build}/${row.budget}` === stratum); + const candidates = [...new Set(subset.map((row) => row.key))]; + const opponents = [...new Set(subset.map((row) => row.opponent))]; + lines.push( + `### ${stratum}`, + '', + `| Candidate | ${opponents.map((id) => name(String(id))).join(' | ')} |`, + `| --- | ${opponents.map(() => '---:').join(' | ')} |` + ); + for (const candidate of candidates) + lines.push( + `| ${name(candidate)} | ${opponents.map((opponent) => percent(mean(subset.filter((row) => row.key === candidate && row.opponent === opponent).map((row) => row.treatment.exchange.mean)))).join(' | ')} |` + ); + lines.push(''); + } +} +lines.push( + '', + '## Siege (do not combine with field exchange)', + '', + '| Candidate | Build | Budget / city defence | Wall damage | Change vs control | Rice | Own dead |', + '| --- | --- | --- | ---: | ---: | ---: | ---: |' +); +for (const row of rows + .filter((entry) => entry.mode === 'siege') + .sort((a, b) => b.treatment.wallDamage.mean - a.treatment.wallDamage.mean)) { + lines.push( + `| ${name(row.key)} | ${row.build}${row.condition ? '/' + row.condition : ''} | ${row.budget ?? 'equal-crew'}/${row.wall ?? 1000} | ${row.treatment.wallDamage.mean.toFixed(0)} | ${row.delta ? row.delta.wallDamage.mean.toFixed(0) : 'n/a'} | ${row.treatment.rice.mean.toFixed(0)} | ${row.treatment.dead.mean.toFixed(0)} |` + ); +} +if (manifest.suite === 'interactions') { + lines.push( + '', + '## Factorial interactions', + '', + 'Synergy = both - trait alone - item alone + neither.', + '', + '| Pair | Mode | Condition | Exchange delta | Exchange synergy | Wall delta | Wall synergy |', + '| --- | --- | --- | ---: | ---: | ---: | ---: |' + ); + for (const row of rows) + lines.push( + `| ${row.key} | ${row.mode} | ${row.condition}/city-${row.wall ?? 1000} | ${row.delta.exchange ? percent(row.delta.exchange.mean) : 'n/a'} | ${row.synergy.exchange ? percent(row.synergy.exchange.mean) : 'n/a'} | ${row.delta.wallDamage.mean.toFixed(0)} | ${row.synergy.wallDamage.mean.toFixed(0)} |` + ); +} +await writeFile(path.join(directory, 'screen.json'), JSON.stringify(ranking, null, 2)); +await writeFile(path.join(directory, 'screen.md'), lines.join('\n') + '\n'); +console.log(JSON.stringify({ directory, groups: ranking.length, largest: ranking.slice(0, 5) })); diff --git a/tools/balance-lab/design.mjs b/tools/balance-lab/design.mjs new file mode 100644 index 00000000..564da997 --- /dev/null +++ b/tools/balance-lab/design.mjs @@ -0,0 +1,366 @@ +// Internal experiment design. No router, database, server seed or gameplay mutation. +export const DESIGN_VERSION = 'balance-lab-v2'; +export const BUILDS = { + martial: { leadership: 90, strength: 90, intel: 30 }, + scholar: { leadership: 90, strength: 30, intel: 90 }, + command: { leadership: 150, strength: 30, intel: 30 }, + hybrid: { leadership: 30, strength: 90, intel: 90 }, +}; +export const BASE_UNITS = [1100, 1200, 1300, 1400, 1500]; +export const nativeBuild = (unit) => + Math.floor(unit / 100) === 14 ? 'scholar' : Math.floor(unit / 100) === 15 ? 'command' : 'martial'; + +export function general(id, unit, build, crew = 7000) { + return { + no: id, + name: `fixture-${id}`, + nation: id, + city: id, + turntime: '2000-01-01 00:00:00', + personal: null, + special: 'None', + special2: null, + crew, + crewtype: unit, + atmos: 100, + train: 100, + ...BUILDS[build], + intel_exp: 0, + book: null, + strength_exp: 0, + weapon: null, + injury: 0, + leadership_exp: 0, + horse: null, + item: null, + explevel: 0, + experience: 1000, + dedication: 1000, + officer_level: 1, + officer_city: 0, + gold: 100000, + rice: 100000, + dex1: 1000, + dex2: 1000, + dex3: 1000, + dex4: 1000, + dex5: 1000, + defence_train: 0, + recent_war: null, + warnum: 0, + killnum: 0, + killcrew: 0, + }; +} + +function city(id, wall) { + return { + city: id, + nation: id, + supply: 1, + name: `fixture-city-${id}`, + pop: 50000, + agri: 1000, + comm: 1000, + secu: 1000, + def: wall, + wall, + trust: 100, + level: 5, + pop_max: 100000, + agri_max: 2000, + comm_max: 2000, + secu_max: 2000, + def_max: Math.max(2000, wall), + wall_max: Math.max(2000, wall), + dead: 0, + state: 0, + conflict: '{}', + }; +} + +function nation(id, type = 'che_중립') { + return { + type, + tech: 3000, + level: 1, + capital: id, + nation: id, + name: `fixture-nation-${id}`, + gold: 100000, + rice: 100000, + gennum: 10, + }; +} + +export function makePayload(unitSet, cell, seed, treatment = true) { + const { build, opponentBuild = build, unit, opponent, mode, role, budget = 'equal-crew' } = cell; + if (!BUILDS[build] || !BUILDS[opponentBuild]) throw new Error('Unknown build'); + const lookup = (id) => { + const found = unitSet.crewTypes.find((entry) => entry.id === id); + if (!found) throw new Error(`Unknown unit ${id}`); + return found; + }; + const count = (id, buildKey) => + budget === 'capacity' + ? BUILDS[buildKey].leadership * 100 + : budget === 'equal-gold' + ? Math.min(BUILDS[buildKey].leadership * 100, Math.floor((7000 * 9) / lookup(id).cost)) + : Math.min(7000, BUILDS[buildKey].leadership * 100); + const subject = general(role === 'attack' ? 1 : 2, unit, build, count(unit, build)); + const other = general(role === 'attack' ? 2 : 1, opponent, opponentBuild, count(opponent, opponentBuild)); + if (treatment) Object.assign(subject, cell.generalPatch ?? {}); + if (cell.condition === 'depleted') { + subject.crew = Math.floor(subject.crew * 0.5); + subject.train = 70; + subject.atmos = 70; + } + const attack = role === 'attack' ? subject : other; + const defend = role === 'attack' ? other : subject; + const payload = { + action: 'battle', + repeatCnt: 1, + seed, + year: 200, + month: 1, + attackerGeneral: attack, + attackerCity: city(1, 1000), + attackerNation: nation(1), + defenderGenerals: + mode === 'siege' + ? [] + : mode === 'line' + ? [defend, { ...defend, no: 3, name: 'fixture-3' }, { ...defend, no: 4, name: 'fixture-4' }] + : [defend], + defenderCity: city(2, mode === 'field' ? 0 : (cell.wall ?? 1000)), + defenderNation: nation(2), + unitSet, + scenarioEffect: null, + config: { + armPerPhase: 500, + maxTrainByCommand: 100, + maxAtmosByCommand: 100, + maxTrainByWar: 110, + maxAtmosByWar: 150, + maxGeneralStat: 255, + statUpgradeLimit: 30, + maxTechLevel: 12, + castleCrewTypeId: 1000, + armTypes: { footman: 1, archer: 2, cavalry: 3, wizard: 4, siege: 5, misc: 6, castle: 0 }, + }, + time: { year: 200, month: 1, startYear: 180 }, + }; + if (treatment && cell.nationTrait) { + payload[role === 'attack' ? 'attackerNation' : 'defenderNation'].type = cell.nationTrait; + } + if (mode === 'siege' && role !== 'attack') throw new Error('Siege subject must attack'); + return payload; +} + +export function summarize(values) { + if (!values.length || values.some((value) => !Number.isFinite(value))) throw new Error('Invalid samples'); + const n = values.length; + const mean = values.reduce((a, b) => a + b, 0) / n; + const variance = n > 1 ? values.reduce((a, b) => a + (b - mean) ** 2, 0) / (n - 1) : null; + // Normal-approximation interval, descriptive screening only; never a multiple-testing gate. + const half = variance === null ? null : 1.96 * Math.sqrt(variance / n); + const sorted = values.toSorted((a, b) => a - b); + return { + n, + mean, + sd: variance === null ? null : Math.sqrt(variance), + ci95: half === null ? null : [mean - half, mean + half], + p10: sorted[Math.floor((n - 1) * 0.1)], + p50: sorted[Math.floor((n - 1) * 0.5)], + p90: sorted[Math.floor((n - 1) * 0.9)], + min: sorted[0], + max: sorted[n - 1], + }; +} + +export function observe(outcome, payload, role) { + const subjectId = role === 'attack' ? 1 : 2; + const report = outcome.reports.find((entry) => entry.type === 'general' && entry.id === subjectId); + if (!report) throw new Error('Subject report missing'); + const ownStart = role === 'attack' ? payload.attackerGeneral : payload.defenderGenerals[0]; + const ownEnd = role === 'attack' ? outcome.attacker : outcome.defenders[0]; + const enemyReports = outcome.reports.filter((entry) => entry.type === 'general' && entry.id !== subjectId); + const fieldKills = enemyReports.reduce((sum, entry) => sum + entry.dead, 0); + const wall = outcome.reports.find((entry) => entry.type === 'city'); + const enemyStart = + role === 'attack' + ? payload.defenderGenerals.reduce((sum, entry) => sum + entry.crew, 0) + : payload.attackerGeneral.crew; + const enemyEnd = + role === 'attack' ? outcome.defenders.reduce((sum, entry) => sum + entry.crew, 0) : outcome.attacker.crew; + return { + // Fractional loss difference is bounded and comparable across troop budgets, unlike kill/death ratios. + exchange: enemyStart ? fieldKills / enemyStart - report.dead / ownStart.crew : null, + fieldKills, + dead: report.dead, + remaining: ownEnd.crew, + rice: ownStart.rice - ownEnd.rice, + goldGain: ownEnd.gold - ownStart.gold, + experienceGain: ownEnd.experience - ownStart.experience, + dedicationGain: ownEnd.dedication - ownStart.dedication, + injury: ownEnd.injury - ownStart.injury, + phase: report.phase ?? 0, + wallDamage: role === 'attack' ? (wall?.dead ?? 0) : 0, + breakthrough: Number(outcome.conquered), + annihilated: Number(ownEnd.crew === 0), + enemyAnnihilated: enemyStart ? Number(enemyEnd === 0) : null, + }; +} + +export function buildPlan(catalog, suite) { + const cells = []; + const add = (cell) => cells.push({ id: String(cells.length).padStart(5, '0'), ...cell }); + if (suite === 'families' || suite === 'tiers') { + const candidates = + suite === 'families' + ? catalog.units.filter((unit) => BASE_UNITS.includes(unit.id)) + : catalog.units.filter((unit) => + unit.requirements.some((req) => req.type === 'ReqTech' && req.tech === 3000) + ); + for (const unit of candidates) + for (const enemy of candidates) { + for (const budget of ['equal-crew', 'equal-gold', 'capacity']) + for (const role of ['attack', 'defend']) { + add({ + category: suite, + key: String(unit.id), + unit: unit.id, + opponent: enemy.id, + build: nativeBuild(unit.id), + opponentBuild: nativeBuild(enemy.id), + budget, + role, + mode: 'field', + }); + } + } + } else if (suite === 'interactions') { + const pairs = [ + ['che_공성', 'che_공성_묵자', 1501], + ['che_공성', 'che_행동_서촉지형도', 1501], + ['che_반계', 'che_서적_07_사마법', 1400], + ['che_반계', 'che_서적_12_산해경', 1400], + ['che_격노', 'che_격노_구정신단경', 1100], + ['che_집중', 'che_환술_논어집해', 1400], + ['che_환술', 'che_집중_전국책', 1400], + ['che_척사', 'che_척사_오악진형도', 1100], + ['che_위압', 'che_위압_조목삭', 1100], + ['che_징병', 'che_징병_낙주', 1501], + ]; + for (const [trait, key, unit] of pairs) { + const item = catalog.items.find((entry) => entry.key === key); + if (!item) throw new Error(`Missing interaction item ${key}`); + for (const mode of ['field', 'siege', 'line']) + for (const condition of ['ready', 'depleted']) { + for (const wall of mode === 'siege' ? [1000, 5000] : [1000]) + add({ + category: 'interaction', + key: `${trait}+${key}`, + unit, + build: nativeBuild(unit), + opponent: 1400, + opponentBuild: 'scholar', + role: 'attack', + mode, + condition, + paired: true, + wall, + generalPatch: { special2: trait, [item.slot]: key }, + factorialPatches: [{ special2: trait }, { [item.slot]: key }], + }); + } + } + } else if (suite === 'units') { + for (const unit of catalog.units) + for (const build of Object.keys(BUILDS)) { + for (const budget of ['equal-crew', 'equal-gold']) { + for (const opponent of BASE_UNITS) + for (const role of ['attack', 'defend']) { + add({ + category: 'unit', + key: String(unit.id), + unit: unit.id, + build, + budget, + opponent, + role, + mode: 'field', + }); + } + for (const wall of [1000, 5000]) + add({ + category: 'unit', + key: String(unit.id), + unit: unit.id, + build, + budget, + opponent: 1100, + role: 'attack', + mode: 'siege', + wall, + }); + } + } + } else { + const treatments = + suite === 'items' + ? catalog.items.map((item) => ({ + category: 'item', + key: item.key, + generalPatch: { [item.slot]: item.key }, + })) + : suite === 'traits' + ? catalog.war.map((trait) => ({ + category: 'war', + key: trait.key, + generalPatch: { special2: trait.key }, + })) + : suite === 'nations' + ? catalog.nations.map((trait) => ({ + category: 'nation', + key: trait.key, + nationTrait: trait.key, + })) + : null; + if (!treatments) throw new Error(`Unknown suite ${suite}`); + for (const treatment of treatments) + for (const [build, unit] of [ + ['martial', 1100], + ['martial', 1200], + ['martial', 1300], + ['scholar', 1400], + ['command', 1501], + ]) { + for (const opponent of [1100, 1200, 1300, 1400, 1500]) + for (const role of ['attack', 'defend']) { + add({ + ...treatment, + unit, + build, + opponent, + opponentBuild: nativeBuild(opponent), + role, + mode: 'field', + paired: true, + }); + } + for (const wall of [1000, 5000]) + add({ + ...treatment, + unit, + build, + opponent: 1100, + role: 'attack', + mode: 'siege', + wall, + paired: true, + }); + } + } + return cells; +} diff --git a/tools/balance-lab/design.test.mjs b/tools/balance-lab/design.test.mjs new file mode 100644 index 00000000..28869869 --- /dev/null +++ b/tools/balance-lab/design.test.mjs @@ -0,0 +1,108 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { readFile } from 'node:fs/promises'; +import { BUILDS, makePayload, buildPlan, summarize, observe } from './design.mjs'; + +const unitSet = JSON.parse( + await readFile(new URL('../../resources/unitset/unitset_che.json', import.meta.url), 'utf8') +); +const cell = { + unit: 1100, + opponent: 1200, + build: 'martial', + role: 'attack', + mode: 'field', + generalPatch: { weapon: 'test-item' }, +}; + +test('allocation preserves total stat budget and caps troops by leadership', () => { + for (const [build, stats] of Object.entries(BUILDS)) { + assert.equal( + Object.values(stats).reduce((a, b) => a + b), + 210 + ); + const payload = makePayload(unitSet, { ...cell, build }, 's'); + assert.ok(payload.attackerGeneral.crew <= stats.leadership * 100); + } +}); +test('treatment changes only the subject and paired control removes it in both roles', () => { + for (const role of ['attack', 'defend']) { + const treatment = makePayload(unitSet, { ...cell, role, nationTrait: 'che_병가' }, 's'); + const control = makePayload(unitSet, { ...cell, role, nationTrait: 'che_병가' }, 's', false); + const own = role === 'attack' ? treatment.attackerGeneral : treatment.defenderGenerals[0]; + const enemy = role === 'attack' ? treatment.defenderGenerals[0] : treatment.attackerGeneral; + assert.equal(own.crewtype, cell.unit); + assert.equal(enemy.crewtype, cell.opponent); + assert.equal(own.weapon, 'test-item'); + assert.equal(enemy.weapon, null); + own.weapon = null; + treatment[role === 'attack' ? 'attackerNation' : 'defenderNation'].type = 'che_중립'; + assert.deepEqual(treatment, control); + } +}); +test('field and siege are separate and economic budget respects expensive units', () => { + assert.equal(makePayload(unitSet, cell, 's').defenderCity.wall, 0); + const siege = makePayload(unitSet, { ...cell, mode: 'siege' }, 's'); + assert.equal(siege.defenderGenerals.length, 0); + assert.equal(siege.defenderCity.wall, 1000); + const expensive = makePayload(unitSet, { ...cell, unit: 1503, budget: 'equal-gold' }, 's'); + assert.ok(expensive.attackerGeneral.crew < 7000); + assert.throws(() => makePayload(unitSet, { ...cell, mode: 'siege', role: 'defend' }, 's')); +}); +test('observer excludes wall damage from field exchange and preserves unknown siege exchange', () => { + const payload = makePayload(unitSet, cell, 's'); + const outcome = { + attacker: { crew: 6000, rice: 99900, injury: 0 }, + defenders: [{ crew: 5000 }], + conquered: false, + reports: [ + { id: 1, type: 'general', dead: 1000, killed: 9000 }, + { id: 2, type: 'general', dead: 2000 }, + { id: null, type: 'city', dead: 7000 }, + ], + }; + assert.equal(observe(outcome, payload, 'attack').fieldKills, 2000); + assert.equal(observe(outcome, payload, 'attack').exchange, 1000 / 7000); + const siege = makePayload(unitSet, { ...cell, mode: 'siege' }, 's'); + assert.equal(observe({ ...outcome, defenders: [] }, siege, 'attack').exchange, null); +}); +test('statistics expose n, interval and tails; invalid samples fail closed', () => { + assert.deepEqual(summarize([2, 2]).ci95, [2, 2]); + assert.equal(summarize([1, 2, 3]).mean, 2); + assert.equal(summarize([1]).ci95, null); + assert.throws(() => summarize([])); + assert.throws(() => summarize([NaN])); +}); +test('design covers every registered unit, both budgets, four builds, roles and siege', () => { + const catalog = { units: unitSet.crewTypes.filter((unit) => unit.armType > 0) }; + const plan = buildPlan(catalog, 'units'); + assert.equal(plan.length, catalog.units.length * 4 * 2 * 12); + assert.equal(new Set(plan.map((cell) => cell.id)).size, plan.length); + assert.throws(() => buildPlan(catalog, 'no-such-suite')); +}); + +test('native family and tier comparisons use matched total stats and directly compare tier-three units', () => { + const catalog = { units: unitSet.crewTypes.filter((unit) => unit.armType > 0) }; + const families = buildPlan(catalog, 'families'); + assert.equal(families.length, 5 * 5 * 3 * 2); + const pair = families.find((row) => row.unit === 1400 && row.opponent === 1100 && row.role === 'attack'); + const payload = makePayload(unitSet, pair, 's'); + assert.equal(payload.attackerGeneral.intel, 90); + assert.equal(payload.defenderGenerals[0].strength, 90); + const capacity = makePayload(unitSet, { ...pair, unit: 1500, build: 'command', budget: 'capacity' }, 's'); + assert.equal(capacity.attackerGeneral.crew, 15000); + const tiers = buildPlan(catalog, 'tiers'); + assert.ok(tiers.some((row) => row.unit === 1502 && row.opponent === 1503)); + assert.ok(tiers.every((row) => ![1100, 1306].includes(row.unit))); +}); + +test('depleted line encounters preserve defender ownership and vary only subject readiness', () => { + const payload = makePayload(unitSet, { ...cell, condition: 'depleted', mode: 'line' }, 's'); + assert.equal(payload.attackerGeneral.crew, 3500); + assert.equal(payload.attackerGeneral.train, 70); + assert.deepEqual( + payload.defenderGenerals.map((row) => row.no), + [2, 3, 4] + ); + assert.ok(payload.defenderGenerals.every((row) => row.nation === 2 && row.city === 2 && row.train === 100)); +}); diff --git a/tools/balance-lab/engine.test.mjs b/tools/balance-lab/engine.test.mjs new file mode 100644 index 00000000..7beac397 --- /dev/null +++ b/tools/balance-lab/engine.test.mjs @@ -0,0 +1,45 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { readFile } from 'node:fs/promises'; +import { processBattleSimJob, parseUnitSetDefinition } from '../../packages/logic/dist/index.js'; +import { makePayload, observe } from './design.mjs'; + +const units = parseUnitSetDefinition( + JSON.parse(await readFile(new URL('../../resources/unitset/unitset_che.json', import.meta.url), 'utf8')) +); +const cell = { unit: 1100, opponent: 1200, build: 'martial', role: 'attack', mode: 'field' }; +function run(payload, role = 'attack') { + let observation; + processBattleSimJob(payload, { + onBattleResolved: (outcome) => { + observation = observe(outcome, payload, role); + }, + }); + assert.ok(observation); + assert.ok(Object.values(observation).every((value) => value === null || Number.isFinite(value))); + return observation; +} +test('actual engine is deterministic and leaves the input fixture immutable', () => { + const payload = makePayload(units, cell, 'balance-test'); + const saved = structuredClone(payload); + assert.deepEqual(run(payload), run(payload)); + assert.deepEqual(payload, saved); +}); +test('neutral nation control is exactly equal, not just statistically similar', () => { + const paired = { ...cell, nationTrait: 'che_중립' }; + assert.deepEqual(run(makePayload(units, paired, 's')), run(makePayload(units, paired, 's', false))); +}); +test('same-unit mirror results reverse exchange and retain zero wall damage', () => { + const mirror = { ...cell, opponent: 1100 }; + const attack = run(makePayload(units, mirror, 's')); + const defend = run(makePayload(units, { ...mirror, role: 'defend' }, 's'), 'defend'); + assert.ok(Math.abs(attack.exchange + defend.exchange) < 1e-12); + assert.equal(attack.wallDamage, 0); +}); +test('real siege item is equipped and changes wall damage in the production action stack', () => { + const siege = { ...cell, unit: 1501, build: 'command', mode: 'siege', generalPatch: { item: 'che_공성_묵자' } }; + const control = run(makePayload(units, { ...siege, wall: 5000 }, 'balance-test', false)); + const treatment = run(makePayload(units, { ...siege, wall: 5000 }, 'balance-test')); + assert.equal(treatment.exchange, null); + assert.ok(treatment.wallDamage > control.wallDamage); +}); diff --git a/tools/balance-lab/run.mjs b/tools/balance-lab/run.mjs new file mode 100644 index 00000000..456ba97e --- /dev/null +++ b/tools/balance-lab/run.mjs @@ -0,0 +1,247 @@ +import { createHash } from 'node:crypto'; +import { execFileSync } from 'node:child_process'; +import { mkdir, readFile, writeFile, readdir, open } from 'node:fs/promises'; +import { fileURLToPath } from 'node:url'; +import path from 'node:path'; +import { parseArgs } from 'node:util'; +import { DESIGN_VERSION, BUILDS, buildPlan, makePayload, observe, summarize } from './design.mjs'; + +const root = fileURLToPath(new URL('../../', import.meta.url)); +const { values } = parseArgs({ + options: { + suite: { type: 'string', default: 'units' }, + samples: { type: 'string', default: '64' }, + seed: { type: 'string', default: 'balance-2026-09-v1' }, + out: { type: 'string' }, + match: { type: 'string' }, + plan: { type: 'boolean', default: false }, + }, +}); +const samples = Number(values.samples); +if (!Number.isSafeInteger(samples) || samples < 2 || samples > 10000) throw new Error('--samples must be 2..10000'); +if (!['units', 'items', 'traits', 'nations', 'families', 'tiers', 'interactions'].includes(values.suite)) + throw new Error('Unknown suite'); +const out = path.resolve(root, values.out ?? `test-results/balance-lab/${values.suite}`); +// Build before importing; a previous branch's dist must never become measurement evidence. +for (const pkg of ['common', 'logic']) { + execFileSync('pnpm', ['--filter', `@sammo-ts/${pkg}`, 'build'], { cwd: root, stdio: ['ignore', 'ignore', 'pipe'] }); +} +const logic = await import('../../packages/logic/dist/index.js'); +const unitSet = logic.parseUnitSetDefinition( + JSON.parse(await readFile(path.join(root, 'resources/unitset/unitset_che.json'), 'utf8')) +); +const allItems = await logic.loadItemModules([...logic.ITEM_KEYS]); +const war = await logic.loadWarTraitModules([...logic.WAR_TRAIT_KEYS]); +const nations = await logic.loadNationTraitModules([...logic.NATION_TRAIT_KEYS]); +const spec = (entry) => ({ + key: entry.key, + name: entry.name, + info: entry.info, + slot: entry.slot, + unique: entry.unique, + cost: entry.cost, + selection: entry.selection, + hooks: Object.keys(entry).filter( + (key) => typeof entry[key] === 'function' && !['getName', 'getInfo'].includes(key) + ), + events: Object.keys(entry.eventHandlers ?? {}), +}); +const catalog = { + scope: 'CHE unitset + all registered unique modules; scenario acquisition eligibility is NOT asserted', + units: unitSet.crewTypes.filter((unit) => unit.armType > 0), + items: allItems.filter((item) => item.unique).map(spec), + war: war.map(spec), + nations: nations.map(spec), +}; +let cells = buildPlan(catalog, values.suite); +if (values.match) + cells = cells.filter((cell) => + `${cell.key}/${cell.build}/${cell.mode}/${cell.role}/${cell.budget ?? ''}`.includes(values.match) + ); +if (!cells.length) throw new Error('No experiment cells selected'); +async function hashTree(directories) { + const hash = createHash('sha256'); + const walk = async (relative) => { + for (const entry of (await readdir(path.join(root, relative), { withFileTypes: true })).sort((a, b) => + a.name.localeCompare(b.name) + )) { + const child = path.join(relative, entry.name); + if (entry.isDirectory()) await walk(child); + else if (entry.isFile()) hash.update(child).update(await readFile(path.join(root, child))); + } + }; + for (const dir of directories) await walk(dir); + return hash.digest('hex'); +} +const sourceHash = await hashTree([ + 'packages/common/src', + 'packages/logic/src', + 'resources/unitset', + 'tools/balance-lab', +]); +const manifest = { + design: DESIGN_VERSION, + suite: values.suite, + seed: values.seed, + samples, + commit: execFileSync('git', ['rev-parse', 'HEAD'], { cwd: root, encoding: 'utf8' }).trim(), + dirty: execFileSync('git', ['status', '--porcelain'], { cwd: root, encoding: 'utf8' }).trim(), + sourceHash, + node: process.version, + builds: BUILDS, + cells: cells.length, + battles: cells.reduce((sum, cell) => sum + samples * (cell.factorialPatches ? 4 : cell.paired ? 2 : 1), 0), + protocol: { + seedPairing: 'same seed within cell and treatment/control; RNG consumption may diverge', + inference: + 'per-cell descriptive normal CI; correlated cells and multiple tests prevent global significance claims', + economics: 'equal-gold uses base crew cost only, capped by leadership; not total lifetime cost', + outcome: 'fraction enemy troops lost minus fraction own troops lost; siege uses wall damage separately', + environment: 'synthetic CHE, tech 3000, year 200, no scenario effect; no production data or mutation', + }, + completed: false, +}; +await mkdir(out, { recursive: true }); +// Never silently truncate a previous run (including a partially completed run). +const claim = await open(path.join(out, 'manifest.json'), 'wx'); +await claim.writeFile(JSON.stringify(manifest, null, 2)); +await claim.close(); +await writeFile(path.join(out, 'catalog.json'), JSON.stringify(catalog, null, 2)); +await writeFile(path.join(out, 'plan.json'), JSON.stringify(cells, null, 2)); +if (values.plan) { + console.log(JSON.stringify({ out, ...manifest })); + process.exit(0); +} + +// Nation coefficients are observed through the actual module hooks. Fail on newly context-dependent +// hooks rather than silently inventing a partial domain context for them. +const noContext = new Proxy( + {}, + { + get: (_, key) => { + throw new Error(`Nation probe needs context.${String(key)}`); + }, + } +); +const nationProfiles = nations.map((module) => { + const domestic = {}; + for (const action of ['농업', '상업', '기술', '치안', '민심', '인구', '수비', '성벽', '계략', '징병', '모병']) { + domestic[action] = Object.fromEntries( + ['score', 'cost', 'success'].map((kind) => { + const base = kind === 'success' ? 0.5 : 100; + return [kind, module.onCalcDomestic?.(noContext, action, kind, base) ?? base]; + }) + ); + } + return { + key: module.key, + domestic, + income: Object.fromEntries( + ['gold', 'rice', 'pop'].map((kind) => [kind, module.onCalcNationalIncome?.(noContext, kind, 100) ?? 100]) + ), + populationLoss: module.onCalcNationalIncome?.(noContext, 'pop', -100) ?? -100, + strategy: Object.fromEntries( + ['의병모집', '급습', '수몰'].map((action) => [ + action, + Object.fromEntries( + ['delay', 'globalDelay'].map((kind) => [ + kind, + [12, 24, 36].map((base) => ({ + base, + actual: module.onCalcStrategic?.(noContext, action, kind, base) ?? base, + })), + ]) + ), + ]) + ), + }; +}); +await writeFile(path.join(out, 'nation-profiles.json'), JSON.stringify(nationProfiles, null, 2)); + +const raw = await open(path.join(out, 'samples.jsonl'), 'wx'); +const summary = []; +const started = Date.now(); +try { + for (const cell of cells) { + const rows = []; + for (let index = 0; index < samples; index++) { + // Omit candidate key and role: common random numbers across candidate and role strata. + const seed = JSON.stringify([ + values.seed, + cell.build, + cell.opponent, + cell.mode, + cell.budget ?? 'equal-crew', + cell.condition ?? 'ready', + index, + ]); + const run = (treatment, patch) => { + const payload = makePayload(unitSet, patch ? { ...cell, generalPatch: patch } : cell, seed, treatment); + let result; + logic.processBattleSimJob(payload, { + onBattleResolved: (battle) => { + result = observe(battle, payload, cell.role); + }, + }); + if (!result) throw new Error('Battle did not execute'); + return result; + }; + const treatment = run(true); + const control = cell.paired ? run(false) : null; + const delta = control + ? Object.fromEntries( + Object.entries(treatment).map(([key, value]) => [ + key, + value === null ? null : value - control[key], + ]) + ) + : null; + const singles = cell.factorialPatches?.map((patch) => run(true, patch)); + const synergy = singles + ? Object.fromEntries( + Object.entries(treatment).map(([key, value]) => [ + key, + value === null ? null : value - singles[0][key] - singles[1][key] + control[key], + ]) + ) + : null; + const row = { cell: cell.id, index, seed, treatment, control, delta, singles, synergy }; + rows.push(row); + await raw.write(JSON.stringify(row) + '\n'); + } + const aggregate = (field) => + Object.fromEntries( + Object.keys(rows[0][field]).map((metric) => { + const values = rows.map((row) => row[field][metric]); + return [metric, values[0] === null ? null : summarize(values)]; + }) + ); + summary.push({ + ...cell, + treatment: aggregate('treatment'), + control: cell.paired ? aggregate('control') : null, + delta: cell.paired ? aggregate('delta') : null, + synergy: cell.factorialPatches ? aggregate('synergy') : null, + }); + if (summary.length % 25 === 0) + console.error( + `${values.suite}: ${summary.length}/${cells.length} cells, ${Math.round((Date.now() - started) / 1000)}s` + ); + } +} finally { + await raw.close(); +} +const finalHash = await hashTree([ + 'packages/common/src', + 'packages/logic/src', + 'resources/unitset', + 'tools/balance-lab', +]); +if (finalHash !== sourceHash) throw new Error('Source changed during run; result is not complete'); +await writeFile(path.join(out, 'summary.json'), JSON.stringify(summary, null, 2)); +manifest.completed = true; +manifest.elapsedSeconds = (Date.now() - started) / 1000; +await writeFile(path.join(out, 'manifest.json'), JSON.stringify(manifest, null, 2)); +console.log( + JSON.stringify({ out, cells: cells.length, battles: manifest.battles, elapsedSeconds: manifest.elapsedSeconds }) +);