운영 안정화 변경을 최신 main과 통합한다

tools/host-recovery/README.md

@@ -0,0 +1,12 @@
+# Windows host boot identity agent
+
+`windows-boot-agent.mjs`는 외부 npm package 없이 실행하는 Node.js 에이전트다.
+Windows OS의 부팅 시각 기반 식별자를 인증된 Linux 게임 데몬에 제공한다.
+
+설치, key 생성, 자동 시작과 선택 기능 장애 시 동작은
+[Hyper-V 복구 운영 안내](../../docs/developer/host-boot-recovery.md)를 따른다.
+설정 예시는 `windows-boot-agent.example.json`이며 실제 key와 설정은 Git에 넣지 않는다.
+
+```sh
+node --test tools/host-recovery/windows-boot-agent.test.mjs
+```

tools/host-recovery/windows-boot-agent.example.json

@@ -0,0 +1,5 @@
+{
+    "keyFile": "C:\\ProgramData\\Sammo\\host-boot.key",
+    "bindAddress": "127.0.0.1",
+    "port": 17781
+}

tools/host-recovery/windows-boot-agent.mjs

@@ -0,0 +1,131 @@
+import { execFile } from 'node:child_process';
+import { createHash, createHmac, randomBytes, timingSafeEqual } from 'node:crypto';
+import { readFile, writeFile } from 'node:fs/promises';
+import { createServer } from 'node:http';
+import { hostname } from 'node:os';
+import { win32, resolve } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { promisify } from 'node:util';
+
+const mac = (key, value) => createHmac('sha256', key).update(value).digest('hex');
+
+export const bootIdentity = (host, ticks) => {
+    if (!/^\d{17,20}$/.test(ticks)) throw new Error('Windows returned an invalid boot time.');
+    return createHash('sha256').update(`${host.toLowerCase()}\n${ticks}`).digest('hex');
+};
+
+export const readWindowsBootIdentity = async () => {
+    if (process.platform !== 'win32') throw new Error('Run the host boot agent on Windows.');
+    const executable = win32.join(
+        process.env.SystemRoot ?? 'C:\\Windows',
+        'System32',
+        'WindowsPowerShell',
+        'v1.0',
+        'powershell.exe'
+    );
+    // Node.js가 OS 내장 CIM 조회를 실행한다. 별도 PowerShell 스크립트/모듈은 필요 없다.
+    const { stdout } = await promisify(execFile)(
+        executable,
+        [
+            '-NoProfile',
+            '-NonInteractive',
+            '-Command',
+            '(Get-CimInstance -ClassName Win32_OperatingSystem -Property LastBootUpTime).LastBootUpTime.ToUniversalTime().Ticks.ToString()',
+        ],
+        { windowsHide: true, timeout: 10_000, maxBuffer: 2_048 }
+    );
+    return bootIdentity(hostname(), stdout.trim());
+};
+
+export const createBootIdentityServer = ({ key, getBootId }) => {
+    if (!Buffer.isBuffer(key) || key.length !== 32) throw new Error('A 32-byte host boot key is required.');
+    const server = createServer((request, response) => {
+        response.setHeader('Cache-Control', 'no-store');
+        if (request.method !== 'GET' || request.url !== '/v1/boot-id') {
+            response.writeHead(404).end();
+            return;
+        }
+        const challenge = request.headers['x-sammo-challenge'];
+        const signature = request.headers['x-sammo-signature'];
+        if (
+            typeof challenge !== 'string' ||
+            !/^[a-f0-9]{64}$/.test(challenge) ||
+            typeof signature !== 'string' ||
+            !/^[a-f0-9]{64}$/.test(signature) ||
+            !timingSafeEqual(
+                Buffer.from(signature, 'hex'),
+                Buffer.from(mac(key, `sammo-host-boot-request-v1\n${challenge}`), 'hex')
+            )
+        ) {
+            response.writeHead(401).end();
+            return;
+        }
+        const bootId = getBootId();
+        if (typeof bootId !== 'string' || !/^[a-f0-9]{64}$/.test(bootId)) {
+            response.writeHead(503).end();
+            return;
+        }
+        response.writeHead(200, { 'Content-Type': 'application/json' });
+        response.end(
+            JSON.stringify({
+                version: 1,
+                challenge,
+                bootId,
+                signature: mac(key, `sammo-host-boot-response-v1\n${challenge}\n${bootId}`),
+            })
+        );
+    });
+    server.headersTimeout = 5_000;
+    server.requestTimeout = 5_000;
+    server.keepAliveTimeout = 1_000;
+    server.maxHeadersCount = 20;
+    return server;
+};
+
+export const main = async (args = process.argv.slice(2), env = process.env) => {
+    if (args.length === 2 && args[0] === '--create-key') {
+        await writeFile(args[1], `${randomBytes(32).toString('hex')}\n`, { flag: 'wx', mode: 0o600 });
+        console.info('Host boot key created. Copy it privately to the Linux secret file.');
+        return;
+    }
+    if (args.length === 2 && args[0] === '--config') {
+        const config = JSON.parse(await readFile(args[1], 'utf8'));
+        if (
+            !config ||
+            typeof config.keyFile !== 'string' ||
+            typeof config.bindAddress !== 'string' ||
+            !Number.isInteger(config.port)
+        ) {
+            throw new Error('Host boot agent configuration is invalid.');
+        }
+        env = {
+            ...env,
+            HOST_BOOT_KEY_FILE: config.keyFile,
+            HOST_BOOT_BIND_ADDRESS: config.bindAddress,
+            HOST_BOOT_PORT: String(config.port),
+        };
+    } else if (args.length) throw new Error('Usage: node windows-boot-agent.mjs [--create-key path | --config path]');
+    if (!env.HOST_BOOT_KEY_FILE) throw new Error('HOST_BOOT_KEY_FILE is required.');
+    const encoded = (await readFile(env.HOST_BOOT_KEY_FILE, 'utf8')).trim();
+    if (!/^[a-f0-9]{64}$/.test(encoded)) throw new Error('Host boot key file is invalid.');
+    const port = Number(env.HOST_BOOT_PORT ?? '17781');
+    if (!Number.isInteger(port) || port < 1 || port > 65535) throw new Error('HOST_BOOT_PORT is invalid.');
+    // OS 부팅 시각은 agent 재시작에도 같으며, 시각 차이 계산으로 재부팅을 추정하지 않는다.
+    const bootId = await readWindowsBootIdentity();
+    const server = createBootIdentityServer({ key: Buffer.from(encoded, 'hex'), getBootId: () => bootId });
+    server.on('error', () => {
+        console.error('Host boot agent listener failed.');
+        process.exitCode = 1;
+    });
+    server.listen(port, env.HOST_BOOT_BIND_ADDRESS ?? '127.0.0.1', () => {
+        console.info('Host boot identity agent is listening.');
+    });
+    for (const signal of ['SIGINT', 'SIGTERM']) process.once(signal, () => server.close());
+};
+
+if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) {
+    main().catch(() => {
+        console.error('Host boot agent startup failed; check OS, private key file, and listener configuration.');
+        process.exitCode = 1;
+    });
+}

tools/host-recovery/windows-boot-agent.test.mjs

@@ -0,0 +1,42 @@
+import { after, test } from 'node:test';
+import assert from 'node:assert/strict';
+import { createHmac, randomBytes, timingSafeEqual } from 'node:crypto';
+import { once } from 'node:events';
+import { bootIdentity, createBootIdentityServer } from './windows-boot-agent.mjs';
+
+test('agent restart preserves OS boot identity; a new OS boot changes it', () => {
+    assert.equal(bootIdentity('HOST', '638985120000000000'), bootIdentity('host', '638985120000000000'));
+    assert.notEqual(bootIdentity('HOST', '638985120000000000'), bootIdentity('HOST', '638985120000000001'));
+    assert.throws(() => bootIdentity('HOST', 'invalid'));
+});
+
+test('HTTP transport rejects anonymous requests and signs the current challenge and boot identity', async () => {
+    const key = randomBytes(32);
+    let bootId = 'a'.repeat(64);
+    const server = createBootIdentityServer({ key, getBootId: () => bootId });
+    server.listen(0, '127.0.0.1');
+    await once(server, 'listening');
+    const url = `http://127.0.0.1:${server.address().port}/v1/boot-id`;
+    after(
+        () =>
+            new Promise((resolve) => {
+                server.close(resolve);
+                server.closeAllConnections();
+            })
+    );
+    assert.equal((await fetch(url)).status, 401);
+    assert.equal((await fetch(`${url}?bypass=1`)).status, 404);
+    const challenge = randomBytes(32).toString('hex');
+    const signature = createHmac('sha256', key).update(`sammo-host-boot-request-v1\n${challenge}`).digest('hex');
+    const headers = { 'x-sammo-challenge': challenge, 'x-sammo-signature': signature };
+    const response = await fetch(url, { headers });
+    assert.equal(response.status, 200);
+    assert.equal(response.headers.get('cache-control'), 'no-store');
+    const body = await response.json();
+    assert.equal(body.challenge, challenge);
+    const expected = createHmac('sha256', key).update(`sammo-host-boot-response-v1\n${challenge}\n${bootId}`).digest();
+    assert.ok(timingSafeEqual(expected, Buffer.from(body.signature, 'hex')));
+    bootId = 'b'.repeat(64);
+    assert.equal((await (await fetch(url, { headers })).json()).bootId, bootId);
+    assert.equal((await fetch(url, { headers: { ...headers, 'x-sammo-signature': '0'.repeat(64) } })).status, 401);
+});

tools/integration-tests/scripts/play-audit-npc-lifecycle.ts

@@ -3,7 +3,7 @@ import assert from 'node:assert/strict';
 import { mkdir, readFile, writeFile } from 'node:fs/promises';
 import { resolve } from 'node:path';
 import { asRecord, type TurnCheckpoint } from '@sammo-ts/common';
-import { createGamePostgresConnector } from '@sammo-ts/infra';
+import { createGamePostgresConnector, decodeAuditDecisionSteps } from '@sammo-ts/infra';
 import { createTurnDaemonRuntime, seedScenarioToDatabase } from '@sammo-ts/game-engine';
 
 const databaseUrl = process.env.DATABASE_URL;
@@ -39,7 +39,7 @@ const verify = async (progress: unknown[]) => {
             include: { chunks: { orderBy: { ordinal: 'asc' } } },
         });
         assert(row && row.chunks.length, `Missing actual NPC ${JSON.stringify(selection)} decision`);
-        const steps = row.chunks.flatMap((chunk) => (Array.isArray(chunk.steps) ? chunk.steps : []));
+        const steps = row.chunks.flatMap((chunk) => decodeAuditDecisionSteps(chunk.steps));
         assert.equal(steps.length, row.stepCount);
         assert.equal(asRecord(steps[0]).kind, 'DECISION_START');
         assert(steps.some((step) => asRecord(step).kind === 'DECISION_END'));

tools/play-audit-maintenance.mjs

@@ -0,0 +1,95 @@
+#!/usr/bin/env node
+// stopped/reserved profile도 같은 fenced batch로 정리한다. 기본은 관찰만 하며 외부 scheduler가 실행한다.
+import { performance } from 'node:perf_hooks';
+import { setTimeout } from 'node:timers/promises';
+import { createGamePostgresConnector, pruneDeliveredReadModelOutbox } from '../packages/infra/dist/index.js';
+
+// PM2/호출자가 넘긴 turn-daemon 역할로 engine root의 자동 기동 경로에 들어가지 않는다.
+process.env.GAME_ENGINE_ROLE = 'audit-maintenance';
+const { prunePreviousAuditBatch } = await import('../app/game-engine/dist/index.js');
+
+const args = process.argv.slice(2);
+const apply = args.includes('--apply');
+const schemas =
+    args
+        .find((arg) => arg.startsWith('--schemas='))
+        ?.slice(10)
+        .split(',') ?? [];
+const budgetMs = Number(args.find((arg) => arg.startsWith('--budget-ms='))?.slice(12) ?? 3000);
+if (
+    !schemas.length ||
+    schemas.some((schema) => !/^[a-z][a-z0-9_]{0,62}$/.test(schema)) ||
+    !Number.isSafeInteger(budgetMs) ||
+    budgetMs < 1 ||
+    budgetMs > 60000 ||
+    args.some((arg) => arg !== '--apply' && !arg.startsWith('--schemas=') && !arg.startsWith('--budget-ms='))
+) {
+    throw new Error('Usage: --schemas=che,kwe,... [--budget-ms=3000] [--apply]');
+}
+let failed = false;
+for (const schema of schemas) {
+    const url = process.env.DATABASE_URL ? new URL(process.env.DATABASE_URL) : new URL('postgresql://localhost');
+    if (!process.env.DATABASE_URL) {
+        url.hostname = process.env.POSTGRES_HOST ?? 'localhost';
+        url.port = process.env.POSTGRES_PORT ?? '5432';
+        url.username = process.env.POSTGRES_USER ?? 'sammo';
+        url.password = process.env.POSTGRES_PASSWORD ?? '';
+        url.pathname = `/${process.env.POSTGRES_DB ?? 'sammo'}`;
+    }
+    url.searchParams.set('schema', schema);
+    const connector = createGamePostgresConnector({ url: url.href, maxConnections: 1 });
+    try {
+        await connector.connect();
+        const db = connector.prisma;
+        const world = await db.worldState.findFirst({ select: { meta: true } });
+        const serverId =
+            world?.meta && typeof world.meta === 'object' && !Array.isArray(world.meta) ? world.meta.serverId : null;
+        if (typeof serverId !== 'string' || !serverId.trim()) {
+            console.log(JSON.stringify({ schema, status: 'identityMissing', deleted: 0 }));
+            continue;
+        }
+        if (!apply) {
+            console.log(JSON.stringify({ schema, status: 'dryRun', apply: false }));
+            continue;
+        }
+        const deadline = performance.now() + budgetMs;
+        let auditDeleted = 0;
+        let outboxDeleted = 0;
+        let status = 'progress';
+        let auditComplete = false;
+        let outboxComplete = false;
+        do {
+            if (!auditComplete) {
+                const result = await prunePreviousAuditBatch(db, serverId);
+                auditDeleted += result.deleted;
+                status = result.status;
+                auditComplete = result.status === 'complete';
+                if (result.status === 'identityChanged' || result.status === 'busy') break;
+            }
+            if (!outboxComplete) {
+                outboxDeleted += await db.$transaction(async (tx) => {
+                    await tx.$executeRaw`SET LOCAL statement_timeout='2000ms'`;
+                    await tx.$executeRaw`SET LOCAL lock_timeout='250ms'`;
+                    const deleted = await pruneDeliveredReadModelOutbox(tx, {
+                        deliveredBefore: new Date(Date.now() - 86400000),
+                        limit: 500,
+                    });
+                    outboxComplete = deleted < 500;
+                    return deleted;
+                });
+            }
+            if (auditComplete && outboxComplete) break;
+            await setTimeout(25);
+        } while (performance.now() < deadline);
+        console.log(
+            JSON.stringify({ schema, status, auditDeleted, outboxDeleted, complete: auditComplete && outboxComplete })
+        );
+    } catch {
+        // 연결 URL과 원문 SQL을 출력하지 않고 scheduler가 감지할 수 있는 종료 코드를 남긴다.
+        failed = true;
+        console.error(JSON.stringify({ schema, status: 'failed' }));
+    } finally {
+        await connector.disconnect();
+    }
+}
+if (failed) process.exitCode = 1;