From 078b9aa2055ac8986f8fbd017593a66f51e19ccf Mon Sep 17 00:00:00 2001 From: hide_d Date: Mon, 9 Nov 2026 20:43:05 +0900 Subject: [PATCH 1/4] =?UTF-8?q?=EA=B0=90=EC=82=AC=20=EC=83=81=EC=84=B8?= =?UTF-8?q?=EB=A5=BC=20=EB=AC=B4=EC=86=90=EC=8B=A4=20=EC=95=95=EC=B6=95?= =?UTF-8?q?=ED=95=98=EA=B3=A0=20=EC=9D=B4=EC=A0=84=20=EA=B8=B0=EC=88=98=20?= =?UTF-8?q?=EC=A0=95=EB=A6=AC=20=EC=B2=98=EB=A6=AC=EB=9F=89=EC=9D=84=20?= =?UTF-8?q?=EA=B0=9C=EC=84=A0=ED=95=9C=EB=8B=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/game-api/src/realtime/outboxWorker.ts | 18 ++-- .../src/router/playAudit/decisions.ts | 4 +- .../npcPolicyLifecycle.integration.test.ts | 17 ++-- .../test/readModelOutboxWorker.test.ts | 22 +++++ .../securityTransport.integration.test.ts | 85 ++++++++--------- app/game-engine/src/index.ts | 1 + .../src/playAudit/decisionPersistence.ts | 6 +- app/game-engine/src/playAudit/retention.ts | 59 +++++++----- .../src/playAudit/retentionWorker.ts | 2 +- .../playAuditDecisionCost.integration.test.ts | 16 +++- ...ditDecisionPersistence.integration.test.ts | 67 +++++++++++++- .../test/playAuditRetentionWorker.test.ts | 2 +- docs/design/play-audit.md | 3 + docs/play-audit-operations.md | 6 +- docs/play-audit-storage.md | 73 +++++++++++++++ .../migration.sql | 18 ++++ packages/infra/src/index.ts | 1 + packages/infra/src/playAuditDecisionCodec.ts | 44 +++++++++ .../infra/src/readModelOutboxDispatcher.ts | 3 +- .../infra/test/playAuditDecisionCodec.test.ts | 54 +++++++++++ release-manifest.json | 2 +- .../scripts/play-audit-npc-lifecycle.ts | 4 +- tools/play-audit-maintenance.mjs | 92 +++++++++++++++++++ 23 files changed, 504 insertions(+), 95 deletions(-) create mode 100644 docs/play-audit-storage.md create mode 100644 packages/infra/prisma/migrations/20261109000000_compact_play_audit_storage/migration.sql create mode 100644 packages/infra/src/playAuditDecisionCodec.ts create mode 100644 packages/infra/test/playAuditDecisionCodec.test.ts create mode 100644 tools/play-audit-maintenance.mjs diff --git a/app/game-api/src/realtime/outboxWorker.ts b/app/game-api/src/realtime/outboxWorker.ts index c54ef73d..a1be796d 100644 --- a/app/game-api/src/realtime/outboxWorker.ts +++ b/app/game-api/src/realtime/outboxWorker.ts @@ -76,7 +76,7 @@ export class ReadModelOutboxWorker implements ReadModelOutboxWakeup { this.leaseMs = normalizePositiveInteger(options.leaseMs, 30_000); this.retentionMs = normalizePositiveInteger(options.retentionMs, 24 * 60 * 60 * 1_000); this.pruneIntervalMs = normalizePositiveInteger(options.pruneIntervalMs, 60_000); - this.pruneLimit = normalizePositiveInteger(options.pruneLimit, 100); + this.pruneLimit = Math.min(500, normalizePositiveInteger(options.pruneLimit, 500)); this.owner = options.owner ?? `game-api:${profileName}:${process.pid}:${randomUUID()}`; this.now = options.now ?? (() => new Date()); this.onError = options.onError ?? (() => undefined); @@ -109,11 +109,17 @@ export class ReadModelOutboxWorker implements ReadModelOutboxWakeup { const now = this.now(); if (now.getTime() >= this.nextPruneAt) { - this.nextPruneAt = now.getTime() + this.pruneIntervalMs; - await pruneDeliveredReadModelOutbox(this.db, { - deliveredBefore: new Date(now.getTime() - this.retentionMs), - limit: this.pruneLimit, - }); + // backlog가 있으면 다음 dispatch에서도 이어서 정리한다. 평상시에는 낮은 빈도로 확인한다. + // 한 호출은 최대 10 batch/50ms이며 각 SQL도 최대 500행이다. + const started = performance.now(); + for (let batch = 0; batch < 10; batch += 1) { + const deleted = await pruneDeliveredReadModelOutbox(this.db, { + deliveredBefore: new Date(now.getTime() - this.retentionMs), + limit: this.pruneLimit, + }); + this.nextPruneAt = deleted < this.pruneLimit ? now.getTime() + this.pruneIntervalMs : 0; + if (deleted < this.pruneLimit || performance.now() - started >= 50) break; + } } return result; } diff --git a/app/game-api/src/router/playAudit/decisions.ts b/app/game-api/src/router/playAudit/decisions.ts index af604ed2..d1202a88 100644 --- a/app/game-api/src/router/playAudit/decisions.ts +++ b/app/game-api/src/router/playAudit/decisions.ts @@ -1,6 +1,6 @@ import { TRPCError } from '@trpc/server'; import { z } from 'zod'; -import type { GamePrisma } from '@sammo-ts/infra'; +import { decodeAuditDecisionSteps, type GamePrisma } from '@sammo-ts/infra'; import { auditProcedure, monthOrdinal, readAudit, readAuditWorld, zAuditMonth } from './shared.js'; const zEffectivePolicy = z.object({ @@ -275,7 +275,7 @@ export const decisionDetail = auditProcedure decision: project(row), chunks: chunks.slice(0, input.limit).map((chunk) => ({ ordinal: chunk.ordinal, - steps: z.array(zStep).max(128).parse(chunk.steps), + steps: z.array(zStep).max(128).parse(decodeAuditDecisionSteps(chunk.steps)), })), nextCursor: chunks.length && chunks.at(-1)!.ordinal + 1 < Math.ceil(row.stepCount / 128) diff --git a/app/game-api/test/npcPolicyLifecycle.integration.test.ts b/app/game-api/test/npcPolicyLifecycle.integration.test.ts index edb24f27..d6a979c3 100644 --- a/app/game-api/test/npcPolicyLifecycle.integration.test.ts +++ b/app/game-api/test/npcPolicyLifecycle.integration.test.ts @@ -2,7 +2,12 @@ import { afterAll, beforeAll, expect, it } from 'vitest'; import fastify from 'fastify'; import { fastifyTRPCPlugin } from '@trpc/server/adapters/fastify'; import { asRecord } from '@sammo-ts/common'; -import { createGamePostgresConnector, createRedisConnector, GamePrisma } from '@sammo-ts/infra'; +import { + decodeAuditDecisionSteps, + createGamePostgresConnector, + createRedisConnector, + GamePrisma, +} from '@sammo-ts/infra'; import { createTurnDaemonRuntime, seedScenarioToDatabase, type TurnDaemonRuntime } from '@sammo-ts/game-engine'; import { appRouter } from '../src/router.js'; import { DatabaseTurnDaemonTransport } from '../src/daemon/databaseTransport.js'; @@ -255,12 +260,10 @@ integration( include: { chunks: { orderBy: { ordinal: 'asc' } } }, }); const procedures = nationDecision.chunks.flatMap((chunk) => - Array.isArray(chunk.steps) - ? chunk.steps - .map(asRecord) - .filter((step) => step.kind === 'PROCEDURE_START') - .map((step) => step.procedure) - : [] + decodeAuditDecisionSteps(chunk.steps) + .map(asRecord) + .filter((step) => step.kind === 'PROCEDURE_START') + .map((step) => step.procedure) ); expect(procedures).toEqual(['NPC포상']); expect(configuration(await call('getPolicy', {}))).toEqual(configuration(withoutRecruitment)); diff --git a/app/game-api/test/readModelOutboxWorker.test.ts b/app/game-api/test/readModelOutboxWorker.test.ts index 60aaf67c..f917a335 100644 --- a/app/game-api/test/readModelOutboxWorker.test.ts +++ b/app/game-api/test/readModelOutboxWorker.test.ts @@ -45,6 +45,28 @@ const createFixture = (rows: readonly object[]) => { }; describe('ReadModelOutboxWorker', () => { + it('drains more than one full retention batch rather than leaving a permanent 100-per-minute backlog', async () => { + let now = new Date('2026-11-09T00:00:00Z'); + const fixture = createFixture([]); + fixture.queryRaw + .mockReset() + .mockResolvedValueOnce([]) + .mockResolvedValueOnce([{ id: 1n }, { id: 2n }]) + .mockResolvedValueOnce([{ id: 3n }, { id: 4n }]) + .mockResolvedValueOnce([]); + const worker = new ReadModelOutboxWorker(fixture.db, fixture.redis, 'che:default', { + intervalMs: 60000, + pruneLimit: 2, + now: () => now, + }); + now = new Date('2026-11-09T00:01:00Z'); + worker.start(); + await vi.waitFor(() => expect(fixture.queryRaw).toHaveBeenCalledTimes(4)); + await worker.stop(); + const query = fixture.queryRaw.mock.calls[1]?.[0] as { sql: string }; + expect(query.sql).toContain('FOR UPDATE SKIP LOCKED'); + expect(query.sql).toContain('ORDER BY "delivered_at", "available_at", "id"'); + }); it('publishes a survey-style global front-status invalidation without a turn daemon', async () => { const fixture = createFixture([{ id: 10n, payload: payload('front.global'), attempts: 1 }]); const worker = new ReadModelOutboxWorker(fixture.db, fixture.redis, 'che:default', { diff --git a/app/game-api/test/securityTransport.integration.test.ts b/app/game-api/test/securityTransport.integration.test.ts index e43ca296..7360a9d7 100644 --- a/app/game-api/test/securityTransport.integration.test.ts +++ b/app/game-api/test/securityTransport.integration.test.ts @@ -13,6 +13,7 @@ import { afterAll, beforeAll, beforeEach, describe, expect, it } from 'vitest'; import { encryptGameSessionToken, type GameSessionTokenPayload } from '@sammo-ts/common/auth/gameToken'; import { createGamePostgresConnector, + encodeAuditDecisionSteps, GamePrisma, hashAuditDiplomacyDocument, createRedisConnector, @@ -2345,50 +2346,52 @@ integration('game API security over HTTP transport', () => { { decisionId: decisionIds[0]!, ordinal: 0, - steps: Array.from({ length: 128 }, (_, sequence) => - sequence === 127 - ? { - ...step, - sequence, - kind: 'DECISION_START', - reservedAction: '휴식', - effectivePolicy: { - schemaVersion: 1, - general: { priority: ['징병'], flags: { 징병: true, 출병: false } }, - nation: { - priority: ['천도'], - flags: { 천도: true }, - values: { - reqNationGold: 4321, - reqNationRice: 100, - reqHumanWarUrgentGold: 100, - reqHumanWarUrgentRice: 100, - reqHumanWarRecommandGold: 100, - reqHumanWarRecommandRice: 100, - reqHumanDevelGold: 100, - reqHumanDevelRice: 100, - reqNpcWarGold: 100, - reqNpcWarRice: 100, - reqNpcDevelGold: 100, - reqNpcDevelRice: 100, - minimumResourceActionAmount: 100, - maximumResourceActionAmount: 100, - minNpcWarLeadership: 100, - minWarCrew: 100, - minNpcRecruitCityPopulation: 100, - safeRecruitCityPopulationRatio: 100, - properWarTrainAtmos: 100, - cureThreshold: 100, + steps: encodeAuditDecisionSteps( + Array.from({ length: 128 }, (_, sequence) => + sequence === 127 + ? { + ...step, + sequence, + kind: 'DECISION_START', + reservedAction: '휴식', + effectivePolicy: { + schemaVersion: 1, + general: { priority: ['징병'], flags: { 징병: true, 출병: false } }, + nation: { + priority: ['천도'], + flags: { 천도: true }, + values: { + reqNationGold: 4321, + reqNationRice: 100, + reqHumanWarUrgentGold: 100, + reqHumanWarUrgentRice: 100, + reqHumanWarRecommandGold: 100, + reqHumanWarRecommandRice: 100, + reqHumanDevelGold: 100, + reqHumanDevelRice: 100, + reqNpcWarGold: 100, + reqNpcWarRice: 100, + reqNpcDevelGold: 100, + reqNpcDevelRice: 100, + minimumResourceActionAmount: 100, + maximumResourceActionAmount: 100, + minNpcWarLeadership: 100, + minWarCrew: 100, + minNpcRecruitCityPopulation: 100, + safeRecruitCityPopulationRatio: 100, + properWarTrainAtmos: 100, + cureThreshold: 100, + }, + combatForce: { '1': [2, 3] }, + supportForce: [4], + developForce: [5], + secret: 'decision-secret', }, - combatForce: { '1': [2, 3] }, - supportForce: [4], - developForce: [5], secret: 'decision-secret', }, - secret: 'decision-secret', - }, - } - : { ...step, sequence } + } + : { ...step, sequence } + ) ), }, { diff --git a/app/game-engine/src/index.ts b/app/game-engine/src/index.ts index 5bf90c79..89da4739 100644 --- a/app/game-engine/src/index.ts +++ b/app/game-engine/src/index.ts @@ -13,6 +13,7 @@ export * from './scenario/databaseUrl.js'; export * from './scenario/mapLoader.js'; export * from './scenario/scenarioSeeder.js'; export * from './scenario/gameCancellation.js'; +export * from './playAudit/retention.js'; export * from './turn/types.js'; export * from './turn/worldLoader.js'; export * from './turn/inMemoryWorld.js'; diff --git a/app/game-engine/src/playAudit/decisionPersistence.ts b/app/game-engine/src/playAudit/decisionPersistence.ts index 26b7f503..e98a2169 100644 --- a/app/game-engine/src/playAudit/decisionPersistence.ts +++ b/app/game-engine/src/playAudit/decisionPersistence.ts @@ -1,4 +1,4 @@ -import type { GamePrisma, InputJsonValue } from '@sammo-ts/infra'; +import { encodeAuditDecisionSteps, type GamePrisma, type InputJsonValue } from '@sammo-ts/infra'; import { auditPolicyHash } from './policy.js'; import type { PendingAuditDecision } from './decision.js'; @@ -50,7 +50,9 @@ export const persistAuditDecisions = async ( chunks.push({ decisionId: decision.id, ordinal: start / AUDIT_DECISION_CHUNK_STEPS, - steps: decision.steps.slice(start, start + AUDIT_DECISION_CHUNK_STEPS) as InputJsonValue, + steps: encodeAuditDecisionSteps( + decision.steps.slice(start, start + AUDIT_DECISION_CHUNK_STEPS) as InputJsonValue[] + ), }); if (chunks.length === BATCH) { await tx.playAuditDecisionChunk.createMany({ data: chunks, skipDuplicates: true }); diff --git a/app/game-engine/src/playAudit/retention.ts b/app/game-engine/src/playAudit/retention.ts index 92149c71..729aa8ed 100644 --- a/app/game-engine/src/playAudit/retention.ts +++ b/app/game-engine/src/playAudit/retention.ts @@ -1,5 +1,5 @@ import { asRecord } from '@sammo-ts/common'; -import type { GamePrismaClient } from '@sammo-ts/infra'; +import { GamePrisma, type GamePrismaClient } from '@sammo-ts/infra'; export const AUDIT_RETENTION_BATCH_SIZE = 200; export type AuditRetentionResult = { status: 'progress' | 'complete' | 'busy' | 'identityChanged'; deleted: number }; @@ -13,6 +13,7 @@ export const prunePreviousAuditBatch = async ( return db.$transaction( async (tx) => { await tx.$executeRaw`SET LOCAL statement_timeout = '2000ms'`; + await tx.$executeRaw`SET LOCAL lock_timeout = '250ms'`; // seeder와 동일한 잠금이다. RESET을 기다리게 하지 않고 다음 batch에서 재시도한다. const [lock] = await tx.$queryRaw<{ locked: boolean }[]>` SELECT pg_try_advisory_xact_lock(hashtextextended(current_schema(), 0)) AS locked @@ -32,29 +33,43 @@ export const prunePreviousAuditBatch = async ( }); return { status: 'progress', deleted: deleted.count }; } - const [decision] = await tx.$queryRaw<{ id: string }[]>` - SELECT id FROM play_audit_decision WHERE id = COALESCE( - (SELECT id FROM play_audit_decision WHERE server_id < ${expectedServerId} - ORDER BY server_id, general_id, tick, id LIMIT 1), - (SELECT id FROM play_audit_decision WHERE server_id > ${expectedServerId} - ORDER BY server_id, general_id, tick, id LIMIT 1) - ) FOR UPDATE + // 이미 있는 (server_id,nation_id,tick,id) 인덱스 순서로 여러 부모를 잠근다. + // chunk 총량과 빈 header 삭제량의 합도 BATCH 이하로 제한한다. + let decisions = await tx.$queryRaw<{ id: string }[]>` + SELECT id FROM play_audit_decision WHERE server_id < ${expectedServerId} + ORDER BY server_id, nation_id, tick, id LIMIT ${AUDIT_RETENTION_BATCH_SIZE} FOR UPDATE `; - if (decision) { - const chunks = await tx.playAuditDecisionChunk.findMany({ - where: { decisionId: decision.id }, - orderBy: { ordinal: 'asc' }, - take: AUDIT_RETENTION_BATCH_SIZE, - select: { ordinal: true }, + if (!decisions.length) { + decisions = await tx.$queryRaw<{ id: string }[]>` + SELECT id FROM play_audit_decision WHERE server_id > ${expectedServerId} + ORDER BY server_id, nation_id, tick, id LIMIT ${AUDIT_RETENTION_BATCH_SIZE} FOR UPDATE + `; + } + if (decisions.length) { + const ids = decisions.map(({ id }) => id); + const [chunks] = await tx.$queryRaw<{ deleted: number }[]>(GamePrisma.sql` + WITH batch AS ( + SELECT decision_id, ordinal FROM play_audit_decision_chunk + WHERE decision_id IN (${GamePrisma.join(ids)}) + ORDER BY decision_id, ordinal LIMIT ${AUDIT_RETENTION_BATCH_SIZE} + ), removed AS ( + DELETE FROM play_audit_decision_chunk c USING batch b + WHERE c.decision_id=b.decision_id AND c.ordinal=b.ordinal RETURNING 1 + ) SELECT count(*)::integer AS deleted FROM removed + `); + const count = chunks?.deleted ?? 0; + const remaining = AUDIT_RETENTION_BATCH_SIZE - count; + if (!remaining) return { status: 'progress', deleted: count }; + const empty = await tx.playAuditDecision.findMany({ + where: { id: { in: ids }, chunks: { none: {} } }, + select: { id: true }, + orderBy: { id: 'asc' }, + take: remaining, }); - if (chunks.length) { - const deleted = await tx.playAuditDecisionChunk.deleteMany({ - where: { decisionId: decision.id, ordinal: { in: chunks.map((row) => row.ordinal) } }, - }); - return { status: 'progress', deleted: deleted.count }; - } - await tx.playAuditDecision.delete({ where: { id: decision.id } }); - return { status: 'progress', deleted: 1 }; + const headers = await tx.playAuditDecision.deleteMany({ + where: { id: { in: empty.map(({ id }) => id) } }, + }); + return { status: 'progress', deleted: count + headers.count }; } const policies = await tx.playAuditPolicy.findMany({ where: { serverId: { not: expectedServerId } }, diff --git a/app/game-engine/src/playAudit/retentionWorker.ts b/app/game-engine/src/playAudit/retentionWorker.ts index 4c8a8670..2b030db4 100644 --- a/app/game-engine/src/playAudit/retentionWorker.ts +++ b/app/game-engine/src/playAudit/retentionWorker.ts @@ -17,7 +17,7 @@ export const startAuditRetentionWorker = (options: { timer.unref(); }; const run = async () => { - let delay = 1000; + let delay = 100; try { const result = await options.prune(); if (result.status === 'complete' || result.status === 'identityChanged') stopped = true; diff --git a/app/game-engine/test/playAuditDecisionCost.integration.test.ts b/app/game-engine/test/playAuditDecisionCost.integration.test.ts index 91026408..6472c959 100644 --- a/app/game-engine/test/playAuditDecisionCost.integration.test.ts +++ b/app/game-engine/test/playAuditDecisionCost.integration.test.ts @@ -1,7 +1,12 @@ import { writeFile } from 'node:fs/promises'; import { performance } from 'node:perf_hooks'; import { afterAll, beforeAll, describe, expect, it } from 'vitest'; -import { createGamePostgresConnector, type GamePrisma, type GamePrismaClient } from '@sammo-ts/infra'; +import { + decodeAuditDecisionSteps, + createGamePostgresConnector, + type GamePrisma, + type GamePrismaClient, +} from '@sammo-ts/infra'; import { persistAuditDecisions } from '../src/playAudit/decisionPersistence.js'; import { buildAuditDecisionFixture } from './fixtures/playAuditDecision.js'; @@ -61,11 +66,15 @@ describe.skipIf(!databaseUrl)('decision storage cost probe', () => { >` SELECT (SELECT count(*) FROM play_audit_decision) AS headers, - (SELECT sum(jsonb_array_length(steps)) FROM play_audit_decision_chunk) AS steps, + (SELECT sum(step_count) FROM play_audit_decision) AS steps, (SELECT sum(pg_column_size(d)) FROM play_audit_decision d) AS "headerBytes", (SELECT sum(pg_column_size(c)) FROM play_audit_decision_chunk c) AS "chunkBytes" `; expect(Number(storage!.steps)).toBe(201 * 302); + const chunks = await db.playAuditDecisionChunk.findMany({ select: { steps: true } }); + expect(chunks.reduce((total, chunk) => total + decodeAuditDecisionSteps(chunk.steps).length, 0)).toBe( + 201 * 302 + ); const relations = await db.$queryRaw< { name: string; tableBytes: bigint; indexBytes: bigint; totalBytes: bigint }[] >` @@ -99,7 +108,8 @@ describe.skipIf(!databaseUrl)('decision storage cost probe', () => { `; const metrics = { scope: 'synthetic 201 decisions, 302 steps each; isolated schema, warm local reads', - limitations: 'Not production p95; no gameplay baseline, WAL or retained heap measurement. SQL counts cover only the persistence transaction. Repetitive synthetic steps compress well. Relation allocation can retain space from previous runs.', + limitations: + 'Not production p95; no gameplay baseline, WAL or retained heap measurement. SQL counts cover only the persistence transaction. Repetitive synthetic steps compress well. Relation allocation can retain space from previous runs.', decisions: 201, steps: Number(storage!.steps), chunks: 603, diff --git a/app/game-engine/test/playAuditDecisionPersistence.integration.test.ts b/app/game-engine/test/playAuditDecisionPersistence.integration.test.ts index 81d93e2a..8c23af7b 100644 --- a/app/game-engine/test/playAuditDecisionPersistence.integration.test.ts +++ b/app/game-engine/test/playAuditDecisionPersistence.integration.test.ts @@ -1,5 +1,5 @@ import { afterAll, beforeAll, describe, expect, it } from 'vitest'; -import { createGamePostgresConnector, type GamePrismaClient } from '@sammo-ts/infra'; +import { decodeAuditDecisionSteps, createGamePostgresConnector, type GamePrismaClient } from '@sammo-ts/infra'; import { persistAuditDecisions } from '../src/playAudit/decisionPersistence.js'; import { buildAuditExecutionFixture, buildAuditDecisionFixture as draft } from './fixtures/playAuditDecision.js'; import { prunePreviousAuditBatch } from '../src/playAudit/retention.js'; @@ -36,7 +36,40 @@ integration('decision persistence and bounded retention', () => { }); it('rolls back gameplay/header/chunks on insert failure, retries and rejects divergent replay', async () => { const decision = draft('decision-one'); - if (decision.steps[0]?.kind === 'DECISION_START') decision.steps[0].effectivePolicy = {"schemaVersion":1,"general":{"priority":["징병"],"flags":{"징병":true,"출병":false}},"nation":{"priority":["천도"],"flags":{"천도":true},"values":{"reqNationGold":4321,"reqNationRice":100,"reqHumanWarUrgentGold":100,"reqHumanWarUrgentRice":100,"reqHumanWarRecommandGold":100,"reqHumanWarRecommandRice":100,"reqHumanDevelGold":100,"reqHumanDevelRice":100,"reqNpcWarGold":100,"reqNpcWarRice":100,"reqNpcDevelGold":100,"reqNpcDevelRice":100,"minimumResourceActionAmount":100,"maximumResourceActionAmount":100,"minNpcWarLeadership":100,"minWarCrew":100,"minNpcRecruitCityPopulation":100,"safeRecruitCityPopulationRatio":100,"properWarTrainAtmos":100,"cureThreshold":100},"combatForce":{"1":[2,3]},"supportForce":[4],"developForce":[5]}}; + if (decision.steps[0]?.kind === 'DECISION_START') + decision.steps[0].effectivePolicy = { + schemaVersion: 1, + general: { priority: ['징병'], flags: { 징병: true, 출병: false } }, + nation: { + priority: ['천도'], + flags: { 천도: true }, + values: { + reqNationGold: 4321, + reqNationRice: 100, + reqHumanWarUrgentGold: 100, + reqHumanWarUrgentRice: 100, + reqHumanWarRecommandGold: 100, + reqHumanWarRecommandRice: 100, + reqHumanDevelGold: 100, + reqHumanDevelRice: 100, + reqNpcWarGold: 100, + reqNpcWarRice: 100, + reqNpcDevelGold: 100, + reqNpcDevelRice: 100, + minimumResourceActionAmount: 100, + maximumResourceActionAmount: 100, + minNpcWarLeadership: 100, + minWarCrew: 100, + minNpcRecruitCityPopulation: 100, + safeRecruitCityPopulationRatio: 100, + properWarTrainAtmos: 100, + cureThreshold: 100, + }, + combatForce: { '1': [2, 3] }, + supportForce: [4], + developForce: [5], + }, + }; decision.summary.codeVersion = 'a'.repeat(40); decision.summary.executionCoverage = 'ATTEMPTS'; decision.steps.push({ ...decision.steps[0]!, ...buildAuditExecutionFixture(), sequence: 302 }); @@ -85,7 +118,7 @@ integration('decision persistence and bounded retention', () => { orderBy: { ordinal: 'asc' }, }); expect(chunks).toHaveLength(3); - expect(chunks.flatMap((chunk) => chunk.steps)).toEqual(decision.steps); + expect(chunks.flatMap((chunk) => decodeAuditDecisionSteps(chunk.steps))).toEqual(decision.steps); await expect( db.$transaction((tx) => persistAuditDecisions(tx, [{ ...decision, summary: { ...decision.summary, completed: true } }]) @@ -109,10 +142,34 @@ integration('decision persistence and bounded retention', () => { expect(await db.playAuditDecisionChunk.count({ where: { decisionId: 'decision-one' } })).toBe(204); expect(await db.playAuditDecision.count({ where: { id: 'decision-one' } })).toBe(1); expect(await prunePreviousAuditBatch(db, 'decision-current')).toEqual({ status: 'progress', deleted: 200 }); - expect(await prunePreviousAuditBatch(db, 'decision-current')).toEqual({ status: 'progress', deleted: 4 }); - expect(await prunePreviousAuditBatch(db, 'decision-current')).toEqual({ status: 'progress', deleted: 1 }); + expect(await prunePreviousAuditBatch(db, 'decision-current')).toEqual({ status: 'progress', deleted: 5 }); expect(await prunePreviousAuditBatch(db, 'decision-current')).toEqual({ status: 'complete', deleted: 0 }); expect(await db.playAuditDecision.count()).toBe(1); expect(await db.playAuditDecisionChunk.count()).toBe(3); }); + it('removes many small decisions in one bounded batch and rolls chunks back on header failure', async () => { + const decisions = Array.from({ length: 100 }, (_, index) => { + const row = draft(`batch-old-${index}`, 'batch-old'); + return { ...row, steps: [row.steps[0]!, row.steps.at(-1)!] }; + }); + await db.$transaction((tx) => persistAuditDecisions(tx, decisions)); + await db.$executeRawUnsafe( + `CREATE OR REPLACE FUNCTION decision_header_failure() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN RAISE EXCEPTION 'header fixture rollback'; END $$` + ); + await db.$executeRawUnsafe( + `CREATE TRIGGER decision_header_failure BEFORE DELETE ON play_audit_decision FOR EACH ROW EXECUTE FUNCTION decision_header_failure()` + ); + try { + await expect(prunePreviousAuditBatch(db, 'decision-current')).rejects.toThrow(); + expect( + await db.playAuditDecisionChunk.count({ where: { decisionId: { in: decisions.map(({ id }) => id) } } }) + ).toBe(100); + } finally { + await db.$executeRawUnsafe('DROP TRIGGER decision_header_failure ON play_audit_decision'); + await db.$executeRawUnsafe('DROP FUNCTION decision_header_failure()'); + } + expect(await prunePreviousAuditBatch(db, 'decision-current')).toEqual({ status: 'progress', deleted: 200 }); + expect(await db.playAuditDecision.count()).toBe(1); + expect(await prunePreviousAuditBatch(db, 'decision-current')).toEqual({ status: 'complete', deleted: 0 }); + }); }); diff --git a/app/game-engine/test/playAuditRetentionWorker.test.ts b/app/game-engine/test/playAuditRetentionWorker.test.ts index 83c762de..039f5cfb 100644 --- a/app/game-engine/test/playAuditRetentionWorker.test.ts +++ b/app/game-engine/test/playAuditRetentionWorker.test.ts @@ -20,7 +20,7 @@ describe('audit retention scheduling', () => { expect(prune).toHaveBeenCalledTimes(1); await vi.advanceTimersByTimeAsync(30_001); expect(prune).toHaveBeenCalledTimes(3); - await vi.advanceTimersByTimeAsync(1000); + await vi.advanceTimersByTimeAsync(100); expect(prune).toHaveBeenCalledTimes(4); await vi.advanceTimersByTimeAsync(600_000); expect(prune).toHaveBeenCalledTimes(4); diff --git a/docs/design/play-audit.md b/docs/design/play-audit.md index 7e251c15..f5001f4e 100644 --- a/docs/design/play-audit.md +++ b/docs/design/play-audit.md @@ -2,6 +2,9 @@ ## 문서 상태와 사용법 +2026-11-09 저장 최적화: [물리 압축·정리 처리량·ZFS 운영 경계](../play-audit-storage.md). +판단 원문과 현재 기수 보존 계약을 유지하며 저장 표현과 이전 기수 정리 구현을 변경한다. + 이 문서는 플레이 감사의 구조·현재 구현 경계·남은 요구사항을 관리하는 기준 문서다. 사용 방법과 DB 적용은 [운영 안내](../play-audit-operations.md)를 따른다. 날짜별 구현·측정 기록은 Git 이력과 상위 작업공간의 `report/`에서 확인한다. diff --git a/docs/play-audit-operations.md b/docs/play-audit-operations.md index b4233f8c..816caff1 100644 --- a/docs/play-audit-operations.md +++ b/docs/play-audit-operations.md @@ -5,6 +5,9 @@ 관리한다. 아래 적용 절차는 각 프로필의 DB 보존 업데이트를 위한 것이며, 문서에 기능이 기재되어 있다고 운영 DB에 적용된 것은 아니다. +저장량 최적화, 중지된 프로필 정리, ZFS 이전과 rollback은 +[저장량과 운영 정리](play-audit-storage.md)를 따른다. + ## 지금 사용할 수 있는 기능 | 화면 | 사용할 수 있는 정보 | 읽을 때 주의할 점 | @@ -55,7 +58,7 @@ ## DB migration과 적용 순서 정식 game migration에 감사 테이블과 인덱스가 포함되어 있다. `prisma db push`나 -수동 CREATE TABLE로 대신 적용하지 않는다. 현재 game chain은58개이며 다음 감사 +수동 CREATE TABLE로 대신 적용하지 않는다. 현재 game chain은59개이며 다음 감사 migration들을 포함한다. 기존 기록을 삭제하거나 지난달 상세를 역산하지 않는다. | migration | 준비되는 저장소/제약 | @@ -69,6 +72,7 @@ migration들을 포함한다. 기존 기록을 삭제하거나 지난달 상세 | `20260916060000_widen_play_audit_ticks` | 월 표본/정책 tick을 BIGINT로 확장하여 약60개월 이후 INTEGER 초과 방지 | | `20260916070000_add_play_audit_decision` | NPC/자동턴 결정 요약과 순서별 상세 chunk, bounded 정리용 FK/index | | `20260916080000_index_play_audit_decision_month` | 기존 장수 인덱스를 월 조건 포함 인덱스로 교체하여 기수 전체 조회 방지 | +| `20261109000000_compact_play_audit_storage` | 무손실 압축 chunk 형식과 작은 JSON inline 압축 설정 | 운영은 [릴리스 절차](release-operations.md)의 **DB 보존 버전 업데이트**로 해당 고정 commit을 적용한다. 이 기능을 켜기 위해 시나리오를 초기화할 필요는 없다. 수동 환경의 diff --git a/docs/play-audit-storage.md b/docs/play-audit-storage.md new file mode 100644 index 00000000..780239ac --- /dev/null +++ b/docs/play-audit-storage.md @@ -0,0 +1,73 @@ +# 플레이 감사 저장량과 운영 정리 + +2026-11-09 저장 최적화는 Core 전용 감사의 물리 표현과 정리 구현을 바꾼다. +현재 기수 전체 기록·판단 순서·정책 참조·hash·API 응답·권한은 유지한다. +원문 샘플링이나 보존기간 축소를 하지 않는다. + +## 물리 표현 + +- 결정 chunk는 기존 `steps` JSONB 배열 또는 `{encoding:"gzip-json-v1",data:"base64"}`로 저장한다. + 128 step 단위는 그대로이며 gzip level 3을 사용한다. 압축 envelope가 더 크면 원래 배열을 쓴다. +- replay hash는 압축 전 원문으로 계산한다. API는 공용 decoder 뒤 기존 step schema를 검증한다. + 손상·알 수 없는 버전은 조회 오류이며 빈 자료로 바꾸지 않는다. +- 압축 해제는 chunk당 16MiB로 제한한다. 더 큰 원문은 기존 배열로 저장해 잘라내지 않는다. +- 장수 snapshot `data`와 결정 `summary`는 PostgreSQL `STORAGE MAIN`, + `toast_tuple_target=512`로 작은 반복 JSON도 inline 압축한다. JSON 값은 바뀌지 않는다. +- migration `20261109000000_compact_play_audit_storage`는 기존 테이블을 재작성하지 않는다. + chunk CHECK는 새 쓰기에 즉시 적용하고 대형 과거 자료 검사는 `NOT VALID`로 미룬다. + 정리 후 `ALTER TABLE ... VALIDATE CONSTRAINT play_audit_decision_chunk_steps_check`를 실행한다. + 기존 자료의 압축 변경은 별도 bounded backfill 또는 검증된 export/restore가 필요하다. + +## 정리 처리량과 실행 수명주기 + +- 이전 기수 결정 부모를 기존 `(server_id,nation_id,tick,id)` 인덱스로 최대 200개 잠근다. + 한 transaction의 chunk+header 삭제 합도 200행 이하이며 큰 결정 하나가 있어도 상한을 지킨다. +- 다음 성공 batch까지 100ms를 기다린다. RESET과 동일 advisory lock, identity 재검사, + 2초 statement/250ms lock 제한, rollback을 유지한다. 현재 기수의 결정과 표본은 삭제하지 않는다. +- API outbox는 전달 후 24시간을 보존한다. 기존 인덱스 순서로 최대 500행씩 정리하고, + backlog가 있으면 다음 dispatch에서도 이어간다. 한 호출은 최대 10 batch/50ms이며 + 미전달 자료는 삭제하지 않는다. 빈 상태에서는 60초 주기로 돌아간다. +- game API/daemon이 중지된 프로필도 host scheduler가 다음 CLI로 정리한다. + CLI는 접속 URL이나 SQL 원문을 출력하지 않는다. 기본은 dry-run이다. + +```sh +# 먼저 common/logic/infra/game-engine을 빌드한다. 접속 정보는 기존 process 환경으로 전달한다. +node tools/play-audit-maintenance.mjs --schemas=che,kwe,pwe,twe,nya,pya,hwe +node tools/play-audit-maintenance.mjs --schemas=che,kwe,pwe,twe,nya,pya,hwe --budget-ms=3000 --apply +``` + +schema마다 최신 identity를 읽고 각 삭제 transaction에서 다시 확인한다. 예상 identity가 바뀌면 +그 schema의 처리를 중단한다. host timer는 다음 실행에서 새 identity를 읽는다. +장수/국가/명령/input_event 같은 gameplay 테이블은 이 CLI의 삭제 대상이 아니다. + +## ZFS 운영 경계 + +호스트는 감사 전용 tablespace와 backup 전용 dataset을 별도로 제공한다. +hot gameplay, WAL, legacy archive를 임의로 이동하지 않는다. DB와 tablespace는 한 cluster이므로 +tablespace 파일만 백업하는 것으로 복구를 보장하지 않는다. + +1. 전체 DB의 일관된 압축 backup을 ZFS에 만들고 checksum, archive 읽기, 실제 복원을 확인한다. +2. DB 보존 DEPLOY로 writer/reader/migration을 함께 적용한다. +3. profile 상태를 기록하고 공식 STOP으로 쓰기를 중단한다. 원래 STOPPED인 profile은 유지한다. +4. 각 schema에서 현재 serverId의 8개 감사 테이블을 일관되게 export한다. + 월 child는 현재 월 header와 join하고 결정 chunk는 현재 결정 header와 join한다. +5. 격리 DB에서 복원·건수·내용 hash를 검증한다. 이전 기수는 전체 ZFS backup에 남는다. +6. 하나의 transaction에서 감사 테이블만 TRUNCATE하고 ZFS tablespace로 옮긴 뒤 + 현재 기수 export를 다시 넣는다. 부모를 먼저 넣고 모든 건수/hash와 identity를 확인한 뒤 commit한다. + index도 별도로 ZFS로 옮긴다. 실패하면 전체 transaction을 rollback한다. +7. CHECK 검증과 ANALYZE를 수행하고 원래 RUNNING이었던 profile만 공식 START한다. + 현재 기록, 새 기록의 decode/hash, lease/clock, 공개 HTTPS와 루트/ZFS 실제 사용량을 확인한다. + +기존 데이터가 적을 때는 bounded DELETE 뒤 별도 재작성으로 공간을 회수할 수도 있다. +수천만 건 backlog에서 export/TRUNCATE/restore는 현재 자료를 보존하며 물리 공간을 즉시 +회수하기 위한 계획된 유지보수 절차다. backup 검증이나 profile 정지가 빠진 자동 작업으로 쓰지 않는다. +ZFS dataset이 없을 때 루트 디렉터리에 쓰지 않도록 host mount guard와 재현 가능한 Compose 설정이 필요하다. + +## rollback + +이전 reader는 압축 envelope를 이해하지 못한다. 구형 버전으로 돌아가기 전 해당 기록을 +새 decoder로 원래 JSON 배열로 변환하고 같은 hash/개수를 검증한다. 물리 압축 설정은 +이전 애플리케이션과 호환되며 원래 TOAST 설정으로 돌릴 필요가 없다. +현재 기수 복원 중 실패하면 transaction rollback으로 기존 table/data/tablespace를 보존한다. +기수 전체 과거 자료가 필요하면 ZFS의 변경 전 backup을 별도 DB에 복원한다. +운영 DB 전체를 과거 dump로 덮어써 새 gameplay를 잃게 하지 않는다. diff --git a/packages/infra/prisma/migrations/20261109000000_compact_play_audit_storage/migration.sql b/packages/infra/prisma/migrations/20261109000000_compact_play_audit_storage/migration.sql new file mode 100644 index 00000000..3a8fb5fa --- /dev/null +++ b/packages/infra/prisma/migrations/20261109000000_compact_play_audit_storage/migration.sql @@ -0,0 +1,18 @@ +-- 대형 기존 기록을 migration 중 재작성/전체 검사하지 않는다. 이전 배열과 새 압축 형식을 함께 허용한다. +BEGIN; +SET LOCAL lock_timeout = '2s'; +ALTER TABLE "play_audit_decision_chunk" DROP CONSTRAINT "play_audit_decision_chunk_steps_check"; +ALTER TABLE "play_audit_decision_chunk" ADD CONSTRAINT "play_audit_decision_chunk_steps_check" + CHECK (COALESCE( + jsonb_typeof("steps") = 'array' OR ( + jsonb_typeof("steps") = 'object' + AND "steps"->>'encoding' = 'gzip-json-v1' + AND jsonb_typeof("steps"->'data') = 'string' + ), false + )) NOT VALID; +-- 기본 약 2KiB 문턱 아래의 반복 JSON도 압축하되 작은 snapshot은 TOAST 밖으로 쪼개지 않는다. +ALTER TABLE "play_audit_general" ALTER COLUMN "data" SET STORAGE MAIN; +ALTER TABLE "play_audit_general" SET (toast_tuple_target = 512); +ALTER TABLE "play_audit_decision" ALTER COLUMN "summary" SET STORAGE MAIN; +ALTER TABLE "play_audit_decision" SET (toast_tuple_target = 512); +COMMIT; diff --git a/packages/infra/src/index.ts b/packages/infra/src/index.ts index 8567c91b..ea155cb0 100644 --- a/packages/infra/src/index.ts +++ b/packages/infra/src/index.ts @@ -15,3 +15,4 @@ export * from './playAuditDiplomacy.js'; export * from './messageEnvelope.js'; export * from './webPushOutbox.js'; export * from './playAuditBestEffort.js'; +export * from './playAuditDecisionCodec.js'; diff --git a/packages/infra/src/playAuditDecisionCodec.ts b/packages/infra/src/playAuditDecisionCodec.ts new file mode 100644 index 00000000..143c85a1 --- /dev/null +++ b/packages/infra/src/playAuditDecisionCodec.ts @@ -0,0 +1,44 @@ +import { gzipSync, gunzipSync } from 'node:zlib'; +import type { InputJsonValue } from './turnEngineDb.js'; + +const ENCODING = 'gzip-json-v1'; +const MAX_STEPS = 128; +const MAX_BYTES = 16 * 1024 * 1024; + +/** 조회/재실행 hash에 쓰는 원문은 바꾸지 않고 저장할 때만 무손실 압축한다. */ +export const encodeAuditDecisionSteps = (steps: readonly InputJsonValue[]): InputJsonValue => { + if (steps.length > MAX_STEPS) throw new Error('Audit decision chunk exceeds its step limit'); + const json = JSON.stringify(steps); + // 드문 초대형 기존 payload도 버리거나 자르지 않는다. 압축 reader 상한 밖이면 원래 형식으로 저장한다. + if (Buffer.byteLength(json) > MAX_BYTES) return steps.slice(); + const encoded = { encoding: ENCODING, data: gzipSync(json, { level: 3 }).toString('base64') }; + return Buffer.byteLength(JSON.stringify(encoded)) < Buffer.byteLength(json) ? encoded : steps.slice(); +}; + +/** 이전 JSONB 배열과 압축 형식을 함께 읽는다. 손상/알 수 없는 버전은 빈 기록으로 숨기지 않는다. */ +export const decodeAuditDecisionSteps = (value: unknown): unknown[] => { + if (Array.isArray(value)) { + if (value.length > MAX_STEPS) throw new Error('Audit decision chunk exceeds its step limit'); + return value; + } + if (!value || typeof value !== 'object' || !('encoding' in value) || !('data' in value)) + throw new Error('Invalid audit decision storage'); + if ( + value.encoding !== ENCODING || + typeof value.data !== 'string' || + value.data.length > Math.ceil(MAX_BYTES / 3) * 4 + 256 || + !/^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/.test(value.data) + ) + throw new Error('Invalid audit decision encoding'); + let decoded: unknown; + try { + decoded = JSON.parse( + gunzipSync(Buffer.from(value.data, 'base64'), { maxOutputLength: MAX_BYTES }).toString('utf8') + ); + } catch { + // DB 원문이나 압축 payload가 오류 응답/로그에 포함되지 않게 한다. + throw new Error('Corrupt audit decision storage'); + } + if (!Array.isArray(decoded) || decoded.length > MAX_STEPS) throw new Error('Invalid audit decision steps'); + return decoded; +}; diff --git a/packages/infra/src/readModelOutboxDispatcher.ts b/packages/infra/src/readModelOutboxDispatcher.ts index 285dd0e9..474f10f1 100644 --- a/packages/infra/src/readModelOutboxDispatcher.ts +++ b/packages/infra/src/readModelOutboxDispatcher.ts @@ -195,8 +195,9 @@ export const pruneDeliveredReadModelOutbox = async ( SELECT "id" FROM "read_model_outbox" WHERE "delivered_at" < ${input.deliveredBefore} - ORDER BY "id" + ORDER BY "delivered_at", "available_at", "id" LIMIT ${limit} + FOR UPDATE SKIP LOCKED ) RETURNING "id" `); diff --git a/packages/infra/test/playAuditDecisionCodec.test.ts b/packages/infra/test/playAuditDecisionCodec.test.ts new file mode 100644 index 00000000..bf1170d6 --- /dev/null +++ b/packages/infra/test/playAuditDecisionCodec.test.ts @@ -0,0 +1,54 @@ +import { gzipSync } from 'node:zlib'; +import { describe, expect, it } from 'vitest'; +import { decodeAuditDecisionSteps, encodeAuditDecisionSteps } from '../src/playAuditDecisionCodec.js'; + +describe('lossless audit decision storage', () => { + it('compresses repeated context while preserving every value and its order', () => { + const steps = Array.from({ length: 128 }, (_, sequence) => ({ + sequence, + phase: 'general', + generalId: 51, + nationId: 3, + cityId: 12, + year: 190, + month: 7, + kind: 'RNG', + value: sequence / 128, + text: '판단·줄바꿈\n따옴표 " 그대로', + empty: null, + })); + const stored = encodeAuditDecisionSteps(steps); + expect(Array.isArray(stored)).toBe(false); + expect(Buffer.byteLength(JSON.stringify(stored))).toBeLessThan(Buffer.byteLength(JSON.stringify(steps)) / 3); + expect(decodeAuditDecisionSteps(stored)).toEqual(steps); + }); + it('keeps tiny and historical uncompressed arrays readable', () => { + expect(encodeAuditDecisionSteps([])).toEqual([]); + expect(decodeAuditDecisionSteps([{ marker: 'legacy', sequence: 0 }])).toEqual([ + { marker: 'legacy', sequence: 0 }, + ]); + }); + it('rejects unknown versions, bad base64, truncated gzip and invalid contents', () => { + for (const value of [ + null, + {}, + { encoding: 'gzip-json-v2', data: '' }, + { encoding: 'gzip-json-v1', data: '%%%' }, + { encoding: 'gzip-json-v1', data: 'YQ==' }, + { encoding: 'gzip-json-v1', data: gzipSync('{}').toString('base64') }, + { encoding: 'gzip-json-v1', data: gzipSync(JSON.stringify(Array(129).fill(null))).toString('base64') }, + ]) + expect(() => decodeAuditDecisionSteps(value)).toThrow(); + expect(() => encodeAuditDecisionSteps(Array(129).fill(null))).toThrow(); + }); + it('bounds decompression and preserves oversized raw records rather than truncating them', () => { + const steps = [{ text: 'x'.repeat(16 * 1024 * 1024) }]; + expect(encodeAuditDecisionSteps(steps)).toEqual(steps); + expect(() => + decodeAuditDecisionSteps({ + encoding: 'gzip-json-v1', + data: gzipSync(JSON.stringify(steps)).toString('base64'), + }) + ).toThrow('Corrupt'); + }); +}); diff --git a/release-manifest.json b/release-manifest.json index e05c2b48..1d6f3b25 100644 --- a/release-manifest.json +++ b/release-manifest.json @@ -2,6 +2,6 @@ "formatVersion": 1, "controllerProtocol": 2, "gatewaySchemaHead": "20261022000000_add_open_requests", - "gameSchemaHead": "20260916080000_index_play_audit_decision_month", + "gameSchemaHead": "20261109000000_compact_play_audit_storage", "components": ["gateway-api", "gateway-frontend", "release-controller", "game-api", "game-engine", "game-frontend"] } diff --git a/tools/integration-tests/scripts/play-audit-npc-lifecycle.ts b/tools/integration-tests/scripts/play-audit-npc-lifecycle.ts index f724cb4c..5152862f 100644 --- a/tools/integration-tests/scripts/play-audit-npc-lifecycle.ts +++ b/tools/integration-tests/scripts/play-audit-npc-lifecycle.ts @@ -3,7 +3,7 @@ import assert from 'node:assert/strict'; import { mkdir, readFile, writeFile } from 'node:fs/promises'; import { resolve } from 'node:path'; import { asRecord, type TurnCheckpoint } from '@sammo-ts/common'; -import { createGamePostgresConnector } from '@sammo-ts/infra'; +import { createGamePostgresConnector, decodeAuditDecisionSteps } from '@sammo-ts/infra'; import { createTurnDaemonRuntime, seedScenarioToDatabase } from '@sammo-ts/game-engine'; const databaseUrl = process.env.DATABASE_URL; @@ -39,7 +39,7 @@ const verify = async (progress: unknown[]) => { include: { chunks: { orderBy: { ordinal: 'asc' } } }, }); assert(row && row.chunks.length, `Missing actual NPC ${JSON.stringify(selection)} decision`); - const steps = row.chunks.flatMap((chunk) => (Array.isArray(chunk.steps) ? chunk.steps : [])); + const steps = row.chunks.flatMap((chunk) => decodeAuditDecisionSteps(chunk.steps)); assert.equal(steps.length, row.stepCount); assert.equal(asRecord(steps[0]).kind, 'DECISION_START'); assert(steps.some((step) => asRecord(step).kind === 'DECISION_END')); diff --git a/tools/play-audit-maintenance.mjs b/tools/play-audit-maintenance.mjs new file mode 100644 index 00000000..20cfceca --- /dev/null +++ b/tools/play-audit-maintenance.mjs @@ -0,0 +1,92 @@ +#!/usr/bin/env node +// stopped/reserved profile도 같은 fenced batch로 정리한다. 기본은 관찰만 하며 외부 scheduler가 실행한다. +import { performance } from 'node:perf_hooks'; +import { setTimeout } from 'node:timers/promises'; +import { createGamePostgresConnector, pruneDeliveredReadModelOutbox } from '../packages/infra/dist/index.js'; +import { prunePreviousAuditBatch } from '../app/game-engine/dist/index.js'; + +const args = process.argv.slice(2); +const apply = args.includes('--apply'); +const schemas = + args + .find((arg) => arg.startsWith('--schemas=')) + ?.slice(10) + .split(',') ?? []; +const budgetMs = Number(args.find((arg) => arg.startsWith('--budget-ms='))?.slice(12) ?? 3000); +if ( + !schemas.length || + schemas.some((schema) => !/^[a-z][a-z0-9_]{0,62}$/.test(schema)) || + !Number.isSafeInteger(budgetMs) || + budgetMs < 1 || + budgetMs > 60000 || + args.some((arg) => arg !== '--apply' && !arg.startsWith('--schemas=') && !arg.startsWith('--budget-ms=')) +) { + throw new Error('Usage: --schemas=che,kwe,... [--budget-ms=3000] [--apply]'); +} +let failed = false; +for (const schema of schemas) { + const url = process.env.DATABASE_URL ? new URL(process.env.DATABASE_URL) : new URL('postgresql://localhost'); + if (!process.env.DATABASE_URL) { + url.hostname = process.env.POSTGRES_HOST ?? 'localhost'; + url.port = process.env.POSTGRES_PORT ?? '5432'; + url.username = process.env.POSTGRES_USER ?? 'sammo'; + url.password = process.env.POSTGRES_PASSWORD ?? ''; + url.pathname = `/${process.env.POSTGRES_DB ?? 'sammo'}`; + } + url.searchParams.set('schema', schema); + const connector = createGamePostgresConnector({ url: url.href, maxConnections: 1 }); + try { + await connector.connect(); + const db = connector.prisma; + const world = await db.worldState.findFirst({ select: { meta: true } }); + const serverId = + world?.meta && typeof world.meta === 'object' && !Array.isArray(world.meta) ? world.meta.serverId : null; + if (typeof serverId !== 'string' || !serverId.trim()) { + console.log(JSON.stringify({ schema, status: 'identityMissing', deleted: 0 })); + continue; + } + if (!apply) { + console.log(JSON.stringify({ schema, status: 'dryRun', apply: false })); + continue; + } + const deadline = performance.now() + budgetMs; + let auditDeleted = 0; + let outboxDeleted = 0; + let status = 'progress'; + let auditComplete = false; + let outboxComplete = false; + do { + if (!auditComplete) { + const result = await prunePreviousAuditBatch(db, serverId); + auditDeleted += result.deleted; + status = result.status; + auditComplete = result.status === 'complete'; + if (result.status === 'identityChanged' || result.status === 'busy') break; + } + if (!outboxComplete) { + outboxDeleted += await db.$transaction(async (tx) => { + await tx.$executeRaw`SET LOCAL statement_timeout='2000ms'`; + await tx.$executeRaw`SET LOCAL lock_timeout='250ms'`; + const deleted = await pruneDeliveredReadModelOutbox(tx, { + deliveredBefore: new Date(Date.now() - 86400000), + limit: 500, + }); + outboxComplete = deleted < 500; + return deleted; + }); + } + if (auditComplete && outboxComplete) break; + await setTimeout(25); + } while (performance.now() < deadline); + console.log( + JSON.stringify({ schema, status, auditDeleted, outboxDeleted, complete: auditComplete && outboxComplete }) + ); + } catch { + // 연결 URL과 원문 SQL을 출력하지 않고 scheduler가 감지할 수 있는 종료 코드를 남긴다. + failed = true; + console.error(JSON.stringify({ schema, status: 'failed' })); + } finally { + await connector.disconnect(); + } +} +if (failed) process.exitCode = 1; From 79de6d484c43098b09a104479a1d0270f626fb11 Mon Sep 17 00:00:00 2001 From: hide_d Date: Mon, 9 Nov 2026 20:49:20 +0900 Subject: [PATCH 2/4] =?UTF-8?q?=EA=B0=90=EC=82=AC=20=EC=A0=95=EB=A6=AC=20C?= =?UTF-8?q?LI=EA=B0=80=20=EA=B2=8C=EC=9E=84=20=EB=8D=B0=EB=AA=AC=20?= =?UTF-8?q?=EC=8B=A4=ED=96=89=20=EC=97=AD=ED=95=A0=EC=9D=84=20=EC=83=81?= =?UTF-8?q?=EC=86=8D=ED=95=98=EC=A7=80=20=EC=95=8A=EA=B2=8C=20=ED=95=9C?= =?UTF-8?q?=EB=8B=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tools/play-audit-maintenance.mjs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tools/play-audit-maintenance.mjs b/tools/play-audit-maintenance.mjs index 20cfceca..e9454c27 100644 --- a/tools/play-audit-maintenance.mjs +++ b/tools/play-audit-maintenance.mjs @@ -3,7 +3,10 @@ import { performance } from 'node:perf_hooks'; import { setTimeout } from 'node:timers/promises'; import { createGamePostgresConnector, pruneDeliveredReadModelOutbox } from '../packages/infra/dist/index.js'; -import { prunePreviousAuditBatch } from '../app/game-engine/dist/index.js'; + +// PM2/호출자가 넘긴 turn-daemon 역할로 engine root의 자동 기동 경로에 들어가지 않는다. +process.env.GAME_ENGINE_ROLE = 'audit-maintenance'; +const { prunePreviousAuditBatch } = await import('../app/game-engine/dist/index.js'); const args = process.argv.slice(2); const apply = args.includes('--apply'); From fd8145d6ebe7e451e9027af54899a1997cc2dd93 Mon Sep 17 00:00:00 2001 From: hide_d Date: Mon, 9 Nov 2026 21:54:57 +0900 Subject: [PATCH 3/4] =?UTF-8?q?=EA=B0=90=EC=82=AC=20=EC=9C=A0=EC=A7=80?= =?UTF-8?q?=EB=B3=B4=EC=88=98=20=EC=A4=91=20=EC=83=88=20=EA=B8=B0=EC=88=98?= =?UTF-8?q?=20=EC=98=88=EC=95=BD=EA=B3=BC=20identity=20=EB=B3=80=EA=B2=BD?= =?UTF-8?q?=20=EB=8C=80=EC=9D=91=EC=9D=84=20=EB=AC=B8=EC=84=9C=ED=99=94?= =?UTF-8?q?=ED=95=9C=EB=8B=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/play-audit-storage.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/play-audit-storage.md b/docs/play-audit-storage.md index 780239ac..f458ed7c 100644 --- a/docs/play-audit-storage.md +++ b/docs/play-audit-storage.md @@ -49,6 +49,9 @@ tablespace 파일만 백업하는 것으로 복구를 보장하지 않는다. 1. 전체 DB의 일관된 압축 backup을 ZFS에 만들고 checksum, archive 읽기, 실제 복원을 확인한다. 2. DB 보존 DEPLOY로 writer/reader/migration을 함께 적용한다. 3. profile 상태를 기록하고 공식 STOP으로 쓰기를 중단한다. 원래 STOPPED인 profile은 유지한다. + 가오픈까지 1시간 이상 남은 RESERVED는 runtimeExpected=false이므로 예약을 유지할 수 있다. + 실제 process 비기동과 identity/hash 안정성을 확인하고 export의 예상 상태를 RESERVED로 고정한다. + 유지보수 중 RESET 등으로 identity가 바뀌면 해당 profile을 새로 export/검증하며 이전 자료를 덮어쓰지 않는다. 4. 각 schema에서 현재 serverId의 8개 감사 테이블을 일관되게 export한다. 월 child는 현재 월 header와 join하고 결정 chunk는 현재 결정 header와 join한다. 5. 격리 DB에서 복원·건수·내용 hash를 검증한다. 이전 기수는 전체 ZFS backup에 남는다. From f36bc2c8d7639c94440e07acf9a8b0fedf3f9a56 Mon Sep 17 00:00:00 2001 From: hide_d Date: Thu, 12 Nov 2026 09:16:04 +0900 Subject: [PATCH 4/4] =?UTF-8?q?VM=20=EB=B3=B5=EA=B7=80=20=ED=9B=84=20?= =?UTF-8?q?=EC=98=88=EC=95=BD=ED=84=B4=20=EC=9E=A0=EA=B8=88=20=EB=B3=B5?= =?UTF-8?q?=EA=B5=AC=EC=99=80=20Windows=20=EB=B6=80=ED=8C=85=20=EC=8B=9D?= =?UTF-8?q?=EB=B3=84=EC=9E=90=20=ED=99=95=EC=9D=B8=EC=9D=84=20=EC=B6=94?= =?UTF-8?q?=EA=B0=80=ED=95=9C=EB=8B=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .env.example | 3 + .env.ops.example | 3 + .gitignore | 1 + .../src/lifecycle/databaseTurnDaemonLease.ts | 20 ++- .../src/lifecycle/turnDaemonLifecycle.ts | 16 +- app/game-engine/src/turn/cli.ts | 3 + app/game-engine/src/turn/hostBootMonitor.ts | 142 +++++++++++++++++ .../src/turn/reservedTurnLeaseRetry.ts | 38 +++++ app/game-engine/src/turn/reservedTurnStore.ts | 44 +++++- app/game-engine/src/turn/runtimePauseGate.ts | 4 +- app/game-engine/src/turn/turnDaemon.ts | 48 ++++-- app/game-engine/test/hostBootMonitor.test.ts | 134 ++++++++++++++++ .../reservedTurnLease.integration.test.ts | 96 +++++++++++ .../test/reservedTurnLease.test.ts | 12 ++ .../test/reservedTurnLeaseRetry.test.ts | 38 +++++ app/game-engine/test/runtimePauseGate.test.ts | 26 ++- .../test/turnDaemonLease.integration.test.ts | 33 ++++ .../test/turnDaemonLifecycle.test.ts | 64 ++++++++ docs/developer/host-boot-recovery.md | 149 ++++++++++++++++++ docs/release-operations.md | 12 ++ tools/host-recovery/README.md | 12 ++ .../windows-boot-agent.example.json | 5 + tools/host-recovery/windows-boot-agent.mjs | 131 +++++++++++++++ .../host-recovery/windows-boot-agent.test.mjs | 42 +++++ 24 files changed, 1048 insertions(+), 28 deletions(-) create mode 100644 app/game-engine/src/turn/hostBootMonitor.ts create mode 100644 app/game-engine/src/turn/reservedTurnLeaseRetry.ts create mode 100644 app/game-engine/test/hostBootMonitor.test.ts create mode 100644 app/game-engine/test/reservedTurnLeaseRetry.test.ts create mode 100644 docs/developer/host-boot-recovery.md create mode 100644 tools/host-recovery/README.md create mode 100644 tools/host-recovery/windows-boot-agent.example.json create mode 100644 tools/host-recovery/windows-boot-agent.mjs create mode 100644 tools/host-recovery/windows-boot-agent.test.mjs diff --git a/.env.example b/.env.example index 5e4917d4..a061ec5e 100644 --- a/.env.example +++ b/.env.example @@ -91,6 +91,9 @@ DAEMON_REQUEST_TIMEOUT_MS=5000 # realtime은 벽시계 경과를 game tick으로 투영합니다. manual은 daemon이 # 다음 월 snapshot을 대기 없이 실행하며 테스트/장기 시뮬레이션에 사용합니다. GAME_CLOCK_MODE=realtime +# Optional Windows host boot identity. Key is a private file, never a VITE_* value. +# TURN_HOST_BOOT_URL=http://windows-private-address:17781/v1/boot-id +# TURN_HOST_BOOT_KEY_FILE=/run/secrets/host_boot_key TRPC_PATH=/trpc # Frontend public URLs diff --git a/.env.ops.example b/.env.ops.example index c148519e..778e0894 100644 --- a/.env.ops.example +++ b/.env.ops.example @@ -16,6 +16,9 @@ GAME_API_HOST=0.0.0.0 GAME_API_PORT=14000 PROFILE=hwe SCENARIO=default +# Optional Windows observation; unavailable or unconfigured service never blocks gameplay. +# TURN_HOST_BOOT_URL=http://windows-private-address:17781/v1/boot-id +# TURN_HOST_BOOT_KEY_FILE=/run/sammo-host-recovery/host_boot.key KAKAO_REST_KEY=your-kakao-rest-key KAKAO_ADMIN_KEY=your-kakao-admin-key KAKAO_REDIRECT_URI=https://example.com/oauth/kakao/callback diff --git a/.gitignore b/.gitignore index ad85bb6b..2120f18f 100644 --- a/.gitignore +++ b/.gitignore @@ -89,6 +89,7 @@ web_modules/ .env.local .env.ci .env.ops +tools/host-recovery/*.key tools/legacy-db-migration/migration-plan.json tools/legacy-db-migration/secrets/ diff --git a/app/game-engine/src/lifecycle/databaseTurnDaemonLease.ts b/app/game-engine/src/lifecycle/databaseTurnDaemonLease.ts index 81e58276..fb783b59 100644 --- a/app/game-engine/src/lifecycle/databaseTurnDaemonLease.ts +++ b/app/game-engine/src/lifecycle/databaseTurnDaemonLease.ts @@ -2,6 +2,7 @@ import { randomUUID } from 'node:crypto'; import { performance } from 'node:perf_hooks'; import { createGamePostgresConnector, GamePrisma, type GamePrismaClient } from '@sammo-ts/infra'; +import type { HostBootMonitor } from '../turn/hostBootMonitor.js'; export interface TurnDaemonLeaseToken { profile: string; @@ -14,6 +15,7 @@ export interface DatabaseTurnDaemonLeaseOptions { ownerId?: string; leaseDurationMs?: number; heartbeat?: boolean; + hostBootMonitor?: HostBootMonitor; } export class TurnDaemonLeaseUnavailableError extends Error { @@ -45,6 +47,7 @@ export class DatabaseTurnDaemonLease { private readonly ownerId: string; private readonly leaseDurationMs: number; private readonly heartbeatEnabled: boolean; + private readonly hostBootMonitor?: HostBootMonitor; private token: TurnDaemonLeaseToken | null = null; private heartbeatTimer: NodeJS.Timeout | null = null; private expiryTimer: NodeJS.Timeout | null = null; @@ -63,6 +66,7 @@ export class DatabaseTurnDaemonLease { this.ownerId = options.ownerId ?? randomUUID(); this.leaseDurationMs = normalizeLeaseDuration(options.leaseDurationMs); this.heartbeatEnabled = options.heartbeat ?? true; + this.hostBootMonitor = options.hostBootMonitor; } static async connect( @@ -75,6 +79,8 @@ export class DatabaseTurnDaemonLease { } async acquire(): Promise { + await this.checkHostBoot(); + if (this.lost) throw this.getLossError(); const requestStartedAt = performance.now(); const rows = await this.db.$queryRaw(GamePrisma.sql` INSERT INTO "turn_daemon_lease" ( @@ -145,6 +151,12 @@ export class DatabaseTurnDaemonLease { return new TurnDaemonLeaseLostError(this.profile, this.lossReason); } + async checkHostBoot(): Promise { + if (!this.lost && (await this.hostBootMonitor?.hasChanged())) { + this.markLost('Host boot identity changed; restart with a fresh database snapshot.'); + } + } + async renew(): Promise { const token = this.token; if (!token || this.lost || this.renewalInFlight) { @@ -153,6 +165,8 @@ export class DatabaseTurnDaemonLease { const requestStartedAt = performance.now(); this.renewalInFlight = true; try { + await this.checkHostBoot(); + if (this.lost) return false; const rows = await this.db.$queryRaw(GamePrisma.sql` UPDATE "turn_daemon_lease" SET @@ -180,6 +194,7 @@ export class DatabaseTurnDaemonLease { } async assertActive(transaction?: GamePrisma.TransactionClient): Promise { + await this.checkHostBoot(); const token = this.token; if (!token || this.lost) { throw this.getLossError(); @@ -207,12 +222,13 @@ export class DatabaseTurnDaemonLease { this.stopExpiryWatchdog(); const token = this.token; this.token = null; - if (!token || this.lost) { + if (!token) { return; } await this.db.$executeRaw(GamePrisma.sql` UPDATE "turn_daemon_lease" - SET "lease_until" = CURRENT_TIMESTAMP AT TIME ZONE 'UTC', + SET "clock_ready" = FALSE, + "lease_until" = CURRENT_TIMESTAMP AT TIME ZONE 'UTC', "heartbeat_at" = CURRENT_TIMESTAMP AT TIME ZONE 'UTC' WHERE "profile" = ${token.profile} diff --git a/app/game-engine/src/lifecycle/turnDaemonLifecycle.ts b/app/game-engine/src/lifecycle/turnDaemonLifecycle.ts index cddf1914..bb7c7c31 100644 --- a/app/game-engine/src/lifecycle/turnDaemonLifecycle.ts +++ b/app/game-engine/src/lifecycle/turnDaemonLifecycle.ts @@ -45,6 +45,7 @@ export interface TurnDaemonLifecycleDeps { commandResponder?: TurnDaemonCommandResponder; pauseGate?: () => Promise; stateManager?: Pick; + runErrorRetry?: { wait(error: unknown): Promise; reset(): void }; } export class TurnDaemonLifecycle { @@ -59,6 +60,7 @@ export class TurnDaemonLifecycle { private readonly commandResponder?: TurnDaemonCommandResponder; private readonly pauseGate?: () => Promise; private readonly stateManager?: Pick; + private readonly runErrorRetry?: TurnDaemonLifecycleDeps['runErrorRetry']; private readonly options: TurnDaemonLifecycleOptions; private status: TurnDaemonStatus; @@ -79,6 +81,7 @@ export class TurnDaemonLifecycle { this.commandResponder = deps.commandResponder; this.pauseGate = deps.pauseGate; this.stateManager = deps.stateManager; + this.runErrorRetry = deps.runErrorRetry; this.options = options; this.status = { state: 'idle', @@ -192,8 +195,9 @@ export class TurnDaemonLifecycle { // 수동 실행 요청도 가오픈·정지·재조정의 턴 실행 gate를 통과해야 한다. // 사용자 명령 처리는 루프 시작에서 계속하되 시간 진행은 여기서 분리한다. if (this.pendingRun) { - await this.runOnce(this.pendingRun); + const pending = this.pendingRun; this.pendingRun = null; + await this.runOnce(pending); continue; } @@ -447,6 +451,15 @@ export class TurnDaemonLifecycle { }; result = this.stateManager ? await this.stateManager.transaction(runAndFlush) : await runAndFlush(); } catch (error) { + // savepoint 복원이 성공한 획득 충돌만 대기한다. 복원 실패/AggregateError나 + // 저장 중 소유권 상실을 자동 재실행하면 중복 효과를 숨길 수 있다. + if (this.stateManager && (await this.runErrorRetry?.wait(error))) { + this.status.running = false; + this.status.state = 'idle'; + this.pendingRun = pending; + return; + } + this.runErrorRetry?.reset(); this.status.running = false; this.status.state = 'paused'; this.status.paused = true; @@ -459,6 +472,7 @@ export class TurnDaemonLifecycle { } await this.applyRunResult(result, startMs); + this.runErrorRetry?.reset(); this.status.state = 'idle'; try { await this.hooks?.publishEvents?.(result); diff --git a/app/game-engine/src/turn/cli.ts b/app/game-engine/src/turn/cli.ts index 4f273d36..5b2b9c59 100644 --- a/app/game-engine/src/turn/cli.ts +++ b/app/game-engine/src/turn/cli.ts @@ -8,6 +8,7 @@ import { createTurnDaemonMemoryReporter } from './turnDaemonMemoryReporter.js'; import { createGatewayProfileGate } from './gatewayProfileGate.js'; import { retryTurnDaemonLeaseStartup } from './leaseStartupRetry.js'; import { TurnDaemonLeaseUnavailableError } from '../lifecycle/databaseTurnDaemonLease.js'; +import { loadHostBootMonitor } from './hostBootMonitor.js'; export interface TurnDaemonCliOptions { profile?: string; @@ -80,6 +81,7 @@ export const runTurnDaemonCli = async (options: TurnDaemonCliOptions = {}): Prom throw new Error(`GAME_CLOCK_MODE must be realtime or manual: ${rawGameClockMode}`); } const gameClockMode = rawGameClockMode as GameClockMode | undefined; + const hostBootMonitor = await loadHostBootMonitor(env); const runtime = await retryTurnDaemonLeaseStartup(() => createTurnDaemonRuntime({ @@ -95,6 +97,7 @@ export const runTurnDaemonCli = async (options: TurnDaemonCliOptions = {}): Prom pauseGateIntervalMs, adminActionIntervalMs, gameClockMode, + hostBootMonitor, }) ).catch(async (error: unknown) => { // 중복 starter가 정상 owner를 멈추면 안 된다. 그 밖의 초기화 실패는 diff --git a/app/game-engine/src/turn/hostBootMonitor.ts b/app/game-engine/src/turn/hostBootMonitor.ts new file mode 100644 index 00000000..c8d33ed0 --- /dev/null +++ b/app/game-engine/src/turn/hostBootMonitor.ts @@ -0,0 +1,142 @@ +import { createHmac, randomBytes, timingSafeEqual } from 'node:crypto'; +import { readFile } from 'node:fs/promises'; +import { performance } from 'node:perf_hooks'; +import { isRecord } from '@sammo-ts/common'; + +export interface HostBootMonitor { + hasChanged(): Promise; +} + +const mac = (key: Buffer, value: string): string => createHmac('sha256', key).update(value).digest('hex'); + +/** 부팅 세대는 첫 인증 응답에 고정한다. 통신 실패는 기존 lease 복구를 막지 않는다. */ +export const createHostBootMonitor = (options: { + url: string; + key: Buffer; + pollMs?: number; + timeoutMs?: number; + now?: () => number; + wallNow?: () => number; + onUnavailable?: () => void; + onVerified?: () => void; +}): HostBootMonitor => { + const url = new URL(options.url); + if ( + !['http:', 'https:'].includes(url.protocol) || + url.username || + url.password || + url.search || + url.hash || + url.pathname !== '/v1/boot-id' + ) { + throw new Error('Host boot URL must use HTTP(S) and /v1/boot-id without credentials or query.'); + } + if (options.key.length !== 32) throw new Error('Host boot key must contain exactly 32 bytes.'); + const now = options.now ?? (() => performance.now()); + const wallNow = options.wallNow ?? (() => Date.now()); + let baseline: string | null = null; + let changed = false; + let lastAttempt = -Infinity; + let lastWall = 0; + let lastWarning = -Infinity; + let inFlight: Promise | null = null; + const sample = async (): Promise => { + const nonce = randomBytes(32).toString('hex'); + const response = await fetch(url, { + method: 'GET', + redirect: 'error', + signal: AbortSignal.timeout(options.timeoutMs ?? 1_000), + headers: { + 'x-sammo-challenge': nonce, + 'x-sammo-signature': mac(options.key, `sammo-host-boot-request-v1\n${nonce}`), + }, + }); + if (!response.ok || !response.body) { + await response.body?.cancel(); + throw new Error('Host boot response unavailable.'); + } + const reader = response.body.getReader(); + const chunks: Uint8Array[] = []; + let size = 0; + try { + for (;;) { + const { done, value } = await reader.read(); + if (done) break; + size += value.length; + if (size > 2_048) throw new Error('Host boot response exceeds its limit.'); + chunks.push(value); + } + } finally { + await reader.cancel(); + } + const body: unknown = JSON.parse(Buffer.concat(chunks).toString('utf8')); + if ( + !isRecord(body) || + body.version !== 1 || + body.challenge !== nonce || + typeof body.bootId !== 'string' || + !/^[a-f0-9]{64}$/.test(body.bootId) || + typeof body.signature !== 'string' || + !/^[a-f0-9]{64}$/.test(body.signature) + ) { + throw new Error('Host boot response is invalid.'); + } + const expected = mac(options.key, `sammo-host-boot-response-v1\n${nonce}\n${body.bootId}`); + if (!timingSafeEqual(Buffer.from(expected, 'hex'), Buffer.from(body.signature, 'hex'))) { + throw new Error('Host boot response signature is invalid.'); + } + if (baseline === null) { + baseline = body.bootId; + options.onVerified?.(); + } + changed ||= body.bootId !== baseline; + return changed; + }; + return { + hasChanged(): Promise { + if (changed) return Promise.resolve(true); + if (inFlight) return inFlight; + const current = now(); + const wall = wallNow(); + const discontinuity = Math.abs(wall - lastWall - (current - lastAttempt)) > 1_000; + if (current - lastAttempt < (options.pollMs ?? 2_000) && !discontinuity) return Promise.resolve(false); + lastAttempt = current; + lastWall = wall; + inFlight = sample() + .catch(() => { + if (current - lastWarning >= 60_000) { + options.onUnavailable?.(); + lastWarning = current; + } + return false; + }) + .finally(() => { + inFlight = null; + }); + return inFlight; + }, + }; +}; + +export const loadHostBootMonitor = async (env: NodeJS.ProcessEnv): Promise => { + const url = env.TURN_HOST_BOOT_URL?.trim(); + const keyFile = env.TURN_HOST_BOOT_KEY_FILE?.trim(); + if (!url && !keyFile) return undefined; + try { + if (!url || !keyFile) throw new Error('Incomplete optional configuration.'); + const encoded = (await readFile(keyFile, 'utf8')).trim(); + if (!/^[a-f0-9]{64}$/.test(encoded)) throw new Error('Invalid optional key file.'); + return createHostBootMonitor({ + url, + key: Buffer.from(encoded, 'hex'), + onVerified: () => console.info('[turn-daemon] host boot identity verified.'), + onUnavailable: () => { + console.warn('[turn-daemon] host boot identity unavailable; ordinary lease recovery remains active.'); + }, + }); + } catch { + // 보조 관측기의 설정/파일 부재도 게임 기동의 필수 의존성이 되지 않는다. + console.warn('[turn-daemon] host boot observation disabled; check optional configuration and key file.'); + return undefined; + } +}; diff --git a/app/game-engine/src/turn/reservedTurnLeaseRetry.ts b/app/game-engine/src/turn/reservedTurnLeaseRetry.ts new file mode 100644 index 00000000..73574e4d --- /dev/null +++ b/app/game-engine/src/turn/reservedTurnLeaseRetry.ts @@ -0,0 +1,38 @@ +import { performance } from 'node:perf_hooks'; +import { setTimeout } from 'node:timers/promises'; +import { ReservedTurnLeaseBusyError } from './reservedTurnStore.js'; + +/** rollback 완료 뒤에만 대기한다. 다음 loop에서 pause/lease gate와 최신 큐를 다시 읽는다. */ +export const createReservedTurnLeaseRetry = ( + options: { + now?: () => number; + sleep?: (ms: number) => Promise; + maxWaitMs?: number; + onWait?: (queueKey: string) => void; + } = {} +) => { + const now = options.now ?? (() => performance.now()); + const sleep = options.sleep ?? ((ms: number) => setTimeout(ms)); + const maxWaitMs = options.maxWaitMs ?? 5 * 60_000 + 1_000; + let startedAt: number | null = null; + let lastReportedAt = -Infinity; + return { + reset(): void { + startedAt = null; + lastReportedAt = -Infinity; + }, + async wait(error: unknown): Promise { + if (!(error instanceof ReservedTurnLeaseBusyError)) return false; + const current = now(); + startedAt ??= current; + const remaining = maxWaitMs - (current - startedAt); + if (remaining <= 0) return false; + if (current - lastReportedAt >= 30_000) { + options.onWait?.(error.queueKey); + lastReportedAt = current; + } + await sleep(Math.min(1_000, remaining)); + return true; + }, + }; +}; diff --git a/app/game-engine/src/turn/reservedTurnStore.ts b/app/game-engine/src/turn/reservedTurnStore.ts index 3c42c6c2..3a9f537a 100644 --- a/app/game-engine/src/turn/reservedTurnStore.ts +++ b/app/game-engine/src/turn/reservedTurnStore.ts @@ -129,6 +129,14 @@ export class ReservedTurnLeaseConflictError extends Error { } } +/** 실행 전 획득 충돌만 재시도한다. flush/finalize의 소유권 상실과 구분한다. */ +export class ReservedTurnLeaseBusyError extends ReservedTurnLeaseConflictError { + constructor(queueKey: string) { + super(queueKey); + this.name = 'ReservedTurnLeaseBusyError'; + } +} + export class InMemoryReservedTurnStore { private readonly generalTurns = new Map(); private readonly nationTurns = new Map(); @@ -249,11 +257,7 @@ export class InMemoryReservedTurnStore { } }; pruneNationKeys(Array.from(this.nationTurns.keys()), (key) => this.nationTurns.delete(key)); - for (const keys of [ - this.dirtyNationKeys, - this.pendingNationInitializationKeys, - this.leasedNationKeys, - ]) { + for (const keys of [this.dirtyNationKeys, this.pendingNationInitializationKeys, this.leasedNationKeys]) { for (const key of Array.from(keys)) { const nationId = Number(key.split(':', 1)[0]); if (deletedNations.has(nationId)) { @@ -376,7 +380,7 @@ export class InMemoryReservedTurnStore { } } if (claimed.count !== 1) { - throw new ReservedTurnLeaseConflictError(`general:${generalId}`); + throw new ReservedTurnLeaseBusyError(`general:${generalId}`); } this.leasedGeneralIds.add(generalId); return !retainedExistingLease; @@ -427,7 +431,7 @@ export class InMemoryReservedTurnStore { } } if (claimed.count !== 1) { - throw new ReservedTurnLeaseConflictError(`nation:${nationId}:${officerLevel}`); + throw new ReservedTurnLeaseBusyError(`nation:${nationId}:${officerLevel}`); } this.leasedNationKeys.add(buildNationKey(nationId, officerLevel)); return !retainedExistingLease; @@ -904,6 +908,24 @@ export class InMemoryReservedTurnStore { await this.persistChanges(this.prisma, changes); this.acknowledgeDirtyState(changes); } + + /** 실행 루프 종료 뒤 호출한다. rollback으로 journal에서 빠진 DB 임대도 해제한다. */ + async releaseOwnedLeases(): Promise { + const results = await Promise.allSettled([ + this.prisma.generalTurnRevision?.updateMany({ + where: { leaseOwner: this.leaseOwner }, + data: { leaseOwner: null, leaseExpiresAt: null }, + }), + this.prisma.nationTurnRevision?.updateMany({ + where: { leaseOwner: this.leaseOwner }, + data: { leaseOwner: null, leaseExpiresAt: null }, + }), + ]); + const errors = results.flatMap((result) => (result.status === 'rejected' ? [result.reason] : [])); + if (errors.length) throw new AggregateError(errors, 'Reserved turn lease cleanup failed.'); + this.leasedGeneralIds.clear(); + this.leasedNationKeys.clear(); + } } export const createReservedTurnStore = async (options: ReservedTurnStoreOptions): Promise => { @@ -918,6 +940,12 @@ export const createReservedTurnStore = async (options: ReservedTurnStoreOptions) await store.loadAll(); return { store, - close: () => connector.disconnect(), + close: async () => { + try { + await store.releaseOwnedLeases(); + } finally { + await connector.disconnect(); + } + }, }; }; diff --git a/app/game-engine/src/turn/runtimePauseGate.ts b/app/game-engine/src/turn/runtimePauseGate.ts index 4c106873..1ecbff66 100644 --- a/app/game-engine/src/turn/runtimePauseGate.ts +++ b/app/game-engine/src/turn/runtimePauseGate.ts @@ -2,7 +2,7 @@ import type { GameClockPhase } from '@sammo-ts/common'; /** Gateway의 실행 gate와 durable 시계를 명령 claim 전에 맞춘다. */ export const createRuntimePauseGate = (options: { - assertLease(): void; + assertLease(): void | Promise; shouldPause(): Promise; getPhase(): GameClockPhase; isExplicitlyPaused(): boolean; @@ -11,7 +11,7 @@ export const createRuntimePauseGate = (options: { }): (() => Promise) => { let lastPaused: boolean | null = null; return async () => { - options.assertLease(); + await options.assertLease(); const paused = await options.shouldPause(); const phase = options.getPhase(); // 오류 정지는 Gateway 상태만 PAUSED로 바꿀 수 있다. 그대로 두면 가입은 diff --git a/app/game-engine/src/turn/turnDaemon.ts b/app/game-engine/src/turn/turnDaemon.ts index f8324aa2..aa7718d3 100644 --- a/app/game-engine/src/turn/turnDaemon.ts +++ b/app/game-engine/src/turn/turnDaemon.ts @@ -108,6 +108,8 @@ import { DatabaseTurnDaemonLease, TurnDaemonLeaseUnavailableError } from '../lif import { EngineStateManager } from './engineStateManager.js'; import { applyRuntimeClockShift } from './runtimeClockShift.js'; import { applyRuntimeGameSettings } from './runtimeGameSettings.js'; +import { createReservedTurnLeaseRetry } from './reservedTurnLeaseRetry.js'; +import type { HostBootMonitor } from './hostBootMonitor.js'; export interface TurnDaemonRuntimeOptions { profile: string; @@ -134,6 +136,7 @@ export interface TurnDaemonRuntimeOptions { leaseDurationMs?: number; leaseOwnerId?: string; enableLeaseHeartbeat?: boolean; + hostBootMonitor?: HostBootMonitor; /** * Isolated, single-process fixture acceleration only. Reserved turns are * loaded once and no concurrent API writer may touch this database. @@ -985,6 +988,7 @@ const createTurnDaemonRuntimeWithLease = async ( prepareClockRecovery = dbHooks.prepareRealtimeRecovery; prunePreviousAudit = dbHooks.prunePreviousAudit; close = async () => { + await adminActionConsumer?.stop(); if (auctionBidder) { await auctionBidder.close(); } @@ -995,12 +999,15 @@ const createTurnDaemonRuntimeWithLease = async ( await tournamentRewardFinalizer.close(); } await dbHooks.close(); - if (reservedTurnStoreHandle) { - await reservedTurnStoreHandle.close(); + try { + await reservedTurnStoreHandle?.close(); + } finally { + try { + await gatewayGate?.close(); + } finally { + await turnDaemonLease?.close(); + } } - await gatewayGate?.close(); - await adminActionConsumer?.stop(); - await turnDaemonLease?.close(); }; } else if (reservedTurnStoreHandle) { hooks = { @@ -1009,9 +1016,12 @@ const createTurnDaemonRuntimeWithLease = async ( }, }; close = async () => { - await reservedTurnStoreHandle.close(); - await gatewayGate?.close(); await adminActionConsumer?.stop(); + try { + await reservedTurnStoreHandle.close(); + } finally { + await gatewayGate?.close(); + } }; } else if (gatewayGate) { hooks = { @@ -1070,12 +1080,17 @@ const createTurnDaemonRuntimeWithLease = async ( const baseClose = close; close = async () => { stopClockProjectionWorker(); - await baseClose(); - await neutralAuctionRegistrar.close(); - if (redisConnector) { - await redisConnector.disconnect(); - } - await commandConnector?.disconnect(); + const baseResults = await Promise.allSettled([baseClose()]); + // queue 임대 정리 실패도 다른 연결/worker의 종료를 막지 않게 한다. + const results = await Promise.allSettled([ + neutralAuctionRegistrar.close(), + redisConnector?.disconnect(), + commandConnector?.disconnect(), + ]); + const failures = [...baseResults, ...results].flatMap((result) => + result.status === 'rejected' ? [result.reason] : [] + ); + if (failures.length) throw new AggregateError(failures, 'Turn daemon connection cleanup failed.'); }; const resolvedControlQueue = options.controlQueue ?? databaseCommandQueue ?? controlQueue; @@ -1109,7 +1124,8 @@ const createTurnDaemonRuntimeWithLease = async ( processor, hooks, pauseGate: createRuntimePauseGate({ - assertLease: () => { + assertLease: async () => { + await turnDaemonLease?.checkHostBoot(); if (turnDaemonLease?.isLost()) throw turnDaemonLease.getLossError(); }, shouldPause: async () => (await pauseGate?.()) ?? false, @@ -1127,6 +1143,9 @@ const createTurnDaemonRuntimeWithLease = async ( // accumulated world before every due tick in that isolated mode; // production and gateway-managed runtimes keep rollback savepoints. stateManager: options.exclusiveFastForward ? undefined : stateManager, + runErrorRetry: createReservedTurnLeaseRetry({ + onWait: (queueKey) => console.info(`[turn-daemon] waiting for reserved turn lease: ${queueKey}`), + }), }, { profile: options.profile, defaultBudget } ); @@ -1179,6 +1198,7 @@ export const createTurnDaemonRuntime = async (options: TurnDaemonRuntimeOptions) ownerId: options.leaseOwnerId, leaseDurationMs: options.leaseDurationMs, heartbeat: options.enableLeaseHeartbeat, + hostBootMonitor: options.hostBootMonitor, }) : null; if (turnDaemonLease && !(await turnDaemonLease.acquire())) { diff --git a/app/game-engine/test/hostBootMonitor.test.ts b/app/game-engine/test/hostBootMonitor.test.ts new file mode 100644 index 00000000..a090945f --- /dev/null +++ b/app/game-engine/test/hostBootMonitor.test.ts @@ -0,0 +1,134 @@ +import { createHmac, randomBytes } from 'node:crypto'; +import { once } from 'node:events'; +import { createServer } from 'node:http'; +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { createHostBootMonitor, loadHostBootMonitor } from '../src/turn/hostBootMonitor.js'; + +const cleanups: Array<() => Promise> = []; +afterEach(async () => { + await Promise.all(cleanups.splice(0).map((cleanup) => cleanup())); +}); + +const fixture = async () => { + const key = randomBytes(32); + let bootId = 'a'.repeat(64); + let replay: string | null = null; + let mode: 'valid' | 'replay' | 'forged' | 'oversized' | 'offline' | 'hang' = 'valid'; + const requests: string[] = []; + const server = createServer((request, response) => { + const nonce = String(request.headers['x-sammo-challenge']); + requests.push(nonce); + expect(request.headers['x-sammo-signature']).toBe( + createHmac('sha256', key).update(`sammo-host-boot-request-v1\n${nonce}`).digest('hex') + ); + if (mode === 'hang') return; + if (mode === 'offline') { + response.writeHead(503).end(); + return; + } + if (mode === 'oversized') { + response.end('x'.repeat(4096)); + return; + } + if (mode === 'replay') { + response.end(replay); + return; + } + const body = JSON.stringify({ + version: 1, + challenge: nonce, + bootId, + signature: + mode === 'forged' + ? '0'.repeat(64) + : createHmac('sha256', key) + .update(`sammo-host-boot-response-v1\n${nonce}\n${bootId}`) + .digest('hex'), + }); + replay ??= body; + response.end(body); + }); + server.listen(0, '127.0.0.1'); + await once(server, 'listening'); + cleanups.push( + () => + new Promise((resolve) => { + server.close(() => resolve()); + server.closeAllConnections(); + }) + ); + const address = server.address(); + if (!address || typeof address === 'string') throw new Error('Missing fixture listener'); + return { + key, + requests, + url: `http://127.0.0.1:${address.port}/v1/boot-id`, + setBoot: (value: string) => { + bootId = value; + }, + setMode: (value: typeof mode) => { + mode = value; + }, + }; +}; + +describe('Windows boot observation', () => { + it('uses a stable baseline, survives outages, and latches a signed boot change', async () => { + const f = await fixture(); + const monitor = createHostBootMonitor({ ...f, pollMs: 0 }); + expect(await monitor.hasChanged()).toBe(false); + f.setMode('offline'); + expect(await monitor.hasChanged()).toBe(false); + f.setBoot('b'.repeat(64)); + f.setMode('valid'); + expect(await monitor.hasChanged()).toBe(true); + f.setMode('offline'); + expect(await monitor.hasChanged()).toBe(true); + }); + + it.each(['replay', 'forged', 'oversized', 'hang'] as const)( + 'rejects %s without changing the baseline', + async (mode) => { + const f = await fixture(); + const onUnavailable = vi.fn(); + const monitor = createHostBootMonitor({ ...f, pollMs: 0, timeoutMs: 100, onUnavailable }); + expect(await monitor.hasChanged()).toBe(false); + f.setBoot('b'.repeat(64)); + f.setMode(mode); + expect(await monitor.hasChanged()).toBe(false); + expect(onUnavailable).toHaveBeenCalledOnce(); + f.setMode('valid'); + expect(await monitor.hasChanged()).toBe(true); + } + ); + + it('samples immediately after a wall-clock jump despite a frozen monotonic cache and coalesces requests', async () => { + const f = await fixture(); + let wall = 10_000; + const monitor = createHostBootMonitor({ ...f, now: () => 0, wallNow: () => wall }); + expect(await monitor.hasChanged()).toBe(false); + expect(await monitor.hasChanged()).toBe(false); + expect(f.requests).toHaveLength(1); + wall += 300_000; + f.setBoot('b'.repeat(64)); + expect(await Promise.all([monitor.hasChanged(), monitor.hasChanged()])).toEqual([true, true]); + expect(f.requests).toHaveLength(2); + expect(new Set(f.requests).size).toBe(2); + }); + + it('does not block game startup when optional configuration or its key file is absent', async () => { + const warning = vi.spyOn(console, 'warn').mockImplementation(() => {}); + expect(await loadHostBootMonitor({})).toBeUndefined(); + expect(await loadHostBootMonitor({ TURN_HOST_BOOT_URL: 'http://host/v1/boot-id' })).toBeUndefined(); + expect( + await loadHostBootMonitor({ + TURN_HOST_BOOT_URL: 'http://host/v1/boot-id', + TURN_HOST_BOOT_KEY_FILE: '/nonexistent-host-boot-key', + }) + ).toBeUndefined(); + expect(() => + createHostBootMonitor({ url: 'http://user:password@host/v1/boot-id', key: randomBytes(32) }) + ).toThrow('without credentials'); + warning.mockRestore(); + }); +}); diff --git a/app/game-engine/test/reservedTurnLease.integration.test.ts b/app/game-engine/test/reservedTurnLease.integration.test.ts index 5b96fadf..0bee424d 100644 --- a/app/game-engine/test/reservedTurnLease.integration.test.ts +++ b/app/game-engine/test/reservedTurnLease.integration.test.ts @@ -4,7 +4,9 @@ import { createGamePostgresConnector } from '@sammo-ts/infra'; import { InMemoryReservedTurnStore, ReservedTurnLeaseConflictError, + createReservedTurnStore, } from '../src/turn/reservedTurnStore.js'; +import { createReservedTurnLeaseRetry } from '../src/turn/reservedTurnLeaseRetry.js'; const databaseUrl = process.env.RESERVED_TURN_DATABASE_URL; const describeIntegration = databaseUrl ? describe : describe.skip; @@ -132,4 +134,98 @@ describeIntegration('reserved turn daemon/API lease integration', () => { }); expect(after.map((turn) => turn.actionCode)).toEqual(before.map((turn) => turn.actionCode)); }); + + it('cleans persisted ownership after rollback loses the journal, preserves other writers, and reloads once', async () => { + if (!connector || !databaseUrl) throw new Error('integration connector is unavailable'); + await connector.prisma.generalTurnRevision.update({ + where: { generalId: GENERAL_ID }, + data: { leaseOwner: null, leaseExpiresAt: null }, + }); + const before = await connector.prisma.generalTurn.findMany({ + where: { generalId: GENERAL_ID }, + orderBy: { turnIdx: 'asc' }, + }); + const revision = await connector.prisma.generalTurnRevision.findUniqueOrThrow({ + where: { generalId: GENERAL_ID }, + }); + const old = await createReservedTurnStore({ databaseUrl, leaseOwner: 'interrupted-daemon' }); + const savepoint = old.store.captureTransactionState(); + await old.store.prepareTurnsForExecution(GENERAL_ID); + old.store.shiftGeneralTurns(GENERAL_ID, -1); + old.store.restoreState(savepoint); + expect(old.store.getQueueCounts().leasedGeneralQueues).toBe(0); + const next = await createReservedTurnStore({ databaseUrl, leaseOwner: 'new-daemon' }); + try { + await expect(next.store.prepareTurnsForExecution(GENERAL_ID)).rejects.toBeInstanceOf( + ReservedTurnLeaseConflictError + ); + await old.close(); + const released = await connector.prisma.generalTurnRevision.findUniqueOrThrow({ + where: { generalId: GENERAL_ID }, + }); + expect(released).toMatchObject({ leaseOwner: null, leaseExpiresAt: null, revision: revision.revision }); + await next.store.prepareTurnsForExecution(GENERAL_ID); + // A delayed close of an old owner cannot clear the new owner's lease. + const delayedCleaner = new InMemoryReservedTurnStore(connector.prisma, { + maxGeneralTurns: 2, + maxNationTurns: 1, + leaseOwner: 'interrupted-daemon', + }); + await delayedCleaner.releaseOwnedLeases(); + expect( + await connector.prisma.generalTurnRevision.findUniqueOrThrow({ where: { generalId: GENERAL_ID } }) + ).toMatchObject({ leaseOwner: 'new-daemon', revision: revision.revision }); + await next.store.releaseOwnedLeases(); + const actual = await connector.prisma.generalTurn.findMany({ + where: { generalId: GENERAL_ID }, + orderBy: { turnIdx: 'asc' }, + }); + expect(actual.map((row) => row.actionCode)).toEqual(before.map((row) => row.actionCode)); + } finally { + await next.close(); + } + }); + + it('waits out an orphaned lease without deleting it and reloads an intervening API edit', async () => { + if (!connector) throw new Error('integration connector is unavailable'); + await connector.prisma.generalTurnRevision.update({ + where: { generalId: GENERAL_ID }, + data: { leaseOwner: 'killed-daemon', leaseExpiresAt: new Date(Date.now() + 300_000) }, + }); + const store = new InMemoryReservedTurnStore(connector.prisma, { + maxGeneralTurns: 2, + maxNationTurns: 1, + leaseOwner: 'recovery-daemon', + }); + await store.loadAll(); + const retry = createReservedTurnLeaseRetry({ + sleep: async () => { + // Simulate the recorded expiry, then an authorized API edit before the next claim. + expect( + await connector.prisma.generalTurnRevision.findUniqueOrThrow({ where: { generalId: GENERAL_ID } }) + ).toMatchObject({ leaseOwner: 'killed-daemon' }); + await connector.prisma.generalTurnRevision.update({ + where: { generalId: GENERAL_ID }, + data: { leaseExpiresAt: new Date(0) }, + }); + await connector.prisma.generalTurn.updateMany({ + where: { generalId: GENERAL_ID, turnIdx: 0 }, + data: { actionCode: 'che_수련' }, + }); + }, + }); + try { + let failure: unknown; + try { + await store.prepareTurnsForExecution(GENERAL_ID); + } catch (error) { + failure = error; + } + expect(await retry.wait(failure)).toBe(true); + await store.prepareTurnsForExecution(GENERAL_ID); + expect(store.getGeneralTurns(GENERAL_ID)[0]?.action).toBe('che_수련'); + } finally { + await store.releaseOwnedLeases(); + } + }); }); diff --git a/app/game-engine/test/reservedTurnLease.test.ts b/app/game-engine/test/reservedTurnLease.test.ts index a6eb98b3..1a87b00c 100644 --- a/app/game-engine/test/reservedTurnLease.test.ts +++ b/app/game-engine/test/reservedTurnLease.test.ts @@ -169,6 +169,18 @@ const buildHarness = (initialRevision: RevisionRow | null = null) => { }; describe('reserved turn daemon lease', () => { + it('cleans both persisted lease kinds even after their journal is rolled back', async () => { + const harness = buildHarness(); + const snapshot = harness.store.captureTransactionState(); + await harness.store.prepareTurnsForExecution(7, { nationId: 3, officerLevel: 12 }); + harness.store.restoreState(snapshot); + expect(harness.store.getQueueCounts()).toMatchObject({ leasedGeneralQueues: 0, leasedNationQueues: 0 }); + await harness.store.releaseOwnedLeases(); + expect(harness.getRevision()).toMatchObject({ leaseOwner: null, leaseExpiresAt: null, revision: 0 }); + expect(harness.getNationRevision()).toMatchObject({ leaseOwner: null, leaseExpiresAt: null, revision: 0 }); + expect(harness.generalDeleteMany).not.toHaveBeenCalled(); + }); + it('prunes deleted general and nation queues together with their journal state', () => { const harness = buildHarness(); harness.store.ensureGeneralTurns(7); diff --git a/app/game-engine/test/reservedTurnLeaseRetry.test.ts b/app/game-engine/test/reservedTurnLeaseRetry.test.ts new file mode 100644 index 00000000..768b88e9 --- /dev/null +++ b/app/game-engine/test/reservedTurnLeaseRetry.test.ts @@ -0,0 +1,38 @@ +import { describe, expect, it, vi } from 'vitest'; +import { createReservedTurnLeaseRetry } from '../src/turn/reservedTurnLeaseRetry.js'; +import { ReservedTurnLeaseBusyError, ReservedTurnLeaseConflictError } from '../src/turn/reservedTurnStore.js'; + +describe('reserved turn acquisition retry', () => { + it('bounds the entire contention episode across different queues and resets after progress', async () => { + let now = 0; + const onWait = vi.fn(); + const retry = createReservedTurnLeaseRetry({ + now: () => now, + sleep: async (ms) => { + now += ms; + }, + maxWaitMs: 2_500, + onWait, + }); + expect(await retry.wait(new ReservedTurnLeaseBusyError('general:1'))).toBe(true); + expect(await retry.wait(new ReservedTurnLeaseBusyError('nation:2:12'))).toBe(true); + expect(await retry.wait(new ReservedTurnLeaseBusyError('general:3'))).toBe(true); + expect(now).toBe(2_500); + expect(await retry.wait(new ReservedTurnLeaseBusyError('general:1'))).toBe(false); + expect(onWait).toHaveBeenCalledTimes(1); + retry.reset(); + expect(await retry.wait(new ReservedTurnLeaseBusyError('general:1'))).toBe(true); + }); + + it('never retries flush ownership loss, calculation errors, or failed savepoint restoration', async () => { + const sleep = vi.fn(); + const retry = createReservedTurnLeaseRetry({ sleep }); + for (const error of [ + new ReservedTurnLeaseConflictError('general:1'), + new Error('calculation failed'), + new AggregateError([new ReservedTurnLeaseBusyError('general:1')], 'restore failed'), + ]) + expect(await retry.wait(error)).toBe(false); + expect(sleep).not.toHaveBeenCalled(); + }); +}); diff --git a/app/game-engine/test/runtimePauseGate.test.ts b/app/game-engine/test/runtimePauseGate.test.ts index 364b2dc0..8f115f5f 100644 --- a/app/game-engine/test/runtimePauseGate.test.ts +++ b/app/game-engine/test/runtimePauseGate.test.ts @@ -3,12 +3,36 @@ import type { GameClockPhase } from '@sammo-ts/common'; import { createRuntimePauseGate } from '../src/turn/runtimePauseGate.js'; describe('runtime pause clock boundary', () => { + it('awaits a host authority check before reading profile state or changing the clock', async () => { + const calls: string[] = []; + const gate = createRuntimePauseGate({ + assertLease: async () => { + await Promise.resolve(); + calls.push('host'); + throw new Error('host changed'); + }, + shouldPause: async () => { + calls.push('profile'); + return false; + }, + isExplicitlyPaused: () => false, + getPhase: () => 'RUNNING', + prepareRecovery: async () => { + calls.push('recovery'); + }, + synchronize: async () => {}, + }); + await expect(gate()).rejects.toThrow('host changed'); + expect(calls).toEqual(['host']); + }); it('freezes a live error pause before commands, resumes without restart, and does not freeze twice', async () => { let phase: GameClockPhase = 'RUNNING'; let paused = true; const calls: string[] = []; const gate = createRuntimePauseGate({ - assertLease: () => calls.push('lease'), + assertLease: () => { + calls.push('lease'); + }, shouldPause: async () => paused, isExplicitlyPaused: () => paused, getPhase: () => phase, diff --git a/app/game-engine/test/turnDaemonLease.integration.test.ts b/app/game-engine/test/turnDaemonLease.integration.test.ts index 2bc81666..e2222c53 100644 --- a/app/game-engine/test/turnDaemonLease.integration.test.ts +++ b/app/game-engine/test/turnDaemonLease.integration.test.ts @@ -207,6 +207,39 @@ integration('database turn daemon lease and fencing', () => { expect(row.ownerId).toBe(firstToken ? 'owner-a' : 'owner-b'); }); + it('fences a host boot change inside a transaction before mutation, without expiring the DB lease', async () => { + const profile = `${profilePrefix}host-boot`; + let changed = false; + const lease = await DatabaseTurnDaemonLease.connect(databaseUrl!, { + profile, + ownerId: 'saved-vm-owner', + heartbeat: false, + leaseDurationMs: 60_000, + hostBootMonitor: { hasChanged: async () => changed }, + }); + leases.push(lease); + await lease.acquire(); + await lease.markClockReady(); + changed = true; + let committed = false; + await expect( + db.$transaction(async (transaction) => { + await lease.assertActive(transaction); + committed = true; + }) + ).rejects.toBeInstanceOf(TurnDaemonLeaseLostError); + expect(committed).toBe(false); + expect(lease.isLost()).toBe(true); + expect(await lease.renew()).toBe(false); + // The monitor does not force-clear ownership or bypass the successor's fencing. + expect((await db.turnDaemonLease.findUniqueOrThrow({ where: { profile } })).ownerId).toBe('saved-vm-owner'); + const next = await createLease(profile, 'fresh-vm-owner'); + expect(await next.acquire()).toBeNull(); + await lease.release(); + expect((await db.turnDaemonLease.findUniqueOrThrow({ where: { profile } })).clockReady).toBe(false); + expect((await next.acquire())?.fencingEpoch).toBe(2n); + }); + it('persists and evaluates lease wall time as UTC under a non-UTC database session', async () => { const profile = `${profilePrefix}utc-wall`; const zonedUrl = new URL(databaseUrl!); diff --git a/app/game-engine/test/turnDaemonLifecycle.test.ts b/app/game-engine/test/turnDaemonLifecycle.test.ts index 68058da1..957bec07 100644 --- a/app/game-engine/test/turnDaemonLifecycle.test.ts +++ b/app/game-engine/test/turnDaemonLifecycle.test.ts @@ -1,6 +1,8 @@ import { describe, expect, it, vi } from 'vitest'; import { ManualClock } from '@sammo-ts/common'; import { TurnDaemonLeaseLostError } from '../src/lifecycle/databaseTurnDaemonLease.js'; +import { ReservedTurnLeaseBusyError } from '../src/turn/reservedTurnStore.js'; +import { createReservedTurnLeaseRetry } from '../src/turn/reservedTurnLeaseRetry.js'; import { InMemoryControlQueue, @@ -15,6 +17,68 @@ import { const addMinutes = (time: Date, minutes: number): Date => new Date(time.getTime() + minutes * 60_000); describe('TurnDaemonLifecycle', () => { + it('rolls back a busy queue, checks the operator gate, and retries an explicit run exactly once', async () => { + const now = new Date('2026-11-12T00:00:00Z'); + const queue = new InMemoryControlQueue(); + queue.enqueue({ type: 'run', reason: 'manual', targetTime: now.toISOString() }); + let resource = 10; + let runs = 0; + const stateManager = new EngineStateManager(); + stateManager.register('resource', { + capture: () => resource, + restore: (value) => { + resource = value; + }, + }); + const onRunError = vi.fn(); + const flushChanges = vi.fn(); + const gate = vi.fn(async () => false); + const retry = createReservedTurnLeaseRetry({ + sleep: async () => { + expect(resource).toBe(10); + }, + }); + const lifecycle = new TurnDaemonLifecycle( + { + clock: new ManualClock(now.getTime()), + controlQueue: queue, + stateManager, + runErrorRetry: retry, + getNextTickTime: (value) => addMinutes(value, 5), + pauseGate: gate, + stateStore: { + loadLastTurnTime: async () => now, + loadNextGeneralTurnTime: async () => null, + saveLastTurnTime: async () => {}, + loadCheckpoint: async () => undefined, + saveCheckpoint: async () => {}, + }, + processor: { + run: async () => { + resource -= 2; + if (++runs === 1) throw new ReservedTurnLeaseBusyError('general:7'); + queue.enqueue({ type: 'shutdown' }); + return { + lastTurnTime: now.toISOString(), + processedGenerals: 1, + processedTurns: 0, + durationMs: 0, + partial: false, + }; + }, + }, + hooks: { onRunError, flushChanges }, + }, + { profile: 'che', defaultBudget: { budgetMs: 100, maxGenerals: 1, catchUpCap: 1 } } + ); + await lifecycle.start(); + expect(runs).toBe(2); + expect(resource).toBe(8); + expect(gate.mock.calls.length).toBeGreaterThanOrEqual(2); + expect(flushChanges).toHaveBeenCalledOnce(); + expect(onRunError).not.toHaveBeenCalled(); + }); + it.each([false, true])('reports a fatal lease gate and exits even if reporting fails (%s)', async (reportFails) => { const now = new Date('2026-09-09T17:30:00Z'); const error = new TurnDaemonLeaseLostError('che:default'); diff --git a/docs/developer/host-boot-recovery.md b/docs/developer/host-boot-recovery.md new file mode 100644 index 00000000..f98e19de --- /dev/null +++ b/docs/developer/host-boot-recovery.md @@ -0,0 +1,149 @@ +# Hyper-V 저장·복원 이후 게임 복구 + +Windows 재부팅 때 Linux VM이 저장·복원되면 게스트의 shutdown/resume hook이 +실행되지 않을 수 있다. 게임은 종료 이벤트 없이도 기존 lease 만료와 DB snapshot +재적재로 복구한다. Windows 식별자 확인은 이 경로에 추가하는 선택 기능이다. + +## 예약턴 임대 복구 + +- 실행 루프가 종료되면 DB에서 자기 owner의 장수·국가 예약턴 임대를 해제한다. + In-memory rollback으로 leased set에서 빠진 row도 정리한다. 큐 내용과 revision, + 다른 실행자/API의 임대는 변경하지 않는다. +- 계산 전 임대 획득 충돌은 전체 메모리 savepoint를 복원한 뒤 1초 단위로 재시도한다. + 한 충돌 구간의 상한은 301초다(기본 queue lease 5분 + 여유 1초). 그동안 다음 + loop마다 운영자 pause/stop과 daemon 권위를 확인한다. 성공한 batch에서 상한을 + 초기화하며 큐가 바뀌어도 같은 실패 구간의 상한을 연장하지 않는다. +- 저장·finalize 단계의 소유권 상실, 메모리 복원 실패와 다른 게임 오류는 자동 + 재시도하지 않는다. 상한을 넘은 획득 충돌도 기존 PAUSED/장애 이력 경로로 간다. +- daemon의 30초 lease와 fencing epoch, 기존 10분 이하 catch-up 및 장시간 복구 + 정책은 유지한다. 운영자가 정지한 profile을 자동 START하지 않는다. + +## Windows Node.js 에이전트 + +필요 파일은 `windows-boot-agent.mjs` 하나이며 외부 npm package가 필요 없다. +Node.js 24.22.1 이상을 사용한다. 설정 예시는 +`tools/host-recovery/windows-boot-agent.example.json`이다. + +Node.js가 Windows 내장 PowerShell의 CIM 조회로 `Win32_OperatingSystem.LastBootUpTime` +UTC ticks를 읽고 hostname과 함께 hash한다. 별도의 PowerShell 스크립트 설치나 +Linux에서 Windows 관리자 명령 실행은 필요 없다. Node 에이전트를 재시작해도 +같은 Windows boot의 식별자는 같고, Windows를 재부팅하면 달라진다. +실제 Windows에서 CIM 조회가 실패하면 가짜 식별자를 만들지 않고 시작을 거부한다. + +1. 파일들을 `C:\ProgramData\Sammo` 같은 고정 경로에 복사한다. +2. 디렉터리를 만든 뒤 Windows 터미널에서 private key를 생성한다. 기존 파일을 + 덮어쓰지 않으며 key 내용은 출력하지 않는다. + + ```text + node C:\ProgramData\Sammo\windows-boot-agent.mjs --create-key C:\ProgramData\Sammo\host-boot.key + ``` + +3. 설정 예시를 `C:\ProgramData\Sammo\host-boot.json`으로 복사한다. `keyFile`은 + 위 경로, `bindAddress`는 **Linux VM에서 접근할 수 있는 Windows의 사설 주소**로 + 설정한다. 예시의 `127.0.0.1`은 안전한 기본값이며 원격 Linux에서 접근할 수 없다. + `port` 기본값은 17781이다. Windows 방화벽은 해당 사설 listener에 Linux VM + 주소에서 오는 TCP 연결만 허용하도록 설정한다. 기존 공개 Caddy는 변경하지 않는다. +4. key 파일은 실행 계정만 읽도록 NTFS 권한을 제한하고, 같은 내용을 Linux의 + `/run/secrets/host_boot_key` 같은 private file로 비공개 전달한다. Git·메신저· + CLI 인자에 key 내용을 넣지 않는다. 에이전트는 key를 시작할 때 읽으므로 key + 교체 시 양쪽 프로세스를 함께 재시작한다. +5. 실행한다. + + ```text + node C:\ProgramData\Sammo\windows-boot-agent.mjs --config C:\ProgramData\Sammo\host-boot.json + ``` + +6. Windows 작업 스케줄러에 **시스템 시작 시** task를 등록한다. 프로그램은 Node의 + 절대 `node.exe` 경로, 인자는 위 `.mjs --config ...` 부분을 사용한다. 로그인 없이 + 실행되고 실패 시 재시작하도록 구성한다. 설정 파일을 사용하므로 터미널 세션의 + 임시 환경변수에 의존하지 않는다. 실제 Windows 설치·방화벽 변경·task 등록은 + 대상 host에서 별도 적용하고 확인해야 한다. + +원한다면 설정 파일 대신 `HOST_BOOT_KEY_FILE`, `HOST_BOOT_BIND_ADDRESS`, +`HOST_BOOT_PORT` 환경변수로도 실행할 수 있다. + +## Linux 연결 + +기존 game runtime/PM2가 상속하는 서버 환경에 아래 두 값을 함께 설정한다. +설정이 없으면 기능을 사용하지 않는다. 설정이 불완전하거나 key 파일이 없거나 잘못돼도 +고정 경고와 함께 보조 관측만 비활성화하며 게임 기동은 계속한다. + +```dotenv +TURN_HOST_BOOT_URL=http://windows-private-address:17781/v1/boot-id +TURN_HOST_BOOT_KEY_FILE=/run/secrets/host_boot_key +``` + +Docker에서는 private key가 들어 있는 전용 디렉터리를 runtime에 read-only mount하고 두 환경변수를 +**runtime container environment**에 전달해야 한다. 호스트의 `.env`에만 추가하면 +자동 전달되지 않는다. 기존 Compose와 host overlay를 함께 적용하는 전용 wrapper를 +사용한다. 실제 secret 파일의 host 경로와 Windows 사설 주소를 확인하기 전에는 +운영 stack 설정을 추측해 바꾸지 않는다. + +예를 들어 host overlay의 runtime 설정에는 다음 형태로 연결할 수 있다. host 경로는 +placeholder이며 실제 운영 경로로 교체해야 한다. + +```yaml +services: + runtime: + environment: + TURN_HOST_BOOT_URL: ${TURN_HOST_BOOT_URL:-} + TURN_HOST_BOOT_KEY_FILE: /run/sammo-host-recovery/host_boot.key + volumes: + - type: bind + source: /absolute/private/path/host-recovery + target: /run/sammo-host-recovery + read_only: true + bind: + create_host_path: true +``` + +위 디렉터리는 보조 기능 전용이므로 없으면 빈 디렉터리를 만들 수 있다. 파일 부재는 +관측만 비활성화한다. DB·backup 같은 필수 저장소에는 이 fallback을 적용하지 않는다. +운영에는 이 전용 디렉터리의 `host_boot.key` 파일을 전달한다. + +처음 인증에 성공한 식별자를 프로세스의 기준으로 삼고 고정된 성공 로그를 남긴다. 이후 다른 식별자를 +확인하면 `TurnDaemonLeaseLostError`로 기존 실행자를 중단하고 DB를 다시 읽는다. +종료 시 owner/epoch가 여전히 자기 것인 경우에만 DB lease와 `clock_ready`를 +해제하므로 다른 실행자를 중지시키지 않는다. 재시작은 기존 profile 상태를 +보존하며 PAUSED/STOPPED를 임의로 RUNNING으로 바꾸지 않는다. + +보통 최대 2초 간격으로 확인한다. 벽시계와 monotonic 경과가 1초 이상 어긋나면 +cache를 무시하고 다시 조회한다. 조회 timeout은 1초다. 네트워크·인증·응답 오류는 +분당 한 번의 고정 경고만 남기고 기존 DB lease 복구를 유지한다. 처음 인증받기 전 +발생한 reboot는 이 신호만으로 확정할 수 없으며 기존 복구 경로가 담당한다. + +## 인증 프로토콜과 검증 + +key 파일은 32바이트 난수를 lowercase hex 64자로 저장한다. 매 요청마다 새 32바이트 +challenge를 만들며, key 자체를 네트워크에 보내지 않는다. + +- GET `/v1/boot-id`, query string 없음. +- `x-sammo-challenge`: lowercase hex 64자. +- `x-sammo-signature`: `HMAC-SHA256(key, "sammo-host-boot-request-v1\n" + challenge)`. +- JSON: `version=1`, `challenge`, `bootId`(hex 64자), `signature`. +- 응답 signature: + `HMAC-SHA256(key, "sammo-host-boot-response-v1\n" + challenge + "\n" + bootId)`. + +위 `\n`은 실제 LF다. Linux는 challenge와 응답 signature를 검증하고, redirect와 +2048바이트 초과 응답을 거부한다. 서명된 옛 응답도 새 challenge에는 유효하지 않다. +시각 정확도나 요청 시각 만료에 의존하지 않는다. 이 프로토콜은 인증과 무결성을 +제공하며 암호화는 하지 않는다. 사설 망에서만 사용하며 공개 무인증 재개 API를 +만들지 않는다. + +```sh +node --test tools/host-recovery/windows-boot-agent.test.mjs +pnpm --filter @sammo-ts/game-engine test hostBootMonitor.test.ts runtimePauseGate.test.ts reservedTurnLeaseRetry.test.ts +``` + +CI에서는 Windows boot를 모사한 HTTP 응답, 위조·재사용·timeout, 실제 PostgreSQL +lease fencing과 queue cleanup을 검증한다. 실제 Windows CIM·Task Scheduler와 +Hyper-V 재부팅 검증을 대체하지 않는다. 배포 후 기존 장애 진단 화면의 lease/ +epoch와 public lobby `turnEngineRunning`, 실제 tick/queue revision 진행을 확인한다. + +기능을 해제하려면 Linux의 두 환경변수를 함께 제거하고 해당 runtime을 정상 +재기동한다. Windows task도 중지할 수 있다. DB migration이나 게임 초기화는 없다. + +근거: [Microsoft LastBootUpTime](https://learn.microsoft.com/en-us/windows/win32/cimwin32prov/win32-operatingsystem), +[Node.js HMAC와 timingSafeEqual](https://nodejs.org/api/crypto.html), +[게임 시계 복구 계약](../architecture/game-clock-reconciliation.md), +[릴리스 운영](../release-operations.md). diff --git a/docs/release-operations.md b/docs/release-operations.md index f5678dcb..adfcdf82 100644 --- a/docs/release-operations.md +++ b/docs/release-operations.md @@ -363,6 +363,18 @@ Hyper-V의 VM 정지/재개, 체크포인트 또는 host 시간 보정은 게스 NTP 로그와 Windows Hyper-V/백업 작업 이력을 대조합니다. lease 기간을 늘리거나 시간 동기화를 임의로 끄는 것으로 해결하지 않습니다. +예약턴의 실행 전 임대 충돌은 메모리 rollback 뒤 1초 간격, 한 구간 최대 301초 동안 +해제·만료를 기다립니다. flush/finalize의 ownership 상실은 재시도하지 않습니다. +종료 시에는 rollback으로 journal에서 사라진 자기 owner의 queue 임대도 DB에서 +해제합니다. 다른 owner와 queue 내용/revision은 보존합니다. 대기 중에도 매 loop의 +운영자 pause/stop과 daemon gate를 통과하며 상한을 넘으면 기존 오류 정지를 유지합니다. + +Windows의 저장·복원으로 guest 이벤트가 없을 때를 위한 선택적 boot identity 확인은 +[Windows Node.js 에이전트 설치와 Linux 연결](./developer/host-boot-recovery.md)을 따릅니다. +`TURN_HOST_BOOT_URL`과 private `TURN_HOST_BOOT_KEY_FILE`을 함께 설정하면 인증된 +식별자 변경을 기존 lease 상실/재기동으로 처리합니다. 통신 실패는 기존 복구를 +막지 않으며 운영자의 PAUSED/STOPPED를 자동 해제하지 않습니다. + Turn daemon의 DB persistence interactive transaction은 연결 획득 대기(`maxWait`) 10초와 callback 실행(`timeout`) 30초를 각각 제한합니다. Prisma 기본 연결 대기 2초는 일시적인 DB 지연에도 callback 시작 전 `P2028`로 서버를 PAUSED로 만들 수 diff --git a/tools/host-recovery/README.md b/tools/host-recovery/README.md new file mode 100644 index 00000000..8088b60f --- /dev/null +++ b/tools/host-recovery/README.md @@ -0,0 +1,12 @@ +# Windows host boot identity agent + +`windows-boot-agent.mjs`는 외부 npm package 없이 실행하는 Node.js 에이전트다. +Windows OS의 부팅 시각 기반 식별자를 인증된 Linux 게임 데몬에 제공한다. + +설치, key 생성, 자동 시작과 선택 기능 장애 시 동작은 +[Hyper-V 복구 운영 안내](../../docs/developer/host-boot-recovery.md)를 따른다. +설정 예시는 `windows-boot-agent.example.json`이며 실제 key와 설정은 Git에 넣지 않는다. + +```sh +node --test tools/host-recovery/windows-boot-agent.test.mjs +``` diff --git a/tools/host-recovery/windows-boot-agent.example.json b/tools/host-recovery/windows-boot-agent.example.json new file mode 100644 index 00000000..47a77db4 --- /dev/null +++ b/tools/host-recovery/windows-boot-agent.example.json @@ -0,0 +1,5 @@ +{ + "keyFile": "C:\\ProgramData\\Sammo\\host-boot.key", + "bindAddress": "127.0.0.1", + "port": 17781 +} diff --git a/tools/host-recovery/windows-boot-agent.mjs b/tools/host-recovery/windows-boot-agent.mjs new file mode 100644 index 00000000..fd8f2cf0 --- /dev/null +++ b/tools/host-recovery/windows-boot-agent.mjs @@ -0,0 +1,131 @@ +import { execFile } from 'node:child_process'; +import { createHash, createHmac, randomBytes, timingSafeEqual } from 'node:crypto'; +import { readFile, writeFile } from 'node:fs/promises'; +import { createServer } from 'node:http'; +import { hostname } from 'node:os'; +import { win32, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { promisify } from 'node:util'; + +const mac = (key, value) => createHmac('sha256', key).update(value).digest('hex'); + +export const bootIdentity = (host, ticks) => { + if (!/^\d{17,20}$/.test(ticks)) throw new Error('Windows returned an invalid boot time.'); + return createHash('sha256').update(`${host.toLowerCase()}\n${ticks}`).digest('hex'); +}; + +export const readWindowsBootIdentity = async () => { + if (process.platform !== 'win32') throw new Error('Run the host boot agent on Windows.'); + const executable = win32.join( + process.env.SystemRoot ?? 'C:\\Windows', + 'System32', + 'WindowsPowerShell', + 'v1.0', + 'powershell.exe' + ); + // Node.js가 OS 내장 CIM 조회를 실행한다. 별도 PowerShell 스크립트/모듈은 필요 없다. + const { stdout } = await promisify(execFile)( + executable, + [ + '-NoProfile', + '-NonInteractive', + '-Command', + '(Get-CimInstance -ClassName Win32_OperatingSystem -Property LastBootUpTime).LastBootUpTime.ToUniversalTime().Ticks.ToString()', + ], + { windowsHide: true, timeout: 10_000, maxBuffer: 2_048 } + ); + return bootIdentity(hostname(), stdout.trim()); +}; + +export const createBootIdentityServer = ({ key, getBootId }) => { + if (!Buffer.isBuffer(key) || key.length !== 32) throw new Error('A 32-byte host boot key is required.'); + const server = createServer((request, response) => { + response.setHeader('Cache-Control', 'no-store'); + if (request.method !== 'GET' || request.url !== '/v1/boot-id') { + response.writeHead(404).end(); + return; + } + const challenge = request.headers['x-sammo-challenge']; + const signature = request.headers['x-sammo-signature']; + if ( + typeof challenge !== 'string' || + !/^[a-f0-9]{64}$/.test(challenge) || + typeof signature !== 'string' || + !/^[a-f0-9]{64}$/.test(signature) || + !timingSafeEqual( + Buffer.from(signature, 'hex'), + Buffer.from(mac(key, `sammo-host-boot-request-v1\n${challenge}`), 'hex') + ) + ) { + response.writeHead(401).end(); + return; + } + const bootId = getBootId(); + if (typeof bootId !== 'string' || !/^[a-f0-9]{64}$/.test(bootId)) { + response.writeHead(503).end(); + return; + } + response.writeHead(200, { 'Content-Type': 'application/json' }); + response.end( + JSON.stringify({ + version: 1, + challenge, + bootId, + signature: mac(key, `sammo-host-boot-response-v1\n${challenge}\n${bootId}`), + }) + ); + }); + server.headersTimeout = 5_000; + server.requestTimeout = 5_000; + server.keepAliveTimeout = 1_000; + server.maxHeadersCount = 20; + return server; +}; + +export const main = async (args = process.argv.slice(2), env = process.env) => { + if (args.length === 2 && args[0] === '--create-key') { + await writeFile(args[1], `${randomBytes(32).toString('hex')}\n`, { flag: 'wx', mode: 0o600 }); + console.info('Host boot key created. Copy it privately to the Linux secret file.'); + return; + } + if (args.length === 2 && args[0] === '--config') { + const config = JSON.parse(await readFile(args[1], 'utf8')); + if ( + !config || + typeof config.keyFile !== 'string' || + typeof config.bindAddress !== 'string' || + !Number.isInteger(config.port) + ) { + throw new Error('Host boot agent configuration is invalid.'); + } + env = { + ...env, + HOST_BOOT_KEY_FILE: config.keyFile, + HOST_BOOT_BIND_ADDRESS: config.bindAddress, + HOST_BOOT_PORT: String(config.port), + }; + } else if (args.length) throw new Error('Usage: node windows-boot-agent.mjs [--create-key path | --config path]'); + if (!env.HOST_BOOT_KEY_FILE) throw new Error('HOST_BOOT_KEY_FILE is required.'); + const encoded = (await readFile(env.HOST_BOOT_KEY_FILE, 'utf8')).trim(); + if (!/^[a-f0-9]{64}$/.test(encoded)) throw new Error('Host boot key file is invalid.'); + const port = Number(env.HOST_BOOT_PORT ?? '17781'); + if (!Number.isInteger(port) || port < 1 || port > 65535) throw new Error('HOST_BOOT_PORT is invalid.'); + // OS 부팅 시각은 agent 재시작에도 같으며, 시각 차이 계산으로 재부팅을 추정하지 않는다. + const bootId = await readWindowsBootIdentity(); + const server = createBootIdentityServer({ key: Buffer.from(encoded, 'hex'), getBootId: () => bootId }); + server.on('error', () => { + console.error('Host boot agent listener failed.'); + process.exitCode = 1; + }); + server.listen(port, env.HOST_BOOT_BIND_ADDRESS ?? '127.0.0.1', () => { + console.info('Host boot identity agent is listening.'); + }); + for (const signal of ['SIGINT', 'SIGTERM']) process.once(signal, () => server.close()); +}; + +if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { + main().catch(() => { + console.error('Host boot agent startup failed; check OS, private key file, and listener configuration.'); + process.exitCode = 1; + }); +} diff --git a/tools/host-recovery/windows-boot-agent.test.mjs b/tools/host-recovery/windows-boot-agent.test.mjs new file mode 100644 index 00000000..9e053d6d --- /dev/null +++ b/tools/host-recovery/windows-boot-agent.test.mjs @@ -0,0 +1,42 @@ +import { after, test } from 'node:test'; +import assert from 'node:assert/strict'; +import { createHmac, randomBytes, timingSafeEqual } from 'node:crypto'; +import { once } from 'node:events'; +import { bootIdentity, createBootIdentityServer } from './windows-boot-agent.mjs'; + +test('agent restart preserves OS boot identity; a new OS boot changes it', () => { + assert.equal(bootIdentity('HOST', '638985120000000000'), bootIdentity('host', '638985120000000000')); + assert.notEqual(bootIdentity('HOST', '638985120000000000'), bootIdentity('HOST', '638985120000000001')); + assert.throws(() => bootIdentity('HOST', 'invalid')); +}); + +test('HTTP transport rejects anonymous requests and signs the current challenge and boot identity', async () => { + const key = randomBytes(32); + let bootId = 'a'.repeat(64); + const server = createBootIdentityServer({ key, getBootId: () => bootId }); + server.listen(0, '127.0.0.1'); + await once(server, 'listening'); + const url = `http://127.0.0.1:${server.address().port}/v1/boot-id`; + after( + () => + new Promise((resolve) => { + server.close(resolve); + server.closeAllConnections(); + }) + ); + assert.equal((await fetch(url)).status, 401); + assert.equal((await fetch(`${url}?bypass=1`)).status, 404); + const challenge = randomBytes(32).toString('hex'); + const signature = createHmac('sha256', key).update(`sammo-host-boot-request-v1\n${challenge}`).digest('hex'); + const headers = { 'x-sammo-challenge': challenge, 'x-sammo-signature': signature }; + const response = await fetch(url, { headers }); + assert.equal(response.status, 200); + assert.equal(response.headers.get('cache-control'), 'no-store'); + const body = await response.json(); + assert.equal(body.challenge, challenge); + const expected = createHmac('sha256', key).update(`sammo-host-boot-response-v1\n${challenge}\n${bootId}`).digest(); + assert.ok(timingSafeEqual(expected, Buffer.from(body.signature, 'hex'))); + bootId = 'b'.repeat(64); + assert.equal((await (await fetch(url, { headers })).json()).bootId, bootId); + assert.equal((await fetch(url, { headers: { ...headers, 'x-sammo-signature': '0'.repeat(64) } })).status, 401); +});