운영 안정화 변경을 최신 main과 통합한다

This commit is contained in:
Hide_D committed 2026-11-14 15:13:53 +09:00
commit 087517821e
47 files changed
+1558 -123

No files matched your search

+3
View File
@@ -91,6 +91,9 @@ DAEMON_REQUEST_TIMEOUT_MS=5000
# realtime은 벽시계 경과를 game tick으로 투영합니다. manual은 daemon이 # realtime은 벽시계 경과를 game tick으로 투영합니다. manual은 daemon이
# 다음 월 snapshot을 대기 없이 실행하며 테스트/장기 시뮬레이션에 사용합니다. # 다음 월 snapshot을 대기 없이 실행하며 테스트/장기 시뮬레이션에 사용합니다.
GAME_CLOCK_MODE=realtime GAME_CLOCK_MODE=realtime
# Optional Windows host boot identity. Key is a private file, never a VITE_* value.
# TURN_HOST_BOOT_URL=http://windows-private-address:17781/v1/boot-id
# TURN_HOST_BOOT_KEY_FILE=/run/secrets/host_boot_key
TRPC_PATH=/trpc TRPC_PATH=/trpc
# Frontend public URLs # Frontend public URLs
+3
View File
@@ -16,6 +16,9 @@ GAME_API_HOST=0.0.0.0
GAME_API_PORT=14000 GAME_API_PORT=14000
PROFILE=hwe PROFILE=hwe
SCENARIO=default SCENARIO=default
# Optional Windows observation; unavailable or unconfigured service never blocks gameplay.
# TURN_HOST_BOOT_URL=http://windows-private-address:17781/v1/boot-id
# TURN_HOST_BOOT_KEY_FILE=/run/sammo-host-recovery/host_boot.key
KAKAO_REST_KEY=your-kakao-rest-key KAKAO_REST_KEY=your-kakao-rest-key
KAKAO_ADMIN_KEY=your-kakao-admin-key KAKAO_ADMIN_KEY=your-kakao-admin-key
KAKAO_REDIRECT_URI=https://example.com/oauth/kakao/callback KAKAO_REDIRECT_URI=https://example.com/oauth/kakao/callback
+1
View File
@@ -89,6 +89,7 @@ web_modules/
.env.local .env.local
.env.ci .env.ci
.env.ops .env.ops
tools/host-recovery/*.key
tools/legacy-db-migration/migration-plan.json tools/legacy-db-migration/migration-plan.json
tools/legacy-db-migration/secrets/ tools/legacy-db-migration/secrets/
+12 -6
View File
@@ -76,7 +76,7 @@ export class ReadModelOutboxWorker implements ReadModelOutboxWakeup {
this.leaseMs = normalizePositiveInteger(options.leaseMs, 30_000); this.leaseMs = normalizePositiveInteger(options.leaseMs, 30_000);
this.retentionMs = normalizePositiveInteger(options.retentionMs, 24 * 60 * 60 * 1_000); this.retentionMs = normalizePositiveInteger(options.retentionMs, 24 * 60 * 60 * 1_000);
this.pruneIntervalMs = normalizePositiveInteger(options.pruneIntervalMs, 60_000); this.pruneIntervalMs = normalizePositiveInteger(options.pruneIntervalMs, 60_000);
this.pruneLimit = normalizePositiveInteger(options.pruneLimit, 100); this.pruneLimit = Math.min(500, normalizePositiveInteger(options.pruneLimit, 500));
this.owner = options.owner ?? `game-api:${profileName}:${process.pid}:${randomUUID()}`; this.owner = options.owner ?? `game-api:${profileName}:${process.pid}:${randomUUID()}`;
this.now = options.now ?? (() => new Date()); this.now = options.now ?? (() => new Date());
this.onError = options.onError ?? (() => undefined); this.onError = options.onError ?? (() => undefined);
@@ -109,11 +109,17 @@ export class ReadModelOutboxWorker implements ReadModelOutboxWakeup {
const now = this.now(); const now = this.now();
if (now.getTime() >= this.nextPruneAt) { if (now.getTime() >= this.nextPruneAt) {
this.nextPruneAt = now.getTime() + this.pruneIntervalMs; // backlog가 있으면 다음 dispatch에서도 이어서 정리한다. 평상시에는 낮은 빈도로 확인한다.
await pruneDeliveredReadModelOutbox(this.db, { // 한 호출은 최대 10 batch/50ms이며 각 SQL도 최대 500행이다.
deliveredBefore: new Date(now.getTime() - this.retentionMs), const started = performance.now();
limit: this.pruneLimit, for (let batch = 0; batch < 10; batch += 1) {
}); const deleted = await pruneDeliveredReadModelOutbox(this.db, {
deliveredBefore: new Date(now.getTime() - this.retentionMs),
limit: this.pruneLimit,
});
this.nextPruneAt = deleted < this.pruneLimit ? now.getTime() + this.pruneIntervalMs : 0;
if (deleted < this.pruneLimit || performance.now() - started >= 50) break;
}
} }
return result; return result;
} }
@@ -1,6 +1,6 @@
import { TRPCError } from '@trpc/server'; import { TRPCError } from '@trpc/server';
import { z } from 'zod'; import { z } from 'zod';
import type { GamePrisma } from '@sammo-ts/infra'; import { decodeAuditDecisionSteps, type GamePrisma } from '@sammo-ts/infra';
import { auditProcedure, monthOrdinal, readAudit, readAuditWorld, zAuditMonth } from './shared.js'; import { auditProcedure, monthOrdinal, readAudit, readAuditWorld, zAuditMonth } from './shared.js';
const zEffectivePolicy = z.object({ const zEffectivePolicy = z.object({
@@ -275,7 +275,7 @@ export const decisionDetail = auditProcedure
decision: project(row), decision: project(row),
chunks: chunks.slice(0, input.limit).map((chunk) => ({ chunks: chunks.slice(0, input.limit).map((chunk) => ({
ordinal: chunk.ordinal, ordinal: chunk.ordinal,
steps: z.array(zStep).max(128).parse(chunk.steps), steps: z.array(zStep).max(128).parse(decodeAuditDecisionSteps(chunk.steps)),
})), })),
nextCursor: nextCursor:
chunks.length && chunks.at(-1)!.ordinal + 1 < Math.ceil(row.stepCount / 128) chunks.length && chunks.at(-1)!.ordinal + 1 < Math.ceil(row.stepCount / 128)
@@ -2,7 +2,12 @@ import { afterAll, beforeAll, expect, it } from 'vitest';
import fastify from 'fastify'; import fastify from 'fastify';
import { fastifyTRPCPlugin } from '@trpc/server/adapters/fastify'; import { fastifyTRPCPlugin } from '@trpc/server/adapters/fastify';
import { asRecord } from '@sammo-ts/common'; import { asRecord } from '@sammo-ts/common';
import { createGamePostgresConnector, createRedisConnector, GamePrisma } from '@sammo-ts/infra'; import {
decodeAuditDecisionSteps,
createGamePostgresConnector,
createRedisConnector,
GamePrisma,
} from '@sammo-ts/infra';
import { createTurnDaemonRuntime, seedScenarioToDatabase, type TurnDaemonRuntime } from '@sammo-ts/game-engine'; import { createTurnDaemonRuntime, seedScenarioToDatabase, type TurnDaemonRuntime } from '@sammo-ts/game-engine';
import { appRouter } from '../src/router.js'; import { appRouter } from '../src/router.js';
import { DatabaseTurnDaemonTransport } from '../src/daemon/databaseTransport.js'; import { DatabaseTurnDaemonTransport } from '../src/daemon/databaseTransport.js';
@@ -255,12 +260,10 @@ integration(
include: { chunks: { orderBy: { ordinal: 'asc' } } }, include: { chunks: { orderBy: { ordinal: 'asc' } } },
}); });
const procedures = nationDecision.chunks.flatMap((chunk) => const procedures = nationDecision.chunks.flatMap((chunk) =>
Array.isArray(chunk.steps) decodeAuditDecisionSteps(chunk.steps)
? chunk.steps .map(asRecord)
.map(asRecord) .filter((step) => step.kind === 'PROCEDURE_START')
.filter((step) => step.kind === 'PROCEDURE_START') .map((step) => step.procedure)
.map((step) => step.procedure)
: []
); );
expect(procedures).toEqual(['NPC포상']); expect(procedures).toEqual(['NPC포상']);
expect(configuration(await call('getPolicy', {}))).toEqual(configuration(withoutRecruitment)); expect(configuration(await call('getPolicy', {}))).toEqual(configuration(withoutRecruitment));
@@ -45,6 +45,28 @@ const createFixture = (rows: readonly object[]) => {
}; };
describe('ReadModelOutboxWorker', () => { describe('ReadModelOutboxWorker', () => {
it('drains more than one full retention batch rather than leaving a permanent 100-per-minute backlog', async () => {
let now = new Date('2026-11-09T00:00:00Z');
const fixture = createFixture([]);
fixture.queryRaw
.mockReset()
.mockResolvedValueOnce([])
.mockResolvedValueOnce([{ id: 1n }, { id: 2n }])
.mockResolvedValueOnce([{ id: 3n }, { id: 4n }])
.mockResolvedValueOnce([]);
const worker = new ReadModelOutboxWorker(fixture.db, fixture.redis, 'che:default', {
intervalMs: 60000,
pruneLimit: 2,
now: () => now,
});
now = new Date('2026-11-09T00:01:00Z');
worker.start();
await vi.waitFor(() => expect(fixture.queryRaw).toHaveBeenCalledTimes(4));
await worker.stop();
const query = fixture.queryRaw.mock.calls[1]?.[0] as { sql: string };
expect(query.sql).toContain('FOR UPDATE SKIP LOCKED');
expect(query.sql).toContain('ORDER BY "delivered_at", "available_at", "id"');
});
it('publishes a survey-style global front-status invalidation without a turn daemon', async () => { it('publishes a survey-style global front-status invalidation without a turn daemon', async () => {
const fixture = createFixture([{ id: 10n, payload: payload('front.global'), attempts: 1 }]); const fixture = createFixture([{ id: 10n, payload: payload('front.global'), attempts: 1 }]);
const worker = new ReadModelOutboxWorker(fixture.db, fixture.redis, 'che:default', { const worker = new ReadModelOutboxWorker(fixture.db, fixture.redis, 'che:default', {
@@ -13,6 +13,7 @@ import { afterAll, beforeAll, beforeEach, describe, expect, it } from 'vitest';
import { encryptGameSessionToken, type GameSessionTokenPayload } from '@sammo-ts/common/auth/gameToken'; import { encryptGameSessionToken, type GameSessionTokenPayload } from '@sammo-ts/common/auth/gameToken';
import { import {
createGamePostgresConnector, createGamePostgresConnector,
encodeAuditDecisionSteps,
GamePrisma, GamePrisma,
hashAuditDiplomacyDocument, hashAuditDiplomacyDocument,
createRedisConnector, createRedisConnector,
@@ -2345,50 +2346,52 @@ integration('game API security over HTTP transport', () => {
{ {
decisionId: decisionIds[0]!, decisionId: decisionIds[0]!,
ordinal: 0, ordinal: 0,
steps: Array.from({ length: 128 }, (_, sequence) => steps: encodeAuditDecisionSteps(
sequence === 127 Array.from({ length: 128 }, (_, sequence) =>
? { sequence === 127
...step, ? {
sequence, ...step,
kind: 'DECISION_START', sequence,
reservedAction: '휴식', kind: 'DECISION_START',
effectivePolicy: { reservedAction: '휴식',
schemaVersion: 1, effectivePolicy: {
general: { priority: ['징병'], flags: { 징병: true, 출병: false } }, schemaVersion: 1,
nation: { general: { priority: ['징병'], flags: { 징병: true, 출병: false } },
priority: ['천도'], nation: {
flags: { 천도: true }, priority: ['천도'],
values: { flags: { 천도: true },
reqNationGold: 4321, values: {
reqNationRice: 100, reqNationGold: 4321,
reqHumanWarUrgentGold: 100, reqNationRice: 100,
reqHumanWarUrgentRice: 100, reqHumanWarUrgentGold: 100,
reqHumanWarRecommandGold: 100, reqHumanWarUrgentRice: 100,
reqHumanWarRecommandRice: 100, reqHumanWarRecommandGold: 100,
reqHumanDevelGold: 100, reqHumanWarRecommandRice: 100,
reqHumanDevelRice: 100, reqHumanDevelGold: 100,
reqNpcWarGold: 100, reqHumanDevelRice: 100,
reqNpcWarRice: 100, reqNpcWarGold: 100,
reqNpcDevelGold: 100, reqNpcWarRice: 100,
reqNpcDevelRice: 100, reqNpcDevelGold: 100,
minimumResourceActionAmount: 100, reqNpcDevelRice: 100,
maximumResourceActionAmount: 100, minimumResourceActionAmount: 100,
minNpcWarLeadership: 100, maximumResourceActionAmount: 100,
minWarCrew: 100, minNpcWarLeadership: 100,
minNpcRecruitCityPopulation: 100, minWarCrew: 100,
safeRecruitCityPopulationRatio: 100, minNpcRecruitCityPopulation: 100,
properWarTrainAtmos: 100, safeRecruitCityPopulationRatio: 100,
cureThreshold: 100, properWarTrainAtmos: 100,
cureThreshold: 100,
},
combatForce: { '1': [2, 3] },
supportForce: [4],
developForce: [5],
secret: 'decision-secret',
}, },
combatForce: { '1': [2, 3] },
supportForce: [4],
developForce: [5],
secret: 'decision-secret', secret: 'decision-secret',
}, },
secret: 'decision-secret', }
}, : { ...step, sequence }
} )
: { ...step, sequence }
), ),
}, },
{ {
+1
View File
@@ -13,6 +13,7 @@ export * from './scenario/databaseUrl.js';
export * from './scenario/mapLoader.js'; export * from './scenario/mapLoader.js';
export * from './scenario/scenarioSeeder.js'; export * from './scenario/scenarioSeeder.js';
export * from './scenario/gameCancellation.js'; export * from './scenario/gameCancellation.js';
export * from './playAudit/retention.js';
export * from './turn/types.js'; export * from './turn/types.js';
export * from './turn/worldLoader.js'; export * from './turn/worldLoader.js';
export * from './turn/inMemoryWorld.js'; export * from './turn/inMemoryWorld.js';
@@ -2,6 +2,7 @@ import { randomUUID } from 'node:crypto';
import { performance } from 'node:perf_hooks'; import { performance } from 'node:perf_hooks';
import { createGamePostgresConnector, GamePrisma, type GamePrismaClient } from '@sammo-ts/infra'; import { createGamePostgresConnector, GamePrisma, type GamePrismaClient } from '@sammo-ts/infra';
import type { HostBootMonitor } from '../turn/hostBootMonitor.js';
export interface TurnDaemonLeaseToken { export interface TurnDaemonLeaseToken {
profile: string; profile: string;
@@ -14,6 +15,7 @@ export interface DatabaseTurnDaemonLeaseOptions {
ownerId?: string; ownerId?: string;
leaseDurationMs?: number; leaseDurationMs?: number;
heartbeat?: boolean; heartbeat?: boolean;
hostBootMonitor?: HostBootMonitor;
} }
export class TurnDaemonLeaseUnavailableError extends Error { export class TurnDaemonLeaseUnavailableError extends Error {
@@ -45,6 +47,7 @@ export class DatabaseTurnDaemonLease {
private readonly ownerId: string; private readonly ownerId: string;
private readonly leaseDurationMs: number; private readonly leaseDurationMs: number;
private readonly heartbeatEnabled: boolean; private readonly heartbeatEnabled: boolean;
private readonly hostBootMonitor?: HostBootMonitor;
private token: TurnDaemonLeaseToken | null = null; private token: TurnDaemonLeaseToken | null = null;
private heartbeatTimer: NodeJS.Timeout | null = null; private heartbeatTimer: NodeJS.Timeout | null = null;
private expiryTimer: NodeJS.Timeout | null = null; private expiryTimer: NodeJS.Timeout | null = null;
@@ -63,6 +66,7 @@ export class DatabaseTurnDaemonLease {
this.ownerId = options.ownerId ?? randomUUID(); this.ownerId = options.ownerId ?? randomUUID();
this.leaseDurationMs = normalizeLeaseDuration(options.leaseDurationMs); this.leaseDurationMs = normalizeLeaseDuration(options.leaseDurationMs);
this.heartbeatEnabled = options.heartbeat ?? true; this.heartbeatEnabled = options.heartbeat ?? true;
this.hostBootMonitor = options.hostBootMonitor;
} }
static async connect( static async connect(
@@ -75,6 +79,8 @@ export class DatabaseTurnDaemonLease {
} }
async acquire(): Promise<TurnDaemonLeaseToken | null> { async acquire(): Promise<TurnDaemonLeaseToken | null> {
await this.checkHostBoot();
if (this.lost) throw this.getLossError();
const requestStartedAt = performance.now(); const requestStartedAt = performance.now();
const rows = await this.db.$queryRaw<LeaseRow[]>(GamePrisma.sql` const rows = await this.db.$queryRaw<LeaseRow[]>(GamePrisma.sql`
INSERT INTO "turn_daemon_lease" ( INSERT INTO "turn_daemon_lease" (
@@ -145,6 +151,12 @@ export class DatabaseTurnDaemonLease {
return new TurnDaemonLeaseLostError(this.profile, this.lossReason); return new TurnDaemonLeaseLostError(this.profile, this.lossReason);
} }
async checkHostBoot(): Promise<void> {
if (!this.lost && (await this.hostBootMonitor?.hasChanged())) {
this.markLost('Host boot identity changed; restart with a fresh database snapshot.');
}
}
async renew(): Promise<boolean> { async renew(): Promise<boolean> {
const token = this.token; const token = this.token;
if (!token || this.lost || this.renewalInFlight) { if (!token || this.lost || this.renewalInFlight) {
@@ -153,6 +165,8 @@ export class DatabaseTurnDaemonLease {
const requestStartedAt = performance.now(); const requestStartedAt = performance.now();
this.renewalInFlight = true; this.renewalInFlight = true;
try { try {
await this.checkHostBoot();
if (this.lost) return false;
const rows = await this.db.$queryRaw<LeaseRow[]>(GamePrisma.sql` const rows = await this.db.$queryRaw<LeaseRow[]>(GamePrisma.sql`
UPDATE "turn_daemon_lease" UPDATE "turn_daemon_lease"
SET SET
@@ -180,6 +194,7 @@ export class DatabaseTurnDaemonLease {
} }
async assertActive(transaction?: GamePrisma.TransactionClient): Promise<void> { async assertActive(transaction?: GamePrisma.TransactionClient): Promise<void> {
await this.checkHostBoot();
const token = this.token; const token = this.token;
if (!token || this.lost) { if (!token || this.lost) {
throw this.getLossError(); throw this.getLossError();
@@ -207,12 +222,13 @@ export class DatabaseTurnDaemonLease {
this.stopExpiryWatchdog(); this.stopExpiryWatchdog();
const token = this.token; const token = this.token;
this.token = null; this.token = null;
if (!token || this.lost) { if (!token) {
return; return;
} }
await this.db.$executeRaw(GamePrisma.sql` await this.db.$executeRaw(GamePrisma.sql`
UPDATE "turn_daemon_lease" UPDATE "turn_daemon_lease"
SET "lease_until" = CURRENT_TIMESTAMP AT TIME ZONE 'UTC', SET "clock_ready" = FALSE,
"lease_until" = CURRENT_TIMESTAMP AT TIME ZONE 'UTC',
"heartbeat_at" = CURRENT_TIMESTAMP AT TIME ZONE 'UTC' "heartbeat_at" = CURRENT_TIMESTAMP AT TIME ZONE 'UTC'
WHERE WHERE
"profile" = ${token.profile} "profile" = ${token.profile}
@@ -45,6 +45,7 @@ export interface TurnDaemonLifecycleDeps {
commandResponder?: TurnDaemonCommandResponder; commandResponder?: TurnDaemonCommandResponder;
pauseGate?: () => Promise<boolean>; pauseGate?: () => Promise<boolean>;
stateManager?: Pick<EngineStateManager, 'transaction'>; stateManager?: Pick<EngineStateManager, 'transaction'>;
runErrorRetry?: { wait(error: unknown): Promise<boolean>; reset(): void };
} }
export class TurnDaemonLifecycle { export class TurnDaemonLifecycle {
@@ -59,6 +60,7 @@ export class TurnDaemonLifecycle {
private readonly commandResponder?: TurnDaemonCommandResponder; private readonly commandResponder?: TurnDaemonCommandResponder;
private readonly pauseGate?: () => Promise<boolean>; private readonly pauseGate?: () => Promise<boolean>;
private readonly stateManager?: Pick<EngineStateManager, 'transaction'>; private readonly stateManager?: Pick<EngineStateManager, 'transaction'>;
private readonly runErrorRetry?: TurnDaemonLifecycleDeps['runErrorRetry'];
private readonly options: TurnDaemonLifecycleOptions; private readonly options: TurnDaemonLifecycleOptions;
private status: TurnDaemonStatus; private status: TurnDaemonStatus;
@@ -79,6 +81,7 @@ export class TurnDaemonLifecycle {
this.commandResponder = deps.commandResponder; this.commandResponder = deps.commandResponder;
this.pauseGate = deps.pauseGate; this.pauseGate = deps.pauseGate;
this.stateManager = deps.stateManager; this.stateManager = deps.stateManager;
this.runErrorRetry = deps.runErrorRetry;
this.options = options; this.options = options;
this.status = { this.status = {
state: 'idle', state: 'idle',
@@ -192,8 +195,9 @@ export class TurnDaemonLifecycle {
// 수동 실행 요청도 가오픈·정지·재조정의 턴 실행 gate를 통과해야 한다. // 수동 실행 요청도 가오픈·정지·재조정의 턴 실행 gate를 통과해야 한다.
// 사용자 명령 처리는 루프 시작에서 계속하되 시간 진행은 여기서 분리한다. // 사용자 명령 처리는 루프 시작에서 계속하되 시간 진행은 여기서 분리한다.
if (this.pendingRun) { if (this.pendingRun) {
await this.runOnce(this.pendingRun); const pending = this.pendingRun;
this.pendingRun = null; this.pendingRun = null;
await this.runOnce(pending);
continue; continue;
} }
@@ -447,6 +451,15 @@ export class TurnDaemonLifecycle {
}; };
result = this.stateManager ? await this.stateManager.transaction(runAndFlush) : await runAndFlush(); result = this.stateManager ? await this.stateManager.transaction(runAndFlush) : await runAndFlush();
} catch (error) { } catch (error) {
// savepoint 복원이 성공한 획득 충돌만 대기한다. 복원 실패/AggregateError나
// 저장 중 소유권 상실을 자동 재실행하면 중복 효과를 숨길 수 있다.
if (this.stateManager && (await this.runErrorRetry?.wait(error))) {
this.status.running = false;
this.status.state = 'idle';
this.pendingRun = pending;
return;
}
this.runErrorRetry?.reset();
this.status.running = false; this.status.running = false;
this.status.state = 'paused'; this.status.state = 'paused';
this.status.paused = true; this.status.paused = true;
@@ -459,6 +472,7 @@ export class TurnDaemonLifecycle {
} }
await this.applyRunResult(result, startMs); await this.applyRunResult(result, startMs);
this.runErrorRetry?.reset();
this.status.state = 'idle'; this.status.state = 'idle';
try { try {
await this.hooks?.publishEvents?.(result); await this.hooks?.publishEvents?.(result);
@@ -1,4 +1,4 @@
import type { GamePrisma, InputJsonValue } from '@sammo-ts/infra'; import { encodeAuditDecisionSteps, type GamePrisma, type InputJsonValue } from '@sammo-ts/infra';
import { auditPolicyHash } from './policy.js'; import { auditPolicyHash } from './policy.js';
import type { PendingAuditDecision } from './decision.js'; import type { PendingAuditDecision } from './decision.js';
@@ -50,7 +50,9 @@ export const persistAuditDecisions = async (
chunks.push({ chunks.push({
decisionId: decision.id, decisionId: decision.id,
ordinal: start / AUDIT_DECISION_CHUNK_STEPS, ordinal: start / AUDIT_DECISION_CHUNK_STEPS,
steps: decision.steps.slice(start, start + AUDIT_DECISION_CHUNK_STEPS) as InputJsonValue, steps: encodeAuditDecisionSteps(
decision.steps.slice(start, start + AUDIT_DECISION_CHUNK_STEPS) as InputJsonValue[]
),
}); });
if (chunks.length === BATCH) { if (chunks.length === BATCH) {
await tx.playAuditDecisionChunk.createMany({ data: chunks, skipDuplicates: true }); await tx.playAuditDecisionChunk.createMany({ data: chunks, skipDuplicates: true });
+37 -22
View File
@@ -1,5 +1,5 @@
import { asRecord } from '@sammo-ts/common'; import { asRecord } from '@sammo-ts/common';
import type { GamePrismaClient } from '@sammo-ts/infra'; import { GamePrisma, type GamePrismaClient } from '@sammo-ts/infra';
export const AUDIT_RETENTION_BATCH_SIZE = 200; export const AUDIT_RETENTION_BATCH_SIZE = 200;
export type AuditRetentionResult = { status: 'progress' | 'complete' | 'busy' | 'identityChanged'; deleted: number }; export type AuditRetentionResult = { status: 'progress' | 'complete' | 'busy' | 'identityChanged'; deleted: number };
@@ -13,6 +13,7 @@ export const prunePreviousAuditBatch = async (
return db.$transaction( return db.$transaction(
async (tx) => { async (tx) => {
await tx.$executeRaw`SET LOCAL statement_timeout = '2000ms'`; await tx.$executeRaw`SET LOCAL statement_timeout = '2000ms'`;
await tx.$executeRaw`SET LOCAL lock_timeout = '250ms'`;
// seeder와 동일한 잠금이다. RESET을 기다리게 하지 않고 다음 batch에서 재시도한다. // seeder와 동일한 잠금이다. RESET을 기다리게 하지 않고 다음 batch에서 재시도한다.
const [lock] = await tx.$queryRaw<{ locked: boolean }[]>` const [lock] = await tx.$queryRaw<{ locked: boolean }[]>`
SELECT pg_try_advisory_xact_lock(hashtextextended(current_schema(), 0)) AS locked SELECT pg_try_advisory_xact_lock(hashtextextended(current_schema(), 0)) AS locked
@@ -32,29 +33,43 @@ export const prunePreviousAuditBatch = async (
}); });
return { status: 'progress', deleted: deleted.count }; return { status: 'progress', deleted: deleted.count };
} }
const [decision] = await tx.$queryRaw<{ id: string }[]>` // 이미 있는 (server_id,nation_id,tick,id) 인덱스 순서로 여러 부모를 잠근다.
SELECT id FROM play_audit_decision WHERE id = COALESCE( // chunk 총량과 빈 header 삭제량의 합도 BATCH 이하로 제한한다.
(SELECT id FROM play_audit_decision WHERE server_id < ${expectedServerId} let decisions = await tx.$queryRaw<{ id: string }[]>`
ORDER BY server_id, general_id, tick, id LIMIT 1), SELECT id FROM play_audit_decision WHERE server_id < ${expectedServerId}
(SELECT id FROM play_audit_decision WHERE server_id > ${expectedServerId} ORDER BY server_id, nation_id, tick, id LIMIT ${AUDIT_RETENTION_BATCH_SIZE} FOR UPDATE
ORDER BY server_id, general_id, tick, id LIMIT 1)
) FOR UPDATE
`; `;
if (decision) { if (!decisions.length) {
const chunks = await tx.playAuditDecisionChunk.findMany({ decisions = await tx.$queryRaw<{ id: string }[]>`
where: { decisionId: decision.id }, SELECT id FROM play_audit_decision WHERE server_id > ${expectedServerId}
orderBy: { ordinal: 'asc' }, ORDER BY server_id, nation_id, tick, id LIMIT ${AUDIT_RETENTION_BATCH_SIZE} FOR UPDATE
take: AUDIT_RETENTION_BATCH_SIZE, `;
select: { ordinal: true }, }
if (decisions.length) {
const ids = decisions.map(({ id }) => id);
const [chunks] = await tx.$queryRaw<{ deleted: number }[]>(GamePrisma.sql`
WITH batch AS (
SELECT decision_id, ordinal FROM play_audit_decision_chunk
WHERE decision_id IN (${GamePrisma.join(ids)})
ORDER BY decision_id, ordinal LIMIT ${AUDIT_RETENTION_BATCH_SIZE}
), removed AS (
DELETE FROM play_audit_decision_chunk c USING batch b
WHERE c.decision_id=b.decision_id AND c.ordinal=b.ordinal RETURNING 1
) SELECT count(*)::integer AS deleted FROM removed
`);
const count = chunks?.deleted ?? 0;
const remaining = AUDIT_RETENTION_BATCH_SIZE - count;
if (!remaining) return { status: 'progress', deleted: count };
const empty = await tx.playAuditDecision.findMany({
where: { id: { in: ids }, chunks: { none: {} } },
select: { id: true },
orderBy: { id: 'asc' },
take: remaining,
}); });
if (chunks.length) { const headers = await tx.playAuditDecision.deleteMany({
const deleted = await tx.playAuditDecisionChunk.deleteMany({ where: { id: { in: empty.map(({ id }) => id) } },
where: { decisionId: decision.id, ordinal: { in: chunks.map((row) => row.ordinal) } }, });
}); return { status: 'progress', deleted: count + headers.count };
return { status: 'progress', deleted: deleted.count };
}
await tx.playAuditDecision.delete({ where: { id: decision.id } });
return { status: 'progress', deleted: 1 };
} }
const policies = await tx.playAuditPolicy.findMany({ const policies = await tx.playAuditPolicy.findMany({
where: { serverId: { not: expectedServerId } }, where: { serverId: { not: expectedServerId } },
@@ -17,7 +17,7 @@ export const startAuditRetentionWorker = (options: {
timer.unref(); timer.unref();
}; };
const run = async () => { const run = async () => {
let delay = 1000; let delay = 100;
try { try {
const result = await options.prune(); const result = await options.prune();
if (result.status === 'complete' || result.status === 'identityChanged') stopped = true; if (result.status === 'complete' || result.status === 'identityChanged') stopped = true;
+3
View File
@@ -8,6 +8,7 @@ import { createTurnDaemonMemoryReporter } from './turnDaemonMemoryReporter.js';
import { createGatewayProfileGate } from './gatewayProfileGate.js'; import { createGatewayProfileGate } from './gatewayProfileGate.js';
import { retryTurnDaemonLeaseStartup } from './leaseStartupRetry.js'; import { retryTurnDaemonLeaseStartup } from './leaseStartupRetry.js';
import { TurnDaemonLeaseUnavailableError } from '../lifecycle/databaseTurnDaemonLease.js'; import { TurnDaemonLeaseUnavailableError } from '../lifecycle/databaseTurnDaemonLease.js';
import { loadHostBootMonitor } from './hostBootMonitor.js';
export interface TurnDaemonCliOptions { export interface TurnDaemonCliOptions {
profile?: string; profile?: string;
@@ -80,6 +81,7 @@ export const runTurnDaemonCli = async (options: TurnDaemonCliOptions = {}): Prom
throw new Error(`GAME_CLOCK_MODE must be realtime or manual: ${rawGameClockMode}`); throw new Error(`GAME_CLOCK_MODE must be realtime or manual: ${rawGameClockMode}`);
} }
const gameClockMode = rawGameClockMode as GameClockMode | undefined; const gameClockMode = rawGameClockMode as GameClockMode | undefined;
const hostBootMonitor = await loadHostBootMonitor(env);
const runtime = await retryTurnDaemonLeaseStartup(() => const runtime = await retryTurnDaemonLeaseStartup(() =>
createTurnDaemonRuntime({ createTurnDaemonRuntime({
@@ -95,6 +97,7 @@ export const runTurnDaemonCli = async (options: TurnDaemonCliOptions = {}): Prom
pauseGateIntervalMs, pauseGateIntervalMs,
adminActionIntervalMs, adminActionIntervalMs,
gameClockMode, gameClockMode,
hostBootMonitor,
}) })
).catch(async (error: unknown) => { ).catch(async (error: unknown) => {
// 중복 starter가 정상 owner를 멈추면 안 된다. 그 밖의 초기화 실패는 // 중복 starter가 정상 owner를 멈추면 안 된다. 그 밖의 초기화 실패는
+142
View File
@@ -0,0 +1,142 @@
import { createHmac, randomBytes, timingSafeEqual } from 'node:crypto';
import { readFile } from 'node:fs/promises';
import { performance } from 'node:perf_hooks';
import { isRecord } from '@sammo-ts/common';
export interface HostBootMonitor {
hasChanged(): Promise<boolean>;
}
const mac = (key: Buffer, value: string): string => createHmac('sha256', key).update(value).digest('hex');
/** 부팅 세대는 첫 인증 응답에 고정한다. 통신 실패는 기존 lease 복구를 막지 않는다. */
export const createHostBootMonitor = (options: {
url: string;
key: Buffer;
pollMs?: number;
timeoutMs?: number;
now?: () => number;
wallNow?: () => number;
onUnavailable?: () => void;
onVerified?: () => void;
}): HostBootMonitor => {
const url = new URL(options.url);
if (
!['http:', 'https:'].includes(url.protocol) ||
url.username ||
url.password ||
url.search ||
url.hash ||
url.pathname !== '/v1/boot-id'
) {
throw new Error('Host boot URL must use HTTP(S) and /v1/boot-id without credentials or query.');
}
if (options.key.length !== 32) throw new Error('Host boot key must contain exactly 32 bytes.');
const now = options.now ?? (() => performance.now());
const wallNow = options.wallNow ?? (() => Date.now());
let baseline: string | null = null;
let changed = false;
let lastAttempt = -Infinity;
let lastWall = 0;
let lastWarning = -Infinity;
let inFlight: Promise<boolean> | null = null;
const sample = async (): Promise<boolean> => {
const nonce = randomBytes(32).toString('hex');
const response = await fetch(url, {
method: 'GET',
redirect: 'error',
signal: AbortSignal.timeout(options.timeoutMs ?? 1_000),
headers: {
'x-sammo-challenge': nonce,
'x-sammo-signature': mac(options.key, `sammo-host-boot-request-v1\n${nonce}`),
},
});
if (!response.ok || !response.body) {
await response.body?.cancel();
throw new Error('Host boot response unavailable.');
}
const reader = response.body.getReader();
const chunks: Uint8Array[] = [];
let size = 0;
try {
for (;;) {
const { done, value } = await reader.read();
if (done) break;
size += value.length;
if (size > 2_048) throw new Error('Host boot response exceeds its limit.');
chunks.push(value);
}
} finally {
await reader.cancel();
}
const body: unknown = JSON.parse(Buffer.concat(chunks).toString('utf8'));
if (
!isRecord(body) ||
body.version !== 1 ||
body.challenge !== nonce ||
typeof body.bootId !== 'string' ||
!/^[a-f0-9]{64}$/.test(body.bootId) ||
typeof body.signature !== 'string' ||
!/^[a-f0-9]{64}$/.test(body.signature)
) {
throw new Error('Host boot response is invalid.');
}
const expected = mac(options.key, `sammo-host-boot-response-v1\n${nonce}\n${body.bootId}`);
if (!timingSafeEqual(Buffer.from(expected, 'hex'), Buffer.from(body.signature, 'hex'))) {
throw new Error('Host boot response signature is invalid.');
}
if (baseline === null) {
baseline = body.bootId;
options.onVerified?.();
}
changed ||= body.bootId !== baseline;
return changed;
};
return {
hasChanged(): Promise<boolean> {
if (changed) return Promise.resolve(true);
if (inFlight) return inFlight;
const current = now();
const wall = wallNow();
const discontinuity = Math.abs(wall - lastWall - (current - lastAttempt)) > 1_000;
if (current - lastAttempt < (options.pollMs ?? 2_000) && !discontinuity) return Promise.resolve(false);
lastAttempt = current;
lastWall = wall;
inFlight = sample()
.catch(() => {
if (current - lastWarning >= 60_000) {
options.onUnavailable?.();
lastWarning = current;
}
return false;
})
.finally(() => {
inFlight = null;
});
return inFlight;
},
};
};
export const loadHostBootMonitor = async (env: NodeJS.ProcessEnv): Promise<HostBootMonitor | undefined> => {
const url = env.TURN_HOST_BOOT_URL?.trim();
const keyFile = env.TURN_HOST_BOOT_KEY_FILE?.trim();
if (!url && !keyFile) return undefined;
try {
if (!url || !keyFile) throw new Error('Incomplete optional configuration.');
const encoded = (await readFile(keyFile, 'utf8')).trim();
if (!/^[a-f0-9]{64}$/.test(encoded)) throw new Error('Invalid optional key file.');
return createHostBootMonitor({
url,
key: Buffer.from(encoded, 'hex'),
onVerified: () => console.info('[turn-daemon] host boot identity verified.'),
onUnavailable: () => {
console.warn('[turn-daemon] host boot identity unavailable; ordinary lease recovery remains active.');
},
});
} catch {
// 보조 관측기의 설정/파일 부재도 게임 기동의 필수 의존성이 되지 않는다.
console.warn('[turn-daemon] host boot observation disabled; check optional configuration and key file.');
return undefined;
}
};
@@ -0,0 +1,38 @@
import { performance } from 'node:perf_hooks';
import { setTimeout } from 'node:timers/promises';
import { ReservedTurnLeaseBusyError } from './reservedTurnStore.js';
/** rollback 완료 뒤에만 대기한다. 다음 loop에서 pause/lease gate와 최신 큐를 다시 읽는다. */
export const createReservedTurnLeaseRetry = (
options: {
now?: () => number;
sleep?: (ms: number) => Promise<void>;
maxWaitMs?: number;
onWait?: (queueKey: string) => void;
} = {}
) => {
const now = options.now ?? (() => performance.now());
const sleep = options.sleep ?? ((ms: number) => setTimeout(ms));
const maxWaitMs = options.maxWaitMs ?? 5 * 60_000 + 1_000;
let startedAt: number | null = null;
let lastReportedAt = -Infinity;
return {
reset(): void {
startedAt = null;
lastReportedAt = -Infinity;
},
async wait(error: unknown): Promise<boolean> {
if (!(error instanceof ReservedTurnLeaseBusyError)) return false;
const current = now();
startedAt ??= current;
const remaining = maxWaitMs - (current - startedAt);
if (remaining <= 0) return false;
if (current - lastReportedAt >= 30_000) {
options.onWait?.(error.queueKey);
lastReportedAt = current;
}
await sleep(Math.min(1_000, remaining));
return true;
},
};
};
+36 -8
View File
@@ -129,6 +129,14 @@ export class ReservedTurnLeaseConflictError extends Error {
} }
} }
/** 실행 전 획득 충돌만 재시도한다. flush/finalize의 소유권 상실과 구분한다. */
export class ReservedTurnLeaseBusyError extends ReservedTurnLeaseConflictError {
constructor(queueKey: string) {
super(queueKey);
this.name = 'ReservedTurnLeaseBusyError';
}
}
export class InMemoryReservedTurnStore { export class InMemoryReservedTurnStore {
private readonly generalTurns = new Map<number, ReservedTurnEntry[]>(); private readonly generalTurns = new Map<number, ReservedTurnEntry[]>();
private readonly nationTurns = new Map<string, ReservedTurnEntry[]>(); private readonly nationTurns = new Map<string, ReservedTurnEntry[]>();
@@ -249,11 +257,7 @@ export class InMemoryReservedTurnStore {
} }
}; };
pruneNationKeys(Array.from(this.nationTurns.keys()), (key) => this.nationTurns.delete(key)); pruneNationKeys(Array.from(this.nationTurns.keys()), (key) => this.nationTurns.delete(key));
for (const keys of [ for (const keys of [this.dirtyNationKeys, this.pendingNationInitializationKeys, this.leasedNationKeys]) {
this.dirtyNationKeys,
this.pendingNationInitializationKeys,
this.leasedNationKeys,
]) {
for (const key of Array.from(keys)) { for (const key of Array.from(keys)) {
const nationId = Number(key.split(':', 1)[0]); const nationId = Number(key.split(':', 1)[0]);
if (deletedNations.has(nationId)) { if (deletedNations.has(nationId)) {
@@ -376,7 +380,7 @@ export class InMemoryReservedTurnStore {
} }
} }
if (claimed.count !== 1) { if (claimed.count !== 1) {
throw new ReservedTurnLeaseConflictError(`general:${generalId}`); throw new ReservedTurnLeaseBusyError(`general:${generalId}`);
} }
this.leasedGeneralIds.add(generalId); this.leasedGeneralIds.add(generalId);
return !retainedExistingLease; return !retainedExistingLease;
@@ -427,7 +431,7 @@ export class InMemoryReservedTurnStore {
} }
} }
if (claimed.count !== 1) { if (claimed.count !== 1) {
throw new ReservedTurnLeaseConflictError(`nation:${nationId}:${officerLevel}`); throw new ReservedTurnLeaseBusyError(`nation:${nationId}:${officerLevel}`);
} }
this.leasedNationKeys.add(buildNationKey(nationId, officerLevel)); this.leasedNationKeys.add(buildNationKey(nationId, officerLevel));
return !retainedExistingLease; return !retainedExistingLease;
@@ -904,6 +908,24 @@ export class InMemoryReservedTurnStore {
await this.persistChanges(this.prisma, changes); await this.persistChanges(this.prisma, changes);
this.acknowledgeDirtyState(changes); this.acknowledgeDirtyState(changes);
} }
/** 실행 루프 종료 뒤 호출한다. rollback으로 journal에서 빠진 DB 임대도 해제한다. */
async releaseOwnedLeases(): Promise<void> {
const results = await Promise.allSettled([
this.prisma.generalTurnRevision?.updateMany({
where: { leaseOwner: this.leaseOwner },
data: { leaseOwner: null, leaseExpiresAt: null },
}),
this.prisma.nationTurnRevision?.updateMany({
where: { leaseOwner: this.leaseOwner },
data: { leaseOwner: null, leaseExpiresAt: null },
}),
]);
const errors = results.flatMap((result) => (result.status === 'rejected' ? [result.reason] : []));
if (errors.length) throw new AggregateError(errors, 'Reserved turn lease cleanup failed.');
this.leasedGeneralIds.clear();
this.leasedNationKeys.clear();
}
} }
export const createReservedTurnStore = async (options: ReservedTurnStoreOptions): Promise<ReservedTurnStoreHandle> => { export const createReservedTurnStore = async (options: ReservedTurnStoreOptions): Promise<ReservedTurnStoreHandle> => {
@@ -918,6 +940,12 @@ export const createReservedTurnStore = async (options: ReservedTurnStoreOptions)
await store.loadAll(); await store.loadAll();
return { return {
store, store,
close: () => connector.disconnect(), close: async () => {
try {
await store.releaseOwnedLeases();
} finally {
await connector.disconnect();
}
},
}; };
}; };
+2 -2
View File
@@ -2,7 +2,7 @@ import type { GameClockPhase } from '@sammo-ts/common';
/** Gateway의 실행 gate와 durable 시계를 명령 claim 전에 맞춘다. */ /** Gateway의 실행 gate와 durable 시계를 명령 claim 전에 맞춘다. */
export const createRuntimePauseGate = (options: { export const createRuntimePauseGate = (options: {
assertLease(): void; assertLease(): void | Promise<void>;
shouldPause(): Promise<boolean>; shouldPause(): Promise<boolean>;
getPhase(): GameClockPhase; getPhase(): GameClockPhase;
isExplicitlyPaused(): boolean; isExplicitlyPaused(): boolean;
@@ -11,7 +11,7 @@ export const createRuntimePauseGate = (options: {
}): (() => Promise<boolean>) => { }): (() => Promise<boolean>) => {
let lastPaused: boolean | null = null; let lastPaused: boolean | null = null;
return async () => { return async () => {
options.assertLease(); await options.assertLease();
const paused = await options.shouldPause(); const paused = await options.shouldPause();
const phase = options.getPhase(); const phase = options.getPhase();
// 오류 정지는 Gateway 상태만 PAUSED로 바꿀 수 있다. 그대로 두면 가입은 // 오류 정지는 Gateway 상태만 PAUSED로 바꿀 수 있다. 그대로 두면 가입은
+34 -14
View File
@@ -108,6 +108,8 @@ import { DatabaseTurnDaemonLease, TurnDaemonLeaseUnavailableError } from '../lif
import { EngineStateManager } from './engineStateManager.js'; import { EngineStateManager } from './engineStateManager.js';
import { applyRuntimeClockShift } from './runtimeClockShift.js'; import { applyRuntimeClockShift } from './runtimeClockShift.js';
import { applyRuntimeGameSettings } from './runtimeGameSettings.js'; import { applyRuntimeGameSettings } from './runtimeGameSettings.js';
import { createReservedTurnLeaseRetry } from './reservedTurnLeaseRetry.js';
import type { HostBootMonitor } from './hostBootMonitor.js';
export interface TurnDaemonRuntimeOptions { export interface TurnDaemonRuntimeOptions {
profile: string; profile: string;
@@ -134,6 +136,7 @@ export interface TurnDaemonRuntimeOptions {
leaseDurationMs?: number; leaseDurationMs?: number;
leaseOwnerId?: string; leaseOwnerId?: string;
enableLeaseHeartbeat?: boolean; enableLeaseHeartbeat?: boolean;
hostBootMonitor?: HostBootMonitor;
/** /**
* Isolated, single-process fixture acceleration only. Reserved turns are * Isolated, single-process fixture acceleration only. Reserved turns are
* loaded once and no concurrent API writer may touch this database. * loaded once and no concurrent API writer may touch this database.
@@ -985,6 +988,7 @@ const createTurnDaemonRuntimeWithLease = async (
prepareClockRecovery = dbHooks.prepareRealtimeRecovery; prepareClockRecovery = dbHooks.prepareRealtimeRecovery;
prunePreviousAudit = dbHooks.prunePreviousAudit; prunePreviousAudit = dbHooks.prunePreviousAudit;
close = async () => { close = async () => {
await adminActionConsumer?.stop();
if (auctionBidder) { if (auctionBidder) {
await auctionBidder.close(); await auctionBidder.close();
} }
@@ -995,12 +999,15 @@ const createTurnDaemonRuntimeWithLease = async (
await tournamentRewardFinalizer.close(); await tournamentRewardFinalizer.close();
} }
await dbHooks.close(); await dbHooks.close();
if (reservedTurnStoreHandle) { try {
await reservedTurnStoreHandle.close(); await reservedTurnStoreHandle?.close();
} finally {
try {
await gatewayGate?.close();
} finally {
await turnDaemonLease?.close();
}
} }
await gatewayGate?.close();
await adminActionConsumer?.stop();
await turnDaemonLease?.close();
}; };
} else if (reservedTurnStoreHandle) { } else if (reservedTurnStoreHandle) {
hooks = { hooks = {
@@ -1009,9 +1016,12 @@ const createTurnDaemonRuntimeWithLease = async (
}, },
}; };
close = async () => { close = async () => {
await reservedTurnStoreHandle.close();
await gatewayGate?.close();
await adminActionConsumer?.stop(); await adminActionConsumer?.stop();
try {
await reservedTurnStoreHandle.close();
} finally {
await gatewayGate?.close();
}
}; };
} else if (gatewayGate) { } else if (gatewayGate) {
hooks = { hooks = {
@@ -1070,12 +1080,17 @@ const createTurnDaemonRuntimeWithLease = async (
const baseClose = close; const baseClose = close;
close = async () => { close = async () => {
stopClockProjectionWorker(); stopClockProjectionWorker();
await baseClose(); const baseResults = await Promise.allSettled([baseClose()]);
await neutralAuctionRegistrar.close(); // queue 임대 정리 실패도 다른 연결/worker의 종료를 막지 않게 한다.
if (redisConnector) { const results = await Promise.allSettled([
await redisConnector.disconnect(); neutralAuctionRegistrar.close(),
} redisConnector?.disconnect(),
await commandConnector?.disconnect(); commandConnector?.disconnect(),
]);
const failures = [...baseResults, ...results].flatMap((result) =>
result.status === 'rejected' ? [result.reason] : []
);
if (failures.length) throw new AggregateError(failures, 'Turn daemon connection cleanup failed.');
}; };
const resolvedControlQueue = options.controlQueue ?? databaseCommandQueue ?? controlQueue; const resolvedControlQueue = options.controlQueue ?? databaseCommandQueue ?? controlQueue;
@@ -1109,7 +1124,8 @@ const createTurnDaemonRuntimeWithLease = async (
processor, processor,
hooks, hooks,
pauseGate: createRuntimePauseGate({ pauseGate: createRuntimePauseGate({
assertLease: () => { assertLease: async () => {
await turnDaemonLease?.checkHostBoot();
if (turnDaemonLease?.isLost()) throw turnDaemonLease.getLossError(); if (turnDaemonLease?.isLost()) throw turnDaemonLease.getLossError();
}, },
shouldPause: async () => (await pauseGate?.()) ?? false, shouldPause: async () => (await pauseGate?.()) ?? false,
@@ -1127,6 +1143,9 @@ const createTurnDaemonRuntimeWithLease = async (
// accumulated world before every due tick in that isolated mode; // accumulated world before every due tick in that isolated mode;
// production and gateway-managed runtimes keep rollback savepoints. // production and gateway-managed runtimes keep rollback savepoints.
stateManager: options.exclusiveFastForward ? undefined : stateManager, stateManager: options.exclusiveFastForward ? undefined : stateManager,
runErrorRetry: createReservedTurnLeaseRetry({
onWait: (queueKey) => console.info(`[turn-daemon] waiting for reserved turn lease: ${queueKey}`),
}),
}, },
{ profile: options.profile, defaultBudget } { profile: options.profile, defaultBudget }
); );
@@ -1179,6 +1198,7 @@ export const createTurnDaemonRuntime = async (options: TurnDaemonRuntimeOptions)
ownerId: options.leaseOwnerId, ownerId: options.leaseOwnerId,
leaseDurationMs: options.leaseDurationMs, leaseDurationMs: options.leaseDurationMs,
heartbeat: options.enableLeaseHeartbeat, heartbeat: options.enableLeaseHeartbeat,
hostBootMonitor: options.hostBootMonitor,
}) })
: null; : null;
if (turnDaemonLease && !(await turnDaemonLease.acquire())) { if (turnDaemonLease && !(await turnDaemonLease.acquire())) {
@@ -0,0 +1,134 @@
import { createHmac, randomBytes } from 'node:crypto';
import { once } from 'node:events';
import { createServer } from 'node:http';
import { afterEach, describe, expect, it, vi } from 'vitest';
import { createHostBootMonitor, loadHostBootMonitor } from '../src/turn/hostBootMonitor.js';
const cleanups: Array<() => Promise<void>> = [];
afterEach(async () => {
await Promise.all(cleanups.splice(0).map((cleanup) => cleanup()));
});
const fixture = async () => {
const key = randomBytes(32);
let bootId = 'a'.repeat(64);
let replay: string | null = null;
let mode: 'valid' | 'replay' | 'forged' | 'oversized' | 'offline' | 'hang' = 'valid';
const requests: string[] = [];
const server = createServer((request, response) => {
const nonce = String(request.headers['x-sammo-challenge']);
requests.push(nonce);
expect(request.headers['x-sammo-signature']).toBe(
createHmac('sha256', key).update(`sammo-host-boot-request-v1\n${nonce}`).digest('hex')
);
if (mode === 'hang') return;
if (mode === 'offline') {
response.writeHead(503).end();
return;
}
if (mode === 'oversized') {
response.end('x'.repeat(4096));
return;
}
if (mode === 'replay') {
response.end(replay);
return;
}
const body = JSON.stringify({
version: 1,
challenge: nonce,
bootId,
signature:
mode === 'forged'
? '0'.repeat(64)
: createHmac('sha256', key)
.update(`sammo-host-boot-response-v1\n${nonce}\n${bootId}`)
.digest('hex'),
});
replay ??= body;
response.end(body);
});
server.listen(0, '127.0.0.1');
await once(server, 'listening');
cleanups.push(
() =>
new Promise<void>((resolve) => {
server.close(() => resolve());
server.closeAllConnections();
})
);
const address = server.address();
if (!address || typeof address === 'string') throw new Error('Missing fixture listener');
return {
key,
requests,
url: `http://127.0.0.1:${address.port}/v1/boot-id`,
setBoot: (value: string) => {
bootId = value;
},
setMode: (value: typeof mode) => {
mode = value;
},
};
};
describe('Windows boot observation', () => {
it('uses a stable baseline, survives outages, and latches a signed boot change', async () => {
const f = await fixture();
const monitor = createHostBootMonitor({ ...f, pollMs: 0 });
expect(await monitor.hasChanged()).toBe(false);
f.setMode('offline');
expect(await monitor.hasChanged()).toBe(false);
f.setBoot('b'.repeat(64));
f.setMode('valid');
expect(await monitor.hasChanged()).toBe(true);
f.setMode('offline');
expect(await monitor.hasChanged()).toBe(true);
});
it.each(['replay', 'forged', 'oversized', 'hang'] as const)(
'rejects %s without changing the baseline',
async (mode) => {
const f = await fixture();
const onUnavailable = vi.fn();
const monitor = createHostBootMonitor({ ...f, pollMs: 0, timeoutMs: 100, onUnavailable });
expect(await monitor.hasChanged()).toBe(false);
f.setBoot('b'.repeat(64));
f.setMode(mode);
expect(await monitor.hasChanged()).toBe(false);
expect(onUnavailable).toHaveBeenCalledOnce();
f.setMode('valid');
expect(await monitor.hasChanged()).toBe(true);
}
);
it('samples immediately after a wall-clock jump despite a frozen monotonic cache and coalesces requests', async () => {
const f = await fixture();
let wall = 10_000;
const monitor = createHostBootMonitor({ ...f, now: () => 0, wallNow: () => wall });
expect(await monitor.hasChanged()).toBe(false);
expect(await monitor.hasChanged()).toBe(false);
expect(f.requests).toHaveLength(1);
wall += 300_000;
f.setBoot('b'.repeat(64));
expect(await Promise.all([monitor.hasChanged(), monitor.hasChanged()])).toEqual([true, true]);
expect(f.requests).toHaveLength(2);
expect(new Set(f.requests).size).toBe(2);
});
it('does not block game startup when optional configuration or its key file is absent', async () => {
const warning = vi.spyOn(console, 'warn').mockImplementation(() => {});
expect(await loadHostBootMonitor({})).toBeUndefined();
expect(await loadHostBootMonitor({ TURN_HOST_BOOT_URL: 'http://host/v1/boot-id' })).toBeUndefined();
expect(
await loadHostBootMonitor({
TURN_HOST_BOOT_URL: 'http://host/v1/boot-id',
TURN_HOST_BOOT_KEY_FILE: '/nonexistent-host-boot-key',
})
).toBeUndefined();
expect(() =>
createHostBootMonitor({ url: 'http://user:password@host/v1/boot-id', key: randomBytes(32) })
).toThrow('without credentials');
warning.mockRestore();
});
});
@@ -1,7 +1,12 @@
import { writeFile } from 'node:fs/promises'; import { writeFile } from 'node:fs/promises';
import { performance } from 'node:perf_hooks'; import { performance } from 'node:perf_hooks';
import { afterAll, beforeAll, describe, expect, it } from 'vitest'; import { afterAll, beforeAll, describe, expect, it } from 'vitest';
import { createGamePostgresConnector, type GamePrisma, type GamePrismaClient } from '@sammo-ts/infra'; import {
decodeAuditDecisionSteps,
createGamePostgresConnector,
type GamePrisma,
type GamePrismaClient,
} from '@sammo-ts/infra';
import { persistAuditDecisions } from '../src/playAudit/decisionPersistence.js'; import { persistAuditDecisions } from '../src/playAudit/decisionPersistence.js';
import { buildAuditDecisionFixture } from './fixtures/playAuditDecision.js'; import { buildAuditDecisionFixture } from './fixtures/playAuditDecision.js';
@@ -61,11 +66,15 @@ describe.skipIf(!databaseUrl)('decision storage cost probe', () => {
>` >`
SELECT SELECT
(SELECT count(*) FROM play_audit_decision) AS headers, (SELECT count(*) FROM play_audit_decision) AS headers,
(SELECT sum(jsonb_array_length(steps)) FROM play_audit_decision_chunk) AS steps, (SELECT sum(step_count) FROM play_audit_decision) AS steps,
(SELECT sum(pg_column_size(d)) FROM play_audit_decision d) AS "headerBytes", (SELECT sum(pg_column_size(d)) FROM play_audit_decision d) AS "headerBytes",
(SELECT sum(pg_column_size(c)) FROM play_audit_decision_chunk c) AS "chunkBytes" (SELECT sum(pg_column_size(c)) FROM play_audit_decision_chunk c) AS "chunkBytes"
`; `;
expect(Number(storage!.steps)).toBe(201 * 302); expect(Number(storage!.steps)).toBe(201 * 302);
const chunks = await db.playAuditDecisionChunk.findMany({ select: { steps: true } });
expect(chunks.reduce((total, chunk) => total + decodeAuditDecisionSteps(chunk.steps).length, 0)).toBe(
201 * 302
);
const relations = await db.$queryRaw< const relations = await db.$queryRaw<
{ name: string; tableBytes: bigint; indexBytes: bigint; totalBytes: bigint }[] { name: string; tableBytes: bigint; indexBytes: bigint; totalBytes: bigint }[]
>` >`
@@ -99,7 +108,8 @@ describe.skipIf(!databaseUrl)('decision storage cost probe', () => {
`; `;
const metrics = { const metrics = {
scope: 'synthetic 201 decisions, 302 steps each; isolated schema, warm local reads', scope: 'synthetic 201 decisions, 302 steps each; isolated schema, warm local reads',
limitations: 'Not production p95; no gameplay baseline, WAL or retained heap measurement. SQL counts cover only the persistence transaction. Repetitive synthetic steps compress well. Relation allocation can retain space from previous runs.', limitations:
'Not production p95; no gameplay baseline, WAL or retained heap measurement. SQL counts cover only the persistence transaction. Repetitive synthetic steps compress well. Relation allocation can retain space from previous runs.',
decisions: 201, decisions: 201,
steps: Number(storage!.steps), steps: Number(storage!.steps),
chunks: 603, chunks: 603,
@@ -1,5 +1,5 @@
import { afterAll, beforeAll, describe, expect, it } from 'vitest'; import { afterAll, beforeAll, describe, expect, it } from 'vitest';
import { createGamePostgresConnector, type GamePrismaClient } from '@sammo-ts/infra'; import { decodeAuditDecisionSteps, createGamePostgresConnector, type GamePrismaClient } from '@sammo-ts/infra';
import { persistAuditDecisions } from '../src/playAudit/decisionPersistence.js'; import { persistAuditDecisions } from '../src/playAudit/decisionPersistence.js';
import { buildAuditExecutionFixture, buildAuditDecisionFixture as draft } from './fixtures/playAuditDecision.js'; import { buildAuditExecutionFixture, buildAuditDecisionFixture as draft } from './fixtures/playAuditDecision.js';
import { prunePreviousAuditBatch } from '../src/playAudit/retention.js'; import { prunePreviousAuditBatch } from '../src/playAudit/retention.js';
@@ -36,7 +36,40 @@ integration('decision persistence and bounded retention', () => {
}); });
it('rolls back gameplay/header/chunks on insert failure, retries and rejects divergent replay', async () => { it('rolls back gameplay/header/chunks on insert failure, retries and rejects divergent replay', async () => {
const decision = draft('decision-one'); const decision = draft('decision-one');
if (decision.steps[0]?.kind === 'DECISION_START') decision.steps[0].effectivePolicy = {"schemaVersion":1,"general":{"priority":["징병"],"flags":{"징병":true,"출병":false}},"nation":{"priority":["천도"],"flags":{"천도":true},"values":{"reqNationGold":4321,"reqNationRice":100,"reqHumanWarUrgentGold":100,"reqHumanWarUrgentRice":100,"reqHumanWarRecommandGold":100,"reqHumanWarRecommandRice":100,"reqHumanDevelGold":100,"reqHumanDevelRice":100,"reqNpcWarGold":100,"reqNpcWarRice":100,"reqNpcDevelGold":100,"reqNpcDevelRice":100,"minimumResourceActionAmount":100,"maximumResourceActionAmount":100,"minNpcWarLeadership":100,"minWarCrew":100,"minNpcRecruitCityPopulation":100,"safeRecruitCityPopulationRatio":100,"properWarTrainAtmos":100,"cureThreshold":100},"combatForce":{"1":[2,3]},"supportForce":[4],"developForce":[5]}}; if (decision.steps[0]?.kind === 'DECISION_START')
decision.steps[0].effectivePolicy = {
schemaVersion: 1,
general: { priority: ['징병'], flags: { 징병: true, 출병: false } },
nation: {
priority: ['천도'],
flags: { 천도: true },
values: {
reqNationGold: 4321,
reqNationRice: 100,
reqHumanWarUrgentGold: 100,
reqHumanWarUrgentRice: 100,
reqHumanWarRecommandGold: 100,
reqHumanWarRecommandRice: 100,
reqHumanDevelGold: 100,
reqHumanDevelRice: 100,
reqNpcWarGold: 100,
reqNpcWarRice: 100,
reqNpcDevelGold: 100,
reqNpcDevelRice: 100,
minimumResourceActionAmount: 100,
maximumResourceActionAmount: 100,
minNpcWarLeadership: 100,
minWarCrew: 100,
minNpcRecruitCityPopulation: 100,
safeRecruitCityPopulationRatio: 100,
properWarTrainAtmos: 100,
cureThreshold: 100,
},
combatForce: { '1': [2, 3] },
supportForce: [4],
developForce: [5],
},
};
decision.summary.codeVersion = 'a'.repeat(40); decision.summary.codeVersion = 'a'.repeat(40);
decision.summary.executionCoverage = 'ATTEMPTS'; decision.summary.executionCoverage = 'ATTEMPTS';
decision.steps.push({ ...decision.steps[0]!, ...buildAuditExecutionFixture(), sequence: 302 }); decision.steps.push({ ...decision.steps[0]!, ...buildAuditExecutionFixture(), sequence: 302 });
@@ -85,7 +118,7 @@ integration('decision persistence and bounded retention', () => {
orderBy: { ordinal: 'asc' }, orderBy: { ordinal: 'asc' },
}); });
expect(chunks).toHaveLength(3); expect(chunks).toHaveLength(3);
expect(chunks.flatMap((chunk) => chunk.steps)).toEqual(decision.steps); expect(chunks.flatMap((chunk) => decodeAuditDecisionSteps(chunk.steps))).toEqual(decision.steps);
await expect( await expect(
db.$transaction((tx) => db.$transaction((tx) =>
persistAuditDecisions(tx, [{ ...decision, summary: { ...decision.summary, completed: true } }]) persistAuditDecisions(tx, [{ ...decision, summary: { ...decision.summary, completed: true } }])
@@ -109,10 +142,34 @@ integration('decision persistence and bounded retention', () => {
expect(await db.playAuditDecisionChunk.count({ where: { decisionId: 'decision-one' } })).toBe(204); expect(await db.playAuditDecisionChunk.count({ where: { decisionId: 'decision-one' } })).toBe(204);
expect(await db.playAuditDecision.count({ where: { id: 'decision-one' } })).toBe(1); expect(await db.playAuditDecision.count({ where: { id: 'decision-one' } })).toBe(1);
expect(await prunePreviousAuditBatch(db, 'decision-current')).toEqual({ status: 'progress', deleted: 200 }); expect(await prunePreviousAuditBatch(db, 'decision-current')).toEqual({ status: 'progress', deleted: 200 });
expect(await prunePreviousAuditBatch(db, 'decision-current')).toEqual({ status: 'progress', deleted: 4 }); expect(await prunePreviousAuditBatch(db, 'decision-current')).toEqual({ status: 'progress', deleted: 5 });
expect(await prunePreviousAuditBatch(db, 'decision-current')).toEqual({ status: 'progress', deleted: 1 });
expect(await prunePreviousAuditBatch(db, 'decision-current')).toEqual({ status: 'complete', deleted: 0 }); expect(await prunePreviousAuditBatch(db, 'decision-current')).toEqual({ status: 'complete', deleted: 0 });
expect(await db.playAuditDecision.count()).toBe(1); expect(await db.playAuditDecision.count()).toBe(1);
expect(await db.playAuditDecisionChunk.count()).toBe(3); expect(await db.playAuditDecisionChunk.count()).toBe(3);
}); });
it('removes many small decisions in one bounded batch and rolls chunks back on header failure', async () => {
const decisions = Array.from({ length: 100 }, (_, index) => {
const row = draft(`batch-old-${index}`, 'batch-old');
return { ...row, steps: [row.steps[0]!, row.steps.at(-1)!] };
});
await db.$transaction((tx) => persistAuditDecisions(tx, decisions));
await db.$executeRawUnsafe(
`CREATE OR REPLACE FUNCTION decision_header_failure() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN RAISE EXCEPTION 'header fixture rollback'; END $$`
);
await db.$executeRawUnsafe(
`CREATE TRIGGER decision_header_failure BEFORE DELETE ON play_audit_decision FOR EACH ROW EXECUTE FUNCTION decision_header_failure()`
);
try {
await expect(prunePreviousAuditBatch(db, 'decision-current')).rejects.toThrow();
expect(
await db.playAuditDecisionChunk.count({ where: { decisionId: { in: decisions.map(({ id }) => id) } } })
).toBe(100);
} finally {
await db.$executeRawUnsafe('DROP TRIGGER decision_header_failure ON play_audit_decision');
await db.$executeRawUnsafe('DROP FUNCTION decision_header_failure()');
}
expect(await prunePreviousAuditBatch(db, 'decision-current')).toEqual({ status: 'progress', deleted: 200 });
expect(await db.playAuditDecision.count()).toBe(1);
expect(await prunePreviousAuditBatch(db, 'decision-current')).toEqual({ status: 'complete', deleted: 0 });
});
}); });
@@ -20,7 +20,7 @@ describe('audit retention scheduling', () => {
expect(prune).toHaveBeenCalledTimes(1); expect(prune).toHaveBeenCalledTimes(1);
await vi.advanceTimersByTimeAsync(30_001); await vi.advanceTimersByTimeAsync(30_001);
expect(prune).toHaveBeenCalledTimes(3); expect(prune).toHaveBeenCalledTimes(3);
await vi.advanceTimersByTimeAsync(1000); await vi.advanceTimersByTimeAsync(100);
expect(prune).toHaveBeenCalledTimes(4); expect(prune).toHaveBeenCalledTimes(4);
await vi.advanceTimersByTimeAsync(600_000); await vi.advanceTimersByTimeAsync(600_000);
expect(prune).toHaveBeenCalledTimes(4); expect(prune).toHaveBeenCalledTimes(4);
@@ -4,7 +4,9 @@ import { createGamePostgresConnector } from '@sammo-ts/infra';
import { import {
InMemoryReservedTurnStore, InMemoryReservedTurnStore,
ReservedTurnLeaseConflictError, ReservedTurnLeaseConflictError,
createReservedTurnStore,
} from '../src/turn/reservedTurnStore.js'; } from '../src/turn/reservedTurnStore.js';
import { createReservedTurnLeaseRetry } from '../src/turn/reservedTurnLeaseRetry.js';
const databaseUrl = process.env.RESERVED_TURN_DATABASE_URL; const databaseUrl = process.env.RESERVED_TURN_DATABASE_URL;
const describeIntegration = databaseUrl ? describe : describe.skip; const describeIntegration = databaseUrl ? describe : describe.skip;
@@ -132,4 +134,98 @@ describeIntegration('reserved turn daemon/API lease integration', () => {
}); });
expect(after.map((turn) => turn.actionCode)).toEqual(before.map((turn) => turn.actionCode)); expect(after.map((turn) => turn.actionCode)).toEqual(before.map((turn) => turn.actionCode));
}); });
it('cleans persisted ownership after rollback loses the journal, preserves other writers, and reloads once', async () => {
if (!connector || !databaseUrl) throw new Error('integration connector is unavailable');
await connector.prisma.generalTurnRevision.update({
where: { generalId: GENERAL_ID },
data: { leaseOwner: null, leaseExpiresAt: null },
});
const before = await connector.prisma.generalTurn.findMany({
where: { generalId: GENERAL_ID },
orderBy: { turnIdx: 'asc' },
});
const revision = await connector.prisma.generalTurnRevision.findUniqueOrThrow({
where: { generalId: GENERAL_ID },
});
const old = await createReservedTurnStore({ databaseUrl, leaseOwner: 'interrupted-daemon' });
const savepoint = old.store.captureTransactionState();
await old.store.prepareTurnsForExecution(GENERAL_ID);
old.store.shiftGeneralTurns(GENERAL_ID, -1);
old.store.restoreState(savepoint);
expect(old.store.getQueueCounts().leasedGeneralQueues).toBe(0);
const next = await createReservedTurnStore({ databaseUrl, leaseOwner: 'new-daemon' });
try {
await expect(next.store.prepareTurnsForExecution(GENERAL_ID)).rejects.toBeInstanceOf(
ReservedTurnLeaseConflictError
);
await old.close();
const released = await connector.prisma.generalTurnRevision.findUniqueOrThrow({
where: { generalId: GENERAL_ID },
});
expect(released).toMatchObject({ leaseOwner: null, leaseExpiresAt: null, revision: revision.revision });
await next.store.prepareTurnsForExecution(GENERAL_ID);
// A delayed close of an old owner cannot clear the new owner's lease.
const delayedCleaner = new InMemoryReservedTurnStore(connector.prisma, {
maxGeneralTurns: 2,
maxNationTurns: 1,
leaseOwner: 'interrupted-daemon',
});
await delayedCleaner.releaseOwnedLeases();
expect(
await connector.prisma.generalTurnRevision.findUniqueOrThrow({ where: { generalId: GENERAL_ID } })
).toMatchObject({ leaseOwner: 'new-daemon', revision: revision.revision });
await next.store.releaseOwnedLeases();
const actual = await connector.prisma.generalTurn.findMany({
where: { generalId: GENERAL_ID },
orderBy: { turnIdx: 'asc' },
});
expect(actual.map((row) => row.actionCode)).toEqual(before.map((row) => row.actionCode));
} finally {
await next.close();
}
});
it('waits out an orphaned lease without deleting it and reloads an intervening API edit', async () => {
if (!connector) throw new Error('integration connector is unavailable');
await connector.prisma.generalTurnRevision.update({
where: { generalId: GENERAL_ID },
data: { leaseOwner: 'killed-daemon', leaseExpiresAt: new Date(Date.now() + 300_000) },
});
const store = new InMemoryReservedTurnStore(connector.prisma, {
maxGeneralTurns: 2,
maxNationTurns: 1,
leaseOwner: 'recovery-daemon',
});
await store.loadAll();
const retry = createReservedTurnLeaseRetry({
sleep: async () => {
// Simulate the recorded expiry, then an authorized API edit before the next claim.
expect(
await connector.prisma.generalTurnRevision.findUniqueOrThrow({ where: { generalId: GENERAL_ID } })
).toMatchObject({ leaseOwner: 'killed-daemon' });
await connector.prisma.generalTurnRevision.update({
where: { generalId: GENERAL_ID },
data: { leaseExpiresAt: new Date(0) },
});
await connector.prisma.generalTurn.updateMany({
where: { generalId: GENERAL_ID, turnIdx: 0 },
data: { actionCode: 'che_수련' },
});
},
});
try {
let failure: unknown;
try {
await store.prepareTurnsForExecution(GENERAL_ID);
} catch (error) {
failure = error;
}
expect(await retry.wait(failure)).toBe(true);
await store.prepareTurnsForExecution(GENERAL_ID);
expect(store.getGeneralTurns(GENERAL_ID)[0]?.action).toBe('che_수련');
} finally {
await store.releaseOwnedLeases();
}
});
}); });
@@ -169,6 +169,18 @@ const buildHarness = (initialRevision: RevisionRow | null = null) => {
}; };
describe('reserved turn daemon lease', () => { describe('reserved turn daemon lease', () => {
it('cleans both persisted lease kinds even after their journal is rolled back', async () => {
const harness = buildHarness();
const snapshot = harness.store.captureTransactionState();
await harness.store.prepareTurnsForExecution(7, { nationId: 3, officerLevel: 12 });
harness.store.restoreState(snapshot);
expect(harness.store.getQueueCounts()).toMatchObject({ leasedGeneralQueues: 0, leasedNationQueues: 0 });
await harness.store.releaseOwnedLeases();
expect(harness.getRevision()).toMatchObject({ leaseOwner: null, leaseExpiresAt: null, revision: 0 });
expect(harness.getNationRevision()).toMatchObject({ leaseOwner: null, leaseExpiresAt: null, revision: 0 });
expect(harness.generalDeleteMany).not.toHaveBeenCalled();
});
it('prunes deleted general and nation queues together with their journal state', () => { it('prunes deleted general and nation queues together with their journal state', () => {
const harness = buildHarness(); const harness = buildHarness();
harness.store.ensureGeneralTurns(7); harness.store.ensureGeneralTurns(7);
@@ -0,0 +1,38 @@
import { describe, expect, it, vi } from 'vitest';
import { createReservedTurnLeaseRetry } from '../src/turn/reservedTurnLeaseRetry.js';
import { ReservedTurnLeaseBusyError, ReservedTurnLeaseConflictError } from '../src/turn/reservedTurnStore.js';
describe('reserved turn acquisition retry', () => {
it('bounds the entire contention episode across different queues and resets after progress', async () => {
let now = 0;
const onWait = vi.fn();
const retry = createReservedTurnLeaseRetry({
now: () => now,
sleep: async (ms) => {
now += ms;
},
maxWaitMs: 2_500,
onWait,
});
expect(await retry.wait(new ReservedTurnLeaseBusyError('general:1'))).toBe(true);
expect(await retry.wait(new ReservedTurnLeaseBusyError('nation:2:12'))).toBe(true);
expect(await retry.wait(new ReservedTurnLeaseBusyError('general:3'))).toBe(true);
expect(now).toBe(2_500);
expect(await retry.wait(new ReservedTurnLeaseBusyError('general:1'))).toBe(false);
expect(onWait).toHaveBeenCalledTimes(1);
retry.reset();
expect(await retry.wait(new ReservedTurnLeaseBusyError('general:1'))).toBe(true);
});
it('never retries flush ownership loss, calculation errors, or failed savepoint restoration', async () => {
const sleep = vi.fn();
const retry = createReservedTurnLeaseRetry({ sleep });
for (const error of [
new ReservedTurnLeaseConflictError('general:1'),
new Error('calculation failed'),
new AggregateError([new ReservedTurnLeaseBusyError('general:1')], 'restore failed'),
])
expect(await retry.wait(error)).toBe(false);
expect(sleep).not.toHaveBeenCalled();
});
});
+25 -1
View File
@@ -3,12 +3,36 @@ import type { GameClockPhase } from '@sammo-ts/common';
import { createRuntimePauseGate } from '../src/turn/runtimePauseGate.js'; import { createRuntimePauseGate } from '../src/turn/runtimePauseGate.js';
describe('runtime pause clock boundary', () => { describe('runtime pause clock boundary', () => {
it('awaits a host authority check before reading profile state or changing the clock', async () => {
const calls: string[] = [];
const gate = createRuntimePauseGate({
assertLease: async () => {
await Promise.resolve();
calls.push('host');
throw new Error('host changed');
},
shouldPause: async () => {
calls.push('profile');
return false;
},
isExplicitlyPaused: () => false,
getPhase: () => 'RUNNING',
prepareRecovery: async () => {
calls.push('recovery');
},
synchronize: async () => {},
});
await expect(gate()).rejects.toThrow('host changed');
expect(calls).toEqual(['host']);
});
it('freezes a live error pause before commands, resumes without restart, and does not freeze twice', async () => { it('freezes a live error pause before commands, resumes without restart, and does not freeze twice', async () => {
let phase: GameClockPhase = 'RUNNING'; let phase: GameClockPhase = 'RUNNING';
let paused = true; let paused = true;
const calls: string[] = []; const calls: string[] = [];
const gate = createRuntimePauseGate({ const gate = createRuntimePauseGate({
assertLease: () => calls.push('lease'), assertLease: () => {
calls.push('lease');
},
shouldPause: async () => paused, shouldPause: async () => paused,
isExplicitlyPaused: () => paused, isExplicitlyPaused: () => paused,
getPhase: () => phase, getPhase: () => phase,
@@ -207,6 +207,39 @@ integration('database turn daemon lease and fencing', () => {
expect(row.ownerId).toBe(firstToken ? 'owner-a' : 'owner-b'); expect(row.ownerId).toBe(firstToken ? 'owner-a' : 'owner-b');
}); });
it('fences a host boot change inside a transaction before mutation, without expiring the DB lease', async () => {
const profile = `${profilePrefix}host-boot`;
let changed = false;
const lease = await DatabaseTurnDaemonLease.connect(databaseUrl!, {
profile,
ownerId: 'saved-vm-owner',
heartbeat: false,
leaseDurationMs: 60_000,
hostBootMonitor: { hasChanged: async () => changed },
});
leases.push(lease);
await lease.acquire();
await lease.markClockReady();
changed = true;
let committed = false;
await expect(
db.$transaction(async (transaction) => {
await lease.assertActive(transaction);
committed = true;
})
).rejects.toBeInstanceOf(TurnDaemonLeaseLostError);
expect(committed).toBe(false);
expect(lease.isLost()).toBe(true);
expect(await lease.renew()).toBe(false);
// The monitor does not force-clear ownership or bypass the successor's fencing.
expect((await db.turnDaemonLease.findUniqueOrThrow({ where: { profile } })).ownerId).toBe('saved-vm-owner');
const next = await createLease(profile, 'fresh-vm-owner');
expect(await next.acquire()).toBeNull();
await lease.release();
expect((await db.turnDaemonLease.findUniqueOrThrow({ where: { profile } })).clockReady).toBe(false);
expect((await next.acquire())?.fencingEpoch).toBe(2n);
});
it('persists and evaluates lease wall time as UTC under a non-UTC database session', async () => { it('persists and evaluates lease wall time as UTC under a non-UTC database session', async () => {
const profile = `${profilePrefix}utc-wall`; const profile = `${profilePrefix}utc-wall`;
const zonedUrl = new URL(databaseUrl!); const zonedUrl = new URL(databaseUrl!);
@@ -1,6 +1,8 @@
import { describe, expect, it, vi } from 'vitest'; import { describe, expect, it, vi } from 'vitest';
import { ManualClock } from '@sammo-ts/common'; import { ManualClock } from '@sammo-ts/common';
import { TurnDaemonLeaseLostError } from '../src/lifecycle/databaseTurnDaemonLease.js'; import { TurnDaemonLeaseLostError } from '../src/lifecycle/databaseTurnDaemonLease.js';
import { ReservedTurnLeaseBusyError } from '../src/turn/reservedTurnStore.js';
import { createReservedTurnLeaseRetry } from '../src/turn/reservedTurnLeaseRetry.js';
import { import {
InMemoryControlQueue, InMemoryControlQueue,
@@ -15,6 +17,68 @@ import {
const addMinutes = (time: Date, minutes: number): Date => new Date(time.getTime() + minutes * 60_000); const addMinutes = (time: Date, minutes: number): Date => new Date(time.getTime() + minutes * 60_000);
describe('TurnDaemonLifecycle', () => { describe('TurnDaemonLifecycle', () => {
it('rolls back a busy queue, checks the operator gate, and retries an explicit run exactly once', async () => {
const now = new Date('2026-11-12T00:00:00Z');
const queue = new InMemoryControlQueue();
queue.enqueue({ type: 'run', reason: 'manual', targetTime: now.toISOString() });
let resource = 10;
let runs = 0;
const stateManager = new EngineStateManager();
stateManager.register('resource', {
capture: () => resource,
restore: (value) => {
resource = value;
},
});
const onRunError = vi.fn();
const flushChanges = vi.fn();
const gate = vi.fn(async () => false);
const retry = createReservedTurnLeaseRetry({
sleep: async () => {
expect(resource).toBe(10);
},
});
const lifecycle = new TurnDaemonLifecycle(
{
clock: new ManualClock(now.getTime()),
controlQueue: queue,
stateManager,
runErrorRetry: retry,
getNextTickTime: (value) => addMinutes(value, 5),
pauseGate: gate,
stateStore: {
loadLastTurnTime: async () => now,
loadNextGeneralTurnTime: async () => null,
saveLastTurnTime: async () => {},
loadCheckpoint: async () => undefined,
saveCheckpoint: async () => {},
},
processor: {
run: async () => {
resource -= 2;
if (++runs === 1) throw new ReservedTurnLeaseBusyError('general:7');
queue.enqueue({ type: 'shutdown' });
return {
lastTurnTime: now.toISOString(),
processedGenerals: 1,
processedTurns: 0,
durationMs: 0,
partial: false,
};
},
},
hooks: { onRunError, flushChanges },
},
{ profile: 'che', defaultBudget: { budgetMs: 100, maxGenerals: 1, catchUpCap: 1 } }
);
await lifecycle.start();
expect(runs).toBe(2);
expect(resource).toBe(8);
expect(gate.mock.calls.length).toBeGreaterThanOrEqual(2);
expect(flushChanges).toHaveBeenCalledOnce();
expect(onRunError).not.toHaveBeenCalled();
});
it.each([false, true])('reports a fatal lease gate and exits even if reporting fails (%s)', async (reportFails) => { it.each([false, true])('reports a fatal lease gate and exits even if reporting fails (%s)', async (reportFails) => {
const now = new Date('2026-09-09T17:30:00Z'); const now = new Date('2026-09-09T17:30:00Z');
const error = new TurnDaemonLeaseLostError('che:default'); const error = new TurnDaemonLeaseLostError('che:default');
+3
View File
@@ -2,6 +2,9 @@
## 문서 상태와 사용법 ## 문서 상태와 사용법
2026-11-09 저장 최적화: [물리 압축·정리 처리량·ZFS 운영 경계](../play-audit-storage.md).
판단 원문과 현재 기수 보존 계약을 유지하며 저장 표현과 이전 기수 정리 구현을 변경한다.
이 문서는 플레이 감사의 구조·현재 구현 경계·남은 요구사항을 관리하는 기준 문서다. 이 문서는 플레이 감사의 구조·현재 구현 경계·남은 요구사항을 관리하는 기준 문서다.
사용 방법과 DB 적용은 [운영 안내](../play-audit-operations.md)를 따른다. 사용 방법과 DB 적용은 [운영 안내](../play-audit-operations.md)를 따른다.
날짜별 구현·측정 기록은 Git 이력과 상위 작업공간의 `report/`에서 확인한다. 날짜별 구현·측정 기록은 Git 이력과 상위 작업공간의 `report/`에서 확인한다.
+149
View File
@@ -0,0 +1,149 @@
# Hyper-V 저장·복원 이후 게임 복구
Windows 재부팅 때 Linux VM이 저장·복원되면 게스트의 shutdown/resume hook이
실행되지 않을 수 있다. 게임은 종료 이벤트 없이도 기존 lease 만료와 DB snapshot
재적재로 복구한다. Windows 식별자 확인은 이 경로에 추가하는 선택 기능이다.
## 예약턴 임대 복구
- 실행 루프가 종료되면 DB에서 자기 owner의 장수·국가 예약턴 임대를 해제한다.
In-memory rollback으로 leased set에서 빠진 row도 정리한다. 큐 내용과 revision,
다른 실행자/API의 임대는 변경하지 않는다.
- 계산 전 임대 획득 충돌은 전체 메모리 savepoint를 복원한 뒤 1초 단위로 재시도한다.
한 충돌 구간의 상한은 301초다(기본 queue lease 5분 + 여유 1초). 그동안 다음
loop마다 운영자 pause/stop과 daemon 권위를 확인한다. 성공한 batch에서 상한을
초기화하며 큐가 바뀌어도 같은 실패 구간의 상한을 연장하지 않는다.
- 저장·finalize 단계의 소유권 상실, 메모리 복원 실패와 다른 게임 오류는 자동
재시도하지 않는다. 상한을 넘은 획득 충돌도 기존 PAUSED/장애 이력 경로로 간다.
- daemon의 30초 lease와 fencing epoch, 기존 10분 이하 catch-up 및 장시간 복구
정책은 유지한다. 운영자가 정지한 profile을 자동 START하지 않는다.
## Windows Node.js 에이전트
필요 파일은 `windows-boot-agent.mjs` 하나이며 외부 npm package가 필요 없다.
Node.js 24.22.1 이상을 사용한다. 설정 예시는
`tools/host-recovery/windows-boot-agent.example.json`이다.
Node.js가 Windows 내장 PowerShell의 CIM 조회로 `Win32_OperatingSystem.LastBootUpTime`
UTC ticks를 읽고 hostname과 함께 hash한다. 별도의 PowerShell 스크립트 설치나
Linux에서 Windows 관리자 명령 실행은 필요 없다. Node 에이전트를 재시작해도
같은 Windows boot의 식별자는 같고, Windows를 재부팅하면 달라진다.
실제 Windows에서 CIM 조회가 실패하면 가짜 식별자를 만들지 않고 시작을 거부한다.
1. 파일들을 `C:\ProgramData\Sammo` 같은 고정 경로에 복사한다.
2. 디렉터리를 만든 뒤 Windows 터미널에서 private key를 생성한다. 기존 파일을
덮어쓰지 않으며 key 내용은 출력하지 않는다.
```text
node C:\ProgramData\Sammo\windows-boot-agent.mjs --create-key C:\ProgramData\Sammo\host-boot.key
```
3. 설정 예시를 `C:\ProgramData\Sammo\host-boot.json`으로 복사한다. `keyFile`은
위 경로, `bindAddress`는 **Linux VM에서 접근할 수 있는 Windows의 사설 주소**로
설정한다. 예시의 `127.0.0.1`은 안전한 기본값이며 원격 Linux에서 접근할 수 없다.
`port` 기본값은 17781이다. Windows 방화벽은 해당 사설 listener에 Linux VM
주소에서 오는 TCP 연결만 허용하도록 설정한다. 기존 공개 Caddy는 변경하지 않는다.
4. key 파일은 실행 계정만 읽도록 NTFS 권한을 제한하고, 같은 내용을 Linux의
`/run/secrets/host_boot_key` 같은 private file로 비공개 전달한다. Git·메신저·
CLI 인자에 key 내용을 넣지 않는다. 에이전트는 key를 시작할 때 읽으므로 key
교체 시 양쪽 프로세스를 함께 재시작한다.
5. 실행한다.
```text
node C:\ProgramData\Sammo\windows-boot-agent.mjs --config C:\ProgramData\Sammo\host-boot.json
```
6. Windows 작업 스케줄러에 **시스템 시작 시** task를 등록한다. 프로그램은 Node의
절대 `node.exe` 경로, 인자는 위 `.mjs --config ...` 부분을 사용한다. 로그인 없이
실행되고 실패 시 재시작하도록 구성한다. 설정 파일을 사용하므로 터미널 세션의
임시 환경변수에 의존하지 않는다. 실제 Windows 설치·방화벽 변경·task 등록은
대상 host에서 별도 적용하고 확인해야 한다.
원한다면 설정 파일 대신 `HOST_BOOT_KEY_FILE`, `HOST_BOOT_BIND_ADDRESS`,
`HOST_BOOT_PORT` 환경변수로도 실행할 수 있다.
## Linux 연결
기존 game runtime/PM2가 상속하는 서버 환경에 아래 두 값을 함께 설정한다.
설정이 없으면 기능을 사용하지 않는다. 설정이 불완전하거나 key 파일이 없거나 잘못돼도
고정 경고와 함께 보조 관측만 비활성화하며 게임 기동은 계속한다.
```dotenv
TURN_HOST_BOOT_URL=http://windows-private-address:17781/v1/boot-id
TURN_HOST_BOOT_KEY_FILE=/run/secrets/host_boot_key
```
Docker에서는 private key가 들어 있는 전용 디렉터리를 runtime에 read-only mount하고 두 환경변수를
**runtime container environment**에 전달해야 한다. 호스트의 `.env`에만 추가하면
자동 전달되지 않는다. 기존 Compose와 host overlay를 함께 적용하는 전용 wrapper를
사용한다. 실제 secret 파일의 host 경로와 Windows 사설 주소를 확인하기 전에는
운영 stack 설정을 추측해 바꾸지 않는다.
예를 들어 host overlay의 runtime 설정에는 다음 형태로 연결할 수 있다. host 경로는
placeholder이며 실제 운영 경로로 교체해야 한다.
```yaml
services:
runtime:
environment:
TURN_HOST_BOOT_URL: ${TURN_HOST_BOOT_URL:-}
TURN_HOST_BOOT_KEY_FILE: /run/sammo-host-recovery/host_boot.key
volumes:
- type: bind
source: /absolute/private/path/host-recovery
target: /run/sammo-host-recovery
read_only: true
bind:
create_host_path: true
```
위 디렉터리는 보조 기능 전용이므로 없으면 빈 디렉터리를 만들 수 있다. 파일 부재는
관측만 비활성화한다. DB·backup 같은 필수 저장소에는 이 fallback을 적용하지 않는다.
운영에는 이 전용 디렉터리의 `host_boot.key` 파일을 전달한다.
처음 인증에 성공한 식별자를 프로세스의 기준으로 삼고 고정된 성공 로그를 남긴다. 이후 다른 식별자를
확인하면 `TurnDaemonLeaseLostError`로 기존 실행자를 중단하고 DB를 다시 읽는다.
종료 시 owner/epoch가 여전히 자기 것인 경우에만 DB lease와 `clock_ready`를
해제하므로 다른 실행자를 중지시키지 않는다. 재시작은 기존 profile 상태를
보존하며 PAUSED/STOPPED를 임의로 RUNNING으로 바꾸지 않는다.
보통 최대 2초 간격으로 확인한다. 벽시계와 monotonic 경과가 1초 이상 어긋나면
cache를 무시하고 다시 조회한다. 조회 timeout은 1초다. 네트워크·인증·응답 오류는
분당 한 번의 고정 경고만 남기고 기존 DB lease 복구를 유지한다. 처음 인증받기 전
발생한 reboot는 이 신호만으로 확정할 수 없으며 기존 복구 경로가 담당한다.
## 인증 프로토콜과 검증
key 파일은 32바이트 난수를 lowercase hex 64자로 저장한다. 매 요청마다 새 32바이트
challenge를 만들며, key 자체를 네트워크에 보내지 않는다.
- GET `/v1/boot-id`, query string 없음.
- `x-sammo-challenge`: lowercase hex 64자.
- `x-sammo-signature`: `HMAC-SHA256(key, "sammo-host-boot-request-v1\n" + challenge)`.
- JSON: `version=1`, `challenge`, `bootId`(hex 64자), `signature`.
- 응답 signature:
`HMAC-SHA256(key, "sammo-host-boot-response-v1\n" + challenge + "\n" + bootId)`.
위 `\n`은 실제 LF다. Linux는 challenge와 응답 signature를 검증하고, redirect와
2048바이트 초과 응답을 거부한다. 서명된 옛 응답도 새 challenge에는 유효하지 않다.
시각 정확도나 요청 시각 만료에 의존하지 않는다. 이 프로토콜은 인증과 무결성을
제공하며 암호화는 하지 않는다. 사설 망에서만 사용하며 공개 무인증 재개 API를
만들지 않는다.
```sh
node --test tools/host-recovery/windows-boot-agent.test.mjs
pnpm --filter @sammo-ts/game-engine test hostBootMonitor.test.ts runtimePauseGate.test.ts reservedTurnLeaseRetry.test.ts
```
CI에서는 Windows boot를 모사한 HTTP 응답, 위조·재사용·timeout, 실제 PostgreSQL
lease fencing과 queue cleanup을 검증한다. 실제 Windows CIM·Task Scheduler와
Hyper-V 재부팅 검증을 대체하지 않는다. 배포 후 기존 장애 진단 화면의 lease/
epoch와 public lobby `turnEngineRunning`, 실제 tick/queue revision 진행을 확인한다.
기능을 해제하려면 Linux의 두 환경변수를 함께 제거하고 해당 runtime을 정상
재기동한다. Windows task도 중지할 수 있다. DB migration이나 게임 초기화는 없다.
근거: [Microsoft LastBootUpTime](https://learn.microsoft.com/en-us/windows/win32/cimwin32prov/win32-operatingsystem),
[Node.js HMAC와 timingSafeEqual](https://nodejs.org/api/crypto.html),
[게임 시계 복구 계약](../architecture/game-clock-reconciliation.md),
[릴리스 운영](../release-operations.md).
+5 -1
View File
@@ -5,6 +5,9 @@
관리한다. 아래 적용 절차는 각 프로필의 DB 보존 업데이트를 위한 것이며, 관리한다. 아래 적용 절차는 각 프로필의 DB 보존 업데이트를 위한 것이며,
문서에 기능이 기재되어 있다고 운영 DB에 적용된 것은 아니다. 문서에 기능이 기재되어 있다고 운영 DB에 적용된 것은 아니다.
저장량 최적화, 중지된 프로필 정리, ZFS 이전과 rollback은
[저장량과 운영 정리](play-audit-storage.md)를 따른다.
## 지금 사용할 수 있는 기능 ## 지금 사용할 수 있는 기능
| 화면 | 사용할 수 있는 정보 | 읽을 때 주의할 점 | | 화면 | 사용할 수 있는 정보 | 읽을 때 주의할 점 |
@@ -55,7 +58,7 @@
## DB migration과 적용 순서 ## DB migration과 적용 순서
정식 game migration에 감사 테이블과 인덱스가 포함되어 있다. `prisma db push`나 정식 game migration에 감사 테이블과 인덱스가 포함되어 있다. `prisma db push`나
수동 CREATE TABLE로 대신 적용하지 않는다. 현재 game chain은58개이며 다음 감사 수동 CREATE TABLE로 대신 적용하지 않는다. 현재 game chain은59개이며 다음 감사
migration들을 포함한다. 기존 기록을 삭제하거나 지난달 상세를 역산하지 않는다. migration들을 포함한다. 기존 기록을 삭제하거나 지난달 상세를 역산하지 않는다.
| migration | 준비되는 저장소/제약 | | migration | 준비되는 저장소/제약 |
@@ -69,6 +72,7 @@ migration들을 포함한다. 기존 기록을 삭제하거나 지난달 상세
| `20260916060000_widen_play_audit_ticks` | 월 표본/정책 tick을 BIGINT로 확장하여 약60개월 이후 INTEGER 초과 방지 | | `20260916060000_widen_play_audit_ticks` | 월 표본/정책 tick을 BIGINT로 확장하여 약60개월 이후 INTEGER 초과 방지 |
| `20260916070000_add_play_audit_decision` | NPC/자동턴 결정 요약과 순서별 상세 chunk, bounded 정리용 FK/index | | `20260916070000_add_play_audit_decision` | NPC/자동턴 결정 요약과 순서별 상세 chunk, bounded 정리용 FK/index |
| `20260916080000_index_play_audit_decision_month` | 기존 장수 인덱스를 월 조건 포함 인덱스로 교체하여 기수 전체 조회 방지 | | `20260916080000_index_play_audit_decision_month` | 기존 장수 인덱스를 월 조건 포함 인덱스로 교체하여 기수 전체 조회 방지 |
| `20261109000000_compact_play_audit_storage` | 무손실 압축 chunk 형식과 작은 JSON inline 압축 설정 |
운영은 [릴리스 절차](release-operations.md)의 **DB 보존 버전 업데이트**로 해당 고정 운영은 [릴리스 절차](release-operations.md)의 **DB 보존 버전 업데이트**로 해당 고정
commit을 적용한다. 이 기능을 켜기 위해 시나리오를 초기화할 필요는 없다. 수동 환경의 commit을 적용한다. 이 기능을 켜기 위해 시나리오를 초기화할 필요는 없다. 수동 환경의
+76
View File
@@ -0,0 +1,76 @@
# 플레이 감사 저장량과 운영 정리
2026-11-09 저장 최적화는 Core 전용 감사의 물리 표현과 정리 구현을 바꾼다.
현재 기수 전체 기록·판단 순서·정책 참조·hash·API 응답·권한은 유지한다.
원문 샘플링이나 보존기간 축소를 하지 않는다.
## 물리 표현
- 결정 chunk는 기존 `steps` JSONB 배열 또는 `{encoding:"gzip-json-v1",data:"base64"}`로 저장한다.
128 step 단위는 그대로이며 gzip level 3을 사용한다. 압축 envelope가 더 크면 원래 배열을 쓴다.
- replay hash는 압축 전 원문으로 계산한다. API는 공용 decoder 뒤 기존 step schema를 검증한다.
손상·알 수 없는 버전은 조회 오류이며 빈 자료로 바꾸지 않는다.
- 압축 해제는 chunk당 16MiB로 제한한다. 더 큰 원문은 기존 배열로 저장해 잘라내지 않는다.
- 장수 snapshot `data`와 결정 `summary`는 PostgreSQL `STORAGE MAIN`,
`toast_tuple_target=512`로 작은 반복 JSON도 inline 압축한다. JSON 값은 바뀌지 않는다.
- migration `20261109000000_compact_play_audit_storage`는 기존 테이블을 재작성하지 않는다.
chunk CHECK는 새 쓰기에 즉시 적용하고 대형 과거 자료 검사는 `NOT VALID`로 미룬다.
정리 후 `ALTER TABLE ... VALIDATE CONSTRAINT play_audit_decision_chunk_steps_check`를 실행한다.
기존 자료의 압축 변경은 별도 bounded backfill 또는 검증된 export/restore가 필요하다.
## 정리 처리량과 실행 수명주기
- 이전 기수 결정 부모를 기존 `(server_id,nation_id,tick,id)` 인덱스로 최대 200개 잠근다.
한 transaction의 chunk+header 삭제 합도 200행 이하이며 큰 결정 하나가 있어도 상한을 지킨다.
- 다음 성공 batch까지 100ms를 기다린다. RESET과 동일 advisory lock, identity 재검사,
2초 statement/250ms lock 제한, rollback을 유지한다. 현재 기수의 결정과 표본은 삭제하지 않는다.
- API outbox는 전달 후 24시간을 보존한다. 기존 인덱스 순서로 최대 500행씩 정리하고,
backlog가 있으면 다음 dispatch에서도 이어간다. 한 호출은 최대 10 batch/50ms이며
미전달 자료는 삭제하지 않는다. 빈 상태에서는 60초 주기로 돌아간다.
- game API/daemon이 중지된 프로필도 host scheduler가 다음 CLI로 정리한다.
CLI는 접속 URL이나 SQL 원문을 출력하지 않는다. 기본은 dry-run이다.
```sh
# 먼저 common/logic/infra/game-engine을 빌드한다. 접속 정보는 기존 process 환경으로 전달한다.
node tools/play-audit-maintenance.mjs --schemas=che,kwe,pwe,twe,nya,pya,hwe
node tools/play-audit-maintenance.mjs --schemas=che,kwe,pwe,twe,nya,pya,hwe --budget-ms=3000 --apply
```
schema마다 최신 identity를 읽고 각 삭제 transaction에서 다시 확인한다. 예상 identity가 바뀌면
그 schema의 처리를 중단한다. host timer는 다음 실행에서 새 identity를 읽는다.
장수/국가/명령/input_event 같은 gameplay 테이블은 이 CLI의 삭제 대상이 아니다.
## ZFS 운영 경계
호스트는 감사 전용 tablespace와 backup 전용 dataset을 별도로 제공한다.
hot gameplay, WAL, legacy archive를 임의로 이동하지 않는다. DB와 tablespace는 한 cluster이므로
tablespace 파일만 백업하는 것으로 복구를 보장하지 않는다.
1. 전체 DB의 일관된 압축 backup을 ZFS에 만들고 checksum, archive 읽기, 실제 복원을 확인한다.
2. DB 보존 DEPLOY로 writer/reader/migration을 함께 적용한다.
3. profile 상태를 기록하고 공식 STOP으로 쓰기를 중단한다. 원래 STOPPED인 profile은 유지한다.
가오픈까지 1시간 이상 남은 RESERVED는 runtimeExpected=false이므로 예약을 유지할 수 있다.
실제 process 비기동과 identity/hash 안정성을 확인하고 export의 예상 상태를 RESERVED로 고정한다.
유지보수 중 RESET 등으로 identity가 바뀌면 해당 profile을 새로 export/검증하며 이전 자료를 덮어쓰지 않는다.
4. 각 schema에서 현재 serverId의 8개 감사 테이블을 일관되게 export한다.
월 child는 현재 월 header와 join하고 결정 chunk는 현재 결정 header와 join한다.
5. 격리 DB에서 복원·건수·내용 hash를 검증한다. 이전 기수는 전체 ZFS backup에 남는다.
6. 하나의 transaction에서 감사 테이블만 TRUNCATE하고 ZFS tablespace로 옮긴 뒤
현재 기수 export를 다시 넣는다. 부모를 먼저 넣고 모든 건수/hash와 identity를 확인한 뒤 commit한다.
index도 별도로 ZFS로 옮긴다. 실패하면 전체 transaction을 rollback한다.
7. CHECK 검증과 ANALYZE를 수행하고 원래 RUNNING이었던 profile만 공식 START한다.
현재 기록, 새 기록의 decode/hash, lease/clock, 공개 HTTPS와 루트/ZFS 실제 사용량을 확인한다.
기존 데이터가 적을 때는 bounded DELETE 뒤 별도 재작성으로 공간을 회수할 수도 있다.
수천만 건 backlog에서 export/TRUNCATE/restore는 현재 자료를 보존하며 물리 공간을 즉시
회수하기 위한 계획된 유지보수 절차다. backup 검증이나 profile 정지가 빠진 자동 작업으로 쓰지 않는다.
ZFS dataset이 없을 때 루트 디렉터리에 쓰지 않도록 host mount guard와 재현 가능한 Compose 설정이 필요하다.
## rollback
이전 reader는 압축 envelope를 이해하지 못한다. 구형 버전으로 돌아가기 전 해당 기록을
새 decoder로 원래 JSON 배열로 변환하고 같은 hash/개수를 검증한다. 물리 압축 설정은
이전 애플리케이션과 호환되며 원래 TOAST 설정으로 돌릴 필요가 없다.
현재 기수 복원 중 실패하면 transaction rollback으로 기존 table/data/tablespace를 보존한다.
기수 전체 과거 자료가 필요하면 ZFS의 변경 전 backup을 별도 DB에 복원한다.
운영 DB 전체를 과거 dump로 덮어써 새 gameplay를 잃게 하지 않는다.
+12
View File
@@ -376,6 +376,18 @@ Hyper-V의 VM 정지/재개, 체크포인트 또는 host 시간 보정은 게스
NTP 로그와 Windows Hyper-V/백업 작업 이력을 대조합니다. lease 기간을 늘리거나 NTP 로그와 Windows Hyper-V/백업 작업 이력을 대조합니다. lease 기간을 늘리거나
시간 동기화를 임의로 끄는 것으로 해결하지 않습니다. 시간 동기화를 임의로 끄는 것으로 해결하지 않습니다.
예약턴의 실행 전 임대 충돌은 메모리 rollback 뒤 1초 간격, 한 구간 최대 301초 동안
해제·만료를 기다립니다. flush/finalize의 ownership 상실은 재시도하지 않습니다.
종료 시에는 rollback으로 journal에서 사라진 자기 owner의 queue 임대도 DB에서
해제합니다. 다른 owner와 queue 내용/revision은 보존합니다. 대기 중에도 매 loop의
운영자 pause/stop과 daemon gate를 통과하며 상한을 넘으면 기존 오류 정지를 유지합니다.
Windows의 저장·복원으로 guest 이벤트가 없을 때를 위한 선택적 boot identity 확인은
[Windows Node.js 에이전트 설치와 Linux 연결](./developer/host-boot-recovery.md)을 따릅니다.
`TURN_HOST_BOOT_URL`과 private `TURN_HOST_BOOT_KEY_FILE`을 함께 설정하면 인증된
식별자 변경을 기존 lease 상실/재기동으로 처리합니다. 통신 실패는 기존 복구를
막지 않으며 운영자의 PAUSED/STOPPED를 자동 해제하지 않습니다.
Turn daemon의 DB persistence interactive transaction은 연결 획득 대기(`maxWait`) Turn daemon의 DB persistence interactive transaction은 연결 획득 대기(`maxWait`)
10초와 callback 실행(`timeout`) 30초를 각각 제한합니다. Prisma 기본 연결 대기 10초와 callback 실행(`timeout`) 30초를 각각 제한합니다. Prisma 기본 연결 대기
2초는 일시적인 DB 지연에도 callback 시작 전 `P2028`로 서버를 PAUSED로 만들 수 2초는 일시적인 DB 지연에도 callback 시작 전 `P2028`로 서버를 PAUSED로 만들 수
@@ -0,0 +1,18 @@
-- 대형 기존 기록을 migration 중 재작성/전체 검사하지 않는다. 이전 배열과 새 압축 형식을 함께 허용한다.
BEGIN;
SET LOCAL lock_timeout = '2s';
ALTER TABLE "play_audit_decision_chunk" DROP CONSTRAINT "play_audit_decision_chunk_steps_check";
ALTER TABLE "play_audit_decision_chunk" ADD CONSTRAINT "play_audit_decision_chunk_steps_check"
CHECK (COALESCE(
jsonb_typeof("steps") = 'array' OR (
jsonb_typeof("steps") = 'object'
AND "steps"->>'encoding' = 'gzip-json-v1'
AND jsonb_typeof("steps"->'data') = 'string'
), false
)) NOT VALID;
-- 기본 약 2KiB 문턱 아래의 반복 JSON도 압축하되 작은 snapshot은 TOAST 밖으로 쪼개지 않는다.
ALTER TABLE "play_audit_general" ALTER COLUMN "data" SET STORAGE MAIN;
ALTER TABLE "play_audit_general" SET (toast_tuple_target = 512);
ALTER TABLE "play_audit_decision" ALTER COLUMN "summary" SET STORAGE MAIN;
ALTER TABLE "play_audit_decision" SET (toast_tuple_target = 512);
COMMIT;
+1
View File
@@ -15,3 +15,4 @@ export * from './playAuditDiplomacy.js';
export * from './messageEnvelope.js'; export * from './messageEnvelope.js';
export * from './webPushOutbox.js'; export * from './webPushOutbox.js';
export * from './playAuditBestEffort.js'; export * from './playAuditBestEffort.js';
export * from './playAuditDecisionCodec.js';
@@ -0,0 +1,44 @@
import { gzipSync, gunzipSync } from 'node:zlib';
import type { InputJsonValue } from './turnEngineDb.js';
const ENCODING = 'gzip-json-v1';
const MAX_STEPS = 128;
const MAX_BYTES = 16 * 1024 * 1024;
/** 조회/재실행 hash에 쓰는 원문은 바꾸지 않고 저장할 때만 무손실 압축한다. */
export const encodeAuditDecisionSteps = (steps: readonly InputJsonValue[]): InputJsonValue => {
if (steps.length > MAX_STEPS) throw new Error('Audit decision chunk exceeds its step limit');
const json = JSON.stringify(steps);
// 드문 초대형 기존 payload도 버리거나 자르지 않는다. 압축 reader 상한 밖이면 원래 형식으로 저장한다.
if (Buffer.byteLength(json) > MAX_BYTES) return steps.slice();
const encoded = { encoding: ENCODING, data: gzipSync(json, { level: 3 }).toString('base64') };
return Buffer.byteLength(JSON.stringify(encoded)) < Buffer.byteLength(json) ? encoded : steps.slice();
};
/** 이전 JSONB 배열과 압축 형식을 함께 읽는다. 손상/알 수 없는 버전은 빈 기록으로 숨기지 않는다. */
export const decodeAuditDecisionSteps = (value: unknown): unknown[] => {
if (Array.isArray(value)) {
if (value.length > MAX_STEPS) throw new Error('Audit decision chunk exceeds its step limit');
return value;
}
if (!value || typeof value !== 'object' || !('encoding' in value) || !('data' in value))
throw new Error('Invalid audit decision storage');
if (
value.encoding !== ENCODING ||
typeof value.data !== 'string' ||
value.data.length > Math.ceil(MAX_BYTES / 3) * 4 + 256 ||
!/^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/.test(value.data)
)
throw new Error('Invalid audit decision encoding');
let decoded: unknown;
try {
decoded = JSON.parse(
gunzipSync(Buffer.from(value.data, 'base64'), { maxOutputLength: MAX_BYTES }).toString('utf8')
);
} catch {
// DB 원문이나 압축 payload가 오류 응답/로그에 포함되지 않게 한다.
throw new Error('Corrupt audit decision storage');
}
if (!Array.isArray(decoded) || decoded.length > MAX_STEPS) throw new Error('Invalid audit decision steps');
return decoded;
};
@@ -195,8 +195,9 @@ export const pruneDeliveredReadModelOutbox = async (
SELECT "id" SELECT "id"
FROM "read_model_outbox" FROM "read_model_outbox"
WHERE "delivered_at" < ${input.deliveredBefore} WHERE "delivered_at" < ${input.deliveredBefore}
ORDER BY "id" ORDER BY "delivered_at", "available_at", "id"
LIMIT ${limit} LIMIT ${limit}
FOR UPDATE SKIP LOCKED
) )
RETURNING "id" RETURNING "id"
`); `);
@@ -0,0 +1,54 @@
import { gzipSync } from 'node:zlib';
import { describe, expect, it } from 'vitest';
import { decodeAuditDecisionSteps, encodeAuditDecisionSteps } from '../src/playAuditDecisionCodec.js';
describe('lossless audit decision storage', () => {
it('compresses repeated context while preserving every value and its order', () => {
const steps = Array.from({ length: 128 }, (_, sequence) => ({
sequence,
phase: 'general',
generalId: 51,
nationId: 3,
cityId: 12,
year: 190,
month: 7,
kind: 'RNG',
value: sequence / 128,
text: '판단·줄바꿈\n따옴표 " 그대로',
empty: null,
}));
const stored = encodeAuditDecisionSteps(steps);
expect(Array.isArray(stored)).toBe(false);
expect(Buffer.byteLength(JSON.stringify(stored))).toBeLessThan(Buffer.byteLength(JSON.stringify(steps)) / 3);
expect(decodeAuditDecisionSteps(stored)).toEqual(steps);
});
it('keeps tiny and historical uncompressed arrays readable', () => {
expect(encodeAuditDecisionSteps([])).toEqual([]);
expect(decodeAuditDecisionSteps([{ marker: 'legacy', sequence: 0 }])).toEqual([
{ marker: 'legacy', sequence: 0 },
]);
});
it('rejects unknown versions, bad base64, truncated gzip and invalid contents', () => {
for (const value of [
null,
{},
{ encoding: 'gzip-json-v2', data: '' },
{ encoding: 'gzip-json-v1', data: '%%%' },
{ encoding: 'gzip-json-v1', data: 'YQ==' },
{ encoding: 'gzip-json-v1', data: gzipSync('{}').toString('base64') },
{ encoding: 'gzip-json-v1', data: gzipSync(JSON.stringify(Array(129).fill(null))).toString('base64') },
])
expect(() => decodeAuditDecisionSteps(value)).toThrow();
expect(() => encodeAuditDecisionSteps(Array(129).fill(null))).toThrow();
});
it('bounds decompression and preserves oversized raw records rather than truncating them', () => {
const steps = [{ text: 'x'.repeat(16 * 1024 * 1024) }];
expect(encodeAuditDecisionSteps(steps)).toEqual(steps);
expect(() =>
decodeAuditDecisionSteps({
encoding: 'gzip-json-v1',
data: gzipSync(JSON.stringify(steps)).toString('base64'),
})
).toThrow('Corrupt');
});
});
+1 -1
View File
@@ -2,6 +2,6 @@
"formatVersion": 1, "formatVersion": 1,
"controllerProtocol": 2, "controllerProtocol": 2,
"gatewaySchemaHead": "20261107000000_kakao_identity_link_history", "gatewaySchemaHead": "20261107000000_kakao_identity_link_history",
"gameSchemaHead": "20260916080000_index_play_audit_decision_month", "gameSchemaHead": "20261109000000_compact_play_audit_storage",
"components": ["gateway-api", "gateway-frontend", "release-controller", "game-api", "game-engine", "game-frontend"] "components": ["gateway-api", "gateway-frontend", "release-controller", "game-api", "game-engine", "game-frontend"]
} }
+12
View File
@@ -0,0 +1,12 @@
# Windows host boot identity agent
`windows-boot-agent.mjs`는 외부 npm package 없이 실행하는 Node.js 에이전트다.
Windows OS의 부팅 시각 기반 식별자를 인증된 Linux 게임 데몬에 제공한다.
설치, key 생성, 자동 시작과 선택 기능 장애 시 동작은
[Hyper-V 복구 운영 안내](../../docs/developer/host-boot-recovery.md)를 따른다.
설정 예시는 `windows-boot-agent.example.json`이며 실제 key와 설정은 Git에 넣지 않는다.
```sh
node --test tools/host-recovery/windows-boot-agent.test.mjs
```
@@ -0,0 +1,5 @@
{
"keyFile": "C:\\ProgramData\\Sammo\\host-boot.key",
"bindAddress": "127.0.0.1",
"port": 17781
}
+131
View File
@@ -0,0 +1,131 @@
import { execFile } from 'node:child_process';
import { createHash, createHmac, randomBytes, timingSafeEqual } from 'node:crypto';
import { readFile, writeFile } from 'node:fs/promises';
import { createServer } from 'node:http';
import { hostname } from 'node:os';
import { win32, resolve } from 'node:path';
import { fileURLToPath } from 'node:url';
import { promisify } from 'node:util';
const mac = (key, value) => createHmac('sha256', key).update(value).digest('hex');
export const bootIdentity = (host, ticks) => {
if (!/^\d{17,20}$/.test(ticks)) throw new Error('Windows returned an invalid boot time.');
return createHash('sha256').update(`${host.toLowerCase()}\n${ticks}`).digest('hex');
};
export const readWindowsBootIdentity = async () => {
if (process.platform !== 'win32') throw new Error('Run the host boot agent on Windows.');
const executable = win32.join(
process.env.SystemRoot ?? 'C:\\Windows',
'System32',
'WindowsPowerShell',
'v1.0',
'powershell.exe'
);
// Node.js가 OS 내장 CIM 조회를 실행한다. 별도 PowerShell 스크립트/모듈은 필요 없다.
const { stdout } = await promisify(execFile)(
executable,
[
'-NoProfile',
'-NonInteractive',
'-Command',
'(Get-CimInstance -ClassName Win32_OperatingSystem -Property LastBootUpTime).LastBootUpTime.ToUniversalTime().Ticks.ToString()',
],
{ windowsHide: true, timeout: 10_000, maxBuffer: 2_048 }
);
return bootIdentity(hostname(), stdout.trim());
};
export const createBootIdentityServer = ({ key, getBootId }) => {
if (!Buffer.isBuffer(key) || key.length !== 32) throw new Error('A 32-byte host boot key is required.');
const server = createServer((request, response) => {
response.setHeader('Cache-Control', 'no-store');
if (request.method !== 'GET' || request.url !== '/v1/boot-id') {
response.writeHead(404).end();
return;
}
const challenge = request.headers['x-sammo-challenge'];
const signature = request.headers['x-sammo-signature'];
if (
typeof challenge !== 'string' ||
!/^[a-f0-9]{64}$/.test(challenge) ||
typeof signature !== 'string' ||
!/^[a-f0-9]{64}$/.test(signature) ||
!timingSafeEqual(
Buffer.from(signature, 'hex'),
Buffer.from(mac(key, `sammo-host-boot-request-v1\n${challenge}`), 'hex')
)
) {
response.writeHead(401).end();
return;
}
const bootId = getBootId();
if (typeof bootId !== 'string' || !/^[a-f0-9]{64}$/.test(bootId)) {
response.writeHead(503).end();
return;
}
response.writeHead(200, { 'Content-Type': 'application/json' });
response.end(
JSON.stringify({
version: 1,
challenge,
bootId,
signature: mac(key, `sammo-host-boot-response-v1\n${challenge}\n${bootId}`),
})
);
});
server.headersTimeout = 5_000;
server.requestTimeout = 5_000;
server.keepAliveTimeout = 1_000;
server.maxHeadersCount = 20;
return server;
};
export const main = async (args = process.argv.slice(2), env = process.env) => {
if (args.length === 2 && args[0] === '--create-key') {
await writeFile(args[1], `${randomBytes(32).toString('hex')}\n`, { flag: 'wx', mode: 0o600 });
console.info('Host boot key created. Copy it privately to the Linux secret file.');
return;
}
if (args.length === 2 && args[0] === '--config') {
const config = JSON.parse(await readFile(args[1], 'utf8'));
if (
!config ||
typeof config.keyFile !== 'string' ||
typeof config.bindAddress !== 'string' ||
!Number.isInteger(config.port)
) {
throw new Error('Host boot agent configuration is invalid.');
}
env = {
...env,
HOST_BOOT_KEY_FILE: config.keyFile,
HOST_BOOT_BIND_ADDRESS: config.bindAddress,
HOST_BOOT_PORT: String(config.port),
};
} else if (args.length) throw new Error('Usage: node windows-boot-agent.mjs [--create-key path | --config path]');
if (!env.HOST_BOOT_KEY_FILE) throw new Error('HOST_BOOT_KEY_FILE is required.');
const encoded = (await readFile(env.HOST_BOOT_KEY_FILE, 'utf8')).trim();
if (!/^[a-f0-9]{64}$/.test(encoded)) throw new Error('Host boot key file is invalid.');
const port = Number(env.HOST_BOOT_PORT ?? '17781');
if (!Number.isInteger(port) || port < 1 || port > 65535) throw new Error('HOST_BOOT_PORT is invalid.');
// OS 부팅 시각은 agent 재시작에도 같으며, 시각 차이 계산으로 재부팅을 추정하지 않는다.
const bootId = await readWindowsBootIdentity();
const server = createBootIdentityServer({ key: Buffer.from(encoded, 'hex'), getBootId: () => bootId });
server.on('error', () => {
console.error('Host boot agent listener failed.');
process.exitCode = 1;
});
server.listen(port, env.HOST_BOOT_BIND_ADDRESS ?? '127.0.0.1', () => {
console.info('Host boot identity agent is listening.');
});
for (const signal of ['SIGINT', 'SIGTERM']) process.once(signal, () => server.close());
};
if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) {
main().catch(() => {
console.error('Host boot agent startup failed; check OS, private key file, and listener configuration.');
process.exitCode = 1;
});
}
@@ -0,0 +1,42 @@
import { after, test } from 'node:test';
import assert from 'node:assert/strict';
import { createHmac, randomBytes, timingSafeEqual } from 'node:crypto';
import { once } from 'node:events';
import { bootIdentity, createBootIdentityServer } from './windows-boot-agent.mjs';
test('agent restart preserves OS boot identity; a new OS boot changes it', () => {
assert.equal(bootIdentity('HOST', '638985120000000000'), bootIdentity('host', '638985120000000000'));
assert.notEqual(bootIdentity('HOST', '638985120000000000'), bootIdentity('HOST', '638985120000000001'));
assert.throws(() => bootIdentity('HOST', 'invalid'));
});
test('HTTP transport rejects anonymous requests and signs the current challenge and boot identity', async () => {
const key = randomBytes(32);
let bootId = 'a'.repeat(64);
const server = createBootIdentityServer({ key, getBootId: () => bootId });
server.listen(0, '127.0.0.1');
await once(server, 'listening');
const url = `http://127.0.0.1:${server.address().port}/v1/boot-id`;
after(
() =>
new Promise((resolve) => {
server.close(resolve);
server.closeAllConnections();
})
);
assert.equal((await fetch(url)).status, 401);
assert.equal((await fetch(`${url}?bypass=1`)).status, 404);
const challenge = randomBytes(32).toString('hex');
const signature = createHmac('sha256', key).update(`sammo-host-boot-request-v1\n${challenge}`).digest('hex');
const headers = { 'x-sammo-challenge': challenge, 'x-sammo-signature': signature };
const response = await fetch(url, { headers });
assert.equal(response.status, 200);
assert.equal(response.headers.get('cache-control'), 'no-store');
const body = await response.json();
assert.equal(body.challenge, challenge);
const expected = createHmac('sha256', key).update(`sammo-host-boot-response-v1\n${challenge}\n${bootId}`).digest();
assert.ok(timingSafeEqual(expected, Buffer.from(body.signature, 'hex')));
bootId = 'b'.repeat(64);
assert.equal((await (await fetch(url, { headers })).json()).bootId, bootId);
assert.equal((await fetch(url, { headers: { ...headers, 'x-sammo-signature': '0'.repeat(64) } })).status, 401);
});
@@ -3,7 +3,7 @@ import assert from 'node:assert/strict';
import { mkdir, readFile, writeFile } from 'node:fs/promises'; import { mkdir, readFile, writeFile } from 'node:fs/promises';
import { resolve } from 'node:path'; import { resolve } from 'node:path';
import { asRecord, type TurnCheckpoint } from '@sammo-ts/common'; import { asRecord, type TurnCheckpoint } from '@sammo-ts/common';
import { createGamePostgresConnector } from '@sammo-ts/infra'; import { createGamePostgresConnector, decodeAuditDecisionSteps } from '@sammo-ts/infra';
import { createTurnDaemonRuntime, seedScenarioToDatabase } from '@sammo-ts/game-engine'; import { createTurnDaemonRuntime, seedScenarioToDatabase } from '@sammo-ts/game-engine';
const databaseUrl = process.env.DATABASE_URL; const databaseUrl = process.env.DATABASE_URL;
@@ -39,7 +39,7 @@ const verify = async (progress: unknown[]) => {
include: { chunks: { orderBy: { ordinal: 'asc' } } }, include: { chunks: { orderBy: { ordinal: 'asc' } } },
}); });
assert(row && row.chunks.length, `Missing actual NPC ${JSON.stringify(selection)} decision`); assert(row && row.chunks.length, `Missing actual NPC ${JSON.stringify(selection)} decision`);
const steps = row.chunks.flatMap((chunk) => (Array.isArray(chunk.steps) ? chunk.steps : [])); const steps = row.chunks.flatMap((chunk) => decodeAuditDecisionSteps(chunk.steps));
assert.equal(steps.length, row.stepCount); assert.equal(steps.length, row.stepCount);
assert.equal(asRecord(steps[0]).kind, 'DECISION_START'); assert.equal(asRecord(steps[0]).kind, 'DECISION_START');
assert(steps.some((step) => asRecord(step).kind === 'DECISION_END')); assert(steps.some((step) => asRecord(step).kind === 'DECISION_END'));
+95
View File
@@ -0,0 +1,95 @@
#!/usr/bin/env node
// stopped/reserved profile도 같은 fenced batch로 정리한다. 기본은 관찰만 하며 외부 scheduler가 실행한다.
import { performance } from 'node:perf_hooks';
import { setTimeout } from 'node:timers/promises';
import { createGamePostgresConnector, pruneDeliveredReadModelOutbox } from '../packages/infra/dist/index.js';
// PM2/호출자가 넘긴 turn-daemon 역할로 engine root의 자동 기동 경로에 들어가지 않는다.
process.env.GAME_ENGINE_ROLE = 'audit-maintenance';
const { prunePreviousAuditBatch } = await import('../app/game-engine/dist/index.js');
const args = process.argv.slice(2);
const apply = args.includes('--apply');
const schemas =
args
.find((arg) => arg.startsWith('--schemas='))
?.slice(10)
.split(',') ?? [];
const budgetMs = Number(args.find((arg) => arg.startsWith('--budget-ms='))?.slice(12) ?? 3000);
if (
!schemas.length ||
schemas.some((schema) => !/^[a-z][a-z0-9_]{0,62}$/.test(schema)) ||
!Number.isSafeInteger(budgetMs) ||
budgetMs < 1 ||
budgetMs > 60000 ||
args.some((arg) => arg !== '--apply' && !arg.startsWith('--schemas=') && !arg.startsWith('--budget-ms='))
) {
throw new Error('Usage: --schemas=che,kwe,... [--budget-ms=3000] [--apply]');
}
let failed = false;
for (const schema of schemas) {
const url = process.env.DATABASE_URL ? new URL(process.env.DATABASE_URL) : new URL('postgresql://localhost');
if (!process.env.DATABASE_URL) {
url.hostname = process.env.POSTGRES_HOST ?? 'localhost';
url.port = process.env.POSTGRES_PORT ?? '5432';
url.username = process.env.POSTGRES_USER ?? 'sammo';
url.password = process.env.POSTGRES_PASSWORD ?? '';
url.pathname = `/${process.env.POSTGRES_DB ?? 'sammo'}`;
}
url.searchParams.set('schema', schema);
const connector = createGamePostgresConnector({ url: url.href, maxConnections: 1 });
try {
await connector.connect();
const db = connector.prisma;
const world = await db.worldState.findFirst({ select: { meta: true } });
const serverId =
world?.meta && typeof world.meta === 'object' && !Array.isArray(world.meta) ? world.meta.serverId : null;
if (typeof serverId !== 'string' || !serverId.trim()) {
console.log(JSON.stringify({ schema, status: 'identityMissing', deleted: 0 }));
continue;
}
if (!apply) {
console.log(JSON.stringify({ schema, status: 'dryRun', apply: false }));
continue;
}
const deadline = performance.now() + budgetMs;
let auditDeleted = 0;
let outboxDeleted = 0;
let status = 'progress';
let auditComplete = false;
let outboxComplete = false;
do {
if (!auditComplete) {
const result = await prunePreviousAuditBatch(db, serverId);
auditDeleted += result.deleted;
status = result.status;
auditComplete = result.status === 'complete';
if (result.status === 'identityChanged' || result.status === 'busy') break;
}
if (!outboxComplete) {
outboxDeleted += await db.$transaction(async (tx) => {
await tx.$executeRaw`SET LOCAL statement_timeout='2000ms'`;
await tx.$executeRaw`SET LOCAL lock_timeout='250ms'`;
const deleted = await pruneDeliveredReadModelOutbox(tx, {
deliveredBefore: new Date(Date.now() - 86400000),
limit: 500,
});
outboxComplete = deleted < 500;
return deleted;
});
}
if (auditComplete && outboxComplete) break;
await setTimeout(25);
} while (performance.now() < deadline);
console.log(
JSON.stringify({ schema, status, auditDeleted, outboxDeleted, complete: auditComplete && outboxComplete })
);
} catch {
// 연결 URL과 원문 SQL을 출력하지 않고 scheduler가 감지할 수 있는 종료 코드를 남긴다.
failed = true;
console.error(JSON.stringify({ schema, status: 'failed' }));
} finally {
await connector.disconnect();
}
}
if (failed) process.exitCode = 1;