VM 복귀 후 예약턴 잠금 복구와 Windows 부팅 식별자 확인을 추가한다

app/game-engine/src/turn/cli.ts

@@ -8,6 +8,7 @@ import { createTurnDaemonMemoryReporter } from './turnDaemonMemoryReporter.js';
 import { createGatewayProfileGate } from './gatewayProfileGate.js';
 import { retryTurnDaemonLeaseStartup } from './leaseStartupRetry.js';
 import { TurnDaemonLeaseUnavailableError } from '../lifecycle/databaseTurnDaemonLease.js';
+import { loadHostBootMonitor } from './hostBootMonitor.js';
 
 export interface TurnDaemonCliOptions {
     profile?: string;
@@ -80,6 +81,7 @@ export const runTurnDaemonCli = async (options: TurnDaemonCliOptions = {}): Prom
         throw new Error(`GAME_CLOCK_MODE must be realtime or manual: ${rawGameClockMode}`);
     }
     const gameClockMode = rawGameClockMode as GameClockMode | undefined;
+    const hostBootMonitor = await loadHostBootMonitor(env);
 
     const runtime = await retryTurnDaemonLeaseStartup(() =>
         createTurnDaemonRuntime({
@@ -95,6 +97,7 @@ export const runTurnDaemonCli = async (options: TurnDaemonCliOptions = {}): Prom
             pauseGateIntervalMs,
             adminActionIntervalMs,
             gameClockMode,
+            hostBootMonitor,
         })
     ).catch(async (error: unknown) => {
         // 중복 starter가 정상 owner를 멈추면 안 된다. 그 밖의 초기화 실패는

app/game-engine/src/turn/hostBootMonitor.ts

@@ -0,0 +1,142 @@
+import { createHmac, randomBytes, timingSafeEqual } from 'node:crypto';
+import { readFile } from 'node:fs/promises';
+import { performance } from 'node:perf_hooks';
+import { isRecord } from '@sammo-ts/common';
+
+export interface HostBootMonitor {
+    hasChanged(): Promise<boolean>;
+}
+
+const mac = (key: Buffer, value: string): string => createHmac('sha256', key).update(value).digest('hex');
+
+/** 부팅 세대는 첫 인증 응답에 고정한다. 통신 실패는 기존 lease 복구를 막지 않는다. */
+export const createHostBootMonitor = (options: {
+    url: string;
+    key: Buffer;
+    pollMs?: number;
+    timeoutMs?: number;
+    now?: () => number;
+    wallNow?: () => number;
+    onUnavailable?: () => void;
+    onVerified?: () => void;
+}): HostBootMonitor => {
+    const url = new URL(options.url);
+    if (
+        !['http:', 'https:'].includes(url.protocol) ||
+        url.username ||
+        url.password ||
+        url.search ||
+        url.hash ||
+        url.pathname !== '/v1/boot-id'
+    ) {
+        throw new Error('Host boot URL must use HTTP(S) and /v1/boot-id without credentials or query.');
+    }
+    if (options.key.length !== 32) throw new Error('Host boot key must contain exactly 32 bytes.');
+    const now = options.now ?? (() => performance.now());
+    const wallNow = options.wallNow ?? (() => Date.now());
+    let baseline: string | null = null;
+    let changed = false;
+    let lastAttempt = -Infinity;
+    let lastWall = 0;
+    let lastWarning = -Infinity;
+    let inFlight: Promise<boolean> | null = null;
+    const sample = async (): Promise<boolean> => {
+        const nonce = randomBytes(32).toString('hex');
+        const response = await fetch(url, {
+            method: 'GET',
+            redirect: 'error',
+            signal: AbortSignal.timeout(options.timeoutMs ?? 1_000),
+            headers: {
+                'x-sammo-challenge': nonce,
+                'x-sammo-signature': mac(options.key, `sammo-host-boot-request-v1\n${nonce}`),
+            },
+        });
+        if (!response.ok || !response.body) {
+            await response.body?.cancel();
+            throw new Error('Host boot response unavailable.');
+        }
+        const reader = response.body.getReader();
+        const chunks: Uint8Array[] = [];
+        let size = 0;
+        try {
+            for (;;) {
+                const { done, value } = await reader.read();
+                if (done) break;
+                size += value.length;
+                if (size > 2_048) throw new Error('Host boot response exceeds its limit.');
+                chunks.push(value);
+            }
+        } finally {
+            await reader.cancel();
+        }
+        const body: unknown = JSON.parse(Buffer.concat(chunks).toString('utf8'));
+        if (
+            !isRecord(body) ||
+            body.version !== 1 ||
+            body.challenge !== nonce ||
+            typeof body.bootId !== 'string' ||
+            !/^[a-f0-9]{64}$/.test(body.bootId) ||
+            typeof body.signature !== 'string' ||
+            !/^[a-f0-9]{64}$/.test(body.signature)
+        ) {
+            throw new Error('Host boot response is invalid.');
+        }
+        const expected = mac(options.key, `sammo-host-boot-response-v1\n${nonce}\n${body.bootId}`);
+        if (!timingSafeEqual(Buffer.from(expected, 'hex'), Buffer.from(body.signature, 'hex'))) {
+            throw new Error('Host boot response signature is invalid.');
+        }
+        if (baseline === null) {
+            baseline = body.bootId;
+            options.onVerified?.();
+        }
+        changed ||= body.bootId !== baseline;
+        return changed;
+    };
+    return {
+        hasChanged(): Promise<boolean> {
+            if (changed) return Promise.resolve(true);
+            if (inFlight) return inFlight;
+            const current = now();
+            const wall = wallNow();
+            const discontinuity = Math.abs(wall - lastWall - (current - lastAttempt)) > 1_000;
+            if (current - lastAttempt < (options.pollMs ?? 2_000) && !discontinuity) return Promise.resolve(false);
+            lastAttempt = current;
+            lastWall = wall;
+            inFlight = sample()
+                .catch(() => {
+                    if (current - lastWarning >= 60_000) {
+                        options.onUnavailable?.();
+                        lastWarning = current;
+                    }
+                    return false;
+                })
+                .finally(() => {
+                    inFlight = null;
+                });
+            return inFlight;
+        },
+    };
+};
+
+export const loadHostBootMonitor = async (env: NodeJS.ProcessEnv): Promise<HostBootMonitor | undefined> => {
+    const url = env.TURN_HOST_BOOT_URL?.trim();
+    const keyFile = env.TURN_HOST_BOOT_KEY_FILE?.trim();
+    if (!url && !keyFile) return undefined;
+    try {
+        if (!url || !keyFile) throw new Error('Incomplete optional configuration.');
+        const encoded = (await readFile(keyFile, 'utf8')).trim();
+        if (!/^[a-f0-9]{64}$/.test(encoded)) throw new Error('Invalid optional key file.');
+        return createHostBootMonitor({
+            url,
+            key: Buffer.from(encoded, 'hex'),
+            onVerified: () => console.info('[turn-daemon] host boot identity verified.'),
+            onUnavailable: () => {
+                console.warn('[turn-daemon] host boot identity unavailable; ordinary lease recovery remains active.');
+            },
+        });
+    } catch {
+        // 보조 관측기의 설정/파일 부재도 게임 기동의 필수 의존성이 되지 않는다.
+        console.warn('[turn-daemon] host boot observation disabled; check optional configuration and key file.');
+        return undefined;
+    }
+};

app/game-engine/src/turn/reservedTurnLeaseRetry.ts

@@ -0,0 +1,38 @@
+import { performance } from 'node:perf_hooks';
+import { setTimeout } from 'node:timers/promises';
+import { ReservedTurnLeaseBusyError } from './reservedTurnStore.js';
+
+/** rollback 완료 뒤에만 대기한다. 다음 loop에서 pause/lease gate와 최신 큐를 다시 읽는다. */
+export const createReservedTurnLeaseRetry = (
+    options: {
+        now?: () => number;
+        sleep?: (ms: number) => Promise<void>;
+        maxWaitMs?: number;
+        onWait?: (queueKey: string) => void;
+    } = {}
+) => {
+    const now = options.now ?? (() => performance.now());
+    const sleep = options.sleep ?? ((ms: number) => setTimeout(ms));
+    const maxWaitMs = options.maxWaitMs ?? 5 * 60_000 + 1_000;
+    let startedAt: number | null = null;
+    let lastReportedAt = -Infinity;
+    return {
+        reset(): void {
+            startedAt = null;
+            lastReportedAt = -Infinity;
+        },
+        async wait(error: unknown): Promise<boolean> {
+            if (!(error instanceof ReservedTurnLeaseBusyError)) return false;
+            const current = now();
+            startedAt ??= current;
+            const remaining = maxWaitMs - (current - startedAt);
+            if (remaining <= 0) return false;
+            if (current - lastReportedAt >= 30_000) {
+                options.onWait?.(error.queueKey);
+                lastReportedAt = current;
+            }
+            await sleep(Math.min(1_000, remaining));
+            return true;
+        },
+    };
+};

app/game-engine/src/turn/reservedTurnStore.ts

@@ -129,6 +129,14 @@ export class ReservedTurnLeaseConflictError extends Error {
     }
 }
 
+/** 실행 전 획득 충돌만 재시도한다. flush/finalize의 소유권 상실과 구분한다. */
+export class ReservedTurnLeaseBusyError extends ReservedTurnLeaseConflictError {
+    constructor(queueKey: string) {
+        super(queueKey);
+        this.name = 'ReservedTurnLeaseBusyError';
+    }
+}
+
 export class InMemoryReservedTurnStore {
     private readonly generalTurns = new Map<number, ReservedTurnEntry[]>();
     private readonly nationTurns = new Map<string, ReservedTurnEntry[]>();
@@ -249,11 +257,7 @@ export class InMemoryReservedTurnStore {
             }
         };
         pruneNationKeys(Array.from(this.nationTurns.keys()), (key) => this.nationTurns.delete(key));
-        for (const keys of [
-            this.dirtyNationKeys,
-            this.pendingNationInitializationKeys,
-            this.leasedNationKeys,
-        ]) {
+        for (const keys of [this.dirtyNationKeys, this.pendingNationInitializationKeys, this.leasedNationKeys]) {
             for (const key of Array.from(keys)) {
                 const nationId = Number(key.split(':', 1)[0]);
                 if (deletedNations.has(nationId)) {
@@ -376,7 +380,7 @@ export class InMemoryReservedTurnStore {
             }
         }
         if (claimed.count !== 1) {
-            throw new ReservedTurnLeaseConflictError(`general:${generalId}`);
+            throw new ReservedTurnLeaseBusyError(`general:${generalId}`);
         }
         this.leasedGeneralIds.add(generalId);
         return !retainedExistingLease;
@@ -427,7 +431,7 @@ export class InMemoryReservedTurnStore {
             }
         }
         if (claimed.count !== 1) {
-            throw new ReservedTurnLeaseConflictError(`nation:${nationId}:${officerLevel}`);
+            throw new ReservedTurnLeaseBusyError(`nation:${nationId}:${officerLevel}`);
         }
         this.leasedNationKeys.add(buildNationKey(nationId, officerLevel));
         return !retainedExistingLease;
@@ -904,6 +908,24 @@ export class InMemoryReservedTurnStore {
         await this.persistChanges(this.prisma, changes);
         this.acknowledgeDirtyState(changes);
     }
+
+    /** 실행 루프 종료 뒤 호출한다. rollback으로 journal에서 빠진 DB 임대도 해제한다. */
+    async releaseOwnedLeases(): Promise<void> {
+        const results = await Promise.allSettled([
+            this.prisma.generalTurnRevision?.updateMany({
+                where: { leaseOwner: this.leaseOwner },
+                data: { leaseOwner: null, leaseExpiresAt: null },
+            }),
+            this.prisma.nationTurnRevision?.updateMany({
+                where: { leaseOwner: this.leaseOwner },
+                data: { leaseOwner: null, leaseExpiresAt: null },
+            }),
+        ]);
+        const errors = results.flatMap((result) => (result.status === 'rejected' ? [result.reason] : []));
+        if (errors.length) throw new AggregateError(errors, 'Reserved turn lease cleanup failed.');
+        this.leasedGeneralIds.clear();
+        this.leasedNationKeys.clear();
+    }
 }
 
 export const createReservedTurnStore = async (options: ReservedTurnStoreOptions): Promise<ReservedTurnStoreHandle> => {
@@ -918,6 +940,12 @@ export const createReservedTurnStore = async (options: ReservedTurnStoreOptions)
     await store.loadAll();
     return {
         store,
-        close: () => connector.disconnect(),
+        close: async () => {
+            try {
+                await store.releaseOwnedLeases();
+            } finally {
+                await connector.disconnect();
+            }
+        },
     };
 };

app/game-engine/src/turn/runtimePauseGate.ts

@@ -2,7 +2,7 @@ import type { GameClockPhase } from '@sammo-ts/common';
 
 /** Gateway의 실행 gate와 durable 시계를 명령 claim 전에 맞춘다. */
 export const createRuntimePauseGate = (options: {
-    assertLease(): void;
+    assertLease(): void | Promise<void>;
     shouldPause(): Promise<boolean>;
     getPhase(): GameClockPhase;
     isExplicitlyPaused(): boolean;
@@ -11,7 +11,7 @@ export const createRuntimePauseGate = (options: {
 }): (() => Promise<boolean>) => {
     let lastPaused: boolean | null = null;
     return async () => {
-        options.assertLease();
+        await options.assertLease();
         const paused = await options.shouldPause();
         const phase = options.getPhase();
         // 오류 정지는 Gateway 상태만 PAUSED로 바꿀 수 있다. 그대로 두면 가입은

app/game-engine/src/turn/turnDaemon.ts

@@ -108,6 +108,8 @@ import { DatabaseTurnDaemonLease, TurnDaemonLeaseUnavailableError } from '../lif
 import { EngineStateManager } from './engineStateManager.js';
 import { applyRuntimeClockShift } from './runtimeClockShift.js';
 import { applyRuntimeGameSettings } from './runtimeGameSettings.js';
+import { createReservedTurnLeaseRetry } from './reservedTurnLeaseRetry.js';
+import type { HostBootMonitor } from './hostBootMonitor.js';
 
 export interface TurnDaemonRuntimeOptions {
     profile: string;
@@ -134,6 +136,7 @@ export interface TurnDaemonRuntimeOptions {
     leaseDurationMs?: number;
     leaseOwnerId?: string;
     enableLeaseHeartbeat?: boolean;
+    hostBootMonitor?: HostBootMonitor;
     /**
      * Isolated, single-process fixture acceleration only. Reserved turns are
      * loaded once and no concurrent API writer may touch this database.
@@ -985,6 +988,7 @@ const createTurnDaemonRuntimeWithLease = async (
         prepareClockRecovery = dbHooks.prepareRealtimeRecovery;
         prunePreviousAudit = dbHooks.prunePreviousAudit;
         close = async () => {
+            await adminActionConsumer?.stop();
             if (auctionBidder) {
                 await auctionBidder.close();
             }
@@ -995,12 +999,15 @@ const createTurnDaemonRuntimeWithLease = async (
                 await tournamentRewardFinalizer.close();
             }
             await dbHooks.close();
-            if (reservedTurnStoreHandle) {
-                await reservedTurnStoreHandle.close();
+            try {
+                await reservedTurnStoreHandle?.close();
+            } finally {
+                try {
+                    await gatewayGate?.close();
+                } finally {
+                    await turnDaemonLease?.close();
+                }
             }
-            await gatewayGate?.close();
-            await adminActionConsumer?.stop();
-            await turnDaemonLease?.close();
         };
     } else if (reservedTurnStoreHandle) {
         hooks = {
@@ -1009,9 +1016,12 @@ const createTurnDaemonRuntimeWithLease = async (
             },
         };
         close = async () => {
-            await reservedTurnStoreHandle.close();
-            await gatewayGate?.close();
             await adminActionConsumer?.stop();
+            try {
+                await reservedTurnStoreHandle.close();
+            } finally {
+                await gatewayGate?.close();
+            }
         };
     } else if (gatewayGate) {
         hooks = {
@@ -1070,12 +1080,17 @@ const createTurnDaemonRuntimeWithLease = async (
     const baseClose = close;
     close = async () => {
         stopClockProjectionWorker();
-        await baseClose();
-        await neutralAuctionRegistrar.close();
-        if (redisConnector) {
-            await redisConnector.disconnect();
-        }
-        await commandConnector?.disconnect();
+        const baseResults = await Promise.allSettled([baseClose()]);
+        // queue 임대 정리 실패도 다른 연결/worker의 종료를 막지 않게 한다.
+        const results = await Promise.allSettled([
+            neutralAuctionRegistrar.close(),
+            redisConnector?.disconnect(),
+            commandConnector?.disconnect(),
+        ]);
+        const failures = [...baseResults, ...results].flatMap((result) =>
+            result.status === 'rejected' ? [result.reason] : []
+        );
+        if (failures.length) throw new AggregateError(failures, 'Turn daemon connection cleanup failed.');
     };
 
     const resolvedControlQueue = options.controlQueue ?? databaseCommandQueue ?? controlQueue;
@@ -1109,7 +1124,8 @@ const createTurnDaemonRuntimeWithLease = async (
             processor,
             hooks,
             pauseGate: createRuntimePauseGate({
-                assertLease: () => {
+                assertLease: async () => {
+                    await turnDaemonLease?.checkHostBoot();
                     if (turnDaemonLease?.isLost()) throw turnDaemonLease.getLossError();
                 },
                 shouldPause: async () => (await pauseGate?.()) ?? false,
@@ -1127,6 +1143,9 @@ const createTurnDaemonRuntimeWithLease = async (
             // accumulated world before every due tick in that isolated mode;
             // production and gateway-managed runtimes keep rollback savepoints.
             stateManager: options.exclusiveFastForward ? undefined : stateManager,
+            runErrorRetry: createReservedTurnLeaseRetry({
+                onWait: (queueKey) => console.info(`[turn-daemon] waiting for reserved turn lease: ${queueKey}`),
+            }),
         },
         { profile: options.profile, defaultBudget }
     );
@@ -1179,6 +1198,7 @@ export const createTurnDaemonRuntime = async (options: TurnDaemonRuntimeOptions)
               ownerId: options.leaseOwnerId,
               leaseDurationMs: options.leaseDurationMs,
               heartbeat: options.enableLeaseHeartbeat,
+              hostBootMonitor: options.hostBootMonitor,
           })
         : null;
     if (turnDaemonLease && !(await turnDaemonLease.acquire())) {