관리자 서버 진단에 lease 상태와 영속 장애 이력 추가

app/game-engine/src/lifecycle/databaseTurnDaemonLease.ts

@@ -24,8 +24,8 @@ export class TurnDaemonLeaseUnavailableError extends Error {
 }
 
 export class TurnDaemonLeaseLostError extends Error {
-    constructor(profile: string) {
-        super(`Turn daemon lease was lost for profile "${profile}".`);
+    constructor(profile: string, reason?: string) {
+        super(`Turn daemon lease was lost for profile "${profile}".${reason ? ` ${reason}` : ''}`);
         this.name = 'TurnDaemonLeaseLostError';
     }
 }
@@ -50,6 +50,7 @@ export class DatabaseTurnDaemonLease {
     private expiryTimer: NodeJS.Timeout | null = null;
     private renewalInFlight = false;
     private lost = false;
+    private lossReason: string | undefined;
 
     private constructor(
         db: GamePrismaClient,
@@ -116,6 +117,7 @@ export class DatabaseTurnDaemonLease {
             fencingEpoch: BigInt(row.fencing_epoch),
         };
         this.lost = false;
+        this.lossReason = undefined;
         this.scheduleExpiryWatchdog(requestStartedAt);
         if (this.heartbeatEnabled) {
             this.startHeartbeat();
@@ -139,6 +141,10 @@ export class DatabaseTurnDaemonLease {
         return this.lost;
     }
 
+    getLossError(): TurnDaemonLeaseLostError {
+        return new TurnDaemonLeaseLostError(this.profile, this.lossReason);
+    }
+
     async renew(): Promise<boolean> {
         const token = this.token;
         if (!token || this.lost || this.renewalInFlight) {
@@ -160,7 +166,7 @@ export class DatabaseTurnDaemonLease {
                 RETURNING "profile", "owner_id", "fencing_epoch"
             `);
             if (rows.length === 0) {
-                this.markLost();
+                this.markLost('Heartbeat renewal rejected: lease expired or owner/epoch changed.');
                 return false;
             }
             if (this.lost) {
@@ -176,7 +182,7 @@ export class DatabaseTurnDaemonLease {
     async assertActive(transaction?: GamePrisma.TransactionClient): Promise<void> {
         const token = this.token;
         if (!token || this.lost) {
-            throw new TurnDaemonLeaseLostError(this.profile);
+            throw this.getLossError();
         }
         const db = transaction ?? this.db;
         const rows = await db.$queryRaw<LeaseRow[]>(GamePrisma.sql`
@@ -190,8 +196,8 @@ export class DatabaseTurnDaemonLease {
             FOR UPDATE
         `);
         if (rows.length === 0) {
-            this.markLost();
-            throw new TurnDaemonLeaseLostError(this.profile);
+            this.markLost('Transaction fencing rejected: lease expired or owner/epoch changed.');
+            throw this.getLossError();
         }
     }
 
@@ -228,8 +234,10 @@ export class DatabaseTurnDaemonLease {
         }
         const intervalMs = Math.max(250, Math.floor(this.leaseDurationMs / 3));
         this.heartbeatTimer = setInterval(() => {
-            void this.renew().catch(() => {
-                this.markLost();
+            void this.renew().catch((error: unknown) => {
+                this.markLost(
+                    `Heartbeat database request failed (${error instanceof Error ? error.name : 'unknown error'}).`
+                );
             });
         }, intervalMs);
         this.heartbeatTimer.unref();
@@ -248,7 +256,9 @@ export class DatabaseTurnDaemonLease {
         this.stopExpiryWatchdog();
         const remainingMs = Math.max(0, this.leaseDurationMs - (performance.now() - requestStartedAt));
         this.expiryTimer = setTimeout(() => {
-            this.markLost();
+            this.markLost(
+                `Heartbeat deadline exceeded (${this.leaseDurationMs}ms; renewal in flight: ${this.renewalInFlight}).`
+            );
         }, remainingMs);
         this.expiryTimer.unref();
     }
@@ -260,7 +270,9 @@ export class DatabaseTurnDaemonLease {
         }
     }
 
-    private markLost(): void {
+    private markLost(reason: string): void {
+        if (this.lost) return;
+        this.lossReason = reason;
         this.lost = true;
         this.stopHeartbeat();
         this.stopExpiryWatchdog();

app/game-engine/src/turn/cli.ts

@@ -5,6 +5,8 @@ import type { TurnRunBudget } from '../lifecycle/types.js';
 import { resolveDatabaseUrl } from '../scenario/databaseUrl.js';
 import { createTurnDaemonRuntime } from './turnDaemon.js';
 import { createTurnDaemonMemoryReporter } from './turnDaemonMemoryReporter.js';
+import { createGatewayProfileGate } from './gatewayProfileGate.js';
+import { TurnDaemonLeaseUnavailableError } from '../lifecycle/databaseTurnDaemonLease.js';
 
 export interface TurnDaemonCliOptions {
     profile?: string;
@@ -89,6 +91,27 @@ export const runTurnDaemonCli = async (options: TurnDaemonCliOptions = {}): Prom
         pauseGateIntervalMs,
         adminActionIntervalMs,
         gameClockMode,
+    }).catch(async (error: unknown) => {
+        // 중복 starter가 정상 owner를 멈추면 안 된다. 그 밖의 초기화 실패는
+        // lifecycle hook이 아직 없으므로 여기서 별도로 관리자에게 기록한다.
+        if (!(error instanceof TurnDaemonLeaseUnavailableError)) {
+            try {
+                const gate = await createGatewayProfileGate({
+                    databaseUrl,
+                    gatewayDatabaseUrl,
+                    profileName,
+                    incidentContext: () => ({ stage: 'startup' }),
+                });
+                try {
+                    await gate.markPaused(error);
+                } finally {
+                    await gate.close();
+                }
+            } catch {
+                /* 원래 시작 실패를 보존한다. */
+            }
+        }
+        throw error;
     });
 
     const memoryReporter = createTurnDaemonMemoryReporter({

app/game-engine/src/turn/gatewayProfileGate.ts

@@ -1,6 +1,7 @@
 import { performance } from 'node:perf_hooks';
+import { randomUUID } from 'node:crypto';
 
-import { gatewayProfileCapabilities, type GatewayProfileStatus } from '@sammo-ts/common';
+import { describeRuntimeError, gatewayProfileCapabilities, type GatewayProfileStatus } from '@sammo-ts/common';
 import { createGatewayPostgresConnector } from '@sammo-ts/infra';
 
 export interface GatewayProfileGateOptions {
@@ -8,6 +9,7 @@ export interface GatewayProfileGateOptions {
     gatewayDatabaseUrl?: string;
     profileName: string;
     cacheMs?: number;
+    incidentContext?: () => Record<string, string | number | boolean | null>;
 }
 
 export interface GatewayProfileGate {
@@ -22,6 +24,7 @@ const PROFILE_STATUSES_MARKABLE_AS_PAUSED = ['PREOPEN', 'RUNNING', 'PAUSED'] as
 export const createGatewayProfileGate = async (options: GatewayProfileGateOptions): Promise<GatewayProfileGate> => {
     const connector = createGatewayPostgresConnector({
         url: options.gatewayDatabaseUrl ?? options.databaseUrl,
+        connectionTimeoutMillis: 3000,
     });
     await connector.connect();
     const prisma = connector.prisma;
@@ -54,19 +57,44 @@ export const createGatewayProfileGate = async (options: GatewayProfileGateOption
             return cachedPause;
         },
         async markPaused(error?: unknown): Promise<void> {
-            const message = error instanceof Error ? error.message : error ? String(error) : null;
+            const failure = error ? describeRuntimeError(error) : null;
+            const message = failure?.message ?? null;
             try {
-                await prisma.gatewayProfile.updateMany({
-                    where: {
-                        profileName: options.profileName,
-                        status: { in: [...PROFILE_STATUSES_MARKABLE_AS_PAUSED] },
-                    },
-                    data: {
-                        status: 'PAUSED',
-                        lastError: message,
-                    },
+                await prisma.$transaction(async (tx) => {
+                    const updated = await tx.gatewayProfile.updateMany({
+                        where: {
+                            profileName: options.profileName,
+                            status: { in: [...PROFILE_STATUSES_MARKABLE_AS_PAUSED] },
+                            OR: [{ status: { not: 'PAUSED' } }, { lastError: { not: message } }, { lastError: null }],
+                        },
+                        data: {
+                            status: 'PAUSED',
+                            lastError: message,
+                        },
+                    });
+                    if (updated.count && failure) {
+                        // 상태와 이력을 함께 commit한다. 재개가 lastError를 지워도
+                        // 당시 원인과 실행 좌표는 관리자 감사 저장소에 남는다.
+                        await tx.adminAuditEvent.create({
+                            data: {
+                                correlationId: randomUUID(),
+                                actorUserId: 'system:turn-daemon',
+                                actorUsername: 'turn-daemon',
+                                credentialKind: 'DAEMON',
+                                action: 'runtime.failure',
+                                targetType: 'profile-runtime',
+                                targetId: options.profileName,
+                                profileName: options.profileName,
+                                outcome: 'FAILED',
+                                errorCode: failure.code,
+                                errorMessage: failure.message,
+                                summary: { frames: failure.frames, ...options.incidentContext?.() },
+                            },
+                        });
+                    }
                 });
             } catch {
+                if (failure) console.error('[turn-daemon] failed to persist runtime incident', failure);
                 return;
             }
         },

app/game-engine/src/turn/turnDaemon.ts

@@ -93,11 +93,7 @@ import {
     createResetOfficerLockHandler,
 } from './monthlyCoreEventAction.js';
 import { buildCommandEnv } from './reservedTurnCommands.js';
-import {
-    DatabaseTurnDaemonLease,
-    TurnDaemonLeaseLostError,
-    TurnDaemonLeaseUnavailableError,
-} from '../lifecycle/databaseTurnDaemonLease.js';
+import { DatabaseTurnDaemonLease, TurnDaemonLeaseUnavailableError } from '../lifecycle/databaseTurnDaemonLease.js';
 import { EngineStateManager } from './engineStateManager.js';
 import { applyRuntimeClockShift } from './runtimeClockShift.js';
 import { applyRuntimeGameSettings } from './runtimeGameSettings.js';
@@ -892,6 +888,19 @@ const createTurnDaemonRuntimeWithLease = async (
               gatewayDatabaseUrl: options.gatewayDatabaseUrl,
               profileName: options.profileName,
               cacheMs: options.pauseGateIntervalMs,
+              incidentContext: () => {
+                  const state = world.getState();
+                  const clock = world.getGameClockState();
+                  const token = turnDaemonLease?.getToken();
+                  return {
+                      year: state.currentYear,
+                      month: state.currentMonth,
+                      clockPhase: clock.phase,
+                      clockTick: clock.tick,
+                      ownerId: token?.ownerId ?? null,
+                      fencingEpoch: token?.fencingEpoch.toString() ?? null,
+                  };
+              },
           })
         : null;
     if (gatewayGate) {
@@ -1066,7 +1075,7 @@ const createTurnDaemonRuntimeWithLease = async (
                 if (turnDaemonLease?.isLost()) {
                     // 만료된 owner는 재개 명령도 처리할 수 없다. 현재 runtime을
                     // 끝내 PM2가 새 owner와 DB snapshot으로 시작하도록 한다.
-                    throw new TurnDaemonLeaseLostError(options.profileName ?? options.profile);
+                    throw turnDaemonLease.getLossError();
                 }
                 const gatewayPaused = (await pauseGate?.()) ?? false;
                 const phase = world.getGameClockState().phase;

app/game-engine/test/gatewayRuntimeAction.integration.test.ts

@@ -112,6 +112,7 @@ integration('gateway runtime action consumer', () => {
     });
 
     it('does not overwrite a terminal operator status while reporting a daemon error', async () => {
+        const existingIncidents = await db.adminAuditEvent.count({ where: { profileName, action: 'runtime.failure' } });
         const gate = await createGatewayProfileGate({
             databaseUrl: databaseUrl!,
             gatewayDatabaseUrl: databaseUrl!,
@@ -127,6 +128,19 @@ integration('gateway runtime action consumer', () => {
                 status: 'PAUSED',
                 lastError: 'running failure',
             });
+            await gate.markPaused(new Error('running failure'));
+            const incidents = await db.adminAuditEvent.findMany({ where: { profileName, action: 'runtime.failure' } });
+            expect(incidents).toHaveLength(existingIncidents + 1);
+            expect(incidents[0]).toMatchObject({
+                credentialKind: 'DAEMON',
+                errorCode: 'Error',
+                errorMessage: 'running failure',
+            });
+
+            await db.gatewayProfile.update({ where: { profileName }, data: { status: 'RUNNING', lastError: null } });
+            expect(await db.adminAuditEvent.count({ where: { profileName, action: 'runtime.failure' } })).toBe(
+                existingIncidents + 1
+            );
 
             await db.gatewayProfile.update({
                 where: { profileName },
@@ -137,6 +151,9 @@ integration('gateway runtime action consumer', () => {
                 status: 'STOPPED',
                 lastError: null,
             });
+            expect(await db.adminAuditEvent.count({ where: { profileName, action: 'runtime.failure' } })).toBe(
+                existingIncidents + 1
+            );
         } finally {
             await gate.close();
         }