diff --git a/app/game-engine/src/lifecycle/databaseTurnDaemonLease.ts b/app/game-engine/src/lifecycle/databaseTurnDaemonLease.ts index 7dac9839..069936c1 100644 --- a/app/game-engine/src/lifecycle/databaseTurnDaemonLease.ts +++ b/app/game-engine/src/lifecycle/databaseTurnDaemonLease.ts @@ -24,8 +24,8 @@ export class TurnDaemonLeaseUnavailableError extends Error { } export class TurnDaemonLeaseLostError extends Error { - constructor(profile: string) { - super(`Turn daemon lease was lost for profile "${profile}".`); + constructor(profile: string, reason?: string) { + super(`Turn daemon lease was lost for profile "${profile}".${reason ? ` ${reason}` : ''}`); this.name = 'TurnDaemonLeaseLostError'; } } @@ -50,6 +50,7 @@ export class DatabaseTurnDaemonLease { private expiryTimer: NodeJS.Timeout | null = null; private renewalInFlight = false; private lost = false; + private lossReason: string | undefined; private constructor( db: GamePrismaClient, @@ -116,6 +117,7 @@ export class DatabaseTurnDaemonLease { fencingEpoch: BigInt(row.fencing_epoch), }; this.lost = false; + this.lossReason = undefined; this.scheduleExpiryWatchdog(requestStartedAt); if (this.heartbeatEnabled) { this.startHeartbeat(); @@ -139,6 +141,10 @@ export class DatabaseTurnDaemonLease { return this.lost; } + getLossError(): TurnDaemonLeaseLostError { + return new TurnDaemonLeaseLostError(this.profile, this.lossReason); + } + async renew(): Promise { const token = this.token; if (!token || this.lost || this.renewalInFlight) { @@ -160,7 +166,7 @@ export class DatabaseTurnDaemonLease { RETURNING "profile", "owner_id", "fencing_epoch" `); if (rows.length === 0) { - this.markLost(); + this.markLost('Heartbeat renewal rejected: lease expired or owner/epoch changed.'); return false; } if (this.lost) { @@ -176,7 +182,7 @@ export class DatabaseTurnDaemonLease { async assertActive(transaction?: GamePrisma.TransactionClient): Promise { const token = this.token; if (!token || this.lost) { - throw new TurnDaemonLeaseLostError(this.profile); + throw this.getLossError(); } const db = transaction ?? this.db; const rows = await db.$queryRaw(GamePrisma.sql` @@ -190,8 +196,8 @@ export class DatabaseTurnDaemonLease { FOR UPDATE `); if (rows.length === 0) { - this.markLost(); - throw new TurnDaemonLeaseLostError(this.profile); + this.markLost('Transaction fencing rejected: lease expired or owner/epoch changed.'); + throw this.getLossError(); } } @@ -228,8 +234,10 @@ export class DatabaseTurnDaemonLease { } const intervalMs = Math.max(250, Math.floor(this.leaseDurationMs / 3)); this.heartbeatTimer = setInterval(() => { - void this.renew().catch(() => { - this.markLost(); + void this.renew().catch((error: unknown) => { + this.markLost( + `Heartbeat database request failed (${error instanceof Error ? error.name : 'unknown error'}).` + ); }); }, intervalMs); this.heartbeatTimer.unref(); @@ -248,7 +256,9 @@ export class DatabaseTurnDaemonLease { this.stopExpiryWatchdog(); const remainingMs = Math.max(0, this.leaseDurationMs - (performance.now() - requestStartedAt)); this.expiryTimer = setTimeout(() => { - this.markLost(); + this.markLost( + `Heartbeat deadline exceeded (${this.leaseDurationMs}ms; renewal in flight: ${this.renewalInFlight}).` + ); }, remainingMs); this.expiryTimer.unref(); } @@ -260,7 +270,9 @@ export class DatabaseTurnDaemonLease { } } - private markLost(): void { + private markLost(reason: string): void { + if (this.lost) return; + this.lossReason = reason; this.lost = true; this.stopHeartbeat(); this.stopExpiryWatchdog(); diff --git a/app/game-engine/src/turn/cli.ts b/app/game-engine/src/turn/cli.ts index a00b68ca..c5f04c56 100644 --- a/app/game-engine/src/turn/cli.ts +++ b/app/game-engine/src/turn/cli.ts @@ -5,6 +5,8 @@ import type { TurnRunBudget } from '../lifecycle/types.js'; import { resolveDatabaseUrl } from '../scenario/databaseUrl.js'; import { createTurnDaemonRuntime } from './turnDaemon.js'; import { createTurnDaemonMemoryReporter } from './turnDaemonMemoryReporter.js'; +import { createGatewayProfileGate } from './gatewayProfileGate.js'; +import { TurnDaemonLeaseUnavailableError } from '../lifecycle/databaseTurnDaemonLease.js'; export interface TurnDaemonCliOptions { profile?: string; @@ -89,6 +91,27 @@ export const runTurnDaemonCli = async (options: TurnDaemonCliOptions = {}): Prom pauseGateIntervalMs, adminActionIntervalMs, gameClockMode, + }).catch(async (error: unknown) => { + // 중복 starter가 정상 owner를 멈추면 안 된다. 그 밖의 초기화 실패는 + // lifecycle hook이 아직 없으므로 여기서 별도로 관리자에게 기록한다. + if (!(error instanceof TurnDaemonLeaseUnavailableError)) { + try { + const gate = await createGatewayProfileGate({ + databaseUrl, + gatewayDatabaseUrl, + profileName, + incidentContext: () => ({ stage: 'startup' }), + }); + try { + await gate.markPaused(error); + } finally { + await gate.close(); + } + } catch { + /* 원래 시작 실패를 보존한다. */ + } + } + throw error; }); const memoryReporter = createTurnDaemonMemoryReporter({ diff --git a/app/game-engine/src/turn/gatewayProfileGate.ts b/app/game-engine/src/turn/gatewayProfileGate.ts index 0b625325..ee756038 100644 --- a/app/game-engine/src/turn/gatewayProfileGate.ts +++ b/app/game-engine/src/turn/gatewayProfileGate.ts @@ -1,6 +1,7 @@ import { performance } from 'node:perf_hooks'; +import { randomUUID } from 'node:crypto'; -import { gatewayProfileCapabilities, type GatewayProfileStatus } from '@sammo-ts/common'; +import { describeRuntimeError, gatewayProfileCapabilities, type GatewayProfileStatus } from '@sammo-ts/common'; import { createGatewayPostgresConnector } from '@sammo-ts/infra'; export interface GatewayProfileGateOptions { @@ -8,6 +9,7 @@ export interface GatewayProfileGateOptions { gatewayDatabaseUrl?: string; profileName: string; cacheMs?: number; + incidentContext?: () => Record; } export interface GatewayProfileGate { @@ -22,6 +24,7 @@ const PROFILE_STATUSES_MARKABLE_AS_PAUSED = ['PREOPEN', 'RUNNING', 'PAUSED'] as export const createGatewayProfileGate = async (options: GatewayProfileGateOptions): Promise => { const connector = createGatewayPostgresConnector({ url: options.gatewayDatabaseUrl ?? options.databaseUrl, + connectionTimeoutMillis: 3000, }); await connector.connect(); const prisma = connector.prisma; @@ -54,19 +57,44 @@ export const createGatewayProfileGate = async (options: GatewayProfileGateOption return cachedPause; }, async markPaused(error?: unknown): Promise { - const message = error instanceof Error ? error.message : error ? String(error) : null; + const failure = error ? describeRuntimeError(error) : null; + const message = failure?.message ?? null; try { - await prisma.gatewayProfile.updateMany({ - where: { - profileName: options.profileName, - status: { in: [...PROFILE_STATUSES_MARKABLE_AS_PAUSED] }, - }, - data: { - status: 'PAUSED', - lastError: message, - }, + await prisma.$transaction(async (tx) => { + const updated = await tx.gatewayProfile.updateMany({ + where: { + profileName: options.profileName, + status: { in: [...PROFILE_STATUSES_MARKABLE_AS_PAUSED] }, + OR: [{ status: { not: 'PAUSED' } }, { lastError: { not: message } }, { lastError: null }], + }, + data: { + status: 'PAUSED', + lastError: message, + }, + }); + if (updated.count && failure) { + // 상태와 이력을 함께 commit한다. 재개가 lastError를 지워도 + // 당시 원인과 실행 좌표는 관리자 감사 저장소에 남는다. + await tx.adminAuditEvent.create({ + data: { + correlationId: randomUUID(), + actorUserId: 'system:turn-daemon', + actorUsername: 'turn-daemon', + credentialKind: 'DAEMON', + action: 'runtime.failure', + targetType: 'profile-runtime', + targetId: options.profileName, + profileName: options.profileName, + outcome: 'FAILED', + errorCode: failure.code, + errorMessage: failure.message, + summary: { frames: failure.frames, ...options.incidentContext?.() }, + }, + }); + } }); } catch { + if (failure) console.error('[turn-daemon] failed to persist runtime incident', failure); return; } }, diff --git a/app/game-engine/src/turn/turnDaemon.ts b/app/game-engine/src/turn/turnDaemon.ts index ba86c296..4a1a22f8 100644 --- a/app/game-engine/src/turn/turnDaemon.ts +++ b/app/game-engine/src/turn/turnDaemon.ts @@ -93,11 +93,7 @@ import { createResetOfficerLockHandler, } from './monthlyCoreEventAction.js'; import { buildCommandEnv } from './reservedTurnCommands.js'; -import { - DatabaseTurnDaemonLease, - TurnDaemonLeaseLostError, - TurnDaemonLeaseUnavailableError, -} from '../lifecycle/databaseTurnDaemonLease.js'; +import { DatabaseTurnDaemonLease, TurnDaemonLeaseUnavailableError } from '../lifecycle/databaseTurnDaemonLease.js'; import { EngineStateManager } from './engineStateManager.js'; import { applyRuntimeClockShift } from './runtimeClockShift.js'; import { applyRuntimeGameSettings } from './runtimeGameSettings.js'; @@ -892,6 +888,19 @@ const createTurnDaemonRuntimeWithLease = async ( gatewayDatabaseUrl: options.gatewayDatabaseUrl, profileName: options.profileName, cacheMs: options.pauseGateIntervalMs, + incidentContext: () => { + const state = world.getState(); + const clock = world.getGameClockState(); + const token = turnDaemonLease?.getToken(); + return { + year: state.currentYear, + month: state.currentMonth, + clockPhase: clock.phase, + clockTick: clock.tick, + ownerId: token?.ownerId ?? null, + fencingEpoch: token?.fencingEpoch.toString() ?? null, + }; + }, }) : null; if (gatewayGate) { @@ -1066,7 +1075,7 @@ const createTurnDaemonRuntimeWithLease = async ( if (turnDaemonLease?.isLost()) { // 만료된 owner는 재개 명령도 처리할 수 없다. 현재 runtime을 // 끝내 PM2가 새 owner와 DB snapshot으로 시작하도록 한다. - throw new TurnDaemonLeaseLostError(options.profileName ?? options.profile); + throw turnDaemonLease.getLossError(); } const gatewayPaused = (await pauseGate?.()) ?? false; const phase = world.getGameClockState().phase; diff --git a/app/game-engine/test/gatewayRuntimeAction.integration.test.ts b/app/game-engine/test/gatewayRuntimeAction.integration.test.ts index 363dc471..97fde950 100644 --- a/app/game-engine/test/gatewayRuntimeAction.integration.test.ts +++ b/app/game-engine/test/gatewayRuntimeAction.integration.test.ts @@ -112,6 +112,7 @@ integration('gateway runtime action consumer', () => { }); it('does not overwrite a terminal operator status while reporting a daemon error', async () => { + const existingIncidents = await db.adminAuditEvent.count({ where: { profileName, action: 'runtime.failure' } }); const gate = await createGatewayProfileGate({ databaseUrl: databaseUrl!, gatewayDatabaseUrl: databaseUrl!, @@ -127,6 +128,19 @@ integration('gateway runtime action consumer', () => { status: 'PAUSED', lastError: 'running failure', }); + await gate.markPaused(new Error('running failure')); + const incidents = await db.adminAuditEvent.findMany({ where: { profileName, action: 'runtime.failure' } }); + expect(incidents).toHaveLength(existingIncidents + 1); + expect(incidents[0]).toMatchObject({ + credentialKind: 'DAEMON', + errorCode: 'Error', + errorMessage: 'running failure', + }); + + await db.gatewayProfile.update({ where: { profileName }, data: { status: 'RUNNING', lastError: null } }); + expect(await db.adminAuditEvent.count({ where: { profileName, action: 'runtime.failure' } })).toBe( + existingIncidents + 1 + ); await db.gatewayProfile.update({ where: { profileName }, @@ -137,6 +151,9 @@ integration('gateway runtime action consumer', () => { status: 'STOPPED', lastError: null, }); + expect(await db.adminAuditEvent.count({ where: { profileName, action: 'runtime.failure' } })).toBe( + existingIncidents + 1 + ); } finally { await gate.close(); } diff --git a/app/gateway-api/src/adminRouter.ts b/app/gateway-api/src/adminRouter.ts index 0b3a9980..d57875bb 100644 --- a/app/gateway-api/src/adminRouter.ts +++ b/app/gateway-api/src/adminRouter.ts @@ -2078,6 +2078,26 @@ export const adminRouter = router({ }), }), profiles: router({ + diagnostics: adminProcedure + .input(z.object({ profileName: z.string().min(1).max(100) })) + .query(async ({ ctx, input }) => { + const auth = requireAdminAuth(ctx); + if (!canReadProfile(auth, input.profileName)) throw new TRPCError({ code: 'FORBIDDEN' }); + const profile = await ctx.profiles.getProfile(input.profileName); + if (!profile) throw new TRPCError({ code: 'NOT_FOUND' }); + const [observation, incidents, processes] = await Promise.all([ + ctx.orchestrator.inspectRuntime?.(input.profileName) ?? Promise.resolve(null), + ctx.adminAudit.list({ profileName: input.profileName, targetType: 'profile-runtime', limit: 20 }), + ctx.orchestrator.listRuntimeStates([input.profileName]).catch(() => []), + ]); + return { + profileName: input.profileName, + status: profile.status, + observation, + runtime: processes[0] ?? null, + incidents, + }; + }), getResetDefaults: adminProcedure .input(z.object({ profileName: z.string().min(1) })) .query(async ({ ctx, input }) => { diff --git a/app/gateway-api/src/orchestrator/gatewayOrchestrator.ts b/app/gateway-api/src/orchestrator/gatewayOrchestrator.ts index 0b2abb57..09eb0af2 100644 --- a/app/gateway-api/src/orchestrator/gatewayOrchestrator.ts +++ b/app/gateway-api/src/orchestrator/gatewayOrchestrator.ts @@ -20,7 +20,7 @@ import { type GameCancellationHistoryMode, type GameCancellationResult, } from '@sammo-ts/game-engine/scenario/gameCancellation.js'; -import { gatewayProfileCapabilities } from '@sammo-ts/common'; +import { gatewayProfileCapabilities, type ProfileRuntimeDiagnostics } from '@sammo-ts/common'; import { createGamePostgresConnector, createRedisConnector, @@ -177,6 +177,7 @@ export interface GatewayOrchestratorHandle { }>; listRuntimeStates(profileNames: string[]): Promise; listRuntimeSettings?(profileNames: string[]): Promise; + inspectRuntime?(profileName: string): Promise; transitionProfileClock( profileName: string, action: 'SUSPEND' | 'RESUME', @@ -1164,6 +1165,94 @@ export class GatewayOrchestrator implements GatewayOrchestratorHandle { return snapshots; } + async inspectRuntime(profileName: string): Promise { + const processes = await this.processManager.list().catch(() => null); + const empty: ProfileRuntimeDiagnostics = { + profileName, + checkedAt: new Date().toISOString(), + database: 'UNINITIALIZED', + processObservation: processes ? 'AVAILABLE' : 'UNAVAILABLE', + processes: (processes ?? []) + .filter((process) => process.name.startsWith(`sammo:${profileName}:`)) + .map((process) => ({ + name: process.name, + status: process.status, + restartCount: process.restartCount ?? 0, + exitCode: process.exitCode ?? null, + })), + lease: null, + clock: null, + }; + const profile = await this.repository.getProfile(profileName); + if (!profile || profile.currentScenario === null) return empty; + const connector = createGamePostgresConnector({ + url: this.resolveProfileDatabaseUrl(profile), + maxConnections: 1, + connectionTimeoutMillis: 3000, + }); + try { + await connector.connect(); + return await connector.prisma.$transaction( + async (db) => { + await db.$executeRaw`SET LOCAL statement_timeout = '3000ms'`; + const [time] = await db.$queryRaw< + Array<{ now: Date }> + >`SELECT clock_timestamp() AT TIME ZONE 'UTC' AS now`; + const lease = await db.turnDaemonLease.findUnique({ where: { profile: profileName } }); + const clock = await db.worldState.findFirst({ + orderBy: { id: 'asc' }, + select: { + clockPhase: true, + clockRevision: true, + clockTick: true, + lastTurnTick: true, + currentYear: true, + currentMonth: true, + clockWallAnchor: true, + clockRecoveryStartWallAt: true, + clockRecoveryEndTick: true, + }, + }); + const now = time!.now; + return { + ...empty, + checkedAt: now.toISOString(), + database: 'AVAILABLE' as const, + lease: lease + ? { + ownerId: lease.ownerId, + fencingEpoch: lease.fencingEpoch.toString(), + heartbeatAt: lease.heartbeatAt.toISOString(), + leaseUntil: lease.leaseUntil.toISOString(), + heartbeatAgeMs: Math.max(0, now.getTime() - lease.heartbeatAt.getTime()), + valid: lease.leaseUntil > now, + clockReady: lease.clockReady, + } + : null, + clock: clock + ? { + phase: clock.clockPhase, + revision: clock.clockRevision.toString(), + tick: clock.clockTick?.toString() ?? null, + lastTurnTick: clock.lastTurnTick?.toString() ?? null, + year: clock.currentYear, + month: clock.currentMonth, + wallAnchor: clock.clockWallAnchor?.toISOString() ?? null, + recoveryStartWallAt: clock.clockRecoveryStartWallAt?.toISOString() ?? null, + recoveryEndTick: clock.clockRecoveryEndTick?.toString() ?? null, + } + : null, + }; + }, + { timeout: 5000, maxWait: 3000 } + ); + } catch { + return { ...empty, checkedAt: new Date().toISOString(), database: 'UNAVAILABLE' }; + } finally { + await connector.disconnect().catch(() => undefined); + } + } + async listRuntimeSettings(profileNames: string[]): Promise { const allowedAutorunOptions = new Set([ 'develop', diff --git a/app/gateway-api/src/orchestrator/pm2ProcessManager.ts b/app/gateway-api/src/orchestrator/pm2ProcessManager.ts index 0f45ac2e..3e8d8391 100644 --- a/app/gateway-api/src/orchestrator/pm2ProcessManager.ts +++ b/app/gateway-api/src/orchestrator/pm2ProcessManager.ts @@ -123,6 +123,12 @@ export class Pm2ProcessManager implements ProcessManager { cwd: item.pm2_env?.pm_cwd ?? undefined, script: item.pm2_env?.pm_exec_path ?? undefined, restartCount: item.pm2_env?.restart_time ?? 0, + exitCode: + item.pm2_env && + 'exit_code' in item.pm2_env && + typeof item.pm2_env.exit_code === 'number' + ? item.pm2_env.exit_code + : undefined, })) ?? []; resolve(normalized); }); @@ -145,16 +151,13 @@ export class Pm2ProcessManager implements ProcessManager { reject(new Error(`PM2 process name already exists: ${definition.name}`)); return; } - pm2.start( - buildPm2StartOptions(definition), - (error) => { - if (error) { - reject(error); - return; - } - resolve(); + pm2.start(buildPm2StartOptions(definition), (error) => { + if (error) { + reject(error); + return; } - ); + resolve(); + }); }); }) ); diff --git a/app/gateway-api/src/orchestrator/processManager.ts b/app/gateway-api/src/orchestrator/processManager.ts index 942f82cc..63bf1141 100644 --- a/app/gateway-api/src/orchestrator/processManager.ts +++ b/app/gateway-api/src/orchestrator/processManager.ts @@ -5,6 +5,7 @@ export interface ManagedProcessInfo { cwd?: string; script?: string; restartCount?: number; + exitCode?: number; } export interface ProcessDefinition { diff --git a/app/gateway-api/test/adminOperations.test.ts b/app/gateway-api/test/adminOperations.test.ts index 5474b90a..3dff259b 100644 --- a/app/gateway-api/test/adminOperations.test.ts +++ b/app/gateway-api/test/adminOperations.test.ts @@ -435,6 +435,35 @@ describe('admin profile navigation API', () => { }); }); +describe('runtime diagnostics authorization', () => { + it('allows the scoped administrator and rejects another profile scope', async () => { + const harness = await buildCaller( + async () => { + throw new Error('not used'); + }, + { adminRoles: ['admin.profiles.runtime:che:2'], firstUserIsAdmin: false } + ); + await expect(harness.caller.admin.profiles.diagnostics({ profileName: 'che:2' })).resolves.toMatchObject({ + profileName: 'che:2', + incidents: [], + }); + await expect(harness.caller.admin.profiles.diagnostics({ profileName: 'kwe:2' })).rejects.toMatchObject({ + code: 'FORBIDDEN', + }); + }); + it('rejects users without profile administration permission', async () => { + const harness = await buildCaller( + async () => { + throw new Error('not used'); + }, + { adminRoles: [], firstUserIsAdmin: false } + ); + await expect(harness.caller.admin.profiles.diagnostics({ profileName: 'che:2' })).rejects.toMatchObject({ + code: 'FORBIDDEN', + }); + }); +}); + describe('admin scenario catalog API', () => { it('marks scenario zero as the current selectable scenario', async () => { const harness = await buildCaller( diff --git a/app/gateway-api/test/adminSecurityTransport.e2e.test.ts b/app/gateway-api/test/adminSecurityTransport.e2e.test.ts index 3df14b75..a675ec77 100644 --- a/app/gateway-api/test/adminSecurityTransport.e2e.test.ts +++ b/app/gateway-api/test/adminSecurityTransport.e2e.test.ts @@ -174,6 +174,36 @@ const postTrpc = async ( }; describe('admin security over HTTP transport', () => { + it('protects runtime diagnostics at the HTTP authentication and profile scope boundaries', async () => { + const harness = await createHarness(['admin.profiles.runtime:che:default']); + const input = { profileName: 'che:default' }; + expect((await postTrpc(harness.baseUrl, 'admin.profiles.diagnostics', input)).response.status).toBe(401); + expect( + (await postTrpc(harness.baseUrl, 'admin.profiles.diagnostics', input, harness.adminSessionToken)).response + .status + ).toBe(200); + expect( + ( + await postTrpc( + harness.baseUrl, + 'admin.profiles.diagnostics', + { profileName: 'kwe:default' }, + harness.adminSessionToken + ) + ).response.status + ).toBe(403); + expect( + ( + await postTrpc( + harness.baseUrl, + 'admin.profiles.diagnostics', + { profileName: '' }, + harness.adminSessionToken + ) + ).response.status + ).toBe(400); + }); + it('accepts query input from a POST JSON body but still rejects a mutation sent as GET', async () => { const harness = await createHarness(); diff --git a/app/gateway-frontend/e2e/admin-runtime-actions.spec.ts b/app/gateway-frontend/e2e/admin-runtime-actions.spec.ts index f737baf1..5f8a9f19 100644 --- a/app/gateway-frontend/e2e/admin-runtime-actions.spec.ts +++ b/app/gateway-frontend/e2e/admin-runtime-actions.spec.ts @@ -42,9 +42,11 @@ const installFixture = async ( currentScenario?: string | null; gameIsUnited?: number; openerOnly?: boolean; + diagnosticsFixture?: boolean; } = {} ) => { let requested = false; + let diagnosticReads = 0; let installRequested = false; let installActive = false; let postRequestProfileReads = 0; @@ -84,6 +86,57 @@ const installFixture = async ( installActive = true; } const results = operations.map((operation) => { + if (operation === 'admin.profiles.diagnostics' && options.diagnosticsFixture) { + diagnosticReads += 1; + return response({ + profileName: 'che:default', + status: diagnosticReads > 1 ? 'RUNNING' : 'PAUSED', + runtime: { daemonRunning: true }, + observation: { + profileName: 'che:default', + checkedAt: '2026-09-09T18:00:00.000Z', + database: 'AVAILABLE', + processObservation: 'AVAILABLE', + processes: [ + { name: 'sammo:hwe:default:turn-daemon', status: 'online', restartCount: 1, exitCode: 1 }, + ], + lease: { + ownerId: 'owner-0123456789-0123456789-0123456789', + fencingEpoch: '72', + heartbeatAt: '2026-09-09T17:30:00.000Z', + leaseUntil: '2026-09-09T17:30:30.000Z', + heartbeatAgeMs: 1800000, + valid: diagnosticReads > 1, + clockReady: true, + }, + clock: { + phase: 'RUNNING', + revision: '6', + tick: '11286094560', + lastTurnTick: '11268000000', + year: 206, + month: 2, + wallAnchor: '2026-09-09T17:00:00.000Z', + recoveryStartWallAt: null, + recoveryEndTick: null, + }, + }, + incidents: [ + { + id: 'incident-1', + createdAt: '2026-09-09T17:30:31.000Z', + errorCode: 'TurnDaemonLeaseLostError', + errorMessage: 'Heartbeat deadline exceeded (30000ms; renewal in flight: true).', + summary: { + year: 206, + month: 2, + clockPhase: 'RUNNING', + frames: ['at flush (/srv/app/flush.ts:42:7)'], + }, + }, + ], + }); + } if (operation === 'me') { return response({ id: 'admin-user', @@ -390,7 +443,9 @@ test('updates live game options from the authoritative database snapshot', async await page.screenshot({ path: testInfo.outputPath('runtime-settings-mobile.png'), fullPage: true }); }); -test('distinguishes a turn pause from an inaccessible stopped server in operator controls', async ({ page }, testInfo) => { +test('distinguishes a turn pause from an inaccessible stopped server in operator controls', async ({ + page, +}, testInfo) => { await installFixture(page, { profileStatus: 'PAUSED', pauseReason: "Cannot assign to read only property 'charges' of object '#'", @@ -583,3 +638,48 @@ test('directs profile deployment to the selected server version tab', async ({ p expect(linkGeometry.right).toBeLessThanOrEqual(linkGeometry.viewportWidth); await page.screenshot({ path: testInfo.outputPath('status-tabs-mobile.png'), fullPage: true }); }); + +test('runtime diagnostics shows expired lease and retains history after recovery on desktop and mobile', async ({ + page, +}, testInfo) => { + await installFixture(page, { profileStatus: 'PAUSED', diagnosticsFixture: true }); + await page.goto('/gateway/admin/servers/hwe%3Adefault'); + await page.getByRole('button', { name: '장애 진단과 이력' }).click(); + const panel = page.getByTestId('runtime-diagnostics'); + await expect(panel).toContainText('턴 실행 권한 만료'); + await panel.locator('summary').filter({ hasText: 'TurnDaemonLeaseLostError' }).click(); + await expect(panel).toContainText('Heartbeat deadline exceeded'); + await expect(panel).toContainText('flush.ts:42:7'); + await panel.getByText('프로세스 상태와 종료 코드', { exact: true }).click(); + await expect(panel).toContainText('마지막 종료 코드 1'); + for (const width of [1200, 390]) { + await page.setViewportSize({ width, height: 900 }); + await page.evaluate(() => document.fonts.ready); + const measured = await panel.evaluate((element) => { + const rect = element.getBoundingClientRect(); + const style = getComputedStyle(element); + return { + left: rect.left, + right: rect.right, + width: rect.width, + viewport: innerWidth, + scrollWidth: element.scrollWidth, + clientWidth: element.clientWidth, + font: style.font, + border: style.border, + html: element.outerHTML, + }; + }); + expect(measured.left).toBeGreaterThanOrEqual(0); + expect(measured.right).toBeLessThanOrEqual(width); + expect(measured.scrollWidth).toBeLessThanOrEqual(measured.clientWidth); + await testInfo.attach(`diagnostics-${width}.json`, { + body: JSON.stringify(measured), + contentType: 'application/json', + }); + await page.screenshot({ path: testInfo.outputPath(`diagnostics-${width}.png`), fullPage: true }); + } + await page.getByRole('button', { name: '장애 진단 새로고침' }).click(); + await expect(panel).toContainText('턴 프로세스와 실행 권한 정상'); + await expect(panel).toContainText('Heartbeat deadline exceeded'); +}); diff --git a/app/gateway-frontend/src/components/ProfileRuntimeDiagnostics.vue b/app/gateway-frontend/src/components/ProfileRuntimeDiagnostics.vue new file mode 100644 index 00000000..102c27f4 --- /dev/null +++ b/app/gateway-frontend/src/components/ProfileRuntimeDiagnostics.vue @@ -0,0 +1,115 @@ + + + diff --git a/app/gateway-frontend/src/views/AdminView.vue b/app/gateway-frontend/src/views/AdminView.vue index dca8251c..e0611a43 100644 --- a/app/gateway-frontend/src/views/AdminView.vue +++ b/app/gateway-frontend/src/views/AdminView.vue @@ -7,6 +7,7 @@ import { import { gatewayProfileCapabilities, type GatewayProfileStatus } from '@sammo-ts/common/gateway/profileStatus'; import { computed, onMounted, ref, watch } from 'vue'; import ServerProfileTabs from '../components/ServerProfileTabs.vue'; +import ProfileRuntimeDiagnostics from '../components/ProfileRuntimeDiagnostics.vue'; import AdminConsoleLayout from '../layouts/AdminConsoleLayout.vue'; import { useToast } from '../composables/useToast'; import { @@ -2443,6 +2444,7 @@ onMounted(() => {
빌드 커밋: {{ profile.buildCommitSha ?? '미지정' }}
+
+ text + .replace(/\b(?:https?|postgres(?:ql)?|rediss?):\/\/[^\s"'<>]+/gi, '[REDACTED_URL]') + .replace(/\bBearer\s+[^\s"',;]+/gi, 'Bearer [REDACTED]') + .replace( + /((?:password|passwd|token|secret|authorization|cookie|api[_-]?key)["']?\s*[:=]\s*)(?:"[^"\n]*"|'[^'\n]*'|[^\s,;]+)/gi, + '$1[REDACTED]' + ) + .slice(0, 2000); + +export const describeRuntimeError = (error: unknown): { code: string; message: string; frames: string[] } => ({ + code: error instanceof Error ? error.name.slice(0, 100) : 'RuntimeError', + message: sanitizeRuntimeErrorText(error instanceof Error ? error.message : String(error)), + frames: + error instanceof Error + ? (error.stack ?? '') + .split('\n') + .filter((line) => /^\s*at\s/.test(line)) + .slice(0, 8) + .map(sanitizeRuntimeErrorText) + : [], +}); + +export interface ProfileRuntimeDiagnostics { + profileName: string; + checkedAt: string; + database: 'AVAILABLE' | 'UNAVAILABLE' | 'UNINITIALIZED'; + processObservation: 'AVAILABLE' | 'UNAVAILABLE'; + processes: Array<{ name: string; status: string; restartCount: number; exitCode: number | null }>; + lease: { + ownerId: string; + fencingEpoch: string; + heartbeatAt: string; + leaseUntil: string; + heartbeatAgeMs: number; + valid: boolean; + clockReady: boolean; + } | null; + clock: { + phase: string; + revision: string; + tick: string | null; + lastTurnTick: string | null; + year: number; + month: number; + wallAnchor: string | null; + recoveryStartWallAt: string | null; + recoveryEndTick: string | null; + } | null; +} diff --git a/packages/common/src/index.ts b/packages/common/src/index.ts index f131188e..63c6f087 100644 --- a/packages/common/src/index.ts +++ b/packages/common/src/index.ts @@ -28,6 +28,7 @@ export * from './auth/accountIconProjection.js'; export * from './logging/formatLegacyLogHtml.js'; export * from './legacyArchive/ArchivedGeneralSnapshot.js'; export * from './gateway/profileStatus.js'; +export * from './gateway/runtimeDiagnostics.js'; export * from './game/accessPenalty.js'; export * from './http/trpcTransport.js'; export * from './webPush/types.js'; diff --git a/packages/common/test/runtimeDiagnostics.test.ts b/packages/common/test/runtimeDiagnostics.test.ts new file mode 100644 index 00000000..80dd3870 --- /dev/null +++ b/packages/common/test/runtimeDiagnostics.test.ts @@ -0,0 +1,23 @@ +import { describe, expect, it } from 'vitest'; +import { describeRuntimeError } from '../src/gateway/runtimeDiagnostics.js'; + +describe('runtime failure records', () => { + it('keeps the cause and frames while removing connection and authentication values', () => { + const error = new Error( + 'database failed postgresql://admin:private@host/db password="hidden value" token=abc Bearer xyz' + ); + error.stack = `${error.message}\n at flush (/srv/app/flush.ts:42:7)`; + const record = describeRuntimeError(error); + expect(record.code).toBe('Error'); + expect(record.message).toContain('database failed'); + for (const secret of ['private', 'hidden value', 'abc', 'xyz']) + expect(JSON.stringify(record)).not.toContain(secret); + expect(record.frames).toEqual([' at flush (/srv/app/flush.ts:42:7)']); + }); + it('bounds untrusted messages and stack depth', () => { + const error = new Error('x'.repeat(4000)); + error.stack = Array.from({ length: 30 }, () => ' at run (/app/run.ts:1:1)').join('\n'); + expect(describeRuntimeError(error).message).toHaveLength(2000); + expect(describeRuntimeError(error).frames).toHaveLength(8); + }); +}); diff --git a/packages/infra/src/postgres.ts b/packages/infra/src/postgres.ts index 8126dea9..dadf55e8 100644 --- a/packages/infra/src/postgres.ts +++ b/packages/infra/src/postgres.ts @@ -15,6 +15,7 @@ export interface PostgresConfig { log?: PostgresLogOption[]; maxConnections?: number; sessionTimezone?: 'UTC'; + connectionTimeoutMillis?: number; } export interface PostgresPoolStats { @@ -65,16 +66,18 @@ const buildSharedPoolKey = ( url: string, schema: string | undefined, maxConnections: number, - sessionTimezone: 'UTC' | undefined -): string => JSON.stringify([url, schema ?? '', maxConnections, sessionTimezone ?? '']); + sessionTimezone: 'UTC' | undefined, + connectionTimeoutMillis: number | undefined +): string => JSON.stringify([url, schema ?? '', maxConnections, sessionTimezone ?? '', connectionTimeoutMillis ?? 0]); const acquireSharedPool = ( url: string, schema: string | undefined, maxConnections: number, - sessionTimezone: 'UTC' | undefined + sessionTimezone: 'UTC' | undefined, + connectionTimeoutMillis: number | undefined ): { entry: SharedPoolEntry; release: () => Promise } => { - const key = buildSharedPoolKey(url, schema, maxConnections, sessionTimezone); + const key = buildSharedPoolKey(url, schema, maxConnections, sessionTimezone, connectionTimeoutMillis); let entry = sharedPools.get(key); if (!entry) { const connectionOptions = [ @@ -86,6 +89,7 @@ const acquireSharedPool = ( const pool = new pg.Pool({ connectionString: url, max: maxConnections, + ...(connectionTimeoutMillis !== undefined ? { connectionTimeoutMillis } : {}), ...(connectionOptions ? { options: connectionOptions } : {}), }); entry = { pool, references: 0, maxConnections }; @@ -173,7 +177,13 @@ export const createPostgresConnector = ( const schema = extractSchemaFromDatabaseUrl(config.url) ?? process.env.POSTGRES_SCHEMA ?? process.env.DATABASE_SCHEMA; const maxConnections = resolvePostgresPoolMax(config.maxConnections ?? process.env.POSTGRES_POOL_MAX); - const sharedPool = acquireSharedPool(config.url, schema, maxConnections, config.sessionTimezone); + const sharedPool = acquireSharedPool( + config.url, + schema, + maxConnections, + config.sessionTimezone, + config.connectionTimeoutMillis + ); const adapter = new PrismaPg(sharedPool.entry.pool, schema ? { schema } : undefined); const prisma = createClient({ adapter,