fix: VM 시간 변경 후 턴 lease를 안전하게 자동 복구

app/game-engine/test/gatewayRuntimeAction.integration.test.ts

@@ -3,6 +3,7 @@ import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest';
 import { createGatewayPostgresConnector, type GatewayPrismaClient } from '@sammo-ts/infra';
 
 import { createGatewayAdminActionConsumer } from '../src/turn/gatewayAdminActions.js';
+import { TurnDaemonLeaseLostError } from '../src/lifecycle/databaseTurnDaemonLease.js';
 import { createGatewayProfileGate } from '../src/turn/gatewayProfileGate.js';
 
 const databaseUrl = process.env.GATEWAY_RUNTIME_ACTION_DATABASE_URL;
@@ -111,6 +112,36 @@ integration('gateway runtime action consumer', () => {
         expect(onActionApplied).toHaveBeenCalledTimes(1);
     });
 
+    it.each(['RUNNING', 'PREOPEN', 'PAUSED', 'STOPPED', 'COMPLETED'] as const)(
+        'preserves %s and its operator error when a lease-lost owner reports failure',
+        async (status) => {
+            await db.gatewayProfile.update({ where: { profileName }, data: { status, lastError: 'operator context' } });
+            const before = await db.adminAuditEvent.count({ where: { profileName, action: 'runtime.failure' } });
+            const gate = await createGatewayProfileGate({ databaseUrl: databaseUrl!, profileName, cacheMs: 0 });
+            try {
+                const paused = await gate.shouldPause();
+                const error = new TurnDaemonLeaseLostError(profileName, 'simulated VM resume');
+                await gate.reportFailure(error);
+                await gate.reportFailure(error);
+                expect(await db.gatewayProfile.findUniqueOrThrow({ where: { profileName } })).toMatchObject({
+                    status,
+                    lastError: 'operator context',
+                });
+                expect(await gate.shouldPause()).toBe(paused);
+                expect(await db.adminAuditEvent.count({ where: { profileName, action: 'runtime.failure' } })).toBe(
+                    before + 1
+                );
+                const incident = await db.adminAuditEvent.findFirstOrThrow({
+                    where: { profileName, errorCode: 'TurnDaemonLeaseLostError' },
+                    orderBy: { createdAt: 'desc' },
+                });
+                expect(incident.summary).toMatchObject({ recovery: 'RESTART' });
+            } finally {
+                await gate.close();
+            }
+        }
+    );
+
     it('does not overwrite a terminal operator status while reporting a daemon error', async () => {
         const existingIncidents = await db.adminAuditEvent.count({ where: { profileName, action: 'runtime.failure' } });
         const gate = await createGatewayProfileGate({
@@ -123,15 +154,15 @@ integration('gateway runtime action consumer', () => {
                 where: { profileName },
                 data: { status: 'RUNNING', lastError: null },
             });
-            await gate.markPaused(new Error('running failure'));
+            await gate.reportFailure(new Error('running failure'));
             expect(await db.gatewayProfile.findUniqueOrThrow({ where: { profileName } })).toMatchObject({
                 status: 'PAUSED',
                 lastError: 'running failure',
             });
-            await gate.markPaused(new Error('running failure'));
+            await gate.reportFailure(new Error('running failure'));
             const incidents = await db.adminAuditEvent.findMany({ where: { profileName, action: 'runtime.failure' } });
             expect(incidents).toHaveLength(existingIncidents + 1);
-            expect(incidents[0]).toMatchObject({
+            expect(incidents.find((incident) => incident.errorMessage === 'running failure')).toMatchObject({
                 credentialKind: 'DAEMON',
                 errorCode: 'Error',
                 errorMessage: 'running failure',
@@ -146,7 +177,7 @@ integration('gateway runtime action consumer', () => {
                 where: { profileName },
                 data: { status: 'STOPPED', lastError: null },
             });
-            await gate.markPaused(new Error('late shutdown failure'));
+            await gate.reportFailure(new Error('late shutdown failure'));
             expect(await db.gatewayProfile.findUniqueOrThrow({ where: { profileName } })).toMatchObject({
                 status: 'STOPPED',
                 lastError: null,

app/game-engine/test/leaseStartupRetry.test.ts

@@ -0,0 +1,28 @@
+import { describe, expect, it, vi } from 'vitest';
+import { TurnDaemonLeaseUnavailableError } from '../src/lifecycle/databaseTurnDaemonLease.js';
+import { retryTurnDaemonLeaseStartup } from '../src/turn/leaseStartupRetry.js';
+
+describe('lease startup retry', () => {
+    it('survives more than the PM2 startup failure budget and returns only a fresh runtime', async () => {
+        const freshRuntime = { owner: 'new-owner' };
+        let attempts = 0;
+        const create = vi.fn(async () => {
+            if (++attempts <= 20) throw new TurnDaemonLeaseUnavailableError('test');
+            return freshRuntime;
+        });
+        const wait = vi.fn(async () => {});
+        expect(await retryTurnDaemonLeaseStartup(create, wait)).toBe(freshRuntime);
+        expect(create).toHaveBeenCalledTimes(21);
+        expect(wait).toHaveBeenCalledTimes(20);
+    });
+    it('propagates gameplay or startup faults instead of hiding them in a retry loop', async () => {
+        const error = new Error('invalid world');
+        const create = vi.fn(async () => {
+            throw error;
+        });
+        const wait = vi.fn(async () => {});
+        await expect(retryTurnDaemonLeaseStartup(create, wait)).rejects.toBe(error);
+        expect(create).toHaveBeenCalledTimes(1);
+        expect(wait).not.toHaveBeenCalled();
+    });
+});

app/game-engine/test/turnDaemonLease.integration.test.ts

@@ -308,6 +308,30 @@ integration('database turn daemon lease and fencing', () => {
         expect(await db.inputEvent.findUnique({ where: { requestId } })).toBeNull();
     });
 
+    it('fences expiry during a transaction using current DB time rather than transaction start time', async () => {
+        const profile = `${profilePrefix}transaction-expiry`;
+        const requestId = `${profilePrefix}transaction-expiry-write`;
+        const lease = await createLease(profile, 'stalled-owner');
+        await lease.acquire();
+        await expect(
+            db.$transaction(async (tx) => {
+                // transaction 시작 뒤에 만료되는 짧은 DB lease를 만든다. local watchdog은
+                // 60초이므로 이 검증은 DB의 실제 시간 fence만으로 통과해야 한다.
+                await tx.$executeRaw`
+                UPDATE turn_daemon_lease
+                SET lease_until = (clock_timestamp() AT TIME ZONE 'UTC') + INTERVAL '100 milliseconds'
+                WHERE profile = ${profile}
+            `;
+                await tx.inputEvent.create({
+                    data: { requestId, target: 'ENGINE', eventType: 'fenced-test', payload: {} },
+                });
+                await tx.$executeRaw`SELECT pg_sleep(0.2)`;
+                await lease.assertActive(tx);
+            })
+        ).rejects.toBeInstanceOf(TurnDaemonLeaseLostError);
+        expect(await db.inputEvent.findUnique({ where: { requestId } })).toBeNull();
+    });
+
     it('permits a clean successor after release while fencing a resumed old token', async () => {
         const profile = `${profilePrefix}release`;
         const first = await createLease(profile, 'owner-a');

app/game-engine/test/turnDaemonLifecycle.test.ts

@@ -51,6 +51,39 @@ describe('TurnDaemonLifecycle', () => {
         expect(lifecycle.getStatus()).toMatchObject({ state: 'stopping', paused: true, lastError: error.message });
     });
 
+    it('rechecks the lease within one second while a wall deadline remains far in the future', async () => {
+        const now = new Date('2026-09-15T00:00:00Z');
+        const error = new TurnDaemonLeaseLostError('che:default');
+        const queue = new InMemoryControlQueue();
+        const wait = vi.spyOn(queue, 'waitFor').mockResolvedValue(null);
+        let checks = 0;
+        const processor = { run: vi.fn() };
+        const lifecycle = new TurnDaemonLifecycle(
+            {
+                clock: new ManualClock(now.getTime()),
+                controlQueue: queue,
+                processor,
+                getNextTickTime: (value) => addMinutes(value, 60),
+                stateStore: {
+                    loadLastTurnTime: async () => now,
+                    loadNextGeneralTurnTime: async () => addMinutes(now, 60),
+                    projectGameDeadline: async () => addMinutes(now, 180),
+                    saveLastTurnTime: async () => {},
+                    loadCheckpoint: async () => undefined,
+                    saveCheckpoint: async () => {},
+                },
+                pauseGate: async () => {
+                    if (++checks === 2) throw error;
+                    return false;
+                },
+            },
+            { profile: 'che', defaultBudget: { budgetMs: 100, maxGenerals: 10, catchUpCap: 1 } }
+        );
+        await expect(lifecycle.start()).rejects.toBe(error);
+        expect(wait).toHaveBeenCalledExactlyOnceWith(1000);
+        expect(processor.run).not.toHaveBeenCalled();
+    });
+
     it.each(['PREOPEN', 'SUSPENDED', 'RECONCILING', 'COMPLETED'] as const)(
         'does not dispatch an explicit run while the clock phase is %s',
         async (phase) => {