fix: 10분 중단 즉시 복구와 대기 중 토너먼트 참가 허용

This commit is contained in:
2026-09-15 23:36:50 +00:00
parent 4fb1343943
commit ab5a1d240c
12 changed files with 447 additions and 87 deletions
+73 -63
View File
@@ -11,7 +11,11 @@ import { assignManualApplicantGroup } from '../../tournament/workerHelpers.js';
import { accessAuthedProcedure, authedProcedure, engineAuthedProcedure, procedure, router } from '../../trpc.js'; import { accessAuthedProcedure, authedProcedure, engineAuthedProcedure, procedure, router } from '../../trpc.js';
import { getMyGeneral } from '../shared/general.js'; import { getMyGeneral } from '../shared/general.js';
import { loadCurrentGameTime } from '../../services/gameClock.js'; import { loadCurrentGameTime } from '../../services/gameClock.js';
import { ensureActiveRedisClockFence, ensureBettingRedisClockFence } from '../../services/redisClockFence.js'; import {
ensureActiveRedisClockFence,
ensureBettingRedisClockFence,
ensureTournamentParticipationRedisClockFence,
} from '../../services/redisClockFence.js';
import { loadClockAdminStatus } from '../../services/clockReadiness.js'; import { loadClockAdminStatus } from '../../services/clockReadiness.js';
const hasAdminRole = (roles: string[], profileName: string): boolean => { const hasAdminRole = (roles: string[], profileName: string): boolean => {
@@ -63,10 +67,11 @@ const withTournamentClockMutation = async <T>(
tournamentMutationLockHeld?: boolean; tournamentMutationLockHeld?: boolean;
}, },
store: TournamentStore, store: TournamentStore,
operation: () => Promise<T> operation: () => Promise<T>,
ensureFence = ensureActiveRedisClockFence
): Promise<T> => { ): Promise<T> => {
const gameTime = await loadCurrentGameTime(ctx.db); const gameTime = await loadCurrentGameTime(ctx.db);
const fence = await ensureActiveRedisClockFence(ctx.redis, ctx.profile.name, gameTime); const fence = await ensureFence(ctx.redis, ctx.profile.name, gameTime);
if (!fence) { if (!fence) {
throw new TRPCError({ throw new TRPCError({
code: 'PRECONDITION_FAILED', code: 'PRECONDITION_FAILED',
@@ -473,70 +478,75 @@ export const tournamentRouter = router({
join: engineAuthedProcedure.mutation(async ({ ctx }) => { join: engineAuthedProcedure.mutation(async ({ ctx }) => {
const general = await getMyGeneral(ctx); const general = await getMyGeneral(ctx);
const store = new TournamentStore(ctx.redis, buildTournamentKeys(ctx.profile.name)); const store = new TournamentStore(ctx.redis, buildTournamentKeys(ctx.profile.name));
return withTournamentClockMutation(ctx, store, async () => { return withTournamentClockMutation(
const state = await store.getState(); ctx,
if (!state || state.stage !== 1 || state.participantsLockedAt) { store,
throw new TRPCError({ code: 'BAD_REQUEST', message: '참가 신청 기간이 아닙니다.' }); async () => {
} const state = await store.getState();
if (!state || state.stage !== 1 || state.participantsLockedAt) {
throw new TRPCError({ code: 'BAD_REQUEST', message: '참가 신청 기간이 아닙니다.' });
}
const [participants, worldState] = await Promise.all([ const [participants, worldState] = await Promise.all([
store.getParticipants(), store.getParticipants(),
ctx.db.worldState.findFirst(), ctx.db.worldState.findFirst(),
]); ]);
if (participants.some((entry) => entry.id === general.id)) { if (participants.some((entry) => entry.id === general.id)) {
return { ok: true, count: participants.length }; return { ok: true, count: participants.length };
} }
if (participants.length >= 64) { if (participants.length >= 64) {
throw new TRPCError({ code: 'BAD_REQUEST', message: '참가 인원이 가득 찼습니다.' }); throw new TRPCError({ code: 'BAD_REQUEST', message: '참가 인원이 가득 찼습니다.' });
} }
const develCost = resolveCurrentDevelCost(worldState); const develCost = resolveCurrentDevelCost(worldState);
const feeResult = await ctx.turnDaemon.requestCommand({ const feeResult = await ctx.turnDaemon.requestCommand({
type: 'adjustGeneralResources',
requestId: tournamentJoinCommandRequestId(ctx.requestId, 'resources'),
reason: 'tournamentJoin',
adjustments: [{ generalId: general.id, goldDelta: -develCost, minGoldAfter: 0 }],
});
if (!feeResult || feeResult.type !== 'adjustGeneralResources') {
throw new TRPCError({ code: 'INTERNAL_SERVER_ERROR', message: 'Unexpected response' });
}
if (!feeResult.ok || feeResult.processed !== 1) {
throw new TRPCError({
code: 'BAD_REQUEST',
message: feeResult.ok ? '금이 부족합니다.' : feeResult.reason,
});
}
const meta = asRecord(general.meta);
const level = typeof meta.explevel === 'number' ? meta.explevel : 0;
const applicant = assignManualApplicantGroup({
state,
baseSeed: String(asRecord(worldState?.meta).hiddenSeed ?? 'tournament'),
current: participants,
applicant: {
id: general.id,
name: general.name,
leadership: general.leadership,
strength: general.strength,
intel: general.intel,
level,
},
});
const next = participants.concat(applicant);
try {
await store.setParticipants(next);
} catch (error) {
await ctx.turnDaemon.requestCommand({
type: 'adjustGeneralResources', type: 'adjustGeneralResources',
requestId: tournamentJoinCommandRequestId(ctx.requestId, 'projection-rollback-resources'), requestId: tournamentJoinCommandRequestId(ctx.requestId, 'resources'),
reason: 'tournamentJoinRollback', reason: 'tournamentJoin',
adjustments: [{ generalId: general.id, goldDelta: develCost }], adjustments: [{ generalId: general.id, goldDelta: -develCost, minGoldAfter: 0 }],
}); });
throw error; if (!feeResult || feeResult.type !== 'adjustGeneralResources') {
} throw new TRPCError({ code: 'INTERNAL_SERVER_ERROR', message: 'Unexpected response' });
return { ok: true, count: next.length }; }
}); if (!feeResult.ok || feeResult.processed !== 1) {
throw new TRPCError({
code: 'BAD_REQUEST',
message: feeResult.ok ? '금이 부족합니다.' : feeResult.reason,
});
}
const meta = asRecord(general.meta);
const level = typeof meta.explevel === 'number' ? meta.explevel : 0;
const applicant = assignManualApplicantGroup({
state,
baseSeed: String(asRecord(worldState?.meta).hiddenSeed ?? 'tournament'),
current: participants,
applicant: {
id: general.id,
name: general.name,
leadership: general.leadership,
strength: general.strength,
intel: general.intel,
level,
},
});
const next = participants.concat(applicant);
try {
await store.setParticipants(next);
} catch (error) {
await ctx.turnDaemon.requestCommand({
type: 'adjustGeneralResources',
requestId: tournamentJoinCommandRequestId(ctx.requestId, 'projection-rollback-resources'),
reason: 'tournamentJoinRollback',
adjustments: [{ generalId: general.id, goldDelta: develCost }],
});
throw error;
}
return { ok: true, count: next.length };
},
ensureTournamentParticipationRedisClockFence
);
}), }),
cancel: adminProcedure.mutation(async ({ ctx }) => { cancel: adminProcedure.mutation(async ({ ctx }) => {
const store = new TournamentStore(ctx.redis, buildTournamentKeys(ctx.profile.name)); const store = new TournamentStore(ctx.redis, buildTournamentKeys(ctx.profile.name));
@@ -73,6 +73,13 @@ export const ensureActiveRedisClockFence = async (
return ensureRedisClockFence(redis, profileName, gameTime, ['RUNNING']); return ensureRedisClockFence(redis, profileName, gameTime, ['RUNNING']);
}; };
/** 복구 대기 중에도 참가 신청은 가능하다. 자동 진행/정산에는 사용하지 않는다. */
export const ensureTournamentParticipationRedisClockFence = async (
redis: ClockFenceRedis,
profileName: string,
gameTime: CurrentGameTime
): Promise<ActiveRedisClockFence | null> => ensureRedisClockFence(redis, profileName, gameTime, ['RUNNING']);
/** /**
* User betting is allowed against a frozen tournament deadline while the game * User betting is allowed against a frozen tournament deadline while the game
* clock is suspended. Stage progression and settlement continue to use the * clock is suspended. Stage progression and settlement continue to use the
@@ -0,0 +1,214 @@
import { expect, it } from 'vitest';
import fastify from 'fastify';
import { fastifyTRPCPlugin } from '@trpc/server/adapters/fastify';
import { createGamePostgresConnector, createRedisConnector } from '@sammo-ts/infra';
import { DatabaseTurnDaemonCommandQueue, DatabaseTurnDaemonLease } from '@sammo-ts/game-engine';
import { appRouter } from '../src/router.js';
import { DatabaseTurnDaemonTransport } from '../src/daemon/databaseTransport.js';
import { RedisAccessTokenStore } from '../src/auth/accessTokenStore.js';
import { InMemoryFlushStore } from '../src/auth/flushStore.js';
import type { GameApiContext } from '../src/context.js';
import { loadCurrentGameTime } from '../src/services/gameClock.js';
const databaseUrl = process.env.TOURNAMENT_RECOVERY_DATABASE_URL;
const integration = it.skipIf(!databaseUrl || !process.env.REDIS_URL);
integration(
'accepts an authenticated HTTP tournament join during recovery wait with durable fee and Redis fences',
async () => {
const schema = new URL(databaseUrl!).searchParams.get('schema');
if (!schema?.endsWith('_tournament_recovery_integration'))
throw new Error('Dedicated tournament fixture schema required');
const connector = createGamePostgresConnector({ url: databaseUrl! });
const redisConnector = createRedisConnector({ url: process.env.REDIS_URL! });
await connector.connect();
await redisConnector.connect();
const db = connector.prisma;
const redis = redisConnector.client;
const profile = 'test:recovery-wait';
const prefix = `sammo:${profile}`;
const tokenStore = new RedisAccessTokenStore(redis, profile);
const transport = new DatabaseTurnDaemonTransport(db, 4000);
const lease = await DatabaseTurnDaemonLease.connect(databaseUrl!, { profile, heartbeat: false });
const queue = new DatabaseTurnDaemonCommandQueue(db);
const app = fastify();
let workerError: unknown;
let workerBusy = false;
let timer: ReturnType<typeof setInterval> | undefined;
try {
await db.inputEvent.deleteMany();
await db.general.deleteMany();
await db.turnDaemonLease.deleteMany();
await db.worldState.deleteMany();
const baseTime = new Date('2026-01-01T00:00:00Z');
const waitAt = new Date(Date.now() + 1_800_000);
const world = await db.worldState.create({
data: {
scenarioCode: 'recovery-wait',
currentYear: 200,
currentMonth: 1,
tickSeconds: 3600,
clockBaseTime: baseTime,
clockTick: 0n,
lastTurnTick: 0n,
clockWallAnchor: waitAt,
clockRecoveryStartTick: 0n,
clockRecoveryEndTick: 36_000_000n,
clockRecoveryStartWallAt: waitAt,
clockMode: 'realtime',
clockPhase: 'RUNNING',
clockRevision: 1n,
deadlineGeneration: 1n,
config: { const: { develCost: 100 } },
},
});
await db.general.create({
data: { id: 901, userId: 'fixture-user', name: 'fixture', gold: 1000, turnTime: baseTime },
});
await lease.acquire();
await lease.markClockReady();
await queue.initialize();
await redis.set(
`${prefix}:tournament:state`,
JSON.stringify({
stage: 1,
phase: 0,
type: 0,
auto: true,
openYear: 200,
openMonth: 1,
termSeconds: 60,
nextAt: baseTime.toISOString(),
})
);
await redis.set(`${prefix}:tournament:participants`, '[]');
await redis.del([
`${prefix}:clock:active-revision`,
`${prefix}:clock:deadline-generation`,
`${prefix}:clock:phase`,
]);
const auth: NonNullable<GameApiContext['auth']> = {
version: 1,
profile,
sessionId: 'fixture-session',
issuedAt: new Date().toISOString(),
expiresAt: '2999-01-01T00:00:00Z',
user: { id: 'fixture-user', username: 'fixture', displayName: 'fixture', roles: [] },
sanctions: {},
};
await app.register(fastifyTRPCPlugin, {
prefix: '/trpc',
trpcOptions: {
router: appRouter,
createContext: ({
req,
}: {
req: { headers: Record<string, string | string[] | undefined> };
}): GameApiContext => ({
db,
redis,
profile: { id: 'test', scenario: 'recovery-wait', name: profile },
auth: req.headers.authorization === 'Bearer fixture-auth' ? auth : null,
requestId:
typeof req.headers['x-test-request'] === 'string'
? req.headers['x-test-request']
: undefined,
turnDaemon: transport,
accessTokenStore: tokenStore,
flushStore: new InMemoryFlushStore(),
battleSim: {} as GameApiContext['battleSim'],
uploadDir: '/tmp',
uploadPath: '/uploads',
uploadPublicUrl: null,
gameTokenSecret: 'fixture-only',
}),
},
});
await app.listen({ host: '127.0.0.1', port: 0 });
const address = app.server.address();
if (!address || typeof address === 'string') throw new Error('Missing fixture listener');
const url = `http://127.0.0.1:${address.port}/trpc/tournament.join`;
const post = (authenticated = true) =>
fetch(url, {
method: 'POST',
headers: {
'content-type': 'application/json',
...(authenticated ? { authorization: 'Bearer fixture-auth' } : {}),
'x-test-request': 'recovery-join',
},
body: '{}',
});
expect((await post(false)).status).toBe(401);
expect(await loadCurrentGameTime(db)).toMatchObject({
phase: 'RUNNING',
running: false,
runtimeReady: true,
});
// Fixture worker: production queue/lease and real PostgreSQL transaction; no auto turns.
timer = setInterval(() => {
if (workerBusy) return;
workerBusy = true;
void (async () => {
for (const command of await queue.drain()) {
if (command.type !== 'adjustGeneralResources' || !command.requestId)
throw new Error('Unexpected fixture command');
await db.$transaction(async (tx) => {
await lease.assertActive(tx);
const adjustment = command.adjustments[0]!;
await tx.general.update({
where: { id: adjustment.generalId },
data: { gold: { increment: adjustment.goldDelta ?? 0 } },
});
await tx.inputEvent.update({
where: { requestId: command.requestId },
data: {
status: 'SUCCEEDED',
result: {
type: 'adjustGeneralResources',
ok: true,
processed: 1,
missing: 0,
totalGoldDelta: adjustment.goldDelta ?? 0,
totalRiceDelta: 0,
},
},
});
});
}
})()
.catch((error: unknown) => {
workerError = error;
})
.finally(() => {
workerBusy = false;
});
}, 10);
expect((await post()).status).toBe(200);
expect((await post()).status).toBe(200);
expect(workerError).toBeUndefined();
expect((await db.general.findUniqueOrThrow({ where: { id: 901 } })).gold).toBe(900);
expect(JSON.parse((await redis.get(`${prefix}:tournament:participants`))!)).toHaveLength(1);
const events = await db.inputEvent.findMany({ where: { eventType: 'adjustGeneralResources' } });
expect(events).toHaveLength(1);
expect(events[0]).toMatchObject({ status: 'SUCCEEDED', acceptedGameTick: 0n, acceptedClockRevision: 1n });
await redis.set(`${prefix}:clock:active-revision`, '2');
expect((await post()).status).toBe(412);
await redis.set(`${prefix}:clock:active-revision`, '1');
await db.worldState.update({ where: { id: world.id }, data: { clockPhase: 'RECONCILING' } });
expect((await post()).status).toBe(412);
await db.worldState.update({ where: { id: world.id }, data: { clockPhase: 'RUNNING' } });
await lease.release();
expect((await post()).status).toBe(412);
expect((await db.general.findUniqueOrThrow({ where: { id: 901 } })).gold).toBe(900);
expect((await db.worldState.findUniqueOrThrow({ where: { id: world.id } })).clockTick).toBe(0n);
} finally {
if (timer) clearInterval(timer);
while (workerBusy) await new Promise((resolve) => setTimeout(resolve, 10));
await app.close();
await lease.close();
await connector.disconnect();
await redisConnector.disconnect();
}
},
15000
);
+18 -2
View File
@@ -153,6 +153,7 @@ const buildContext = (options: {
rankRows?: Array<{ generalId: number; type: string; value: number }>; rankRows?: Array<{ generalId: number; type: string; value: number }>;
clockPhase?: 'PREOPEN' | 'RUNNING' | 'MANUAL' | 'SUSPENDED' | 'RECONCILING'; clockPhase?: 'PREOPEN' | 'RUNNING' | 'MANUAL' | 'SUSPENDED' | 'RECONCILING';
requestId?: string; requestId?: string;
clockWallAnchor?: Date;
}): GameApiContext => { }): GameApiContext => {
const db = { const db = {
general: { general: {
@@ -169,7 +170,7 @@ const buildContext = (options: {
clockBaseTime: new Date('2026-01-01T00:00:00.000Z'), clockBaseTime: new Date('2026-01-01T00:00:00.000Z'),
clockTick: 0n, clockTick: 0n,
clockMode: 'realtime', clockMode: 'realtime',
clockWallAnchor: new Date('2026-01-01T00:00:00.000Z'), clockWallAnchor: options.clockWallAnchor ?? new Date('2026-01-01T00:00:00.000Z'),
clockPhase: options.clockPhase ?? 'RUNNING', clockPhase: options.clockPhase ?? 'RUNNING',
clockRevision: 1n, clockRevision: 1n,
deadlineGeneration: 1n, deadlineGeneration: 1n,
@@ -261,7 +262,21 @@ describe('tournament router permissions and mutations', () => {
]); ]);
}); });
it('charges the authenticated general once when joining', async () => { it.each(['PREOPEN', 'SUSPENDED', 'RECONCILING'] as const)(
'rejects joining in %s without charging',
async (clockPhase) => {
const redis = new MemoryRedis();
const transport = new TournamentTransport();
const general = buildGeneral(1, 'user-1');
const caller = appRouter.createCaller(
buildContext({ redis, transport, generals: [general], userId: 'user-1', clockPhase })
);
await expect(caller.tournament.join()).rejects.toMatchObject({ code: 'PRECONDITION_FAILED' });
expect(transport.commands).toHaveLength(0);
}
);
it.each([false, true])('charges the authenticated general once when joining; recovery wait=%s', async (waiting) => {
const redis = new MemoryRedis(); const redis = new MemoryRedis();
const transport = new TournamentTransport(); const transport = new TournamentTransport();
const general = buildGeneral(1, 'user-1'); const general = buildGeneral(1, 'user-1');
@@ -284,6 +299,7 @@ describe('tournament router permissions and mutations', () => {
userId: 'user-1', userId: 'user-1',
develCost: 200, develCost: 200,
requestId: 'http:tournament-join', requestId: 'http:tournament-join',
...(waiting ? { clockWallAnchor: new Date(Date.now() + 1_800_000) } : {}),
}); });
const outerApiTransaction = vi.fn(async () => { const outerApiTransaction = vi.fn(async () => {
throw new Error('tournament join must not hold an API transaction while waiting for the daemon'); throw new Error('tournament join must not hold an API transaction while waiting for the daemon');
@@ -28,11 +28,8 @@ export const prepareRealtimeRecovery = async (
if (world.clockPhase !== 'RUNNING' || !world.clockWallAnchor || world.clockTick === null) return; if (world.clockPhase !== 'RUNNING' || !world.clockWallAnchor || world.clockTick === null) return;
const now = await readClockDatabaseWall(db); const now = await readClockDatabaseWall(db);
// 가속 중 정상적인 프로세스 교체는 기존 창을 그대로 재사용한다. // 가속 중 정상적인 프로세스 교체는 기존 창을 그대로 재사용한다.
// 짧은 중단만 즉시 처리한다. 기준값과 같으면 대기 후 복구한다. // 전체 중단 10분까지는 기존 엔진이 밀린 턴을 순서대로 처리한다.
if ( if (!options.paused && now.getTime() - world.clockWallAnchor.getTime() <= immediateRecoveryLimitSeconds() * 1_000)
!options.paused &&
now.getTime() - world.clockWallAnchor.getTime() < immediateRecoveryLimitSeconds(world.tickSeconds) * 1_000
)
return; return;
const suspensionId = `recovery-${randomUUID()}`; const suspensionId = `recovery-${randomUUID()}`;
await startClockSuspension({ await startClockSuspension({
@@ -231,7 +231,7 @@ describeIntegration('durable clock reconciliation', () => {
expect((await db.worldState.findUniqueOrThrow({ where: { id: row.id } })).clockRecoveryStartTick).toBe(1n); expect((await db.worldState.findUniqueOrThrow({ where: { id: row.id } })).clockRecoveryStartTick).toBe(1n);
}); });
it.each([300, 420])('applies startup recovery after %i seconds on a 60-minute server', async (delay) => { it.each([180, 420, 590, 610])('applies startup recovery after %i seconds on a 60-minute server', async (delay) => {
const profile = 'short-startup'; const profile = 'short-startup';
await db.worldState.create({ await db.worldState.create({
data: { data: {
@@ -259,8 +259,8 @@ describeIntegration('durable clock reconciliation', () => {
fencingEpoch: token.fencingEpoch, fencingEpoch: token.fencingEpoch,
}); });
const world = await db.worldState.findFirstOrThrow(); const world = await db.worldState.findFirstOrThrow();
expect(world.clockPhase).toBe(delay < 360 ? 'RUNNING' : 'RECONCILING'); expect(world.clockPhase).toBe(delay <= 600 ? 'RUNNING' : 'RECONCILING');
expect(readTurnRecovery(world) === null).toBe(delay < 360); expect(readTurnRecovery(world) === null).toBe(delay <= 600);
} finally { } finally {
await lease.close(); await lease.close();
} }
@@ -345,6 +345,61 @@ describeIntegration('durable clock reconciliation', () => {
} }
}); });
it.each([300, 1200, 3600])(
'keeps pending turns and operator pause across a short VM outage (%i second turn)',
async (turnSeconds) => {
const profile = 'short-vm-outage';
const anchor = new Date(Date.now() - 590_000);
await db.worldState.create({
data: {
scenarioCode: profile,
currentYear: 199,
currentMonth: 1,
tickSeconds: turnSeconds,
clockBaseTime: anchor,
clockWallAnchor: anchor,
clockTick: 0n,
lastTurnTick: 0n,
clockMode: 'realtime',
clockPhase: 'RUNNING',
clockRevision: 1n,
deadlineGeneration: 1n,
},
});
const general = await db.general.create({
data: { id: 901, name: 'pending-turn', turnTick: 12345n, turnTime: anchor },
});
const lease = await DatabaseTurnDaemonLease.connect(databaseUrl!, { profile, heartbeat: false });
try {
const token = await lease.acquire();
const authority = {
kind: 'DAEMON' as const,
profileName: profile,
ownerId: token!.ownerId,
fencingEpoch: token!.fencingEpoch,
};
await prepareRealtimeRecovery(db, authority);
expect(await db.worldState.findFirstOrThrow()).toMatchObject({
clockPhase: 'RUNNING',
clockRevision: 1n,
clockTick: 0n,
lastTurnTick: 0n,
clockRecoveryStartTick: null,
});
expect(await db.general.findUniqueOrThrow({ where: { id: general.id } })).toMatchObject({
turnTick: general.turnTick,
turnTime: general.turnTime,
});
expect(await db.clockSuspension.count()).toBe(0);
// 짧은 host 중단도 운영자가 정지한 서버를 자동 재개하는 근거가 되지 않는다.
await prepareRealtimeRecovery(db, authority, { paused: true });
expect(await db.worldState.findFirstOrThrow()).toMatchObject({ clockPhase: 'SUSPENDED' });
} finally {
await lease.close();
}
}
);
it.each([false, true])('fences outage recovery and reuses its window; repeated outage=%s', async (repeated) => { it.each([false, true])('fences outage recovery and reuses its window; repeated outage=%s', async (repeated) => {
const profile = 'recovery-startup'; const profile = 'recovery-startup';
await db.worldState.create({ await db.worldState.create({
@@ -513,8 +568,8 @@ describeIntegration('durable clock reconciliation', () => {
} }
); );
it.each([359999, 360000, 360001, 840000, 12 * 3600000 + 1000])( it.each([180000, 599999, 600000, 600001, 840000, 12 * 3600000 + 1000])(
'persists strict recovery boundaries for %i ms', 'persists the inclusive ten-minute recovery boundary for %i ms',
async (gap) => { async (gap) => {
const observed = T / 6; const observed = T / 6;
const future = new Date(Date.now() + 3600000); const future = new Date(Date.now() + 3600000);
@@ -549,7 +604,7 @@ describeIntegration('durable clock reconciliation', () => {
authority, authority,
testResumeWallAt: now, testResumeWallAt: now,
}); });
expect(plan.recovery === null).toBe(gap < 360000); expect(plan.recovery === null).toBe(gap <= 600000);
expect(await db.message.count()).toBe(0); expect(await db.message.count()).toBe(0);
// Redis 장애 후에도 알림은 DB의 RUNNING 전이와 함께 한 번만 저장한다. // Redis 장애 후에도 알림은 DB의 RUNNING 전이와 함께 한 번만 저장한다.
await expect( await expect(
@@ -84,6 +84,52 @@ const buildWorld = (stateOverride: Partial<TurnWorldState> = {}): InMemoryTurnWo
}; };
describe('runtime clock shift', () => { describe('runtime clock shift', () => {
it('executes ten minutes of pending work in bounded batches without skipping or repeating generals', async () => {
const base = new Date('2026-07-30T10:00:00Z');
const resumed = new Date(base.getTime() + 600_000);
const plan = planTurnRecovery({
observedTick: 0,
normalTick: GAME_TICKS_PER_TURN,
wallNow: resumed,
turnSeconds: 600,
});
expect(plan).toMatchObject({ skippedTurns: 0, recovery: null });
const world = buildWorld({
clockBaseTime: base,
clockTick: 0,
clockWallAnchor: base,
clockMode: 'realtime',
clockPhase: 'RUNNING',
lastTurnTick: 0,
});
for (const [id, offset] of [
[1, 19_902],
[2, 42_001],
] as const) {
world.updateGeneral(id, { turnTime: new Date(base.getTime() + offset), turnTick: offset * 60 });
}
const executed: number[] = [];
const processor = new InMemoryTurnProcessor(world, {
afterExecuteGeneral: async (general) => {
executed.push(general.id);
},
});
const target = world.getGameNow(resumed);
world.advanceGameClockTo(target, resumed);
const budget = { budgetMs: 10_000, maxGenerals: 1, catchUpCap: 1 };
let result = await processor.run(target, budget);
expect(result.processedGenerals).toBe(1);
for (let batch = 0; result.partial && batch < 4; batch++) {
result = await processor.run(target, budget, result.checkpoint);
expect(result.processedGenerals).toBeLessThanOrEqual(1);
}
expect(result.partial).toBe(false);
expect(executed).toEqual([1, 2]);
expect(world.getState().currentMonth).toBe(2);
expect(world.getGeneralById(1)!.turnTick).toBe(GAME_TICKS_PER_TURN + 19_902 * 60);
expect(world.getGeneralById(2)!.turnTick).toBe(GAME_TICKS_PER_TURN + 42_001 * 60);
});
it('runs two real monthly cycles per normal interval, survives reload, and returns to one cycle', async () => { it('runs two real monthly cycles per normal interval, survives reload, and returns to one cycle', async () => {
const base = new Date('2026-07-30T10:00:00Z'); const base = new Date('2026-07-30T10:00:00Z');
const wallAt = (minutes: number) => new Date(base.getTime() + minutes * 60_000); const wallAt = (minutes: number) => new Date(base.getTime() + minutes * 60_000);
+12 -3
View File
@@ -44,9 +44,18 @@ From 2026-09-07, maintenance and crash recovery use the following
`RECOVER_TURNS` policy. The base turn length does not change. One turn remains `RECOVER_TURNS` policy. The base turn length does not change. One turn remains
36,000,000 ticks; a persisted `TurnRecoveryWindow` changes only wall execution. 36,000,000 ticks; a persisted `TurnRecoveryWindow` changes only wall execution.
- An entire delay strictly below `min(600 seconds, turnSeconds / 10)` catches - From 2026-09-15, an entire delay up to and including 600 seconds catches
up immediately through the ordinary engine. Equality uses recovery. The up immediately through the ordinary engine, independently of the server's
limit is 30 seconds on a 5-minute server and 6 minutes on a 60-minute server. turn length. This accommodates host updates that pause and resume a VM.
Pending turns are executed in order with the ordinary per-run budget and
month boundary cap; they are not marked complete or skipped. The lease still
expires after 30 seconds and a fresh owner must fence the old process and
reload durable state before catching up. Operator pauses remain pauses.
- During the persisted wait before 2x recovery, tournament participation is
allowed when the phase is `RUNNING`, the daemon lease is ready, and DB/Redis
revision and deadline generation agree. Registration stage, capacity,
duplicate entry, fee and ENGINE transaction checks still apply. Tournament
stage advancement and settlement continue to require a ticking clock.
- For longer delays, skip only complete 12-turn blocks, moving future - For longer delays, skip only complete 12-turn blocks, moving future
schedules and the execution cursor by the same integer delta. Never apply schedules and the execution cursor by the same integer delta. Never apply
the short-delay exception again to the remainder. An exact multiple of the short-delay exception again to the remainder. An exact multiple of
+5
View File
@@ -336,6 +336,11 @@ PM2가 새 owner/epoch와 DB snapshot으로 재시작합니다. lease 오류만
않습니다. 이전 버전에서 이미 lease 오류로 PAUSED가 된 서버는 원인을 확인한 뒤 않습니다. 이전 버전에서 이미 lease 오류로 PAUSED가 된 서버는 원인을 확인한 뒤
한 번 `START`/재개해야 합니다. 한 번 `START`/재개해야 합니다.
Host 업데이트 등으로 VM이 최대 10분 중단되면 새 runtime은 밀린 턴을 기존 엔진의
순서·batch budget으로 즉시 따라잡습니다(정확히 10분 포함). 10분 초과는 기존의
12턴 묶음 이동 및 대기 후 2배속 복구 정책을 사용합니다. 이미 저장한 복구 창과
운영자의 PAUSED/STOPPED는 변경하지 않습니다.
30초 lease와 이전 owner의 fencing은 유지합니다. transaction의 만료 검사는 시작에 30초 lease와 이전 owner의 fencing은 유지합니다. transaction의 만료 검사는 시작에
고정된 `CURRENT_TIMESTAMP`가 아닌 검사 순간의 `clock_timestamp()`를 사용합니다. 고정된 `CURRENT_TIMESTAMP`가 아닌 검사 순간의 `clock_timestamp()`를 사용합니다.
역방향 시계 보정으로 기존 lease가 아직 유효하면 startup은 2초 간격으로 기다리며 역방향 시계 보정으로 기존 lease가 아직 유효하면 startup은 2초 간격으로 기다리며
+4 -4
View File
@@ -68,8 +68,8 @@ export const turnShiftTicks = (turns: number): GameTick => {
return asGameTick(turns * GAME_TICKS_PER_TURN); return asGameTick(turns * GAME_TICKS_PER_TURN);
}; };
/** 즉시 처리 여부는 12턴 묶음 생략 전의 전체 지연으로 판정한다. */ /** VM/host 업데이트의 전체 중단 10분까지는 기존 엔진 순서와 budget으로 따라잡는다. */
export const immediateRecoveryLimitSeconds = (turnSeconds: number): number => Math.min(600, turnSeconds / 10); export const immediateRecoveryLimitSeconds = (): number => 600;
/** /**
* observedTick은 중단 전에 저장한 관측 지점, normalTick은 기존 시간표의 현재 지점이다. * observedTick은 중단 전에 저장한 관측 지점, normalTick은 기존 시간표의 현재 지점이다.
@@ -91,8 +91,8 @@ export const planTurnRecovery = (input: {
if (!Number.isFinite(wallNow.getTime())) throw new Error('Recovery wall instant is invalid.'); if (!Number.isFinite(wallNow.getTime())) throw new Error('Recovery wall instant is invalid.');
const gap = Math.max(0, normalTick - observedTick); const gap = Math.max(0, normalTick - observedTick);
const ticksPerSecond = GAME_TICKS_PER_TURN / turnSeconds; const ticksPerSecond = GAME_TICKS_PER_TURN / turnSeconds;
const immediateLimit = immediateRecoveryLimitSeconds(turnSeconds) * ticksPerSecond; const immediateLimit = immediateRecoveryLimitSeconds() * ticksPerSecond;
if (gap < immediateLimit) { if (gap <= immediateLimit) {
return { return {
skippedTurns: 0, skippedTurns: 0,
recoveryTurns: 0, recoveryTurns: 0,
+4 -4
View File
@@ -160,9 +160,9 @@ describe('turn-aligned double-speed recovery', () => {
expect(clock.executionRate(end)).toBe(1); expect(clock.executionRate(end)).toBe(1);
}); });
it.each([300, 3600, 6000, 7200])('uses the strict whole-delay threshold for a %i second turn', (turnSeconds) => { it.each([60, 300, 1200, 3600, 6000, 7200])('catches up through ten minutes for a %i second turn', (turnSeconds) => {
const rate = T / turnSeconds; const rate = T / turnSeconds;
const limitMs = Math.min(600, turnSeconds / 10) * 1000; const limitMs = 600_000;
for (const delta of [-1, 0, 1]) { for (const delta of [-1, 0, 1]) {
const delayMs = limitMs + delta; const delayMs = limitMs + delta;
const normal = Math.trunc((delayMs * rate) / 1000); const normal = Math.trunc((delayMs * rate) / 1000);
@@ -172,8 +172,8 @@ describe('turn-aligned double-speed recovery', () => {
wallNow: new Date(base + delayMs), wallNow: new Date(base + delayMs),
turnSeconds, turnSeconds,
}); });
expect(plan.recovery === null).toBe(delta < 0); expect(plan.recovery === null).toBe(delta <= 0);
expect(plan.initialTick).toBe(delta < 0 ? normal : 0); expect(plan.initialTick).toBe(delta <= 0 ? normal : 0);
} }
// 장시간 중단의 작은 나머지에는 즉시 처리 예외를 다시 적용하지 않는다. // 장시간 중단의 작은 나머지에는 즉시 처리 예외를 다시 적용하지 않는다.
const long = planTurnRecovery({ observedTick: 0, normalTick: 12 * T + rate, wallNow: wall(20), turnSeconds }); const long = planTurnRecovery({ observedTick: 0, normalTick: 12 * T + rate, wallNow: wall(20), turnSeconds });
@@ -25,3 +25,4 @@ TURN_COMMAND_DURABLE_MATRIX_DATABASE_URL reference_command_durable_matrix
TURN_DIFFERENTIAL_DATABASE_URL core TURN_DIFFERENTIAL_DATABASE_URL core
TURN_FULL_LIFECYCLE_PERSISTENCE_DATABASE_URL reference_full_lifecycle TURN_FULL_LIFECYCLE_PERSISTENCE_DATABASE_URL reference_full_lifecycle
WEB_PUSH_GATEWAY_DATABASE_URL web_push_gateway WEB_PUSH_GATEWAY_DATABASE_URL web_push_gateway
TOURNAMENT_RECOVERY_DATABASE_URL external_fixture
1 # Environment variable Execution mode
25 TURN_DIFFERENTIAL_DATABASE_URL core
26 TURN_FULL_LIFECYCLE_PERSISTENCE_DATABASE_URL reference_full_lifecycle
27 WEB_PUSH_GATEWAY_DATABASE_URL web_push_gateway
28 TOURNAMENT_RECOVERY_DATABASE_URL external_fixture