diff --git a/app/game-api/src/auction/worker.ts b/app/game-api/src/auction/worker.ts index eff528d6..2a660348 100644 --- a/app/game-api/src/auction/worker.ts +++ b/app/game-api/src/auction/worker.ts @@ -1,4 +1,7 @@ import { + CLOCK_OPERATION_PERSISTENCE_LOCK, + GamePrisma, + acquireGameSchemaAdvisoryXactLock, createGamePostgresConnector, createRedisConnector, type GamePrismaClient, @@ -10,6 +13,7 @@ import { resolveGameApiConfigFromEnv } from '../config.js'; import { createBestEffortResourceCloser } from '../services/bestEffortResourceCloser.js'; import { loadCurrentGameTime, type CurrentGameTime } from '../services/gameClock.js'; import { createPollingWorkerControl, waitForWorkerPoll } from '../services/pollingWorkerLifecycle.js'; +import { ensureActiveRedisClockFence } from '../services/redisClockFence.js'; import { buildAuctionTimerKeys } from './keys.js'; import { resolveAuctionTimerScore, seedAuctionTimers } from './scheduler.js'; @@ -24,8 +28,22 @@ interface RedisTimerClient { zAdd(key: string, values: Array<{ score: number; value: string }>): Promise; zRem(key: string, values: string | string[]): Promise; zRemRangeByScore(key: string, min: number, max: number): Promise; + eval?(script: string, options: { keys: string[]; arguments: string[] }): Promise; } +const POP_DUE_AUCTIONS_SCRIPT = ` +if redis.call('GET', KEYS[2]) ~= ARGV[1] + or redis.call('GET', KEYS[3]) ~= ARGV[2] + or redis.call('GET', KEYS[4]) ~= 'RUNNING' then + return { '__CLOCK_FENCE__' } +end +local ids = redis.call('ZRANGEBYSCORE', KEYS[1], '-inf', ARGV[3], 'LIMIT', 0, ARGV[4]) +if #ids > 0 then + redis.call('ZREM', KEYS[1], unpack(ids)) +end +return ids +`; + const AUCTION_FINALIZE_RECOVERY_LIMIT = 1; interface AuctionFinalizeDeadline { @@ -115,12 +133,29 @@ const isSuccessfulAuctionFinalizeResult = (result: unknown, auctionId: number): return resultRecord.type === 'auctionFinalize' && resultRecord.ok === true && resultRecord.auctionId === auctionId; }; -const popDueAuctionIds = async ( +export const popDueAuctionIds = async ( redis: RedisTimerClient, timerKey: string, nowMs: number, - batchSize: number + batchSize: number, + clockFence?: { + activeRevisionKey: string; + deadlineGenerationKey: string; + phaseKey: string; + revision: number; + generation: number; + } ): Promise => { + if (clockFence) { + if (!redis.eval) throw new Error('Redis EVAL is required for revision-fenced auction due-pop.'); + const result = await redis.eval(POP_DUE_AUCTIONS_SCRIPT, { + keys: [timerKey, clockFence.activeRevisionKey, clockFence.deadlineGenerationKey, clockFence.phaseKey], + arguments: [String(clockFence.revision), String(clockFence.generation), String(nowMs), String(batchSize)], + }); + if (!Array.isArray(result)) throw new Error('Auction due-pop returned an invalid Redis result.'); + if (result[0] === '__CLOCK_FENCE__') return []; + return result.map(String); + } const ids = await redis.zRangeByScore(timerKey, 0, nowMs, { LIMIT: { offset: 0, count: batchSize } }); if (ids.length > 0) { await redis.zRem(timerKey, ids); @@ -205,6 +240,8 @@ export const processDueAuctionId = async (options: { nowMs: number; nowTick?: number | null; historyNowMs?: number; + expectedClockRevision?: number; + expectedDeadlineGeneration?: number; }): Promise<'PENDING' | 'RESCHEDULED' | 'IGNORED'> => { const { db, redis, timerKey, historyKey, id, nowMs, nowTick = null, historyNowMs = nowMs } = options; const auctionId = Number(id); @@ -213,6 +250,29 @@ export const processDueAuctionId = async (options: { } const now = new Date(nowMs); const outcome = await db.$transaction(async (transaction) => { + if (options.expectedClockRevision !== undefined || options.expectedDeadlineGeneration !== undefined) { + await acquireGameSchemaAdvisoryXactLock(transaction, CLOCK_OPERATION_PERSISTENCE_LOCK); + const [world] = await transaction.$queryRaw< + Array<{ clockPhase: string | null; clockRevision: bigint; deadlineGeneration: bigint }> + >(GamePrisma.sql` + SELECT + clock_phase AS "clockPhase", + clock_revision AS "clockRevision", + deadline_generation AS "deadlineGeneration" + FROM world_state + ORDER BY id ASC + LIMIT 1 + FOR UPDATE + `); + if ( + !world || + world.clockPhase !== 'RUNNING' || + world.clockRevision !== BigInt(options.expectedClockRevision ?? -1) || + world.deadlineGeneration !== BigInt(options.expectedDeadlineGeneration ?? -1) + ) { + return { status: 'RESCHEDULED' as const, clockFenceFailed: true }; + } + } const current = await transaction.auction.findUnique({ where: { id: auctionId }, select: { status: true, closeAt: true, closeTick: true }, @@ -259,6 +319,13 @@ export const processDueAuctionId = async (options: { target: 'ENGINE', eventType: nextCommand.type, payload: { ...nextCommand }, + ...(nowTick === null ? {} : { acceptedGameTick: BigInt(nowTick) }), + ...(options.expectedClockRevision === undefined + ? {} + : { acceptedClockRevision: BigInt(options.expectedClockRevision) }), + ...(options.expectedDeadlineGeneration === undefined + ? {} + : { acceptedDeadlineGeneration: BigInt(options.expectedDeadlineGeneration) }), }, }); return { status: 'PENDING' as const }; @@ -284,6 +351,10 @@ export const processDueAuctionId = async (options: { return 'PENDING'; } if (outcome.status === 'RESCHEDULED') { + if ('clockFenceFailed' in outcome) { + await redis.zAdd(timerKey, [{ score: nowTick ?? nowMs, value: id }]); + return 'RESCHEDULED'; + } const gameTime = await loadCurrentGameTime(db, now); await redis.zAdd(timerKey, [ { @@ -324,6 +395,17 @@ export const runAuctionWorker = async (options: AuctionWorkerOptions = {}): Prom const gameTime = await loadCurrentGameTime(postgres.prisma, new Date(operationalNowMs)); const gameNowMs = gameTime.now.getTime(); const dueScore = gameTime.tick ?? gameNowMs; + if (gameTime.phase && gameTime.phase !== 'RUNNING') { + await waitForWorkerPoll(control.signal, config.auctionTimerPollMs); + continue; + } + const clockFence = gameTime.phase + ? await ensureActiveRedisClockFence(redis.client, config.profileName, gameTime) + : null; + if (gameTime.phase && !clockFence) { + await waitForWorkerPoll(control.signal, config.auctionTimerPollMs); + continue; + } if (operationalNowMs >= nextResyncAt) { await seedAuctionTimers(postgres.prisma, redis.client, keys); nextResyncAt = operationalNowMs + config.auctionTimerResyncMs; @@ -345,7 +427,7 @@ export const runAuctionWorker = async (options: AuctionWorkerOptions = {}): Prom if (historyTrimBefore > 0) { await redis.client.zRemRangeByScore(keys.historyKey, 0, historyTrimBefore); } - const dueIds = await popDueAuctionIds(redis.client, keys.timerKey, dueScore, 100); + const dueIds = await popDueAuctionIds(redis.client, keys.timerKey, dueScore, 100, clockFence ?? undefined); if (dueIds.length > 0) { for (const id of dueIds) { try { @@ -358,6 +440,12 @@ export const runAuctionWorker = async (options: AuctionWorkerOptions = {}): Prom nowMs: gameNowMs, nowTick: gameTime.tick, historyNowMs: operationalNowMs, + ...(clockFence + ? { + expectedClockRevision: clockFence.revision, + expectedDeadlineGeneration: clockFence.generation, + } + : {}), }); if (outcome === 'PENDING') { pendingFinalizationIds.add(Number(id)); diff --git a/app/game-api/src/daemon/databaseTransport.ts b/app/game-api/src/daemon/databaseTransport.ts index e2ae7b6d..a7d42461 100644 --- a/app/game-api/src/daemon/databaseTransport.ts +++ b/app/game-api/src/daemon/databaseTransport.ts @@ -1,6 +1,12 @@ import { randomUUID } from 'node:crypto'; -import { acquireGameSchemaAdvisoryXactLock, type DatabaseClient, type GamePrisma } from '@sammo-ts/infra'; +import { + acquireGameSchemaAdvisoryXactLock, + readInputEventClockCoordinate, + type DatabaseClient, + type GamePrisma, + type InputEventClockCoordinate, +} from '@sammo-ts/infra'; import type { TurnDaemonTransport } from './transport.js'; import type { TurnDaemonCommand, TurnDaemonCommandResult, TurnDaemonStatus } from './types.js'; @@ -103,10 +109,10 @@ export class DatabaseTurnDaemonTransport implements TurnDaemonTransport { try { if (command.type === 'npcPossessGeneral' && this.db.$transaction) { const rejectionReason = await this.db.$transaction(async (transaction) => { + const coordinate = await readInputEventClockCoordinate(transaction); await acquireGameSchemaAdvisoryXactLock(transaction, 'npc-possession:global'); await acquireGameSchemaAdvisoryXactLock(transaction, `npc-possession:user:${command.userId}`); - const acceptedAt = new Date(Math.floor(Date.now() / 1000) * 1000); - const acceptedGameAt = (await loadCurrentGameTime(transaction, acceptedAt)).now; + const acceptedGameAt = coordinate.gameAt; const token = await transaction.npcSelectionToken.findFirst({ where: { ownerUserId: command.userId, @@ -130,14 +136,21 @@ export class DatabaseTurnDaemonTransport implements TurnDaemonTransport { ...(durableCommand as Extract), acceptedGameAt: acceptedGameAt.toISOString(), }; - await this.createInputEvent(transaction, acceptedCommand, requestId, acceptedAt); + await this.createInputEvent(transaction, acceptedCommand, requestId, coordinate); return null; }); if (rejectionReason) { throw new RejectedNpcPossessionCommandError('PRECONDITION_FAILED', rejectionReason); } } else { - await this.createInputEvent(this.db, durableCommand, requestId); + if (this.db.$transaction) { + await this.db.$transaction(async (transaction) => { + const coordinate = await readInputEventClockCoordinate(transaction); + await this.createInputEvent(transaction, durableCommand, requestId, coordinate); + }); + } else { + await this.createInputEvent(this.db, durableCommand, requestId); + } } } catch (error) { if (error instanceof RejectedNpcPossessionCommandError) { @@ -166,8 +179,20 @@ export class DatabaseTurnDaemonTransport implements TurnDaemonTransport { db: DatabaseClient, command: TurnDaemonCommand, requestId: string, - createdAt?: Date + coordinate?: InputEventClockCoordinate ): Promise { + const gameTime = coordinate ? null : await loadCurrentGameTime(db, new Date()); + const commandAcceptedTick = Reflect.get(command, 'acceptedGameTick'); + const acceptedGameTick = + typeof commandAcceptedTick === 'number' && Number.isSafeInteger(commandAcceptedTick) + ? commandAcceptedTick + : coordinate + ? Number(coordinate.gameTick) + : gameTime!.tick; + const acceptedClockRevision = coordinate ? Number(coordinate.clockRevision) : gameTime!.revision; + const acceptedDeadlineGeneration = coordinate + ? Number(coordinate.deadlineGeneration) + : gameTime!.deadlineGeneration; await db.inputEvent.create({ data: { requestId, @@ -175,7 +200,14 @@ export class DatabaseTurnDaemonTransport implements TurnDaemonTransport { eventType: command.type, payload: asJson(command), actorUserId: 'userId' in command && typeof command.userId === 'string' ? command.userId : null, - ...(createdAt ? { createdAt } : {}), + ...(acceptedGameTick === null ? {} : { acceptedGameTick: BigInt(acceptedGameTick) }), + ...(acceptedClockRevision === null || acceptedClockRevision === undefined + ? {} + : { acceptedClockRevision: BigInt(acceptedClockRevision) }), + ...(acceptedDeadlineGeneration === null || acceptedDeadlineGeneration === undefined + ? {} + : { acceptedDeadlineGeneration: BigInt(acceptedDeadlineGeneration) }), + ...(coordinate ? { createdAt: coordinate.wallAt } : {}), }, }); } diff --git a/app/game-api/src/inputEventBoundary.ts b/app/game-api/src/inputEventBoundary.ts index 1a0993dd..b868ab37 100644 --- a/app/game-api/src/inputEventBoundary.ts +++ b/app/game-api/src/inputEventBoundary.ts @@ -1,6 +1,11 @@ import { createHash } from 'node:crypto'; -import { GamePrisma, type DatabaseClient as InfraDatabaseClient } from '@sammo-ts/infra'; +import { + CLOCK_OPERATION_PERSISTENCE_LOCK, + GamePrisma, + acquireGameSchemaAdvisoryXactLock, + type DatabaseClient as InfraDatabaseClient, +} from '@sammo-ts/infra'; import type { DatabaseClient } from './context.js'; @@ -20,6 +25,9 @@ interface LockedInputEvent { status: 'PENDING' | 'PROCESSING' | 'SUCCEEDED' | 'FAILED'; result: GamePrisma.JsonValue | null; attempts: number; + acceptedGameTick: bigint | null; + acceptedClockRevision: bigint | null; + acceptedDeadlineGeneration: bigint | null; } type InputEventOutcome = @@ -85,6 +93,9 @@ const insertPendingIfAbsent = async ( actor_user_id, status, attempts, + accepted_game_tick, + accepted_clock_revision, + accepted_deadline_generation, created_at ) VALUES ( @@ -95,6 +106,9 @@ const insertPendingIfAbsent = async ( ${options.actorUserId}, 'PENDING'::"InputEventStatus", 0, + (SELECT clock_tick FROM world_state ORDER BY id ASC LIMIT 1), + (SELECT clock_revision FROM world_state ORDER BY id ASC LIMIT 1), + (SELECT deadline_generation FROM world_state ORDER BY id ASC LIMIT 1), CURRENT_TIMESTAMP AT TIME ZONE 'UTC' ) ON CONFLICT (request_id) DO NOTHING @@ -112,7 +126,10 @@ const lockInputEvent = async (db: DatabaseClient, requestId: string): Promise => { await db.inputEvent.update({ where: { requestId }, @@ -164,6 +182,9 @@ const claimInputEvent = async ( lockedBy: null, leaseUntil: null, processingAt: new Date(), + processingGameTick: row.acceptedGameTick, + processingClockRevision: row.acceptedClockRevision, + processingDeadlineGeneration: row.acceptedDeadlineGeneration, completedAt: null, }, }); @@ -184,6 +205,7 @@ const markUnexpectedFailure = async ( try { await db.$transaction(async (transaction) => { + await acquireGameSchemaAdvisoryXactLock(transaction, CLOCK_OPERATION_PERSISTENCE_LOCK); await insertPendingIfAbsent(transaction, options); const row = await lockInputEvent(transaction, options.requestId); const identityMatches = isMatchingIdentity(row, options) || canAdoptLegacyFailedPayload(row, options); @@ -233,6 +255,7 @@ export const executeInputEvent = async (options: { let outcome: InputEventOutcome; try { outcome = await db.$transaction(async (transaction) => { + await acquireGameSchemaAdvisoryXactLock(transaction, CLOCK_OPERATION_PERSISTENCE_LOCK); await insertPendingIfAbsent(transaction, { requestId, eventType, actorUserId, payloadIdentity }); const row = await lockInputEvent(transaction, requestId); const identityMatches = isMatchingIdentity(row, { eventType, actorUserId, payloadIdentity }); @@ -251,7 +274,7 @@ export const executeInputEvent = async (options: { throw new DuplicateInputEventError(requestId); } - await claimInputEvent(transaction, requestId, payloadIdentity); + await claimInputEvent(transaction, requestId, payloadIdentity, row); const savepointDb = transaction as SavepointDatabaseClient; await savepointDb.$executeRawUnsafe(`SAVEPOINT ${BUSINESS_SAVEPOINT}`); businessStarted = true; diff --git a/app/game-api/src/router/tournament/index.ts b/app/game-api/src/router/tournament/index.ts index 8e61d694..8e1c24d0 100644 --- a/app/game-api/src/router/tournament/index.ts +++ b/app/game-api/src/router/tournament/index.ts @@ -5,12 +5,14 @@ import { asRecord } from '@sammo-ts/common'; import type { TournamentType } from '@sammo-ts/logic'; import type { TournamentState } from '../../tournament/types.js'; -import { TournamentStore } from '../../tournament/store.js'; +import { TournamentStore, type TournamentClockContext } from '../../tournament/store.js'; import { buildTournamentKeys } from '../../tournament/keys.js'; import { assignManualApplicantGroup } from '../../tournament/workerHelpers.js'; import { accessAuthedProcedure, authedProcedure, router } from '../../trpc.js'; import { getMyGeneral } from '../shared/general.js'; import { loadCurrentGameTime } from '../../services/gameClock.js'; +import { ensureActiveRedisClockFence } from '../../services/redisClockFence.js'; +import { loadClockAdminStatus } from '../../services/clockReadiness.js'; const hasAdminRole = (roles: string[], profileName: string): boolean => { if (roles.includes('superuser') || roles.includes('admin') || roles.includes('admin.superuser')) { @@ -44,6 +46,32 @@ const adminProcedure = authedProcedure.use(({ ctx, next }) => { return next(); }); +const withTournamentClockMutation = async ( + ctx: { + db: Parameters[0]; + redis: Parameters[0]; + profile: { name: string }; + }, + store: TournamentStore, + operation: () => Promise +): Promise => { + const gameTime = await loadCurrentGameTime(ctx.db); + const fence = await ensureActiveRedisClockFence(ctx.redis, ctx.profile.name, gameTime); + if (!fence) { + throw new TRPCError({ + code: 'PRECONDITION_FAILED', + message: 'Clock reconciliation is incomplete; tournament mutation is disabled.', + }); + } + const clockContext: TournamentClockContext = { + phase: 'RUNNING', + revision: fence.revision, + deadlineGeneration: fence.generation, + dateToTick: gameTime.dateToTick, + }; + return store.withClockContext(clockContext, () => store.withMutationLock(operation)); +}; + const zTournamentState = z.object({ stage: z.number().int().min(0), phase: z.number().int().min(0), @@ -143,7 +171,10 @@ export const tournamentRouter = router({ const store = new TournamentStore(ctx.redis, buildTournamentKeys(ctx.profile.name)); return store.getState(); }), - getAdminStatus: adminProcedure.query(async () => ({ ok: true })), + getAdminStatus: adminProcedure.query(async ({ ctx }) => ({ + ok: true, + clock: await loadClockAdminStatus(ctx.db), + })), getSnapshot: accessAuthedProcedure.query(async ({ ctx }) => { await getMyGeneral(ctx); const store = new TournamentStore(ctx.redis, buildTournamentKeys(ctx.profile.name)); @@ -271,7 +302,7 @@ export const tournamentRouter = router({ }), setState: adminProcedure.input(zTournamentState).mutation(async ({ ctx, input }) => { const store = new TournamentStore(ctx.redis, buildTournamentKeys(ctx.profile.name)); - return store.withMutationLock(async () => { + return withTournamentClockMutation(ctx, store, async () => { await store.setState({ ...input, type: input.type as TournamentType, @@ -281,7 +312,7 @@ export const tournamentRouter = router({ }), patchState: adminProcedure.input(zTournamentState.partial()).mutation(async ({ ctx, input }) => { const store = new TournamentStore(ctx.redis, buildTournamentKeys(ctx.profile.name)); - return store.withMutationLock(async () => { + return withTournamentClockMutation(ctx, store, async () => { const current = await store.getState(); if (!current) { throw new TRPCError({ code: 'NOT_FOUND', message: 'Tournament state not found.' }); @@ -297,21 +328,21 @@ export const tournamentRouter = router({ }), setParticipants: adminProcedure.input(z.array(zParticipant)).mutation(async ({ ctx, input }) => { const store = new TournamentStore(ctx.redis, buildTournamentKeys(ctx.profile.name)); - return store.withMutationLock(async () => { + return withTournamentClockMutation(ctx, store, async () => { await store.setParticipants(input); return { ok: true, count: input.length }; }); }), setMatches: adminProcedure.input(z.array(zMatch)).mutation(async ({ ctx, input }) => { const store = new TournamentStore(ctx.redis, buildTournamentKeys(ctx.profile.name)); - return store.withMutationLock(async () => { + return withTournamentClockMutation(ctx, store, async () => { await store.setMatches(input); return { ok: true, count: input.length }; }); }), setBettingEntries: adminProcedure.input(z.array(zBetEntry)).mutation(async ({ ctx, input }) => { const store = new TournamentStore(ctx.redis, buildTournamentKeys(ctx.profile.name)); - return store.withMutationLock(async () => { + return withTournamentClockMutation(ctx, store, async () => { await store.setBettingEntries(input); return { ok: true, count: input.length }; }); @@ -347,7 +378,7 @@ export const tournamentRouter = router({ }; }); - return store.withMutationLock(async () => { + return withTournamentClockMutation(ctx, store, async () => { await store.setParticipants(participants); return { ok: true, count: participants.length }; }); @@ -394,7 +425,7 @@ export const tournamentRouter = router({ join: authedProcedure.mutation(async ({ ctx }) => { const general = await getMyGeneral(ctx); const store = new TournamentStore(ctx.redis, buildTournamentKeys(ctx.profile.name)); - return store.withMutationLock(async () => { + return withTournamentClockMutation(ctx, store, async () => { const state = await store.getState(); if (!state || state.stage !== 1 || state.participantsLockedAt) { throw new TRPCError({ code: 'BAD_REQUEST', message: '참가 신청 기간이 아닙니다.' }); @@ -459,7 +490,7 @@ export const tournamentRouter = router({ }), cancel: adminProcedure.mutation(async ({ ctx }) => { const store = new TournamentStore(ctx.redis, buildTournamentKeys(ctx.profile.name)); - return store.withMutationLock(async () => { + return withTournamentClockMutation(ctx, store, async () => { const state = await store.getState(); if (!state) { throw new TRPCError({ code: 'NOT_FOUND', message: 'Tournament state not found.' }); @@ -523,7 +554,7 @@ export const tournamentRouter = router({ .mutation(async ({ ctx, input }) => { const general = await getMyGeneral(ctx); const store = new TournamentStore(ctx.redis, buildTournamentKeys(ctx.profile.name)); - return store.withMutationLock(async () => { + return withTournamentClockMutation(ctx, store, async () => { const state = await store.getState(); if (!state || state.stage !== 6) { throw new TRPCError({ code: 'BAD_REQUEST', message: '베팅 기간이 아닙니다.' }); diff --git a/app/game-api/src/server.ts b/app/game-api/src/server.ts index da0517a0..2dc4550e 100644 --- a/app/game-api/src/server.ts +++ b/app/game-api/src/server.ts @@ -50,6 +50,7 @@ import { ReadModelOutboxWorker } from './realtime/outboxWorker.js'; import { DeferredGeneralAccessWorker } from './services/deferredGeneralAccess.js'; import { WebPushOutboxWorker } from './services/webPushOutboxWorker.js'; import { scopeHttpIdempotencyKey } from './requestId.js'; +import { loadClockReadiness } from './services/clockReadiness.js'; const extractBearerToken = (value: string | string[] | undefined): string | null => { if (!value) { @@ -420,12 +421,17 @@ export const createGameApiServer = async () => { request.raw.on('aborted', close); }); - app.get('/healthz', async () => ({ - ok: true, - profile: config.profileName, - postgresPool: postgres.getPoolStats(), - accountIconReconciliation: accountIconResetReconciler.getHealth(), - })); + app.get('/healthz', async (_request, reply) => { + const clock = await loadClockReadiness(postgres.prisma); + if (!clock.reconciliationComplete) reply.code(503); + return { + ok: clock.reconciliationComplete, + profile: config.profileName, + postgresPool: postgres.getPoolStats(), + accountIconReconciliation: accountIconResetReconciler.getHealth(), + clock, + }; + }); try { await realtimeHub.start(); diff --git a/app/game-api/src/services/clockReadiness.ts b/app/game-api/src/services/clockReadiness.ts new file mode 100644 index 00000000..b457245a --- /dev/null +++ b/app/game-api/src/services/clockReadiness.ts @@ -0,0 +1,92 @@ +import { parseGameClockPhase } from '@sammo-ts/common'; + +import type { DatabaseClient } from '../context.js'; + +const safeInteger = (value: bigint, label: string): number => { + const result = Number(value); + if (!Number.isSafeInteger(result)) throw new Error(`${label} is outside the safe integer range.`); + return result; +}; + +export const loadClockReadiness = async (db: DatabaseClient) => { + if (!db.clockProjectionOutbox) { + return { + reconciliationComplete: false, + gameplayEnabled: false, + phase: null, + revision: null, + deadlineGeneration: null, + incompleteOutboxCount: null, + }; + } + const [world, incompleteOutboxCount] = await Promise.all([ + db.worldState.findFirst({ + orderBy: { id: 'asc' }, + select: { clockPhase: true, clockRevision: true, deadlineGeneration: true }, + }), + db.clockProjectionOutbox.count({ where: { status: { not: 'APPLIED' } } }), + ]); + if (!world) { + return { + reconciliationComplete: false, + gameplayEnabled: false, + phase: null, + revision: null, + deadlineGeneration: null, + incompleteOutboxCount, + }; + } + const phase = parseGameClockPhase(world.clockPhase); + return { + reconciliationComplete: phase !== 'RECONCILING' && incompleteOutboxCount === 0, + gameplayEnabled: phase === 'RUNNING' || phase === 'MANUAL', + phase, + revision: safeInteger(world.clockRevision, 'clock revision'), + deadlineGeneration: safeInteger(world.deadlineGeneration, 'deadline generation'), + incompleteOutboxCount, + }; +}; + +export const loadClockAdminStatus = async (db: DatabaseClient) => { + const readiness = await loadClockReadiness(db); + if (!db.clockSuspension) { + return { ...readiness, latestReconciliation: null }; + } + const latest = await db.clockSuspension.findFirst({ + orderBy: { createdAt: 'desc' }, + include: { + participants: { orderBy: { participantKey: 'asc' } }, + projectionOutbox: { orderBy: { id: 'asc' } }, + }, + }); + if (!latest) return { ...readiness, latestReconciliation: null }; + return { + ...readiness, + latestReconciliation: { + id: latest.id, + source: latest.source, + policy: latest.policy, + status: latest.status, + sourceRevision: safeInteger(latest.sourceRevision, 'source revision'), + targetRevision: safeInteger(latest.targetRevision, 'target revision'), + cutTick: safeInteger(latest.cutTick, 'cut tick'), + alignedTick: latest.alignedTick === null ? null : safeInteger(latest.alignedTick, 'aligned tick'), + participantChecksumBefore: latest.participantChecksumBefore, + participantChecksumAfter: latest.participantChecksumAfter, + participants: latest.participants.map((participant) => ({ + key: participant.participantKey, + policy: participant.policy, + beforeChecksum: participant.beforeChecksum, + afterChecksum: participant.afterChecksum, + affectedCount: participant.affectedCount, + })), + outbox: latest.projectionOutbox.map((entry) => ({ + id: entry.id.toString(), + targetRevision: safeInteger(entry.targetRevision, 'outbox target revision'), + status: entry.status, + attempts: entry.attempts, + lastError: entry.lastError, + })), + }, + }; +}; diff --git a/app/game-api/src/services/redisClockFence.ts b/app/game-api/src/services/redisClockFence.ts new file mode 100644 index 00000000..783bb4a4 --- /dev/null +++ b/app/game-api/src/services/redisClockFence.ts @@ -0,0 +1,55 @@ +import type { CurrentGameTime } from './gameClock.js'; + +interface ClockFenceRedis { + eval(script: string, options: { keys: string[]; arguments: string[] }): Promise; +} + +const BOOTSTRAP_CLOCK_FENCE_SCRIPT = ` +local revision = redis.call('GET', KEYS[1]) +local generation = redis.call('GET', KEYS[2]) +local phase = redis.call('GET', KEYS[3]) +if not revision and not generation and not phase then + redis.call('SET', KEYS[1], ARGV[1]) + redis.call('SET', KEYS[2], ARGV[2]) + redis.call('SET', KEYS[3], ARGV[3]) + return 1 +end +if revision == ARGV[1] and generation == ARGV[2] and phase == ARGV[3] then + return 2 +end +return 0 +`; + +export interface ActiveRedisClockFence { + activeRevisionKey: string; + deadlineGenerationKey: string; + phaseKey: string; + revision: number; + generation: number; +} + +export const ensureActiveRedisClockFence = async ( + redis: ClockFenceRedis, + profileName: string, + gameTime: CurrentGameTime +): Promise => { + if ( + gameTime.phase !== 'RUNNING' || + !Number.isSafeInteger(gameTime.revision) || + !Number.isSafeInteger(gameTime.deadlineGeneration) + ) { + return null; + } + const fence: ActiveRedisClockFence = { + activeRevisionKey: `sammo:${profileName}:clock:active-revision`, + deadlineGenerationKey: `sammo:${profileName}:clock:deadline-generation`, + phaseKey: `sammo:${profileName}:clock:phase`, + revision: gameTime.revision!, + generation: gameTime.deadlineGeneration!, + }; + const result = await redis.eval(BOOTSTRAP_CLOCK_FENCE_SCRIPT, { + keys: [fence.activeRevisionKey, fence.deadlineGenerationKey, fence.phaseKey], + arguments: [String(fence.revision), String(fence.generation), 'RUNNING'], + }); + return Number(result) === 1 || Number(result) === 2 ? fence : null; +}; diff --git a/app/game-api/src/tournament/keys.ts b/app/game-api/src/tournament/keys.ts index e560483d..0938a773 100644 --- a/app/game-api/src/tournament/keys.ts +++ b/app/game-api/src/tournament/keys.ts @@ -8,6 +8,9 @@ export interface TournamentKeys { sourceRevisionKey: string; sourceRevisionChannel: string; realtimeEventChannel: string; + activeClockRevisionKey: string; + deadlineGenerationKey: string; + clockPhaseKey: string; } export const buildTournamentKeys = (profileName: string): TournamentKeys => ({ @@ -18,4 +21,7 @@ export const buildTournamentKeys = (profileName: string): TournamentKeys => ({ sourceRevisionKey: `sammo:${profileName}:tournament:source-revision`, sourceRevisionChannel: `sammo:${profileName}:tournament:source-changed`, realtimeEventChannel: buildGameEventChannel(profileName), + activeClockRevisionKey: `sammo:${profileName}:clock:active-revision`, + deadlineGenerationKey: `sammo:${profileName}:clock:deadline-generation`, + clockPhaseKey: `sammo:${profileName}:clock:phase`, }); diff --git a/app/game-api/src/tournament/store.ts b/app/game-api/src/tournament/store.ts index ba7d656d..59abeea7 100644 --- a/app/game-api/src/tournament/store.ts +++ b/app/game-api/src/tournament/store.ts @@ -1,5 +1,5 @@ import { randomUUID } from 'node:crypto'; -import { parseTournamentSourceRevision, writeTournamentProjection } from '@sammo-ts/common'; +import { parseTournamentSourceRevision, writeTournamentProjection, type TournamentClockFence } from '@sammo-ts/common'; import { z } from 'zod'; import type { TournamentKeys } from './keys.js'; @@ -37,8 +37,12 @@ const zTournamentState = z openMonth: z.number().int(), termSeconds: z.number(), nextAt: z.string(), + nextTick: z.number().int().safe().optional(), + clockRevision: z.number().int().positive().safe().optional(), + deadlineGeneration: z.number().int().positive().safe().optional(), bettingId: z.number().int().optional(), bettingCloseAt: z.string().optional(), + bettingCloseTick: z.number().int().safe().optional(), winnerId: z.number().int().optional(), bettingSettled: z.boolean().optional(), rewardSettled: z.boolean().optional(), @@ -131,12 +135,62 @@ const parseProjection = (raw: string | null, key: string, schema: z.ZodType { + const date = new Date(value); + if (Number.isNaN(date.getTime())) { + throw new Error(`Tournament ${field} is not a valid instant.`); + } + const tick = context.dateToTick(date); + if (tick === null || !Number.isSafeInteger(tick)) { + throw new Error(`Tournament ${field} cannot be represented in the active game clock.`); + } + return tick; +}; + +export const stampTournamentClock = (state: TournamentState, context: TournamentClockContext): TournamentState => ({ + ...state, + nextTick: parseDeadlineTick(state.nextAt, context, 'nextAt'), + clockRevision: context.revision, + deadlineGeneration: context.deadlineGeneration, + ...(state.bettingCloseAt + ? { bettingCloseTick: parseDeadlineTick(state.bettingCloseAt, context, 'bettingCloseAt') } + : { bettingCloseTick: undefined }), +}); + +const toClockFence = (keys: TournamentKeys, context: TournamentClockContext): TournamentClockFence => ({ + activeRevisionKey: keys.activeClockRevisionKey, + deadlineGenerationKey: keys.deadlineGenerationKey, + phaseKey: keys.clockPhaseKey, + revision: context.revision, + deadlineGeneration: context.deadlineGeneration, + phase: context.phase, +}); + export class TournamentStore { + private clockContext: TournamentClockContext | null = null; + constructor( private readonly redis: RedisClientLike, private readonly keys: TournamentKeys ) {} + async withClockContext(context: TournamentClockContext, operation: () => Promise): Promise { + const previous = this.clockContext; + this.clockContext = context; + try { + return await operation(); + } finally { + this.clockContext = previous; + } + } + async withMutationLock(operation: () => Promise, timeoutMs = 2_000): Promise { if (!this.redis.del) { return operation(); @@ -174,11 +228,15 @@ export class TournamentStore { } private async writeWithSourceRevision(key: string, value: unknown): Promise { - return writeTournamentProjection(this.redis, this.keys, [{ key, value }]); + const fence = this.clockContext ? toClockFence(this.keys, this.clockContext) : undefined; + return writeTournamentProjection(this.redis, this.keys, [{ key, value }], fence); } async setState(state: TournamentState): Promise { - return this.writeWithSourceRevision(this.keys.stateKey, state); + return this.writeWithSourceRevision( + this.keys.stateKey, + this.clockContext ? stampTournamentClock(state, this.clockContext) : state + ); } async getParticipants(): Promise { diff --git a/app/game-api/src/tournament/types.ts b/app/game-api/src/tournament/types.ts index 3efde50b..0511bb3c 100644 --- a/app/game-api/src/tournament/types.ts +++ b/app/game-api/src/tournament/types.ts @@ -9,8 +9,12 @@ export interface TournamentState { openMonth: number; termSeconds: number; nextAt: string; + nextTick?: number; + clockRevision?: number; + deadlineGeneration?: number; bettingId?: number; bettingCloseAt?: string; + bettingCloseTick?: number; winnerId?: number; bettingSettled?: boolean; rewardSettled?: boolean; diff --git a/app/game-api/src/tournament/worker.ts b/app/game-api/src/tournament/worker.ts index 5c41f890..2ce052be 100644 --- a/app/game-api/src/tournament/worker.ts +++ b/app/game-api/src/tournament/worker.ts @@ -13,9 +13,10 @@ import { DatabaseTurnDaemonTransport } from '../daemon/databaseTransport.js'; import { createBestEffortResourceCloser } from '../services/bestEffortResourceCloser.js'; import { loadCurrentGameTime } from '../services/gameClock.js'; import { createPollingWorkerControl, waitForWorkerPoll } from '../services/pollingWorkerLifecycle.js'; +import { ensureActiveRedisClockFence } from '../services/redisClockFence.js'; import type { TurnDaemonTransport } from '../daemon/transport.js'; import { buildTournamentKeys } from './keys.js'; -import { TournamentStore } from './store.js'; +import { TournamentStore, stampTournamentClock, type TournamentClockContext } from './store.js'; import type { TournamentMatchEntry, TournamentState } from './types.js'; import { applyGroupMatch, @@ -595,41 +596,63 @@ export const processTournamentTick = async (options: { prisma: GamePrismaClient; daemonTransport: TurnDaemonTransport; now?: () => number; + clockContext?: TournamentClockContext; }): Promise => { const { store, prisma, daemonTransport } = options; const now = options.now ?? Date.now; let processedState: TournamentState | null = null; - await store.withMutationLock(async () => { - const state = await store.getState(); - if (!state || (!state.auto && !needsSettlement(state))) { - return; - } - const nextAt = new Date(state.nextAt).getTime(); - if (state.auto && Number.isFinite(nextAt) && nextAt > now()) { - return; - } + const processWithLock = async (): Promise => + store.withMutationLock(async () => { + const state = await store.getState(); + if (!state || (!state.auto && !needsSettlement(state))) { + return; + } + if ( + options.clockContext && + (state.clockRevision !== options.clockContext.revision || + state.deadlineGeneration !== options.clockContext.deadlineGeneration) + ) { + throw new Error('Tournament state does not match the active clock revision.'); + } + const nextAt = new Date(state.nextAt).getTime(); + const notDue = options.clockContext + ? !Number.isSafeInteger(state.nextTick) || + state.nextTick! > options.clockContext.dateToTick(new Date(now()))! + : Number.isFinite(nextAt) && nextAt > now(); + if (state.auto && notDue) { + if (options.clockContext && !Number.isSafeInteger(state.nextTick)) { + throw new Error('Active tournament nextAt lacks the authoritative nextTick dual-write.'); + } + return; + } - if (needsSettlement(state)) { - processedState = (await settleTournamentOutcome({ store, daemonTransport, state })) ?? state; - return; - } + if (needsSettlement(state)) { + processedState = (await settleTournamentOutcome({ store, daemonTransport, state })) ?? state; + return; + } - const worldState = await prisma.worldState.findFirst(); - const baseSeed = (worldState?.meta as Record | null)?.hiddenSeed ?? 'tournament'; - let nextState = state; - if (isBattleStage(state.stage)) { - nextState = await applyBattle(store, state, String(baseSeed), daemonTransport); - } else if (isPreBattleStage(state.stage)) { - nextState = await applyPreBattleStage(store, prisma, state, String(baseSeed), daemonTransport, now); - } - processedState = - (await settleTournamentOutcome({ - store, - daemonTransport, - state: nextState, - })) ?? nextState; - }); + const worldState = await prisma.worldState.findFirst(); + const baseSeed = (worldState?.meta as Record | null)?.hiddenSeed ?? 'tournament'; + let nextState = state; + if (isBattleStage(state.stage)) { + nextState = await applyBattle(store, state, String(baseSeed), daemonTransport); + } else if (isPreBattleStage(state.stage)) { + nextState = await applyPreBattleStage(store, prisma, state, String(baseSeed), daemonTransport, now); + } + processedState = + (await settleTournamentOutcome({ + store, + daemonTransport, + state: nextState, + })) ?? nextState; + }); + + if (options.clockContext) { + await store.withClockContext(options.clockContext, processWithLock); + } else { + await processWithLock(); + } return processedState; }; @@ -656,16 +679,60 @@ export const runTournamentWorker = async (options: TournamentWorkerOptions = {}) try { while (!control.signal.aborted) { - const state = await store.getState(); + let state = await store.getState(); if (!state || (!state.auto && !needsSettlement(state))) { await waitForWorkerPoll(control.signal, config.tournamentPollMs); continue; } + const gameTime = await loadCurrentGameTime(postgres.prisma); + if (gameTime.phase && gameTime.phase !== 'RUNNING') { + await waitForWorkerPoll(control.signal, config.tournamentPollMs); + continue; + } + const clockFence = gameTime.phase + ? await ensureActiveRedisClockFence(redis.client, config.profileName, gameTime) + : null; + if (gameTime.phase && !clockFence) { + await waitForWorkerPoll(control.signal, config.tournamentPollMs); + continue; + } + const clockContext: TournamentClockContext | undefined = clockFence + ? { + phase: 'RUNNING', + revision: clockFence.revision, + deadlineGeneration: clockFence.generation, + dateToTick: gameTime.dateToTick, + } + : undefined; + if ( + clockContext && + (!Number.isSafeInteger(state.nextTick) || + state.clockRevision === undefined || + state.deadlineGeneration === undefined) + ) { + state = stampTournamentClock(state, clockContext); + await store.withClockContext(clockContext, () => store.setState(state!)); + } + if ( + clockContext && + (state.clockRevision !== clockContext.revision || + state.deadlineGeneration !== clockContext.deadlineGeneration) + ) { + await waitForWorkerPoll(control.signal, config.tournamentPollMs); + continue; + } + const nextAt = new Date(state.nextAt).getTime(); - const gameNow = (await loadCurrentGameTime(postgres.prisma)).now.getTime(); - if (state.auto && Number.isFinite(nextAt) && nextAt > gameNow) { - await waitForWorkerPoll(control.signal, Math.min(config.tournamentPollMs, nextAt - gameNow)); + const gameNow = gameTime.now.getTime(); + const notDue = clockContext + ? state.nextTick! > gameTime.tick! + : Number.isFinite(nextAt) && nextAt > gameNow; + if (state.auto && notDue) { + await waitForWorkerPoll( + control.signal, + Math.min(config.tournamentPollMs, Math.max(1, nextAt - gameNow)) + ); continue; } @@ -675,6 +742,7 @@ export const runTournamentWorker = async (options: TournamentWorkerOptions = {}) prisma: postgres.prisma, daemonTransport, now: () => gameNow, + clockContext, }); } catch (error) { const message = error instanceof Error ? error.message : 'Unknown error'; @@ -700,7 +768,11 @@ export const runTournamentWorker = async (options: TournamentWorkerOptions = {}) lastError: message, lastErrorAt: failedAt, }; - await store.setState(nextState); + if (clockContext) { + await store.withClockContext(clockContext, () => store.setState(nextState)); + } else { + await store.setState(nextState); + } } await waitForWorkerPoll(control.signal, config.tournamentPollMs); diff --git a/app/game-api/test/auctionWorker.test.ts b/app/game-api/test/auctionWorker.test.ts index a3ae0b86..b55ab267 100644 --- a/app/game-api/test/auctionWorker.test.ts +++ b/app/game-api/test/auctionWorker.test.ts @@ -2,7 +2,7 @@ import { describe, expect, it, vi } from 'vitest'; import type { GamePrismaClient } from '@sammo-ts/infra'; -import { processDueAuctionId, reconcilePendingAuctionTimers } from '../src/auction/worker.js'; +import { popDueAuctionIds, processDueAuctionId, reconcilePendingAuctionTimers } from '../src/auction/worker.js'; import { resolveAuctionSeedScore } from '../src/auction/scheduler.js'; const buildRedis = () => ({ @@ -55,6 +55,38 @@ const buildDb = (options: { }; describe('auction worker clock-shift race', () => { + it('uses one Redis script for revision, generation, phase, due-read, and removal', async () => { + const redis = { ...buildRedis(), eval: vi.fn(async () => ['7', '9']) }; + await expect( + popDueAuctionIds(redis, 'auction-timer', 123_456, 100, { + activeRevisionKey: 'clock-revision', + deadlineGenerationKey: 'deadline-generation', + phaseKey: 'clock-phase', + revision: 4, + generation: 8, + }) + ).resolves.toEqual(['7', '9']); + expect(redis.eval).toHaveBeenCalledWith(expect.stringContaining('ZRANGEBYSCORE'), { + keys: ['auction-timer', 'clock-revision', 'deadline-generation', 'clock-phase'], + arguments: ['4', '8', '123456', '100'], + }); + expect(redis.zRangeByScore).not.toHaveBeenCalled(); + expect(redis.zRem).not.toHaveBeenCalled(); + }); + + it('returns no due member when the atomic Redis clock fence rejects the pop', async () => { + const redis = { ...buildRedis(), eval: vi.fn(async () => ['__CLOCK_FENCE__']) }; + await expect( + popDueAuctionIds(redis, 'auction-timer', 123_456, 100, { + activeRevisionKey: 'clock-revision', + deadlineGenerationKey: 'deadline-generation', + phaseKey: 'clock-phase', + revision: 4, + generation: 8, + }) + ).resolves.toEqual([]); + }); + it('seeds OPEN at its deadline but retries FINALIZING at the current logical tick', () => { const now = new Date('2026-07-30T12:00:00.000Z'); const time = { @@ -292,6 +324,7 @@ describe('auction worker clock-shift race', () => { requestId, target: 'ENGINE', eventType: 'auctionFinalize', + acceptedGameTick: 72_000_000n, payload: { type: 'auctionFinalize', requestId, diff --git a/app/game-api/test/clockReadiness.test.ts b/app/game-api/test/clockReadiness.test.ts new file mode 100644 index 00000000..e910b5f8 --- /dev/null +++ b/app/game-api/test/clockReadiness.test.ts @@ -0,0 +1,87 @@ +import { describe, expect, it, vi } from 'vitest'; + +import type { DatabaseClient } from '../src/context.js'; +import { loadClockAdminStatus, loadClockReadiness } from '../src/services/clockReadiness.js'; + +describe('clock reconciliation readiness', () => { + it('fails closed when the reconciliation schema is not available', async () => { + const db = {} as DatabaseClient; + await expect(loadClockReadiness(db)).resolves.toEqual({ + reconciliationComplete: false, + gameplayEnabled: false, + phase: null, + revision: null, + deadlineGeneration: null, + incompleteOutboxCount: null, + }); + }); + + it('blocks readiness for RECONCILING or incomplete outbox state', async () => { + const db = { + worldState: { + findFirst: vi.fn(async () => ({ + clockPhase: 'RECONCILING', + clockRevision: 9n, + deadlineGeneration: 4n, + })), + }, + clockProjectionOutbox: { count: vi.fn(async () => 1) }, + } as unknown as DatabaseClient; + await expect(loadClockReadiness(db)).resolves.toMatchObject({ + reconciliationComplete: false, + gameplayEnabled: false, + phase: 'RECONCILING', + revision: 9, + deadlineGeneration: 4, + incompleteOutboxCount: 1, + }); + }); + + it('exposes participant checksums and incomplete outbox detail to admins', async () => { + const db = { + worldState: { + findFirst: vi.fn(async () => ({ + clockPhase: 'RUNNING', + clockRevision: 3n, + deadlineGeneration: 2n, + })), + }, + clockProjectionOutbox: { count: vi.fn(async () => 0) }, + clockSuspension: { + findFirst: vi.fn(async () => ({ + id: 'maintenance-1', + source: 'MAINTENANCE', + policy: 'EXACT', + status: 'APPLIED', + sourceRevision: 2n, + targetRevision: 3n, + cutTick: 100n, + alignedTick: 130n, + participantChecksumBefore: 'before-all', + participantChecksumAfter: 'after-all', + participants: [ + { + participantKey: 'general-turn', + policy: 'SHIFT', + beforeChecksum: 'before', + afterChecksum: 'after', + affectedCount: 2, + }, + ], + projectionOutbox: [{ id: 8n, targetRevision: 3n, status: 'APPLIED', attempts: 1, lastError: null }], + })), + }, + } as unknown as DatabaseClient; + + await expect(loadClockAdminStatus(db)).resolves.toMatchObject({ + reconciliationComplete: true, + latestReconciliation: { + id: 'maintenance-1', + participantChecksumBefore: 'before-all', + participantChecksumAfter: 'after-all', + participants: [{ key: 'general-turn', policy: 'SHIFT', affectedCount: 2 }], + outbox: [{ id: '8', status: 'APPLIED' }], + }, + }); + }); +}); diff --git a/app/game-api/test/inputEventBoundary.integration.test.ts b/app/game-api/test/inputEventBoundary.integration.test.ts index d1dfb8cf..205d782e 100644 --- a/app/game-api/test/inputEventBoundary.integration.test.ts +++ b/app/game-api/test/inputEventBoundary.integration.test.ts @@ -511,6 +511,10 @@ integration('API input event boundary', () => { it('reuses the same engine child event but rejects a changed retry payload', async () => { const transport = new DatabaseTurnDaemonTransport(db, 100); const requestId = 'integration:api:engine-child'; + const worldClock = await db.worldState.findFirst({ + orderBy: { id: 'asc' }, + select: { clockRevision: true, deadlineGeneration: true }, + }); const acceptedWindowStart = Date.now(); await transport.sendCommand({ type: 'vacation', requestId, userId: 'user-7', generalId: 7 }); const acceptedWindowEnd = Date.now(); @@ -518,6 +522,9 @@ integration('API input event boundary', () => { expect(event.actorUserId).toBe('user-7'); expect(event.createdAt.getTime()).toBeGreaterThanOrEqual(acceptedWindowStart); expect(event.createdAt.getTime()).toBeLessThanOrEqual(acceptedWindowEnd); + expect(event.acceptedGameTick).not.toBeNull(); + expect(event.acceptedClockRevision).toBe(worldClock?.clockRevision ?? null); + expect(event.acceptedDeadlineGeneration).toBe(worldClock?.deadlineGeneration ?? null); await expect( transport.sendCommand({ type: 'vacation', requestId, userId: 'user-7', generalId: 7 }) ).resolves.toBe(requestId); diff --git a/app/game-api/test/inputEventJournal.test.ts b/app/game-api/test/inputEventJournal.test.ts index 4046fb47..631db1fd 100644 --- a/app/game-api/test/inputEventJournal.test.ts +++ b/app/game-api/test/inputEventJournal.test.ts @@ -28,6 +28,9 @@ const createContext = (payload: unknown = {}) => { status: 'PENDING', result: null, attempts: 0, + acceptedGameTick: 100n, + acceptedClockRevision: 3n, + acceptedDeadlineGeneration: 2n, }, ]; } @@ -36,8 +39,8 @@ const createContext = (payload: unknown = {}) => { }); const transaction = { $queryRaw: queryRaw, - $executeRaw: vi.fn(async () => { - order.push('accepted'); + $executeRaw: vi.fn(async (query: { sql?: string }) => { + order.push(query.sql?.includes('pg_advisory_xact_lock') ? 'clock-fence' : 'accepted'); return 1; }), $executeRawUnsafe: vi.fn(async (statement: string) => { @@ -88,6 +91,7 @@ describe('API input-event change journal boundary', () => { expect(fixture.order).toEqual([ 'transaction-begin', + 'clock-fence', 'accepted', 'locked', 'processing', @@ -112,6 +116,7 @@ describe('API input-event change journal boundary', () => { expect(fixture.order).toEqual([ 'transaction-begin', + 'clock-fence', 'accepted', 'locked', 'processing', diff --git a/app/game-api/test/tournamentRouter.test.ts b/app/game-api/test/tournamentRouter.test.ts index b4635eec..6947cb6e 100644 --- a/app/game-api/test/tournamentRouter.test.ts +++ b/app/game-api/test/tournamentRouter.test.ts @@ -30,11 +30,31 @@ class MemoryRedis { } async eval(_script: string, options: { keys: string[]; arguments: string[] }): Promise { - const [valueKey, revisionKey] = options.keys; - const [value] = options.arguments; + if (options.keys.length === 3 && options.keys[0]?.endsWith(':clock:active-revision')) { + const current = options.keys.map((key) => this.values.get(key)); + if (current.every((value) => value === undefined)) { + options.keys.forEach((key, index) => this.values.set(key, options.arguments[index]!)); + return '1'; + } + return current.every((value, index) => value === options.arguments[index]) ? '2' : '0'; + } + const fenced = options.keys.at(-1)?.endsWith(':clock:phase') === true; + const writeCount = options.keys.length - (fenced ? 4 : 1); + if (fenced) { + const clockKeys = options.keys.slice(-3); + const expected = options.arguments.slice(-3); + if (!clockKeys.every((key, index) => this.values.get(key) === expected[index])) { + return '__CLOCK_FENCE__'; + } + } + const valueKey = options.keys[0]; + const revisionKey = options.keys[writeCount]; + const value = options.arguments[0]; if (!valueKey || !revisionKey || value === undefined) throw new Error('invalid eval arguments'); const revision = Number(this.values.get(revisionKey) ?? '0') + 1; - this.values.set(valueKey, value); + for (let index = 0; index < writeCount; index += 1) { + this.values.set(options.keys[index]!, options.arguments[index]!); + } this.values.set(revisionKey, String(revision)); return String(revision); } @@ -144,6 +164,14 @@ const buildContext = (options: { }, worldState: { findFirst: async () => ({ + clockBaseTime: new Date('2026-01-01T00:00:00.000Z'), + clockTick: 0n, + clockMode: 'realtime', + clockWallAnchor: new Date('2026-01-01T00:00:00.000Z'), + clockPhase: 'RUNNING', + clockRevision: 1n, + deadlineGeneration: 1n, + tickSeconds: 60, config: { const: { develCost: options.develCost ?? 200 } }, ...(options.currentDevelCost === undefined ? {} : { meta: { develcost: options.currentDevelCost } }), }), @@ -394,7 +422,10 @@ describe('tournament router permissions and mutations', () => { roles: ['admin.tournament:che:default'], }) ); - await expect(adminCaller.tournament.getAdminStatus()).resolves.toEqual({ ok: true }); + await expect(adminCaller.tournament.getAdminStatus()).resolves.toMatchObject({ + ok: true, + clock: { reconciliationComplete: false, latestReconciliation: null }, + }); }); it('applies the admin role boundary to every tournament mutation', async () => { @@ -479,9 +510,7 @@ describe('tournament router permissions and mutations', () => { ]) ).resolves.toEqual({ ok: true, count: 1 }); await expect( - caller.tournament.setBettingEntries([ - { generalId: general.id, targetId: rival.id, amount: 100 }, - ]) + caller.tournament.setBettingEntries([{ generalId: general.id, targetId: rival.id, amount: 100 }]) ).resolves.toEqual({ ok: true, count: 1 }); await expect(caller.tournament.seedParticipants({ generalIds: [general.id, rival.id] })).resolves.toEqual({ ok: true, diff --git a/app/game-api/test/tournamentStoreRevision.integration.test.ts b/app/game-api/test/tournamentStoreRevision.integration.test.ts index 5acf6dd6..8ecb1448 100644 --- a/app/game-api/test/tournamentStoreRevision.integration.test.ts +++ b/app/game-api/test/tournamentStoreRevision.integration.test.ts @@ -41,6 +41,9 @@ integration('TournamentStore Redis source revision', () => { keys.matchesKey, keys.bettingKey, keys.sourceRevisionKey, + keys.activeClockRevisionKey, + keys.deadlineGenerationKey, + keys.clockPhaseKey, ]); if (subscriber) { await subscriber.client.unsubscribe(keys.sourceRevisionChannel); @@ -124,4 +127,48 @@ integration('TournamentStore Redis source revision', () => { }), ]); }); + + it('dual-writes deadline ticks and rejects a Redis clock revision race atomically', async () => { + const store = new TournamentStore(connector.client, keys); + await Promise.all([ + connector.client.set(keys.activeClockRevisionKey, '7'), + connector.client.set(keys.deadlineGenerationKey, '3'), + connector.client.set(keys.clockPhaseKey, 'RUNNING'), + ]); + const clockContext = { + phase: 'RUNNING' as const, + revision: 7, + deadlineGeneration: 3, + dateToTick: (date: Date) => Math.trunc(date.getTime() / 1_000), + }; + const nextAt = '2026-09-03T10:00:00.000Z'; + await store.withClockContext(clockContext, () => + store.setState({ + stage: 6, + phase: 0, + type: 0, + auto: true, + openYear: 200, + openMonth: 1, + termSeconds: 10, + nextAt, + bettingCloseAt: '2026-09-03T09:59:50.000Z', + }) + ); + await expect(store.getState()).resolves.toMatchObject({ + nextTick: Math.trunc(new Date(nextAt).getTime() / 1_000), + bettingCloseTick: Math.trunc(new Date('2026-09-03T09:59:50.000Z').getTime() / 1_000), + clockRevision: 7, + deadlineGeneration: 3, + }); + + const beforeRevision = await store.getSourceRevision(); + await connector.client.set(keys.activeClockRevisionKey, '8'); + await expect( + store.withClockContext(clockContext, () => + store.setMatches([{ id: 99, stage: 7, roundIndex: 0, attackerId: 1, defenderId: 2 }]) + ) + ).rejects.toThrow('clock revision fence failed'); + await expect(store.getSourceRevision()).resolves.toBe(beforeRevision); + }); }); diff --git a/app/game-engine/src/auction/bidder.ts b/app/game-engine/src/auction/bidder.ts index 9c6a6586..7877b431 100644 --- a/app/game-engine/src/auction/bidder.ts +++ b/app/game-engine/src/auction/bidder.ts @@ -269,7 +269,14 @@ export const createAuctionBidder = async (options: { }; } const processingNow = world.getGameNow(new Date()); - const { bidAt, bidTick } = resolveAuctionBidTiming(world, processingNow, command.acceptedGameTick); + const convertedProcessingTick = Reflect.get(command, 'processingGameTick'); + const { bidAt, bidTick } = resolveAuctionBidTiming( + world, + processingNow, + typeof convertedProcessingTick === 'number' && Number.isSafeInteger(convertedProcessingTick) + ? convertedProcessingTick + : command.acceptedGameTick + ); if (hasAuctionClosePassed(auction, bidAt, bidTick)) { return { type: 'auctionBid', diff --git a/app/game-engine/src/index.ts b/app/game-engine/src/index.ts index aeab1633..5bf90c79 100644 --- a/app/game-engine/src/index.ts +++ b/app/game-engine/src/index.ts @@ -20,6 +20,8 @@ export * from './turn/engineStateManager.js'; export * from './turn/inMemoryStateStore.js'; export * from './turn/inMemoryTurnProcessor.js'; export * from './turn/databaseHooks.js'; +export * from './turn/clockReconciliation.js'; +export * from './turn/clockProjectionOutbox.js'; export * from './turn/joinCreateGeneralService.js'; export * from './turn/npcPossessionService.js'; export * from './turn/selectPoolService.js'; diff --git a/app/game-engine/src/lifecycle/databaseCommandQueue.ts b/app/game-engine/src/lifecycle/databaseCommandQueue.ts index 6e4d8343..a3c60035 100644 --- a/app/game-engine/src/lifecycle/databaseCommandQueue.ts +++ b/app/game-engine/src/lifecycle/databaseCommandQueue.ts @@ -108,6 +108,13 @@ export class DatabaseTurnDaemonCommandQueue implements TurnDaemonControlQueue, T private async claimPending(limit = 100): Promise { await this.recoverExpiredLeases(); return this.db.$transaction(async (transaction) => { + const world = await transaction.worldState.findFirst({ + orderBy: { id: 'asc' }, + select: { clockPhase: true, clockRevision: true, deadlineGeneration: true, clockTick: true }, + }); + const gameplayAllowed = + !world || world.clockPhase === 'RUNNING' || world.clockPhase === 'MANUAL'; + const currentRevision = world?.clockRevision ?? null; const rows = await transaction.$queryRaw< Array<{ sequence: bigint; @@ -115,6 +122,9 @@ export class DatabaseTurnDaemonCommandQueue implements TurnDaemonControlQueue, T eventType: string; payload: unknown; createdAt: Date; + acceptedGameTick: bigint | null; + acceptedClockRevision: bigint | null; + acceptedDeadlineGeneration: bigint | null; }> >(GamePrisma.sql` SELECT @@ -122,10 +132,14 @@ export class DatabaseTurnDaemonCommandQueue implements TurnDaemonControlQueue, T "request_id" AS "requestId", "event_type" AS "eventType", "payload", - "created_at" AS "createdAt" + "created_at" AS "createdAt", + "accepted_game_tick" AS "acceptedGameTick", + "accepted_clock_revision" AS "acceptedClockRevision", + "accepted_deadline_generation" AS "acceptedDeadlineGeneration" FROM "input_event" WHERE "target" = 'ENGINE'::"InputEventTarget" AND "status" = 'PENDING'::"InputEventStatus" + AND (${gameplayAllowed} OR "event_type" = 'getStatus') ORDER BY "sequence" ASC FOR UPDATE SKIP LOCKED LIMIT ${limit} @@ -133,23 +147,53 @@ export class DatabaseTurnDaemonCommandQueue implements TurnDaemonControlQueue, T if (rows.length === 0) { return []; } - await transaction.inputEvent.updateMany({ - where: { - sequence: { in: rows.map((row) => row.sequence) }, - target: 'ENGINE', - status: 'PENDING', - }, - data: { - status: 'PROCESSING', - processingAt: new Date(), - lockedBy: this.workerId, - leaseUntil: new Date(Date.now() + this.leaseDurationMs), - attempts: { increment: 1 }, - }, - }); + const appliedSuspensions = currentRevision + ? await transaction.clockSuspension.findMany({ + where: { status: 'APPLIED', targetRevision: { lte: currentRevision } }, + orderBy: { sourceRevision: 'asc' }, + select: { sourceRevision: true, targetRevision: true, shiftTicks: true }, + }) + : []; + const convertTick = (row: (typeof rows)[number]): bigint | null | undefined => { + if (row.eventType === 'getStatus') return row.acceptedGameTick; + if (row.acceptedGameTick === null || row.acceptedClockRevision === null || currentRevision === null) { + return row.acceptedGameTick ?? world?.clockTick ?? null; + } + if (row.acceptedClockRevision > currentRevision) return undefined; + let revision = row.acceptedClockRevision; + let tick = row.acceptedGameTick; + while (revision < currentRevision) { + const step = appliedSuspensions.find((entry) => entry.sourceRevision === revision); + if (!step || step.shiftTicks === null || step.targetRevision !== revision + 1n) return undefined; + tick += step.shiftTicks; + revision = step.targetRevision; + } + return tick; + }; + const processableRows = rows + .map((row) => ({ row, processingGameTick: convertTick(row) })) + .filter( + (entry): entry is { row: (typeof rows)[number]; processingGameTick: bigint | null } => + entry.processingGameTick !== undefined + ); + for (const { row, processingGameTick } of processableRows) { + await transaction.inputEvent.update({ + where: { sequence: row.sequence }, + data: { + status: 'PROCESSING', + processingAt: new Date(), + processingGameTick, + processingClockRevision: currentRevision, + processingDeadlineGeneration: world?.deadlineGeneration ?? null, + lockedBy: this.workerId, + leaseUntil: new Date(Date.now() + this.leaseDurationMs), + attempts: { increment: 1 }, + }, + }); + } const commands: TurnDaemonCommand[] = []; - for (const row of rows) { + for (const { row, processingGameTick } of processableRows) { const command = normalizeTurnDaemonCommand({ requestId: row.requestId, sentAt: row.createdAt.toISOString(), @@ -168,6 +212,27 @@ export class DatabaseTurnDaemonCommandQueue implements TurnDaemonControlQueue, T }); continue; } + if ( + processingGameTick !== null && + row.acceptedGameTick !== null && + processingGameTick !== row.acceptedGameTick + ) { + const value = Number(processingGameTick); + if (!Number.isSafeInteger(value)) { + await transaction.inputEvent.update({ + where: { sequence: row.sequence }, + data: { + status: 'FAILED', + error: 'Converted processing game tick is outside the safe integer range.', + completedAt: new Date(), + lockedBy: null, + leaseUntil: null, + }, + }); + continue; + } + Reflect.set(command, 'processingGameTick', value); + } commands.push(command); } return commands; diff --git a/app/game-engine/src/turn/clockProjectionOutbox.ts b/app/game-engine/src/turn/clockProjectionOutbox.ts new file mode 100644 index 00000000..1c3ca1f0 --- /dev/null +++ b/app/game-engine/src/turn/clockProjectionOutbox.ts @@ -0,0 +1,343 @@ +import { createHash } from 'node:crypto'; + +import { GameClock, parseGameClockPhase } from '@sammo-ts/common'; +import { + CLOCK_OPERATION_PERSISTENCE_LOCK, + GamePrisma, + acquireGameSchemaAdvisoryXactLock, + type GamePrismaClient, +} from '@sammo-ts/infra'; + +interface ClockProjectionRedis { + get(key: string): Promise; + eval(script: string, options: { keys: string[]; arguments: string[] }): Promise; +} + +interface ClaimedOutboxRow { + id: bigint; +} + +interface DbWallRow { + wallNow: Date; +} + +interface ProjectionPayload { + version: 1; + profileName: string; + suspensionId: string; + sourceRevision: number; + targetRevision: number; + deadlineGeneration: number; + shiftTicks: number; + projectionDeltaMilliseconds: number; + clockBaseTime: string; + ticksPerSecond: number; +} + +interface TournamentProjectionState { + stage?: number; + nextAt?: string; + nextTick?: number; + bettingCloseAt?: string; + bettingCloseTick?: number; + clockRevision?: number; + deadlineGeneration?: number; + [key: string]: unknown; +} + +const APPLY_CLOCK_PROJECTION_SCRIPT = ` +local active = redis.call('GET', KEYS[1]) +if active == ARGV[2] then + if redis.call('GET', KEYS[5]) == ARGV[4] and redis.call('GET', KEYS[2]) == ARGV[3] then + return 2 + end + return -3 +end +if active and active ~= ARGV[1] then + return -1 +end +if ARGV[5] ~= '__NONE__' and redis.call('GET', KEYS[4]) ~= ARGV[5] then + return -2 +end +redis.call('DEL', KEYS[3]) +local count = tonumber(ARGV[7]) +local offset = 8 +for index = 1, count do + redis.call('ZADD', KEYS[3], ARGV[offset], ARGV[offset + 1]) + offset = offset + 2 +end +if ARGV[5] ~= '__NONE__' then + redis.call('SET', KEYS[4], ARGV[6]) +end +redis.call('SET', KEYS[1], ARGV[2]) +redis.call('SET', KEYS[2], ARGV[3]) +redis.call('SET', KEYS[5], ARGV[4]) +redis.call('SET', KEYS[6], 'RUNNING') +return 1 +`; + +const safeInteger = (value: unknown, label: string): number => { + const result = typeof value === 'bigint' ? Number(value) : value; + if (typeof result !== 'number' || !Number.isSafeInteger(result)) { + throw new Error(`${label} must be a safe integer.`); + } + return result; +}; + +const canonicalize = (value: unknown): unknown => { + if (typeof value === 'bigint') return value.toString(); + if (Array.isArray(value)) return value.map(canonicalize); + if (value && typeof value === 'object') { + return Object.fromEntries( + Object.entries(value as Record) + .sort(([left], [right]) => left.localeCompare(right)) + .map(([key, item]) => [key, canonicalize(item)]) + ); + } + return value; +}; + +const stableJson = (value: unknown): string => JSON.stringify(canonicalize(value)); + +const checksum = (value: unknown): string => createHash('sha256').update(stableJson(value)).digest('hex'); + +const readDbWall = async (db: GamePrisma.TransactionClient): Promise => { + const rows = await db.$queryRaw(GamePrisma.sql` + SELECT (CURRENT_TIMESTAMP AT TIME ZONE 'UTC')::timestamp(3) AS "wallNow" + `); + if (!rows[0]?.wallNow) throw new Error('Failed to read PostgreSQL wall time for the projection outbox.'); + return rows[0].wallNow; +}; + +const parsePayload = (value: GamePrisma.JsonValue): ProjectionPayload => { + if (!value || typeof value !== 'object' || Array.isArray(value)) { + throw new Error('Clock projection outbox payload must be an object.'); + } + const payload = value as Record; + if (payload.version !== 1 || typeof payload.profileName !== 'string' || typeof payload.suspensionId !== 'string') { + throw new Error('Clock projection outbox payload identity is invalid.'); + } + if (typeof payload.clockBaseTime !== 'string') { + throw new Error('Clock projection outbox is missing its projection base.'); + } + return { + version: 1, + profileName: payload.profileName, + suspensionId: payload.suspensionId, + sourceRevision: safeInteger(payload.sourceRevision, 'sourceRevision'), + targetRevision: safeInteger(payload.targetRevision, 'targetRevision'), + deadlineGeneration: safeInteger(payload.deadlineGeneration, 'deadlineGeneration'), + shiftTicks: safeInteger(payload.shiftTicks, 'shiftTicks'), + projectionDeltaMilliseconds: safeInteger( + payload.projectionDeltaMilliseconds, + 'projectionDeltaMilliseconds' + ), + clockBaseTime: payload.clockBaseTime, + ticksPerSecond: safeInteger(payload.ticksPerSecond, 'ticksPerSecond'), + }; +}; + +const claimNext = async (db: GamePrismaClient, workerId: string) => + db.$transaction(async (transaction) => { + const rows = await transaction.$queryRaw(GamePrisma.sql` + SELECT id + FROM clock_projection_outbox + WHERE available_at <= (CURRENT_TIMESTAMP AT TIME ZONE 'UTC') + AND ( + status IN ('PENDING', 'FAILED') + OR (status = 'APPLYING' AND locked_at < (CURRENT_TIMESTAMP AT TIME ZONE 'UTC') - INTERVAL '30 seconds') + ) + ORDER BY id + LIMIT 1 + FOR UPDATE SKIP LOCKED + `); + const id = rows[0]?.id; + if (id === undefined) return null; + const lockedAt = await readDbWall(transaction); + return transaction.clockProjectionOutbox.update({ + where: { id }, + data: { + status: 'APPLYING', + attempts: { increment: 1 }, + lockedAt, + lockedBy: workerId, + lastError: null, + }, + }); + }); + +const projectTournamentState = ( + raw: string | null, + payload: ProjectionPayload, + clock: GameClock +): { expected: string; next: string } | null => { + if (!raw) return null; + const parsed = JSON.parse(raw) as TournamentProjectionState; + const active = typeof parsed.stage === 'number' && parsed.stage > 0; + if (active && parsed.nextAt && !Number.isSafeInteger(parsed.nextTick)) { + throw new Error('Active tournament nextAt lacks the authoritative nextTick dual-write.'); + } + if (active && parsed.bettingCloseAt && !Number.isSafeInteger(parsed.bettingCloseTick)) { + throw new Error('Active tournament bettingCloseAt lacks the authoritative bettingCloseTick dual-write.'); + } + const next: TournamentProjectionState = { + ...parsed, + clockRevision: payload.targetRevision, + deadlineGeneration: payload.deadlineGeneration, + }; + if (Number.isSafeInteger(parsed.nextTick)) { + next.nextTick = parsed.nextTick! + payload.shiftTicks; + next.nextAt = clock.tickToDate(next.nextTick).toISOString(); + } + if (Number.isSafeInteger(parsed.bettingCloseTick)) { + next.bettingCloseTick = parsed.bettingCloseTick! + payload.shiftTicks; + next.bettingCloseAt = clock.tickToDate(next.bettingCloseTick).toISOString(); + } + return { expected: raw, next: JSON.stringify(next) }; +}; + +const recordFailure = async (db: GamePrismaClient, outboxId: bigint, error: unknown): Promise => { + const message = error instanceof Error ? error.message : String(error); + await db.$executeRaw(GamePrisma.sql` + UPDATE clock_projection_outbox + SET status = 'FAILED', + locked_at = NULL, + locked_by = NULL, + last_error = ${message.slice(0, 4_000)}, + available_at = (CURRENT_TIMESTAMP AT TIME ZONE 'UTC') + INTERVAL '1 second' + WHERE id = ${outboxId} AND status = 'APPLYING' + `); +}; + +export const applyNextClockProjection = async (options: { + db: GamePrismaClient; + redis: ClockProjectionRedis; + workerId: string; +}): Promise<'IDLE' | 'APPLIED' | 'RECOVERED'> => { + if (!options.workerId.trim()) throw new Error('Clock projection worker ID is required.'); + const outbox = await claimNext(options.db, options.workerId); + if (!outbox) return 'IDLE'; + try { + const payload = parsePayload(outbox.payload); + if (checksum(outbox.payload) !== outbox.checksum) { + throw new Error('Clock projection outbox checksum verification failed.'); + } + if (outbox.targetRevision !== BigInt(payload.targetRevision)) { + throw new Error('Clock projection payload revision differs from its outbox row.'); + } + const world = await options.db.worldState.findUniqueOrThrow({ where: { id: outbox.worldStateId } }); + if ( + parseGameClockPhase(world.clockPhase) !== 'RECONCILING' || + world.clockRevision !== outbox.targetRevision || + world.deadlineGeneration !== BigInt(payload.deadlineGeneration) || + !world.clockBaseTime + ) { + throw new Error('Clock projection DB phase/revision/generation fence failed.'); + } + const clock = new GameClock({ + baseTime: new Date(payload.clockBaseTime), + tick: safeInteger(world.clockTick, 'world clock tick'), + mode: world.clockMode === 'manual' ? 'manual' : 'realtime', + wallAnchor: world.clockWallAnchor ?? new Date(), + turnSeconds: world.tickSeconds, + phase: 'RECONCILING', + revision: payload.targetRevision, + }); + if (clock.ticksPerSecond !== payload.ticksPerSecond) { + throw new Error('Clock projection rate differs from the durable outbox payload.'); + } + const auctions = await options.db.auction.findMany({ + where: { status: { in: ['OPEN', 'FINALIZING'] } }, + orderBy: { id: 'asc' }, + select: { id: true, closeTick: true }, + }); + const timers = auctions.map((auction) => { + if (auction.closeTick === null) { + throw new Error(`Active auction ${auction.id} lacks closeTick during projection rebuild.`); + } + return { score: safeInteger(auction.closeTick, `auction ${auction.id} closeTick`), id: String(auction.id) }; + }); + const prefix = `sammo:${payload.profileName}`; + const tournamentKey = `${prefix}:tournament:state`; + const tournament = projectTournamentState(await options.redis.get(tournamentKey), payload, clock); + const result = await options.redis.eval(APPLY_CLOCK_PROJECTION_SCRIPT, { + keys: [ + `${prefix}:clock:active-revision`, + `${prefix}:clock:deadline-generation`, + `${prefix}:auction:timer`, + tournamentKey, + `${prefix}:clock:projection-checksum`, + `${prefix}:clock:phase`, + ], + arguments: [ + String(payload.sourceRevision), + String(payload.targetRevision), + String(payload.deadlineGeneration), + outbox.checksum, + tournament?.expected ?? '__NONE__', + tournament?.next ?? '__NONE__', + String(timers.length), + ...timers.flatMap(({ score, id }) => [String(score), id]), + ], + }); + const applied = Number(result); + if (applied === -1) throw new Error('Redis active clock revision does not match the outbox source revision.'); + if (applied === -2) throw new Error('Redis tournament state changed while rebuilding its projection.'); + if (applied === -3) throw new Error('Redis target revision exists without the expected projection checksum.'); + if (applied !== 1 && applied !== 2) throw new Error(`Unexpected Redis clock projection result: ${String(result)}`); + + await options.db.$transaction(async (transaction) => { + await acquireGameSchemaAdvisoryXactLock(transaction, CLOCK_OPERATION_PERSISTENCE_LOCK); + await transaction.$queryRaw(GamePrisma.sql` + SELECT id FROM world_state WHERE id = ${outbox.worldStateId} FOR UPDATE + `); + const finalized = await transaction.worldState.updateMany({ + where: { + id: outbox.worldStateId, + clockPhase: 'RECONCILING', + clockRevision: outbox.targetRevision, + deadlineGeneration: BigInt(payload.deadlineGeneration), + }, + data: { clockPhase: 'RUNNING' }, + }); + if (finalized.count !== 1) { + throw new Error('Clock projection final RUNNING transition fence failed.'); + } + const appliedAt = await readDbWall(transaction); + await transaction.clockProjectionOutbox.update({ + where: { id: outbox.id }, + data: { status: 'APPLIED', appliedAt, lockedAt: null, lockedBy: null, lastError: null }, + }); + if (outbox.suspensionId) { + await transaction.clockSuspension.update({ + where: { id: outbox.suspensionId }, + data: { status: 'APPLIED' }, + }); + } + }); + return applied === 2 ? 'RECOVERED' : 'APPLIED'; + } catch (error) { + await recordFailure(options.db, outbox.id, error); + throw error; + } +}; + +export const loadClockReconciliationReadiness = async (db: GamePrismaClient) => { + const world = await db.worldState.findFirst({ + orderBy: { id: 'asc' }, + select: { clockPhase: true, clockRevision: true, deadlineGeneration: true }, + }); + const incompleteOutboxCount = await db.clockProjectionOutbox.count({ where: { status: { not: 'APPLIED' } } }); + if (!world) { + return { ready: false, phase: null, revision: null, deadlineGeneration: null, incompleteOutboxCount }; + } + const phase = parseGameClockPhase(world.clockPhase); + return { + ready: phase !== 'RECONCILING' && incompleteOutboxCount === 0, + gameplayEnabled: phase === 'RUNNING' || phase === 'MANUAL', + phase, + revision: safeInteger(world.clockRevision, 'clock revision'), + deadlineGeneration: safeInteger(world.deadlineGeneration, 'deadline generation'), + incompleteOutboxCount, + }; +}; diff --git a/app/game-engine/src/turn/clockReconciliation.ts b/app/game-engine/src/turn/clockReconciliation.ts new file mode 100644 index 00000000..ae06c761 --- /dev/null +++ b/app/game-engine/src/turn/clockReconciliation.ts @@ -0,0 +1,763 @@ +import { createHash } from 'node:crypto'; + +import { + GAME_TICKS_PER_TURN, + MAX_SAFE_GAME_TICK, + GameClock, + buildClockAlignmentPlan, + parseClockAlignmentPolicy, + parseGameClockPhase, + type ClockAlignmentPolicy, +} from '@sammo-ts/common'; +import { + CLOCK_OPERATION_PERSISTENCE_LOCK, + GENERAL_ACCESS_PERSISTENCE_LOCK, + GamePrisma, + acquireGameSchemaAdvisoryXactLock, + type GamePrismaClient, +} from '@sammo-ts/infra'; + +export type ClockSuspensionSource = 'MAINTENANCE' | 'OPEN_DELAY' | 'UNIFICATION_WAIT' | 'RECOVERY'; + +export type ClockOperationAuthority = + | { kind: 'DAEMON'; profileName: string; ownerId: string; fencingEpoch: bigint } + | { kind: 'OFFLINE'; profileName: string; reason: string }; + +export interface ClockSuspensionResult { + suspensionId: string; + phase: 'SUSPENDED'; + sourceRevision: number; + targetRevision: number; + cutTick: number; + cutWallAt: Date; +} + +export interface ClockReconciliationResult { + suspensionId: string; + phase: 'RECONCILING'; + sourceRevision: number; + targetRevision: number; + deadlineGeneration: number; + gapTicks: number; + catchUpTicks: number; + shiftTicks: number; + alignedTick: number; + resumeWallAt: Date; +} + +interface DbWallRow { + wallNow: Date; +} + +interface LeaseFenceRow { + ownerId: string; + fencingEpoch: bigint; + valid: boolean; +} + +interface IdRow { + id: number; +} + +interface TextIdRow { + id: string; +} + +interface ParticipantSnapshot { + key: string; + policy: 'SHIFT' | 'KEEP' | 'REBUILD'; + checksum: string; + count: number; +} + +const asJson = (value: unknown): GamePrisma.InputJsonValue => value as GamePrisma.InputJsonValue; + +const safeNumber = (value: bigint, label: string): number => { + const result = Number(value); + if (!Number.isSafeInteger(result)) { + throw new Error(`${label} is outside the JavaScript safe integer range: ${value}`); + } + return result; +}; + +const canonicalize = (value: unknown): unknown => { + if (typeof value === 'bigint') return value.toString(); + if (Array.isArray(value)) return value.map(canonicalize); + if (value && typeof value === 'object') { + return Object.fromEntries( + Object.entries(value as Record) + .sort(([left], [right]) => left.localeCompare(right)) + .map(([key, item]) => [key, canonicalize(item)]) + ); + } + return value; +}; + +const stableJson = (value: unknown): string => JSON.stringify(canonicalize(value)); + +const checksum = (value: unknown): string => createHash('sha256').update(stableJson(value)).digest('hex'); + +const aggregateChecksum = (participants: readonly ParticipantSnapshot[]): string => + checksum(participants.map(({ key, policy, checksum: value, count }) => ({ key, policy, checksum: value, count }))); + +const readDbWall = async (db: GamePrisma.TransactionClient): Promise => { + const rows = await db.$queryRaw(GamePrisma.sql` + SELECT (CURRENT_TIMESTAMP AT TIME ZONE 'UTC')::timestamp(3) AS "wallNow" + `); + const wallNow = rows[0]?.wallNow; + if (!wallNow || Number.isNaN(wallNow.getTime())) { + throw new Error('Failed to read the PostgreSQL wall clock.'); + } + return wallNow; +}; + +const verifyAuthority = async ( + db: GamePrisma.TransactionClient, + authority: ClockOperationAuthority +): Promise => { + const rows = await db.$queryRaw(GamePrisma.sql` + SELECT owner_id AS "ownerId", + fencing_epoch AS "fencingEpoch", + lease_until > (CURRENT_TIMESTAMP AT TIME ZONE 'UTC') AS valid + FROM turn_daemon_lease + WHERE profile = ${authority.profileName} + FOR UPDATE + `); + const lease = rows[0]; + if (authority.kind === 'OFFLINE') { + if (!authority.reason.trim()) { + throw new Error('Offline clock operations require an audit reason.'); + } + if (lease?.valid) { + throw new Error(`Clock operation requires the ${authority.profileName} daemon lease to be offline.`); + } + return; + } + if ( + !lease?.valid || + lease.ownerId !== authority.ownerId || + lease.fencingEpoch !== authority.fencingEpoch + ) { + throw new Error(`Stale turn-daemon fencing authority for profile ${authority.profileName}.`); + } +}; + +const lockWorld = async (db: GamePrisma.TransactionClient): Promise => { + const rows = await db.$queryRaw(GamePrisma.sql` + SELECT id FROM world_state ORDER BY id LIMIT 2 FOR UPDATE + `); + if (rows.length !== 1) { + throw new Error(`Clock reconciliation requires exactly one world_state row; found ${rows.length}.`); + } + return rows[0]!.id; +}; + +const lockParticipants = async (db: GamePrisma.TransactionClient, cutTick: bigint): Promise => { + await db.$queryRaw(GamePrisma.sql`SELECT id FROM general ORDER BY id FOR UPDATE`); + await db.$queryRaw(GamePrisma.sql` + SELECT id FROM auction + WHERE status IN ('OPEN'::auction_status, 'FINALIZING'::auction_status) + ORDER BY id FOR UPDATE + `); + await db.$queryRaw(GamePrisma.sql` + SELECT id FROM message + WHERE valid_until_tick IS NOT NULL AND valid_until_tick >= ${cutTick} + ORDER BY id FOR UPDATE + `); + await db.$queryRaw(GamePrisma.sql` + SELECT id FROM vote_poll WHERE closed_at IS NULL ORDER BY id FOR UPDATE + `); + await db.$queryRaw(GamePrisma.sql` + SELECT id FROM select_pool WHERE general_id IS NULL ORDER BY id FOR UPDATE + `); + await db.$queryRaw(GamePrisma.sql` + SELECT owner_user_id AS id FROM select_npc_token ORDER BY owner_user_id FOR UPDATE + `); +}; + +const readParticipantSnapshots = async ( + db: GamePrisma.TransactionClient, + worldStateId: number, + cutTick: bigint +): Promise => { + const [world, generals, auctions, messages, votes, pool, npcTokens, commands] = await Promise.all([ + db.worldState.findUniqueOrThrow({ + where: { id: worldStateId }, + select: { + clockTick: true, + clockRevision: true, + deadlineGeneration: true, + lastTurnTick: true, + meta: true, + }, + }), + db.general.findMany({ + orderBy: { id: 'asc' }, + select: { id: true, turnTick: true, recentWarTick: true }, + }), + db.auction.findMany({ + where: { status: { in: ['OPEN', 'FINALIZING'] } }, + orderBy: { id: 'asc' }, + select: { id: true, status: true, openTick: true, closeTick: true }, + }), + db.message.findMany({ + where: { validUntilTick: { not: null, gte: cutTick } }, + orderBy: { id: 'asc' }, + select: { id: true, timeTick: true, validUntilTick: true }, + }), + db.votePoll.findMany({ + where: { closedAt: null }, + orderBy: { id: 'asc' }, + select: { id: true, startTick: true, endTick: true }, + }), + db.selectPoolEntry.findMany({ + where: { generalId: null }, + orderBy: { id: 'asc' }, + select: { id: true, reservedUntilTick: true }, + }), + db.npcSelectionToken.findMany({ + orderBy: { ownerUserId: 'asc' }, + select: { ownerUserId: true, validUntilTick: true, pickMoreFromTick: true }, + }), + db.inputEvent.findMany({ + where: { status: { in: ['PENDING', 'PROCESSING'] } }, + orderBy: { sequence: 'asc' }, + select: { sequence: true, acceptedGameTick: true, acceptedClockRevision: true }, + }), + ]); + const snapshot = (key: string, policy: ParticipantSnapshot['policy'], rows: unknown[]): ParticipantSnapshot => ({ + key, + policy, + checksum: checksum(rows), + count: rows.length, + }); + const meta = world.meta && typeof world.meta === 'object' && !Array.isArray(world.meta) ? world.meta : {}; + return [ + snapshot('world-clock', 'REBUILD', [ + { + clockTick: world.clockTick, + clockRevision: world.clockRevision, + deadlineGeneration: world.deadlineGeneration, + }, + ]), + snapshot('turn-cursor', 'SHIFT', [{ lastTurnTick: world.lastTurnTick }]), + snapshot( + 'general-next-turn', + 'SHIFT', + generals.map(({ id, turnTick }) => ({ id, turnTick })) + ), + snapshot( + 'general-recent-war-occurrence', + 'KEEP', + generals.map(({ id, recentWarTick }) => ({ id, recentWarTick })) + ), + snapshot( + 'auction-open-occurrence', + 'KEEP', + auctions.map(({ id, openTick }) => ({ id, openTick })) + ), + snapshot( + 'auction-deadline', + 'SHIFT', + auctions.map(({ id, status, closeTick }) => ({ id, status, closeTick })) + ), + snapshot( + 'auction-finalizing-recovery', + 'REBUILD', + auctions.map(({ id, status }) => ({ id, status })) + ), + snapshot( + 'message-occurrence', + 'KEEP', + messages.map(({ id, timeTick }) => ({ id, timeTick })) + ), + snapshot( + 'message-expiry', + 'SHIFT', + messages.map(({ id, validUntilTick }) => ({ id, validUntilTick })) + ), + snapshot( + 'vote-start-occurrence', + 'KEEP', + votes.map(({ id, startTick }) => ({ id, startTick })) + ), + snapshot( + 'vote-end-deadline', + 'SHIFT', + votes.map(({ id, endTick }) => ({ id, endTick })) + ), + snapshot('select-pool-reservation', 'SHIFT', pool), + snapshot('npc-selection-window', 'SHIFT', npcTokens), + snapshot('accepted-command-coordinate', 'KEEP', commands), + snapshot('movable-json-rule-anchors', 'SHIFT', [ + { + lastTurnTime: Reflect.get(meta, 'lastTurnTime'), + turntime: Reflect.get(meta, 'turntime'), + starttime: Reflect.get(meta, 'starttime'), + tnmt_time: Reflect.get(meta, 'tnmt_time'), + }, + ]), + ]; +}; + +const persistInitialParticipants = async ( + db: GamePrisma.TransactionClient, + suspensionId: string, + participants: readonly ParticipantSnapshot[] +): Promise => { + for (const participant of participants) { + await db.clockReconciliationParticipant.create({ + data: { + suspensionId, + participantKey: participant.key, + policy: participant.policy, + beforeChecksum: participant.checksum, + afterChecksum: participant.checksum, + affectedCount: 0, + }, + }); + } +}; + +const shiftMetaDate = (value: unknown, deltaMilliseconds: number): unknown => { + if (typeof value !== 'string' || !value.trim()) return value; + const parsed = new Date(value); + if (Number.isNaN(parsed.getTime())) return value; + return new Date(parsed.getTime() + deltaMilliseconds).toISOString(); +}; + +const shiftedMeta = (value: GamePrisma.JsonValue, deltaMilliseconds: number): GamePrisma.InputJsonValue => { + const meta = value && typeof value === 'object' && !Array.isArray(value) ? { ...value } : {}; + for (const key of ['lastTurnTime', 'turntime', 'starttime', 'tnmt_time'] as const) { + if (Object.hasOwn(meta, key)) { + Reflect.set(meta, key, shiftMetaDate(Reflect.get(meta, key), deltaMilliseconds)); + } + } + return asJson(meta); +}; + +const assertShiftFits = (participants: readonly ParticipantSnapshot[], shiftTicks: number): void => { + if (!Number.isSafeInteger(shiftTicks) || shiftTicks < 0) { + throw new Error(`Invalid reconciliation shift: ${shiftTicks}`); + } + // Checksums retain stringified values for audit; actual row ranges are + // checked by PostgreSQL BIGINT and the world aligned tick is checked by the + // shared GameClock plan. The sentinel expiry is deliberately never shifted. + if (participants.some((participant) => !participant.checksum)) { + throw new Error('Participant snapshot is incomplete.'); + } +}; + +const assertScheduleRanges = async (db: GamePrisma.TransactionClient, shiftTicks: number): Promise => { + const shift = BigInt(shiftTicks); + const maximum = BigInt(MAX_SAFE_GAME_TICK) - shift; + const [general, auction, message, vote, pool, npcValid, npcMore] = await Promise.all([ + db.general.aggregate({ _max: { turnTick: true }, where: { turnTick: { not: null } } }), + db.auction.aggregate({ + _max: { closeTick: true }, + where: { status: { in: ['OPEN', 'FINALIZING'] }, closeTick: { not: null } }, + }), + db.message.aggregate({ + _max: { validUntilTick: true }, + where: { validUntilTick: { not: null, lt: BigInt(MAX_SAFE_GAME_TICK) } }, + }), + db.votePoll.aggregate({ _max: { endTick: true }, where: { closedAt: null, endTick: { not: null } } }), + db.selectPoolEntry.aggregate({ + _max: { reservedUntilTick: true }, + where: { generalId: null, reservedUntilTick: { not: null } }, + }), + db.npcSelectionToken.aggregate({ _max: { validUntilTick: true }, where: { validUntilTick: { not: null } } }), + db.npcSelectionToken.aggregate({ + _max: { pickMoreFromTick: true }, + where: { pickMoreFromTick: { not: null } }, + }), + ]); + const values: Array<[string, bigint | null]> = [ + ['general.turn_tick', general._max.turnTick], + ['auction.close_tick', auction._max.closeTick], + ['message.valid_until_tick', message._max.validUntilTick], + ['vote_poll.end_tick', vote._max.endTick], + ['select_pool.reserved_until_tick', pool._max.reservedUntilTick], + ['select_npc_token.valid_until_tick', npcValid._max.validUntilTick], + ['select_npc_token.pick_more_from_tick', npcMore._max.pickMoreFromTick], + ]; + for (const [label, value] of values) { + if (value !== null && value > maximum) { + throw new Error(`${label} would exceed the safe game tick range after reconciliation.`); + } + } +}; + +const applyParticipantShift = async ( + db: GamePrisma.TransactionClient, + worldStateId: number, + cutTick: bigint, + alignedTick: bigint, + targetRevision: bigint, + targetGeneration: bigint, + shiftTicks: bigint, + projectionDeltaMilliseconds: number, + resumeWallAt: Date +): Promise> => { + const affected = new Map(); + const world = await db.worldState.findUniqueOrThrow({ where: { id: worldStateId }, select: { meta: true } }); + const cursor = await db.worldState.updateMany({ + where: { id: worldStateId, lastTurnTick: { not: null } }, + data: { lastTurnTick: { increment: shiftTicks } }, + }); + affected.set('turn-cursor', cursor.count); + affected.set( + 'general-next-turn', + await db.$executeRaw(GamePrisma.sql` + UPDATE general + SET turn_tick = turn_tick + ${shiftTicks}, + turn_time = turn_time + ${projectionDeltaMilliseconds} * INTERVAL '1 millisecond' + WHERE turn_tick IS NOT NULL + `) + ); + affected.set( + 'auction-deadline', + await db.$executeRaw(GamePrisma.sql` + UPDATE auction + SET close_tick = close_tick + ${shiftTicks}, + close_at = close_at + ${projectionDeltaMilliseconds} * INTERVAL '1 millisecond' + WHERE status IN ('OPEN'::auction_status, 'FINALIZING'::auction_status) + AND close_tick IS NOT NULL + `) + ); + affected.set( + 'message-expiry', + await db.$executeRaw(GamePrisma.sql` + UPDATE message + SET valid_until_tick = valid_until_tick + ${shiftTicks}, + valid_until = valid_until + ${projectionDeltaMilliseconds} * INTERVAL '1 millisecond' + WHERE valid_until_tick IS NOT NULL + AND valid_until_tick >= ${cutTick} + AND valid_until_tick < ${BigInt(MAX_SAFE_GAME_TICK)} + `) + ); + affected.set( + 'vote-end-deadline', + await db.$executeRaw(GamePrisma.sql` + UPDATE vote_poll + SET end_tick = end_tick + ${shiftTicks}, + end_at = end_at + ${projectionDeltaMilliseconds} * INTERVAL '1 millisecond' + WHERE closed_at IS NULL AND end_tick IS NOT NULL + `) + ); + affected.set( + 'select-pool-reservation', + await db.$executeRaw(GamePrisma.sql` + UPDATE select_pool + SET reserved_until_tick = reserved_until_tick + ${shiftTicks}, + reserved_until = reserved_until + ${projectionDeltaMilliseconds} * INTERVAL '1 millisecond' + WHERE general_id IS NULL AND reserved_until_tick IS NOT NULL + `) + ); + affected.set( + 'npc-selection-window', + await db.$executeRaw(GamePrisma.sql` + UPDATE select_npc_token + SET valid_until_tick = CASE + WHEN valid_until_tick IS NULL THEN NULL ELSE valid_until_tick + ${shiftTicks} END, + valid_until = CASE + WHEN valid_until_tick IS NULL THEN valid_until + ELSE valid_until + ${projectionDeltaMilliseconds} * INTERVAL '1 millisecond' END, + pick_more_from_tick = CASE + WHEN pick_more_from_tick IS NULL THEN NULL ELSE pick_more_from_tick + ${shiftTicks} END, + pick_more_from = CASE + WHEN pick_more_from_tick IS NULL THEN pick_more_from + ELSE pick_more_from + ${projectionDeltaMilliseconds} * INTERVAL '1 millisecond' END + WHERE valid_until_tick IS NOT NULL OR pick_more_from_tick IS NOT NULL + `) + ); + await db.worldState.update({ + where: { id: worldStateId }, + data: { + clockTick: alignedTick, + clockWallAnchor: resumeWallAt, + clockPhase: 'RECONCILING', + clockRevision: targetRevision, + deadlineGeneration: targetGeneration, + meta: shiftedMeta(world.meta, projectionDeltaMilliseconds), + }, + }); + affected.set('world-clock', 1); + affected.set('movable-json-rule-anchors', 1); + affected.set('auction-finalizing-recovery', 0); + return affected; +}; + +export const startClockSuspension = async (options: { + db: GamePrismaClient; + suspensionId: string; + source: ClockSuspensionSource; + authority: ClockOperationAuthority; + policy?: ClockAlignmentPolicy; + catchUpTicks?: number; +}): Promise => { + if (!options.suspensionId.trim() || options.suspensionId.length > 64) { + throw new Error('Clock suspension ID must contain 1-64 characters.'); + } + const policy = options.policy ?? 'EXACT'; + const catchUpTicks = options.catchUpTicks ?? 0; + if (!Number.isSafeInteger(catchUpTicks) || catchUpTicks < 0) { + throw new Error('Clock suspension catch-up ticks must be a non-negative safe integer.'); + } + return options.db.$transaction( + async (db) => { + await verifyAuthority(db, options.authority); + await acquireGameSchemaAdvisoryXactLock(db, CLOCK_OPERATION_PERSISTENCE_LOCK); + await acquireGameSchemaAdvisoryXactLock(db, GENERAL_ACCESS_PERSISTENCE_LOCK); + const worldStateId = await lockWorld(db); + const existing = await db.clockSuspension.findUnique({ where: { id: options.suspensionId } }); + if (existing) { + if (existing.worldStateId !== worldStateId || existing.source !== options.source || existing.policy !== policy) { + throw new Error(`Clock suspension ID ${options.suspensionId} is already bound to another operation.`); + } + if (existing.status !== 'SUSPENDED') { + throw new Error(`Clock suspension ${options.suspensionId} already advanced to ${existing.status}.`); + } + return { + suspensionId: existing.id, + phase: 'SUSPENDED' as const, + sourceRevision: safeNumber(existing.sourceRevision, 'source revision'), + targetRevision: safeNumber(existing.targetRevision, 'target revision'), + cutTick: safeNumber(existing.cutTick, 'cut tick'), + cutWallAt: existing.cutWallAt, + }; + } + const world = await db.worldState.findUniqueOrThrow({ where: { id: worldStateId } }); + const phase = parseGameClockPhase(world.clockPhase); + if (phase !== 'RUNNING') { + throw new Error(`Clock suspension can start only from RUNNING; current phase is ${phase}.`); + } + if (!world.clockBaseTime || world.clockTick === null || !world.clockWallAnchor) { + throw new Error('Clock suspension requires a fully initialized logical game clock.'); + } + const cutWallAt = await readDbWall(db); + const storedTick = safeNumber(world.clockTick, 'world clock tick'); + const sourceRevision = safeNumber(world.clockRevision, 'world clock revision'); + const clock = new GameClock({ + baseTime: world.clockBaseTime, + tick: storedTick, + mode: world.clockMode === 'manual' ? 'manual' : 'realtime', + wallAnchor: world.clockWallAnchor, + turnSeconds: world.tickSeconds, + phase, + revision: sourceRevision, + }); + const cutTick = clock.nowTick(cutWallAt); + await lockParticipants(db, BigInt(cutTick)); + await db.worldState.update({ + where: { id: worldStateId }, + data: { clockPhase: 'SUSPENDED', clockTick: BigInt(cutTick), clockWallAnchor: cutWallAt }, + }); + const participants = await readParticipantSnapshots(db, worldStateId, BigInt(cutTick)); + await db.clockSuspension.create({ + data: { + id: options.suspensionId, + worldStateId, + source: options.source, + policy, + status: 'SUSPENDED', + sourceRevision: BigInt(sourceRevision), + targetRevision: BigInt(sourceRevision + 1), + cutTick: BigInt(cutTick), + cutWallAt, + rateTicksPerSecond: GAME_TICKS_PER_TURN / world.tickSeconds, + catchUpTicks: BigInt(catchUpTicks), + participantChecksumBefore: aggregateChecksum(participants), + detail: asJson({ authority: options.authority.kind, profileName: options.authority.profileName }), + }, + }); + await persistInitialParticipants(db, options.suspensionId, participants); + return { + suspensionId: options.suspensionId, + phase: 'SUSPENDED', + sourceRevision, + targetRevision: sourceRevision + 1, + cutTick, + cutWallAt, + }; + }, + { isolationLevel: 'Serializable', maxWait: 10_000, timeout: 30_000 } + ); +}; + +export const reconcileClockSuspension = async (options: { + db: GamePrismaClient; + suspensionId: string; + authority: ClockOperationAuthority; + /** Deterministic fixture seam; production must always use PostgreSQL CURRENT_TIMESTAMP. */ + testResumeWallAt?: Date; +}): Promise => + options.db.$transaction( + async (db) => { + await verifyAuthority(db, options.authority); + await acquireGameSchemaAdvisoryXactLock(db, CLOCK_OPERATION_PERSISTENCE_LOCK); + await acquireGameSchemaAdvisoryXactLock(db, GENERAL_ACCESS_PERSISTENCE_LOCK); + const worldStateId = await lockWorld(db); + const suspension = await db.clockSuspension.findUniqueOrThrow({ where: { id: options.suspensionId } }); + if (suspension.worldStateId !== worldStateId) { + throw new Error('Clock suspension belongs to another world state.'); + } + if (suspension.status === 'RECONCILING' || suspension.status === 'APPLIED') { + if ( + suspension.gapTicks === null || + suspension.shiftTicks === null || + suspension.alignedTick === null || + !suspension.resumeWallAt + ) { + throw new Error('Persisted clock reconciliation result is incomplete.'); + } + const world = await db.worldState.findUniqueOrThrow({ where: { id: worldStateId } }); + return { + suspensionId: suspension.id, + phase: 'RECONCILING' as const, + sourceRevision: safeNumber(suspension.sourceRevision, 'source revision'), + targetRevision: safeNumber(suspension.targetRevision, 'target revision'), + deadlineGeneration: safeNumber(world.deadlineGeneration, 'deadline generation'), + gapTicks: safeNumber(suspension.gapTicks, 'gap ticks'), + catchUpTicks: safeNumber(suspension.catchUpTicks, 'catch-up ticks'), + shiftTicks: safeNumber(suspension.shiftTicks, 'shift ticks'), + alignedTick: safeNumber(suspension.alignedTick, 'aligned tick'), + resumeWallAt: suspension.resumeWallAt, + }; + } + if (suspension.status !== 'SUSPENDED') { + throw new Error(`Clock suspension cannot reconcile from status ${suspension.status}.`); + } + const world = await db.worldState.findUniqueOrThrow({ where: { id: worldStateId } }); + const phase = parseGameClockPhase(world.clockPhase); + if (phase !== 'SUSPENDED' || world.clockRevision !== suspension.sourceRevision) { + throw new Error('Clock reconciliation phase or source revision fence failed.'); + } + const worldMeta = + world.meta && typeof world.meta === 'object' && !Array.isArray(world.meta) + ? (world.meta as Record) + : {}; + const united = Number(worldMeta.isunited ?? worldMeta.isUnited ?? 0); + if (suspension.source === 'UNIFICATION_WAIT' || united >= 2) { + throw new Error( + 'Unification wait requires the atomic alignment-and-invader workflow; generic resume is forbidden.' + ); + } + const cutTick = safeNumber(suspension.cutTick, 'cut tick'); + await lockParticipants(db, suspension.cutTick); + if (options.testResumeWallAt && process.env.NODE_ENV !== 'test') { + throw new Error('A clock reconciliation wall override is allowed only in tests.'); + } + const resumeWallAt = options.testResumeWallAt + ? new Date(options.testResumeWallAt.getTime()) + : await readDbWall(db); + const plan = buildClockAlignmentPlan({ + policy: parseClockAlignmentPolicy(suspension.policy), + sourceRevision: safeNumber(suspension.sourceRevision, 'source revision'), + cutTick, + cutWall: suspension.cutWallAt, + resumeWall: resumeWallAt, + ticksPerSecond: suspension.rateTicksPerSecond, + catchUpTicks: safeNumber(suspension.catchUpTicks, 'catch-up ticks'), + }); + const before = await readParticipantSnapshots(db, worldStateId, suspension.cutTick); + assertShiftFits(before, plan.shiftTicks); + await assertScheduleRanges(db, plan.shiftTicks); + const projectionDeltaMilliseconds = Math.trunc( + (plan.shiftTicks * 1_000) / suspension.rateTicksPerSecond + ); + if (!Number.isSafeInteger(projectionDeltaMilliseconds)) { + throw new Error('Clock reconciliation projection delta is outside the safe integer range.'); + } + const targetGeneration = world.deadlineGeneration + 1n; + const affected = await applyParticipantShift( + db, + worldStateId, + suspension.cutTick, + BigInt(plan.alignedTick), + BigInt(plan.targetRevision), + targetGeneration, + BigInt(plan.shiftTicks), + projectionDeltaMilliseconds, + resumeWallAt + ); + const after = await readParticipantSnapshots(db, worldStateId, suspension.cutTick); + const afterByKey = new Map(after.map((participant) => [participant.key, participant])); + for (const participant of before) { + const next = afterByKey.get(participant.key); + if (!next) throw new Error(`Missing post-reconciliation participant: ${participant.key}`); + if (participant.policy === 'KEEP' && participant.checksum !== next.checksum) { + throw new Error(`KEEP participant changed during reconciliation: ${participant.key}`); + } + await db.clockReconciliationParticipant.upsert({ + where: { + suspensionId_participantKey: { + suspensionId: suspension.id, + participantKey: participant.key, + }, + }, + create: { + suspensionId: suspension.id, + participantKey: participant.key, + policy: participant.policy, + beforeChecksum: participant.checksum, + afterChecksum: next.checksum, + affectedCount: affected.get(participant.key) ?? 0, + }, + update: { + policy: participant.policy, + beforeChecksum: participant.checksum, + afterChecksum: next.checksum, + affectedCount: affected.get(participant.key) ?? 0, + }, + }); + } + const outboxPayload = { + version: 1, + profileName: options.authority.profileName, + suspensionId: suspension.id, + sourceRevision: plan.sourceRevision, + targetRevision: plan.targetRevision, + deadlineGeneration: safeNumber(targetGeneration, 'deadline generation'), + shiftTicks: plan.shiftTicks, + projectionDeltaMilliseconds, + clockBaseTime: world.clockBaseTime!.toISOString(), + ticksPerSecond: suspension.rateTicksPerSecond, + }; + await db.clockProjectionOutbox.create({ + data: { + worldStateId, + suspensionId: suspension.id, + targetRevision: BigInt(plan.targetRevision), + status: 'PENDING', + payload: asJson(outboxPayload), + checksum: checksum(outboxPayload), + }, + }); + await db.clockSuspension.update({ + where: { id: suspension.id }, + data: { + status: 'RECONCILING', + resumeWallAt, + gapTicks: BigInt(plan.gapTicks), + shiftTicks: BigInt(plan.shiftTicks), + alignedTick: BigInt(plan.alignedTick), + participantChecksumBefore: aggregateChecksum(before), + participantChecksumAfter: aggregateChecksum(after), + }, + }); + return { + suspensionId: suspension.id, + phase: 'RECONCILING', + sourceRevision: plan.sourceRevision, + targetRevision: plan.targetRevision, + deadlineGeneration: safeNumber(targetGeneration, 'deadline generation'), + gapTicks: plan.gapTicks, + catchUpTicks: plan.catchUpTicks, + shiftTicks: plan.shiftTicks, + alignedTick: plan.alignedTick, + resumeWallAt, + }; + }, + { isolationLevel: 'Serializable', maxWait: 10_000, timeout: 30_000 } + ); diff --git a/app/game-engine/src/turn/databaseHooks.ts b/app/game-engine/src/turn/databaseHooks.ts index 79442707..3103df06 100644 --- a/app/game-engine/src/turn/databaseHooks.ts +++ b/app/game-engine/src/turn/databaseHooks.ts @@ -1218,6 +1218,34 @@ export const createDatabaseTurnHooks = async ( `found ${durableClock.clock_phase}@${durableClock.clock_revision}/${durableClock.deadline_generation}.` ); } + if (commandCompletion) { + const commandFence = await prisma.$queryRaw< + Array<{ + status: string; + processing_clock_revision: bigint | null; + processing_deadline_generation: bigint | null; + }> + >(GamePrisma.sql` + SELECT status, + processing_clock_revision, + processing_deadline_generation + FROM input_event + WHERE request_id = ${commandCompletion.requestId} + AND target = 'ENGINE'::"InputEventTarget" + FOR UPDATE + `); + const event = commandFence[0]; + if ( + !event || + event.status !== 'PROCESSING' || + event.processing_clock_revision !== expectedRevision || + event.processing_deadline_generation !== expectedGeneration + ) { + throw new Error( + `Input event processing clock fence changed before commit: ${commandCompletion.requestId}.` + ); + } + } let neutralAuctionsToCreate = pendingNeutralAuctions; if (pendingNeutralAuctions.length > 0) { const latestRegistrationKey = diff --git a/app/game-engine/src/turn/runtimeClockShift.ts b/app/game-engine/src/turn/runtimeClockShift.ts index 5b730e81..e8311a3f 100644 --- a/app/game-engine/src/turn/runtimeClockShift.ts +++ b/app/game-engine/src/turn/runtimeClockShift.ts @@ -1,4 +1,4 @@ -import type { GamePrisma, GamePrismaClient } from '@sammo-ts/infra'; +import { readInputEventClockCoordinate, type GamePrisma, type GamePrismaClient } from '@sammo-ts/infra'; import { randomUUID } from 'node:crypto'; import { @@ -137,13 +137,20 @@ const ensureEngineCommand = async ( deltaMinutes, }; try { - await db.inputEvent.create({ - data: { - requestId, - target: 'ENGINE', - eventType: command.type, - payload: asJson(command), - }, + await db.$transaction(async (transaction) => { + const coordinate = await readInputEventClockCoordinate(transaction); + await transaction.inputEvent.create({ + data: { + requestId, + target: 'ENGINE', + eventType: command.type, + payload: asJson(command), + acceptedGameTick: coordinate.gameTick, + acceptedClockRevision: coordinate.clockRevision, + acceptedDeadlineGeneration: coordinate.deadlineGeneration, + createdAt: coordinate.wallAt, + }, + }); }); } catch (error) { if (!isUniqueConflict(error)) { diff --git a/app/game-engine/src/turn/runtimeGameSettings.ts b/app/game-engine/src/turn/runtimeGameSettings.ts index 43a186e9..74332427 100644 --- a/app/game-engine/src/turn/runtimeGameSettings.ts +++ b/app/game-engine/src/turn/runtimeGameSettings.ts @@ -10,7 +10,7 @@ import { type TurnDaemonCommand, type TurnDaemonCommandResult, } from '@sammo-ts/common'; -import type { GamePrisma, GamePrismaClient } from '@sammo-ts/infra'; +import { readInputEventClockCoordinate, type GamePrisma, type GamePrismaClient } from '@sammo-ts/infra'; import type { GatewayAdminActionRecord, GatewayAdminActionResult } from './gatewayAdminActions.js'; @@ -97,13 +97,20 @@ const ensureEngineCommand = async ( settings, }; try { - await db.inputEvent.create({ - data: { - requestId, - target: 'ENGINE', - eventType: command.type, - payload: asJson(command), - }, + await db.$transaction(async (transaction) => { + const coordinate = await readInputEventClockCoordinate(transaction); + await transaction.inputEvent.create({ + data: { + requestId, + target: 'ENGINE', + eventType: command.type, + payload: asJson(command), + acceptedGameTick: coordinate.gameTick, + acceptedClockRevision: coordinate.clockRevision, + acceptedDeadlineGeneration: coordinate.deadlineGeneration, + createdAt: coordinate.wallAt, + }, + }); }); } catch (error) { if (!isUniqueConflict(error)) throw error; diff --git a/app/game-engine/src/turn/selectPoolService.ts b/app/game-engine/src/turn/selectPoolService.ts index 7ad581db..9352404e 100644 --- a/app/game-engine/src/turn/selectPoolService.ts +++ b/app/game-engine/src/turn/selectPoolService.ts @@ -439,15 +439,20 @@ export const reserveSelectionPool = async (options: { userId: string; now?: Date; acceptedGameTick?: number; + processingGameTick?: number; seedOwnerIdentity?: string | number; }): Promise => { const { db, world, worldState, userId } = options; requirePoolWorld(worldState); const now = options.now ?? new Date(); const acceptedGameTick = options.acceptedGameTick ?? resolveAcceptedGameTick(world, now); + const processingGameTick = options.processingGameTick ?? acceptedGameTick; if (!Number.isSafeInteger(acceptedGameTick)) { fail('INTERNAL_SERVER_ERROR', '장수 선택 예약 tick이 안전한 정수 범위를 벗어났습니다.'); } + if (!Number.isSafeInteger(processingGameTick)) { + fail('INTERNAL_SERVER_ERROR', '장수 선택 처리 tick이 안전한 정수 범위를 벗어났습니다.'); + } await lockSelectionUser(db, userId); await lockSelectionMutationTables(db); const general = await db.general.findFirst({ @@ -461,7 +466,7 @@ export const reserveSelectionPool = async (options: { let currentRows = await synchronizeSelectionPoolWorld(db, world); const existing = currentRows.filter( - (row) => row.ownerUserId === userId && row.generalId === null && isReservationActive(row, now, acceptedGameTick) + (row) => row.ownerUserId === userId && row.generalId === null && isReservationActive(row, now, processingGameTick) ); if (existing.length > 0) { return toReservationDto(existing, Boolean(general), worldState, world); @@ -471,7 +476,7 @@ export const reserveSelectionPool = async (options: { where: { generalId: null, OR: [ - { reservedUntilTick: { lt: BigInt(acceptedGameTick) } }, + { reservedUntilTick: { lt: BigInt(processingGameTick) } }, { reservedUntilTick: null, reservedUntil: { lt: now } }, ], }, @@ -483,7 +488,7 @@ export const reserveSelectionPool = async (options: { }); currentRows = await synchronizeSelectionPoolWorld(db, world); const availableIds = new Set( - world.listGeneralPoolCandidates(now, acceptedGameTick)?.map((candidate) => candidate.poolEntryId) ?? [] + world.listGeneralPoolCandidates(now, processingGameTick)?.map((candidate) => candidate.poolEntryId) ?? [] ); const available = currentRows.filter( (row) => @@ -507,7 +512,7 @@ export const reserveSelectionPool = async (options: { (row) => [row, calculateSelectionCandidateWeight(poolName, parseCandidate(row), true)] as [SelectPoolRow, number] ); - const reservedUntilTick = acceptedGameTick + RESERVATION_TURN_MULTIPLIER * GAME_TICKS_PER_TURN; + const reservedUntilTick = processingGameTick + RESERVATION_TURN_MULTIPLIER * GAME_TICKS_PER_TURN; if (!Number.isSafeInteger(reservedUntilTick)) { fail('INTERNAL_SERVER_ERROR', '장수 선택 예약 tick이 안전한 정수 범위를 벗어났습니다.'); } diff --git a/app/game-engine/src/turn/tournamentAutoStart.ts b/app/game-engine/src/turn/tournamentAutoStart.ts index 8da5370f..666edacc 100644 --- a/app/game-engine/src/turn/tournamentAutoStart.ts +++ b/app/game-engine/src/turn/tournamentAutoStart.ts @@ -14,8 +14,12 @@ interface TournamentState { openMonth: number; termSeconds: number; nextAt: string; + nextTick?: number; + clockRevision?: number; + deadlineGeneration?: number; bettingId?: number; bettingCloseAt?: string; + bettingCloseTick?: number; winnerId?: number; bettingSettled?: boolean; rewardSettled?: boolean; @@ -76,6 +80,9 @@ export const createTournamentAutoStartHandler = (options: { sourceRevisionKey: `sammo:${options.profileName}:tournament:source-revision`, sourceRevisionChannel: `sammo:${options.profileName}:tournament:source-changed`, realtimeEventChannel: buildGameEventChannel(options.profileName), + activeClockRevisionKey: `sammo:${options.profileName}:clock:active-revision`, + deadlineGenerationKey: `sammo:${options.profileName}:clock:deadline-generation`, + clockPhaseKey: `sammo:${options.profileName}:clock:phase`, }; return { onMonthChanged: async (context) => { @@ -118,6 +125,8 @@ export const createTournamentAutoStartHandler = (options: { previousState && Number.isFinite(previousState.termSeconds) && previousState.termSeconds > 0 ? previousState.termSeconds : resolveTermSeconds(state.tickSeconds); + const nextAt = new Date(now.getTime() + termSeconds * 60_000); + const clockState = world.getGameClockState(); const nextState: TournamentState = { stage: 1, phase: 0, @@ -129,7 +138,10 @@ export const createTournamentAutoStartHandler = (options: { // Ref startTournament() passes calcTournamentTerm()'s seconds // value to DateInterval's minute field. Preserve that historical // initial enrollment delay; later tournament phases use seconds. - nextAt: new Date(now.getTime() + termSeconds * 60_000).toISOString(), + nextAt: nextAt.toISOString(), + nextTick: world.dateToGameTick(nextAt), + clockRevision: clockState.revision, + deadlineGeneration: clockState.deadlineGeneration, bettingId: typeof previousState?.bettingId === 'number' && Number.isFinite(previousState.bettingId) ? previousState.bettingId + 1 @@ -142,12 +154,26 @@ export const createTournamentAutoStartHandler = (options: { lastError: undefined, lastErrorAt: undefined, }; - await writeTournamentProjection(redis, keys, [ - { key: keys.participantsKey, value: [] }, - { key: keys.matchesKey, value: [] }, - { key: keys.bettingKey, value: [] }, - { key: keys.stateKey, value: nextState }, - ]); + await writeTournamentProjection( + redis, + keys, + [ + { key: keys.participantsKey, value: [] }, + { key: keys.matchesKey, value: [] }, + { key: keys.bettingKey, value: [] }, + { key: keys.stateKey, value: nextState }, + ], + clockState.phase === 'RUNNING' + ? { + activeRevisionKey: keys.activeClockRevisionKey, + deadlineGenerationKey: keys.deadlineGenerationKey, + phaseKey: keys.clockPhaseKey, + revision: clockState.revision, + deadlineGeneration: clockState.deadlineGeneration, + phase: 'RUNNING', + } + : undefined + ); const [typeText, generalTypeText] = TOURNAMENT_TEXT[type] ?? TOURNAMENT_TEXT[0]; const emperor = world diff --git a/app/game-engine/src/turn/worldCommandHandler.ts b/app/game-engine/src/turn/worldCommandHandler.ts index 829cdef6..2a4b29c9 100644 --- a/app/game-engine/src/turn/worldCommandHandler.ts +++ b/app/game-engine/src/turn/worldCommandHandler.ts @@ -274,6 +274,10 @@ const resolveSelectionCommandAcceptedAt = async ( command: Extract ): Promise => { const operationalAcceptedAt = await resolveCommandAcceptedAt(db, command); + const processingGameTick = Reflect.get(command, 'processingGameTick'); + if (typeof processingGameTick === 'number' && Number.isSafeInteger(processingGameTick)) { + return world.gameTickToDate(processingGameTick); + } if (command.acceptedGameTick !== undefined) { return world.gameTickToDate(command.acceptedGameTick); } @@ -407,9 +411,13 @@ async function handleNpcPossessGeneral( throw new Error('NPC possession world state is missing.'); } const operationalAcceptedAt = await resolveCommandAcceptedAt(db, command); - const acceptedAt = command.acceptedGameAt - ? new Date(command.acceptedGameAt) - : ctx.world.getGameNow(operationalAcceptedAt); + const processingGameTick = Reflect.get(command, 'processingGameTick'); + const acceptedAt = + typeof processingGameTick === 'number' && Number.isSafeInteger(processingGameTick) + ? ctx.world.gameTickToDate(processingGameTick) + : command.acceptedGameAt + ? new Date(command.acceptedGameAt) + : ctx.world.getGameNow(operationalAcceptedAt); try { return { type: 'npcPossessGeneral', @@ -451,8 +459,11 @@ async function handleSelectPoolCreate( throw new Error('Selection-pool world state is missing.'); } const operationalAcceptedAt = await resolveCommandAcceptedAt(db, command); + const processingGameTick = Reflect.get(command, 'processingGameTick'); const acceptedAt = - command.acceptedGameTick !== undefined + typeof processingGameTick === 'number' && Number.isSafeInteger(processingGameTick) + ? ctx.world.gameTickToDate(processingGameTick) + : command.acceptedGameTick !== undefined ? ctx.world.gameTickToDate(command.acceptedGameTick) : command.acceptedGameAt !== undefined ? new Date(command.acceptedGameAt) @@ -515,6 +526,9 @@ async function handleSelectPoolReserve( seedOwnerIdentity: command.seedOwnerIdentity, now: acceptedAt, ...(command.acceptedGameTick === undefined ? {} : { acceptedGameTick: command.acceptedGameTick }), + ...(typeof Reflect.get(command, 'processingGameTick') === 'number' + ? { processingGameTick: Reflect.get(command, 'processingGameTick') as number } + : {}), }), }; } catch (error) { @@ -2770,9 +2784,15 @@ const validateVoteSelectionInTransaction = async ( if (!poll) return '설문조사가 없습니다.'; const processingNow = ctx.world.getGameNow(new Date()); - const acceptedGameTick = command.acceptedGameTick ?? ctx.world.dateToGameTick(processingNow); + const convertedProcessingTick = Reflect.get(command, 'processingGameTick'); + const acceptedGameTick = + typeof convertedProcessingTick === 'number' && Number.isSafeInteger(convertedProcessingTick) + ? convertedProcessingTick + : (command.acceptedGameTick ?? ctx.world.dateToGameTick(processingNow)); const acceptedGameAt = - command.acceptedGameTick === undefined ? processingNow : ctx.world.gameTickToDate(command.acceptedGameTick); + command.acceptedGameTick === undefined && convertedProcessingTick === undefined + ? processingNow + : ctx.world.gameTickToDate(acceptedGameTick); if (hasVotePollDeadlinePassed(poll, acceptedGameAt, acceptedGameTick)) { return '설문조사가 종료되었습니다.'; } diff --git a/app/game-engine/test/clockReconciliation.integration.test.ts b/app/game-engine/test/clockReconciliation.integration.test.ts new file mode 100644 index 00000000..3cf349b6 --- /dev/null +++ b/app/game-engine/test/clockReconciliation.integration.test.ts @@ -0,0 +1,364 @@ +import { afterAll, beforeAll, beforeEach, describe, expect, it } from 'vitest'; + +import { GameClock } from '@sammo-ts/common'; +import { + createGamePostgresConnector, + createRedisConnector, + type GamePrismaClient, + type RedisConnector, +} from '@sammo-ts/infra'; + +import { reconcileClockSuspension, startClockSuspension } from '../src/turn/clockReconciliation.js'; +import { applyNextClockProjection } from '../src/turn/clockProjectionOutbox.js'; + +const enabled = + process.env.CLOCK_RECONCILIATION_INTEGRATION === '1' && + Boolean(process.env.DATABASE_URL) && + Boolean(process.env.REDIS_URL); +const describeIntegration = enabled ? describe : describe.skip; + +describeIntegration('durable clock reconciliation', () => { + let db: GamePrismaClient; + let disconnect: (() => Promise) | undefined; + let redis: RedisConnector; + + beforeAll(async () => { + const connector = createGamePostgresConnector({ url: process.env.DATABASE_URL! }); + db = connector.prisma; + disconnect = connector.disconnect; + redis = createRedisConnector({ url: process.env.REDIS_URL! }); + await redis.connect(); + }); + + afterAll(async () => { + await redis.disconnect(); + await disconnect?.(); + }); + + beforeEach(async () => { + await redis.client.flushDb(); + await db.$transaction([ + db.clockProjectionOutbox.deleteMany(), + db.clockReconciliationParticipant.deleteMany(), + db.clockSuspension.deleteMany(), + db.inputEvent.deleteMany(), + db.vote.deleteMany(), + db.voteComment.deleteMany(), + db.votePoll.deleteMany(), + db.message.deleteMany(), + db.auctionBid.deleteMany(), + db.auction.deleteMany(), + db.npcSelectionToken.deleteMany(), + db.selectPoolEntry.deleteMany(), + db.general.deleteMany(), + db.turnDaemonLease.deleteMany(), + db.worldState.deleteMany(), + ]); + }); + + it('preserves every remaining deadline and occurrence across a 65m17.250s exact gap', async () => { + const baseTime = new Date('2026-01-01T00:00:00.000Z'); + const futureAnchor = new Date(Date.now() + 3_600_000); + const initialTick = 1_000_000; + const lastTurnTick = 900_000; + const clock = new GameClock({ + baseTime, + tick: initialTick, + mode: 'realtime', + wallAnchor: futureAnchor, + turnSeconds: 600, + phase: 'RUNNING', + revision: 1, + }); + const generalTicks = [initialTick + 1_234, initialTick + 36_000_123]; + const auctionCloseTick = initialTick + 72_000_777; + const messageOccurrenceTick = initialTick - 500; + const messageExpiryTick = initialTick + 90_000_999; + const voteStartTick = initialTick - 200; + const voteEndTick = initialTick + 18_000_321; + const poolTick = initialTick + 2_000_111; + const npcValidTick = initialTick + 3_000_222; + const npcMoreTick = initialTick + 1_000_333; + + const world = await db.worldState.create({ + data: { + scenarioCode: 'clock-test', + currentYear: 180, + currentMonth: 1, + tickSeconds: 600, + clockBaseTime: baseTime, + clockTick: BigInt(initialTick), + clockMode: 'realtime', + clockWallAnchor: futureAnchor, + lastTurnTick: BigInt(lastTurnTick), + clockPhase: 'RUNNING', + clockRevision: 1n, + deadlineGeneration: 7n, + meta: { + lastTurnTime: clock.tickToDate(lastTurnTick).toISOString(), + starttime: clock.tickToDate(initialTick + 100).toISOString(), + }, + }, + }); + await db.general.createMany({ + data: generalTicks.map((turnTick, index) => ({ + id: index + 1, + name: `general-${index + 1}`, + turnTick: BigInt(turnTick), + turnTime: clock.tickToDate(turnTick), + recentWarTick: BigInt(initialTick - 100 - index), + recentWarTime: clock.tickToDate(initialTick - 100 - index), + })), + }); + await db.auction.create({ + data: { + type: 'BUY_RICE', + hostGeneralId: 1, + status: 'FINALIZING', + openTick: BigInt(initialTick - 300), + closeTick: BigInt(auctionCloseTick), + closeAt: clock.tickToDate(auctionCloseTick), + }, + }); + await db.message.create({ + data: { + mailbox: 1, + type: 'private', + src: 1, + dest: 2, + time: clock.tickToDate(messageOccurrenceTick), + timeTick: BigInt(messageOccurrenceTick), + validUntil: clock.tickToDate(messageExpiryTick), + validUntilTick: BigInt(messageExpiryTick), + message: {}, + }, + }); + await db.votePoll.create({ + data: { + title: 'clock vote', + options: ['yes', 'no'], + revealMode: 'AFTER_VOTE', + openerGeneralId: 1, + openerName: 'general-1', + startAt: clock.tickToDate(voteStartTick), + startTick: BigInt(voteStartTick), + endAt: clock.tickToDate(voteEndTick), + endTick: BigInt(voteEndTick), + }, + }); + await db.selectPoolEntry.create({ + data: { + uniqueName: 'clock-pool', + reservedUntil: clock.tickToDate(poolTick), + reservedUntilTick: BigInt(poolTick), + info: {}, + }, + }); + await db.npcSelectionToken.create({ + data: { + ownerUserId: 'clock-user', + validUntil: clock.tickToDate(npcValidTick), + validUntilTick: BigInt(npcValidTick), + pickMoreFrom: clock.tickToDate(npcMoreTick), + pickMoreFromTick: BigInt(npcMoreTick), + pickResult: [], + nonce: 1, + }, + }); + + const authority = { kind: 'OFFLINE' as const, profileName: 'clock-test', reason: 'integration fixture' }; + const suspended = await startClockSuspension({ + db, + suspensionId: 'clock-gap-65m17s250', + source: 'MAINTENANCE', + authority, + }); + expect(suspended.cutTick).toBe(initialTick); + expect((await db.worldState.findUniqueOrThrow({ where: { id: world.id } })).clockPhase).toBe('SUSPENDED'); + + const resumeWallAt = new Date(suspended.cutWallAt.getTime() + 65 * 60_000 + 17_250); + const reconciled = await reconcileClockSuspension({ + db, + suspensionId: suspended.suspensionId, + authority, + testResumeWallAt: resumeWallAt, + }); + expect(reconciled).toMatchObject({ + phase: 'RECONCILING', + sourceRevision: 1, + targetRevision: 2, + deadlineGeneration: 8, + gapTicks: 235_035_000, + shiftTicks: 235_035_000, + alignedTick: 236_035_000, + }); + + const [afterWorld, generals, auction, message, vote, pool, token, ledger, outboxes] = await Promise.all([ + db.worldState.findUniqueOrThrow({ where: { id: world.id } }), + db.general.findMany({ orderBy: { id: 'asc' } }), + db.auction.findFirstOrThrow(), + db.message.findFirstOrThrow(), + db.votePoll.findFirstOrThrow(), + db.selectPoolEntry.findFirstOrThrow(), + db.npcSelectionToken.findFirstOrThrow(), + db.clockSuspension.findUniqueOrThrow({ where: { id: suspended.suspensionId } }), + db.clockProjectionOutbox.findMany(), + ]); + const alignedTick = BigInt(reconciled.alignedTick); + expect(afterWorld).toMatchObject({ + clockPhase: 'RECONCILING', + clockRevision: 2n, + deadlineGeneration: 8n, + clockTick: alignedTick, + lastTurnTick: BigInt(lastTurnTick + reconciled.shiftTicks), + }); + expect(generals.map((general) => general.turnTick! - alignedTick)).toEqual( + generalTicks.map((tick) => BigInt(tick - initialTick)) + ); + expect(auction.closeTick! - alignedTick).toBe(BigInt(auctionCloseTick - initialTick)); + expect(message.validUntilTick! - alignedTick).toBe(BigInt(messageExpiryTick - initialTick)); + expect(vote.endTick! - alignedTick).toBe(BigInt(voteEndTick - initialTick)); + expect(pool.reservedUntilTick! - alignedTick).toBe(BigInt(poolTick - initialTick)); + expect(token.validUntilTick! - alignedTick).toBe(BigInt(npcValidTick - initialTick)); + expect(token.pickMoreFromTick! - alignedTick).toBe(BigInt(npcMoreTick - initialTick)); + expect(generals.map((general) => general.recentWarTick)).toEqual([ + BigInt(initialTick - 100), + BigInt(initialTick - 101), + ]); + expect(auction.openTick).toBe(BigInt(initialTick - 300)); + expect(message.timeTick).toBe(BigInt(messageOccurrenceTick)); + expect(vote.startTick).toBe(BigInt(voteStartTick)); + expect(ledger.status).toBe('RECONCILING'); + expect(outboxes).toHaveLength(1); + expect(outboxes[0]).toMatchObject({ status: 'PENDING', targetRevision: 2n }); + + const retried = await reconcileClockSuspension({ + db, + suspensionId: suspended.suspensionId, + authority, + testResumeWallAt: new Date(resumeWallAt.getTime() + 10_000), + }); + expect(retried).toEqual(reconciled); + expect(await db.clockProjectionOutbox.count()).toBe(1); + const keepParticipants = await db.clockReconciliationParticipant.findMany({ where: { policy: 'KEEP' } }); + expect(keepParticipants.every((participant) => participant.beforeChecksum === participant.afterChecksum)).toBe( + true + ); + + await redis.client.set('sammo:clock-test:clock:active-revision', '1'); + expect( + await applyNextClockProjection({ db, redis: redis.client, workerId: 'clock-projection-success' }) + ).toBe('APPLIED'); + expect(await redis.client.get('sammo:clock-test:clock:active-revision')).toBe('2'); + expect(await redis.client.get('sammo:clock-test:clock:deadline-generation')).toBe('8'); + expect(await redis.client.get('sammo:clock-test:clock:phase')).toBe('RUNNING'); + expect(await redis.client.zRangeWithScores('sammo:clock-test:auction:timer', 0, -1)).toEqual([ + { value: String(auction.id), score: Number(auction.closeTick) }, + ]); + expect(await db.worldState.findUniqueOrThrow({ where: { id: world.id } })).toMatchObject({ + clockPhase: 'RUNNING', + clockRevision: 2n, + }); + expect(await db.clockProjectionOutbox.findFirstOrThrow()).toMatchObject({ status: 'APPLIED' }); + }); + + it('rejects a live offline fence and preserves a turn deadline across an exact 24-hour gap', async () => { + const baseTime = new Date('2026-02-01T00:00:00.000Z'); + const futureAnchor = new Date(Date.now() + 3_600_000); + const initialTick = 5 * 36_000_000; + const turnTick = initialTick + 17_000_007; + const clock = new GameClock({ + baseTime, + tick: initialTick, + mode: 'realtime', + wallAnchor: futureAnchor, + turnSeconds: 3_600, + phase: 'RUNNING', + }); + await db.worldState.create({ + data: { + scenarioCode: 'clock-day-test', + currentYear: 180, + currentMonth: 1, + tickSeconds: 3_600, + clockBaseTime: baseTime, + clockTick: BigInt(initialTick), + clockMode: 'realtime', + clockWallAnchor: futureAnchor, + lastTurnTick: BigInt(initialTick), + clockPhase: 'RUNNING', + clockRevision: 3n, + deadlineGeneration: 2n, + }, + }); + await db.general.create({ + data: { id: 1, name: 'day-general', turnTick: BigInt(turnTick), turnTime: clock.tickToDate(turnTick) }, + }); + await db.turnDaemonLease.create({ + data: { + profile: 'clock-day-test', + ownerId: 'other-daemon', + fencingEpoch: 9n, + leaseUntil: new Date(Date.now() + 60_000), + }, + }); + const authority = { + kind: 'OFFLINE' as const, + profileName: 'clock-day-test', + reason: '24-hour integration fixture', + }; + await expect( + startClockSuspension({ + db, + suspensionId: 'clock-gap-24h', + source: 'MAINTENANCE', + authority, + }) + ).rejects.toThrow('daemon lease to be offline'); + await db.turnDaemonLease.delete({ where: { profile: 'clock-day-test' } }); + const suspended = await startClockSuspension({ + db, + suspensionId: 'clock-gap-24h', + source: 'MAINTENANCE', + authority, + }); + const reconciled = await reconcileClockSuspension({ + db, + suspensionId: suspended.suspensionId, + authority, + testResumeWallAt: new Date(suspended.cutWallAt.getTime() + 24 * 60 * 60_000), + }); + + expect(reconciled).toMatchObject({ + sourceRevision: 3, + targetRevision: 4, + gapTicks: 24 * 36_000_000, + shiftTicks: 24 * 36_000_000, + alignedTick: initialTick + 24 * 36_000_000, + }); + const shifted = await db.general.findUniqueOrThrow({ where: { id: 1 } }); + expect(shifted.turnTick! - BigInt(reconciled.alignedTick)).toBe(BigInt(turnTick - initialTick)); + + await redis.client.set('sammo:clock-day-test:clock:active-revision', '3'); + const redisThenCrash = { + get: redis.client.get.bind(redis.client), + eval: async (script: string, options: { keys: string[]; arguments: string[] }) => { + await redis.client.eval(script, options); + throw new Error('fixture crash after Redis commit'); + }, + }; + await expect( + applyNextClockProjection({ db, redis: redisThenCrash, workerId: 'clock-projection-crash' }) + ).rejects.toThrow('fixture crash after Redis commit'); + expect(await redis.client.get('sammo:clock-day-test:clock:active-revision')).toBe('4'); + expect(await db.worldState.findFirstOrThrow()).toMatchObject({ clockPhase: 'RECONCILING' }); + expect(await db.clockProjectionOutbox.findFirstOrThrow()).toMatchObject({ status: 'FAILED', attempts: 1 }); + + await db.clockProjectionOutbox.updateMany({ data: { availableAt: new Date(0) } }); + expect( + await applyNextClockProjection({ db, redis: redis.client, workerId: 'clock-projection-restart' }) + ).toBe('RECOVERED'); + expect(await db.worldState.findFirstOrThrow()).toMatchObject({ clockPhase: 'RUNNING', clockRevision: 4n }); + expect(await db.clockProjectionOutbox.findFirstOrThrow()).toMatchObject({ status: 'APPLIED', attempts: 2 }); + }); +}); diff --git a/app/game-engine/test/databaseCommandQueue.integration.test.ts b/app/game-engine/test/databaseCommandQueue.integration.test.ts index 0ca4a18a..d1714621 100644 --- a/app/game-engine/test/databaseCommandQueue.integration.test.ts +++ b/app/game-engine/test/databaseCommandQueue.integration.test.ts @@ -1,4 +1,4 @@ -import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest'; +import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'; import type { TurnDaemonCommand } from '@sammo-ts/common'; import { createGamePostgresConnector } from '@sammo-ts/infra'; @@ -25,6 +25,12 @@ integration('database command queue', () => { }); }); + beforeEach(async () => { + await db.inputEvent.deleteMany({ where: { requestId: { startsWith: 'integration:engine:' } } }); + await db.clockSuspension.deleteMany({ where: { id: 'integration-queue-revision-8-9' } }); + await db.worldState.updateMany({ data: { clockPhase: 'RUNNING' } }); + }); + afterAll(async () => { await db.inputEvent.deleteMany({ where: { requestId: { startsWith: 'integration:engine:' } }, @@ -229,4 +235,119 @@ integration('database command queue', () => { expect(handle).toHaveBeenCalledOnce(); expect(mutation).not.toHaveBeenCalled(); }); + + it('dequeues gameplay only in an executable phase and records the processing clock generation', async () => { + const existingWorld = await db.worldState.findFirst({ orderBy: { id: 'asc' } }); + const world = existingWorld + ? await db.worldState.update({ + where: { id: existingWorld.id }, + data: { clockPhase: 'SUSPENDED', clockRevision: 9n, deadlineGeneration: 4n, clockTick: 123n }, + }) + : await db.worldState.create({ + data: { + scenarioCode: 'queue-clock-test', + currentYear: 180, + currentMonth: 1, + tickSeconds: 600, + clockPhase: 'SUSPENDED', + clockRevision: 9n, + deadlineGeneration: 4n, + clockTick: 123n, + }, + }); + const gameplayId = 'integration:engine:clock-gated-gameplay'; + const statusId = 'integration:engine:clock-gated-status'; + const staleId = 'integration:engine:clock-gated-stale'; + await db.inputEvent.createMany({ + data: [ + { + requestId: gameplayId, + target: 'ENGINE', + eventType: 'vacation', + actorUserId: 'user-7', + acceptedGameTick: 100n, + acceptedClockRevision: 9n, + acceptedDeadlineGeneration: 4n, + payload: { type: 'vacation', requestId: gameplayId, userId: 'user-7', generalId: 7 }, + }, + { + requestId: statusId, + target: 'ENGINE', + eventType: 'getStatus', + acceptedGameTick: 100n, + acceptedClockRevision: 9n, + acceptedDeadlineGeneration: 4n, + payload: { type: 'getStatus', requestId: statusId }, + }, + { + requestId: staleId, + target: 'ENGINE', + eventType: 'vacation', + actorUserId: 'user-8', + acceptedGameTick: 90n, + acceptedClockRevision: 8n, + acceptedDeadlineGeneration: 3n, + payload: { type: 'vacation', requestId: staleId, userId: 'user-8', generalId: 8 }, + }, + ], + }); + const queue = new DatabaseTurnDaemonCommandQueue(db); + + expect(await queue.drain()).toEqual([{ type: 'getStatus', requestId: statusId }]); + expect(await db.inputEvent.findUniqueOrThrow({ where: { requestId: gameplayId } })).toMatchObject({ + status: 'PENDING', + processingClockRevision: null, + }); + await db.worldState.update({ where: { id: world.id }, data: { clockPhase: 'RUNNING' } }); + + expect(await queue.drain()).toEqual([ + { type: 'vacation', requestId: gameplayId, userId: 'user-7', generalId: 7 }, + ]); + expect(await db.inputEvent.findUniqueOrThrow({ where: { requestId: gameplayId } })).toMatchObject({ + status: 'PROCESSING', + processingGameTick: 100n, + processingClockRevision: 9n, + processingDeadlineGeneration: 4n, + }); + expect(await db.inputEvent.findUniqueOrThrow({ where: { requestId: staleId } })).toMatchObject({ + status: 'PENDING', + processingClockRevision: null, + }); + await db.clockSuspension.deleteMany({ where: { id: 'integration-queue-revision-8-9' } }); + await db.clockSuspension.create({ + data: { + id: 'integration-queue-revision-8-9', + worldStateId: world.id, + source: 'MAINTENANCE', + policy: 'EXACT', + status: 'APPLIED', + sourceRevision: 8n, + targetRevision: 9n, + cutTick: 90n, + cutWallAt: new Date(), + resumeWallAt: new Date(), + rateTicksPerSecond: 60_000, + gapTicks: 33n, + shiftTicks: 33n, + alignedTick: 123n, + }, + }); + expect(await queue.drain()).toEqual([ + { + type: 'vacation', + requestId: staleId, + userId: 'user-8', + generalId: 8, + processingGameTick: 123, + }, + ]); + expect(await db.inputEvent.findUniqueOrThrow({ where: { requestId: staleId } })).toMatchObject({ + status: 'PROCESSING', + acceptedGameTick: 90n, + acceptedClockRevision: 8n, + processingGameTick: 123n, + processingClockRevision: 9n, + processingDeadlineGeneration: 4n, + }); + }); }); diff --git a/app/game-engine/test/runtimeClockShift.test.ts b/app/game-engine/test/runtimeClockShift.test.ts index a3cf08a3..097cb22c 100644 --- a/app/game-engine/test/runtimeClockShift.test.ts +++ b/app/game-engine/test/runtimeClockShift.test.ts @@ -448,6 +448,21 @@ describe('runtime clock shift projection', () => { return {}; }); const db = { + $transaction: async (operation: (transaction: GamePrismaClient) => Promise) => operation(db), + $executeRaw: vi.fn(async () => 1), + $queryRaw: vi.fn(async () => [{ wallNow: new Date('2026-07-30T10:00:00.000Z') }]), + worldState: { + findFirst: vi.fn(async () => ({ + clockBaseTime: new Date('2026-07-30T10:00:00.000Z'), + clockTick: 0n, + clockMode: 'realtime', + clockWallAnchor: new Date('2026-07-30T10:00:00.000Z'), + tickSeconds: 600, + clockPhase: 'RUNNING', + clockRevision: 1n, + deadlineGeneration: 1n, + })), + }, inputEvent: { create: inputEventCreate, findUniqueOrThrow: vi.fn(async () => @@ -567,6 +582,21 @@ describe('runtime game settings projection', () => { let eventStatus: 'PENDING' | 'SUCCEEDED' = 'PENDING'; let created = false; const db = { + $transaction: async (operation: (transaction: GamePrismaClient) => Promise) => operation(db), + $executeRaw: vi.fn(async () => 1), + $queryRaw: vi.fn(async () => [{ wallNow: new Date('2026-07-30T10:00:00.000Z') }]), + worldState: { + findFirst: vi.fn(async () => ({ + clockBaseTime: new Date('2026-07-30T10:00:00.000Z'), + clockTick: 0n, + clockMode: 'realtime', + clockWallAnchor: new Date('2026-07-30T10:00:00.000Z'), + tickSeconds: 600, + clockPhase: 'RUNNING', + clockRevision: 1n, + deadlineGeneration: 1n, + })), + }, inputEvent: { create: vi.fn(async () => { if (created) throw { code: 'P2002' }; diff --git a/app/game-engine/test/runtimeClockShiftPersistence.integration.test.ts b/app/game-engine/test/runtimeClockShiftPersistence.integration.test.ts index f7c49205..6bdd5f90 100644 --- a/app/game-engine/test/runtimeClockShiftPersistence.integration.test.ts +++ b/app/game-engine/test/runtimeClockShiftPersistence.integration.test.ts @@ -1,4 +1,4 @@ -import { afterAll, beforeAll, describe, expect, it } from 'vitest'; +import { afterAll, beforeAll, beforeEach, describe, expect, it } from 'vitest'; import { createGamePostgresConnector, type GamePrisma, type GamePrismaClient } from '@sammo-ts/infra'; import { GAME_TICKS_PER_TURN } from '@sammo-ts/common'; @@ -77,38 +77,32 @@ integration('runtime clock shift persistence', () => { let db: GamePrismaClient; let closeDb: (() => Promise) | undefined; + const cleanupFixtures = async (): Promise => { + await db.inputEvent.deleteMany({ where: { requestId: { in: [requestId, runtimeSettingsRequestId] } } }); + await db.votePoll.deleteMany({ where: { openerGeneralId: generalIds[2] } }); + await db.message.deleteMany({ where: { mailbox: generalIds[2] } }); + await db.logEntry.deleteMany({ where: { text: runtimeSettingsLogText } }); + await db.auction.deleteMany({ where: { hostGeneralId: { in: [...generalIds] } } }); + await db.selectPoolEntry.deleteMany({ where: { uniqueName: backlogPoolUniqueName } }); + await db.general.deleteMany({ where: { id: { in: [...generalIds] } } }); + await db.worldState.deleteMany({ + where: { + scenarioCode: { in: ['runtime-clock-shift', 'runtime-game-settings', 'realtime-backlog-rebase'] }, + }, + }); + }; + beforeAll(async () => { const connector = createGamePostgresConnector({ url: databaseUrl! }); await connector.connect(); db = connector.prisma; closeDb = () => connector.disconnect(); - await db.inputEvent.deleteMany({ where: { requestId: { in: [requestId, runtimeSettingsRequestId] } } }); - await db.votePoll.deleteMany({ where: { openerGeneralId: generalIds[2] } }); - await db.message.deleteMany({ where: { mailbox: generalIds[2] } }); - await db.logEntry.deleteMany({ where: { text: runtimeSettingsLogText } }); - await db.auction.deleteMany({ where: { hostGeneralId: { in: [...generalIds] } } }); - await db.selectPoolEntry.deleteMany({ where: { uniqueName: backlogPoolUniqueName } }); - await db.general.deleteMany({ where: { id: { in: [...generalIds] } } }); - await db.worldState.deleteMany({ - where: { - scenarioCode: { in: ['runtime-clock-shift', 'runtime-game-settings', 'realtime-backlog-rebase'] }, - }, - }); }); + beforeEach(cleanupFixtures); + afterAll(async () => { - await db.inputEvent.deleteMany({ where: { requestId: { in: [requestId, runtimeSettingsRequestId] } } }); - await db.votePoll.deleteMany({ where: { openerGeneralId: generalIds[2] } }); - await db.message.deleteMany({ where: { mailbox: generalIds[2] } }); - await db.logEntry.deleteMany({ where: { text: runtimeSettingsLogText } }); - await db.auction.deleteMany({ where: { hostGeneralId: { in: [...generalIds] } } }); - await db.selectPoolEntry.deleteMany({ where: { uniqueName: backlogPoolUniqueName } }); - await db.general.deleteMany({ where: { id: { in: [...generalIds] } } }); - await db.worldState.deleteMany({ - where: { - scenarioCode: { in: ['runtime-clock-shift', 'runtime-game-settings', 'realtime-backlog-rebase'] }, - }, - }); + await cleanupFixtures(); await closeDb?.(); }); @@ -469,6 +463,7 @@ integration('runtime clock shift persistence', () => { clockBaseTime: base, clockTick: 0, clockMode: 'manual', + clockPhase: 'MANUAL', clockWallAnchor: base, lastTurnTick: 0, config: { turnTermMinutes: 10, blockGeneralCreate: 0 }, diff --git a/app/gateway-api/test/releaseManifest.test.ts b/app/gateway-api/test/releaseManifest.test.ts index 8ec6a71a..b343fa08 100644 --- a/app/gateway-api/test/releaseManifest.test.ts +++ b/app/gateway-api/test/releaseManifest.test.ts @@ -39,7 +39,7 @@ describe('readReleaseManifest', () => { await expect(readReleaseManifest(workspaceRoot)).resolves.toMatchObject({ controllerProtocol: RELEASE_CONTROLLER_PROTOCOL, gatewaySchemaHead: '20260825000000_add_bulk_release_batches', - gameSchemaHead: '20260903090000_add_game_clock_reconciliation', + gameSchemaHead: '20260903103000_add_input_event_clock_processing', }); }); diff --git a/docs/architecture/game-clock-participants.json b/docs/architecture/game-clock-participants.json index f0835457..b85bbbb4 100644 --- a/docs/architecture/game-clock-participants.json +++ b/docs/architecture/game-clock-participants.json @@ -6,6 +6,10 @@ "coveredFields": [ "input_event.accepted_game_tick", "input_event.accepted_clock_revision", + "input_event.accepted_deadline_generation", + "input_event.processing_game_tick", + "input_event.processing_clock_revision", + "input_event.processing_deadline_generation", "world_state.clock_tick", "world_state.last_turn_tick", "world_state.clock_revision", @@ -125,8 +129,16 @@ { "key": "accepted-command-coordinate", "policy": "KEEP", - "authorityFields": ["input_event.accepted_game_tick", "input_event.accepted_clock_revision"], - "projectionFields": [], + "authorityFields": [ + "input_event.accepted_game_tick", + "input_event.accepted_clock_revision", + "input_event.accepted_deadline_generation" + ], + "projectionFields": [ + "input_event.processing_game_tick", + "input_event.processing_clock_revision", + "input_event.processing_deadline_generation" + ], "owner": "game-api/input-event" }, { @@ -139,11 +151,15 @@ }, { "key": "movable-json-rule-anchors", - "policy": "FORBID", - "authorityFields": ["world_state.meta.turntime", "world_state.meta.starttime", "world_state.meta.tnmt_time"], + "policy": "SHIFT", + "authorityFields": [ + "world_state.meta.turntime", + "world_state.meta.starttime", + "world_state.meta.tnmt_time" + ], "projectionFields": [], - "owner": "game-engine/world-meta", - "migration": "Register typed columns or explicit participant adapters before exact reconciliation can complete." + "owner": "game-engine/clock-operation", + "migration": "Explicit adapter shifts registered ISO projections while typed tick columns remain authoritative." }, { "key": "unification-wait", @@ -179,12 +195,12 @@ { "keyPattern": "sammo:{profile}:auction:timer", "policy": "REBUILD", - "status": "implemented-without-clock-revision-fence" + "status": "implemented-with-clock-revision-phase-generation-fence" }, { "keyPattern": "sammo:{profile}:tournament:state", "policy": "REBUILD", - "status": "legacy-date-dual-write-required" + "status": "implemented-with-tick-dual-write-and-clock-fence" } ], "wallOnly": [ diff --git a/docs/architecture/game-clock-reconciliation.md b/docs/architecture/game-clock-reconciliation.md index 0cfd8ea2..bad247e9 100644 --- a/docs/architecture/game-clock-reconciliation.md +++ b/docs/architecture/game-clock-reconciliation.md @@ -65,6 +65,19 @@ pop is one Redis operation: verify revision/phase, read `-inf..nowTick`, and remove the claimed members. A failed Redis rebuild therefore leaves the game in `RECONCILING`; process liveness alone is not readiness. +`applyNextClockProjection()` claims rows with `FOR UPDATE SKIP LOCKED` and uses +PostgreSQL UTC wall time for claim/retry timestamps. One Redis Lua operation +compares the active source revision, rebuilds the auction timer, conditionally +replaces the exact tournament source snapshot, and writes target revision plus +deadline generation. The DB finalizer then re-acquires the clock-operation lock +and changes `RECONCILING -> RUNNING` only when target revision and generation +still match. If the process dies after the Lua commit, retry observes the +already-active target revision and performs only the DB finalizer. + +An active legacy tournament containing only `nextAt`/`bettingCloseAt` is a +fail-closed migration boundary. Reconciliation remains incomplete until its +authoritative `nextTick`/`bettingCloseTick` dual-write exists. + ## Lock order All mutation paths use this order: @@ -101,7 +114,9 @@ future anchored realtime profiles as `PREOPEN`, and other profiles as `RUNNING`. Existing DateTime columns remain projections while tick columns are authoritative. -Exact reconciliation stays disabled while any registry participant is -`FORBID`. In particular, Redis-only tournament dates and unification wait must -be moved to durable tick/revision contracts before the operation can reach -`RUNNING`. Removing these guards to make a partial operation pass is prohibited. +Exact reconciliation stays disabled while an active registry participant is +`FORBID`. Tournament writes now carry tick/revision/generation coordinates and +are revision-fenced in Redis. The remaining unification wait participant must +be moved from its `lastTurnTime` workaround to a durable suspension before that +workflow can reach `RUNNING`. Removing this guard to make a partial operation +pass is prohibited. diff --git a/docs/developer/game-clock-reconciliation-plan.md b/docs/developer/game-clock-reconciliation-plan.md index 937711d3..66597a6b 100644 --- a/docs/developer/game-clock-reconciliation-plan.md +++ b/docs/developer/game-clock-reconciliation-plan.md @@ -11,7 +11,7 @@ mean deployment or production validation. ## Milestone 1 - authority and inventory - [x] Branded `GameTick`, `ObservedGameInstant`, `ScheduleInstant`, - `WallInstant`, and `ClockRevision` boundaries. + `WallInstant`, and `ClockRevision` boundaries. - [x] Explicit clock phase and monotonic RUNNING projection. - [x] Exact alignment arithmetic preserving millisecond/sub-turn remainder. - [x] Opening tick zero and PREOPEN executable floor in the shared seeder. @@ -19,49 +19,50 @@ mean deployment or production validation. - [x] Suspension, participant-checksum, and Redis projection outbox tables. - [x] Machine-readable DB/Redis/JSON participant inventory and architecture gate. - [x] Turn flush lock prefix and phase/revision/generation fence. -- [ ] Empty and upgraded database migration execution evidence. +- [x] Empty and upgraded database migration execution evidence. ## Milestone 2 - exact DB reconciliation -- [ ] Suspension start command with DB wall time and idempotent source revision. -- [ ] Exact resume plan transaction with deterministic participant lock order. -- [ ] SHIFT adapters for cursor, generals, active auctions, message expiry, vote - end, select pool, and NPC selection windows. -- [ ] KEEP checksum adapters for occurrences and history. -- [ ] Explicit `LEGACY_COMPLETE_TURNS` and bounded `CATCH_UP` policies. -- [ ] Property tests for remaining distance, ordering, and history invariants. -- [ ] 24-hour and 65m17.250s PostgreSQL integration evidence. +- [x] Suspension start command with DB wall time and idempotent source revision. +- [x] Exact resume plan transaction with deterministic participant lock order. +- [x] SHIFT adapters for cursor, generals, active auctions, message expiry, vote + end, select pool, and NPC selection windows. +- [x] KEEP checksum adapters for occurrences and accepted command coordinates. +- [x] Explicit `LEGACY_COMPLETE_TURNS` and bounded `CATCH_UP` policies. +- [x] Property tests for remaining distance, ordering, and history invariants. +- [x] 24-hour and 65m17.250s PostgreSQL integration evidence. ## Milestone 3 - revisioned Redis and workers -- [ ] Projection outbox claimer/retry/recovery state machine. -- [ ] Redis active revision and atomic due-pop script. -- [ ] Auction OPEN/FINALIZING revision and generation fence. -- [ ] Tournament durable tick dual-write and projection rebuild. -- [ ] DB-commit/Redis-failure restart tests and readiness integration. +- [x] Projection outbox claimer/retry/recovery state machine. +- [x] Redis active revision and atomic due-pop script. +- [x] Auction OPEN/FINALIZING revision and generation fence. +- [x] Tournament durable tick dual-write and projection rebuild. +- [x] DB-commit/Redis-failure crash-restart test. +- [x] Readiness integration in Gateway/API process health. ## Milestone 4 - command and lifecycle workflows -- [ ] All durable input events record accepted tick and accepted revision. -- [ ] Processing converts accepted coordinates across revisions or fails closed. +- [x] All durable input events record accepted tick and accepted revision. +- [x] Processing converts accepted coordinates across revisions or fails closed. - [ ] Gateway pause/resume/open orchestration writes the DB clock phase. - [ ] Unification wait becomes a durable `UNIFICATION_WAIT` suspension. - [ ] Alignment, optional rate change, invader IDs/RNG, creation, first schedule, - outbox, verification, and RUNNING transition form one retry-safe workflow. + outbox, verification, and RUNNING transition form one retry-safe workflow. - [ ] Multi-host drift and general-access/clock-operation deadlock tests. ## Milestone 5 - test-branch release gate - [ ] Full typecheck, architecture, lint, unit, build, and non-conditional - integration suites. + integration suites. - [ ] Dedicated PostgreSQL/Redis conditional integration suite with skip count - recorded. + recorded. - [ ] Recovery runbook exercised from each incomplete status. -- [ ] Admin status/readiness exposes revision, phase, participant checksums, and - incomplete outbox state. +- [x] Admin status/readiness exposes revision, phase, participant checksums, and + incomplete outbox state. - [ ] User-test deployment evidence is recorded separately from Git push. - [ ] All `FORBID` inventory entries are removed by typed migrations or proven - inactive preconditions. + inactive preconditions. ## Evidence log @@ -78,4 +79,25 @@ mean deployment or production validation. - `pnpm check:architecture`: package boundaries passed; 21 authoritative clock fields and 18 participants were registered. - Migration SQL was generated, formatted, validated, and registered as the - release manifest head. Empty/upgraded PostgreSQL execution is still pending. + release manifest head. All 43 migrations applied to the empty dedicated + PostgreSQL instance. A second schema upgraded from the previous head with an + existing manual `world_state` row and verified `MANUAL:1:1` plus all three + reconciliation tables. +- Dedicated PostgreSQL/Redis fixtures proved exact 24-hour and 65m17.250s gaps, + schedule distance/order preservation, occurrence checksums, live-lease + fencing, a single durable outbox, Redis target revision, and recovery after a + crash between Redis commit and DB finalization. + +### 2026-09-03 - revisioned workers and input coordinates + +- `CI=1 TURBO_CONCURRENCY=1 pnpm typecheck`: 21/21 tasks passed after the + worker/input contract changes. +- `CI=1 TURBO_CONCURRENCY=1 pnpm test`: 12/12 package tasks passed. +- Dedicated PostgreSQL runs passed `databaseCommandQueue.integration.test.ts` + 6/6 and `runtimeClockShiftPersistence.integration.test.ts` 3/3 when executed + sequentially; the latter now clears each single-world fixture before the next + case. Dedicated Redis passed tournament source revision 4/4, including an + atomic stale-clock rejection. API input-event integration passed 13/13. +- The 24-hour/65m17.250s reconciliation suite passed 2/2 after the other DB + suites. Conditional files share a deliberately dedicated schema and are run + sequentially to prevent their fixture cleanup from racing another file. diff --git a/docs/developer/game-clock-recovery.md b/docs/developer/game-clock-recovery.md index a48097a2..311d223b 100644 --- a/docs/developer/game-clock-recovery.md +++ b/docs/developer/game-clock-recovery.md @@ -28,6 +28,18 @@ service. The service must re-read participant checksums and either return the already-applied result or resume the pending outbox. Never create a replacement revision to hide a failed target revision. +When an outbox row is `FAILED`, the profile must remain `RECONCILING`. A retry +is safe in both crash locations: + +- before Redis commit, the Lua operation reapplies from the source revision; +- after Redis commit but before DB finalization, the Lua operation returns the + already-active target result and DB finalization resumes without shifting any + tournament deadline twice. + +`lastError` naming a legacy tournament deadline means the tournament lacks its +tick dual-write. Do not force the active revision; migrate or prove the +tournament inactive, then retry the same outbox. + ## Rollback There is no blind inverse update. Before enabling exact reconciliation in an diff --git a/packages/common/src/time/GameClock.ts b/packages/common/src/time/GameClock.ts index 1468aa98..a52eb7bd 100644 --- a/packages/common/src/time/GameClock.ts +++ b/packages/common/src/time/GameClock.ts @@ -17,8 +17,8 @@ export type ScheduleInstant = GameTick & { readonly [scheduleInstantBrand]: 'Sch export type ClockRevision = number & { readonly [clockRevisionBrand]: 'ClockRevision' }; export type WallInstant = Date & { readonly [wallInstantBrand]: 'WallInstant' }; -export interface ExactClockAlignmentPlan { - policy: 'EXACT'; +export interface ClockAlignmentPlan { + policy: ClockAlignmentPolicy; sourceRevision: ClockRevision; targetRevision: ClockRevision; cutTick: GameTick; @@ -84,6 +84,15 @@ export const parseGameClockPhase = (value: string): GameClockPhase => { throw new Error(`Unknown game clock phase: ${value}`); }; +const CLOCK_ALIGNMENT_POLICIES: readonly ClockAlignmentPolicy[] = ['EXACT', 'LEGACY_COMPLETE_TURNS', 'CATCH_UP']; + +export const parseClockAlignmentPolicy = (value: string): ClockAlignmentPolicy => { + if ((CLOCK_ALIGNMENT_POLICIES as readonly string[]).includes(value)) { + return value as ClockAlignmentPolicy; + } + throw new Error(`Unknown clock alignment policy: ${value}`); +}; + export const scheduleNotBefore = (instant: ObservedGameInstant, phase: GameClockPhase): ScheduleInstant => { if (phase === 'PREOPEN') { return asScheduleInstant(Math.max(0, instant)); @@ -103,14 +112,15 @@ export const assertGameplayCommitAllowed = (phase: GameClockPhase): void => { } }; -export const buildExactClockAlignmentPlan = (input: { +const buildAlignmentPlan = (input: { + policy: ClockAlignmentPolicy; sourceRevision: number; cutTick: number; cutWall: Date; resumeWall: Date; ticksPerSecond: number; catchUpTicks?: number; -}): ExactClockAlignmentPlan => { +}): ClockAlignmentPlan => { const sourceRevision = asClockRevision(input.sourceRevision); const cutTick = asGameTick(input.cutTick); const cutWall = asWallInstant(input.cutWall); @@ -136,7 +146,7 @@ export const buildExactClockAlignmentPlan = (input: { } const shiftTicks = asGameTick(gapTicks - catchUpTicks); return { - policy: 'EXACT', + policy: input.policy, sourceRevision, targetRevision: asClockRevision(sourceRevision + 1), cutTick, @@ -147,6 +157,37 @@ export const buildExactClockAlignmentPlan = (input: { }; }; +export const buildExactClockAlignmentPlan = ( + input: Omit[0], 'policy'> +): ClockAlignmentPlan => buildAlignmentPlan({ ...input, policy: 'EXACT' }); + +export const buildClockAlignmentPlan = (input: { + policy: ClockAlignmentPolicy; + sourceRevision: number; + cutTick: number; + cutWall: Date; + resumeWall: Date; + ticksPerSecond: number; + catchUpTicks?: number; +}): ClockAlignmentPlan => { + if (input.policy === 'EXACT') { + if ((input.catchUpTicks ?? 0) !== 0) { + throw new Error('EXACT alignment does not allow catch-up ticks.'); + } + return buildAlignmentPlan({ ...input, policy: 'EXACT', catchUpTicks: 0 }); + } + if (input.policy === 'CATCH_UP') { + return buildAlignmentPlan({ ...input, policy: 'CATCH_UP' }); + } + const exact = buildAlignmentPlan({ ...input, policy: 'LEGACY_COMPLETE_TURNS', catchUpTicks: 0 }); + const shiftTicks = asGameTick(Math.floor(exact.gapTicks / GAME_TICKS_PER_TURN) * GAME_TICKS_PER_TURN); + return { + ...exact, + catchUpTicks: asGameTick(exact.gapTicks - shiftTicks), + shiftTicks, + }; +}; + const tickOffsetMilliseconds = (tick: number, ticksPerSecond: number): number => { const wholeSeconds = Math.floor(tick / ticksPerSecond); const remainingTicks = tick - wholeSeconds * ticksPerSecond; diff --git a/packages/common/src/tournament/sourceRevision.ts b/packages/common/src/tournament/sourceRevision.ts index 29cc6b83..d4ab9d6f 100644 --- a/packages/common/src/tournament/sourceRevision.ts +++ b/packages/common/src/tournament/sourceRevision.ts @@ -18,6 +18,15 @@ export interface TournamentProjectionWrite { value: unknown; } +export interface TournamentClockFence { + activeRevisionKey: string; + deadlineGenerationKey: string; + phaseKey: string; + revision: number; + deadlineGeneration: number; + phase: 'RUNNING'; +} + const WRITE_TOURNAMENT_PROJECTION_SCRIPT = ` local revision_key = KEYS[#KEYS] local current = redis.call('GET', revision_key) @@ -55,6 +64,49 @@ local revision = redis.call('INCR', revision_key) return tostring(revision) .. ':' .. (stage_changed and '1' or '0') .. ':' .. (rankings_changed and '1' or '0') `; +const WRITE_FENCED_TOURNAMENT_PROJECTION_SCRIPT = ` +local revision_key_index = #KEYS - 3 +if redis.call('GET', KEYS[#KEYS - 2]) ~= ARGV[#ARGV - 2] + or redis.call('GET', KEYS[#KEYS - 1]) ~= ARGV[#ARGV - 1] + or redis.call('GET', KEYS[#KEYS]) ~= ARGV[#ARGV] then + return '__CLOCK_FENCE__' +end +local revision_key = KEYS[revision_key_index] +local current = redis.call('GET', revision_key) +if current then + if not string.match(current, '^%d+$') then + return redis.error_reply('invalid tournament source revision') + end + if string.len(current) > 18 then + return redis.error_reply('tournament source revision exhausted') + end +end +local stage_changed = false +local rankings_changed = false +for index = 1, revision_key_index - 1 do + local next_ok, next_value = pcall(cjson.decode, ARGV[index]) + if next_ok and type(next_value) == 'table' and next_value['stage'] ~= nil then + local previous = redis.call('GET', KEYS[index]) + local previous_stage = nil + local previous_value = nil + if previous then + local previous_ok + previous_ok, previous_value = pcall(cjson.decode, previous) + if previous_ok and type(previous_value) == 'table' then + previous_stage = previous_value['stage'] + end + end + local next_stage = next_value['stage'] + stage_changed = (not previous) or previous_stage ~= next_stage + local previous_reward_settled = previous_value and previous_value['rewardSettled'] or false + rankings_changed = next_value['rewardSettled'] == true and previous_reward_settled ~= true + end + redis.call('SET', KEYS[index], ARGV[index]) +end +local revision = redis.call('INCR', revision_key) +return tostring(revision) .. ':' .. (stage_changed and '1' or '0') .. ':' .. (rankings_changed and '1' or '0') +`; + export const parseTournamentSourceRevision = (value: unknown): string | null => { if (typeof value === 'number') { return Number.isSafeInteger(value) && value >= 0 ? String(value) : null; @@ -69,7 +121,8 @@ export const parseTournamentSourceRevision = (value: unknown): string | null => export const writeTournamentProjection = async ( redis: TournamentProjectionRedis, keys: TournamentSourceKeys, - writes: readonly TournamentProjectionWrite[] + writes: readonly TournamentProjectionWrite[], + clockFence?: TournamentClockFence ): Promise => { if (writes.length === 0) { throw new Error('Tournament projection write must contain at least one payload.'); @@ -90,10 +143,27 @@ export const writeTournamentProjection = async ( value !== null && (value as { rewardSettled?: unknown }).rewardSettled === true ); - const result = await redis.eval(WRITE_TOURNAMENT_PROJECTION_SCRIPT, { - keys: [...writes.map(({ key }) => key), keys.sourceRevisionKey], - arguments: writes.map(({ value }) => JSON.stringify(value)), - }); + const result = await redis.eval( + clockFence ? WRITE_FENCED_TOURNAMENT_PROJECTION_SCRIPT : WRITE_TOURNAMENT_PROJECTION_SCRIPT, + { + keys: [ + ...writes.map(({ key }) => key), + keys.sourceRevisionKey, + ...(clockFence + ? [clockFence.activeRevisionKey, clockFence.deadlineGenerationKey, clockFence.phaseKey] + : []), + ], + arguments: [ + ...writes.map(({ value }) => JSON.stringify(value)), + ...(clockFence + ? [String(clockFence.revision), String(clockFence.deadlineGeneration), clockFence.phase] + : []), + ], + } + ); + if (result === '__CLOCK_FENCE__') { + throw new Error('Tournament projection clock revision fence failed.'); + } const scriptResult = typeof result === 'string' ? /^(\d+):([01])(?::([01]))?$/u.exec(result) : null; const sourceRevision = parseTournamentSourceRevision(scriptResult?.[1] ?? result); // Plain revision results remain accepted for rolling deployments and small diff --git a/packages/common/test/gameClock.test.ts b/packages/common/test/gameClock.test.ts index 9af0d7aa..6672cae8 100644 --- a/packages/common/test/gameClock.test.ts +++ b/packages/common/test/gameClock.test.ts @@ -6,6 +6,7 @@ import { MAX_SAFE_GAME_TICK, asGameTick, asObservedGameInstant, + buildClockAlignmentPlan, buildExactClockAlignmentPlan, createDeadline, scheduleNotBefore, @@ -113,6 +114,64 @@ describe('GameClock', () => { expect(plan.alignedTick).toBe(26 * GAME_TICKS_PER_TURN); }); + it('keeps legacy complete-turn rebasing separate from bounded catch-up', () => { + const common = { + sourceRevision: 4, + cutTick: 100, + cutWall: new Date('2026-01-01T00:00:00.000Z'), + resumeWall: new Date('2026-01-01T01:05:17.250Z'), + ticksPerSecond: 10_000, + }; + const legacy = buildClockAlignmentPlan({ ...common, policy: 'LEGACY_COMPLETE_TURNS' }); + const catchUp = buildClockAlignmentPlan({ ...common, policy: 'CATCH_UP', catchUpTicks: 1_000_000 }); + + expect(legacy).toMatchObject({ + policy: 'LEGACY_COMPLETE_TURNS', + gapTicks: 39_172_500, + shiftTicks: GAME_TICKS_PER_TURN, + catchUpTicks: 3_172_500, + }); + expect(catchUp).toMatchObject({ + policy: 'CATCH_UP', + gapTicks: 39_172_500, + shiftTicks: 38_172_500, + catchUpTicks: 1_000_000, + }); + expect(() => buildClockAlignmentPlan({ ...common, policy: 'EXACT', catchUpTicks: 1 })).toThrow( + 'EXACT alignment does not allow catch-up ticks' + ); + }); + + it('preserves schedule ordering, remaining distance, and occurrence ticks across generated exact gaps', () => { + let seed = 0x5eed1234; + const next = (): number => { + seed = (Math.imul(seed, 1_664_525) + 1_013_904_223) >>> 0; + return seed; + }; + for (let iteration = 0; iteration < 500; iteration += 1) { + const cutTick = next() % 1_000_000_000; + const gapMilliseconds = next() % (7 * 24 * 60 * 60 * 1_000); + const ticksPerSecond = [5_000, 10_000, 60_000][next() % 3]!; + const offsets = Array.from({ length: 8 }, () => next() % (3 * GAME_TICKS_PER_TURN)).sort( + (left, right) => left - right + ); + const occurrenceTicks = Array.from({ length: 4 }, () => cutTick - (next() % GAME_TICKS_PER_TURN)); + const plan = buildClockAlignmentPlan({ + policy: 'EXACT', + sourceRevision: 1 + (next() % 10_000), + cutTick, + cutWall: new Date(0), + resumeWall: new Date(gapMilliseconds), + ticksPerSecond, + }); + const shifted = offsets.map((offset) => cutTick + offset + plan.shiftTicks); + + expect(shifted.map((deadline) => deadline - plan.alignedTick)).toEqual(offsets); + expect([...shifted].sort((left, right) => left - right)).toEqual(shifted); + expect(occurrenceTicks).toEqual([...occurrenceTicks]); + } + }); + it('projects near the safe tick boundary without unsafe intermediate multiplication', () => { const clock = new GameClock({ baseTime: new Date(0), diff --git a/packages/infra/prisma/game.prisma b/packages/infra/prisma/game.prisma index 196523dc..67b05cea 100644 --- a/packages/infra/prisma/game.prisma +++ b/packages/infra/prisma/game.prisma @@ -63,23 +63,27 @@ enum InputEventTarget { } model InputEvent { - sequence BigInt @id @default(autoincrement()) - requestId String @unique @map("request_id") - target InputEventTarget - eventType String @map("event_type") - payload Json @default(dbgenerated("'{}'::jsonb")) - actorUserId String? @map("actor_user_id") - acceptedGameTick BigInt? @map("accepted_game_tick") - acceptedClockRevision BigInt? @map("accepted_clock_revision") - status InputEventStatus @default(PENDING) - result Json? - error String? - attempts Int @default(0) - lockedBy String? @map("locked_by") - leaseUntil DateTime? @map("lease_until") - createdAt DateTime @default(now()) @map("created_at") - processingAt DateTime? @map("processing_at") - completedAt DateTime? @map("completed_at") + sequence BigInt @id @default(autoincrement()) + requestId String @unique @map("request_id") + target InputEventTarget + eventType String @map("event_type") + payload Json @default(dbgenerated("'{}'::jsonb")) + actorUserId String? @map("actor_user_id") + acceptedGameTick BigInt? @map("accepted_game_tick") + acceptedClockRevision BigInt? @map("accepted_clock_revision") + acceptedDeadlineGeneration BigInt? @map("accepted_deadline_generation") + processingGameTick BigInt? @map("processing_game_tick") + processingClockRevision BigInt? @map("processing_clock_revision") + processingDeadlineGeneration BigInt? @map("processing_deadline_generation") + status InputEventStatus @default(PENDING) + result Json? + error String? + attempts Int @default(0) + lockedBy String? @map("locked_by") + leaseUntil DateTime? @map("lease_until") + createdAt DateTime @default(now()) @map("created_at") + processingAt DateTime? @map("processing_at") + completedAt DateTime? @map("completed_at") @@index([target, status, sequence]) @@map("input_event") diff --git a/packages/infra/prisma/migrations/20260903103000_add_input_event_clock_processing/migration.sql b/packages/infra/prisma/migrations/20260903103000_add_input_event_clock_processing/migration.sql new file mode 100644 index 00000000..0f24fcf8 --- /dev/null +++ b/packages/infra/prisma/migrations/20260903103000_add_input_event_clock_processing/migration.sql @@ -0,0 +1,15 @@ +ALTER TABLE input_event + ADD COLUMN accepted_deadline_generation BIGINT, + ADD COLUMN processing_game_tick BIGINT, + ADD COLUMN processing_clock_revision BIGINT, + ADD COLUMN processing_deadline_generation BIGINT; + +ALTER TABLE input_event + ADD CONSTRAINT input_event_accepted_clock_revision_positive + CHECK (accepted_clock_revision IS NULL OR accepted_clock_revision > 0), + ADD CONSTRAINT input_event_accepted_deadline_generation_positive + CHECK (accepted_deadline_generation IS NULL OR accepted_deadline_generation > 0), + ADD CONSTRAINT input_event_processing_clock_revision_positive + CHECK (processing_clock_revision IS NULL OR processing_clock_revision > 0), + ADD CONSTRAINT input_event_processing_deadline_generation_positive + CHECK (processing_deadline_generation IS NULL OR processing_deadline_generation > 0); diff --git a/packages/infra/src/db.ts b/packages/infra/src/db.ts index aa1dfede..c6205460 100644 --- a/packages/infra/src/db.ts +++ b/packages/infra/src/db.ts @@ -46,6 +46,9 @@ export interface DatabaseClient { vote: GamePrisma.VoteDelegate; inputEvent: GamePrisma.InputEventDelegate; turnDaemonLease: GamePrisma.TurnDaemonLeaseDelegate; + clockSuspension?: GamePrisma.ClockSuspensionDelegate; + clockReconciliationParticipant?: GamePrisma.ClockReconciliationParticipantDelegate; + clockProjectionOutbox?: GamePrisma.ClockProjectionOutboxDelegate; readModelOutbox: GamePrisma.ReadModelOutboxDelegate; webPushOutbox: GamePrisma.WebPushOutboxDelegate; } diff --git a/packages/infra/src/index.ts b/packages/infra/src/index.ts index 63c8ddfa..51abe88c 100644 --- a/packages/infra/src/index.ts +++ b/packages/infra/src/index.ts @@ -10,4 +10,5 @@ export * from './readModelChangeJournal.js'; export * from './readModelOutboxDispatcher.js'; export * from './readModelCoverageActivation.js'; export * from './gameSchemaAdvisoryLock.js'; +export * from './inputEventClock.js'; export * from './webPushOutbox.js'; diff --git a/packages/infra/src/inputEventClock.ts b/packages/infra/src/inputEventClock.ts new file mode 100644 index 00000000..c4eb121f --- /dev/null +++ b/packages/infra/src/inputEventClock.ts @@ -0,0 +1,76 @@ +import { GameClock, inferClockPhase, parseGameClockPhase } from '@sammo-ts/common'; + +import { GamePrisma, type GamePrismaClient } from './gamePrisma.js'; +import { acquireGameSchemaAdvisoryXactLock, CLOCK_OPERATION_PERSISTENCE_LOCK } from './gameSchemaAdvisoryLock.js'; + +type ClockAcceptanceDatabase = Pick; + +interface DbWallRow { + wallNow: Date; +} + +export interface InputEventClockCoordinate { + wallAt: Date; + gameAt: Date; + gameTick: bigint; + clockRevision: bigint; + deadlineGeneration: bigint; + phase: string; +} + +/** + * Reads one input-event acceptance coordinate while holding the same schema + * clock-operation fence used by reconciliation. The caller must create the + * input_event in this transaction before releasing the lock. + */ +export const readInputEventClockCoordinate = async ( + db: ClockAcceptanceDatabase +): Promise => { + await acquireGameSchemaAdvisoryXactLock(db, CLOCK_OPERATION_PERSISTENCE_LOCK); + const [wall] = await db.$queryRaw(GamePrisma.sql` + SELECT timezone('UTC', clock_timestamp()) AS "wallNow" + `); + if (!wall) throw new Error('PostgreSQL did not return its authoritative wall clock.'); + const world = await db.worldState.findFirst({ + orderBy: { id: 'asc' }, + select: { + clockBaseTime: true, + clockTick: true, + clockMode: true, + clockWallAnchor: true, + tickSeconds: true, + clockPhase: true, + clockRevision: true, + deadlineGeneration: true, + }, + }); + if (!world?.clockBaseTime || world.clockTick === null || !world.clockWallAnchor) { + throw new Error('The authoritative game clock is not initialized.'); + } + const tick = Number(world.clockTick); + const revision = Number(world.clockRevision); + const generation = Number(world.deadlineGeneration); + if (!Number.isSafeInteger(tick) || !Number.isSafeInteger(revision) || !Number.isSafeInteger(generation)) { + throw new Error('The authoritative game clock coordinate is outside the safe integer range.'); + } + const mode = world.clockMode === 'manual' ? 'manual' : 'realtime'; + const phase = world.clockPhase ? parseGameClockPhase(world.clockPhase) : inferClockPhase(mode); + const clock = new GameClock({ + baseTime: world.clockBaseTime, + tick, + mode, + wallAnchor: world.clockWallAnchor, + turnSeconds: world.tickSeconds, + phase, + revision, + }); + const observedTick = clock.nowTick(wall.wallNow); + return { + wallAt: wall.wallNow, + gameAt: clock.tickToDate(observedTick), + gameTick: BigInt(observedTick), + clockRevision: BigInt(revision), + deadlineGeneration: BigInt(generation), + phase, + }; +}; diff --git a/release-manifest.json b/release-manifest.json index cc0a6058..1f3d023c 100644 --- a/release-manifest.json +++ b/release-manifest.json @@ -2,6 +2,6 @@ "formatVersion": 1, "controllerProtocol": 2, "gatewaySchemaHead": "20260825000000_add_bulk_release_batches", - "gameSchemaHead": "20260903090000_add_game_clock_reconciliation", + "gameSchemaHead": "20260903103000_add_input_event_clock_processing", "components": ["gateway-api", "gateway-frontend", "release-controller", "game-api", "game-engine", "game-frontend"] }