시간 작동 방식 변경 #1

Merged
Hide_D merged 20 commits from test/game-clock-reconciliation-20260903 into main 2026-09-04 08:37:06 +09:00
46 changed files with 3011 additions and 219 deletions
Showing only changes of commit a3e2bf90ae - Show all commits
+91 -3
View File
@@ -1,4 +1,7 @@
import {
CLOCK_OPERATION_PERSISTENCE_LOCK,
GamePrisma,
acquireGameSchemaAdvisoryXactLock,
createGamePostgresConnector,
createRedisConnector,
type GamePrismaClient,
@@ -10,6 +13,7 @@ import { resolveGameApiConfigFromEnv } from '../config.js';
import { createBestEffortResourceCloser } from '../services/bestEffortResourceCloser.js';
import { loadCurrentGameTime, type CurrentGameTime } from '../services/gameClock.js';
import { createPollingWorkerControl, waitForWorkerPoll } from '../services/pollingWorkerLifecycle.js';
import { ensureActiveRedisClockFence } from '../services/redisClockFence.js';
import { buildAuctionTimerKeys } from './keys.js';
import { resolveAuctionTimerScore, seedAuctionTimers } from './scheduler.js';
@@ -24,8 +28,22 @@ interface RedisTimerClient {
zAdd(key: string, values: Array<{ score: number; value: string }>): Promise<number>;
zRem(key: string, values: string | string[]): Promise<number>;
zRemRangeByScore(key: string, min: number, max: number): Promise<number>;
eval?(script: string, options: { keys: string[]; arguments: string[] }): Promise<unknown>;
}
const POP_DUE_AUCTIONS_SCRIPT = `
if redis.call('GET', KEYS[2]) ~= ARGV[1]
or redis.call('GET', KEYS[3]) ~= ARGV[2]
or redis.call('GET', KEYS[4]) ~= 'RUNNING' then
return { '__CLOCK_FENCE__' }
end
local ids = redis.call('ZRANGEBYSCORE', KEYS[1], '-inf', ARGV[3], 'LIMIT', 0, ARGV[4])
if #ids > 0 then
redis.call('ZREM', KEYS[1], unpack(ids))
end
return ids
`;
const AUCTION_FINALIZE_RECOVERY_LIMIT = 1;
interface AuctionFinalizeDeadline {
@@ -115,12 +133,29 @@ const isSuccessfulAuctionFinalizeResult = (result: unknown, auctionId: number):
return resultRecord.type === 'auctionFinalize' && resultRecord.ok === true && resultRecord.auctionId === auctionId;
};
const popDueAuctionIds = async (
export const popDueAuctionIds = async (
redis: RedisTimerClient,
timerKey: string,
nowMs: number,
batchSize: number
batchSize: number,
clockFence?: {
activeRevisionKey: string;
deadlineGenerationKey: string;
phaseKey: string;
revision: number;
generation: number;
}
): Promise<string[]> => {
if (clockFence) {
if (!redis.eval) throw new Error('Redis EVAL is required for revision-fenced auction due-pop.');
const result = await redis.eval(POP_DUE_AUCTIONS_SCRIPT, {
keys: [timerKey, clockFence.activeRevisionKey, clockFence.deadlineGenerationKey, clockFence.phaseKey],
arguments: [String(clockFence.revision), String(clockFence.generation), String(nowMs), String(batchSize)],
});
if (!Array.isArray(result)) throw new Error('Auction due-pop returned an invalid Redis result.');
if (result[0] === '__CLOCK_FENCE__') return [];
return result.map(String);
}
const ids = await redis.zRangeByScore(timerKey, 0, nowMs, { LIMIT: { offset: 0, count: batchSize } });
if (ids.length > 0) {
await redis.zRem(timerKey, ids);
@@ -205,6 +240,8 @@ export const processDueAuctionId = async (options: {
nowMs: number;
nowTick?: number | null;
historyNowMs?: number;
expectedClockRevision?: number;
expectedDeadlineGeneration?: number;
}): Promise<'PENDING' | 'RESCHEDULED' | 'IGNORED'> => {
const { db, redis, timerKey, historyKey, id, nowMs, nowTick = null, historyNowMs = nowMs } = options;
const auctionId = Number(id);
@@ -213,6 +250,29 @@ export const processDueAuctionId = async (options: {
}
const now = new Date(nowMs);
const outcome = await db.$transaction(async (transaction) => {
if (options.expectedClockRevision !== undefined || options.expectedDeadlineGeneration !== undefined) {
await acquireGameSchemaAdvisoryXactLock(transaction, CLOCK_OPERATION_PERSISTENCE_LOCK);
const [world] = await transaction.$queryRaw<
Array<{ clockPhase: string | null; clockRevision: bigint; deadlineGeneration: bigint }>
>(GamePrisma.sql`
SELECT
clock_phase AS "clockPhase",
clock_revision AS "clockRevision",
deadline_generation AS "deadlineGeneration"
FROM world_state
ORDER BY id ASC
LIMIT 1
FOR UPDATE
`);
if (
!world ||
world.clockPhase !== 'RUNNING' ||
world.clockRevision !== BigInt(options.expectedClockRevision ?? -1) ||
world.deadlineGeneration !== BigInt(options.expectedDeadlineGeneration ?? -1)
) {
return { status: 'RESCHEDULED' as const, clockFenceFailed: true };
}
}
const current = await transaction.auction.findUnique({
where: { id: auctionId },
select: { status: true, closeAt: true, closeTick: true },
@@ -259,6 +319,13 @@ export const processDueAuctionId = async (options: {
target: 'ENGINE',
eventType: nextCommand.type,
payload: { ...nextCommand },
...(nowTick === null ? {} : { acceptedGameTick: BigInt(nowTick) }),
...(options.expectedClockRevision === undefined
? {}
: { acceptedClockRevision: BigInt(options.expectedClockRevision) }),
...(options.expectedDeadlineGeneration === undefined
? {}
: { acceptedDeadlineGeneration: BigInt(options.expectedDeadlineGeneration) }),
},
});
return { status: 'PENDING' as const };
@@ -284,6 +351,10 @@ export const processDueAuctionId = async (options: {
return 'PENDING';
}
if (outcome.status === 'RESCHEDULED') {
if ('clockFenceFailed' in outcome) {
await redis.zAdd(timerKey, [{ score: nowTick ?? nowMs, value: id }]);
return 'RESCHEDULED';
}
const gameTime = await loadCurrentGameTime(db, now);
await redis.zAdd(timerKey, [
{
@@ -324,6 +395,17 @@ export const runAuctionWorker = async (options: AuctionWorkerOptions = {}): Prom
const gameTime = await loadCurrentGameTime(postgres.prisma, new Date(operationalNowMs));
const gameNowMs = gameTime.now.getTime();
const dueScore = gameTime.tick ?? gameNowMs;
if (gameTime.phase && gameTime.phase !== 'RUNNING') {
await waitForWorkerPoll(control.signal, config.auctionTimerPollMs);
continue;
}
const clockFence = gameTime.phase
? await ensureActiveRedisClockFence(redis.client, config.profileName, gameTime)
: null;
if (gameTime.phase && !clockFence) {
await waitForWorkerPoll(control.signal, config.auctionTimerPollMs);
continue;
}
if (operationalNowMs >= nextResyncAt) {
await seedAuctionTimers(postgres.prisma, redis.client, keys);
nextResyncAt = operationalNowMs + config.auctionTimerResyncMs;
@@ -345,7 +427,7 @@ export const runAuctionWorker = async (options: AuctionWorkerOptions = {}): Prom
if (historyTrimBefore > 0) {
await redis.client.zRemRangeByScore(keys.historyKey, 0, historyTrimBefore);
}
const dueIds = await popDueAuctionIds(redis.client, keys.timerKey, dueScore, 100);
const dueIds = await popDueAuctionIds(redis.client, keys.timerKey, dueScore, 100, clockFence ?? undefined);
if (dueIds.length > 0) {
for (const id of dueIds) {
try {
@@ -358,6 +440,12 @@ export const runAuctionWorker = async (options: AuctionWorkerOptions = {}): Prom
nowMs: gameNowMs,
nowTick: gameTime.tick,
historyNowMs: operationalNowMs,
...(clockFence
? {
expectedClockRevision: clockFence.revision,
expectedDeadlineGeneration: clockFence.generation,
}
: {}),
});
if (outcome === 'PENDING') {
pendingFinalizationIds.add(Number(id));
+39 -7
View File
@@ -1,6 +1,12 @@
import { randomUUID } from 'node:crypto';
import { acquireGameSchemaAdvisoryXactLock, type DatabaseClient, type GamePrisma } from '@sammo-ts/infra';
import {
acquireGameSchemaAdvisoryXactLock,
readInputEventClockCoordinate,
type DatabaseClient,
type GamePrisma,
type InputEventClockCoordinate,
} from '@sammo-ts/infra';
import type { TurnDaemonTransport } from './transport.js';
import type { TurnDaemonCommand, TurnDaemonCommandResult, TurnDaemonStatus } from './types.js';
@@ -103,10 +109,10 @@ export class DatabaseTurnDaemonTransport implements TurnDaemonTransport {
try {
if (command.type === 'npcPossessGeneral' && this.db.$transaction) {
const rejectionReason = await this.db.$transaction(async (transaction) => {
const coordinate = await readInputEventClockCoordinate(transaction);
await acquireGameSchemaAdvisoryXactLock(transaction, 'npc-possession:global');
await acquireGameSchemaAdvisoryXactLock(transaction, `npc-possession:user:${command.userId}`);
const acceptedAt = new Date(Math.floor(Date.now() / 1000) * 1000);
const acceptedGameAt = (await loadCurrentGameTime(transaction, acceptedAt)).now;
const acceptedGameAt = coordinate.gameAt;
const token = await transaction.npcSelectionToken.findFirst({
where: {
ownerUserId: command.userId,
@@ -130,14 +136,21 @@ export class DatabaseTurnDaemonTransport implements TurnDaemonTransport {
...(durableCommand as Extract<TurnDaemonCommand, { type: 'npcPossessGeneral' }>),
acceptedGameAt: acceptedGameAt.toISOString(),
};
await this.createInputEvent(transaction, acceptedCommand, requestId, acceptedAt);
await this.createInputEvent(transaction, acceptedCommand, requestId, coordinate);
return null;
});
if (rejectionReason) {
throw new RejectedNpcPossessionCommandError('PRECONDITION_FAILED', rejectionReason);
}
} else {
await this.createInputEvent(this.db, durableCommand, requestId);
if (this.db.$transaction) {
await this.db.$transaction(async (transaction) => {
const coordinate = await readInputEventClockCoordinate(transaction);
await this.createInputEvent(transaction, durableCommand, requestId, coordinate);
});
} else {
await this.createInputEvent(this.db, durableCommand, requestId);
}
}
} catch (error) {
if (error instanceof RejectedNpcPossessionCommandError) {
@@ -166,8 +179,20 @@ export class DatabaseTurnDaemonTransport implements TurnDaemonTransport {
db: DatabaseClient,
command: TurnDaemonCommand,
requestId: string,
createdAt?: Date
coordinate?: InputEventClockCoordinate
): Promise<void> {
const gameTime = coordinate ? null : await loadCurrentGameTime(db, new Date());
const commandAcceptedTick = Reflect.get(command, 'acceptedGameTick');
const acceptedGameTick =
typeof commandAcceptedTick === 'number' && Number.isSafeInteger(commandAcceptedTick)
? commandAcceptedTick
: coordinate
? Number(coordinate.gameTick)
: gameTime!.tick;
const acceptedClockRevision = coordinate ? Number(coordinate.clockRevision) : gameTime!.revision;
const acceptedDeadlineGeneration = coordinate
? Number(coordinate.deadlineGeneration)
: gameTime!.deadlineGeneration;
await db.inputEvent.create({
data: {
requestId,
@@ -175,7 +200,14 @@ export class DatabaseTurnDaemonTransport implements TurnDaemonTransport {
eventType: command.type,
payload: asJson(command),
actorUserId: 'userId' in command && typeof command.userId === 'string' ? command.userId : null,
...(createdAt ? { createdAt } : {}),
...(acceptedGameTick === null ? {} : { acceptedGameTick: BigInt(acceptedGameTick) }),
...(acceptedClockRevision === null || acceptedClockRevision === undefined
? {}
: { acceptedClockRevision: BigInt(acceptedClockRevision) }),
...(acceptedDeadlineGeneration === null || acceptedDeadlineGeneration === undefined
? {}
: { acceptedDeadlineGeneration: BigInt(acceptedDeadlineGeneration) }),
...(coordinate ? { createdAt: coordinate.wallAt } : {}),
},
});
}
+27 -4
View File
@@ -1,6 +1,11 @@
import { createHash } from 'node:crypto';
import { GamePrisma, type DatabaseClient as InfraDatabaseClient } from '@sammo-ts/infra';
import {
CLOCK_OPERATION_PERSISTENCE_LOCK,
GamePrisma,
acquireGameSchemaAdvisoryXactLock,
type DatabaseClient as InfraDatabaseClient,
} from '@sammo-ts/infra';
import type { DatabaseClient } from './context.js';
@@ -20,6 +25,9 @@ interface LockedInputEvent {
status: 'PENDING' | 'PROCESSING' | 'SUCCEEDED' | 'FAILED';
result: GamePrisma.JsonValue | null;
attempts: number;
acceptedGameTick: bigint | null;
acceptedClockRevision: bigint | null;
acceptedDeadlineGeneration: bigint | null;
}
type InputEventOutcome<T> =
@@ -85,6 +93,9 @@ const insertPendingIfAbsent = async (
actor_user_id,
status,
attempts,
accepted_game_tick,
accepted_clock_revision,
accepted_deadline_generation,
created_at
)
VALUES (
@@ -95,6 +106,9 @@ const insertPendingIfAbsent = async (
${options.actorUserId},
'PENDING'::"InputEventStatus",
0,
(SELECT clock_tick FROM world_state ORDER BY id ASC LIMIT 1),
(SELECT clock_revision FROM world_state ORDER BY id ASC LIMIT 1),
(SELECT deadline_generation FROM world_state ORDER BY id ASC LIMIT 1),
CURRENT_TIMESTAMP AT TIME ZONE 'UTC'
)
ON CONFLICT (request_id) DO NOTHING
@@ -112,7 +126,10 @@ const lockInputEvent = async (db: DatabaseClient, requestId: string): Promise<Lo
actor_user_id AS "actorUserId",
status,
result,
attempts
attempts,
accepted_game_tick AS "acceptedGameTick",
accepted_clock_revision AS "acceptedClockRevision",
accepted_deadline_generation AS "acceptedDeadlineGeneration"
FROM input_event
WHERE request_id = ${requestId}
FOR UPDATE
@@ -151,7 +168,8 @@ const isMatchingIdentity = (
const claimInputEvent = async (
db: DatabaseClient,
requestId: string,
payloadIdentity: ApiInputPayloadIdentity
payloadIdentity: ApiInputPayloadIdentity,
row: LockedInputEvent
): Promise<void> => {
await db.inputEvent.update({
where: { requestId },
@@ -164,6 +182,9 @@ const claimInputEvent = async (
lockedBy: null,
leaseUntil: null,
processingAt: new Date(),
processingGameTick: row.acceptedGameTick,
processingClockRevision: row.acceptedClockRevision,
processingDeadlineGeneration: row.acceptedDeadlineGeneration,
completedAt: null,
},
});
@@ -184,6 +205,7 @@ const markUnexpectedFailure = async (
try {
await db.$transaction(async (transaction) => {
await acquireGameSchemaAdvisoryXactLock(transaction, CLOCK_OPERATION_PERSISTENCE_LOCK);
await insertPendingIfAbsent(transaction, options);
const row = await lockInputEvent(transaction, options.requestId);
const identityMatches = isMatchingIdentity(row, options) || canAdoptLegacyFailedPayload(row, options);
@@ -233,6 +255,7 @@ export const executeInputEvent = async <T>(options: {
let outcome: InputEventOutcome<T>;
try {
outcome = await db.$transaction(async (transaction) => {
await acquireGameSchemaAdvisoryXactLock(transaction, CLOCK_OPERATION_PERSISTENCE_LOCK);
await insertPendingIfAbsent(transaction, { requestId, eventType, actorUserId, payloadIdentity });
const row = await lockInputEvent(transaction, requestId);
const identityMatches = isMatchingIdentity(row, { eventType, actorUserId, payloadIdentity });
@@ -251,7 +274,7 @@ export const executeInputEvent = async <T>(options: {
throw new DuplicateInputEventError(requestId);
}
await claimInputEvent(transaction, requestId, payloadIdentity);
await claimInputEvent(transaction, requestId, payloadIdentity, row);
const savepointDb = transaction as SavepointDatabaseClient;
await savepointDb.$executeRawUnsafe(`SAVEPOINT ${BUSINESS_SAVEPOINT}`);
businessStarted = true;
+42 -11
View File
@@ -5,12 +5,14 @@ import { asRecord } from '@sammo-ts/common';
import type { TournamentType } from '@sammo-ts/logic';
import type { TournamentState } from '../../tournament/types.js';
import { TournamentStore } from '../../tournament/store.js';
import { TournamentStore, type TournamentClockContext } from '../../tournament/store.js';
import { buildTournamentKeys } from '../../tournament/keys.js';
import { assignManualApplicantGroup } from '../../tournament/workerHelpers.js';
import { accessAuthedProcedure, authedProcedure, router } from '../../trpc.js';
import { getMyGeneral } from '../shared/general.js';
import { loadCurrentGameTime } from '../../services/gameClock.js';
import { ensureActiveRedisClockFence } from '../../services/redisClockFence.js';
import { loadClockAdminStatus } from '../../services/clockReadiness.js';
const hasAdminRole = (roles: string[], profileName: string): boolean => {
if (roles.includes('superuser') || roles.includes('admin') || roles.includes('admin.superuser')) {
@@ -44,6 +46,32 @@ const adminProcedure = authedProcedure.use(({ ctx, next }) => {
return next();
});
const withTournamentClockMutation = async <T>(
ctx: {
db: Parameters<typeof loadCurrentGameTime>[0];
redis: Parameters<typeof ensureActiveRedisClockFence>[0];
profile: { name: string };
},
store: TournamentStore,
operation: () => Promise<T>
): Promise<T> => {
const gameTime = await loadCurrentGameTime(ctx.db);
const fence = await ensureActiveRedisClockFence(ctx.redis, ctx.profile.name, gameTime);
if (!fence) {
throw new TRPCError({
code: 'PRECONDITION_FAILED',
message: 'Clock reconciliation is incomplete; tournament mutation is disabled.',
});
}
const clockContext: TournamentClockContext = {
phase: 'RUNNING',
revision: fence.revision,
deadlineGeneration: fence.generation,
dateToTick: gameTime.dateToTick,
};
return store.withClockContext(clockContext, () => store.withMutationLock(operation));
};
const zTournamentState = z.object({
stage: z.number().int().min(0),
phase: z.number().int().min(0),
@@ -143,7 +171,10 @@ export const tournamentRouter = router({
const store = new TournamentStore(ctx.redis, buildTournamentKeys(ctx.profile.name));
return store.getState();
}),
getAdminStatus: adminProcedure.query(async () => ({ ok: true })),
getAdminStatus: adminProcedure.query(async ({ ctx }) => ({
ok: true,
clock: await loadClockAdminStatus(ctx.db),
})),
getSnapshot: accessAuthedProcedure.query(async ({ ctx }) => {
await getMyGeneral(ctx);
const store = new TournamentStore(ctx.redis, buildTournamentKeys(ctx.profile.name));
@@ -271,7 +302,7 @@ export const tournamentRouter = router({
}),
setState: adminProcedure.input(zTournamentState).mutation(async ({ ctx, input }) => {
const store = new TournamentStore(ctx.redis, buildTournamentKeys(ctx.profile.name));
return store.withMutationLock(async () => {
return withTournamentClockMutation(ctx, store, async () => {
await store.setState({
...input,
type: input.type as TournamentType,
@@ -281,7 +312,7 @@ export const tournamentRouter = router({
}),
patchState: adminProcedure.input(zTournamentState.partial()).mutation(async ({ ctx, input }) => {
const store = new TournamentStore(ctx.redis, buildTournamentKeys(ctx.profile.name));
return store.withMutationLock(async () => {
return withTournamentClockMutation(ctx, store, async () => {
const current = await store.getState();
if (!current) {
throw new TRPCError({ code: 'NOT_FOUND', message: 'Tournament state not found.' });
@@ -297,21 +328,21 @@ export const tournamentRouter = router({
}),
setParticipants: adminProcedure.input(z.array(zParticipant)).mutation(async ({ ctx, input }) => {
const store = new TournamentStore(ctx.redis, buildTournamentKeys(ctx.profile.name));
return store.withMutationLock(async () => {
return withTournamentClockMutation(ctx, store, async () => {
await store.setParticipants(input);
return { ok: true, count: input.length };
});
}),
setMatches: adminProcedure.input(z.array(zMatch)).mutation(async ({ ctx, input }) => {
const store = new TournamentStore(ctx.redis, buildTournamentKeys(ctx.profile.name));
return store.withMutationLock(async () => {
return withTournamentClockMutation(ctx, store, async () => {
await store.setMatches(input);
return { ok: true, count: input.length };
});
}),
setBettingEntries: adminProcedure.input(z.array(zBetEntry)).mutation(async ({ ctx, input }) => {
const store = new TournamentStore(ctx.redis, buildTournamentKeys(ctx.profile.name));
return store.withMutationLock(async () => {
return withTournamentClockMutation(ctx, store, async () => {
await store.setBettingEntries(input);
return { ok: true, count: input.length };
});
@@ -347,7 +378,7 @@ export const tournamentRouter = router({
};
});
return store.withMutationLock(async () => {
return withTournamentClockMutation(ctx, store, async () => {
await store.setParticipants(participants);
return { ok: true, count: participants.length };
});
@@ -394,7 +425,7 @@ export const tournamentRouter = router({
join: authedProcedure.mutation(async ({ ctx }) => {
const general = await getMyGeneral(ctx);
const store = new TournamentStore(ctx.redis, buildTournamentKeys(ctx.profile.name));
return store.withMutationLock(async () => {
return withTournamentClockMutation(ctx, store, async () => {
const state = await store.getState();
if (!state || state.stage !== 1 || state.participantsLockedAt) {
throw new TRPCError({ code: 'BAD_REQUEST', message: '참가 신청 기간이 아닙니다.' });
@@ -459,7 +490,7 @@ export const tournamentRouter = router({
}),
cancel: adminProcedure.mutation(async ({ ctx }) => {
const store = new TournamentStore(ctx.redis, buildTournamentKeys(ctx.profile.name));
return store.withMutationLock(async () => {
return withTournamentClockMutation(ctx, store, async () => {
const state = await store.getState();
if (!state) {
throw new TRPCError({ code: 'NOT_FOUND', message: 'Tournament state not found.' });
@@ -523,7 +554,7 @@ export const tournamentRouter = router({
.mutation(async ({ ctx, input }) => {
const general = await getMyGeneral(ctx);
const store = new TournamentStore(ctx.redis, buildTournamentKeys(ctx.profile.name));
return store.withMutationLock(async () => {
return withTournamentClockMutation(ctx, store, async () => {
const state = await store.getState();
if (!state || state.stage !== 6) {
throw new TRPCError({ code: 'BAD_REQUEST', message: '베팅 기간이 아닙니다.' });
+12 -6
View File
@@ -50,6 +50,7 @@ import { ReadModelOutboxWorker } from './realtime/outboxWorker.js';
import { DeferredGeneralAccessWorker } from './services/deferredGeneralAccess.js';
import { WebPushOutboxWorker } from './services/webPushOutboxWorker.js';
import { scopeHttpIdempotencyKey } from './requestId.js';
import { loadClockReadiness } from './services/clockReadiness.js';
const extractBearerToken = (value: string | string[] | undefined): string | null => {
if (!value) {
@@ -420,12 +421,17 @@ export const createGameApiServer = async () => {
request.raw.on('aborted', close);
});
app.get('/healthz', async () => ({
ok: true,
profile: config.profileName,
postgresPool: postgres.getPoolStats(),
accountIconReconciliation: accountIconResetReconciler.getHealth(),
}));
app.get('/healthz', async (_request, reply) => {
const clock = await loadClockReadiness(postgres.prisma);
if (!clock.reconciliationComplete) reply.code(503);
return {
ok: clock.reconciliationComplete,
profile: config.profileName,
postgresPool: postgres.getPoolStats(),
accountIconReconciliation: accountIconResetReconciler.getHealth(),
clock,
};
});
try {
await realtimeHub.start();
@@ -0,0 +1,92 @@
import { parseGameClockPhase } from '@sammo-ts/common';
import type { DatabaseClient } from '../context.js';
const safeInteger = (value: bigint, label: string): number => {
const result = Number(value);
if (!Number.isSafeInteger(result)) throw new Error(`${label} is outside the safe integer range.`);
return result;
};
export const loadClockReadiness = async (db: DatabaseClient) => {
if (!db.clockProjectionOutbox) {
return {
reconciliationComplete: false,
gameplayEnabled: false,
phase: null,
revision: null,
deadlineGeneration: null,
incompleteOutboxCount: null,
};
}
const [world, incompleteOutboxCount] = await Promise.all([
db.worldState.findFirst({
orderBy: { id: 'asc' },
select: { clockPhase: true, clockRevision: true, deadlineGeneration: true },
}),
db.clockProjectionOutbox.count({ where: { status: { not: 'APPLIED' } } }),
]);
if (!world) {
return {
reconciliationComplete: false,
gameplayEnabled: false,
phase: null,
revision: null,
deadlineGeneration: null,
incompleteOutboxCount,
};
}
const phase = parseGameClockPhase(world.clockPhase);
return {
reconciliationComplete: phase !== 'RECONCILING' && incompleteOutboxCount === 0,
gameplayEnabled: phase === 'RUNNING' || phase === 'MANUAL',
phase,
revision: safeInteger(world.clockRevision, 'clock revision'),
deadlineGeneration: safeInteger(world.deadlineGeneration, 'deadline generation'),
incompleteOutboxCount,
};
};
export const loadClockAdminStatus = async (db: DatabaseClient) => {
const readiness = await loadClockReadiness(db);
if (!db.clockSuspension) {
return { ...readiness, latestReconciliation: null };
}
const latest = await db.clockSuspension.findFirst({
orderBy: { createdAt: 'desc' },
include: {
participants: { orderBy: { participantKey: 'asc' } },
projectionOutbox: { orderBy: { id: 'asc' } },
},
});
if (!latest) return { ...readiness, latestReconciliation: null };
return {
...readiness,
latestReconciliation: {
id: latest.id,
source: latest.source,
policy: latest.policy,
status: latest.status,
sourceRevision: safeInteger(latest.sourceRevision, 'source revision'),
targetRevision: safeInteger(latest.targetRevision, 'target revision'),
cutTick: safeInteger(latest.cutTick, 'cut tick'),
alignedTick: latest.alignedTick === null ? null : safeInteger(latest.alignedTick, 'aligned tick'),
participantChecksumBefore: latest.participantChecksumBefore,
participantChecksumAfter: latest.participantChecksumAfter,
participants: latest.participants.map((participant) => ({
key: participant.participantKey,
policy: participant.policy,
beforeChecksum: participant.beforeChecksum,
afterChecksum: participant.afterChecksum,
affectedCount: participant.affectedCount,
})),
outbox: latest.projectionOutbox.map((entry) => ({
id: entry.id.toString(),
targetRevision: safeInteger(entry.targetRevision, 'outbox target revision'),
status: entry.status,
attempts: entry.attempts,
lastError: entry.lastError,
})),
},
};
};
@@ -0,0 +1,55 @@
import type { CurrentGameTime } from './gameClock.js';
interface ClockFenceRedis {
eval(script: string, options: { keys: string[]; arguments: string[] }): Promise<unknown>;
}
const BOOTSTRAP_CLOCK_FENCE_SCRIPT = `
local revision = redis.call('GET', KEYS[1])
local generation = redis.call('GET', KEYS[2])
local phase = redis.call('GET', KEYS[3])
if not revision and not generation and not phase then
redis.call('SET', KEYS[1], ARGV[1])
redis.call('SET', KEYS[2], ARGV[2])
redis.call('SET', KEYS[3], ARGV[3])
return 1
end
if revision == ARGV[1] and generation == ARGV[2] and phase == ARGV[3] then
return 2
end
return 0
`;
export interface ActiveRedisClockFence {
activeRevisionKey: string;
deadlineGenerationKey: string;
phaseKey: string;
revision: number;
generation: number;
}
export const ensureActiveRedisClockFence = async (
redis: ClockFenceRedis,
profileName: string,
gameTime: CurrentGameTime
): Promise<ActiveRedisClockFence | null> => {
if (
gameTime.phase !== 'RUNNING' ||
!Number.isSafeInteger(gameTime.revision) ||
!Number.isSafeInteger(gameTime.deadlineGeneration)
) {
return null;
}
const fence: ActiveRedisClockFence = {
activeRevisionKey: `sammo:${profileName}:clock:active-revision`,
deadlineGenerationKey: `sammo:${profileName}:clock:deadline-generation`,
phaseKey: `sammo:${profileName}:clock:phase`,
revision: gameTime.revision!,
generation: gameTime.deadlineGeneration!,
};
const result = await redis.eval(BOOTSTRAP_CLOCK_FENCE_SCRIPT, {
keys: [fence.activeRevisionKey, fence.deadlineGenerationKey, fence.phaseKey],
arguments: [String(fence.revision), String(fence.generation), 'RUNNING'],
});
return Number(result) === 1 || Number(result) === 2 ? fence : null;
};
+6
View File
@@ -8,6 +8,9 @@ export interface TournamentKeys {
sourceRevisionKey: string;
sourceRevisionChannel: string;
realtimeEventChannel: string;
activeClockRevisionKey: string;
deadlineGenerationKey: string;
clockPhaseKey: string;
}
export const buildTournamentKeys = (profileName: string): TournamentKeys => ({
@@ -18,4 +21,7 @@ export const buildTournamentKeys = (profileName: string): TournamentKeys => ({
sourceRevisionKey: `sammo:${profileName}:tournament:source-revision`,
sourceRevisionChannel: `sammo:${profileName}:tournament:source-changed`,
realtimeEventChannel: buildGameEventChannel(profileName),
activeClockRevisionKey: `sammo:${profileName}:clock:active-revision`,
deadlineGenerationKey: `sammo:${profileName}:clock:deadline-generation`,
clockPhaseKey: `sammo:${profileName}:clock:phase`,
});
+61 -3
View File
@@ -1,5 +1,5 @@
import { randomUUID } from 'node:crypto';
import { parseTournamentSourceRevision, writeTournamentProjection } from '@sammo-ts/common';
import { parseTournamentSourceRevision, writeTournamentProjection, type TournamentClockFence } from '@sammo-ts/common';
import { z } from 'zod';
import type { TournamentKeys } from './keys.js';
@@ -37,8 +37,12 @@ const zTournamentState = z
openMonth: z.number().int(),
termSeconds: z.number(),
nextAt: z.string(),
nextTick: z.number().int().safe().optional(),
clockRevision: z.number().int().positive().safe().optional(),
deadlineGeneration: z.number().int().positive().safe().optional(),
bettingId: z.number().int().optional(),
bettingCloseAt: z.string().optional(),
bettingCloseTick: z.number().int().safe().optional(),
winnerId: z.number().int().optional(),
bettingSettled: z.boolean().optional(),
rewardSettled: z.boolean().optional(),
@@ -131,12 +135,62 @@ const parseProjection = <T>(raw: string | null, key: string, schema: z.ZodType<T
return parsed.data;
};
export interface TournamentClockContext {
phase: 'RUNNING';
revision: number;
deadlineGeneration: number;
dateToTick(date: Date): number | null;
}
const parseDeadlineTick = (value: string, context: TournamentClockContext, field: string): number => {
const date = new Date(value);
if (Number.isNaN(date.getTime())) {
throw new Error(`Tournament ${field} is not a valid instant.`);
}
const tick = context.dateToTick(date);
if (tick === null || !Number.isSafeInteger(tick)) {
throw new Error(`Tournament ${field} cannot be represented in the active game clock.`);
}
return tick;
};
export const stampTournamentClock = (state: TournamentState, context: TournamentClockContext): TournamentState => ({
...state,
nextTick: parseDeadlineTick(state.nextAt, context, 'nextAt'),
clockRevision: context.revision,
deadlineGeneration: context.deadlineGeneration,
...(state.bettingCloseAt
? { bettingCloseTick: parseDeadlineTick(state.bettingCloseAt, context, 'bettingCloseAt') }
: { bettingCloseTick: undefined }),
});
const toClockFence = (keys: TournamentKeys, context: TournamentClockContext): TournamentClockFence => ({
activeRevisionKey: keys.activeClockRevisionKey,
deadlineGenerationKey: keys.deadlineGenerationKey,
phaseKey: keys.clockPhaseKey,
revision: context.revision,
deadlineGeneration: context.deadlineGeneration,
phase: context.phase,
});
export class TournamentStore {
private clockContext: TournamentClockContext | null = null;
constructor(
private readonly redis: RedisClientLike,
private readonly keys: TournamentKeys
) {}
async withClockContext<T>(context: TournamentClockContext, operation: () => Promise<T>): Promise<T> {
const previous = this.clockContext;
this.clockContext = context;
try {
return await operation();
} finally {
this.clockContext = previous;
}
}
async withMutationLock<T>(operation: () => Promise<T>, timeoutMs = 2_000): Promise<T> {
if (!this.redis.del) {
return operation();
@@ -174,11 +228,15 @@ export class TournamentStore {
}
private async writeWithSourceRevision(key: string, value: unknown): Promise<string> {
return writeTournamentProjection(this.redis, this.keys, [{ key, value }]);
const fence = this.clockContext ? toClockFence(this.keys, this.clockContext) : undefined;
return writeTournamentProjection(this.redis, this.keys, [{ key, value }], fence);
}
async setState(state: TournamentState): Promise<string> {
return this.writeWithSourceRevision(this.keys.stateKey, state);
return this.writeWithSourceRevision(
this.keys.stateKey,
this.clockContext ? stampTournamentClock(state, this.clockContext) : state
);
}
async getParticipants(): Promise<TournamentParticipantEntry[]> {
+4
View File
@@ -9,8 +9,12 @@ export interface TournamentState {
openMonth: number;
termSeconds: number;
nextAt: string;
nextTick?: number;
clockRevision?: number;
deadlineGeneration?: number;
bettingId?: number;
bettingCloseAt?: string;
bettingCloseTick?: number;
winnerId?: number;
bettingSettled?: boolean;
rewardSettled?: boolean;
+106 -34
View File
@@ -13,9 +13,10 @@ import { DatabaseTurnDaemonTransport } from '../daemon/databaseTransport.js';
import { createBestEffortResourceCloser } from '../services/bestEffortResourceCloser.js';
import { loadCurrentGameTime } from '../services/gameClock.js';
import { createPollingWorkerControl, waitForWorkerPoll } from '../services/pollingWorkerLifecycle.js';
import { ensureActiveRedisClockFence } from '../services/redisClockFence.js';
import type { TurnDaemonTransport } from '../daemon/transport.js';
import { buildTournamentKeys } from './keys.js';
import { TournamentStore } from './store.js';
import { TournamentStore, stampTournamentClock, type TournamentClockContext } from './store.js';
import type { TournamentMatchEntry, TournamentState } from './types.js';
import {
applyGroupMatch,
@@ -595,41 +596,63 @@ export const processTournamentTick = async (options: {
prisma: GamePrismaClient;
daemonTransport: TurnDaemonTransport;
now?: () => number;
clockContext?: TournamentClockContext;
}): Promise<TournamentState | null> => {
const { store, prisma, daemonTransport } = options;
const now = options.now ?? Date.now;
let processedState: TournamentState | null = null;
await store.withMutationLock(async () => {
const state = await store.getState();
if (!state || (!state.auto && !needsSettlement(state))) {
return;
}
const nextAt = new Date(state.nextAt).getTime();
if (state.auto && Number.isFinite(nextAt) && nextAt > now()) {
return;
}
const processWithLock = async (): Promise<void> =>
store.withMutationLock(async () => {
const state = await store.getState();
if (!state || (!state.auto && !needsSettlement(state))) {
return;
}
if (
options.clockContext &&
(state.clockRevision !== options.clockContext.revision ||
state.deadlineGeneration !== options.clockContext.deadlineGeneration)
) {
throw new Error('Tournament state does not match the active clock revision.');
}
const nextAt = new Date(state.nextAt).getTime();
const notDue = options.clockContext
? !Number.isSafeInteger(state.nextTick) ||
state.nextTick! > options.clockContext.dateToTick(new Date(now()))!
: Number.isFinite(nextAt) && nextAt > now();
if (state.auto && notDue) {
if (options.clockContext && !Number.isSafeInteger(state.nextTick)) {
throw new Error('Active tournament nextAt lacks the authoritative nextTick dual-write.');
}
return;
}
if (needsSettlement(state)) {
processedState = (await settleTournamentOutcome({ store, daemonTransport, state })) ?? state;
return;
}
if (needsSettlement(state)) {
processedState = (await settleTournamentOutcome({ store, daemonTransport, state })) ?? state;
return;
}
const worldState = await prisma.worldState.findFirst();
const baseSeed = (worldState?.meta as Record<string, unknown> | null)?.hiddenSeed ?? 'tournament';
let nextState = state;
if (isBattleStage(state.stage)) {
nextState = await applyBattle(store, state, String(baseSeed), daemonTransport);
} else if (isPreBattleStage(state.stage)) {
nextState = await applyPreBattleStage(store, prisma, state, String(baseSeed), daemonTransport, now);
}
processedState =
(await settleTournamentOutcome({
store,
daemonTransport,
state: nextState,
})) ?? nextState;
});
const worldState = await prisma.worldState.findFirst();
const baseSeed = (worldState?.meta as Record<string, unknown> | null)?.hiddenSeed ?? 'tournament';
let nextState = state;
if (isBattleStage(state.stage)) {
nextState = await applyBattle(store, state, String(baseSeed), daemonTransport);
} else if (isPreBattleStage(state.stage)) {
nextState = await applyPreBattleStage(store, prisma, state, String(baseSeed), daemonTransport, now);
}
processedState =
(await settleTournamentOutcome({
store,
daemonTransport,
state: nextState,
})) ?? nextState;
});
if (options.clockContext) {
await store.withClockContext(options.clockContext, processWithLock);
} else {
await processWithLock();
}
return processedState;
};
@@ -656,16 +679,60 @@ export const runTournamentWorker = async (options: TournamentWorkerOptions = {})
try {
while (!control.signal.aborted) {
const state = await store.getState();
let state = await store.getState();
if (!state || (!state.auto && !needsSettlement(state))) {
await waitForWorkerPoll(control.signal, config.tournamentPollMs);
continue;
}
const gameTime = await loadCurrentGameTime(postgres.prisma);
if (gameTime.phase && gameTime.phase !== 'RUNNING') {
await waitForWorkerPoll(control.signal, config.tournamentPollMs);
continue;
}
const clockFence = gameTime.phase
? await ensureActiveRedisClockFence(redis.client, config.profileName, gameTime)
: null;
if (gameTime.phase && !clockFence) {
await waitForWorkerPoll(control.signal, config.tournamentPollMs);
continue;
}
const clockContext: TournamentClockContext | undefined = clockFence
? {
phase: 'RUNNING',
revision: clockFence.revision,
deadlineGeneration: clockFence.generation,
dateToTick: gameTime.dateToTick,
}
: undefined;
if (
clockContext &&
(!Number.isSafeInteger(state.nextTick) ||
state.clockRevision === undefined ||
state.deadlineGeneration === undefined)
) {
state = stampTournamentClock(state, clockContext);
await store.withClockContext(clockContext, () => store.setState(state!));
}
if (
clockContext &&
(state.clockRevision !== clockContext.revision ||
state.deadlineGeneration !== clockContext.deadlineGeneration)
) {
await waitForWorkerPoll(control.signal, config.tournamentPollMs);
continue;
}
const nextAt = new Date(state.nextAt).getTime();
const gameNow = (await loadCurrentGameTime(postgres.prisma)).now.getTime();
if (state.auto && Number.isFinite(nextAt) && nextAt > gameNow) {
await waitForWorkerPoll(control.signal, Math.min(config.tournamentPollMs, nextAt - gameNow));
const gameNow = gameTime.now.getTime();
const notDue = clockContext
? state.nextTick! > gameTime.tick!
: Number.isFinite(nextAt) && nextAt > gameNow;
if (state.auto && notDue) {
await waitForWorkerPoll(
control.signal,
Math.min(config.tournamentPollMs, Math.max(1, nextAt - gameNow))
);
continue;
}
@@ -675,6 +742,7 @@ export const runTournamentWorker = async (options: TournamentWorkerOptions = {})
prisma: postgres.prisma,
daemonTransport,
now: () => gameNow,
clockContext,
});
} catch (error) {
const message = error instanceof Error ? error.message : 'Unknown error';
@@ -700,7 +768,11 @@ export const runTournamentWorker = async (options: TournamentWorkerOptions = {})
lastError: message,
lastErrorAt: failedAt,
};
await store.setState(nextState);
if (clockContext) {
await store.withClockContext(clockContext, () => store.setState(nextState));
} else {
await store.setState(nextState);
}
}
await waitForWorkerPoll(control.signal, config.tournamentPollMs);
+34 -1
View File
@@ -2,7 +2,7 @@ import { describe, expect, it, vi } from 'vitest';
import type { GamePrismaClient } from '@sammo-ts/infra';
import { processDueAuctionId, reconcilePendingAuctionTimers } from '../src/auction/worker.js';
import { popDueAuctionIds, processDueAuctionId, reconcilePendingAuctionTimers } from '../src/auction/worker.js';
import { resolveAuctionSeedScore } from '../src/auction/scheduler.js';
const buildRedis = () => ({
@@ -55,6 +55,38 @@ const buildDb = (options: {
};
describe('auction worker clock-shift race', () => {
it('uses one Redis script for revision, generation, phase, due-read, and removal', async () => {
const redis = { ...buildRedis(), eval: vi.fn(async () => ['7', '9']) };
await expect(
popDueAuctionIds(redis, 'auction-timer', 123_456, 100, {
activeRevisionKey: 'clock-revision',
deadlineGenerationKey: 'deadline-generation',
phaseKey: 'clock-phase',
revision: 4,
generation: 8,
})
).resolves.toEqual(['7', '9']);
expect(redis.eval).toHaveBeenCalledWith(expect.stringContaining('ZRANGEBYSCORE'), {
keys: ['auction-timer', 'clock-revision', 'deadline-generation', 'clock-phase'],
arguments: ['4', '8', '123456', '100'],
});
expect(redis.zRangeByScore).not.toHaveBeenCalled();
expect(redis.zRem).not.toHaveBeenCalled();
});
it('returns no due member when the atomic Redis clock fence rejects the pop', async () => {
const redis = { ...buildRedis(), eval: vi.fn(async () => ['__CLOCK_FENCE__']) };
await expect(
popDueAuctionIds(redis, 'auction-timer', 123_456, 100, {
activeRevisionKey: 'clock-revision',
deadlineGenerationKey: 'deadline-generation',
phaseKey: 'clock-phase',
revision: 4,
generation: 8,
})
).resolves.toEqual([]);
});
it('seeds OPEN at its deadline but retries FINALIZING at the current logical tick', () => {
const now = new Date('2026-07-30T12:00:00.000Z');
const time = {
@@ -292,6 +324,7 @@ describe('auction worker clock-shift race', () => {
requestId,
target: 'ENGINE',
eventType: 'auctionFinalize',
acceptedGameTick: 72_000_000n,
payload: {
type: 'auctionFinalize',
requestId,
+87
View File
@@ -0,0 +1,87 @@
import { describe, expect, it, vi } from 'vitest';
import type { DatabaseClient } from '../src/context.js';
import { loadClockAdminStatus, loadClockReadiness } from '../src/services/clockReadiness.js';
describe('clock reconciliation readiness', () => {
it('fails closed when the reconciliation schema is not available', async () => {
const db = {} as DatabaseClient;
await expect(loadClockReadiness(db)).resolves.toEqual({
reconciliationComplete: false,
gameplayEnabled: false,
phase: null,
revision: null,
deadlineGeneration: null,
incompleteOutboxCount: null,
});
});
it('blocks readiness for RECONCILING or incomplete outbox state', async () => {
const db = {
worldState: {
findFirst: vi.fn(async () => ({
clockPhase: 'RECONCILING',
clockRevision: 9n,
deadlineGeneration: 4n,
})),
},
clockProjectionOutbox: { count: vi.fn(async () => 1) },
} as unknown as DatabaseClient;
await expect(loadClockReadiness(db)).resolves.toMatchObject({
reconciliationComplete: false,
gameplayEnabled: false,
phase: 'RECONCILING',
revision: 9,
deadlineGeneration: 4,
incompleteOutboxCount: 1,
});
});
it('exposes participant checksums and incomplete outbox detail to admins', async () => {
const db = {
worldState: {
findFirst: vi.fn(async () => ({
clockPhase: 'RUNNING',
clockRevision: 3n,
deadlineGeneration: 2n,
})),
},
clockProjectionOutbox: { count: vi.fn(async () => 0) },
clockSuspension: {
findFirst: vi.fn(async () => ({
id: 'maintenance-1',
source: 'MAINTENANCE',
policy: 'EXACT',
status: 'APPLIED',
sourceRevision: 2n,
targetRevision: 3n,
cutTick: 100n,
alignedTick: 130n,
participantChecksumBefore: 'before-all',
participantChecksumAfter: 'after-all',
participants: [
{
participantKey: 'general-turn',
policy: 'SHIFT',
beforeChecksum: 'before',
afterChecksum: 'after',
affectedCount: 2,
},
],
projectionOutbox: [{ id: 8n, targetRevision: 3n, status: 'APPLIED', attempts: 1, lastError: null }],
})),
},
} as unknown as DatabaseClient;
await expect(loadClockAdminStatus(db)).resolves.toMatchObject({
reconciliationComplete: true,
latestReconciliation: {
id: 'maintenance-1',
participantChecksumBefore: 'before-all',
participantChecksumAfter: 'after-all',
participants: [{ key: 'general-turn', policy: 'SHIFT', affectedCount: 2 }],
outbox: [{ id: '8', status: 'APPLIED' }],
},
});
});
});
@@ -511,6 +511,10 @@ integration('API input event boundary', () => {
it('reuses the same engine child event but rejects a changed retry payload', async () => {
const transport = new DatabaseTurnDaemonTransport(db, 100);
const requestId = 'integration:api:engine-child';
const worldClock = await db.worldState.findFirst({
orderBy: { id: 'asc' },
select: { clockRevision: true, deadlineGeneration: true },
});
const acceptedWindowStart = Date.now();
await transport.sendCommand({ type: 'vacation', requestId, userId: 'user-7', generalId: 7 });
const acceptedWindowEnd = Date.now();
@@ -518,6 +522,9 @@ integration('API input event boundary', () => {
expect(event.actorUserId).toBe('user-7');
expect(event.createdAt.getTime()).toBeGreaterThanOrEqual(acceptedWindowStart);
expect(event.createdAt.getTime()).toBeLessThanOrEqual(acceptedWindowEnd);
expect(event.acceptedGameTick).not.toBeNull();
expect(event.acceptedClockRevision).toBe(worldClock?.clockRevision ?? null);
expect(event.acceptedDeadlineGeneration).toBe(worldClock?.deadlineGeneration ?? null);
await expect(
transport.sendCommand({ type: 'vacation', requestId, userId: 'user-7', generalId: 7 })
).resolves.toBe(requestId);
+7 -2
View File
@@ -28,6 +28,9 @@ const createContext = (payload: unknown = {}) => {
status: 'PENDING',
result: null,
attempts: 0,
acceptedGameTick: 100n,
acceptedClockRevision: 3n,
acceptedDeadlineGeneration: 2n,
},
];
}
@@ -36,8 +39,8 @@ const createContext = (payload: unknown = {}) => {
});
const transaction = {
$queryRaw: queryRaw,
$executeRaw: vi.fn(async () => {
order.push('accepted');
$executeRaw: vi.fn(async (query: { sql?: string }) => {
order.push(query.sql?.includes('pg_advisory_xact_lock') ? 'clock-fence' : 'accepted');
return 1;
}),
$executeRawUnsafe: vi.fn(async (statement: string) => {
@@ -88,6 +91,7 @@ describe('API input-event change journal boundary', () => {
expect(fixture.order).toEqual([
'transaction-begin',
'clock-fence',
'accepted',
'locked',
'processing',
@@ -112,6 +116,7 @@ describe('API input-event change journal boundary', () => {
expect(fixture.order).toEqual([
'transaction-begin',
'clock-fence',
'accepted',
'locked',
'processing',
+36 -7
View File
@@ -30,11 +30,31 @@ class MemoryRedis {
}
async eval(_script: string, options: { keys: string[]; arguments: string[] }): Promise<string> {
const [valueKey, revisionKey] = options.keys;
const [value] = options.arguments;
if (options.keys.length === 3 && options.keys[0]?.endsWith(':clock:active-revision')) {
const current = options.keys.map((key) => this.values.get(key));
if (current.every((value) => value === undefined)) {
options.keys.forEach((key, index) => this.values.set(key, options.arguments[index]!));
return '1';
}
return current.every((value, index) => value === options.arguments[index]) ? '2' : '0';
}
const fenced = options.keys.at(-1)?.endsWith(':clock:phase') === true;
const writeCount = options.keys.length - (fenced ? 4 : 1);
if (fenced) {
const clockKeys = options.keys.slice(-3);
const expected = options.arguments.slice(-3);
if (!clockKeys.every((key, index) => this.values.get(key) === expected[index])) {
return '__CLOCK_FENCE__';
}
}
const valueKey = options.keys[0];
const revisionKey = options.keys[writeCount];
const value = options.arguments[0];
if (!valueKey || !revisionKey || value === undefined) throw new Error('invalid eval arguments');
const revision = Number(this.values.get(revisionKey) ?? '0') + 1;
this.values.set(valueKey, value);
for (let index = 0; index < writeCount; index += 1) {
this.values.set(options.keys[index]!, options.arguments[index]!);
}
this.values.set(revisionKey, String(revision));
return String(revision);
}
@@ -144,6 +164,14 @@ const buildContext = (options: {
},
worldState: {
findFirst: async () => ({
clockBaseTime: new Date('2026-01-01T00:00:00.000Z'),
clockTick: 0n,
clockMode: 'realtime',
clockWallAnchor: new Date('2026-01-01T00:00:00.000Z'),
clockPhase: 'RUNNING',
clockRevision: 1n,
deadlineGeneration: 1n,
tickSeconds: 60,
config: { const: { develCost: options.develCost ?? 200 } },
...(options.currentDevelCost === undefined ? {} : { meta: { develcost: options.currentDevelCost } }),
}),
@@ -394,7 +422,10 @@ describe('tournament router permissions and mutations', () => {
roles: ['admin.tournament:che:default'],
})
);
await expect(adminCaller.tournament.getAdminStatus()).resolves.toEqual({ ok: true });
await expect(adminCaller.tournament.getAdminStatus()).resolves.toMatchObject({
ok: true,
clock: { reconciliationComplete: false, latestReconciliation: null },
});
});
it('applies the admin role boundary to every tournament mutation', async () => {
@@ -479,9 +510,7 @@ describe('tournament router permissions and mutations', () => {
])
).resolves.toEqual({ ok: true, count: 1 });
await expect(
caller.tournament.setBettingEntries([
{ generalId: general.id, targetId: rival.id, amount: 100 },
])
caller.tournament.setBettingEntries([{ generalId: general.id, targetId: rival.id, amount: 100 }])
).resolves.toEqual({ ok: true, count: 1 });
await expect(caller.tournament.seedParticipants({ generalIds: [general.id, rival.id] })).resolves.toEqual({
ok: true,
@@ -41,6 +41,9 @@ integration('TournamentStore Redis source revision', () => {
keys.matchesKey,
keys.bettingKey,
keys.sourceRevisionKey,
keys.activeClockRevisionKey,
keys.deadlineGenerationKey,
keys.clockPhaseKey,
]);
if (subscriber) {
await subscriber.client.unsubscribe(keys.sourceRevisionChannel);
@@ -124,4 +127,48 @@ integration('TournamentStore Redis source revision', () => {
}),
]);
});
it('dual-writes deadline ticks and rejects a Redis clock revision race atomically', async () => {
const store = new TournamentStore(connector.client, keys);
await Promise.all([
connector.client.set(keys.activeClockRevisionKey, '7'),
connector.client.set(keys.deadlineGenerationKey, '3'),
connector.client.set(keys.clockPhaseKey, 'RUNNING'),
]);
const clockContext = {
phase: 'RUNNING' as const,
revision: 7,
deadlineGeneration: 3,
dateToTick: (date: Date) => Math.trunc(date.getTime() / 1_000),
};
const nextAt = '2026-09-03T10:00:00.000Z';
await store.withClockContext(clockContext, () =>
store.setState({
stage: 6,
phase: 0,
type: 0,
auto: true,
openYear: 200,
openMonth: 1,
termSeconds: 10,
nextAt,
bettingCloseAt: '2026-09-03T09:59:50.000Z',
})
);
await expect(store.getState()).resolves.toMatchObject({
nextTick: Math.trunc(new Date(nextAt).getTime() / 1_000),
bettingCloseTick: Math.trunc(new Date('2026-09-03T09:59:50.000Z').getTime() / 1_000),
clockRevision: 7,
deadlineGeneration: 3,
});
const beforeRevision = await store.getSourceRevision();
await connector.client.set(keys.activeClockRevisionKey, '8');
await expect(
store.withClockContext(clockContext, () =>
store.setMatches([{ id: 99, stage: 7, roundIndex: 0, attackerId: 1, defenderId: 2 }])
)
).rejects.toThrow('clock revision fence failed');
await expect(store.getSourceRevision()).resolves.toBe(beforeRevision);
});
});
+8 -1
View File
@@ -269,7 +269,14 @@ export const createAuctionBidder = async (options: {
};
}
const processingNow = world.getGameNow(new Date());
const { bidAt, bidTick } = resolveAuctionBidTiming(world, processingNow, command.acceptedGameTick);
const convertedProcessingTick = Reflect.get(command, 'processingGameTick');
const { bidAt, bidTick } = resolveAuctionBidTiming(
world,
processingNow,
typeof convertedProcessingTick === 'number' && Number.isSafeInteger(convertedProcessingTick)
? convertedProcessingTick
: command.acceptedGameTick
);
if (hasAuctionClosePassed(auction, bidAt, bidTick)) {
return {
type: 'auctionBid',
+2
View File
@@ -20,6 +20,8 @@ export * from './turn/engineStateManager.js';
export * from './turn/inMemoryStateStore.js';
export * from './turn/inMemoryTurnProcessor.js';
export * from './turn/databaseHooks.js';
export * from './turn/clockReconciliation.js';
export * from './turn/clockProjectionOutbox.js';
export * from './turn/joinCreateGeneralService.js';
export * from './turn/npcPossessionService.js';
export * from './turn/selectPoolService.js';
@@ -108,6 +108,13 @@ export class DatabaseTurnDaemonCommandQueue implements TurnDaemonControlQueue, T
private async claimPending(limit = 100): Promise<TurnDaemonCommand[]> {
await this.recoverExpiredLeases();
return this.db.$transaction(async (transaction) => {
const world = await transaction.worldState.findFirst({
orderBy: { id: 'asc' },
select: { clockPhase: true, clockRevision: true, deadlineGeneration: true, clockTick: true },
});
const gameplayAllowed =
!world || world.clockPhase === 'RUNNING' || world.clockPhase === 'MANUAL';
const currentRevision = world?.clockRevision ?? null;
const rows = await transaction.$queryRaw<
Array<{
sequence: bigint;
@@ -115,6 +122,9 @@ export class DatabaseTurnDaemonCommandQueue implements TurnDaemonControlQueue, T
eventType: string;
payload: unknown;
createdAt: Date;
acceptedGameTick: bigint | null;
acceptedClockRevision: bigint | null;
acceptedDeadlineGeneration: bigint | null;
}>
>(GamePrisma.sql`
SELECT
@@ -122,10 +132,14 @@ export class DatabaseTurnDaemonCommandQueue implements TurnDaemonControlQueue, T
"request_id" AS "requestId",
"event_type" AS "eventType",
"payload",
"created_at" AS "createdAt"
"created_at" AS "createdAt",
"accepted_game_tick" AS "acceptedGameTick",
"accepted_clock_revision" AS "acceptedClockRevision",
"accepted_deadline_generation" AS "acceptedDeadlineGeneration"
FROM "input_event"
WHERE "target" = 'ENGINE'::"InputEventTarget"
AND "status" = 'PENDING'::"InputEventStatus"
AND (${gameplayAllowed} OR "event_type" = 'getStatus')
ORDER BY "sequence" ASC
FOR UPDATE SKIP LOCKED
LIMIT ${limit}
@@ -133,23 +147,53 @@ export class DatabaseTurnDaemonCommandQueue implements TurnDaemonControlQueue, T
if (rows.length === 0) {
return [];
}
await transaction.inputEvent.updateMany({
where: {
sequence: { in: rows.map((row) => row.sequence) },
target: 'ENGINE',
status: 'PENDING',
},
data: {
status: 'PROCESSING',
processingAt: new Date(),
lockedBy: this.workerId,
leaseUntil: new Date(Date.now() + this.leaseDurationMs),
attempts: { increment: 1 },
},
});
const appliedSuspensions = currentRevision
? await transaction.clockSuspension.findMany({
where: { status: 'APPLIED', targetRevision: { lte: currentRevision } },
orderBy: { sourceRevision: 'asc' },
select: { sourceRevision: true, targetRevision: true, shiftTicks: true },
})
: [];
const convertTick = (row: (typeof rows)[number]): bigint | null | undefined => {
if (row.eventType === 'getStatus') return row.acceptedGameTick;
if (row.acceptedGameTick === null || row.acceptedClockRevision === null || currentRevision === null) {
return row.acceptedGameTick ?? world?.clockTick ?? null;
}
if (row.acceptedClockRevision > currentRevision) return undefined;
let revision = row.acceptedClockRevision;
let tick = row.acceptedGameTick;
while (revision < currentRevision) {
const step = appliedSuspensions.find((entry) => entry.sourceRevision === revision);
if (!step || step.shiftTicks === null || step.targetRevision !== revision + 1n) return undefined;
tick += step.shiftTicks;
revision = step.targetRevision;
}
return tick;
};
const processableRows = rows
.map((row) => ({ row, processingGameTick: convertTick(row) }))
.filter(
(entry): entry is { row: (typeof rows)[number]; processingGameTick: bigint | null } =>
entry.processingGameTick !== undefined
);
for (const { row, processingGameTick } of processableRows) {
await transaction.inputEvent.update({
where: { sequence: row.sequence },
data: {
status: 'PROCESSING',
processingAt: new Date(),
processingGameTick,
processingClockRevision: currentRevision,
processingDeadlineGeneration: world?.deadlineGeneration ?? null,
lockedBy: this.workerId,
leaseUntil: new Date(Date.now() + this.leaseDurationMs),
attempts: { increment: 1 },
},
});
}
const commands: TurnDaemonCommand[] = [];
for (const row of rows) {
for (const { row, processingGameTick } of processableRows) {
const command = normalizeTurnDaemonCommand({
requestId: row.requestId,
sentAt: row.createdAt.toISOString(),
@@ -168,6 +212,27 @@ export class DatabaseTurnDaemonCommandQueue implements TurnDaemonControlQueue, T
});
continue;
}
if (
processingGameTick !== null &&
row.acceptedGameTick !== null &&
processingGameTick !== row.acceptedGameTick
) {
const value = Number(processingGameTick);
if (!Number.isSafeInteger(value)) {
await transaction.inputEvent.update({
where: { sequence: row.sequence },
data: {
status: 'FAILED',
error: 'Converted processing game tick is outside the safe integer range.',
completedAt: new Date(),
lockedBy: null,
leaseUntil: null,
},
});
continue;
}
Reflect.set(command, 'processingGameTick', value);
}
commands.push(command);
}
return commands;
@@ -0,0 +1,343 @@
import { createHash } from 'node:crypto';
import { GameClock, parseGameClockPhase } from '@sammo-ts/common';
import {
CLOCK_OPERATION_PERSISTENCE_LOCK,
GamePrisma,
acquireGameSchemaAdvisoryXactLock,
type GamePrismaClient,
} from '@sammo-ts/infra';
interface ClockProjectionRedis {
get(key: string): Promise<string | null>;
eval(script: string, options: { keys: string[]; arguments: string[] }): Promise<unknown>;
}
interface ClaimedOutboxRow {
id: bigint;
}
interface DbWallRow {
wallNow: Date;
}
interface ProjectionPayload {
version: 1;
profileName: string;
suspensionId: string;
sourceRevision: number;
targetRevision: number;
deadlineGeneration: number;
shiftTicks: number;
projectionDeltaMilliseconds: number;
clockBaseTime: string;
ticksPerSecond: number;
}
interface TournamentProjectionState {
stage?: number;
nextAt?: string;
nextTick?: number;
bettingCloseAt?: string;
bettingCloseTick?: number;
clockRevision?: number;
deadlineGeneration?: number;
[key: string]: unknown;
}
const APPLY_CLOCK_PROJECTION_SCRIPT = `
local active = redis.call('GET', KEYS[1])
if active == ARGV[2] then
if redis.call('GET', KEYS[5]) == ARGV[4] and redis.call('GET', KEYS[2]) == ARGV[3] then
return 2
end
return -3
end
if active and active ~= ARGV[1] then
return -1
end
if ARGV[5] ~= '__NONE__' and redis.call('GET', KEYS[4]) ~= ARGV[5] then
return -2
end
redis.call('DEL', KEYS[3])
local count = tonumber(ARGV[7])
local offset = 8
for index = 1, count do
redis.call('ZADD', KEYS[3], ARGV[offset], ARGV[offset + 1])
offset = offset + 2
end
if ARGV[5] ~= '__NONE__' then
redis.call('SET', KEYS[4], ARGV[6])
end
redis.call('SET', KEYS[1], ARGV[2])
redis.call('SET', KEYS[2], ARGV[3])
redis.call('SET', KEYS[5], ARGV[4])
redis.call('SET', KEYS[6], 'RUNNING')
return 1
`;
const safeInteger = (value: unknown, label: string): number => {
const result = typeof value === 'bigint' ? Number(value) : value;
if (typeof result !== 'number' || !Number.isSafeInteger(result)) {
throw new Error(`${label} must be a safe integer.`);
}
return result;
};
const canonicalize = (value: unknown): unknown => {
if (typeof value === 'bigint') return value.toString();
if (Array.isArray(value)) return value.map(canonicalize);
if (value && typeof value === 'object') {
return Object.fromEntries(
Object.entries(value as Record<string, unknown>)
.sort(([left], [right]) => left.localeCompare(right))
.map(([key, item]) => [key, canonicalize(item)])
);
}
return value;
};
const stableJson = (value: unknown): string => JSON.stringify(canonicalize(value));
const checksum = (value: unknown): string => createHash('sha256').update(stableJson(value)).digest('hex');
const readDbWall = async (db: GamePrisma.TransactionClient): Promise<Date> => {
const rows = await db.$queryRaw<DbWallRow[]>(GamePrisma.sql`
SELECT (CURRENT_TIMESTAMP AT TIME ZONE 'UTC')::timestamp(3) AS "wallNow"
`);
if (!rows[0]?.wallNow) throw new Error('Failed to read PostgreSQL wall time for the projection outbox.');
return rows[0].wallNow;
};
const parsePayload = (value: GamePrisma.JsonValue): ProjectionPayload => {
if (!value || typeof value !== 'object' || Array.isArray(value)) {
throw new Error('Clock projection outbox payload must be an object.');
}
const payload = value as Record<string, unknown>;
if (payload.version !== 1 || typeof payload.profileName !== 'string' || typeof payload.suspensionId !== 'string') {
throw new Error('Clock projection outbox payload identity is invalid.');
}
if (typeof payload.clockBaseTime !== 'string') {
throw new Error('Clock projection outbox is missing its projection base.');
}
return {
version: 1,
profileName: payload.profileName,
suspensionId: payload.suspensionId,
sourceRevision: safeInteger(payload.sourceRevision, 'sourceRevision'),
targetRevision: safeInteger(payload.targetRevision, 'targetRevision'),
deadlineGeneration: safeInteger(payload.deadlineGeneration, 'deadlineGeneration'),
shiftTicks: safeInteger(payload.shiftTicks, 'shiftTicks'),
projectionDeltaMilliseconds: safeInteger(
payload.projectionDeltaMilliseconds,
'projectionDeltaMilliseconds'
),
clockBaseTime: payload.clockBaseTime,
ticksPerSecond: safeInteger(payload.ticksPerSecond, 'ticksPerSecond'),
};
};
const claimNext = async (db: GamePrismaClient, workerId: string) =>
db.$transaction(async (transaction) => {
const rows = await transaction.$queryRaw<ClaimedOutboxRow[]>(GamePrisma.sql`
SELECT id
FROM clock_projection_outbox
WHERE available_at <= (CURRENT_TIMESTAMP AT TIME ZONE 'UTC')
AND (
status IN ('PENDING', 'FAILED')
OR (status = 'APPLYING' AND locked_at < (CURRENT_TIMESTAMP AT TIME ZONE 'UTC') - INTERVAL '30 seconds')
)
ORDER BY id
LIMIT 1
FOR UPDATE SKIP LOCKED
`);
const id = rows[0]?.id;
if (id === undefined) return null;
const lockedAt = await readDbWall(transaction);
return transaction.clockProjectionOutbox.update({
where: { id },
data: {
status: 'APPLYING',
attempts: { increment: 1 },
lockedAt,
lockedBy: workerId,
lastError: null,
},
});
});
const projectTournamentState = (
raw: string | null,
payload: ProjectionPayload,
clock: GameClock
): { expected: string; next: string } | null => {
if (!raw) return null;
const parsed = JSON.parse(raw) as TournamentProjectionState;
const active = typeof parsed.stage === 'number' && parsed.stage > 0;
if (active && parsed.nextAt && !Number.isSafeInteger(parsed.nextTick)) {
throw new Error('Active tournament nextAt lacks the authoritative nextTick dual-write.');
}
if (active && parsed.bettingCloseAt && !Number.isSafeInteger(parsed.bettingCloseTick)) {
throw new Error('Active tournament bettingCloseAt lacks the authoritative bettingCloseTick dual-write.');
}
const next: TournamentProjectionState = {
...parsed,
clockRevision: payload.targetRevision,
deadlineGeneration: payload.deadlineGeneration,
};
if (Number.isSafeInteger(parsed.nextTick)) {
next.nextTick = parsed.nextTick! + payload.shiftTicks;
next.nextAt = clock.tickToDate(next.nextTick).toISOString();
}
if (Number.isSafeInteger(parsed.bettingCloseTick)) {
next.bettingCloseTick = parsed.bettingCloseTick! + payload.shiftTicks;
next.bettingCloseAt = clock.tickToDate(next.bettingCloseTick).toISOString();
}
return { expected: raw, next: JSON.stringify(next) };
};
const recordFailure = async (db: GamePrismaClient, outboxId: bigint, error: unknown): Promise<void> => {
const message = error instanceof Error ? error.message : String(error);
await db.$executeRaw(GamePrisma.sql`
UPDATE clock_projection_outbox
SET status = 'FAILED',
locked_at = NULL,
locked_by = NULL,
last_error = ${message.slice(0, 4_000)},
available_at = (CURRENT_TIMESTAMP AT TIME ZONE 'UTC') + INTERVAL '1 second'
WHERE id = ${outboxId} AND status = 'APPLYING'
`);
};
export const applyNextClockProjection = async (options: {
db: GamePrismaClient;
redis: ClockProjectionRedis;
workerId: string;
}): Promise<'IDLE' | 'APPLIED' | 'RECOVERED'> => {
if (!options.workerId.trim()) throw new Error('Clock projection worker ID is required.');
const outbox = await claimNext(options.db, options.workerId);
if (!outbox) return 'IDLE';
try {
const payload = parsePayload(outbox.payload);
if (checksum(outbox.payload) !== outbox.checksum) {
throw new Error('Clock projection outbox checksum verification failed.');
}
if (outbox.targetRevision !== BigInt(payload.targetRevision)) {
throw new Error('Clock projection payload revision differs from its outbox row.');
}
const world = await options.db.worldState.findUniqueOrThrow({ where: { id: outbox.worldStateId } });
if (
parseGameClockPhase(world.clockPhase) !== 'RECONCILING' ||
world.clockRevision !== outbox.targetRevision ||
world.deadlineGeneration !== BigInt(payload.deadlineGeneration) ||
!world.clockBaseTime
) {
throw new Error('Clock projection DB phase/revision/generation fence failed.');
}
const clock = new GameClock({
baseTime: new Date(payload.clockBaseTime),
tick: safeInteger(world.clockTick, 'world clock tick'),
mode: world.clockMode === 'manual' ? 'manual' : 'realtime',
wallAnchor: world.clockWallAnchor ?? new Date(),
turnSeconds: world.tickSeconds,
phase: 'RECONCILING',
revision: payload.targetRevision,
});
if (clock.ticksPerSecond !== payload.ticksPerSecond) {
throw new Error('Clock projection rate differs from the durable outbox payload.');
}
const auctions = await options.db.auction.findMany({
where: { status: { in: ['OPEN', 'FINALIZING'] } },
orderBy: { id: 'asc' },
select: { id: true, closeTick: true },
});
const timers = auctions.map((auction) => {
if (auction.closeTick === null) {
throw new Error(`Active auction ${auction.id} lacks closeTick during projection rebuild.`);
}
return { score: safeInteger(auction.closeTick, `auction ${auction.id} closeTick`), id: String(auction.id) };
});
const prefix = `sammo:${payload.profileName}`;
const tournamentKey = `${prefix}:tournament:state`;
const tournament = projectTournamentState(await options.redis.get(tournamentKey), payload, clock);
const result = await options.redis.eval(APPLY_CLOCK_PROJECTION_SCRIPT, {
keys: [
`${prefix}:clock:active-revision`,
`${prefix}:clock:deadline-generation`,
`${prefix}:auction:timer`,
tournamentKey,
`${prefix}:clock:projection-checksum`,
`${prefix}:clock:phase`,
],
arguments: [
String(payload.sourceRevision),
String(payload.targetRevision),
String(payload.deadlineGeneration),
outbox.checksum,
tournament?.expected ?? '__NONE__',
tournament?.next ?? '__NONE__',
String(timers.length),
...timers.flatMap(({ score, id }) => [String(score), id]),
],
});
const applied = Number(result);
if (applied === -1) throw new Error('Redis active clock revision does not match the outbox source revision.');
if (applied === -2) throw new Error('Redis tournament state changed while rebuilding its projection.');
if (applied === -3) throw new Error('Redis target revision exists without the expected projection checksum.');
if (applied !== 1 && applied !== 2) throw new Error(`Unexpected Redis clock projection result: ${String(result)}`);
await options.db.$transaction(async (transaction) => {
await acquireGameSchemaAdvisoryXactLock(transaction, CLOCK_OPERATION_PERSISTENCE_LOCK);
await transaction.$queryRaw<ClaimedOutboxRow[]>(GamePrisma.sql`
SELECT id FROM world_state WHERE id = ${outbox.worldStateId} FOR UPDATE
`);
const finalized = await transaction.worldState.updateMany({
where: {
id: outbox.worldStateId,
clockPhase: 'RECONCILING',
clockRevision: outbox.targetRevision,
deadlineGeneration: BigInt(payload.deadlineGeneration),
},
data: { clockPhase: 'RUNNING' },
});
if (finalized.count !== 1) {
throw new Error('Clock projection final RUNNING transition fence failed.');
}
const appliedAt = await readDbWall(transaction);
await transaction.clockProjectionOutbox.update({
where: { id: outbox.id },
data: { status: 'APPLIED', appliedAt, lockedAt: null, lockedBy: null, lastError: null },
});
if (outbox.suspensionId) {
await transaction.clockSuspension.update({
where: { id: outbox.suspensionId },
data: { status: 'APPLIED' },
});
}
});
return applied === 2 ? 'RECOVERED' : 'APPLIED';
} catch (error) {
await recordFailure(options.db, outbox.id, error);
throw error;
}
};
export const loadClockReconciliationReadiness = async (db: GamePrismaClient) => {
const world = await db.worldState.findFirst({
orderBy: { id: 'asc' },
select: { clockPhase: true, clockRevision: true, deadlineGeneration: true },
});
const incompleteOutboxCount = await db.clockProjectionOutbox.count({ where: { status: { not: 'APPLIED' } } });
if (!world) {
return { ready: false, phase: null, revision: null, deadlineGeneration: null, incompleteOutboxCount };
}
const phase = parseGameClockPhase(world.clockPhase);
return {
ready: phase !== 'RECONCILING' && incompleteOutboxCount === 0,
gameplayEnabled: phase === 'RUNNING' || phase === 'MANUAL',
phase,
revision: safeInteger(world.clockRevision, 'clock revision'),
deadlineGeneration: safeInteger(world.deadlineGeneration, 'deadline generation'),
incompleteOutboxCount,
};
};
@@ -0,0 +1,763 @@
import { createHash } from 'node:crypto';
import {
GAME_TICKS_PER_TURN,
MAX_SAFE_GAME_TICK,
GameClock,
buildClockAlignmentPlan,
parseClockAlignmentPolicy,
parseGameClockPhase,
type ClockAlignmentPolicy,
} from '@sammo-ts/common';
import {
CLOCK_OPERATION_PERSISTENCE_LOCK,
GENERAL_ACCESS_PERSISTENCE_LOCK,
GamePrisma,
acquireGameSchemaAdvisoryXactLock,
type GamePrismaClient,
} from '@sammo-ts/infra';
export type ClockSuspensionSource = 'MAINTENANCE' | 'OPEN_DELAY' | 'UNIFICATION_WAIT' | 'RECOVERY';
export type ClockOperationAuthority =
| { kind: 'DAEMON'; profileName: string; ownerId: string; fencingEpoch: bigint }
| { kind: 'OFFLINE'; profileName: string; reason: string };
export interface ClockSuspensionResult {
suspensionId: string;
phase: 'SUSPENDED';
sourceRevision: number;
targetRevision: number;
cutTick: number;
cutWallAt: Date;
}
export interface ClockReconciliationResult {
suspensionId: string;
phase: 'RECONCILING';
sourceRevision: number;
targetRevision: number;
deadlineGeneration: number;
gapTicks: number;
catchUpTicks: number;
shiftTicks: number;
alignedTick: number;
resumeWallAt: Date;
}
interface DbWallRow {
wallNow: Date;
}
interface LeaseFenceRow {
ownerId: string;
fencingEpoch: bigint;
valid: boolean;
}
interface IdRow {
id: number;
}
interface TextIdRow {
id: string;
}
interface ParticipantSnapshot {
key: string;
policy: 'SHIFT' | 'KEEP' | 'REBUILD';
checksum: string;
count: number;
}
const asJson = (value: unknown): GamePrisma.InputJsonValue => value as GamePrisma.InputJsonValue;
const safeNumber = (value: bigint, label: string): number => {
const result = Number(value);
if (!Number.isSafeInteger(result)) {
throw new Error(`${label} is outside the JavaScript safe integer range: ${value}`);
}
return result;
};
const canonicalize = (value: unknown): unknown => {
if (typeof value === 'bigint') return value.toString();
if (Array.isArray(value)) return value.map(canonicalize);
if (value && typeof value === 'object') {
return Object.fromEntries(
Object.entries(value as Record<string, unknown>)
.sort(([left], [right]) => left.localeCompare(right))
.map(([key, item]) => [key, canonicalize(item)])
);
}
return value;
};
const stableJson = (value: unknown): string => JSON.stringify(canonicalize(value));
const checksum = (value: unknown): string => createHash('sha256').update(stableJson(value)).digest('hex');
const aggregateChecksum = (participants: readonly ParticipantSnapshot[]): string =>
checksum(participants.map(({ key, policy, checksum: value, count }) => ({ key, policy, checksum: value, count })));
const readDbWall = async (db: GamePrisma.TransactionClient): Promise<Date> => {
const rows = await db.$queryRaw<DbWallRow[]>(GamePrisma.sql`
SELECT (CURRENT_TIMESTAMP AT TIME ZONE 'UTC')::timestamp(3) AS "wallNow"
`);
const wallNow = rows[0]?.wallNow;
if (!wallNow || Number.isNaN(wallNow.getTime())) {
throw new Error('Failed to read the PostgreSQL wall clock.');
}
return wallNow;
};
const verifyAuthority = async (
db: GamePrisma.TransactionClient,
authority: ClockOperationAuthority
): Promise<void> => {
const rows = await db.$queryRaw<LeaseFenceRow[]>(GamePrisma.sql`
SELECT owner_id AS "ownerId",
fencing_epoch AS "fencingEpoch",
lease_until > (CURRENT_TIMESTAMP AT TIME ZONE 'UTC') AS valid
FROM turn_daemon_lease
WHERE profile = ${authority.profileName}
FOR UPDATE
`);
const lease = rows[0];
if (authority.kind === 'OFFLINE') {
if (!authority.reason.trim()) {
throw new Error('Offline clock operations require an audit reason.');
}
if (lease?.valid) {
throw new Error(`Clock operation requires the ${authority.profileName} daemon lease to be offline.`);
}
return;
}
if (
!lease?.valid ||
lease.ownerId !== authority.ownerId ||
lease.fencingEpoch !== authority.fencingEpoch
) {
throw new Error(`Stale turn-daemon fencing authority for profile ${authority.profileName}.`);
}
};
const lockWorld = async (db: GamePrisma.TransactionClient): Promise<number> => {
const rows = await db.$queryRaw<IdRow[]>(GamePrisma.sql`
SELECT id FROM world_state ORDER BY id LIMIT 2 FOR UPDATE
`);
if (rows.length !== 1) {
throw new Error(`Clock reconciliation requires exactly one world_state row; found ${rows.length}.`);
}
return rows[0]!.id;
};
const lockParticipants = async (db: GamePrisma.TransactionClient, cutTick: bigint): Promise<void> => {
await db.$queryRaw<IdRow[]>(GamePrisma.sql`SELECT id FROM general ORDER BY id FOR UPDATE`);
await db.$queryRaw<IdRow[]>(GamePrisma.sql`
SELECT id FROM auction
WHERE status IN ('OPEN'::auction_status, 'FINALIZING'::auction_status)
ORDER BY id FOR UPDATE
`);
await db.$queryRaw<IdRow[]>(GamePrisma.sql`
SELECT id FROM message
WHERE valid_until_tick IS NOT NULL AND valid_until_tick >= ${cutTick}
ORDER BY id FOR UPDATE
`);
await db.$queryRaw<IdRow[]>(GamePrisma.sql`
SELECT id FROM vote_poll WHERE closed_at IS NULL ORDER BY id FOR UPDATE
`);
await db.$queryRaw<IdRow[]>(GamePrisma.sql`
SELECT id FROM select_pool WHERE general_id IS NULL ORDER BY id FOR UPDATE
`);
await db.$queryRaw<TextIdRow[]>(GamePrisma.sql`
SELECT owner_user_id AS id FROM select_npc_token ORDER BY owner_user_id FOR UPDATE
`);
};
const readParticipantSnapshots = async (
db: GamePrisma.TransactionClient,
worldStateId: number,
cutTick: bigint
): Promise<ParticipantSnapshot[]> => {
const [world, generals, auctions, messages, votes, pool, npcTokens, commands] = await Promise.all([
db.worldState.findUniqueOrThrow({
where: { id: worldStateId },
select: {
clockTick: true,
clockRevision: true,
deadlineGeneration: true,
lastTurnTick: true,
meta: true,
},
}),
db.general.findMany({
orderBy: { id: 'asc' },
select: { id: true, turnTick: true, recentWarTick: true },
}),
db.auction.findMany({
where: { status: { in: ['OPEN', 'FINALIZING'] } },
orderBy: { id: 'asc' },
select: { id: true, status: true, openTick: true, closeTick: true },
}),
db.message.findMany({
where: { validUntilTick: { not: null, gte: cutTick } },
orderBy: { id: 'asc' },
select: { id: true, timeTick: true, validUntilTick: true },
}),
db.votePoll.findMany({
where: { closedAt: null },
orderBy: { id: 'asc' },
select: { id: true, startTick: true, endTick: true },
}),
db.selectPoolEntry.findMany({
where: { generalId: null },
orderBy: { id: 'asc' },
select: { id: true, reservedUntilTick: true },
}),
db.npcSelectionToken.findMany({
orderBy: { ownerUserId: 'asc' },
select: { ownerUserId: true, validUntilTick: true, pickMoreFromTick: true },
}),
db.inputEvent.findMany({
where: { status: { in: ['PENDING', 'PROCESSING'] } },
orderBy: { sequence: 'asc' },
select: { sequence: true, acceptedGameTick: true, acceptedClockRevision: true },
}),
]);
const snapshot = (key: string, policy: ParticipantSnapshot['policy'], rows: unknown[]): ParticipantSnapshot => ({
key,
policy,
checksum: checksum(rows),
count: rows.length,
});
const meta = world.meta && typeof world.meta === 'object' && !Array.isArray(world.meta) ? world.meta : {};
return [
snapshot('world-clock', 'REBUILD', [
{
clockTick: world.clockTick,
clockRevision: world.clockRevision,
deadlineGeneration: world.deadlineGeneration,
},
]),
snapshot('turn-cursor', 'SHIFT', [{ lastTurnTick: world.lastTurnTick }]),
snapshot(
'general-next-turn',
'SHIFT',
generals.map(({ id, turnTick }) => ({ id, turnTick }))
),
snapshot(
'general-recent-war-occurrence',
'KEEP',
generals.map(({ id, recentWarTick }) => ({ id, recentWarTick }))
),
snapshot(
'auction-open-occurrence',
'KEEP',
auctions.map(({ id, openTick }) => ({ id, openTick }))
),
snapshot(
'auction-deadline',
'SHIFT',
auctions.map(({ id, status, closeTick }) => ({ id, status, closeTick }))
),
snapshot(
'auction-finalizing-recovery',
'REBUILD',
auctions.map(({ id, status }) => ({ id, status }))
),
snapshot(
'message-occurrence',
'KEEP',
messages.map(({ id, timeTick }) => ({ id, timeTick }))
),
snapshot(
'message-expiry',
'SHIFT',
messages.map(({ id, validUntilTick }) => ({ id, validUntilTick }))
),
snapshot(
'vote-start-occurrence',
'KEEP',
votes.map(({ id, startTick }) => ({ id, startTick }))
),
snapshot(
'vote-end-deadline',
'SHIFT',
votes.map(({ id, endTick }) => ({ id, endTick }))
),
snapshot('select-pool-reservation', 'SHIFT', pool),
snapshot('npc-selection-window', 'SHIFT', npcTokens),
snapshot('accepted-command-coordinate', 'KEEP', commands),
snapshot('movable-json-rule-anchors', 'SHIFT', [
{
lastTurnTime: Reflect.get(meta, 'lastTurnTime'),
turntime: Reflect.get(meta, 'turntime'),
starttime: Reflect.get(meta, 'starttime'),
tnmt_time: Reflect.get(meta, 'tnmt_time'),
},
]),
];
};
const persistInitialParticipants = async (
db: GamePrisma.TransactionClient,
suspensionId: string,
participants: readonly ParticipantSnapshot[]
): Promise<void> => {
for (const participant of participants) {
await db.clockReconciliationParticipant.create({
data: {
suspensionId,
participantKey: participant.key,
policy: participant.policy,
beforeChecksum: participant.checksum,
afterChecksum: participant.checksum,
affectedCount: 0,
},
});
}
};
const shiftMetaDate = (value: unknown, deltaMilliseconds: number): unknown => {
if (typeof value !== 'string' || !value.trim()) return value;
const parsed = new Date(value);
if (Number.isNaN(parsed.getTime())) return value;
return new Date(parsed.getTime() + deltaMilliseconds).toISOString();
};
const shiftedMeta = (value: GamePrisma.JsonValue, deltaMilliseconds: number): GamePrisma.InputJsonValue => {
const meta = value && typeof value === 'object' && !Array.isArray(value) ? { ...value } : {};
for (const key of ['lastTurnTime', 'turntime', 'starttime', 'tnmt_time'] as const) {
if (Object.hasOwn(meta, key)) {
Reflect.set(meta, key, shiftMetaDate(Reflect.get(meta, key), deltaMilliseconds));
}
}
return asJson(meta);
};
const assertShiftFits = (participants: readonly ParticipantSnapshot[], shiftTicks: number): void => {
if (!Number.isSafeInteger(shiftTicks) || shiftTicks < 0) {
throw new Error(`Invalid reconciliation shift: ${shiftTicks}`);
}
// Checksums retain stringified values for audit; actual row ranges are
// checked by PostgreSQL BIGINT and the world aligned tick is checked by the
// shared GameClock plan. The sentinel expiry is deliberately never shifted.
if (participants.some((participant) => !participant.checksum)) {
throw new Error('Participant snapshot is incomplete.');
}
};
const assertScheduleRanges = async (db: GamePrisma.TransactionClient, shiftTicks: number): Promise<void> => {
const shift = BigInt(shiftTicks);
const maximum = BigInt(MAX_SAFE_GAME_TICK) - shift;
const [general, auction, message, vote, pool, npcValid, npcMore] = await Promise.all([
db.general.aggregate({ _max: { turnTick: true }, where: { turnTick: { not: null } } }),
db.auction.aggregate({
_max: { closeTick: true },
where: { status: { in: ['OPEN', 'FINALIZING'] }, closeTick: { not: null } },
}),
db.message.aggregate({
_max: { validUntilTick: true },
where: { validUntilTick: { not: null, lt: BigInt(MAX_SAFE_GAME_TICK) } },
}),
db.votePoll.aggregate({ _max: { endTick: true }, where: { closedAt: null, endTick: { not: null } } }),
db.selectPoolEntry.aggregate({
_max: { reservedUntilTick: true },
where: { generalId: null, reservedUntilTick: { not: null } },
}),
db.npcSelectionToken.aggregate({ _max: { validUntilTick: true }, where: { validUntilTick: { not: null } } }),
db.npcSelectionToken.aggregate({
_max: { pickMoreFromTick: true },
where: { pickMoreFromTick: { not: null } },
}),
]);
const values: Array<[string, bigint | null]> = [
['general.turn_tick', general._max.turnTick],
['auction.close_tick', auction._max.closeTick],
['message.valid_until_tick', message._max.validUntilTick],
['vote_poll.end_tick', vote._max.endTick],
['select_pool.reserved_until_tick', pool._max.reservedUntilTick],
['select_npc_token.valid_until_tick', npcValid._max.validUntilTick],
['select_npc_token.pick_more_from_tick', npcMore._max.pickMoreFromTick],
];
for (const [label, value] of values) {
if (value !== null && value > maximum) {
throw new Error(`${label} would exceed the safe game tick range after reconciliation.`);
}
}
};
const applyParticipantShift = async (
db: GamePrisma.TransactionClient,
worldStateId: number,
cutTick: bigint,
alignedTick: bigint,
targetRevision: bigint,
targetGeneration: bigint,
shiftTicks: bigint,
projectionDeltaMilliseconds: number,
resumeWallAt: Date
): Promise<Map<string, number>> => {
const affected = new Map<string, number>();
const world = await db.worldState.findUniqueOrThrow({ where: { id: worldStateId }, select: { meta: true } });
const cursor = await db.worldState.updateMany({
where: { id: worldStateId, lastTurnTick: { not: null } },
data: { lastTurnTick: { increment: shiftTicks } },
});
affected.set('turn-cursor', cursor.count);
affected.set(
'general-next-turn',
await db.$executeRaw(GamePrisma.sql`
UPDATE general
SET turn_tick = turn_tick + ${shiftTicks},
turn_time = turn_time + ${projectionDeltaMilliseconds} * INTERVAL '1 millisecond'
WHERE turn_tick IS NOT NULL
`)
);
affected.set(
'auction-deadline',
await db.$executeRaw(GamePrisma.sql`
UPDATE auction
SET close_tick = close_tick + ${shiftTicks},
close_at = close_at + ${projectionDeltaMilliseconds} * INTERVAL '1 millisecond'
WHERE status IN ('OPEN'::auction_status, 'FINALIZING'::auction_status)
AND close_tick IS NOT NULL
`)
);
affected.set(
'message-expiry',
await db.$executeRaw(GamePrisma.sql`
UPDATE message
SET valid_until_tick = valid_until_tick + ${shiftTicks},
valid_until = valid_until + ${projectionDeltaMilliseconds} * INTERVAL '1 millisecond'
WHERE valid_until_tick IS NOT NULL
AND valid_until_tick >= ${cutTick}
AND valid_until_tick < ${BigInt(MAX_SAFE_GAME_TICK)}
`)
);
affected.set(
'vote-end-deadline',
await db.$executeRaw(GamePrisma.sql`
UPDATE vote_poll
SET end_tick = end_tick + ${shiftTicks},
end_at = end_at + ${projectionDeltaMilliseconds} * INTERVAL '1 millisecond'
WHERE closed_at IS NULL AND end_tick IS NOT NULL
`)
);
affected.set(
'select-pool-reservation',
await db.$executeRaw(GamePrisma.sql`
UPDATE select_pool
SET reserved_until_tick = reserved_until_tick + ${shiftTicks},
reserved_until = reserved_until + ${projectionDeltaMilliseconds} * INTERVAL '1 millisecond'
WHERE general_id IS NULL AND reserved_until_tick IS NOT NULL
`)
);
affected.set(
'npc-selection-window',
await db.$executeRaw(GamePrisma.sql`
UPDATE select_npc_token
SET valid_until_tick = CASE
WHEN valid_until_tick IS NULL THEN NULL ELSE valid_until_tick + ${shiftTicks} END,
valid_until = CASE
WHEN valid_until_tick IS NULL THEN valid_until
ELSE valid_until + ${projectionDeltaMilliseconds} * INTERVAL '1 millisecond' END,
pick_more_from_tick = CASE
WHEN pick_more_from_tick IS NULL THEN NULL ELSE pick_more_from_tick + ${shiftTicks} END,
pick_more_from = CASE
WHEN pick_more_from_tick IS NULL THEN pick_more_from
ELSE pick_more_from + ${projectionDeltaMilliseconds} * INTERVAL '1 millisecond' END
WHERE valid_until_tick IS NOT NULL OR pick_more_from_tick IS NOT NULL
`)
);
await db.worldState.update({
where: { id: worldStateId },
data: {
clockTick: alignedTick,
clockWallAnchor: resumeWallAt,
clockPhase: 'RECONCILING',
clockRevision: targetRevision,
deadlineGeneration: targetGeneration,
meta: shiftedMeta(world.meta, projectionDeltaMilliseconds),
},
});
affected.set('world-clock', 1);
affected.set('movable-json-rule-anchors', 1);
affected.set('auction-finalizing-recovery', 0);
return affected;
};
export const startClockSuspension = async (options: {
db: GamePrismaClient;
suspensionId: string;
source: ClockSuspensionSource;
authority: ClockOperationAuthority;
policy?: ClockAlignmentPolicy;
catchUpTicks?: number;
}): Promise<ClockSuspensionResult> => {
if (!options.suspensionId.trim() || options.suspensionId.length > 64) {
throw new Error('Clock suspension ID must contain 1-64 characters.');
}
const policy = options.policy ?? 'EXACT';
const catchUpTicks = options.catchUpTicks ?? 0;
if (!Number.isSafeInteger(catchUpTicks) || catchUpTicks < 0) {
throw new Error('Clock suspension catch-up ticks must be a non-negative safe integer.');
}
return options.db.$transaction(
async (db) => {
await verifyAuthority(db, options.authority);
await acquireGameSchemaAdvisoryXactLock(db, CLOCK_OPERATION_PERSISTENCE_LOCK);
await acquireGameSchemaAdvisoryXactLock(db, GENERAL_ACCESS_PERSISTENCE_LOCK);
const worldStateId = await lockWorld(db);
const existing = await db.clockSuspension.findUnique({ where: { id: options.suspensionId } });
if (existing) {
if (existing.worldStateId !== worldStateId || existing.source !== options.source || existing.policy !== policy) {
throw new Error(`Clock suspension ID ${options.suspensionId} is already bound to another operation.`);
}
if (existing.status !== 'SUSPENDED') {
throw new Error(`Clock suspension ${options.suspensionId} already advanced to ${existing.status}.`);
}
return {
suspensionId: existing.id,
phase: 'SUSPENDED' as const,
sourceRevision: safeNumber(existing.sourceRevision, 'source revision'),
targetRevision: safeNumber(existing.targetRevision, 'target revision'),
cutTick: safeNumber(existing.cutTick, 'cut tick'),
cutWallAt: existing.cutWallAt,
};
}
const world = await db.worldState.findUniqueOrThrow({ where: { id: worldStateId } });
const phase = parseGameClockPhase(world.clockPhase);
if (phase !== 'RUNNING') {
throw new Error(`Clock suspension can start only from RUNNING; current phase is ${phase}.`);
}
if (!world.clockBaseTime || world.clockTick === null || !world.clockWallAnchor) {
throw new Error('Clock suspension requires a fully initialized logical game clock.');
}
const cutWallAt = await readDbWall(db);
const storedTick = safeNumber(world.clockTick, 'world clock tick');
const sourceRevision = safeNumber(world.clockRevision, 'world clock revision');
const clock = new GameClock({
baseTime: world.clockBaseTime,
tick: storedTick,
mode: world.clockMode === 'manual' ? 'manual' : 'realtime',
wallAnchor: world.clockWallAnchor,
turnSeconds: world.tickSeconds,
phase,
revision: sourceRevision,
});
const cutTick = clock.nowTick(cutWallAt);
await lockParticipants(db, BigInt(cutTick));
await db.worldState.update({
where: { id: worldStateId },
data: { clockPhase: 'SUSPENDED', clockTick: BigInt(cutTick), clockWallAnchor: cutWallAt },
});
const participants = await readParticipantSnapshots(db, worldStateId, BigInt(cutTick));
await db.clockSuspension.create({
data: {
id: options.suspensionId,
worldStateId,
source: options.source,
policy,
status: 'SUSPENDED',
sourceRevision: BigInt(sourceRevision),
targetRevision: BigInt(sourceRevision + 1),
cutTick: BigInt(cutTick),
cutWallAt,
rateTicksPerSecond: GAME_TICKS_PER_TURN / world.tickSeconds,
catchUpTicks: BigInt(catchUpTicks),
participantChecksumBefore: aggregateChecksum(participants),
detail: asJson({ authority: options.authority.kind, profileName: options.authority.profileName }),
},
});
await persistInitialParticipants(db, options.suspensionId, participants);
return {
suspensionId: options.suspensionId,
phase: 'SUSPENDED',
sourceRevision,
targetRevision: sourceRevision + 1,
cutTick,
cutWallAt,
};
},
{ isolationLevel: 'Serializable', maxWait: 10_000, timeout: 30_000 }
);
};
export const reconcileClockSuspension = async (options: {
db: GamePrismaClient;
suspensionId: string;
authority: ClockOperationAuthority;
/** Deterministic fixture seam; production must always use PostgreSQL CURRENT_TIMESTAMP. */
testResumeWallAt?: Date;
}): Promise<ClockReconciliationResult> =>
options.db.$transaction(
async (db) => {
await verifyAuthority(db, options.authority);
await acquireGameSchemaAdvisoryXactLock(db, CLOCK_OPERATION_PERSISTENCE_LOCK);
await acquireGameSchemaAdvisoryXactLock(db, GENERAL_ACCESS_PERSISTENCE_LOCK);
const worldStateId = await lockWorld(db);
const suspension = await db.clockSuspension.findUniqueOrThrow({ where: { id: options.suspensionId } });
if (suspension.worldStateId !== worldStateId) {
throw new Error('Clock suspension belongs to another world state.');
}
if (suspension.status === 'RECONCILING' || suspension.status === 'APPLIED') {
if (
suspension.gapTicks === null ||
suspension.shiftTicks === null ||
suspension.alignedTick === null ||
!suspension.resumeWallAt
) {
throw new Error('Persisted clock reconciliation result is incomplete.');
}
const world = await db.worldState.findUniqueOrThrow({ where: { id: worldStateId } });
return {
suspensionId: suspension.id,
phase: 'RECONCILING' as const,
sourceRevision: safeNumber(suspension.sourceRevision, 'source revision'),
targetRevision: safeNumber(suspension.targetRevision, 'target revision'),
deadlineGeneration: safeNumber(world.deadlineGeneration, 'deadline generation'),
gapTicks: safeNumber(suspension.gapTicks, 'gap ticks'),
catchUpTicks: safeNumber(suspension.catchUpTicks, 'catch-up ticks'),
shiftTicks: safeNumber(suspension.shiftTicks, 'shift ticks'),
alignedTick: safeNumber(suspension.alignedTick, 'aligned tick'),
resumeWallAt: suspension.resumeWallAt,
};
}
if (suspension.status !== 'SUSPENDED') {
throw new Error(`Clock suspension cannot reconcile from status ${suspension.status}.`);
}
const world = await db.worldState.findUniqueOrThrow({ where: { id: worldStateId } });
const phase = parseGameClockPhase(world.clockPhase);
if (phase !== 'SUSPENDED' || world.clockRevision !== suspension.sourceRevision) {
throw new Error('Clock reconciliation phase or source revision fence failed.');
}
const worldMeta =
world.meta && typeof world.meta === 'object' && !Array.isArray(world.meta)
? (world.meta as Record<string, unknown>)
: {};
const united = Number(worldMeta.isunited ?? worldMeta.isUnited ?? 0);
if (suspension.source === 'UNIFICATION_WAIT' || united >= 2) {
throw new Error(
'Unification wait requires the atomic alignment-and-invader workflow; generic resume is forbidden.'
);
}
const cutTick = safeNumber(suspension.cutTick, 'cut tick');
await lockParticipants(db, suspension.cutTick);
if (options.testResumeWallAt && process.env.NODE_ENV !== 'test') {
throw new Error('A clock reconciliation wall override is allowed only in tests.');
}
const resumeWallAt = options.testResumeWallAt
? new Date(options.testResumeWallAt.getTime())
: await readDbWall(db);
const plan = buildClockAlignmentPlan({
policy: parseClockAlignmentPolicy(suspension.policy),
sourceRevision: safeNumber(suspension.sourceRevision, 'source revision'),
cutTick,
cutWall: suspension.cutWallAt,
resumeWall: resumeWallAt,
ticksPerSecond: suspension.rateTicksPerSecond,
catchUpTicks: safeNumber(suspension.catchUpTicks, 'catch-up ticks'),
});
const before = await readParticipantSnapshots(db, worldStateId, suspension.cutTick);
assertShiftFits(before, plan.shiftTicks);
await assertScheduleRanges(db, plan.shiftTicks);
const projectionDeltaMilliseconds = Math.trunc(
(plan.shiftTicks * 1_000) / suspension.rateTicksPerSecond
);
if (!Number.isSafeInteger(projectionDeltaMilliseconds)) {
throw new Error('Clock reconciliation projection delta is outside the safe integer range.');
}
const targetGeneration = world.deadlineGeneration + 1n;
const affected = await applyParticipantShift(
db,
worldStateId,
suspension.cutTick,
BigInt(plan.alignedTick),
BigInt(plan.targetRevision),
targetGeneration,
BigInt(plan.shiftTicks),
projectionDeltaMilliseconds,
resumeWallAt
);
const after = await readParticipantSnapshots(db, worldStateId, suspension.cutTick);
const afterByKey = new Map(after.map((participant) => [participant.key, participant]));
for (const participant of before) {
const next = afterByKey.get(participant.key);
if (!next) throw new Error(`Missing post-reconciliation participant: ${participant.key}`);
if (participant.policy === 'KEEP' && participant.checksum !== next.checksum) {
throw new Error(`KEEP participant changed during reconciliation: ${participant.key}`);
}
await db.clockReconciliationParticipant.upsert({
where: {
suspensionId_participantKey: {
suspensionId: suspension.id,
participantKey: participant.key,
},
},
create: {
suspensionId: suspension.id,
participantKey: participant.key,
policy: participant.policy,
beforeChecksum: participant.checksum,
afterChecksum: next.checksum,
affectedCount: affected.get(participant.key) ?? 0,
},
update: {
policy: participant.policy,
beforeChecksum: participant.checksum,
afterChecksum: next.checksum,
affectedCount: affected.get(participant.key) ?? 0,
},
});
}
const outboxPayload = {
version: 1,
profileName: options.authority.profileName,
suspensionId: suspension.id,
sourceRevision: plan.sourceRevision,
targetRevision: plan.targetRevision,
deadlineGeneration: safeNumber(targetGeneration, 'deadline generation'),
shiftTicks: plan.shiftTicks,
projectionDeltaMilliseconds,
clockBaseTime: world.clockBaseTime!.toISOString(),
ticksPerSecond: suspension.rateTicksPerSecond,
};
await db.clockProjectionOutbox.create({
data: {
worldStateId,
suspensionId: suspension.id,
targetRevision: BigInt(plan.targetRevision),
status: 'PENDING',
payload: asJson(outboxPayload),
checksum: checksum(outboxPayload),
},
});
await db.clockSuspension.update({
where: { id: suspension.id },
data: {
status: 'RECONCILING',
resumeWallAt,
gapTicks: BigInt(plan.gapTicks),
shiftTicks: BigInt(plan.shiftTicks),
alignedTick: BigInt(plan.alignedTick),
participantChecksumBefore: aggregateChecksum(before),
participantChecksumAfter: aggregateChecksum(after),
},
});
return {
suspensionId: suspension.id,
phase: 'RECONCILING',
sourceRevision: plan.sourceRevision,
targetRevision: plan.targetRevision,
deadlineGeneration: safeNumber(targetGeneration, 'deadline generation'),
gapTicks: plan.gapTicks,
catchUpTicks: plan.catchUpTicks,
shiftTicks: plan.shiftTicks,
alignedTick: plan.alignedTick,
resumeWallAt,
};
},
{ isolationLevel: 'Serializable', maxWait: 10_000, timeout: 30_000 }
);
+28
View File
@@ -1218,6 +1218,34 @@ export const createDatabaseTurnHooks = async (
`found ${durableClock.clock_phase}@${durableClock.clock_revision}/${durableClock.deadline_generation}.`
);
}
if (commandCompletion) {
const commandFence = await prisma.$queryRaw<
Array<{
status: string;
processing_clock_revision: bigint | null;
processing_deadline_generation: bigint | null;
}>
>(GamePrisma.sql`
SELECT status,
processing_clock_revision,
processing_deadline_generation
FROM input_event
WHERE request_id = ${commandCompletion.requestId}
AND target = 'ENGINE'::"InputEventTarget"
FOR UPDATE
`);
const event = commandFence[0];
if (
!event ||
event.status !== 'PROCESSING' ||
event.processing_clock_revision !== expectedRevision ||
event.processing_deadline_generation !== expectedGeneration
) {
throw new Error(
`Input event processing clock fence changed before commit: ${commandCompletion.requestId}.`
);
}
}
let neutralAuctionsToCreate = pendingNeutralAuctions;
if (pendingNeutralAuctions.length > 0) {
const latestRegistrationKey =
+15 -8
View File
@@ -1,4 +1,4 @@
import type { GamePrisma, GamePrismaClient } from '@sammo-ts/infra';
import { readInputEventClockCoordinate, type GamePrisma, type GamePrismaClient } from '@sammo-ts/infra';
import { randomUUID } from 'node:crypto';
import {
@@ -137,13 +137,20 @@ const ensureEngineCommand = async (
deltaMinutes,
};
try {
await db.inputEvent.create({
data: {
requestId,
target: 'ENGINE',
eventType: command.type,
payload: asJson(command),
},
await db.$transaction(async (transaction) => {
const coordinate = await readInputEventClockCoordinate(transaction);
await transaction.inputEvent.create({
data: {
requestId,
target: 'ENGINE',
eventType: command.type,
payload: asJson(command),
acceptedGameTick: coordinate.gameTick,
acceptedClockRevision: coordinate.clockRevision,
acceptedDeadlineGeneration: coordinate.deadlineGeneration,
createdAt: coordinate.wallAt,
},
});
});
} catch (error) {
if (!isUniqueConflict(error)) {
@@ -10,7 +10,7 @@ import {
type TurnDaemonCommand,
type TurnDaemonCommandResult,
} from '@sammo-ts/common';
import type { GamePrisma, GamePrismaClient } from '@sammo-ts/infra';
import { readInputEventClockCoordinate, type GamePrisma, type GamePrismaClient } from '@sammo-ts/infra';
import type { GatewayAdminActionRecord, GatewayAdminActionResult } from './gatewayAdminActions.js';
@@ -97,13 +97,20 @@ const ensureEngineCommand = async (
settings,
};
try {
await db.inputEvent.create({
data: {
requestId,
target: 'ENGINE',
eventType: command.type,
payload: asJson(command),
},
await db.$transaction(async (transaction) => {
const coordinate = await readInputEventClockCoordinate(transaction);
await transaction.inputEvent.create({
data: {
requestId,
target: 'ENGINE',
eventType: command.type,
payload: asJson(command),
acceptedGameTick: coordinate.gameTick,
acceptedClockRevision: coordinate.clockRevision,
acceptedDeadlineGeneration: coordinate.deadlineGeneration,
createdAt: coordinate.wallAt,
},
});
});
} catch (error) {
if (!isUniqueConflict(error)) throw error;
@@ -439,15 +439,20 @@ export const reserveSelectionPool = async (options: {
userId: string;
now?: Date;
acceptedGameTick?: number;
processingGameTick?: number;
seedOwnerIdentity?: string | number;
}): Promise<SelectPoolReservationDto> => {
const { db, world, worldState, userId } = options;
requirePoolWorld(worldState);
const now = options.now ?? new Date();
const acceptedGameTick = options.acceptedGameTick ?? resolveAcceptedGameTick(world, now);
const processingGameTick = options.processingGameTick ?? acceptedGameTick;
if (!Number.isSafeInteger(acceptedGameTick)) {
fail('INTERNAL_SERVER_ERROR', '장수 선택 예약 tick이 안전한 정수 범위를 벗어났습니다.');
}
if (!Number.isSafeInteger(processingGameTick)) {
fail('INTERNAL_SERVER_ERROR', '장수 선택 처리 tick이 안전한 정수 범위를 벗어났습니다.');
}
await lockSelectionUser(db, userId);
await lockSelectionMutationTables(db);
const general = await db.general.findFirst({
@@ -461,7 +466,7 @@ export const reserveSelectionPool = async (options: {
let currentRows = await synchronizeSelectionPoolWorld(db, world);
const existing = currentRows.filter(
(row) => row.ownerUserId === userId && row.generalId === null && isReservationActive(row, now, acceptedGameTick)
(row) => row.ownerUserId === userId && row.generalId === null && isReservationActive(row, now, processingGameTick)
);
if (existing.length > 0) {
return toReservationDto(existing, Boolean(general), worldState, world);
@@ -471,7 +476,7 @@ export const reserveSelectionPool = async (options: {
where: {
generalId: null,
OR: [
{ reservedUntilTick: { lt: BigInt(acceptedGameTick) } },
{ reservedUntilTick: { lt: BigInt(processingGameTick) } },
{ reservedUntilTick: null, reservedUntil: { lt: now } },
],
},
@@ -483,7 +488,7 @@ export const reserveSelectionPool = async (options: {
});
currentRows = await synchronizeSelectionPoolWorld(db, world);
const availableIds = new Set(
world.listGeneralPoolCandidates(now, acceptedGameTick)?.map((candidate) => candidate.poolEntryId) ?? []
world.listGeneralPoolCandidates(now, processingGameTick)?.map((candidate) => candidate.poolEntryId) ?? []
);
const available = currentRows.filter(
(row) =>
@@ -507,7 +512,7 @@ export const reserveSelectionPool = async (options: {
(row) =>
[row, calculateSelectionCandidateWeight(poolName, parseCandidate(row), true)] as [SelectPoolRow, number]
);
const reservedUntilTick = acceptedGameTick + RESERVATION_TURN_MULTIPLIER * GAME_TICKS_PER_TURN;
const reservedUntilTick = processingGameTick + RESERVATION_TURN_MULTIPLIER * GAME_TICKS_PER_TURN;
if (!Number.isSafeInteger(reservedUntilTick)) {
fail('INTERNAL_SERVER_ERROR', '장수 선택 예약 tick이 안전한 정수 범위를 벗어났습니다.');
}
@@ -14,8 +14,12 @@ interface TournamentState {
openMonth: number;
termSeconds: number;
nextAt: string;
nextTick?: number;
clockRevision?: number;
deadlineGeneration?: number;
bettingId?: number;
bettingCloseAt?: string;
bettingCloseTick?: number;
winnerId?: number;
bettingSettled?: boolean;
rewardSettled?: boolean;
@@ -76,6 +80,9 @@ export const createTournamentAutoStartHandler = (options: {
sourceRevisionKey: `sammo:${options.profileName}:tournament:source-revision`,
sourceRevisionChannel: `sammo:${options.profileName}:tournament:source-changed`,
realtimeEventChannel: buildGameEventChannel(options.profileName),
activeClockRevisionKey: `sammo:${options.profileName}:clock:active-revision`,
deadlineGenerationKey: `sammo:${options.profileName}:clock:deadline-generation`,
clockPhaseKey: `sammo:${options.profileName}:clock:phase`,
};
return {
onMonthChanged: async (context) => {
@@ -118,6 +125,8 @@ export const createTournamentAutoStartHandler = (options: {
previousState && Number.isFinite(previousState.termSeconds) && previousState.termSeconds > 0
? previousState.termSeconds
: resolveTermSeconds(state.tickSeconds);
const nextAt = new Date(now.getTime() + termSeconds * 60_000);
const clockState = world.getGameClockState();
const nextState: TournamentState = {
stage: 1,
phase: 0,
@@ -129,7 +138,10 @@ export const createTournamentAutoStartHandler = (options: {
// Ref startTournament() passes calcTournamentTerm()'s seconds
// value to DateInterval's minute field. Preserve that historical
// initial enrollment delay; later tournament phases use seconds.
nextAt: new Date(now.getTime() + termSeconds * 60_000).toISOString(),
nextAt: nextAt.toISOString(),
nextTick: world.dateToGameTick(nextAt),
clockRevision: clockState.revision,
deadlineGeneration: clockState.deadlineGeneration,
bettingId:
typeof previousState?.bettingId === 'number' && Number.isFinite(previousState.bettingId)
? previousState.bettingId + 1
@@ -142,12 +154,26 @@ export const createTournamentAutoStartHandler = (options: {
lastError: undefined,
lastErrorAt: undefined,
};
await writeTournamentProjection(redis, keys, [
{ key: keys.participantsKey, value: [] },
{ key: keys.matchesKey, value: [] },
{ key: keys.bettingKey, value: [] },
{ key: keys.stateKey, value: nextState },
]);
await writeTournamentProjection(
redis,
keys,
[
{ key: keys.participantsKey, value: [] },
{ key: keys.matchesKey, value: [] },
{ key: keys.bettingKey, value: [] },
{ key: keys.stateKey, value: nextState },
],
clockState.phase === 'RUNNING'
? {
activeRevisionKey: keys.activeClockRevisionKey,
deadlineGenerationKey: keys.deadlineGenerationKey,
phaseKey: keys.clockPhaseKey,
revision: clockState.revision,
deadlineGeneration: clockState.deadlineGeneration,
phase: 'RUNNING',
}
: undefined
);
const [typeText, generalTypeText] = TOURNAMENT_TEXT[type] ?? TOURNAMENT_TEXT[0];
const emperor = world
@@ -274,6 +274,10 @@ const resolveSelectionCommandAcceptedAt = async (
command: Extract<TurnDaemonCommand, { type: 'selectPoolReserve' | 'selectPoolCreate' | 'selectPoolReselect' }>
): Promise<Date> => {
const operationalAcceptedAt = await resolveCommandAcceptedAt(db, command);
const processingGameTick = Reflect.get(command, 'processingGameTick');
if (typeof processingGameTick === 'number' && Number.isSafeInteger(processingGameTick)) {
return world.gameTickToDate(processingGameTick);
}
if (command.acceptedGameTick !== undefined) {
return world.gameTickToDate(command.acceptedGameTick);
}
@@ -407,9 +411,13 @@ async function handleNpcPossessGeneral(
throw new Error('NPC possession world state is missing.');
}
const operationalAcceptedAt = await resolveCommandAcceptedAt(db, command);
const acceptedAt = command.acceptedGameAt
? new Date(command.acceptedGameAt)
: ctx.world.getGameNow(operationalAcceptedAt);
const processingGameTick = Reflect.get(command, 'processingGameTick');
const acceptedAt =
typeof processingGameTick === 'number' && Number.isSafeInteger(processingGameTick)
? ctx.world.gameTickToDate(processingGameTick)
: command.acceptedGameAt
? new Date(command.acceptedGameAt)
: ctx.world.getGameNow(operationalAcceptedAt);
try {
return {
type: 'npcPossessGeneral',
@@ -451,8 +459,11 @@ async function handleSelectPoolCreate(
throw new Error('Selection-pool world state is missing.');
}
const operationalAcceptedAt = await resolveCommandAcceptedAt(db, command);
const processingGameTick = Reflect.get(command, 'processingGameTick');
const acceptedAt =
command.acceptedGameTick !== undefined
typeof processingGameTick === 'number' && Number.isSafeInteger(processingGameTick)
? ctx.world.gameTickToDate(processingGameTick)
: command.acceptedGameTick !== undefined
? ctx.world.gameTickToDate(command.acceptedGameTick)
: command.acceptedGameAt !== undefined
? new Date(command.acceptedGameAt)
@@ -515,6 +526,9 @@ async function handleSelectPoolReserve(
seedOwnerIdentity: command.seedOwnerIdentity,
now: acceptedAt,
...(command.acceptedGameTick === undefined ? {} : { acceptedGameTick: command.acceptedGameTick }),
...(typeof Reflect.get(command, 'processingGameTick') === 'number'
? { processingGameTick: Reflect.get(command, 'processingGameTick') as number }
: {}),
}),
};
} catch (error) {
@@ -2770,9 +2784,15 @@ const validateVoteSelectionInTransaction = async (
if (!poll) return '설문조사가 없습니다.';
const processingNow = ctx.world.getGameNow(new Date());
const acceptedGameTick = command.acceptedGameTick ?? ctx.world.dateToGameTick(processingNow);
const convertedProcessingTick = Reflect.get(command, 'processingGameTick');
const acceptedGameTick =
typeof convertedProcessingTick === 'number' && Number.isSafeInteger(convertedProcessingTick)
? convertedProcessingTick
: (command.acceptedGameTick ?? ctx.world.dateToGameTick(processingNow));
const acceptedGameAt =
command.acceptedGameTick === undefined ? processingNow : ctx.world.gameTickToDate(command.acceptedGameTick);
command.acceptedGameTick === undefined && convertedProcessingTick === undefined
? processingNow
: ctx.world.gameTickToDate(acceptedGameTick);
if (hasVotePollDeadlinePassed(poll, acceptedGameAt, acceptedGameTick)) {
return '설문조사가 종료되었습니다.';
}
@@ -0,0 +1,364 @@
import { afterAll, beforeAll, beforeEach, describe, expect, it } from 'vitest';
import { GameClock } from '@sammo-ts/common';
import {
createGamePostgresConnector,
createRedisConnector,
type GamePrismaClient,
type RedisConnector,
} from '@sammo-ts/infra';
import { reconcileClockSuspension, startClockSuspension } from '../src/turn/clockReconciliation.js';
import { applyNextClockProjection } from '../src/turn/clockProjectionOutbox.js';
const enabled =
process.env.CLOCK_RECONCILIATION_INTEGRATION === '1' &&
Boolean(process.env.DATABASE_URL) &&
Boolean(process.env.REDIS_URL);
const describeIntegration = enabled ? describe : describe.skip;
describeIntegration('durable clock reconciliation', () => {
let db: GamePrismaClient;
let disconnect: (() => Promise<void>) | undefined;
let redis: RedisConnector;
beforeAll(async () => {
const connector = createGamePostgresConnector({ url: process.env.DATABASE_URL! });
db = connector.prisma;
disconnect = connector.disconnect;
redis = createRedisConnector({ url: process.env.REDIS_URL! });
await redis.connect();
});
afterAll(async () => {
await redis.disconnect();
await disconnect?.();
});
beforeEach(async () => {
await redis.client.flushDb();
await db.$transaction([
db.clockProjectionOutbox.deleteMany(),
db.clockReconciliationParticipant.deleteMany(),
db.clockSuspension.deleteMany(),
db.inputEvent.deleteMany(),
db.vote.deleteMany(),
db.voteComment.deleteMany(),
db.votePoll.deleteMany(),
db.message.deleteMany(),
db.auctionBid.deleteMany(),
db.auction.deleteMany(),
db.npcSelectionToken.deleteMany(),
db.selectPoolEntry.deleteMany(),
db.general.deleteMany(),
db.turnDaemonLease.deleteMany(),
db.worldState.deleteMany(),
]);
});
it('preserves every remaining deadline and occurrence across a 65m17.250s exact gap', async () => {
const baseTime = new Date('2026-01-01T00:00:00.000Z');
const futureAnchor = new Date(Date.now() + 3_600_000);
const initialTick = 1_000_000;
const lastTurnTick = 900_000;
const clock = new GameClock({
baseTime,
tick: initialTick,
mode: 'realtime',
wallAnchor: futureAnchor,
turnSeconds: 600,
phase: 'RUNNING',
revision: 1,
});
const generalTicks = [initialTick + 1_234, initialTick + 36_000_123];
const auctionCloseTick = initialTick + 72_000_777;
const messageOccurrenceTick = initialTick - 500;
const messageExpiryTick = initialTick + 90_000_999;
const voteStartTick = initialTick - 200;
const voteEndTick = initialTick + 18_000_321;
const poolTick = initialTick + 2_000_111;
const npcValidTick = initialTick + 3_000_222;
const npcMoreTick = initialTick + 1_000_333;
const world = await db.worldState.create({
data: {
scenarioCode: 'clock-test',
currentYear: 180,
currentMonth: 1,
tickSeconds: 600,
clockBaseTime: baseTime,
clockTick: BigInt(initialTick),
clockMode: 'realtime',
clockWallAnchor: futureAnchor,
lastTurnTick: BigInt(lastTurnTick),
clockPhase: 'RUNNING',
clockRevision: 1n,
deadlineGeneration: 7n,
meta: {
lastTurnTime: clock.tickToDate(lastTurnTick).toISOString(),
starttime: clock.tickToDate(initialTick + 100).toISOString(),
},
},
});
await db.general.createMany({
data: generalTicks.map((turnTick, index) => ({
id: index + 1,
name: `general-${index + 1}`,
turnTick: BigInt(turnTick),
turnTime: clock.tickToDate(turnTick),
recentWarTick: BigInt(initialTick - 100 - index),
recentWarTime: clock.tickToDate(initialTick - 100 - index),
})),
});
await db.auction.create({
data: {
type: 'BUY_RICE',
hostGeneralId: 1,
status: 'FINALIZING',
openTick: BigInt(initialTick - 300),
closeTick: BigInt(auctionCloseTick),
closeAt: clock.tickToDate(auctionCloseTick),
},
});
await db.message.create({
data: {
mailbox: 1,
type: 'private',
src: 1,
dest: 2,
time: clock.tickToDate(messageOccurrenceTick),
timeTick: BigInt(messageOccurrenceTick),
validUntil: clock.tickToDate(messageExpiryTick),
validUntilTick: BigInt(messageExpiryTick),
message: {},
},
});
await db.votePoll.create({
data: {
title: 'clock vote',
options: ['yes', 'no'],
revealMode: 'AFTER_VOTE',
openerGeneralId: 1,
openerName: 'general-1',
startAt: clock.tickToDate(voteStartTick),
startTick: BigInt(voteStartTick),
endAt: clock.tickToDate(voteEndTick),
endTick: BigInt(voteEndTick),
},
});
await db.selectPoolEntry.create({
data: {
uniqueName: 'clock-pool',
reservedUntil: clock.tickToDate(poolTick),
reservedUntilTick: BigInt(poolTick),
info: {},
},
});
await db.npcSelectionToken.create({
data: {
ownerUserId: 'clock-user',
validUntil: clock.tickToDate(npcValidTick),
validUntilTick: BigInt(npcValidTick),
pickMoreFrom: clock.tickToDate(npcMoreTick),
pickMoreFromTick: BigInt(npcMoreTick),
pickResult: [],
nonce: 1,
},
});
const authority = { kind: 'OFFLINE' as const, profileName: 'clock-test', reason: 'integration fixture' };
const suspended = await startClockSuspension({
db,
suspensionId: 'clock-gap-65m17s250',
source: 'MAINTENANCE',
authority,
});
expect(suspended.cutTick).toBe(initialTick);
expect((await db.worldState.findUniqueOrThrow({ where: { id: world.id } })).clockPhase).toBe('SUSPENDED');
const resumeWallAt = new Date(suspended.cutWallAt.getTime() + 65 * 60_000 + 17_250);
const reconciled = await reconcileClockSuspension({
db,
suspensionId: suspended.suspensionId,
authority,
testResumeWallAt: resumeWallAt,
});
expect(reconciled).toMatchObject({
phase: 'RECONCILING',
sourceRevision: 1,
targetRevision: 2,
deadlineGeneration: 8,
gapTicks: 235_035_000,
shiftTicks: 235_035_000,
alignedTick: 236_035_000,
});
const [afterWorld, generals, auction, message, vote, pool, token, ledger, outboxes] = await Promise.all([
db.worldState.findUniqueOrThrow({ where: { id: world.id } }),
db.general.findMany({ orderBy: { id: 'asc' } }),
db.auction.findFirstOrThrow(),
db.message.findFirstOrThrow(),
db.votePoll.findFirstOrThrow(),
db.selectPoolEntry.findFirstOrThrow(),
db.npcSelectionToken.findFirstOrThrow(),
db.clockSuspension.findUniqueOrThrow({ where: { id: suspended.suspensionId } }),
db.clockProjectionOutbox.findMany(),
]);
const alignedTick = BigInt(reconciled.alignedTick);
expect(afterWorld).toMatchObject({
clockPhase: 'RECONCILING',
clockRevision: 2n,
deadlineGeneration: 8n,
clockTick: alignedTick,
lastTurnTick: BigInt(lastTurnTick + reconciled.shiftTicks),
});
expect(generals.map((general) => general.turnTick! - alignedTick)).toEqual(
generalTicks.map((tick) => BigInt(tick - initialTick))
);
expect(auction.closeTick! - alignedTick).toBe(BigInt(auctionCloseTick - initialTick));
expect(message.validUntilTick! - alignedTick).toBe(BigInt(messageExpiryTick - initialTick));
expect(vote.endTick! - alignedTick).toBe(BigInt(voteEndTick - initialTick));
expect(pool.reservedUntilTick! - alignedTick).toBe(BigInt(poolTick - initialTick));
expect(token.validUntilTick! - alignedTick).toBe(BigInt(npcValidTick - initialTick));
expect(token.pickMoreFromTick! - alignedTick).toBe(BigInt(npcMoreTick - initialTick));
expect(generals.map((general) => general.recentWarTick)).toEqual([
BigInt(initialTick - 100),
BigInt(initialTick - 101),
]);
expect(auction.openTick).toBe(BigInt(initialTick - 300));
expect(message.timeTick).toBe(BigInt(messageOccurrenceTick));
expect(vote.startTick).toBe(BigInt(voteStartTick));
expect(ledger.status).toBe('RECONCILING');
expect(outboxes).toHaveLength(1);
expect(outboxes[0]).toMatchObject({ status: 'PENDING', targetRevision: 2n });
const retried = await reconcileClockSuspension({
db,
suspensionId: suspended.suspensionId,
authority,
testResumeWallAt: new Date(resumeWallAt.getTime() + 10_000),
});
expect(retried).toEqual(reconciled);
expect(await db.clockProjectionOutbox.count()).toBe(1);
const keepParticipants = await db.clockReconciliationParticipant.findMany({ where: { policy: 'KEEP' } });
expect(keepParticipants.every((participant) => participant.beforeChecksum === participant.afterChecksum)).toBe(
true
);
await redis.client.set('sammo:clock-test:clock:active-revision', '1');
expect(
await applyNextClockProjection({ db, redis: redis.client, workerId: 'clock-projection-success' })
).toBe('APPLIED');
expect(await redis.client.get('sammo:clock-test:clock:active-revision')).toBe('2');
expect(await redis.client.get('sammo:clock-test:clock:deadline-generation')).toBe('8');
expect(await redis.client.get('sammo:clock-test:clock:phase')).toBe('RUNNING');
expect(await redis.client.zRangeWithScores('sammo:clock-test:auction:timer', 0, -1)).toEqual([
{ value: String(auction.id), score: Number(auction.closeTick) },
]);
expect(await db.worldState.findUniqueOrThrow({ where: { id: world.id } })).toMatchObject({
clockPhase: 'RUNNING',
clockRevision: 2n,
});
expect(await db.clockProjectionOutbox.findFirstOrThrow()).toMatchObject({ status: 'APPLIED' });
});
it('rejects a live offline fence and preserves a turn deadline across an exact 24-hour gap', async () => {
const baseTime = new Date('2026-02-01T00:00:00.000Z');
const futureAnchor = new Date(Date.now() + 3_600_000);
const initialTick = 5 * 36_000_000;
const turnTick = initialTick + 17_000_007;
const clock = new GameClock({
baseTime,
tick: initialTick,
mode: 'realtime',
wallAnchor: futureAnchor,
turnSeconds: 3_600,
phase: 'RUNNING',
});
await db.worldState.create({
data: {
scenarioCode: 'clock-day-test',
currentYear: 180,
currentMonth: 1,
tickSeconds: 3_600,
clockBaseTime: baseTime,
clockTick: BigInt(initialTick),
clockMode: 'realtime',
clockWallAnchor: futureAnchor,
lastTurnTick: BigInt(initialTick),
clockPhase: 'RUNNING',
clockRevision: 3n,
deadlineGeneration: 2n,
},
});
await db.general.create({
data: { id: 1, name: 'day-general', turnTick: BigInt(turnTick), turnTime: clock.tickToDate(turnTick) },
});
await db.turnDaemonLease.create({
data: {
profile: 'clock-day-test',
ownerId: 'other-daemon',
fencingEpoch: 9n,
leaseUntil: new Date(Date.now() + 60_000),
},
});
const authority = {
kind: 'OFFLINE' as const,
profileName: 'clock-day-test',
reason: '24-hour integration fixture',
};
await expect(
startClockSuspension({
db,
suspensionId: 'clock-gap-24h',
source: 'MAINTENANCE',
authority,
})
).rejects.toThrow('daemon lease to be offline');
await db.turnDaemonLease.delete({ where: { profile: 'clock-day-test' } });
const suspended = await startClockSuspension({
db,
suspensionId: 'clock-gap-24h',
source: 'MAINTENANCE',
authority,
});
const reconciled = await reconcileClockSuspension({
db,
suspensionId: suspended.suspensionId,
authority,
testResumeWallAt: new Date(suspended.cutWallAt.getTime() + 24 * 60 * 60_000),
});
expect(reconciled).toMatchObject({
sourceRevision: 3,
targetRevision: 4,
gapTicks: 24 * 36_000_000,
shiftTicks: 24 * 36_000_000,
alignedTick: initialTick + 24 * 36_000_000,
});
const shifted = await db.general.findUniqueOrThrow({ where: { id: 1 } });
expect(shifted.turnTick! - BigInt(reconciled.alignedTick)).toBe(BigInt(turnTick - initialTick));
await redis.client.set('sammo:clock-day-test:clock:active-revision', '3');
const redisThenCrash = {
get: redis.client.get.bind(redis.client),
eval: async (script: string, options: { keys: string[]; arguments: string[] }) => {
await redis.client.eval(script, options);
throw new Error('fixture crash after Redis commit');
},
};
await expect(
applyNextClockProjection({ db, redis: redisThenCrash, workerId: 'clock-projection-crash' })
).rejects.toThrow('fixture crash after Redis commit');
expect(await redis.client.get('sammo:clock-day-test:clock:active-revision')).toBe('4');
expect(await db.worldState.findFirstOrThrow()).toMatchObject({ clockPhase: 'RECONCILING' });
expect(await db.clockProjectionOutbox.findFirstOrThrow()).toMatchObject({ status: 'FAILED', attempts: 1 });
await db.clockProjectionOutbox.updateMany({ data: { availableAt: new Date(0) } });
expect(
await applyNextClockProjection({ db, redis: redis.client, workerId: 'clock-projection-restart' })
).toBe('RECOVERED');
expect(await db.worldState.findFirstOrThrow()).toMatchObject({ clockPhase: 'RUNNING', clockRevision: 4n });
expect(await db.clockProjectionOutbox.findFirstOrThrow()).toMatchObject({ status: 'APPLIED', attempts: 2 });
});
});
@@ -1,4 +1,4 @@
import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest';
import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest';
import type { TurnDaemonCommand } from '@sammo-ts/common';
import { createGamePostgresConnector } from '@sammo-ts/infra';
@@ -25,6 +25,12 @@ integration('database command queue', () => {
});
});
beforeEach(async () => {
await db.inputEvent.deleteMany({ where: { requestId: { startsWith: 'integration:engine:' } } });
await db.clockSuspension.deleteMany({ where: { id: 'integration-queue-revision-8-9' } });
await db.worldState.updateMany({ data: { clockPhase: 'RUNNING' } });
});
afterAll(async () => {
await db.inputEvent.deleteMany({
where: { requestId: { startsWith: 'integration:engine:' } },
@@ -229,4 +235,119 @@ integration('database command queue', () => {
expect(handle).toHaveBeenCalledOnce();
expect(mutation).not.toHaveBeenCalled();
});
it('dequeues gameplay only in an executable phase and records the processing clock generation', async () => {
const existingWorld = await db.worldState.findFirst({ orderBy: { id: 'asc' } });
const world = existingWorld
? await db.worldState.update({
where: { id: existingWorld.id },
data: { clockPhase: 'SUSPENDED', clockRevision: 9n, deadlineGeneration: 4n, clockTick: 123n },
})
: await db.worldState.create({
data: {
scenarioCode: 'queue-clock-test',
currentYear: 180,
currentMonth: 1,
tickSeconds: 600,
clockPhase: 'SUSPENDED',
clockRevision: 9n,
deadlineGeneration: 4n,
clockTick: 123n,
},
});
const gameplayId = 'integration:engine:clock-gated-gameplay';
const statusId = 'integration:engine:clock-gated-status';
const staleId = 'integration:engine:clock-gated-stale';
await db.inputEvent.createMany({
data: [
{
requestId: gameplayId,
target: 'ENGINE',
eventType: 'vacation',
actorUserId: 'user-7',
acceptedGameTick: 100n,
acceptedClockRevision: 9n,
acceptedDeadlineGeneration: 4n,
payload: { type: 'vacation', requestId: gameplayId, userId: 'user-7', generalId: 7 },
},
{
requestId: statusId,
target: 'ENGINE',
eventType: 'getStatus',
acceptedGameTick: 100n,
acceptedClockRevision: 9n,
acceptedDeadlineGeneration: 4n,
payload: { type: 'getStatus', requestId: statusId },
},
{
requestId: staleId,
target: 'ENGINE',
eventType: 'vacation',
actorUserId: 'user-8',
acceptedGameTick: 90n,
acceptedClockRevision: 8n,
acceptedDeadlineGeneration: 3n,
payload: { type: 'vacation', requestId: staleId, userId: 'user-8', generalId: 8 },
},
],
});
const queue = new DatabaseTurnDaemonCommandQueue(db);
expect(await queue.drain()).toEqual([{ type: 'getStatus', requestId: statusId }]);
expect(await db.inputEvent.findUniqueOrThrow({ where: { requestId: gameplayId } })).toMatchObject({
status: 'PENDING',
processingClockRevision: null,
});
await db.worldState.update({ where: { id: world.id }, data: { clockPhase: 'RUNNING' } });
expect(await queue.drain()).toEqual([
{ type: 'vacation', requestId: gameplayId, userId: 'user-7', generalId: 7 },
]);
expect(await db.inputEvent.findUniqueOrThrow({ where: { requestId: gameplayId } })).toMatchObject({
status: 'PROCESSING',
processingGameTick: 100n,
processingClockRevision: 9n,
processingDeadlineGeneration: 4n,
});
expect(await db.inputEvent.findUniqueOrThrow({ where: { requestId: staleId } })).toMatchObject({
status: 'PENDING',
processingClockRevision: null,
});
await db.clockSuspension.deleteMany({ where: { id: 'integration-queue-revision-8-9' } });
await db.clockSuspension.create({
data: {
id: 'integration-queue-revision-8-9',
worldStateId: world.id,
source: 'MAINTENANCE',
policy: 'EXACT',
status: 'APPLIED',
sourceRevision: 8n,
targetRevision: 9n,
cutTick: 90n,
cutWallAt: new Date(),
resumeWallAt: new Date(),
rateTicksPerSecond: 60_000,
gapTicks: 33n,
shiftTicks: 33n,
alignedTick: 123n,
},
});
expect(await queue.drain()).toEqual([
{
type: 'vacation',
requestId: staleId,
userId: 'user-8',
generalId: 8,
processingGameTick: 123,
},
]);
expect(await db.inputEvent.findUniqueOrThrow({ where: { requestId: staleId } })).toMatchObject({
status: 'PROCESSING',
acceptedGameTick: 90n,
acceptedClockRevision: 8n,
processingGameTick: 123n,
processingClockRevision: 9n,
processingDeadlineGeneration: 4n,
});
});
});
@@ -448,6 +448,21 @@ describe('runtime clock shift projection', () => {
return {};
});
const db = {
$transaction: async (operation: (transaction: GamePrismaClient) => Promise<unknown>) => operation(db),
$executeRaw: vi.fn(async () => 1),
$queryRaw: vi.fn(async () => [{ wallNow: new Date('2026-07-30T10:00:00.000Z') }]),
worldState: {
findFirst: vi.fn(async () => ({
clockBaseTime: new Date('2026-07-30T10:00:00.000Z'),
clockTick: 0n,
clockMode: 'realtime',
clockWallAnchor: new Date('2026-07-30T10:00:00.000Z'),
tickSeconds: 600,
clockPhase: 'RUNNING',
clockRevision: 1n,
deadlineGeneration: 1n,
})),
},
inputEvent: {
create: inputEventCreate,
findUniqueOrThrow: vi.fn(async () =>
@@ -567,6 +582,21 @@ describe('runtime game settings projection', () => {
let eventStatus: 'PENDING' | 'SUCCEEDED' = 'PENDING';
let created = false;
const db = {
$transaction: async (operation: (transaction: GamePrismaClient) => Promise<unknown>) => operation(db),
$executeRaw: vi.fn(async () => 1),
$queryRaw: vi.fn(async () => [{ wallNow: new Date('2026-07-30T10:00:00.000Z') }]),
worldState: {
findFirst: vi.fn(async () => ({
clockBaseTime: new Date('2026-07-30T10:00:00.000Z'),
clockTick: 0n,
clockMode: 'realtime',
clockWallAnchor: new Date('2026-07-30T10:00:00.000Z'),
tickSeconds: 600,
clockPhase: 'RUNNING',
clockRevision: 1n,
deadlineGeneration: 1n,
})),
},
inputEvent: {
create: vi.fn(async () => {
if (created) throw { code: 'P2002' };
@@ -1,4 +1,4 @@
import { afterAll, beforeAll, describe, expect, it } from 'vitest';
import { afterAll, beforeAll, beforeEach, describe, expect, it } from 'vitest';
import { createGamePostgresConnector, type GamePrisma, type GamePrismaClient } from '@sammo-ts/infra';
import { GAME_TICKS_PER_TURN } from '@sammo-ts/common';
@@ -77,38 +77,32 @@ integration('runtime clock shift persistence', () => {
let db: GamePrismaClient;
let closeDb: (() => Promise<void>) | undefined;
const cleanupFixtures = async (): Promise<void> => {
await db.inputEvent.deleteMany({ where: { requestId: { in: [requestId, runtimeSettingsRequestId] } } });
await db.votePoll.deleteMany({ where: { openerGeneralId: generalIds[2] } });
await db.message.deleteMany({ where: { mailbox: generalIds[2] } });
await db.logEntry.deleteMany({ where: { text: runtimeSettingsLogText } });
await db.auction.deleteMany({ where: { hostGeneralId: { in: [...generalIds] } } });
await db.selectPoolEntry.deleteMany({ where: { uniqueName: backlogPoolUniqueName } });
await db.general.deleteMany({ where: { id: { in: [...generalIds] } } });
await db.worldState.deleteMany({
where: {
scenarioCode: { in: ['runtime-clock-shift', 'runtime-game-settings', 'realtime-backlog-rebase'] },
},
});
};
beforeAll(async () => {
const connector = createGamePostgresConnector({ url: databaseUrl! });
await connector.connect();
db = connector.prisma;
closeDb = () => connector.disconnect();
await db.inputEvent.deleteMany({ where: { requestId: { in: [requestId, runtimeSettingsRequestId] } } });
await db.votePoll.deleteMany({ where: { openerGeneralId: generalIds[2] } });
await db.message.deleteMany({ where: { mailbox: generalIds[2] } });
await db.logEntry.deleteMany({ where: { text: runtimeSettingsLogText } });
await db.auction.deleteMany({ where: { hostGeneralId: { in: [...generalIds] } } });
await db.selectPoolEntry.deleteMany({ where: { uniqueName: backlogPoolUniqueName } });
await db.general.deleteMany({ where: { id: { in: [...generalIds] } } });
await db.worldState.deleteMany({
where: {
scenarioCode: { in: ['runtime-clock-shift', 'runtime-game-settings', 'realtime-backlog-rebase'] },
},
});
});
beforeEach(cleanupFixtures);
afterAll(async () => {
await db.inputEvent.deleteMany({ where: { requestId: { in: [requestId, runtimeSettingsRequestId] } } });
await db.votePoll.deleteMany({ where: { openerGeneralId: generalIds[2] } });
await db.message.deleteMany({ where: { mailbox: generalIds[2] } });
await db.logEntry.deleteMany({ where: { text: runtimeSettingsLogText } });
await db.auction.deleteMany({ where: { hostGeneralId: { in: [...generalIds] } } });
await db.selectPoolEntry.deleteMany({ where: { uniqueName: backlogPoolUniqueName } });
await db.general.deleteMany({ where: { id: { in: [...generalIds] } } });
await db.worldState.deleteMany({
where: {
scenarioCode: { in: ['runtime-clock-shift', 'runtime-game-settings', 'realtime-backlog-rebase'] },
},
});
await cleanupFixtures();
await closeDb?.();
});
@@ -469,6 +463,7 @@ integration('runtime clock shift persistence', () => {
clockBaseTime: base,
clockTick: 0,
clockMode: 'manual',
clockPhase: 'MANUAL',
clockWallAnchor: base,
lastTurnTick: 0,
config: { turnTermMinutes: 10, blockGeneralCreate: 0 },
+1 -1
View File
@@ -39,7 +39,7 @@ describe('readReleaseManifest', () => {
await expect(readReleaseManifest(workspaceRoot)).resolves.toMatchObject({
controllerProtocol: RELEASE_CONTROLLER_PROTOCOL,
gatewaySchemaHead: '20260825000000_add_bulk_release_batches',
gameSchemaHead: '20260903090000_add_game_clock_reconciliation',
gameSchemaHead: '20260903103000_add_input_event_clock_processing',
});
});
+24 -8
View File
@@ -6,6 +6,10 @@
"coveredFields": [
"input_event.accepted_game_tick",
"input_event.accepted_clock_revision",
"input_event.accepted_deadline_generation",
"input_event.processing_game_tick",
"input_event.processing_clock_revision",
"input_event.processing_deadline_generation",
"world_state.clock_tick",
"world_state.last_turn_tick",
"world_state.clock_revision",
@@ -125,8 +129,16 @@
{
"key": "accepted-command-coordinate",
"policy": "KEEP",
"authorityFields": ["input_event.accepted_game_tick", "input_event.accepted_clock_revision"],
"projectionFields": [],
"authorityFields": [
"input_event.accepted_game_tick",
"input_event.accepted_clock_revision",
"input_event.accepted_deadline_generation"
],
"projectionFields": [
"input_event.processing_game_tick",
"input_event.processing_clock_revision",
"input_event.processing_deadline_generation"
],
"owner": "game-api/input-event"
},
{
@@ -139,11 +151,15 @@
},
{
"key": "movable-json-rule-anchors",
"policy": "FORBID",
"authorityFields": ["world_state.meta.turntime", "world_state.meta.starttime", "world_state.meta.tnmt_time"],
"policy": "SHIFT",
"authorityFields": [
"world_state.meta.turntime",
"world_state.meta.starttime",
"world_state.meta.tnmt_time"
],
"projectionFields": [],
"owner": "game-engine/world-meta",
"migration": "Register typed columns or explicit participant adapters before exact reconciliation can complete."
"owner": "game-engine/clock-operation",
"migration": "Explicit adapter shifts registered ISO projections while typed tick columns remain authoritative."
},
{
"key": "unification-wait",
@@ -179,12 +195,12 @@
{
"keyPattern": "sammo:{profile}:auction:timer",
"policy": "REBUILD",
"status": "implemented-without-clock-revision-fence"
"status": "implemented-with-clock-revision-phase-generation-fence"
},
{
"keyPattern": "sammo:{profile}:tournament:state",
"policy": "REBUILD",
"status": "legacy-date-dual-write-required"
"status": "implemented-with-tick-dual-write-and-clock-fence"
}
],
"wallOnly": [
+19 -4
View File
@@ -65,6 +65,19 @@ pop is one Redis operation: verify revision/phase, read `-inf..nowTick`, and
remove the claimed members. A failed Redis rebuild therefore leaves the game in
`RECONCILING`; process liveness alone is not readiness.
`applyNextClockProjection()` claims rows with `FOR UPDATE SKIP LOCKED` and uses
PostgreSQL UTC wall time for claim/retry timestamps. One Redis Lua operation
compares the active source revision, rebuilds the auction timer, conditionally
replaces the exact tournament source snapshot, and writes target revision plus
deadline generation. The DB finalizer then re-acquires the clock-operation lock
and changes `RECONCILING -> RUNNING` only when target revision and generation
still match. If the process dies after the Lua commit, retry observes the
already-active target revision and performs only the DB finalizer.
An active legacy tournament containing only `nextAt`/`bettingCloseAt` is a
fail-closed migration boundary. Reconciliation remains incomplete until its
authoritative `nextTick`/`bettingCloseTick` dual-write exists.
## Lock order
All mutation paths use this order:
@@ -101,7 +114,9 @@ future anchored realtime profiles as `PREOPEN`, and other profiles as
`RUNNING`. Existing DateTime columns remain projections while tick columns are
authoritative.
Exact reconciliation stays disabled while any registry participant is
`FORBID`. In particular, Redis-only tournament dates and unification wait must
be moved to durable tick/revision contracts before the operation can reach
`RUNNING`. Removing these guards to make a partial operation pass is prohibited.
Exact reconciliation stays disabled while an active registry participant is
`FORBID`. Tournament writes now carry tick/revision/generation coordinates and
are revision-fenced in Redis. The remaining unification wait participant must
be moved from its `lastTurnTime` workaround to a durable suspension before that
workflow can reach `RUNNING`. Removing this guard to make a partial operation
pass is prohibited.
@@ -11,7 +11,7 @@ mean deployment or production validation.
## Milestone 1 - authority and inventory
- [x] Branded `GameTick`, `ObservedGameInstant`, `ScheduleInstant`,
`WallInstant`, and `ClockRevision` boundaries.
`WallInstant`, and `ClockRevision` boundaries.
- [x] Explicit clock phase and monotonic RUNNING projection.
- [x] Exact alignment arithmetic preserving millisecond/sub-turn remainder.
- [x] Opening tick zero and PREOPEN executable floor in the shared seeder.
@@ -19,49 +19,50 @@ mean deployment or production validation.
- [x] Suspension, participant-checksum, and Redis projection outbox tables.
- [x] Machine-readable DB/Redis/JSON participant inventory and architecture gate.
- [x] Turn flush lock prefix and phase/revision/generation fence.
- [ ] Empty and upgraded database migration execution evidence.
- [x] Empty and upgraded database migration execution evidence.
## Milestone 2 - exact DB reconciliation
- [ ] Suspension start command with DB wall time and idempotent source revision.
- [ ] Exact resume plan transaction with deterministic participant lock order.
- [ ] SHIFT adapters for cursor, generals, active auctions, message expiry, vote
end, select pool, and NPC selection windows.
- [ ] KEEP checksum adapters for occurrences and history.
- [ ] Explicit `LEGACY_COMPLETE_TURNS` and bounded `CATCH_UP` policies.
- [ ] Property tests for remaining distance, ordering, and history invariants.
- [ ] 24-hour and 65m17.250s PostgreSQL integration evidence.
- [x] Suspension start command with DB wall time and idempotent source revision.
- [x] Exact resume plan transaction with deterministic participant lock order.
- [x] SHIFT adapters for cursor, generals, active auctions, message expiry, vote
end, select pool, and NPC selection windows.
- [x] KEEP checksum adapters for occurrences and accepted command coordinates.
- [x] Explicit `LEGACY_COMPLETE_TURNS` and bounded `CATCH_UP` policies.
- [x] Property tests for remaining distance, ordering, and history invariants.
- [x] 24-hour and 65m17.250s PostgreSQL integration evidence.
## Milestone 3 - revisioned Redis and workers
- [ ] Projection outbox claimer/retry/recovery state machine.
- [ ] Redis active revision and atomic due-pop script.
- [ ] Auction OPEN/FINALIZING revision and generation fence.
- [ ] Tournament durable tick dual-write and projection rebuild.
- [ ] DB-commit/Redis-failure restart tests and readiness integration.
- [x] Projection outbox claimer/retry/recovery state machine.
- [x] Redis active revision and atomic due-pop script.
- [x] Auction OPEN/FINALIZING revision and generation fence.
- [x] Tournament durable tick dual-write and projection rebuild.
- [x] DB-commit/Redis-failure crash-restart test.
- [x] Readiness integration in Gateway/API process health.
## Milestone 4 - command and lifecycle workflows
- [ ] All durable input events record accepted tick and accepted revision.
- [ ] Processing converts accepted coordinates across revisions or fails closed.
- [x] All durable input events record accepted tick and accepted revision.
- [x] Processing converts accepted coordinates across revisions or fails closed.
- [ ] Gateway pause/resume/open orchestration writes the DB clock phase.
- [ ] Unification wait becomes a durable `UNIFICATION_WAIT` suspension.
- [ ] Alignment, optional rate change, invader IDs/RNG, creation, first schedule,
outbox, verification, and RUNNING transition form one retry-safe workflow.
outbox, verification, and RUNNING transition form one retry-safe workflow.
- [ ] Multi-host drift and general-access/clock-operation deadlock tests.
## Milestone 5 - test-branch release gate
- [ ] Full typecheck, architecture, lint, unit, build, and non-conditional
integration suites.
integration suites.
- [ ] Dedicated PostgreSQL/Redis conditional integration suite with skip count
recorded.
recorded.
- [ ] Recovery runbook exercised from each incomplete status.
- [ ] Admin status/readiness exposes revision, phase, participant checksums, and
incomplete outbox state.
- [x] Admin status/readiness exposes revision, phase, participant checksums, and
incomplete outbox state.
- [ ] User-test deployment evidence is recorded separately from Git push.
- [ ] All `FORBID` inventory entries are removed by typed migrations or proven
inactive preconditions.
inactive preconditions.
## Evidence log
@@ -78,4 +79,25 @@ mean deployment or production validation.
- `pnpm check:architecture`: package boundaries passed; 21 authoritative clock
fields and 18 participants were registered.
- Migration SQL was generated, formatted, validated, and registered as the
release manifest head. Empty/upgraded PostgreSQL execution is still pending.
release manifest head. All 43 migrations applied to the empty dedicated
PostgreSQL instance. A second schema upgraded from the previous head with an
existing manual `world_state` row and verified `MANUAL:1:1` plus all three
reconciliation tables.
- Dedicated PostgreSQL/Redis fixtures proved exact 24-hour and 65m17.250s gaps,
schedule distance/order preservation, occurrence checksums, live-lease
fencing, a single durable outbox, Redis target revision, and recovery after a
crash between Redis commit and DB finalization.
### 2026-09-03 - revisioned workers and input coordinates
- `CI=1 TURBO_CONCURRENCY=1 pnpm typecheck`: 21/21 tasks passed after the
worker/input contract changes.
- `CI=1 TURBO_CONCURRENCY=1 pnpm test`: 12/12 package tasks passed.
- Dedicated PostgreSQL runs passed `databaseCommandQueue.integration.test.ts`
6/6 and `runtimeClockShiftPersistence.integration.test.ts` 3/3 when executed
sequentially; the latter now clears each single-world fixture before the next
case. Dedicated Redis passed tournament source revision 4/4, including an
atomic stale-clock rejection. API input-event integration passed 13/13.
- The 24-hour/65m17.250s reconciliation suite passed 2/2 after the other DB
suites. Conditional files share a deliberately dedicated schema and are run
sequentially to prevent their fixture cleanup from racing another file.
+12
View File
@@ -28,6 +28,18 @@ service. The service must re-read participant checksums and either return the
already-applied result or resume the pending outbox. Never create a replacement
revision to hide a failed target revision.
When an outbox row is `FAILED`, the profile must remain `RECONCILING`. A retry
is safe in both crash locations:
- before Redis commit, the Lua operation reapplies from the source revision;
- after Redis commit but before DB finalization, the Lua operation returns the
already-active target result and DB finalization resumes without shifting any
tournament deadline twice.
`lastError` naming a legacy tournament deadline means the tournament lacks its
tick dual-write. Do not force the active revision; migrate or prove the
tournament inactive, then retry the same outbox.
## Rollback
There is no blind inverse update. Before enabling exact reconciliation in an
+46 -5
View File
@@ -17,8 +17,8 @@ export type ScheduleInstant = GameTick & { readonly [scheduleInstantBrand]: 'Sch
export type ClockRevision = number & { readonly [clockRevisionBrand]: 'ClockRevision' };
export type WallInstant = Date & { readonly [wallInstantBrand]: 'WallInstant' };
export interface ExactClockAlignmentPlan {
policy: 'EXACT';
export interface ClockAlignmentPlan {
policy: ClockAlignmentPolicy;
sourceRevision: ClockRevision;
targetRevision: ClockRevision;
cutTick: GameTick;
@@ -84,6 +84,15 @@ export const parseGameClockPhase = (value: string): GameClockPhase => {
throw new Error(`Unknown game clock phase: ${value}`);
};
const CLOCK_ALIGNMENT_POLICIES: readonly ClockAlignmentPolicy[] = ['EXACT', 'LEGACY_COMPLETE_TURNS', 'CATCH_UP'];
export const parseClockAlignmentPolicy = (value: string): ClockAlignmentPolicy => {
if ((CLOCK_ALIGNMENT_POLICIES as readonly string[]).includes(value)) {
return value as ClockAlignmentPolicy;
}
throw new Error(`Unknown clock alignment policy: ${value}`);
};
export const scheduleNotBefore = (instant: ObservedGameInstant, phase: GameClockPhase): ScheduleInstant => {
if (phase === 'PREOPEN') {
return asScheduleInstant(Math.max(0, instant));
@@ -103,14 +112,15 @@ export const assertGameplayCommitAllowed = (phase: GameClockPhase): void => {
}
};
export const buildExactClockAlignmentPlan = (input: {
const buildAlignmentPlan = (input: {
policy: ClockAlignmentPolicy;
sourceRevision: number;
cutTick: number;
cutWall: Date;
resumeWall: Date;
ticksPerSecond: number;
catchUpTicks?: number;
}): ExactClockAlignmentPlan => {
}): ClockAlignmentPlan => {
const sourceRevision = asClockRevision(input.sourceRevision);
const cutTick = asGameTick(input.cutTick);
const cutWall = asWallInstant(input.cutWall);
@@ -136,7 +146,7 @@ export const buildExactClockAlignmentPlan = (input: {
}
const shiftTicks = asGameTick(gapTicks - catchUpTicks);
return {
policy: 'EXACT',
policy: input.policy,
sourceRevision,
targetRevision: asClockRevision(sourceRevision + 1),
cutTick,
@@ -147,6 +157,37 @@ export const buildExactClockAlignmentPlan = (input: {
};
};
export const buildExactClockAlignmentPlan = (
input: Omit<Parameters<typeof buildAlignmentPlan>[0], 'policy'>
): ClockAlignmentPlan => buildAlignmentPlan({ ...input, policy: 'EXACT' });
export const buildClockAlignmentPlan = (input: {
policy: ClockAlignmentPolicy;
sourceRevision: number;
cutTick: number;
cutWall: Date;
resumeWall: Date;
ticksPerSecond: number;
catchUpTicks?: number;
}): ClockAlignmentPlan => {
if (input.policy === 'EXACT') {
if ((input.catchUpTicks ?? 0) !== 0) {
throw new Error('EXACT alignment does not allow catch-up ticks.');
}
return buildAlignmentPlan({ ...input, policy: 'EXACT', catchUpTicks: 0 });
}
if (input.policy === 'CATCH_UP') {
return buildAlignmentPlan({ ...input, policy: 'CATCH_UP' });
}
const exact = buildAlignmentPlan({ ...input, policy: 'LEGACY_COMPLETE_TURNS', catchUpTicks: 0 });
const shiftTicks = asGameTick(Math.floor(exact.gapTicks / GAME_TICKS_PER_TURN) * GAME_TICKS_PER_TURN);
return {
...exact,
catchUpTicks: asGameTick(exact.gapTicks - shiftTicks),
shiftTicks,
};
};
const tickOffsetMilliseconds = (tick: number, ticksPerSecond: number): number => {
const wholeSeconds = Math.floor(tick / ticksPerSecond);
const remainingTicks = tick - wholeSeconds * ticksPerSecond;
@@ -18,6 +18,15 @@ export interface TournamentProjectionWrite {
value: unknown;
}
export interface TournamentClockFence {
activeRevisionKey: string;
deadlineGenerationKey: string;
phaseKey: string;
revision: number;
deadlineGeneration: number;
phase: 'RUNNING';
}
const WRITE_TOURNAMENT_PROJECTION_SCRIPT = `
local revision_key = KEYS[#KEYS]
local current = redis.call('GET', revision_key)
@@ -55,6 +64,49 @@ local revision = redis.call('INCR', revision_key)
return tostring(revision) .. ':' .. (stage_changed and '1' or '0') .. ':' .. (rankings_changed and '1' or '0')
`;
const WRITE_FENCED_TOURNAMENT_PROJECTION_SCRIPT = `
local revision_key_index = #KEYS - 3
if redis.call('GET', KEYS[#KEYS - 2]) ~= ARGV[#ARGV - 2]
or redis.call('GET', KEYS[#KEYS - 1]) ~= ARGV[#ARGV - 1]
or redis.call('GET', KEYS[#KEYS]) ~= ARGV[#ARGV] then
return '__CLOCK_FENCE__'
end
local revision_key = KEYS[revision_key_index]
local current = redis.call('GET', revision_key)
if current then
if not string.match(current, '^%d+$') then
return redis.error_reply('invalid tournament source revision')
end
if string.len(current) > 18 then
return redis.error_reply('tournament source revision exhausted')
end
end
local stage_changed = false
local rankings_changed = false
for index = 1, revision_key_index - 1 do
local next_ok, next_value = pcall(cjson.decode, ARGV[index])
if next_ok and type(next_value) == 'table' and next_value['stage'] ~= nil then
local previous = redis.call('GET', KEYS[index])
local previous_stage = nil
local previous_value = nil
if previous then
local previous_ok
previous_ok, previous_value = pcall(cjson.decode, previous)
if previous_ok and type(previous_value) == 'table' then
previous_stage = previous_value['stage']
end
end
local next_stage = next_value['stage']
stage_changed = (not previous) or previous_stage ~= next_stage
local previous_reward_settled = previous_value and previous_value['rewardSettled'] or false
rankings_changed = next_value['rewardSettled'] == true and previous_reward_settled ~= true
end
redis.call('SET', KEYS[index], ARGV[index])
end
local revision = redis.call('INCR', revision_key)
return tostring(revision) .. ':' .. (stage_changed and '1' or '0') .. ':' .. (rankings_changed and '1' or '0')
`;
export const parseTournamentSourceRevision = (value: unknown): string | null => {
if (typeof value === 'number') {
return Number.isSafeInteger(value) && value >= 0 ? String(value) : null;
@@ -69,7 +121,8 @@ export const parseTournamentSourceRevision = (value: unknown): string | null =>
export const writeTournamentProjection = async (
redis: TournamentProjectionRedis,
keys: TournamentSourceKeys,
writes: readonly TournamentProjectionWrite[]
writes: readonly TournamentProjectionWrite[],
clockFence?: TournamentClockFence
): Promise<string> => {
if (writes.length === 0) {
throw new Error('Tournament projection write must contain at least one payload.');
@@ -90,10 +143,27 @@ export const writeTournamentProjection = async (
value !== null &&
(value as { rewardSettled?: unknown }).rewardSettled === true
);
const result = await redis.eval(WRITE_TOURNAMENT_PROJECTION_SCRIPT, {
keys: [...writes.map(({ key }) => key), keys.sourceRevisionKey],
arguments: writes.map(({ value }) => JSON.stringify(value)),
});
const result = await redis.eval(
clockFence ? WRITE_FENCED_TOURNAMENT_PROJECTION_SCRIPT : WRITE_TOURNAMENT_PROJECTION_SCRIPT,
{
keys: [
...writes.map(({ key }) => key),
keys.sourceRevisionKey,
...(clockFence
? [clockFence.activeRevisionKey, clockFence.deadlineGenerationKey, clockFence.phaseKey]
: []),
],
arguments: [
...writes.map(({ value }) => JSON.stringify(value)),
...(clockFence
? [String(clockFence.revision), String(clockFence.deadlineGeneration), clockFence.phase]
: []),
],
}
);
if (result === '__CLOCK_FENCE__') {
throw new Error('Tournament projection clock revision fence failed.');
}
const scriptResult = typeof result === 'string' ? /^(\d+):([01])(?::([01]))?$/u.exec(result) : null;
const sourceRevision = parseTournamentSourceRevision(scriptResult?.[1] ?? result);
// Plain revision results remain accepted for rolling deployments and small
+59
View File
@@ -6,6 +6,7 @@ import {
MAX_SAFE_GAME_TICK,
asGameTick,
asObservedGameInstant,
buildClockAlignmentPlan,
buildExactClockAlignmentPlan,
createDeadline,
scheduleNotBefore,
@@ -113,6 +114,64 @@ describe('GameClock', () => {
expect(plan.alignedTick).toBe(26 * GAME_TICKS_PER_TURN);
});
it('keeps legacy complete-turn rebasing separate from bounded catch-up', () => {
const common = {
sourceRevision: 4,
cutTick: 100,
cutWall: new Date('2026-01-01T00:00:00.000Z'),
resumeWall: new Date('2026-01-01T01:05:17.250Z'),
ticksPerSecond: 10_000,
};
const legacy = buildClockAlignmentPlan({ ...common, policy: 'LEGACY_COMPLETE_TURNS' });
const catchUp = buildClockAlignmentPlan({ ...common, policy: 'CATCH_UP', catchUpTicks: 1_000_000 });
expect(legacy).toMatchObject({
policy: 'LEGACY_COMPLETE_TURNS',
gapTicks: 39_172_500,
shiftTicks: GAME_TICKS_PER_TURN,
catchUpTicks: 3_172_500,
});
expect(catchUp).toMatchObject({
policy: 'CATCH_UP',
gapTicks: 39_172_500,
shiftTicks: 38_172_500,
catchUpTicks: 1_000_000,
});
expect(() => buildClockAlignmentPlan({ ...common, policy: 'EXACT', catchUpTicks: 1 })).toThrow(
'EXACT alignment does not allow catch-up ticks'
);
});
it('preserves schedule ordering, remaining distance, and occurrence ticks across generated exact gaps', () => {
let seed = 0x5eed1234;
const next = (): number => {
seed = (Math.imul(seed, 1_664_525) + 1_013_904_223) >>> 0;
return seed;
};
for (let iteration = 0; iteration < 500; iteration += 1) {
const cutTick = next() % 1_000_000_000;
const gapMilliseconds = next() % (7 * 24 * 60 * 60 * 1_000);
const ticksPerSecond = [5_000, 10_000, 60_000][next() % 3]!;
const offsets = Array.from({ length: 8 }, () => next() % (3 * GAME_TICKS_PER_TURN)).sort(
(left, right) => left - right
);
const occurrenceTicks = Array.from({ length: 4 }, () => cutTick - (next() % GAME_TICKS_PER_TURN));
const plan = buildClockAlignmentPlan({
policy: 'EXACT',
sourceRevision: 1 + (next() % 10_000),
cutTick,
cutWall: new Date(0),
resumeWall: new Date(gapMilliseconds),
ticksPerSecond,
});
const shifted = offsets.map((offset) => cutTick + offset + plan.shiftTicks);
expect(shifted.map((deadline) => deadline - plan.alignedTick)).toEqual(offsets);
expect([...shifted].sort((left, right) => left - right)).toEqual(shifted);
expect(occurrenceTicks).toEqual([...occurrenceTicks]);
}
});
it('projects near the safe tick boundary without unsafe intermediate multiplication', () => {
const clock = new GameClock({
baseTime: new Date(0),
+21 -17
View File
@@ -63,23 +63,27 @@ enum InputEventTarget {
}
model InputEvent {
sequence BigInt @id @default(autoincrement())
requestId String @unique @map("request_id")
target InputEventTarget
eventType String @map("event_type")
payload Json @default(dbgenerated("'{}'::jsonb"))
actorUserId String? @map("actor_user_id")
acceptedGameTick BigInt? @map("accepted_game_tick")
acceptedClockRevision BigInt? @map("accepted_clock_revision")
status InputEventStatus @default(PENDING)
result Json?
error String?
attempts Int @default(0)
lockedBy String? @map("locked_by")
leaseUntil DateTime? @map("lease_until")
createdAt DateTime @default(now()) @map("created_at")
processingAt DateTime? @map("processing_at")
completedAt DateTime? @map("completed_at")
sequence BigInt @id @default(autoincrement())
requestId String @unique @map("request_id")
target InputEventTarget
eventType String @map("event_type")
payload Json @default(dbgenerated("'{}'::jsonb"))
actorUserId String? @map("actor_user_id")
acceptedGameTick BigInt? @map("accepted_game_tick")
acceptedClockRevision BigInt? @map("accepted_clock_revision")
acceptedDeadlineGeneration BigInt? @map("accepted_deadline_generation")
processingGameTick BigInt? @map("processing_game_tick")
processingClockRevision BigInt? @map("processing_clock_revision")
processingDeadlineGeneration BigInt? @map("processing_deadline_generation")
status InputEventStatus @default(PENDING)
result Json?
error String?
attempts Int @default(0)
lockedBy String? @map("locked_by")
leaseUntil DateTime? @map("lease_until")
createdAt DateTime @default(now()) @map("created_at")
processingAt DateTime? @map("processing_at")
completedAt DateTime? @map("completed_at")
@@index([target, status, sequence])
@@map("input_event")
@@ -0,0 +1,15 @@
ALTER TABLE input_event
ADD COLUMN accepted_deadline_generation BIGINT,
ADD COLUMN processing_game_tick BIGINT,
ADD COLUMN processing_clock_revision BIGINT,
ADD COLUMN processing_deadline_generation BIGINT;
ALTER TABLE input_event
ADD CONSTRAINT input_event_accepted_clock_revision_positive
CHECK (accepted_clock_revision IS NULL OR accepted_clock_revision > 0),
ADD CONSTRAINT input_event_accepted_deadline_generation_positive
CHECK (accepted_deadline_generation IS NULL OR accepted_deadline_generation > 0),
ADD CONSTRAINT input_event_processing_clock_revision_positive
CHECK (processing_clock_revision IS NULL OR processing_clock_revision > 0),
ADD CONSTRAINT input_event_processing_deadline_generation_positive
CHECK (processing_deadline_generation IS NULL OR processing_deadline_generation > 0);
+3
View File
@@ -46,6 +46,9 @@ export interface DatabaseClient {
vote: GamePrisma.VoteDelegate;
inputEvent: GamePrisma.InputEventDelegate;
turnDaemonLease: GamePrisma.TurnDaemonLeaseDelegate;
clockSuspension?: GamePrisma.ClockSuspensionDelegate;
clockReconciliationParticipant?: GamePrisma.ClockReconciliationParticipantDelegate;
clockProjectionOutbox?: GamePrisma.ClockProjectionOutboxDelegate;
readModelOutbox: GamePrisma.ReadModelOutboxDelegate;
webPushOutbox: GamePrisma.WebPushOutboxDelegate;
}
+1
View File
@@ -10,4 +10,5 @@ export * from './readModelChangeJournal.js';
export * from './readModelOutboxDispatcher.js';
export * from './readModelCoverageActivation.js';
export * from './gameSchemaAdvisoryLock.js';
export * from './inputEventClock.js';
export * from './webPushOutbox.js';
+76
View File
@@ -0,0 +1,76 @@
import { GameClock, inferClockPhase, parseGameClockPhase } from '@sammo-ts/common';
import { GamePrisma, type GamePrismaClient } from './gamePrisma.js';
import { acquireGameSchemaAdvisoryXactLock, CLOCK_OPERATION_PERSISTENCE_LOCK } from './gameSchemaAdvisoryLock.js';
type ClockAcceptanceDatabase = Pick<GamePrismaClient, '$executeRaw' | '$queryRaw' | 'worldState'>;
interface DbWallRow {
wallNow: Date;
}
export interface InputEventClockCoordinate {
wallAt: Date;
gameAt: Date;
gameTick: bigint;
clockRevision: bigint;
deadlineGeneration: bigint;
phase: string;
}
/**
* Reads one input-event acceptance coordinate while holding the same schema
* clock-operation fence used by reconciliation. The caller must create the
* input_event in this transaction before releasing the lock.
*/
export const readInputEventClockCoordinate = async (
db: ClockAcceptanceDatabase
): Promise<InputEventClockCoordinate> => {
await acquireGameSchemaAdvisoryXactLock(db, CLOCK_OPERATION_PERSISTENCE_LOCK);
const [wall] = await db.$queryRaw<DbWallRow[]>(GamePrisma.sql`
SELECT timezone('UTC', clock_timestamp()) AS "wallNow"
`);
if (!wall) throw new Error('PostgreSQL did not return its authoritative wall clock.');
const world = await db.worldState.findFirst({
orderBy: { id: 'asc' },
select: {
clockBaseTime: true,
clockTick: true,
clockMode: true,
clockWallAnchor: true,
tickSeconds: true,
clockPhase: true,
clockRevision: true,
deadlineGeneration: true,
},
});
if (!world?.clockBaseTime || world.clockTick === null || !world.clockWallAnchor) {
throw new Error('The authoritative game clock is not initialized.');
}
const tick = Number(world.clockTick);
const revision = Number(world.clockRevision);
const generation = Number(world.deadlineGeneration);
if (!Number.isSafeInteger(tick) || !Number.isSafeInteger(revision) || !Number.isSafeInteger(generation)) {
throw new Error('The authoritative game clock coordinate is outside the safe integer range.');
}
const mode = world.clockMode === 'manual' ? 'manual' : 'realtime';
const phase = world.clockPhase ? parseGameClockPhase(world.clockPhase) : inferClockPhase(mode);
const clock = new GameClock({
baseTime: world.clockBaseTime,
tick,
mode,
wallAnchor: world.clockWallAnchor,
turnSeconds: world.tickSeconds,
phase,
revision,
});
const observedTick = clock.nowTick(wall.wallNow);
return {
wallAt: wall.wallNow,
gameAt: clock.tickToDate(observedTick),
gameTick: BigInt(observedTick),
clockRevision: BigInt(revision),
deadlineGeneration: BigInt(generation),
phase,
};
};
+1 -1
View File
@@ -2,6 +2,6 @@
"formatVersion": 1,
"controllerProtocol": 2,
"gatewaySchemaHead": "20260825000000_add_bulk_release_batches",
"gameSchemaHead": "20260903090000_add_game_clock_reconciliation",
"gameSchemaHead": "20260903103000_add_input_event_clock_processing",
"components": ["gateway-api", "gateway-frontend", "release-controller", "game-api", "game-engine", "game-frontend"]
}