feat: 시계 reconciliation 워커와 명령 경계 완성

This commit is contained in:
2026-09-03 09:40:48 +00:00
parent ae7d55ef47
commit a3e2bf90ae
46 changed files with 3011 additions and 219 deletions
+91 -3
View File
@@ -1,4 +1,7 @@
import { import {
CLOCK_OPERATION_PERSISTENCE_LOCK,
GamePrisma,
acquireGameSchemaAdvisoryXactLock,
createGamePostgresConnector, createGamePostgresConnector,
createRedisConnector, createRedisConnector,
type GamePrismaClient, type GamePrismaClient,
@@ -10,6 +13,7 @@ import { resolveGameApiConfigFromEnv } from '../config.js';
import { createBestEffortResourceCloser } from '../services/bestEffortResourceCloser.js'; import { createBestEffortResourceCloser } from '../services/bestEffortResourceCloser.js';
import { loadCurrentGameTime, type CurrentGameTime } from '../services/gameClock.js'; import { loadCurrentGameTime, type CurrentGameTime } from '../services/gameClock.js';
import { createPollingWorkerControl, waitForWorkerPoll } from '../services/pollingWorkerLifecycle.js'; import { createPollingWorkerControl, waitForWorkerPoll } from '../services/pollingWorkerLifecycle.js';
import { ensureActiveRedisClockFence } from '../services/redisClockFence.js';
import { buildAuctionTimerKeys } from './keys.js'; import { buildAuctionTimerKeys } from './keys.js';
import { resolveAuctionTimerScore, seedAuctionTimers } from './scheduler.js'; import { resolveAuctionTimerScore, seedAuctionTimers } from './scheduler.js';
@@ -24,8 +28,22 @@ interface RedisTimerClient {
zAdd(key: string, values: Array<{ score: number; value: string }>): Promise<number>; zAdd(key: string, values: Array<{ score: number; value: string }>): Promise<number>;
zRem(key: string, values: string | string[]): Promise<number>; zRem(key: string, values: string | string[]): Promise<number>;
zRemRangeByScore(key: string, min: number, max: number): Promise<number>; zRemRangeByScore(key: string, min: number, max: number): Promise<number>;
eval?(script: string, options: { keys: string[]; arguments: string[] }): Promise<unknown>;
} }
const POP_DUE_AUCTIONS_SCRIPT = `
if redis.call('GET', KEYS[2]) ~= ARGV[1]
or redis.call('GET', KEYS[3]) ~= ARGV[2]
or redis.call('GET', KEYS[4]) ~= 'RUNNING' then
return { '__CLOCK_FENCE__' }
end
local ids = redis.call('ZRANGEBYSCORE', KEYS[1], '-inf', ARGV[3], 'LIMIT', 0, ARGV[4])
if #ids > 0 then
redis.call('ZREM', KEYS[1], unpack(ids))
end
return ids
`;
const AUCTION_FINALIZE_RECOVERY_LIMIT = 1; const AUCTION_FINALIZE_RECOVERY_LIMIT = 1;
interface AuctionFinalizeDeadline { interface AuctionFinalizeDeadline {
@@ -115,12 +133,29 @@ const isSuccessfulAuctionFinalizeResult = (result: unknown, auctionId: number):
return resultRecord.type === 'auctionFinalize' && resultRecord.ok === true && resultRecord.auctionId === auctionId; return resultRecord.type === 'auctionFinalize' && resultRecord.ok === true && resultRecord.auctionId === auctionId;
}; };
const popDueAuctionIds = async ( export const popDueAuctionIds = async (
redis: RedisTimerClient, redis: RedisTimerClient,
timerKey: string, timerKey: string,
nowMs: number, nowMs: number,
batchSize: number batchSize: number,
clockFence?: {
activeRevisionKey: string;
deadlineGenerationKey: string;
phaseKey: string;
revision: number;
generation: number;
}
): Promise<string[]> => { ): Promise<string[]> => {
if (clockFence) {
if (!redis.eval) throw new Error('Redis EVAL is required for revision-fenced auction due-pop.');
const result = await redis.eval(POP_DUE_AUCTIONS_SCRIPT, {
keys: [timerKey, clockFence.activeRevisionKey, clockFence.deadlineGenerationKey, clockFence.phaseKey],
arguments: [String(clockFence.revision), String(clockFence.generation), String(nowMs), String(batchSize)],
});
if (!Array.isArray(result)) throw new Error('Auction due-pop returned an invalid Redis result.');
if (result[0] === '__CLOCK_FENCE__') return [];
return result.map(String);
}
const ids = await redis.zRangeByScore(timerKey, 0, nowMs, { LIMIT: { offset: 0, count: batchSize } }); const ids = await redis.zRangeByScore(timerKey, 0, nowMs, { LIMIT: { offset: 0, count: batchSize } });
if (ids.length > 0) { if (ids.length > 0) {
await redis.zRem(timerKey, ids); await redis.zRem(timerKey, ids);
@@ -205,6 +240,8 @@ export const processDueAuctionId = async (options: {
nowMs: number; nowMs: number;
nowTick?: number | null; nowTick?: number | null;
historyNowMs?: number; historyNowMs?: number;
expectedClockRevision?: number;
expectedDeadlineGeneration?: number;
}): Promise<'PENDING' | 'RESCHEDULED' | 'IGNORED'> => { }): Promise<'PENDING' | 'RESCHEDULED' | 'IGNORED'> => {
const { db, redis, timerKey, historyKey, id, nowMs, nowTick = null, historyNowMs = nowMs } = options; const { db, redis, timerKey, historyKey, id, nowMs, nowTick = null, historyNowMs = nowMs } = options;
const auctionId = Number(id); const auctionId = Number(id);
@@ -213,6 +250,29 @@ export const processDueAuctionId = async (options: {
} }
const now = new Date(nowMs); const now = new Date(nowMs);
const outcome = await db.$transaction(async (transaction) => { const outcome = await db.$transaction(async (transaction) => {
if (options.expectedClockRevision !== undefined || options.expectedDeadlineGeneration !== undefined) {
await acquireGameSchemaAdvisoryXactLock(transaction, CLOCK_OPERATION_PERSISTENCE_LOCK);
const [world] = await transaction.$queryRaw<
Array<{ clockPhase: string | null; clockRevision: bigint; deadlineGeneration: bigint }>
>(GamePrisma.sql`
SELECT
clock_phase AS "clockPhase",
clock_revision AS "clockRevision",
deadline_generation AS "deadlineGeneration"
FROM world_state
ORDER BY id ASC
LIMIT 1
FOR UPDATE
`);
if (
!world ||
world.clockPhase !== 'RUNNING' ||
world.clockRevision !== BigInt(options.expectedClockRevision ?? -1) ||
world.deadlineGeneration !== BigInt(options.expectedDeadlineGeneration ?? -1)
) {
return { status: 'RESCHEDULED' as const, clockFenceFailed: true };
}
}
const current = await transaction.auction.findUnique({ const current = await transaction.auction.findUnique({
where: { id: auctionId }, where: { id: auctionId },
select: { status: true, closeAt: true, closeTick: true }, select: { status: true, closeAt: true, closeTick: true },
@@ -259,6 +319,13 @@ export const processDueAuctionId = async (options: {
target: 'ENGINE', target: 'ENGINE',
eventType: nextCommand.type, eventType: nextCommand.type,
payload: { ...nextCommand }, payload: { ...nextCommand },
...(nowTick === null ? {} : { acceptedGameTick: BigInt(nowTick) }),
...(options.expectedClockRevision === undefined
? {}
: { acceptedClockRevision: BigInt(options.expectedClockRevision) }),
...(options.expectedDeadlineGeneration === undefined
? {}
: { acceptedDeadlineGeneration: BigInt(options.expectedDeadlineGeneration) }),
}, },
}); });
return { status: 'PENDING' as const }; return { status: 'PENDING' as const };
@@ -284,6 +351,10 @@ export const processDueAuctionId = async (options: {
return 'PENDING'; return 'PENDING';
} }
if (outcome.status === 'RESCHEDULED') { if (outcome.status === 'RESCHEDULED') {
if ('clockFenceFailed' in outcome) {
await redis.zAdd(timerKey, [{ score: nowTick ?? nowMs, value: id }]);
return 'RESCHEDULED';
}
const gameTime = await loadCurrentGameTime(db, now); const gameTime = await loadCurrentGameTime(db, now);
await redis.zAdd(timerKey, [ await redis.zAdd(timerKey, [
{ {
@@ -324,6 +395,17 @@ export const runAuctionWorker = async (options: AuctionWorkerOptions = {}): Prom
const gameTime = await loadCurrentGameTime(postgres.prisma, new Date(operationalNowMs)); const gameTime = await loadCurrentGameTime(postgres.prisma, new Date(operationalNowMs));
const gameNowMs = gameTime.now.getTime(); const gameNowMs = gameTime.now.getTime();
const dueScore = gameTime.tick ?? gameNowMs; const dueScore = gameTime.tick ?? gameNowMs;
if (gameTime.phase && gameTime.phase !== 'RUNNING') {
await waitForWorkerPoll(control.signal, config.auctionTimerPollMs);
continue;
}
const clockFence = gameTime.phase
? await ensureActiveRedisClockFence(redis.client, config.profileName, gameTime)
: null;
if (gameTime.phase && !clockFence) {
await waitForWorkerPoll(control.signal, config.auctionTimerPollMs);
continue;
}
if (operationalNowMs >= nextResyncAt) { if (operationalNowMs >= nextResyncAt) {
await seedAuctionTimers(postgres.prisma, redis.client, keys); await seedAuctionTimers(postgres.prisma, redis.client, keys);
nextResyncAt = operationalNowMs + config.auctionTimerResyncMs; nextResyncAt = operationalNowMs + config.auctionTimerResyncMs;
@@ -345,7 +427,7 @@ export const runAuctionWorker = async (options: AuctionWorkerOptions = {}): Prom
if (historyTrimBefore > 0) { if (historyTrimBefore > 0) {
await redis.client.zRemRangeByScore(keys.historyKey, 0, historyTrimBefore); await redis.client.zRemRangeByScore(keys.historyKey, 0, historyTrimBefore);
} }
const dueIds = await popDueAuctionIds(redis.client, keys.timerKey, dueScore, 100); const dueIds = await popDueAuctionIds(redis.client, keys.timerKey, dueScore, 100, clockFence ?? undefined);
if (dueIds.length > 0) { if (dueIds.length > 0) {
for (const id of dueIds) { for (const id of dueIds) {
try { try {
@@ -358,6 +440,12 @@ export const runAuctionWorker = async (options: AuctionWorkerOptions = {}): Prom
nowMs: gameNowMs, nowMs: gameNowMs,
nowTick: gameTime.tick, nowTick: gameTime.tick,
historyNowMs: operationalNowMs, historyNowMs: operationalNowMs,
...(clockFence
? {
expectedClockRevision: clockFence.revision,
expectedDeadlineGeneration: clockFence.generation,
}
: {}),
}); });
if (outcome === 'PENDING') { if (outcome === 'PENDING') {
pendingFinalizationIds.add(Number(id)); pendingFinalizationIds.add(Number(id));
+39 -7
View File
@@ -1,6 +1,12 @@
import { randomUUID } from 'node:crypto'; import { randomUUID } from 'node:crypto';
import { acquireGameSchemaAdvisoryXactLock, type DatabaseClient, type GamePrisma } from '@sammo-ts/infra'; import {
acquireGameSchemaAdvisoryXactLock,
readInputEventClockCoordinate,
type DatabaseClient,
type GamePrisma,
type InputEventClockCoordinate,
} from '@sammo-ts/infra';
import type { TurnDaemonTransport } from './transport.js'; import type { TurnDaemonTransport } from './transport.js';
import type { TurnDaemonCommand, TurnDaemonCommandResult, TurnDaemonStatus } from './types.js'; import type { TurnDaemonCommand, TurnDaemonCommandResult, TurnDaemonStatus } from './types.js';
@@ -103,10 +109,10 @@ export class DatabaseTurnDaemonTransport implements TurnDaemonTransport {
try { try {
if (command.type === 'npcPossessGeneral' && this.db.$transaction) { if (command.type === 'npcPossessGeneral' && this.db.$transaction) {
const rejectionReason = await this.db.$transaction(async (transaction) => { const rejectionReason = await this.db.$transaction(async (transaction) => {
const coordinate = await readInputEventClockCoordinate(transaction);
await acquireGameSchemaAdvisoryXactLock(transaction, 'npc-possession:global'); await acquireGameSchemaAdvisoryXactLock(transaction, 'npc-possession:global');
await acquireGameSchemaAdvisoryXactLock(transaction, `npc-possession:user:${command.userId}`); await acquireGameSchemaAdvisoryXactLock(transaction, `npc-possession:user:${command.userId}`);
const acceptedAt = new Date(Math.floor(Date.now() / 1000) * 1000); const acceptedGameAt = coordinate.gameAt;
const acceptedGameAt = (await loadCurrentGameTime(transaction, acceptedAt)).now;
const token = await transaction.npcSelectionToken.findFirst({ const token = await transaction.npcSelectionToken.findFirst({
where: { where: {
ownerUserId: command.userId, ownerUserId: command.userId,
@@ -130,14 +136,21 @@ export class DatabaseTurnDaemonTransport implements TurnDaemonTransport {
...(durableCommand as Extract<TurnDaemonCommand, { type: 'npcPossessGeneral' }>), ...(durableCommand as Extract<TurnDaemonCommand, { type: 'npcPossessGeneral' }>),
acceptedGameAt: acceptedGameAt.toISOString(), acceptedGameAt: acceptedGameAt.toISOString(),
}; };
await this.createInputEvent(transaction, acceptedCommand, requestId, acceptedAt); await this.createInputEvent(transaction, acceptedCommand, requestId, coordinate);
return null; return null;
}); });
if (rejectionReason) { if (rejectionReason) {
throw new RejectedNpcPossessionCommandError('PRECONDITION_FAILED', rejectionReason); throw new RejectedNpcPossessionCommandError('PRECONDITION_FAILED', rejectionReason);
} }
} else { } else {
await this.createInputEvent(this.db, durableCommand, requestId); if (this.db.$transaction) {
await this.db.$transaction(async (transaction) => {
const coordinate = await readInputEventClockCoordinate(transaction);
await this.createInputEvent(transaction, durableCommand, requestId, coordinate);
});
} else {
await this.createInputEvent(this.db, durableCommand, requestId);
}
} }
} catch (error) { } catch (error) {
if (error instanceof RejectedNpcPossessionCommandError) { if (error instanceof RejectedNpcPossessionCommandError) {
@@ -166,8 +179,20 @@ export class DatabaseTurnDaemonTransport implements TurnDaemonTransport {
db: DatabaseClient, db: DatabaseClient,
command: TurnDaemonCommand, command: TurnDaemonCommand,
requestId: string, requestId: string,
createdAt?: Date coordinate?: InputEventClockCoordinate
): Promise<void> { ): Promise<void> {
const gameTime = coordinate ? null : await loadCurrentGameTime(db, new Date());
const commandAcceptedTick = Reflect.get(command, 'acceptedGameTick');
const acceptedGameTick =
typeof commandAcceptedTick === 'number' && Number.isSafeInteger(commandAcceptedTick)
? commandAcceptedTick
: coordinate
? Number(coordinate.gameTick)
: gameTime!.tick;
const acceptedClockRevision = coordinate ? Number(coordinate.clockRevision) : gameTime!.revision;
const acceptedDeadlineGeneration = coordinate
? Number(coordinate.deadlineGeneration)
: gameTime!.deadlineGeneration;
await db.inputEvent.create({ await db.inputEvent.create({
data: { data: {
requestId, requestId,
@@ -175,7 +200,14 @@ export class DatabaseTurnDaemonTransport implements TurnDaemonTransport {
eventType: command.type, eventType: command.type,
payload: asJson(command), payload: asJson(command),
actorUserId: 'userId' in command && typeof command.userId === 'string' ? command.userId : null, actorUserId: 'userId' in command && typeof command.userId === 'string' ? command.userId : null,
...(createdAt ? { createdAt } : {}), ...(acceptedGameTick === null ? {} : { acceptedGameTick: BigInt(acceptedGameTick) }),
...(acceptedClockRevision === null || acceptedClockRevision === undefined
? {}
: { acceptedClockRevision: BigInt(acceptedClockRevision) }),
...(acceptedDeadlineGeneration === null || acceptedDeadlineGeneration === undefined
? {}
: { acceptedDeadlineGeneration: BigInt(acceptedDeadlineGeneration) }),
...(coordinate ? { createdAt: coordinate.wallAt } : {}),
}, },
}); });
} }
+27 -4
View File
@@ -1,6 +1,11 @@
import { createHash } from 'node:crypto'; import { createHash } from 'node:crypto';
import { GamePrisma, type DatabaseClient as InfraDatabaseClient } from '@sammo-ts/infra'; import {
CLOCK_OPERATION_PERSISTENCE_LOCK,
GamePrisma,
acquireGameSchemaAdvisoryXactLock,
type DatabaseClient as InfraDatabaseClient,
} from '@sammo-ts/infra';
import type { DatabaseClient } from './context.js'; import type { DatabaseClient } from './context.js';
@@ -20,6 +25,9 @@ interface LockedInputEvent {
status: 'PENDING' | 'PROCESSING' | 'SUCCEEDED' | 'FAILED'; status: 'PENDING' | 'PROCESSING' | 'SUCCEEDED' | 'FAILED';
result: GamePrisma.JsonValue | null; result: GamePrisma.JsonValue | null;
attempts: number; attempts: number;
acceptedGameTick: bigint | null;
acceptedClockRevision: bigint | null;
acceptedDeadlineGeneration: bigint | null;
} }
type InputEventOutcome<T> = type InputEventOutcome<T> =
@@ -85,6 +93,9 @@ const insertPendingIfAbsent = async (
actor_user_id, actor_user_id,
status, status,
attempts, attempts,
accepted_game_tick,
accepted_clock_revision,
accepted_deadline_generation,
created_at created_at
) )
VALUES ( VALUES (
@@ -95,6 +106,9 @@ const insertPendingIfAbsent = async (
${options.actorUserId}, ${options.actorUserId},
'PENDING'::"InputEventStatus", 'PENDING'::"InputEventStatus",
0, 0,
(SELECT clock_tick FROM world_state ORDER BY id ASC LIMIT 1),
(SELECT clock_revision FROM world_state ORDER BY id ASC LIMIT 1),
(SELECT deadline_generation FROM world_state ORDER BY id ASC LIMIT 1),
CURRENT_TIMESTAMP AT TIME ZONE 'UTC' CURRENT_TIMESTAMP AT TIME ZONE 'UTC'
) )
ON CONFLICT (request_id) DO NOTHING ON CONFLICT (request_id) DO NOTHING
@@ -112,7 +126,10 @@ const lockInputEvent = async (db: DatabaseClient, requestId: string): Promise<Lo
actor_user_id AS "actorUserId", actor_user_id AS "actorUserId",
status, status,
result, result,
attempts attempts,
accepted_game_tick AS "acceptedGameTick",
accepted_clock_revision AS "acceptedClockRevision",
accepted_deadline_generation AS "acceptedDeadlineGeneration"
FROM input_event FROM input_event
WHERE request_id = ${requestId} WHERE request_id = ${requestId}
FOR UPDATE FOR UPDATE
@@ -151,7 +168,8 @@ const isMatchingIdentity = (
const claimInputEvent = async ( const claimInputEvent = async (
db: DatabaseClient, db: DatabaseClient,
requestId: string, requestId: string,
payloadIdentity: ApiInputPayloadIdentity payloadIdentity: ApiInputPayloadIdentity,
row: LockedInputEvent
): Promise<void> => { ): Promise<void> => {
await db.inputEvent.update({ await db.inputEvent.update({
where: { requestId }, where: { requestId },
@@ -164,6 +182,9 @@ const claimInputEvent = async (
lockedBy: null, lockedBy: null,
leaseUntil: null, leaseUntil: null,
processingAt: new Date(), processingAt: new Date(),
processingGameTick: row.acceptedGameTick,
processingClockRevision: row.acceptedClockRevision,
processingDeadlineGeneration: row.acceptedDeadlineGeneration,
completedAt: null, completedAt: null,
}, },
}); });
@@ -184,6 +205,7 @@ const markUnexpectedFailure = async (
try { try {
await db.$transaction(async (transaction) => { await db.$transaction(async (transaction) => {
await acquireGameSchemaAdvisoryXactLock(transaction, CLOCK_OPERATION_PERSISTENCE_LOCK);
await insertPendingIfAbsent(transaction, options); await insertPendingIfAbsent(transaction, options);
const row = await lockInputEvent(transaction, options.requestId); const row = await lockInputEvent(transaction, options.requestId);
const identityMatches = isMatchingIdentity(row, options) || canAdoptLegacyFailedPayload(row, options); const identityMatches = isMatchingIdentity(row, options) || canAdoptLegacyFailedPayload(row, options);
@@ -233,6 +255,7 @@ export const executeInputEvent = async <T>(options: {
let outcome: InputEventOutcome<T>; let outcome: InputEventOutcome<T>;
try { try {
outcome = await db.$transaction(async (transaction) => { outcome = await db.$transaction(async (transaction) => {
await acquireGameSchemaAdvisoryXactLock(transaction, CLOCK_OPERATION_PERSISTENCE_LOCK);
await insertPendingIfAbsent(transaction, { requestId, eventType, actorUserId, payloadIdentity }); await insertPendingIfAbsent(transaction, { requestId, eventType, actorUserId, payloadIdentity });
const row = await lockInputEvent(transaction, requestId); const row = await lockInputEvent(transaction, requestId);
const identityMatches = isMatchingIdentity(row, { eventType, actorUserId, payloadIdentity }); const identityMatches = isMatchingIdentity(row, { eventType, actorUserId, payloadIdentity });
@@ -251,7 +274,7 @@ export const executeInputEvent = async <T>(options: {
throw new DuplicateInputEventError(requestId); throw new DuplicateInputEventError(requestId);
} }
await claimInputEvent(transaction, requestId, payloadIdentity); await claimInputEvent(transaction, requestId, payloadIdentity, row);
const savepointDb = transaction as SavepointDatabaseClient; const savepointDb = transaction as SavepointDatabaseClient;
await savepointDb.$executeRawUnsafe(`SAVEPOINT ${BUSINESS_SAVEPOINT}`); await savepointDb.$executeRawUnsafe(`SAVEPOINT ${BUSINESS_SAVEPOINT}`);
businessStarted = true; businessStarted = true;
+42 -11
View File
@@ -5,12 +5,14 @@ import { asRecord } from '@sammo-ts/common';
import type { TournamentType } from '@sammo-ts/logic'; import type { TournamentType } from '@sammo-ts/logic';
import type { TournamentState } from '../../tournament/types.js'; import type { TournamentState } from '../../tournament/types.js';
import { TournamentStore } from '../../tournament/store.js'; import { TournamentStore, type TournamentClockContext } from '../../tournament/store.js';
import { buildTournamentKeys } from '../../tournament/keys.js'; import { buildTournamentKeys } from '../../tournament/keys.js';
import { assignManualApplicantGroup } from '../../tournament/workerHelpers.js'; import { assignManualApplicantGroup } from '../../tournament/workerHelpers.js';
import { accessAuthedProcedure, authedProcedure, router } from '../../trpc.js'; import { accessAuthedProcedure, authedProcedure, router } from '../../trpc.js';
import { getMyGeneral } from '../shared/general.js'; import { getMyGeneral } from '../shared/general.js';
import { loadCurrentGameTime } from '../../services/gameClock.js'; import { loadCurrentGameTime } from '../../services/gameClock.js';
import { ensureActiveRedisClockFence } from '../../services/redisClockFence.js';
import { loadClockAdminStatus } from '../../services/clockReadiness.js';
const hasAdminRole = (roles: string[], profileName: string): boolean => { const hasAdminRole = (roles: string[], profileName: string): boolean => {
if (roles.includes('superuser') || roles.includes('admin') || roles.includes('admin.superuser')) { if (roles.includes('superuser') || roles.includes('admin') || roles.includes('admin.superuser')) {
@@ -44,6 +46,32 @@ const adminProcedure = authedProcedure.use(({ ctx, next }) => {
return next(); return next();
}); });
const withTournamentClockMutation = async <T>(
ctx: {
db: Parameters<typeof loadCurrentGameTime>[0];
redis: Parameters<typeof ensureActiveRedisClockFence>[0];
profile: { name: string };
},
store: TournamentStore,
operation: () => Promise<T>
): Promise<T> => {
const gameTime = await loadCurrentGameTime(ctx.db);
const fence = await ensureActiveRedisClockFence(ctx.redis, ctx.profile.name, gameTime);
if (!fence) {
throw new TRPCError({
code: 'PRECONDITION_FAILED',
message: 'Clock reconciliation is incomplete; tournament mutation is disabled.',
});
}
const clockContext: TournamentClockContext = {
phase: 'RUNNING',
revision: fence.revision,
deadlineGeneration: fence.generation,
dateToTick: gameTime.dateToTick,
};
return store.withClockContext(clockContext, () => store.withMutationLock(operation));
};
const zTournamentState = z.object({ const zTournamentState = z.object({
stage: z.number().int().min(0), stage: z.number().int().min(0),
phase: z.number().int().min(0), phase: z.number().int().min(0),
@@ -143,7 +171,10 @@ export const tournamentRouter = router({
const store = new TournamentStore(ctx.redis, buildTournamentKeys(ctx.profile.name)); const store = new TournamentStore(ctx.redis, buildTournamentKeys(ctx.profile.name));
return store.getState(); return store.getState();
}), }),
getAdminStatus: adminProcedure.query(async () => ({ ok: true })), getAdminStatus: adminProcedure.query(async ({ ctx }) => ({
ok: true,
clock: await loadClockAdminStatus(ctx.db),
})),
getSnapshot: accessAuthedProcedure.query(async ({ ctx }) => { getSnapshot: accessAuthedProcedure.query(async ({ ctx }) => {
await getMyGeneral(ctx); await getMyGeneral(ctx);
const store = new TournamentStore(ctx.redis, buildTournamentKeys(ctx.profile.name)); const store = new TournamentStore(ctx.redis, buildTournamentKeys(ctx.profile.name));
@@ -271,7 +302,7 @@ export const tournamentRouter = router({
}), }),
setState: adminProcedure.input(zTournamentState).mutation(async ({ ctx, input }) => { setState: adminProcedure.input(zTournamentState).mutation(async ({ ctx, input }) => {
const store = new TournamentStore(ctx.redis, buildTournamentKeys(ctx.profile.name)); const store = new TournamentStore(ctx.redis, buildTournamentKeys(ctx.profile.name));
return store.withMutationLock(async () => { return withTournamentClockMutation(ctx, store, async () => {
await store.setState({ await store.setState({
...input, ...input,
type: input.type as TournamentType, type: input.type as TournamentType,
@@ -281,7 +312,7 @@ export const tournamentRouter = router({
}), }),
patchState: adminProcedure.input(zTournamentState.partial()).mutation(async ({ ctx, input }) => { patchState: adminProcedure.input(zTournamentState.partial()).mutation(async ({ ctx, input }) => {
const store = new TournamentStore(ctx.redis, buildTournamentKeys(ctx.profile.name)); const store = new TournamentStore(ctx.redis, buildTournamentKeys(ctx.profile.name));
return store.withMutationLock(async () => { return withTournamentClockMutation(ctx, store, async () => {
const current = await store.getState(); const current = await store.getState();
if (!current) { if (!current) {
throw new TRPCError({ code: 'NOT_FOUND', message: 'Tournament state not found.' }); throw new TRPCError({ code: 'NOT_FOUND', message: 'Tournament state not found.' });
@@ -297,21 +328,21 @@ export const tournamentRouter = router({
}), }),
setParticipants: adminProcedure.input(z.array(zParticipant)).mutation(async ({ ctx, input }) => { setParticipants: adminProcedure.input(z.array(zParticipant)).mutation(async ({ ctx, input }) => {
const store = new TournamentStore(ctx.redis, buildTournamentKeys(ctx.profile.name)); const store = new TournamentStore(ctx.redis, buildTournamentKeys(ctx.profile.name));
return store.withMutationLock(async () => { return withTournamentClockMutation(ctx, store, async () => {
await store.setParticipants(input); await store.setParticipants(input);
return { ok: true, count: input.length }; return { ok: true, count: input.length };
}); });
}), }),
setMatches: adminProcedure.input(z.array(zMatch)).mutation(async ({ ctx, input }) => { setMatches: adminProcedure.input(z.array(zMatch)).mutation(async ({ ctx, input }) => {
const store = new TournamentStore(ctx.redis, buildTournamentKeys(ctx.profile.name)); const store = new TournamentStore(ctx.redis, buildTournamentKeys(ctx.profile.name));
return store.withMutationLock(async () => { return withTournamentClockMutation(ctx, store, async () => {
await store.setMatches(input); await store.setMatches(input);
return { ok: true, count: input.length }; return { ok: true, count: input.length };
}); });
}), }),
setBettingEntries: adminProcedure.input(z.array(zBetEntry)).mutation(async ({ ctx, input }) => { setBettingEntries: adminProcedure.input(z.array(zBetEntry)).mutation(async ({ ctx, input }) => {
const store = new TournamentStore(ctx.redis, buildTournamentKeys(ctx.profile.name)); const store = new TournamentStore(ctx.redis, buildTournamentKeys(ctx.profile.name));
return store.withMutationLock(async () => { return withTournamentClockMutation(ctx, store, async () => {
await store.setBettingEntries(input); await store.setBettingEntries(input);
return { ok: true, count: input.length }; return { ok: true, count: input.length };
}); });
@@ -347,7 +378,7 @@ export const tournamentRouter = router({
}; };
}); });
return store.withMutationLock(async () => { return withTournamentClockMutation(ctx, store, async () => {
await store.setParticipants(participants); await store.setParticipants(participants);
return { ok: true, count: participants.length }; return { ok: true, count: participants.length };
}); });
@@ -394,7 +425,7 @@ export const tournamentRouter = router({
join: authedProcedure.mutation(async ({ ctx }) => { join: authedProcedure.mutation(async ({ ctx }) => {
const general = await getMyGeneral(ctx); const general = await getMyGeneral(ctx);
const store = new TournamentStore(ctx.redis, buildTournamentKeys(ctx.profile.name)); const store = new TournamentStore(ctx.redis, buildTournamentKeys(ctx.profile.name));
return store.withMutationLock(async () => { return withTournamentClockMutation(ctx, store, async () => {
const state = await store.getState(); const state = await store.getState();
if (!state || state.stage !== 1 || state.participantsLockedAt) { if (!state || state.stage !== 1 || state.participantsLockedAt) {
throw new TRPCError({ code: 'BAD_REQUEST', message: '참가 신청 기간이 아닙니다.' }); throw new TRPCError({ code: 'BAD_REQUEST', message: '참가 신청 기간이 아닙니다.' });
@@ -459,7 +490,7 @@ export const tournamentRouter = router({
}), }),
cancel: adminProcedure.mutation(async ({ ctx }) => { cancel: adminProcedure.mutation(async ({ ctx }) => {
const store = new TournamentStore(ctx.redis, buildTournamentKeys(ctx.profile.name)); const store = new TournamentStore(ctx.redis, buildTournamentKeys(ctx.profile.name));
return store.withMutationLock(async () => { return withTournamentClockMutation(ctx, store, async () => {
const state = await store.getState(); const state = await store.getState();
if (!state) { if (!state) {
throw new TRPCError({ code: 'NOT_FOUND', message: 'Tournament state not found.' }); throw new TRPCError({ code: 'NOT_FOUND', message: 'Tournament state not found.' });
@@ -523,7 +554,7 @@ export const tournamentRouter = router({
.mutation(async ({ ctx, input }) => { .mutation(async ({ ctx, input }) => {
const general = await getMyGeneral(ctx); const general = await getMyGeneral(ctx);
const store = new TournamentStore(ctx.redis, buildTournamentKeys(ctx.profile.name)); const store = new TournamentStore(ctx.redis, buildTournamentKeys(ctx.profile.name));
return store.withMutationLock(async () => { return withTournamentClockMutation(ctx, store, async () => {
const state = await store.getState(); const state = await store.getState();
if (!state || state.stage !== 6) { if (!state || state.stage !== 6) {
throw new TRPCError({ code: 'BAD_REQUEST', message: '베팅 기간이 아닙니다.' }); throw new TRPCError({ code: 'BAD_REQUEST', message: '베팅 기간이 아닙니다.' });
+12 -6
View File
@@ -50,6 +50,7 @@ import { ReadModelOutboxWorker } from './realtime/outboxWorker.js';
import { DeferredGeneralAccessWorker } from './services/deferredGeneralAccess.js'; import { DeferredGeneralAccessWorker } from './services/deferredGeneralAccess.js';
import { WebPushOutboxWorker } from './services/webPushOutboxWorker.js'; import { WebPushOutboxWorker } from './services/webPushOutboxWorker.js';
import { scopeHttpIdempotencyKey } from './requestId.js'; import { scopeHttpIdempotencyKey } from './requestId.js';
import { loadClockReadiness } from './services/clockReadiness.js';
const extractBearerToken = (value: string | string[] | undefined): string | null => { const extractBearerToken = (value: string | string[] | undefined): string | null => {
if (!value) { if (!value) {
@@ -420,12 +421,17 @@ export const createGameApiServer = async () => {
request.raw.on('aborted', close); request.raw.on('aborted', close);
}); });
app.get('/healthz', async () => ({ app.get('/healthz', async (_request, reply) => {
ok: true, const clock = await loadClockReadiness(postgres.prisma);
profile: config.profileName, if (!clock.reconciliationComplete) reply.code(503);
postgresPool: postgres.getPoolStats(), return {
accountIconReconciliation: accountIconResetReconciler.getHealth(), ok: clock.reconciliationComplete,
})); profile: config.profileName,
postgresPool: postgres.getPoolStats(),
accountIconReconciliation: accountIconResetReconciler.getHealth(),
clock,
};
});
try { try {
await realtimeHub.start(); await realtimeHub.start();
@@ -0,0 +1,92 @@
import { parseGameClockPhase } from '@sammo-ts/common';
import type { DatabaseClient } from '../context.js';
const safeInteger = (value: bigint, label: string): number => {
const result = Number(value);
if (!Number.isSafeInteger(result)) throw new Error(`${label} is outside the safe integer range.`);
return result;
};
export const loadClockReadiness = async (db: DatabaseClient) => {
if (!db.clockProjectionOutbox) {
return {
reconciliationComplete: false,
gameplayEnabled: false,
phase: null,
revision: null,
deadlineGeneration: null,
incompleteOutboxCount: null,
};
}
const [world, incompleteOutboxCount] = await Promise.all([
db.worldState.findFirst({
orderBy: { id: 'asc' },
select: { clockPhase: true, clockRevision: true, deadlineGeneration: true },
}),
db.clockProjectionOutbox.count({ where: { status: { not: 'APPLIED' } } }),
]);
if (!world) {
return {
reconciliationComplete: false,
gameplayEnabled: false,
phase: null,
revision: null,
deadlineGeneration: null,
incompleteOutboxCount,
};
}
const phase = parseGameClockPhase(world.clockPhase);
return {
reconciliationComplete: phase !== 'RECONCILING' && incompleteOutboxCount === 0,
gameplayEnabled: phase === 'RUNNING' || phase === 'MANUAL',
phase,
revision: safeInteger(world.clockRevision, 'clock revision'),
deadlineGeneration: safeInteger(world.deadlineGeneration, 'deadline generation'),
incompleteOutboxCount,
};
};
export const loadClockAdminStatus = async (db: DatabaseClient) => {
const readiness = await loadClockReadiness(db);
if (!db.clockSuspension) {
return { ...readiness, latestReconciliation: null };
}
const latest = await db.clockSuspension.findFirst({
orderBy: { createdAt: 'desc' },
include: {
participants: { orderBy: { participantKey: 'asc' } },
projectionOutbox: { orderBy: { id: 'asc' } },
},
});
if (!latest) return { ...readiness, latestReconciliation: null };
return {
...readiness,
latestReconciliation: {
id: latest.id,
source: latest.source,
policy: latest.policy,
status: latest.status,
sourceRevision: safeInteger(latest.sourceRevision, 'source revision'),
targetRevision: safeInteger(latest.targetRevision, 'target revision'),
cutTick: safeInteger(latest.cutTick, 'cut tick'),
alignedTick: latest.alignedTick === null ? null : safeInteger(latest.alignedTick, 'aligned tick'),
participantChecksumBefore: latest.participantChecksumBefore,
participantChecksumAfter: latest.participantChecksumAfter,
participants: latest.participants.map((participant) => ({
key: participant.participantKey,
policy: participant.policy,
beforeChecksum: participant.beforeChecksum,
afterChecksum: participant.afterChecksum,
affectedCount: participant.affectedCount,
})),
outbox: latest.projectionOutbox.map((entry) => ({
id: entry.id.toString(),
targetRevision: safeInteger(entry.targetRevision, 'outbox target revision'),
status: entry.status,
attempts: entry.attempts,
lastError: entry.lastError,
})),
},
};
};
@@ -0,0 +1,55 @@
import type { CurrentGameTime } from './gameClock.js';
interface ClockFenceRedis {
eval(script: string, options: { keys: string[]; arguments: string[] }): Promise<unknown>;
}
const BOOTSTRAP_CLOCK_FENCE_SCRIPT = `
local revision = redis.call('GET', KEYS[1])
local generation = redis.call('GET', KEYS[2])
local phase = redis.call('GET', KEYS[3])
if not revision and not generation and not phase then
redis.call('SET', KEYS[1], ARGV[1])
redis.call('SET', KEYS[2], ARGV[2])
redis.call('SET', KEYS[3], ARGV[3])
return 1
end
if revision == ARGV[1] and generation == ARGV[2] and phase == ARGV[3] then
return 2
end
return 0
`;
export interface ActiveRedisClockFence {
activeRevisionKey: string;
deadlineGenerationKey: string;
phaseKey: string;
revision: number;
generation: number;
}
export const ensureActiveRedisClockFence = async (
redis: ClockFenceRedis,
profileName: string,
gameTime: CurrentGameTime
): Promise<ActiveRedisClockFence | null> => {
if (
gameTime.phase !== 'RUNNING' ||
!Number.isSafeInteger(gameTime.revision) ||
!Number.isSafeInteger(gameTime.deadlineGeneration)
) {
return null;
}
const fence: ActiveRedisClockFence = {
activeRevisionKey: `sammo:${profileName}:clock:active-revision`,
deadlineGenerationKey: `sammo:${profileName}:clock:deadline-generation`,
phaseKey: `sammo:${profileName}:clock:phase`,
revision: gameTime.revision!,
generation: gameTime.deadlineGeneration!,
};
const result = await redis.eval(BOOTSTRAP_CLOCK_FENCE_SCRIPT, {
keys: [fence.activeRevisionKey, fence.deadlineGenerationKey, fence.phaseKey],
arguments: [String(fence.revision), String(fence.generation), 'RUNNING'],
});
return Number(result) === 1 || Number(result) === 2 ? fence : null;
};
+6
View File
@@ -8,6 +8,9 @@ export interface TournamentKeys {
sourceRevisionKey: string; sourceRevisionKey: string;
sourceRevisionChannel: string; sourceRevisionChannel: string;
realtimeEventChannel: string; realtimeEventChannel: string;
activeClockRevisionKey: string;
deadlineGenerationKey: string;
clockPhaseKey: string;
} }
export const buildTournamentKeys = (profileName: string): TournamentKeys => ({ export const buildTournamentKeys = (profileName: string): TournamentKeys => ({
@@ -18,4 +21,7 @@ export const buildTournamentKeys = (profileName: string): TournamentKeys => ({
sourceRevisionKey: `sammo:${profileName}:tournament:source-revision`, sourceRevisionKey: `sammo:${profileName}:tournament:source-revision`,
sourceRevisionChannel: `sammo:${profileName}:tournament:source-changed`, sourceRevisionChannel: `sammo:${profileName}:tournament:source-changed`,
realtimeEventChannel: buildGameEventChannel(profileName), realtimeEventChannel: buildGameEventChannel(profileName),
activeClockRevisionKey: `sammo:${profileName}:clock:active-revision`,
deadlineGenerationKey: `sammo:${profileName}:clock:deadline-generation`,
clockPhaseKey: `sammo:${profileName}:clock:phase`,
}); });
+61 -3
View File
@@ -1,5 +1,5 @@
import { randomUUID } from 'node:crypto'; import { randomUUID } from 'node:crypto';
import { parseTournamentSourceRevision, writeTournamentProjection } from '@sammo-ts/common'; import { parseTournamentSourceRevision, writeTournamentProjection, type TournamentClockFence } from '@sammo-ts/common';
import { z } from 'zod'; import { z } from 'zod';
import type { TournamentKeys } from './keys.js'; import type { TournamentKeys } from './keys.js';
@@ -37,8 +37,12 @@ const zTournamentState = z
openMonth: z.number().int(), openMonth: z.number().int(),
termSeconds: z.number(), termSeconds: z.number(),
nextAt: z.string(), nextAt: z.string(),
nextTick: z.number().int().safe().optional(),
clockRevision: z.number().int().positive().safe().optional(),
deadlineGeneration: z.number().int().positive().safe().optional(),
bettingId: z.number().int().optional(), bettingId: z.number().int().optional(),
bettingCloseAt: z.string().optional(), bettingCloseAt: z.string().optional(),
bettingCloseTick: z.number().int().safe().optional(),
winnerId: z.number().int().optional(), winnerId: z.number().int().optional(),
bettingSettled: z.boolean().optional(), bettingSettled: z.boolean().optional(),
rewardSettled: z.boolean().optional(), rewardSettled: z.boolean().optional(),
@@ -131,12 +135,62 @@ const parseProjection = <T>(raw: string | null, key: string, schema: z.ZodType<T
return parsed.data; return parsed.data;
}; };
export interface TournamentClockContext {
phase: 'RUNNING';
revision: number;
deadlineGeneration: number;
dateToTick(date: Date): number | null;
}
const parseDeadlineTick = (value: string, context: TournamentClockContext, field: string): number => {
const date = new Date(value);
if (Number.isNaN(date.getTime())) {
throw new Error(`Tournament ${field} is not a valid instant.`);
}
const tick = context.dateToTick(date);
if (tick === null || !Number.isSafeInteger(tick)) {
throw new Error(`Tournament ${field} cannot be represented in the active game clock.`);
}
return tick;
};
export const stampTournamentClock = (state: TournamentState, context: TournamentClockContext): TournamentState => ({
...state,
nextTick: parseDeadlineTick(state.nextAt, context, 'nextAt'),
clockRevision: context.revision,
deadlineGeneration: context.deadlineGeneration,
...(state.bettingCloseAt
? { bettingCloseTick: parseDeadlineTick(state.bettingCloseAt, context, 'bettingCloseAt') }
: { bettingCloseTick: undefined }),
});
const toClockFence = (keys: TournamentKeys, context: TournamentClockContext): TournamentClockFence => ({
activeRevisionKey: keys.activeClockRevisionKey,
deadlineGenerationKey: keys.deadlineGenerationKey,
phaseKey: keys.clockPhaseKey,
revision: context.revision,
deadlineGeneration: context.deadlineGeneration,
phase: context.phase,
});
export class TournamentStore { export class TournamentStore {
private clockContext: TournamentClockContext | null = null;
constructor( constructor(
private readonly redis: RedisClientLike, private readonly redis: RedisClientLike,
private readonly keys: TournamentKeys private readonly keys: TournamentKeys
) {} ) {}
async withClockContext<T>(context: TournamentClockContext, operation: () => Promise<T>): Promise<T> {
const previous = this.clockContext;
this.clockContext = context;
try {
return await operation();
} finally {
this.clockContext = previous;
}
}
async withMutationLock<T>(operation: () => Promise<T>, timeoutMs = 2_000): Promise<T> { async withMutationLock<T>(operation: () => Promise<T>, timeoutMs = 2_000): Promise<T> {
if (!this.redis.del) { if (!this.redis.del) {
return operation(); return operation();
@@ -174,11 +228,15 @@ export class TournamentStore {
} }
private async writeWithSourceRevision(key: string, value: unknown): Promise<string> { private async writeWithSourceRevision(key: string, value: unknown): Promise<string> {
return writeTournamentProjection(this.redis, this.keys, [{ key, value }]); const fence = this.clockContext ? toClockFence(this.keys, this.clockContext) : undefined;
return writeTournamentProjection(this.redis, this.keys, [{ key, value }], fence);
} }
async setState(state: TournamentState): Promise<string> { async setState(state: TournamentState): Promise<string> {
return this.writeWithSourceRevision(this.keys.stateKey, state); return this.writeWithSourceRevision(
this.keys.stateKey,
this.clockContext ? stampTournamentClock(state, this.clockContext) : state
);
} }
async getParticipants(): Promise<TournamentParticipantEntry[]> { async getParticipants(): Promise<TournamentParticipantEntry[]> {
+4
View File
@@ -9,8 +9,12 @@ export interface TournamentState {
openMonth: number; openMonth: number;
termSeconds: number; termSeconds: number;
nextAt: string; nextAt: string;
nextTick?: number;
clockRevision?: number;
deadlineGeneration?: number;
bettingId?: number; bettingId?: number;
bettingCloseAt?: string; bettingCloseAt?: string;
bettingCloseTick?: number;
winnerId?: number; winnerId?: number;
bettingSettled?: boolean; bettingSettled?: boolean;
rewardSettled?: boolean; rewardSettled?: boolean;
+106 -34
View File
@@ -13,9 +13,10 @@ import { DatabaseTurnDaemonTransport } from '../daemon/databaseTransport.js';
import { createBestEffortResourceCloser } from '../services/bestEffortResourceCloser.js'; import { createBestEffortResourceCloser } from '../services/bestEffortResourceCloser.js';
import { loadCurrentGameTime } from '../services/gameClock.js'; import { loadCurrentGameTime } from '../services/gameClock.js';
import { createPollingWorkerControl, waitForWorkerPoll } from '../services/pollingWorkerLifecycle.js'; import { createPollingWorkerControl, waitForWorkerPoll } from '../services/pollingWorkerLifecycle.js';
import { ensureActiveRedisClockFence } from '../services/redisClockFence.js';
import type { TurnDaemonTransport } from '../daemon/transport.js'; import type { TurnDaemonTransport } from '../daemon/transport.js';
import { buildTournamentKeys } from './keys.js'; import { buildTournamentKeys } from './keys.js';
import { TournamentStore } from './store.js'; import { TournamentStore, stampTournamentClock, type TournamentClockContext } from './store.js';
import type { TournamentMatchEntry, TournamentState } from './types.js'; import type { TournamentMatchEntry, TournamentState } from './types.js';
import { import {
applyGroupMatch, applyGroupMatch,
@@ -595,41 +596,63 @@ export const processTournamentTick = async (options: {
prisma: GamePrismaClient; prisma: GamePrismaClient;
daemonTransport: TurnDaemonTransport; daemonTransport: TurnDaemonTransport;
now?: () => number; now?: () => number;
clockContext?: TournamentClockContext;
}): Promise<TournamentState | null> => { }): Promise<TournamentState | null> => {
const { store, prisma, daemonTransport } = options; const { store, prisma, daemonTransport } = options;
const now = options.now ?? Date.now; const now = options.now ?? Date.now;
let processedState: TournamentState | null = null; let processedState: TournamentState | null = null;
await store.withMutationLock(async () => { const processWithLock = async (): Promise<void> =>
const state = await store.getState(); store.withMutationLock(async () => {
if (!state || (!state.auto && !needsSettlement(state))) { const state = await store.getState();
return; if (!state || (!state.auto && !needsSettlement(state))) {
} return;
const nextAt = new Date(state.nextAt).getTime(); }
if (state.auto && Number.isFinite(nextAt) && nextAt > now()) { if (
return; options.clockContext &&
} (state.clockRevision !== options.clockContext.revision ||
state.deadlineGeneration !== options.clockContext.deadlineGeneration)
) {
throw new Error('Tournament state does not match the active clock revision.');
}
const nextAt = new Date(state.nextAt).getTime();
const notDue = options.clockContext
? !Number.isSafeInteger(state.nextTick) ||
state.nextTick! > options.clockContext.dateToTick(new Date(now()))!
: Number.isFinite(nextAt) && nextAt > now();
if (state.auto && notDue) {
if (options.clockContext && !Number.isSafeInteger(state.nextTick)) {
throw new Error('Active tournament nextAt lacks the authoritative nextTick dual-write.');
}
return;
}
if (needsSettlement(state)) { if (needsSettlement(state)) {
processedState = (await settleTournamentOutcome({ store, daemonTransport, state })) ?? state; processedState = (await settleTournamentOutcome({ store, daemonTransport, state })) ?? state;
return; return;
} }
const worldState = await prisma.worldState.findFirst(); const worldState = await prisma.worldState.findFirst();
const baseSeed = (worldState?.meta as Record<string, unknown> | null)?.hiddenSeed ?? 'tournament'; const baseSeed = (worldState?.meta as Record<string, unknown> | null)?.hiddenSeed ?? 'tournament';
let nextState = state; let nextState = state;
if (isBattleStage(state.stage)) { if (isBattleStage(state.stage)) {
nextState = await applyBattle(store, state, String(baseSeed), daemonTransport); nextState = await applyBattle(store, state, String(baseSeed), daemonTransport);
} else if (isPreBattleStage(state.stage)) { } else if (isPreBattleStage(state.stage)) {
nextState = await applyPreBattleStage(store, prisma, state, String(baseSeed), daemonTransport, now); nextState = await applyPreBattleStage(store, prisma, state, String(baseSeed), daemonTransport, now);
} }
processedState = processedState =
(await settleTournamentOutcome({ (await settleTournamentOutcome({
store, store,
daemonTransport, daemonTransport,
state: nextState, state: nextState,
})) ?? nextState; })) ?? nextState;
}); });
if (options.clockContext) {
await store.withClockContext(options.clockContext, processWithLock);
} else {
await processWithLock();
}
return processedState; return processedState;
}; };
@@ -656,16 +679,60 @@ export const runTournamentWorker = async (options: TournamentWorkerOptions = {})
try { try {
while (!control.signal.aborted) { while (!control.signal.aborted) {
const state = await store.getState(); let state = await store.getState();
if (!state || (!state.auto && !needsSettlement(state))) { if (!state || (!state.auto && !needsSettlement(state))) {
await waitForWorkerPoll(control.signal, config.tournamentPollMs); await waitForWorkerPoll(control.signal, config.tournamentPollMs);
continue; continue;
} }
const gameTime = await loadCurrentGameTime(postgres.prisma);
if (gameTime.phase && gameTime.phase !== 'RUNNING') {
await waitForWorkerPoll(control.signal, config.tournamentPollMs);
continue;
}
const clockFence = gameTime.phase
? await ensureActiveRedisClockFence(redis.client, config.profileName, gameTime)
: null;
if (gameTime.phase && !clockFence) {
await waitForWorkerPoll(control.signal, config.tournamentPollMs);
continue;
}
const clockContext: TournamentClockContext | undefined = clockFence
? {
phase: 'RUNNING',
revision: clockFence.revision,
deadlineGeneration: clockFence.generation,
dateToTick: gameTime.dateToTick,
}
: undefined;
if (
clockContext &&
(!Number.isSafeInteger(state.nextTick) ||
state.clockRevision === undefined ||
state.deadlineGeneration === undefined)
) {
state = stampTournamentClock(state, clockContext);
await store.withClockContext(clockContext, () => store.setState(state!));
}
if (
clockContext &&
(state.clockRevision !== clockContext.revision ||
state.deadlineGeneration !== clockContext.deadlineGeneration)
) {
await waitForWorkerPoll(control.signal, config.tournamentPollMs);
continue;
}
const nextAt = new Date(state.nextAt).getTime(); const nextAt = new Date(state.nextAt).getTime();
const gameNow = (await loadCurrentGameTime(postgres.prisma)).now.getTime(); const gameNow = gameTime.now.getTime();
if (state.auto && Number.isFinite(nextAt) && nextAt > gameNow) { const notDue = clockContext
await waitForWorkerPoll(control.signal, Math.min(config.tournamentPollMs, nextAt - gameNow)); ? state.nextTick! > gameTime.tick!
: Number.isFinite(nextAt) && nextAt > gameNow;
if (state.auto && notDue) {
await waitForWorkerPoll(
control.signal,
Math.min(config.tournamentPollMs, Math.max(1, nextAt - gameNow))
);
continue; continue;
} }
@@ -675,6 +742,7 @@ export const runTournamentWorker = async (options: TournamentWorkerOptions = {})
prisma: postgres.prisma, prisma: postgres.prisma,
daemonTransport, daemonTransport,
now: () => gameNow, now: () => gameNow,
clockContext,
}); });
} catch (error) { } catch (error) {
const message = error instanceof Error ? error.message : 'Unknown error'; const message = error instanceof Error ? error.message : 'Unknown error';
@@ -700,7 +768,11 @@ export const runTournamentWorker = async (options: TournamentWorkerOptions = {})
lastError: message, lastError: message,
lastErrorAt: failedAt, lastErrorAt: failedAt,
}; };
await store.setState(nextState); if (clockContext) {
await store.withClockContext(clockContext, () => store.setState(nextState));
} else {
await store.setState(nextState);
}
} }
await waitForWorkerPoll(control.signal, config.tournamentPollMs); await waitForWorkerPoll(control.signal, config.tournamentPollMs);
+34 -1
View File
@@ -2,7 +2,7 @@ import { describe, expect, it, vi } from 'vitest';
import type { GamePrismaClient } from '@sammo-ts/infra'; import type { GamePrismaClient } from '@sammo-ts/infra';
import { processDueAuctionId, reconcilePendingAuctionTimers } from '../src/auction/worker.js'; import { popDueAuctionIds, processDueAuctionId, reconcilePendingAuctionTimers } from '../src/auction/worker.js';
import { resolveAuctionSeedScore } from '../src/auction/scheduler.js'; import { resolveAuctionSeedScore } from '../src/auction/scheduler.js';
const buildRedis = () => ({ const buildRedis = () => ({
@@ -55,6 +55,38 @@ const buildDb = (options: {
}; };
describe('auction worker clock-shift race', () => { describe('auction worker clock-shift race', () => {
it('uses one Redis script for revision, generation, phase, due-read, and removal', async () => {
const redis = { ...buildRedis(), eval: vi.fn(async () => ['7', '9']) };
await expect(
popDueAuctionIds(redis, 'auction-timer', 123_456, 100, {
activeRevisionKey: 'clock-revision',
deadlineGenerationKey: 'deadline-generation',
phaseKey: 'clock-phase',
revision: 4,
generation: 8,
})
).resolves.toEqual(['7', '9']);
expect(redis.eval).toHaveBeenCalledWith(expect.stringContaining('ZRANGEBYSCORE'), {
keys: ['auction-timer', 'clock-revision', 'deadline-generation', 'clock-phase'],
arguments: ['4', '8', '123456', '100'],
});
expect(redis.zRangeByScore).not.toHaveBeenCalled();
expect(redis.zRem).not.toHaveBeenCalled();
});
it('returns no due member when the atomic Redis clock fence rejects the pop', async () => {
const redis = { ...buildRedis(), eval: vi.fn(async () => ['__CLOCK_FENCE__']) };
await expect(
popDueAuctionIds(redis, 'auction-timer', 123_456, 100, {
activeRevisionKey: 'clock-revision',
deadlineGenerationKey: 'deadline-generation',
phaseKey: 'clock-phase',
revision: 4,
generation: 8,
})
).resolves.toEqual([]);
});
it('seeds OPEN at its deadline but retries FINALIZING at the current logical tick', () => { it('seeds OPEN at its deadline but retries FINALIZING at the current logical tick', () => {
const now = new Date('2026-07-30T12:00:00.000Z'); const now = new Date('2026-07-30T12:00:00.000Z');
const time = { const time = {
@@ -292,6 +324,7 @@ describe('auction worker clock-shift race', () => {
requestId, requestId,
target: 'ENGINE', target: 'ENGINE',
eventType: 'auctionFinalize', eventType: 'auctionFinalize',
acceptedGameTick: 72_000_000n,
payload: { payload: {
type: 'auctionFinalize', type: 'auctionFinalize',
requestId, requestId,
+87
View File
@@ -0,0 +1,87 @@
import { describe, expect, it, vi } from 'vitest';
import type { DatabaseClient } from '../src/context.js';
import { loadClockAdminStatus, loadClockReadiness } from '../src/services/clockReadiness.js';
describe('clock reconciliation readiness', () => {
it('fails closed when the reconciliation schema is not available', async () => {
const db = {} as DatabaseClient;
await expect(loadClockReadiness(db)).resolves.toEqual({
reconciliationComplete: false,
gameplayEnabled: false,
phase: null,
revision: null,
deadlineGeneration: null,
incompleteOutboxCount: null,
});
});
it('blocks readiness for RECONCILING or incomplete outbox state', async () => {
const db = {
worldState: {
findFirst: vi.fn(async () => ({
clockPhase: 'RECONCILING',
clockRevision: 9n,
deadlineGeneration: 4n,
})),
},
clockProjectionOutbox: { count: vi.fn(async () => 1) },
} as unknown as DatabaseClient;
await expect(loadClockReadiness(db)).resolves.toMatchObject({
reconciliationComplete: false,
gameplayEnabled: false,
phase: 'RECONCILING',
revision: 9,
deadlineGeneration: 4,
incompleteOutboxCount: 1,
});
});
it('exposes participant checksums and incomplete outbox detail to admins', async () => {
const db = {
worldState: {
findFirst: vi.fn(async () => ({
clockPhase: 'RUNNING',
clockRevision: 3n,
deadlineGeneration: 2n,
})),
},
clockProjectionOutbox: { count: vi.fn(async () => 0) },
clockSuspension: {
findFirst: vi.fn(async () => ({
id: 'maintenance-1',
source: 'MAINTENANCE',
policy: 'EXACT',
status: 'APPLIED',
sourceRevision: 2n,
targetRevision: 3n,
cutTick: 100n,
alignedTick: 130n,
participantChecksumBefore: 'before-all',
participantChecksumAfter: 'after-all',
participants: [
{
participantKey: 'general-turn',
policy: 'SHIFT',
beforeChecksum: 'before',
afterChecksum: 'after',
affectedCount: 2,
},
],
projectionOutbox: [{ id: 8n, targetRevision: 3n, status: 'APPLIED', attempts: 1, lastError: null }],
})),
},
} as unknown as DatabaseClient;
await expect(loadClockAdminStatus(db)).resolves.toMatchObject({
reconciliationComplete: true,
latestReconciliation: {
id: 'maintenance-1',
participantChecksumBefore: 'before-all',
participantChecksumAfter: 'after-all',
participants: [{ key: 'general-turn', policy: 'SHIFT', affectedCount: 2 }],
outbox: [{ id: '8', status: 'APPLIED' }],
},
});
});
});
@@ -511,6 +511,10 @@ integration('API input event boundary', () => {
it('reuses the same engine child event but rejects a changed retry payload', async () => { it('reuses the same engine child event but rejects a changed retry payload', async () => {
const transport = new DatabaseTurnDaemonTransport(db, 100); const transport = new DatabaseTurnDaemonTransport(db, 100);
const requestId = 'integration:api:engine-child'; const requestId = 'integration:api:engine-child';
const worldClock = await db.worldState.findFirst({
orderBy: { id: 'asc' },
select: { clockRevision: true, deadlineGeneration: true },
});
const acceptedWindowStart = Date.now(); const acceptedWindowStart = Date.now();
await transport.sendCommand({ type: 'vacation', requestId, userId: 'user-7', generalId: 7 }); await transport.sendCommand({ type: 'vacation', requestId, userId: 'user-7', generalId: 7 });
const acceptedWindowEnd = Date.now(); const acceptedWindowEnd = Date.now();
@@ -518,6 +522,9 @@ integration('API input event boundary', () => {
expect(event.actorUserId).toBe('user-7'); expect(event.actorUserId).toBe('user-7');
expect(event.createdAt.getTime()).toBeGreaterThanOrEqual(acceptedWindowStart); expect(event.createdAt.getTime()).toBeGreaterThanOrEqual(acceptedWindowStart);
expect(event.createdAt.getTime()).toBeLessThanOrEqual(acceptedWindowEnd); expect(event.createdAt.getTime()).toBeLessThanOrEqual(acceptedWindowEnd);
expect(event.acceptedGameTick).not.toBeNull();
expect(event.acceptedClockRevision).toBe(worldClock?.clockRevision ?? null);
expect(event.acceptedDeadlineGeneration).toBe(worldClock?.deadlineGeneration ?? null);
await expect( await expect(
transport.sendCommand({ type: 'vacation', requestId, userId: 'user-7', generalId: 7 }) transport.sendCommand({ type: 'vacation', requestId, userId: 'user-7', generalId: 7 })
).resolves.toBe(requestId); ).resolves.toBe(requestId);
+7 -2
View File
@@ -28,6 +28,9 @@ const createContext = (payload: unknown = {}) => {
status: 'PENDING', status: 'PENDING',
result: null, result: null,
attempts: 0, attempts: 0,
acceptedGameTick: 100n,
acceptedClockRevision: 3n,
acceptedDeadlineGeneration: 2n,
}, },
]; ];
} }
@@ -36,8 +39,8 @@ const createContext = (payload: unknown = {}) => {
}); });
const transaction = { const transaction = {
$queryRaw: queryRaw, $queryRaw: queryRaw,
$executeRaw: vi.fn(async () => { $executeRaw: vi.fn(async (query: { sql?: string }) => {
order.push('accepted'); order.push(query.sql?.includes('pg_advisory_xact_lock') ? 'clock-fence' : 'accepted');
return 1; return 1;
}), }),
$executeRawUnsafe: vi.fn(async (statement: string) => { $executeRawUnsafe: vi.fn(async (statement: string) => {
@@ -88,6 +91,7 @@ describe('API input-event change journal boundary', () => {
expect(fixture.order).toEqual([ expect(fixture.order).toEqual([
'transaction-begin', 'transaction-begin',
'clock-fence',
'accepted', 'accepted',
'locked', 'locked',
'processing', 'processing',
@@ -112,6 +116,7 @@ describe('API input-event change journal boundary', () => {
expect(fixture.order).toEqual([ expect(fixture.order).toEqual([
'transaction-begin', 'transaction-begin',
'clock-fence',
'accepted', 'accepted',
'locked', 'locked',
'processing', 'processing',
+36 -7
View File
@@ -30,11 +30,31 @@ class MemoryRedis {
} }
async eval(_script: string, options: { keys: string[]; arguments: string[] }): Promise<string> { async eval(_script: string, options: { keys: string[]; arguments: string[] }): Promise<string> {
const [valueKey, revisionKey] = options.keys; if (options.keys.length === 3 && options.keys[0]?.endsWith(':clock:active-revision')) {
const [value] = options.arguments; const current = options.keys.map((key) => this.values.get(key));
if (current.every((value) => value === undefined)) {
options.keys.forEach((key, index) => this.values.set(key, options.arguments[index]!));
return '1';
}
return current.every((value, index) => value === options.arguments[index]) ? '2' : '0';
}
const fenced = options.keys.at(-1)?.endsWith(':clock:phase') === true;
const writeCount = options.keys.length - (fenced ? 4 : 1);
if (fenced) {
const clockKeys = options.keys.slice(-3);
const expected = options.arguments.slice(-3);
if (!clockKeys.every((key, index) => this.values.get(key) === expected[index])) {
return '__CLOCK_FENCE__';
}
}
const valueKey = options.keys[0];
const revisionKey = options.keys[writeCount];
const value = options.arguments[0];
if (!valueKey || !revisionKey || value === undefined) throw new Error('invalid eval arguments'); if (!valueKey || !revisionKey || value === undefined) throw new Error('invalid eval arguments');
const revision = Number(this.values.get(revisionKey) ?? '0') + 1; const revision = Number(this.values.get(revisionKey) ?? '0') + 1;
this.values.set(valueKey, value); for (let index = 0; index < writeCount; index += 1) {
this.values.set(options.keys[index]!, options.arguments[index]!);
}
this.values.set(revisionKey, String(revision)); this.values.set(revisionKey, String(revision));
return String(revision); return String(revision);
} }
@@ -144,6 +164,14 @@ const buildContext = (options: {
}, },
worldState: { worldState: {
findFirst: async () => ({ findFirst: async () => ({
clockBaseTime: new Date('2026-01-01T00:00:00.000Z'),
clockTick: 0n,
clockMode: 'realtime',
clockWallAnchor: new Date('2026-01-01T00:00:00.000Z'),
clockPhase: 'RUNNING',
clockRevision: 1n,
deadlineGeneration: 1n,
tickSeconds: 60,
config: { const: { develCost: options.develCost ?? 200 } }, config: { const: { develCost: options.develCost ?? 200 } },
...(options.currentDevelCost === undefined ? {} : { meta: { develcost: options.currentDevelCost } }), ...(options.currentDevelCost === undefined ? {} : { meta: { develcost: options.currentDevelCost } }),
}), }),
@@ -394,7 +422,10 @@ describe('tournament router permissions and mutations', () => {
roles: ['admin.tournament:che:default'], roles: ['admin.tournament:che:default'],
}) })
); );
await expect(adminCaller.tournament.getAdminStatus()).resolves.toEqual({ ok: true }); await expect(adminCaller.tournament.getAdminStatus()).resolves.toMatchObject({
ok: true,
clock: { reconciliationComplete: false, latestReconciliation: null },
});
}); });
it('applies the admin role boundary to every tournament mutation', async () => { it('applies the admin role boundary to every tournament mutation', async () => {
@@ -479,9 +510,7 @@ describe('tournament router permissions and mutations', () => {
]) ])
).resolves.toEqual({ ok: true, count: 1 }); ).resolves.toEqual({ ok: true, count: 1 });
await expect( await expect(
caller.tournament.setBettingEntries([ caller.tournament.setBettingEntries([{ generalId: general.id, targetId: rival.id, amount: 100 }])
{ generalId: general.id, targetId: rival.id, amount: 100 },
])
).resolves.toEqual({ ok: true, count: 1 }); ).resolves.toEqual({ ok: true, count: 1 });
await expect(caller.tournament.seedParticipants({ generalIds: [general.id, rival.id] })).resolves.toEqual({ await expect(caller.tournament.seedParticipants({ generalIds: [general.id, rival.id] })).resolves.toEqual({
ok: true, ok: true,
@@ -41,6 +41,9 @@ integration('TournamentStore Redis source revision', () => {
keys.matchesKey, keys.matchesKey,
keys.bettingKey, keys.bettingKey,
keys.sourceRevisionKey, keys.sourceRevisionKey,
keys.activeClockRevisionKey,
keys.deadlineGenerationKey,
keys.clockPhaseKey,
]); ]);
if (subscriber) { if (subscriber) {
await subscriber.client.unsubscribe(keys.sourceRevisionChannel); await subscriber.client.unsubscribe(keys.sourceRevisionChannel);
@@ -124,4 +127,48 @@ integration('TournamentStore Redis source revision', () => {
}), }),
]); ]);
}); });
it('dual-writes deadline ticks and rejects a Redis clock revision race atomically', async () => {
const store = new TournamentStore(connector.client, keys);
await Promise.all([
connector.client.set(keys.activeClockRevisionKey, '7'),
connector.client.set(keys.deadlineGenerationKey, '3'),
connector.client.set(keys.clockPhaseKey, 'RUNNING'),
]);
const clockContext = {
phase: 'RUNNING' as const,
revision: 7,
deadlineGeneration: 3,
dateToTick: (date: Date) => Math.trunc(date.getTime() / 1_000),
};
const nextAt = '2026-09-03T10:00:00.000Z';
await store.withClockContext(clockContext, () =>
store.setState({
stage: 6,
phase: 0,
type: 0,
auto: true,
openYear: 200,
openMonth: 1,
termSeconds: 10,
nextAt,
bettingCloseAt: '2026-09-03T09:59:50.000Z',
})
);
await expect(store.getState()).resolves.toMatchObject({
nextTick: Math.trunc(new Date(nextAt).getTime() / 1_000),
bettingCloseTick: Math.trunc(new Date('2026-09-03T09:59:50.000Z').getTime() / 1_000),
clockRevision: 7,
deadlineGeneration: 3,
});
const beforeRevision = await store.getSourceRevision();
await connector.client.set(keys.activeClockRevisionKey, '8');
await expect(
store.withClockContext(clockContext, () =>
store.setMatches([{ id: 99, stage: 7, roundIndex: 0, attackerId: 1, defenderId: 2 }])
)
).rejects.toThrow('clock revision fence failed');
await expect(store.getSourceRevision()).resolves.toBe(beforeRevision);
});
}); });
+8 -1
View File
@@ -269,7 +269,14 @@ export const createAuctionBidder = async (options: {
}; };
} }
const processingNow = world.getGameNow(new Date()); const processingNow = world.getGameNow(new Date());
const { bidAt, bidTick } = resolveAuctionBidTiming(world, processingNow, command.acceptedGameTick); const convertedProcessingTick = Reflect.get(command, 'processingGameTick');
const { bidAt, bidTick } = resolveAuctionBidTiming(
world,
processingNow,
typeof convertedProcessingTick === 'number' && Number.isSafeInteger(convertedProcessingTick)
? convertedProcessingTick
: command.acceptedGameTick
);
if (hasAuctionClosePassed(auction, bidAt, bidTick)) { if (hasAuctionClosePassed(auction, bidAt, bidTick)) {
return { return {
type: 'auctionBid', type: 'auctionBid',
+2
View File
@@ -20,6 +20,8 @@ export * from './turn/engineStateManager.js';
export * from './turn/inMemoryStateStore.js'; export * from './turn/inMemoryStateStore.js';
export * from './turn/inMemoryTurnProcessor.js'; export * from './turn/inMemoryTurnProcessor.js';
export * from './turn/databaseHooks.js'; export * from './turn/databaseHooks.js';
export * from './turn/clockReconciliation.js';
export * from './turn/clockProjectionOutbox.js';
export * from './turn/joinCreateGeneralService.js'; export * from './turn/joinCreateGeneralService.js';
export * from './turn/npcPossessionService.js'; export * from './turn/npcPossessionService.js';
export * from './turn/selectPoolService.js'; export * from './turn/selectPoolService.js';
@@ -108,6 +108,13 @@ export class DatabaseTurnDaemonCommandQueue implements TurnDaemonControlQueue, T
private async claimPending(limit = 100): Promise<TurnDaemonCommand[]> { private async claimPending(limit = 100): Promise<TurnDaemonCommand[]> {
await this.recoverExpiredLeases(); await this.recoverExpiredLeases();
return this.db.$transaction(async (transaction) => { return this.db.$transaction(async (transaction) => {
const world = await transaction.worldState.findFirst({
orderBy: { id: 'asc' },
select: { clockPhase: true, clockRevision: true, deadlineGeneration: true, clockTick: true },
});
const gameplayAllowed =
!world || world.clockPhase === 'RUNNING' || world.clockPhase === 'MANUAL';
const currentRevision = world?.clockRevision ?? null;
const rows = await transaction.$queryRaw< const rows = await transaction.$queryRaw<
Array<{ Array<{
sequence: bigint; sequence: bigint;
@@ -115,6 +122,9 @@ export class DatabaseTurnDaemonCommandQueue implements TurnDaemonControlQueue, T
eventType: string; eventType: string;
payload: unknown; payload: unknown;
createdAt: Date; createdAt: Date;
acceptedGameTick: bigint | null;
acceptedClockRevision: bigint | null;
acceptedDeadlineGeneration: bigint | null;
}> }>
>(GamePrisma.sql` >(GamePrisma.sql`
SELECT SELECT
@@ -122,10 +132,14 @@ export class DatabaseTurnDaemonCommandQueue implements TurnDaemonControlQueue, T
"request_id" AS "requestId", "request_id" AS "requestId",
"event_type" AS "eventType", "event_type" AS "eventType",
"payload", "payload",
"created_at" AS "createdAt" "created_at" AS "createdAt",
"accepted_game_tick" AS "acceptedGameTick",
"accepted_clock_revision" AS "acceptedClockRevision",
"accepted_deadline_generation" AS "acceptedDeadlineGeneration"
FROM "input_event" FROM "input_event"
WHERE "target" = 'ENGINE'::"InputEventTarget" WHERE "target" = 'ENGINE'::"InputEventTarget"
AND "status" = 'PENDING'::"InputEventStatus" AND "status" = 'PENDING'::"InputEventStatus"
AND (${gameplayAllowed} OR "event_type" = 'getStatus')
ORDER BY "sequence" ASC ORDER BY "sequence" ASC
FOR UPDATE SKIP LOCKED FOR UPDATE SKIP LOCKED
LIMIT ${limit} LIMIT ${limit}
@@ -133,23 +147,53 @@ export class DatabaseTurnDaemonCommandQueue implements TurnDaemonControlQueue, T
if (rows.length === 0) { if (rows.length === 0) {
return []; return [];
} }
await transaction.inputEvent.updateMany({ const appliedSuspensions = currentRevision
where: { ? await transaction.clockSuspension.findMany({
sequence: { in: rows.map((row) => row.sequence) }, where: { status: 'APPLIED', targetRevision: { lte: currentRevision } },
target: 'ENGINE', orderBy: { sourceRevision: 'asc' },
status: 'PENDING', select: { sourceRevision: true, targetRevision: true, shiftTicks: true },
}, })
data: { : [];
status: 'PROCESSING', const convertTick = (row: (typeof rows)[number]): bigint | null | undefined => {
processingAt: new Date(), if (row.eventType === 'getStatus') return row.acceptedGameTick;
lockedBy: this.workerId, if (row.acceptedGameTick === null || row.acceptedClockRevision === null || currentRevision === null) {
leaseUntil: new Date(Date.now() + this.leaseDurationMs), return row.acceptedGameTick ?? world?.clockTick ?? null;
attempts: { increment: 1 }, }
}, if (row.acceptedClockRevision > currentRevision) return undefined;
}); let revision = row.acceptedClockRevision;
let tick = row.acceptedGameTick;
while (revision < currentRevision) {
const step = appliedSuspensions.find((entry) => entry.sourceRevision === revision);
if (!step || step.shiftTicks === null || step.targetRevision !== revision + 1n) return undefined;
tick += step.shiftTicks;
revision = step.targetRevision;
}
return tick;
};
const processableRows = rows
.map((row) => ({ row, processingGameTick: convertTick(row) }))
.filter(
(entry): entry is { row: (typeof rows)[number]; processingGameTick: bigint | null } =>
entry.processingGameTick !== undefined
);
for (const { row, processingGameTick } of processableRows) {
await transaction.inputEvent.update({
where: { sequence: row.sequence },
data: {
status: 'PROCESSING',
processingAt: new Date(),
processingGameTick,
processingClockRevision: currentRevision,
processingDeadlineGeneration: world?.deadlineGeneration ?? null,
lockedBy: this.workerId,
leaseUntil: new Date(Date.now() + this.leaseDurationMs),
attempts: { increment: 1 },
},
});
}
const commands: TurnDaemonCommand[] = []; const commands: TurnDaemonCommand[] = [];
for (const row of rows) { for (const { row, processingGameTick } of processableRows) {
const command = normalizeTurnDaemonCommand({ const command = normalizeTurnDaemonCommand({
requestId: row.requestId, requestId: row.requestId,
sentAt: row.createdAt.toISOString(), sentAt: row.createdAt.toISOString(),
@@ -168,6 +212,27 @@ export class DatabaseTurnDaemonCommandQueue implements TurnDaemonControlQueue, T
}); });
continue; continue;
} }
if (
processingGameTick !== null &&
row.acceptedGameTick !== null &&
processingGameTick !== row.acceptedGameTick
) {
const value = Number(processingGameTick);
if (!Number.isSafeInteger(value)) {
await transaction.inputEvent.update({
where: { sequence: row.sequence },
data: {
status: 'FAILED',
error: 'Converted processing game tick is outside the safe integer range.',
completedAt: new Date(),
lockedBy: null,
leaseUntil: null,
},
});
continue;
}
Reflect.set(command, 'processingGameTick', value);
}
commands.push(command); commands.push(command);
} }
return commands; return commands;
@@ -0,0 +1,343 @@
import { createHash } from 'node:crypto';
import { GameClock, parseGameClockPhase } from '@sammo-ts/common';
import {
CLOCK_OPERATION_PERSISTENCE_LOCK,
GamePrisma,
acquireGameSchemaAdvisoryXactLock,
type GamePrismaClient,
} from '@sammo-ts/infra';
interface ClockProjectionRedis {
get(key: string): Promise<string | null>;
eval(script: string, options: { keys: string[]; arguments: string[] }): Promise<unknown>;
}
interface ClaimedOutboxRow {
id: bigint;
}
interface DbWallRow {
wallNow: Date;
}
interface ProjectionPayload {
version: 1;
profileName: string;
suspensionId: string;
sourceRevision: number;
targetRevision: number;
deadlineGeneration: number;
shiftTicks: number;
projectionDeltaMilliseconds: number;
clockBaseTime: string;
ticksPerSecond: number;
}
interface TournamentProjectionState {
stage?: number;
nextAt?: string;
nextTick?: number;
bettingCloseAt?: string;
bettingCloseTick?: number;
clockRevision?: number;
deadlineGeneration?: number;
[key: string]: unknown;
}
const APPLY_CLOCK_PROJECTION_SCRIPT = `
local active = redis.call('GET', KEYS[1])
if active == ARGV[2] then
if redis.call('GET', KEYS[5]) == ARGV[4] and redis.call('GET', KEYS[2]) == ARGV[3] then
return 2
end
return -3
end
if active and active ~= ARGV[1] then
return -1
end
if ARGV[5] ~= '__NONE__' and redis.call('GET', KEYS[4]) ~= ARGV[5] then
return -2
end
redis.call('DEL', KEYS[3])
local count = tonumber(ARGV[7])
local offset = 8
for index = 1, count do
redis.call('ZADD', KEYS[3], ARGV[offset], ARGV[offset + 1])
offset = offset + 2
end
if ARGV[5] ~= '__NONE__' then
redis.call('SET', KEYS[4], ARGV[6])
end
redis.call('SET', KEYS[1], ARGV[2])
redis.call('SET', KEYS[2], ARGV[3])
redis.call('SET', KEYS[5], ARGV[4])
redis.call('SET', KEYS[6], 'RUNNING')
return 1
`;
const safeInteger = (value: unknown, label: string): number => {
const result = typeof value === 'bigint' ? Number(value) : value;
if (typeof result !== 'number' || !Number.isSafeInteger(result)) {
throw new Error(`${label} must be a safe integer.`);
}
return result;
};
const canonicalize = (value: unknown): unknown => {
if (typeof value === 'bigint') return value.toString();
if (Array.isArray(value)) return value.map(canonicalize);
if (value && typeof value === 'object') {
return Object.fromEntries(
Object.entries(value as Record<string, unknown>)
.sort(([left], [right]) => left.localeCompare(right))
.map(([key, item]) => [key, canonicalize(item)])
);
}
return value;
};
const stableJson = (value: unknown): string => JSON.stringify(canonicalize(value));
const checksum = (value: unknown): string => createHash('sha256').update(stableJson(value)).digest('hex');
const readDbWall = async (db: GamePrisma.TransactionClient): Promise<Date> => {
const rows = await db.$queryRaw<DbWallRow[]>(GamePrisma.sql`
SELECT (CURRENT_TIMESTAMP AT TIME ZONE 'UTC')::timestamp(3) AS "wallNow"
`);
if (!rows[0]?.wallNow) throw new Error('Failed to read PostgreSQL wall time for the projection outbox.');
return rows[0].wallNow;
};
const parsePayload = (value: GamePrisma.JsonValue): ProjectionPayload => {
if (!value || typeof value !== 'object' || Array.isArray(value)) {
throw new Error('Clock projection outbox payload must be an object.');
}
const payload = value as Record<string, unknown>;
if (payload.version !== 1 || typeof payload.profileName !== 'string' || typeof payload.suspensionId !== 'string') {
throw new Error('Clock projection outbox payload identity is invalid.');
}
if (typeof payload.clockBaseTime !== 'string') {
throw new Error('Clock projection outbox is missing its projection base.');
}
return {
version: 1,
profileName: payload.profileName,
suspensionId: payload.suspensionId,
sourceRevision: safeInteger(payload.sourceRevision, 'sourceRevision'),
targetRevision: safeInteger(payload.targetRevision, 'targetRevision'),
deadlineGeneration: safeInteger(payload.deadlineGeneration, 'deadlineGeneration'),
shiftTicks: safeInteger(payload.shiftTicks, 'shiftTicks'),
projectionDeltaMilliseconds: safeInteger(
payload.projectionDeltaMilliseconds,
'projectionDeltaMilliseconds'
),
clockBaseTime: payload.clockBaseTime,
ticksPerSecond: safeInteger(payload.ticksPerSecond, 'ticksPerSecond'),
};
};
const claimNext = async (db: GamePrismaClient, workerId: string) =>
db.$transaction(async (transaction) => {
const rows = await transaction.$queryRaw<ClaimedOutboxRow[]>(GamePrisma.sql`
SELECT id
FROM clock_projection_outbox
WHERE available_at <= (CURRENT_TIMESTAMP AT TIME ZONE 'UTC')
AND (
status IN ('PENDING', 'FAILED')
OR (status = 'APPLYING' AND locked_at < (CURRENT_TIMESTAMP AT TIME ZONE 'UTC') - INTERVAL '30 seconds')
)
ORDER BY id
LIMIT 1
FOR UPDATE SKIP LOCKED
`);
const id = rows[0]?.id;
if (id === undefined) return null;
const lockedAt = await readDbWall(transaction);
return transaction.clockProjectionOutbox.update({
where: { id },
data: {
status: 'APPLYING',
attempts: { increment: 1 },
lockedAt,
lockedBy: workerId,
lastError: null,
},
});
});
const projectTournamentState = (
raw: string | null,
payload: ProjectionPayload,
clock: GameClock
): { expected: string; next: string } | null => {
if (!raw) return null;
const parsed = JSON.parse(raw) as TournamentProjectionState;
const active = typeof parsed.stage === 'number' && parsed.stage > 0;
if (active && parsed.nextAt && !Number.isSafeInteger(parsed.nextTick)) {
throw new Error('Active tournament nextAt lacks the authoritative nextTick dual-write.');
}
if (active && parsed.bettingCloseAt && !Number.isSafeInteger(parsed.bettingCloseTick)) {
throw new Error('Active tournament bettingCloseAt lacks the authoritative bettingCloseTick dual-write.');
}
const next: TournamentProjectionState = {
...parsed,
clockRevision: payload.targetRevision,
deadlineGeneration: payload.deadlineGeneration,
};
if (Number.isSafeInteger(parsed.nextTick)) {
next.nextTick = parsed.nextTick! + payload.shiftTicks;
next.nextAt = clock.tickToDate(next.nextTick).toISOString();
}
if (Number.isSafeInteger(parsed.bettingCloseTick)) {
next.bettingCloseTick = parsed.bettingCloseTick! + payload.shiftTicks;
next.bettingCloseAt = clock.tickToDate(next.bettingCloseTick).toISOString();
}
return { expected: raw, next: JSON.stringify(next) };
};
const recordFailure = async (db: GamePrismaClient, outboxId: bigint, error: unknown): Promise<void> => {
const message = error instanceof Error ? error.message : String(error);
await db.$executeRaw(GamePrisma.sql`
UPDATE clock_projection_outbox
SET status = 'FAILED',
locked_at = NULL,
locked_by = NULL,
last_error = ${message.slice(0, 4_000)},
available_at = (CURRENT_TIMESTAMP AT TIME ZONE 'UTC') + INTERVAL '1 second'
WHERE id = ${outboxId} AND status = 'APPLYING'
`);
};
export const applyNextClockProjection = async (options: {
db: GamePrismaClient;
redis: ClockProjectionRedis;
workerId: string;
}): Promise<'IDLE' | 'APPLIED' | 'RECOVERED'> => {
if (!options.workerId.trim()) throw new Error('Clock projection worker ID is required.');
const outbox = await claimNext(options.db, options.workerId);
if (!outbox) return 'IDLE';
try {
const payload = parsePayload(outbox.payload);
if (checksum(outbox.payload) !== outbox.checksum) {
throw new Error('Clock projection outbox checksum verification failed.');
}
if (outbox.targetRevision !== BigInt(payload.targetRevision)) {
throw new Error('Clock projection payload revision differs from its outbox row.');
}
const world = await options.db.worldState.findUniqueOrThrow({ where: { id: outbox.worldStateId } });
if (
parseGameClockPhase(world.clockPhase) !== 'RECONCILING' ||
world.clockRevision !== outbox.targetRevision ||
world.deadlineGeneration !== BigInt(payload.deadlineGeneration) ||
!world.clockBaseTime
) {
throw new Error('Clock projection DB phase/revision/generation fence failed.');
}
const clock = new GameClock({
baseTime: new Date(payload.clockBaseTime),
tick: safeInteger(world.clockTick, 'world clock tick'),
mode: world.clockMode === 'manual' ? 'manual' : 'realtime',
wallAnchor: world.clockWallAnchor ?? new Date(),
turnSeconds: world.tickSeconds,
phase: 'RECONCILING',
revision: payload.targetRevision,
});
if (clock.ticksPerSecond !== payload.ticksPerSecond) {
throw new Error('Clock projection rate differs from the durable outbox payload.');
}
const auctions = await options.db.auction.findMany({
where: { status: { in: ['OPEN', 'FINALIZING'] } },
orderBy: { id: 'asc' },
select: { id: true, closeTick: true },
});
const timers = auctions.map((auction) => {
if (auction.closeTick === null) {
throw new Error(`Active auction ${auction.id} lacks closeTick during projection rebuild.`);
}
return { score: safeInteger(auction.closeTick, `auction ${auction.id} closeTick`), id: String(auction.id) };
});
const prefix = `sammo:${payload.profileName}`;
const tournamentKey = `${prefix}:tournament:state`;
const tournament = projectTournamentState(await options.redis.get(tournamentKey), payload, clock);
const result = await options.redis.eval(APPLY_CLOCK_PROJECTION_SCRIPT, {
keys: [
`${prefix}:clock:active-revision`,
`${prefix}:clock:deadline-generation`,
`${prefix}:auction:timer`,
tournamentKey,
`${prefix}:clock:projection-checksum`,
`${prefix}:clock:phase`,
],
arguments: [
String(payload.sourceRevision),
String(payload.targetRevision),
String(payload.deadlineGeneration),
outbox.checksum,
tournament?.expected ?? '__NONE__',
tournament?.next ?? '__NONE__',
String(timers.length),
...timers.flatMap(({ score, id }) => [String(score), id]),
],
});
const applied = Number(result);
if (applied === -1) throw new Error('Redis active clock revision does not match the outbox source revision.');
if (applied === -2) throw new Error('Redis tournament state changed while rebuilding its projection.');
if (applied === -3) throw new Error('Redis target revision exists without the expected projection checksum.');
if (applied !== 1 && applied !== 2) throw new Error(`Unexpected Redis clock projection result: ${String(result)}`);
await options.db.$transaction(async (transaction) => {
await acquireGameSchemaAdvisoryXactLock(transaction, CLOCK_OPERATION_PERSISTENCE_LOCK);
await transaction.$queryRaw<ClaimedOutboxRow[]>(GamePrisma.sql`
SELECT id FROM world_state WHERE id = ${outbox.worldStateId} FOR UPDATE
`);
const finalized = await transaction.worldState.updateMany({
where: {
id: outbox.worldStateId,
clockPhase: 'RECONCILING',
clockRevision: outbox.targetRevision,
deadlineGeneration: BigInt(payload.deadlineGeneration),
},
data: { clockPhase: 'RUNNING' },
});
if (finalized.count !== 1) {
throw new Error('Clock projection final RUNNING transition fence failed.');
}
const appliedAt = await readDbWall(transaction);
await transaction.clockProjectionOutbox.update({
where: { id: outbox.id },
data: { status: 'APPLIED', appliedAt, lockedAt: null, lockedBy: null, lastError: null },
});
if (outbox.suspensionId) {
await transaction.clockSuspension.update({
where: { id: outbox.suspensionId },
data: { status: 'APPLIED' },
});
}
});
return applied === 2 ? 'RECOVERED' : 'APPLIED';
} catch (error) {
await recordFailure(options.db, outbox.id, error);
throw error;
}
};
export const loadClockReconciliationReadiness = async (db: GamePrismaClient) => {
const world = await db.worldState.findFirst({
orderBy: { id: 'asc' },
select: { clockPhase: true, clockRevision: true, deadlineGeneration: true },
});
const incompleteOutboxCount = await db.clockProjectionOutbox.count({ where: { status: { not: 'APPLIED' } } });
if (!world) {
return { ready: false, phase: null, revision: null, deadlineGeneration: null, incompleteOutboxCount };
}
const phase = parseGameClockPhase(world.clockPhase);
return {
ready: phase !== 'RECONCILING' && incompleteOutboxCount === 0,
gameplayEnabled: phase === 'RUNNING' || phase === 'MANUAL',
phase,
revision: safeInteger(world.clockRevision, 'clock revision'),
deadlineGeneration: safeInteger(world.deadlineGeneration, 'deadline generation'),
incompleteOutboxCount,
};
};
@@ -0,0 +1,763 @@
import { createHash } from 'node:crypto';
import {
GAME_TICKS_PER_TURN,
MAX_SAFE_GAME_TICK,
GameClock,
buildClockAlignmentPlan,
parseClockAlignmentPolicy,
parseGameClockPhase,
type ClockAlignmentPolicy,
} from '@sammo-ts/common';
import {
CLOCK_OPERATION_PERSISTENCE_LOCK,
GENERAL_ACCESS_PERSISTENCE_LOCK,
GamePrisma,
acquireGameSchemaAdvisoryXactLock,
type GamePrismaClient,
} from '@sammo-ts/infra';
export type ClockSuspensionSource = 'MAINTENANCE' | 'OPEN_DELAY' | 'UNIFICATION_WAIT' | 'RECOVERY';
export type ClockOperationAuthority =
| { kind: 'DAEMON'; profileName: string; ownerId: string; fencingEpoch: bigint }
| { kind: 'OFFLINE'; profileName: string; reason: string };
export interface ClockSuspensionResult {
suspensionId: string;
phase: 'SUSPENDED';
sourceRevision: number;
targetRevision: number;
cutTick: number;
cutWallAt: Date;
}
export interface ClockReconciliationResult {
suspensionId: string;
phase: 'RECONCILING';
sourceRevision: number;
targetRevision: number;
deadlineGeneration: number;
gapTicks: number;
catchUpTicks: number;
shiftTicks: number;
alignedTick: number;
resumeWallAt: Date;
}
interface DbWallRow {
wallNow: Date;
}
interface LeaseFenceRow {
ownerId: string;
fencingEpoch: bigint;
valid: boolean;
}
interface IdRow {
id: number;
}
interface TextIdRow {
id: string;
}
interface ParticipantSnapshot {
key: string;
policy: 'SHIFT' | 'KEEP' | 'REBUILD';
checksum: string;
count: number;
}
const asJson = (value: unknown): GamePrisma.InputJsonValue => value as GamePrisma.InputJsonValue;
const safeNumber = (value: bigint, label: string): number => {
const result = Number(value);
if (!Number.isSafeInteger(result)) {
throw new Error(`${label} is outside the JavaScript safe integer range: ${value}`);
}
return result;
};
const canonicalize = (value: unknown): unknown => {
if (typeof value === 'bigint') return value.toString();
if (Array.isArray(value)) return value.map(canonicalize);
if (value && typeof value === 'object') {
return Object.fromEntries(
Object.entries(value as Record<string, unknown>)
.sort(([left], [right]) => left.localeCompare(right))
.map(([key, item]) => [key, canonicalize(item)])
);
}
return value;
};
const stableJson = (value: unknown): string => JSON.stringify(canonicalize(value));
const checksum = (value: unknown): string => createHash('sha256').update(stableJson(value)).digest('hex');
const aggregateChecksum = (participants: readonly ParticipantSnapshot[]): string =>
checksum(participants.map(({ key, policy, checksum: value, count }) => ({ key, policy, checksum: value, count })));
const readDbWall = async (db: GamePrisma.TransactionClient): Promise<Date> => {
const rows = await db.$queryRaw<DbWallRow[]>(GamePrisma.sql`
SELECT (CURRENT_TIMESTAMP AT TIME ZONE 'UTC')::timestamp(3) AS "wallNow"
`);
const wallNow = rows[0]?.wallNow;
if (!wallNow || Number.isNaN(wallNow.getTime())) {
throw new Error('Failed to read the PostgreSQL wall clock.');
}
return wallNow;
};
const verifyAuthority = async (
db: GamePrisma.TransactionClient,
authority: ClockOperationAuthority
): Promise<void> => {
const rows = await db.$queryRaw<LeaseFenceRow[]>(GamePrisma.sql`
SELECT owner_id AS "ownerId",
fencing_epoch AS "fencingEpoch",
lease_until > (CURRENT_TIMESTAMP AT TIME ZONE 'UTC') AS valid
FROM turn_daemon_lease
WHERE profile = ${authority.profileName}
FOR UPDATE
`);
const lease = rows[0];
if (authority.kind === 'OFFLINE') {
if (!authority.reason.trim()) {
throw new Error('Offline clock operations require an audit reason.');
}
if (lease?.valid) {
throw new Error(`Clock operation requires the ${authority.profileName} daemon lease to be offline.`);
}
return;
}
if (
!lease?.valid ||
lease.ownerId !== authority.ownerId ||
lease.fencingEpoch !== authority.fencingEpoch
) {
throw new Error(`Stale turn-daemon fencing authority for profile ${authority.profileName}.`);
}
};
const lockWorld = async (db: GamePrisma.TransactionClient): Promise<number> => {
const rows = await db.$queryRaw<IdRow[]>(GamePrisma.sql`
SELECT id FROM world_state ORDER BY id LIMIT 2 FOR UPDATE
`);
if (rows.length !== 1) {
throw new Error(`Clock reconciliation requires exactly one world_state row; found ${rows.length}.`);
}
return rows[0]!.id;
};
const lockParticipants = async (db: GamePrisma.TransactionClient, cutTick: bigint): Promise<void> => {
await db.$queryRaw<IdRow[]>(GamePrisma.sql`SELECT id FROM general ORDER BY id FOR UPDATE`);
await db.$queryRaw<IdRow[]>(GamePrisma.sql`
SELECT id FROM auction
WHERE status IN ('OPEN'::auction_status, 'FINALIZING'::auction_status)
ORDER BY id FOR UPDATE
`);
await db.$queryRaw<IdRow[]>(GamePrisma.sql`
SELECT id FROM message
WHERE valid_until_tick IS NOT NULL AND valid_until_tick >= ${cutTick}
ORDER BY id FOR UPDATE
`);
await db.$queryRaw<IdRow[]>(GamePrisma.sql`
SELECT id FROM vote_poll WHERE closed_at IS NULL ORDER BY id FOR UPDATE
`);
await db.$queryRaw<IdRow[]>(GamePrisma.sql`
SELECT id FROM select_pool WHERE general_id IS NULL ORDER BY id FOR UPDATE
`);
await db.$queryRaw<TextIdRow[]>(GamePrisma.sql`
SELECT owner_user_id AS id FROM select_npc_token ORDER BY owner_user_id FOR UPDATE
`);
};
const readParticipantSnapshots = async (
db: GamePrisma.TransactionClient,
worldStateId: number,
cutTick: bigint
): Promise<ParticipantSnapshot[]> => {
const [world, generals, auctions, messages, votes, pool, npcTokens, commands] = await Promise.all([
db.worldState.findUniqueOrThrow({
where: { id: worldStateId },
select: {
clockTick: true,
clockRevision: true,
deadlineGeneration: true,
lastTurnTick: true,
meta: true,
},
}),
db.general.findMany({
orderBy: { id: 'asc' },
select: { id: true, turnTick: true, recentWarTick: true },
}),
db.auction.findMany({
where: { status: { in: ['OPEN', 'FINALIZING'] } },
orderBy: { id: 'asc' },
select: { id: true, status: true, openTick: true, closeTick: true },
}),
db.message.findMany({
where: { validUntilTick: { not: null, gte: cutTick } },
orderBy: { id: 'asc' },
select: { id: true, timeTick: true, validUntilTick: true },
}),
db.votePoll.findMany({
where: { closedAt: null },
orderBy: { id: 'asc' },
select: { id: true, startTick: true, endTick: true },
}),
db.selectPoolEntry.findMany({
where: { generalId: null },
orderBy: { id: 'asc' },
select: { id: true, reservedUntilTick: true },
}),
db.npcSelectionToken.findMany({
orderBy: { ownerUserId: 'asc' },
select: { ownerUserId: true, validUntilTick: true, pickMoreFromTick: true },
}),
db.inputEvent.findMany({
where: { status: { in: ['PENDING', 'PROCESSING'] } },
orderBy: { sequence: 'asc' },
select: { sequence: true, acceptedGameTick: true, acceptedClockRevision: true },
}),
]);
const snapshot = (key: string, policy: ParticipantSnapshot['policy'], rows: unknown[]): ParticipantSnapshot => ({
key,
policy,
checksum: checksum(rows),
count: rows.length,
});
const meta = world.meta && typeof world.meta === 'object' && !Array.isArray(world.meta) ? world.meta : {};
return [
snapshot('world-clock', 'REBUILD', [
{
clockTick: world.clockTick,
clockRevision: world.clockRevision,
deadlineGeneration: world.deadlineGeneration,
},
]),
snapshot('turn-cursor', 'SHIFT', [{ lastTurnTick: world.lastTurnTick }]),
snapshot(
'general-next-turn',
'SHIFT',
generals.map(({ id, turnTick }) => ({ id, turnTick }))
),
snapshot(
'general-recent-war-occurrence',
'KEEP',
generals.map(({ id, recentWarTick }) => ({ id, recentWarTick }))
),
snapshot(
'auction-open-occurrence',
'KEEP',
auctions.map(({ id, openTick }) => ({ id, openTick }))
),
snapshot(
'auction-deadline',
'SHIFT',
auctions.map(({ id, status, closeTick }) => ({ id, status, closeTick }))
),
snapshot(
'auction-finalizing-recovery',
'REBUILD',
auctions.map(({ id, status }) => ({ id, status }))
),
snapshot(
'message-occurrence',
'KEEP',
messages.map(({ id, timeTick }) => ({ id, timeTick }))
),
snapshot(
'message-expiry',
'SHIFT',
messages.map(({ id, validUntilTick }) => ({ id, validUntilTick }))
),
snapshot(
'vote-start-occurrence',
'KEEP',
votes.map(({ id, startTick }) => ({ id, startTick }))
),
snapshot(
'vote-end-deadline',
'SHIFT',
votes.map(({ id, endTick }) => ({ id, endTick }))
),
snapshot('select-pool-reservation', 'SHIFT', pool),
snapshot('npc-selection-window', 'SHIFT', npcTokens),
snapshot('accepted-command-coordinate', 'KEEP', commands),
snapshot('movable-json-rule-anchors', 'SHIFT', [
{
lastTurnTime: Reflect.get(meta, 'lastTurnTime'),
turntime: Reflect.get(meta, 'turntime'),
starttime: Reflect.get(meta, 'starttime'),
tnmt_time: Reflect.get(meta, 'tnmt_time'),
},
]),
];
};
const persistInitialParticipants = async (
db: GamePrisma.TransactionClient,
suspensionId: string,
participants: readonly ParticipantSnapshot[]
): Promise<void> => {
for (const participant of participants) {
await db.clockReconciliationParticipant.create({
data: {
suspensionId,
participantKey: participant.key,
policy: participant.policy,
beforeChecksum: participant.checksum,
afterChecksum: participant.checksum,
affectedCount: 0,
},
});
}
};
const shiftMetaDate = (value: unknown, deltaMilliseconds: number): unknown => {
if (typeof value !== 'string' || !value.trim()) return value;
const parsed = new Date(value);
if (Number.isNaN(parsed.getTime())) return value;
return new Date(parsed.getTime() + deltaMilliseconds).toISOString();
};
const shiftedMeta = (value: GamePrisma.JsonValue, deltaMilliseconds: number): GamePrisma.InputJsonValue => {
const meta = value && typeof value === 'object' && !Array.isArray(value) ? { ...value } : {};
for (const key of ['lastTurnTime', 'turntime', 'starttime', 'tnmt_time'] as const) {
if (Object.hasOwn(meta, key)) {
Reflect.set(meta, key, shiftMetaDate(Reflect.get(meta, key), deltaMilliseconds));
}
}
return asJson(meta);
};
const assertShiftFits = (participants: readonly ParticipantSnapshot[], shiftTicks: number): void => {
if (!Number.isSafeInteger(shiftTicks) || shiftTicks < 0) {
throw new Error(`Invalid reconciliation shift: ${shiftTicks}`);
}
// Checksums retain stringified values for audit; actual row ranges are
// checked by PostgreSQL BIGINT and the world aligned tick is checked by the
// shared GameClock plan. The sentinel expiry is deliberately never shifted.
if (participants.some((participant) => !participant.checksum)) {
throw new Error('Participant snapshot is incomplete.');
}
};
const assertScheduleRanges = async (db: GamePrisma.TransactionClient, shiftTicks: number): Promise<void> => {
const shift = BigInt(shiftTicks);
const maximum = BigInt(MAX_SAFE_GAME_TICK) - shift;
const [general, auction, message, vote, pool, npcValid, npcMore] = await Promise.all([
db.general.aggregate({ _max: { turnTick: true }, where: { turnTick: { not: null } } }),
db.auction.aggregate({
_max: { closeTick: true },
where: { status: { in: ['OPEN', 'FINALIZING'] }, closeTick: { not: null } },
}),
db.message.aggregate({
_max: { validUntilTick: true },
where: { validUntilTick: { not: null, lt: BigInt(MAX_SAFE_GAME_TICK) } },
}),
db.votePoll.aggregate({ _max: { endTick: true }, where: { closedAt: null, endTick: { not: null } } }),
db.selectPoolEntry.aggregate({
_max: { reservedUntilTick: true },
where: { generalId: null, reservedUntilTick: { not: null } },
}),
db.npcSelectionToken.aggregate({ _max: { validUntilTick: true }, where: { validUntilTick: { not: null } } }),
db.npcSelectionToken.aggregate({
_max: { pickMoreFromTick: true },
where: { pickMoreFromTick: { not: null } },
}),
]);
const values: Array<[string, bigint | null]> = [
['general.turn_tick', general._max.turnTick],
['auction.close_tick', auction._max.closeTick],
['message.valid_until_tick', message._max.validUntilTick],
['vote_poll.end_tick', vote._max.endTick],
['select_pool.reserved_until_tick', pool._max.reservedUntilTick],
['select_npc_token.valid_until_tick', npcValid._max.validUntilTick],
['select_npc_token.pick_more_from_tick', npcMore._max.pickMoreFromTick],
];
for (const [label, value] of values) {
if (value !== null && value > maximum) {
throw new Error(`${label} would exceed the safe game tick range after reconciliation.`);
}
}
};
const applyParticipantShift = async (
db: GamePrisma.TransactionClient,
worldStateId: number,
cutTick: bigint,
alignedTick: bigint,
targetRevision: bigint,
targetGeneration: bigint,
shiftTicks: bigint,
projectionDeltaMilliseconds: number,
resumeWallAt: Date
): Promise<Map<string, number>> => {
const affected = new Map<string, number>();
const world = await db.worldState.findUniqueOrThrow({ where: { id: worldStateId }, select: { meta: true } });
const cursor = await db.worldState.updateMany({
where: { id: worldStateId, lastTurnTick: { not: null } },
data: { lastTurnTick: { increment: shiftTicks } },
});
affected.set('turn-cursor', cursor.count);
affected.set(
'general-next-turn',
await db.$executeRaw(GamePrisma.sql`
UPDATE general
SET turn_tick = turn_tick + ${shiftTicks},
turn_time = turn_time + ${projectionDeltaMilliseconds} * INTERVAL '1 millisecond'
WHERE turn_tick IS NOT NULL
`)
);
affected.set(
'auction-deadline',
await db.$executeRaw(GamePrisma.sql`
UPDATE auction
SET close_tick = close_tick + ${shiftTicks},
close_at = close_at + ${projectionDeltaMilliseconds} * INTERVAL '1 millisecond'
WHERE status IN ('OPEN'::auction_status, 'FINALIZING'::auction_status)
AND close_tick IS NOT NULL
`)
);
affected.set(
'message-expiry',
await db.$executeRaw(GamePrisma.sql`
UPDATE message
SET valid_until_tick = valid_until_tick + ${shiftTicks},
valid_until = valid_until + ${projectionDeltaMilliseconds} * INTERVAL '1 millisecond'
WHERE valid_until_tick IS NOT NULL
AND valid_until_tick >= ${cutTick}
AND valid_until_tick < ${BigInt(MAX_SAFE_GAME_TICK)}
`)
);
affected.set(
'vote-end-deadline',
await db.$executeRaw(GamePrisma.sql`
UPDATE vote_poll
SET end_tick = end_tick + ${shiftTicks},
end_at = end_at + ${projectionDeltaMilliseconds} * INTERVAL '1 millisecond'
WHERE closed_at IS NULL AND end_tick IS NOT NULL
`)
);
affected.set(
'select-pool-reservation',
await db.$executeRaw(GamePrisma.sql`
UPDATE select_pool
SET reserved_until_tick = reserved_until_tick + ${shiftTicks},
reserved_until = reserved_until + ${projectionDeltaMilliseconds} * INTERVAL '1 millisecond'
WHERE general_id IS NULL AND reserved_until_tick IS NOT NULL
`)
);
affected.set(
'npc-selection-window',
await db.$executeRaw(GamePrisma.sql`
UPDATE select_npc_token
SET valid_until_tick = CASE
WHEN valid_until_tick IS NULL THEN NULL ELSE valid_until_tick + ${shiftTicks} END,
valid_until = CASE
WHEN valid_until_tick IS NULL THEN valid_until
ELSE valid_until + ${projectionDeltaMilliseconds} * INTERVAL '1 millisecond' END,
pick_more_from_tick = CASE
WHEN pick_more_from_tick IS NULL THEN NULL ELSE pick_more_from_tick + ${shiftTicks} END,
pick_more_from = CASE
WHEN pick_more_from_tick IS NULL THEN pick_more_from
ELSE pick_more_from + ${projectionDeltaMilliseconds} * INTERVAL '1 millisecond' END
WHERE valid_until_tick IS NOT NULL OR pick_more_from_tick IS NOT NULL
`)
);
await db.worldState.update({
where: { id: worldStateId },
data: {
clockTick: alignedTick,
clockWallAnchor: resumeWallAt,
clockPhase: 'RECONCILING',
clockRevision: targetRevision,
deadlineGeneration: targetGeneration,
meta: shiftedMeta(world.meta, projectionDeltaMilliseconds),
},
});
affected.set('world-clock', 1);
affected.set('movable-json-rule-anchors', 1);
affected.set('auction-finalizing-recovery', 0);
return affected;
};
export const startClockSuspension = async (options: {
db: GamePrismaClient;
suspensionId: string;
source: ClockSuspensionSource;
authority: ClockOperationAuthority;
policy?: ClockAlignmentPolicy;
catchUpTicks?: number;
}): Promise<ClockSuspensionResult> => {
if (!options.suspensionId.trim() || options.suspensionId.length > 64) {
throw new Error('Clock suspension ID must contain 1-64 characters.');
}
const policy = options.policy ?? 'EXACT';
const catchUpTicks = options.catchUpTicks ?? 0;
if (!Number.isSafeInteger(catchUpTicks) || catchUpTicks < 0) {
throw new Error('Clock suspension catch-up ticks must be a non-negative safe integer.');
}
return options.db.$transaction(
async (db) => {
await verifyAuthority(db, options.authority);
await acquireGameSchemaAdvisoryXactLock(db, CLOCK_OPERATION_PERSISTENCE_LOCK);
await acquireGameSchemaAdvisoryXactLock(db, GENERAL_ACCESS_PERSISTENCE_LOCK);
const worldStateId = await lockWorld(db);
const existing = await db.clockSuspension.findUnique({ where: { id: options.suspensionId } });
if (existing) {
if (existing.worldStateId !== worldStateId || existing.source !== options.source || existing.policy !== policy) {
throw new Error(`Clock suspension ID ${options.suspensionId} is already bound to another operation.`);
}
if (existing.status !== 'SUSPENDED') {
throw new Error(`Clock suspension ${options.suspensionId} already advanced to ${existing.status}.`);
}
return {
suspensionId: existing.id,
phase: 'SUSPENDED' as const,
sourceRevision: safeNumber(existing.sourceRevision, 'source revision'),
targetRevision: safeNumber(existing.targetRevision, 'target revision'),
cutTick: safeNumber(existing.cutTick, 'cut tick'),
cutWallAt: existing.cutWallAt,
};
}
const world = await db.worldState.findUniqueOrThrow({ where: { id: worldStateId } });
const phase = parseGameClockPhase(world.clockPhase);
if (phase !== 'RUNNING') {
throw new Error(`Clock suspension can start only from RUNNING; current phase is ${phase}.`);
}
if (!world.clockBaseTime || world.clockTick === null || !world.clockWallAnchor) {
throw new Error('Clock suspension requires a fully initialized logical game clock.');
}
const cutWallAt = await readDbWall(db);
const storedTick = safeNumber(world.clockTick, 'world clock tick');
const sourceRevision = safeNumber(world.clockRevision, 'world clock revision');
const clock = new GameClock({
baseTime: world.clockBaseTime,
tick: storedTick,
mode: world.clockMode === 'manual' ? 'manual' : 'realtime',
wallAnchor: world.clockWallAnchor,
turnSeconds: world.tickSeconds,
phase,
revision: sourceRevision,
});
const cutTick = clock.nowTick(cutWallAt);
await lockParticipants(db, BigInt(cutTick));
await db.worldState.update({
where: { id: worldStateId },
data: { clockPhase: 'SUSPENDED', clockTick: BigInt(cutTick), clockWallAnchor: cutWallAt },
});
const participants = await readParticipantSnapshots(db, worldStateId, BigInt(cutTick));
await db.clockSuspension.create({
data: {
id: options.suspensionId,
worldStateId,
source: options.source,
policy,
status: 'SUSPENDED',
sourceRevision: BigInt(sourceRevision),
targetRevision: BigInt(sourceRevision + 1),
cutTick: BigInt(cutTick),
cutWallAt,
rateTicksPerSecond: GAME_TICKS_PER_TURN / world.tickSeconds,
catchUpTicks: BigInt(catchUpTicks),
participantChecksumBefore: aggregateChecksum(participants),
detail: asJson({ authority: options.authority.kind, profileName: options.authority.profileName }),
},
});
await persistInitialParticipants(db, options.suspensionId, participants);
return {
suspensionId: options.suspensionId,
phase: 'SUSPENDED',
sourceRevision,
targetRevision: sourceRevision + 1,
cutTick,
cutWallAt,
};
},
{ isolationLevel: 'Serializable', maxWait: 10_000, timeout: 30_000 }
);
};
export const reconcileClockSuspension = async (options: {
db: GamePrismaClient;
suspensionId: string;
authority: ClockOperationAuthority;
/** Deterministic fixture seam; production must always use PostgreSQL CURRENT_TIMESTAMP. */
testResumeWallAt?: Date;
}): Promise<ClockReconciliationResult> =>
options.db.$transaction(
async (db) => {
await verifyAuthority(db, options.authority);
await acquireGameSchemaAdvisoryXactLock(db, CLOCK_OPERATION_PERSISTENCE_LOCK);
await acquireGameSchemaAdvisoryXactLock(db, GENERAL_ACCESS_PERSISTENCE_LOCK);
const worldStateId = await lockWorld(db);
const suspension = await db.clockSuspension.findUniqueOrThrow({ where: { id: options.suspensionId } });
if (suspension.worldStateId !== worldStateId) {
throw new Error('Clock suspension belongs to another world state.');
}
if (suspension.status === 'RECONCILING' || suspension.status === 'APPLIED') {
if (
suspension.gapTicks === null ||
suspension.shiftTicks === null ||
suspension.alignedTick === null ||
!suspension.resumeWallAt
) {
throw new Error('Persisted clock reconciliation result is incomplete.');
}
const world = await db.worldState.findUniqueOrThrow({ where: { id: worldStateId } });
return {
suspensionId: suspension.id,
phase: 'RECONCILING' as const,
sourceRevision: safeNumber(suspension.sourceRevision, 'source revision'),
targetRevision: safeNumber(suspension.targetRevision, 'target revision'),
deadlineGeneration: safeNumber(world.deadlineGeneration, 'deadline generation'),
gapTicks: safeNumber(suspension.gapTicks, 'gap ticks'),
catchUpTicks: safeNumber(suspension.catchUpTicks, 'catch-up ticks'),
shiftTicks: safeNumber(suspension.shiftTicks, 'shift ticks'),
alignedTick: safeNumber(suspension.alignedTick, 'aligned tick'),
resumeWallAt: suspension.resumeWallAt,
};
}
if (suspension.status !== 'SUSPENDED') {
throw new Error(`Clock suspension cannot reconcile from status ${suspension.status}.`);
}
const world = await db.worldState.findUniqueOrThrow({ where: { id: worldStateId } });
const phase = parseGameClockPhase(world.clockPhase);
if (phase !== 'SUSPENDED' || world.clockRevision !== suspension.sourceRevision) {
throw new Error('Clock reconciliation phase or source revision fence failed.');
}
const worldMeta =
world.meta && typeof world.meta === 'object' && !Array.isArray(world.meta)
? (world.meta as Record<string, unknown>)
: {};
const united = Number(worldMeta.isunited ?? worldMeta.isUnited ?? 0);
if (suspension.source === 'UNIFICATION_WAIT' || united >= 2) {
throw new Error(
'Unification wait requires the atomic alignment-and-invader workflow; generic resume is forbidden.'
);
}
const cutTick = safeNumber(suspension.cutTick, 'cut tick');
await lockParticipants(db, suspension.cutTick);
if (options.testResumeWallAt && process.env.NODE_ENV !== 'test') {
throw new Error('A clock reconciliation wall override is allowed only in tests.');
}
const resumeWallAt = options.testResumeWallAt
? new Date(options.testResumeWallAt.getTime())
: await readDbWall(db);
const plan = buildClockAlignmentPlan({
policy: parseClockAlignmentPolicy(suspension.policy),
sourceRevision: safeNumber(suspension.sourceRevision, 'source revision'),
cutTick,
cutWall: suspension.cutWallAt,
resumeWall: resumeWallAt,
ticksPerSecond: suspension.rateTicksPerSecond,
catchUpTicks: safeNumber(suspension.catchUpTicks, 'catch-up ticks'),
});
const before = await readParticipantSnapshots(db, worldStateId, suspension.cutTick);
assertShiftFits(before, plan.shiftTicks);
await assertScheduleRanges(db, plan.shiftTicks);
const projectionDeltaMilliseconds = Math.trunc(
(plan.shiftTicks * 1_000) / suspension.rateTicksPerSecond
);
if (!Number.isSafeInteger(projectionDeltaMilliseconds)) {
throw new Error('Clock reconciliation projection delta is outside the safe integer range.');
}
const targetGeneration = world.deadlineGeneration + 1n;
const affected = await applyParticipantShift(
db,
worldStateId,
suspension.cutTick,
BigInt(plan.alignedTick),
BigInt(plan.targetRevision),
targetGeneration,
BigInt(plan.shiftTicks),
projectionDeltaMilliseconds,
resumeWallAt
);
const after = await readParticipantSnapshots(db, worldStateId, suspension.cutTick);
const afterByKey = new Map(after.map((participant) => [participant.key, participant]));
for (const participant of before) {
const next = afterByKey.get(participant.key);
if (!next) throw new Error(`Missing post-reconciliation participant: ${participant.key}`);
if (participant.policy === 'KEEP' && participant.checksum !== next.checksum) {
throw new Error(`KEEP participant changed during reconciliation: ${participant.key}`);
}
await db.clockReconciliationParticipant.upsert({
where: {
suspensionId_participantKey: {
suspensionId: suspension.id,
participantKey: participant.key,
},
},
create: {
suspensionId: suspension.id,
participantKey: participant.key,
policy: participant.policy,
beforeChecksum: participant.checksum,
afterChecksum: next.checksum,
affectedCount: affected.get(participant.key) ?? 0,
},
update: {
policy: participant.policy,
beforeChecksum: participant.checksum,
afterChecksum: next.checksum,
affectedCount: affected.get(participant.key) ?? 0,
},
});
}
const outboxPayload = {
version: 1,
profileName: options.authority.profileName,
suspensionId: suspension.id,
sourceRevision: plan.sourceRevision,
targetRevision: plan.targetRevision,
deadlineGeneration: safeNumber(targetGeneration, 'deadline generation'),
shiftTicks: plan.shiftTicks,
projectionDeltaMilliseconds,
clockBaseTime: world.clockBaseTime!.toISOString(),
ticksPerSecond: suspension.rateTicksPerSecond,
};
await db.clockProjectionOutbox.create({
data: {
worldStateId,
suspensionId: suspension.id,
targetRevision: BigInt(plan.targetRevision),
status: 'PENDING',
payload: asJson(outboxPayload),
checksum: checksum(outboxPayload),
},
});
await db.clockSuspension.update({
where: { id: suspension.id },
data: {
status: 'RECONCILING',
resumeWallAt,
gapTicks: BigInt(plan.gapTicks),
shiftTicks: BigInt(plan.shiftTicks),
alignedTick: BigInt(plan.alignedTick),
participantChecksumBefore: aggregateChecksum(before),
participantChecksumAfter: aggregateChecksum(after),
},
});
return {
suspensionId: suspension.id,
phase: 'RECONCILING',
sourceRevision: plan.sourceRevision,
targetRevision: plan.targetRevision,
deadlineGeneration: safeNumber(targetGeneration, 'deadline generation'),
gapTicks: plan.gapTicks,
catchUpTicks: plan.catchUpTicks,
shiftTicks: plan.shiftTicks,
alignedTick: plan.alignedTick,
resumeWallAt,
};
},
{ isolationLevel: 'Serializable', maxWait: 10_000, timeout: 30_000 }
);
+28
View File
@@ -1218,6 +1218,34 @@ export const createDatabaseTurnHooks = async (
`found ${durableClock.clock_phase}@${durableClock.clock_revision}/${durableClock.deadline_generation}.` `found ${durableClock.clock_phase}@${durableClock.clock_revision}/${durableClock.deadline_generation}.`
); );
} }
if (commandCompletion) {
const commandFence = await prisma.$queryRaw<
Array<{
status: string;
processing_clock_revision: bigint | null;
processing_deadline_generation: bigint | null;
}>
>(GamePrisma.sql`
SELECT status,
processing_clock_revision,
processing_deadline_generation
FROM input_event
WHERE request_id = ${commandCompletion.requestId}
AND target = 'ENGINE'::"InputEventTarget"
FOR UPDATE
`);
const event = commandFence[0];
if (
!event ||
event.status !== 'PROCESSING' ||
event.processing_clock_revision !== expectedRevision ||
event.processing_deadline_generation !== expectedGeneration
) {
throw new Error(
`Input event processing clock fence changed before commit: ${commandCompletion.requestId}.`
);
}
}
let neutralAuctionsToCreate = pendingNeutralAuctions; let neutralAuctionsToCreate = pendingNeutralAuctions;
if (pendingNeutralAuctions.length > 0) { if (pendingNeutralAuctions.length > 0) {
const latestRegistrationKey = const latestRegistrationKey =
+15 -8
View File
@@ -1,4 +1,4 @@
import type { GamePrisma, GamePrismaClient } from '@sammo-ts/infra'; import { readInputEventClockCoordinate, type GamePrisma, type GamePrismaClient } from '@sammo-ts/infra';
import { randomUUID } from 'node:crypto'; import { randomUUID } from 'node:crypto';
import { import {
@@ -137,13 +137,20 @@ const ensureEngineCommand = async (
deltaMinutes, deltaMinutes,
}; };
try { try {
await db.inputEvent.create({ await db.$transaction(async (transaction) => {
data: { const coordinate = await readInputEventClockCoordinate(transaction);
requestId, await transaction.inputEvent.create({
target: 'ENGINE', data: {
eventType: command.type, requestId,
payload: asJson(command), target: 'ENGINE',
}, eventType: command.type,
payload: asJson(command),
acceptedGameTick: coordinate.gameTick,
acceptedClockRevision: coordinate.clockRevision,
acceptedDeadlineGeneration: coordinate.deadlineGeneration,
createdAt: coordinate.wallAt,
},
});
}); });
} catch (error) { } catch (error) {
if (!isUniqueConflict(error)) { if (!isUniqueConflict(error)) {
@@ -10,7 +10,7 @@ import {
type TurnDaemonCommand, type TurnDaemonCommand,
type TurnDaemonCommandResult, type TurnDaemonCommandResult,
} from '@sammo-ts/common'; } from '@sammo-ts/common';
import type { GamePrisma, GamePrismaClient } from '@sammo-ts/infra'; import { readInputEventClockCoordinate, type GamePrisma, type GamePrismaClient } from '@sammo-ts/infra';
import type { GatewayAdminActionRecord, GatewayAdminActionResult } from './gatewayAdminActions.js'; import type { GatewayAdminActionRecord, GatewayAdminActionResult } from './gatewayAdminActions.js';
@@ -97,13 +97,20 @@ const ensureEngineCommand = async (
settings, settings,
}; };
try { try {
await db.inputEvent.create({ await db.$transaction(async (transaction) => {
data: { const coordinate = await readInputEventClockCoordinate(transaction);
requestId, await transaction.inputEvent.create({
target: 'ENGINE', data: {
eventType: command.type, requestId,
payload: asJson(command), target: 'ENGINE',
}, eventType: command.type,
payload: asJson(command),
acceptedGameTick: coordinate.gameTick,
acceptedClockRevision: coordinate.clockRevision,
acceptedDeadlineGeneration: coordinate.deadlineGeneration,
createdAt: coordinate.wallAt,
},
});
}); });
} catch (error) { } catch (error) {
if (!isUniqueConflict(error)) throw error; if (!isUniqueConflict(error)) throw error;
@@ -439,15 +439,20 @@ export const reserveSelectionPool = async (options: {
userId: string; userId: string;
now?: Date; now?: Date;
acceptedGameTick?: number; acceptedGameTick?: number;
processingGameTick?: number;
seedOwnerIdentity?: string | number; seedOwnerIdentity?: string | number;
}): Promise<SelectPoolReservationDto> => { }): Promise<SelectPoolReservationDto> => {
const { db, world, worldState, userId } = options; const { db, world, worldState, userId } = options;
requirePoolWorld(worldState); requirePoolWorld(worldState);
const now = options.now ?? new Date(); const now = options.now ?? new Date();
const acceptedGameTick = options.acceptedGameTick ?? resolveAcceptedGameTick(world, now); const acceptedGameTick = options.acceptedGameTick ?? resolveAcceptedGameTick(world, now);
const processingGameTick = options.processingGameTick ?? acceptedGameTick;
if (!Number.isSafeInteger(acceptedGameTick)) { if (!Number.isSafeInteger(acceptedGameTick)) {
fail('INTERNAL_SERVER_ERROR', '장수 선택 예약 tick이 안전한 정수 범위를 벗어났습니다.'); fail('INTERNAL_SERVER_ERROR', '장수 선택 예약 tick이 안전한 정수 범위를 벗어났습니다.');
} }
if (!Number.isSafeInteger(processingGameTick)) {
fail('INTERNAL_SERVER_ERROR', '장수 선택 처리 tick이 안전한 정수 범위를 벗어났습니다.');
}
await lockSelectionUser(db, userId); await lockSelectionUser(db, userId);
await lockSelectionMutationTables(db); await lockSelectionMutationTables(db);
const general = await db.general.findFirst({ const general = await db.general.findFirst({
@@ -461,7 +466,7 @@ export const reserveSelectionPool = async (options: {
let currentRows = await synchronizeSelectionPoolWorld(db, world); let currentRows = await synchronizeSelectionPoolWorld(db, world);
const existing = currentRows.filter( const existing = currentRows.filter(
(row) => row.ownerUserId === userId && row.generalId === null && isReservationActive(row, now, acceptedGameTick) (row) => row.ownerUserId === userId && row.generalId === null && isReservationActive(row, now, processingGameTick)
); );
if (existing.length > 0) { if (existing.length > 0) {
return toReservationDto(existing, Boolean(general), worldState, world); return toReservationDto(existing, Boolean(general), worldState, world);
@@ -471,7 +476,7 @@ export const reserveSelectionPool = async (options: {
where: { where: {
generalId: null, generalId: null,
OR: [ OR: [
{ reservedUntilTick: { lt: BigInt(acceptedGameTick) } }, { reservedUntilTick: { lt: BigInt(processingGameTick) } },
{ reservedUntilTick: null, reservedUntil: { lt: now } }, { reservedUntilTick: null, reservedUntil: { lt: now } },
], ],
}, },
@@ -483,7 +488,7 @@ export const reserveSelectionPool = async (options: {
}); });
currentRows = await synchronizeSelectionPoolWorld(db, world); currentRows = await synchronizeSelectionPoolWorld(db, world);
const availableIds = new Set( const availableIds = new Set(
world.listGeneralPoolCandidates(now, acceptedGameTick)?.map((candidate) => candidate.poolEntryId) ?? [] world.listGeneralPoolCandidates(now, processingGameTick)?.map((candidate) => candidate.poolEntryId) ?? []
); );
const available = currentRows.filter( const available = currentRows.filter(
(row) => (row) =>
@@ -507,7 +512,7 @@ export const reserveSelectionPool = async (options: {
(row) => (row) =>
[row, calculateSelectionCandidateWeight(poolName, parseCandidate(row), true)] as [SelectPoolRow, number] [row, calculateSelectionCandidateWeight(poolName, parseCandidate(row), true)] as [SelectPoolRow, number]
); );
const reservedUntilTick = acceptedGameTick + RESERVATION_TURN_MULTIPLIER * GAME_TICKS_PER_TURN; const reservedUntilTick = processingGameTick + RESERVATION_TURN_MULTIPLIER * GAME_TICKS_PER_TURN;
if (!Number.isSafeInteger(reservedUntilTick)) { if (!Number.isSafeInteger(reservedUntilTick)) {
fail('INTERNAL_SERVER_ERROR', '장수 선택 예약 tick이 안전한 정수 범위를 벗어났습니다.'); fail('INTERNAL_SERVER_ERROR', '장수 선택 예약 tick이 안전한 정수 범위를 벗어났습니다.');
} }
@@ -14,8 +14,12 @@ interface TournamentState {
openMonth: number; openMonth: number;
termSeconds: number; termSeconds: number;
nextAt: string; nextAt: string;
nextTick?: number;
clockRevision?: number;
deadlineGeneration?: number;
bettingId?: number; bettingId?: number;
bettingCloseAt?: string; bettingCloseAt?: string;
bettingCloseTick?: number;
winnerId?: number; winnerId?: number;
bettingSettled?: boolean; bettingSettled?: boolean;
rewardSettled?: boolean; rewardSettled?: boolean;
@@ -76,6 +80,9 @@ export const createTournamentAutoStartHandler = (options: {
sourceRevisionKey: `sammo:${options.profileName}:tournament:source-revision`, sourceRevisionKey: `sammo:${options.profileName}:tournament:source-revision`,
sourceRevisionChannel: `sammo:${options.profileName}:tournament:source-changed`, sourceRevisionChannel: `sammo:${options.profileName}:tournament:source-changed`,
realtimeEventChannel: buildGameEventChannel(options.profileName), realtimeEventChannel: buildGameEventChannel(options.profileName),
activeClockRevisionKey: `sammo:${options.profileName}:clock:active-revision`,
deadlineGenerationKey: `sammo:${options.profileName}:clock:deadline-generation`,
clockPhaseKey: `sammo:${options.profileName}:clock:phase`,
}; };
return { return {
onMonthChanged: async (context) => { onMonthChanged: async (context) => {
@@ -118,6 +125,8 @@ export const createTournamentAutoStartHandler = (options: {
previousState && Number.isFinite(previousState.termSeconds) && previousState.termSeconds > 0 previousState && Number.isFinite(previousState.termSeconds) && previousState.termSeconds > 0
? previousState.termSeconds ? previousState.termSeconds
: resolveTermSeconds(state.tickSeconds); : resolveTermSeconds(state.tickSeconds);
const nextAt = new Date(now.getTime() + termSeconds * 60_000);
const clockState = world.getGameClockState();
const nextState: TournamentState = { const nextState: TournamentState = {
stage: 1, stage: 1,
phase: 0, phase: 0,
@@ -129,7 +138,10 @@ export const createTournamentAutoStartHandler = (options: {
// Ref startTournament() passes calcTournamentTerm()'s seconds // Ref startTournament() passes calcTournamentTerm()'s seconds
// value to DateInterval's minute field. Preserve that historical // value to DateInterval's minute field. Preserve that historical
// initial enrollment delay; later tournament phases use seconds. // initial enrollment delay; later tournament phases use seconds.
nextAt: new Date(now.getTime() + termSeconds * 60_000).toISOString(), nextAt: nextAt.toISOString(),
nextTick: world.dateToGameTick(nextAt),
clockRevision: clockState.revision,
deadlineGeneration: clockState.deadlineGeneration,
bettingId: bettingId:
typeof previousState?.bettingId === 'number' && Number.isFinite(previousState.bettingId) typeof previousState?.bettingId === 'number' && Number.isFinite(previousState.bettingId)
? previousState.bettingId + 1 ? previousState.bettingId + 1
@@ -142,12 +154,26 @@ export const createTournamentAutoStartHandler = (options: {
lastError: undefined, lastError: undefined,
lastErrorAt: undefined, lastErrorAt: undefined,
}; };
await writeTournamentProjection(redis, keys, [ await writeTournamentProjection(
{ key: keys.participantsKey, value: [] }, redis,
{ key: keys.matchesKey, value: [] }, keys,
{ key: keys.bettingKey, value: [] }, [
{ key: keys.stateKey, value: nextState }, { key: keys.participantsKey, value: [] },
]); { key: keys.matchesKey, value: [] },
{ key: keys.bettingKey, value: [] },
{ key: keys.stateKey, value: nextState },
],
clockState.phase === 'RUNNING'
? {
activeRevisionKey: keys.activeClockRevisionKey,
deadlineGenerationKey: keys.deadlineGenerationKey,
phaseKey: keys.clockPhaseKey,
revision: clockState.revision,
deadlineGeneration: clockState.deadlineGeneration,
phase: 'RUNNING',
}
: undefined
);
const [typeText, generalTypeText] = TOURNAMENT_TEXT[type] ?? TOURNAMENT_TEXT[0]; const [typeText, generalTypeText] = TOURNAMENT_TEXT[type] ?? TOURNAMENT_TEXT[0];
const emperor = world const emperor = world
@@ -274,6 +274,10 @@ const resolveSelectionCommandAcceptedAt = async (
command: Extract<TurnDaemonCommand, { type: 'selectPoolReserve' | 'selectPoolCreate' | 'selectPoolReselect' }> command: Extract<TurnDaemonCommand, { type: 'selectPoolReserve' | 'selectPoolCreate' | 'selectPoolReselect' }>
): Promise<Date> => { ): Promise<Date> => {
const operationalAcceptedAt = await resolveCommandAcceptedAt(db, command); const operationalAcceptedAt = await resolveCommandAcceptedAt(db, command);
const processingGameTick = Reflect.get(command, 'processingGameTick');
if (typeof processingGameTick === 'number' && Number.isSafeInteger(processingGameTick)) {
return world.gameTickToDate(processingGameTick);
}
if (command.acceptedGameTick !== undefined) { if (command.acceptedGameTick !== undefined) {
return world.gameTickToDate(command.acceptedGameTick); return world.gameTickToDate(command.acceptedGameTick);
} }
@@ -407,9 +411,13 @@ async function handleNpcPossessGeneral(
throw new Error('NPC possession world state is missing.'); throw new Error('NPC possession world state is missing.');
} }
const operationalAcceptedAt = await resolveCommandAcceptedAt(db, command); const operationalAcceptedAt = await resolveCommandAcceptedAt(db, command);
const acceptedAt = command.acceptedGameAt const processingGameTick = Reflect.get(command, 'processingGameTick');
? new Date(command.acceptedGameAt) const acceptedAt =
: ctx.world.getGameNow(operationalAcceptedAt); typeof processingGameTick === 'number' && Number.isSafeInteger(processingGameTick)
? ctx.world.gameTickToDate(processingGameTick)
: command.acceptedGameAt
? new Date(command.acceptedGameAt)
: ctx.world.getGameNow(operationalAcceptedAt);
try { try {
return { return {
type: 'npcPossessGeneral', type: 'npcPossessGeneral',
@@ -451,8 +459,11 @@ async function handleSelectPoolCreate(
throw new Error('Selection-pool world state is missing.'); throw new Error('Selection-pool world state is missing.');
} }
const operationalAcceptedAt = await resolveCommandAcceptedAt(db, command); const operationalAcceptedAt = await resolveCommandAcceptedAt(db, command);
const processingGameTick = Reflect.get(command, 'processingGameTick');
const acceptedAt = const acceptedAt =
command.acceptedGameTick !== undefined typeof processingGameTick === 'number' && Number.isSafeInteger(processingGameTick)
? ctx.world.gameTickToDate(processingGameTick)
: command.acceptedGameTick !== undefined
? ctx.world.gameTickToDate(command.acceptedGameTick) ? ctx.world.gameTickToDate(command.acceptedGameTick)
: command.acceptedGameAt !== undefined : command.acceptedGameAt !== undefined
? new Date(command.acceptedGameAt) ? new Date(command.acceptedGameAt)
@@ -515,6 +526,9 @@ async function handleSelectPoolReserve(
seedOwnerIdentity: command.seedOwnerIdentity, seedOwnerIdentity: command.seedOwnerIdentity,
now: acceptedAt, now: acceptedAt,
...(command.acceptedGameTick === undefined ? {} : { acceptedGameTick: command.acceptedGameTick }), ...(command.acceptedGameTick === undefined ? {} : { acceptedGameTick: command.acceptedGameTick }),
...(typeof Reflect.get(command, 'processingGameTick') === 'number'
? { processingGameTick: Reflect.get(command, 'processingGameTick') as number }
: {}),
}), }),
}; };
} catch (error) { } catch (error) {
@@ -2770,9 +2784,15 @@ const validateVoteSelectionInTransaction = async (
if (!poll) return '설문조사가 없습니다.'; if (!poll) return '설문조사가 없습니다.';
const processingNow = ctx.world.getGameNow(new Date()); const processingNow = ctx.world.getGameNow(new Date());
const acceptedGameTick = command.acceptedGameTick ?? ctx.world.dateToGameTick(processingNow); const convertedProcessingTick = Reflect.get(command, 'processingGameTick');
const acceptedGameTick =
typeof convertedProcessingTick === 'number' && Number.isSafeInteger(convertedProcessingTick)
? convertedProcessingTick
: (command.acceptedGameTick ?? ctx.world.dateToGameTick(processingNow));
const acceptedGameAt = const acceptedGameAt =
command.acceptedGameTick === undefined ? processingNow : ctx.world.gameTickToDate(command.acceptedGameTick); command.acceptedGameTick === undefined && convertedProcessingTick === undefined
? processingNow
: ctx.world.gameTickToDate(acceptedGameTick);
if (hasVotePollDeadlinePassed(poll, acceptedGameAt, acceptedGameTick)) { if (hasVotePollDeadlinePassed(poll, acceptedGameAt, acceptedGameTick)) {
return '설문조사가 종료되었습니다.'; return '설문조사가 종료되었습니다.';
} }
@@ -0,0 +1,364 @@
import { afterAll, beforeAll, beforeEach, describe, expect, it } from 'vitest';
import { GameClock } from '@sammo-ts/common';
import {
createGamePostgresConnector,
createRedisConnector,
type GamePrismaClient,
type RedisConnector,
} from '@sammo-ts/infra';
import { reconcileClockSuspension, startClockSuspension } from '../src/turn/clockReconciliation.js';
import { applyNextClockProjection } from '../src/turn/clockProjectionOutbox.js';
const enabled =
process.env.CLOCK_RECONCILIATION_INTEGRATION === '1' &&
Boolean(process.env.DATABASE_URL) &&
Boolean(process.env.REDIS_URL);
const describeIntegration = enabled ? describe : describe.skip;
describeIntegration('durable clock reconciliation', () => {
let db: GamePrismaClient;
let disconnect: (() => Promise<void>) | undefined;
let redis: RedisConnector;
beforeAll(async () => {
const connector = createGamePostgresConnector({ url: process.env.DATABASE_URL! });
db = connector.prisma;
disconnect = connector.disconnect;
redis = createRedisConnector({ url: process.env.REDIS_URL! });
await redis.connect();
});
afterAll(async () => {
await redis.disconnect();
await disconnect?.();
});
beforeEach(async () => {
await redis.client.flushDb();
await db.$transaction([
db.clockProjectionOutbox.deleteMany(),
db.clockReconciliationParticipant.deleteMany(),
db.clockSuspension.deleteMany(),
db.inputEvent.deleteMany(),
db.vote.deleteMany(),
db.voteComment.deleteMany(),
db.votePoll.deleteMany(),
db.message.deleteMany(),
db.auctionBid.deleteMany(),
db.auction.deleteMany(),
db.npcSelectionToken.deleteMany(),
db.selectPoolEntry.deleteMany(),
db.general.deleteMany(),
db.turnDaemonLease.deleteMany(),
db.worldState.deleteMany(),
]);
});
it('preserves every remaining deadline and occurrence across a 65m17.250s exact gap', async () => {
const baseTime = new Date('2026-01-01T00:00:00.000Z');
const futureAnchor = new Date(Date.now() + 3_600_000);
const initialTick = 1_000_000;
const lastTurnTick = 900_000;
const clock = new GameClock({
baseTime,
tick: initialTick,
mode: 'realtime',
wallAnchor: futureAnchor,
turnSeconds: 600,
phase: 'RUNNING',
revision: 1,
});
const generalTicks = [initialTick + 1_234, initialTick + 36_000_123];
const auctionCloseTick = initialTick + 72_000_777;
const messageOccurrenceTick = initialTick - 500;
const messageExpiryTick = initialTick + 90_000_999;
const voteStartTick = initialTick - 200;
const voteEndTick = initialTick + 18_000_321;
const poolTick = initialTick + 2_000_111;
const npcValidTick = initialTick + 3_000_222;
const npcMoreTick = initialTick + 1_000_333;
const world = await db.worldState.create({
data: {
scenarioCode: 'clock-test',
currentYear: 180,
currentMonth: 1,
tickSeconds: 600,
clockBaseTime: baseTime,
clockTick: BigInt(initialTick),
clockMode: 'realtime',
clockWallAnchor: futureAnchor,
lastTurnTick: BigInt(lastTurnTick),
clockPhase: 'RUNNING',
clockRevision: 1n,
deadlineGeneration: 7n,
meta: {
lastTurnTime: clock.tickToDate(lastTurnTick).toISOString(),
starttime: clock.tickToDate(initialTick + 100).toISOString(),
},
},
});
await db.general.createMany({
data: generalTicks.map((turnTick, index) => ({
id: index + 1,
name: `general-${index + 1}`,
turnTick: BigInt(turnTick),
turnTime: clock.tickToDate(turnTick),
recentWarTick: BigInt(initialTick - 100 - index),
recentWarTime: clock.tickToDate(initialTick - 100 - index),
})),
});
await db.auction.create({
data: {
type: 'BUY_RICE',
hostGeneralId: 1,
status: 'FINALIZING',
openTick: BigInt(initialTick - 300),
closeTick: BigInt(auctionCloseTick),
closeAt: clock.tickToDate(auctionCloseTick),
},
});
await db.message.create({
data: {
mailbox: 1,
type: 'private',
src: 1,
dest: 2,
time: clock.tickToDate(messageOccurrenceTick),
timeTick: BigInt(messageOccurrenceTick),
validUntil: clock.tickToDate(messageExpiryTick),
validUntilTick: BigInt(messageExpiryTick),
message: {},
},
});
await db.votePoll.create({
data: {
title: 'clock vote',
options: ['yes', 'no'],
revealMode: 'AFTER_VOTE',
openerGeneralId: 1,
openerName: 'general-1',
startAt: clock.tickToDate(voteStartTick),
startTick: BigInt(voteStartTick),
endAt: clock.tickToDate(voteEndTick),
endTick: BigInt(voteEndTick),
},
});
await db.selectPoolEntry.create({
data: {
uniqueName: 'clock-pool',
reservedUntil: clock.tickToDate(poolTick),
reservedUntilTick: BigInt(poolTick),
info: {},
},
});
await db.npcSelectionToken.create({
data: {
ownerUserId: 'clock-user',
validUntil: clock.tickToDate(npcValidTick),
validUntilTick: BigInt(npcValidTick),
pickMoreFrom: clock.tickToDate(npcMoreTick),
pickMoreFromTick: BigInt(npcMoreTick),
pickResult: [],
nonce: 1,
},
});
const authority = { kind: 'OFFLINE' as const, profileName: 'clock-test', reason: 'integration fixture' };
const suspended = await startClockSuspension({
db,
suspensionId: 'clock-gap-65m17s250',
source: 'MAINTENANCE',
authority,
});
expect(suspended.cutTick).toBe(initialTick);
expect((await db.worldState.findUniqueOrThrow({ where: { id: world.id } })).clockPhase).toBe('SUSPENDED');
const resumeWallAt = new Date(suspended.cutWallAt.getTime() + 65 * 60_000 + 17_250);
const reconciled = await reconcileClockSuspension({
db,
suspensionId: suspended.suspensionId,
authority,
testResumeWallAt: resumeWallAt,
});
expect(reconciled).toMatchObject({
phase: 'RECONCILING',
sourceRevision: 1,
targetRevision: 2,
deadlineGeneration: 8,
gapTicks: 235_035_000,
shiftTicks: 235_035_000,
alignedTick: 236_035_000,
});
const [afterWorld, generals, auction, message, vote, pool, token, ledger, outboxes] = await Promise.all([
db.worldState.findUniqueOrThrow({ where: { id: world.id } }),
db.general.findMany({ orderBy: { id: 'asc' } }),
db.auction.findFirstOrThrow(),
db.message.findFirstOrThrow(),
db.votePoll.findFirstOrThrow(),
db.selectPoolEntry.findFirstOrThrow(),
db.npcSelectionToken.findFirstOrThrow(),
db.clockSuspension.findUniqueOrThrow({ where: { id: suspended.suspensionId } }),
db.clockProjectionOutbox.findMany(),
]);
const alignedTick = BigInt(reconciled.alignedTick);
expect(afterWorld).toMatchObject({
clockPhase: 'RECONCILING',
clockRevision: 2n,
deadlineGeneration: 8n,
clockTick: alignedTick,
lastTurnTick: BigInt(lastTurnTick + reconciled.shiftTicks),
});
expect(generals.map((general) => general.turnTick! - alignedTick)).toEqual(
generalTicks.map((tick) => BigInt(tick - initialTick))
);
expect(auction.closeTick! - alignedTick).toBe(BigInt(auctionCloseTick - initialTick));
expect(message.validUntilTick! - alignedTick).toBe(BigInt(messageExpiryTick - initialTick));
expect(vote.endTick! - alignedTick).toBe(BigInt(voteEndTick - initialTick));
expect(pool.reservedUntilTick! - alignedTick).toBe(BigInt(poolTick - initialTick));
expect(token.validUntilTick! - alignedTick).toBe(BigInt(npcValidTick - initialTick));
expect(token.pickMoreFromTick! - alignedTick).toBe(BigInt(npcMoreTick - initialTick));
expect(generals.map((general) => general.recentWarTick)).toEqual([
BigInt(initialTick - 100),
BigInt(initialTick - 101),
]);
expect(auction.openTick).toBe(BigInt(initialTick - 300));
expect(message.timeTick).toBe(BigInt(messageOccurrenceTick));
expect(vote.startTick).toBe(BigInt(voteStartTick));
expect(ledger.status).toBe('RECONCILING');
expect(outboxes).toHaveLength(1);
expect(outboxes[0]).toMatchObject({ status: 'PENDING', targetRevision: 2n });
const retried = await reconcileClockSuspension({
db,
suspensionId: suspended.suspensionId,
authority,
testResumeWallAt: new Date(resumeWallAt.getTime() + 10_000),
});
expect(retried).toEqual(reconciled);
expect(await db.clockProjectionOutbox.count()).toBe(1);
const keepParticipants = await db.clockReconciliationParticipant.findMany({ where: { policy: 'KEEP' } });
expect(keepParticipants.every((participant) => participant.beforeChecksum === participant.afterChecksum)).toBe(
true
);
await redis.client.set('sammo:clock-test:clock:active-revision', '1');
expect(
await applyNextClockProjection({ db, redis: redis.client, workerId: 'clock-projection-success' })
).toBe('APPLIED');
expect(await redis.client.get('sammo:clock-test:clock:active-revision')).toBe('2');
expect(await redis.client.get('sammo:clock-test:clock:deadline-generation')).toBe('8');
expect(await redis.client.get('sammo:clock-test:clock:phase')).toBe('RUNNING');
expect(await redis.client.zRangeWithScores('sammo:clock-test:auction:timer', 0, -1)).toEqual([
{ value: String(auction.id), score: Number(auction.closeTick) },
]);
expect(await db.worldState.findUniqueOrThrow({ where: { id: world.id } })).toMatchObject({
clockPhase: 'RUNNING',
clockRevision: 2n,
});
expect(await db.clockProjectionOutbox.findFirstOrThrow()).toMatchObject({ status: 'APPLIED' });
});
it('rejects a live offline fence and preserves a turn deadline across an exact 24-hour gap', async () => {
const baseTime = new Date('2026-02-01T00:00:00.000Z');
const futureAnchor = new Date(Date.now() + 3_600_000);
const initialTick = 5 * 36_000_000;
const turnTick = initialTick + 17_000_007;
const clock = new GameClock({
baseTime,
tick: initialTick,
mode: 'realtime',
wallAnchor: futureAnchor,
turnSeconds: 3_600,
phase: 'RUNNING',
});
await db.worldState.create({
data: {
scenarioCode: 'clock-day-test',
currentYear: 180,
currentMonth: 1,
tickSeconds: 3_600,
clockBaseTime: baseTime,
clockTick: BigInt(initialTick),
clockMode: 'realtime',
clockWallAnchor: futureAnchor,
lastTurnTick: BigInt(initialTick),
clockPhase: 'RUNNING',
clockRevision: 3n,
deadlineGeneration: 2n,
},
});
await db.general.create({
data: { id: 1, name: 'day-general', turnTick: BigInt(turnTick), turnTime: clock.tickToDate(turnTick) },
});
await db.turnDaemonLease.create({
data: {
profile: 'clock-day-test',
ownerId: 'other-daemon',
fencingEpoch: 9n,
leaseUntil: new Date(Date.now() + 60_000),
},
});
const authority = {
kind: 'OFFLINE' as const,
profileName: 'clock-day-test',
reason: '24-hour integration fixture',
};
await expect(
startClockSuspension({
db,
suspensionId: 'clock-gap-24h',
source: 'MAINTENANCE',
authority,
})
).rejects.toThrow('daemon lease to be offline');
await db.turnDaemonLease.delete({ where: { profile: 'clock-day-test' } });
const suspended = await startClockSuspension({
db,
suspensionId: 'clock-gap-24h',
source: 'MAINTENANCE',
authority,
});
const reconciled = await reconcileClockSuspension({
db,
suspensionId: suspended.suspensionId,
authority,
testResumeWallAt: new Date(suspended.cutWallAt.getTime() + 24 * 60 * 60_000),
});
expect(reconciled).toMatchObject({
sourceRevision: 3,
targetRevision: 4,
gapTicks: 24 * 36_000_000,
shiftTicks: 24 * 36_000_000,
alignedTick: initialTick + 24 * 36_000_000,
});
const shifted = await db.general.findUniqueOrThrow({ where: { id: 1 } });
expect(shifted.turnTick! - BigInt(reconciled.alignedTick)).toBe(BigInt(turnTick - initialTick));
await redis.client.set('sammo:clock-day-test:clock:active-revision', '3');
const redisThenCrash = {
get: redis.client.get.bind(redis.client),
eval: async (script: string, options: { keys: string[]; arguments: string[] }) => {
await redis.client.eval(script, options);
throw new Error('fixture crash after Redis commit');
},
};
await expect(
applyNextClockProjection({ db, redis: redisThenCrash, workerId: 'clock-projection-crash' })
).rejects.toThrow('fixture crash after Redis commit');
expect(await redis.client.get('sammo:clock-day-test:clock:active-revision')).toBe('4');
expect(await db.worldState.findFirstOrThrow()).toMatchObject({ clockPhase: 'RECONCILING' });
expect(await db.clockProjectionOutbox.findFirstOrThrow()).toMatchObject({ status: 'FAILED', attempts: 1 });
await db.clockProjectionOutbox.updateMany({ data: { availableAt: new Date(0) } });
expect(
await applyNextClockProjection({ db, redis: redis.client, workerId: 'clock-projection-restart' })
).toBe('RECOVERED');
expect(await db.worldState.findFirstOrThrow()).toMatchObject({ clockPhase: 'RUNNING', clockRevision: 4n });
expect(await db.clockProjectionOutbox.findFirstOrThrow()).toMatchObject({ status: 'APPLIED', attempts: 2 });
});
});
@@ -1,4 +1,4 @@
import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest'; import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest';
import type { TurnDaemonCommand } from '@sammo-ts/common'; import type { TurnDaemonCommand } from '@sammo-ts/common';
import { createGamePostgresConnector } from '@sammo-ts/infra'; import { createGamePostgresConnector } from '@sammo-ts/infra';
@@ -25,6 +25,12 @@ integration('database command queue', () => {
}); });
}); });
beforeEach(async () => {
await db.inputEvent.deleteMany({ where: { requestId: { startsWith: 'integration:engine:' } } });
await db.clockSuspension.deleteMany({ where: { id: 'integration-queue-revision-8-9' } });
await db.worldState.updateMany({ data: { clockPhase: 'RUNNING' } });
});
afterAll(async () => { afterAll(async () => {
await db.inputEvent.deleteMany({ await db.inputEvent.deleteMany({
where: { requestId: { startsWith: 'integration:engine:' } }, where: { requestId: { startsWith: 'integration:engine:' } },
@@ -229,4 +235,119 @@ integration('database command queue', () => {
expect(handle).toHaveBeenCalledOnce(); expect(handle).toHaveBeenCalledOnce();
expect(mutation).not.toHaveBeenCalled(); expect(mutation).not.toHaveBeenCalled();
}); });
it('dequeues gameplay only in an executable phase and records the processing clock generation', async () => {
const existingWorld = await db.worldState.findFirst({ orderBy: { id: 'asc' } });
const world = existingWorld
? await db.worldState.update({
where: { id: existingWorld.id },
data: { clockPhase: 'SUSPENDED', clockRevision: 9n, deadlineGeneration: 4n, clockTick: 123n },
})
: await db.worldState.create({
data: {
scenarioCode: 'queue-clock-test',
currentYear: 180,
currentMonth: 1,
tickSeconds: 600,
clockPhase: 'SUSPENDED',
clockRevision: 9n,
deadlineGeneration: 4n,
clockTick: 123n,
},
});
const gameplayId = 'integration:engine:clock-gated-gameplay';
const statusId = 'integration:engine:clock-gated-status';
const staleId = 'integration:engine:clock-gated-stale';
await db.inputEvent.createMany({
data: [
{
requestId: gameplayId,
target: 'ENGINE',
eventType: 'vacation',
actorUserId: 'user-7',
acceptedGameTick: 100n,
acceptedClockRevision: 9n,
acceptedDeadlineGeneration: 4n,
payload: { type: 'vacation', requestId: gameplayId, userId: 'user-7', generalId: 7 },
},
{
requestId: statusId,
target: 'ENGINE',
eventType: 'getStatus',
acceptedGameTick: 100n,
acceptedClockRevision: 9n,
acceptedDeadlineGeneration: 4n,
payload: { type: 'getStatus', requestId: statusId },
},
{
requestId: staleId,
target: 'ENGINE',
eventType: 'vacation',
actorUserId: 'user-8',
acceptedGameTick: 90n,
acceptedClockRevision: 8n,
acceptedDeadlineGeneration: 3n,
payload: { type: 'vacation', requestId: staleId, userId: 'user-8', generalId: 8 },
},
],
});
const queue = new DatabaseTurnDaemonCommandQueue(db);
expect(await queue.drain()).toEqual([{ type: 'getStatus', requestId: statusId }]);
expect(await db.inputEvent.findUniqueOrThrow({ where: { requestId: gameplayId } })).toMatchObject({
status: 'PENDING',
processingClockRevision: null,
});
await db.worldState.update({ where: { id: world.id }, data: { clockPhase: 'RUNNING' } });
expect(await queue.drain()).toEqual([
{ type: 'vacation', requestId: gameplayId, userId: 'user-7', generalId: 7 },
]);
expect(await db.inputEvent.findUniqueOrThrow({ where: { requestId: gameplayId } })).toMatchObject({
status: 'PROCESSING',
processingGameTick: 100n,
processingClockRevision: 9n,
processingDeadlineGeneration: 4n,
});
expect(await db.inputEvent.findUniqueOrThrow({ where: { requestId: staleId } })).toMatchObject({
status: 'PENDING',
processingClockRevision: null,
});
await db.clockSuspension.deleteMany({ where: { id: 'integration-queue-revision-8-9' } });
await db.clockSuspension.create({
data: {
id: 'integration-queue-revision-8-9',
worldStateId: world.id,
source: 'MAINTENANCE',
policy: 'EXACT',
status: 'APPLIED',
sourceRevision: 8n,
targetRevision: 9n,
cutTick: 90n,
cutWallAt: new Date(),
resumeWallAt: new Date(),
rateTicksPerSecond: 60_000,
gapTicks: 33n,
shiftTicks: 33n,
alignedTick: 123n,
},
});
expect(await queue.drain()).toEqual([
{
type: 'vacation',
requestId: staleId,
userId: 'user-8',
generalId: 8,
processingGameTick: 123,
},
]);
expect(await db.inputEvent.findUniqueOrThrow({ where: { requestId: staleId } })).toMatchObject({
status: 'PROCESSING',
acceptedGameTick: 90n,
acceptedClockRevision: 8n,
processingGameTick: 123n,
processingClockRevision: 9n,
processingDeadlineGeneration: 4n,
});
});
}); });
@@ -448,6 +448,21 @@ describe('runtime clock shift projection', () => {
return {}; return {};
}); });
const db = { const db = {
$transaction: async (operation: (transaction: GamePrismaClient) => Promise<unknown>) => operation(db),
$executeRaw: vi.fn(async () => 1),
$queryRaw: vi.fn(async () => [{ wallNow: new Date('2026-07-30T10:00:00.000Z') }]),
worldState: {
findFirst: vi.fn(async () => ({
clockBaseTime: new Date('2026-07-30T10:00:00.000Z'),
clockTick: 0n,
clockMode: 'realtime',
clockWallAnchor: new Date('2026-07-30T10:00:00.000Z'),
tickSeconds: 600,
clockPhase: 'RUNNING',
clockRevision: 1n,
deadlineGeneration: 1n,
})),
},
inputEvent: { inputEvent: {
create: inputEventCreate, create: inputEventCreate,
findUniqueOrThrow: vi.fn(async () => findUniqueOrThrow: vi.fn(async () =>
@@ -567,6 +582,21 @@ describe('runtime game settings projection', () => {
let eventStatus: 'PENDING' | 'SUCCEEDED' = 'PENDING'; let eventStatus: 'PENDING' | 'SUCCEEDED' = 'PENDING';
let created = false; let created = false;
const db = { const db = {
$transaction: async (operation: (transaction: GamePrismaClient) => Promise<unknown>) => operation(db),
$executeRaw: vi.fn(async () => 1),
$queryRaw: vi.fn(async () => [{ wallNow: new Date('2026-07-30T10:00:00.000Z') }]),
worldState: {
findFirst: vi.fn(async () => ({
clockBaseTime: new Date('2026-07-30T10:00:00.000Z'),
clockTick: 0n,
clockMode: 'realtime',
clockWallAnchor: new Date('2026-07-30T10:00:00.000Z'),
tickSeconds: 600,
clockPhase: 'RUNNING',
clockRevision: 1n,
deadlineGeneration: 1n,
})),
},
inputEvent: { inputEvent: {
create: vi.fn(async () => { create: vi.fn(async () => {
if (created) throw { code: 'P2002' }; if (created) throw { code: 'P2002' };
@@ -1,4 +1,4 @@
import { afterAll, beforeAll, describe, expect, it } from 'vitest'; import { afterAll, beforeAll, beforeEach, describe, expect, it } from 'vitest';
import { createGamePostgresConnector, type GamePrisma, type GamePrismaClient } from '@sammo-ts/infra'; import { createGamePostgresConnector, type GamePrisma, type GamePrismaClient } from '@sammo-ts/infra';
import { GAME_TICKS_PER_TURN } from '@sammo-ts/common'; import { GAME_TICKS_PER_TURN } from '@sammo-ts/common';
@@ -77,38 +77,32 @@ integration('runtime clock shift persistence', () => {
let db: GamePrismaClient; let db: GamePrismaClient;
let closeDb: (() => Promise<void>) | undefined; let closeDb: (() => Promise<void>) | undefined;
const cleanupFixtures = async (): Promise<void> => {
await db.inputEvent.deleteMany({ where: { requestId: { in: [requestId, runtimeSettingsRequestId] } } });
await db.votePoll.deleteMany({ where: { openerGeneralId: generalIds[2] } });
await db.message.deleteMany({ where: { mailbox: generalIds[2] } });
await db.logEntry.deleteMany({ where: { text: runtimeSettingsLogText } });
await db.auction.deleteMany({ where: { hostGeneralId: { in: [...generalIds] } } });
await db.selectPoolEntry.deleteMany({ where: { uniqueName: backlogPoolUniqueName } });
await db.general.deleteMany({ where: { id: { in: [...generalIds] } } });
await db.worldState.deleteMany({
where: {
scenarioCode: { in: ['runtime-clock-shift', 'runtime-game-settings', 'realtime-backlog-rebase'] },
},
});
};
beforeAll(async () => { beforeAll(async () => {
const connector = createGamePostgresConnector({ url: databaseUrl! }); const connector = createGamePostgresConnector({ url: databaseUrl! });
await connector.connect(); await connector.connect();
db = connector.prisma; db = connector.prisma;
closeDb = () => connector.disconnect(); closeDb = () => connector.disconnect();
await db.inputEvent.deleteMany({ where: { requestId: { in: [requestId, runtimeSettingsRequestId] } } });
await db.votePoll.deleteMany({ where: { openerGeneralId: generalIds[2] } });
await db.message.deleteMany({ where: { mailbox: generalIds[2] } });
await db.logEntry.deleteMany({ where: { text: runtimeSettingsLogText } });
await db.auction.deleteMany({ where: { hostGeneralId: { in: [...generalIds] } } });
await db.selectPoolEntry.deleteMany({ where: { uniqueName: backlogPoolUniqueName } });
await db.general.deleteMany({ where: { id: { in: [...generalIds] } } });
await db.worldState.deleteMany({
where: {
scenarioCode: { in: ['runtime-clock-shift', 'runtime-game-settings', 'realtime-backlog-rebase'] },
},
});
}); });
beforeEach(cleanupFixtures);
afterAll(async () => { afterAll(async () => {
await db.inputEvent.deleteMany({ where: { requestId: { in: [requestId, runtimeSettingsRequestId] } } }); await cleanupFixtures();
await db.votePoll.deleteMany({ where: { openerGeneralId: generalIds[2] } });
await db.message.deleteMany({ where: { mailbox: generalIds[2] } });
await db.logEntry.deleteMany({ where: { text: runtimeSettingsLogText } });
await db.auction.deleteMany({ where: { hostGeneralId: { in: [...generalIds] } } });
await db.selectPoolEntry.deleteMany({ where: { uniqueName: backlogPoolUniqueName } });
await db.general.deleteMany({ where: { id: { in: [...generalIds] } } });
await db.worldState.deleteMany({
where: {
scenarioCode: { in: ['runtime-clock-shift', 'runtime-game-settings', 'realtime-backlog-rebase'] },
},
});
await closeDb?.(); await closeDb?.();
}); });
@@ -469,6 +463,7 @@ integration('runtime clock shift persistence', () => {
clockBaseTime: base, clockBaseTime: base,
clockTick: 0, clockTick: 0,
clockMode: 'manual', clockMode: 'manual',
clockPhase: 'MANUAL',
clockWallAnchor: base, clockWallAnchor: base,
lastTurnTick: 0, lastTurnTick: 0,
config: { turnTermMinutes: 10, blockGeneralCreate: 0 }, config: { turnTermMinutes: 10, blockGeneralCreate: 0 },
+1 -1
View File
@@ -39,7 +39,7 @@ describe('readReleaseManifest', () => {
await expect(readReleaseManifest(workspaceRoot)).resolves.toMatchObject({ await expect(readReleaseManifest(workspaceRoot)).resolves.toMatchObject({
controllerProtocol: RELEASE_CONTROLLER_PROTOCOL, controllerProtocol: RELEASE_CONTROLLER_PROTOCOL,
gatewaySchemaHead: '20260825000000_add_bulk_release_batches', gatewaySchemaHead: '20260825000000_add_bulk_release_batches',
gameSchemaHead: '20260903090000_add_game_clock_reconciliation', gameSchemaHead: '20260903103000_add_input_event_clock_processing',
}); });
}); });
+24 -8
View File
@@ -6,6 +6,10 @@
"coveredFields": [ "coveredFields": [
"input_event.accepted_game_tick", "input_event.accepted_game_tick",
"input_event.accepted_clock_revision", "input_event.accepted_clock_revision",
"input_event.accepted_deadline_generation",
"input_event.processing_game_tick",
"input_event.processing_clock_revision",
"input_event.processing_deadline_generation",
"world_state.clock_tick", "world_state.clock_tick",
"world_state.last_turn_tick", "world_state.last_turn_tick",
"world_state.clock_revision", "world_state.clock_revision",
@@ -125,8 +129,16 @@
{ {
"key": "accepted-command-coordinate", "key": "accepted-command-coordinate",
"policy": "KEEP", "policy": "KEEP",
"authorityFields": ["input_event.accepted_game_tick", "input_event.accepted_clock_revision"], "authorityFields": [
"projectionFields": [], "input_event.accepted_game_tick",
"input_event.accepted_clock_revision",
"input_event.accepted_deadline_generation"
],
"projectionFields": [
"input_event.processing_game_tick",
"input_event.processing_clock_revision",
"input_event.processing_deadline_generation"
],
"owner": "game-api/input-event" "owner": "game-api/input-event"
}, },
{ {
@@ -139,11 +151,15 @@
}, },
{ {
"key": "movable-json-rule-anchors", "key": "movable-json-rule-anchors",
"policy": "FORBID", "policy": "SHIFT",
"authorityFields": ["world_state.meta.turntime", "world_state.meta.starttime", "world_state.meta.tnmt_time"], "authorityFields": [
"world_state.meta.turntime",
"world_state.meta.starttime",
"world_state.meta.tnmt_time"
],
"projectionFields": [], "projectionFields": [],
"owner": "game-engine/world-meta", "owner": "game-engine/clock-operation",
"migration": "Register typed columns or explicit participant adapters before exact reconciliation can complete." "migration": "Explicit adapter shifts registered ISO projections while typed tick columns remain authoritative."
}, },
{ {
"key": "unification-wait", "key": "unification-wait",
@@ -179,12 +195,12 @@
{ {
"keyPattern": "sammo:{profile}:auction:timer", "keyPattern": "sammo:{profile}:auction:timer",
"policy": "REBUILD", "policy": "REBUILD",
"status": "implemented-without-clock-revision-fence" "status": "implemented-with-clock-revision-phase-generation-fence"
}, },
{ {
"keyPattern": "sammo:{profile}:tournament:state", "keyPattern": "sammo:{profile}:tournament:state",
"policy": "REBUILD", "policy": "REBUILD",
"status": "legacy-date-dual-write-required" "status": "implemented-with-tick-dual-write-and-clock-fence"
} }
], ],
"wallOnly": [ "wallOnly": [
+19 -4
View File
@@ -65,6 +65,19 @@ pop is one Redis operation: verify revision/phase, read `-inf..nowTick`, and
remove the claimed members. A failed Redis rebuild therefore leaves the game in remove the claimed members. A failed Redis rebuild therefore leaves the game in
`RECONCILING`; process liveness alone is not readiness. `RECONCILING`; process liveness alone is not readiness.
`applyNextClockProjection()` claims rows with `FOR UPDATE SKIP LOCKED` and uses
PostgreSQL UTC wall time for claim/retry timestamps. One Redis Lua operation
compares the active source revision, rebuilds the auction timer, conditionally
replaces the exact tournament source snapshot, and writes target revision plus
deadline generation. The DB finalizer then re-acquires the clock-operation lock
and changes `RECONCILING -> RUNNING` only when target revision and generation
still match. If the process dies after the Lua commit, retry observes the
already-active target revision and performs only the DB finalizer.
An active legacy tournament containing only `nextAt`/`bettingCloseAt` is a
fail-closed migration boundary. Reconciliation remains incomplete until its
authoritative `nextTick`/`bettingCloseTick` dual-write exists.
## Lock order ## Lock order
All mutation paths use this order: All mutation paths use this order:
@@ -101,7 +114,9 @@ future anchored realtime profiles as `PREOPEN`, and other profiles as
`RUNNING`. Existing DateTime columns remain projections while tick columns are `RUNNING`. Existing DateTime columns remain projections while tick columns are
authoritative. authoritative.
Exact reconciliation stays disabled while any registry participant is Exact reconciliation stays disabled while an active registry participant is
`FORBID`. In particular, Redis-only tournament dates and unification wait must `FORBID`. Tournament writes now carry tick/revision/generation coordinates and
be moved to durable tick/revision contracts before the operation can reach are revision-fenced in Redis. The remaining unification wait participant must
`RUNNING`. Removing these guards to make a partial operation pass is prohibited. be moved from its `lastTurnTime` workaround to a durable suspension before that
workflow can reach `RUNNING`. Removing this guard to make a partial operation
pass is prohibited.
@@ -11,7 +11,7 @@ mean deployment or production validation.
## Milestone 1 - authority and inventory ## Milestone 1 - authority and inventory
- [x] Branded `GameTick`, `ObservedGameInstant`, `ScheduleInstant`, - [x] Branded `GameTick`, `ObservedGameInstant`, `ScheduleInstant`,
`WallInstant`, and `ClockRevision` boundaries. `WallInstant`, and `ClockRevision` boundaries.
- [x] Explicit clock phase and monotonic RUNNING projection. - [x] Explicit clock phase and monotonic RUNNING projection.
- [x] Exact alignment arithmetic preserving millisecond/sub-turn remainder. - [x] Exact alignment arithmetic preserving millisecond/sub-turn remainder.
- [x] Opening tick zero and PREOPEN executable floor in the shared seeder. - [x] Opening tick zero and PREOPEN executable floor in the shared seeder.
@@ -19,49 +19,50 @@ mean deployment or production validation.
- [x] Suspension, participant-checksum, and Redis projection outbox tables. - [x] Suspension, participant-checksum, and Redis projection outbox tables.
- [x] Machine-readable DB/Redis/JSON participant inventory and architecture gate. - [x] Machine-readable DB/Redis/JSON participant inventory and architecture gate.
- [x] Turn flush lock prefix and phase/revision/generation fence. - [x] Turn flush lock prefix and phase/revision/generation fence.
- [ ] Empty and upgraded database migration execution evidence. - [x] Empty and upgraded database migration execution evidence.
## Milestone 2 - exact DB reconciliation ## Milestone 2 - exact DB reconciliation
- [ ] Suspension start command with DB wall time and idempotent source revision. - [x] Suspension start command with DB wall time and idempotent source revision.
- [ ] Exact resume plan transaction with deterministic participant lock order. - [x] Exact resume plan transaction with deterministic participant lock order.
- [ ] SHIFT adapters for cursor, generals, active auctions, message expiry, vote - [x] SHIFT adapters for cursor, generals, active auctions, message expiry, vote
end, select pool, and NPC selection windows. end, select pool, and NPC selection windows.
- [ ] KEEP checksum adapters for occurrences and history. - [x] KEEP checksum adapters for occurrences and accepted command coordinates.
- [ ] Explicit `LEGACY_COMPLETE_TURNS` and bounded `CATCH_UP` policies. - [x] Explicit `LEGACY_COMPLETE_TURNS` and bounded `CATCH_UP` policies.
- [ ] Property tests for remaining distance, ordering, and history invariants. - [x] Property tests for remaining distance, ordering, and history invariants.
- [ ] 24-hour and 65m17.250s PostgreSQL integration evidence. - [x] 24-hour and 65m17.250s PostgreSQL integration evidence.
## Milestone 3 - revisioned Redis and workers ## Milestone 3 - revisioned Redis and workers
- [ ] Projection outbox claimer/retry/recovery state machine. - [x] Projection outbox claimer/retry/recovery state machine.
- [ ] Redis active revision and atomic due-pop script. - [x] Redis active revision and atomic due-pop script.
- [ ] Auction OPEN/FINALIZING revision and generation fence. - [x] Auction OPEN/FINALIZING revision and generation fence.
- [ ] Tournament durable tick dual-write and projection rebuild. - [x] Tournament durable tick dual-write and projection rebuild.
- [ ] DB-commit/Redis-failure restart tests and readiness integration. - [x] DB-commit/Redis-failure crash-restart test.
- [x] Readiness integration in Gateway/API process health.
## Milestone 4 - command and lifecycle workflows ## Milestone 4 - command and lifecycle workflows
- [ ] All durable input events record accepted tick and accepted revision. - [x] All durable input events record accepted tick and accepted revision.
- [ ] Processing converts accepted coordinates across revisions or fails closed. - [x] Processing converts accepted coordinates across revisions or fails closed.
- [ ] Gateway pause/resume/open orchestration writes the DB clock phase. - [ ] Gateway pause/resume/open orchestration writes the DB clock phase.
- [ ] Unification wait becomes a durable `UNIFICATION_WAIT` suspension. - [ ] Unification wait becomes a durable `UNIFICATION_WAIT` suspension.
- [ ] Alignment, optional rate change, invader IDs/RNG, creation, first schedule, - [ ] Alignment, optional rate change, invader IDs/RNG, creation, first schedule,
outbox, verification, and RUNNING transition form one retry-safe workflow. outbox, verification, and RUNNING transition form one retry-safe workflow.
- [ ] Multi-host drift and general-access/clock-operation deadlock tests. - [ ] Multi-host drift and general-access/clock-operation deadlock tests.
## Milestone 5 - test-branch release gate ## Milestone 5 - test-branch release gate
- [ ] Full typecheck, architecture, lint, unit, build, and non-conditional - [ ] Full typecheck, architecture, lint, unit, build, and non-conditional
integration suites. integration suites.
- [ ] Dedicated PostgreSQL/Redis conditional integration suite with skip count - [ ] Dedicated PostgreSQL/Redis conditional integration suite with skip count
recorded. recorded.
- [ ] Recovery runbook exercised from each incomplete status. - [ ] Recovery runbook exercised from each incomplete status.
- [ ] Admin status/readiness exposes revision, phase, participant checksums, and - [x] Admin status/readiness exposes revision, phase, participant checksums, and
incomplete outbox state. incomplete outbox state.
- [ ] User-test deployment evidence is recorded separately from Git push. - [ ] User-test deployment evidence is recorded separately from Git push.
- [ ] All `FORBID` inventory entries are removed by typed migrations or proven - [ ] All `FORBID` inventory entries are removed by typed migrations or proven
inactive preconditions. inactive preconditions.
## Evidence log ## Evidence log
@@ -78,4 +79,25 @@ mean deployment or production validation.
- `pnpm check:architecture`: package boundaries passed; 21 authoritative clock - `pnpm check:architecture`: package boundaries passed; 21 authoritative clock
fields and 18 participants were registered. fields and 18 participants were registered.
- Migration SQL was generated, formatted, validated, and registered as the - Migration SQL was generated, formatted, validated, and registered as the
release manifest head. Empty/upgraded PostgreSQL execution is still pending. release manifest head. All 43 migrations applied to the empty dedicated
PostgreSQL instance. A second schema upgraded from the previous head with an
existing manual `world_state` row and verified `MANUAL:1:1` plus all three
reconciliation tables.
- Dedicated PostgreSQL/Redis fixtures proved exact 24-hour and 65m17.250s gaps,
schedule distance/order preservation, occurrence checksums, live-lease
fencing, a single durable outbox, Redis target revision, and recovery after a
crash between Redis commit and DB finalization.
### 2026-09-03 - revisioned workers and input coordinates
- `CI=1 TURBO_CONCURRENCY=1 pnpm typecheck`: 21/21 tasks passed after the
worker/input contract changes.
- `CI=1 TURBO_CONCURRENCY=1 pnpm test`: 12/12 package tasks passed.
- Dedicated PostgreSQL runs passed `databaseCommandQueue.integration.test.ts`
6/6 and `runtimeClockShiftPersistence.integration.test.ts` 3/3 when executed
sequentially; the latter now clears each single-world fixture before the next
case. Dedicated Redis passed tournament source revision 4/4, including an
atomic stale-clock rejection. API input-event integration passed 13/13.
- The 24-hour/65m17.250s reconciliation suite passed 2/2 after the other DB
suites. Conditional files share a deliberately dedicated schema and are run
sequentially to prevent their fixture cleanup from racing another file.
+12
View File
@@ -28,6 +28,18 @@ service. The service must re-read participant checksums and either return the
already-applied result or resume the pending outbox. Never create a replacement already-applied result or resume the pending outbox. Never create a replacement
revision to hide a failed target revision. revision to hide a failed target revision.
When an outbox row is `FAILED`, the profile must remain `RECONCILING`. A retry
is safe in both crash locations:
- before Redis commit, the Lua operation reapplies from the source revision;
- after Redis commit but before DB finalization, the Lua operation returns the
already-active target result and DB finalization resumes without shifting any
tournament deadline twice.
`lastError` naming a legacy tournament deadline means the tournament lacks its
tick dual-write. Do not force the active revision; migrate or prove the
tournament inactive, then retry the same outbox.
## Rollback ## Rollback
There is no blind inverse update. Before enabling exact reconciliation in an There is no blind inverse update. Before enabling exact reconciliation in an
+46 -5
View File
@@ -17,8 +17,8 @@ export type ScheduleInstant = GameTick & { readonly [scheduleInstantBrand]: 'Sch
export type ClockRevision = number & { readonly [clockRevisionBrand]: 'ClockRevision' }; export type ClockRevision = number & { readonly [clockRevisionBrand]: 'ClockRevision' };
export type WallInstant = Date & { readonly [wallInstantBrand]: 'WallInstant' }; export type WallInstant = Date & { readonly [wallInstantBrand]: 'WallInstant' };
export interface ExactClockAlignmentPlan { export interface ClockAlignmentPlan {
policy: 'EXACT'; policy: ClockAlignmentPolicy;
sourceRevision: ClockRevision; sourceRevision: ClockRevision;
targetRevision: ClockRevision; targetRevision: ClockRevision;
cutTick: GameTick; cutTick: GameTick;
@@ -84,6 +84,15 @@ export const parseGameClockPhase = (value: string): GameClockPhase => {
throw new Error(`Unknown game clock phase: ${value}`); throw new Error(`Unknown game clock phase: ${value}`);
}; };
const CLOCK_ALIGNMENT_POLICIES: readonly ClockAlignmentPolicy[] = ['EXACT', 'LEGACY_COMPLETE_TURNS', 'CATCH_UP'];
export const parseClockAlignmentPolicy = (value: string): ClockAlignmentPolicy => {
if ((CLOCK_ALIGNMENT_POLICIES as readonly string[]).includes(value)) {
return value as ClockAlignmentPolicy;
}
throw new Error(`Unknown clock alignment policy: ${value}`);
};
export const scheduleNotBefore = (instant: ObservedGameInstant, phase: GameClockPhase): ScheduleInstant => { export const scheduleNotBefore = (instant: ObservedGameInstant, phase: GameClockPhase): ScheduleInstant => {
if (phase === 'PREOPEN') { if (phase === 'PREOPEN') {
return asScheduleInstant(Math.max(0, instant)); return asScheduleInstant(Math.max(0, instant));
@@ -103,14 +112,15 @@ export const assertGameplayCommitAllowed = (phase: GameClockPhase): void => {
} }
}; };
export const buildExactClockAlignmentPlan = (input: { const buildAlignmentPlan = (input: {
policy: ClockAlignmentPolicy;
sourceRevision: number; sourceRevision: number;
cutTick: number; cutTick: number;
cutWall: Date; cutWall: Date;
resumeWall: Date; resumeWall: Date;
ticksPerSecond: number; ticksPerSecond: number;
catchUpTicks?: number; catchUpTicks?: number;
}): ExactClockAlignmentPlan => { }): ClockAlignmentPlan => {
const sourceRevision = asClockRevision(input.sourceRevision); const sourceRevision = asClockRevision(input.sourceRevision);
const cutTick = asGameTick(input.cutTick); const cutTick = asGameTick(input.cutTick);
const cutWall = asWallInstant(input.cutWall); const cutWall = asWallInstant(input.cutWall);
@@ -136,7 +146,7 @@ export const buildExactClockAlignmentPlan = (input: {
} }
const shiftTicks = asGameTick(gapTicks - catchUpTicks); const shiftTicks = asGameTick(gapTicks - catchUpTicks);
return { return {
policy: 'EXACT', policy: input.policy,
sourceRevision, sourceRevision,
targetRevision: asClockRevision(sourceRevision + 1), targetRevision: asClockRevision(sourceRevision + 1),
cutTick, cutTick,
@@ -147,6 +157,37 @@ export const buildExactClockAlignmentPlan = (input: {
}; };
}; };
export const buildExactClockAlignmentPlan = (
input: Omit<Parameters<typeof buildAlignmentPlan>[0], 'policy'>
): ClockAlignmentPlan => buildAlignmentPlan({ ...input, policy: 'EXACT' });
export const buildClockAlignmentPlan = (input: {
policy: ClockAlignmentPolicy;
sourceRevision: number;
cutTick: number;
cutWall: Date;
resumeWall: Date;
ticksPerSecond: number;
catchUpTicks?: number;
}): ClockAlignmentPlan => {
if (input.policy === 'EXACT') {
if ((input.catchUpTicks ?? 0) !== 0) {
throw new Error('EXACT alignment does not allow catch-up ticks.');
}
return buildAlignmentPlan({ ...input, policy: 'EXACT', catchUpTicks: 0 });
}
if (input.policy === 'CATCH_UP') {
return buildAlignmentPlan({ ...input, policy: 'CATCH_UP' });
}
const exact = buildAlignmentPlan({ ...input, policy: 'LEGACY_COMPLETE_TURNS', catchUpTicks: 0 });
const shiftTicks = asGameTick(Math.floor(exact.gapTicks / GAME_TICKS_PER_TURN) * GAME_TICKS_PER_TURN);
return {
...exact,
catchUpTicks: asGameTick(exact.gapTicks - shiftTicks),
shiftTicks,
};
};
const tickOffsetMilliseconds = (tick: number, ticksPerSecond: number): number => { const tickOffsetMilliseconds = (tick: number, ticksPerSecond: number): number => {
const wholeSeconds = Math.floor(tick / ticksPerSecond); const wholeSeconds = Math.floor(tick / ticksPerSecond);
const remainingTicks = tick - wholeSeconds * ticksPerSecond; const remainingTicks = tick - wholeSeconds * ticksPerSecond;
@@ -18,6 +18,15 @@ export interface TournamentProjectionWrite {
value: unknown; value: unknown;
} }
export interface TournamentClockFence {
activeRevisionKey: string;
deadlineGenerationKey: string;
phaseKey: string;
revision: number;
deadlineGeneration: number;
phase: 'RUNNING';
}
const WRITE_TOURNAMENT_PROJECTION_SCRIPT = ` const WRITE_TOURNAMENT_PROJECTION_SCRIPT = `
local revision_key = KEYS[#KEYS] local revision_key = KEYS[#KEYS]
local current = redis.call('GET', revision_key) local current = redis.call('GET', revision_key)
@@ -55,6 +64,49 @@ local revision = redis.call('INCR', revision_key)
return tostring(revision) .. ':' .. (stage_changed and '1' or '0') .. ':' .. (rankings_changed and '1' or '0') return tostring(revision) .. ':' .. (stage_changed and '1' or '0') .. ':' .. (rankings_changed and '1' or '0')
`; `;
const WRITE_FENCED_TOURNAMENT_PROJECTION_SCRIPT = `
local revision_key_index = #KEYS - 3
if redis.call('GET', KEYS[#KEYS - 2]) ~= ARGV[#ARGV - 2]
or redis.call('GET', KEYS[#KEYS - 1]) ~= ARGV[#ARGV - 1]
or redis.call('GET', KEYS[#KEYS]) ~= ARGV[#ARGV] then
return '__CLOCK_FENCE__'
end
local revision_key = KEYS[revision_key_index]
local current = redis.call('GET', revision_key)
if current then
if not string.match(current, '^%d+$') then
return redis.error_reply('invalid tournament source revision')
end
if string.len(current) > 18 then
return redis.error_reply('tournament source revision exhausted')
end
end
local stage_changed = false
local rankings_changed = false
for index = 1, revision_key_index - 1 do
local next_ok, next_value = pcall(cjson.decode, ARGV[index])
if next_ok and type(next_value) == 'table' and next_value['stage'] ~= nil then
local previous = redis.call('GET', KEYS[index])
local previous_stage = nil
local previous_value = nil
if previous then
local previous_ok
previous_ok, previous_value = pcall(cjson.decode, previous)
if previous_ok and type(previous_value) == 'table' then
previous_stage = previous_value['stage']
end
end
local next_stage = next_value['stage']
stage_changed = (not previous) or previous_stage ~= next_stage
local previous_reward_settled = previous_value and previous_value['rewardSettled'] or false
rankings_changed = next_value['rewardSettled'] == true and previous_reward_settled ~= true
end
redis.call('SET', KEYS[index], ARGV[index])
end
local revision = redis.call('INCR', revision_key)
return tostring(revision) .. ':' .. (stage_changed and '1' or '0') .. ':' .. (rankings_changed and '1' or '0')
`;
export const parseTournamentSourceRevision = (value: unknown): string | null => { export const parseTournamentSourceRevision = (value: unknown): string | null => {
if (typeof value === 'number') { if (typeof value === 'number') {
return Number.isSafeInteger(value) && value >= 0 ? String(value) : null; return Number.isSafeInteger(value) && value >= 0 ? String(value) : null;
@@ -69,7 +121,8 @@ export const parseTournamentSourceRevision = (value: unknown): string | null =>
export const writeTournamentProjection = async ( export const writeTournamentProjection = async (
redis: TournamentProjectionRedis, redis: TournamentProjectionRedis,
keys: TournamentSourceKeys, keys: TournamentSourceKeys,
writes: readonly TournamentProjectionWrite[] writes: readonly TournamentProjectionWrite[],
clockFence?: TournamentClockFence
): Promise<string> => { ): Promise<string> => {
if (writes.length === 0) { if (writes.length === 0) {
throw new Error('Tournament projection write must contain at least one payload.'); throw new Error('Tournament projection write must contain at least one payload.');
@@ -90,10 +143,27 @@ export const writeTournamentProjection = async (
value !== null && value !== null &&
(value as { rewardSettled?: unknown }).rewardSettled === true (value as { rewardSettled?: unknown }).rewardSettled === true
); );
const result = await redis.eval(WRITE_TOURNAMENT_PROJECTION_SCRIPT, { const result = await redis.eval(
keys: [...writes.map(({ key }) => key), keys.sourceRevisionKey], clockFence ? WRITE_FENCED_TOURNAMENT_PROJECTION_SCRIPT : WRITE_TOURNAMENT_PROJECTION_SCRIPT,
arguments: writes.map(({ value }) => JSON.stringify(value)), {
}); keys: [
...writes.map(({ key }) => key),
keys.sourceRevisionKey,
...(clockFence
? [clockFence.activeRevisionKey, clockFence.deadlineGenerationKey, clockFence.phaseKey]
: []),
],
arguments: [
...writes.map(({ value }) => JSON.stringify(value)),
...(clockFence
? [String(clockFence.revision), String(clockFence.deadlineGeneration), clockFence.phase]
: []),
],
}
);
if (result === '__CLOCK_FENCE__') {
throw new Error('Tournament projection clock revision fence failed.');
}
const scriptResult = typeof result === 'string' ? /^(\d+):([01])(?::([01]))?$/u.exec(result) : null; const scriptResult = typeof result === 'string' ? /^(\d+):([01])(?::([01]))?$/u.exec(result) : null;
const sourceRevision = parseTournamentSourceRevision(scriptResult?.[1] ?? result); const sourceRevision = parseTournamentSourceRevision(scriptResult?.[1] ?? result);
// Plain revision results remain accepted for rolling deployments and small // Plain revision results remain accepted for rolling deployments and small
+59
View File
@@ -6,6 +6,7 @@ import {
MAX_SAFE_GAME_TICK, MAX_SAFE_GAME_TICK,
asGameTick, asGameTick,
asObservedGameInstant, asObservedGameInstant,
buildClockAlignmentPlan,
buildExactClockAlignmentPlan, buildExactClockAlignmentPlan,
createDeadline, createDeadline,
scheduleNotBefore, scheduleNotBefore,
@@ -113,6 +114,64 @@ describe('GameClock', () => {
expect(plan.alignedTick).toBe(26 * GAME_TICKS_PER_TURN); expect(plan.alignedTick).toBe(26 * GAME_TICKS_PER_TURN);
}); });
it('keeps legacy complete-turn rebasing separate from bounded catch-up', () => {
const common = {
sourceRevision: 4,
cutTick: 100,
cutWall: new Date('2026-01-01T00:00:00.000Z'),
resumeWall: new Date('2026-01-01T01:05:17.250Z'),
ticksPerSecond: 10_000,
};
const legacy = buildClockAlignmentPlan({ ...common, policy: 'LEGACY_COMPLETE_TURNS' });
const catchUp = buildClockAlignmentPlan({ ...common, policy: 'CATCH_UP', catchUpTicks: 1_000_000 });
expect(legacy).toMatchObject({
policy: 'LEGACY_COMPLETE_TURNS',
gapTicks: 39_172_500,
shiftTicks: GAME_TICKS_PER_TURN,
catchUpTicks: 3_172_500,
});
expect(catchUp).toMatchObject({
policy: 'CATCH_UP',
gapTicks: 39_172_500,
shiftTicks: 38_172_500,
catchUpTicks: 1_000_000,
});
expect(() => buildClockAlignmentPlan({ ...common, policy: 'EXACT', catchUpTicks: 1 })).toThrow(
'EXACT alignment does not allow catch-up ticks'
);
});
it('preserves schedule ordering, remaining distance, and occurrence ticks across generated exact gaps', () => {
let seed = 0x5eed1234;
const next = (): number => {
seed = (Math.imul(seed, 1_664_525) + 1_013_904_223) >>> 0;
return seed;
};
for (let iteration = 0; iteration < 500; iteration += 1) {
const cutTick = next() % 1_000_000_000;
const gapMilliseconds = next() % (7 * 24 * 60 * 60 * 1_000);
const ticksPerSecond = [5_000, 10_000, 60_000][next() % 3]!;
const offsets = Array.from({ length: 8 }, () => next() % (3 * GAME_TICKS_PER_TURN)).sort(
(left, right) => left - right
);
const occurrenceTicks = Array.from({ length: 4 }, () => cutTick - (next() % GAME_TICKS_PER_TURN));
const plan = buildClockAlignmentPlan({
policy: 'EXACT',
sourceRevision: 1 + (next() % 10_000),
cutTick,
cutWall: new Date(0),
resumeWall: new Date(gapMilliseconds),
ticksPerSecond,
});
const shifted = offsets.map((offset) => cutTick + offset + plan.shiftTicks);
expect(shifted.map((deadline) => deadline - plan.alignedTick)).toEqual(offsets);
expect([...shifted].sort((left, right) => left - right)).toEqual(shifted);
expect(occurrenceTicks).toEqual([...occurrenceTicks]);
}
});
it('projects near the safe tick boundary without unsafe intermediate multiplication', () => { it('projects near the safe tick boundary without unsafe intermediate multiplication', () => {
const clock = new GameClock({ const clock = new GameClock({
baseTime: new Date(0), baseTime: new Date(0),
+21 -17
View File
@@ -63,23 +63,27 @@ enum InputEventTarget {
} }
model InputEvent { model InputEvent {
sequence BigInt @id @default(autoincrement()) sequence BigInt @id @default(autoincrement())
requestId String @unique @map("request_id") requestId String @unique @map("request_id")
target InputEventTarget target InputEventTarget
eventType String @map("event_type") eventType String @map("event_type")
payload Json @default(dbgenerated("'{}'::jsonb")) payload Json @default(dbgenerated("'{}'::jsonb"))
actorUserId String? @map("actor_user_id") actorUserId String? @map("actor_user_id")
acceptedGameTick BigInt? @map("accepted_game_tick") acceptedGameTick BigInt? @map("accepted_game_tick")
acceptedClockRevision BigInt? @map("accepted_clock_revision") acceptedClockRevision BigInt? @map("accepted_clock_revision")
status InputEventStatus @default(PENDING) acceptedDeadlineGeneration BigInt? @map("accepted_deadline_generation")
result Json? processingGameTick BigInt? @map("processing_game_tick")
error String? processingClockRevision BigInt? @map("processing_clock_revision")
attempts Int @default(0) processingDeadlineGeneration BigInt? @map("processing_deadline_generation")
lockedBy String? @map("locked_by") status InputEventStatus @default(PENDING)
leaseUntil DateTime? @map("lease_until") result Json?
createdAt DateTime @default(now()) @map("created_at") error String?
processingAt DateTime? @map("processing_at") attempts Int @default(0)
completedAt DateTime? @map("completed_at") lockedBy String? @map("locked_by")
leaseUntil DateTime? @map("lease_until")
createdAt DateTime @default(now()) @map("created_at")
processingAt DateTime? @map("processing_at")
completedAt DateTime? @map("completed_at")
@@index([target, status, sequence]) @@index([target, status, sequence])
@@map("input_event") @@map("input_event")
@@ -0,0 +1,15 @@
ALTER TABLE input_event
ADD COLUMN accepted_deadline_generation BIGINT,
ADD COLUMN processing_game_tick BIGINT,
ADD COLUMN processing_clock_revision BIGINT,
ADD COLUMN processing_deadline_generation BIGINT;
ALTER TABLE input_event
ADD CONSTRAINT input_event_accepted_clock_revision_positive
CHECK (accepted_clock_revision IS NULL OR accepted_clock_revision > 0),
ADD CONSTRAINT input_event_accepted_deadline_generation_positive
CHECK (accepted_deadline_generation IS NULL OR accepted_deadline_generation > 0),
ADD CONSTRAINT input_event_processing_clock_revision_positive
CHECK (processing_clock_revision IS NULL OR processing_clock_revision > 0),
ADD CONSTRAINT input_event_processing_deadline_generation_positive
CHECK (processing_deadline_generation IS NULL OR processing_deadline_generation > 0);
+3
View File
@@ -46,6 +46,9 @@ export interface DatabaseClient {
vote: GamePrisma.VoteDelegate; vote: GamePrisma.VoteDelegate;
inputEvent: GamePrisma.InputEventDelegate; inputEvent: GamePrisma.InputEventDelegate;
turnDaemonLease: GamePrisma.TurnDaemonLeaseDelegate; turnDaemonLease: GamePrisma.TurnDaemonLeaseDelegate;
clockSuspension?: GamePrisma.ClockSuspensionDelegate;
clockReconciliationParticipant?: GamePrisma.ClockReconciliationParticipantDelegate;
clockProjectionOutbox?: GamePrisma.ClockProjectionOutboxDelegate;
readModelOutbox: GamePrisma.ReadModelOutboxDelegate; readModelOutbox: GamePrisma.ReadModelOutboxDelegate;
webPushOutbox: GamePrisma.WebPushOutboxDelegate; webPushOutbox: GamePrisma.WebPushOutboxDelegate;
} }
+1
View File
@@ -10,4 +10,5 @@ export * from './readModelChangeJournal.js';
export * from './readModelOutboxDispatcher.js'; export * from './readModelOutboxDispatcher.js';
export * from './readModelCoverageActivation.js'; export * from './readModelCoverageActivation.js';
export * from './gameSchemaAdvisoryLock.js'; export * from './gameSchemaAdvisoryLock.js';
export * from './inputEventClock.js';
export * from './webPushOutbox.js'; export * from './webPushOutbox.js';
+76
View File
@@ -0,0 +1,76 @@
import { GameClock, inferClockPhase, parseGameClockPhase } from '@sammo-ts/common';
import { GamePrisma, type GamePrismaClient } from './gamePrisma.js';
import { acquireGameSchemaAdvisoryXactLock, CLOCK_OPERATION_PERSISTENCE_LOCK } from './gameSchemaAdvisoryLock.js';
type ClockAcceptanceDatabase = Pick<GamePrismaClient, '$executeRaw' | '$queryRaw' | 'worldState'>;
interface DbWallRow {
wallNow: Date;
}
export interface InputEventClockCoordinate {
wallAt: Date;
gameAt: Date;
gameTick: bigint;
clockRevision: bigint;
deadlineGeneration: bigint;
phase: string;
}
/**
* Reads one input-event acceptance coordinate while holding the same schema
* clock-operation fence used by reconciliation. The caller must create the
* input_event in this transaction before releasing the lock.
*/
export const readInputEventClockCoordinate = async (
db: ClockAcceptanceDatabase
): Promise<InputEventClockCoordinate> => {
await acquireGameSchemaAdvisoryXactLock(db, CLOCK_OPERATION_PERSISTENCE_LOCK);
const [wall] = await db.$queryRaw<DbWallRow[]>(GamePrisma.sql`
SELECT timezone('UTC', clock_timestamp()) AS "wallNow"
`);
if (!wall) throw new Error('PostgreSQL did not return its authoritative wall clock.');
const world = await db.worldState.findFirst({
orderBy: { id: 'asc' },
select: {
clockBaseTime: true,
clockTick: true,
clockMode: true,
clockWallAnchor: true,
tickSeconds: true,
clockPhase: true,
clockRevision: true,
deadlineGeneration: true,
},
});
if (!world?.clockBaseTime || world.clockTick === null || !world.clockWallAnchor) {
throw new Error('The authoritative game clock is not initialized.');
}
const tick = Number(world.clockTick);
const revision = Number(world.clockRevision);
const generation = Number(world.deadlineGeneration);
if (!Number.isSafeInteger(tick) || !Number.isSafeInteger(revision) || !Number.isSafeInteger(generation)) {
throw new Error('The authoritative game clock coordinate is outside the safe integer range.');
}
const mode = world.clockMode === 'manual' ? 'manual' : 'realtime';
const phase = world.clockPhase ? parseGameClockPhase(world.clockPhase) : inferClockPhase(mode);
const clock = new GameClock({
baseTime: world.clockBaseTime,
tick,
mode,
wallAnchor: world.clockWallAnchor,
turnSeconds: world.tickSeconds,
phase,
revision,
});
const observedTick = clock.nowTick(wall.wallNow);
return {
wallAt: wall.wallNow,
gameAt: clock.tickToDate(observedTick),
gameTick: BigInt(observedTick),
clockRevision: BigInt(revision),
deadlineGeneration: BigInt(generation),
phase,
};
};
+1 -1
View File
@@ -2,6 +2,6 @@
"formatVersion": 1, "formatVersion": 1,
"controllerProtocol": 2, "controllerProtocol": 2,
"gatewaySchemaHead": "20260825000000_add_bulk_release_batches", "gatewaySchemaHead": "20260825000000_add_bulk_release_batches",
"gameSchemaHead": "20260903090000_add_game_clock_reconciliation", "gameSchemaHead": "20260903103000_add_input_event_clock_processing",
"components": ["gateway-api", "gateway-frontend", "release-controller", "game-api", "game-engine", "game-frontend"] "components": ["gateway-api", "gateway-frontend", "release-controller", "game-api", "game-engine", "game-frontend"]
} }