feat: 시계 reconciliation 워커와 명령 경계 완성
This commit is contained in:
@@ -1,4 +1,7 @@
|
|||||||
import {
|
import {
|
||||||
|
CLOCK_OPERATION_PERSISTENCE_LOCK,
|
||||||
|
GamePrisma,
|
||||||
|
acquireGameSchemaAdvisoryXactLock,
|
||||||
createGamePostgresConnector,
|
createGamePostgresConnector,
|
||||||
createRedisConnector,
|
createRedisConnector,
|
||||||
type GamePrismaClient,
|
type GamePrismaClient,
|
||||||
@@ -10,6 +13,7 @@ import { resolveGameApiConfigFromEnv } from '../config.js';
|
|||||||
import { createBestEffortResourceCloser } from '../services/bestEffortResourceCloser.js';
|
import { createBestEffortResourceCloser } from '../services/bestEffortResourceCloser.js';
|
||||||
import { loadCurrentGameTime, type CurrentGameTime } from '../services/gameClock.js';
|
import { loadCurrentGameTime, type CurrentGameTime } from '../services/gameClock.js';
|
||||||
import { createPollingWorkerControl, waitForWorkerPoll } from '../services/pollingWorkerLifecycle.js';
|
import { createPollingWorkerControl, waitForWorkerPoll } from '../services/pollingWorkerLifecycle.js';
|
||||||
|
import { ensureActiveRedisClockFence } from '../services/redisClockFence.js';
|
||||||
import { buildAuctionTimerKeys } from './keys.js';
|
import { buildAuctionTimerKeys } from './keys.js';
|
||||||
import { resolveAuctionTimerScore, seedAuctionTimers } from './scheduler.js';
|
import { resolveAuctionTimerScore, seedAuctionTimers } from './scheduler.js';
|
||||||
|
|
||||||
@@ -24,8 +28,22 @@ interface RedisTimerClient {
|
|||||||
zAdd(key: string, values: Array<{ score: number; value: string }>): Promise<number>;
|
zAdd(key: string, values: Array<{ score: number; value: string }>): Promise<number>;
|
||||||
zRem(key: string, values: string | string[]): Promise<number>;
|
zRem(key: string, values: string | string[]): Promise<number>;
|
||||||
zRemRangeByScore(key: string, min: number, max: number): Promise<number>;
|
zRemRangeByScore(key: string, min: number, max: number): Promise<number>;
|
||||||
|
eval?(script: string, options: { keys: string[]; arguments: string[] }): Promise<unknown>;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const POP_DUE_AUCTIONS_SCRIPT = `
|
||||||
|
if redis.call('GET', KEYS[2]) ~= ARGV[1]
|
||||||
|
or redis.call('GET', KEYS[3]) ~= ARGV[2]
|
||||||
|
or redis.call('GET', KEYS[4]) ~= 'RUNNING' then
|
||||||
|
return { '__CLOCK_FENCE__' }
|
||||||
|
end
|
||||||
|
local ids = redis.call('ZRANGEBYSCORE', KEYS[1], '-inf', ARGV[3], 'LIMIT', 0, ARGV[4])
|
||||||
|
if #ids > 0 then
|
||||||
|
redis.call('ZREM', KEYS[1], unpack(ids))
|
||||||
|
end
|
||||||
|
return ids
|
||||||
|
`;
|
||||||
|
|
||||||
const AUCTION_FINALIZE_RECOVERY_LIMIT = 1;
|
const AUCTION_FINALIZE_RECOVERY_LIMIT = 1;
|
||||||
|
|
||||||
interface AuctionFinalizeDeadline {
|
interface AuctionFinalizeDeadline {
|
||||||
@@ -115,12 +133,29 @@ const isSuccessfulAuctionFinalizeResult = (result: unknown, auctionId: number):
|
|||||||
return resultRecord.type === 'auctionFinalize' && resultRecord.ok === true && resultRecord.auctionId === auctionId;
|
return resultRecord.type === 'auctionFinalize' && resultRecord.ok === true && resultRecord.auctionId === auctionId;
|
||||||
};
|
};
|
||||||
|
|
||||||
const popDueAuctionIds = async (
|
export const popDueAuctionIds = async (
|
||||||
redis: RedisTimerClient,
|
redis: RedisTimerClient,
|
||||||
timerKey: string,
|
timerKey: string,
|
||||||
nowMs: number,
|
nowMs: number,
|
||||||
batchSize: number
|
batchSize: number,
|
||||||
|
clockFence?: {
|
||||||
|
activeRevisionKey: string;
|
||||||
|
deadlineGenerationKey: string;
|
||||||
|
phaseKey: string;
|
||||||
|
revision: number;
|
||||||
|
generation: number;
|
||||||
|
}
|
||||||
): Promise<string[]> => {
|
): Promise<string[]> => {
|
||||||
|
if (clockFence) {
|
||||||
|
if (!redis.eval) throw new Error('Redis EVAL is required for revision-fenced auction due-pop.');
|
||||||
|
const result = await redis.eval(POP_DUE_AUCTIONS_SCRIPT, {
|
||||||
|
keys: [timerKey, clockFence.activeRevisionKey, clockFence.deadlineGenerationKey, clockFence.phaseKey],
|
||||||
|
arguments: [String(clockFence.revision), String(clockFence.generation), String(nowMs), String(batchSize)],
|
||||||
|
});
|
||||||
|
if (!Array.isArray(result)) throw new Error('Auction due-pop returned an invalid Redis result.');
|
||||||
|
if (result[0] === '__CLOCK_FENCE__') return [];
|
||||||
|
return result.map(String);
|
||||||
|
}
|
||||||
const ids = await redis.zRangeByScore(timerKey, 0, nowMs, { LIMIT: { offset: 0, count: batchSize } });
|
const ids = await redis.zRangeByScore(timerKey, 0, nowMs, { LIMIT: { offset: 0, count: batchSize } });
|
||||||
if (ids.length > 0) {
|
if (ids.length > 0) {
|
||||||
await redis.zRem(timerKey, ids);
|
await redis.zRem(timerKey, ids);
|
||||||
@@ -205,6 +240,8 @@ export const processDueAuctionId = async (options: {
|
|||||||
nowMs: number;
|
nowMs: number;
|
||||||
nowTick?: number | null;
|
nowTick?: number | null;
|
||||||
historyNowMs?: number;
|
historyNowMs?: number;
|
||||||
|
expectedClockRevision?: number;
|
||||||
|
expectedDeadlineGeneration?: number;
|
||||||
}): Promise<'PENDING' | 'RESCHEDULED' | 'IGNORED'> => {
|
}): Promise<'PENDING' | 'RESCHEDULED' | 'IGNORED'> => {
|
||||||
const { db, redis, timerKey, historyKey, id, nowMs, nowTick = null, historyNowMs = nowMs } = options;
|
const { db, redis, timerKey, historyKey, id, nowMs, nowTick = null, historyNowMs = nowMs } = options;
|
||||||
const auctionId = Number(id);
|
const auctionId = Number(id);
|
||||||
@@ -213,6 +250,29 @@ export const processDueAuctionId = async (options: {
|
|||||||
}
|
}
|
||||||
const now = new Date(nowMs);
|
const now = new Date(nowMs);
|
||||||
const outcome = await db.$transaction(async (transaction) => {
|
const outcome = await db.$transaction(async (transaction) => {
|
||||||
|
if (options.expectedClockRevision !== undefined || options.expectedDeadlineGeneration !== undefined) {
|
||||||
|
await acquireGameSchemaAdvisoryXactLock(transaction, CLOCK_OPERATION_PERSISTENCE_LOCK);
|
||||||
|
const [world] = await transaction.$queryRaw<
|
||||||
|
Array<{ clockPhase: string | null; clockRevision: bigint; deadlineGeneration: bigint }>
|
||||||
|
>(GamePrisma.sql`
|
||||||
|
SELECT
|
||||||
|
clock_phase AS "clockPhase",
|
||||||
|
clock_revision AS "clockRevision",
|
||||||
|
deadline_generation AS "deadlineGeneration"
|
||||||
|
FROM world_state
|
||||||
|
ORDER BY id ASC
|
||||||
|
LIMIT 1
|
||||||
|
FOR UPDATE
|
||||||
|
`);
|
||||||
|
if (
|
||||||
|
!world ||
|
||||||
|
world.clockPhase !== 'RUNNING' ||
|
||||||
|
world.clockRevision !== BigInt(options.expectedClockRevision ?? -1) ||
|
||||||
|
world.deadlineGeneration !== BigInt(options.expectedDeadlineGeneration ?? -1)
|
||||||
|
) {
|
||||||
|
return { status: 'RESCHEDULED' as const, clockFenceFailed: true };
|
||||||
|
}
|
||||||
|
}
|
||||||
const current = await transaction.auction.findUnique({
|
const current = await transaction.auction.findUnique({
|
||||||
where: { id: auctionId },
|
where: { id: auctionId },
|
||||||
select: { status: true, closeAt: true, closeTick: true },
|
select: { status: true, closeAt: true, closeTick: true },
|
||||||
@@ -259,6 +319,13 @@ export const processDueAuctionId = async (options: {
|
|||||||
target: 'ENGINE',
|
target: 'ENGINE',
|
||||||
eventType: nextCommand.type,
|
eventType: nextCommand.type,
|
||||||
payload: { ...nextCommand },
|
payload: { ...nextCommand },
|
||||||
|
...(nowTick === null ? {} : { acceptedGameTick: BigInt(nowTick) }),
|
||||||
|
...(options.expectedClockRevision === undefined
|
||||||
|
? {}
|
||||||
|
: { acceptedClockRevision: BigInt(options.expectedClockRevision) }),
|
||||||
|
...(options.expectedDeadlineGeneration === undefined
|
||||||
|
? {}
|
||||||
|
: { acceptedDeadlineGeneration: BigInt(options.expectedDeadlineGeneration) }),
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
return { status: 'PENDING' as const };
|
return { status: 'PENDING' as const };
|
||||||
@@ -284,6 +351,10 @@ export const processDueAuctionId = async (options: {
|
|||||||
return 'PENDING';
|
return 'PENDING';
|
||||||
}
|
}
|
||||||
if (outcome.status === 'RESCHEDULED') {
|
if (outcome.status === 'RESCHEDULED') {
|
||||||
|
if ('clockFenceFailed' in outcome) {
|
||||||
|
await redis.zAdd(timerKey, [{ score: nowTick ?? nowMs, value: id }]);
|
||||||
|
return 'RESCHEDULED';
|
||||||
|
}
|
||||||
const gameTime = await loadCurrentGameTime(db, now);
|
const gameTime = await loadCurrentGameTime(db, now);
|
||||||
await redis.zAdd(timerKey, [
|
await redis.zAdd(timerKey, [
|
||||||
{
|
{
|
||||||
@@ -324,6 +395,17 @@ export const runAuctionWorker = async (options: AuctionWorkerOptions = {}): Prom
|
|||||||
const gameTime = await loadCurrentGameTime(postgres.prisma, new Date(operationalNowMs));
|
const gameTime = await loadCurrentGameTime(postgres.prisma, new Date(operationalNowMs));
|
||||||
const gameNowMs = gameTime.now.getTime();
|
const gameNowMs = gameTime.now.getTime();
|
||||||
const dueScore = gameTime.tick ?? gameNowMs;
|
const dueScore = gameTime.tick ?? gameNowMs;
|
||||||
|
if (gameTime.phase && gameTime.phase !== 'RUNNING') {
|
||||||
|
await waitForWorkerPoll(control.signal, config.auctionTimerPollMs);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
const clockFence = gameTime.phase
|
||||||
|
? await ensureActiveRedisClockFence(redis.client, config.profileName, gameTime)
|
||||||
|
: null;
|
||||||
|
if (gameTime.phase && !clockFence) {
|
||||||
|
await waitForWorkerPoll(control.signal, config.auctionTimerPollMs);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
if (operationalNowMs >= nextResyncAt) {
|
if (operationalNowMs >= nextResyncAt) {
|
||||||
await seedAuctionTimers(postgres.prisma, redis.client, keys);
|
await seedAuctionTimers(postgres.prisma, redis.client, keys);
|
||||||
nextResyncAt = operationalNowMs + config.auctionTimerResyncMs;
|
nextResyncAt = operationalNowMs + config.auctionTimerResyncMs;
|
||||||
@@ -345,7 +427,7 @@ export const runAuctionWorker = async (options: AuctionWorkerOptions = {}): Prom
|
|||||||
if (historyTrimBefore > 0) {
|
if (historyTrimBefore > 0) {
|
||||||
await redis.client.zRemRangeByScore(keys.historyKey, 0, historyTrimBefore);
|
await redis.client.zRemRangeByScore(keys.historyKey, 0, historyTrimBefore);
|
||||||
}
|
}
|
||||||
const dueIds = await popDueAuctionIds(redis.client, keys.timerKey, dueScore, 100);
|
const dueIds = await popDueAuctionIds(redis.client, keys.timerKey, dueScore, 100, clockFence ?? undefined);
|
||||||
if (dueIds.length > 0) {
|
if (dueIds.length > 0) {
|
||||||
for (const id of dueIds) {
|
for (const id of dueIds) {
|
||||||
try {
|
try {
|
||||||
@@ -358,6 +440,12 @@ export const runAuctionWorker = async (options: AuctionWorkerOptions = {}): Prom
|
|||||||
nowMs: gameNowMs,
|
nowMs: gameNowMs,
|
||||||
nowTick: gameTime.tick,
|
nowTick: gameTime.tick,
|
||||||
historyNowMs: operationalNowMs,
|
historyNowMs: operationalNowMs,
|
||||||
|
...(clockFence
|
||||||
|
? {
|
||||||
|
expectedClockRevision: clockFence.revision,
|
||||||
|
expectedDeadlineGeneration: clockFence.generation,
|
||||||
|
}
|
||||||
|
: {}),
|
||||||
});
|
});
|
||||||
if (outcome === 'PENDING') {
|
if (outcome === 'PENDING') {
|
||||||
pendingFinalizationIds.add(Number(id));
|
pendingFinalizationIds.add(Number(id));
|
||||||
|
|||||||
@@ -1,6 +1,12 @@
|
|||||||
import { randomUUID } from 'node:crypto';
|
import { randomUUID } from 'node:crypto';
|
||||||
|
|
||||||
import { acquireGameSchemaAdvisoryXactLock, type DatabaseClient, type GamePrisma } from '@sammo-ts/infra';
|
import {
|
||||||
|
acquireGameSchemaAdvisoryXactLock,
|
||||||
|
readInputEventClockCoordinate,
|
||||||
|
type DatabaseClient,
|
||||||
|
type GamePrisma,
|
||||||
|
type InputEventClockCoordinate,
|
||||||
|
} from '@sammo-ts/infra';
|
||||||
|
|
||||||
import type { TurnDaemonTransport } from './transport.js';
|
import type { TurnDaemonTransport } from './transport.js';
|
||||||
import type { TurnDaemonCommand, TurnDaemonCommandResult, TurnDaemonStatus } from './types.js';
|
import type { TurnDaemonCommand, TurnDaemonCommandResult, TurnDaemonStatus } from './types.js';
|
||||||
@@ -103,10 +109,10 @@ export class DatabaseTurnDaemonTransport implements TurnDaemonTransport {
|
|||||||
try {
|
try {
|
||||||
if (command.type === 'npcPossessGeneral' && this.db.$transaction) {
|
if (command.type === 'npcPossessGeneral' && this.db.$transaction) {
|
||||||
const rejectionReason = await this.db.$transaction(async (transaction) => {
|
const rejectionReason = await this.db.$transaction(async (transaction) => {
|
||||||
|
const coordinate = await readInputEventClockCoordinate(transaction);
|
||||||
await acquireGameSchemaAdvisoryXactLock(transaction, 'npc-possession:global');
|
await acquireGameSchemaAdvisoryXactLock(transaction, 'npc-possession:global');
|
||||||
await acquireGameSchemaAdvisoryXactLock(transaction, `npc-possession:user:${command.userId}`);
|
await acquireGameSchemaAdvisoryXactLock(transaction, `npc-possession:user:${command.userId}`);
|
||||||
const acceptedAt = new Date(Math.floor(Date.now() / 1000) * 1000);
|
const acceptedGameAt = coordinate.gameAt;
|
||||||
const acceptedGameAt = (await loadCurrentGameTime(transaction, acceptedAt)).now;
|
|
||||||
const token = await transaction.npcSelectionToken.findFirst({
|
const token = await transaction.npcSelectionToken.findFirst({
|
||||||
where: {
|
where: {
|
||||||
ownerUserId: command.userId,
|
ownerUserId: command.userId,
|
||||||
@@ -130,15 +136,22 @@ export class DatabaseTurnDaemonTransport implements TurnDaemonTransport {
|
|||||||
...(durableCommand as Extract<TurnDaemonCommand, { type: 'npcPossessGeneral' }>),
|
...(durableCommand as Extract<TurnDaemonCommand, { type: 'npcPossessGeneral' }>),
|
||||||
acceptedGameAt: acceptedGameAt.toISOString(),
|
acceptedGameAt: acceptedGameAt.toISOString(),
|
||||||
};
|
};
|
||||||
await this.createInputEvent(transaction, acceptedCommand, requestId, acceptedAt);
|
await this.createInputEvent(transaction, acceptedCommand, requestId, coordinate);
|
||||||
return null;
|
return null;
|
||||||
});
|
});
|
||||||
if (rejectionReason) {
|
if (rejectionReason) {
|
||||||
throw new RejectedNpcPossessionCommandError('PRECONDITION_FAILED', rejectionReason);
|
throw new RejectedNpcPossessionCommandError('PRECONDITION_FAILED', rejectionReason);
|
||||||
}
|
}
|
||||||
|
} else {
|
||||||
|
if (this.db.$transaction) {
|
||||||
|
await this.db.$transaction(async (transaction) => {
|
||||||
|
const coordinate = await readInputEventClockCoordinate(transaction);
|
||||||
|
await this.createInputEvent(transaction, durableCommand, requestId, coordinate);
|
||||||
|
});
|
||||||
} else {
|
} else {
|
||||||
await this.createInputEvent(this.db, durableCommand, requestId);
|
await this.createInputEvent(this.db, durableCommand, requestId);
|
||||||
}
|
}
|
||||||
|
}
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
if (error instanceof RejectedNpcPossessionCommandError) {
|
if (error instanceof RejectedNpcPossessionCommandError) {
|
||||||
throw error;
|
throw error;
|
||||||
@@ -166,8 +179,20 @@ export class DatabaseTurnDaemonTransport implements TurnDaemonTransport {
|
|||||||
db: DatabaseClient,
|
db: DatabaseClient,
|
||||||
command: TurnDaemonCommand,
|
command: TurnDaemonCommand,
|
||||||
requestId: string,
|
requestId: string,
|
||||||
createdAt?: Date
|
coordinate?: InputEventClockCoordinate
|
||||||
): Promise<void> {
|
): Promise<void> {
|
||||||
|
const gameTime = coordinate ? null : await loadCurrentGameTime(db, new Date());
|
||||||
|
const commandAcceptedTick = Reflect.get(command, 'acceptedGameTick');
|
||||||
|
const acceptedGameTick =
|
||||||
|
typeof commandAcceptedTick === 'number' && Number.isSafeInteger(commandAcceptedTick)
|
||||||
|
? commandAcceptedTick
|
||||||
|
: coordinate
|
||||||
|
? Number(coordinate.gameTick)
|
||||||
|
: gameTime!.tick;
|
||||||
|
const acceptedClockRevision = coordinate ? Number(coordinate.clockRevision) : gameTime!.revision;
|
||||||
|
const acceptedDeadlineGeneration = coordinate
|
||||||
|
? Number(coordinate.deadlineGeneration)
|
||||||
|
: gameTime!.deadlineGeneration;
|
||||||
await db.inputEvent.create({
|
await db.inputEvent.create({
|
||||||
data: {
|
data: {
|
||||||
requestId,
|
requestId,
|
||||||
@@ -175,7 +200,14 @@ export class DatabaseTurnDaemonTransport implements TurnDaemonTransport {
|
|||||||
eventType: command.type,
|
eventType: command.type,
|
||||||
payload: asJson(command),
|
payload: asJson(command),
|
||||||
actorUserId: 'userId' in command && typeof command.userId === 'string' ? command.userId : null,
|
actorUserId: 'userId' in command && typeof command.userId === 'string' ? command.userId : null,
|
||||||
...(createdAt ? { createdAt } : {}),
|
...(acceptedGameTick === null ? {} : { acceptedGameTick: BigInt(acceptedGameTick) }),
|
||||||
|
...(acceptedClockRevision === null || acceptedClockRevision === undefined
|
||||||
|
? {}
|
||||||
|
: { acceptedClockRevision: BigInt(acceptedClockRevision) }),
|
||||||
|
...(acceptedDeadlineGeneration === null || acceptedDeadlineGeneration === undefined
|
||||||
|
? {}
|
||||||
|
: { acceptedDeadlineGeneration: BigInt(acceptedDeadlineGeneration) }),
|
||||||
|
...(coordinate ? { createdAt: coordinate.wallAt } : {}),
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,6 +1,11 @@
|
|||||||
import { createHash } from 'node:crypto';
|
import { createHash } from 'node:crypto';
|
||||||
|
|
||||||
import { GamePrisma, type DatabaseClient as InfraDatabaseClient } from '@sammo-ts/infra';
|
import {
|
||||||
|
CLOCK_OPERATION_PERSISTENCE_LOCK,
|
||||||
|
GamePrisma,
|
||||||
|
acquireGameSchemaAdvisoryXactLock,
|
||||||
|
type DatabaseClient as InfraDatabaseClient,
|
||||||
|
} from '@sammo-ts/infra';
|
||||||
|
|
||||||
import type { DatabaseClient } from './context.js';
|
import type { DatabaseClient } from './context.js';
|
||||||
|
|
||||||
@@ -20,6 +25,9 @@ interface LockedInputEvent {
|
|||||||
status: 'PENDING' | 'PROCESSING' | 'SUCCEEDED' | 'FAILED';
|
status: 'PENDING' | 'PROCESSING' | 'SUCCEEDED' | 'FAILED';
|
||||||
result: GamePrisma.JsonValue | null;
|
result: GamePrisma.JsonValue | null;
|
||||||
attempts: number;
|
attempts: number;
|
||||||
|
acceptedGameTick: bigint | null;
|
||||||
|
acceptedClockRevision: bigint | null;
|
||||||
|
acceptedDeadlineGeneration: bigint | null;
|
||||||
}
|
}
|
||||||
|
|
||||||
type InputEventOutcome<T> =
|
type InputEventOutcome<T> =
|
||||||
@@ -85,6 +93,9 @@ const insertPendingIfAbsent = async (
|
|||||||
actor_user_id,
|
actor_user_id,
|
||||||
status,
|
status,
|
||||||
attempts,
|
attempts,
|
||||||
|
accepted_game_tick,
|
||||||
|
accepted_clock_revision,
|
||||||
|
accepted_deadline_generation,
|
||||||
created_at
|
created_at
|
||||||
)
|
)
|
||||||
VALUES (
|
VALUES (
|
||||||
@@ -95,6 +106,9 @@ const insertPendingIfAbsent = async (
|
|||||||
${options.actorUserId},
|
${options.actorUserId},
|
||||||
'PENDING'::"InputEventStatus",
|
'PENDING'::"InputEventStatus",
|
||||||
0,
|
0,
|
||||||
|
(SELECT clock_tick FROM world_state ORDER BY id ASC LIMIT 1),
|
||||||
|
(SELECT clock_revision FROM world_state ORDER BY id ASC LIMIT 1),
|
||||||
|
(SELECT deadline_generation FROM world_state ORDER BY id ASC LIMIT 1),
|
||||||
CURRENT_TIMESTAMP AT TIME ZONE 'UTC'
|
CURRENT_TIMESTAMP AT TIME ZONE 'UTC'
|
||||||
)
|
)
|
||||||
ON CONFLICT (request_id) DO NOTHING
|
ON CONFLICT (request_id) DO NOTHING
|
||||||
@@ -112,7 +126,10 @@ const lockInputEvent = async (db: DatabaseClient, requestId: string): Promise<Lo
|
|||||||
actor_user_id AS "actorUserId",
|
actor_user_id AS "actorUserId",
|
||||||
status,
|
status,
|
||||||
result,
|
result,
|
||||||
attempts
|
attempts,
|
||||||
|
accepted_game_tick AS "acceptedGameTick",
|
||||||
|
accepted_clock_revision AS "acceptedClockRevision",
|
||||||
|
accepted_deadline_generation AS "acceptedDeadlineGeneration"
|
||||||
FROM input_event
|
FROM input_event
|
||||||
WHERE request_id = ${requestId}
|
WHERE request_id = ${requestId}
|
||||||
FOR UPDATE
|
FOR UPDATE
|
||||||
@@ -151,7 +168,8 @@ const isMatchingIdentity = (
|
|||||||
const claimInputEvent = async (
|
const claimInputEvent = async (
|
||||||
db: DatabaseClient,
|
db: DatabaseClient,
|
||||||
requestId: string,
|
requestId: string,
|
||||||
payloadIdentity: ApiInputPayloadIdentity
|
payloadIdentity: ApiInputPayloadIdentity,
|
||||||
|
row: LockedInputEvent
|
||||||
): Promise<void> => {
|
): Promise<void> => {
|
||||||
await db.inputEvent.update({
|
await db.inputEvent.update({
|
||||||
where: { requestId },
|
where: { requestId },
|
||||||
@@ -164,6 +182,9 @@ const claimInputEvent = async (
|
|||||||
lockedBy: null,
|
lockedBy: null,
|
||||||
leaseUntil: null,
|
leaseUntil: null,
|
||||||
processingAt: new Date(),
|
processingAt: new Date(),
|
||||||
|
processingGameTick: row.acceptedGameTick,
|
||||||
|
processingClockRevision: row.acceptedClockRevision,
|
||||||
|
processingDeadlineGeneration: row.acceptedDeadlineGeneration,
|
||||||
completedAt: null,
|
completedAt: null,
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
@@ -184,6 +205,7 @@ const markUnexpectedFailure = async (
|
|||||||
|
|
||||||
try {
|
try {
|
||||||
await db.$transaction(async (transaction) => {
|
await db.$transaction(async (transaction) => {
|
||||||
|
await acquireGameSchemaAdvisoryXactLock(transaction, CLOCK_OPERATION_PERSISTENCE_LOCK);
|
||||||
await insertPendingIfAbsent(transaction, options);
|
await insertPendingIfAbsent(transaction, options);
|
||||||
const row = await lockInputEvent(transaction, options.requestId);
|
const row = await lockInputEvent(transaction, options.requestId);
|
||||||
const identityMatches = isMatchingIdentity(row, options) || canAdoptLegacyFailedPayload(row, options);
|
const identityMatches = isMatchingIdentity(row, options) || canAdoptLegacyFailedPayload(row, options);
|
||||||
@@ -233,6 +255,7 @@ export const executeInputEvent = async <T>(options: {
|
|||||||
let outcome: InputEventOutcome<T>;
|
let outcome: InputEventOutcome<T>;
|
||||||
try {
|
try {
|
||||||
outcome = await db.$transaction(async (transaction) => {
|
outcome = await db.$transaction(async (transaction) => {
|
||||||
|
await acquireGameSchemaAdvisoryXactLock(transaction, CLOCK_OPERATION_PERSISTENCE_LOCK);
|
||||||
await insertPendingIfAbsent(transaction, { requestId, eventType, actorUserId, payloadIdentity });
|
await insertPendingIfAbsent(transaction, { requestId, eventType, actorUserId, payloadIdentity });
|
||||||
const row = await lockInputEvent(transaction, requestId);
|
const row = await lockInputEvent(transaction, requestId);
|
||||||
const identityMatches = isMatchingIdentity(row, { eventType, actorUserId, payloadIdentity });
|
const identityMatches = isMatchingIdentity(row, { eventType, actorUserId, payloadIdentity });
|
||||||
@@ -251,7 +274,7 @@ export const executeInputEvent = async <T>(options: {
|
|||||||
throw new DuplicateInputEventError(requestId);
|
throw new DuplicateInputEventError(requestId);
|
||||||
}
|
}
|
||||||
|
|
||||||
await claimInputEvent(transaction, requestId, payloadIdentity);
|
await claimInputEvent(transaction, requestId, payloadIdentity, row);
|
||||||
const savepointDb = transaction as SavepointDatabaseClient;
|
const savepointDb = transaction as SavepointDatabaseClient;
|
||||||
await savepointDb.$executeRawUnsafe(`SAVEPOINT ${BUSINESS_SAVEPOINT}`);
|
await savepointDb.$executeRawUnsafe(`SAVEPOINT ${BUSINESS_SAVEPOINT}`);
|
||||||
businessStarted = true;
|
businessStarted = true;
|
||||||
|
|||||||
@@ -5,12 +5,14 @@ import { asRecord } from '@sammo-ts/common';
|
|||||||
import type { TournamentType } from '@sammo-ts/logic';
|
import type { TournamentType } from '@sammo-ts/logic';
|
||||||
import type { TournamentState } from '../../tournament/types.js';
|
import type { TournamentState } from '../../tournament/types.js';
|
||||||
|
|
||||||
import { TournamentStore } from '../../tournament/store.js';
|
import { TournamentStore, type TournamentClockContext } from '../../tournament/store.js';
|
||||||
import { buildTournamentKeys } from '../../tournament/keys.js';
|
import { buildTournamentKeys } from '../../tournament/keys.js';
|
||||||
import { assignManualApplicantGroup } from '../../tournament/workerHelpers.js';
|
import { assignManualApplicantGroup } from '../../tournament/workerHelpers.js';
|
||||||
import { accessAuthedProcedure, authedProcedure, router } from '../../trpc.js';
|
import { accessAuthedProcedure, authedProcedure, router } from '../../trpc.js';
|
||||||
import { getMyGeneral } from '../shared/general.js';
|
import { getMyGeneral } from '../shared/general.js';
|
||||||
import { loadCurrentGameTime } from '../../services/gameClock.js';
|
import { loadCurrentGameTime } from '../../services/gameClock.js';
|
||||||
|
import { ensureActiveRedisClockFence } from '../../services/redisClockFence.js';
|
||||||
|
import { loadClockAdminStatus } from '../../services/clockReadiness.js';
|
||||||
|
|
||||||
const hasAdminRole = (roles: string[], profileName: string): boolean => {
|
const hasAdminRole = (roles: string[], profileName: string): boolean => {
|
||||||
if (roles.includes('superuser') || roles.includes('admin') || roles.includes('admin.superuser')) {
|
if (roles.includes('superuser') || roles.includes('admin') || roles.includes('admin.superuser')) {
|
||||||
@@ -44,6 +46,32 @@ const adminProcedure = authedProcedure.use(({ ctx, next }) => {
|
|||||||
return next();
|
return next();
|
||||||
});
|
});
|
||||||
|
|
||||||
|
const withTournamentClockMutation = async <T>(
|
||||||
|
ctx: {
|
||||||
|
db: Parameters<typeof loadCurrentGameTime>[0];
|
||||||
|
redis: Parameters<typeof ensureActiveRedisClockFence>[0];
|
||||||
|
profile: { name: string };
|
||||||
|
},
|
||||||
|
store: TournamentStore,
|
||||||
|
operation: () => Promise<T>
|
||||||
|
): Promise<T> => {
|
||||||
|
const gameTime = await loadCurrentGameTime(ctx.db);
|
||||||
|
const fence = await ensureActiveRedisClockFence(ctx.redis, ctx.profile.name, gameTime);
|
||||||
|
if (!fence) {
|
||||||
|
throw new TRPCError({
|
||||||
|
code: 'PRECONDITION_FAILED',
|
||||||
|
message: 'Clock reconciliation is incomplete; tournament mutation is disabled.',
|
||||||
|
});
|
||||||
|
}
|
||||||
|
const clockContext: TournamentClockContext = {
|
||||||
|
phase: 'RUNNING',
|
||||||
|
revision: fence.revision,
|
||||||
|
deadlineGeneration: fence.generation,
|
||||||
|
dateToTick: gameTime.dateToTick,
|
||||||
|
};
|
||||||
|
return store.withClockContext(clockContext, () => store.withMutationLock(operation));
|
||||||
|
};
|
||||||
|
|
||||||
const zTournamentState = z.object({
|
const zTournamentState = z.object({
|
||||||
stage: z.number().int().min(0),
|
stage: z.number().int().min(0),
|
||||||
phase: z.number().int().min(0),
|
phase: z.number().int().min(0),
|
||||||
@@ -143,7 +171,10 @@ export const tournamentRouter = router({
|
|||||||
const store = new TournamentStore(ctx.redis, buildTournamentKeys(ctx.profile.name));
|
const store = new TournamentStore(ctx.redis, buildTournamentKeys(ctx.profile.name));
|
||||||
return store.getState();
|
return store.getState();
|
||||||
}),
|
}),
|
||||||
getAdminStatus: adminProcedure.query(async () => ({ ok: true })),
|
getAdminStatus: adminProcedure.query(async ({ ctx }) => ({
|
||||||
|
ok: true,
|
||||||
|
clock: await loadClockAdminStatus(ctx.db),
|
||||||
|
})),
|
||||||
getSnapshot: accessAuthedProcedure.query(async ({ ctx }) => {
|
getSnapshot: accessAuthedProcedure.query(async ({ ctx }) => {
|
||||||
await getMyGeneral(ctx);
|
await getMyGeneral(ctx);
|
||||||
const store = new TournamentStore(ctx.redis, buildTournamentKeys(ctx.profile.name));
|
const store = new TournamentStore(ctx.redis, buildTournamentKeys(ctx.profile.name));
|
||||||
@@ -271,7 +302,7 @@ export const tournamentRouter = router({
|
|||||||
}),
|
}),
|
||||||
setState: adminProcedure.input(zTournamentState).mutation(async ({ ctx, input }) => {
|
setState: adminProcedure.input(zTournamentState).mutation(async ({ ctx, input }) => {
|
||||||
const store = new TournamentStore(ctx.redis, buildTournamentKeys(ctx.profile.name));
|
const store = new TournamentStore(ctx.redis, buildTournamentKeys(ctx.profile.name));
|
||||||
return store.withMutationLock(async () => {
|
return withTournamentClockMutation(ctx, store, async () => {
|
||||||
await store.setState({
|
await store.setState({
|
||||||
...input,
|
...input,
|
||||||
type: input.type as TournamentType,
|
type: input.type as TournamentType,
|
||||||
@@ -281,7 +312,7 @@ export const tournamentRouter = router({
|
|||||||
}),
|
}),
|
||||||
patchState: adminProcedure.input(zTournamentState.partial()).mutation(async ({ ctx, input }) => {
|
patchState: adminProcedure.input(zTournamentState.partial()).mutation(async ({ ctx, input }) => {
|
||||||
const store = new TournamentStore(ctx.redis, buildTournamentKeys(ctx.profile.name));
|
const store = new TournamentStore(ctx.redis, buildTournamentKeys(ctx.profile.name));
|
||||||
return store.withMutationLock(async () => {
|
return withTournamentClockMutation(ctx, store, async () => {
|
||||||
const current = await store.getState();
|
const current = await store.getState();
|
||||||
if (!current) {
|
if (!current) {
|
||||||
throw new TRPCError({ code: 'NOT_FOUND', message: 'Tournament state not found.' });
|
throw new TRPCError({ code: 'NOT_FOUND', message: 'Tournament state not found.' });
|
||||||
@@ -297,21 +328,21 @@ export const tournamentRouter = router({
|
|||||||
}),
|
}),
|
||||||
setParticipants: adminProcedure.input(z.array(zParticipant)).mutation(async ({ ctx, input }) => {
|
setParticipants: adminProcedure.input(z.array(zParticipant)).mutation(async ({ ctx, input }) => {
|
||||||
const store = new TournamentStore(ctx.redis, buildTournamentKeys(ctx.profile.name));
|
const store = new TournamentStore(ctx.redis, buildTournamentKeys(ctx.profile.name));
|
||||||
return store.withMutationLock(async () => {
|
return withTournamentClockMutation(ctx, store, async () => {
|
||||||
await store.setParticipants(input);
|
await store.setParticipants(input);
|
||||||
return { ok: true, count: input.length };
|
return { ok: true, count: input.length };
|
||||||
});
|
});
|
||||||
}),
|
}),
|
||||||
setMatches: adminProcedure.input(z.array(zMatch)).mutation(async ({ ctx, input }) => {
|
setMatches: adminProcedure.input(z.array(zMatch)).mutation(async ({ ctx, input }) => {
|
||||||
const store = new TournamentStore(ctx.redis, buildTournamentKeys(ctx.profile.name));
|
const store = new TournamentStore(ctx.redis, buildTournamentKeys(ctx.profile.name));
|
||||||
return store.withMutationLock(async () => {
|
return withTournamentClockMutation(ctx, store, async () => {
|
||||||
await store.setMatches(input);
|
await store.setMatches(input);
|
||||||
return { ok: true, count: input.length };
|
return { ok: true, count: input.length };
|
||||||
});
|
});
|
||||||
}),
|
}),
|
||||||
setBettingEntries: adminProcedure.input(z.array(zBetEntry)).mutation(async ({ ctx, input }) => {
|
setBettingEntries: adminProcedure.input(z.array(zBetEntry)).mutation(async ({ ctx, input }) => {
|
||||||
const store = new TournamentStore(ctx.redis, buildTournamentKeys(ctx.profile.name));
|
const store = new TournamentStore(ctx.redis, buildTournamentKeys(ctx.profile.name));
|
||||||
return store.withMutationLock(async () => {
|
return withTournamentClockMutation(ctx, store, async () => {
|
||||||
await store.setBettingEntries(input);
|
await store.setBettingEntries(input);
|
||||||
return { ok: true, count: input.length };
|
return { ok: true, count: input.length };
|
||||||
});
|
});
|
||||||
@@ -347,7 +378,7 @@ export const tournamentRouter = router({
|
|||||||
};
|
};
|
||||||
});
|
});
|
||||||
|
|
||||||
return store.withMutationLock(async () => {
|
return withTournamentClockMutation(ctx, store, async () => {
|
||||||
await store.setParticipants(participants);
|
await store.setParticipants(participants);
|
||||||
return { ok: true, count: participants.length };
|
return { ok: true, count: participants.length };
|
||||||
});
|
});
|
||||||
@@ -394,7 +425,7 @@ export const tournamentRouter = router({
|
|||||||
join: authedProcedure.mutation(async ({ ctx }) => {
|
join: authedProcedure.mutation(async ({ ctx }) => {
|
||||||
const general = await getMyGeneral(ctx);
|
const general = await getMyGeneral(ctx);
|
||||||
const store = new TournamentStore(ctx.redis, buildTournamentKeys(ctx.profile.name));
|
const store = new TournamentStore(ctx.redis, buildTournamentKeys(ctx.profile.name));
|
||||||
return store.withMutationLock(async () => {
|
return withTournamentClockMutation(ctx, store, async () => {
|
||||||
const state = await store.getState();
|
const state = await store.getState();
|
||||||
if (!state || state.stage !== 1 || state.participantsLockedAt) {
|
if (!state || state.stage !== 1 || state.participantsLockedAt) {
|
||||||
throw new TRPCError({ code: 'BAD_REQUEST', message: '참가 신청 기간이 아닙니다.' });
|
throw new TRPCError({ code: 'BAD_REQUEST', message: '참가 신청 기간이 아닙니다.' });
|
||||||
@@ -459,7 +490,7 @@ export const tournamentRouter = router({
|
|||||||
}),
|
}),
|
||||||
cancel: adminProcedure.mutation(async ({ ctx }) => {
|
cancel: adminProcedure.mutation(async ({ ctx }) => {
|
||||||
const store = new TournamentStore(ctx.redis, buildTournamentKeys(ctx.profile.name));
|
const store = new TournamentStore(ctx.redis, buildTournamentKeys(ctx.profile.name));
|
||||||
return store.withMutationLock(async () => {
|
return withTournamentClockMutation(ctx, store, async () => {
|
||||||
const state = await store.getState();
|
const state = await store.getState();
|
||||||
if (!state) {
|
if (!state) {
|
||||||
throw new TRPCError({ code: 'NOT_FOUND', message: 'Tournament state not found.' });
|
throw new TRPCError({ code: 'NOT_FOUND', message: 'Tournament state not found.' });
|
||||||
@@ -523,7 +554,7 @@ export const tournamentRouter = router({
|
|||||||
.mutation(async ({ ctx, input }) => {
|
.mutation(async ({ ctx, input }) => {
|
||||||
const general = await getMyGeneral(ctx);
|
const general = await getMyGeneral(ctx);
|
||||||
const store = new TournamentStore(ctx.redis, buildTournamentKeys(ctx.profile.name));
|
const store = new TournamentStore(ctx.redis, buildTournamentKeys(ctx.profile.name));
|
||||||
return store.withMutationLock(async () => {
|
return withTournamentClockMutation(ctx, store, async () => {
|
||||||
const state = await store.getState();
|
const state = await store.getState();
|
||||||
if (!state || state.stage !== 6) {
|
if (!state || state.stage !== 6) {
|
||||||
throw new TRPCError({ code: 'BAD_REQUEST', message: '베팅 기간이 아닙니다.' });
|
throw new TRPCError({ code: 'BAD_REQUEST', message: '베팅 기간이 아닙니다.' });
|
||||||
|
|||||||
@@ -50,6 +50,7 @@ import { ReadModelOutboxWorker } from './realtime/outboxWorker.js';
|
|||||||
import { DeferredGeneralAccessWorker } from './services/deferredGeneralAccess.js';
|
import { DeferredGeneralAccessWorker } from './services/deferredGeneralAccess.js';
|
||||||
import { WebPushOutboxWorker } from './services/webPushOutboxWorker.js';
|
import { WebPushOutboxWorker } from './services/webPushOutboxWorker.js';
|
||||||
import { scopeHttpIdempotencyKey } from './requestId.js';
|
import { scopeHttpIdempotencyKey } from './requestId.js';
|
||||||
|
import { loadClockReadiness } from './services/clockReadiness.js';
|
||||||
|
|
||||||
const extractBearerToken = (value: string | string[] | undefined): string | null => {
|
const extractBearerToken = (value: string | string[] | undefined): string | null => {
|
||||||
if (!value) {
|
if (!value) {
|
||||||
@@ -420,12 +421,17 @@ export const createGameApiServer = async () => {
|
|||||||
request.raw.on('aborted', close);
|
request.raw.on('aborted', close);
|
||||||
});
|
});
|
||||||
|
|
||||||
app.get('/healthz', async () => ({
|
app.get('/healthz', async (_request, reply) => {
|
||||||
ok: true,
|
const clock = await loadClockReadiness(postgres.prisma);
|
||||||
|
if (!clock.reconciliationComplete) reply.code(503);
|
||||||
|
return {
|
||||||
|
ok: clock.reconciliationComplete,
|
||||||
profile: config.profileName,
|
profile: config.profileName,
|
||||||
postgresPool: postgres.getPoolStats(),
|
postgresPool: postgres.getPoolStats(),
|
||||||
accountIconReconciliation: accountIconResetReconciler.getHealth(),
|
accountIconReconciliation: accountIconResetReconciler.getHealth(),
|
||||||
}));
|
clock,
|
||||||
|
};
|
||||||
|
});
|
||||||
|
|
||||||
try {
|
try {
|
||||||
await realtimeHub.start();
|
await realtimeHub.start();
|
||||||
|
|||||||
@@ -0,0 +1,92 @@
|
|||||||
|
import { parseGameClockPhase } from '@sammo-ts/common';
|
||||||
|
|
||||||
|
import type { DatabaseClient } from '../context.js';
|
||||||
|
|
||||||
|
const safeInteger = (value: bigint, label: string): number => {
|
||||||
|
const result = Number(value);
|
||||||
|
if (!Number.isSafeInteger(result)) throw new Error(`${label} is outside the safe integer range.`);
|
||||||
|
return result;
|
||||||
|
};
|
||||||
|
|
||||||
|
export const loadClockReadiness = async (db: DatabaseClient) => {
|
||||||
|
if (!db.clockProjectionOutbox) {
|
||||||
|
return {
|
||||||
|
reconciliationComplete: false,
|
||||||
|
gameplayEnabled: false,
|
||||||
|
phase: null,
|
||||||
|
revision: null,
|
||||||
|
deadlineGeneration: null,
|
||||||
|
incompleteOutboxCount: null,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
const [world, incompleteOutboxCount] = await Promise.all([
|
||||||
|
db.worldState.findFirst({
|
||||||
|
orderBy: { id: 'asc' },
|
||||||
|
select: { clockPhase: true, clockRevision: true, deadlineGeneration: true },
|
||||||
|
}),
|
||||||
|
db.clockProjectionOutbox.count({ where: { status: { not: 'APPLIED' } } }),
|
||||||
|
]);
|
||||||
|
if (!world) {
|
||||||
|
return {
|
||||||
|
reconciliationComplete: false,
|
||||||
|
gameplayEnabled: false,
|
||||||
|
phase: null,
|
||||||
|
revision: null,
|
||||||
|
deadlineGeneration: null,
|
||||||
|
incompleteOutboxCount,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
const phase = parseGameClockPhase(world.clockPhase);
|
||||||
|
return {
|
||||||
|
reconciliationComplete: phase !== 'RECONCILING' && incompleteOutboxCount === 0,
|
||||||
|
gameplayEnabled: phase === 'RUNNING' || phase === 'MANUAL',
|
||||||
|
phase,
|
||||||
|
revision: safeInteger(world.clockRevision, 'clock revision'),
|
||||||
|
deadlineGeneration: safeInteger(world.deadlineGeneration, 'deadline generation'),
|
||||||
|
incompleteOutboxCount,
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
export const loadClockAdminStatus = async (db: DatabaseClient) => {
|
||||||
|
const readiness = await loadClockReadiness(db);
|
||||||
|
if (!db.clockSuspension) {
|
||||||
|
return { ...readiness, latestReconciliation: null };
|
||||||
|
}
|
||||||
|
const latest = await db.clockSuspension.findFirst({
|
||||||
|
orderBy: { createdAt: 'desc' },
|
||||||
|
include: {
|
||||||
|
participants: { orderBy: { participantKey: 'asc' } },
|
||||||
|
projectionOutbox: { orderBy: { id: 'asc' } },
|
||||||
|
},
|
||||||
|
});
|
||||||
|
if (!latest) return { ...readiness, latestReconciliation: null };
|
||||||
|
return {
|
||||||
|
...readiness,
|
||||||
|
latestReconciliation: {
|
||||||
|
id: latest.id,
|
||||||
|
source: latest.source,
|
||||||
|
policy: latest.policy,
|
||||||
|
status: latest.status,
|
||||||
|
sourceRevision: safeInteger(latest.sourceRevision, 'source revision'),
|
||||||
|
targetRevision: safeInteger(latest.targetRevision, 'target revision'),
|
||||||
|
cutTick: safeInteger(latest.cutTick, 'cut tick'),
|
||||||
|
alignedTick: latest.alignedTick === null ? null : safeInteger(latest.alignedTick, 'aligned tick'),
|
||||||
|
participantChecksumBefore: latest.participantChecksumBefore,
|
||||||
|
participantChecksumAfter: latest.participantChecksumAfter,
|
||||||
|
participants: latest.participants.map((participant) => ({
|
||||||
|
key: participant.participantKey,
|
||||||
|
policy: participant.policy,
|
||||||
|
beforeChecksum: participant.beforeChecksum,
|
||||||
|
afterChecksum: participant.afterChecksum,
|
||||||
|
affectedCount: participant.affectedCount,
|
||||||
|
})),
|
||||||
|
outbox: latest.projectionOutbox.map((entry) => ({
|
||||||
|
id: entry.id.toString(),
|
||||||
|
targetRevision: safeInteger(entry.targetRevision, 'outbox target revision'),
|
||||||
|
status: entry.status,
|
||||||
|
attempts: entry.attempts,
|
||||||
|
lastError: entry.lastError,
|
||||||
|
})),
|
||||||
|
},
|
||||||
|
};
|
||||||
|
};
|
||||||
@@ -0,0 +1,55 @@
|
|||||||
|
import type { CurrentGameTime } from './gameClock.js';
|
||||||
|
|
||||||
|
interface ClockFenceRedis {
|
||||||
|
eval(script: string, options: { keys: string[]; arguments: string[] }): Promise<unknown>;
|
||||||
|
}
|
||||||
|
|
||||||
|
const BOOTSTRAP_CLOCK_FENCE_SCRIPT = `
|
||||||
|
local revision = redis.call('GET', KEYS[1])
|
||||||
|
local generation = redis.call('GET', KEYS[2])
|
||||||
|
local phase = redis.call('GET', KEYS[3])
|
||||||
|
if not revision and not generation and not phase then
|
||||||
|
redis.call('SET', KEYS[1], ARGV[1])
|
||||||
|
redis.call('SET', KEYS[2], ARGV[2])
|
||||||
|
redis.call('SET', KEYS[3], ARGV[3])
|
||||||
|
return 1
|
||||||
|
end
|
||||||
|
if revision == ARGV[1] and generation == ARGV[2] and phase == ARGV[3] then
|
||||||
|
return 2
|
||||||
|
end
|
||||||
|
return 0
|
||||||
|
`;
|
||||||
|
|
||||||
|
export interface ActiveRedisClockFence {
|
||||||
|
activeRevisionKey: string;
|
||||||
|
deadlineGenerationKey: string;
|
||||||
|
phaseKey: string;
|
||||||
|
revision: number;
|
||||||
|
generation: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export const ensureActiveRedisClockFence = async (
|
||||||
|
redis: ClockFenceRedis,
|
||||||
|
profileName: string,
|
||||||
|
gameTime: CurrentGameTime
|
||||||
|
): Promise<ActiveRedisClockFence | null> => {
|
||||||
|
if (
|
||||||
|
gameTime.phase !== 'RUNNING' ||
|
||||||
|
!Number.isSafeInteger(gameTime.revision) ||
|
||||||
|
!Number.isSafeInteger(gameTime.deadlineGeneration)
|
||||||
|
) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
const fence: ActiveRedisClockFence = {
|
||||||
|
activeRevisionKey: `sammo:${profileName}:clock:active-revision`,
|
||||||
|
deadlineGenerationKey: `sammo:${profileName}:clock:deadline-generation`,
|
||||||
|
phaseKey: `sammo:${profileName}:clock:phase`,
|
||||||
|
revision: gameTime.revision!,
|
||||||
|
generation: gameTime.deadlineGeneration!,
|
||||||
|
};
|
||||||
|
const result = await redis.eval(BOOTSTRAP_CLOCK_FENCE_SCRIPT, {
|
||||||
|
keys: [fence.activeRevisionKey, fence.deadlineGenerationKey, fence.phaseKey],
|
||||||
|
arguments: [String(fence.revision), String(fence.generation), 'RUNNING'],
|
||||||
|
});
|
||||||
|
return Number(result) === 1 || Number(result) === 2 ? fence : null;
|
||||||
|
};
|
||||||
@@ -8,6 +8,9 @@ export interface TournamentKeys {
|
|||||||
sourceRevisionKey: string;
|
sourceRevisionKey: string;
|
||||||
sourceRevisionChannel: string;
|
sourceRevisionChannel: string;
|
||||||
realtimeEventChannel: string;
|
realtimeEventChannel: string;
|
||||||
|
activeClockRevisionKey: string;
|
||||||
|
deadlineGenerationKey: string;
|
||||||
|
clockPhaseKey: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
export const buildTournamentKeys = (profileName: string): TournamentKeys => ({
|
export const buildTournamentKeys = (profileName: string): TournamentKeys => ({
|
||||||
@@ -18,4 +21,7 @@ export const buildTournamentKeys = (profileName: string): TournamentKeys => ({
|
|||||||
sourceRevisionKey: `sammo:${profileName}:tournament:source-revision`,
|
sourceRevisionKey: `sammo:${profileName}:tournament:source-revision`,
|
||||||
sourceRevisionChannel: `sammo:${profileName}:tournament:source-changed`,
|
sourceRevisionChannel: `sammo:${profileName}:tournament:source-changed`,
|
||||||
realtimeEventChannel: buildGameEventChannel(profileName),
|
realtimeEventChannel: buildGameEventChannel(profileName),
|
||||||
|
activeClockRevisionKey: `sammo:${profileName}:clock:active-revision`,
|
||||||
|
deadlineGenerationKey: `sammo:${profileName}:clock:deadline-generation`,
|
||||||
|
clockPhaseKey: `sammo:${profileName}:clock:phase`,
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
import { randomUUID } from 'node:crypto';
|
import { randomUUID } from 'node:crypto';
|
||||||
import { parseTournamentSourceRevision, writeTournamentProjection } from '@sammo-ts/common';
|
import { parseTournamentSourceRevision, writeTournamentProjection, type TournamentClockFence } from '@sammo-ts/common';
|
||||||
import { z } from 'zod';
|
import { z } from 'zod';
|
||||||
|
|
||||||
import type { TournamentKeys } from './keys.js';
|
import type { TournamentKeys } from './keys.js';
|
||||||
@@ -37,8 +37,12 @@ const zTournamentState = z
|
|||||||
openMonth: z.number().int(),
|
openMonth: z.number().int(),
|
||||||
termSeconds: z.number(),
|
termSeconds: z.number(),
|
||||||
nextAt: z.string(),
|
nextAt: z.string(),
|
||||||
|
nextTick: z.number().int().safe().optional(),
|
||||||
|
clockRevision: z.number().int().positive().safe().optional(),
|
||||||
|
deadlineGeneration: z.number().int().positive().safe().optional(),
|
||||||
bettingId: z.number().int().optional(),
|
bettingId: z.number().int().optional(),
|
||||||
bettingCloseAt: z.string().optional(),
|
bettingCloseAt: z.string().optional(),
|
||||||
|
bettingCloseTick: z.number().int().safe().optional(),
|
||||||
winnerId: z.number().int().optional(),
|
winnerId: z.number().int().optional(),
|
||||||
bettingSettled: z.boolean().optional(),
|
bettingSettled: z.boolean().optional(),
|
||||||
rewardSettled: z.boolean().optional(),
|
rewardSettled: z.boolean().optional(),
|
||||||
@@ -131,12 +135,62 @@ const parseProjection = <T>(raw: string | null, key: string, schema: z.ZodType<T
|
|||||||
return parsed.data;
|
return parsed.data;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
export interface TournamentClockContext {
|
||||||
|
phase: 'RUNNING';
|
||||||
|
revision: number;
|
||||||
|
deadlineGeneration: number;
|
||||||
|
dateToTick(date: Date): number | null;
|
||||||
|
}
|
||||||
|
|
||||||
|
const parseDeadlineTick = (value: string, context: TournamentClockContext, field: string): number => {
|
||||||
|
const date = new Date(value);
|
||||||
|
if (Number.isNaN(date.getTime())) {
|
||||||
|
throw new Error(`Tournament ${field} is not a valid instant.`);
|
||||||
|
}
|
||||||
|
const tick = context.dateToTick(date);
|
||||||
|
if (tick === null || !Number.isSafeInteger(tick)) {
|
||||||
|
throw new Error(`Tournament ${field} cannot be represented in the active game clock.`);
|
||||||
|
}
|
||||||
|
return tick;
|
||||||
|
};
|
||||||
|
|
||||||
|
export const stampTournamentClock = (state: TournamentState, context: TournamentClockContext): TournamentState => ({
|
||||||
|
...state,
|
||||||
|
nextTick: parseDeadlineTick(state.nextAt, context, 'nextAt'),
|
||||||
|
clockRevision: context.revision,
|
||||||
|
deadlineGeneration: context.deadlineGeneration,
|
||||||
|
...(state.bettingCloseAt
|
||||||
|
? { bettingCloseTick: parseDeadlineTick(state.bettingCloseAt, context, 'bettingCloseAt') }
|
||||||
|
: { bettingCloseTick: undefined }),
|
||||||
|
});
|
||||||
|
|
||||||
|
const toClockFence = (keys: TournamentKeys, context: TournamentClockContext): TournamentClockFence => ({
|
||||||
|
activeRevisionKey: keys.activeClockRevisionKey,
|
||||||
|
deadlineGenerationKey: keys.deadlineGenerationKey,
|
||||||
|
phaseKey: keys.clockPhaseKey,
|
||||||
|
revision: context.revision,
|
||||||
|
deadlineGeneration: context.deadlineGeneration,
|
||||||
|
phase: context.phase,
|
||||||
|
});
|
||||||
|
|
||||||
export class TournamentStore {
|
export class TournamentStore {
|
||||||
|
private clockContext: TournamentClockContext | null = null;
|
||||||
|
|
||||||
constructor(
|
constructor(
|
||||||
private readonly redis: RedisClientLike,
|
private readonly redis: RedisClientLike,
|
||||||
private readonly keys: TournamentKeys
|
private readonly keys: TournamentKeys
|
||||||
) {}
|
) {}
|
||||||
|
|
||||||
|
async withClockContext<T>(context: TournamentClockContext, operation: () => Promise<T>): Promise<T> {
|
||||||
|
const previous = this.clockContext;
|
||||||
|
this.clockContext = context;
|
||||||
|
try {
|
||||||
|
return await operation();
|
||||||
|
} finally {
|
||||||
|
this.clockContext = previous;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
async withMutationLock<T>(operation: () => Promise<T>, timeoutMs = 2_000): Promise<T> {
|
async withMutationLock<T>(operation: () => Promise<T>, timeoutMs = 2_000): Promise<T> {
|
||||||
if (!this.redis.del) {
|
if (!this.redis.del) {
|
||||||
return operation();
|
return operation();
|
||||||
@@ -174,11 +228,15 @@ export class TournamentStore {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private async writeWithSourceRevision(key: string, value: unknown): Promise<string> {
|
private async writeWithSourceRevision(key: string, value: unknown): Promise<string> {
|
||||||
return writeTournamentProjection(this.redis, this.keys, [{ key, value }]);
|
const fence = this.clockContext ? toClockFence(this.keys, this.clockContext) : undefined;
|
||||||
|
return writeTournamentProjection(this.redis, this.keys, [{ key, value }], fence);
|
||||||
}
|
}
|
||||||
|
|
||||||
async setState(state: TournamentState): Promise<string> {
|
async setState(state: TournamentState): Promise<string> {
|
||||||
return this.writeWithSourceRevision(this.keys.stateKey, state);
|
return this.writeWithSourceRevision(
|
||||||
|
this.keys.stateKey,
|
||||||
|
this.clockContext ? stampTournamentClock(state, this.clockContext) : state
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
async getParticipants(): Promise<TournamentParticipantEntry[]> {
|
async getParticipants(): Promise<TournamentParticipantEntry[]> {
|
||||||
|
|||||||
@@ -9,8 +9,12 @@ export interface TournamentState {
|
|||||||
openMonth: number;
|
openMonth: number;
|
||||||
termSeconds: number;
|
termSeconds: number;
|
||||||
nextAt: string;
|
nextAt: string;
|
||||||
|
nextTick?: number;
|
||||||
|
clockRevision?: number;
|
||||||
|
deadlineGeneration?: number;
|
||||||
bettingId?: number;
|
bettingId?: number;
|
||||||
bettingCloseAt?: string;
|
bettingCloseAt?: string;
|
||||||
|
bettingCloseTick?: number;
|
||||||
winnerId?: number;
|
winnerId?: number;
|
||||||
bettingSettled?: boolean;
|
bettingSettled?: boolean;
|
||||||
rewardSettled?: boolean;
|
rewardSettled?: boolean;
|
||||||
|
|||||||
@@ -13,9 +13,10 @@ import { DatabaseTurnDaemonTransport } from '../daemon/databaseTransport.js';
|
|||||||
import { createBestEffortResourceCloser } from '../services/bestEffortResourceCloser.js';
|
import { createBestEffortResourceCloser } from '../services/bestEffortResourceCloser.js';
|
||||||
import { loadCurrentGameTime } from '../services/gameClock.js';
|
import { loadCurrentGameTime } from '../services/gameClock.js';
|
||||||
import { createPollingWorkerControl, waitForWorkerPoll } from '../services/pollingWorkerLifecycle.js';
|
import { createPollingWorkerControl, waitForWorkerPoll } from '../services/pollingWorkerLifecycle.js';
|
||||||
|
import { ensureActiveRedisClockFence } from '../services/redisClockFence.js';
|
||||||
import type { TurnDaemonTransport } from '../daemon/transport.js';
|
import type { TurnDaemonTransport } from '../daemon/transport.js';
|
||||||
import { buildTournamentKeys } from './keys.js';
|
import { buildTournamentKeys } from './keys.js';
|
||||||
import { TournamentStore } from './store.js';
|
import { TournamentStore, stampTournamentClock, type TournamentClockContext } from './store.js';
|
||||||
import type { TournamentMatchEntry, TournamentState } from './types.js';
|
import type { TournamentMatchEntry, TournamentState } from './types.js';
|
||||||
import {
|
import {
|
||||||
applyGroupMatch,
|
applyGroupMatch,
|
||||||
@@ -595,18 +596,34 @@ export const processTournamentTick = async (options: {
|
|||||||
prisma: GamePrismaClient;
|
prisma: GamePrismaClient;
|
||||||
daemonTransport: TurnDaemonTransport;
|
daemonTransport: TurnDaemonTransport;
|
||||||
now?: () => number;
|
now?: () => number;
|
||||||
|
clockContext?: TournamentClockContext;
|
||||||
}): Promise<TournamentState | null> => {
|
}): Promise<TournamentState | null> => {
|
||||||
const { store, prisma, daemonTransport } = options;
|
const { store, prisma, daemonTransport } = options;
|
||||||
const now = options.now ?? Date.now;
|
const now = options.now ?? Date.now;
|
||||||
let processedState: TournamentState | null = null;
|
let processedState: TournamentState | null = null;
|
||||||
|
|
||||||
await store.withMutationLock(async () => {
|
const processWithLock = async (): Promise<void> =>
|
||||||
|
store.withMutationLock(async () => {
|
||||||
const state = await store.getState();
|
const state = await store.getState();
|
||||||
if (!state || (!state.auto && !needsSettlement(state))) {
|
if (!state || (!state.auto && !needsSettlement(state))) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
if (
|
||||||
|
options.clockContext &&
|
||||||
|
(state.clockRevision !== options.clockContext.revision ||
|
||||||
|
state.deadlineGeneration !== options.clockContext.deadlineGeneration)
|
||||||
|
) {
|
||||||
|
throw new Error('Tournament state does not match the active clock revision.');
|
||||||
|
}
|
||||||
const nextAt = new Date(state.nextAt).getTime();
|
const nextAt = new Date(state.nextAt).getTime();
|
||||||
if (state.auto && Number.isFinite(nextAt) && nextAt > now()) {
|
const notDue = options.clockContext
|
||||||
|
? !Number.isSafeInteger(state.nextTick) ||
|
||||||
|
state.nextTick! > options.clockContext.dateToTick(new Date(now()))!
|
||||||
|
: Number.isFinite(nextAt) && nextAt > now();
|
||||||
|
if (state.auto && notDue) {
|
||||||
|
if (options.clockContext && !Number.isSafeInteger(state.nextTick)) {
|
||||||
|
throw new Error('Active tournament nextAt lacks the authoritative nextTick dual-write.');
|
||||||
|
}
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -631,6 +648,12 @@ export const processTournamentTick = async (options: {
|
|||||||
})) ?? nextState;
|
})) ?? nextState;
|
||||||
});
|
});
|
||||||
|
|
||||||
|
if (options.clockContext) {
|
||||||
|
await store.withClockContext(options.clockContext, processWithLock);
|
||||||
|
} else {
|
||||||
|
await processWithLock();
|
||||||
|
}
|
||||||
|
|
||||||
return processedState;
|
return processedState;
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -656,16 +679,60 @@ export const runTournamentWorker = async (options: TournamentWorkerOptions = {})
|
|||||||
|
|
||||||
try {
|
try {
|
||||||
while (!control.signal.aborted) {
|
while (!control.signal.aborted) {
|
||||||
const state = await store.getState();
|
let state = await store.getState();
|
||||||
if (!state || (!state.auto && !needsSettlement(state))) {
|
if (!state || (!state.auto && !needsSettlement(state))) {
|
||||||
await waitForWorkerPoll(control.signal, config.tournamentPollMs);
|
await waitForWorkerPoll(control.signal, config.tournamentPollMs);
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const gameTime = await loadCurrentGameTime(postgres.prisma);
|
||||||
|
if (gameTime.phase && gameTime.phase !== 'RUNNING') {
|
||||||
|
await waitForWorkerPoll(control.signal, config.tournamentPollMs);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
const clockFence = gameTime.phase
|
||||||
|
? await ensureActiveRedisClockFence(redis.client, config.profileName, gameTime)
|
||||||
|
: null;
|
||||||
|
if (gameTime.phase && !clockFence) {
|
||||||
|
await waitForWorkerPoll(control.signal, config.tournamentPollMs);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
const clockContext: TournamentClockContext | undefined = clockFence
|
||||||
|
? {
|
||||||
|
phase: 'RUNNING',
|
||||||
|
revision: clockFence.revision,
|
||||||
|
deadlineGeneration: clockFence.generation,
|
||||||
|
dateToTick: gameTime.dateToTick,
|
||||||
|
}
|
||||||
|
: undefined;
|
||||||
|
if (
|
||||||
|
clockContext &&
|
||||||
|
(!Number.isSafeInteger(state.nextTick) ||
|
||||||
|
state.clockRevision === undefined ||
|
||||||
|
state.deadlineGeneration === undefined)
|
||||||
|
) {
|
||||||
|
state = stampTournamentClock(state, clockContext);
|
||||||
|
await store.withClockContext(clockContext, () => store.setState(state!));
|
||||||
|
}
|
||||||
|
if (
|
||||||
|
clockContext &&
|
||||||
|
(state.clockRevision !== clockContext.revision ||
|
||||||
|
state.deadlineGeneration !== clockContext.deadlineGeneration)
|
||||||
|
) {
|
||||||
|
await waitForWorkerPoll(control.signal, config.tournamentPollMs);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
const nextAt = new Date(state.nextAt).getTime();
|
const nextAt = new Date(state.nextAt).getTime();
|
||||||
const gameNow = (await loadCurrentGameTime(postgres.prisma)).now.getTime();
|
const gameNow = gameTime.now.getTime();
|
||||||
if (state.auto && Number.isFinite(nextAt) && nextAt > gameNow) {
|
const notDue = clockContext
|
||||||
await waitForWorkerPoll(control.signal, Math.min(config.tournamentPollMs, nextAt - gameNow));
|
? state.nextTick! > gameTime.tick!
|
||||||
|
: Number.isFinite(nextAt) && nextAt > gameNow;
|
||||||
|
if (state.auto && notDue) {
|
||||||
|
await waitForWorkerPoll(
|
||||||
|
control.signal,
|
||||||
|
Math.min(config.tournamentPollMs, Math.max(1, nextAt - gameNow))
|
||||||
|
);
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -675,6 +742,7 @@ export const runTournamentWorker = async (options: TournamentWorkerOptions = {})
|
|||||||
prisma: postgres.prisma,
|
prisma: postgres.prisma,
|
||||||
daemonTransport,
|
daemonTransport,
|
||||||
now: () => gameNow,
|
now: () => gameNow,
|
||||||
|
clockContext,
|
||||||
});
|
});
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
const message = error instanceof Error ? error.message : 'Unknown error';
|
const message = error instanceof Error ? error.message : 'Unknown error';
|
||||||
@@ -700,8 +768,12 @@ export const runTournamentWorker = async (options: TournamentWorkerOptions = {})
|
|||||||
lastError: message,
|
lastError: message,
|
||||||
lastErrorAt: failedAt,
|
lastErrorAt: failedAt,
|
||||||
};
|
};
|
||||||
|
if (clockContext) {
|
||||||
|
await store.withClockContext(clockContext, () => store.setState(nextState));
|
||||||
|
} else {
|
||||||
await store.setState(nextState);
|
await store.setState(nextState);
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
await waitForWorkerPoll(control.signal, config.tournamentPollMs);
|
await waitForWorkerPoll(control.signal, config.tournamentPollMs);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -2,7 +2,7 @@ import { describe, expect, it, vi } from 'vitest';
|
|||||||
|
|
||||||
import type { GamePrismaClient } from '@sammo-ts/infra';
|
import type { GamePrismaClient } from '@sammo-ts/infra';
|
||||||
|
|
||||||
import { processDueAuctionId, reconcilePendingAuctionTimers } from '../src/auction/worker.js';
|
import { popDueAuctionIds, processDueAuctionId, reconcilePendingAuctionTimers } from '../src/auction/worker.js';
|
||||||
import { resolveAuctionSeedScore } from '../src/auction/scheduler.js';
|
import { resolveAuctionSeedScore } from '../src/auction/scheduler.js';
|
||||||
|
|
||||||
const buildRedis = () => ({
|
const buildRedis = () => ({
|
||||||
@@ -55,6 +55,38 @@ const buildDb = (options: {
|
|||||||
};
|
};
|
||||||
|
|
||||||
describe('auction worker clock-shift race', () => {
|
describe('auction worker clock-shift race', () => {
|
||||||
|
it('uses one Redis script for revision, generation, phase, due-read, and removal', async () => {
|
||||||
|
const redis = { ...buildRedis(), eval: vi.fn(async () => ['7', '9']) };
|
||||||
|
await expect(
|
||||||
|
popDueAuctionIds(redis, 'auction-timer', 123_456, 100, {
|
||||||
|
activeRevisionKey: 'clock-revision',
|
||||||
|
deadlineGenerationKey: 'deadline-generation',
|
||||||
|
phaseKey: 'clock-phase',
|
||||||
|
revision: 4,
|
||||||
|
generation: 8,
|
||||||
|
})
|
||||||
|
).resolves.toEqual(['7', '9']);
|
||||||
|
expect(redis.eval).toHaveBeenCalledWith(expect.stringContaining('ZRANGEBYSCORE'), {
|
||||||
|
keys: ['auction-timer', 'clock-revision', 'deadline-generation', 'clock-phase'],
|
||||||
|
arguments: ['4', '8', '123456', '100'],
|
||||||
|
});
|
||||||
|
expect(redis.zRangeByScore).not.toHaveBeenCalled();
|
||||||
|
expect(redis.zRem).not.toHaveBeenCalled();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('returns no due member when the atomic Redis clock fence rejects the pop', async () => {
|
||||||
|
const redis = { ...buildRedis(), eval: vi.fn(async () => ['__CLOCK_FENCE__']) };
|
||||||
|
await expect(
|
||||||
|
popDueAuctionIds(redis, 'auction-timer', 123_456, 100, {
|
||||||
|
activeRevisionKey: 'clock-revision',
|
||||||
|
deadlineGenerationKey: 'deadline-generation',
|
||||||
|
phaseKey: 'clock-phase',
|
||||||
|
revision: 4,
|
||||||
|
generation: 8,
|
||||||
|
})
|
||||||
|
).resolves.toEqual([]);
|
||||||
|
});
|
||||||
|
|
||||||
it('seeds OPEN at its deadline but retries FINALIZING at the current logical tick', () => {
|
it('seeds OPEN at its deadline but retries FINALIZING at the current logical tick', () => {
|
||||||
const now = new Date('2026-07-30T12:00:00.000Z');
|
const now = new Date('2026-07-30T12:00:00.000Z');
|
||||||
const time = {
|
const time = {
|
||||||
@@ -292,6 +324,7 @@ describe('auction worker clock-shift race', () => {
|
|||||||
requestId,
|
requestId,
|
||||||
target: 'ENGINE',
|
target: 'ENGINE',
|
||||||
eventType: 'auctionFinalize',
|
eventType: 'auctionFinalize',
|
||||||
|
acceptedGameTick: 72_000_000n,
|
||||||
payload: {
|
payload: {
|
||||||
type: 'auctionFinalize',
|
type: 'auctionFinalize',
|
||||||
requestId,
|
requestId,
|
||||||
|
|||||||
@@ -0,0 +1,87 @@
|
|||||||
|
import { describe, expect, it, vi } from 'vitest';
|
||||||
|
|
||||||
|
import type { DatabaseClient } from '../src/context.js';
|
||||||
|
import { loadClockAdminStatus, loadClockReadiness } from '../src/services/clockReadiness.js';
|
||||||
|
|
||||||
|
describe('clock reconciliation readiness', () => {
|
||||||
|
it('fails closed when the reconciliation schema is not available', async () => {
|
||||||
|
const db = {} as DatabaseClient;
|
||||||
|
await expect(loadClockReadiness(db)).resolves.toEqual({
|
||||||
|
reconciliationComplete: false,
|
||||||
|
gameplayEnabled: false,
|
||||||
|
phase: null,
|
||||||
|
revision: null,
|
||||||
|
deadlineGeneration: null,
|
||||||
|
incompleteOutboxCount: null,
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it('blocks readiness for RECONCILING or incomplete outbox state', async () => {
|
||||||
|
const db = {
|
||||||
|
worldState: {
|
||||||
|
findFirst: vi.fn(async () => ({
|
||||||
|
clockPhase: 'RECONCILING',
|
||||||
|
clockRevision: 9n,
|
||||||
|
deadlineGeneration: 4n,
|
||||||
|
})),
|
||||||
|
},
|
||||||
|
clockProjectionOutbox: { count: vi.fn(async () => 1) },
|
||||||
|
} as unknown as DatabaseClient;
|
||||||
|
await expect(loadClockReadiness(db)).resolves.toMatchObject({
|
||||||
|
reconciliationComplete: false,
|
||||||
|
gameplayEnabled: false,
|
||||||
|
phase: 'RECONCILING',
|
||||||
|
revision: 9,
|
||||||
|
deadlineGeneration: 4,
|
||||||
|
incompleteOutboxCount: 1,
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it('exposes participant checksums and incomplete outbox detail to admins', async () => {
|
||||||
|
const db = {
|
||||||
|
worldState: {
|
||||||
|
findFirst: vi.fn(async () => ({
|
||||||
|
clockPhase: 'RUNNING',
|
||||||
|
clockRevision: 3n,
|
||||||
|
deadlineGeneration: 2n,
|
||||||
|
})),
|
||||||
|
},
|
||||||
|
clockProjectionOutbox: { count: vi.fn(async () => 0) },
|
||||||
|
clockSuspension: {
|
||||||
|
findFirst: vi.fn(async () => ({
|
||||||
|
id: 'maintenance-1',
|
||||||
|
source: 'MAINTENANCE',
|
||||||
|
policy: 'EXACT',
|
||||||
|
status: 'APPLIED',
|
||||||
|
sourceRevision: 2n,
|
||||||
|
targetRevision: 3n,
|
||||||
|
cutTick: 100n,
|
||||||
|
alignedTick: 130n,
|
||||||
|
participantChecksumBefore: 'before-all',
|
||||||
|
participantChecksumAfter: 'after-all',
|
||||||
|
participants: [
|
||||||
|
{
|
||||||
|
participantKey: 'general-turn',
|
||||||
|
policy: 'SHIFT',
|
||||||
|
beforeChecksum: 'before',
|
||||||
|
afterChecksum: 'after',
|
||||||
|
affectedCount: 2,
|
||||||
|
},
|
||||||
|
],
|
||||||
|
projectionOutbox: [{ id: 8n, targetRevision: 3n, status: 'APPLIED', attempts: 1, lastError: null }],
|
||||||
|
})),
|
||||||
|
},
|
||||||
|
} as unknown as DatabaseClient;
|
||||||
|
|
||||||
|
await expect(loadClockAdminStatus(db)).resolves.toMatchObject({
|
||||||
|
reconciliationComplete: true,
|
||||||
|
latestReconciliation: {
|
||||||
|
id: 'maintenance-1',
|
||||||
|
participantChecksumBefore: 'before-all',
|
||||||
|
participantChecksumAfter: 'after-all',
|
||||||
|
participants: [{ key: 'general-turn', policy: 'SHIFT', affectedCount: 2 }],
|
||||||
|
outbox: [{ id: '8', status: 'APPLIED' }],
|
||||||
|
},
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -511,6 +511,10 @@ integration('API input event boundary', () => {
|
|||||||
it('reuses the same engine child event but rejects a changed retry payload', async () => {
|
it('reuses the same engine child event but rejects a changed retry payload', async () => {
|
||||||
const transport = new DatabaseTurnDaemonTransport(db, 100);
|
const transport = new DatabaseTurnDaemonTransport(db, 100);
|
||||||
const requestId = 'integration:api:engine-child';
|
const requestId = 'integration:api:engine-child';
|
||||||
|
const worldClock = await db.worldState.findFirst({
|
||||||
|
orderBy: { id: 'asc' },
|
||||||
|
select: { clockRevision: true, deadlineGeneration: true },
|
||||||
|
});
|
||||||
const acceptedWindowStart = Date.now();
|
const acceptedWindowStart = Date.now();
|
||||||
await transport.sendCommand({ type: 'vacation', requestId, userId: 'user-7', generalId: 7 });
|
await transport.sendCommand({ type: 'vacation', requestId, userId: 'user-7', generalId: 7 });
|
||||||
const acceptedWindowEnd = Date.now();
|
const acceptedWindowEnd = Date.now();
|
||||||
@@ -518,6 +522,9 @@ integration('API input event boundary', () => {
|
|||||||
expect(event.actorUserId).toBe('user-7');
|
expect(event.actorUserId).toBe('user-7');
|
||||||
expect(event.createdAt.getTime()).toBeGreaterThanOrEqual(acceptedWindowStart);
|
expect(event.createdAt.getTime()).toBeGreaterThanOrEqual(acceptedWindowStart);
|
||||||
expect(event.createdAt.getTime()).toBeLessThanOrEqual(acceptedWindowEnd);
|
expect(event.createdAt.getTime()).toBeLessThanOrEqual(acceptedWindowEnd);
|
||||||
|
expect(event.acceptedGameTick).not.toBeNull();
|
||||||
|
expect(event.acceptedClockRevision).toBe(worldClock?.clockRevision ?? null);
|
||||||
|
expect(event.acceptedDeadlineGeneration).toBe(worldClock?.deadlineGeneration ?? null);
|
||||||
await expect(
|
await expect(
|
||||||
transport.sendCommand({ type: 'vacation', requestId, userId: 'user-7', generalId: 7 })
|
transport.sendCommand({ type: 'vacation', requestId, userId: 'user-7', generalId: 7 })
|
||||||
).resolves.toBe(requestId);
|
).resolves.toBe(requestId);
|
||||||
|
|||||||
@@ -28,6 +28,9 @@ const createContext = (payload: unknown = {}) => {
|
|||||||
status: 'PENDING',
|
status: 'PENDING',
|
||||||
result: null,
|
result: null,
|
||||||
attempts: 0,
|
attempts: 0,
|
||||||
|
acceptedGameTick: 100n,
|
||||||
|
acceptedClockRevision: 3n,
|
||||||
|
acceptedDeadlineGeneration: 2n,
|
||||||
},
|
},
|
||||||
];
|
];
|
||||||
}
|
}
|
||||||
@@ -36,8 +39,8 @@ const createContext = (payload: unknown = {}) => {
|
|||||||
});
|
});
|
||||||
const transaction = {
|
const transaction = {
|
||||||
$queryRaw: queryRaw,
|
$queryRaw: queryRaw,
|
||||||
$executeRaw: vi.fn(async () => {
|
$executeRaw: vi.fn(async (query: { sql?: string }) => {
|
||||||
order.push('accepted');
|
order.push(query.sql?.includes('pg_advisory_xact_lock') ? 'clock-fence' : 'accepted');
|
||||||
return 1;
|
return 1;
|
||||||
}),
|
}),
|
||||||
$executeRawUnsafe: vi.fn(async (statement: string) => {
|
$executeRawUnsafe: vi.fn(async (statement: string) => {
|
||||||
@@ -88,6 +91,7 @@ describe('API input-event change journal boundary', () => {
|
|||||||
|
|
||||||
expect(fixture.order).toEqual([
|
expect(fixture.order).toEqual([
|
||||||
'transaction-begin',
|
'transaction-begin',
|
||||||
|
'clock-fence',
|
||||||
'accepted',
|
'accepted',
|
||||||
'locked',
|
'locked',
|
||||||
'processing',
|
'processing',
|
||||||
@@ -112,6 +116,7 @@ describe('API input-event change journal boundary', () => {
|
|||||||
|
|
||||||
expect(fixture.order).toEqual([
|
expect(fixture.order).toEqual([
|
||||||
'transaction-begin',
|
'transaction-begin',
|
||||||
|
'clock-fence',
|
||||||
'accepted',
|
'accepted',
|
||||||
'locked',
|
'locked',
|
||||||
'processing',
|
'processing',
|
||||||
|
|||||||
@@ -30,11 +30,31 @@ class MemoryRedis {
|
|||||||
}
|
}
|
||||||
|
|
||||||
async eval(_script: string, options: { keys: string[]; arguments: string[] }): Promise<string> {
|
async eval(_script: string, options: { keys: string[]; arguments: string[] }): Promise<string> {
|
||||||
const [valueKey, revisionKey] = options.keys;
|
if (options.keys.length === 3 && options.keys[0]?.endsWith(':clock:active-revision')) {
|
||||||
const [value] = options.arguments;
|
const current = options.keys.map((key) => this.values.get(key));
|
||||||
|
if (current.every((value) => value === undefined)) {
|
||||||
|
options.keys.forEach((key, index) => this.values.set(key, options.arguments[index]!));
|
||||||
|
return '1';
|
||||||
|
}
|
||||||
|
return current.every((value, index) => value === options.arguments[index]) ? '2' : '0';
|
||||||
|
}
|
||||||
|
const fenced = options.keys.at(-1)?.endsWith(':clock:phase') === true;
|
||||||
|
const writeCount = options.keys.length - (fenced ? 4 : 1);
|
||||||
|
if (fenced) {
|
||||||
|
const clockKeys = options.keys.slice(-3);
|
||||||
|
const expected = options.arguments.slice(-3);
|
||||||
|
if (!clockKeys.every((key, index) => this.values.get(key) === expected[index])) {
|
||||||
|
return '__CLOCK_FENCE__';
|
||||||
|
}
|
||||||
|
}
|
||||||
|
const valueKey = options.keys[0];
|
||||||
|
const revisionKey = options.keys[writeCount];
|
||||||
|
const value = options.arguments[0];
|
||||||
if (!valueKey || !revisionKey || value === undefined) throw new Error('invalid eval arguments');
|
if (!valueKey || !revisionKey || value === undefined) throw new Error('invalid eval arguments');
|
||||||
const revision = Number(this.values.get(revisionKey) ?? '0') + 1;
|
const revision = Number(this.values.get(revisionKey) ?? '0') + 1;
|
||||||
this.values.set(valueKey, value);
|
for (let index = 0; index < writeCount; index += 1) {
|
||||||
|
this.values.set(options.keys[index]!, options.arguments[index]!);
|
||||||
|
}
|
||||||
this.values.set(revisionKey, String(revision));
|
this.values.set(revisionKey, String(revision));
|
||||||
return String(revision);
|
return String(revision);
|
||||||
}
|
}
|
||||||
@@ -144,6 +164,14 @@ const buildContext = (options: {
|
|||||||
},
|
},
|
||||||
worldState: {
|
worldState: {
|
||||||
findFirst: async () => ({
|
findFirst: async () => ({
|
||||||
|
clockBaseTime: new Date('2026-01-01T00:00:00.000Z'),
|
||||||
|
clockTick: 0n,
|
||||||
|
clockMode: 'realtime',
|
||||||
|
clockWallAnchor: new Date('2026-01-01T00:00:00.000Z'),
|
||||||
|
clockPhase: 'RUNNING',
|
||||||
|
clockRevision: 1n,
|
||||||
|
deadlineGeneration: 1n,
|
||||||
|
tickSeconds: 60,
|
||||||
config: { const: { develCost: options.develCost ?? 200 } },
|
config: { const: { develCost: options.develCost ?? 200 } },
|
||||||
...(options.currentDevelCost === undefined ? {} : { meta: { develcost: options.currentDevelCost } }),
|
...(options.currentDevelCost === undefined ? {} : { meta: { develcost: options.currentDevelCost } }),
|
||||||
}),
|
}),
|
||||||
@@ -394,7 +422,10 @@ describe('tournament router permissions and mutations', () => {
|
|||||||
roles: ['admin.tournament:che:default'],
|
roles: ['admin.tournament:che:default'],
|
||||||
})
|
})
|
||||||
);
|
);
|
||||||
await expect(adminCaller.tournament.getAdminStatus()).resolves.toEqual({ ok: true });
|
await expect(adminCaller.tournament.getAdminStatus()).resolves.toMatchObject({
|
||||||
|
ok: true,
|
||||||
|
clock: { reconciliationComplete: false, latestReconciliation: null },
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
it('applies the admin role boundary to every tournament mutation', async () => {
|
it('applies the admin role boundary to every tournament mutation', async () => {
|
||||||
@@ -479,9 +510,7 @@ describe('tournament router permissions and mutations', () => {
|
|||||||
])
|
])
|
||||||
).resolves.toEqual({ ok: true, count: 1 });
|
).resolves.toEqual({ ok: true, count: 1 });
|
||||||
await expect(
|
await expect(
|
||||||
caller.tournament.setBettingEntries([
|
caller.tournament.setBettingEntries([{ generalId: general.id, targetId: rival.id, amount: 100 }])
|
||||||
{ generalId: general.id, targetId: rival.id, amount: 100 },
|
|
||||||
])
|
|
||||||
).resolves.toEqual({ ok: true, count: 1 });
|
).resolves.toEqual({ ok: true, count: 1 });
|
||||||
await expect(caller.tournament.seedParticipants({ generalIds: [general.id, rival.id] })).resolves.toEqual({
|
await expect(caller.tournament.seedParticipants({ generalIds: [general.id, rival.id] })).resolves.toEqual({
|
||||||
ok: true,
|
ok: true,
|
||||||
|
|||||||
@@ -41,6 +41,9 @@ integration('TournamentStore Redis source revision', () => {
|
|||||||
keys.matchesKey,
|
keys.matchesKey,
|
||||||
keys.bettingKey,
|
keys.bettingKey,
|
||||||
keys.sourceRevisionKey,
|
keys.sourceRevisionKey,
|
||||||
|
keys.activeClockRevisionKey,
|
||||||
|
keys.deadlineGenerationKey,
|
||||||
|
keys.clockPhaseKey,
|
||||||
]);
|
]);
|
||||||
if (subscriber) {
|
if (subscriber) {
|
||||||
await subscriber.client.unsubscribe(keys.sourceRevisionChannel);
|
await subscriber.client.unsubscribe(keys.sourceRevisionChannel);
|
||||||
@@ -124,4 +127,48 @@ integration('TournamentStore Redis source revision', () => {
|
|||||||
}),
|
}),
|
||||||
]);
|
]);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('dual-writes deadline ticks and rejects a Redis clock revision race atomically', async () => {
|
||||||
|
const store = new TournamentStore(connector.client, keys);
|
||||||
|
await Promise.all([
|
||||||
|
connector.client.set(keys.activeClockRevisionKey, '7'),
|
||||||
|
connector.client.set(keys.deadlineGenerationKey, '3'),
|
||||||
|
connector.client.set(keys.clockPhaseKey, 'RUNNING'),
|
||||||
|
]);
|
||||||
|
const clockContext = {
|
||||||
|
phase: 'RUNNING' as const,
|
||||||
|
revision: 7,
|
||||||
|
deadlineGeneration: 3,
|
||||||
|
dateToTick: (date: Date) => Math.trunc(date.getTime() / 1_000),
|
||||||
|
};
|
||||||
|
const nextAt = '2026-09-03T10:00:00.000Z';
|
||||||
|
await store.withClockContext(clockContext, () =>
|
||||||
|
store.setState({
|
||||||
|
stage: 6,
|
||||||
|
phase: 0,
|
||||||
|
type: 0,
|
||||||
|
auto: true,
|
||||||
|
openYear: 200,
|
||||||
|
openMonth: 1,
|
||||||
|
termSeconds: 10,
|
||||||
|
nextAt,
|
||||||
|
bettingCloseAt: '2026-09-03T09:59:50.000Z',
|
||||||
|
})
|
||||||
|
);
|
||||||
|
await expect(store.getState()).resolves.toMatchObject({
|
||||||
|
nextTick: Math.trunc(new Date(nextAt).getTime() / 1_000),
|
||||||
|
bettingCloseTick: Math.trunc(new Date('2026-09-03T09:59:50.000Z').getTime() / 1_000),
|
||||||
|
clockRevision: 7,
|
||||||
|
deadlineGeneration: 3,
|
||||||
|
});
|
||||||
|
|
||||||
|
const beforeRevision = await store.getSourceRevision();
|
||||||
|
await connector.client.set(keys.activeClockRevisionKey, '8');
|
||||||
|
await expect(
|
||||||
|
store.withClockContext(clockContext, () =>
|
||||||
|
store.setMatches([{ id: 99, stage: 7, roundIndex: 0, attackerId: 1, defenderId: 2 }])
|
||||||
|
)
|
||||||
|
).rejects.toThrow('clock revision fence failed');
|
||||||
|
await expect(store.getSourceRevision()).resolves.toBe(beforeRevision);
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -269,7 +269,14 @@ export const createAuctionBidder = async (options: {
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
const processingNow = world.getGameNow(new Date());
|
const processingNow = world.getGameNow(new Date());
|
||||||
const { bidAt, bidTick } = resolveAuctionBidTiming(world, processingNow, command.acceptedGameTick);
|
const convertedProcessingTick = Reflect.get(command, 'processingGameTick');
|
||||||
|
const { bidAt, bidTick } = resolveAuctionBidTiming(
|
||||||
|
world,
|
||||||
|
processingNow,
|
||||||
|
typeof convertedProcessingTick === 'number' && Number.isSafeInteger(convertedProcessingTick)
|
||||||
|
? convertedProcessingTick
|
||||||
|
: command.acceptedGameTick
|
||||||
|
);
|
||||||
if (hasAuctionClosePassed(auction, bidAt, bidTick)) {
|
if (hasAuctionClosePassed(auction, bidAt, bidTick)) {
|
||||||
return {
|
return {
|
||||||
type: 'auctionBid',
|
type: 'auctionBid',
|
||||||
|
|||||||
@@ -20,6 +20,8 @@ export * from './turn/engineStateManager.js';
|
|||||||
export * from './turn/inMemoryStateStore.js';
|
export * from './turn/inMemoryStateStore.js';
|
||||||
export * from './turn/inMemoryTurnProcessor.js';
|
export * from './turn/inMemoryTurnProcessor.js';
|
||||||
export * from './turn/databaseHooks.js';
|
export * from './turn/databaseHooks.js';
|
||||||
|
export * from './turn/clockReconciliation.js';
|
||||||
|
export * from './turn/clockProjectionOutbox.js';
|
||||||
export * from './turn/joinCreateGeneralService.js';
|
export * from './turn/joinCreateGeneralService.js';
|
||||||
export * from './turn/npcPossessionService.js';
|
export * from './turn/npcPossessionService.js';
|
||||||
export * from './turn/selectPoolService.js';
|
export * from './turn/selectPoolService.js';
|
||||||
|
|||||||
@@ -108,6 +108,13 @@ export class DatabaseTurnDaemonCommandQueue implements TurnDaemonControlQueue, T
|
|||||||
private async claimPending(limit = 100): Promise<TurnDaemonCommand[]> {
|
private async claimPending(limit = 100): Promise<TurnDaemonCommand[]> {
|
||||||
await this.recoverExpiredLeases();
|
await this.recoverExpiredLeases();
|
||||||
return this.db.$transaction(async (transaction) => {
|
return this.db.$transaction(async (transaction) => {
|
||||||
|
const world = await transaction.worldState.findFirst({
|
||||||
|
orderBy: { id: 'asc' },
|
||||||
|
select: { clockPhase: true, clockRevision: true, deadlineGeneration: true, clockTick: true },
|
||||||
|
});
|
||||||
|
const gameplayAllowed =
|
||||||
|
!world || world.clockPhase === 'RUNNING' || world.clockPhase === 'MANUAL';
|
||||||
|
const currentRevision = world?.clockRevision ?? null;
|
||||||
const rows = await transaction.$queryRaw<
|
const rows = await transaction.$queryRaw<
|
||||||
Array<{
|
Array<{
|
||||||
sequence: bigint;
|
sequence: bigint;
|
||||||
@@ -115,6 +122,9 @@ export class DatabaseTurnDaemonCommandQueue implements TurnDaemonControlQueue, T
|
|||||||
eventType: string;
|
eventType: string;
|
||||||
payload: unknown;
|
payload: unknown;
|
||||||
createdAt: Date;
|
createdAt: Date;
|
||||||
|
acceptedGameTick: bigint | null;
|
||||||
|
acceptedClockRevision: bigint | null;
|
||||||
|
acceptedDeadlineGeneration: bigint | null;
|
||||||
}>
|
}>
|
||||||
>(GamePrisma.sql`
|
>(GamePrisma.sql`
|
||||||
SELECT
|
SELECT
|
||||||
@@ -122,10 +132,14 @@ export class DatabaseTurnDaemonCommandQueue implements TurnDaemonControlQueue, T
|
|||||||
"request_id" AS "requestId",
|
"request_id" AS "requestId",
|
||||||
"event_type" AS "eventType",
|
"event_type" AS "eventType",
|
||||||
"payload",
|
"payload",
|
||||||
"created_at" AS "createdAt"
|
"created_at" AS "createdAt",
|
||||||
|
"accepted_game_tick" AS "acceptedGameTick",
|
||||||
|
"accepted_clock_revision" AS "acceptedClockRevision",
|
||||||
|
"accepted_deadline_generation" AS "acceptedDeadlineGeneration"
|
||||||
FROM "input_event"
|
FROM "input_event"
|
||||||
WHERE "target" = 'ENGINE'::"InputEventTarget"
|
WHERE "target" = 'ENGINE'::"InputEventTarget"
|
||||||
AND "status" = 'PENDING'::"InputEventStatus"
|
AND "status" = 'PENDING'::"InputEventStatus"
|
||||||
|
AND (${gameplayAllowed} OR "event_type" = 'getStatus')
|
||||||
ORDER BY "sequence" ASC
|
ORDER BY "sequence" ASC
|
||||||
FOR UPDATE SKIP LOCKED
|
FOR UPDATE SKIP LOCKED
|
||||||
LIMIT ${limit}
|
LIMIT ${limit}
|
||||||
@@ -133,23 +147,53 @@ export class DatabaseTurnDaemonCommandQueue implements TurnDaemonControlQueue, T
|
|||||||
if (rows.length === 0) {
|
if (rows.length === 0) {
|
||||||
return [];
|
return [];
|
||||||
}
|
}
|
||||||
await transaction.inputEvent.updateMany({
|
const appliedSuspensions = currentRevision
|
||||||
where: {
|
? await transaction.clockSuspension.findMany({
|
||||||
sequence: { in: rows.map((row) => row.sequence) },
|
where: { status: 'APPLIED', targetRevision: { lte: currentRevision } },
|
||||||
target: 'ENGINE',
|
orderBy: { sourceRevision: 'asc' },
|
||||||
status: 'PENDING',
|
select: { sourceRevision: true, targetRevision: true, shiftTicks: true },
|
||||||
},
|
})
|
||||||
|
: [];
|
||||||
|
const convertTick = (row: (typeof rows)[number]): bigint | null | undefined => {
|
||||||
|
if (row.eventType === 'getStatus') return row.acceptedGameTick;
|
||||||
|
if (row.acceptedGameTick === null || row.acceptedClockRevision === null || currentRevision === null) {
|
||||||
|
return row.acceptedGameTick ?? world?.clockTick ?? null;
|
||||||
|
}
|
||||||
|
if (row.acceptedClockRevision > currentRevision) return undefined;
|
||||||
|
let revision = row.acceptedClockRevision;
|
||||||
|
let tick = row.acceptedGameTick;
|
||||||
|
while (revision < currentRevision) {
|
||||||
|
const step = appliedSuspensions.find((entry) => entry.sourceRevision === revision);
|
||||||
|
if (!step || step.shiftTicks === null || step.targetRevision !== revision + 1n) return undefined;
|
||||||
|
tick += step.shiftTicks;
|
||||||
|
revision = step.targetRevision;
|
||||||
|
}
|
||||||
|
return tick;
|
||||||
|
};
|
||||||
|
const processableRows = rows
|
||||||
|
.map((row) => ({ row, processingGameTick: convertTick(row) }))
|
||||||
|
.filter(
|
||||||
|
(entry): entry is { row: (typeof rows)[number]; processingGameTick: bigint | null } =>
|
||||||
|
entry.processingGameTick !== undefined
|
||||||
|
);
|
||||||
|
for (const { row, processingGameTick } of processableRows) {
|
||||||
|
await transaction.inputEvent.update({
|
||||||
|
where: { sequence: row.sequence },
|
||||||
data: {
|
data: {
|
||||||
status: 'PROCESSING',
|
status: 'PROCESSING',
|
||||||
processingAt: new Date(),
|
processingAt: new Date(),
|
||||||
|
processingGameTick,
|
||||||
|
processingClockRevision: currentRevision,
|
||||||
|
processingDeadlineGeneration: world?.deadlineGeneration ?? null,
|
||||||
lockedBy: this.workerId,
|
lockedBy: this.workerId,
|
||||||
leaseUntil: new Date(Date.now() + this.leaseDurationMs),
|
leaseUntil: new Date(Date.now() + this.leaseDurationMs),
|
||||||
attempts: { increment: 1 },
|
attempts: { increment: 1 },
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
|
}
|
||||||
|
|
||||||
const commands: TurnDaemonCommand[] = [];
|
const commands: TurnDaemonCommand[] = [];
|
||||||
for (const row of rows) {
|
for (const { row, processingGameTick } of processableRows) {
|
||||||
const command = normalizeTurnDaemonCommand({
|
const command = normalizeTurnDaemonCommand({
|
||||||
requestId: row.requestId,
|
requestId: row.requestId,
|
||||||
sentAt: row.createdAt.toISOString(),
|
sentAt: row.createdAt.toISOString(),
|
||||||
@@ -168,6 +212,27 @@ export class DatabaseTurnDaemonCommandQueue implements TurnDaemonControlQueue, T
|
|||||||
});
|
});
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
if (
|
||||||
|
processingGameTick !== null &&
|
||||||
|
row.acceptedGameTick !== null &&
|
||||||
|
processingGameTick !== row.acceptedGameTick
|
||||||
|
) {
|
||||||
|
const value = Number(processingGameTick);
|
||||||
|
if (!Number.isSafeInteger(value)) {
|
||||||
|
await transaction.inputEvent.update({
|
||||||
|
where: { sequence: row.sequence },
|
||||||
|
data: {
|
||||||
|
status: 'FAILED',
|
||||||
|
error: 'Converted processing game tick is outside the safe integer range.',
|
||||||
|
completedAt: new Date(),
|
||||||
|
lockedBy: null,
|
||||||
|
leaseUntil: null,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
Reflect.set(command, 'processingGameTick', value);
|
||||||
|
}
|
||||||
commands.push(command);
|
commands.push(command);
|
||||||
}
|
}
|
||||||
return commands;
|
return commands;
|
||||||
|
|||||||
@@ -0,0 +1,343 @@
|
|||||||
|
import { createHash } from 'node:crypto';
|
||||||
|
|
||||||
|
import { GameClock, parseGameClockPhase } from '@sammo-ts/common';
|
||||||
|
import {
|
||||||
|
CLOCK_OPERATION_PERSISTENCE_LOCK,
|
||||||
|
GamePrisma,
|
||||||
|
acquireGameSchemaAdvisoryXactLock,
|
||||||
|
type GamePrismaClient,
|
||||||
|
} from '@sammo-ts/infra';
|
||||||
|
|
||||||
|
interface ClockProjectionRedis {
|
||||||
|
get(key: string): Promise<string | null>;
|
||||||
|
eval(script: string, options: { keys: string[]; arguments: string[] }): Promise<unknown>;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface ClaimedOutboxRow {
|
||||||
|
id: bigint;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface DbWallRow {
|
||||||
|
wallNow: Date;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface ProjectionPayload {
|
||||||
|
version: 1;
|
||||||
|
profileName: string;
|
||||||
|
suspensionId: string;
|
||||||
|
sourceRevision: number;
|
||||||
|
targetRevision: number;
|
||||||
|
deadlineGeneration: number;
|
||||||
|
shiftTicks: number;
|
||||||
|
projectionDeltaMilliseconds: number;
|
||||||
|
clockBaseTime: string;
|
||||||
|
ticksPerSecond: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface TournamentProjectionState {
|
||||||
|
stage?: number;
|
||||||
|
nextAt?: string;
|
||||||
|
nextTick?: number;
|
||||||
|
bettingCloseAt?: string;
|
||||||
|
bettingCloseTick?: number;
|
||||||
|
clockRevision?: number;
|
||||||
|
deadlineGeneration?: number;
|
||||||
|
[key: string]: unknown;
|
||||||
|
}
|
||||||
|
|
||||||
|
const APPLY_CLOCK_PROJECTION_SCRIPT = `
|
||||||
|
local active = redis.call('GET', KEYS[1])
|
||||||
|
if active == ARGV[2] then
|
||||||
|
if redis.call('GET', KEYS[5]) == ARGV[4] and redis.call('GET', KEYS[2]) == ARGV[3] then
|
||||||
|
return 2
|
||||||
|
end
|
||||||
|
return -3
|
||||||
|
end
|
||||||
|
if active and active ~= ARGV[1] then
|
||||||
|
return -1
|
||||||
|
end
|
||||||
|
if ARGV[5] ~= '__NONE__' and redis.call('GET', KEYS[4]) ~= ARGV[5] then
|
||||||
|
return -2
|
||||||
|
end
|
||||||
|
redis.call('DEL', KEYS[3])
|
||||||
|
local count = tonumber(ARGV[7])
|
||||||
|
local offset = 8
|
||||||
|
for index = 1, count do
|
||||||
|
redis.call('ZADD', KEYS[3], ARGV[offset], ARGV[offset + 1])
|
||||||
|
offset = offset + 2
|
||||||
|
end
|
||||||
|
if ARGV[5] ~= '__NONE__' then
|
||||||
|
redis.call('SET', KEYS[4], ARGV[6])
|
||||||
|
end
|
||||||
|
redis.call('SET', KEYS[1], ARGV[2])
|
||||||
|
redis.call('SET', KEYS[2], ARGV[3])
|
||||||
|
redis.call('SET', KEYS[5], ARGV[4])
|
||||||
|
redis.call('SET', KEYS[6], 'RUNNING')
|
||||||
|
return 1
|
||||||
|
`;
|
||||||
|
|
||||||
|
const safeInteger = (value: unknown, label: string): number => {
|
||||||
|
const result = typeof value === 'bigint' ? Number(value) : value;
|
||||||
|
if (typeof result !== 'number' || !Number.isSafeInteger(result)) {
|
||||||
|
throw new Error(`${label} must be a safe integer.`);
|
||||||
|
}
|
||||||
|
return result;
|
||||||
|
};
|
||||||
|
|
||||||
|
const canonicalize = (value: unknown): unknown => {
|
||||||
|
if (typeof value === 'bigint') return value.toString();
|
||||||
|
if (Array.isArray(value)) return value.map(canonicalize);
|
||||||
|
if (value && typeof value === 'object') {
|
||||||
|
return Object.fromEntries(
|
||||||
|
Object.entries(value as Record<string, unknown>)
|
||||||
|
.sort(([left], [right]) => left.localeCompare(right))
|
||||||
|
.map(([key, item]) => [key, canonicalize(item)])
|
||||||
|
);
|
||||||
|
}
|
||||||
|
return value;
|
||||||
|
};
|
||||||
|
|
||||||
|
const stableJson = (value: unknown): string => JSON.stringify(canonicalize(value));
|
||||||
|
|
||||||
|
const checksum = (value: unknown): string => createHash('sha256').update(stableJson(value)).digest('hex');
|
||||||
|
|
||||||
|
const readDbWall = async (db: GamePrisma.TransactionClient): Promise<Date> => {
|
||||||
|
const rows = await db.$queryRaw<DbWallRow[]>(GamePrisma.sql`
|
||||||
|
SELECT (CURRENT_TIMESTAMP AT TIME ZONE 'UTC')::timestamp(3) AS "wallNow"
|
||||||
|
`);
|
||||||
|
if (!rows[0]?.wallNow) throw new Error('Failed to read PostgreSQL wall time for the projection outbox.');
|
||||||
|
return rows[0].wallNow;
|
||||||
|
};
|
||||||
|
|
||||||
|
const parsePayload = (value: GamePrisma.JsonValue): ProjectionPayload => {
|
||||||
|
if (!value || typeof value !== 'object' || Array.isArray(value)) {
|
||||||
|
throw new Error('Clock projection outbox payload must be an object.');
|
||||||
|
}
|
||||||
|
const payload = value as Record<string, unknown>;
|
||||||
|
if (payload.version !== 1 || typeof payload.profileName !== 'string' || typeof payload.suspensionId !== 'string') {
|
||||||
|
throw new Error('Clock projection outbox payload identity is invalid.');
|
||||||
|
}
|
||||||
|
if (typeof payload.clockBaseTime !== 'string') {
|
||||||
|
throw new Error('Clock projection outbox is missing its projection base.');
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
version: 1,
|
||||||
|
profileName: payload.profileName,
|
||||||
|
suspensionId: payload.suspensionId,
|
||||||
|
sourceRevision: safeInteger(payload.sourceRevision, 'sourceRevision'),
|
||||||
|
targetRevision: safeInteger(payload.targetRevision, 'targetRevision'),
|
||||||
|
deadlineGeneration: safeInteger(payload.deadlineGeneration, 'deadlineGeneration'),
|
||||||
|
shiftTicks: safeInteger(payload.shiftTicks, 'shiftTicks'),
|
||||||
|
projectionDeltaMilliseconds: safeInteger(
|
||||||
|
payload.projectionDeltaMilliseconds,
|
||||||
|
'projectionDeltaMilliseconds'
|
||||||
|
),
|
||||||
|
clockBaseTime: payload.clockBaseTime,
|
||||||
|
ticksPerSecond: safeInteger(payload.ticksPerSecond, 'ticksPerSecond'),
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
const claimNext = async (db: GamePrismaClient, workerId: string) =>
|
||||||
|
db.$transaction(async (transaction) => {
|
||||||
|
const rows = await transaction.$queryRaw<ClaimedOutboxRow[]>(GamePrisma.sql`
|
||||||
|
SELECT id
|
||||||
|
FROM clock_projection_outbox
|
||||||
|
WHERE available_at <= (CURRENT_TIMESTAMP AT TIME ZONE 'UTC')
|
||||||
|
AND (
|
||||||
|
status IN ('PENDING', 'FAILED')
|
||||||
|
OR (status = 'APPLYING' AND locked_at < (CURRENT_TIMESTAMP AT TIME ZONE 'UTC') - INTERVAL '30 seconds')
|
||||||
|
)
|
||||||
|
ORDER BY id
|
||||||
|
LIMIT 1
|
||||||
|
FOR UPDATE SKIP LOCKED
|
||||||
|
`);
|
||||||
|
const id = rows[0]?.id;
|
||||||
|
if (id === undefined) return null;
|
||||||
|
const lockedAt = await readDbWall(transaction);
|
||||||
|
return transaction.clockProjectionOutbox.update({
|
||||||
|
where: { id },
|
||||||
|
data: {
|
||||||
|
status: 'APPLYING',
|
||||||
|
attempts: { increment: 1 },
|
||||||
|
lockedAt,
|
||||||
|
lockedBy: workerId,
|
||||||
|
lastError: null,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
const projectTournamentState = (
|
||||||
|
raw: string | null,
|
||||||
|
payload: ProjectionPayload,
|
||||||
|
clock: GameClock
|
||||||
|
): { expected: string; next: string } | null => {
|
||||||
|
if (!raw) return null;
|
||||||
|
const parsed = JSON.parse(raw) as TournamentProjectionState;
|
||||||
|
const active = typeof parsed.stage === 'number' && parsed.stage > 0;
|
||||||
|
if (active && parsed.nextAt && !Number.isSafeInteger(parsed.nextTick)) {
|
||||||
|
throw new Error('Active tournament nextAt lacks the authoritative nextTick dual-write.');
|
||||||
|
}
|
||||||
|
if (active && parsed.bettingCloseAt && !Number.isSafeInteger(parsed.bettingCloseTick)) {
|
||||||
|
throw new Error('Active tournament bettingCloseAt lacks the authoritative bettingCloseTick dual-write.');
|
||||||
|
}
|
||||||
|
const next: TournamentProjectionState = {
|
||||||
|
...parsed,
|
||||||
|
clockRevision: payload.targetRevision,
|
||||||
|
deadlineGeneration: payload.deadlineGeneration,
|
||||||
|
};
|
||||||
|
if (Number.isSafeInteger(parsed.nextTick)) {
|
||||||
|
next.nextTick = parsed.nextTick! + payload.shiftTicks;
|
||||||
|
next.nextAt = clock.tickToDate(next.nextTick).toISOString();
|
||||||
|
}
|
||||||
|
if (Number.isSafeInteger(parsed.bettingCloseTick)) {
|
||||||
|
next.bettingCloseTick = parsed.bettingCloseTick! + payload.shiftTicks;
|
||||||
|
next.bettingCloseAt = clock.tickToDate(next.bettingCloseTick).toISOString();
|
||||||
|
}
|
||||||
|
return { expected: raw, next: JSON.stringify(next) };
|
||||||
|
};
|
||||||
|
|
||||||
|
const recordFailure = async (db: GamePrismaClient, outboxId: bigint, error: unknown): Promise<void> => {
|
||||||
|
const message = error instanceof Error ? error.message : String(error);
|
||||||
|
await db.$executeRaw(GamePrisma.sql`
|
||||||
|
UPDATE clock_projection_outbox
|
||||||
|
SET status = 'FAILED',
|
||||||
|
locked_at = NULL,
|
||||||
|
locked_by = NULL,
|
||||||
|
last_error = ${message.slice(0, 4_000)},
|
||||||
|
available_at = (CURRENT_TIMESTAMP AT TIME ZONE 'UTC') + INTERVAL '1 second'
|
||||||
|
WHERE id = ${outboxId} AND status = 'APPLYING'
|
||||||
|
`);
|
||||||
|
};
|
||||||
|
|
||||||
|
export const applyNextClockProjection = async (options: {
|
||||||
|
db: GamePrismaClient;
|
||||||
|
redis: ClockProjectionRedis;
|
||||||
|
workerId: string;
|
||||||
|
}): Promise<'IDLE' | 'APPLIED' | 'RECOVERED'> => {
|
||||||
|
if (!options.workerId.trim()) throw new Error('Clock projection worker ID is required.');
|
||||||
|
const outbox = await claimNext(options.db, options.workerId);
|
||||||
|
if (!outbox) return 'IDLE';
|
||||||
|
try {
|
||||||
|
const payload = parsePayload(outbox.payload);
|
||||||
|
if (checksum(outbox.payload) !== outbox.checksum) {
|
||||||
|
throw new Error('Clock projection outbox checksum verification failed.');
|
||||||
|
}
|
||||||
|
if (outbox.targetRevision !== BigInt(payload.targetRevision)) {
|
||||||
|
throw new Error('Clock projection payload revision differs from its outbox row.');
|
||||||
|
}
|
||||||
|
const world = await options.db.worldState.findUniqueOrThrow({ where: { id: outbox.worldStateId } });
|
||||||
|
if (
|
||||||
|
parseGameClockPhase(world.clockPhase) !== 'RECONCILING' ||
|
||||||
|
world.clockRevision !== outbox.targetRevision ||
|
||||||
|
world.deadlineGeneration !== BigInt(payload.deadlineGeneration) ||
|
||||||
|
!world.clockBaseTime
|
||||||
|
) {
|
||||||
|
throw new Error('Clock projection DB phase/revision/generation fence failed.');
|
||||||
|
}
|
||||||
|
const clock = new GameClock({
|
||||||
|
baseTime: new Date(payload.clockBaseTime),
|
||||||
|
tick: safeInteger(world.clockTick, 'world clock tick'),
|
||||||
|
mode: world.clockMode === 'manual' ? 'manual' : 'realtime',
|
||||||
|
wallAnchor: world.clockWallAnchor ?? new Date(),
|
||||||
|
turnSeconds: world.tickSeconds,
|
||||||
|
phase: 'RECONCILING',
|
||||||
|
revision: payload.targetRevision,
|
||||||
|
});
|
||||||
|
if (clock.ticksPerSecond !== payload.ticksPerSecond) {
|
||||||
|
throw new Error('Clock projection rate differs from the durable outbox payload.');
|
||||||
|
}
|
||||||
|
const auctions = await options.db.auction.findMany({
|
||||||
|
where: { status: { in: ['OPEN', 'FINALIZING'] } },
|
||||||
|
orderBy: { id: 'asc' },
|
||||||
|
select: { id: true, closeTick: true },
|
||||||
|
});
|
||||||
|
const timers = auctions.map((auction) => {
|
||||||
|
if (auction.closeTick === null) {
|
||||||
|
throw new Error(`Active auction ${auction.id} lacks closeTick during projection rebuild.`);
|
||||||
|
}
|
||||||
|
return { score: safeInteger(auction.closeTick, `auction ${auction.id} closeTick`), id: String(auction.id) };
|
||||||
|
});
|
||||||
|
const prefix = `sammo:${payload.profileName}`;
|
||||||
|
const tournamentKey = `${prefix}:tournament:state`;
|
||||||
|
const tournament = projectTournamentState(await options.redis.get(tournamentKey), payload, clock);
|
||||||
|
const result = await options.redis.eval(APPLY_CLOCK_PROJECTION_SCRIPT, {
|
||||||
|
keys: [
|
||||||
|
`${prefix}:clock:active-revision`,
|
||||||
|
`${prefix}:clock:deadline-generation`,
|
||||||
|
`${prefix}:auction:timer`,
|
||||||
|
tournamentKey,
|
||||||
|
`${prefix}:clock:projection-checksum`,
|
||||||
|
`${prefix}:clock:phase`,
|
||||||
|
],
|
||||||
|
arguments: [
|
||||||
|
String(payload.sourceRevision),
|
||||||
|
String(payload.targetRevision),
|
||||||
|
String(payload.deadlineGeneration),
|
||||||
|
outbox.checksum,
|
||||||
|
tournament?.expected ?? '__NONE__',
|
||||||
|
tournament?.next ?? '__NONE__',
|
||||||
|
String(timers.length),
|
||||||
|
...timers.flatMap(({ score, id }) => [String(score), id]),
|
||||||
|
],
|
||||||
|
});
|
||||||
|
const applied = Number(result);
|
||||||
|
if (applied === -1) throw new Error('Redis active clock revision does not match the outbox source revision.');
|
||||||
|
if (applied === -2) throw new Error('Redis tournament state changed while rebuilding its projection.');
|
||||||
|
if (applied === -3) throw new Error('Redis target revision exists without the expected projection checksum.');
|
||||||
|
if (applied !== 1 && applied !== 2) throw new Error(`Unexpected Redis clock projection result: ${String(result)}`);
|
||||||
|
|
||||||
|
await options.db.$transaction(async (transaction) => {
|
||||||
|
await acquireGameSchemaAdvisoryXactLock(transaction, CLOCK_OPERATION_PERSISTENCE_LOCK);
|
||||||
|
await transaction.$queryRaw<ClaimedOutboxRow[]>(GamePrisma.sql`
|
||||||
|
SELECT id FROM world_state WHERE id = ${outbox.worldStateId} FOR UPDATE
|
||||||
|
`);
|
||||||
|
const finalized = await transaction.worldState.updateMany({
|
||||||
|
where: {
|
||||||
|
id: outbox.worldStateId,
|
||||||
|
clockPhase: 'RECONCILING',
|
||||||
|
clockRevision: outbox.targetRevision,
|
||||||
|
deadlineGeneration: BigInt(payload.deadlineGeneration),
|
||||||
|
},
|
||||||
|
data: { clockPhase: 'RUNNING' },
|
||||||
|
});
|
||||||
|
if (finalized.count !== 1) {
|
||||||
|
throw new Error('Clock projection final RUNNING transition fence failed.');
|
||||||
|
}
|
||||||
|
const appliedAt = await readDbWall(transaction);
|
||||||
|
await transaction.clockProjectionOutbox.update({
|
||||||
|
where: { id: outbox.id },
|
||||||
|
data: { status: 'APPLIED', appliedAt, lockedAt: null, lockedBy: null, lastError: null },
|
||||||
|
});
|
||||||
|
if (outbox.suspensionId) {
|
||||||
|
await transaction.clockSuspension.update({
|
||||||
|
where: { id: outbox.suspensionId },
|
||||||
|
data: { status: 'APPLIED' },
|
||||||
|
});
|
||||||
|
}
|
||||||
|
});
|
||||||
|
return applied === 2 ? 'RECOVERED' : 'APPLIED';
|
||||||
|
} catch (error) {
|
||||||
|
await recordFailure(options.db, outbox.id, error);
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
export const loadClockReconciliationReadiness = async (db: GamePrismaClient) => {
|
||||||
|
const world = await db.worldState.findFirst({
|
||||||
|
orderBy: { id: 'asc' },
|
||||||
|
select: { clockPhase: true, clockRevision: true, deadlineGeneration: true },
|
||||||
|
});
|
||||||
|
const incompleteOutboxCount = await db.clockProjectionOutbox.count({ where: { status: { not: 'APPLIED' } } });
|
||||||
|
if (!world) {
|
||||||
|
return { ready: false, phase: null, revision: null, deadlineGeneration: null, incompleteOutboxCount };
|
||||||
|
}
|
||||||
|
const phase = parseGameClockPhase(world.clockPhase);
|
||||||
|
return {
|
||||||
|
ready: phase !== 'RECONCILING' && incompleteOutboxCount === 0,
|
||||||
|
gameplayEnabled: phase === 'RUNNING' || phase === 'MANUAL',
|
||||||
|
phase,
|
||||||
|
revision: safeInteger(world.clockRevision, 'clock revision'),
|
||||||
|
deadlineGeneration: safeInteger(world.deadlineGeneration, 'deadline generation'),
|
||||||
|
incompleteOutboxCount,
|
||||||
|
};
|
||||||
|
};
|
||||||
@@ -0,0 +1,763 @@
|
|||||||
|
import { createHash } from 'node:crypto';
|
||||||
|
|
||||||
|
import {
|
||||||
|
GAME_TICKS_PER_TURN,
|
||||||
|
MAX_SAFE_GAME_TICK,
|
||||||
|
GameClock,
|
||||||
|
buildClockAlignmentPlan,
|
||||||
|
parseClockAlignmentPolicy,
|
||||||
|
parseGameClockPhase,
|
||||||
|
type ClockAlignmentPolicy,
|
||||||
|
} from '@sammo-ts/common';
|
||||||
|
import {
|
||||||
|
CLOCK_OPERATION_PERSISTENCE_LOCK,
|
||||||
|
GENERAL_ACCESS_PERSISTENCE_LOCK,
|
||||||
|
GamePrisma,
|
||||||
|
acquireGameSchemaAdvisoryXactLock,
|
||||||
|
type GamePrismaClient,
|
||||||
|
} from '@sammo-ts/infra';
|
||||||
|
|
||||||
|
export type ClockSuspensionSource = 'MAINTENANCE' | 'OPEN_DELAY' | 'UNIFICATION_WAIT' | 'RECOVERY';
|
||||||
|
|
||||||
|
export type ClockOperationAuthority =
|
||||||
|
| { kind: 'DAEMON'; profileName: string; ownerId: string; fencingEpoch: bigint }
|
||||||
|
| { kind: 'OFFLINE'; profileName: string; reason: string };
|
||||||
|
|
||||||
|
export interface ClockSuspensionResult {
|
||||||
|
suspensionId: string;
|
||||||
|
phase: 'SUSPENDED';
|
||||||
|
sourceRevision: number;
|
||||||
|
targetRevision: number;
|
||||||
|
cutTick: number;
|
||||||
|
cutWallAt: Date;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface ClockReconciliationResult {
|
||||||
|
suspensionId: string;
|
||||||
|
phase: 'RECONCILING';
|
||||||
|
sourceRevision: number;
|
||||||
|
targetRevision: number;
|
||||||
|
deadlineGeneration: number;
|
||||||
|
gapTicks: number;
|
||||||
|
catchUpTicks: number;
|
||||||
|
shiftTicks: number;
|
||||||
|
alignedTick: number;
|
||||||
|
resumeWallAt: Date;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface DbWallRow {
|
||||||
|
wallNow: Date;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface LeaseFenceRow {
|
||||||
|
ownerId: string;
|
||||||
|
fencingEpoch: bigint;
|
||||||
|
valid: boolean;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface IdRow {
|
||||||
|
id: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface TextIdRow {
|
||||||
|
id: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface ParticipantSnapshot {
|
||||||
|
key: string;
|
||||||
|
policy: 'SHIFT' | 'KEEP' | 'REBUILD';
|
||||||
|
checksum: string;
|
||||||
|
count: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
const asJson = (value: unknown): GamePrisma.InputJsonValue => value as GamePrisma.InputJsonValue;
|
||||||
|
|
||||||
|
const safeNumber = (value: bigint, label: string): number => {
|
||||||
|
const result = Number(value);
|
||||||
|
if (!Number.isSafeInteger(result)) {
|
||||||
|
throw new Error(`${label} is outside the JavaScript safe integer range: ${value}`);
|
||||||
|
}
|
||||||
|
return result;
|
||||||
|
};
|
||||||
|
|
||||||
|
const canonicalize = (value: unknown): unknown => {
|
||||||
|
if (typeof value === 'bigint') return value.toString();
|
||||||
|
if (Array.isArray(value)) return value.map(canonicalize);
|
||||||
|
if (value && typeof value === 'object') {
|
||||||
|
return Object.fromEntries(
|
||||||
|
Object.entries(value as Record<string, unknown>)
|
||||||
|
.sort(([left], [right]) => left.localeCompare(right))
|
||||||
|
.map(([key, item]) => [key, canonicalize(item)])
|
||||||
|
);
|
||||||
|
}
|
||||||
|
return value;
|
||||||
|
};
|
||||||
|
|
||||||
|
const stableJson = (value: unknown): string => JSON.stringify(canonicalize(value));
|
||||||
|
|
||||||
|
const checksum = (value: unknown): string => createHash('sha256').update(stableJson(value)).digest('hex');
|
||||||
|
|
||||||
|
const aggregateChecksum = (participants: readonly ParticipantSnapshot[]): string =>
|
||||||
|
checksum(participants.map(({ key, policy, checksum: value, count }) => ({ key, policy, checksum: value, count })));
|
||||||
|
|
||||||
|
const readDbWall = async (db: GamePrisma.TransactionClient): Promise<Date> => {
|
||||||
|
const rows = await db.$queryRaw<DbWallRow[]>(GamePrisma.sql`
|
||||||
|
SELECT (CURRENT_TIMESTAMP AT TIME ZONE 'UTC')::timestamp(3) AS "wallNow"
|
||||||
|
`);
|
||||||
|
const wallNow = rows[0]?.wallNow;
|
||||||
|
if (!wallNow || Number.isNaN(wallNow.getTime())) {
|
||||||
|
throw new Error('Failed to read the PostgreSQL wall clock.');
|
||||||
|
}
|
||||||
|
return wallNow;
|
||||||
|
};
|
||||||
|
|
||||||
|
const verifyAuthority = async (
|
||||||
|
db: GamePrisma.TransactionClient,
|
||||||
|
authority: ClockOperationAuthority
|
||||||
|
): Promise<void> => {
|
||||||
|
const rows = await db.$queryRaw<LeaseFenceRow[]>(GamePrisma.sql`
|
||||||
|
SELECT owner_id AS "ownerId",
|
||||||
|
fencing_epoch AS "fencingEpoch",
|
||||||
|
lease_until > (CURRENT_TIMESTAMP AT TIME ZONE 'UTC') AS valid
|
||||||
|
FROM turn_daemon_lease
|
||||||
|
WHERE profile = ${authority.profileName}
|
||||||
|
FOR UPDATE
|
||||||
|
`);
|
||||||
|
const lease = rows[0];
|
||||||
|
if (authority.kind === 'OFFLINE') {
|
||||||
|
if (!authority.reason.trim()) {
|
||||||
|
throw new Error('Offline clock operations require an audit reason.');
|
||||||
|
}
|
||||||
|
if (lease?.valid) {
|
||||||
|
throw new Error(`Clock operation requires the ${authority.profileName} daemon lease to be offline.`);
|
||||||
|
}
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (
|
||||||
|
!lease?.valid ||
|
||||||
|
lease.ownerId !== authority.ownerId ||
|
||||||
|
lease.fencingEpoch !== authority.fencingEpoch
|
||||||
|
) {
|
||||||
|
throw new Error(`Stale turn-daemon fencing authority for profile ${authority.profileName}.`);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
const lockWorld = async (db: GamePrisma.TransactionClient): Promise<number> => {
|
||||||
|
const rows = await db.$queryRaw<IdRow[]>(GamePrisma.sql`
|
||||||
|
SELECT id FROM world_state ORDER BY id LIMIT 2 FOR UPDATE
|
||||||
|
`);
|
||||||
|
if (rows.length !== 1) {
|
||||||
|
throw new Error(`Clock reconciliation requires exactly one world_state row; found ${rows.length}.`);
|
||||||
|
}
|
||||||
|
return rows[0]!.id;
|
||||||
|
};
|
||||||
|
|
||||||
|
const lockParticipants = async (db: GamePrisma.TransactionClient, cutTick: bigint): Promise<void> => {
|
||||||
|
await db.$queryRaw<IdRow[]>(GamePrisma.sql`SELECT id FROM general ORDER BY id FOR UPDATE`);
|
||||||
|
await db.$queryRaw<IdRow[]>(GamePrisma.sql`
|
||||||
|
SELECT id FROM auction
|
||||||
|
WHERE status IN ('OPEN'::auction_status, 'FINALIZING'::auction_status)
|
||||||
|
ORDER BY id FOR UPDATE
|
||||||
|
`);
|
||||||
|
await db.$queryRaw<IdRow[]>(GamePrisma.sql`
|
||||||
|
SELECT id FROM message
|
||||||
|
WHERE valid_until_tick IS NOT NULL AND valid_until_tick >= ${cutTick}
|
||||||
|
ORDER BY id FOR UPDATE
|
||||||
|
`);
|
||||||
|
await db.$queryRaw<IdRow[]>(GamePrisma.sql`
|
||||||
|
SELECT id FROM vote_poll WHERE closed_at IS NULL ORDER BY id FOR UPDATE
|
||||||
|
`);
|
||||||
|
await db.$queryRaw<IdRow[]>(GamePrisma.sql`
|
||||||
|
SELECT id FROM select_pool WHERE general_id IS NULL ORDER BY id FOR UPDATE
|
||||||
|
`);
|
||||||
|
await db.$queryRaw<TextIdRow[]>(GamePrisma.sql`
|
||||||
|
SELECT owner_user_id AS id FROM select_npc_token ORDER BY owner_user_id FOR UPDATE
|
||||||
|
`);
|
||||||
|
};
|
||||||
|
|
||||||
|
const readParticipantSnapshots = async (
|
||||||
|
db: GamePrisma.TransactionClient,
|
||||||
|
worldStateId: number,
|
||||||
|
cutTick: bigint
|
||||||
|
): Promise<ParticipantSnapshot[]> => {
|
||||||
|
const [world, generals, auctions, messages, votes, pool, npcTokens, commands] = await Promise.all([
|
||||||
|
db.worldState.findUniqueOrThrow({
|
||||||
|
where: { id: worldStateId },
|
||||||
|
select: {
|
||||||
|
clockTick: true,
|
||||||
|
clockRevision: true,
|
||||||
|
deadlineGeneration: true,
|
||||||
|
lastTurnTick: true,
|
||||||
|
meta: true,
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
db.general.findMany({
|
||||||
|
orderBy: { id: 'asc' },
|
||||||
|
select: { id: true, turnTick: true, recentWarTick: true },
|
||||||
|
}),
|
||||||
|
db.auction.findMany({
|
||||||
|
where: { status: { in: ['OPEN', 'FINALIZING'] } },
|
||||||
|
orderBy: { id: 'asc' },
|
||||||
|
select: { id: true, status: true, openTick: true, closeTick: true },
|
||||||
|
}),
|
||||||
|
db.message.findMany({
|
||||||
|
where: { validUntilTick: { not: null, gte: cutTick } },
|
||||||
|
orderBy: { id: 'asc' },
|
||||||
|
select: { id: true, timeTick: true, validUntilTick: true },
|
||||||
|
}),
|
||||||
|
db.votePoll.findMany({
|
||||||
|
where: { closedAt: null },
|
||||||
|
orderBy: { id: 'asc' },
|
||||||
|
select: { id: true, startTick: true, endTick: true },
|
||||||
|
}),
|
||||||
|
db.selectPoolEntry.findMany({
|
||||||
|
where: { generalId: null },
|
||||||
|
orderBy: { id: 'asc' },
|
||||||
|
select: { id: true, reservedUntilTick: true },
|
||||||
|
}),
|
||||||
|
db.npcSelectionToken.findMany({
|
||||||
|
orderBy: { ownerUserId: 'asc' },
|
||||||
|
select: { ownerUserId: true, validUntilTick: true, pickMoreFromTick: true },
|
||||||
|
}),
|
||||||
|
db.inputEvent.findMany({
|
||||||
|
where: { status: { in: ['PENDING', 'PROCESSING'] } },
|
||||||
|
orderBy: { sequence: 'asc' },
|
||||||
|
select: { sequence: true, acceptedGameTick: true, acceptedClockRevision: true },
|
||||||
|
}),
|
||||||
|
]);
|
||||||
|
const snapshot = (key: string, policy: ParticipantSnapshot['policy'], rows: unknown[]): ParticipantSnapshot => ({
|
||||||
|
key,
|
||||||
|
policy,
|
||||||
|
checksum: checksum(rows),
|
||||||
|
count: rows.length,
|
||||||
|
});
|
||||||
|
const meta = world.meta && typeof world.meta === 'object' && !Array.isArray(world.meta) ? world.meta : {};
|
||||||
|
return [
|
||||||
|
snapshot('world-clock', 'REBUILD', [
|
||||||
|
{
|
||||||
|
clockTick: world.clockTick,
|
||||||
|
clockRevision: world.clockRevision,
|
||||||
|
deadlineGeneration: world.deadlineGeneration,
|
||||||
|
},
|
||||||
|
]),
|
||||||
|
snapshot('turn-cursor', 'SHIFT', [{ lastTurnTick: world.lastTurnTick }]),
|
||||||
|
snapshot(
|
||||||
|
'general-next-turn',
|
||||||
|
'SHIFT',
|
||||||
|
generals.map(({ id, turnTick }) => ({ id, turnTick }))
|
||||||
|
),
|
||||||
|
snapshot(
|
||||||
|
'general-recent-war-occurrence',
|
||||||
|
'KEEP',
|
||||||
|
generals.map(({ id, recentWarTick }) => ({ id, recentWarTick }))
|
||||||
|
),
|
||||||
|
snapshot(
|
||||||
|
'auction-open-occurrence',
|
||||||
|
'KEEP',
|
||||||
|
auctions.map(({ id, openTick }) => ({ id, openTick }))
|
||||||
|
),
|
||||||
|
snapshot(
|
||||||
|
'auction-deadline',
|
||||||
|
'SHIFT',
|
||||||
|
auctions.map(({ id, status, closeTick }) => ({ id, status, closeTick }))
|
||||||
|
),
|
||||||
|
snapshot(
|
||||||
|
'auction-finalizing-recovery',
|
||||||
|
'REBUILD',
|
||||||
|
auctions.map(({ id, status }) => ({ id, status }))
|
||||||
|
),
|
||||||
|
snapshot(
|
||||||
|
'message-occurrence',
|
||||||
|
'KEEP',
|
||||||
|
messages.map(({ id, timeTick }) => ({ id, timeTick }))
|
||||||
|
),
|
||||||
|
snapshot(
|
||||||
|
'message-expiry',
|
||||||
|
'SHIFT',
|
||||||
|
messages.map(({ id, validUntilTick }) => ({ id, validUntilTick }))
|
||||||
|
),
|
||||||
|
snapshot(
|
||||||
|
'vote-start-occurrence',
|
||||||
|
'KEEP',
|
||||||
|
votes.map(({ id, startTick }) => ({ id, startTick }))
|
||||||
|
),
|
||||||
|
snapshot(
|
||||||
|
'vote-end-deadline',
|
||||||
|
'SHIFT',
|
||||||
|
votes.map(({ id, endTick }) => ({ id, endTick }))
|
||||||
|
),
|
||||||
|
snapshot('select-pool-reservation', 'SHIFT', pool),
|
||||||
|
snapshot('npc-selection-window', 'SHIFT', npcTokens),
|
||||||
|
snapshot('accepted-command-coordinate', 'KEEP', commands),
|
||||||
|
snapshot('movable-json-rule-anchors', 'SHIFT', [
|
||||||
|
{
|
||||||
|
lastTurnTime: Reflect.get(meta, 'lastTurnTime'),
|
||||||
|
turntime: Reflect.get(meta, 'turntime'),
|
||||||
|
starttime: Reflect.get(meta, 'starttime'),
|
||||||
|
tnmt_time: Reflect.get(meta, 'tnmt_time'),
|
||||||
|
},
|
||||||
|
]),
|
||||||
|
];
|
||||||
|
};
|
||||||
|
|
||||||
|
const persistInitialParticipants = async (
|
||||||
|
db: GamePrisma.TransactionClient,
|
||||||
|
suspensionId: string,
|
||||||
|
participants: readonly ParticipantSnapshot[]
|
||||||
|
): Promise<void> => {
|
||||||
|
for (const participant of participants) {
|
||||||
|
await db.clockReconciliationParticipant.create({
|
||||||
|
data: {
|
||||||
|
suspensionId,
|
||||||
|
participantKey: participant.key,
|
||||||
|
policy: participant.policy,
|
||||||
|
beforeChecksum: participant.checksum,
|
||||||
|
afterChecksum: participant.checksum,
|
||||||
|
affectedCount: 0,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
const shiftMetaDate = (value: unknown, deltaMilliseconds: number): unknown => {
|
||||||
|
if (typeof value !== 'string' || !value.trim()) return value;
|
||||||
|
const parsed = new Date(value);
|
||||||
|
if (Number.isNaN(parsed.getTime())) return value;
|
||||||
|
return new Date(parsed.getTime() + deltaMilliseconds).toISOString();
|
||||||
|
};
|
||||||
|
|
||||||
|
const shiftedMeta = (value: GamePrisma.JsonValue, deltaMilliseconds: number): GamePrisma.InputJsonValue => {
|
||||||
|
const meta = value && typeof value === 'object' && !Array.isArray(value) ? { ...value } : {};
|
||||||
|
for (const key of ['lastTurnTime', 'turntime', 'starttime', 'tnmt_time'] as const) {
|
||||||
|
if (Object.hasOwn(meta, key)) {
|
||||||
|
Reflect.set(meta, key, shiftMetaDate(Reflect.get(meta, key), deltaMilliseconds));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return asJson(meta);
|
||||||
|
};
|
||||||
|
|
||||||
|
const assertShiftFits = (participants: readonly ParticipantSnapshot[], shiftTicks: number): void => {
|
||||||
|
if (!Number.isSafeInteger(shiftTicks) || shiftTicks < 0) {
|
||||||
|
throw new Error(`Invalid reconciliation shift: ${shiftTicks}`);
|
||||||
|
}
|
||||||
|
// Checksums retain stringified values for audit; actual row ranges are
|
||||||
|
// checked by PostgreSQL BIGINT and the world aligned tick is checked by the
|
||||||
|
// shared GameClock plan. The sentinel expiry is deliberately never shifted.
|
||||||
|
if (participants.some((participant) => !participant.checksum)) {
|
||||||
|
throw new Error('Participant snapshot is incomplete.');
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
const assertScheduleRanges = async (db: GamePrisma.TransactionClient, shiftTicks: number): Promise<void> => {
|
||||||
|
const shift = BigInt(shiftTicks);
|
||||||
|
const maximum = BigInt(MAX_SAFE_GAME_TICK) - shift;
|
||||||
|
const [general, auction, message, vote, pool, npcValid, npcMore] = await Promise.all([
|
||||||
|
db.general.aggregate({ _max: { turnTick: true }, where: { turnTick: { not: null } } }),
|
||||||
|
db.auction.aggregate({
|
||||||
|
_max: { closeTick: true },
|
||||||
|
where: { status: { in: ['OPEN', 'FINALIZING'] }, closeTick: { not: null } },
|
||||||
|
}),
|
||||||
|
db.message.aggregate({
|
||||||
|
_max: { validUntilTick: true },
|
||||||
|
where: { validUntilTick: { not: null, lt: BigInt(MAX_SAFE_GAME_TICK) } },
|
||||||
|
}),
|
||||||
|
db.votePoll.aggregate({ _max: { endTick: true }, where: { closedAt: null, endTick: { not: null } } }),
|
||||||
|
db.selectPoolEntry.aggregate({
|
||||||
|
_max: { reservedUntilTick: true },
|
||||||
|
where: { generalId: null, reservedUntilTick: { not: null } },
|
||||||
|
}),
|
||||||
|
db.npcSelectionToken.aggregate({ _max: { validUntilTick: true }, where: { validUntilTick: { not: null } } }),
|
||||||
|
db.npcSelectionToken.aggregate({
|
||||||
|
_max: { pickMoreFromTick: true },
|
||||||
|
where: { pickMoreFromTick: { not: null } },
|
||||||
|
}),
|
||||||
|
]);
|
||||||
|
const values: Array<[string, bigint | null]> = [
|
||||||
|
['general.turn_tick', general._max.turnTick],
|
||||||
|
['auction.close_tick', auction._max.closeTick],
|
||||||
|
['message.valid_until_tick', message._max.validUntilTick],
|
||||||
|
['vote_poll.end_tick', vote._max.endTick],
|
||||||
|
['select_pool.reserved_until_tick', pool._max.reservedUntilTick],
|
||||||
|
['select_npc_token.valid_until_tick', npcValid._max.validUntilTick],
|
||||||
|
['select_npc_token.pick_more_from_tick', npcMore._max.pickMoreFromTick],
|
||||||
|
];
|
||||||
|
for (const [label, value] of values) {
|
||||||
|
if (value !== null && value > maximum) {
|
||||||
|
throw new Error(`${label} would exceed the safe game tick range after reconciliation.`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
const applyParticipantShift = async (
|
||||||
|
db: GamePrisma.TransactionClient,
|
||||||
|
worldStateId: number,
|
||||||
|
cutTick: bigint,
|
||||||
|
alignedTick: bigint,
|
||||||
|
targetRevision: bigint,
|
||||||
|
targetGeneration: bigint,
|
||||||
|
shiftTicks: bigint,
|
||||||
|
projectionDeltaMilliseconds: number,
|
||||||
|
resumeWallAt: Date
|
||||||
|
): Promise<Map<string, number>> => {
|
||||||
|
const affected = new Map<string, number>();
|
||||||
|
const world = await db.worldState.findUniqueOrThrow({ where: { id: worldStateId }, select: { meta: true } });
|
||||||
|
const cursor = await db.worldState.updateMany({
|
||||||
|
where: { id: worldStateId, lastTurnTick: { not: null } },
|
||||||
|
data: { lastTurnTick: { increment: shiftTicks } },
|
||||||
|
});
|
||||||
|
affected.set('turn-cursor', cursor.count);
|
||||||
|
affected.set(
|
||||||
|
'general-next-turn',
|
||||||
|
await db.$executeRaw(GamePrisma.sql`
|
||||||
|
UPDATE general
|
||||||
|
SET turn_tick = turn_tick + ${shiftTicks},
|
||||||
|
turn_time = turn_time + ${projectionDeltaMilliseconds} * INTERVAL '1 millisecond'
|
||||||
|
WHERE turn_tick IS NOT NULL
|
||||||
|
`)
|
||||||
|
);
|
||||||
|
affected.set(
|
||||||
|
'auction-deadline',
|
||||||
|
await db.$executeRaw(GamePrisma.sql`
|
||||||
|
UPDATE auction
|
||||||
|
SET close_tick = close_tick + ${shiftTicks},
|
||||||
|
close_at = close_at + ${projectionDeltaMilliseconds} * INTERVAL '1 millisecond'
|
||||||
|
WHERE status IN ('OPEN'::auction_status, 'FINALIZING'::auction_status)
|
||||||
|
AND close_tick IS NOT NULL
|
||||||
|
`)
|
||||||
|
);
|
||||||
|
affected.set(
|
||||||
|
'message-expiry',
|
||||||
|
await db.$executeRaw(GamePrisma.sql`
|
||||||
|
UPDATE message
|
||||||
|
SET valid_until_tick = valid_until_tick + ${shiftTicks},
|
||||||
|
valid_until = valid_until + ${projectionDeltaMilliseconds} * INTERVAL '1 millisecond'
|
||||||
|
WHERE valid_until_tick IS NOT NULL
|
||||||
|
AND valid_until_tick >= ${cutTick}
|
||||||
|
AND valid_until_tick < ${BigInt(MAX_SAFE_GAME_TICK)}
|
||||||
|
`)
|
||||||
|
);
|
||||||
|
affected.set(
|
||||||
|
'vote-end-deadline',
|
||||||
|
await db.$executeRaw(GamePrisma.sql`
|
||||||
|
UPDATE vote_poll
|
||||||
|
SET end_tick = end_tick + ${shiftTicks},
|
||||||
|
end_at = end_at + ${projectionDeltaMilliseconds} * INTERVAL '1 millisecond'
|
||||||
|
WHERE closed_at IS NULL AND end_tick IS NOT NULL
|
||||||
|
`)
|
||||||
|
);
|
||||||
|
affected.set(
|
||||||
|
'select-pool-reservation',
|
||||||
|
await db.$executeRaw(GamePrisma.sql`
|
||||||
|
UPDATE select_pool
|
||||||
|
SET reserved_until_tick = reserved_until_tick + ${shiftTicks},
|
||||||
|
reserved_until = reserved_until + ${projectionDeltaMilliseconds} * INTERVAL '1 millisecond'
|
||||||
|
WHERE general_id IS NULL AND reserved_until_tick IS NOT NULL
|
||||||
|
`)
|
||||||
|
);
|
||||||
|
affected.set(
|
||||||
|
'npc-selection-window',
|
||||||
|
await db.$executeRaw(GamePrisma.sql`
|
||||||
|
UPDATE select_npc_token
|
||||||
|
SET valid_until_tick = CASE
|
||||||
|
WHEN valid_until_tick IS NULL THEN NULL ELSE valid_until_tick + ${shiftTicks} END,
|
||||||
|
valid_until = CASE
|
||||||
|
WHEN valid_until_tick IS NULL THEN valid_until
|
||||||
|
ELSE valid_until + ${projectionDeltaMilliseconds} * INTERVAL '1 millisecond' END,
|
||||||
|
pick_more_from_tick = CASE
|
||||||
|
WHEN pick_more_from_tick IS NULL THEN NULL ELSE pick_more_from_tick + ${shiftTicks} END,
|
||||||
|
pick_more_from = CASE
|
||||||
|
WHEN pick_more_from_tick IS NULL THEN pick_more_from
|
||||||
|
ELSE pick_more_from + ${projectionDeltaMilliseconds} * INTERVAL '1 millisecond' END
|
||||||
|
WHERE valid_until_tick IS NOT NULL OR pick_more_from_tick IS NOT NULL
|
||||||
|
`)
|
||||||
|
);
|
||||||
|
await db.worldState.update({
|
||||||
|
where: { id: worldStateId },
|
||||||
|
data: {
|
||||||
|
clockTick: alignedTick,
|
||||||
|
clockWallAnchor: resumeWallAt,
|
||||||
|
clockPhase: 'RECONCILING',
|
||||||
|
clockRevision: targetRevision,
|
||||||
|
deadlineGeneration: targetGeneration,
|
||||||
|
meta: shiftedMeta(world.meta, projectionDeltaMilliseconds),
|
||||||
|
},
|
||||||
|
});
|
||||||
|
affected.set('world-clock', 1);
|
||||||
|
affected.set('movable-json-rule-anchors', 1);
|
||||||
|
affected.set('auction-finalizing-recovery', 0);
|
||||||
|
return affected;
|
||||||
|
};
|
||||||
|
|
||||||
|
export const startClockSuspension = async (options: {
|
||||||
|
db: GamePrismaClient;
|
||||||
|
suspensionId: string;
|
||||||
|
source: ClockSuspensionSource;
|
||||||
|
authority: ClockOperationAuthority;
|
||||||
|
policy?: ClockAlignmentPolicy;
|
||||||
|
catchUpTicks?: number;
|
||||||
|
}): Promise<ClockSuspensionResult> => {
|
||||||
|
if (!options.suspensionId.trim() || options.suspensionId.length > 64) {
|
||||||
|
throw new Error('Clock suspension ID must contain 1-64 characters.');
|
||||||
|
}
|
||||||
|
const policy = options.policy ?? 'EXACT';
|
||||||
|
const catchUpTicks = options.catchUpTicks ?? 0;
|
||||||
|
if (!Number.isSafeInteger(catchUpTicks) || catchUpTicks < 0) {
|
||||||
|
throw new Error('Clock suspension catch-up ticks must be a non-negative safe integer.');
|
||||||
|
}
|
||||||
|
return options.db.$transaction(
|
||||||
|
async (db) => {
|
||||||
|
await verifyAuthority(db, options.authority);
|
||||||
|
await acquireGameSchemaAdvisoryXactLock(db, CLOCK_OPERATION_PERSISTENCE_LOCK);
|
||||||
|
await acquireGameSchemaAdvisoryXactLock(db, GENERAL_ACCESS_PERSISTENCE_LOCK);
|
||||||
|
const worldStateId = await lockWorld(db);
|
||||||
|
const existing = await db.clockSuspension.findUnique({ where: { id: options.suspensionId } });
|
||||||
|
if (existing) {
|
||||||
|
if (existing.worldStateId !== worldStateId || existing.source !== options.source || existing.policy !== policy) {
|
||||||
|
throw new Error(`Clock suspension ID ${options.suspensionId} is already bound to another operation.`);
|
||||||
|
}
|
||||||
|
if (existing.status !== 'SUSPENDED') {
|
||||||
|
throw new Error(`Clock suspension ${options.suspensionId} already advanced to ${existing.status}.`);
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
suspensionId: existing.id,
|
||||||
|
phase: 'SUSPENDED' as const,
|
||||||
|
sourceRevision: safeNumber(existing.sourceRevision, 'source revision'),
|
||||||
|
targetRevision: safeNumber(existing.targetRevision, 'target revision'),
|
||||||
|
cutTick: safeNumber(existing.cutTick, 'cut tick'),
|
||||||
|
cutWallAt: existing.cutWallAt,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
const world = await db.worldState.findUniqueOrThrow({ where: { id: worldStateId } });
|
||||||
|
const phase = parseGameClockPhase(world.clockPhase);
|
||||||
|
if (phase !== 'RUNNING') {
|
||||||
|
throw new Error(`Clock suspension can start only from RUNNING; current phase is ${phase}.`);
|
||||||
|
}
|
||||||
|
if (!world.clockBaseTime || world.clockTick === null || !world.clockWallAnchor) {
|
||||||
|
throw new Error('Clock suspension requires a fully initialized logical game clock.');
|
||||||
|
}
|
||||||
|
const cutWallAt = await readDbWall(db);
|
||||||
|
const storedTick = safeNumber(world.clockTick, 'world clock tick');
|
||||||
|
const sourceRevision = safeNumber(world.clockRevision, 'world clock revision');
|
||||||
|
const clock = new GameClock({
|
||||||
|
baseTime: world.clockBaseTime,
|
||||||
|
tick: storedTick,
|
||||||
|
mode: world.clockMode === 'manual' ? 'manual' : 'realtime',
|
||||||
|
wallAnchor: world.clockWallAnchor,
|
||||||
|
turnSeconds: world.tickSeconds,
|
||||||
|
phase,
|
||||||
|
revision: sourceRevision,
|
||||||
|
});
|
||||||
|
const cutTick = clock.nowTick(cutWallAt);
|
||||||
|
await lockParticipants(db, BigInt(cutTick));
|
||||||
|
await db.worldState.update({
|
||||||
|
where: { id: worldStateId },
|
||||||
|
data: { clockPhase: 'SUSPENDED', clockTick: BigInt(cutTick), clockWallAnchor: cutWallAt },
|
||||||
|
});
|
||||||
|
const participants = await readParticipantSnapshots(db, worldStateId, BigInt(cutTick));
|
||||||
|
await db.clockSuspension.create({
|
||||||
|
data: {
|
||||||
|
id: options.suspensionId,
|
||||||
|
worldStateId,
|
||||||
|
source: options.source,
|
||||||
|
policy,
|
||||||
|
status: 'SUSPENDED',
|
||||||
|
sourceRevision: BigInt(sourceRevision),
|
||||||
|
targetRevision: BigInt(sourceRevision + 1),
|
||||||
|
cutTick: BigInt(cutTick),
|
||||||
|
cutWallAt,
|
||||||
|
rateTicksPerSecond: GAME_TICKS_PER_TURN / world.tickSeconds,
|
||||||
|
catchUpTicks: BigInt(catchUpTicks),
|
||||||
|
participantChecksumBefore: aggregateChecksum(participants),
|
||||||
|
detail: asJson({ authority: options.authority.kind, profileName: options.authority.profileName }),
|
||||||
|
},
|
||||||
|
});
|
||||||
|
await persistInitialParticipants(db, options.suspensionId, participants);
|
||||||
|
return {
|
||||||
|
suspensionId: options.suspensionId,
|
||||||
|
phase: 'SUSPENDED',
|
||||||
|
sourceRevision,
|
||||||
|
targetRevision: sourceRevision + 1,
|
||||||
|
cutTick,
|
||||||
|
cutWallAt,
|
||||||
|
};
|
||||||
|
},
|
||||||
|
{ isolationLevel: 'Serializable', maxWait: 10_000, timeout: 30_000 }
|
||||||
|
);
|
||||||
|
};
|
||||||
|
|
||||||
|
export const reconcileClockSuspension = async (options: {
|
||||||
|
db: GamePrismaClient;
|
||||||
|
suspensionId: string;
|
||||||
|
authority: ClockOperationAuthority;
|
||||||
|
/** Deterministic fixture seam; production must always use PostgreSQL CURRENT_TIMESTAMP. */
|
||||||
|
testResumeWallAt?: Date;
|
||||||
|
}): Promise<ClockReconciliationResult> =>
|
||||||
|
options.db.$transaction(
|
||||||
|
async (db) => {
|
||||||
|
await verifyAuthority(db, options.authority);
|
||||||
|
await acquireGameSchemaAdvisoryXactLock(db, CLOCK_OPERATION_PERSISTENCE_LOCK);
|
||||||
|
await acquireGameSchemaAdvisoryXactLock(db, GENERAL_ACCESS_PERSISTENCE_LOCK);
|
||||||
|
const worldStateId = await lockWorld(db);
|
||||||
|
const suspension = await db.clockSuspension.findUniqueOrThrow({ where: { id: options.suspensionId } });
|
||||||
|
if (suspension.worldStateId !== worldStateId) {
|
||||||
|
throw new Error('Clock suspension belongs to another world state.');
|
||||||
|
}
|
||||||
|
if (suspension.status === 'RECONCILING' || suspension.status === 'APPLIED') {
|
||||||
|
if (
|
||||||
|
suspension.gapTicks === null ||
|
||||||
|
suspension.shiftTicks === null ||
|
||||||
|
suspension.alignedTick === null ||
|
||||||
|
!suspension.resumeWallAt
|
||||||
|
) {
|
||||||
|
throw new Error('Persisted clock reconciliation result is incomplete.');
|
||||||
|
}
|
||||||
|
const world = await db.worldState.findUniqueOrThrow({ where: { id: worldStateId } });
|
||||||
|
return {
|
||||||
|
suspensionId: suspension.id,
|
||||||
|
phase: 'RECONCILING' as const,
|
||||||
|
sourceRevision: safeNumber(suspension.sourceRevision, 'source revision'),
|
||||||
|
targetRevision: safeNumber(suspension.targetRevision, 'target revision'),
|
||||||
|
deadlineGeneration: safeNumber(world.deadlineGeneration, 'deadline generation'),
|
||||||
|
gapTicks: safeNumber(suspension.gapTicks, 'gap ticks'),
|
||||||
|
catchUpTicks: safeNumber(suspension.catchUpTicks, 'catch-up ticks'),
|
||||||
|
shiftTicks: safeNumber(suspension.shiftTicks, 'shift ticks'),
|
||||||
|
alignedTick: safeNumber(suspension.alignedTick, 'aligned tick'),
|
||||||
|
resumeWallAt: suspension.resumeWallAt,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
if (suspension.status !== 'SUSPENDED') {
|
||||||
|
throw new Error(`Clock suspension cannot reconcile from status ${suspension.status}.`);
|
||||||
|
}
|
||||||
|
const world = await db.worldState.findUniqueOrThrow({ where: { id: worldStateId } });
|
||||||
|
const phase = parseGameClockPhase(world.clockPhase);
|
||||||
|
if (phase !== 'SUSPENDED' || world.clockRevision !== suspension.sourceRevision) {
|
||||||
|
throw new Error('Clock reconciliation phase or source revision fence failed.');
|
||||||
|
}
|
||||||
|
const worldMeta =
|
||||||
|
world.meta && typeof world.meta === 'object' && !Array.isArray(world.meta)
|
||||||
|
? (world.meta as Record<string, unknown>)
|
||||||
|
: {};
|
||||||
|
const united = Number(worldMeta.isunited ?? worldMeta.isUnited ?? 0);
|
||||||
|
if (suspension.source === 'UNIFICATION_WAIT' || united >= 2) {
|
||||||
|
throw new Error(
|
||||||
|
'Unification wait requires the atomic alignment-and-invader workflow; generic resume is forbidden.'
|
||||||
|
);
|
||||||
|
}
|
||||||
|
const cutTick = safeNumber(suspension.cutTick, 'cut tick');
|
||||||
|
await lockParticipants(db, suspension.cutTick);
|
||||||
|
if (options.testResumeWallAt && process.env.NODE_ENV !== 'test') {
|
||||||
|
throw new Error('A clock reconciliation wall override is allowed only in tests.');
|
||||||
|
}
|
||||||
|
const resumeWallAt = options.testResumeWallAt
|
||||||
|
? new Date(options.testResumeWallAt.getTime())
|
||||||
|
: await readDbWall(db);
|
||||||
|
const plan = buildClockAlignmentPlan({
|
||||||
|
policy: parseClockAlignmentPolicy(suspension.policy),
|
||||||
|
sourceRevision: safeNumber(suspension.sourceRevision, 'source revision'),
|
||||||
|
cutTick,
|
||||||
|
cutWall: suspension.cutWallAt,
|
||||||
|
resumeWall: resumeWallAt,
|
||||||
|
ticksPerSecond: suspension.rateTicksPerSecond,
|
||||||
|
catchUpTicks: safeNumber(suspension.catchUpTicks, 'catch-up ticks'),
|
||||||
|
});
|
||||||
|
const before = await readParticipantSnapshots(db, worldStateId, suspension.cutTick);
|
||||||
|
assertShiftFits(before, plan.shiftTicks);
|
||||||
|
await assertScheduleRanges(db, plan.shiftTicks);
|
||||||
|
const projectionDeltaMilliseconds = Math.trunc(
|
||||||
|
(plan.shiftTicks * 1_000) / suspension.rateTicksPerSecond
|
||||||
|
);
|
||||||
|
if (!Number.isSafeInteger(projectionDeltaMilliseconds)) {
|
||||||
|
throw new Error('Clock reconciliation projection delta is outside the safe integer range.');
|
||||||
|
}
|
||||||
|
const targetGeneration = world.deadlineGeneration + 1n;
|
||||||
|
const affected = await applyParticipantShift(
|
||||||
|
db,
|
||||||
|
worldStateId,
|
||||||
|
suspension.cutTick,
|
||||||
|
BigInt(plan.alignedTick),
|
||||||
|
BigInt(plan.targetRevision),
|
||||||
|
targetGeneration,
|
||||||
|
BigInt(plan.shiftTicks),
|
||||||
|
projectionDeltaMilliseconds,
|
||||||
|
resumeWallAt
|
||||||
|
);
|
||||||
|
const after = await readParticipantSnapshots(db, worldStateId, suspension.cutTick);
|
||||||
|
const afterByKey = new Map(after.map((participant) => [participant.key, participant]));
|
||||||
|
for (const participant of before) {
|
||||||
|
const next = afterByKey.get(participant.key);
|
||||||
|
if (!next) throw new Error(`Missing post-reconciliation participant: ${participant.key}`);
|
||||||
|
if (participant.policy === 'KEEP' && participant.checksum !== next.checksum) {
|
||||||
|
throw new Error(`KEEP participant changed during reconciliation: ${participant.key}`);
|
||||||
|
}
|
||||||
|
await db.clockReconciliationParticipant.upsert({
|
||||||
|
where: {
|
||||||
|
suspensionId_participantKey: {
|
||||||
|
suspensionId: suspension.id,
|
||||||
|
participantKey: participant.key,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
create: {
|
||||||
|
suspensionId: suspension.id,
|
||||||
|
participantKey: participant.key,
|
||||||
|
policy: participant.policy,
|
||||||
|
beforeChecksum: participant.checksum,
|
||||||
|
afterChecksum: next.checksum,
|
||||||
|
affectedCount: affected.get(participant.key) ?? 0,
|
||||||
|
},
|
||||||
|
update: {
|
||||||
|
policy: participant.policy,
|
||||||
|
beforeChecksum: participant.checksum,
|
||||||
|
afterChecksum: next.checksum,
|
||||||
|
affectedCount: affected.get(participant.key) ?? 0,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
}
|
||||||
|
const outboxPayload = {
|
||||||
|
version: 1,
|
||||||
|
profileName: options.authority.profileName,
|
||||||
|
suspensionId: suspension.id,
|
||||||
|
sourceRevision: plan.sourceRevision,
|
||||||
|
targetRevision: plan.targetRevision,
|
||||||
|
deadlineGeneration: safeNumber(targetGeneration, 'deadline generation'),
|
||||||
|
shiftTicks: plan.shiftTicks,
|
||||||
|
projectionDeltaMilliseconds,
|
||||||
|
clockBaseTime: world.clockBaseTime!.toISOString(),
|
||||||
|
ticksPerSecond: suspension.rateTicksPerSecond,
|
||||||
|
};
|
||||||
|
await db.clockProjectionOutbox.create({
|
||||||
|
data: {
|
||||||
|
worldStateId,
|
||||||
|
suspensionId: suspension.id,
|
||||||
|
targetRevision: BigInt(plan.targetRevision),
|
||||||
|
status: 'PENDING',
|
||||||
|
payload: asJson(outboxPayload),
|
||||||
|
checksum: checksum(outboxPayload),
|
||||||
|
},
|
||||||
|
});
|
||||||
|
await db.clockSuspension.update({
|
||||||
|
where: { id: suspension.id },
|
||||||
|
data: {
|
||||||
|
status: 'RECONCILING',
|
||||||
|
resumeWallAt,
|
||||||
|
gapTicks: BigInt(plan.gapTicks),
|
||||||
|
shiftTicks: BigInt(plan.shiftTicks),
|
||||||
|
alignedTick: BigInt(plan.alignedTick),
|
||||||
|
participantChecksumBefore: aggregateChecksum(before),
|
||||||
|
participantChecksumAfter: aggregateChecksum(after),
|
||||||
|
},
|
||||||
|
});
|
||||||
|
return {
|
||||||
|
suspensionId: suspension.id,
|
||||||
|
phase: 'RECONCILING',
|
||||||
|
sourceRevision: plan.sourceRevision,
|
||||||
|
targetRevision: plan.targetRevision,
|
||||||
|
deadlineGeneration: safeNumber(targetGeneration, 'deadline generation'),
|
||||||
|
gapTicks: plan.gapTicks,
|
||||||
|
catchUpTicks: plan.catchUpTicks,
|
||||||
|
shiftTicks: plan.shiftTicks,
|
||||||
|
alignedTick: plan.alignedTick,
|
||||||
|
resumeWallAt,
|
||||||
|
};
|
||||||
|
},
|
||||||
|
{ isolationLevel: 'Serializable', maxWait: 10_000, timeout: 30_000 }
|
||||||
|
);
|
||||||
@@ -1218,6 +1218,34 @@ export const createDatabaseTurnHooks = async (
|
|||||||
`found ${durableClock.clock_phase}@${durableClock.clock_revision}/${durableClock.deadline_generation}.`
|
`found ${durableClock.clock_phase}@${durableClock.clock_revision}/${durableClock.deadline_generation}.`
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
if (commandCompletion) {
|
||||||
|
const commandFence = await prisma.$queryRaw<
|
||||||
|
Array<{
|
||||||
|
status: string;
|
||||||
|
processing_clock_revision: bigint | null;
|
||||||
|
processing_deadline_generation: bigint | null;
|
||||||
|
}>
|
||||||
|
>(GamePrisma.sql`
|
||||||
|
SELECT status,
|
||||||
|
processing_clock_revision,
|
||||||
|
processing_deadline_generation
|
||||||
|
FROM input_event
|
||||||
|
WHERE request_id = ${commandCompletion.requestId}
|
||||||
|
AND target = 'ENGINE'::"InputEventTarget"
|
||||||
|
FOR UPDATE
|
||||||
|
`);
|
||||||
|
const event = commandFence[0];
|
||||||
|
if (
|
||||||
|
!event ||
|
||||||
|
event.status !== 'PROCESSING' ||
|
||||||
|
event.processing_clock_revision !== expectedRevision ||
|
||||||
|
event.processing_deadline_generation !== expectedGeneration
|
||||||
|
) {
|
||||||
|
throw new Error(
|
||||||
|
`Input event processing clock fence changed before commit: ${commandCompletion.requestId}.`
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
let neutralAuctionsToCreate = pendingNeutralAuctions;
|
let neutralAuctionsToCreate = pendingNeutralAuctions;
|
||||||
if (pendingNeutralAuctions.length > 0) {
|
if (pendingNeutralAuctions.length > 0) {
|
||||||
const latestRegistrationKey =
|
const latestRegistrationKey =
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
import type { GamePrisma, GamePrismaClient } from '@sammo-ts/infra';
|
import { readInputEventClockCoordinate, type GamePrisma, type GamePrismaClient } from '@sammo-ts/infra';
|
||||||
import { randomUUID } from 'node:crypto';
|
import { randomUUID } from 'node:crypto';
|
||||||
|
|
||||||
import {
|
import {
|
||||||
@@ -137,14 +137,21 @@ const ensureEngineCommand = async (
|
|||||||
deltaMinutes,
|
deltaMinutes,
|
||||||
};
|
};
|
||||||
try {
|
try {
|
||||||
await db.inputEvent.create({
|
await db.$transaction(async (transaction) => {
|
||||||
|
const coordinate = await readInputEventClockCoordinate(transaction);
|
||||||
|
await transaction.inputEvent.create({
|
||||||
data: {
|
data: {
|
||||||
requestId,
|
requestId,
|
||||||
target: 'ENGINE',
|
target: 'ENGINE',
|
||||||
eventType: command.type,
|
eventType: command.type,
|
||||||
payload: asJson(command),
|
payload: asJson(command),
|
||||||
|
acceptedGameTick: coordinate.gameTick,
|
||||||
|
acceptedClockRevision: coordinate.clockRevision,
|
||||||
|
acceptedDeadlineGeneration: coordinate.deadlineGeneration,
|
||||||
|
createdAt: coordinate.wallAt,
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
|
});
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
if (!isUniqueConflict(error)) {
|
if (!isUniqueConflict(error)) {
|
||||||
throw error;
|
throw error;
|
||||||
|
|||||||
@@ -10,7 +10,7 @@ import {
|
|||||||
type TurnDaemonCommand,
|
type TurnDaemonCommand,
|
||||||
type TurnDaemonCommandResult,
|
type TurnDaemonCommandResult,
|
||||||
} from '@sammo-ts/common';
|
} from '@sammo-ts/common';
|
||||||
import type { GamePrisma, GamePrismaClient } from '@sammo-ts/infra';
|
import { readInputEventClockCoordinate, type GamePrisma, type GamePrismaClient } from '@sammo-ts/infra';
|
||||||
|
|
||||||
import type { GatewayAdminActionRecord, GatewayAdminActionResult } from './gatewayAdminActions.js';
|
import type { GatewayAdminActionRecord, GatewayAdminActionResult } from './gatewayAdminActions.js';
|
||||||
|
|
||||||
@@ -97,14 +97,21 @@ const ensureEngineCommand = async (
|
|||||||
settings,
|
settings,
|
||||||
};
|
};
|
||||||
try {
|
try {
|
||||||
await db.inputEvent.create({
|
await db.$transaction(async (transaction) => {
|
||||||
|
const coordinate = await readInputEventClockCoordinate(transaction);
|
||||||
|
await transaction.inputEvent.create({
|
||||||
data: {
|
data: {
|
||||||
requestId,
|
requestId,
|
||||||
target: 'ENGINE',
|
target: 'ENGINE',
|
||||||
eventType: command.type,
|
eventType: command.type,
|
||||||
payload: asJson(command),
|
payload: asJson(command),
|
||||||
|
acceptedGameTick: coordinate.gameTick,
|
||||||
|
acceptedClockRevision: coordinate.clockRevision,
|
||||||
|
acceptedDeadlineGeneration: coordinate.deadlineGeneration,
|
||||||
|
createdAt: coordinate.wallAt,
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
|
});
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
if (!isUniqueConflict(error)) throw error;
|
if (!isUniqueConflict(error)) throw error;
|
||||||
const existing = await db.inputEvent.findUniqueOrThrow({
|
const existing = await db.inputEvent.findUniqueOrThrow({
|
||||||
|
|||||||
@@ -439,15 +439,20 @@ export const reserveSelectionPool = async (options: {
|
|||||||
userId: string;
|
userId: string;
|
||||||
now?: Date;
|
now?: Date;
|
||||||
acceptedGameTick?: number;
|
acceptedGameTick?: number;
|
||||||
|
processingGameTick?: number;
|
||||||
seedOwnerIdentity?: string | number;
|
seedOwnerIdentity?: string | number;
|
||||||
}): Promise<SelectPoolReservationDto> => {
|
}): Promise<SelectPoolReservationDto> => {
|
||||||
const { db, world, worldState, userId } = options;
|
const { db, world, worldState, userId } = options;
|
||||||
requirePoolWorld(worldState);
|
requirePoolWorld(worldState);
|
||||||
const now = options.now ?? new Date();
|
const now = options.now ?? new Date();
|
||||||
const acceptedGameTick = options.acceptedGameTick ?? resolveAcceptedGameTick(world, now);
|
const acceptedGameTick = options.acceptedGameTick ?? resolveAcceptedGameTick(world, now);
|
||||||
|
const processingGameTick = options.processingGameTick ?? acceptedGameTick;
|
||||||
if (!Number.isSafeInteger(acceptedGameTick)) {
|
if (!Number.isSafeInteger(acceptedGameTick)) {
|
||||||
fail('INTERNAL_SERVER_ERROR', '장수 선택 예약 tick이 안전한 정수 범위를 벗어났습니다.');
|
fail('INTERNAL_SERVER_ERROR', '장수 선택 예약 tick이 안전한 정수 범위를 벗어났습니다.');
|
||||||
}
|
}
|
||||||
|
if (!Number.isSafeInteger(processingGameTick)) {
|
||||||
|
fail('INTERNAL_SERVER_ERROR', '장수 선택 처리 tick이 안전한 정수 범위를 벗어났습니다.');
|
||||||
|
}
|
||||||
await lockSelectionUser(db, userId);
|
await lockSelectionUser(db, userId);
|
||||||
await lockSelectionMutationTables(db);
|
await lockSelectionMutationTables(db);
|
||||||
const general = await db.general.findFirst({
|
const general = await db.general.findFirst({
|
||||||
@@ -461,7 +466,7 @@ export const reserveSelectionPool = async (options: {
|
|||||||
|
|
||||||
let currentRows = await synchronizeSelectionPoolWorld(db, world);
|
let currentRows = await synchronizeSelectionPoolWorld(db, world);
|
||||||
const existing = currentRows.filter(
|
const existing = currentRows.filter(
|
||||||
(row) => row.ownerUserId === userId && row.generalId === null && isReservationActive(row, now, acceptedGameTick)
|
(row) => row.ownerUserId === userId && row.generalId === null && isReservationActive(row, now, processingGameTick)
|
||||||
);
|
);
|
||||||
if (existing.length > 0) {
|
if (existing.length > 0) {
|
||||||
return toReservationDto(existing, Boolean(general), worldState, world);
|
return toReservationDto(existing, Boolean(general), worldState, world);
|
||||||
@@ -471,7 +476,7 @@ export const reserveSelectionPool = async (options: {
|
|||||||
where: {
|
where: {
|
||||||
generalId: null,
|
generalId: null,
|
||||||
OR: [
|
OR: [
|
||||||
{ reservedUntilTick: { lt: BigInt(acceptedGameTick) } },
|
{ reservedUntilTick: { lt: BigInt(processingGameTick) } },
|
||||||
{ reservedUntilTick: null, reservedUntil: { lt: now } },
|
{ reservedUntilTick: null, reservedUntil: { lt: now } },
|
||||||
],
|
],
|
||||||
},
|
},
|
||||||
@@ -483,7 +488,7 @@ export const reserveSelectionPool = async (options: {
|
|||||||
});
|
});
|
||||||
currentRows = await synchronizeSelectionPoolWorld(db, world);
|
currentRows = await synchronizeSelectionPoolWorld(db, world);
|
||||||
const availableIds = new Set(
|
const availableIds = new Set(
|
||||||
world.listGeneralPoolCandidates(now, acceptedGameTick)?.map((candidate) => candidate.poolEntryId) ?? []
|
world.listGeneralPoolCandidates(now, processingGameTick)?.map((candidate) => candidate.poolEntryId) ?? []
|
||||||
);
|
);
|
||||||
const available = currentRows.filter(
|
const available = currentRows.filter(
|
||||||
(row) =>
|
(row) =>
|
||||||
@@ -507,7 +512,7 @@ export const reserveSelectionPool = async (options: {
|
|||||||
(row) =>
|
(row) =>
|
||||||
[row, calculateSelectionCandidateWeight(poolName, parseCandidate(row), true)] as [SelectPoolRow, number]
|
[row, calculateSelectionCandidateWeight(poolName, parseCandidate(row), true)] as [SelectPoolRow, number]
|
||||||
);
|
);
|
||||||
const reservedUntilTick = acceptedGameTick + RESERVATION_TURN_MULTIPLIER * GAME_TICKS_PER_TURN;
|
const reservedUntilTick = processingGameTick + RESERVATION_TURN_MULTIPLIER * GAME_TICKS_PER_TURN;
|
||||||
if (!Number.isSafeInteger(reservedUntilTick)) {
|
if (!Number.isSafeInteger(reservedUntilTick)) {
|
||||||
fail('INTERNAL_SERVER_ERROR', '장수 선택 예약 tick이 안전한 정수 범위를 벗어났습니다.');
|
fail('INTERNAL_SERVER_ERROR', '장수 선택 예약 tick이 안전한 정수 범위를 벗어났습니다.');
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -14,8 +14,12 @@ interface TournamentState {
|
|||||||
openMonth: number;
|
openMonth: number;
|
||||||
termSeconds: number;
|
termSeconds: number;
|
||||||
nextAt: string;
|
nextAt: string;
|
||||||
|
nextTick?: number;
|
||||||
|
clockRevision?: number;
|
||||||
|
deadlineGeneration?: number;
|
||||||
bettingId?: number;
|
bettingId?: number;
|
||||||
bettingCloseAt?: string;
|
bettingCloseAt?: string;
|
||||||
|
bettingCloseTick?: number;
|
||||||
winnerId?: number;
|
winnerId?: number;
|
||||||
bettingSettled?: boolean;
|
bettingSettled?: boolean;
|
||||||
rewardSettled?: boolean;
|
rewardSettled?: boolean;
|
||||||
@@ -76,6 +80,9 @@ export const createTournamentAutoStartHandler = (options: {
|
|||||||
sourceRevisionKey: `sammo:${options.profileName}:tournament:source-revision`,
|
sourceRevisionKey: `sammo:${options.profileName}:tournament:source-revision`,
|
||||||
sourceRevisionChannel: `sammo:${options.profileName}:tournament:source-changed`,
|
sourceRevisionChannel: `sammo:${options.profileName}:tournament:source-changed`,
|
||||||
realtimeEventChannel: buildGameEventChannel(options.profileName),
|
realtimeEventChannel: buildGameEventChannel(options.profileName),
|
||||||
|
activeClockRevisionKey: `sammo:${options.profileName}:clock:active-revision`,
|
||||||
|
deadlineGenerationKey: `sammo:${options.profileName}:clock:deadline-generation`,
|
||||||
|
clockPhaseKey: `sammo:${options.profileName}:clock:phase`,
|
||||||
};
|
};
|
||||||
return {
|
return {
|
||||||
onMonthChanged: async (context) => {
|
onMonthChanged: async (context) => {
|
||||||
@@ -118,6 +125,8 @@ export const createTournamentAutoStartHandler = (options: {
|
|||||||
previousState && Number.isFinite(previousState.termSeconds) && previousState.termSeconds > 0
|
previousState && Number.isFinite(previousState.termSeconds) && previousState.termSeconds > 0
|
||||||
? previousState.termSeconds
|
? previousState.termSeconds
|
||||||
: resolveTermSeconds(state.tickSeconds);
|
: resolveTermSeconds(state.tickSeconds);
|
||||||
|
const nextAt = new Date(now.getTime() + termSeconds * 60_000);
|
||||||
|
const clockState = world.getGameClockState();
|
||||||
const nextState: TournamentState = {
|
const nextState: TournamentState = {
|
||||||
stage: 1,
|
stage: 1,
|
||||||
phase: 0,
|
phase: 0,
|
||||||
@@ -129,7 +138,10 @@ export const createTournamentAutoStartHandler = (options: {
|
|||||||
// Ref startTournament() passes calcTournamentTerm()'s seconds
|
// Ref startTournament() passes calcTournamentTerm()'s seconds
|
||||||
// value to DateInterval's minute field. Preserve that historical
|
// value to DateInterval's minute field. Preserve that historical
|
||||||
// initial enrollment delay; later tournament phases use seconds.
|
// initial enrollment delay; later tournament phases use seconds.
|
||||||
nextAt: new Date(now.getTime() + termSeconds * 60_000).toISOString(),
|
nextAt: nextAt.toISOString(),
|
||||||
|
nextTick: world.dateToGameTick(nextAt),
|
||||||
|
clockRevision: clockState.revision,
|
||||||
|
deadlineGeneration: clockState.deadlineGeneration,
|
||||||
bettingId:
|
bettingId:
|
||||||
typeof previousState?.bettingId === 'number' && Number.isFinite(previousState.bettingId)
|
typeof previousState?.bettingId === 'number' && Number.isFinite(previousState.bettingId)
|
||||||
? previousState.bettingId + 1
|
? previousState.bettingId + 1
|
||||||
@@ -142,12 +154,26 @@ export const createTournamentAutoStartHandler = (options: {
|
|||||||
lastError: undefined,
|
lastError: undefined,
|
||||||
lastErrorAt: undefined,
|
lastErrorAt: undefined,
|
||||||
};
|
};
|
||||||
await writeTournamentProjection(redis, keys, [
|
await writeTournamentProjection(
|
||||||
|
redis,
|
||||||
|
keys,
|
||||||
|
[
|
||||||
{ key: keys.participantsKey, value: [] },
|
{ key: keys.participantsKey, value: [] },
|
||||||
{ key: keys.matchesKey, value: [] },
|
{ key: keys.matchesKey, value: [] },
|
||||||
{ key: keys.bettingKey, value: [] },
|
{ key: keys.bettingKey, value: [] },
|
||||||
{ key: keys.stateKey, value: nextState },
|
{ key: keys.stateKey, value: nextState },
|
||||||
]);
|
],
|
||||||
|
clockState.phase === 'RUNNING'
|
||||||
|
? {
|
||||||
|
activeRevisionKey: keys.activeClockRevisionKey,
|
||||||
|
deadlineGenerationKey: keys.deadlineGenerationKey,
|
||||||
|
phaseKey: keys.clockPhaseKey,
|
||||||
|
revision: clockState.revision,
|
||||||
|
deadlineGeneration: clockState.deadlineGeneration,
|
||||||
|
phase: 'RUNNING',
|
||||||
|
}
|
||||||
|
: undefined
|
||||||
|
);
|
||||||
|
|
||||||
const [typeText, generalTypeText] = TOURNAMENT_TEXT[type] ?? TOURNAMENT_TEXT[0];
|
const [typeText, generalTypeText] = TOURNAMENT_TEXT[type] ?? TOURNAMENT_TEXT[0];
|
||||||
const emperor = world
|
const emperor = world
|
||||||
|
|||||||
@@ -274,6 +274,10 @@ const resolveSelectionCommandAcceptedAt = async (
|
|||||||
command: Extract<TurnDaemonCommand, { type: 'selectPoolReserve' | 'selectPoolCreate' | 'selectPoolReselect' }>
|
command: Extract<TurnDaemonCommand, { type: 'selectPoolReserve' | 'selectPoolCreate' | 'selectPoolReselect' }>
|
||||||
): Promise<Date> => {
|
): Promise<Date> => {
|
||||||
const operationalAcceptedAt = await resolveCommandAcceptedAt(db, command);
|
const operationalAcceptedAt = await resolveCommandAcceptedAt(db, command);
|
||||||
|
const processingGameTick = Reflect.get(command, 'processingGameTick');
|
||||||
|
if (typeof processingGameTick === 'number' && Number.isSafeInteger(processingGameTick)) {
|
||||||
|
return world.gameTickToDate(processingGameTick);
|
||||||
|
}
|
||||||
if (command.acceptedGameTick !== undefined) {
|
if (command.acceptedGameTick !== undefined) {
|
||||||
return world.gameTickToDate(command.acceptedGameTick);
|
return world.gameTickToDate(command.acceptedGameTick);
|
||||||
}
|
}
|
||||||
@@ -407,7 +411,11 @@ async function handleNpcPossessGeneral(
|
|||||||
throw new Error('NPC possession world state is missing.');
|
throw new Error('NPC possession world state is missing.');
|
||||||
}
|
}
|
||||||
const operationalAcceptedAt = await resolveCommandAcceptedAt(db, command);
|
const operationalAcceptedAt = await resolveCommandAcceptedAt(db, command);
|
||||||
const acceptedAt = command.acceptedGameAt
|
const processingGameTick = Reflect.get(command, 'processingGameTick');
|
||||||
|
const acceptedAt =
|
||||||
|
typeof processingGameTick === 'number' && Number.isSafeInteger(processingGameTick)
|
||||||
|
? ctx.world.gameTickToDate(processingGameTick)
|
||||||
|
: command.acceptedGameAt
|
||||||
? new Date(command.acceptedGameAt)
|
? new Date(command.acceptedGameAt)
|
||||||
: ctx.world.getGameNow(operationalAcceptedAt);
|
: ctx.world.getGameNow(operationalAcceptedAt);
|
||||||
try {
|
try {
|
||||||
@@ -451,8 +459,11 @@ async function handleSelectPoolCreate(
|
|||||||
throw new Error('Selection-pool world state is missing.');
|
throw new Error('Selection-pool world state is missing.');
|
||||||
}
|
}
|
||||||
const operationalAcceptedAt = await resolveCommandAcceptedAt(db, command);
|
const operationalAcceptedAt = await resolveCommandAcceptedAt(db, command);
|
||||||
|
const processingGameTick = Reflect.get(command, 'processingGameTick');
|
||||||
const acceptedAt =
|
const acceptedAt =
|
||||||
command.acceptedGameTick !== undefined
|
typeof processingGameTick === 'number' && Number.isSafeInteger(processingGameTick)
|
||||||
|
? ctx.world.gameTickToDate(processingGameTick)
|
||||||
|
: command.acceptedGameTick !== undefined
|
||||||
? ctx.world.gameTickToDate(command.acceptedGameTick)
|
? ctx.world.gameTickToDate(command.acceptedGameTick)
|
||||||
: command.acceptedGameAt !== undefined
|
: command.acceptedGameAt !== undefined
|
||||||
? new Date(command.acceptedGameAt)
|
? new Date(command.acceptedGameAt)
|
||||||
@@ -515,6 +526,9 @@ async function handleSelectPoolReserve(
|
|||||||
seedOwnerIdentity: command.seedOwnerIdentity,
|
seedOwnerIdentity: command.seedOwnerIdentity,
|
||||||
now: acceptedAt,
|
now: acceptedAt,
|
||||||
...(command.acceptedGameTick === undefined ? {} : { acceptedGameTick: command.acceptedGameTick }),
|
...(command.acceptedGameTick === undefined ? {} : { acceptedGameTick: command.acceptedGameTick }),
|
||||||
|
...(typeof Reflect.get(command, 'processingGameTick') === 'number'
|
||||||
|
? { processingGameTick: Reflect.get(command, 'processingGameTick') as number }
|
||||||
|
: {}),
|
||||||
}),
|
}),
|
||||||
};
|
};
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
@@ -2770,9 +2784,15 @@ const validateVoteSelectionInTransaction = async (
|
|||||||
if (!poll) return '설문조사가 없습니다.';
|
if (!poll) return '설문조사가 없습니다.';
|
||||||
|
|
||||||
const processingNow = ctx.world.getGameNow(new Date());
|
const processingNow = ctx.world.getGameNow(new Date());
|
||||||
const acceptedGameTick = command.acceptedGameTick ?? ctx.world.dateToGameTick(processingNow);
|
const convertedProcessingTick = Reflect.get(command, 'processingGameTick');
|
||||||
|
const acceptedGameTick =
|
||||||
|
typeof convertedProcessingTick === 'number' && Number.isSafeInteger(convertedProcessingTick)
|
||||||
|
? convertedProcessingTick
|
||||||
|
: (command.acceptedGameTick ?? ctx.world.dateToGameTick(processingNow));
|
||||||
const acceptedGameAt =
|
const acceptedGameAt =
|
||||||
command.acceptedGameTick === undefined ? processingNow : ctx.world.gameTickToDate(command.acceptedGameTick);
|
command.acceptedGameTick === undefined && convertedProcessingTick === undefined
|
||||||
|
? processingNow
|
||||||
|
: ctx.world.gameTickToDate(acceptedGameTick);
|
||||||
if (hasVotePollDeadlinePassed(poll, acceptedGameAt, acceptedGameTick)) {
|
if (hasVotePollDeadlinePassed(poll, acceptedGameAt, acceptedGameTick)) {
|
||||||
return '설문조사가 종료되었습니다.';
|
return '설문조사가 종료되었습니다.';
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,364 @@
|
|||||||
|
import { afterAll, beforeAll, beforeEach, describe, expect, it } from 'vitest';
|
||||||
|
|
||||||
|
import { GameClock } from '@sammo-ts/common';
|
||||||
|
import {
|
||||||
|
createGamePostgresConnector,
|
||||||
|
createRedisConnector,
|
||||||
|
type GamePrismaClient,
|
||||||
|
type RedisConnector,
|
||||||
|
} from '@sammo-ts/infra';
|
||||||
|
|
||||||
|
import { reconcileClockSuspension, startClockSuspension } from '../src/turn/clockReconciliation.js';
|
||||||
|
import { applyNextClockProjection } from '../src/turn/clockProjectionOutbox.js';
|
||||||
|
|
||||||
|
const enabled =
|
||||||
|
process.env.CLOCK_RECONCILIATION_INTEGRATION === '1' &&
|
||||||
|
Boolean(process.env.DATABASE_URL) &&
|
||||||
|
Boolean(process.env.REDIS_URL);
|
||||||
|
const describeIntegration = enabled ? describe : describe.skip;
|
||||||
|
|
||||||
|
describeIntegration('durable clock reconciliation', () => {
|
||||||
|
let db: GamePrismaClient;
|
||||||
|
let disconnect: (() => Promise<void>) | undefined;
|
||||||
|
let redis: RedisConnector;
|
||||||
|
|
||||||
|
beforeAll(async () => {
|
||||||
|
const connector = createGamePostgresConnector({ url: process.env.DATABASE_URL! });
|
||||||
|
db = connector.prisma;
|
||||||
|
disconnect = connector.disconnect;
|
||||||
|
redis = createRedisConnector({ url: process.env.REDIS_URL! });
|
||||||
|
await redis.connect();
|
||||||
|
});
|
||||||
|
|
||||||
|
afterAll(async () => {
|
||||||
|
await redis.disconnect();
|
||||||
|
await disconnect?.();
|
||||||
|
});
|
||||||
|
|
||||||
|
beforeEach(async () => {
|
||||||
|
await redis.client.flushDb();
|
||||||
|
await db.$transaction([
|
||||||
|
db.clockProjectionOutbox.deleteMany(),
|
||||||
|
db.clockReconciliationParticipant.deleteMany(),
|
||||||
|
db.clockSuspension.deleteMany(),
|
||||||
|
db.inputEvent.deleteMany(),
|
||||||
|
db.vote.deleteMany(),
|
||||||
|
db.voteComment.deleteMany(),
|
||||||
|
db.votePoll.deleteMany(),
|
||||||
|
db.message.deleteMany(),
|
||||||
|
db.auctionBid.deleteMany(),
|
||||||
|
db.auction.deleteMany(),
|
||||||
|
db.npcSelectionToken.deleteMany(),
|
||||||
|
db.selectPoolEntry.deleteMany(),
|
||||||
|
db.general.deleteMany(),
|
||||||
|
db.turnDaemonLease.deleteMany(),
|
||||||
|
db.worldState.deleteMany(),
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('preserves every remaining deadline and occurrence across a 65m17.250s exact gap', async () => {
|
||||||
|
const baseTime = new Date('2026-01-01T00:00:00.000Z');
|
||||||
|
const futureAnchor = new Date(Date.now() + 3_600_000);
|
||||||
|
const initialTick = 1_000_000;
|
||||||
|
const lastTurnTick = 900_000;
|
||||||
|
const clock = new GameClock({
|
||||||
|
baseTime,
|
||||||
|
tick: initialTick,
|
||||||
|
mode: 'realtime',
|
||||||
|
wallAnchor: futureAnchor,
|
||||||
|
turnSeconds: 600,
|
||||||
|
phase: 'RUNNING',
|
||||||
|
revision: 1,
|
||||||
|
});
|
||||||
|
const generalTicks = [initialTick + 1_234, initialTick + 36_000_123];
|
||||||
|
const auctionCloseTick = initialTick + 72_000_777;
|
||||||
|
const messageOccurrenceTick = initialTick - 500;
|
||||||
|
const messageExpiryTick = initialTick + 90_000_999;
|
||||||
|
const voteStartTick = initialTick - 200;
|
||||||
|
const voteEndTick = initialTick + 18_000_321;
|
||||||
|
const poolTick = initialTick + 2_000_111;
|
||||||
|
const npcValidTick = initialTick + 3_000_222;
|
||||||
|
const npcMoreTick = initialTick + 1_000_333;
|
||||||
|
|
||||||
|
const world = await db.worldState.create({
|
||||||
|
data: {
|
||||||
|
scenarioCode: 'clock-test',
|
||||||
|
currentYear: 180,
|
||||||
|
currentMonth: 1,
|
||||||
|
tickSeconds: 600,
|
||||||
|
clockBaseTime: baseTime,
|
||||||
|
clockTick: BigInt(initialTick),
|
||||||
|
clockMode: 'realtime',
|
||||||
|
clockWallAnchor: futureAnchor,
|
||||||
|
lastTurnTick: BigInt(lastTurnTick),
|
||||||
|
clockPhase: 'RUNNING',
|
||||||
|
clockRevision: 1n,
|
||||||
|
deadlineGeneration: 7n,
|
||||||
|
meta: {
|
||||||
|
lastTurnTime: clock.tickToDate(lastTurnTick).toISOString(),
|
||||||
|
starttime: clock.tickToDate(initialTick + 100).toISOString(),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
});
|
||||||
|
await db.general.createMany({
|
||||||
|
data: generalTicks.map((turnTick, index) => ({
|
||||||
|
id: index + 1,
|
||||||
|
name: `general-${index + 1}`,
|
||||||
|
turnTick: BigInt(turnTick),
|
||||||
|
turnTime: clock.tickToDate(turnTick),
|
||||||
|
recentWarTick: BigInt(initialTick - 100 - index),
|
||||||
|
recentWarTime: clock.tickToDate(initialTick - 100 - index),
|
||||||
|
})),
|
||||||
|
});
|
||||||
|
await db.auction.create({
|
||||||
|
data: {
|
||||||
|
type: 'BUY_RICE',
|
||||||
|
hostGeneralId: 1,
|
||||||
|
status: 'FINALIZING',
|
||||||
|
openTick: BigInt(initialTick - 300),
|
||||||
|
closeTick: BigInt(auctionCloseTick),
|
||||||
|
closeAt: clock.tickToDate(auctionCloseTick),
|
||||||
|
},
|
||||||
|
});
|
||||||
|
await db.message.create({
|
||||||
|
data: {
|
||||||
|
mailbox: 1,
|
||||||
|
type: 'private',
|
||||||
|
src: 1,
|
||||||
|
dest: 2,
|
||||||
|
time: clock.tickToDate(messageOccurrenceTick),
|
||||||
|
timeTick: BigInt(messageOccurrenceTick),
|
||||||
|
validUntil: clock.tickToDate(messageExpiryTick),
|
||||||
|
validUntilTick: BigInt(messageExpiryTick),
|
||||||
|
message: {},
|
||||||
|
},
|
||||||
|
});
|
||||||
|
await db.votePoll.create({
|
||||||
|
data: {
|
||||||
|
title: 'clock vote',
|
||||||
|
options: ['yes', 'no'],
|
||||||
|
revealMode: 'AFTER_VOTE',
|
||||||
|
openerGeneralId: 1,
|
||||||
|
openerName: 'general-1',
|
||||||
|
startAt: clock.tickToDate(voteStartTick),
|
||||||
|
startTick: BigInt(voteStartTick),
|
||||||
|
endAt: clock.tickToDate(voteEndTick),
|
||||||
|
endTick: BigInt(voteEndTick),
|
||||||
|
},
|
||||||
|
});
|
||||||
|
await db.selectPoolEntry.create({
|
||||||
|
data: {
|
||||||
|
uniqueName: 'clock-pool',
|
||||||
|
reservedUntil: clock.tickToDate(poolTick),
|
||||||
|
reservedUntilTick: BigInt(poolTick),
|
||||||
|
info: {},
|
||||||
|
},
|
||||||
|
});
|
||||||
|
await db.npcSelectionToken.create({
|
||||||
|
data: {
|
||||||
|
ownerUserId: 'clock-user',
|
||||||
|
validUntil: clock.tickToDate(npcValidTick),
|
||||||
|
validUntilTick: BigInt(npcValidTick),
|
||||||
|
pickMoreFrom: clock.tickToDate(npcMoreTick),
|
||||||
|
pickMoreFromTick: BigInt(npcMoreTick),
|
||||||
|
pickResult: [],
|
||||||
|
nonce: 1,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
const authority = { kind: 'OFFLINE' as const, profileName: 'clock-test', reason: 'integration fixture' };
|
||||||
|
const suspended = await startClockSuspension({
|
||||||
|
db,
|
||||||
|
suspensionId: 'clock-gap-65m17s250',
|
||||||
|
source: 'MAINTENANCE',
|
||||||
|
authority,
|
||||||
|
});
|
||||||
|
expect(suspended.cutTick).toBe(initialTick);
|
||||||
|
expect((await db.worldState.findUniqueOrThrow({ where: { id: world.id } })).clockPhase).toBe('SUSPENDED');
|
||||||
|
|
||||||
|
const resumeWallAt = new Date(suspended.cutWallAt.getTime() + 65 * 60_000 + 17_250);
|
||||||
|
const reconciled = await reconcileClockSuspension({
|
||||||
|
db,
|
||||||
|
suspensionId: suspended.suspensionId,
|
||||||
|
authority,
|
||||||
|
testResumeWallAt: resumeWallAt,
|
||||||
|
});
|
||||||
|
expect(reconciled).toMatchObject({
|
||||||
|
phase: 'RECONCILING',
|
||||||
|
sourceRevision: 1,
|
||||||
|
targetRevision: 2,
|
||||||
|
deadlineGeneration: 8,
|
||||||
|
gapTicks: 235_035_000,
|
||||||
|
shiftTicks: 235_035_000,
|
||||||
|
alignedTick: 236_035_000,
|
||||||
|
});
|
||||||
|
|
||||||
|
const [afterWorld, generals, auction, message, vote, pool, token, ledger, outboxes] = await Promise.all([
|
||||||
|
db.worldState.findUniqueOrThrow({ where: { id: world.id } }),
|
||||||
|
db.general.findMany({ orderBy: { id: 'asc' } }),
|
||||||
|
db.auction.findFirstOrThrow(),
|
||||||
|
db.message.findFirstOrThrow(),
|
||||||
|
db.votePoll.findFirstOrThrow(),
|
||||||
|
db.selectPoolEntry.findFirstOrThrow(),
|
||||||
|
db.npcSelectionToken.findFirstOrThrow(),
|
||||||
|
db.clockSuspension.findUniqueOrThrow({ where: { id: suspended.suspensionId } }),
|
||||||
|
db.clockProjectionOutbox.findMany(),
|
||||||
|
]);
|
||||||
|
const alignedTick = BigInt(reconciled.alignedTick);
|
||||||
|
expect(afterWorld).toMatchObject({
|
||||||
|
clockPhase: 'RECONCILING',
|
||||||
|
clockRevision: 2n,
|
||||||
|
deadlineGeneration: 8n,
|
||||||
|
clockTick: alignedTick,
|
||||||
|
lastTurnTick: BigInt(lastTurnTick + reconciled.shiftTicks),
|
||||||
|
});
|
||||||
|
expect(generals.map((general) => general.turnTick! - alignedTick)).toEqual(
|
||||||
|
generalTicks.map((tick) => BigInt(tick - initialTick))
|
||||||
|
);
|
||||||
|
expect(auction.closeTick! - alignedTick).toBe(BigInt(auctionCloseTick - initialTick));
|
||||||
|
expect(message.validUntilTick! - alignedTick).toBe(BigInt(messageExpiryTick - initialTick));
|
||||||
|
expect(vote.endTick! - alignedTick).toBe(BigInt(voteEndTick - initialTick));
|
||||||
|
expect(pool.reservedUntilTick! - alignedTick).toBe(BigInt(poolTick - initialTick));
|
||||||
|
expect(token.validUntilTick! - alignedTick).toBe(BigInt(npcValidTick - initialTick));
|
||||||
|
expect(token.pickMoreFromTick! - alignedTick).toBe(BigInt(npcMoreTick - initialTick));
|
||||||
|
expect(generals.map((general) => general.recentWarTick)).toEqual([
|
||||||
|
BigInt(initialTick - 100),
|
||||||
|
BigInt(initialTick - 101),
|
||||||
|
]);
|
||||||
|
expect(auction.openTick).toBe(BigInt(initialTick - 300));
|
||||||
|
expect(message.timeTick).toBe(BigInt(messageOccurrenceTick));
|
||||||
|
expect(vote.startTick).toBe(BigInt(voteStartTick));
|
||||||
|
expect(ledger.status).toBe('RECONCILING');
|
||||||
|
expect(outboxes).toHaveLength(1);
|
||||||
|
expect(outboxes[0]).toMatchObject({ status: 'PENDING', targetRevision: 2n });
|
||||||
|
|
||||||
|
const retried = await reconcileClockSuspension({
|
||||||
|
db,
|
||||||
|
suspensionId: suspended.suspensionId,
|
||||||
|
authority,
|
||||||
|
testResumeWallAt: new Date(resumeWallAt.getTime() + 10_000),
|
||||||
|
});
|
||||||
|
expect(retried).toEqual(reconciled);
|
||||||
|
expect(await db.clockProjectionOutbox.count()).toBe(1);
|
||||||
|
const keepParticipants = await db.clockReconciliationParticipant.findMany({ where: { policy: 'KEEP' } });
|
||||||
|
expect(keepParticipants.every((participant) => participant.beforeChecksum === participant.afterChecksum)).toBe(
|
||||||
|
true
|
||||||
|
);
|
||||||
|
|
||||||
|
await redis.client.set('sammo:clock-test:clock:active-revision', '1');
|
||||||
|
expect(
|
||||||
|
await applyNextClockProjection({ db, redis: redis.client, workerId: 'clock-projection-success' })
|
||||||
|
).toBe('APPLIED');
|
||||||
|
expect(await redis.client.get('sammo:clock-test:clock:active-revision')).toBe('2');
|
||||||
|
expect(await redis.client.get('sammo:clock-test:clock:deadline-generation')).toBe('8');
|
||||||
|
expect(await redis.client.get('sammo:clock-test:clock:phase')).toBe('RUNNING');
|
||||||
|
expect(await redis.client.zRangeWithScores('sammo:clock-test:auction:timer', 0, -1)).toEqual([
|
||||||
|
{ value: String(auction.id), score: Number(auction.closeTick) },
|
||||||
|
]);
|
||||||
|
expect(await db.worldState.findUniqueOrThrow({ where: { id: world.id } })).toMatchObject({
|
||||||
|
clockPhase: 'RUNNING',
|
||||||
|
clockRevision: 2n,
|
||||||
|
});
|
||||||
|
expect(await db.clockProjectionOutbox.findFirstOrThrow()).toMatchObject({ status: 'APPLIED' });
|
||||||
|
});
|
||||||
|
|
||||||
|
it('rejects a live offline fence and preserves a turn deadline across an exact 24-hour gap', async () => {
|
||||||
|
const baseTime = new Date('2026-02-01T00:00:00.000Z');
|
||||||
|
const futureAnchor = new Date(Date.now() + 3_600_000);
|
||||||
|
const initialTick = 5 * 36_000_000;
|
||||||
|
const turnTick = initialTick + 17_000_007;
|
||||||
|
const clock = new GameClock({
|
||||||
|
baseTime,
|
||||||
|
tick: initialTick,
|
||||||
|
mode: 'realtime',
|
||||||
|
wallAnchor: futureAnchor,
|
||||||
|
turnSeconds: 3_600,
|
||||||
|
phase: 'RUNNING',
|
||||||
|
});
|
||||||
|
await db.worldState.create({
|
||||||
|
data: {
|
||||||
|
scenarioCode: 'clock-day-test',
|
||||||
|
currentYear: 180,
|
||||||
|
currentMonth: 1,
|
||||||
|
tickSeconds: 3_600,
|
||||||
|
clockBaseTime: baseTime,
|
||||||
|
clockTick: BigInt(initialTick),
|
||||||
|
clockMode: 'realtime',
|
||||||
|
clockWallAnchor: futureAnchor,
|
||||||
|
lastTurnTick: BigInt(initialTick),
|
||||||
|
clockPhase: 'RUNNING',
|
||||||
|
clockRevision: 3n,
|
||||||
|
deadlineGeneration: 2n,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
await db.general.create({
|
||||||
|
data: { id: 1, name: 'day-general', turnTick: BigInt(turnTick), turnTime: clock.tickToDate(turnTick) },
|
||||||
|
});
|
||||||
|
await db.turnDaemonLease.create({
|
||||||
|
data: {
|
||||||
|
profile: 'clock-day-test',
|
||||||
|
ownerId: 'other-daemon',
|
||||||
|
fencingEpoch: 9n,
|
||||||
|
leaseUntil: new Date(Date.now() + 60_000),
|
||||||
|
},
|
||||||
|
});
|
||||||
|
const authority = {
|
||||||
|
kind: 'OFFLINE' as const,
|
||||||
|
profileName: 'clock-day-test',
|
||||||
|
reason: '24-hour integration fixture',
|
||||||
|
};
|
||||||
|
await expect(
|
||||||
|
startClockSuspension({
|
||||||
|
db,
|
||||||
|
suspensionId: 'clock-gap-24h',
|
||||||
|
source: 'MAINTENANCE',
|
||||||
|
authority,
|
||||||
|
})
|
||||||
|
).rejects.toThrow('daemon lease to be offline');
|
||||||
|
await db.turnDaemonLease.delete({ where: { profile: 'clock-day-test' } });
|
||||||
|
const suspended = await startClockSuspension({
|
||||||
|
db,
|
||||||
|
suspensionId: 'clock-gap-24h',
|
||||||
|
source: 'MAINTENANCE',
|
||||||
|
authority,
|
||||||
|
});
|
||||||
|
const reconciled = await reconcileClockSuspension({
|
||||||
|
db,
|
||||||
|
suspensionId: suspended.suspensionId,
|
||||||
|
authority,
|
||||||
|
testResumeWallAt: new Date(suspended.cutWallAt.getTime() + 24 * 60 * 60_000),
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(reconciled).toMatchObject({
|
||||||
|
sourceRevision: 3,
|
||||||
|
targetRevision: 4,
|
||||||
|
gapTicks: 24 * 36_000_000,
|
||||||
|
shiftTicks: 24 * 36_000_000,
|
||||||
|
alignedTick: initialTick + 24 * 36_000_000,
|
||||||
|
});
|
||||||
|
const shifted = await db.general.findUniqueOrThrow({ where: { id: 1 } });
|
||||||
|
expect(shifted.turnTick! - BigInt(reconciled.alignedTick)).toBe(BigInt(turnTick - initialTick));
|
||||||
|
|
||||||
|
await redis.client.set('sammo:clock-day-test:clock:active-revision', '3');
|
||||||
|
const redisThenCrash = {
|
||||||
|
get: redis.client.get.bind(redis.client),
|
||||||
|
eval: async (script: string, options: { keys: string[]; arguments: string[] }) => {
|
||||||
|
await redis.client.eval(script, options);
|
||||||
|
throw new Error('fixture crash after Redis commit');
|
||||||
|
},
|
||||||
|
};
|
||||||
|
await expect(
|
||||||
|
applyNextClockProjection({ db, redis: redisThenCrash, workerId: 'clock-projection-crash' })
|
||||||
|
).rejects.toThrow('fixture crash after Redis commit');
|
||||||
|
expect(await redis.client.get('sammo:clock-day-test:clock:active-revision')).toBe('4');
|
||||||
|
expect(await db.worldState.findFirstOrThrow()).toMatchObject({ clockPhase: 'RECONCILING' });
|
||||||
|
expect(await db.clockProjectionOutbox.findFirstOrThrow()).toMatchObject({ status: 'FAILED', attempts: 1 });
|
||||||
|
|
||||||
|
await db.clockProjectionOutbox.updateMany({ data: { availableAt: new Date(0) } });
|
||||||
|
expect(
|
||||||
|
await applyNextClockProjection({ db, redis: redis.client, workerId: 'clock-projection-restart' })
|
||||||
|
).toBe('RECOVERED');
|
||||||
|
expect(await db.worldState.findFirstOrThrow()).toMatchObject({ clockPhase: 'RUNNING', clockRevision: 4n });
|
||||||
|
expect(await db.clockProjectionOutbox.findFirstOrThrow()).toMatchObject({ status: 'APPLIED', attempts: 2 });
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest';
|
import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||||
|
|
||||||
import type { TurnDaemonCommand } from '@sammo-ts/common';
|
import type { TurnDaemonCommand } from '@sammo-ts/common';
|
||||||
import { createGamePostgresConnector } from '@sammo-ts/infra';
|
import { createGamePostgresConnector } from '@sammo-ts/infra';
|
||||||
@@ -25,6 +25,12 @@ integration('database command queue', () => {
|
|||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
beforeEach(async () => {
|
||||||
|
await db.inputEvent.deleteMany({ where: { requestId: { startsWith: 'integration:engine:' } } });
|
||||||
|
await db.clockSuspension.deleteMany({ where: { id: 'integration-queue-revision-8-9' } });
|
||||||
|
await db.worldState.updateMany({ data: { clockPhase: 'RUNNING' } });
|
||||||
|
});
|
||||||
|
|
||||||
afterAll(async () => {
|
afterAll(async () => {
|
||||||
await db.inputEvent.deleteMany({
|
await db.inputEvent.deleteMany({
|
||||||
where: { requestId: { startsWith: 'integration:engine:' } },
|
where: { requestId: { startsWith: 'integration:engine:' } },
|
||||||
@@ -229,4 +235,119 @@ integration('database command queue', () => {
|
|||||||
expect(handle).toHaveBeenCalledOnce();
|
expect(handle).toHaveBeenCalledOnce();
|
||||||
expect(mutation).not.toHaveBeenCalled();
|
expect(mutation).not.toHaveBeenCalled();
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('dequeues gameplay only in an executable phase and records the processing clock generation', async () => {
|
||||||
|
const existingWorld = await db.worldState.findFirst({ orderBy: { id: 'asc' } });
|
||||||
|
const world = existingWorld
|
||||||
|
? await db.worldState.update({
|
||||||
|
where: { id: existingWorld.id },
|
||||||
|
data: { clockPhase: 'SUSPENDED', clockRevision: 9n, deadlineGeneration: 4n, clockTick: 123n },
|
||||||
|
})
|
||||||
|
: await db.worldState.create({
|
||||||
|
data: {
|
||||||
|
scenarioCode: 'queue-clock-test',
|
||||||
|
currentYear: 180,
|
||||||
|
currentMonth: 1,
|
||||||
|
tickSeconds: 600,
|
||||||
|
clockPhase: 'SUSPENDED',
|
||||||
|
clockRevision: 9n,
|
||||||
|
deadlineGeneration: 4n,
|
||||||
|
clockTick: 123n,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
const gameplayId = 'integration:engine:clock-gated-gameplay';
|
||||||
|
const statusId = 'integration:engine:clock-gated-status';
|
||||||
|
const staleId = 'integration:engine:clock-gated-stale';
|
||||||
|
await db.inputEvent.createMany({
|
||||||
|
data: [
|
||||||
|
{
|
||||||
|
requestId: gameplayId,
|
||||||
|
target: 'ENGINE',
|
||||||
|
eventType: 'vacation',
|
||||||
|
actorUserId: 'user-7',
|
||||||
|
acceptedGameTick: 100n,
|
||||||
|
acceptedClockRevision: 9n,
|
||||||
|
acceptedDeadlineGeneration: 4n,
|
||||||
|
payload: { type: 'vacation', requestId: gameplayId, userId: 'user-7', generalId: 7 },
|
||||||
|
},
|
||||||
|
{
|
||||||
|
requestId: statusId,
|
||||||
|
target: 'ENGINE',
|
||||||
|
eventType: 'getStatus',
|
||||||
|
acceptedGameTick: 100n,
|
||||||
|
acceptedClockRevision: 9n,
|
||||||
|
acceptedDeadlineGeneration: 4n,
|
||||||
|
payload: { type: 'getStatus', requestId: statusId },
|
||||||
|
},
|
||||||
|
{
|
||||||
|
requestId: staleId,
|
||||||
|
target: 'ENGINE',
|
||||||
|
eventType: 'vacation',
|
||||||
|
actorUserId: 'user-8',
|
||||||
|
acceptedGameTick: 90n,
|
||||||
|
acceptedClockRevision: 8n,
|
||||||
|
acceptedDeadlineGeneration: 3n,
|
||||||
|
payload: { type: 'vacation', requestId: staleId, userId: 'user-8', generalId: 8 },
|
||||||
|
},
|
||||||
|
],
|
||||||
|
});
|
||||||
|
const queue = new DatabaseTurnDaemonCommandQueue(db);
|
||||||
|
|
||||||
|
expect(await queue.drain()).toEqual([{ type: 'getStatus', requestId: statusId }]);
|
||||||
|
expect(await db.inputEvent.findUniqueOrThrow({ where: { requestId: gameplayId } })).toMatchObject({
|
||||||
|
status: 'PENDING',
|
||||||
|
processingClockRevision: null,
|
||||||
|
});
|
||||||
|
await db.worldState.update({ where: { id: world.id }, data: { clockPhase: 'RUNNING' } });
|
||||||
|
|
||||||
|
expect(await queue.drain()).toEqual([
|
||||||
|
{ type: 'vacation', requestId: gameplayId, userId: 'user-7', generalId: 7 },
|
||||||
|
]);
|
||||||
|
expect(await db.inputEvent.findUniqueOrThrow({ where: { requestId: gameplayId } })).toMatchObject({
|
||||||
|
status: 'PROCESSING',
|
||||||
|
processingGameTick: 100n,
|
||||||
|
processingClockRevision: 9n,
|
||||||
|
processingDeadlineGeneration: 4n,
|
||||||
|
});
|
||||||
|
expect(await db.inputEvent.findUniqueOrThrow({ where: { requestId: staleId } })).toMatchObject({
|
||||||
|
status: 'PENDING',
|
||||||
|
processingClockRevision: null,
|
||||||
|
});
|
||||||
|
await db.clockSuspension.deleteMany({ where: { id: 'integration-queue-revision-8-9' } });
|
||||||
|
await db.clockSuspension.create({
|
||||||
|
data: {
|
||||||
|
id: 'integration-queue-revision-8-9',
|
||||||
|
worldStateId: world.id,
|
||||||
|
source: 'MAINTENANCE',
|
||||||
|
policy: 'EXACT',
|
||||||
|
status: 'APPLIED',
|
||||||
|
sourceRevision: 8n,
|
||||||
|
targetRevision: 9n,
|
||||||
|
cutTick: 90n,
|
||||||
|
cutWallAt: new Date(),
|
||||||
|
resumeWallAt: new Date(),
|
||||||
|
rateTicksPerSecond: 60_000,
|
||||||
|
gapTicks: 33n,
|
||||||
|
shiftTicks: 33n,
|
||||||
|
alignedTick: 123n,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
expect(await queue.drain()).toEqual([
|
||||||
|
{
|
||||||
|
type: 'vacation',
|
||||||
|
requestId: staleId,
|
||||||
|
userId: 'user-8',
|
||||||
|
generalId: 8,
|
||||||
|
processingGameTick: 123,
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
expect(await db.inputEvent.findUniqueOrThrow({ where: { requestId: staleId } })).toMatchObject({
|
||||||
|
status: 'PROCESSING',
|
||||||
|
acceptedGameTick: 90n,
|
||||||
|
acceptedClockRevision: 8n,
|
||||||
|
processingGameTick: 123n,
|
||||||
|
processingClockRevision: 9n,
|
||||||
|
processingDeadlineGeneration: 4n,
|
||||||
|
});
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -448,6 +448,21 @@ describe('runtime clock shift projection', () => {
|
|||||||
return {};
|
return {};
|
||||||
});
|
});
|
||||||
const db = {
|
const db = {
|
||||||
|
$transaction: async (operation: (transaction: GamePrismaClient) => Promise<unknown>) => operation(db),
|
||||||
|
$executeRaw: vi.fn(async () => 1),
|
||||||
|
$queryRaw: vi.fn(async () => [{ wallNow: new Date('2026-07-30T10:00:00.000Z') }]),
|
||||||
|
worldState: {
|
||||||
|
findFirst: vi.fn(async () => ({
|
||||||
|
clockBaseTime: new Date('2026-07-30T10:00:00.000Z'),
|
||||||
|
clockTick: 0n,
|
||||||
|
clockMode: 'realtime',
|
||||||
|
clockWallAnchor: new Date('2026-07-30T10:00:00.000Z'),
|
||||||
|
tickSeconds: 600,
|
||||||
|
clockPhase: 'RUNNING',
|
||||||
|
clockRevision: 1n,
|
||||||
|
deadlineGeneration: 1n,
|
||||||
|
})),
|
||||||
|
},
|
||||||
inputEvent: {
|
inputEvent: {
|
||||||
create: inputEventCreate,
|
create: inputEventCreate,
|
||||||
findUniqueOrThrow: vi.fn(async () =>
|
findUniqueOrThrow: vi.fn(async () =>
|
||||||
@@ -567,6 +582,21 @@ describe('runtime game settings projection', () => {
|
|||||||
let eventStatus: 'PENDING' | 'SUCCEEDED' = 'PENDING';
|
let eventStatus: 'PENDING' | 'SUCCEEDED' = 'PENDING';
|
||||||
let created = false;
|
let created = false;
|
||||||
const db = {
|
const db = {
|
||||||
|
$transaction: async (operation: (transaction: GamePrismaClient) => Promise<unknown>) => operation(db),
|
||||||
|
$executeRaw: vi.fn(async () => 1),
|
||||||
|
$queryRaw: vi.fn(async () => [{ wallNow: new Date('2026-07-30T10:00:00.000Z') }]),
|
||||||
|
worldState: {
|
||||||
|
findFirst: vi.fn(async () => ({
|
||||||
|
clockBaseTime: new Date('2026-07-30T10:00:00.000Z'),
|
||||||
|
clockTick: 0n,
|
||||||
|
clockMode: 'realtime',
|
||||||
|
clockWallAnchor: new Date('2026-07-30T10:00:00.000Z'),
|
||||||
|
tickSeconds: 600,
|
||||||
|
clockPhase: 'RUNNING',
|
||||||
|
clockRevision: 1n,
|
||||||
|
deadlineGeneration: 1n,
|
||||||
|
})),
|
||||||
|
},
|
||||||
inputEvent: {
|
inputEvent: {
|
||||||
create: vi.fn(async () => {
|
create: vi.fn(async () => {
|
||||||
if (created) throw { code: 'P2002' };
|
if (created) throw { code: 'P2002' };
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
import { afterAll, beforeAll, describe, expect, it } from 'vitest';
|
import { afterAll, beforeAll, beforeEach, describe, expect, it } from 'vitest';
|
||||||
|
|
||||||
import { createGamePostgresConnector, type GamePrisma, type GamePrismaClient } from '@sammo-ts/infra';
|
import { createGamePostgresConnector, type GamePrisma, type GamePrismaClient } from '@sammo-ts/infra';
|
||||||
import { GAME_TICKS_PER_TURN } from '@sammo-ts/common';
|
import { GAME_TICKS_PER_TURN } from '@sammo-ts/common';
|
||||||
@@ -77,38 +77,32 @@ integration('runtime clock shift persistence', () => {
|
|||||||
let db: GamePrismaClient;
|
let db: GamePrismaClient;
|
||||||
let closeDb: (() => Promise<void>) | undefined;
|
let closeDb: (() => Promise<void>) | undefined;
|
||||||
|
|
||||||
|
const cleanupFixtures = async (): Promise<void> => {
|
||||||
|
await db.inputEvent.deleteMany({ where: { requestId: { in: [requestId, runtimeSettingsRequestId] } } });
|
||||||
|
await db.votePoll.deleteMany({ where: { openerGeneralId: generalIds[2] } });
|
||||||
|
await db.message.deleteMany({ where: { mailbox: generalIds[2] } });
|
||||||
|
await db.logEntry.deleteMany({ where: { text: runtimeSettingsLogText } });
|
||||||
|
await db.auction.deleteMany({ where: { hostGeneralId: { in: [...generalIds] } } });
|
||||||
|
await db.selectPoolEntry.deleteMany({ where: { uniqueName: backlogPoolUniqueName } });
|
||||||
|
await db.general.deleteMany({ where: { id: { in: [...generalIds] } } });
|
||||||
|
await db.worldState.deleteMany({
|
||||||
|
where: {
|
||||||
|
scenarioCode: { in: ['runtime-clock-shift', 'runtime-game-settings', 'realtime-backlog-rebase'] },
|
||||||
|
},
|
||||||
|
});
|
||||||
|
};
|
||||||
|
|
||||||
beforeAll(async () => {
|
beforeAll(async () => {
|
||||||
const connector = createGamePostgresConnector({ url: databaseUrl! });
|
const connector = createGamePostgresConnector({ url: databaseUrl! });
|
||||||
await connector.connect();
|
await connector.connect();
|
||||||
db = connector.prisma;
|
db = connector.prisma;
|
||||||
closeDb = () => connector.disconnect();
|
closeDb = () => connector.disconnect();
|
||||||
await db.inputEvent.deleteMany({ where: { requestId: { in: [requestId, runtimeSettingsRequestId] } } });
|
|
||||||
await db.votePoll.deleteMany({ where: { openerGeneralId: generalIds[2] } });
|
|
||||||
await db.message.deleteMany({ where: { mailbox: generalIds[2] } });
|
|
||||||
await db.logEntry.deleteMany({ where: { text: runtimeSettingsLogText } });
|
|
||||||
await db.auction.deleteMany({ where: { hostGeneralId: { in: [...generalIds] } } });
|
|
||||||
await db.selectPoolEntry.deleteMany({ where: { uniqueName: backlogPoolUniqueName } });
|
|
||||||
await db.general.deleteMany({ where: { id: { in: [...generalIds] } } });
|
|
||||||
await db.worldState.deleteMany({
|
|
||||||
where: {
|
|
||||||
scenarioCode: { in: ['runtime-clock-shift', 'runtime-game-settings', 'realtime-backlog-rebase'] },
|
|
||||||
},
|
|
||||||
});
|
|
||||||
});
|
});
|
||||||
|
|
||||||
|
beforeEach(cleanupFixtures);
|
||||||
|
|
||||||
afterAll(async () => {
|
afterAll(async () => {
|
||||||
await db.inputEvent.deleteMany({ where: { requestId: { in: [requestId, runtimeSettingsRequestId] } } });
|
await cleanupFixtures();
|
||||||
await db.votePoll.deleteMany({ where: { openerGeneralId: generalIds[2] } });
|
|
||||||
await db.message.deleteMany({ where: { mailbox: generalIds[2] } });
|
|
||||||
await db.logEntry.deleteMany({ where: { text: runtimeSettingsLogText } });
|
|
||||||
await db.auction.deleteMany({ where: { hostGeneralId: { in: [...generalIds] } } });
|
|
||||||
await db.selectPoolEntry.deleteMany({ where: { uniqueName: backlogPoolUniqueName } });
|
|
||||||
await db.general.deleteMany({ where: { id: { in: [...generalIds] } } });
|
|
||||||
await db.worldState.deleteMany({
|
|
||||||
where: {
|
|
||||||
scenarioCode: { in: ['runtime-clock-shift', 'runtime-game-settings', 'realtime-backlog-rebase'] },
|
|
||||||
},
|
|
||||||
});
|
|
||||||
await closeDb?.();
|
await closeDb?.();
|
||||||
});
|
});
|
||||||
|
|
||||||
@@ -469,6 +463,7 @@ integration('runtime clock shift persistence', () => {
|
|||||||
clockBaseTime: base,
|
clockBaseTime: base,
|
||||||
clockTick: 0,
|
clockTick: 0,
|
||||||
clockMode: 'manual',
|
clockMode: 'manual',
|
||||||
|
clockPhase: 'MANUAL',
|
||||||
clockWallAnchor: base,
|
clockWallAnchor: base,
|
||||||
lastTurnTick: 0,
|
lastTurnTick: 0,
|
||||||
config: { turnTermMinutes: 10, blockGeneralCreate: 0 },
|
config: { turnTermMinutes: 10, blockGeneralCreate: 0 },
|
||||||
|
|||||||
@@ -39,7 +39,7 @@ describe('readReleaseManifest', () => {
|
|||||||
await expect(readReleaseManifest(workspaceRoot)).resolves.toMatchObject({
|
await expect(readReleaseManifest(workspaceRoot)).resolves.toMatchObject({
|
||||||
controllerProtocol: RELEASE_CONTROLLER_PROTOCOL,
|
controllerProtocol: RELEASE_CONTROLLER_PROTOCOL,
|
||||||
gatewaySchemaHead: '20260825000000_add_bulk_release_batches',
|
gatewaySchemaHead: '20260825000000_add_bulk_release_batches',
|
||||||
gameSchemaHead: '20260903090000_add_game_clock_reconciliation',
|
gameSchemaHead: '20260903103000_add_input_event_clock_processing',
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|||||||
@@ -6,6 +6,10 @@
|
|||||||
"coveredFields": [
|
"coveredFields": [
|
||||||
"input_event.accepted_game_tick",
|
"input_event.accepted_game_tick",
|
||||||
"input_event.accepted_clock_revision",
|
"input_event.accepted_clock_revision",
|
||||||
|
"input_event.accepted_deadline_generation",
|
||||||
|
"input_event.processing_game_tick",
|
||||||
|
"input_event.processing_clock_revision",
|
||||||
|
"input_event.processing_deadline_generation",
|
||||||
"world_state.clock_tick",
|
"world_state.clock_tick",
|
||||||
"world_state.last_turn_tick",
|
"world_state.last_turn_tick",
|
||||||
"world_state.clock_revision",
|
"world_state.clock_revision",
|
||||||
@@ -125,8 +129,16 @@
|
|||||||
{
|
{
|
||||||
"key": "accepted-command-coordinate",
|
"key": "accepted-command-coordinate",
|
||||||
"policy": "KEEP",
|
"policy": "KEEP",
|
||||||
"authorityFields": ["input_event.accepted_game_tick", "input_event.accepted_clock_revision"],
|
"authorityFields": [
|
||||||
"projectionFields": [],
|
"input_event.accepted_game_tick",
|
||||||
|
"input_event.accepted_clock_revision",
|
||||||
|
"input_event.accepted_deadline_generation"
|
||||||
|
],
|
||||||
|
"projectionFields": [
|
||||||
|
"input_event.processing_game_tick",
|
||||||
|
"input_event.processing_clock_revision",
|
||||||
|
"input_event.processing_deadline_generation"
|
||||||
|
],
|
||||||
"owner": "game-api/input-event"
|
"owner": "game-api/input-event"
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -139,11 +151,15 @@
|
|||||||
},
|
},
|
||||||
{
|
{
|
||||||
"key": "movable-json-rule-anchors",
|
"key": "movable-json-rule-anchors",
|
||||||
"policy": "FORBID",
|
"policy": "SHIFT",
|
||||||
"authorityFields": ["world_state.meta.turntime", "world_state.meta.starttime", "world_state.meta.tnmt_time"],
|
"authorityFields": [
|
||||||
|
"world_state.meta.turntime",
|
||||||
|
"world_state.meta.starttime",
|
||||||
|
"world_state.meta.tnmt_time"
|
||||||
|
],
|
||||||
"projectionFields": [],
|
"projectionFields": [],
|
||||||
"owner": "game-engine/world-meta",
|
"owner": "game-engine/clock-operation",
|
||||||
"migration": "Register typed columns or explicit participant adapters before exact reconciliation can complete."
|
"migration": "Explicit adapter shifts registered ISO projections while typed tick columns remain authoritative."
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"key": "unification-wait",
|
"key": "unification-wait",
|
||||||
@@ -179,12 +195,12 @@
|
|||||||
{
|
{
|
||||||
"keyPattern": "sammo:{profile}:auction:timer",
|
"keyPattern": "sammo:{profile}:auction:timer",
|
||||||
"policy": "REBUILD",
|
"policy": "REBUILD",
|
||||||
"status": "implemented-without-clock-revision-fence"
|
"status": "implemented-with-clock-revision-phase-generation-fence"
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"keyPattern": "sammo:{profile}:tournament:state",
|
"keyPattern": "sammo:{profile}:tournament:state",
|
||||||
"policy": "REBUILD",
|
"policy": "REBUILD",
|
||||||
"status": "legacy-date-dual-write-required"
|
"status": "implemented-with-tick-dual-write-and-clock-fence"
|
||||||
}
|
}
|
||||||
],
|
],
|
||||||
"wallOnly": [
|
"wallOnly": [
|
||||||
|
|||||||
@@ -65,6 +65,19 @@ pop is one Redis operation: verify revision/phase, read `-inf..nowTick`, and
|
|||||||
remove the claimed members. A failed Redis rebuild therefore leaves the game in
|
remove the claimed members. A failed Redis rebuild therefore leaves the game in
|
||||||
`RECONCILING`; process liveness alone is not readiness.
|
`RECONCILING`; process liveness alone is not readiness.
|
||||||
|
|
||||||
|
`applyNextClockProjection()` claims rows with `FOR UPDATE SKIP LOCKED` and uses
|
||||||
|
PostgreSQL UTC wall time for claim/retry timestamps. One Redis Lua operation
|
||||||
|
compares the active source revision, rebuilds the auction timer, conditionally
|
||||||
|
replaces the exact tournament source snapshot, and writes target revision plus
|
||||||
|
deadline generation. The DB finalizer then re-acquires the clock-operation lock
|
||||||
|
and changes `RECONCILING -> RUNNING` only when target revision and generation
|
||||||
|
still match. If the process dies after the Lua commit, retry observes the
|
||||||
|
already-active target revision and performs only the DB finalizer.
|
||||||
|
|
||||||
|
An active legacy tournament containing only `nextAt`/`bettingCloseAt` is a
|
||||||
|
fail-closed migration boundary. Reconciliation remains incomplete until its
|
||||||
|
authoritative `nextTick`/`bettingCloseTick` dual-write exists.
|
||||||
|
|
||||||
## Lock order
|
## Lock order
|
||||||
|
|
||||||
All mutation paths use this order:
|
All mutation paths use this order:
|
||||||
@@ -101,7 +114,9 @@ future anchored realtime profiles as `PREOPEN`, and other profiles as
|
|||||||
`RUNNING`. Existing DateTime columns remain projections while tick columns are
|
`RUNNING`. Existing DateTime columns remain projections while tick columns are
|
||||||
authoritative.
|
authoritative.
|
||||||
|
|
||||||
Exact reconciliation stays disabled while any registry participant is
|
Exact reconciliation stays disabled while an active registry participant is
|
||||||
`FORBID`. In particular, Redis-only tournament dates and unification wait must
|
`FORBID`. Tournament writes now carry tick/revision/generation coordinates and
|
||||||
be moved to durable tick/revision contracts before the operation can reach
|
are revision-fenced in Redis. The remaining unification wait participant must
|
||||||
`RUNNING`. Removing these guards to make a partial operation pass is prohibited.
|
be moved from its `lastTurnTime` workaround to a durable suspension before that
|
||||||
|
workflow can reach `RUNNING`. Removing this guard to make a partial operation
|
||||||
|
pass is prohibited.
|
||||||
|
|||||||
@@ -19,31 +19,32 @@ mean deployment or production validation.
|
|||||||
- [x] Suspension, participant-checksum, and Redis projection outbox tables.
|
- [x] Suspension, participant-checksum, and Redis projection outbox tables.
|
||||||
- [x] Machine-readable DB/Redis/JSON participant inventory and architecture gate.
|
- [x] Machine-readable DB/Redis/JSON participant inventory and architecture gate.
|
||||||
- [x] Turn flush lock prefix and phase/revision/generation fence.
|
- [x] Turn flush lock prefix and phase/revision/generation fence.
|
||||||
- [ ] Empty and upgraded database migration execution evidence.
|
- [x] Empty and upgraded database migration execution evidence.
|
||||||
|
|
||||||
## Milestone 2 - exact DB reconciliation
|
## Milestone 2 - exact DB reconciliation
|
||||||
|
|
||||||
- [ ] Suspension start command with DB wall time and idempotent source revision.
|
- [x] Suspension start command with DB wall time and idempotent source revision.
|
||||||
- [ ] Exact resume plan transaction with deterministic participant lock order.
|
- [x] Exact resume plan transaction with deterministic participant lock order.
|
||||||
- [ ] SHIFT adapters for cursor, generals, active auctions, message expiry, vote
|
- [x] SHIFT adapters for cursor, generals, active auctions, message expiry, vote
|
||||||
end, select pool, and NPC selection windows.
|
end, select pool, and NPC selection windows.
|
||||||
- [ ] KEEP checksum adapters for occurrences and history.
|
- [x] KEEP checksum adapters for occurrences and accepted command coordinates.
|
||||||
- [ ] Explicit `LEGACY_COMPLETE_TURNS` and bounded `CATCH_UP` policies.
|
- [x] Explicit `LEGACY_COMPLETE_TURNS` and bounded `CATCH_UP` policies.
|
||||||
- [ ] Property tests for remaining distance, ordering, and history invariants.
|
- [x] Property tests for remaining distance, ordering, and history invariants.
|
||||||
- [ ] 24-hour and 65m17.250s PostgreSQL integration evidence.
|
- [x] 24-hour and 65m17.250s PostgreSQL integration evidence.
|
||||||
|
|
||||||
## Milestone 3 - revisioned Redis and workers
|
## Milestone 3 - revisioned Redis and workers
|
||||||
|
|
||||||
- [ ] Projection outbox claimer/retry/recovery state machine.
|
- [x] Projection outbox claimer/retry/recovery state machine.
|
||||||
- [ ] Redis active revision and atomic due-pop script.
|
- [x] Redis active revision and atomic due-pop script.
|
||||||
- [ ] Auction OPEN/FINALIZING revision and generation fence.
|
- [x] Auction OPEN/FINALIZING revision and generation fence.
|
||||||
- [ ] Tournament durable tick dual-write and projection rebuild.
|
- [x] Tournament durable tick dual-write and projection rebuild.
|
||||||
- [ ] DB-commit/Redis-failure restart tests and readiness integration.
|
- [x] DB-commit/Redis-failure crash-restart test.
|
||||||
|
- [x] Readiness integration in Gateway/API process health.
|
||||||
|
|
||||||
## Milestone 4 - command and lifecycle workflows
|
## Milestone 4 - command and lifecycle workflows
|
||||||
|
|
||||||
- [ ] All durable input events record accepted tick and accepted revision.
|
- [x] All durable input events record accepted tick and accepted revision.
|
||||||
- [ ] Processing converts accepted coordinates across revisions or fails closed.
|
- [x] Processing converts accepted coordinates across revisions or fails closed.
|
||||||
- [ ] Gateway pause/resume/open orchestration writes the DB clock phase.
|
- [ ] Gateway pause/resume/open orchestration writes the DB clock phase.
|
||||||
- [ ] Unification wait becomes a durable `UNIFICATION_WAIT` suspension.
|
- [ ] Unification wait becomes a durable `UNIFICATION_WAIT` suspension.
|
||||||
- [ ] Alignment, optional rate change, invader IDs/RNG, creation, first schedule,
|
- [ ] Alignment, optional rate change, invader IDs/RNG, creation, first schedule,
|
||||||
@@ -57,7 +58,7 @@ mean deployment or production validation.
|
|||||||
- [ ] Dedicated PostgreSQL/Redis conditional integration suite with skip count
|
- [ ] Dedicated PostgreSQL/Redis conditional integration suite with skip count
|
||||||
recorded.
|
recorded.
|
||||||
- [ ] Recovery runbook exercised from each incomplete status.
|
- [ ] Recovery runbook exercised from each incomplete status.
|
||||||
- [ ] Admin status/readiness exposes revision, phase, participant checksums, and
|
- [x] Admin status/readiness exposes revision, phase, participant checksums, and
|
||||||
incomplete outbox state.
|
incomplete outbox state.
|
||||||
- [ ] User-test deployment evidence is recorded separately from Git push.
|
- [ ] User-test deployment evidence is recorded separately from Git push.
|
||||||
- [ ] All `FORBID` inventory entries are removed by typed migrations or proven
|
- [ ] All `FORBID` inventory entries are removed by typed migrations or proven
|
||||||
@@ -78,4 +79,25 @@ mean deployment or production validation.
|
|||||||
- `pnpm check:architecture`: package boundaries passed; 21 authoritative clock
|
- `pnpm check:architecture`: package boundaries passed; 21 authoritative clock
|
||||||
fields and 18 participants were registered.
|
fields and 18 participants were registered.
|
||||||
- Migration SQL was generated, formatted, validated, and registered as the
|
- Migration SQL was generated, formatted, validated, and registered as the
|
||||||
release manifest head. Empty/upgraded PostgreSQL execution is still pending.
|
release manifest head. All 43 migrations applied to the empty dedicated
|
||||||
|
PostgreSQL instance. A second schema upgraded from the previous head with an
|
||||||
|
existing manual `world_state` row and verified `MANUAL:1:1` plus all three
|
||||||
|
reconciliation tables.
|
||||||
|
- Dedicated PostgreSQL/Redis fixtures proved exact 24-hour and 65m17.250s gaps,
|
||||||
|
schedule distance/order preservation, occurrence checksums, live-lease
|
||||||
|
fencing, a single durable outbox, Redis target revision, and recovery after a
|
||||||
|
crash between Redis commit and DB finalization.
|
||||||
|
|
||||||
|
### 2026-09-03 - revisioned workers and input coordinates
|
||||||
|
|
||||||
|
- `CI=1 TURBO_CONCURRENCY=1 pnpm typecheck`: 21/21 tasks passed after the
|
||||||
|
worker/input contract changes.
|
||||||
|
- `CI=1 TURBO_CONCURRENCY=1 pnpm test`: 12/12 package tasks passed.
|
||||||
|
- Dedicated PostgreSQL runs passed `databaseCommandQueue.integration.test.ts`
|
||||||
|
6/6 and `runtimeClockShiftPersistence.integration.test.ts` 3/3 when executed
|
||||||
|
sequentially; the latter now clears each single-world fixture before the next
|
||||||
|
case. Dedicated Redis passed tournament source revision 4/4, including an
|
||||||
|
atomic stale-clock rejection. API input-event integration passed 13/13.
|
||||||
|
- The 24-hour/65m17.250s reconciliation suite passed 2/2 after the other DB
|
||||||
|
suites. Conditional files share a deliberately dedicated schema and are run
|
||||||
|
sequentially to prevent their fixture cleanup from racing another file.
|
||||||
|
|||||||
@@ -28,6 +28,18 @@ service. The service must re-read participant checksums and either return the
|
|||||||
already-applied result or resume the pending outbox. Never create a replacement
|
already-applied result or resume the pending outbox. Never create a replacement
|
||||||
revision to hide a failed target revision.
|
revision to hide a failed target revision.
|
||||||
|
|
||||||
|
When an outbox row is `FAILED`, the profile must remain `RECONCILING`. A retry
|
||||||
|
is safe in both crash locations:
|
||||||
|
|
||||||
|
- before Redis commit, the Lua operation reapplies from the source revision;
|
||||||
|
- after Redis commit but before DB finalization, the Lua operation returns the
|
||||||
|
already-active target result and DB finalization resumes without shifting any
|
||||||
|
tournament deadline twice.
|
||||||
|
|
||||||
|
`lastError` naming a legacy tournament deadline means the tournament lacks its
|
||||||
|
tick dual-write. Do not force the active revision; migrate or prove the
|
||||||
|
tournament inactive, then retry the same outbox.
|
||||||
|
|
||||||
## Rollback
|
## Rollback
|
||||||
|
|
||||||
There is no blind inverse update. Before enabling exact reconciliation in an
|
There is no blind inverse update. Before enabling exact reconciliation in an
|
||||||
|
|||||||
@@ -17,8 +17,8 @@ export type ScheduleInstant = GameTick & { readonly [scheduleInstantBrand]: 'Sch
|
|||||||
export type ClockRevision = number & { readonly [clockRevisionBrand]: 'ClockRevision' };
|
export type ClockRevision = number & { readonly [clockRevisionBrand]: 'ClockRevision' };
|
||||||
export type WallInstant = Date & { readonly [wallInstantBrand]: 'WallInstant' };
|
export type WallInstant = Date & { readonly [wallInstantBrand]: 'WallInstant' };
|
||||||
|
|
||||||
export interface ExactClockAlignmentPlan {
|
export interface ClockAlignmentPlan {
|
||||||
policy: 'EXACT';
|
policy: ClockAlignmentPolicy;
|
||||||
sourceRevision: ClockRevision;
|
sourceRevision: ClockRevision;
|
||||||
targetRevision: ClockRevision;
|
targetRevision: ClockRevision;
|
||||||
cutTick: GameTick;
|
cutTick: GameTick;
|
||||||
@@ -84,6 +84,15 @@ export const parseGameClockPhase = (value: string): GameClockPhase => {
|
|||||||
throw new Error(`Unknown game clock phase: ${value}`);
|
throw new Error(`Unknown game clock phase: ${value}`);
|
||||||
};
|
};
|
||||||
|
|
||||||
|
const CLOCK_ALIGNMENT_POLICIES: readonly ClockAlignmentPolicy[] = ['EXACT', 'LEGACY_COMPLETE_TURNS', 'CATCH_UP'];
|
||||||
|
|
||||||
|
export const parseClockAlignmentPolicy = (value: string): ClockAlignmentPolicy => {
|
||||||
|
if ((CLOCK_ALIGNMENT_POLICIES as readonly string[]).includes(value)) {
|
||||||
|
return value as ClockAlignmentPolicy;
|
||||||
|
}
|
||||||
|
throw new Error(`Unknown clock alignment policy: ${value}`);
|
||||||
|
};
|
||||||
|
|
||||||
export const scheduleNotBefore = (instant: ObservedGameInstant, phase: GameClockPhase): ScheduleInstant => {
|
export const scheduleNotBefore = (instant: ObservedGameInstant, phase: GameClockPhase): ScheduleInstant => {
|
||||||
if (phase === 'PREOPEN') {
|
if (phase === 'PREOPEN') {
|
||||||
return asScheduleInstant(Math.max(0, instant));
|
return asScheduleInstant(Math.max(0, instant));
|
||||||
@@ -103,14 +112,15 @@ export const assertGameplayCommitAllowed = (phase: GameClockPhase): void => {
|
|||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
export const buildExactClockAlignmentPlan = (input: {
|
const buildAlignmentPlan = (input: {
|
||||||
|
policy: ClockAlignmentPolicy;
|
||||||
sourceRevision: number;
|
sourceRevision: number;
|
||||||
cutTick: number;
|
cutTick: number;
|
||||||
cutWall: Date;
|
cutWall: Date;
|
||||||
resumeWall: Date;
|
resumeWall: Date;
|
||||||
ticksPerSecond: number;
|
ticksPerSecond: number;
|
||||||
catchUpTicks?: number;
|
catchUpTicks?: number;
|
||||||
}): ExactClockAlignmentPlan => {
|
}): ClockAlignmentPlan => {
|
||||||
const sourceRevision = asClockRevision(input.sourceRevision);
|
const sourceRevision = asClockRevision(input.sourceRevision);
|
||||||
const cutTick = asGameTick(input.cutTick);
|
const cutTick = asGameTick(input.cutTick);
|
||||||
const cutWall = asWallInstant(input.cutWall);
|
const cutWall = asWallInstant(input.cutWall);
|
||||||
@@ -136,7 +146,7 @@ export const buildExactClockAlignmentPlan = (input: {
|
|||||||
}
|
}
|
||||||
const shiftTicks = asGameTick(gapTicks - catchUpTicks);
|
const shiftTicks = asGameTick(gapTicks - catchUpTicks);
|
||||||
return {
|
return {
|
||||||
policy: 'EXACT',
|
policy: input.policy,
|
||||||
sourceRevision,
|
sourceRevision,
|
||||||
targetRevision: asClockRevision(sourceRevision + 1),
|
targetRevision: asClockRevision(sourceRevision + 1),
|
||||||
cutTick,
|
cutTick,
|
||||||
@@ -147,6 +157,37 @@ export const buildExactClockAlignmentPlan = (input: {
|
|||||||
};
|
};
|
||||||
};
|
};
|
||||||
|
|
||||||
|
export const buildExactClockAlignmentPlan = (
|
||||||
|
input: Omit<Parameters<typeof buildAlignmentPlan>[0], 'policy'>
|
||||||
|
): ClockAlignmentPlan => buildAlignmentPlan({ ...input, policy: 'EXACT' });
|
||||||
|
|
||||||
|
export const buildClockAlignmentPlan = (input: {
|
||||||
|
policy: ClockAlignmentPolicy;
|
||||||
|
sourceRevision: number;
|
||||||
|
cutTick: number;
|
||||||
|
cutWall: Date;
|
||||||
|
resumeWall: Date;
|
||||||
|
ticksPerSecond: number;
|
||||||
|
catchUpTicks?: number;
|
||||||
|
}): ClockAlignmentPlan => {
|
||||||
|
if (input.policy === 'EXACT') {
|
||||||
|
if ((input.catchUpTicks ?? 0) !== 0) {
|
||||||
|
throw new Error('EXACT alignment does not allow catch-up ticks.');
|
||||||
|
}
|
||||||
|
return buildAlignmentPlan({ ...input, policy: 'EXACT', catchUpTicks: 0 });
|
||||||
|
}
|
||||||
|
if (input.policy === 'CATCH_UP') {
|
||||||
|
return buildAlignmentPlan({ ...input, policy: 'CATCH_UP' });
|
||||||
|
}
|
||||||
|
const exact = buildAlignmentPlan({ ...input, policy: 'LEGACY_COMPLETE_TURNS', catchUpTicks: 0 });
|
||||||
|
const shiftTicks = asGameTick(Math.floor(exact.gapTicks / GAME_TICKS_PER_TURN) * GAME_TICKS_PER_TURN);
|
||||||
|
return {
|
||||||
|
...exact,
|
||||||
|
catchUpTicks: asGameTick(exact.gapTicks - shiftTicks),
|
||||||
|
shiftTicks,
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
const tickOffsetMilliseconds = (tick: number, ticksPerSecond: number): number => {
|
const tickOffsetMilliseconds = (tick: number, ticksPerSecond: number): number => {
|
||||||
const wholeSeconds = Math.floor(tick / ticksPerSecond);
|
const wholeSeconds = Math.floor(tick / ticksPerSecond);
|
||||||
const remainingTicks = tick - wholeSeconds * ticksPerSecond;
|
const remainingTicks = tick - wholeSeconds * ticksPerSecond;
|
||||||
|
|||||||
@@ -18,6 +18,15 @@ export interface TournamentProjectionWrite {
|
|||||||
value: unknown;
|
value: unknown;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export interface TournamentClockFence {
|
||||||
|
activeRevisionKey: string;
|
||||||
|
deadlineGenerationKey: string;
|
||||||
|
phaseKey: string;
|
||||||
|
revision: number;
|
||||||
|
deadlineGeneration: number;
|
||||||
|
phase: 'RUNNING';
|
||||||
|
}
|
||||||
|
|
||||||
const WRITE_TOURNAMENT_PROJECTION_SCRIPT = `
|
const WRITE_TOURNAMENT_PROJECTION_SCRIPT = `
|
||||||
local revision_key = KEYS[#KEYS]
|
local revision_key = KEYS[#KEYS]
|
||||||
local current = redis.call('GET', revision_key)
|
local current = redis.call('GET', revision_key)
|
||||||
@@ -55,6 +64,49 @@ local revision = redis.call('INCR', revision_key)
|
|||||||
return tostring(revision) .. ':' .. (stage_changed and '1' or '0') .. ':' .. (rankings_changed and '1' or '0')
|
return tostring(revision) .. ':' .. (stage_changed and '1' or '0') .. ':' .. (rankings_changed and '1' or '0')
|
||||||
`;
|
`;
|
||||||
|
|
||||||
|
const WRITE_FENCED_TOURNAMENT_PROJECTION_SCRIPT = `
|
||||||
|
local revision_key_index = #KEYS - 3
|
||||||
|
if redis.call('GET', KEYS[#KEYS - 2]) ~= ARGV[#ARGV - 2]
|
||||||
|
or redis.call('GET', KEYS[#KEYS - 1]) ~= ARGV[#ARGV - 1]
|
||||||
|
or redis.call('GET', KEYS[#KEYS]) ~= ARGV[#ARGV] then
|
||||||
|
return '__CLOCK_FENCE__'
|
||||||
|
end
|
||||||
|
local revision_key = KEYS[revision_key_index]
|
||||||
|
local current = redis.call('GET', revision_key)
|
||||||
|
if current then
|
||||||
|
if not string.match(current, '^%d+$') then
|
||||||
|
return redis.error_reply('invalid tournament source revision')
|
||||||
|
end
|
||||||
|
if string.len(current) > 18 then
|
||||||
|
return redis.error_reply('tournament source revision exhausted')
|
||||||
|
end
|
||||||
|
end
|
||||||
|
local stage_changed = false
|
||||||
|
local rankings_changed = false
|
||||||
|
for index = 1, revision_key_index - 1 do
|
||||||
|
local next_ok, next_value = pcall(cjson.decode, ARGV[index])
|
||||||
|
if next_ok and type(next_value) == 'table' and next_value['stage'] ~= nil then
|
||||||
|
local previous = redis.call('GET', KEYS[index])
|
||||||
|
local previous_stage = nil
|
||||||
|
local previous_value = nil
|
||||||
|
if previous then
|
||||||
|
local previous_ok
|
||||||
|
previous_ok, previous_value = pcall(cjson.decode, previous)
|
||||||
|
if previous_ok and type(previous_value) == 'table' then
|
||||||
|
previous_stage = previous_value['stage']
|
||||||
|
end
|
||||||
|
end
|
||||||
|
local next_stage = next_value['stage']
|
||||||
|
stage_changed = (not previous) or previous_stage ~= next_stage
|
||||||
|
local previous_reward_settled = previous_value and previous_value['rewardSettled'] or false
|
||||||
|
rankings_changed = next_value['rewardSettled'] == true and previous_reward_settled ~= true
|
||||||
|
end
|
||||||
|
redis.call('SET', KEYS[index], ARGV[index])
|
||||||
|
end
|
||||||
|
local revision = redis.call('INCR', revision_key)
|
||||||
|
return tostring(revision) .. ':' .. (stage_changed and '1' or '0') .. ':' .. (rankings_changed and '1' or '0')
|
||||||
|
`;
|
||||||
|
|
||||||
export const parseTournamentSourceRevision = (value: unknown): string | null => {
|
export const parseTournamentSourceRevision = (value: unknown): string | null => {
|
||||||
if (typeof value === 'number') {
|
if (typeof value === 'number') {
|
||||||
return Number.isSafeInteger(value) && value >= 0 ? String(value) : null;
|
return Number.isSafeInteger(value) && value >= 0 ? String(value) : null;
|
||||||
@@ -69,7 +121,8 @@ export const parseTournamentSourceRevision = (value: unknown): string | null =>
|
|||||||
export const writeTournamentProjection = async (
|
export const writeTournamentProjection = async (
|
||||||
redis: TournamentProjectionRedis,
|
redis: TournamentProjectionRedis,
|
||||||
keys: TournamentSourceKeys,
|
keys: TournamentSourceKeys,
|
||||||
writes: readonly TournamentProjectionWrite[]
|
writes: readonly TournamentProjectionWrite[],
|
||||||
|
clockFence?: TournamentClockFence
|
||||||
): Promise<string> => {
|
): Promise<string> => {
|
||||||
if (writes.length === 0) {
|
if (writes.length === 0) {
|
||||||
throw new Error('Tournament projection write must contain at least one payload.');
|
throw new Error('Tournament projection write must contain at least one payload.');
|
||||||
@@ -90,10 +143,27 @@ export const writeTournamentProjection = async (
|
|||||||
value !== null &&
|
value !== null &&
|
||||||
(value as { rewardSettled?: unknown }).rewardSettled === true
|
(value as { rewardSettled?: unknown }).rewardSettled === true
|
||||||
);
|
);
|
||||||
const result = await redis.eval(WRITE_TOURNAMENT_PROJECTION_SCRIPT, {
|
const result = await redis.eval(
|
||||||
keys: [...writes.map(({ key }) => key), keys.sourceRevisionKey],
|
clockFence ? WRITE_FENCED_TOURNAMENT_PROJECTION_SCRIPT : WRITE_TOURNAMENT_PROJECTION_SCRIPT,
|
||||||
arguments: writes.map(({ value }) => JSON.stringify(value)),
|
{
|
||||||
});
|
keys: [
|
||||||
|
...writes.map(({ key }) => key),
|
||||||
|
keys.sourceRevisionKey,
|
||||||
|
...(clockFence
|
||||||
|
? [clockFence.activeRevisionKey, clockFence.deadlineGenerationKey, clockFence.phaseKey]
|
||||||
|
: []),
|
||||||
|
],
|
||||||
|
arguments: [
|
||||||
|
...writes.map(({ value }) => JSON.stringify(value)),
|
||||||
|
...(clockFence
|
||||||
|
? [String(clockFence.revision), String(clockFence.deadlineGeneration), clockFence.phase]
|
||||||
|
: []),
|
||||||
|
],
|
||||||
|
}
|
||||||
|
);
|
||||||
|
if (result === '__CLOCK_FENCE__') {
|
||||||
|
throw new Error('Tournament projection clock revision fence failed.');
|
||||||
|
}
|
||||||
const scriptResult = typeof result === 'string' ? /^(\d+):([01])(?::([01]))?$/u.exec(result) : null;
|
const scriptResult = typeof result === 'string' ? /^(\d+):([01])(?::([01]))?$/u.exec(result) : null;
|
||||||
const sourceRevision = parseTournamentSourceRevision(scriptResult?.[1] ?? result);
|
const sourceRevision = parseTournamentSourceRevision(scriptResult?.[1] ?? result);
|
||||||
// Plain revision results remain accepted for rolling deployments and small
|
// Plain revision results remain accepted for rolling deployments and small
|
||||||
|
|||||||
@@ -6,6 +6,7 @@ import {
|
|||||||
MAX_SAFE_GAME_TICK,
|
MAX_SAFE_GAME_TICK,
|
||||||
asGameTick,
|
asGameTick,
|
||||||
asObservedGameInstant,
|
asObservedGameInstant,
|
||||||
|
buildClockAlignmentPlan,
|
||||||
buildExactClockAlignmentPlan,
|
buildExactClockAlignmentPlan,
|
||||||
createDeadline,
|
createDeadline,
|
||||||
scheduleNotBefore,
|
scheduleNotBefore,
|
||||||
@@ -113,6 +114,64 @@ describe('GameClock', () => {
|
|||||||
expect(plan.alignedTick).toBe(26 * GAME_TICKS_PER_TURN);
|
expect(plan.alignedTick).toBe(26 * GAME_TICKS_PER_TURN);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('keeps legacy complete-turn rebasing separate from bounded catch-up', () => {
|
||||||
|
const common = {
|
||||||
|
sourceRevision: 4,
|
||||||
|
cutTick: 100,
|
||||||
|
cutWall: new Date('2026-01-01T00:00:00.000Z'),
|
||||||
|
resumeWall: new Date('2026-01-01T01:05:17.250Z'),
|
||||||
|
ticksPerSecond: 10_000,
|
||||||
|
};
|
||||||
|
const legacy = buildClockAlignmentPlan({ ...common, policy: 'LEGACY_COMPLETE_TURNS' });
|
||||||
|
const catchUp = buildClockAlignmentPlan({ ...common, policy: 'CATCH_UP', catchUpTicks: 1_000_000 });
|
||||||
|
|
||||||
|
expect(legacy).toMatchObject({
|
||||||
|
policy: 'LEGACY_COMPLETE_TURNS',
|
||||||
|
gapTicks: 39_172_500,
|
||||||
|
shiftTicks: GAME_TICKS_PER_TURN,
|
||||||
|
catchUpTicks: 3_172_500,
|
||||||
|
});
|
||||||
|
expect(catchUp).toMatchObject({
|
||||||
|
policy: 'CATCH_UP',
|
||||||
|
gapTicks: 39_172_500,
|
||||||
|
shiftTicks: 38_172_500,
|
||||||
|
catchUpTicks: 1_000_000,
|
||||||
|
});
|
||||||
|
expect(() => buildClockAlignmentPlan({ ...common, policy: 'EXACT', catchUpTicks: 1 })).toThrow(
|
||||||
|
'EXACT alignment does not allow catch-up ticks'
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('preserves schedule ordering, remaining distance, and occurrence ticks across generated exact gaps', () => {
|
||||||
|
let seed = 0x5eed1234;
|
||||||
|
const next = (): number => {
|
||||||
|
seed = (Math.imul(seed, 1_664_525) + 1_013_904_223) >>> 0;
|
||||||
|
return seed;
|
||||||
|
};
|
||||||
|
for (let iteration = 0; iteration < 500; iteration += 1) {
|
||||||
|
const cutTick = next() % 1_000_000_000;
|
||||||
|
const gapMilliseconds = next() % (7 * 24 * 60 * 60 * 1_000);
|
||||||
|
const ticksPerSecond = [5_000, 10_000, 60_000][next() % 3]!;
|
||||||
|
const offsets = Array.from({ length: 8 }, () => next() % (3 * GAME_TICKS_PER_TURN)).sort(
|
||||||
|
(left, right) => left - right
|
||||||
|
);
|
||||||
|
const occurrenceTicks = Array.from({ length: 4 }, () => cutTick - (next() % GAME_TICKS_PER_TURN));
|
||||||
|
const plan = buildClockAlignmentPlan({
|
||||||
|
policy: 'EXACT',
|
||||||
|
sourceRevision: 1 + (next() % 10_000),
|
||||||
|
cutTick,
|
||||||
|
cutWall: new Date(0),
|
||||||
|
resumeWall: new Date(gapMilliseconds),
|
||||||
|
ticksPerSecond,
|
||||||
|
});
|
||||||
|
const shifted = offsets.map((offset) => cutTick + offset + plan.shiftTicks);
|
||||||
|
|
||||||
|
expect(shifted.map((deadline) => deadline - plan.alignedTick)).toEqual(offsets);
|
||||||
|
expect([...shifted].sort((left, right) => left - right)).toEqual(shifted);
|
||||||
|
expect(occurrenceTicks).toEqual([...occurrenceTicks]);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
it('projects near the safe tick boundary without unsafe intermediate multiplication', () => {
|
it('projects near the safe tick boundary without unsafe intermediate multiplication', () => {
|
||||||
const clock = new GameClock({
|
const clock = new GameClock({
|
||||||
baseTime: new Date(0),
|
baseTime: new Date(0),
|
||||||
|
|||||||
@@ -71,6 +71,10 @@ model InputEvent {
|
|||||||
actorUserId String? @map("actor_user_id")
|
actorUserId String? @map("actor_user_id")
|
||||||
acceptedGameTick BigInt? @map("accepted_game_tick")
|
acceptedGameTick BigInt? @map("accepted_game_tick")
|
||||||
acceptedClockRevision BigInt? @map("accepted_clock_revision")
|
acceptedClockRevision BigInt? @map("accepted_clock_revision")
|
||||||
|
acceptedDeadlineGeneration BigInt? @map("accepted_deadline_generation")
|
||||||
|
processingGameTick BigInt? @map("processing_game_tick")
|
||||||
|
processingClockRevision BigInt? @map("processing_clock_revision")
|
||||||
|
processingDeadlineGeneration BigInt? @map("processing_deadline_generation")
|
||||||
status InputEventStatus @default(PENDING)
|
status InputEventStatus @default(PENDING)
|
||||||
result Json?
|
result Json?
|
||||||
error String?
|
error String?
|
||||||
|
|||||||
+15
@@ -0,0 +1,15 @@
|
|||||||
|
ALTER TABLE input_event
|
||||||
|
ADD COLUMN accepted_deadline_generation BIGINT,
|
||||||
|
ADD COLUMN processing_game_tick BIGINT,
|
||||||
|
ADD COLUMN processing_clock_revision BIGINT,
|
||||||
|
ADD COLUMN processing_deadline_generation BIGINT;
|
||||||
|
|
||||||
|
ALTER TABLE input_event
|
||||||
|
ADD CONSTRAINT input_event_accepted_clock_revision_positive
|
||||||
|
CHECK (accepted_clock_revision IS NULL OR accepted_clock_revision > 0),
|
||||||
|
ADD CONSTRAINT input_event_accepted_deadline_generation_positive
|
||||||
|
CHECK (accepted_deadline_generation IS NULL OR accepted_deadline_generation > 0),
|
||||||
|
ADD CONSTRAINT input_event_processing_clock_revision_positive
|
||||||
|
CHECK (processing_clock_revision IS NULL OR processing_clock_revision > 0),
|
||||||
|
ADD CONSTRAINT input_event_processing_deadline_generation_positive
|
||||||
|
CHECK (processing_deadline_generation IS NULL OR processing_deadline_generation > 0);
|
||||||
@@ -46,6 +46,9 @@ export interface DatabaseClient {
|
|||||||
vote: GamePrisma.VoteDelegate;
|
vote: GamePrisma.VoteDelegate;
|
||||||
inputEvent: GamePrisma.InputEventDelegate;
|
inputEvent: GamePrisma.InputEventDelegate;
|
||||||
turnDaemonLease: GamePrisma.TurnDaemonLeaseDelegate;
|
turnDaemonLease: GamePrisma.TurnDaemonLeaseDelegate;
|
||||||
|
clockSuspension?: GamePrisma.ClockSuspensionDelegate;
|
||||||
|
clockReconciliationParticipant?: GamePrisma.ClockReconciliationParticipantDelegate;
|
||||||
|
clockProjectionOutbox?: GamePrisma.ClockProjectionOutboxDelegate;
|
||||||
readModelOutbox: GamePrisma.ReadModelOutboxDelegate;
|
readModelOutbox: GamePrisma.ReadModelOutboxDelegate;
|
||||||
webPushOutbox: GamePrisma.WebPushOutboxDelegate;
|
webPushOutbox: GamePrisma.WebPushOutboxDelegate;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -10,4 +10,5 @@ export * from './readModelChangeJournal.js';
|
|||||||
export * from './readModelOutboxDispatcher.js';
|
export * from './readModelOutboxDispatcher.js';
|
||||||
export * from './readModelCoverageActivation.js';
|
export * from './readModelCoverageActivation.js';
|
||||||
export * from './gameSchemaAdvisoryLock.js';
|
export * from './gameSchemaAdvisoryLock.js';
|
||||||
|
export * from './inputEventClock.js';
|
||||||
export * from './webPushOutbox.js';
|
export * from './webPushOutbox.js';
|
||||||
|
|||||||
@@ -0,0 +1,76 @@
|
|||||||
|
import { GameClock, inferClockPhase, parseGameClockPhase } from '@sammo-ts/common';
|
||||||
|
|
||||||
|
import { GamePrisma, type GamePrismaClient } from './gamePrisma.js';
|
||||||
|
import { acquireGameSchemaAdvisoryXactLock, CLOCK_OPERATION_PERSISTENCE_LOCK } from './gameSchemaAdvisoryLock.js';
|
||||||
|
|
||||||
|
type ClockAcceptanceDatabase = Pick<GamePrismaClient, '$executeRaw' | '$queryRaw' | 'worldState'>;
|
||||||
|
|
||||||
|
interface DbWallRow {
|
||||||
|
wallNow: Date;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface InputEventClockCoordinate {
|
||||||
|
wallAt: Date;
|
||||||
|
gameAt: Date;
|
||||||
|
gameTick: bigint;
|
||||||
|
clockRevision: bigint;
|
||||||
|
deadlineGeneration: bigint;
|
||||||
|
phase: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Reads one input-event acceptance coordinate while holding the same schema
|
||||||
|
* clock-operation fence used by reconciliation. The caller must create the
|
||||||
|
* input_event in this transaction before releasing the lock.
|
||||||
|
*/
|
||||||
|
export const readInputEventClockCoordinate = async (
|
||||||
|
db: ClockAcceptanceDatabase
|
||||||
|
): Promise<InputEventClockCoordinate> => {
|
||||||
|
await acquireGameSchemaAdvisoryXactLock(db, CLOCK_OPERATION_PERSISTENCE_LOCK);
|
||||||
|
const [wall] = await db.$queryRaw<DbWallRow[]>(GamePrisma.sql`
|
||||||
|
SELECT timezone('UTC', clock_timestamp()) AS "wallNow"
|
||||||
|
`);
|
||||||
|
if (!wall) throw new Error('PostgreSQL did not return its authoritative wall clock.');
|
||||||
|
const world = await db.worldState.findFirst({
|
||||||
|
orderBy: { id: 'asc' },
|
||||||
|
select: {
|
||||||
|
clockBaseTime: true,
|
||||||
|
clockTick: true,
|
||||||
|
clockMode: true,
|
||||||
|
clockWallAnchor: true,
|
||||||
|
tickSeconds: true,
|
||||||
|
clockPhase: true,
|
||||||
|
clockRevision: true,
|
||||||
|
deadlineGeneration: true,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
if (!world?.clockBaseTime || world.clockTick === null || !world.clockWallAnchor) {
|
||||||
|
throw new Error('The authoritative game clock is not initialized.');
|
||||||
|
}
|
||||||
|
const tick = Number(world.clockTick);
|
||||||
|
const revision = Number(world.clockRevision);
|
||||||
|
const generation = Number(world.deadlineGeneration);
|
||||||
|
if (!Number.isSafeInteger(tick) || !Number.isSafeInteger(revision) || !Number.isSafeInteger(generation)) {
|
||||||
|
throw new Error('The authoritative game clock coordinate is outside the safe integer range.');
|
||||||
|
}
|
||||||
|
const mode = world.clockMode === 'manual' ? 'manual' : 'realtime';
|
||||||
|
const phase = world.clockPhase ? parseGameClockPhase(world.clockPhase) : inferClockPhase(mode);
|
||||||
|
const clock = new GameClock({
|
||||||
|
baseTime: world.clockBaseTime,
|
||||||
|
tick,
|
||||||
|
mode,
|
||||||
|
wallAnchor: world.clockWallAnchor,
|
||||||
|
turnSeconds: world.tickSeconds,
|
||||||
|
phase,
|
||||||
|
revision,
|
||||||
|
});
|
||||||
|
const observedTick = clock.nowTick(wall.wallNow);
|
||||||
|
return {
|
||||||
|
wallAt: wall.wallNow,
|
||||||
|
gameAt: clock.tickToDate(observedTick),
|
||||||
|
gameTick: BigInt(observedTick),
|
||||||
|
clockRevision: BigInt(revision),
|
||||||
|
deadlineGeneration: BigInt(generation),
|
||||||
|
phase,
|
||||||
|
};
|
||||||
|
};
|
||||||
@@ -2,6 +2,6 @@
|
|||||||
"formatVersion": 1,
|
"formatVersion": 1,
|
||||||
"controllerProtocol": 2,
|
"controllerProtocol": 2,
|
||||||
"gatewaySchemaHead": "20260825000000_add_bulk_release_batches",
|
"gatewaySchemaHead": "20260825000000_add_bulk_release_batches",
|
||||||
"gameSchemaHead": "20260903090000_add_game_clock_reconciliation",
|
"gameSchemaHead": "20260903103000_add_input_event_clock_processing",
|
||||||
"components": ["gateway-api", "gateway-frontend", "release-controller", "game-api", "game-engine", "game-frontend"]
|
"components": ["gateway-api", "gateway-frontend", "release-controller", "game-api", "game-engine", "game-frontend"]
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user