feat: 시계 reconciliation 워커와 명령 경계 완성
This commit is contained in:
@@ -1,4 +1,7 @@
|
||||
import {
|
||||
CLOCK_OPERATION_PERSISTENCE_LOCK,
|
||||
GamePrisma,
|
||||
acquireGameSchemaAdvisoryXactLock,
|
||||
createGamePostgresConnector,
|
||||
createRedisConnector,
|
||||
type GamePrismaClient,
|
||||
@@ -10,6 +13,7 @@ import { resolveGameApiConfigFromEnv } from '../config.js';
|
||||
import { createBestEffortResourceCloser } from '../services/bestEffortResourceCloser.js';
|
||||
import { loadCurrentGameTime, type CurrentGameTime } from '../services/gameClock.js';
|
||||
import { createPollingWorkerControl, waitForWorkerPoll } from '../services/pollingWorkerLifecycle.js';
|
||||
import { ensureActiveRedisClockFence } from '../services/redisClockFence.js';
|
||||
import { buildAuctionTimerKeys } from './keys.js';
|
||||
import { resolveAuctionTimerScore, seedAuctionTimers } from './scheduler.js';
|
||||
|
||||
@@ -24,8 +28,22 @@ interface RedisTimerClient {
|
||||
zAdd(key: string, values: Array<{ score: number; value: string }>): Promise<number>;
|
||||
zRem(key: string, values: string | string[]): Promise<number>;
|
||||
zRemRangeByScore(key: string, min: number, max: number): Promise<number>;
|
||||
eval?(script: string, options: { keys: string[]; arguments: string[] }): Promise<unknown>;
|
||||
}
|
||||
|
||||
const POP_DUE_AUCTIONS_SCRIPT = `
|
||||
if redis.call('GET', KEYS[2]) ~= ARGV[1]
|
||||
or redis.call('GET', KEYS[3]) ~= ARGV[2]
|
||||
or redis.call('GET', KEYS[4]) ~= 'RUNNING' then
|
||||
return { '__CLOCK_FENCE__' }
|
||||
end
|
||||
local ids = redis.call('ZRANGEBYSCORE', KEYS[1], '-inf', ARGV[3], 'LIMIT', 0, ARGV[4])
|
||||
if #ids > 0 then
|
||||
redis.call('ZREM', KEYS[1], unpack(ids))
|
||||
end
|
||||
return ids
|
||||
`;
|
||||
|
||||
const AUCTION_FINALIZE_RECOVERY_LIMIT = 1;
|
||||
|
||||
interface AuctionFinalizeDeadline {
|
||||
@@ -115,12 +133,29 @@ const isSuccessfulAuctionFinalizeResult = (result: unknown, auctionId: number):
|
||||
return resultRecord.type === 'auctionFinalize' && resultRecord.ok === true && resultRecord.auctionId === auctionId;
|
||||
};
|
||||
|
||||
const popDueAuctionIds = async (
|
||||
export const popDueAuctionIds = async (
|
||||
redis: RedisTimerClient,
|
||||
timerKey: string,
|
||||
nowMs: number,
|
||||
batchSize: number
|
||||
batchSize: number,
|
||||
clockFence?: {
|
||||
activeRevisionKey: string;
|
||||
deadlineGenerationKey: string;
|
||||
phaseKey: string;
|
||||
revision: number;
|
||||
generation: number;
|
||||
}
|
||||
): Promise<string[]> => {
|
||||
if (clockFence) {
|
||||
if (!redis.eval) throw new Error('Redis EVAL is required for revision-fenced auction due-pop.');
|
||||
const result = await redis.eval(POP_DUE_AUCTIONS_SCRIPT, {
|
||||
keys: [timerKey, clockFence.activeRevisionKey, clockFence.deadlineGenerationKey, clockFence.phaseKey],
|
||||
arguments: [String(clockFence.revision), String(clockFence.generation), String(nowMs), String(batchSize)],
|
||||
});
|
||||
if (!Array.isArray(result)) throw new Error('Auction due-pop returned an invalid Redis result.');
|
||||
if (result[0] === '__CLOCK_FENCE__') return [];
|
||||
return result.map(String);
|
||||
}
|
||||
const ids = await redis.zRangeByScore(timerKey, 0, nowMs, { LIMIT: { offset: 0, count: batchSize } });
|
||||
if (ids.length > 0) {
|
||||
await redis.zRem(timerKey, ids);
|
||||
@@ -205,6 +240,8 @@ export const processDueAuctionId = async (options: {
|
||||
nowMs: number;
|
||||
nowTick?: number | null;
|
||||
historyNowMs?: number;
|
||||
expectedClockRevision?: number;
|
||||
expectedDeadlineGeneration?: number;
|
||||
}): Promise<'PENDING' | 'RESCHEDULED' | 'IGNORED'> => {
|
||||
const { db, redis, timerKey, historyKey, id, nowMs, nowTick = null, historyNowMs = nowMs } = options;
|
||||
const auctionId = Number(id);
|
||||
@@ -213,6 +250,29 @@ export const processDueAuctionId = async (options: {
|
||||
}
|
||||
const now = new Date(nowMs);
|
||||
const outcome = await db.$transaction(async (transaction) => {
|
||||
if (options.expectedClockRevision !== undefined || options.expectedDeadlineGeneration !== undefined) {
|
||||
await acquireGameSchemaAdvisoryXactLock(transaction, CLOCK_OPERATION_PERSISTENCE_LOCK);
|
||||
const [world] = await transaction.$queryRaw<
|
||||
Array<{ clockPhase: string | null; clockRevision: bigint; deadlineGeneration: bigint }>
|
||||
>(GamePrisma.sql`
|
||||
SELECT
|
||||
clock_phase AS "clockPhase",
|
||||
clock_revision AS "clockRevision",
|
||||
deadline_generation AS "deadlineGeneration"
|
||||
FROM world_state
|
||||
ORDER BY id ASC
|
||||
LIMIT 1
|
||||
FOR UPDATE
|
||||
`);
|
||||
if (
|
||||
!world ||
|
||||
world.clockPhase !== 'RUNNING' ||
|
||||
world.clockRevision !== BigInt(options.expectedClockRevision ?? -1) ||
|
||||
world.deadlineGeneration !== BigInt(options.expectedDeadlineGeneration ?? -1)
|
||||
) {
|
||||
return { status: 'RESCHEDULED' as const, clockFenceFailed: true };
|
||||
}
|
||||
}
|
||||
const current = await transaction.auction.findUnique({
|
||||
where: { id: auctionId },
|
||||
select: { status: true, closeAt: true, closeTick: true },
|
||||
@@ -259,6 +319,13 @@ export const processDueAuctionId = async (options: {
|
||||
target: 'ENGINE',
|
||||
eventType: nextCommand.type,
|
||||
payload: { ...nextCommand },
|
||||
...(nowTick === null ? {} : { acceptedGameTick: BigInt(nowTick) }),
|
||||
...(options.expectedClockRevision === undefined
|
||||
? {}
|
||||
: { acceptedClockRevision: BigInt(options.expectedClockRevision) }),
|
||||
...(options.expectedDeadlineGeneration === undefined
|
||||
? {}
|
||||
: { acceptedDeadlineGeneration: BigInt(options.expectedDeadlineGeneration) }),
|
||||
},
|
||||
});
|
||||
return { status: 'PENDING' as const };
|
||||
@@ -284,6 +351,10 @@ export const processDueAuctionId = async (options: {
|
||||
return 'PENDING';
|
||||
}
|
||||
if (outcome.status === 'RESCHEDULED') {
|
||||
if ('clockFenceFailed' in outcome) {
|
||||
await redis.zAdd(timerKey, [{ score: nowTick ?? nowMs, value: id }]);
|
||||
return 'RESCHEDULED';
|
||||
}
|
||||
const gameTime = await loadCurrentGameTime(db, now);
|
||||
await redis.zAdd(timerKey, [
|
||||
{
|
||||
@@ -324,6 +395,17 @@ export const runAuctionWorker = async (options: AuctionWorkerOptions = {}): Prom
|
||||
const gameTime = await loadCurrentGameTime(postgres.prisma, new Date(operationalNowMs));
|
||||
const gameNowMs = gameTime.now.getTime();
|
||||
const dueScore = gameTime.tick ?? gameNowMs;
|
||||
if (gameTime.phase && gameTime.phase !== 'RUNNING') {
|
||||
await waitForWorkerPoll(control.signal, config.auctionTimerPollMs);
|
||||
continue;
|
||||
}
|
||||
const clockFence = gameTime.phase
|
||||
? await ensureActiveRedisClockFence(redis.client, config.profileName, gameTime)
|
||||
: null;
|
||||
if (gameTime.phase && !clockFence) {
|
||||
await waitForWorkerPoll(control.signal, config.auctionTimerPollMs);
|
||||
continue;
|
||||
}
|
||||
if (operationalNowMs >= nextResyncAt) {
|
||||
await seedAuctionTimers(postgres.prisma, redis.client, keys);
|
||||
nextResyncAt = operationalNowMs + config.auctionTimerResyncMs;
|
||||
@@ -345,7 +427,7 @@ export const runAuctionWorker = async (options: AuctionWorkerOptions = {}): Prom
|
||||
if (historyTrimBefore > 0) {
|
||||
await redis.client.zRemRangeByScore(keys.historyKey, 0, historyTrimBefore);
|
||||
}
|
||||
const dueIds = await popDueAuctionIds(redis.client, keys.timerKey, dueScore, 100);
|
||||
const dueIds = await popDueAuctionIds(redis.client, keys.timerKey, dueScore, 100, clockFence ?? undefined);
|
||||
if (dueIds.length > 0) {
|
||||
for (const id of dueIds) {
|
||||
try {
|
||||
@@ -358,6 +440,12 @@ export const runAuctionWorker = async (options: AuctionWorkerOptions = {}): Prom
|
||||
nowMs: gameNowMs,
|
||||
nowTick: gameTime.tick,
|
||||
historyNowMs: operationalNowMs,
|
||||
...(clockFence
|
||||
? {
|
||||
expectedClockRevision: clockFence.revision,
|
||||
expectedDeadlineGeneration: clockFence.generation,
|
||||
}
|
||||
: {}),
|
||||
});
|
||||
if (outcome === 'PENDING') {
|
||||
pendingFinalizationIds.add(Number(id));
|
||||
|
||||
@@ -1,6 +1,12 @@
|
||||
import { randomUUID } from 'node:crypto';
|
||||
|
||||
import { acquireGameSchemaAdvisoryXactLock, type DatabaseClient, type GamePrisma } from '@sammo-ts/infra';
|
||||
import {
|
||||
acquireGameSchemaAdvisoryXactLock,
|
||||
readInputEventClockCoordinate,
|
||||
type DatabaseClient,
|
||||
type GamePrisma,
|
||||
type InputEventClockCoordinate,
|
||||
} from '@sammo-ts/infra';
|
||||
|
||||
import type { TurnDaemonTransport } from './transport.js';
|
||||
import type { TurnDaemonCommand, TurnDaemonCommandResult, TurnDaemonStatus } from './types.js';
|
||||
@@ -103,10 +109,10 @@ export class DatabaseTurnDaemonTransport implements TurnDaemonTransport {
|
||||
try {
|
||||
if (command.type === 'npcPossessGeneral' && this.db.$transaction) {
|
||||
const rejectionReason = await this.db.$transaction(async (transaction) => {
|
||||
const coordinate = await readInputEventClockCoordinate(transaction);
|
||||
await acquireGameSchemaAdvisoryXactLock(transaction, 'npc-possession:global');
|
||||
await acquireGameSchemaAdvisoryXactLock(transaction, `npc-possession:user:${command.userId}`);
|
||||
const acceptedAt = new Date(Math.floor(Date.now() / 1000) * 1000);
|
||||
const acceptedGameAt = (await loadCurrentGameTime(transaction, acceptedAt)).now;
|
||||
const acceptedGameAt = coordinate.gameAt;
|
||||
const token = await transaction.npcSelectionToken.findFirst({
|
||||
where: {
|
||||
ownerUserId: command.userId,
|
||||
@@ -130,14 +136,21 @@ export class DatabaseTurnDaemonTransport implements TurnDaemonTransport {
|
||||
...(durableCommand as Extract<TurnDaemonCommand, { type: 'npcPossessGeneral' }>),
|
||||
acceptedGameAt: acceptedGameAt.toISOString(),
|
||||
};
|
||||
await this.createInputEvent(transaction, acceptedCommand, requestId, acceptedAt);
|
||||
await this.createInputEvent(transaction, acceptedCommand, requestId, coordinate);
|
||||
return null;
|
||||
});
|
||||
if (rejectionReason) {
|
||||
throw new RejectedNpcPossessionCommandError('PRECONDITION_FAILED', rejectionReason);
|
||||
}
|
||||
} else {
|
||||
await this.createInputEvent(this.db, durableCommand, requestId);
|
||||
if (this.db.$transaction) {
|
||||
await this.db.$transaction(async (transaction) => {
|
||||
const coordinate = await readInputEventClockCoordinate(transaction);
|
||||
await this.createInputEvent(transaction, durableCommand, requestId, coordinate);
|
||||
});
|
||||
} else {
|
||||
await this.createInputEvent(this.db, durableCommand, requestId);
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
if (error instanceof RejectedNpcPossessionCommandError) {
|
||||
@@ -166,8 +179,20 @@ export class DatabaseTurnDaemonTransport implements TurnDaemonTransport {
|
||||
db: DatabaseClient,
|
||||
command: TurnDaemonCommand,
|
||||
requestId: string,
|
||||
createdAt?: Date
|
||||
coordinate?: InputEventClockCoordinate
|
||||
): Promise<void> {
|
||||
const gameTime = coordinate ? null : await loadCurrentGameTime(db, new Date());
|
||||
const commandAcceptedTick = Reflect.get(command, 'acceptedGameTick');
|
||||
const acceptedGameTick =
|
||||
typeof commandAcceptedTick === 'number' && Number.isSafeInteger(commandAcceptedTick)
|
||||
? commandAcceptedTick
|
||||
: coordinate
|
||||
? Number(coordinate.gameTick)
|
||||
: gameTime!.tick;
|
||||
const acceptedClockRevision = coordinate ? Number(coordinate.clockRevision) : gameTime!.revision;
|
||||
const acceptedDeadlineGeneration = coordinate
|
||||
? Number(coordinate.deadlineGeneration)
|
||||
: gameTime!.deadlineGeneration;
|
||||
await db.inputEvent.create({
|
||||
data: {
|
||||
requestId,
|
||||
@@ -175,7 +200,14 @@ export class DatabaseTurnDaemonTransport implements TurnDaemonTransport {
|
||||
eventType: command.type,
|
||||
payload: asJson(command),
|
||||
actorUserId: 'userId' in command && typeof command.userId === 'string' ? command.userId : null,
|
||||
...(createdAt ? { createdAt } : {}),
|
||||
...(acceptedGameTick === null ? {} : { acceptedGameTick: BigInt(acceptedGameTick) }),
|
||||
...(acceptedClockRevision === null || acceptedClockRevision === undefined
|
||||
? {}
|
||||
: { acceptedClockRevision: BigInt(acceptedClockRevision) }),
|
||||
...(acceptedDeadlineGeneration === null || acceptedDeadlineGeneration === undefined
|
||||
? {}
|
||||
: { acceptedDeadlineGeneration: BigInt(acceptedDeadlineGeneration) }),
|
||||
...(coordinate ? { createdAt: coordinate.wallAt } : {}),
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
@@ -1,6 +1,11 @@
|
||||
import { createHash } from 'node:crypto';
|
||||
|
||||
import { GamePrisma, type DatabaseClient as InfraDatabaseClient } from '@sammo-ts/infra';
|
||||
import {
|
||||
CLOCK_OPERATION_PERSISTENCE_LOCK,
|
||||
GamePrisma,
|
||||
acquireGameSchemaAdvisoryXactLock,
|
||||
type DatabaseClient as InfraDatabaseClient,
|
||||
} from '@sammo-ts/infra';
|
||||
|
||||
import type { DatabaseClient } from './context.js';
|
||||
|
||||
@@ -20,6 +25,9 @@ interface LockedInputEvent {
|
||||
status: 'PENDING' | 'PROCESSING' | 'SUCCEEDED' | 'FAILED';
|
||||
result: GamePrisma.JsonValue | null;
|
||||
attempts: number;
|
||||
acceptedGameTick: bigint | null;
|
||||
acceptedClockRevision: bigint | null;
|
||||
acceptedDeadlineGeneration: bigint | null;
|
||||
}
|
||||
|
||||
type InputEventOutcome<T> =
|
||||
@@ -85,6 +93,9 @@ const insertPendingIfAbsent = async (
|
||||
actor_user_id,
|
||||
status,
|
||||
attempts,
|
||||
accepted_game_tick,
|
||||
accepted_clock_revision,
|
||||
accepted_deadline_generation,
|
||||
created_at
|
||||
)
|
||||
VALUES (
|
||||
@@ -95,6 +106,9 @@ const insertPendingIfAbsent = async (
|
||||
${options.actorUserId},
|
||||
'PENDING'::"InputEventStatus",
|
||||
0,
|
||||
(SELECT clock_tick FROM world_state ORDER BY id ASC LIMIT 1),
|
||||
(SELECT clock_revision FROM world_state ORDER BY id ASC LIMIT 1),
|
||||
(SELECT deadline_generation FROM world_state ORDER BY id ASC LIMIT 1),
|
||||
CURRENT_TIMESTAMP AT TIME ZONE 'UTC'
|
||||
)
|
||||
ON CONFLICT (request_id) DO NOTHING
|
||||
@@ -112,7 +126,10 @@ const lockInputEvent = async (db: DatabaseClient, requestId: string): Promise<Lo
|
||||
actor_user_id AS "actorUserId",
|
||||
status,
|
||||
result,
|
||||
attempts
|
||||
attempts,
|
||||
accepted_game_tick AS "acceptedGameTick",
|
||||
accepted_clock_revision AS "acceptedClockRevision",
|
||||
accepted_deadline_generation AS "acceptedDeadlineGeneration"
|
||||
FROM input_event
|
||||
WHERE request_id = ${requestId}
|
||||
FOR UPDATE
|
||||
@@ -151,7 +168,8 @@ const isMatchingIdentity = (
|
||||
const claimInputEvent = async (
|
||||
db: DatabaseClient,
|
||||
requestId: string,
|
||||
payloadIdentity: ApiInputPayloadIdentity
|
||||
payloadIdentity: ApiInputPayloadIdentity,
|
||||
row: LockedInputEvent
|
||||
): Promise<void> => {
|
||||
await db.inputEvent.update({
|
||||
where: { requestId },
|
||||
@@ -164,6 +182,9 @@ const claimInputEvent = async (
|
||||
lockedBy: null,
|
||||
leaseUntil: null,
|
||||
processingAt: new Date(),
|
||||
processingGameTick: row.acceptedGameTick,
|
||||
processingClockRevision: row.acceptedClockRevision,
|
||||
processingDeadlineGeneration: row.acceptedDeadlineGeneration,
|
||||
completedAt: null,
|
||||
},
|
||||
});
|
||||
@@ -184,6 +205,7 @@ const markUnexpectedFailure = async (
|
||||
|
||||
try {
|
||||
await db.$transaction(async (transaction) => {
|
||||
await acquireGameSchemaAdvisoryXactLock(transaction, CLOCK_OPERATION_PERSISTENCE_LOCK);
|
||||
await insertPendingIfAbsent(transaction, options);
|
||||
const row = await lockInputEvent(transaction, options.requestId);
|
||||
const identityMatches = isMatchingIdentity(row, options) || canAdoptLegacyFailedPayload(row, options);
|
||||
@@ -233,6 +255,7 @@ export const executeInputEvent = async <T>(options: {
|
||||
let outcome: InputEventOutcome<T>;
|
||||
try {
|
||||
outcome = await db.$transaction(async (transaction) => {
|
||||
await acquireGameSchemaAdvisoryXactLock(transaction, CLOCK_OPERATION_PERSISTENCE_LOCK);
|
||||
await insertPendingIfAbsent(transaction, { requestId, eventType, actorUserId, payloadIdentity });
|
||||
const row = await lockInputEvent(transaction, requestId);
|
||||
const identityMatches = isMatchingIdentity(row, { eventType, actorUserId, payloadIdentity });
|
||||
@@ -251,7 +274,7 @@ export const executeInputEvent = async <T>(options: {
|
||||
throw new DuplicateInputEventError(requestId);
|
||||
}
|
||||
|
||||
await claimInputEvent(transaction, requestId, payloadIdentity);
|
||||
await claimInputEvent(transaction, requestId, payloadIdentity, row);
|
||||
const savepointDb = transaction as SavepointDatabaseClient;
|
||||
await savepointDb.$executeRawUnsafe(`SAVEPOINT ${BUSINESS_SAVEPOINT}`);
|
||||
businessStarted = true;
|
||||
|
||||
@@ -5,12 +5,14 @@ import { asRecord } from '@sammo-ts/common';
|
||||
import type { TournamentType } from '@sammo-ts/logic';
|
||||
import type { TournamentState } from '../../tournament/types.js';
|
||||
|
||||
import { TournamentStore } from '../../tournament/store.js';
|
||||
import { TournamentStore, type TournamentClockContext } from '../../tournament/store.js';
|
||||
import { buildTournamentKeys } from '../../tournament/keys.js';
|
||||
import { assignManualApplicantGroup } from '../../tournament/workerHelpers.js';
|
||||
import { accessAuthedProcedure, authedProcedure, router } from '../../trpc.js';
|
||||
import { getMyGeneral } from '../shared/general.js';
|
||||
import { loadCurrentGameTime } from '../../services/gameClock.js';
|
||||
import { ensureActiveRedisClockFence } from '../../services/redisClockFence.js';
|
||||
import { loadClockAdminStatus } from '../../services/clockReadiness.js';
|
||||
|
||||
const hasAdminRole = (roles: string[], profileName: string): boolean => {
|
||||
if (roles.includes('superuser') || roles.includes('admin') || roles.includes('admin.superuser')) {
|
||||
@@ -44,6 +46,32 @@ const adminProcedure = authedProcedure.use(({ ctx, next }) => {
|
||||
return next();
|
||||
});
|
||||
|
||||
const withTournamentClockMutation = async <T>(
|
||||
ctx: {
|
||||
db: Parameters<typeof loadCurrentGameTime>[0];
|
||||
redis: Parameters<typeof ensureActiveRedisClockFence>[0];
|
||||
profile: { name: string };
|
||||
},
|
||||
store: TournamentStore,
|
||||
operation: () => Promise<T>
|
||||
): Promise<T> => {
|
||||
const gameTime = await loadCurrentGameTime(ctx.db);
|
||||
const fence = await ensureActiveRedisClockFence(ctx.redis, ctx.profile.name, gameTime);
|
||||
if (!fence) {
|
||||
throw new TRPCError({
|
||||
code: 'PRECONDITION_FAILED',
|
||||
message: 'Clock reconciliation is incomplete; tournament mutation is disabled.',
|
||||
});
|
||||
}
|
||||
const clockContext: TournamentClockContext = {
|
||||
phase: 'RUNNING',
|
||||
revision: fence.revision,
|
||||
deadlineGeneration: fence.generation,
|
||||
dateToTick: gameTime.dateToTick,
|
||||
};
|
||||
return store.withClockContext(clockContext, () => store.withMutationLock(operation));
|
||||
};
|
||||
|
||||
const zTournamentState = z.object({
|
||||
stage: z.number().int().min(0),
|
||||
phase: z.number().int().min(0),
|
||||
@@ -143,7 +171,10 @@ export const tournamentRouter = router({
|
||||
const store = new TournamentStore(ctx.redis, buildTournamentKeys(ctx.profile.name));
|
||||
return store.getState();
|
||||
}),
|
||||
getAdminStatus: adminProcedure.query(async () => ({ ok: true })),
|
||||
getAdminStatus: adminProcedure.query(async ({ ctx }) => ({
|
||||
ok: true,
|
||||
clock: await loadClockAdminStatus(ctx.db),
|
||||
})),
|
||||
getSnapshot: accessAuthedProcedure.query(async ({ ctx }) => {
|
||||
await getMyGeneral(ctx);
|
||||
const store = new TournamentStore(ctx.redis, buildTournamentKeys(ctx.profile.name));
|
||||
@@ -271,7 +302,7 @@ export const tournamentRouter = router({
|
||||
}),
|
||||
setState: adminProcedure.input(zTournamentState).mutation(async ({ ctx, input }) => {
|
||||
const store = new TournamentStore(ctx.redis, buildTournamentKeys(ctx.profile.name));
|
||||
return store.withMutationLock(async () => {
|
||||
return withTournamentClockMutation(ctx, store, async () => {
|
||||
await store.setState({
|
||||
...input,
|
||||
type: input.type as TournamentType,
|
||||
@@ -281,7 +312,7 @@ export const tournamentRouter = router({
|
||||
}),
|
||||
patchState: adminProcedure.input(zTournamentState.partial()).mutation(async ({ ctx, input }) => {
|
||||
const store = new TournamentStore(ctx.redis, buildTournamentKeys(ctx.profile.name));
|
||||
return store.withMutationLock(async () => {
|
||||
return withTournamentClockMutation(ctx, store, async () => {
|
||||
const current = await store.getState();
|
||||
if (!current) {
|
||||
throw new TRPCError({ code: 'NOT_FOUND', message: 'Tournament state not found.' });
|
||||
@@ -297,21 +328,21 @@ export const tournamentRouter = router({
|
||||
}),
|
||||
setParticipants: adminProcedure.input(z.array(zParticipant)).mutation(async ({ ctx, input }) => {
|
||||
const store = new TournamentStore(ctx.redis, buildTournamentKeys(ctx.profile.name));
|
||||
return store.withMutationLock(async () => {
|
||||
return withTournamentClockMutation(ctx, store, async () => {
|
||||
await store.setParticipants(input);
|
||||
return { ok: true, count: input.length };
|
||||
});
|
||||
}),
|
||||
setMatches: adminProcedure.input(z.array(zMatch)).mutation(async ({ ctx, input }) => {
|
||||
const store = new TournamentStore(ctx.redis, buildTournamentKeys(ctx.profile.name));
|
||||
return store.withMutationLock(async () => {
|
||||
return withTournamentClockMutation(ctx, store, async () => {
|
||||
await store.setMatches(input);
|
||||
return { ok: true, count: input.length };
|
||||
});
|
||||
}),
|
||||
setBettingEntries: adminProcedure.input(z.array(zBetEntry)).mutation(async ({ ctx, input }) => {
|
||||
const store = new TournamentStore(ctx.redis, buildTournamentKeys(ctx.profile.name));
|
||||
return store.withMutationLock(async () => {
|
||||
return withTournamentClockMutation(ctx, store, async () => {
|
||||
await store.setBettingEntries(input);
|
||||
return { ok: true, count: input.length };
|
||||
});
|
||||
@@ -347,7 +378,7 @@ export const tournamentRouter = router({
|
||||
};
|
||||
});
|
||||
|
||||
return store.withMutationLock(async () => {
|
||||
return withTournamentClockMutation(ctx, store, async () => {
|
||||
await store.setParticipants(participants);
|
||||
return { ok: true, count: participants.length };
|
||||
});
|
||||
@@ -394,7 +425,7 @@ export const tournamentRouter = router({
|
||||
join: authedProcedure.mutation(async ({ ctx }) => {
|
||||
const general = await getMyGeneral(ctx);
|
||||
const store = new TournamentStore(ctx.redis, buildTournamentKeys(ctx.profile.name));
|
||||
return store.withMutationLock(async () => {
|
||||
return withTournamentClockMutation(ctx, store, async () => {
|
||||
const state = await store.getState();
|
||||
if (!state || state.stage !== 1 || state.participantsLockedAt) {
|
||||
throw new TRPCError({ code: 'BAD_REQUEST', message: '참가 신청 기간이 아닙니다.' });
|
||||
@@ -459,7 +490,7 @@ export const tournamentRouter = router({
|
||||
}),
|
||||
cancel: adminProcedure.mutation(async ({ ctx }) => {
|
||||
const store = new TournamentStore(ctx.redis, buildTournamentKeys(ctx.profile.name));
|
||||
return store.withMutationLock(async () => {
|
||||
return withTournamentClockMutation(ctx, store, async () => {
|
||||
const state = await store.getState();
|
||||
if (!state) {
|
||||
throw new TRPCError({ code: 'NOT_FOUND', message: 'Tournament state not found.' });
|
||||
@@ -523,7 +554,7 @@ export const tournamentRouter = router({
|
||||
.mutation(async ({ ctx, input }) => {
|
||||
const general = await getMyGeneral(ctx);
|
||||
const store = new TournamentStore(ctx.redis, buildTournamentKeys(ctx.profile.name));
|
||||
return store.withMutationLock(async () => {
|
||||
return withTournamentClockMutation(ctx, store, async () => {
|
||||
const state = await store.getState();
|
||||
if (!state || state.stage !== 6) {
|
||||
throw new TRPCError({ code: 'BAD_REQUEST', message: '베팅 기간이 아닙니다.' });
|
||||
|
||||
@@ -50,6 +50,7 @@ import { ReadModelOutboxWorker } from './realtime/outboxWorker.js';
|
||||
import { DeferredGeneralAccessWorker } from './services/deferredGeneralAccess.js';
|
||||
import { WebPushOutboxWorker } from './services/webPushOutboxWorker.js';
|
||||
import { scopeHttpIdempotencyKey } from './requestId.js';
|
||||
import { loadClockReadiness } from './services/clockReadiness.js';
|
||||
|
||||
const extractBearerToken = (value: string | string[] | undefined): string | null => {
|
||||
if (!value) {
|
||||
@@ -420,12 +421,17 @@ export const createGameApiServer = async () => {
|
||||
request.raw.on('aborted', close);
|
||||
});
|
||||
|
||||
app.get('/healthz', async () => ({
|
||||
ok: true,
|
||||
profile: config.profileName,
|
||||
postgresPool: postgres.getPoolStats(),
|
||||
accountIconReconciliation: accountIconResetReconciler.getHealth(),
|
||||
}));
|
||||
app.get('/healthz', async (_request, reply) => {
|
||||
const clock = await loadClockReadiness(postgres.prisma);
|
||||
if (!clock.reconciliationComplete) reply.code(503);
|
||||
return {
|
||||
ok: clock.reconciliationComplete,
|
||||
profile: config.profileName,
|
||||
postgresPool: postgres.getPoolStats(),
|
||||
accountIconReconciliation: accountIconResetReconciler.getHealth(),
|
||||
clock,
|
||||
};
|
||||
});
|
||||
|
||||
try {
|
||||
await realtimeHub.start();
|
||||
|
||||
@@ -0,0 +1,92 @@
|
||||
import { parseGameClockPhase } from '@sammo-ts/common';
|
||||
|
||||
import type { DatabaseClient } from '../context.js';
|
||||
|
||||
const safeInteger = (value: bigint, label: string): number => {
|
||||
const result = Number(value);
|
||||
if (!Number.isSafeInteger(result)) throw new Error(`${label} is outside the safe integer range.`);
|
||||
return result;
|
||||
};
|
||||
|
||||
export const loadClockReadiness = async (db: DatabaseClient) => {
|
||||
if (!db.clockProjectionOutbox) {
|
||||
return {
|
||||
reconciliationComplete: false,
|
||||
gameplayEnabled: false,
|
||||
phase: null,
|
||||
revision: null,
|
||||
deadlineGeneration: null,
|
||||
incompleteOutboxCount: null,
|
||||
};
|
||||
}
|
||||
const [world, incompleteOutboxCount] = await Promise.all([
|
||||
db.worldState.findFirst({
|
||||
orderBy: { id: 'asc' },
|
||||
select: { clockPhase: true, clockRevision: true, deadlineGeneration: true },
|
||||
}),
|
||||
db.clockProjectionOutbox.count({ where: { status: { not: 'APPLIED' } } }),
|
||||
]);
|
||||
if (!world) {
|
||||
return {
|
||||
reconciliationComplete: false,
|
||||
gameplayEnabled: false,
|
||||
phase: null,
|
||||
revision: null,
|
||||
deadlineGeneration: null,
|
||||
incompleteOutboxCount,
|
||||
};
|
||||
}
|
||||
const phase = parseGameClockPhase(world.clockPhase);
|
||||
return {
|
||||
reconciliationComplete: phase !== 'RECONCILING' && incompleteOutboxCount === 0,
|
||||
gameplayEnabled: phase === 'RUNNING' || phase === 'MANUAL',
|
||||
phase,
|
||||
revision: safeInteger(world.clockRevision, 'clock revision'),
|
||||
deadlineGeneration: safeInteger(world.deadlineGeneration, 'deadline generation'),
|
||||
incompleteOutboxCount,
|
||||
};
|
||||
};
|
||||
|
||||
export const loadClockAdminStatus = async (db: DatabaseClient) => {
|
||||
const readiness = await loadClockReadiness(db);
|
||||
if (!db.clockSuspension) {
|
||||
return { ...readiness, latestReconciliation: null };
|
||||
}
|
||||
const latest = await db.clockSuspension.findFirst({
|
||||
orderBy: { createdAt: 'desc' },
|
||||
include: {
|
||||
participants: { orderBy: { participantKey: 'asc' } },
|
||||
projectionOutbox: { orderBy: { id: 'asc' } },
|
||||
},
|
||||
});
|
||||
if (!latest) return { ...readiness, latestReconciliation: null };
|
||||
return {
|
||||
...readiness,
|
||||
latestReconciliation: {
|
||||
id: latest.id,
|
||||
source: latest.source,
|
||||
policy: latest.policy,
|
||||
status: latest.status,
|
||||
sourceRevision: safeInteger(latest.sourceRevision, 'source revision'),
|
||||
targetRevision: safeInteger(latest.targetRevision, 'target revision'),
|
||||
cutTick: safeInteger(latest.cutTick, 'cut tick'),
|
||||
alignedTick: latest.alignedTick === null ? null : safeInteger(latest.alignedTick, 'aligned tick'),
|
||||
participantChecksumBefore: latest.participantChecksumBefore,
|
||||
participantChecksumAfter: latest.participantChecksumAfter,
|
||||
participants: latest.participants.map((participant) => ({
|
||||
key: participant.participantKey,
|
||||
policy: participant.policy,
|
||||
beforeChecksum: participant.beforeChecksum,
|
||||
afterChecksum: participant.afterChecksum,
|
||||
affectedCount: participant.affectedCount,
|
||||
})),
|
||||
outbox: latest.projectionOutbox.map((entry) => ({
|
||||
id: entry.id.toString(),
|
||||
targetRevision: safeInteger(entry.targetRevision, 'outbox target revision'),
|
||||
status: entry.status,
|
||||
attempts: entry.attempts,
|
||||
lastError: entry.lastError,
|
||||
})),
|
||||
},
|
||||
};
|
||||
};
|
||||
@@ -0,0 +1,55 @@
|
||||
import type { CurrentGameTime } from './gameClock.js';
|
||||
|
||||
interface ClockFenceRedis {
|
||||
eval(script: string, options: { keys: string[]; arguments: string[] }): Promise<unknown>;
|
||||
}
|
||||
|
||||
const BOOTSTRAP_CLOCK_FENCE_SCRIPT = `
|
||||
local revision = redis.call('GET', KEYS[1])
|
||||
local generation = redis.call('GET', KEYS[2])
|
||||
local phase = redis.call('GET', KEYS[3])
|
||||
if not revision and not generation and not phase then
|
||||
redis.call('SET', KEYS[1], ARGV[1])
|
||||
redis.call('SET', KEYS[2], ARGV[2])
|
||||
redis.call('SET', KEYS[3], ARGV[3])
|
||||
return 1
|
||||
end
|
||||
if revision == ARGV[1] and generation == ARGV[2] and phase == ARGV[3] then
|
||||
return 2
|
||||
end
|
||||
return 0
|
||||
`;
|
||||
|
||||
export interface ActiveRedisClockFence {
|
||||
activeRevisionKey: string;
|
||||
deadlineGenerationKey: string;
|
||||
phaseKey: string;
|
||||
revision: number;
|
||||
generation: number;
|
||||
}
|
||||
|
||||
export const ensureActiveRedisClockFence = async (
|
||||
redis: ClockFenceRedis,
|
||||
profileName: string,
|
||||
gameTime: CurrentGameTime
|
||||
): Promise<ActiveRedisClockFence | null> => {
|
||||
if (
|
||||
gameTime.phase !== 'RUNNING' ||
|
||||
!Number.isSafeInteger(gameTime.revision) ||
|
||||
!Number.isSafeInteger(gameTime.deadlineGeneration)
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
const fence: ActiveRedisClockFence = {
|
||||
activeRevisionKey: `sammo:${profileName}:clock:active-revision`,
|
||||
deadlineGenerationKey: `sammo:${profileName}:clock:deadline-generation`,
|
||||
phaseKey: `sammo:${profileName}:clock:phase`,
|
||||
revision: gameTime.revision!,
|
||||
generation: gameTime.deadlineGeneration!,
|
||||
};
|
||||
const result = await redis.eval(BOOTSTRAP_CLOCK_FENCE_SCRIPT, {
|
||||
keys: [fence.activeRevisionKey, fence.deadlineGenerationKey, fence.phaseKey],
|
||||
arguments: [String(fence.revision), String(fence.generation), 'RUNNING'],
|
||||
});
|
||||
return Number(result) === 1 || Number(result) === 2 ? fence : null;
|
||||
};
|
||||
@@ -8,6 +8,9 @@ export interface TournamentKeys {
|
||||
sourceRevisionKey: string;
|
||||
sourceRevisionChannel: string;
|
||||
realtimeEventChannel: string;
|
||||
activeClockRevisionKey: string;
|
||||
deadlineGenerationKey: string;
|
||||
clockPhaseKey: string;
|
||||
}
|
||||
|
||||
export const buildTournamentKeys = (profileName: string): TournamentKeys => ({
|
||||
@@ -18,4 +21,7 @@ export const buildTournamentKeys = (profileName: string): TournamentKeys => ({
|
||||
sourceRevisionKey: `sammo:${profileName}:tournament:source-revision`,
|
||||
sourceRevisionChannel: `sammo:${profileName}:tournament:source-changed`,
|
||||
realtimeEventChannel: buildGameEventChannel(profileName),
|
||||
activeClockRevisionKey: `sammo:${profileName}:clock:active-revision`,
|
||||
deadlineGenerationKey: `sammo:${profileName}:clock:deadline-generation`,
|
||||
clockPhaseKey: `sammo:${profileName}:clock:phase`,
|
||||
});
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import { parseTournamentSourceRevision, writeTournamentProjection } from '@sammo-ts/common';
|
||||
import { parseTournamentSourceRevision, writeTournamentProjection, type TournamentClockFence } from '@sammo-ts/common';
|
||||
import { z } from 'zod';
|
||||
|
||||
import type { TournamentKeys } from './keys.js';
|
||||
@@ -37,8 +37,12 @@ const zTournamentState = z
|
||||
openMonth: z.number().int(),
|
||||
termSeconds: z.number(),
|
||||
nextAt: z.string(),
|
||||
nextTick: z.number().int().safe().optional(),
|
||||
clockRevision: z.number().int().positive().safe().optional(),
|
||||
deadlineGeneration: z.number().int().positive().safe().optional(),
|
||||
bettingId: z.number().int().optional(),
|
||||
bettingCloseAt: z.string().optional(),
|
||||
bettingCloseTick: z.number().int().safe().optional(),
|
||||
winnerId: z.number().int().optional(),
|
||||
bettingSettled: z.boolean().optional(),
|
||||
rewardSettled: z.boolean().optional(),
|
||||
@@ -131,12 +135,62 @@ const parseProjection = <T>(raw: string | null, key: string, schema: z.ZodType<T
|
||||
return parsed.data;
|
||||
};
|
||||
|
||||
export interface TournamentClockContext {
|
||||
phase: 'RUNNING';
|
||||
revision: number;
|
||||
deadlineGeneration: number;
|
||||
dateToTick(date: Date): number | null;
|
||||
}
|
||||
|
||||
const parseDeadlineTick = (value: string, context: TournamentClockContext, field: string): number => {
|
||||
const date = new Date(value);
|
||||
if (Number.isNaN(date.getTime())) {
|
||||
throw new Error(`Tournament ${field} is not a valid instant.`);
|
||||
}
|
||||
const tick = context.dateToTick(date);
|
||||
if (tick === null || !Number.isSafeInteger(tick)) {
|
||||
throw new Error(`Tournament ${field} cannot be represented in the active game clock.`);
|
||||
}
|
||||
return tick;
|
||||
};
|
||||
|
||||
export const stampTournamentClock = (state: TournamentState, context: TournamentClockContext): TournamentState => ({
|
||||
...state,
|
||||
nextTick: parseDeadlineTick(state.nextAt, context, 'nextAt'),
|
||||
clockRevision: context.revision,
|
||||
deadlineGeneration: context.deadlineGeneration,
|
||||
...(state.bettingCloseAt
|
||||
? { bettingCloseTick: parseDeadlineTick(state.bettingCloseAt, context, 'bettingCloseAt') }
|
||||
: { bettingCloseTick: undefined }),
|
||||
});
|
||||
|
||||
const toClockFence = (keys: TournamentKeys, context: TournamentClockContext): TournamentClockFence => ({
|
||||
activeRevisionKey: keys.activeClockRevisionKey,
|
||||
deadlineGenerationKey: keys.deadlineGenerationKey,
|
||||
phaseKey: keys.clockPhaseKey,
|
||||
revision: context.revision,
|
||||
deadlineGeneration: context.deadlineGeneration,
|
||||
phase: context.phase,
|
||||
});
|
||||
|
||||
export class TournamentStore {
|
||||
private clockContext: TournamentClockContext | null = null;
|
||||
|
||||
constructor(
|
||||
private readonly redis: RedisClientLike,
|
||||
private readonly keys: TournamentKeys
|
||||
) {}
|
||||
|
||||
async withClockContext<T>(context: TournamentClockContext, operation: () => Promise<T>): Promise<T> {
|
||||
const previous = this.clockContext;
|
||||
this.clockContext = context;
|
||||
try {
|
||||
return await operation();
|
||||
} finally {
|
||||
this.clockContext = previous;
|
||||
}
|
||||
}
|
||||
|
||||
async withMutationLock<T>(operation: () => Promise<T>, timeoutMs = 2_000): Promise<T> {
|
||||
if (!this.redis.del) {
|
||||
return operation();
|
||||
@@ -174,11 +228,15 @@ export class TournamentStore {
|
||||
}
|
||||
|
||||
private async writeWithSourceRevision(key: string, value: unknown): Promise<string> {
|
||||
return writeTournamentProjection(this.redis, this.keys, [{ key, value }]);
|
||||
const fence = this.clockContext ? toClockFence(this.keys, this.clockContext) : undefined;
|
||||
return writeTournamentProjection(this.redis, this.keys, [{ key, value }], fence);
|
||||
}
|
||||
|
||||
async setState(state: TournamentState): Promise<string> {
|
||||
return this.writeWithSourceRevision(this.keys.stateKey, state);
|
||||
return this.writeWithSourceRevision(
|
||||
this.keys.stateKey,
|
||||
this.clockContext ? stampTournamentClock(state, this.clockContext) : state
|
||||
);
|
||||
}
|
||||
|
||||
async getParticipants(): Promise<TournamentParticipantEntry[]> {
|
||||
|
||||
@@ -9,8 +9,12 @@ export interface TournamentState {
|
||||
openMonth: number;
|
||||
termSeconds: number;
|
||||
nextAt: string;
|
||||
nextTick?: number;
|
||||
clockRevision?: number;
|
||||
deadlineGeneration?: number;
|
||||
bettingId?: number;
|
||||
bettingCloseAt?: string;
|
||||
bettingCloseTick?: number;
|
||||
winnerId?: number;
|
||||
bettingSettled?: boolean;
|
||||
rewardSettled?: boolean;
|
||||
|
||||
@@ -13,9 +13,10 @@ import { DatabaseTurnDaemonTransport } from '../daemon/databaseTransport.js';
|
||||
import { createBestEffortResourceCloser } from '../services/bestEffortResourceCloser.js';
|
||||
import { loadCurrentGameTime } from '../services/gameClock.js';
|
||||
import { createPollingWorkerControl, waitForWorkerPoll } from '../services/pollingWorkerLifecycle.js';
|
||||
import { ensureActiveRedisClockFence } from '../services/redisClockFence.js';
|
||||
import type { TurnDaemonTransport } from '../daemon/transport.js';
|
||||
import { buildTournamentKeys } from './keys.js';
|
||||
import { TournamentStore } from './store.js';
|
||||
import { TournamentStore, stampTournamentClock, type TournamentClockContext } from './store.js';
|
||||
import type { TournamentMatchEntry, TournamentState } from './types.js';
|
||||
import {
|
||||
applyGroupMatch,
|
||||
@@ -595,41 +596,63 @@ export const processTournamentTick = async (options: {
|
||||
prisma: GamePrismaClient;
|
||||
daemonTransport: TurnDaemonTransport;
|
||||
now?: () => number;
|
||||
clockContext?: TournamentClockContext;
|
||||
}): Promise<TournamentState | null> => {
|
||||
const { store, prisma, daemonTransport } = options;
|
||||
const now = options.now ?? Date.now;
|
||||
let processedState: TournamentState | null = null;
|
||||
|
||||
await store.withMutationLock(async () => {
|
||||
const state = await store.getState();
|
||||
if (!state || (!state.auto && !needsSettlement(state))) {
|
||||
return;
|
||||
}
|
||||
const nextAt = new Date(state.nextAt).getTime();
|
||||
if (state.auto && Number.isFinite(nextAt) && nextAt > now()) {
|
||||
return;
|
||||
}
|
||||
const processWithLock = async (): Promise<void> =>
|
||||
store.withMutationLock(async () => {
|
||||
const state = await store.getState();
|
||||
if (!state || (!state.auto && !needsSettlement(state))) {
|
||||
return;
|
||||
}
|
||||
if (
|
||||
options.clockContext &&
|
||||
(state.clockRevision !== options.clockContext.revision ||
|
||||
state.deadlineGeneration !== options.clockContext.deadlineGeneration)
|
||||
) {
|
||||
throw new Error('Tournament state does not match the active clock revision.');
|
||||
}
|
||||
const nextAt = new Date(state.nextAt).getTime();
|
||||
const notDue = options.clockContext
|
||||
? !Number.isSafeInteger(state.nextTick) ||
|
||||
state.nextTick! > options.clockContext.dateToTick(new Date(now()))!
|
||||
: Number.isFinite(nextAt) && nextAt > now();
|
||||
if (state.auto && notDue) {
|
||||
if (options.clockContext && !Number.isSafeInteger(state.nextTick)) {
|
||||
throw new Error('Active tournament nextAt lacks the authoritative nextTick dual-write.');
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
if (needsSettlement(state)) {
|
||||
processedState = (await settleTournamentOutcome({ store, daemonTransport, state })) ?? state;
|
||||
return;
|
||||
}
|
||||
if (needsSettlement(state)) {
|
||||
processedState = (await settleTournamentOutcome({ store, daemonTransport, state })) ?? state;
|
||||
return;
|
||||
}
|
||||
|
||||
const worldState = await prisma.worldState.findFirst();
|
||||
const baseSeed = (worldState?.meta as Record<string, unknown> | null)?.hiddenSeed ?? 'tournament';
|
||||
let nextState = state;
|
||||
if (isBattleStage(state.stage)) {
|
||||
nextState = await applyBattle(store, state, String(baseSeed), daemonTransport);
|
||||
} else if (isPreBattleStage(state.stage)) {
|
||||
nextState = await applyPreBattleStage(store, prisma, state, String(baseSeed), daemonTransport, now);
|
||||
}
|
||||
processedState =
|
||||
(await settleTournamentOutcome({
|
||||
store,
|
||||
daemonTransport,
|
||||
state: nextState,
|
||||
})) ?? nextState;
|
||||
});
|
||||
const worldState = await prisma.worldState.findFirst();
|
||||
const baseSeed = (worldState?.meta as Record<string, unknown> | null)?.hiddenSeed ?? 'tournament';
|
||||
let nextState = state;
|
||||
if (isBattleStage(state.stage)) {
|
||||
nextState = await applyBattle(store, state, String(baseSeed), daemonTransport);
|
||||
} else if (isPreBattleStage(state.stage)) {
|
||||
nextState = await applyPreBattleStage(store, prisma, state, String(baseSeed), daemonTransport, now);
|
||||
}
|
||||
processedState =
|
||||
(await settleTournamentOutcome({
|
||||
store,
|
||||
daemonTransport,
|
||||
state: nextState,
|
||||
})) ?? nextState;
|
||||
});
|
||||
|
||||
if (options.clockContext) {
|
||||
await store.withClockContext(options.clockContext, processWithLock);
|
||||
} else {
|
||||
await processWithLock();
|
||||
}
|
||||
|
||||
return processedState;
|
||||
};
|
||||
@@ -656,16 +679,60 @@ export const runTournamentWorker = async (options: TournamentWorkerOptions = {})
|
||||
|
||||
try {
|
||||
while (!control.signal.aborted) {
|
||||
const state = await store.getState();
|
||||
let state = await store.getState();
|
||||
if (!state || (!state.auto && !needsSettlement(state))) {
|
||||
await waitForWorkerPoll(control.signal, config.tournamentPollMs);
|
||||
continue;
|
||||
}
|
||||
|
||||
const gameTime = await loadCurrentGameTime(postgres.prisma);
|
||||
if (gameTime.phase && gameTime.phase !== 'RUNNING') {
|
||||
await waitForWorkerPoll(control.signal, config.tournamentPollMs);
|
||||
continue;
|
||||
}
|
||||
const clockFence = gameTime.phase
|
||||
? await ensureActiveRedisClockFence(redis.client, config.profileName, gameTime)
|
||||
: null;
|
||||
if (gameTime.phase && !clockFence) {
|
||||
await waitForWorkerPoll(control.signal, config.tournamentPollMs);
|
||||
continue;
|
||||
}
|
||||
const clockContext: TournamentClockContext | undefined = clockFence
|
||||
? {
|
||||
phase: 'RUNNING',
|
||||
revision: clockFence.revision,
|
||||
deadlineGeneration: clockFence.generation,
|
||||
dateToTick: gameTime.dateToTick,
|
||||
}
|
||||
: undefined;
|
||||
if (
|
||||
clockContext &&
|
||||
(!Number.isSafeInteger(state.nextTick) ||
|
||||
state.clockRevision === undefined ||
|
||||
state.deadlineGeneration === undefined)
|
||||
) {
|
||||
state = stampTournamentClock(state, clockContext);
|
||||
await store.withClockContext(clockContext, () => store.setState(state!));
|
||||
}
|
||||
if (
|
||||
clockContext &&
|
||||
(state.clockRevision !== clockContext.revision ||
|
||||
state.deadlineGeneration !== clockContext.deadlineGeneration)
|
||||
) {
|
||||
await waitForWorkerPoll(control.signal, config.tournamentPollMs);
|
||||
continue;
|
||||
}
|
||||
|
||||
const nextAt = new Date(state.nextAt).getTime();
|
||||
const gameNow = (await loadCurrentGameTime(postgres.prisma)).now.getTime();
|
||||
if (state.auto && Number.isFinite(nextAt) && nextAt > gameNow) {
|
||||
await waitForWorkerPoll(control.signal, Math.min(config.tournamentPollMs, nextAt - gameNow));
|
||||
const gameNow = gameTime.now.getTime();
|
||||
const notDue = clockContext
|
||||
? state.nextTick! > gameTime.tick!
|
||||
: Number.isFinite(nextAt) && nextAt > gameNow;
|
||||
if (state.auto && notDue) {
|
||||
await waitForWorkerPoll(
|
||||
control.signal,
|
||||
Math.min(config.tournamentPollMs, Math.max(1, nextAt - gameNow))
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
@@ -675,6 +742,7 @@ export const runTournamentWorker = async (options: TournamentWorkerOptions = {})
|
||||
prisma: postgres.prisma,
|
||||
daemonTransport,
|
||||
now: () => gameNow,
|
||||
clockContext,
|
||||
});
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : 'Unknown error';
|
||||
@@ -700,7 +768,11 @@ export const runTournamentWorker = async (options: TournamentWorkerOptions = {})
|
||||
lastError: message,
|
||||
lastErrorAt: failedAt,
|
||||
};
|
||||
await store.setState(nextState);
|
||||
if (clockContext) {
|
||||
await store.withClockContext(clockContext, () => store.setState(nextState));
|
||||
} else {
|
||||
await store.setState(nextState);
|
||||
}
|
||||
}
|
||||
|
||||
await waitForWorkerPoll(control.signal, config.tournamentPollMs);
|
||||
|
||||
@@ -2,7 +2,7 @@ import { describe, expect, it, vi } from 'vitest';
|
||||
|
||||
import type { GamePrismaClient } from '@sammo-ts/infra';
|
||||
|
||||
import { processDueAuctionId, reconcilePendingAuctionTimers } from '../src/auction/worker.js';
|
||||
import { popDueAuctionIds, processDueAuctionId, reconcilePendingAuctionTimers } from '../src/auction/worker.js';
|
||||
import { resolveAuctionSeedScore } from '../src/auction/scheduler.js';
|
||||
|
||||
const buildRedis = () => ({
|
||||
@@ -55,6 +55,38 @@ const buildDb = (options: {
|
||||
};
|
||||
|
||||
describe('auction worker clock-shift race', () => {
|
||||
it('uses one Redis script for revision, generation, phase, due-read, and removal', async () => {
|
||||
const redis = { ...buildRedis(), eval: vi.fn(async () => ['7', '9']) };
|
||||
await expect(
|
||||
popDueAuctionIds(redis, 'auction-timer', 123_456, 100, {
|
||||
activeRevisionKey: 'clock-revision',
|
||||
deadlineGenerationKey: 'deadline-generation',
|
||||
phaseKey: 'clock-phase',
|
||||
revision: 4,
|
||||
generation: 8,
|
||||
})
|
||||
).resolves.toEqual(['7', '9']);
|
||||
expect(redis.eval).toHaveBeenCalledWith(expect.stringContaining('ZRANGEBYSCORE'), {
|
||||
keys: ['auction-timer', 'clock-revision', 'deadline-generation', 'clock-phase'],
|
||||
arguments: ['4', '8', '123456', '100'],
|
||||
});
|
||||
expect(redis.zRangeByScore).not.toHaveBeenCalled();
|
||||
expect(redis.zRem).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('returns no due member when the atomic Redis clock fence rejects the pop', async () => {
|
||||
const redis = { ...buildRedis(), eval: vi.fn(async () => ['__CLOCK_FENCE__']) };
|
||||
await expect(
|
||||
popDueAuctionIds(redis, 'auction-timer', 123_456, 100, {
|
||||
activeRevisionKey: 'clock-revision',
|
||||
deadlineGenerationKey: 'deadline-generation',
|
||||
phaseKey: 'clock-phase',
|
||||
revision: 4,
|
||||
generation: 8,
|
||||
})
|
||||
).resolves.toEqual([]);
|
||||
});
|
||||
|
||||
it('seeds OPEN at its deadline but retries FINALIZING at the current logical tick', () => {
|
||||
const now = new Date('2026-07-30T12:00:00.000Z');
|
||||
const time = {
|
||||
@@ -292,6 +324,7 @@ describe('auction worker clock-shift race', () => {
|
||||
requestId,
|
||||
target: 'ENGINE',
|
||||
eventType: 'auctionFinalize',
|
||||
acceptedGameTick: 72_000_000n,
|
||||
payload: {
|
||||
type: 'auctionFinalize',
|
||||
requestId,
|
||||
|
||||
@@ -0,0 +1,87 @@
|
||||
import { describe, expect, it, vi } from 'vitest';
|
||||
|
||||
import type { DatabaseClient } from '../src/context.js';
|
||||
import { loadClockAdminStatus, loadClockReadiness } from '../src/services/clockReadiness.js';
|
||||
|
||||
describe('clock reconciliation readiness', () => {
|
||||
it('fails closed when the reconciliation schema is not available', async () => {
|
||||
const db = {} as DatabaseClient;
|
||||
await expect(loadClockReadiness(db)).resolves.toEqual({
|
||||
reconciliationComplete: false,
|
||||
gameplayEnabled: false,
|
||||
phase: null,
|
||||
revision: null,
|
||||
deadlineGeneration: null,
|
||||
incompleteOutboxCount: null,
|
||||
});
|
||||
});
|
||||
|
||||
it('blocks readiness for RECONCILING or incomplete outbox state', async () => {
|
||||
const db = {
|
||||
worldState: {
|
||||
findFirst: vi.fn(async () => ({
|
||||
clockPhase: 'RECONCILING',
|
||||
clockRevision: 9n,
|
||||
deadlineGeneration: 4n,
|
||||
})),
|
||||
},
|
||||
clockProjectionOutbox: { count: vi.fn(async () => 1) },
|
||||
} as unknown as DatabaseClient;
|
||||
await expect(loadClockReadiness(db)).resolves.toMatchObject({
|
||||
reconciliationComplete: false,
|
||||
gameplayEnabled: false,
|
||||
phase: 'RECONCILING',
|
||||
revision: 9,
|
||||
deadlineGeneration: 4,
|
||||
incompleteOutboxCount: 1,
|
||||
});
|
||||
});
|
||||
|
||||
it('exposes participant checksums and incomplete outbox detail to admins', async () => {
|
||||
const db = {
|
||||
worldState: {
|
||||
findFirst: vi.fn(async () => ({
|
||||
clockPhase: 'RUNNING',
|
||||
clockRevision: 3n,
|
||||
deadlineGeneration: 2n,
|
||||
})),
|
||||
},
|
||||
clockProjectionOutbox: { count: vi.fn(async () => 0) },
|
||||
clockSuspension: {
|
||||
findFirst: vi.fn(async () => ({
|
||||
id: 'maintenance-1',
|
||||
source: 'MAINTENANCE',
|
||||
policy: 'EXACT',
|
||||
status: 'APPLIED',
|
||||
sourceRevision: 2n,
|
||||
targetRevision: 3n,
|
||||
cutTick: 100n,
|
||||
alignedTick: 130n,
|
||||
participantChecksumBefore: 'before-all',
|
||||
participantChecksumAfter: 'after-all',
|
||||
participants: [
|
||||
{
|
||||
participantKey: 'general-turn',
|
||||
policy: 'SHIFT',
|
||||
beforeChecksum: 'before',
|
||||
afterChecksum: 'after',
|
||||
affectedCount: 2,
|
||||
},
|
||||
],
|
||||
projectionOutbox: [{ id: 8n, targetRevision: 3n, status: 'APPLIED', attempts: 1, lastError: null }],
|
||||
})),
|
||||
},
|
||||
} as unknown as DatabaseClient;
|
||||
|
||||
await expect(loadClockAdminStatus(db)).resolves.toMatchObject({
|
||||
reconciliationComplete: true,
|
||||
latestReconciliation: {
|
||||
id: 'maintenance-1',
|
||||
participantChecksumBefore: 'before-all',
|
||||
participantChecksumAfter: 'after-all',
|
||||
participants: [{ key: 'general-turn', policy: 'SHIFT', affectedCount: 2 }],
|
||||
outbox: [{ id: '8', status: 'APPLIED' }],
|
||||
},
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -511,6 +511,10 @@ integration('API input event boundary', () => {
|
||||
it('reuses the same engine child event but rejects a changed retry payload', async () => {
|
||||
const transport = new DatabaseTurnDaemonTransport(db, 100);
|
||||
const requestId = 'integration:api:engine-child';
|
||||
const worldClock = await db.worldState.findFirst({
|
||||
orderBy: { id: 'asc' },
|
||||
select: { clockRevision: true, deadlineGeneration: true },
|
||||
});
|
||||
const acceptedWindowStart = Date.now();
|
||||
await transport.sendCommand({ type: 'vacation', requestId, userId: 'user-7', generalId: 7 });
|
||||
const acceptedWindowEnd = Date.now();
|
||||
@@ -518,6 +522,9 @@ integration('API input event boundary', () => {
|
||||
expect(event.actorUserId).toBe('user-7');
|
||||
expect(event.createdAt.getTime()).toBeGreaterThanOrEqual(acceptedWindowStart);
|
||||
expect(event.createdAt.getTime()).toBeLessThanOrEqual(acceptedWindowEnd);
|
||||
expect(event.acceptedGameTick).not.toBeNull();
|
||||
expect(event.acceptedClockRevision).toBe(worldClock?.clockRevision ?? null);
|
||||
expect(event.acceptedDeadlineGeneration).toBe(worldClock?.deadlineGeneration ?? null);
|
||||
await expect(
|
||||
transport.sendCommand({ type: 'vacation', requestId, userId: 'user-7', generalId: 7 })
|
||||
).resolves.toBe(requestId);
|
||||
|
||||
@@ -28,6 +28,9 @@ const createContext = (payload: unknown = {}) => {
|
||||
status: 'PENDING',
|
||||
result: null,
|
||||
attempts: 0,
|
||||
acceptedGameTick: 100n,
|
||||
acceptedClockRevision: 3n,
|
||||
acceptedDeadlineGeneration: 2n,
|
||||
},
|
||||
];
|
||||
}
|
||||
@@ -36,8 +39,8 @@ const createContext = (payload: unknown = {}) => {
|
||||
});
|
||||
const transaction = {
|
||||
$queryRaw: queryRaw,
|
||||
$executeRaw: vi.fn(async () => {
|
||||
order.push('accepted');
|
||||
$executeRaw: vi.fn(async (query: { sql?: string }) => {
|
||||
order.push(query.sql?.includes('pg_advisory_xact_lock') ? 'clock-fence' : 'accepted');
|
||||
return 1;
|
||||
}),
|
||||
$executeRawUnsafe: vi.fn(async (statement: string) => {
|
||||
@@ -88,6 +91,7 @@ describe('API input-event change journal boundary', () => {
|
||||
|
||||
expect(fixture.order).toEqual([
|
||||
'transaction-begin',
|
||||
'clock-fence',
|
||||
'accepted',
|
||||
'locked',
|
||||
'processing',
|
||||
@@ -112,6 +116,7 @@ describe('API input-event change journal boundary', () => {
|
||||
|
||||
expect(fixture.order).toEqual([
|
||||
'transaction-begin',
|
||||
'clock-fence',
|
||||
'accepted',
|
||||
'locked',
|
||||
'processing',
|
||||
|
||||
@@ -30,11 +30,31 @@ class MemoryRedis {
|
||||
}
|
||||
|
||||
async eval(_script: string, options: { keys: string[]; arguments: string[] }): Promise<string> {
|
||||
const [valueKey, revisionKey] = options.keys;
|
||||
const [value] = options.arguments;
|
||||
if (options.keys.length === 3 && options.keys[0]?.endsWith(':clock:active-revision')) {
|
||||
const current = options.keys.map((key) => this.values.get(key));
|
||||
if (current.every((value) => value === undefined)) {
|
||||
options.keys.forEach((key, index) => this.values.set(key, options.arguments[index]!));
|
||||
return '1';
|
||||
}
|
||||
return current.every((value, index) => value === options.arguments[index]) ? '2' : '0';
|
||||
}
|
||||
const fenced = options.keys.at(-1)?.endsWith(':clock:phase') === true;
|
||||
const writeCount = options.keys.length - (fenced ? 4 : 1);
|
||||
if (fenced) {
|
||||
const clockKeys = options.keys.slice(-3);
|
||||
const expected = options.arguments.slice(-3);
|
||||
if (!clockKeys.every((key, index) => this.values.get(key) === expected[index])) {
|
||||
return '__CLOCK_FENCE__';
|
||||
}
|
||||
}
|
||||
const valueKey = options.keys[0];
|
||||
const revisionKey = options.keys[writeCount];
|
||||
const value = options.arguments[0];
|
||||
if (!valueKey || !revisionKey || value === undefined) throw new Error('invalid eval arguments');
|
||||
const revision = Number(this.values.get(revisionKey) ?? '0') + 1;
|
||||
this.values.set(valueKey, value);
|
||||
for (let index = 0; index < writeCount; index += 1) {
|
||||
this.values.set(options.keys[index]!, options.arguments[index]!);
|
||||
}
|
||||
this.values.set(revisionKey, String(revision));
|
||||
return String(revision);
|
||||
}
|
||||
@@ -144,6 +164,14 @@ const buildContext = (options: {
|
||||
},
|
||||
worldState: {
|
||||
findFirst: async () => ({
|
||||
clockBaseTime: new Date('2026-01-01T00:00:00.000Z'),
|
||||
clockTick: 0n,
|
||||
clockMode: 'realtime',
|
||||
clockWallAnchor: new Date('2026-01-01T00:00:00.000Z'),
|
||||
clockPhase: 'RUNNING',
|
||||
clockRevision: 1n,
|
||||
deadlineGeneration: 1n,
|
||||
tickSeconds: 60,
|
||||
config: { const: { develCost: options.develCost ?? 200 } },
|
||||
...(options.currentDevelCost === undefined ? {} : { meta: { develcost: options.currentDevelCost } }),
|
||||
}),
|
||||
@@ -394,7 +422,10 @@ describe('tournament router permissions and mutations', () => {
|
||||
roles: ['admin.tournament:che:default'],
|
||||
})
|
||||
);
|
||||
await expect(adminCaller.tournament.getAdminStatus()).resolves.toEqual({ ok: true });
|
||||
await expect(adminCaller.tournament.getAdminStatus()).resolves.toMatchObject({
|
||||
ok: true,
|
||||
clock: { reconciliationComplete: false, latestReconciliation: null },
|
||||
});
|
||||
});
|
||||
|
||||
it('applies the admin role boundary to every tournament mutation', async () => {
|
||||
@@ -479,9 +510,7 @@ describe('tournament router permissions and mutations', () => {
|
||||
])
|
||||
).resolves.toEqual({ ok: true, count: 1 });
|
||||
await expect(
|
||||
caller.tournament.setBettingEntries([
|
||||
{ generalId: general.id, targetId: rival.id, amount: 100 },
|
||||
])
|
||||
caller.tournament.setBettingEntries([{ generalId: general.id, targetId: rival.id, amount: 100 }])
|
||||
).resolves.toEqual({ ok: true, count: 1 });
|
||||
await expect(caller.tournament.seedParticipants({ generalIds: [general.id, rival.id] })).resolves.toEqual({
|
||||
ok: true,
|
||||
|
||||
@@ -41,6 +41,9 @@ integration('TournamentStore Redis source revision', () => {
|
||||
keys.matchesKey,
|
||||
keys.bettingKey,
|
||||
keys.sourceRevisionKey,
|
||||
keys.activeClockRevisionKey,
|
||||
keys.deadlineGenerationKey,
|
||||
keys.clockPhaseKey,
|
||||
]);
|
||||
if (subscriber) {
|
||||
await subscriber.client.unsubscribe(keys.sourceRevisionChannel);
|
||||
@@ -124,4 +127,48 @@ integration('TournamentStore Redis source revision', () => {
|
||||
}),
|
||||
]);
|
||||
});
|
||||
|
||||
it('dual-writes deadline ticks and rejects a Redis clock revision race atomically', async () => {
|
||||
const store = new TournamentStore(connector.client, keys);
|
||||
await Promise.all([
|
||||
connector.client.set(keys.activeClockRevisionKey, '7'),
|
||||
connector.client.set(keys.deadlineGenerationKey, '3'),
|
||||
connector.client.set(keys.clockPhaseKey, 'RUNNING'),
|
||||
]);
|
||||
const clockContext = {
|
||||
phase: 'RUNNING' as const,
|
||||
revision: 7,
|
||||
deadlineGeneration: 3,
|
||||
dateToTick: (date: Date) => Math.trunc(date.getTime() / 1_000),
|
||||
};
|
||||
const nextAt = '2026-09-03T10:00:00.000Z';
|
||||
await store.withClockContext(clockContext, () =>
|
||||
store.setState({
|
||||
stage: 6,
|
||||
phase: 0,
|
||||
type: 0,
|
||||
auto: true,
|
||||
openYear: 200,
|
||||
openMonth: 1,
|
||||
termSeconds: 10,
|
||||
nextAt,
|
||||
bettingCloseAt: '2026-09-03T09:59:50.000Z',
|
||||
})
|
||||
);
|
||||
await expect(store.getState()).resolves.toMatchObject({
|
||||
nextTick: Math.trunc(new Date(nextAt).getTime() / 1_000),
|
||||
bettingCloseTick: Math.trunc(new Date('2026-09-03T09:59:50.000Z').getTime() / 1_000),
|
||||
clockRevision: 7,
|
||||
deadlineGeneration: 3,
|
||||
});
|
||||
|
||||
const beforeRevision = await store.getSourceRevision();
|
||||
await connector.client.set(keys.activeClockRevisionKey, '8');
|
||||
await expect(
|
||||
store.withClockContext(clockContext, () =>
|
||||
store.setMatches([{ id: 99, stage: 7, roundIndex: 0, attackerId: 1, defenderId: 2 }])
|
||||
)
|
||||
).rejects.toThrow('clock revision fence failed');
|
||||
await expect(store.getSourceRevision()).resolves.toBe(beforeRevision);
|
||||
});
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user