fix: 10분 중단 즉시 복구와 대기 중 토너먼트 참가 허용
This commit is contained in:
@@ -11,7 +11,11 @@ import { assignManualApplicantGroup } from '../../tournament/workerHelpers.js';
|
|||||||
import { accessAuthedProcedure, authedProcedure, engineAuthedProcedure, procedure, router } from '../../trpc.js';
|
import { accessAuthedProcedure, authedProcedure, engineAuthedProcedure, procedure, router } from '../../trpc.js';
|
||||||
import { getMyGeneral } from '../shared/general.js';
|
import { getMyGeneral } from '../shared/general.js';
|
||||||
import { loadCurrentGameTime } from '../../services/gameClock.js';
|
import { loadCurrentGameTime } from '../../services/gameClock.js';
|
||||||
import { ensureActiveRedisClockFence, ensureBettingRedisClockFence } from '../../services/redisClockFence.js';
|
import {
|
||||||
|
ensureActiveRedisClockFence,
|
||||||
|
ensureBettingRedisClockFence,
|
||||||
|
ensureTournamentParticipationRedisClockFence,
|
||||||
|
} from '../../services/redisClockFence.js';
|
||||||
import { loadClockAdminStatus } from '../../services/clockReadiness.js';
|
import { loadClockAdminStatus } from '../../services/clockReadiness.js';
|
||||||
|
|
||||||
const hasAdminRole = (roles: string[], profileName: string): boolean => {
|
const hasAdminRole = (roles: string[], profileName: string): boolean => {
|
||||||
@@ -63,10 +67,11 @@ const withTournamentClockMutation = async <T>(
|
|||||||
tournamentMutationLockHeld?: boolean;
|
tournamentMutationLockHeld?: boolean;
|
||||||
},
|
},
|
||||||
store: TournamentStore,
|
store: TournamentStore,
|
||||||
operation: () => Promise<T>
|
operation: () => Promise<T>,
|
||||||
|
ensureFence = ensureActiveRedisClockFence
|
||||||
): Promise<T> => {
|
): Promise<T> => {
|
||||||
const gameTime = await loadCurrentGameTime(ctx.db);
|
const gameTime = await loadCurrentGameTime(ctx.db);
|
||||||
const fence = await ensureActiveRedisClockFence(ctx.redis, ctx.profile.name, gameTime);
|
const fence = await ensureFence(ctx.redis, ctx.profile.name, gameTime);
|
||||||
if (!fence) {
|
if (!fence) {
|
||||||
throw new TRPCError({
|
throw new TRPCError({
|
||||||
code: 'PRECONDITION_FAILED',
|
code: 'PRECONDITION_FAILED',
|
||||||
@@ -473,70 +478,75 @@ export const tournamentRouter = router({
|
|||||||
join: engineAuthedProcedure.mutation(async ({ ctx }) => {
|
join: engineAuthedProcedure.mutation(async ({ ctx }) => {
|
||||||
const general = await getMyGeneral(ctx);
|
const general = await getMyGeneral(ctx);
|
||||||
const store = new TournamentStore(ctx.redis, buildTournamentKeys(ctx.profile.name));
|
const store = new TournamentStore(ctx.redis, buildTournamentKeys(ctx.profile.name));
|
||||||
return withTournamentClockMutation(ctx, store, async () => {
|
return withTournamentClockMutation(
|
||||||
const state = await store.getState();
|
ctx,
|
||||||
if (!state || state.stage !== 1 || state.participantsLockedAt) {
|
store,
|
||||||
throw new TRPCError({ code: 'BAD_REQUEST', message: '참가 신청 기간이 아닙니다.' });
|
async () => {
|
||||||
}
|
const state = await store.getState();
|
||||||
|
if (!state || state.stage !== 1 || state.participantsLockedAt) {
|
||||||
|
throw new TRPCError({ code: 'BAD_REQUEST', message: '참가 신청 기간이 아닙니다.' });
|
||||||
|
}
|
||||||
|
|
||||||
const [participants, worldState] = await Promise.all([
|
const [participants, worldState] = await Promise.all([
|
||||||
store.getParticipants(),
|
store.getParticipants(),
|
||||||
ctx.db.worldState.findFirst(),
|
ctx.db.worldState.findFirst(),
|
||||||
]);
|
]);
|
||||||
if (participants.some((entry) => entry.id === general.id)) {
|
if (participants.some((entry) => entry.id === general.id)) {
|
||||||
return { ok: true, count: participants.length };
|
return { ok: true, count: participants.length };
|
||||||
}
|
}
|
||||||
if (participants.length >= 64) {
|
if (participants.length >= 64) {
|
||||||
throw new TRPCError({ code: 'BAD_REQUEST', message: '참가 인원이 가득 찼습니다.' });
|
throw new TRPCError({ code: 'BAD_REQUEST', message: '참가 인원이 가득 찼습니다.' });
|
||||||
}
|
}
|
||||||
|
|
||||||
const develCost = resolveCurrentDevelCost(worldState);
|
const develCost = resolveCurrentDevelCost(worldState);
|
||||||
const feeResult = await ctx.turnDaemon.requestCommand({
|
const feeResult = await ctx.turnDaemon.requestCommand({
|
||||||
type: 'adjustGeneralResources',
|
|
||||||
requestId: tournamentJoinCommandRequestId(ctx.requestId, 'resources'),
|
|
||||||
reason: 'tournamentJoin',
|
|
||||||
adjustments: [{ generalId: general.id, goldDelta: -develCost, minGoldAfter: 0 }],
|
|
||||||
});
|
|
||||||
if (!feeResult || feeResult.type !== 'adjustGeneralResources') {
|
|
||||||
throw new TRPCError({ code: 'INTERNAL_SERVER_ERROR', message: 'Unexpected response' });
|
|
||||||
}
|
|
||||||
if (!feeResult.ok || feeResult.processed !== 1) {
|
|
||||||
throw new TRPCError({
|
|
||||||
code: 'BAD_REQUEST',
|
|
||||||
message: feeResult.ok ? '금이 부족합니다.' : feeResult.reason,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
const meta = asRecord(general.meta);
|
|
||||||
const level = typeof meta.explevel === 'number' ? meta.explevel : 0;
|
|
||||||
const applicant = assignManualApplicantGroup({
|
|
||||||
state,
|
|
||||||
baseSeed: String(asRecord(worldState?.meta).hiddenSeed ?? 'tournament'),
|
|
||||||
current: participants,
|
|
||||||
applicant: {
|
|
||||||
id: general.id,
|
|
||||||
name: general.name,
|
|
||||||
leadership: general.leadership,
|
|
||||||
strength: general.strength,
|
|
||||||
intel: general.intel,
|
|
||||||
level,
|
|
||||||
},
|
|
||||||
});
|
|
||||||
const next = participants.concat(applicant);
|
|
||||||
|
|
||||||
try {
|
|
||||||
await store.setParticipants(next);
|
|
||||||
} catch (error) {
|
|
||||||
await ctx.turnDaemon.requestCommand({
|
|
||||||
type: 'adjustGeneralResources',
|
type: 'adjustGeneralResources',
|
||||||
requestId: tournamentJoinCommandRequestId(ctx.requestId, 'projection-rollback-resources'),
|
requestId: tournamentJoinCommandRequestId(ctx.requestId, 'resources'),
|
||||||
reason: 'tournamentJoinRollback',
|
reason: 'tournamentJoin',
|
||||||
adjustments: [{ generalId: general.id, goldDelta: develCost }],
|
adjustments: [{ generalId: general.id, goldDelta: -develCost, minGoldAfter: 0 }],
|
||||||
});
|
});
|
||||||
throw error;
|
if (!feeResult || feeResult.type !== 'adjustGeneralResources') {
|
||||||
}
|
throw new TRPCError({ code: 'INTERNAL_SERVER_ERROR', message: 'Unexpected response' });
|
||||||
return { ok: true, count: next.length };
|
}
|
||||||
});
|
if (!feeResult.ok || feeResult.processed !== 1) {
|
||||||
|
throw new TRPCError({
|
||||||
|
code: 'BAD_REQUEST',
|
||||||
|
message: feeResult.ok ? '금이 부족합니다.' : feeResult.reason,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
const meta = asRecord(general.meta);
|
||||||
|
const level = typeof meta.explevel === 'number' ? meta.explevel : 0;
|
||||||
|
const applicant = assignManualApplicantGroup({
|
||||||
|
state,
|
||||||
|
baseSeed: String(asRecord(worldState?.meta).hiddenSeed ?? 'tournament'),
|
||||||
|
current: participants,
|
||||||
|
applicant: {
|
||||||
|
id: general.id,
|
||||||
|
name: general.name,
|
||||||
|
leadership: general.leadership,
|
||||||
|
strength: general.strength,
|
||||||
|
intel: general.intel,
|
||||||
|
level,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
const next = participants.concat(applicant);
|
||||||
|
|
||||||
|
try {
|
||||||
|
await store.setParticipants(next);
|
||||||
|
} catch (error) {
|
||||||
|
await ctx.turnDaemon.requestCommand({
|
||||||
|
type: 'adjustGeneralResources',
|
||||||
|
requestId: tournamentJoinCommandRequestId(ctx.requestId, 'projection-rollback-resources'),
|
||||||
|
reason: 'tournamentJoinRollback',
|
||||||
|
adjustments: [{ generalId: general.id, goldDelta: develCost }],
|
||||||
|
});
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
return { ok: true, count: next.length };
|
||||||
|
},
|
||||||
|
ensureTournamentParticipationRedisClockFence
|
||||||
|
);
|
||||||
}),
|
}),
|
||||||
cancel: adminProcedure.mutation(async ({ ctx }) => {
|
cancel: adminProcedure.mutation(async ({ ctx }) => {
|
||||||
const store = new TournamentStore(ctx.redis, buildTournamentKeys(ctx.profile.name));
|
const store = new TournamentStore(ctx.redis, buildTournamentKeys(ctx.profile.name));
|
||||||
|
|||||||
@@ -73,6 +73,13 @@ export const ensureActiveRedisClockFence = async (
|
|||||||
return ensureRedisClockFence(redis, profileName, gameTime, ['RUNNING']);
|
return ensureRedisClockFence(redis, profileName, gameTime, ['RUNNING']);
|
||||||
};
|
};
|
||||||
|
|
||||||
|
/** 복구 대기 중에도 참가 신청은 가능하다. 자동 진행/정산에는 사용하지 않는다. */
|
||||||
|
export const ensureTournamentParticipationRedisClockFence = async (
|
||||||
|
redis: ClockFenceRedis,
|
||||||
|
profileName: string,
|
||||||
|
gameTime: CurrentGameTime
|
||||||
|
): Promise<ActiveRedisClockFence | null> => ensureRedisClockFence(redis, profileName, gameTime, ['RUNNING']);
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* User betting is allowed against a frozen tournament deadline while the game
|
* User betting is allowed against a frozen tournament deadline while the game
|
||||||
* clock is suspended. Stage progression and settlement continue to use the
|
* clock is suspended. Stage progression and settlement continue to use the
|
||||||
|
|||||||
@@ -0,0 +1,214 @@
|
|||||||
|
import { expect, it } from 'vitest';
|
||||||
|
import fastify from 'fastify';
|
||||||
|
import { fastifyTRPCPlugin } from '@trpc/server/adapters/fastify';
|
||||||
|
import { createGamePostgresConnector, createRedisConnector } from '@sammo-ts/infra';
|
||||||
|
import { DatabaseTurnDaemonCommandQueue, DatabaseTurnDaemonLease } from '@sammo-ts/game-engine';
|
||||||
|
import { appRouter } from '../src/router.js';
|
||||||
|
import { DatabaseTurnDaemonTransport } from '../src/daemon/databaseTransport.js';
|
||||||
|
import { RedisAccessTokenStore } from '../src/auth/accessTokenStore.js';
|
||||||
|
import { InMemoryFlushStore } from '../src/auth/flushStore.js';
|
||||||
|
import type { GameApiContext } from '../src/context.js';
|
||||||
|
import { loadCurrentGameTime } from '../src/services/gameClock.js';
|
||||||
|
|
||||||
|
const databaseUrl = process.env.TOURNAMENT_RECOVERY_DATABASE_URL;
|
||||||
|
const integration = it.skipIf(!databaseUrl || !process.env.REDIS_URL);
|
||||||
|
|
||||||
|
integration(
|
||||||
|
'accepts an authenticated HTTP tournament join during recovery wait with durable fee and Redis fences',
|
||||||
|
async () => {
|
||||||
|
const schema = new URL(databaseUrl!).searchParams.get('schema');
|
||||||
|
if (!schema?.endsWith('_tournament_recovery_integration'))
|
||||||
|
throw new Error('Dedicated tournament fixture schema required');
|
||||||
|
const connector = createGamePostgresConnector({ url: databaseUrl! });
|
||||||
|
const redisConnector = createRedisConnector({ url: process.env.REDIS_URL! });
|
||||||
|
await connector.connect();
|
||||||
|
await redisConnector.connect();
|
||||||
|
const db = connector.prisma;
|
||||||
|
const redis = redisConnector.client;
|
||||||
|
const profile = 'test:recovery-wait';
|
||||||
|
const prefix = `sammo:${profile}`;
|
||||||
|
const tokenStore = new RedisAccessTokenStore(redis, profile);
|
||||||
|
const transport = new DatabaseTurnDaemonTransport(db, 4000);
|
||||||
|
const lease = await DatabaseTurnDaemonLease.connect(databaseUrl!, { profile, heartbeat: false });
|
||||||
|
const queue = new DatabaseTurnDaemonCommandQueue(db);
|
||||||
|
const app = fastify();
|
||||||
|
let workerError: unknown;
|
||||||
|
let workerBusy = false;
|
||||||
|
let timer: ReturnType<typeof setInterval> | undefined;
|
||||||
|
try {
|
||||||
|
await db.inputEvent.deleteMany();
|
||||||
|
await db.general.deleteMany();
|
||||||
|
await db.turnDaemonLease.deleteMany();
|
||||||
|
await db.worldState.deleteMany();
|
||||||
|
const baseTime = new Date('2026-01-01T00:00:00Z');
|
||||||
|
const waitAt = new Date(Date.now() + 1_800_000);
|
||||||
|
const world = await db.worldState.create({
|
||||||
|
data: {
|
||||||
|
scenarioCode: 'recovery-wait',
|
||||||
|
currentYear: 200,
|
||||||
|
currentMonth: 1,
|
||||||
|
tickSeconds: 3600,
|
||||||
|
clockBaseTime: baseTime,
|
||||||
|
clockTick: 0n,
|
||||||
|
lastTurnTick: 0n,
|
||||||
|
clockWallAnchor: waitAt,
|
||||||
|
clockRecoveryStartTick: 0n,
|
||||||
|
clockRecoveryEndTick: 36_000_000n,
|
||||||
|
clockRecoveryStartWallAt: waitAt,
|
||||||
|
clockMode: 'realtime',
|
||||||
|
clockPhase: 'RUNNING',
|
||||||
|
clockRevision: 1n,
|
||||||
|
deadlineGeneration: 1n,
|
||||||
|
config: { const: { develCost: 100 } },
|
||||||
|
},
|
||||||
|
});
|
||||||
|
await db.general.create({
|
||||||
|
data: { id: 901, userId: 'fixture-user', name: 'fixture', gold: 1000, turnTime: baseTime },
|
||||||
|
});
|
||||||
|
await lease.acquire();
|
||||||
|
await lease.markClockReady();
|
||||||
|
await queue.initialize();
|
||||||
|
await redis.set(
|
||||||
|
`${prefix}:tournament:state`,
|
||||||
|
JSON.stringify({
|
||||||
|
stage: 1,
|
||||||
|
phase: 0,
|
||||||
|
type: 0,
|
||||||
|
auto: true,
|
||||||
|
openYear: 200,
|
||||||
|
openMonth: 1,
|
||||||
|
termSeconds: 60,
|
||||||
|
nextAt: baseTime.toISOString(),
|
||||||
|
})
|
||||||
|
);
|
||||||
|
await redis.set(`${prefix}:tournament:participants`, '[]');
|
||||||
|
await redis.del([
|
||||||
|
`${prefix}:clock:active-revision`,
|
||||||
|
`${prefix}:clock:deadline-generation`,
|
||||||
|
`${prefix}:clock:phase`,
|
||||||
|
]);
|
||||||
|
const auth: NonNullable<GameApiContext['auth']> = {
|
||||||
|
version: 1,
|
||||||
|
profile,
|
||||||
|
sessionId: 'fixture-session',
|
||||||
|
issuedAt: new Date().toISOString(),
|
||||||
|
expiresAt: '2999-01-01T00:00:00Z',
|
||||||
|
user: { id: 'fixture-user', username: 'fixture', displayName: 'fixture', roles: [] },
|
||||||
|
sanctions: {},
|
||||||
|
};
|
||||||
|
await app.register(fastifyTRPCPlugin, {
|
||||||
|
prefix: '/trpc',
|
||||||
|
trpcOptions: {
|
||||||
|
router: appRouter,
|
||||||
|
createContext: ({
|
||||||
|
req,
|
||||||
|
}: {
|
||||||
|
req: { headers: Record<string, string | string[] | undefined> };
|
||||||
|
}): GameApiContext => ({
|
||||||
|
db,
|
||||||
|
redis,
|
||||||
|
profile: { id: 'test', scenario: 'recovery-wait', name: profile },
|
||||||
|
auth: req.headers.authorization === 'Bearer fixture-auth' ? auth : null,
|
||||||
|
requestId:
|
||||||
|
typeof req.headers['x-test-request'] === 'string'
|
||||||
|
? req.headers['x-test-request']
|
||||||
|
: undefined,
|
||||||
|
turnDaemon: transport,
|
||||||
|
accessTokenStore: tokenStore,
|
||||||
|
flushStore: new InMemoryFlushStore(),
|
||||||
|
battleSim: {} as GameApiContext['battleSim'],
|
||||||
|
uploadDir: '/tmp',
|
||||||
|
uploadPath: '/uploads',
|
||||||
|
uploadPublicUrl: null,
|
||||||
|
gameTokenSecret: 'fixture-only',
|
||||||
|
}),
|
||||||
|
},
|
||||||
|
});
|
||||||
|
await app.listen({ host: '127.0.0.1', port: 0 });
|
||||||
|
const address = app.server.address();
|
||||||
|
if (!address || typeof address === 'string') throw new Error('Missing fixture listener');
|
||||||
|
const url = `http://127.0.0.1:${address.port}/trpc/tournament.join`;
|
||||||
|
const post = (authenticated = true) =>
|
||||||
|
fetch(url, {
|
||||||
|
method: 'POST',
|
||||||
|
headers: {
|
||||||
|
'content-type': 'application/json',
|
||||||
|
...(authenticated ? { authorization: 'Bearer fixture-auth' } : {}),
|
||||||
|
'x-test-request': 'recovery-join',
|
||||||
|
},
|
||||||
|
body: '{}',
|
||||||
|
});
|
||||||
|
expect((await post(false)).status).toBe(401);
|
||||||
|
expect(await loadCurrentGameTime(db)).toMatchObject({
|
||||||
|
phase: 'RUNNING',
|
||||||
|
running: false,
|
||||||
|
runtimeReady: true,
|
||||||
|
});
|
||||||
|
// Fixture worker: production queue/lease and real PostgreSQL transaction; no auto turns.
|
||||||
|
timer = setInterval(() => {
|
||||||
|
if (workerBusy) return;
|
||||||
|
workerBusy = true;
|
||||||
|
void (async () => {
|
||||||
|
for (const command of await queue.drain()) {
|
||||||
|
if (command.type !== 'adjustGeneralResources' || !command.requestId)
|
||||||
|
throw new Error('Unexpected fixture command');
|
||||||
|
await db.$transaction(async (tx) => {
|
||||||
|
await lease.assertActive(tx);
|
||||||
|
const adjustment = command.adjustments[0]!;
|
||||||
|
await tx.general.update({
|
||||||
|
where: { id: adjustment.generalId },
|
||||||
|
data: { gold: { increment: adjustment.goldDelta ?? 0 } },
|
||||||
|
});
|
||||||
|
await tx.inputEvent.update({
|
||||||
|
where: { requestId: command.requestId },
|
||||||
|
data: {
|
||||||
|
status: 'SUCCEEDED',
|
||||||
|
result: {
|
||||||
|
type: 'adjustGeneralResources',
|
||||||
|
ok: true,
|
||||||
|
processed: 1,
|
||||||
|
missing: 0,
|
||||||
|
totalGoldDelta: adjustment.goldDelta ?? 0,
|
||||||
|
totalRiceDelta: 0,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
});
|
||||||
|
});
|
||||||
|
}
|
||||||
|
})()
|
||||||
|
.catch((error: unknown) => {
|
||||||
|
workerError = error;
|
||||||
|
})
|
||||||
|
.finally(() => {
|
||||||
|
workerBusy = false;
|
||||||
|
});
|
||||||
|
}, 10);
|
||||||
|
expect((await post()).status).toBe(200);
|
||||||
|
expect((await post()).status).toBe(200);
|
||||||
|
expect(workerError).toBeUndefined();
|
||||||
|
expect((await db.general.findUniqueOrThrow({ where: { id: 901 } })).gold).toBe(900);
|
||||||
|
expect(JSON.parse((await redis.get(`${prefix}:tournament:participants`))!)).toHaveLength(1);
|
||||||
|
const events = await db.inputEvent.findMany({ where: { eventType: 'adjustGeneralResources' } });
|
||||||
|
expect(events).toHaveLength(1);
|
||||||
|
expect(events[0]).toMatchObject({ status: 'SUCCEEDED', acceptedGameTick: 0n, acceptedClockRevision: 1n });
|
||||||
|
await redis.set(`${prefix}:clock:active-revision`, '2');
|
||||||
|
expect((await post()).status).toBe(412);
|
||||||
|
await redis.set(`${prefix}:clock:active-revision`, '1');
|
||||||
|
await db.worldState.update({ where: { id: world.id }, data: { clockPhase: 'RECONCILING' } });
|
||||||
|
expect((await post()).status).toBe(412);
|
||||||
|
await db.worldState.update({ where: { id: world.id }, data: { clockPhase: 'RUNNING' } });
|
||||||
|
await lease.release();
|
||||||
|
expect((await post()).status).toBe(412);
|
||||||
|
expect((await db.general.findUniqueOrThrow({ where: { id: 901 } })).gold).toBe(900);
|
||||||
|
expect((await db.worldState.findUniqueOrThrow({ where: { id: world.id } })).clockTick).toBe(0n);
|
||||||
|
} finally {
|
||||||
|
if (timer) clearInterval(timer);
|
||||||
|
while (workerBusy) await new Promise((resolve) => setTimeout(resolve, 10));
|
||||||
|
await app.close();
|
||||||
|
await lease.close();
|
||||||
|
await connector.disconnect();
|
||||||
|
await redisConnector.disconnect();
|
||||||
|
}
|
||||||
|
},
|
||||||
|
15000
|
||||||
|
);
|
||||||
@@ -153,6 +153,7 @@ const buildContext = (options: {
|
|||||||
rankRows?: Array<{ generalId: number; type: string; value: number }>;
|
rankRows?: Array<{ generalId: number; type: string; value: number }>;
|
||||||
clockPhase?: 'PREOPEN' | 'RUNNING' | 'MANUAL' | 'SUSPENDED' | 'RECONCILING';
|
clockPhase?: 'PREOPEN' | 'RUNNING' | 'MANUAL' | 'SUSPENDED' | 'RECONCILING';
|
||||||
requestId?: string;
|
requestId?: string;
|
||||||
|
clockWallAnchor?: Date;
|
||||||
}): GameApiContext => {
|
}): GameApiContext => {
|
||||||
const db = {
|
const db = {
|
||||||
general: {
|
general: {
|
||||||
@@ -169,7 +170,7 @@ const buildContext = (options: {
|
|||||||
clockBaseTime: new Date('2026-01-01T00:00:00.000Z'),
|
clockBaseTime: new Date('2026-01-01T00:00:00.000Z'),
|
||||||
clockTick: 0n,
|
clockTick: 0n,
|
||||||
clockMode: 'realtime',
|
clockMode: 'realtime',
|
||||||
clockWallAnchor: new Date('2026-01-01T00:00:00.000Z'),
|
clockWallAnchor: options.clockWallAnchor ?? new Date('2026-01-01T00:00:00.000Z'),
|
||||||
clockPhase: options.clockPhase ?? 'RUNNING',
|
clockPhase: options.clockPhase ?? 'RUNNING',
|
||||||
clockRevision: 1n,
|
clockRevision: 1n,
|
||||||
deadlineGeneration: 1n,
|
deadlineGeneration: 1n,
|
||||||
@@ -261,7 +262,21 @@ describe('tournament router permissions and mutations', () => {
|
|||||||
]);
|
]);
|
||||||
});
|
});
|
||||||
|
|
||||||
it('charges the authenticated general once when joining', async () => {
|
it.each(['PREOPEN', 'SUSPENDED', 'RECONCILING'] as const)(
|
||||||
|
'rejects joining in %s without charging',
|
||||||
|
async (clockPhase) => {
|
||||||
|
const redis = new MemoryRedis();
|
||||||
|
const transport = new TournamentTransport();
|
||||||
|
const general = buildGeneral(1, 'user-1');
|
||||||
|
const caller = appRouter.createCaller(
|
||||||
|
buildContext({ redis, transport, generals: [general], userId: 'user-1', clockPhase })
|
||||||
|
);
|
||||||
|
await expect(caller.tournament.join()).rejects.toMatchObject({ code: 'PRECONDITION_FAILED' });
|
||||||
|
expect(transport.commands).toHaveLength(0);
|
||||||
|
}
|
||||||
|
);
|
||||||
|
|
||||||
|
it.each([false, true])('charges the authenticated general once when joining; recovery wait=%s', async (waiting) => {
|
||||||
const redis = new MemoryRedis();
|
const redis = new MemoryRedis();
|
||||||
const transport = new TournamentTransport();
|
const transport = new TournamentTransport();
|
||||||
const general = buildGeneral(1, 'user-1');
|
const general = buildGeneral(1, 'user-1');
|
||||||
@@ -284,6 +299,7 @@ describe('tournament router permissions and mutations', () => {
|
|||||||
userId: 'user-1',
|
userId: 'user-1',
|
||||||
develCost: 200,
|
develCost: 200,
|
||||||
requestId: 'http:tournament-join',
|
requestId: 'http:tournament-join',
|
||||||
|
...(waiting ? { clockWallAnchor: new Date(Date.now() + 1_800_000) } : {}),
|
||||||
});
|
});
|
||||||
const outerApiTransaction = vi.fn(async () => {
|
const outerApiTransaction = vi.fn(async () => {
|
||||||
throw new Error('tournament join must not hold an API transaction while waiting for the daemon');
|
throw new Error('tournament join must not hold an API transaction while waiting for the daemon');
|
||||||
|
|||||||
@@ -28,11 +28,8 @@ export const prepareRealtimeRecovery = async (
|
|||||||
if (world.clockPhase !== 'RUNNING' || !world.clockWallAnchor || world.clockTick === null) return;
|
if (world.clockPhase !== 'RUNNING' || !world.clockWallAnchor || world.clockTick === null) return;
|
||||||
const now = await readClockDatabaseWall(db);
|
const now = await readClockDatabaseWall(db);
|
||||||
// 가속 중 정상적인 프로세스 교체는 기존 창을 그대로 재사용한다.
|
// 가속 중 정상적인 프로세스 교체는 기존 창을 그대로 재사용한다.
|
||||||
// 짧은 중단만 즉시 처리한다. 기준값과 같으면 대기 후 복구한다.
|
// 전체 중단 10분까지는 기존 엔진이 밀린 턴을 순서대로 처리한다.
|
||||||
if (
|
if (!options.paused && now.getTime() - world.clockWallAnchor.getTime() <= immediateRecoveryLimitSeconds() * 1_000)
|
||||||
!options.paused &&
|
|
||||||
now.getTime() - world.clockWallAnchor.getTime() < immediateRecoveryLimitSeconds(world.tickSeconds) * 1_000
|
|
||||||
)
|
|
||||||
return;
|
return;
|
||||||
const suspensionId = `recovery-${randomUUID()}`;
|
const suspensionId = `recovery-${randomUUID()}`;
|
||||||
await startClockSuspension({
|
await startClockSuspension({
|
||||||
|
|||||||
@@ -231,7 +231,7 @@ describeIntegration('durable clock reconciliation', () => {
|
|||||||
expect((await db.worldState.findUniqueOrThrow({ where: { id: row.id } })).clockRecoveryStartTick).toBe(1n);
|
expect((await db.worldState.findUniqueOrThrow({ where: { id: row.id } })).clockRecoveryStartTick).toBe(1n);
|
||||||
});
|
});
|
||||||
|
|
||||||
it.each([300, 420])('applies startup recovery after %i seconds on a 60-minute server', async (delay) => {
|
it.each([180, 420, 590, 610])('applies startup recovery after %i seconds on a 60-minute server', async (delay) => {
|
||||||
const profile = 'short-startup';
|
const profile = 'short-startup';
|
||||||
await db.worldState.create({
|
await db.worldState.create({
|
||||||
data: {
|
data: {
|
||||||
@@ -259,8 +259,8 @@ describeIntegration('durable clock reconciliation', () => {
|
|||||||
fencingEpoch: token.fencingEpoch,
|
fencingEpoch: token.fencingEpoch,
|
||||||
});
|
});
|
||||||
const world = await db.worldState.findFirstOrThrow();
|
const world = await db.worldState.findFirstOrThrow();
|
||||||
expect(world.clockPhase).toBe(delay < 360 ? 'RUNNING' : 'RECONCILING');
|
expect(world.clockPhase).toBe(delay <= 600 ? 'RUNNING' : 'RECONCILING');
|
||||||
expect(readTurnRecovery(world) === null).toBe(delay < 360);
|
expect(readTurnRecovery(world) === null).toBe(delay <= 600);
|
||||||
} finally {
|
} finally {
|
||||||
await lease.close();
|
await lease.close();
|
||||||
}
|
}
|
||||||
@@ -345,6 +345,61 @@ describeIntegration('durable clock reconciliation', () => {
|
|||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it.each([300, 1200, 3600])(
|
||||||
|
'keeps pending turns and operator pause across a short VM outage (%i second turn)',
|
||||||
|
async (turnSeconds) => {
|
||||||
|
const profile = 'short-vm-outage';
|
||||||
|
const anchor = new Date(Date.now() - 590_000);
|
||||||
|
await db.worldState.create({
|
||||||
|
data: {
|
||||||
|
scenarioCode: profile,
|
||||||
|
currentYear: 199,
|
||||||
|
currentMonth: 1,
|
||||||
|
tickSeconds: turnSeconds,
|
||||||
|
clockBaseTime: anchor,
|
||||||
|
clockWallAnchor: anchor,
|
||||||
|
clockTick: 0n,
|
||||||
|
lastTurnTick: 0n,
|
||||||
|
clockMode: 'realtime',
|
||||||
|
clockPhase: 'RUNNING',
|
||||||
|
clockRevision: 1n,
|
||||||
|
deadlineGeneration: 1n,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
const general = await db.general.create({
|
||||||
|
data: { id: 901, name: 'pending-turn', turnTick: 12345n, turnTime: anchor },
|
||||||
|
});
|
||||||
|
const lease = await DatabaseTurnDaemonLease.connect(databaseUrl!, { profile, heartbeat: false });
|
||||||
|
try {
|
||||||
|
const token = await lease.acquire();
|
||||||
|
const authority = {
|
||||||
|
kind: 'DAEMON' as const,
|
||||||
|
profileName: profile,
|
||||||
|
ownerId: token!.ownerId,
|
||||||
|
fencingEpoch: token!.fencingEpoch,
|
||||||
|
};
|
||||||
|
await prepareRealtimeRecovery(db, authority);
|
||||||
|
expect(await db.worldState.findFirstOrThrow()).toMatchObject({
|
||||||
|
clockPhase: 'RUNNING',
|
||||||
|
clockRevision: 1n,
|
||||||
|
clockTick: 0n,
|
||||||
|
lastTurnTick: 0n,
|
||||||
|
clockRecoveryStartTick: null,
|
||||||
|
});
|
||||||
|
expect(await db.general.findUniqueOrThrow({ where: { id: general.id } })).toMatchObject({
|
||||||
|
turnTick: general.turnTick,
|
||||||
|
turnTime: general.turnTime,
|
||||||
|
});
|
||||||
|
expect(await db.clockSuspension.count()).toBe(0);
|
||||||
|
// 짧은 host 중단도 운영자가 정지한 서버를 자동 재개하는 근거가 되지 않는다.
|
||||||
|
await prepareRealtimeRecovery(db, authority, { paused: true });
|
||||||
|
expect(await db.worldState.findFirstOrThrow()).toMatchObject({ clockPhase: 'SUSPENDED' });
|
||||||
|
} finally {
|
||||||
|
await lease.close();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
);
|
||||||
|
|
||||||
it.each([false, true])('fences outage recovery and reuses its window; repeated outage=%s', async (repeated) => {
|
it.each([false, true])('fences outage recovery and reuses its window; repeated outage=%s', async (repeated) => {
|
||||||
const profile = 'recovery-startup';
|
const profile = 'recovery-startup';
|
||||||
await db.worldState.create({
|
await db.worldState.create({
|
||||||
@@ -513,8 +568,8 @@ describeIntegration('durable clock reconciliation', () => {
|
|||||||
}
|
}
|
||||||
);
|
);
|
||||||
|
|
||||||
it.each([359999, 360000, 360001, 840000, 12 * 3600000 + 1000])(
|
it.each([180000, 599999, 600000, 600001, 840000, 12 * 3600000 + 1000])(
|
||||||
'persists strict recovery boundaries for %i ms',
|
'persists the inclusive ten-minute recovery boundary for %i ms',
|
||||||
async (gap) => {
|
async (gap) => {
|
||||||
const observed = T / 6;
|
const observed = T / 6;
|
||||||
const future = new Date(Date.now() + 3600000);
|
const future = new Date(Date.now() + 3600000);
|
||||||
@@ -549,7 +604,7 @@ describeIntegration('durable clock reconciliation', () => {
|
|||||||
authority,
|
authority,
|
||||||
testResumeWallAt: now,
|
testResumeWallAt: now,
|
||||||
});
|
});
|
||||||
expect(plan.recovery === null).toBe(gap < 360000);
|
expect(plan.recovery === null).toBe(gap <= 600000);
|
||||||
expect(await db.message.count()).toBe(0);
|
expect(await db.message.count()).toBe(0);
|
||||||
// Redis 장애 후에도 알림은 DB의 RUNNING 전이와 함께 한 번만 저장한다.
|
// Redis 장애 후에도 알림은 DB의 RUNNING 전이와 함께 한 번만 저장한다.
|
||||||
await expect(
|
await expect(
|
||||||
|
|||||||
@@ -84,6 +84,52 @@ const buildWorld = (stateOverride: Partial<TurnWorldState> = {}): InMemoryTurnWo
|
|||||||
};
|
};
|
||||||
|
|
||||||
describe('runtime clock shift', () => {
|
describe('runtime clock shift', () => {
|
||||||
|
it('executes ten minutes of pending work in bounded batches without skipping or repeating generals', async () => {
|
||||||
|
const base = new Date('2026-07-30T10:00:00Z');
|
||||||
|
const resumed = new Date(base.getTime() + 600_000);
|
||||||
|
const plan = planTurnRecovery({
|
||||||
|
observedTick: 0,
|
||||||
|
normalTick: GAME_TICKS_PER_TURN,
|
||||||
|
wallNow: resumed,
|
||||||
|
turnSeconds: 600,
|
||||||
|
});
|
||||||
|
expect(plan).toMatchObject({ skippedTurns: 0, recovery: null });
|
||||||
|
const world = buildWorld({
|
||||||
|
clockBaseTime: base,
|
||||||
|
clockTick: 0,
|
||||||
|
clockWallAnchor: base,
|
||||||
|
clockMode: 'realtime',
|
||||||
|
clockPhase: 'RUNNING',
|
||||||
|
lastTurnTick: 0,
|
||||||
|
});
|
||||||
|
for (const [id, offset] of [
|
||||||
|
[1, 19_902],
|
||||||
|
[2, 42_001],
|
||||||
|
] as const) {
|
||||||
|
world.updateGeneral(id, { turnTime: new Date(base.getTime() + offset), turnTick: offset * 60 });
|
||||||
|
}
|
||||||
|
const executed: number[] = [];
|
||||||
|
const processor = new InMemoryTurnProcessor(world, {
|
||||||
|
afterExecuteGeneral: async (general) => {
|
||||||
|
executed.push(general.id);
|
||||||
|
},
|
||||||
|
});
|
||||||
|
const target = world.getGameNow(resumed);
|
||||||
|
world.advanceGameClockTo(target, resumed);
|
||||||
|
const budget = { budgetMs: 10_000, maxGenerals: 1, catchUpCap: 1 };
|
||||||
|
let result = await processor.run(target, budget);
|
||||||
|
expect(result.processedGenerals).toBe(1);
|
||||||
|
for (let batch = 0; result.partial && batch < 4; batch++) {
|
||||||
|
result = await processor.run(target, budget, result.checkpoint);
|
||||||
|
expect(result.processedGenerals).toBeLessThanOrEqual(1);
|
||||||
|
}
|
||||||
|
expect(result.partial).toBe(false);
|
||||||
|
expect(executed).toEqual([1, 2]);
|
||||||
|
expect(world.getState().currentMonth).toBe(2);
|
||||||
|
expect(world.getGeneralById(1)!.turnTick).toBe(GAME_TICKS_PER_TURN + 19_902 * 60);
|
||||||
|
expect(world.getGeneralById(2)!.turnTick).toBe(GAME_TICKS_PER_TURN + 42_001 * 60);
|
||||||
|
});
|
||||||
|
|
||||||
it('runs two real monthly cycles per normal interval, survives reload, and returns to one cycle', async () => {
|
it('runs two real monthly cycles per normal interval, survives reload, and returns to one cycle', async () => {
|
||||||
const base = new Date('2026-07-30T10:00:00Z');
|
const base = new Date('2026-07-30T10:00:00Z');
|
||||||
const wallAt = (minutes: number) => new Date(base.getTime() + minutes * 60_000);
|
const wallAt = (minutes: number) => new Date(base.getTime() + minutes * 60_000);
|
||||||
|
|||||||
@@ -44,9 +44,18 @@ From 2026-09-07, maintenance and crash recovery use the following
|
|||||||
`RECOVER_TURNS` policy. The base turn length does not change. One turn remains
|
`RECOVER_TURNS` policy. The base turn length does not change. One turn remains
|
||||||
36,000,000 ticks; a persisted `TurnRecoveryWindow` changes only wall execution.
|
36,000,000 ticks; a persisted `TurnRecoveryWindow` changes only wall execution.
|
||||||
|
|
||||||
- An entire delay strictly below `min(600 seconds, turnSeconds / 10)` catches
|
- From 2026-09-15, an entire delay up to and including 600 seconds catches
|
||||||
up immediately through the ordinary engine. Equality uses recovery. The
|
up immediately through the ordinary engine, independently of the server's
|
||||||
limit is 30 seconds on a 5-minute server and 6 minutes on a 60-minute server.
|
turn length. This accommodates host updates that pause and resume a VM.
|
||||||
|
Pending turns are executed in order with the ordinary per-run budget and
|
||||||
|
month boundary cap; they are not marked complete or skipped. The lease still
|
||||||
|
expires after 30 seconds and a fresh owner must fence the old process and
|
||||||
|
reload durable state before catching up. Operator pauses remain pauses.
|
||||||
|
- During the persisted wait before 2x recovery, tournament participation is
|
||||||
|
allowed when the phase is `RUNNING`, the daemon lease is ready, and DB/Redis
|
||||||
|
revision and deadline generation agree. Registration stage, capacity,
|
||||||
|
duplicate entry, fee and ENGINE transaction checks still apply. Tournament
|
||||||
|
stage advancement and settlement continue to require a ticking clock.
|
||||||
- For longer delays, skip only complete 12-turn blocks, moving future
|
- For longer delays, skip only complete 12-turn blocks, moving future
|
||||||
schedules and the execution cursor by the same integer delta. Never apply
|
schedules and the execution cursor by the same integer delta. Never apply
|
||||||
the short-delay exception again to the remainder. An exact multiple of
|
the short-delay exception again to the remainder. An exact multiple of
|
||||||
|
|||||||
@@ -336,6 +336,11 @@ PM2가 새 owner/epoch와 DB snapshot으로 재시작합니다. lease 오류만
|
|||||||
않습니다. 이전 버전에서 이미 lease 오류로 PAUSED가 된 서버는 원인을 확인한 뒤
|
않습니다. 이전 버전에서 이미 lease 오류로 PAUSED가 된 서버는 원인을 확인한 뒤
|
||||||
한 번 `START`/재개해야 합니다.
|
한 번 `START`/재개해야 합니다.
|
||||||
|
|
||||||
|
Host 업데이트 등으로 VM이 최대 10분 중단되면 새 runtime은 밀린 턴을 기존 엔진의
|
||||||
|
순서·batch budget으로 즉시 따라잡습니다(정확히 10분 포함). 10분 초과는 기존의
|
||||||
|
12턴 묶음 이동 및 대기 후 2배속 복구 정책을 사용합니다. 이미 저장한 복구 창과
|
||||||
|
운영자의 PAUSED/STOPPED는 변경하지 않습니다.
|
||||||
|
|
||||||
30초 lease와 이전 owner의 fencing은 유지합니다. transaction의 만료 검사는 시작에
|
30초 lease와 이전 owner의 fencing은 유지합니다. transaction의 만료 검사는 시작에
|
||||||
고정된 `CURRENT_TIMESTAMP`가 아닌 검사 순간의 `clock_timestamp()`를 사용합니다.
|
고정된 `CURRENT_TIMESTAMP`가 아닌 검사 순간의 `clock_timestamp()`를 사용합니다.
|
||||||
역방향 시계 보정으로 기존 lease가 아직 유효하면 startup은 2초 간격으로 기다리며
|
역방향 시계 보정으로 기존 lease가 아직 유효하면 startup은 2초 간격으로 기다리며
|
||||||
|
|||||||
@@ -68,8 +68,8 @@ export const turnShiftTicks = (turns: number): GameTick => {
|
|||||||
return asGameTick(turns * GAME_TICKS_PER_TURN);
|
return asGameTick(turns * GAME_TICKS_PER_TURN);
|
||||||
};
|
};
|
||||||
|
|
||||||
/** 즉시 처리 여부는 12턴 묶음 생략 전의 전체 지연으로 판정한다. */
|
/** VM/host 업데이트의 전체 중단 10분까지는 기존 엔진 순서와 budget으로 따라잡는다. */
|
||||||
export const immediateRecoveryLimitSeconds = (turnSeconds: number): number => Math.min(600, turnSeconds / 10);
|
export const immediateRecoveryLimitSeconds = (): number => 600;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* observedTick은 중단 전에 저장한 관측 지점, normalTick은 기존 시간표의 현재 지점이다.
|
* observedTick은 중단 전에 저장한 관측 지점, normalTick은 기존 시간표의 현재 지점이다.
|
||||||
@@ -91,8 +91,8 @@ export const planTurnRecovery = (input: {
|
|||||||
if (!Number.isFinite(wallNow.getTime())) throw new Error('Recovery wall instant is invalid.');
|
if (!Number.isFinite(wallNow.getTime())) throw new Error('Recovery wall instant is invalid.');
|
||||||
const gap = Math.max(0, normalTick - observedTick);
|
const gap = Math.max(0, normalTick - observedTick);
|
||||||
const ticksPerSecond = GAME_TICKS_PER_TURN / turnSeconds;
|
const ticksPerSecond = GAME_TICKS_PER_TURN / turnSeconds;
|
||||||
const immediateLimit = immediateRecoveryLimitSeconds(turnSeconds) * ticksPerSecond;
|
const immediateLimit = immediateRecoveryLimitSeconds() * ticksPerSecond;
|
||||||
if (gap < immediateLimit) {
|
if (gap <= immediateLimit) {
|
||||||
return {
|
return {
|
||||||
skippedTurns: 0,
|
skippedTurns: 0,
|
||||||
recoveryTurns: 0,
|
recoveryTurns: 0,
|
||||||
|
|||||||
@@ -160,9 +160,9 @@ describe('turn-aligned double-speed recovery', () => {
|
|||||||
expect(clock.executionRate(end)).toBe(1);
|
expect(clock.executionRate(end)).toBe(1);
|
||||||
});
|
});
|
||||||
|
|
||||||
it.each([300, 3600, 6000, 7200])('uses the strict whole-delay threshold for a %i second turn', (turnSeconds) => {
|
it.each([60, 300, 1200, 3600, 6000, 7200])('catches up through ten minutes for a %i second turn', (turnSeconds) => {
|
||||||
const rate = T / turnSeconds;
|
const rate = T / turnSeconds;
|
||||||
const limitMs = Math.min(600, turnSeconds / 10) * 1000;
|
const limitMs = 600_000;
|
||||||
for (const delta of [-1, 0, 1]) {
|
for (const delta of [-1, 0, 1]) {
|
||||||
const delayMs = limitMs + delta;
|
const delayMs = limitMs + delta;
|
||||||
const normal = Math.trunc((delayMs * rate) / 1000);
|
const normal = Math.trunc((delayMs * rate) / 1000);
|
||||||
@@ -172,8 +172,8 @@ describe('turn-aligned double-speed recovery', () => {
|
|||||||
wallNow: new Date(base + delayMs),
|
wallNow: new Date(base + delayMs),
|
||||||
turnSeconds,
|
turnSeconds,
|
||||||
});
|
});
|
||||||
expect(plan.recovery === null).toBe(delta < 0);
|
expect(plan.recovery === null).toBe(delta <= 0);
|
||||||
expect(plan.initialTick).toBe(delta < 0 ? normal : 0);
|
expect(plan.initialTick).toBe(delta <= 0 ? normal : 0);
|
||||||
}
|
}
|
||||||
// 장시간 중단의 작은 나머지에는 즉시 처리 예외를 다시 적용하지 않는다.
|
// 장시간 중단의 작은 나머지에는 즉시 처리 예외를 다시 적용하지 않는다.
|
||||||
const long = planTurnRecovery({ observedTick: 0, normalTick: 12 * T + rate, wallNow: wall(20), turnSeconds });
|
const long = planTurnRecovery({ observedTick: 0, normalTick: 12 * T + rate, wallNow: wall(20), turnSeconds });
|
||||||
|
|||||||
@@ -25,3 +25,4 @@ TURN_COMMAND_DURABLE_MATRIX_DATABASE_URL reference_command_durable_matrix
|
|||||||
TURN_DIFFERENTIAL_DATABASE_URL core
|
TURN_DIFFERENTIAL_DATABASE_URL core
|
||||||
TURN_FULL_LIFECYCLE_PERSISTENCE_DATABASE_URL reference_full_lifecycle
|
TURN_FULL_LIFECYCLE_PERSISTENCE_DATABASE_URL reference_full_lifecycle
|
||||||
WEB_PUSH_GATEWAY_DATABASE_URL web_push_gateway
|
WEB_PUSH_GATEWAY_DATABASE_URL web_push_gateway
|
||||||
|
TOURNAMENT_RECOVERY_DATABASE_URL external_fixture
|
||||||
|
|||||||
|
Reference in New Issue
Block a user