feat: 시계 reconciliation 워커와 명령 경계 완성
This commit is contained in:
@@ -2,7 +2,7 @@ import { describe, expect, it, vi } from 'vitest';
|
||||
|
||||
import type { GamePrismaClient } from '@sammo-ts/infra';
|
||||
|
||||
import { processDueAuctionId, reconcilePendingAuctionTimers } from '../src/auction/worker.js';
|
||||
import { popDueAuctionIds, processDueAuctionId, reconcilePendingAuctionTimers } from '../src/auction/worker.js';
|
||||
import { resolveAuctionSeedScore } from '../src/auction/scheduler.js';
|
||||
|
||||
const buildRedis = () => ({
|
||||
@@ -55,6 +55,38 @@ const buildDb = (options: {
|
||||
};
|
||||
|
||||
describe('auction worker clock-shift race', () => {
|
||||
it('uses one Redis script for revision, generation, phase, due-read, and removal', async () => {
|
||||
const redis = { ...buildRedis(), eval: vi.fn(async () => ['7', '9']) };
|
||||
await expect(
|
||||
popDueAuctionIds(redis, 'auction-timer', 123_456, 100, {
|
||||
activeRevisionKey: 'clock-revision',
|
||||
deadlineGenerationKey: 'deadline-generation',
|
||||
phaseKey: 'clock-phase',
|
||||
revision: 4,
|
||||
generation: 8,
|
||||
})
|
||||
).resolves.toEqual(['7', '9']);
|
||||
expect(redis.eval).toHaveBeenCalledWith(expect.stringContaining('ZRANGEBYSCORE'), {
|
||||
keys: ['auction-timer', 'clock-revision', 'deadline-generation', 'clock-phase'],
|
||||
arguments: ['4', '8', '123456', '100'],
|
||||
});
|
||||
expect(redis.zRangeByScore).not.toHaveBeenCalled();
|
||||
expect(redis.zRem).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('returns no due member when the atomic Redis clock fence rejects the pop', async () => {
|
||||
const redis = { ...buildRedis(), eval: vi.fn(async () => ['__CLOCK_FENCE__']) };
|
||||
await expect(
|
||||
popDueAuctionIds(redis, 'auction-timer', 123_456, 100, {
|
||||
activeRevisionKey: 'clock-revision',
|
||||
deadlineGenerationKey: 'deadline-generation',
|
||||
phaseKey: 'clock-phase',
|
||||
revision: 4,
|
||||
generation: 8,
|
||||
})
|
||||
).resolves.toEqual([]);
|
||||
});
|
||||
|
||||
it('seeds OPEN at its deadline but retries FINALIZING at the current logical tick', () => {
|
||||
const now = new Date('2026-07-30T12:00:00.000Z');
|
||||
const time = {
|
||||
@@ -292,6 +324,7 @@ describe('auction worker clock-shift race', () => {
|
||||
requestId,
|
||||
target: 'ENGINE',
|
||||
eventType: 'auctionFinalize',
|
||||
acceptedGameTick: 72_000_000n,
|
||||
payload: {
|
||||
type: 'auctionFinalize',
|
||||
requestId,
|
||||
|
||||
@@ -0,0 +1,87 @@
|
||||
import { describe, expect, it, vi } from 'vitest';
|
||||
|
||||
import type { DatabaseClient } from '../src/context.js';
|
||||
import { loadClockAdminStatus, loadClockReadiness } from '../src/services/clockReadiness.js';
|
||||
|
||||
describe('clock reconciliation readiness', () => {
|
||||
it('fails closed when the reconciliation schema is not available', async () => {
|
||||
const db = {} as DatabaseClient;
|
||||
await expect(loadClockReadiness(db)).resolves.toEqual({
|
||||
reconciliationComplete: false,
|
||||
gameplayEnabled: false,
|
||||
phase: null,
|
||||
revision: null,
|
||||
deadlineGeneration: null,
|
||||
incompleteOutboxCount: null,
|
||||
});
|
||||
});
|
||||
|
||||
it('blocks readiness for RECONCILING or incomplete outbox state', async () => {
|
||||
const db = {
|
||||
worldState: {
|
||||
findFirst: vi.fn(async () => ({
|
||||
clockPhase: 'RECONCILING',
|
||||
clockRevision: 9n,
|
||||
deadlineGeneration: 4n,
|
||||
})),
|
||||
},
|
||||
clockProjectionOutbox: { count: vi.fn(async () => 1) },
|
||||
} as unknown as DatabaseClient;
|
||||
await expect(loadClockReadiness(db)).resolves.toMatchObject({
|
||||
reconciliationComplete: false,
|
||||
gameplayEnabled: false,
|
||||
phase: 'RECONCILING',
|
||||
revision: 9,
|
||||
deadlineGeneration: 4,
|
||||
incompleteOutboxCount: 1,
|
||||
});
|
||||
});
|
||||
|
||||
it('exposes participant checksums and incomplete outbox detail to admins', async () => {
|
||||
const db = {
|
||||
worldState: {
|
||||
findFirst: vi.fn(async () => ({
|
||||
clockPhase: 'RUNNING',
|
||||
clockRevision: 3n,
|
||||
deadlineGeneration: 2n,
|
||||
})),
|
||||
},
|
||||
clockProjectionOutbox: { count: vi.fn(async () => 0) },
|
||||
clockSuspension: {
|
||||
findFirst: vi.fn(async () => ({
|
||||
id: 'maintenance-1',
|
||||
source: 'MAINTENANCE',
|
||||
policy: 'EXACT',
|
||||
status: 'APPLIED',
|
||||
sourceRevision: 2n,
|
||||
targetRevision: 3n,
|
||||
cutTick: 100n,
|
||||
alignedTick: 130n,
|
||||
participantChecksumBefore: 'before-all',
|
||||
participantChecksumAfter: 'after-all',
|
||||
participants: [
|
||||
{
|
||||
participantKey: 'general-turn',
|
||||
policy: 'SHIFT',
|
||||
beforeChecksum: 'before',
|
||||
afterChecksum: 'after',
|
||||
affectedCount: 2,
|
||||
},
|
||||
],
|
||||
projectionOutbox: [{ id: 8n, targetRevision: 3n, status: 'APPLIED', attempts: 1, lastError: null }],
|
||||
})),
|
||||
},
|
||||
} as unknown as DatabaseClient;
|
||||
|
||||
await expect(loadClockAdminStatus(db)).resolves.toMatchObject({
|
||||
reconciliationComplete: true,
|
||||
latestReconciliation: {
|
||||
id: 'maintenance-1',
|
||||
participantChecksumBefore: 'before-all',
|
||||
participantChecksumAfter: 'after-all',
|
||||
participants: [{ key: 'general-turn', policy: 'SHIFT', affectedCount: 2 }],
|
||||
outbox: [{ id: '8', status: 'APPLIED' }],
|
||||
},
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -511,6 +511,10 @@ integration('API input event boundary', () => {
|
||||
it('reuses the same engine child event but rejects a changed retry payload', async () => {
|
||||
const transport = new DatabaseTurnDaemonTransport(db, 100);
|
||||
const requestId = 'integration:api:engine-child';
|
||||
const worldClock = await db.worldState.findFirst({
|
||||
orderBy: { id: 'asc' },
|
||||
select: { clockRevision: true, deadlineGeneration: true },
|
||||
});
|
||||
const acceptedWindowStart = Date.now();
|
||||
await transport.sendCommand({ type: 'vacation', requestId, userId: 'user-7', generalId: 7 });
|
||||
const acceptedWindowEnd = Date.now();
|
||||
@@ -518,6 +522,9 @@ integration('API input event boundary', () => {
|
||||
expect(event.actorUserId).toBe('user-7');
|
||||
expect(event.createdAt.getTime()).toBeGreaterThanOrEqual(acceptedWindowStart);
|
||||
expect(event.createdAt.getTime()).toBeLessThanOrEqual(acceptedWindowEnd);
|
||||
expect(event.acceptedGameTick).not.toBeNull();
|
||||
expect(event.acceptedClockRevision).toBe(worldClock?.clockRevision ?? null);
|
||||
expect(event.acceptedDeadlineGeneration).toBe(worldClock?.deadlineGeneration ?? null);
|
||||
await expect(
|
||||
transport.sendCommand({ type: 'vacation', requestId, userId: 'user-7', generalId: 7 })
|
||||
).resolves.toBe(requestId);
|
||||
|
||||
@@ -28,6 +28,9 @@ const createContext = (payload: unknown = {}) => {
|
||||
status: 'PENDING',
|
||||
result: null,
|
||||
attempts: 0,
|
||||
acceptedGameTick: 100n,
|
||||
acceptedClockRevision: 3n,
|
||||
acceptedDeadlineGeneration: 2n,
|
||||
},
|
||||
];
|
||||
}
|
||||
@@ -36,8 +39,8 @@ const createContext = (payload: unknown = {}) => {
|
||||
});
|
||||
const transaction = {
|
||||
$queryRaw: queryRaw,
|
||||
$executeRaw: vi.fn(async () => {
|
||||
order.push('accepted');
|
||||
$executeRaw: vi.fn(async (query: { sql?: string }) => {
|
||||
order.push(query.sql?.includes('pg_advisory_xact_lock') ? 'clock-fence' : 'accepted');
|
||||
return 1;
|
||||
}),
|
||||
$executeRawUnsafe: vi.fn(async (statement: string) => {
|
||||
@@ -88,6 +91,7 @@ describe('API input-event change journal boundary', () => {
|
||||
|
||||
expect(fixture.order).toEqual([
|
||||
'transaction-begin',
|
||||
'clock-fence',
|
||||
'accepted',
|
||||
'locked',
|
||||
'processing',
|
||||
@@ -112,6 +116,7 @@ describe('API input-event change journal boundary', () => {
|
||||
|
||||
expect(fixture.order).toEqual([
|
||||
'transaction-begin',
|
||||
'clock-fence',
|
||||
'accepted',
|
||||
'locked',
|
||||
'processing',
|
||||
|
||||
@@ -30,11 +30,31 @@ class MemoryRedis {
|
||||
}
|
||||
|
||||
async eval(_script: string, options: { keys: string[]; arguments: string[] }): Promise<string> {
|
||||
const [valueKey, revisionKey] = options.keys;
|
||||
const [value] = options.arguments;
|
||||
if (options.keys.length === 3 && options.keys[0]?.endsWith(':clock:active-revision')) {
|
||||
const current = options.keys.map((key) => this.values.get(key));
|
||||
if (current.every((value) => value === undefined)) {
|
||||
options.keys.forEach((key, index) => this.values.set(key, options.arguments[index]!));
|
||||
return '1';
|
||||
}
|
||||
return current.every((value, index) => value === options.arguments[index]) ? '2' : '0';
|
||||
}
|
||||
const fenced = options.keys.at(-1)?.endsWith(':clock:phase') === true;
|
||||
const writeCount = options.keys.length - (fenced ? 4 : 1);
|
||||
if (fenced) {
|
||||
const clockKeys = options.keys.slice(-3);
|
||||
const expected = options.arguments.slice(-3);
|
||||
if (!clockKeys.every((key, index) => this.values.get(key) === expected[index])) {
|
||||
return '__CLOCK_FENCE__';
|
||||
}
|
||||
}
|
||||
const valueKey = options.keys[0];
|
||||
const revisionKey = options.keys[writeCount];
|
||||
const value = options.arguments[0];
|
||||
if (!valueKey || !revisionKey || value === undefined) throw new Error('invalid eval arguments');
|
||||
const revision = Number(this.values.get(revisionKey) ?? '0') + 1;
|
||||
this.values.set(valueKey, value);
|
||||
for (let index = 0; index < writeCount; index += 1) {
|
||||
this.values.set(options.keys[index]!, options.arguments[index]!);
|
||||
}
|
||||
this.values.set(revisionKey, String(revision));
|
||||
return String(revision);
|
||||
}
|
||||
@@ -144,6 +164,14 @@ const buildContext = (options: {
|
||||
},
|
||||
worldState: {
|
||||
findFirst: async () => ({
|
||||
clockBaseTime: new Date('2026-01-01T00:00:00.000Z'),
|
||||
clockTick: 0n,
|
||||
clockMode: 'realtime',
|
||||
clockWallAnchor: new Date('2026-01-01T00:00:00.000Z'),
|
||||
clockPhase: 'RUNNING',
|
||||
clockRevision: 1n,
|
||||
deadlineGeneration: 1n,
|
||||
tickSeconds: 60,
|
||||
config: { const: { develCost: options.develCost ?? 200 } },
|
||||
...(options.currentDevelCost === undefined ? {} : { meta: { develcost: options.currentDevelCost } }),
|
||||
}),
|
||||
@@ -394,7 +422,10 @@ describe('tournament router permissions and mutations', () => {
|
||||
roles: ['admin.tournament:che:default'],
|
||||
})
|
||||
);
|
||||
await expect(adminCaller.tournament.getAdminStatus()).resolves.toEqual({ ok: true });
|
||||
await expect(adminCaller.tournament.getAdminStatus()).resolves.toMatchObject({
|
||||
ok: true,
|
||||
clock: { reconciliationComplete: false, latestReconciliation: null },
|
||||
});
|
||||
});
|
||||
|
||||
it('applies the admin role boundary to every tournament mutation', async () => {
|
||||
@@ -479,9 +510,7 @@ describe('tournament router permissions and mutations', () => {
|
||||
])
|
||||
).resolves.toEqual({ ok: true, count: 1 });
|
||||
await expect(
|
||||
caller.tournament.setBettingEntries([
|
||||
{ generalId: general.id, targetId: rival.id, amount: 100 },
|
||||
])
|
||||
caller.tournament.setBettingEntries([{ generalId: general.id, targetId: rival.id, amount: 100 }])
|
||||
).resolves.toEqual({ ok: true, count: 1 });
|
||||
await expect(caller.tournament.seedParticipants({ generalIds: [general.id, rival.id] })).resolves.toEqual({
|
||||
ok: true,
|
||||
|
||||
@@ -41,6 +41,9 @@ integration('TournamentStore Redis source revision', () => {
|
||||
keys.matchesKey,
|
||||
keys.bettingKey,
|
||||
keys.sourceRevisionKey,
|
||||
keys.activeClockRevisionKey,
|
||||
keys.deadlineGenerationKey,
|
||||
keys.clockPhaseKey,
|
||||
]);
|
||||
if (subscriber) {
|
||||
await subscriber.client.unsubscribe(keys.sourceRevisionChannel);
|
||||
@@ -124,4 +127,48 @@ integration('TournamentStore Redis source revision', () => {
|
||||
}),
|
||||
]);
|
||||
});
|
||||
|
||||
it('dual-writes deadline ticks and rejects a Redis clock revision race atomically', async () => {
|
||||
const store = new TournamentStore(connector.client, keys);
|
||||
await Promise.all([
|
||||
connector.client.set(keys.activeClockRevisionKey, '7'),
|
||||
connector.client.set(keys.deadlineGenerationKey, '3'),
|
||||
connector.client.set(keys.clockPhaseKey, 'RUNNING'),
|
||||
]);
|
||||
const clockContext = {
|
||||
phase: 'RUNNING' as const,
|
||||
revision: 7,
|
||||
deadlineGeneration: 3,
|
||||
dateToTick: (date: Date) => Math.trunc(date.getTime() / 1_000),
|
||||
};
|
||||
const nextAt = '2026-09-03T10:00:00.000Z';
|
||||
await store.withClockContext(clockContext, () =>
|
||||
store.setState({
|
||||
stage: 6,
|
||||
phase: 0,
|
||||
type: 0,
|
||||
auto: true,
|
||||
openYear: 200,
|
||||
openMonth: 1,
|
||||
termSeconds: 10,
|
||||
nextAt,
|
||||
bettingCloseAt: '2026-09-03T09:59:50.000Z',
|
||||
})
|
||||
);
|
||||
await expect(store.getState()).resolves.toMatchObject({
|
||||
nextTick: Math.trunc(new Date(nextAt).getTime() / 1_000),
|
||||
bettingCloseTick: Math.trunc(new Date('2026-09-03T09:59:50.000Z').getTime() / 1_000),
|
||||
clockRevision: 7,
|
||||
deadlineGeneration: 3,
|
||||
});
|
||||
|
||||
const beforeRevision = await store.getSourceRevision();
|
||||
await connector.client.set(keys.activeClockRevisionKey, '8');
|
||||
await expect(
|
||||
store.withClockContext(clockContext, () =>
|
||||
store.setMatches([{ id: 99, stage: 7, roundIndex: 0, attackerId: 1, defenderId: 2 }])
|
||||
)
|
||||
).rejects.toThrow('clock revision fence failed');
|
||||
await expect(store.getSourceRevision()).resolves.toBe(beforeRevision);
|
||||
});
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user