중단 후 대기와 2배속 복구 경계 및 전체 공지 적용
This commit is contained in:
@@ -53,6 +53,47 @@ describe('current game time projection', () => {
|
||||
});
|
||||
});
|
||||
|
||||
it('exposes waiting, 2x and normal speed boundaries from a reloaded recovery', async () => {
|
||||
const db = {
|
||||
worldState: {
|
||||
findFirst: vi.fn(async () => ({
|
||||
clockBaseTime: new Date('2026-09-07T00:00:00Z'),
|
||||
clockTick: 6000000n,
|
||||
clockMode: 'realtime',
|
||||
clockWallAnchor: new Date('2026-09-07T00:35:00Z'),
|
||||
tickSeconds: 3600,
|
||||
clockPhase: 'RUNNING',
|
||||
clockRevision: 2n,
|
||||
deadlineGeneration: 2n,
|
||||
clockRecoveryStartTick: 6000000n,
|
||||
clockRecoveryEndTick: 36000000n,
|
||||
clockRecoveryStartWallAt: new Date('2026-09-07T00:35:00Z'),
|
||||
})),
|
||||
},
|
||||
$queryRaw: vi.fn(async () => [{ ready: true }]),
|
||||
} as unknown as DatabaseClient;
|
||||
for (const now of ['00:24:00', '00:34:59.999']) {
|
||||
expect(await loadCurrentGameTime(db, new Date(`2026-09-07T${now}Z`))).toMatchObject({
|
||||
tick: 6000000,
|
||||
running: false,
|
||||
startsAt: new Date('2026-09-07T00:35:00Z'),
|
||||
recovery: { startsAt: '2026-09-07T00:35:00.000Z', endsAt: '2026-09-07T01:00:00.000Z' },
|
||||
});
|
||||
}
|
||||
expect(await loadCurrentGameTime(db, new Date('2026-09-07T00:35:00.001Z'))).toMatchObject({
|
||||
tick: 6000020,
|
||||
running: true,
|
||||
startsAt: null,
|
||||
});
|
||||
expect(await loadCurrentGameTime(db, new Date('2026-09-07T00:59:59.999Z'))).toMatchObject({ tick: 35999980 });
|
||||
expect(await loadCurrentGameTime(db, new Date('2026-09-07T01:00:00Z'))).toMatchObject({
|
||||
tick: 36000000,
|
||||
running: true,
|
||||
recovery: null,
|
||||
});
|
||||
expect(await loadCurrentGameTime(db, new Date('2026-09-07T01:00:00.001Z'))).toMatchObject({ tick: 36000010 });
|
||||
});
|
||||
|
||||
it('holds an invader restart until its future turn boundary', async () => {
|
||||
const db = buildDatabase('realtime', 'RUNNING');
|
||||
const result = await loadCurrentGameTime(db, new Date('2026-08-21T10:59:59Z'));
|
||||
|
||||
@@ -1,9 +1,18 @@
|
||||
import { createHash } from 'node:crypto';
|
||||
|
||||
import { GameClock, parseGameClockPhase } from '@sammo-ts/common';
|
||||
import {
|
||||
ChangeJournal,
|
||||
GameClock,
|
||||
formatServerDateTime,
|
||||
parseGameClockPhase,
|
||||
readTurnRecovery,
|
||||
} from '@sammo-ts/common';
|
||||
import { MESSAGE_MAILBOX_PUBLIC, resolveMessageTargetIcon, sendMessage } from '@sammo-ts/logic';
|
||||
import {
|
||||
CLOCK_OPERATION_PERSISTENCE_LOCK,
|
||||
GamePrisma,
|
||||
persistMessageEnvelope,
|
||||
writeReadModelChangeJournal,
|
||||
acquireGameSchemaAdvisoryXactLock,
|
||||
type GamePrismaClient,
|
||||
} from '@sammo-ts/infra';
|
||||
@@ -302,6 +311,47 @@ export const applyNextClockProjection = async (options: {
|
||||
throw new Error('Clock projection final RUNNING transition fence failed.');
|
||||
}
|
||||
const appliedAt = await readDbWall(transaction);
|
||||
// RUNNING 전이와 같은 transaction에 남겨 재시도 시 전체 공지를 중복 발송하지 않는다.
|
||||
const recovery = readTurnRecovery(world);
|
||||
const journal = new ChangeJournal();
|
||||
journal.mark('world.content').mark('map.world');
|
||||
if (recovery) {
|
||||
const recoveredClock = new GameClock({
|
||||
baseTime: world.clockBaseTime!,
|
||||
tick: Number(world.clockTick),
|
||||
wallAnchor: world.clockWallAnchor!,
|
||||
mode: 'realtime',
|
||||
turnSeconds: world.tickSeconds,
|
||||
recovery,
|
||||
});
|
||||
const endsAt = recoveredClock.tickToWallDate(recovery.endTick);
|
||||
const system = {
|
||||
generalId: 0,
|
||||
generalName: '시스템',
|
||||
nationId: 0,
|
||||
nationName: '',
|
||||
color: '#000000',
|
||||
icon: resolveMessageTargetIcon(),
|
||||
};
|
||||
await sendMessage(
|
||||
{ insertMessage: (draft) => persistMessageEnvelope(transaction, draft) },
|
||||
{
|
||||
msgType: 'public',
|
||||
src: system,
|
||||
dest: system,
|
||||
text: `서버 재개에 따른 2배속 복구 시간: ${formatServerDateTime(recovery.startWallAt)} ~ ${formatServerDateTime(endsAt)} (한국 시각). 시작 전까지 대기하며, 종료 시 정상 속도로 진행합니다.`,
|
||||
time: appliedAt,
|
||||
validUntil: new Date('9999-12-31T00:00:00Z'),
|
||||
option: {
|
||||
recoveryStartsAt: recovery.startWallAt.toISOString(),
|
||||
recoveryEndsAt: endsAt.toISOString(),
|
||||
},
|
||||
}
|
||||
);
|
||||
journal.mark('messages.mailbox', MESSAGE_MAILBOX_PUBLIC);
|
||||
}
|
||||
await writeReadModelChangeJournal(transaction, journal.snapshot());
|
||||
|
||||
await transaction.clockProjectionOutbox.update({
|
||||
where: { id: outbox.id },
|
||||
data: { status: 'APPLIED', appliedAt, lockedAt: null, lockedBy: null, lastError: null },
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import { immediateRecoveryLimitSeconds } from '@sammo-ts/common';
|
||||
import type { GamePrismaClient } from '@sammo-ts/infra';
|
||||
import {
|
||||
readClockDatabaseWall,
|
||||
@@ -27,8 +28,12 @@ export const prepareRealtimeRecovery = async (
|
||||
if (world.clockPhase !== 'RUNNING' || !world.clockWallAnchor || world.clockTick === null) return;
|
||||
const now = await readClockDatabaseWall(db);
|
||||
// 가속 중 정상적인 프로세스 교체는 기존 창을 그대로 재사용한다.
|
||||
// 한 턴 미만의 장애는 잔여 구간 실행만 필요하므로 새 좌표 세대를 만들지 않는다.
|
||||
if (!options.paused && now.getTime() - world.clockWallAnchor.getTime() < world.tickSeconds * 1_000) return;
|
||||
// 짧은 중단만 즉시 처리한다. 기준값과 같으면 대기 후 복구한다.
|
||||
if (
|
||||
!options.paused &&
|
||||
now.getTime() - world.clockWallAnchor.getTime() < immediateRecoveryLimitSeconds(world.tickSeconds) * 1_000
|
||||
)
|
||||
return;
|
||||
const suspensionId = `recovery-${randomUUID()}`;
|
||||
await startClockSuspension({
|
||||
db,
|
||||
|
||||
@@ -29,6 +29,8 @@ describeIntegration('durable clock reconciliation', () => {
|
||||
const clean = async (): Promise<void> => {
|
||||
await redis.client.flushDb();
|
||||
await db.$transaction([
|
||||
db.readModelOutbox.deleteMany(),
|
||||
db.readModelRevision.deleteMany(),
|
||||
db.clockProjectionOutbox.deleteMany(),
|
||||
db.clockReconciliationParticipant.deleteMany(),
|
||||
db.clockSuspension.deleteMany(),
|
||||
@@ -65,6 +67,66 @@ describeIntegration('durable clock reconciliation', () => {
|
||||
await clean();
|
||||
});
|
||||
|
||||
it('accepts partial starts while rejecting incomplete and off-boundary DB windows', async () => {
|
||||
const row = await db.worldState.create({
|
||||
data: {
|
||||
scenarioCode: 'constraint',
|
||||
currentYear: 199,
|
||||
currentMonth: 1,
|
||||
tickSeconds: 3600,
|
||||
clockRecoveryStartTick: 1n,
|
||||
clockRecoveryEndTick: BigInt(T),
|
||||
clockRecoveryStartWallAt: new Date(),
|
||||
},
|
||||
});
|
||||
for (const data of [
|
||||
{ clockRecoveryStartWallAt: null },
|
||||
{ clockRecoveryEndTick: BigInt(T + 1) },
|
||||
{ clockRecoveryStartTick: BigInt(T) },
|
||||
{ clockRecoveryStartTick: 0n, clockRecoveryEndTick: BigInt(25 * T) },
|
||||
]) {
|
||||
await expect(db.worldState.update({ where: { id: row.id }, data })).rejects.toThrow(
|
||||
'world_state_turn_recovery_window_check'
|
||||
);
|
||||
}
|
||||
expect((await db.worldState.findUniqueOrThrow({ where: { id: row.id } })).clockRecoveryStartTick).toBe(1n);
|
||||
});
|
||||
|
||||
it.each([300, 420])('applies startup recovery after %i seconds on a 60-minute server', async (delay) => {
|
||||
const profile = 'short-startup';
|
||||
await db.worldState.create({
|
||||
data: {
|
||||
scenarioCode: profile,
|
||||
currentYear: 199,
|
||||
currentMonth: 1,
|
||||
tickSeconds: 3600,
|
||||
clockBaseTime: new Date('2026-01-01T00:00:00Z'),
|
||||
clockTick: BigInt(T / 6),
|
||||
clockWallAnchor: new Date(Date.now() - delay * 1000),
|
||||
clockMode: 'realtime',
|
||||
clockPhase: 'RUNNING',
|
||||
clockRevision: 1n,
|
||||
deadlineGeneration: 1n,
|
||||
lastTurnTick: 0n,
|
||||
},
|
||||
});
|
||||
const lease = await DatabaseTurnDaemonLease.connect(databaseUrl!, { profile, heartbeat: false });
|
||||
try {
|
||||
const token = (await lease.acquire())!;
|
||||
await prepareRealtimeRecovery(db, {
|
||||
kind: 'DAEMON',
|
||||
profileName: profile,
|
||||
ownerId: token.ownerId,
|
||||
fencingEpoch: token.fencingEpoch,
|
||||
});
|
||||
const world = await db.worldState.findFirstOrThrow();
|
||||
expect(world.clockPhase).toBe(delay < 360 ? 'RUNNING' : 'RECONCILING');
|
||||
expect(readTurnRecovery(world) === null).toBe(delay < 360);
|
||||
} finally {
|
||||
await lease.close();
|
||||
}
|
||||
});
|
||||
|
||||
it.each([false, true])('fences outage recovery and reuses its window; repeated outage=%s', async (repeated) => {
|
||||
const profile = 'recovery-startup';
|
||||
await db.worldState.create({
|
||||
@@ -115,7 +177,7 @@ describeIntegration('durable clock reconciliation', () => {
|
||||
expect(pending.clockPhase).toBe('RECONCILING');
|
||||
const recoveredWindow = readTurnRecovery(pending)!;
|
||||
expect(recoveredWindow).not.toBeNull();
|
||||
expect(recoveredWindow.endTick - recoveredWindow.startTick).toBe((repeated ? 12 : 8) * T);
|
||||
expect(recoveredWindow.endTick - recoveredWindow.startTick).toBe((repeated ? 13 : 9) * T);
|
||||
expect(await readTurnRuntimeReady(db, pending.clockRevision)).toBe(false);
|
||||
await applyNextClockProjection({ db, redis: redis.client, workerId: profile });
|
||||
await lease.markClockReady();
|
||||
@@ -140,7 +202,7 @@ describeIntegration('durable clock reconciliation', () => {
|
||||
it.each([4, 12, 13, 23, 24])(
|
||||
'persists recovery for %i turns and reloads the same normal boundary',
|
||||
async (turns) => {
|
||||
const now = new Date();
|
||||
const now = new Date(Date.now() + 3_600_000);
|
||||
await db.worldState.create({
|
||||
data: {
|
||||
scenarioCode: 'turn-recovery',
|
||||
@@ -214,6 +276,102 @@ describeIntegration('durable clock reconciliation', () => {
|
||||
const retry = await reconcileClockSuspension({ db, suspensionId: suspension.suspensionId, authority });
|
||||
expect(retry.recovery).toEqual(plan.recovery);
|
||||
expect(retry.catchUpTicks).toBe(plan.catchUpTicks);
|
||||
expect(await applyNextClockProjection({ db, redis: redis.client, workerId: 'retry' })).toBe('IDLE');
|
||||
const announcements = await db.message.findMany({ where: { mailbox: 9999 } });
|
||||
expect(announcements).toHaveLength(recovery ? 1 : 0);
|
||||
if (recovery) {
|
||||
expect(announcements[0]!.message).toMatchObject({
|
||||
src: { generalName: '시스템' },
|
||||
option: {
|
||||
recoveryStartsAt: recovery.startWallAt.toISOString(),
|
||||
recoveryEndsAt: reloaded.tickToWallDate(recovery.endTick).toISOString(),
|
||||
},
|
||||
});
|
||||
expect(await db.messageAction.count()).toBe(0);
|
||||
expect(
|
||||
await db.readModelRevision.findFirst({ where: { domain: 'messages.mailbox', entityId: 9999 } })
|
||||
).toMatchObject({ revision: 1n });
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
it.each([359999, 360000, 360001, 840000, 12 * 3600000 + 1000])(
|
||||
'persists strict recovery boundaries for %i ms',
|
||||
async (gap) => {
|
||||
const observed = T / 6;
|
||||
const future = new Date(Date.now() + 3600000);
|
||||
await db.worldState.create({
|
||||
data: {
|
||||
scenarioCode: 'wait-boundary',
|
||||
currentYear: 199,
|
||||
currentMonth: 1,
|
||||
tickSeconds: 3600,
|
||||
clockBaseTime: new Date('2026-01-01T00:00:00Z'),
|
||||
clockTick: BigInt(observed),
|
||||
clockMode: 'realtime',
|
||||
clockWallAnchor: future,
|
||||
lastTurnTick: 0n,
|
||||
clockPhase: 'RUNNING',
|
||||
clockRevision: 1n,
|
||||
deadlineGeneration: 1n,
|
||||
},
|
||||
});
|
||||
const authority = { kind: 'OFFLINE' as const, profileName: 'wait-boundary', reason: 'fixture' };
|
||||
const suspension = await startClockSuspension({
|
||||
db,
|
||||
suspensionId: 'wait-boundary',
|
||||
source: 'MAINTENANCE',
|
||||
policy: 'RECOVER_TURNS',
|
||||
authority,
|
||||
});
|
||||
const now = new Date(suspension.cutWallAt.getTime() + gap);
|
||||
const plan = await reconcileClockSuspension({
|
||||
db,
|
||||
suspensionId: suspension.suspensionId,
|
||||
authority,
|
||||
testResumeWallAt: now,
|
||||
});
|
||||
expect(plan.recovery === null).toBe(gap < 360000);
|
||||
expect(await db.message.count()).toBe(0);
|
||||
// Redis 장애 후에도 알림은 DB의 RUNNING 전이와 함께 한 번만 저장한다.
|
||||
await expect(
|
||||
applyNextClockProjection({
|
||||
db,
|
||||
workerId: 'failure',
|
||||
redis: {
|
||||
get: (key) => redis.client.get(key),
|
||||
eval: async (script, options) => {
|
||||
await redis.client.eval(script, options);
|
||||
throw new Error('fixture Redis outage');
|
||||
},
|
||||
},
|
||||
})
|
||||
).rejects.toThrow('fixture Redis outage');
|
||||
expect(await db.message.count()).toBe(0);
|
||||
await db.clockProjectionOutbox.updateMany({ data: { availableAt: new Date(0) } });
|
||||
expect(await applyNextClockProjection({ db, redis: redis.client, workerId: 'retry' })).toBe('RECOVERED');
|
||||
const row = await db.worldState.findFirstOrThrow();
|
||||
const recovery = readTurnRecovery(row);
|
||||
expect(await db.message.count()).toBe(recovery ? 1 : 0);
|
||||
const clock = new GameClock({
|
||||
baseTime: row.clockBaseTime!,
|
||||
tick: Number(row.clockTick),
|
||||
wallAnchor: row.clockWallAnchor!,
|
||||
turnSeconds: row.tickSeconds,
|
||||
mode: 'realtime',
|
||||
recovery,
|
||||
});
|
||||
if (recovery) {
|
||||
expect(row.clockWallAnchor).toEqual(recovery.startWallAt);
|
||||
expect(clock.nowTick(now)).toBe(observed + plan.shiftTicks);
|
||||
expect(clock.nowTick(new Date(recovery.startWallAt.getTime() - 1))).toBe(observed + plan.shiftTicks);
|
||||
const end = clock.tickToWallDate(recovery.endTick);
|
||||
expect(clock.nowTick(end)).toBe(clock.normalNowTick(end));
|
||||
} else {
|
||||
expect(clock.nowTick(now)).toBe(observed + gap * 10);
|
||||
}
|
||||
expect(await applyNextClockProjection({ db, redis: redis.client, workerId: 'done' })).toBe('IDLE');
|
||||
expect(await db.message.count()).toBe(recovery ? 1 : 0);
|
||||
}
|
||||
);
|
||||
|
||||
|
||||
@@ -49,6 +49,36 @@ describe('TurnDaemonLifecycle', () => {
|
||||
}
|
||||
);
|
||||
|
||||
it('holds even an explicit run during the recovery wait', async () => {
|
||||
const now = new Date('2026-09-07T00:24:00Z');
|
||||
const startsAt = new Date('2026-09-07T00:35:00Z');
|
||||
const controlQueue = new InMemoryControlQueue();
|
||||
controlQueue.enqueue({ type: 'run', reason: 'manual' });
|
||||
const processor = { run: vi.fn() };
|
||||
const lifecycle = new TurnDaemonLifecycle(
|
||||
{
|
||||
clock: new ManualClock(now.getTime()),
|
||||
controlQueue,
|
||||
processor,
|
||||
getNextTickTime: (value) => addMinutes(value, 60),
|
||||
stateStore: {
|
||||
loadLastTurnTime: async () => now,
|
||||
loadNextGeneralTurnTime: async () => now,
|
||||
saveLastTurnTime: async () => {},
|
||||
loadCheckpoint: async () => undefined,
|
||||
saveCheckpoint: async () => {},
|
||||
loadGameClock: async () => {
|
||||
controlQueue.enqueue({ type: 'shutdown' });
|
||||
return { mode: 'realtime', phase: 'RUNNING', now, startsAt };
|
||||
},
|
||||
},
|
||||
},
|
||||
{ profile: 'recovery-wait-gate', defaultBudget: { budgetMs: 100, maxGenerals: 10, catchUpCap: 1 } }
|
||||
);
|
||||
await lifecycle.start();
|
||||
expect(processor.run).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('durably rebases a long realtime backlog before executing another turn', async () => {
|
||||
const wallNow = new Date('2026-08-23T01:35:00.000Z');
|
||||
const clock = new ManualClock(wallNow.getTime());
|
||||
|
||||
@@ -6247,8 +6247,8 @@ for (const viewport of [
|
||||
{ width: 1200, height: 900 },
|
||||
{ width: 390, height: 844 },
|
||||
]) {
|
||||
test(`turn recovery returns to normal speed at the boundary (${viewport.width}px)`, async ({ page }) => {
|
||||
const start = new Date('2026-09-06T07:59:50Z');
|
||||
test(`turn recovery waits then accelerates and returns to normal speed (${viewport.width}px)`, async ({ page }) => {
|
||||
const start = new Date('2026-09-06T07:59:45Z');
|
||||
await page.clock.install({ time: start });
|
||||
await page.setViewportSize(viewport);
|
||||
const state: NavigationFixture = {
|
||||
@@ -6262,15 +6262,18 @@ for (const viewport of [
|
||||
serverTime: '2026-09-06T07:59:40Z',
|
||||
serverWallTime: start.toISOString(),
|
||||
clockMode: 'realtime',
|
||||
clockRunning: true,
|
||||
clockRunning: false,
|
||||
clockStartsAt: '2026-09-06T07:59:50Z',
|
||||
turnEngineRunning: true,
|
||||
clockRecovery: { startsAt: '2026-09-06T04:00:00Z', endsAt: '2026-09-06T08:00:00Z' },
|
||||
clockRecovery: { startsAt: '2026-09-06T07:59:50Z', endsAt: '2026-09-06T08:00:00Z' },
|
||||
};
|
||||
await installFixture(page, state);
|
||||
await page.goto('./');
|
||||
await expect(page.locator('.game-shell__title')).toBeVisible({ timeout: 15_000 });
|
||||
await page.evaluate(() => document.fonts.ready);
|
||||
const status = page.locator('.execution-status:visible');
|
||||
await expect(status).not.toContainText('복구 2배속');
|
||||
await page.clock.runFor(5_000);
|
||||
await expect(status).toContainText('복구 2배속');
|
||||
const root = process.env.TURN_RECOVERY_ARTIFACT_DIR;
|
||||
const measure = () =>
|
||||
@@ -6288,7 +6291,7 @@ for (const viewport of [
|
||||
await mkdir(root, { recursive: true });
|
||||
await page.screenshot({ path: resolve(root, `recovering-${viewport.width}.png`), fullPage: true });
|
||||
}
|
||||
await page.clock.runFor(20_000);
|
||||
await page.clock.runFor(10_000);
|
||||
await expect(status).not.toContainText('복구 2배속');
|
||||
await expect(status).toContainText('17:00');
|
||||
const after = await measure();
|
||||
|
||||
@@ -68,3 +68,27 @@ void test('a single browser sample accelerates only inside the recovery window a
|
||||
assert.equal(projectServerClock(sample, 340 * minute).time.toISOString(), '2026-09-06T10:00:00.000Z');
|
||||
assert.equal(projectServerClock(sample, 280 * minute).rate, 1);
|
||||
});
|
||||
|
||||
void test('waits then accelerates from a partial month and returns to normal without resampling', () => {
|
||||
const sample = sampleServerClock(
|
||||
{
|
||||
serverTime: '2026-09-07T00:10:00Z',
|
||||
serverWallTime: '2026-09-07T00:24:00Z',
|
||||
clockMode: 'realtime',
|
||||
clockRunning: false,
|
||||
clockStartsAt: '2026-09-07T00:35:00Z',
|
||||
clockRecovery: { startsAt: '2026-09-07T00:35:00Z', endsAt: '2026-09-07T01:00:00Z' },
|
||||
},
|
||||
0
|
||||
);
|
||||
assert.ok(sample);
|
||||
for (const elapsed of [0, 660000 - 1, 660000]) {
|
||||
assert.equal(projectServerClock(sample, elapsed).time.toISOString(), '2026-09-07T00:10:00.000Z');
|
||||
}
|
||||
assert.equal(projectServerClock(sample, 660001).time.toISOString(), '2026-09-07T00:10:00.002Z');
|
||||
assert.equal(projectServerClock(sample, 2160000 - 1).time.toISOString(), '2026-09-07T00:59:59.998Z');
|
||||
assert.equal(projectServerClock(sample, 2160000 - 1).rate, 2);
|
||||
assert.equal(projectServerClock(sample, 2160000).time.toISOString(), '2026-09-07T01:00:00.000Z');
|
||||
assert.equal(projectServerClock(sample, 2160000).rate, 1);
|
||||
assert.equal(projectServerClock(sample, 2160001).time.toISOString(), '2026-09-07T01:00:00.001Z');
|
||||
});
|
||||
|
||||
@@ -40,30 +40,53 @@ alignedTick = cutTick + gapTicks
|
||||
deadlineAfter = deadlineBefore + shiftTicks
|
||||
```
|
||||
|
||||
From 2026-09-06, maintenance and crash recovery use `RECOVER_TURNS`.
|
||||
This supersedes the earlier same-day `PRESERVE_SCHEDULE` immediate catch-up
|
||||
policy. The base turn length does not change. One turn remains 36,000,000
|
||||
ticks; a persisted `TurnRecoveryWindow` changes only the wall execution rate.
|
||||
From 2026-09-07, maintenance and crash recovery use the following
|
||||
`RECOVER_TURNS` policy. The base turn length does not change. One turn remains
|
||||
36,000,000 ticks; a persisted `TurnRecoveryWindow` changes only wall execution.
|
||||
|
||||
- Count complete overdue turns from the durable observation to the normal
|
||||
timeline. Skip only `floor(overdueTurns / 12) * 12` turns, moving future
|
||||
schedules and the execution cursor by the same integer delta.
|
||||
- Execute the sub-turn remainder immediately through the ordinary engine.
|
||||
Reach the next normal turn boundary at normal speed, then execute the
|
||||
remaining one to eleven turns of backlog at 2x speed.
|
||||
- Join the original schedule at the recorded end boundary and return to 1x.
|
||||
Four hours of backlog on a 60-minute server needs four hours at 2x; it runs
|
||||
eight turns in that time. Resources, RNG, commands and monthly handlers run
|
||||
normally for those turns, in the existing chronological order.
|
||||
- Purchased within-turn offsets remain logical offsets. Their wall offsets
|
||||
compress during recovery and return to the original minutes/seconds after
|
||||
the end boundary. The API and browser expose the recovery interval.
|
||||
- An entire delay strictly below `min(600 seconds, turnSeconds / 10)` catches
|
||||
up immediately through the ordinary engine. Equality uses recovery. The
|
||||
limit is 30 seconds on a 5-minute server and 6 minutes on a 60-minute server.
|
||||
- For longer delays, skip only complete 12-turn blocks, moving future
|
||||
schedules and the execution cursor by the same integer delta. Never apply
|
||||
the short-delay exception again to the remainder. An exact multiple of
|
||||
12 turns needs no acceleration window.
|
||||
- Preserve the remaining observation, including its partial-month position.
|
||||
Wait without advancing, then execute at 2x until joining the original
|
||||
schedule at a logical month boundary. There is no initial 1x segment or
|
||||
fractional immediate burst in a newly planned window.
|
||||
- Let `S` be the observation after skips, `N` the normal tick at resume,
|
||||
`T` one turn, and `r` ticks per second. Choose
|
||||
`E = ceil((2*N-S)/T)*T`. The end is `resume + (E-N)/r` and the wait is
|
||||
`(E+S-2*N)/(2*r)`. End wall time and 2x duration round up to milliseconds;
|
||||
derive the start by subtraction. This preserves the end boundary with at
|
||||
most one millisecond of wall resolution error.
|
||||
- On a 60-minute server stopped at game 00:10 and resumed at wall 00:24,
|
||||
wait until 00:35, then run 2x until game/wall 01:00. A 240-minute outage
|
||||
from the same stop waits from 04:10 to 04:35 and rejoins at 09:00.
|
||||
- Resources, RNG, commands and monthly handlers execute in the existing
|
||||
chronological order. Purchased within-turn offsets remain logical offsets;
|
||||
their wall offsets compress during recovery and return to normal afterward.
|
||||
|
||||
`clock_recovery_start_tick`, `clock_recovery_end_tick`, and
|
||||
`clock_recovery_start_wall_at` are an all-or-none durable window. Flush/reload
|
||||
preserves it; a short restart reuses it. A new long outage replans against the
|
||||
original normal timeline. Game-date epoch and wall epoch may differ; never
|
||||
convert the real wall instant using `dateToTick` to compute normal time.
|
||||
`clock_recovery_start_wall_at` are an all-or-none durable window. The migration
|
||||
`20260907150000_wait_then_turn_recovery` permits partial-month starts and keeps
|
||||
end-boundary and safe-integer constraints. Existing windows remain valid and
|
||||
retain their old pre-start 1x behavior; new windows use the stored tick as the
|
||||
frozen lower bound and `clockWallAnchor` as the future start gate. Do not roll
|
||||
back to code requiring whole-turn starts while a new window is stored.
|
||||
|
||||
Flush/reload preserves the window; short restarts reuse it. A new long outage
|
||||
replans against the original normal timeline. Game-date epoch and wall epoch
|
||||
may differ; never use `dateToTick(realWallInstant)` to compute normal time.
|
||||
The API and browser expose waiting, start and end boundaries without needing
|
||||
a fresh response at the speed transitions.
|
||||
|
||||
Projection completion stores one public system message with both recovery
|
||||
wall dates (Korean time) and ISO interval metadata. Message creation, mailbox
|
||||
read-model invalidation, and the `RUNNING`/outbox `APPLIED` transition share a
|
||||
PostgreSQL transaction. Retries after Redis application cannot duplicate the
|
||||
message. Existing read-model outbox delivery refreshes connected clients.
|
||||
|
||||
Before a newly leased daemon permits independent workers to advance time, it
|
||||
prepares recovery under the clock lock. `turn_daemon_lease.clock_ready` starts
|
||||
|
||||
@@ -241,6 +241,7 @@ export const buildClockAlignmentPlan = (input: {
|
||||
catchUpTicks: asGameTick(Math.max(0, (input.normalTick ?? exact.alignedTick) - input.cutTick - shiftTicks)),
|
||||
alignedTick: recovery.initialTick,
|
||||
recovery: recovery.recovery,
|
||||
...(recovery.recovery ? { resumeAnchor: recovery.recovery.startWallAt } : {}),
|
||||
...(recovery.initialTick > (input.normalTick ?? exact.alignedTick)
|
||||
? {
|
||||
resumeAnchor: new Date(
|
||||
@@ -380,8 +381,8 @@ export class GameClock {
|
||||
normalNowTick(wallNow: Date): number {
|
||||
if (this.recovery) {
|
||||
return this.addTicks(
|
||||
(this.recovery.startTick + this.recovery.endTick) / 2,
|
||||
this.ticksBetween(this.recovery.startWallAt, wallNow)
|
||||
this.recovery.endTick,
|
||||
this.ticksBetween(this.tickToWallDate(this.recovery.endTick), wallNow)
|
||||
);
|
||||
}
|
||||
return this.addTicks(this.tick, this.ticksBetween(this.wallAnchor, wallNow));
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { asGameTick, GAME_TICKS_PER_TURN, type GameTick } from './gameTimeUnits.js';
|
||||
|
||||
/** 정상 시간표는 바꾸지 않고, 정수 턴의 지연만 두 배 속도로 소진한다. */
|
||||
/** 정상 시간표를 유지하며 대기 후 두 배 속도로 월 경계에 합류한다. */
|
||||
export interface TurnRecoveryWindow {
|
||||
startTick: GameTick;
|
||||
endTick: GameTick;
|
||||
@@ -68,9 +68,12 @@ export const turnShiftTicks = (turns: number): GameTick => {
|
||||
return asGameTick(turns * GAME_TICKS_PER_TURN);
|
||||
};
|
||||
|
||||
/** 즉시 처리 여부는 12턴 묶음 생략 전의 전체 지연으로 판정한다. */
|
||||
export const immediateRecoveryLimitSeconds = (turnSeconds: number): number => Math.min(600, turnSeconds / 10);
|
||||
|
||||
/**
|
||||
* observedTick은 중단 전에 저장한 관측 지점, normalTick은 기존 시간표의 현재 지점이다.
|
||||
* 잔여 한 턴 미만은 정상 실행하고, 다음 경계부터 정수 턴 지연을 두 배속으로 처리한다.
|
||||
* 짧은 전체 지연만 즉시 처리한다. 그 외에는 나머지도 생략하지 않고 대기 후 두 배속으로 처리한다.
|
||||
* 반환한 skip은 호출자가 미래 일정과 실행 cursor에 원자적으로 적용해야 한다.
|
||||
*/
|
||||
export const planTurnRecovery = (input: {
|
||||
@@ -86,26 +89,32 @@ export const planTurnRecovery = (input: {
|
||||
throw new Error('Recovery requires a representable positive turn length.');
|
||||
}
|
||||
if (!Number.isFinite(wallNow.getTime())) throw new Error('Recovery wall instant is invalid.');
|
||||
const overdueTurns = Math.max(0, Math.floor((normalTick - observedTick) / GAME_TICKS_PER_TURN));
|
||||
const skippedTurns = Math.floor(overdueTurns / 12) * 12;
|
||||
const recoveryTurns = overdueTurns % 12;
|
||||
const initialTick = asGameTick(
|
||||
Math.max(observedTick + turnShiftTicks(skippedTurns), normalTick - turnShiftTicks(recoveryTurns))
|
||||
);
|
||||
if (recoveryTurns === 0) return { skippedTurns, recoveryTurns, initialTick, recovery: null };
|
||||
const boundary = nextTurnBoundary(normalTick);
|
||||
const startWallAt = new Date(
|
||||
wallNow.getTime() + Math.ceil(((boundary - normalTick) * turnSeconds * 1_000) / GAME_TICKS_PER_TURN)
|
||||
);
|
||||
const gap = Math.max(0, normalTick - observedTick);
|
||||
const ticksPerSecond = GAME_TICKS_PER_TURN / turnSeconds;
|
||||
const immediateLimit = immediateRecoveryLimitSeconds(turnSeconds) * ticksPerSecond;
|
||||
if (gap < immediateLimit) {
|
||||
return {
|
||||
skippedTurns: 0,
|
||||
recoveryTurns: 0,
|
||||
initialTick: asGameTick(Math.max(observedTick, normalTick)),
|
||||
recovery: null,
|
||||
};
|
||||
}
|
||||
const skippedTurns = Math.floor(gap / (12 * GAME_TICKS_PER_TURN)) * 12;
|
||||
const initialTick = asGameTick(observedTick + turnShiftTicks(skippedTurns));
|
||||
const remaining = normalTick - initialTick;
|
||||
if (remaining === 0) return { skippedTurns, recoveryTurns: 0, initialTick, recovery: null };
|
||||
|
||||
// 즉시 2배속으로 따라잡을 수 있는 가장 이른 지점 이후의 월 경계를 고른다.
|
||||
// 대기도 추가 지연이므로 경계까지 여유의 절반만 기다린다. 밀리초 반올림은 종료 시각을 보존한다.
|
||||
const endTick = nextTurnBoundary(normalTick + remaining);
|
||||
const endWallMs = wallNow.getTime() + Math.ceil(((endTick - normalTick) * 1_000) / ticksPerSecond);
|
||||
const durationMs = Math.ceil(((endTick - initialTick) * 1_000) / (2 * ticksPerSecond));
|
||||
return {
|
||||
skippedTurns,
|
||||
recoveryTurns,
|
||||
recoveryTurns: remaining / GAME_TICKS_PER_TURN,
|
||||
initialTick,
|
||||
recovery: {
|
||||
startTick: asGameTick(boundary - turnShiftTicks(recoveryTurns)),
|
||||
endTick: asGameTick(boundary + turnShiftTicks(recoveryTurns)),
|
||||
startWallAt,
|
||||
},
|
||||
recovery: { startTick: initialTick, endTick, startWallAt: new Date(endWallMs - durationMs) },
|
||||
};
|
||||
};
|
||||
|
||||
@@ -115,23 +124,32 @@ export const validateTurnRecovery = (window: TurnRecoveryWindow): void => {
|
||||
const span = window.endTick - window.startTick;
|
||||
if (
|
||||
!Number.isFinite(window.startWallAt.getTime()) ||
|
||||
window.startTick % GAME_TICKS_PER_TURN !== 0 ||
|
||||
window.endTick % GAME_TICKS_PER_TURN !== 0 ||
|
||||
span <= 0 ||
|
||||
span % (2 * GAME_TICKS_PER_TURN) !== 0 ||
|
||||
span >= 24 * GAME_TICKS_PER_TURN
|
||||
span >= 25 * GAME_TICKS_PER_TURN
|
||||
)
|
||||
throw new Error('Recovery must join turn boundaries after one to eleven turns at double speed.');
|
||||
throw new Error('Recovery must end at a turn boundary with a positive span below twenty-five turns.');
|
||||
};
|
||||
|
||||
/** 경계 전에는 정상 속도, 복구 구간은 두 배, 합류 경계 이후는 정상 속도이다. */
|
||||
export const observeTurnRecovery = (window: TurnRecoveryWindow, wallNow: Date, ticksPerSecond: number): GameTick => {
|
||||
validateTurnRecovery(window);
|
||||
const elapsed = asGameTick(
|
||||
Math.trunc(((wallNow.getTime() - window.startWallAt.getTime()) * ticksPerSecond) / 1_000)
|
||||
Math.trunc(
|
||||
((wallNow.getTime() - window.startWallAt.getTime()) *
|
||||
ticksPerSecond *
|
||||
(wallNow < window.startWallAt ? 1 : 2)) /
|
||||
1_000
|
||||
)
|
||||
);
|
||||
const halfSpan = (window.endTick - window.startTick) / 2;
|
||||
return asGameTick(window.startTick + elapsed + Math.max(0, Math.min(elapsed, halfSpan)));
|
||||
const endWallAt = projectRecoveryDeadline(window, window.endTick, ticksPerSecond);
|
||||
if (wallNow >= endWallAt) {
|
||||
return asGameTick(
|
||||
window.endTick + Math.trunc(((wallNow.getTime() - endWallAt.getTime()) * ticksPerSecond) / 1_000)
|
||||
);
|
||||
}
|
||||
// 이전 복구 창은 시작 전 1배속이었다. 새 창은 GameClock의 저장 tick 하한으로 대기한다.
|
||||
return asGameTick(Math.min(window.endTick, window.startTick + elapsed));
|
||||
};
|
||||
|
||||
/** 게임 좌표의 예정 시각을 사용자에게 표시할 실제 실행 시각으로 투영한다. */
|
||||
@@ -140,6 +158,10 @@ export const projectRecoveryDeadline = (window: TurnRecoveryWindow, tick: number
|
||||
asGameTick(tick);
|
||||
const offset = tick - window.startTick;
|
||||
const span = window.endTick - window.startTick;
|
||||
const elapsed = offset < 0 ? offset : offset <= span ? offset / 2 : offset - span / 2;
|
||||
if (offset > span) {
|
||||
const endWallMs = window.startWallAt.getTime() + Math.ceil((span * 1_000) / (2 * ticksPerSecond));
|
||||
return new Date(endWallMs + Math.ceil(((offset - span) * 1_000) / ticksPerSecond));
|
||||
}
|
||||
const elapsed = offset < 0 ? offset : offset / 2;
|
||||
return new Date(window.startWallAt.getTime() + Math.ceil((elapsed * 1_000) / ticksPerSecond));
|
||||
};
|
||||
|
||||
@@ -35,6 +35,21 @@ describe('turn-aligned double-speed recovery', () => {
|
||||
expect(reloaded.executionRate(wall(8))).toBe(1);
|
||||
});
|
||||
|
||||
it('preserves pre-start normal speed for an existing serialized window', () => {
|
||||
const old = new GameClock({
|
||||
baseTime: wall(0),
|
||||
tick: T / 3,
|
||||
wallAnchor: wall(4 + 1 / 3),
|
||||
mode: 'realtime',
|
||||
turnSeconds: 3600,
|
||||
recovery: { startTick: nextTurnBoundary(T), endTick: nextTurnBoundary(9 * T), startWallAt: wall(5) },
|
||||
});
|
||||
expect(old.nowTick(wall(4.5))).toBe(T / 2);
|
||||
expect(old.nowTick(wall(5))).toBe(T);
|
||||
expect(old.nowTick(wall(9))).toBe(9 * T);
|
||||
expect(old.normalNowTick(wall(4.5))).toBe(4.5 * T);
|
||||
});
|
||||
|
||||
it('resumes a planned wait at a whole-turn boundary without changing purchased phase', () => {
|
||||
const plan = buildClockAlignmentPlan({
|
||||
policy: 'TURN_BOUNDARY',
|
||||
@@ -102,18 +117,104 @@ describe('turn-aligned double-speed recovery', () => {
|
||||
expect(observeTurnRecovery(recovery!, wall(9), 10_000)).toBe(9 * T);
|
||||
});
|
||||
|
||||
it('retains the fractional phase and begins acceleration at the next boundary', () => {
|
||||
it.each([
|
||||
[14, 11, 60],
|
||||
[49, 6, 120],
|
||||
[59, 26, 180],
|
||||
[100, 15, 240],
|
||||
[240, 25, 540],
|
||||
[400, 15, 840],
|
||||
[700, 15, 1440],
|
||||
])('waits then runs 2x after stopping at 00:10 for %i minutes', (delay, wait, endMinute) => {
|
||||
const observed = T / 6;
|
||||
const resumed = wall((10 + delay) / 60);
|
||||
const plan = planTurnRecovery({
|
||||
observedTick: 0,
|
||||
normalTick: 4 * T + T / 3,
|
||||
wallNow: wall(4 + 1 / 3),
|
||||
observedTick: observed,
|
||||
normalTick: ((10 + delay) * T) / 60,
|
||||
wallNow: resumed,
|
||||
turnSeconds: 3600,
|
||||
});
|
||||
expect(plan.initialTick).toBe(T / 3);
|
||||
expect(plan.recovery!.startWallAt).toEqual(wall(5));
|
||||
expect(observeTurnRecovery(plan.recovery!, wall(4.5), 10_000)).toBe(T / 2);
|
||||
expect(observeTurnRecovery(plan.recovery!, wall(5), 10_000)).toBe(T);
|
||||
expect(observeTurnRecovery(plan.recovery!, wall(9), 10_000)).toBe(9 * T);
|
||||
const recovery = plan.recovery!;
|
||||
const start = recovery.startWallAt;
|
||||
const end = wall(endMinute / 60);
|
||||
const clock = new GameClock({
|
||||
baseTime: wall(0),
|
||||
tick: plan.initialTick,
|
||||
wallAnchor: start,
|
||||
mode: 'realtime',
|
||||
turnSeconds: 3600,
|
||||
recovery,
|
||||
});
|
||||
expect(plan.initialTick).toBe(observed);
|
||||
expect(start.getTime() - resumed.getTime()).toBeCloseTo(wait * 60000, 0);
|
||||
expect(clock.nowTick(resumed)).toBe(observed);
|
||||
expect(clock.nowTick(new Date(start.getTime() - 1))).toBe(observed);
|
||||
expect(clock.nowTick(start)).toBe(observed);
|
||||
expect(clock.nowTick(new Date(start.getTime() + 1))).toBe(observed + 20);
|
||||
expect(clock.tickToWallDate(recovery.endTick)).toEqual(end);
|
||||
expect(recovery.endTick % T).toBe(0);
|
||||
expect(clock.nowTick(new Date(end.getTime() - 1))).toBe(recovery.endTick - 20);
|
||||
expect(clock.nowTick(end)).toBe(clock.normalNowTick(end));
|
||||
expect(clock.nowTick(new Date(end.getTime() + 1))).toBe(recovery.endTick + 10);
|
||||
expect(clock.executionRate(new Date(end.getTime() - 1))).toBe(2);
|
||||
expect(clock.executionRate(end)).toBe(1);
|
||||
});
|
||||
|
||||
it.each([300, 3600, 6000, 7200])('uses the strict whole-delay threshold for a %i second turn', (turnSeconds) => {
|
||||
const rate = T / turnSeconds;
|
||||
const limitMs = Math.min(600, turnSeconds / 10) * 1000;
|
||||
for (const delta of [-1, 0, 1]) {
|
||||
const delayMs = limitMs + delta;
|
||||
const normal = Math.trunc((delayMs * rate) / 1000);
|
||||
const plan = planTurnRecovery({
|
||||
observedTick: 0,
|
||||
normalTick: normal,
|
||||
wallNow: new Date(base + delayMs),
|
||||
turnSeconds,
|
||||
});
|
||||
expect(plan.recovery === null).toBe(delta < 0);
|
||||
expect(plan.initialTick).toBe(delta < 0 ? normal : 0);
|
||||
}
|
||||
// 장시간 중단의 작은 나머지에는 즉시 처리 예외를 다시 적용하지 않는다.
|
||||
const long = planTurnRecovery({ observedTick: 0, normalTick: 12 * T + rate, wallNow: wall(20), turnSeconds });
|
||||
expect(long.skippedTurns).toBe(12);
|
||||
expect(long.initialTick).toBe(12 * T);
|
||||
expect(long.recovery).not.toBeNull();
|
||||
});
|
||||
|
||||
it('keeps integer ticks and deadline ordering for sub-millisecond phases', () => {
|
||||
for (const turnSeconds of [300, 3600, 7200, 36000]) {
|
||||
const rate = T / turnSeconds;
|
||||
for (const observed of [1, T / 6 + 1, T - 1]) {
|
||||
const now = wall(4);
|
||||
const normal = observed + 4 * T + 1;
|
||||
const plan = planTurnRecovery({
|
||||
observedTick: observed,
|
||||
normalTick: normal,
|
||||
wallNow: now,
|
||||
turnSeconds,
|
||||
});
|
||||
const recovery = plan.recovery!;
|
||||
const clock = new GameClock({
|
||||
baseTime: wall(0),
|
||||
tick: observed,
|
||||
wallAnchor: recovery.startWallAt,
|
||||
mode: 'realtime',
|
||||
turnSeconds,
|
||||
recovery,
|
||||
});
|
||||
expect(recovery.startWallAt.getTime()).toBeGreaterThanOrEqual(now.getTime());
|
||||
const end = clock.tickToWallDate(recovery.endTick);
|
||||
expect(clock.nowTick(end)).toBe(recovery.endTick);
|
||||
expect(clock.nowTick(new Date(end.getTime() - 1))).toBeLessThan(recovery.endTick);
|
||||
expect(Math.abs(clock.normalNowTick(now) - normal)).toBeLessThanOrEqual(Math.ceil(rate / 1000));
|
||||
for (const tick of [observed + 1, observed + T, recovery.endTick - 1, recovery.endTick + 1]) {
|
||||
const deadline = clock.tickToWallDate(tick);
|
||||
expect(clock.nowTick(deadline)).toBeGreaterThanOrEqual(tick);
|
||||
expect(clock.nowTick(new Date(deadline.getTime() - 1))).toBeLessThan(tick);
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
it('preserves purchased phase coordinates while projecting compressed wall deadlines', () => {
|
||||
|
||||
@@ -0,0 +1,14 @@
|
||||
-- 복구 시작은 중단 당시 월 내부 좌표를 보존하고, 종료만 월 경계에 맞춘다.
|
||||
-- 기존 월 경계 시작 창도 그대로 유효하며 저장 좌표를 변경하지 않는다.
|
||||
ALTER TABLE "world_state"
|
||||
DROP CONSTRAINT "world_state_turn_recovery_window_check",
|
||||
ADD CONSTRAINT "world_state_turn_recovery_window_check" CHECK (
|
||||
("clock_recovery_start_tick" IS NULL AND "clock_recovery_end_tick" IS NULL AND "clock_recovery_start_wall_at" IS NULL)
|
||||
OR (
|
||||
"clock_recovery_start_tick" IS NOT NULL AND "clock_recovery_end_tick" IS NOT NULL AND "clock_recovery_start_wall_at" IS NOT NULL
|
||||
AND "clock_recovery_start_tick" BETWEEN -9007199254740991 AND 9007199254740991
|
||||
AND "clock_recovery_end_tick" BETWEEN -9007199254740991 AND 9007199254740991
|
||||
AND "clock_recovery_end_tick" % 36000000 = 0
|
||||
AND "clock_recovery_end_tick" - "clock_recovery_start_tick" BETWEEN 1 AND 899999999
|
||||
)
|
||||
);
|
||||
Reference in New Issue
Block a user