fix(clock): preserve UTC leases and unification recovery
This commit is contained in:
@@ -86,9 +86,9 @@ export class DatabaseTurnDaemonLease {
|
|||||||
VALUES (
|
VALUES (
|
||||||
${this.profile},
|
${this.profile},
|
||||||
${this.ownerId},
|
${this.ownerId},
|
||||||
CURRENT_TIMESTAMP + (${this.leaseDurationMs} * INTERVAL '1 millisecond'),
|
(CURRENT_TIMESTAMP AT TIME ZONE 'UTC') + (${this.leaseDurationMs} * INTERVAL '1 millisecond'),
|
||||||
1,
|
1,
|
||||||
CURRENT_TIMESTAMP
|
CURRENT_TIMESTAMP AT TIME ZONE 'UTC'
|
||||||
)
|
)
|
||||||
ON CONFLICT ("profile") DO UPDATE
|
ON CONFLICT ("profile") DO UPDATE
|
||||||
SET
|
SET
|
||||||
@@ -99,10 +99,10 @@ export class DatabaseTurnDaemonLease {
|
|||||||
THEN "turn_daemon_lease"."fencing_epoch"
|
THEN "turn_daemon_lease"."fencing_epoch"
|
||||||
ELSE "turn_daemon_lease"."fencing_epoch" + 1
|
ELSE "turn_daemon_lease"."fencing_epoch" + 1
|
||||||
END,
|
END,
|
||||||
"heartbeat_at" = CURRENT_TIMESTAMP
|
"heartbeat_at" = CURRENT_TIMESTAMP AT TIME ZONE 'UTC'
|
||||||
WHERE
|
WHERE
|
||||||
"turn_daemon_lease"."owner_id" = EXCLUDED."owner_id"
|
"turn_daemon_lease"."owner_id" = EXCLUDED."owner_id"
|
||||||
OR "turn_daemon_lease"."lease_until" <= CURRENT_TIMESTAMP
|
OR "turn_daemon_lease"."lease_until" <= CURRENT_TIMESTAMP AT TIME ZONE 'UTC'
|
||||||
RETURNING "profile", "owner_id", "fencing_epoch"
|
RETURNING "profile", "owner_id", "fencing_epoch"
|
||||||
`);
|
`);
|
||||||
const row = rows[0];
|
const row = rows[0];
|
||||||
@@ -141,13 +141,13 @@ export class DatabaseTurnDaemonLease {
|
|||||||
const rows = await this.db.$queryRaw<LeaseRow[]>(GamePrisma.sql`
|
const rows = await this.db.$queryRaw<LeaseRow[]>(GamePrisma.sql`
|
||||||
UPDATE "turn_daemon_lease"
|
UPDATE "turn_daemon_lease"
|
||||||
SET
|
SET
|
||||||
"lease_until" = CURRENT_TIMESTAMP + (${this.leaseDurationMs} * INTERVAL '1 millisecond'),
|
"lease_until" = (CURRENT_TIMESTAMP AT TIME ZONE 'UTC') + (${this.leaseDurationMs} * INTERVAL '1 millisecond'),
|
||||||
"heartbeat_at" = CURRENT_TIMESTAMP
|
"heartbeat_at" = CURRENT_TIMESTAMP AT TIME ZONE 'UTC'
|
||||||
WHERE
|
WHERE
|
||||||
"profile" = ${token.profile}
|
"profile" = ${token.profile}
|
||||||
AND "owner_id" = ${token.ownerId}
|
AND "owner_id" = ${token.ownerId}
|
||||||
AND "fencing_epoch" = ${token.fencingEpoch}
|
AND "fencing_epoch" = ${token.fencingEpoch}
|
||||||
AND "lease_until" > CURRENT_TIMESTAMP
|
AND "lease_until" > CURRENT_TIMESTAMP AT TIME ZONE 'UTC'
|
||||||
RETURNING "profile", "owner_id", "fencing_epoch"
|
RETURNING "profile", "owner_id", "fencing_epoch"
|
||||||
`);
|
`);
|
||||||
if (rows.length === 0) {
|
if (rows.length === 0) {
|
||||||
@@ -177,7 +177,7 @@ export class DatabaseTurnDaemonLease {
|
|||||||
"profile" = ${token.profile}
|
"profile" = ${token.profile}
|
||||||
AND "owner_id" = ${token.ownerId}
|
AND "owner_id" = ${token.ownerId}
|
||||||
AND "fencing_epoch" = ${token.fencingEpoch}
|
AND "fencing_epoch" = ${token.fencingEpoch}
|
||||||
AND "lease_until" > CURRENT_TIMESTAMP
|
AND "lease_until" > CURRENT_TIMESTAMP AT TIME ZONE 'UTC'
|
||||||
FOR UPDATE
|
FOR UPDATE
|
||||||
`);
|
`);
|
||||||
if (rows.length === 0) {
|
if (rows.length === 0) {
|
||||||
@@ -196,7 +196,8 @@ export class DatabaseTurnDaemonLease {
|
|||||||
}
|
}
|
||||||
await this.db.$executeRaw(GamePrisma.sql`
|
await this.db.$executeRaw(GamePrisma.sql`
|
||||||
UPDATE "turn_daemon_lease"
|
UPDATE "turn_daemon_lease"
|
||||||
SET "lease_until" = CURRENT_TIMESTAMP, "heartbeat_at" = CURRENT_TIMESTAMP
|
SET "lease_until" = CURRENT_TIMESTAMP AT TIME ZONE 'UTC',
|
||||||
|
"heartbeat_at" = CURRENT_TIMESTAMP AT TIME ZONE 'UTC'
|
||||||
WHERE
|
WHERE
|
||||||
"profile" = ${token.profile}
|
"profile" = ${token.profile}
|
||||||
AND "owner_id" = ${token.ownerId}
|
AND "owner_id" = ${token.ownerId}
|
||||||
|
|||||||
@@ -98,11 +98,12 @@ const invalidateMessageIds = async (
|
|||||||
db: GamePrisma.TransactionClient,
|
db: GamePrisma.TransactionClient,
|
||||||
world: InMemoryTurnWorld,
|
world: InMemoryTurnWorld,
|
||||||
ids: number[],
|
ids: number[],
|
||||||
now: Date
|
now: Date,
|
||||||
|
authoritativeGameTick?: bigint
|
||||||
): Promise<void> => {
|
): Promise<void> => {
|
||||||
const uniqueIds = [...new Set(ids.filter((id) => Number.isInteger(id) && id > 0))];
|
const uniqueIds = [...new Set(ids.filter((id) => Number.isInteger(id) && id > 0))];
|
||||||
if (uniqueIds.length === 0) return;
|
if (uniqueIds.length === 0) return;
|
||||||
const resolvedGameTick = BigInt(world.dateToGameTick(now));
|
const resolvedGameTick = authoritativeGameTick ?? BigInt(world.dateToGameTick(now));
|
||||||
await db.messageAction.updateMany({
|
await db.messageAction.updateMany({
|
||||||
where: { messageId: { in: uniqueIds }, status: 'PENDING' },
|
where: { messageId: { in: uniqueIds }, status: 'PENDING' },
|
||||||
data: { status: 'RESOLVED', resolvedGameTick },
|
data: { status: 'RESOLVED', resolvedGameTick },
|
||||||
@@ -348,7 +349,27 @@ const respondToRaiseInvader = async (options: {
|
|||||||
},
|
},
|
||||||
event
|
event
|
||||||
);
|
);
|
||||||
await invalidateMessageIds(db, world, [row.id], world.gameTickToDate(alignment.alignedTick));
|
const resolvedGameTick = BigInt(alignment.alignedTick);
|
||||||
|
if (resolvedGameTick < row.createdGameTick) {
|
||||||
|
throw new Error(
|
||||||
|
`RaiseInvader resolved tick ${resolvedGameTick} precedes prompt tick ${row.createdGameTick}; clock authority is inconsistent.`
|
||||||
|
);
|
||||||
|
}
|
||||||
|
const promptRows = await db.$queryRaw<Array<{ id: number }>>(GamePrisma.sql`
|
||||||
|
SELECT message_id AS id
|
||||||
|
FROM message_action
|
||||||
|
WHERE action_type = 'raiseInvader'
|
||||||
|
AND status = 'PENDING'
|
||||||
|
AND created_game_tick = ${row.createdGameTick}
|
||||||
|
FOR UPDATE
|
||||||
|
`);
|
||||||
|
await invalidateMessageIds(
|
||||||
|
db,
|
||||||
|
world,
|
||||||
|
promptRows.map(({ id }) => id),
|
||||||
|
world.gameTickToDate(alignment.alignedTick),
|
||||||
|
resolvedGameTick
|
||||||
|
);
|
||||||
return { ok: true, action: 'raiseInvader', reason: 'success' };
|
return { ok: true, action: 'raiseInvader', reason: 'success' };
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|||||||
@@ -207,6 +207,57 @@ integration('database turn daemon lease and fencing', () => {
|
|||||||
expect(row.ownerId).toBe(firstToken ? 'owner-a' : 'owner-b');
|
expect(row.ownerId).toBe(firstToken ? 'owner-a' : 'owner-b');
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('persists and evaluates lease wall time as UTC under a non-UTC database session', async () => {
|
||||||
|
const profile = `${profilePrefix}utc-wall`;
|
||||||
|
const zonedUrl = new URL(databaseUrl!);
|
||||||
|
const schema = zonedUrl.searchParams.get('schema') ?? 'public';
|
||||||
|
zonedUrl.searchParams.delete('schema');
|
||||||
|
zonedUrl.searchParams.set('options', `-c search_path=${schema} -c TimeZone=Asia/Seoul`);
|
||||||
|
const zonedConnector = createGamePostgresConnector({ url: zonedUrl.toString() });
|
||||||
|
await zonedConnector.connect();
|
||||||
|
const lease = await DatabaseTurnDaemonLease.connect(zonedUrl.toString(), {
|
||||||
|
profile,
|
||||||
|
ownerId: 'utc-owner',
|
||||||
|
leaseDurationMs: 60_000,
|
||||||
|
heartbeat: false,
|
||||||
|
});
|
||||||
|
leases.push(lease);
|
||||||
|
try {
|
||||||
|
await expect(lease.acquire()).resolves.toMatchObject({ profile, ownerId: 'utc-owner' });
|
||||||
|
const [leaseRows, wallRows] = await Promise.all([
|
||||||
|
zonedConnector.prisma.$queryRaw<Array<{ leaseUntil: Date }>>`
|
||||||
|
SELECT lease_until AS "leaseUntil"
|
||||||
|
FROM turn_daemon_lease
|
||||||
|
WHERE profile = ${profile}
|
||||||
|
`,
|
||||||
|
zonedConnector.prisma.$queryRaw<Array<{ zone: string; wallNow: Date }>>`
|
||||||
|
SELECT current_setting('TimeZone') AS zone,
|
||||||
|
(CURRENT_TIMESTAMP AT TIME ZONE 'UTC')::timestamp(3) AS "wallNow"
|
||||||
|
`,
|
||||||
|
]);
|
||||||
|
expect(wallRows[0]?.zone).toBe('Asia/Seoul');
|
||||||
|
expect(leaseRows[0]!.leaseUntil.getTime() - wallRows[0]!.wallNow.getTime()).toBeGreaterThan(50_000);
|
||||||
|
expect(leaseRows[0]!.leaseUntil.getTime() - wallRows[0]!.wallNow.getTime()).toBeLessThanOrEqual(60_000);
|
||||||
|
|
||||||
|
await lease.release();
|
||||||
|
const [releasedRows, releasedWall] = await Promise.all([
|
||||||
|
zonedConnector.prisma.$queryRaw<Array<{ leaseUntil: Date }>>`
|
||||||
|
SELECT lease_until AS "leaseUntil"
|
||||||
|
FROM turn_daemon_lease
|
||||||
|
WHERE profile = ${profile}
|
||||||
|
`,
|
||||||
|
zonedConnector.prisma.$queryRaw<Array<{ wallNow: Date }>>`
|
||||||
|
SELECT (CURRENT_TIMESTAMP AT TIME ZONE 'UTC')::timestamp(3) AS "wallNow"
|
||||||
|
`,
|
||||||
|
]);
|
||||||
|
expect(Math.abs(releasedRows[0]!.leaseUntil.getTime() - releasedWall[0]!.wallNow.getTime())).toBeLessThan(
|
||||||
|
1_000
|
||||||
|
);
|
||||||
|
} finally {
|
||||||
|
await zonedConnector.disconnect();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
it('increments the epoch on expiry takeover and fences the stale owner', async () => {
|
it('increments the epoch on expiry takeover and fences the stale owner', async () => {
|
||||||
const profile = `${profilePrefix}takeover`;
|
const profile = `${profilePrefix}takeover`;
|
||||||
const first = await createLease(profile, 'owner-a');
|
const first = await createLease(profile, 'owner-a');
|
||||||
|
|||||||
@@ -312,7 +312,17 @@ integration('unification finalization transaction', () => {
|
|||||||
const bidWorld = new InMemoryTurnWorld(beforeBid.state, beforeBid.snapshot, {
|
const bidWorld = new InMemoryTurnWorld(beforeBid.state, beforeBid.snapshot, {
|
||||||
schedule: { entries: [{ startMinute: 0, tickMinutes: 10 }] },
|
schedule: { entries: [{ startMinute: 0, tickMinutes: 10 }] },
|
||||||
});
|
});
|
||||||
|
const bidProcessingTick = bidWorld.getGameClockState().tick;
|
||||||
|
const futureCloseTick = BigInt(bidProcessingTick) + 86_400_000n;
|
||||||
|
await db.auction.updateMany({
|
||||||
|
where: { id: { in: [uniqueAuction.id, resourceAuction.id] } },
|
||||||
|
data: { closeTick: futureCloseTick },
|
||||||
|
});
|
||||||
const bidder = await createAuctionBidder({ databaseUrl: databaseUrl!, world: bidWorld });
|
const bidder = await createAuctionBidder({ databaseUrl: databaseUrl!, world: bidWorld });
|
||||||
|
const bidClockContext = {
|
||||||
|
processingGameTick: bidProcessingTick,
|
||||||
|
requestedAtWall: new Date('2026-09-03T00:00:00.000Z'),
|
||||||
|
};
|
||||||
try {
|
try {
|
||||||
await expect(
|
await expect(
|
||||||
bidder.bid({
|
bidder.bid({
|
||||||
@@ -322,7 +332,8 @@ integration('unification finalization transaction', () => {
|
|||||||
generalId: fixtureId,
|
generalId: fixtureId,
|
||||||
amount: 30,
|
amount: 30,
|
||||||
tryExtendCloseDate: false,
|
tryExtendCloseDate: false,
|
||||||
})
|
...bidClockContext,
|
||||||
|
} as Parameters<typeof bidder.bid>[0])
|
||||||
).resolves.toMatchObject({ ok: true, auctionId: uniqueAuction.id });
|
).resolves.toMatchObject({ ok: true, auctionId: uniqueAuction.id });
|
||||||
await expect(
|
await expect(
|
||||||
bidder.bid({
|
bidder.bid({
|
||||||
@@ -332,7 +343,8 @@ integration('unification finalization transaction', () => {
|
|||||||
generalId: fixtureId,
|
generalId: fixtureId,
|
||||||
amount: 50,
|
amount: 50,
|
||||||
tryExtendCloseDate: false,
|
tryExtendCloseDate: false,
|
||||||
})
|
...bidClockContext,
|
||||||
|
} as Parameters<typeof bidder.bid>[0])
|
||||||
).resolves.toMatchObject({ ok: true, auctionId: uniqueAuction.id });
|
).resolves.toMatchObject({ ok: true, auctionId: uniqueAuction.id });
|
||||||
} finally {
|
} finally {
|
||||||
await bidder.close();
|
await bidder.close();
|
||||||
@@ -699,6 +711,24 @@ integration('unification finalization transaction', () => {
|
|||||||
});
|
});
|
||||||
const commandResult = await stateManager.transaction(executeCommand);
|
const commandResult = await stateManager.transaction(executeCommand);
|
||||||
expect(commandResult).toMatchObject({ type: 'messageRespond', ok: true, action: 'raiseInvader' });
|
expect(commandResult).toMatchObject({ type: 'messageRespond', ok: true, action: 'raiseInvader' });
|
||||||
|
const selectedPromptAction = await db.messageAction.findUniqueOrThrow({
|
||||||
|
where: { messageId: invaderPrompt!.id },
|
||||||
|
});
|
||||||
|
const siblingPromptActions = await db.messageAction.findMany({
|
||||||
|
where: {
|
||||||
|
actionType: 'raiseInvader',
|
||||||
|
createdGameTick: selectedPromptAction.createdGameTick,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
expect(siblingPromptActions.length).toBeGreaterThan(1);
|
||||||
|
expect(
|
||||||
|
siblingPromptActions.every(
|
||||||
|
(action) =>
|
||||||
|
action.status === 'RESOLVED' &&
|
||||||
|
action.resolvedGameTick !== null &&
|
||||||
|
action.resolvedGameTick >= action.createdGameTick
|
||||||
|
)
|
||||||
|
).toBe(true);
|
||||||
const reconciledWorld = await db.worldState.findUniqueOrThrow({ where: { id: worldRow.id } });
|
const reconciledWorld = await db.worldState.findUniqueOrThrow({ where: { id: worldRow.id } });
|
||||||
const appliedSuspension = await db.clockSuspension.findUniqueOrThrow({ where: { id: suspension.id } });
|
const appliedSuspension = await db.clockSuspension.findUniqueOrThrow({ where: { id: suspension.id } });
|
||||||
expect(reconciledWorld).toMatchObject({
|
expect(reconciledWorld).toMatchObject({
|
||||||
|
|||||||
@@ -222,6 +222,14 @@ export const planProfileReconcile = (
|
|||||||
};
|
};
|
||||||
};
|
};
|
||||||
|
|
||||||
|
/**
|
||||||
|
* A stopped process may be restarted while the world remains in unification
|
||||||
|
* wait. Only the daemon-authorized raise-invader response may reconcile that
|
||||||
|
* suspension, so Gateway RESUME must start the runtime without consuming it.
|
||||||
|
*/
|
||||||
|
export const shouldStartRuntimeInUnificationWait = (clockPhase: string, suspensionSource: string): boolean =>
|
||||||
|
clockPhase === 'SUSPENDED' && suspensionSource === 'UNIFICATION_WAIT';
|
||||||
|
|
||||||
export const resolveResetLifecycleStatus = (
|
export const resolveResetLifecycleStatus = (
|
||||||
now: Date,
|
now: Date,
|
||||||
preopenAt: Date | null,
|
preopenAt: Date | null,
|
||||||
@@ -1311,6 +1319,9 @@ export class GatewayOrchestrator implements GatewayOrchestratorHandle {
|
|||||||
orderBy: { createdAt: 'desc' },
|
orderBy: { createdAt: 'desc' },
|
||||||
});
|
});
|
||||||
if (!suspension) throw new Error('Profile resume requires a durable suspended clock ledger.');
|
if (!suspension) throw new Error('Profile resume requires a durable suspended clock ledger.');
|
||||||
|
if (shouldStartRuntimeInUnificationWait(world.clockPhase, suspension.source)) {
|
||||||
|
return { phase: 'SUSPENDED', revision: clockRevisionAsNumber(world.clockRevision) };
|
||||||
|
}
|
||||||
if (world.clockPhase === 'SUSPENDED') {
|
if (world.clockPhase === 'SUSPENDED') {
|
||||||
await reconcileClockSuspension({ db: postgres.prisma, suspensionId: suspension.id, authority });
|
await reconcileClockSuspension({ db: postgres.prisma, suspensionId: suspension.id, authority });
|
||||||
} else if (world.clockPhase !== 'RECONCILING') {
|
} else if (world.clockPhase !== 'RECONCILING') {
|
||||||
|
|||||||
@@ -13,6 +13,7 @@ import {
|
|||||||
planProfileReconcile,
|
planProfileReconcile,
|
||||||
resolveProfileArchiveServerName,
|
resolveProfileArchiveServerName,
|
||||||
resolveResetLifecycleStatus,
|
resolveResetLifecycleStatus,
|
||||||
|
shouldStartRuntimeInUnificationWait,
|
||||||
} from '../src/orchestrator/gatewayOrchestrator.js';
|
} from '../src/orchestrator/gatewayOrchestrator.js';
|
||||||
import { GATEWAY_PROFILE_ORDER } from '../src/profileOrder.js';
|
import { GATEWAY_PROFILE_ORDER } from '../src/profileOrder.js';
|
||||||
import { sanitizeManagedProcessEnv } from '../src/orchestrator/processManager.js';
|
import { sanitizeManagedProcessEnv } from '../src/orchestrator/processManager.js';
|
||||||
@@ -114,6 +115,20 @@ describe('planProfileReconcile', () => {
|
|||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
describe('shouldStartRuntimeInUnificationWait', () => {
|
||||||
|
it('starts a stopped runtime without consuming its daemon-authorized unification suspension', () => {
|
||||||
|
expect(shouldStartRuntimeInUnificationWait('SUSPENDED', 'UNIFICATION_WAIT')).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it.each([
|
||||||
|
['SUSPENDED', 'MAINTENANCE'],
|
||||||
|
['RECONCILING', 'UNIFICATION_WAIT'],
|
||||||
|
['RUNNING', 'UNIFICATION_WAIT'],
|
||||||
|
])('keeps ordinary resume reconciliation for %s / %s', (phase, source) => {
|
||||||
|
expect(shouldStartRuntimeInUnificationWait(phase, source)).toBe(false);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
describe('resolveResetLifecycleStatus', () => {
|
describe('resolveResetLifecycleStatus', () => {
|
||||||
const now = new Date('2030-01-01T00:00:00.000Z');
|
const now = new Date('2030-01-01T00:00:00.000Z');
|
||||||
|
|
||||||
@@ -471,7 +486,9 @@ describe('buildWorkspaceCommands', () => {
|
|||||||
);
|
);
|
||||||
const migrationUrl = new URL(command.env?.DATABASE_URL ?? '');
|
const migrationUrl = new URL(command.env?.DATABASE_URL ?? '');
|
||||||
|
|
||||||
expect(migrationUrl.searchParams.getAll('options')).toEqual(['-c statement_timeout=30000 -c TimeZone=Asia/Seoul']);
|
expect(migrationUrl.searchParams.getAll('options')).toEqual([
|
||||||
|
'-c statement_timeout=30000 -c TimeZone=Asia/Seoul',
|
||||||
|
]);
|
||||||
});
|
});
|
||||||
|
|
||||||
it('keeps an already explicit KST migration contract without adding another override', () => {
|
it('keeps an already explicit KST migration contract without adding another override', () => {
|
||||||
|
|||||||
+20
@@ -0,0 +1,20 @@
|
|||||||
|
-- A daemon lease is an operational WALL_TIME deadline. The game database keeps
|
||||||
|
-- legacy DateTime columns as TIMESTAMP(3), while its session timezone can be
|
||||||
|
-- Asia/Seoul, so bare CURRENT_TIMESTAMP writes were nine hours ahead of the UTC
|
||||||
|
-- comparisons used by clock fencing.
|
||||||
|
--
|
||||||
|
-- Lease rows are ephemeral authority, not business history. Expire every row at
|
||||||
|
-- the migration boundary so an old writer is fenced and a new UTC-aware daemon
|
||||||
|
-- must acquire a fresh epoch. This also makes mixed-version deployment fail
|
||||||
|
-- closed instead of preserving a falsely-live lease.
|
||||||
|
BEGIN;
|
||||||
|
|
||||||
|
UPDATE "turn_daemon_lease"
|
||||||
|
SET
|
||||||
|
"lease_until" = CURRENT_TIMESTAMP AT TIME ZONE 'UTC',
|
||||||
|
"heartbeat_at" = CURRENT_TIMESTAMP AT TIME ZONE 'UTC';
|
||||||
|
|
||||||
|
ALTER TABLE "turn_daemon_lease"
|
||||||
|
ALTER COLUMN "heartbeat_at" SET DEFAULT (CURRENT_TIMESTAMP AT TIME ZONE 'UTC');
|
||||||
|
|
||||||
|
COMMIT;
|
||||||
@@ -9,6 +9,8 @@ import { createTRPCProxyClient, httpBatchLink } from '@trpc/client';
|
|||||||
import type { AppRouter as GatewayAppRouter } from '@sammo-ts/gateway-api';
|
import type { AppRouter as GatewayAppRouter } from '@sammo-ts/gateway-api';
|
||||||
import type { AppRouter as GameAppRouter } from '@sammo-ts/game-api';
|
import type { AppRouter as GameAppRouter } from '@sammo-ts/game-api';
|
||||||
import { createGamePostgresConnector, createRedisConnector, type GamePrisma } from '@sammo-ts/infra';
|
import { createGamePostgresConnector, createRedisConnector, type GamePrisma } from '@sammo-ts/infra';
|
||||||
|
import { applyNextClockProjection } from '../../../app/game-engine/src/turn/clockProjectionOutbox.js';
|
||||||
|
import { reconcileClockSuspension } from '../../../app/game-engine/src/turn/clockReconciliation.js';
|
||||||
import { createTurnDaemonRuntime } from '../../../app/game-engine/src/turn/turnDaemon.js';
|
import { createTurnDaemonRuntime } from '../../../app/game-engine/src/turn/turnDaemon.js';
|
||||||
|
|
||||||
const gatewayUrl = process.env.SAMMO_LIVE_GATEWAY_URL ?? 'http://caddy/gateway/api/trpc';
|
const gatewayUrl = process.env.SAMMO_LIVE_GATEWAY_URL ?? 'http://caddy/gateway/api/trpc';
|
||||||
@@ -530,7 +532,10 @@ const databaseStatus = async (): Promise<void> => {
|
|||||||
monitorMessages,
|
monitorMessages,
|
||||||
betting,
|
betting,
|
||||||
unification,
|
unification,
|
||||||
|
unificationActions,
|
||||||
suspensions,
|
suspensions,
|
||||||
|
dbWallRows,
|
||||||
|
daemonLeases,
|
||||||
] = await Promise.all([
|
] = await Promise.all([
|
||||||
db.prisma.worldState.findFirstOrThrow({
|
db.prisma.worldState.findFirstOrThrow({
|
||||||
select: {
|
select: {
|
||||||
@@ -572,6 +577,21 @@ const databaseStatus = async (): Promise<void> => {
|
|||||||
select: { id: true, name: true, finished: true, openYearMonth: true, closeYearMonth: true, bets: true },
|
select: { id: true, name: true, finished: true, openYearMonth: true, closeYearMonth: true, bets: true },
|
||||||
}),
|
}),
|
||||||
db.prisma.unificationFinalization.findMany({ orderBy: { createdAt: 'asc' } }),
|
db.prisma.unificationFinalization.findMany({ orderBy: { createdAt: 'asc' } }),
|
||||||
|
db.prisma.messageAction.findMany({
|
||||||
|
where: { actionType: 'raiseInvader' },
|
||||||
|
orderBy: { messageId: 'asc' },
|
||||||
|
include: {
|
||||||
|
message: {
|
||||||
|
select: {
|
||||||
|
id: true,
|
||||||
|
mailbox: true,
|
||||||
|
createdAtWall: true,
|
||||||
|
occurredGameTick: true,
|
||||||
|
message: true,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}),
|
||||||
db.prisma.clockSuspension.findMany({
|
db.prisma.clockSuspension.findMany({
|
||||||
orderBy: { createdAt: 'asc' },
|
orderBy: { createdAt: 'asc' },
|
||||||
select: {
|
select: {
|
||||||
@@ -586,6 +606,10 @@ const databaseStatus = async (): Promise<void> => {
|
|||||||
alignedTick: true,
|
alignedTick: true,
|
||||||
},
|
},
|
||||||
}),
|
}),
|
||||||
|
db.prisma.$queryRaw<Array<{ dbNow: Date }>>`
|
||||||
|
SELECT (CURRENT_TIMESTAMP AT TIME ZONE 'UTC')::timestamp(3) AS "dbNow"
|
||||||
|
`,
|
||||||
|
db.prisma.turnDaemonLease.findMany({ orderBy: { profile: 'asc' } }),
|
||||||
]);
|
]);
|
||||||
log('database-status', {
|
log('database-status', {
|
||||||
world: {
|
world: {
|
||||||
@@ -623,7 +647,28 @@ const databaseStatus = async (): Promise<void> => {
|
|||||||
closeYearMonth: row.closeYearMonth,
|
closeYearMonth: row.closeYearMonth,
|
||||||
bets: row.bets.length,
|
bets: row.bets.length,
|
||||||
})),
|
})),
|
||||||
unification: unification.length,
|
unification,
|
||||||
|
unificationActions: unificationActions.map((action) => ({
|
||||||
|
messageId: action.messageId,
|
||||||
|
status: action.status,
|
||||||
|
createdGameTick: action.createdGameTick.toString(),
|
||||||
|
expiresGameTick: action.expiresGameTick?.toString() ?? null,
|
||||||
|
resolvedGameTick: action.resolvedGameTick?.toString() ?? null,
|
||||||
|
clockRevision: action.clockRevision.toString(),
|
||||||
|
deadlineGeneration: action.deadlineGeneration.toString(),
|
||||||
|
message: {
|
||||||
|
...action.message,
|
||||||
|
createdAtWall: action.message.createdAtWall.toISOString(),
|
||||||
|
occurredGameTick: action.message.occurredGameTick?.toString() ?? null,
|
||||||
|
},
|
||||||
|
})),
|
||||||
|
dbWallNow: dbWallRows[0]?.dbNow.toISOString() ?? null,
|
||||||
|
daemonLeases: daemonLeases.map((lease) => ({
|
||||||
|
...lease,
|
||||||
|
leaseUntil: lease.leaseUntil.toISOString(),
|
||||||
|
heartbeatAt: lease.heartbeatAt.toISOString(),
|
||||||
|
fencingEpoch: lease.fencingEpoch.toString(),
|
||||||
|
})),
|
||||||
suspensions: suspensions.map((row) => ({
|
suspensions: suspensions.map((row) => ({
|
||||||
...row,
|
...row,
|
||||||
sourceRevision: row.sourceRevision.toString(),
|
sourceRevision: row.sourceRevision.toString(),
|
||||||
@@ -1788,9 +1833,60 @@ const npcActionAudit = async (): Promise<void> => {
|
|||||||
where: { status: 'FAILED' },
|
where: { status: 'FAILED' },
|
||||||
orderBy: { createdAt: 'desc' },
|
orderBy: { createdAt: 'desc' },
|
||||||
take: 30,
|
take: 30,
|
||||||
select: { id: true, eventType: true, error: true, createdAt: true },
|
select: { sequence: true, requestId: true, eventType: true, error: true, createdAt: true },
|
||||||
}),
|
}),
|
||||||
]);
|
]);
|
||||||
|
const activeNations = await db.prisma.nation.findMany({
|
||||||
|
where: { id: { gt: 0 }, level: { gt: 0 } },
|
||||||
|
orderBy: { id: 'asc' },
|
||||||
|
});
|
||||||
|
const activeNationIds = activeNations.map((nation) => nation.id);
|
||||||
|
const [activeNationStats, activeRulers, reservedNationActions, recentNationActions, activeDiplomacy] =
|
||||||
|
await Promise.all([
|
||||||
|
db.prisma.$queryRaw<
|
||||||
|
Array<{
|
||||||
|
nationId: number;
|
||||||
|
cities: bigint;
|
||||||
|
generals: bigint;
|
||||||
|
armedGenerals: bigint;
|
||||||
|
totalCrew: bigint;
|
||||||
|
}>
|
||||||
|
>`
|
||||||
|
SELECT
|
||||||
|
n.id AS "nationId",
|
||||||
|
(SELECT COUNT(*) FROM city c WHERE c.nation_id = n.id) AS cities,
|
||||||
|
(SELECT COUNT(*) FROM general g WHERE g.nation_id = n.id) AS generals,
|
||||||
|
(SELECT COUNT(*) FROM general g WHERE g.nation_id = n.id AND g.crew > 0) AS "armedGenerals",
|
||||||
|
(SELECT COALESCE(SUM(g.crew), 0)::bigint FROM general g WHERE g.nation_id = n.id) AS "totalCrew"
|
||||||
|
FROM nation n
|
||||||
|
WHERE n.id = ANY(${activeNationIds}::int[])
|
||||||
|
ORDER BY n.id
|
||||||
|
`,
|
||||||
|
db.prisma.general.findMany({
|
||||||
|
where: { nationId: { in: activeNationIds }, officerLevel: 12 },
|
||||||
|
orderBy: [{ nationId: 'asc' }, { id: 'asc' }],
|
||||||
|
select: { id: true, name: true, nationId: true, npcState: true, crew: true, lastTurn: true },
|
||||||
|
}),
|
||||||
|
db.prisma.$queryRaw<Array<{ nationId: number; actionCode: string; reservations: bigint }>>`
|
||||||
|
SELECT nation_id AS "nationId", action_code AS "actionCode", COUNT(*) AS reservations
|
||||||
|
FROM nation_turn
|
||||||
|
WHERE nation_id = ANY(${activeNationIds}::int[])
|
||||||
|
GROUP BY nation_id, action_code
|
||||||
|
ORDER BY nation_id, reservations DESC, action_code
|
||||||
|
`,
|
||||||
|
db.prisma.logEntry.findMany({
|
||||||
|
where: { nationId: { in: activeNationIds }, category: 'ACTION' },
|
||||||
|
orderBy: { id: 'desc' },
|
||||||
|
take: 80,
|
||||||
|
select: { id: true, year: true, month: true, nationId: true, generalId: true, text: true },
|
||||||
|
}),
|
||||||
|
db.prisma.diplomacy.findMany({
|
||||||
|
where: { srcNationId: { in: activeNationIds }, destNationId: { in: activeNationIds } },
|
||||||
|
orderBy: [{ srcNationId: 'asc' }, { destNationId: 'asc' }],
|
||||||
|
select: { srcNationId: true, destNationId: true, stateCode: true, term: true, isDead: true },
|
||||||
|
}),
|
||||||
|
]);
|
||||||
|
const statByNation = new Map(activeNationStats.map((entry) => [entry.nationId, entry]));
|
||||||
log('npc-action-audit', {
|
log('npc-action-audit', {
|
||||||
label,
|
label,
|
||||||
world: {
|
world: {
|
||||||
@@ -1814,8 +1910,29 @@ const npcActionAudit = async (): Promise<void> => {
|
|||||||
errors: errors.map((error) => ({ ...error, createdAt: error.createdAt.toISOString() })),
|
errors: errors.map((error) => ({ ...error, createdAt: error.createdAt.toISOString() })),
|
||||||
failedInputEvents: failedInputEvents.map((event) => ({
|
failedInputEvents: failedInputEvents.map((event) => ({
|
||||||
...event,
|
...event,
|
||||||
|
sequence: event.sequence.toString(),
|
||||||
createdAt: event.createdAt.toISOString(),
|
createdAt: event.createdAt.toISOString(),
|
||||||
})),
|
})),
|
||||||
|
activeNationStrategy: activeNations.map((nation) => {
|
||||||
|
const stats = statByNation.get(nation.id);
|
||||||
|
return {
|
||||||
|
id: nation.id,
|
||||||
|
name: nation.name,
|
||||||
|
level: nation.level,
|
||||||
|
gold: nation.gold,
|
||||||
|
rice: nation.rice,
|
||||||
|
cities: Number(stats?.cities ?? 0),
|
||||||
|
generals: Number(stats?.generals ?? 0),
|
||||||
|
armedGenerals: Number(stats?.armedGenerals ?? 0),
|
||||||
|
totalCrew: Number(stats?.totalCrew ?? 0),
|
||||||
|
rulers: activeRulers.filter((general) => general.nationId === nation.id),
|
||||||
|
reservedNationActions: reservedNationActions
|
||||||
|
.filter((entry) => entry.nationId === nation.id)
|
||||||
|
.map((entry) => ({ actionCode: entry.actionCode, reservations: Number(entry.reservations) })),
|
||||||
|
};
|
||||||
|
}),
|
||||||
|
activeDiplomacy,
|
||||||
|
recentNationActions: recentNationActions.reverse(),
|
||||||
});
|
});
|
||||||
} finally {
|
} finally {
|
||||||
await db.disconnect();
|
await db.disconnect();
|
||||||
@@ -1947,6 +2064,57 @@ const fastForward = async (): Promise<void> => {
|
|||||||
if (!Number.isInteger(stopNationCount) || stopNationCount < 0) {
|
if (!Number.isInteger(stopNationCount) || stopNationCount < 0) {
|
||||||
throw new Error('SAMMO_FAST_FORWARD_STOP_NATIONS must be a non-negative integer.');
|
throw new Error('SAMMO_FAST_FORWARD_STOP_NATIONS must be a non-negative integer.');
|
||||||
}
|
}
|
||||||
|
const clockDb = createGamePostgresConnector({ url: gameDatabaseUrl() });
|
||||||
|
const clockRedis = createRedisConnector({ url: redisUrl() });
|
||||||
|
await clockDb.connect();
|
||||||
|
await clockRedis.connect();
|
||||||
|
try {
|
||||||
|
const worldBefore = await clockDb.prisma.worldState.findFirstOrThrow({ orderBy: { id: 'asc' } });
|
||||||
|
let reconciliation = null;
|
||||||
|
let projection: 'IDLE' | 'APPLIED' | 'RECOVERED' | null = null;
|
||||||
|
if (worldBefore.clockPhase === 'SUSPENDED' || worldBefore.clockPhase === 'RECONCILING') {
|
||||||
|
const suspension = await clockDb.prisma.clockSuspension.findFirstOrThrow({
|
||||||
|
where: { status: { in: ['SUSPENDED', 'RECONCILING'] } },
|
||||||
|
orderBy: { createdAt: 'desc' },
|
||||||
|
});
|
||||||
|
if (worldBefore.clockPhase === 'SUSPENDED') {
|
||||||
|
reconciliation = await reconcileClockSuspension({
|
||||||
|
db: clockDb.prisma,
|
||||||
|
suspensionId: suspension.id,
|
||||||
|
authority: {
|
||||||
|
kind: 'OFFLINE',
|
||||||
|
profileName,
|
||||||
|
reason: 'exclusive ten-user lifecycle fast-forward',
|
||||||
|
},
|
||||||
|
});
|
||||||
|
}
|
||||||
|
projection = await applyNextClockProjection({
|
||||||
|
db: clockDb.prisma,
|
||||||
|
redis: clockRedis.client,
|
||||||
|
workerId: `lifecycle-fast-forward:${state.runId.slice(0, 8)}`,
|
||||||
|
});
|
||||||
|
if (projection === 'IDLE') {
|
||||||
|
throw new Error('Offline fast-forward reconciliation had no claimable clock projection.');
|
||||||
|
}
|
||||||
|
} else if (!['RUNNING', 'MANUAL'].includes(worldBefore.clockPhase)) {
|
||||||
|
throw new Error(`Offline fast-forward cannot start from clock phase ${worldBefore.clockPhase}.`);
|
||||||
|
}
|
||||||
|
const worldAfter = await clockDb.prisma.worldState.findFirstOrThrow({ orderBy: { id: 'asc' } });
|
||||||
|
if (!['RUNNING', 'MANUAL'].includes(worldAfter.clockPhase)) {
|
||||||
|
throw new Error(`Offline fast-forward clock preparation ended in ${worldAfter.clockPhase}.`);
|
||||||
|
}
|
||||||
|
log('fast-forward-clock-prepared', {
|
||||||
|
phaseBefore: worldBefore.clockPhase,
|
||||||
|
phaseAfter: worldAfter.clockPhase,
|
||||||
|
revisionBefore: worldBefore.clockRevision.toString(),
|
||||||
|
revisionAfter: worldAfter.clockRevision.toString(),
|
||||||
|
reconciliation,
|
||||||
|
projection,
|
||||||
|
});
|
||||||
|
} finally {
|
||||||
|
await clockRedis.disconnect();
|
||||||
|
await clockDb.disconnect();
|
||||||
|
}
|
||||||
const runtime = await createTurnDaemonRuntime({
|
const runtime = await createTurnDaemonRuntime({
|
||||||
profile: profileName.split(':', 1)[0] ?? 'hwe',
|
profile: profileName.split(':', 1)[0] ?? 'hwe',
|
||||||
databaseUrl: gameDatabaseUrl(),
|
databaseUrl: gameDatabaseUrl(),
|
||||||
@@ -1963,6 +2131,14 @@ const fastForward = async (): Promise<void> => {
|
|||||||
while (months < maxMonths) {
|
while (months < maxMonths) {
|
||||||
const before = runtime.world.getState();
|
const before = runtime.world.getState();
|
||||||
const targetTime = new Date(before.lastTurnTime.getTime() + before.tickSeconds * 1000);
|
const targetTime = new Date(before.lastTurnTime.getTime() + before.tickSeconds * 1000);
|
||||||
|
// The production lifecycle advances the authoritative game clock
|
||||||
|
// before processing a requested target. This fixture drives the
|
||||||
|
// processor directly to reset memory between long batches, so it
|
||||||
|
// must preserve that same boundary explicitly. Otherwise monthly
|
||||||
|
// turns move while world_state.clock_tick stays behind, producing
|
||||||
|
// impossible action histories such as resolvedGameTick <
|
||||||
|
// createdGameTick at the unification/invader hand-off.
|
||||||
|
runtime.world.advanceGameClockTo(targetTime, new Date());
|
||||||
let checkpoint;
|
let checkpoint;
|
||||||
do {
|
do {
|
||||||
const result = await runtime.processor.run(
|
const result = await runtime.processor.run(
|
||||||
@@ -2023,8 +2199,8 @@ const placeInvaderRecipients = async (): Promise<void> => {
|
|||||||
orderBy: [{ level: 'desc' }, { id: 'asc' }],
|
orderBy: [{ level: 'desc' }, { id: 'asc' }],
|
||||||
take: 5,
|
take: 5,
|
||||||
});
|
});
|
||||||
if (nations.length < 2 || nations.length > 5) {
|
if (nations.length < 1 || nations.length > 5) {
|
||||||
throw new Error(`Recipient fixture requires two to five active nations, found ${nations.length}.`);
|
throw new Error(`Recipient fixture requires one to five active nations, found ${nations.length}.`);
|
||||||
}
|
}
|
||||||
const placements = [];
|
const placements = [];
|
||||||
for (const [index, user] of state.users.entries()) {
|
for (const [index, user] of state.users.entries()) {
|
||||||
@@ -2032,7 +2208,7 @@ const placeInvaderRecipients = async (): Promise<void> => {
|
|||||||
const city = nation.capitalCityId
|
const city = nation.capitalCityId
|
||||||
? { id: nation.capitalCityId }
|
? { id: nation.capitalCityId }
|
||||||
: await db.prisma.city.findFirstOrThrow({ where: { nationId: nation.id }, orderBy: { id: 'asc' } });
|
: await db.prisma.city.findFirstOrThrow({ where: { nationId: nation.id }, orderBy: { id: 'asc' } });
|
||||||
const officerLevel = 5 + Math.floor(index / nations.length);
|
const officerLevel = Math.min(11, 5 + Math.floor(index / nations.length));
|
||||||
const general = await db.prisma.general.update({
|
const general = await db.prisma.general.update({
|
||||||
where: { id: (await db.prisma.general.findFirstOrThrow({ where: { name: user.generalName } })).id },
|
where: { id: (await db.prisma.general.findFirstOrThrow({ where: { name: user.generalName } })).id },
|
||||||
data: { nationId: nation.id, cityId: city.id, officerLevel },
|
data: { nationId: nation.id, cityId: city.id, officerLevel },
|
||||||
|
|||||||
@@ -391,7 +391,10 @@ const replaceCurrentSeason = async (
|
|||||||
await client.query('BEGIN');
|
await client.query('BEGIN');
|
||||||
try {
|
try {
|
||||||
const activeLease = await client.query(
|
const activeLease = await client.query(
|
||||||
`SELECT 1 FROM turn_daemon_lease WHERE lease_until > CURRENT_TIMESTAMP LIMIT 1`
|
`SELECT 1
|
||||||
|
FROM turn_daemon_lease
|
||||||
|
WHERE lease_until > CURRENT_TIMESTAMP AT TIME ZONE 'UTC'
|
||||||
|
LIMIT 1`
|
||||||
);
|
);
|
||||||
if (activeLease.rowCount) {
|
if (activeLease.rowCount) {
|
||||||
throw new Error('Refusing to replace a current season while a turn daemon lease is active');
|
throw new Error('Refusing to replace a current season while a turn daemon lease is active');
|
||||||
|
|||||||
Reference in New Issue
Block a user