fix(game-engine): sync clock authority during pause
This commit is contained in:
@@ -171,7 +171,8 @@ export class DatabaseTurnDaemonCommandQueue implements TurnDaemonControlQueue, T
|
|||||||
'changePermission',
|
'changePermission',
|
||||||
'appoint',
|
'appoint',
|
||||||
'setNationSetting',
|
'setNationSetting',
|
||||||
'setNpcPolicy'
|
'setNpcPolicy',
|
||||||
|
'shiftSchedule'
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
OR (
|
OR (
|
||||||
|
|||||||
@@ -113,6 +113,41 @@ const readDbWall = async (db: GamePrisma.TransactionClient): Promise<Date> => {
|
|||||||
|
|
||||||
export const readClockDatabaseWall = readDbWall;
|
export const readClockDatabaseWall = readDbWall;
|
||||||
|
|
||||||
|
const isRetryableSerializableClockError = (error: unknown): boolean => {
|
||||||
|
if (!error || typeof error !== 'object') return false;
|
||||||
|
const value = error as { code?: unknown; message?: unknown; meta?: unknown };
|
||||||
|
const meta =
|
||||||
|
value.meta && typeof value.meta === 'object'
|
||||||
|
? (value.meta as { code?: unknown; message?: unknown })
|
||||||
|
: undefined;
|
||||||
|
const code = typeof value.code === 'string' ? value.code : '';
|
||||||
|
const databaseCode = typeof meta?.code === 'string' ? meta.code : '';
|
||||||
|
const message = [value.message, meta?.message]
|
||||||
|
.filter((entry): entry is string => typeof entry === 'string')
|
||||||
|
.join(' ');
|
||||||
|
return (
|
||||||
|
code === 'P2034' ||
|
||||||
|
code === '40001' ||
|
||||||
|
databaseCode === '40001' ||
|
||||||
|
message.includes('40001') ||
|
||||||
|
message.includes('could not serialize access')
|
||||||
|
);
|
||||||
|
};
|
||||||
|
|
||||||
|
const runSerializableClockOperation = async <T>(operation: () => Promise<T>): Promise<T> => {
|
||||||
|
const maxAttempts = 3;
|
||||||
|
for (let attempt = 1; ; attempt += 1) {
|
||||||
|
try {
|
||||||
|
return await operation();
|
||||||
|
} catch (error) {
|
||||||
|
if (attempt >= maxAttempts || !isRetryableSerializableClockError(error)) {
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
await new Promise<void>((resolve) => setTimeout(resolve, attempt * 10));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
const verifyAuthority = async (db: GamePrisma.TransactionClient, authority: ClockOperationAuthority): Promise<void> => {
|
const verifyAuthority = async (db: GamePrisma.TransactionClient, authority: ClockOperationAuthority): Promise<void> => {
|
||||||
const rows = await db.$queryRaw<LeaseFenceRow[]>(GamePrisma.sql`
|
const rows = await db.$queryRaw<LeaseFenceRow[]>(GamePrisma.sql`
|
||||||
SELECT owner_id AS "ownerId",
|
SELECT owner_id AS "ownerId",
|
||||||
@@ -185,64 +220,64 @@ const readParticipantSnapshots = async (
|
|||||||
): Promise<ParticipantSnapshot[]> => {
|
): Promise<ParticipantSnapshot[]> => {
|
||||||
const [world, generals, auctions, auctionBids, messages, inheritanceEffects, votes, pool, npcTokens, commands] =
|
const [world, generals, auctions, auctionBids, messages, inheritanceEffects, votes, pool, npcTokens, commands] =
|
||||||
await Promise.all([
|
await Promise.all([
|
||||||
db.worldState.findUniqueOrThrow({
|
db.worldState.findUniqueOrThrow({
|
||||||
where: { id: worldStateId },
|
where: { id: worldStateId },
|
||||||
select: {
|
select: {
|
||||||
clockTick: true,
|
clockTick: true,
|
||||||
clockRevision: true,
|
clockRevision: true,
|
||||||
deadlineGeneration: true,
|
deadlineGeneration: true,
|
||||||
lastTurnTick: true,
|
lastTurnTick: true,
|
||||||
meta: true,
|
meta: true,
|
||||||
},
|
},
|
||||||
}),
|
}),
|
||||||
db.general.findMany({
|
db.general.findMany({
|
||||||
orderBy: { id: 'asc' },
|
orderBy: { id: 'asc' },
|
||||||
select: { id: true, turnTick: true, recentWarTick: true, meta: true },
|
select: { id: true, turnTick: true, recentWarTick: true, meta: true },
|
||||||
}),
|
}),
|
||||||
db.auction.findMany({
|
db.auction.findMany({
|
||||||
where: { status: { in: ['OPEN', 'FINALIZING'] } },
|
where: { status: { in: ['OPEN', 'FINALIZING'] } },
|
||||||
orderBy: { id: 'asc' },
|
orderBy: { id: 'asc' },
|
||||||
select: { id: true, status: true, openTick: true, closeTick: true },
|
select: { id: true, status: true, openTick: true, closeTick: true },
|
||||||
}),
|
}),
|
||||||
db.auctionBid.findMany({
|
db.auctionBid.findMany({
|
||||||
where: { auction: { status: { in: ['OPEN', 'FINALIZING'] } } },
|
where: { auction: { status: { in: ['OPEN', 'FINALIZING'] } } },
|
||||||
orderBy: { id: 'asc' },
|
orderBy: { id: 'asc' },
|
||||||
select: { id: true, occurredGameTick: true },
|
select: { id: true, occurredGameTick: true },
|
||||||
}),
|
}),
|
||||||
db.messageAction.findMany({
|
db.messageAction.findMany({
|
||||||
where: { status: 'PENDING' },
|
where: { status: 'PENDING' },
|
||||||
orderBy: { messageId: 'asc' },
|
orderBy: { messageId: 'asc' },
|
||||||
select: {
|
select: {
|
||||||
messageId: true,
|
messageId: true,
|
||||||
createdGameTick: true,
|
createdGameTick: true,
|
||||||
expiresGameTick: true,
|
expiresGameTick: true,
|
||||||
clockRevision: true,
|
clockRevision: true,
|
||||||
deadlineGeneration: true,
|
deadlineGeneration: true,
|
||||||
},
|
},
|
||||||
}),
|
}),
|
||||||
db.inheritanceLedger.findMany({
|
db.inheritanceLedger.findMany({
|
||||||
orderBy: { id: 'asc' },
|
orderBy: { id: 'asc' },
|
||||||
select: { id: true, appliedClockRevision: true, appliedDeadlineGeneration: true },
|
select: { id: true, appliedClockRevision: true, appliedDeadlineGeneration: true },
|
||||||
}),
|
}),
|
||||||
db.votePoll.findMany({
|
db.votePoll.findMany({
|
||||||
where: { closedAt: null },
|
where: { closedAt: null },
|
||||||
orderBy: { id: 'asc' },
|
orderBy: { id: 'asc' },
|
||||||
select: { id: true, startTick: true, endTick: true },
|
select: { id: true, startTick: true, endTick: true },
|
||||||
}),
|
}),
|
||||||
db.selectPoolEntry.findMany({
|
db.selectPoolEntry.findMany({
|
||||||
where: { generalId: null },
|
where: { generalId: null },
|
||||||
orderBy: { id: 'asc' },
|
orderBy: { id: 'asc' },
|
||||||
select: { id: true, reservedUntilTick: true },
|
select: { id: true, reservedUntilTick: true },
|
||||||
}),
|
}),
|
||||||
db.npcSelectionToken.findMany({
|
db.npcSelectionToken.findMany({
|
||||||
orderBy: { ownerUserId: 'asc' },
|
orderBy: { ownerUserId: 'asc' },
|
||||||
select: { ownerUserId: true, validUntilTick: true, pickMoreFromTick: true },
|
select: { ownerUserId: true, validUntilTick: true, pickMoreFromTick: true },
|
||||||
}),
|
}),
|
||||||
db.inputEvent.findMany({
|
db.inputEvent.findMany({
|
||||||
where: { status: { in: ['PENDING', 'PROCESSING'] } },
|
where: { status: { in: ['PENDING', 'PROCESSING'] } },
|
||||||
orderBy: { sequence: 'asc' },
|
orderBy: { sequence: 'asc' },
|
||||||
select: { sequence: true, acceptedGameTick: true, acceptedClockRevision: true },
|
select: { sequence: true, acceptedGameTick: true, acceptedClockRevision: true },
|
||||||
}),
|
}),
|
||||||
]);
|
]);
|
||||||
const snapshot = (key: string, policy: ParticipantSnapshot['policy'], rows: unknown[]): ParticipantSnapshot => ({
|
const snapshot = (key: string, policy: ParticipantSnapshot['policy'], rows: unknown[]): ParticipantSnapshot => ({
|
||||||
key,
|
key,
|
||||||
@@ -686,90 +721,97 @@ export const startClockSuspension = async (options: {
|
|||||||
if (!Number.isSafeInteger(catchUpTicks) || catchUpTicks < 0) {
|
if (!Number.isSafeInteger(catchUpTicks) || catchUpTicks < 0) {
|
||||||
throw new Error('Clock suspension catch-up ticks must be a non-negative safe integer.');
|
throw new Error('Clock suspension catch-up ticks must be a non-negative safe integer.');
|
||||||
}
|
}
|
||||||
return options.db.$transaction(
|
return runSerializableClockOperation(() =>
|
||||||
async (db) => {
|
options.db.$transaction(
|
||||||
await verifyAuthority(db, options.authority);
|
async (db) => {
|
||||||
await acquireGameSchemaAdvisoryXactLock(db, CLOCK_OPERATION_PERSISTENCE_LOCK);
|
await verifyAuthority(db, options.authority);
|
||||||
await acquireGameSchemaAdvisoryXactLock(db, GENERAL_ACCESS_PERSISTENCE_LOCK);
|
await acquireGameSchemaAdvisoryXactLock(db, CLOCK_OPERATION_PERSISTENCE_LOCK);
|
||||||
const worldStateId = await lockWorld(db);
|
await acquireGameSchemaAdvisoryXactLock(db, GENERAL_ACCESS_PERSISTENCE_LOCK);
|
||||||
const existing = await db.clockSuspension.findUnique({ where: { id: options.suspensionId } });
|
const worldStateId = await lockWorld(db);
|
||||||
if (existing) {
|
const existing = await db.clockSuspension.findUnique({ where: { id: options.suspensionId } });
|
||||||
if (
|
if (existing) {
|
||||||
existing.worldStateId !== worldStateId ||
|
if (
|
||||||
existing.source !== options.source ||
|
existing.worldStateId !== worldStateId ||
|
||||||
existing.policy !== policy
|
existing.source !== options.source ||
|
||||||
) {
|
existing.policy !== policy
|
||||||
throw new Error(
|
) {
|
||||||
`Clock suspension ID ${options.suspensionId} is already bound to another operation.`
|
throw new Error(
|
||||||
);
|
`Clock suspension ID ${options.suspensionId} is already bound to another operation.`
|
||||||
|
);
|
||||||
|
}
|
||||||
|
if (existing.status !== 'SUSPENDED') {
|
||||||
|
throw new Error(
|
||||||
|
`Clock suspension ${options.suspensionId} already advanced to ${existing.status}.`
|
||||||
|
);
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
suspensionId: existing.id,
|
||||||
|
phase: 'SUSPENDED' as const,
|
||||||
|
sourceRevision: safeNumber(existing.sourceRevision, 'source revision'),
|
||||||
|
targetRevision: safeNumber(existing.targetRevision, 'target revision'),
|
||||||
|
cutTick: safeNumber(existing.cutTick, 'cut tick'),
|
||||||
|
cutWallAt: existing.cutWallAt,
|
||||||
|
};
|
||||||
}
|
}
|
||||||
if (existing.status !== 'SUSPENDED') {
|
const world = await db.worldState.findUniqueOrThrow({ where: { id: worldStateId } });
|
||||||
throw new Error(`Clock suspension ${options.suspensionId} already advanced to ${existing.status}.`);
|
const phase = parseGameClockPhase(world.clockPhase);
|
||||||
|
if (phase !== 'RUNNING') {
|
||||||
|
throw new Error(`Clock suspension can start only from RUNNING; current phase is ${phase}.`);
|
||||||
}
|
}
|
||||||
|
if (!world.clockBaseTime || world.clockTick === null || !world.clockWallAnchor) {
|
||||||
|
throw new Error('Clock suspension requires a fully initialized logical game clock.');
|
||||||
|
}
|
||||||
|
const cutWallAt = await readDbWall(db);
|
||||||
|
const storedTick = safeNumber(world.clockTick, 'world clock tick');
|
||||||
|
const sourceRevision = safeNumber(world.clockRevision, 'world clock revision');
|
||||||
|
const clock = new GameClock({
|
||||||
|
baseTime: world.clockBaseTime,
|
||||||
|
tick: storedTick,
|
||||||
|
mode: world.clockMode === 'manual' ? 'manual' : 'realtime',
|
||||||
|
wallAnchor: world.clockWallAnchor,
|
||||||
|
turnSeconds: world.tickSeconds,
|
||||||
|
phase,
|
||||||
|
revision: sourceRevision,
|
||||||
|
});
|
||||||
|
const cutTick = clock.nowTick(cutWallAt);
|
||||||
|
await lockParticipants(db, BigInt(cutTick));
|
||||||
|
await db.worldState.update({
|
||||||
|
where: { id: worldStateId },
|
||||||
|
data: { clockPhase: 'SUSPENDED', clockTick: BigInt(cutTick), clockWallAnchor: cutWallAt },
|
||||||
|
});
|
||||||
|
const participants = await readParticipantSnapshots(db, worldStateId, BigInt(cutTick));
|
||||||
|
await db.clockSuspension.create({
|
||||||
|
data: {
|
||||||
|
id: options.suspensionId,
|
||||||
|
worldStateId,
|
||||||
|
source: options.source,
|
||||||
|
policy,
|
||||||
|
status: 'SUSPENDED',
|
||||||
|
sourceRevision: BigInt(sourceRevision),
|
||||||
|
targetRevision: BigInt(sourceRevision + 1),
|
||||||
|
cutTick: BigInt(cutTick),
|
||||||
|
cutWallAt,
|
||||||
|
rateTicksPerSecond: GAME_TICKS_PER_TURN / world.tickSeconds,
|
||||||
|
catchUpTicks: BigInt(catchUpTicks),
|
||||||
|
participantChecksumBefore: aggregateChecksum(participants),
|
||||||
|
detail: asJson({
|
||||||
|
authority: options.authority.kind,
|
||||||
|
profileName: options.authority.profileName,
|
||||||
|
}),
|
||||||
|
},
|
||||||
|
});
|
||||||
|
await persistInitialParticipants(db, options.suspensionId, participants);
|
||||||
return {
|
return {
|
||||||
suspensionId: existing.id,
|
suspensionId: options.suspensionId,
|
||||||
phase: 'SUSPENDED' as const,
|
phase: 'SUSPENDED',
|
||||||
sourceRevision: safeNumber(existing.sourceRevision, 'source revision'),
|
sourceRevision,
|
||||||
targetRevision: safeNumber(existing.targetRevision, 'target revision'),
|
targetRevision: sourceRevision + 1,
|
||||||
cutTick: safeNumber(existing.cutTick, 'cut tick'),
|
cutTick,
|
||||||
cutWallAt: existing.cutWallAt,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
const world = await db.worldState.findUniqueOrThrow({ where: { id: worldStateId } });
|
|
||||||
const phase = parseGameClockPhase(world.clockPhase);
|
|
||||||
if (phase !== 'RUNNING') {
|
|
||||||
throw new Error(`Clock suspension can start only from RUNNING; current phase is ${phase}.`);
|
|
||||||
}
|
|
||||||
if (!world.clockBaseTime || world.clockTick === null || !world.clockWallAnchor) {
|
|
||||||
throw new Error('Clock suspension requires a fully initialized logical game clock.');
|
|
||||||
}
|
|
||||||
const cutWallAt = await readDbWall(db);
|
|
||||||
const storedTick = safeNumber(world.clockTick, 'world clock tick');
|
|
||||||
const sourceRevision = safeNumber(world.clockRevision, 'world clock revision');
|
|
||||||
const clock = new GameClock({
|
|
||||||
baseTime: world.clockBaseTime,
|
|
||||||
tick: storedTick,
|
|
||||||
mode: world.clockMode === 'manual' ? 'manual' : 'realtime',
|
|
||||||
wallAnchor: world.clockWallAnchor,
|
|
||||||
turnSeconds: world.tickSeconds,
|
|
||||||
phase,
|
|
||||||
revision: sourceRevision,
|
|
||||||
});
|
|
||||||
const cutTick = clock.nowTick(cutWallAt);
|
|
||||||
await lockParticipants(db, BigInt(cutTick));
|
|
||||||
await db.worldState.update({
|
|
||||||
where: { id: worldStateId },
|
|
||||||
data: { clockPhase: 'SUSPENDED', clockTick: BigInt(cutTick), clockWallAnchor: cutWallAt },
|
|
||||||
});
|
|
||||||
const participants = await readParticipantSnapshots(db, worldStateId, BigInt(cutTick));
|
|
||||||
await db.clockSuspension.create({
|
|
||||||
data: {
|
|
||||||
id: options.suspensionId,
|
|
||||||
worldStateId,
|
|
||||||
source: options.source,
|
|
||||||
policy,
|
|
||||||
status: 'SUSPENDED',
|
|
||||||
sourceRevision: BigInt(sourceRevision),
|
|
||||||
targetRevision: BigInt(sourceRevision + 1),
|
|
||||||
cutTick: BigInt(cutTick),
|
|
||||||
cutWallAt,
|
cutWallAt,
|
||||||
rateTicksPerSecond: GAME_TICKS_PER_TURN / world.tickSeconds,
|
};
|
||||||
catchUpTicks: BigInt(catchUpTicks),
|
},
|
||||||
participantChecksumBefore: aggregateChecksum(participants),
|
{ isolationLevel: 'Serializable', maxWait: 10_000, timeout: 30_000 }
|
||||||
detail: asJson({ authority: options.authority.kind, profileName: options.authority.profileName }),
|
)
|
||||||
},
|
|
||||||
});
|
|
||||||
await persistInitialParticipants(db, options.suspensionId, participants);
|
|
||||||
return {
|
|
||||||
suspensionId: options.suspensionId,
|
|
||||||
phase: 'SUSPENDED',
|
|
||||||
sourceRevision,
|
|
||||||
targetRevision: sourceRevision + 1,
|
|
||||||
cutTick,
|
|
||||||
cutWallAt,
|
|
||||||
};
|
|
||||||
},
|
|
||||||
{ isolationLevel: 'Serializable', maxWait: 10_000, timeout: 30_000 }
|
|
||||||
);
|
);
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -983,18 +1025,20 @@ export const reconcileClockSuspension = async (options: {
|
|||||||
/** Deterministic fixture seam; production must always use PostgreSQL CURRENT_TIMESTAMP. */
|
/** Deterministic fixture seam; production must always use PostgreSQL CURRENT_TIMESTAMP. */
|
||||||
testResumeWallAt?: Date;
|
testResumeWallAt?: Date;
|
||||||
}): Promise<ClockReconciliationResult> =>
|
}): Promise<ClockReconciliationResult> =>
|
||||||
options.db.$transaction(
|
runSerializableClockOperation(() =>
|
||||||
async (db) => {
|
options.db.$transaction(
|
||||||
await verifyAuthority(db, options.authority);
|
async (db) => {
|
||||||
await acquireGameSchemaAdvisoryXactLock(db, CLOCK_OPERATION_PERSISTENCE_LOCK);
|
await verifyAuthority(db, options.authority);
|
||||||
await acquireGameSchemaAdvisoryXactLock(db, GENERAL_ACCESS_PERSISTENCE_LOCK);
|
await acquireGameSchemaAdvisoryXactLock(db, CLOCK_OPERATION_PERSISTENCE_LOCK);
|
||||||
return reconcileClockSuspensionInTransaction({
|
await acquireGameSchemaAdvisoryXactLock(db, GENERAL_ACCESS_PERSISTENCE_LOCK);
|
||||||
db,
|
return reconcileClockSuspensionInTransaction({
|
||||||
suspensionId: options.suspensionId,
|
db,
|
||||||
profileName: options.authority.profileName,
|
suspensionId: options.suspensionId,
|
||||||
authority: options.authority,
|
profileName: options.authority.profileName,
|
||||||
...(options.testResumeWallAt ? { testResumeWallAt: options.testResumeWallAt } : {}),
|
authority: options.authority,
|
||||||
});
|
...(options.testResumeWallAt ? { testResumeWallAt: options.testResumeWallAt } : {}),
|
||||||
},
|
});
|
||||||
{ isolationLevel: 'Serializable', maxWait: 10_000, timeout: 30_000 }
|
},
|
||||||
|
{ isolationLevel: 'Serializable', maxWait: 10_000, timeout: 30_000 }
|
||||||
|
)
|
||||||
);
|
);
|
||||||
|
|||||||
@@ -64,6 +64,7 @@ import {
|
|||||||
refreshClockProjectionForFinalClockUnderHeldLocks,
|
refreshClockProjectionForFinalClockUnderHeldLocks,
|
||||||
} from './clockReconciliation.js';
|
} from './clockReconciliation.js';
|
||||||
import { applyNextClockProjection, type ClockProjectionRedis } from './clockProjectionOutbox.js';
|
import { applyNextClockProjection, type ClockProjectionRedis } from './clockProjectionOutbox.js';
|
||||||
|
import { synchronizeRuntimeClockAuthorityUnderHeldLock } from './runtimeClockAuthoritySync.js';
|
||||||
|
|
||||||
export interface DatabaseTurnHooks {
|
export interface DatabaseTurnHooks {
|
||||||
hooks: TurnDaemonHooks;
|
hooks: TurnDaemonHooks;
|
||||||
@@ -71,6 +72,7 @@ export interface DatabaseTurnHooks {
|
|||||||
takeCommittedReadModelChangeReceipt(): CommittedReadModelChangeReceipt | null;
|
takeCommittedReadModelChangeReceipt(): CommittedReadModelChangeReceipt | null;
|
||||||
close(): Promise<void>;
|
close(): Promise<void>;
|
||||||
applyClockProjection(redis: ClockProjectionRedis, workerId: string): Promise<boolean>;
|
applyClockProjection(redis: ClockProjectionRedis, workerId: string): Promise<boolean>;
|
||||||
|
synchronizeClockAuthority(): Promise<boolean>;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface CommittedReadModelChangeReceipt {
|
export interface CommittedReadModelChangeReceipt {
|
||||||
@@ -2010,6 +2012,7 @@ export const createDatabaseTurnHooks = async (
|
|||||||
await options?.turnDaemonLease?.assertActive(transaction);
|
await options?.turnDaemonLease?.assertActive(transaction);
|
||||||
await acquireGameSchemaAdvisoryXactLock(transaction, CLOCK_OPERATION_PERSISTENCE_LOCK);
|
await acquireGameSchemaAdvisoryXactLock(transaction, CLOCK_OPERATION_PERSISTENCE_LOCK);
|
||||||
await acquireGameSchemaAdvisoryXactLock(transaction, GENERAL_ACCESS_PERSISTENCE_LOCK);
|
await acquireGameSchemaAdvisoryXactLock(transaction, GENERAL_ACCESS_PERSISTENCE_LOCK);
|
||||||
|
await synchronizeRuntimeClockAuthorityUnderHeldLock(transaction, world);
|
||||||
const leaseToken = options?.turnDaemonLease?.getToken();
|
const leaseToken = options?.turnDaemonLease?.getToken();
|
||||||
const directLogFloor =
|
const directLogFloor =
|
||||||
(
|
(
|
||||||
@@ -2054,6 +2057,12 @@ export const createDatabaseTurnHooks = async (
|
|||||||
});
|
});
|
||||||
return clock?.clockPhase === 'RUNNING' || clock?.clockPhase === 'MANUAL';
|
return clock?.clockPhase === 'RUNNING' || clock?.clockPhase === 'MANUAL';
|
||||||
},
|
},
|
||||||
|
synchronizeClockAuthority: () =>
|
||||||
|
prisma.$transaction(async (transaction) => {
|
||||||
|
await options?.turnDaemonLease?.assertActive(transaction);
|
||||||
|
await acquireGameSchemaAdvisoryXactLock(transaction, CLOCK_OPERATION_PERSISTENCE_LOCK);
|
||||||
|
return synchronizeRuntimeClockAuthorityUnderHeldLock(transaction, world);
|
||||||
|
}, transactionOptions),
|
||||||
close: () => connector.disconnect(),
|
close: () => connector.disconnect(),
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -135,6 +135,20 @@ export interface InMemoryGameClockState {
|
|||||||
deadlineGeneration: number;
|
deadlineGeneration: number;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export interface DurableGameClockSnapshot extends InMemoryGameClockState {
|
||||||
|
lastTurnTick: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface DurableClockReconciliationAlignment {
|
||||||
|
suspensionId: string;
|
||||||
|
sourceRevision: number;
|
||||||
|
targetRevision: number;
|
||||||
|
deadlineGeneration: number;
|
||||||
|
alignedTick: number;
|
||||||
|
shiftTicks: number;
|
||||||
|
resumeWallAt: Date;
|
||||||
|
}
|
||||||
|
|
||||||
export type InheritancePersistencePhase = 'before_lifecycle' | 'after_lifecycle';
|
export type InheritancePersistencePhase = 'before_lifecycle' | 'after_lifecycle';
|
||||||
|
|
||||||
export interface PendingInheritancePointAdjustment {
|
export interface PendingInheritancePointAdjustment {
|
||||||
@@ -723,23 +737,14 @@ export class InMemoryTurnWorld {
|
|||||||
this.state = { ...this.state, clockPhase: 'COMPLETED' };
|
this.state = { ...this.state, clockPhase: 'COMPLETED' };
|
||||||
}
|
}
|
||||||
|
|
||||||
applyClockReconciliation(input: {
|
private applyClockReconciliationAlignment(input: DurableClockReconciliationAlignment): void {
|
||||||
suspensionId: string;
|
|
||||||
alignedTick: number;
|
|
||||||
shiftTicks: number;
|
|
||||||
targetRevision: number;
|
|
||||||
deadlineGeneration: number;
|
|
||||||
resumeWallAt: Date;
|
|
||||||
}): void {
|
|
||||||
const phase = this.state.clockPhase ?? inferClockPhase(this.state.clockMode ?? 'manual');
|
|
||||||
if (phase !== 'SUSPENDED' || this.state.meta.unificationClockSuspensionId !== input.suspensionId) {
|
|
||||||
throw new Error('In-memory clock reconciliation requires the matching UNIFICATION_WAIT suspension.');
|
|
||||||
}
|
|
||||||
if (
|
if (
|
||||||
!Number.isSafeInteger(input.alignedTick) ||
|
!Number.isSafeInteger(input.alignedTick) ||
|
||||||
!Number.isSafeInteger(input.shiftTicks) ||
|
!Number.isSafeInteger(input.shiftTicks) ||
|
||||||
input.shiftTicks < 0 ||
|
input.shiftTicks < 0 ||
|
||||||
|
!Number.isSafeInteger(input.sourceRevision) ||
|
||||||
!Number.isSafeInteger(input.targetRevision) ||
|
!Number.isSafeInteger(input.targetRevision) ||
|
||||||
|
input.targetRevision !== input.sourceRevision + 1 ||
|
||||||
!Number.isSafeInteger(input.deadlineGeneration)
|
!Number.isSafeInteger(input.deadlineGeneration)
|
||||||
) {
|
) {
|
||||||
throw new Error('In-memory clock reconciliation received an unsafe coordinate.');
|
throw new Error('In-memory clock reconciliation received an unsafe coordinate.');
|
||||||
@@ -800,6 +805,72 @@ export class InMemoryTurnWorld {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
applyClockReconciliation(input: Omit<DurableClockReconciliationAlignment, 'sourceRevision'>): void {
|
||||||
|
const phase = this.state.clockPhase ?? inferClockPhase(this.state.clockMode ?? 'manual');
|
||||||
|
if (phase !== 'SUSPENDED' || this.state.meta.unificationClockSuspensionId !== input.suspensionId) {
|
||||||
|
throw new Error('In-memory clock reconciliation requires the matching UNIFICATION_WAIT suspension.');
|
||||||
|
}
|
||||||
|
this.applyClockReconciliationAlignment({
|
||||||
|
...input,
|
||||||
|
sourceRevision: this.state.clockRevision ?? 1,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
applyDurableClockReconciliation(input: DurableClockReconciliationAlignment): void {
|
||||||
|
const clock = this.getGameClockState();
|
||||||
|
if (clock.revision === input.targetRevision && clock.phase === 'RECONCILING') {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (clock.revision !== input.sourceRevision) {
|
||||||
|
throw new Error(
|
||||||
|
`Durable clock reconciliation source mismatch: memory ${clock.revision}, ledger ${input.sourceRevision}.`
|
||||||
|
);
|
||||||
|
}
|
||||||
|
if (clock.phase !== 'RUNNING' && clock.phase !== 'SUSPENDED') {
|
||||||
|
throw new Error(`Durable clock reconciliation cannot apply from in-memory phase ${clock.phase}.`);
|
||||||
|
}
|
||||||
|
this.applyClockReconciliationAlignment(input);
|
||||||
|
}
|
||||||
|
|
||||||
|
synchronizeDurableClockSnapshot(input: DurableGameClockSnapshot): void {
|
||||||
|
const current = this.getGameClockState();
|
||||||
|
if (current.revision !== input.revision || current.deadlineGeneration !== input.deadlineGeneration) {
|
||||||
|
throw new Error(
|
||||||
|
`Durable clock snapshot generation mismatch: memory ${current.revision}/${current.deadlineGeneration}, ` +
|
||||||
|
`database ${input.revision}/${input.deadlineGeneration}.`
|
||||||
|
);
|
||||||
|
}
|
||||||
|
if ((this.state.lastTurnTick ?? 0) !== input.lastTurnTick) {
|
||||||
|
throw new Error(
|
||||||
|
`Durable clock snapshot turn cursor mismatch: memory ${this.state.lastTurnTick ?? 0}, database ${input.lastTurnTick}.`
|
||||||
|
);
|
||||||
|
}
|
||||||
|
const currentBaseTime = this.state.clockBaseTime ?? this.state.lastTurnTime;
|
||||||
|
if (currentBaseTime.getTime() !== input.baseTime.getTime()) {
|
||||||
|
throw new Error(
|
||||||
|
`Durable clock snapshot base mismatch: memory ${currentBaseTime.toISOString()}, ` +
|
||||||
|
`database ${input.baseTime.toISOString()}.`
|
||||||
|
);
|
||||||
|
}
|
||||||
|
const validTransition =
|
||||||
|
current.phase === input.phase ||
|
||||||
|
(current.phase === 'RUNNING' && input.phase === 'SUSPENDED') ||
|
||||||
|
(current.phase === 'RECONCILING' && input.phase === 'RUNNING');
|
||||||
|
if (!validTransition) {
|
||||||
|
throw new Error(`Durable clock snapshot phase mismatch: memory ${current.phase}, database ${input.phase}.`);
|
||||||
|
}
|
||||||
|
this.state = {
|
||||||
|
...this.state,
|
||||||
|
clockBaseTime: new Date(input.baseTime.getTime()),
|
||||||
|
clockTick: input.tick,
|
||||||
|
clockMode: input.mode,
|
||||||
|
clockWallAnchor: new Date(input.wallAnchor.getTime()),
|
||||||
|
clockPhase: input.phase,
|
||||||
|
clockRevision: input.revision,
|
||||||
|
deadlineGeneration: input.deadlineGeneration,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
completeClockReconciliation(): void {
|
completeClockReconciliation(): void {
|
||||||
if (this.state.clockPhase === 'RECONCILING') {
|
if (this.state.clockPhase === 'RECONCILING') {
|
||||||
this.state = { ...this.state, clockPhase: 'RUNNING' };
|
this.state = { ...this.state, clockPhase: 'RUNNING' };
|
||||||
|
|||||||
@@ -0,0 +1,125 @@
|
|||||||
|
import { parseGameClockPhase } from '@sammo-ts/common';
|
||||||
|
import type { GamePrisma } from '@sammo-ts/infra';
|
||||||
|
|
||||||
|
import type { InMemoryTurnWorld } from './inMemoryWorld.js';
|
||||||
|
|
||||||
|
const safeNumber = (value: bigint, label: string): number => {
|
||||||
|
const result = Number(value);
|
||||||
|
if (!Number.isSafeInteger(result)) {
|
||||||
|
throw new Error(`${label} is outside the JavaScript safe integer range: ${value}.`);
|
||||||
|
}
|
||||||
|
return result;
|
||||||
|
};
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Replays durable suspension ledgers into the already-running daemon before it
|
||||||
|
* handles a command or resumes scheduled turns. The caller must hold the clock
|
||||||
|
* operation advisory lock so the world row and ledger chain are one snapshot.
|
||||||
|
*/
|
||||||
|
export const synchronizeRuntimeClockAuthorityUnderHeldLock = async (
|
||||||
|
db: GamePrisma.TransactionClient,
|
||||||
|
world: InMemoryTurnWorld
|
||||||
|
): Promise<boolean> => {
|
||||||
|
const durable = await db.worldState.findFirst({
|
||||||
|
orderBy: { id: 'asc' },
|
||||||
|
select: {
|
||||||
|
id: true,
|
||||||
|
clockBaseTime: true,
|
||||||
|
clockTick: true,
|
||||||
|
clockMode: true,
|
||||||
|
clockWallAnchor: true,
|
||||||
|
lastTurnTick: true,
|
||||||
|
clockPhase: true,
|
||||||
|
clockRevision: true,
|
||||||
|
deadlineGeneration: true,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
if (
|
||||||
|
!durable ||
|
||||||
|
!durable.clockBaseTime ||
|
||||||
|
durable.clockTick === null ||
|
||||||
|
!durable.clockWallAnchor ||
|
||||||
|
durable.lastTurnTick === null
|
||||||
|
) {
|
||||||
|
throw new Error('Runtime clock synchronization requires one fully initialized world clock.');
|
||||||
|
}
|
||||||
|
|
||||||
|
const before = world.getGameClockState();
|
||||||
|
const durableRevision = safeNumber(durable.clockRevision, 'durable clock revision');
|
||||||
|
const durableGeneration = safeNumber(durable.deadlineGeneration, 'durable deadline generation');
|
||||||
|
if (before.revision > durableRevision) {
|
||||||
|
throw new Error(`In-memory clock revision ${before.revision} is ahead of durable revision ${durableRevision}.`);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (before.revision < durableRevision) {
|
||||||
|
const ledgers = await db.clockSuspension.findMany({
|
||||||
|
where: {
|
||||||
|
worldStateId: durable.id,
|
||||||
|
sourceRevision: { gte: BigInt(before.revision) },
|
||||||
|
targetRevision: { lte: durable.clockRevision },
|
||||||
|
status: { in: ['RECONCILING', 'APPLIED'] },
|
||||||
|
},
|
||||||
|
orderBy: { sourceRevision: 'asc' },
|
||||||
|
select: {
|
||||||
|
id: true,
|
||||||
|
sourceRevision: true,
|
||||||
|
targetRevision: true,
|
||||||
|
shiftTicks: true,
|
||||||
|
alignedTick: true,
|
||||||
|
resumeWallAt: true,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
let expectedRevision = before.revision;
|
||||||
|
let expectedGeneration = before.deadlineGeneration;
|
||||||
|
for (const ledger of ledgers) {
|
||||||
|
const sourceRevision = safeNumber(ledger.sourceRevision, `clock suspension ${ledger.id} source revision`);
|
||||||
|
const targetRevision = safeNumber(ledger.targetRevision, `clock suspension ${ledger.id} target revision`);
|
||||||
|
if (sourceRevision !== expectedRevision || targetRevision !== sourceRevision + 1) {
|
||||||
|
throw new Error(
|
||||||
|
`Clock suspension ledger chain is discontinuous at ${ledger.id}: ` +
|
||||||
|
`expected ${expectedRevision}->${expectedRevision + 1}, found ${sourceRevision}->${targetRevision}.`
|
||||||
|
);
|
||||||
|
}
|
||||||
|
if (ledger.shiftTicks === null || ledger.alignedTick === null || !ledger.resumeWallAt) {
|
||||||
|
throw new Error(`Clock suspension ${ledger.id} has no completed reconciliation coordinate.`);
|
||||||
|
}
|
||||||
|
expectedGeneration += 1;
|
||||||
|
world.applyDurableClockReconciliation({
|
||||||
|
suspensionId: ledger.id,
|
||||||
|
sourceRevision,
|
||||||
|
targetRevision,
|
||||||
|
deadlineGeneration: expectedGeneration,
|
||||||
|
alignedTick: safeNumber(ledger.alignedTick, `clock suspension ${ledger.id} aligned tick`),
|
||||||
|
shiftTicks: safeNumber(ledger.shiftTicks, `clock suspension ${ledger.id} shift ticks`),
|
||||||
|
resumeWallAt: ledger.resumeWallAt,
|
||||||
|
});
|
||||||
|
expectedRevision = targetRevision;
|
||||||
|
}
|
||||||
|
if (expectedRevision !== durableRevision || expectedGeneration !== durableGeneration) {
|
||||||
|
throw new Error(
|
||||||
|
`Clock suspension ledger chain ended at ${expectedRevision}/${expectedGeneration}, ` +
|
||||||
|
`but durable clock is ${durableRevision}/${durableGeneration}.`
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
world.synchronizeDurableClockSnapshot({
|
||||||
|
baseTime: durable.clockBaseTime,
|
||||||
|
tick: safeNumber(durable.clockTick, 'durable clock tick'),
|
||||||
|
mode: durable.clockMode === 'manual' ? 'manual' : 'realtime',
|
||||||
|
wallAnchor: durable.clockWallAnchor,
|
||||||
|
lastTurnTick: safeNumber(durable.lastTurnTick, 'durable last turn tick'),
|
||||||
|
phase: parseGameClockPhase(durable.clockPhase),
|
||||||
|
revision: durableRevision,
|
||||||
|
deadlineGeneration: durableGeneration,
|
||||||
|
});
|
||||||
|
|
||||||
|
const after = world.getGameClockState();
|
||||||
|
return (
|
||||||
|
before.phase !== after.phase ||
|
||||||
|
before.revision !== after.revision ||
|
||||||
|
before.deadlineGeneration !== after.deadlineGeneration ||
|
||||||
|
before.tick !== after.tick ||
|
||||||
|
before.wallAnchor.getTime() !== after.wallAnchor.getTime()
|
||||||
|
);
|
||||||
|
};
|
||||||
@@ -715,6 +715,7 @@ const createTurnDaemonRuntimeWithLease = async (
|
|||||||
let redisConnector: RedisConnector | null = null;
|
let redisConnector: RedisConnector | null = null;
|
||||||
let stopClockProjectionWorker = () => {};
|
let stopClockProjectionWorker = () => {};
|
||||||
let applyClockProjection: DatabaseTurnHooks['applyClockProjection'] | undefined;
|
let applyClockProjection: DatabaseTurnHooks['applyClockProjection'] | undefined;
|
||||||
|
let synchronizeClockAuthority: DatabaseTurnHooks['synchronizeClockAuthority'] | undefined;
|
||||||
const nationTraits = await loadNationTraitModules([...NATION_TRAIT_KEYS], new NationTraitLoader());
|
const nationTraits = await loadNationTraitModules([...NATION_TRAIT_KEYS], new NationTraitLoader());
|
||||||
const nationTraitMap = new Map(nationTraits.map((module) => [module.key, module]));
|
const nationTraitMap = new Map(nationTraits.map((module) => [module.key, module]));
|
||||||
const monthlyActionModules = await loadActionModuleBundle(
|
const monthlyActionModules = await loadActionModuleBundle(
|
||||||
@@ -920,6 +921,7 @@ const createTurnDaemonRuntimeWithLease = async (
|
|||||||
};
|
};
|
||||||
takeCommittedReadModelChangeReceipt = dbHooks.takeCommittedReadModelChangeReceipt;
|
takeCommittedReadModelChangeReceipt = dbHooks.takeCommittedReadModelChangeReceipt;
|
||||||
applyClockProjection = dbHooks.applyClockProjection;
|
applyClockProjection = dbHooks.applyClockProjection;
|
||||||
|
synchronizeClockAuthority = dbHooks.synchronizeClockAuthority;
|
||||||
close = async () => {
|
close = async () => {
|
||||||
if (auctionBidder) {
|
if (auctionBidder) {
|
||||||
await auctionBidder.close();
|
await auctionBidder.close();
|
||||||
@@ -1031,6 +1033,7 @@ const createTurnDaemonRuntimeWithLease = async (
|
|||||||
maxGenerals: 200,
|
maxGenerals: 200,
|
||||||
catchUpCap: 1,
|
catchUpCap: 1,
|
||||||
};
|
};
|
||||||
|
let lastObservedGatewayPause: boolean | null = null;
|
||||||
|
|
||||||
const lifecycle = new TurnDaemonLifecycle(
|
const lifecycle = new TurnDaemonLifecycle(
|
||||||
{
|
{
|
||||||
@@ -1041,7 +1044,21 @@ const createTurnDaemonRuntimeWithLease = async (
|
|||||||
stateStore,
|
stateStore,
|
||||||
processor,
|
processor,
|
||||||
hooks,
|
hooks,
|
||||||
pauseGate: async () => turnDaemonLease?.isLost() || ((await pauseGate?.()) ?? false),
|
pauseGate: async () => {
|
||||||
|
if (turnDaemonLease?.isLost()) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
const gatewayPaused = (await pauseGate?.()) ?? false;
|
||||||
|
const phase = world.getGameClockState().phase;
|
||||||
|
const phaseNeedsSync = gatewayPaused
|
||||||
|
? phase !== 'SUSPENDED'
|
||||||
|
: phase === 'SUSPENDED' || phase === 'RECONCILING';
|
||||||
|
if (synchronizeClockAuthority && (lastObservedGatewayPause !== gatewayPaused || phaseNeedsSync)) {
|
||||||
|
await synchronizeClockAuthority();
|
||||||
|
}
|
||||||
|
lastObservedGatewayPause = gatewayPaused;
|
||||||
|
return gatewayPaused;
|
||||||
|
},
|
||||||
commandHandler,
|
commandHandler,
|
||||||
commandResponder: options.controlQueue ? undefined : (databaseCommandQueue ?? undefined),
|
commandResponder: options.controlQueue ? undefined : (databaseCommandQueue ?? undefined),
|
||||||
// The exclusive fixture runner aborts the entire in-memory runtime
|
// The exclusive fixture runner aborts the entire in-memory runtime
|
||||||
|
|||||||
@@ -0,0 +1,71 @@
|
|||||||
|
import { describe, expect, it, vi } from 'vitest';
|
||||||
|
|
||||||
|
import type { GamePrismaClient } from '@sammo-ts/infra';
|
||||||
|
import {
|
||||||
|
reconcileClockSuspension,
|
||||||
|
startClockSuspension,
|
||||||
|
type ClockReconciliationResult,
|
||||||
|
type ClockSuspensionResult,
|
||||||
|
} from '../src/turn/clockReconciliation.js';
|
||||||
|
|
||||||
|
const authority = { kind: 'OFFLINE' as const, profileName: 'retry-test', reason: 'isolated unit test' };
|
||||||
|
|
||||||
|
describe('serializable clock operation retries', () => {
|
||||||
|
it('retries a PostgreSQL serialization conflict while starting suspension', async () => {
|
||||||
|
const result: ClockSuspensionResult = {
|
||||||
|
suspensionId: 'retry-start',
|
||||||
|
phase: 'SUSPENDED',
|
||||||
|
sourceRevision: 7,
|
||||||
|
targetRevision: 8,
|
||||||
|
cutTick: 123,
|
||||||
|
cutWallAt: new Date('2026-09-03T10:00:00.000Z'),
|
||||||
|
};
|
||||||
|
const transaction = vi
|
||||||
|
.fn()
|
||||||
|
.mockRejectedValueOnce(Object.assign(new Error('could not serialize access'), { code: 'P2034' }))
|
||||||
|
.mockResolvedValueOnce(result);
|
||||||
|
const db = { $transaction: transaction } as unknown as GamePrismaClient;
|
||||||
|
|
||||||
|
await expect(
|
||||||
|
startClockSuspension({ db, suspensionId: result.suspensionId, source: 'MAINTENANCE', authority })
|
||||||
|
).resolves.toEqual(result);
|
||||||
|
expect(transaction).toHaveBeenCalledTimes(2);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('retries a raw 40001 conflict while reconciling suspension', async () => {
|
||||||
|
const result: ClockReconciliationResult = {
|
||||||
|
suspensionId: 'retry-resume',
|
||||||
|
phase: 'RECONCILING',
|
||||||
|
sourceRevision: 7,
|
||||||
|
targetRevision: 8,
|
||||||
|
deadlineGeneration: 8,
|
||||||
|
gapTicks: 100,
|
||||||
|
catchUpTicks: 0,
|
||||||
|
shiftTicks: 100,
|
||||||
|
alignedTick: 223,
|
||||||
|
resumeWallAt: new Date('2026-09-03T10:10:00.000Z'),
|
||||||
|
};
|
||||||
|
const transaction = vi
|
||||||
|
.fn()
|
||||||
|
.mockRejectedValueOnce(
|
||||||
|
Object.assign(new Error('SQLSTATE 40001'), { code: 'P2010', meta: { code: '40001' } })
|
||||||
|
)
|
||||||
|
.mockResolvedValueOnce(result);
|
||||||
|
const db = { $transaction: transaction } as unknown as GamePrismaClient;
|
||||||
|
|
||||||
|
await expect(reconcileClockSuspension({ db, suspensionId: result.suspensionId, authority })).resolves.toEqual(
|
||||||
|
result
|
||||||
|
);
|
||||||
|
expect(transaction).toHaveBeenCalledTimes(2);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('does not retry a non-serialization failure', async () => {
|
||||||
|
const transaction = vi.fn().mockRejectedValue(new Error('authority denied'));
|
||||||
|
const db = { $transaction: transaction } as unknown as GamePrismaClient;
|
||||||
|
|
||||||
|
await expect(
|
||||||
|
startClockSuspension({ db, suspensionId: 'no-retry', source: 'MAINTENANCE', authority })
|
||||||
|
).rejects.toThrow('authority denied');
|
||||||
|
expect(transaction).toHaveBeenCalledTimes(1);
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -608,6 +608,12 @@ integration('database command queue', () => {
|
|||||||
expectedUpdatedAt: null,
|
expectedUpdatedAt: null,
|
||||||
mutation: { kind: 'nationPriority', priority: ['develop'] },
|
mutation: { kind: 'nationPriority', priority: ['develop'] },
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
type: 'shiftSchedule',
|
||||||
|
requestId: 'integration:engine:suspended-shift-schedule',
|
||||||
|
actionId: '00000000-0000-4000-8000-000000000023',
|
||||||
|
deltaMinutes: -15,
|
||||||
|
},
|
||||||
];
|
];
|
||||||
await db.inputEvent.createMany({
|
await db.inputEvent.createMany({
|
||||||
data: commands.map((command) => ({
|
data: commands.map((command) => ({
|
||||||
@@ -637,14 +643,18 @@ integration('database command queue', () => {
|
|||||||
const claimed = await new DatabaseTurnDaemonCommandQueue(db).drain();
|
const claimed = await new DatabaseTurnDaemonCommandQueue(db).drain();
|
||||||
expect(claimed.map(({ type }) => type)).toEqual(commands.map(({ type }) => type));
|
expect(claimed.map(({ type }) => type)).toEqual(commands.map(({ type }) => type));
|
||||||
for (const command of commands) {
|
for (const command of commands) {
|
||||||
await expect(db.inputEvent.findUniqueOrThrow({ where: { requestId: command.requestId! } })).resolves.toMatchObject({
|
await expect(
|
||||||
|
db.inputEvent.findUniqueOrThrow({ where: { requestId: command.requestId! } })
|
||||||
|
).resolves.toMatchObject({
|
||||||
status: 'PROCESSING',
|
status: 'PROCESSING',
|
||||||
processingGameTick: 777n,
|
processingGameTick: 777n,
|
||||||
processingClockRevision: 23n,
|
processingClockRevision: 23n,
|
||||||
processingDeadlineGeneration: 9n,
|
processingDeadlineGeneration: 9n,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
await expect(db.inputEvent.findUniqueOrThrow({ where: { requestId: blockedRequestId } })).resolves.toMatchObject({
|
await expect(
|
||||||
|
db.inputEvent.findUniqueOrThrow({ where: { requestId: blockedRequestId } })
|
||||||
|
).resolves.toMatchObject({
|
||||||
status: 'PENDING',
|
status: 'PENDING',
|
||||||
processingClockRevision: null,
|
processingClockRevision: null,
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -0,0 +1,174 @@
|
|||||||
|
import { describe, expect, it, vi } from 'vitest';
|
||||||
|
|
||||||
|
import type { GamePrisma } from '@sammo-ts/infra';
|
||||||
|
import { InMemoryTurnWorld } from '../src/turn/inMemoryWorld.js';
|
||||||
|
import { synchronizeRuntimeClockAuthorityUnderHeldLock } from '../src/turn/runtimeClockAuthoritySync.js';
|
||||||
|
import type { TurnGeneral, TurnWorldSnapshot, TurnWorldState } from '../src/turn/types.js';
|
||||||
|
|
||||||
|
const baseTime = new Date('2026-09-03T10:00:00.000Z');
|
||||||
|
|
||||||
|
const buildWorld = (phase: 'RUNNING' | 'SUSPENDED' = 'RUNNING'): InMemoryTurnWorld => {
|
||||||
|
const general = {
|
||||||
|
id: 1,
|
||||||
|
name: 'clock-sync-general',
|
||||||
|
nationId: 1,
|
||||||
|
cityId: 1,
|
||||||
|
troopId: 0,
|
||||||
|
stats: { leadership: 50, strength: 50, intelligence: 50 },
|
||||||
|
turnTime: new Date('2026-09-03T10:10:00.000Z'),
|
||||||
|
turnTick: 36_000_000,
|
||||||
|
role: { items: { horse: null, weapon: null, book: null, item: null } },
|
||||||
|
triggerState: { flags: {}, counters: {}, modifiers: {}, meta: {} },
|
||||||
|
meta: { killturn: 24 },
|
||||||
|
officerLevel: 5,
|
||||||
|
experience: 0,
|
||||||
|
dedication: 0,
|
||||||
|
injury: 0,
|
||||||
|
gold: 1000,
|
||||||
|
rice: 1000,
|
||||||
|
crew: 0,
|
||||||
|
crewTypeId: 0,
|
||||||
|
train: 0,
|
||||||
|
atmos: 0,
|
||||||
|
age: 30,
|
||||||
|
npcState: 0,
|
||||||
|
} as TurnGeneral;
|
||||||
|
const state: TurnWorldState = {
|
||||||
|
id: 1,
|
||||||
|
currentYear: 190,
|
||||||
|
currentMonth: 1,
|
||||||
|
tickSeconds: 600,
|
||||||
|
lastTurnTime: baseTime,
|
||||||
|
clockBaseTime: baseTime,
|
||||||
|
clockTick: 0,
|
||||||
|
clockMode: 'realtime',
|
||||||
|
clockWallAnchor: baseTime,
|
||||||
|
lastTurnTick: 0,
|
||||||
|
clockPhase: phase,
|
||||||
|
clockRevision: 3,
|
||||||
|
deadlineGeneration: 5,
|
||||||
|
meta: { lastTurnTime: baseTime.toISOString() },
|
||||||
|
};
|
||||||
|
const snapshot: TurnWorldSnapshot = {
|
||||||
|
generals: [general],
|
||||||
|
cities: [],
|
||||||
|
nations: [],
|
||||||
|
troops: [],
|
||||||
|
diplomacy: [],
|
||||||
|
events: [],
|
||||||
|
initialEvents: [],
|
||||||
|
map: {
|
||||||
|
id: 'clock-sync',
|
||||||
|
name: 'clock-sync',
|
||||||
|
cities: [],
|
||||||
|
defaults: { trust: 50, trade: 100, supplyState: 1, frontState: 0 },
|
||||||
|
},
|
||||||
|
scenarioConfig: {
|
||||||
|
stat: { total: 300, min: 10, max: 100, npcTotal: 150, npcMax: 50, npcMin: 10, chiefMin: 70 },
|
||||||
|
iconPath: '',
|
||||||
|
map: {},
|
||||||
|
const: {},
|
||||||
|
environment: { mapName: 'test', unitSet: 'default' },
|
||||||
|
},
|
||||||
|
};
|
||||||
|
return new InMemoryTurnWorld(state, snapshot, {
|
||||||
|
schedule: { entries: [{ startMinute: 0, tickMinutes: 10 }] },
|
||||||
|
});
|
||||||
|
};
|
||||||
|
|
||||||
|
const buildDb = (worldState: Record<string, unknown>, ledgers: unknown[] = []): GamePrisma.TransactionClient =>
|
||||||
|
({
|
||||||
|
worldState: { findFirst: vi.fn().mockResolvedValue(worldState) },
|
||||||
|
clockSuspension: { findMany: vi.fn().mockResolvedValue(ledgers) },
|
||||||
|
}) as unknown as GamePrisma.TransactionClient;
|
||||||
|
|
||||||
|
describe('runtime clock authority synchronization', () => {
|
||||||
|
it('adopts a maintenance suspension cut without advancing the game schedule', async () => {
|
||||||
|
const world = buildWorld('RUNNING');
|
||||||
|
const cutWallAt = new Date('2026-09-03T10:02:00.000Z');
|
||||||
|
const beforeTurnTick = world.getGeneralById(1)!.turnTick;
|
||||||
|
const db = buildDb({
|
||||||
|
id: 1,
|
||||||
|
clockBaseTime: baseTime,
|
||||||
|
clockTick: 7_200_000n,
|
||||||
|
clockMode: 'realtime',
|
||||||
|
clockWallAnchor: cutWallAt,
|
||||||
|
lastTurnTick: 0n,
|
||||||
|
clockPhase: 'SUSPENDED',
|
||||||
|
clockRevision: 3n,
|
||||||
|
deadlineGeneration: 5n,
|
||||||
|
});
|
||||||
|
|
||||||
|
await expect(synchronizeRuntimeClockAuthorityUnderHeldLock(db, world)).resolves.toBe(true);
|
||||||
|
|
||||||
|
expect(world.getGameClockState()).toMatchObject({
|
||||||
|
phase: 'SUSPENDED',
|
||||||
|
tick: 7_200_000,
|
||||||
|
revision: 3,
|
||||||
|
deadlineGeneration: 5,
|
||||||
|
wallAnchor: cutWallAt,
|
||||||
|
});
|
||||||
|
expect(world.getGeneralById(1)!.turnTick).toBe(beforeTurnTick);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('replays the durable reconciliation shift before returning to RUNNING', async () => {
|
||||||
|
const world = buildWorld('SUSPENDED');
|
||||||
|
const resumeWallAt = new Date('2026-09-03T11:00:00.000Z');
|
||||||
|
const shiftTicks = 5_000;
|
||||||
|
const beforeTurnTick = world.getGeneralById(1)!.turnTick!;
|
||||||
|
const db = buildDb(
|
||||||
|
{
|
||||||
|
id: 1,
|
||||||
|
clockBaseTime: baseTime,
|
||||||
|
clockTick: 5_000n,
|
||||||
|
clockMode: 'realtime',
|
||||||
|
clockWallAnchor: resumeWallAt,
|
||||||
|
lastTurnTick: 5_000n,
|
||||||
|
clockPhase: 'RUNNING',
|
||||||
|
clockRevision: 4n,
|
||||||
|
deadlineGeneration: 6n,
|
||||||
|
},
|
||||||
|
[
|
||||||
|
{
|
||||||
|
id: 'maintenance-revision-3',
|
||||||
|
sourceRevision: 3n,
|
||||||
|
targetRevision: 4n,
|
||||||
|
shiftTicks: BigInt(shiftTicks),
|
||||||
|
alignedTick: 5_000n,
|
||||||
|
resumeWallAt,
|
||||||
|
},
|
||||||
|
]
|
||||||
|
);
|
||||||
|
|
||||||
|
await expect(synchronizeRuntimeClockAuthorityUnderHeldLock(db, world)).resolves.toBe(true);
|
||||||
|
|
||||||
|
expect(world.getGameClockState()).toMatchObject({
|
||||||
|
phase: 'RUNNING',
|
||||||
|
tick: 5_000,
|
||||||
|
lastTurnTick: 5_000,
|
||||||
|
revision: 4,
|
||||||
|
deadlineGeneration: 6,
|
||||||
|
wallAnchor: resumeWallAt,
|
||||||
|
});
|
||||||
|
expect(world.getGeneralById(1)!.turnTick).toBe(beforeTurnTick + shiftTicks);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('rejects a revision jump when the durable ledger chain is incomplete', async () => {
|
||||||
|
const world = buildWorld('SUSPENDED');
|
||||||
|
const db = buildDb({
|
||||||
|
id: 1,
|
||||||
|
clockBaseTime: baseTime,
|
||||||
|
clockTick: 1n,
|
||||||
|
clockMode: 'realtime',
|
||||||
|
clockWallAnchor: baseTime,
|
||||||
|
lastTurnTick: 1n,
|
||||||
|
clockPhase: 'RUNNING',
|
||||||
|
clockRevision: 4n,
|
||||||
|
deadlineGeneration: 6n,
|
||||||
|
});
|
||||||
|
|
||||||
|
await expect(synchronizeRuntimeClockAuthorityUnderHeldLock(db, world)).rejects.toThrow(
|
||||||
|
/ledger chain ended at 3\/5/
|
||||||
|
);
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -8,7 +8,7 @@ import { chromium, type BrowserContext, type Page } from '@playwright/test';
|
|||||||
import { createTRPCProxyClient, httpBatchLink } from '@trpc/client';
|
import { createTRPCProxyClient, httpBatchLink } from '@trpc/client';
|
||||||
import type { AppRouter as GatewayAppRouter } from '@sammo-ts/gateway-api';
|
import type { AppRouter as GatewayAppRouter } from '@sammo-ts/gateway-api';
|
||||||
import type { AppRouter as GameAppRouter } from '@sammo-ts/game-api';
|
import type { AppRouter as GameAppRouter } from '@sammo-ts/game-api';
|
||||||
import { createGamePostgresConnector, createRedisConnector } from '@sammo-ts/infra';
|
import { createGamePostgresConnector, createRedisConnector, type GamePrisma } from '@sammo-ts/infra';
|
||||||
import { createTurnDaemonRuntime } from '../../../app/game-engine/src/turn/turnDaemon.js';
|
import { createTurnDaemonRuntime } from '../../../app/game-engine/src/turn/turnDaemon.js';
|
||||||
|
|
||||||
const gatewayUrl = process.env.SAMMO_LIVE_GATEWAY_URL ?? 'http://caddy/gateway/api/trpc';
|
const gatewayUrl = process.env.SAMMO_LIVE_GATEWAY_URL ?? 'http://caddy/gateway/api/trpc';
|
||||||
@@ -923,6 +923,64 @@ const prepareActionFixture = async (): Promise<void> => {
|
|||||||
|
|
||||||
const rotateFirst = <T>(values: readonly T[]): T[] => (values.length < 2 ? [...values] : [...values.slice(1), values[0]!]);
|
const rotateFirst = <T>(values: readonly T[]): T[] => (values.length < 2 ? [...values] : [...values.slice(1), values[0]!]);
|
||||||
|
|
||||||
|
const readHiddenBuffLevel = (rawMeta: unknown, key: string): number => {
|
||||||
|
if (!rawMeta || typeof rawMeta !== 'object' || Array.isArray(rawMeta)) return 0;
|
||||||
|
const rawBuff = Reflect.get(rawMeta, 'inheritBuff');
|
||||||
|
let buff: unknown = rawBuff;
|
||||||
|
if (typeof rawBuff === 'string') {
|
||||||
|
try {
|
||||||
|
buff = JSON.parse(rawBuff) as unknown;
|
||||||
|
} catch {
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (!buff || typeof buff !== 'object' || Array.isArray(buff)) return 0;
|
||||||
|
const value = Reflect.get(buff, key);
|
||||||
|
return typeof value === 'number' && Number.isFinite(value) ? value : 0;
|
||||||
|
};
|
||||||
|
|
||||||
|
const repairActionItemFixture = async (): Promise<void> => {
|
||||||
|
const state = await readState();
|
||||||
|
if (!state.actionFixture) throw new Error('The action fixture is required.');
|
||||||
|
const gateway = await loginAdmin();
|
||||||
|
const profiles = (await gateway.admin.profiles.list.query()) as unknown as Array<{
|
||||||
|
profileName: string;
|
||||||
|
status: string;
|
||||||
|
}>;
|
||||||
|
const profile = profiles.find((entry) => entry.profileName === profileName);
|
||||||
|
if (profile?.status !== 'STOPPED') {
|
||||||
|
throw new Error(`Item fixture repair requires STOPPED, found ${profile?.status ?? 'missing'}.`);
|
||||||
|
}
|
||||||
|
const db = createGamePostgresConnector({ url: gameDatabaseUrl() });
|
||||||
|
await db.connect();
|
||||||
|
try {
|
||||||
|
const targetGeneralId = state.actionFixture.generals[2]!.id;
|
||||||
|
const repaired = await db.prisma.$transaction(async (tx) => {
|
||||||
|
const world = await tx.worldState.findFirstOrThrow({ orderBy: { id: 'asc' } });
|
||||||
|
if (world.clockPhase !== 'SUSPENDED') {
|
||||||
|
throw new Error(`Item fixture repair requires SUSPENDED, found ${world.clockPhase}.`);
|
||||||
|
}
|
||||||
|
const general = await tx.general.findUniqueOrThrow({ where: { id: targetGeneralId } });
|
||||||
|
const meta =
|
||||||
|
general.meta && typeof general.meta === 'object' && !Array.isArray(general.meta)
|
||||||
|
? { ...general.meta }
|
||||||
|
: {};
|
||||||
|
delete meta.itemInventory;
|
||||||
|
return tx.general.update({
|
||||||
|
where: { id: targetGeneralId },
|
||||||
|
data: {
|
||||||
|
weaponCode: 'che_무기_01_단도',
|
||||||
|
meta: meta as GamePrisma.InputJsonValue,
|
||||||
|
},
|
||||||
|
select: { id: true, weaponCode: true },
|
||||||
|
});
|
||||||
|
});
|
||||||
|
log('action-item-fixture-repaired', repaired);
|
||||||
|
} finally {
|
||||||
|
await db.disconnect();
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
const exercisePausedActions = async (): Promise<void> => {
|
const exercisePausedActions = async (): Promise<void> => {
|
||||||
const state = await readState();
|
const state = await readState();
|
||||||
if (state.users?.length !== 10 || !state.actionFixture) {
|
if (state.users?.length !== 10 || !state.actionFixture) {
|
||||||
@@ -986,8 +1044,20 @@ const exercisePausedActions = async (): Promise<void> => {
|
|||||||
expectedRevision: turnSnapshot.revision,
|
expectedRevision: turnSnapshot.revision,
|
||||||
});
|
});
|
||||||
|
|
||||||
const inheritance = await clients[3]!.inherit.buyHiddenBuff.mutate({ type: 'warAvoidRatio', level: 1 });
|
const inheritanceRows = await db.prisma.general.findMany({
|
||||||
const dropped = await clients[2]!.general.dropItem.mutate({ itemType: 'weapon' });
|
where: { id: { in: generals.map(({ id }) => id) } },
|
||||||
|
select: { id: true, meta: true },
|
||||||
|
});
|
||||||
|
const inheritanceById = new Map(inheritanceRows.map((general) => [general.id, general.meta]));
|
||||||
|
const inheritanceGeneralIndex = generals.findIndex(
|
||||||
|
(general, index) =>
|
||||||
|
index >= 3 && readHiddenBuffLevel(inheritanceById.get(general.id), 'warAvoidRatio') < 1
|
||||||
|
);
|
||||||
|
if (inheritanceGeneralIndex < 0) throw new Error('No lifecycle user remains for the inheritance purchase.');
|
||||||
|
const inheritance = await clients[inheritanceGeneralIndex]!.inherit.buyHiddenBuff.mutate({
|
||||||
|
type: 'warAvoidRatio',
|
||||||
|
level: 1,
|
||||||
|
});
|
||||||
const permission = await clients[0]!.nation.changePermission.mutate({
|
const permission = await clients[0]!.nation.changePermission.mutate({
|
||||||
isAmbassador: true,
|
isAmbassador: true,
|
||||||
targetGeneralIds: [generals[2]!.id],
|
targetGeneralIds: [generals[2]!.id],
|
||||||
@@ -1018,6 +1088,7 @@ const exercisePausedActions = async (): Promise<void> => {
|
|||||||
detail: `SUSPENDED 상태의 WALL_TIME 외교문서 ${runSlug}`,
|
detail: `SUSPENDED 상태의 WALL_TIME 외교문서 ${runSlug}`,
|
||||||
});
|
});
|
||||||
const letterResponse = await clients[5]!.diplomacy.respondLetter.mutate({ letterId: letter.id, agree: true });
|
const letterResponse = await clients[5]!.diplomacy.respondLetter.mutate({ letterId: letter.id, agree: true });
|
||||||
|
const dropped = await clients[2]!.general.dropItem.mutate({ itemType: 'weapon' });
|
||||||
|
|
||||||
const [worldAfter, persistedGenerals, persistedMessages, persistedLetter, inheritanceLogs, pendingEvents] =
|
const [worldAfter, persistedGenerals, persistedMessages, persistedLetter, inheritanceLogs, pendingEvents] =
|
||||||
await Promise.all([
|
await Promise.all([
|
||||||
@@ -2186,6 +2257,7 @@ else if (command === 'prepare-paused-betting') await preparePausedBetting();
|
|||||||
else if (command === 'submit-paused-betting') await submitPausedBetting();
|
else if (command === 'submit-paused-betting') await submitPausedBetting();
|
||||||
else if (command === 'reserve-user-enlistments') await reserveUserEnlistments();
|
else if (command === 'reserve-user-enlistments') await reserveUserEnlistments();
|
||||||
else if (command === 'prepare-action-fixture') await prepareActionFixture();
|
else if (command === 'prepare-action-fixture') await prepareActionFixture();
|
||||||
|
else if (command === 'repair-action-item-fixture') await repairActionItemFixture();
|
||||||
else if (command === 'exercise-paused-actions') await exercisePausedActions();
|
else if (command === 'exercise-paused-actions') await exercisePausedActions();
|
||||||
else if (command === 'verify-paused-wall-expiry') await verifyPausedWallExpiry();
|
else if (command === 'verify-paused-wall-expiry') await verifyPausedWallExpiry();
|
||||||
else if (command === 'exercise-paused-tournament-bet') await exercisePausedTournamentBet();
|
else if (command === 'exercise-paused-tournament-bet') await exercisePausedTournamentBet();
|
||||||
@@ -2208,5 +2280,5 @@ else if (command === 'wait-runtime-action') await waitRuntimeAction();
|
|||||||
else if (command === 'monitor-users') await monitorUsers();
|
else if (command === 'monitor-users') await monitorUsers();
|
||||||
else
|
else
|
||||||
throw new Error(
|
throw new Error(
|
||||||
'usage: live-ten-user-lifecycle.ts <reset|wait-reset|deploy|wait-deploy|status|prepare-users|preopen-messages|verified-preopen-messages|repair-preopen-fixture|database-status|prepare-paused-betting|submit-paused-betting|reserve-user-enlistments|prepare-action-fixture|exercise-paused-actions|verify-paused-wall-expiry|exercise-paused-tournament-bet|reserve-actionable-commands|wait-actionable-messages|respond-actionable-messages|reserve-no-aggression-cancellation|wait-no-aggression-cancellation|respond-no-aggression-cancellation|npc-action-audit|repair-opening-clock-runtime|verify-monitor-message|resume-daemon|fast-forward|place-invader-recipients|respond-invader-browser|action|wait-profile-status|wait-runtime-action|monitor-users>'
|
'usage: live-ten-user-lifecycle.ts <reset|wait-reset|deploy|wait-deploy|status|prepare-users|preopen-messages|verified-preopen-messages|repair-preopen-fixture|database-status|prepare-paused-betting|submit-paused-betting|reserve-user-enlistments|prepare-action-fixture|repair-action-item-fixture|exercise-paused-actions|verify-paused-wall-expiry|exercise-paused-tournament-bet|reserve-actionable-commands|wait-actionable-messages|respond-actionable-messages|reserve-no-aggression-cancellation|wait-no-aggression-cancellation|respond-no-aggression-cancellation|npc-action-audit|repair-opening-clock-runtime|verify-monitor-message|resume-daemon|fast-forward|place-invader-recipients|respond-invader-browser|action|wait-profile-status|wait-runtime-action|monitor-users>'
|
||||||
);
|
);
|
||||||
|
|||||||
Reference in New Issue
Block a user