feat: 게이트웨이 시계 생명주기 권위 연결

This commit is contained in:
2026-09-03 09:49:53 +00:00
parent a3e2bf90ae
commit a9f23703d9
7 changed files with 330 additions and 25 deletions
+13
View File
@@ -2750,6 +2750,19 @@ export const adminRouter = router({
};
await ctx.profiles.updateMeta(input.profileName, nextMeta);
if (mappedStatus) {
if (input.action === 'PAUSE' || input.action === 'STOP') {
await ctx.orchestrator.transitionProfileClock(
input.profileName,
'SUSPEND',
input.reason?.trim() || `gateway ${input.action.toLowerCase()} by ${adminAuth.user.id}`
);
} else if (input.action === 'RESUME') {
await ctx.orchestrator.transitionProfileClock(
input.profileName,
'RESUME',
input.reason?.trim() || `gateway resume by ${adminAuth.user.id}`
);
}
await ctx.profiles.updateStatus(input.profileName, mappedStatus);
await ctx.orchestrator.reconcileNow();
const appliedActionRecord = {
@@ -5,6 +5,12 @@ import { createHash, randomBytes, randomUUID } from 'node:crypto';
import { stripVTControlCharacters } from 'node:util';
import type { ScenarioInstallOptions } from '@sammo-ts/game-engine/scenario/scenarioSeeder.js';
import {
applyNextClockProjection,
reconcileClockSuspension,
startClockSuspension,
type ClockOperationAuthority,
} from '@sammo-ts/game-engine';
import {
cancelGame as defaultCancelGame,
GAME_CANCELLATION_GENERAL_MODES,
@@ -17,6 +23,9 @@ import { gatewayProfileCapabilities } from '@sammo-ts/common';
import {
createGamePostgresConnector,
createRedisConnector,
CLOCK_OPERATION_PERSISTENCE_LOCK,
GamePrisma,
acquireGameSchemaAdvisoryXactLock,
resolvePostgresPoolMax,
resolveRedisConfigFromEnv,
} from '@sammo-ts/infra';
@@ -112,6 +121,12 @@ export interface GatewayOrchestratorOptions {
fetchImpl?: typeof fetch;
clearTournamentRuntimeState?: (profileName: string) => Promise<void>;
cancelGame?: typeof defaultCancelGame;
transitionProfileClock?: (
profileName: string,
action: 'SUSPEND' | 'RESUME',
reason: string
) => Promise<{ phase: string; revision: number }>;
promoteProfileOpening?: (profile: GatewayProfileRecord) => Promise<void>;
}
const WORKSPACE_CLEANUP_INTERVAL_MS = 24 * 60 * 60 * 1_000;
@@ -161,6 +176,11 @@ export interface GatewayOrchestratorHandle {
}>;
listRuntimeStates(profileNames: string[]): Promise<ProfileRuntimeSnapshot[]>;
listRuntimeSettings?(profileNames: string[]): Promise<ProfileRuntimeSettingsSnapshot[]>;
transitionProfileClock(
profileName: string,
action: 'SUSPEND' | 'RESUME',
reason: string
): Promise<{ phase: string; revision: number }>;
}
export interface GatewayManagedCleanupResult {
@@ -322,6 +342,14 @@ const readMetaNumber = (meta: Record<string, unknown>, key: string): number | nu
return null;
};
const clockRevisionAsNumber = (revision: bigint): number => {
const value = Number(revision);
if (!Number.isSafeInteger(value) || value < 1) {
throw new Error(`Clock revision is outside the safe API integer range: ${revision.toString()}.`);
}
return value;
};
export const resolveProfileFirstGameIdx = (meta: Record<string, unknown>): number => {
const raw = meta.firstGameIdx;
const configured = typeof raw === 'number' ? raw : typeof raw === 'string' ? Number(raw) : Number.NaN;
@@ -842,10 +870,7 @@ const assertProfileMigrationEnvironmentTimeZone = (env?: Record<string, string>)
if (pgTimeZone) assertProfileMigrationTimeZone(pgTimeZone, 'PGTZ');
};
const buildProfileMigrationEnv = (
profileDatabaseUrl: string,
env?: Record<string, string>
): Record<string, string> => {
const buildProfileMigrationEnv = (profileDatabaseUrl: string, env?: Record<string, string>): Record<string, string> => {
assertProfileMigrationEnvironmentTimeZone(env);
return { ...(env ?? {}), DATABASE_URL: buildProfileMigrationDatabaseUrl(profileDatabaseUrl) };
};
@@ -942,6 +967,8 @@ export class GatewayOrchestrator implements GatewayOrchestratorHandle {
private readonly fetchImpl: typeof fetch;
private readonly clearTournamentRuntimeState: (profileName: string) => Promise<void>;
private readonly cancelGame: typeof defaultCancelGame;
private readonly transitionProfileClockOverride?: GatewayOrchestratorOptions['transitionProfileClock'];
private readonly promoteProfileOpeningOverride?: GatewayOrchestratorOptions['promoteProfileOpening'];
private reconcileTimer?: NodeJS.Timeout;
private scheduleTimer?: NodeJS.Timeout;
private buildTimer?: NodeJS.Timeout;
@@ -986,6 +1013,8 @@ export class GatewayOrchestrator implements GatewayOrchestratorHandle {
options.clearTournamentRuntimeState ??
((profileName) => this.clearTournamentRuntimeStateFromRedis(profileName));
this.cancelGame = options.cancelGame ?? defaultCancelGame;
this.transitionProfileClockOverride = options.transitionProfileClock;
this.promoteProfileOpeningOverride = options.promoteProfileOpening;
}
private sanitizeOperationLogMessage(message: string): string {
@@ -1171,6 +1200,191 @@ export class GatewayOrchestrator implements GatewayOrchestratorHandle {
return snapshots.filter((snapshot): snapshot is ProfileRuntimeSettingsSnapshot => snapshot !== null);
}
async transitionProfileClock(
profileName: string,
action: 'SUSPEND' | 'RESUME',
reason: string
): Promise<{ phase: string; revision: number }> {
if (this.transitionProfileClockOverride) {
return this.transitionProfileClockOverride(profileName, action, reason);
}
const profile = await this.repository.getProfile(profileName);
if (!profile || profile.currentScenario === null) {
throw new Error(`Profile clock is unavailable: ${profileName}`);
}
const postgres = createGamePostgresConnector({ url: this.resolveProfileDatabaseUrl(profile) });
const redis = createRedisConnector(resolveRedisConfigFromEnv(this.processConfig.baseEnv ?? process.env));
await postgres.connect();
await redis.connect();
try {
const authorityRows = await postgres.prisma.$queryRaw<
Array<{ ownerId: string; fencingEpoch: bigint; valid: boolean }>
>(GamePrisma.sql`
SELECT owner_id AS "ownerId",
fencing_epoch AS "fencingEpoch",
lease_until > (CURRENT_TIMESTAMP AT TIME ZONE 'UTC') AS valid
FROM turn_daemon_lease
WHERE profile = ${profileName}
`);
const liveLease = authorityRows.find((row) => row.valid);
const authority: ClockOperationAuthority = liveLease
? {
kind: 'DAEMON',
profileName,
ownerId: liveLease.ownerId,
fencingEpoch: liveLease.fencingEpoch,
}
: { kind: 'OFFLINE', profileName, reason };
const world = await postgres.prisma.worldState.findFirst({
orderBy: { id: 'asc' },
select: { clockPhase: true, clockRevision: true, deadlineGeneration: true },
});
if (!world) throw new Error(`Profile has no world_state: ${profileName}`);
if (action === 'SUSPEND') {
let suspension = await postgres.prisma.clockSuspension.findFirst({
where: { sourceRevision: world.clockRevision, status: 'SUSPENDED' },
orderBy: { createdAt: 'desc' },
});
if (world.clockPhase === 'RUNNING') {
const suffix = createHash('sha256')
.update(`${profileName}:${world.clockRevision.toString()}`)
.digest('hex')
.slice(0, 20);
const started = await startClockSuspension({
db: postgres.prisma,
suspensionId: `gateway-maintenance-${suffix}`,
source: 'MAINTENANCE',
authority,
});
suspension = await postgres.prisma.clockSuspension.findUniqueOrThrow({
where: { id: started.suspensionId },
});
} else if (world.clockPhase !== 'SUSPENDED') {
throw new Error(`Cannot suspend profile clock from ${world.clockPhase}.`);
}
if (!suspension) throw new Error('Suspended profile is missing its durable clock ledger.');
const phaseResult = await redis.client.eval(
`
local active = redis.call('GET', KEYS[1])
if active and active ~= ARGV[1] then return 0 end
redis.call('SET', KEYS[1], ARGV[1])
redis.call('SET', KEYS[2], ARGV[2])
redis.call('SET', KEYS[3], 'SUSPENDED')
return 1
`,
{
keys: [
`sammo:${profileName}:clock:active-revision`,
`sammo:${profileName}:clock:deadline-generation`,
`sammo:${profileName}:clock:phase`,
],
arguments: [world.clockRevision.toString(), world.deadlineGeneration.toString()],
}
);
if (Number(phaseResult) !== 1) throw new Error('Redis clock source revision differs from the DB.');
return { phase: 'SUSPENDED', revision: clockRevisionAsNumber(suspension.sourceRevision) };
}
if (world.clockPhase === 'RUNNING') {
return { phase: 'RUNNING', revision: clockRevisionAsNumber(world.clockRevision) };
}
const suspension = await postgres.prisma.clockSuspension.findFirst({
where: { status: { in: ['SUSPENDED', 'RECONCILING'] } },
orderBy: { createdAt: 'desc' },
});
if (!suspension) throw new Error('Profile resume requires a durable suspended clock ledger.');
if (world.clockPhase === 'SUSPENDED') {
await reconcileClockSuspension({ db: postgres.prisma, suspensionId: suspension.id, authority });
} else if (world.clockPhase !== 'RECONCILING') {
throw new Error(`Cannot resume profile clock from ${world.clockPhase}.`);
}
const projected = await applyNextClockProjection({
db: postgres.prisma,
redis: redis.client,
workerId: `gateway:${this.operationLeaseOwner}`,
});
if (projected === 'IDLE') throw new Error('Clock reconciliation has no claimable projection outbox.');
const resumed = await postgres.prisma.worldState.findFirstOrThrow({
orderBy: { id: 'asc' },
select: { clockPhase: true, clockRevision: true },
});
if (resumed.clockPhase !== 'RUNNING') throw new Error('Clock projection did not reach RUNNING.');
return { phase: resumed.clockPhase, revision: clockRevisionAsNumber(resumed.clockRevision) };
} finally {
await redis.disconnect().catch(() => undefined);
await postgres.disconnect().catch(() => undefined);
}
}
private async promoteProfileOpening(profile: GatewayProfileRecord): Promise<void> {
if (this.promoteProfileOpeningOverride) {
await this.promoteProfileOpeningOverride(profile);
return;
}
const postgres = createGamePostgresConnector({ url: this.resolveProfileDatabaseUrl(profile) });
const redis = createRedisConnector(resolveRedisConfigFromEnv(this.processConfig.baseEnv ?? process.env));
await postgres.connect();
await redis.connect();
try {
const clock = await postgres.prisma.$transaction(async (transaction) => {
await acquireGameSchemaAdvisoryXactLock(transaction, CLOCK_OPERATION_PERSISTENCE_LOCK);
const rows = await transaction.$queryRaw<Array<{ id: number }>>(GamePrisma.sql`
SELECT id FROM world_state ORDER BY id LIMIT 2 FOR UPDATE
`);
if (rows.length !== 1) {
throw new Error(`Opening promotion requires exactly one world_state row; found ${rows.length}.`);
}
const world = await transaction.worldState.findUniqueOrThrow({ where: { id: rows[0]!.id } });
if (world.clockPhase === 'RUNNING') {
return { revision: world.clockRevision, generation: world.deadlineGeneration };
}
if (world.clockPhase !== 'PREOPEN' || world.clockTick !== 0n || !world.clockWallAnchor) {
throw new Error(
`Opening promotion requires PREOPEN at tick 0; found ${world.clockPhase}@${world.clockTick?.toString() ?? 'null'}.`
);
}
const [wall] = await transaction.$queryRaw<Array<{ wallNow: Date }>>(GamePrisma.sql`
SELECT (CURRENT_TIMESTAMP AT TIME ZONE 'UTC')::timestamp(3) AS "wallNow"
`);
if (!wall || wall.wallNow.getTime() < world.clockWallAnchor.getTime()) {
throw new Error('Opening promotion was requested before the scheduled wall instant.');
}
await transaction.worldState.update({
where: { id: world.id },
data: { clockPhase: 'RUNNING' },
});
return { revision: world.clockRevision, generation: world.deadlineGeneration };
});
const result = await redis.client.eval(
`
local revision = redis.call('GET', KEYS[1])
local generation = redis.call('GET', KEYS[2])
if revision and revision ~= ARGV[1] then return 0 end
if generation and generation ~= ARGV[2] then return 0 end
redis.call('SET', KEYS[1], ARGV[1])
redis.call('SET', KEYS[2], ARGV[2])
redis.call('SET', KEYS[3], 'RUNNING')
return 1
`,
{
keys: [
`sammo:${profile.profileName}:clock:active-revision`,
`sammo:${profile.profileName}:clock:deadline-generation`,
`sammo:${profile.profileName}:clock:phase`,
],
arguments: [clock.revision.toString(), clock.generation.toString()],
}
);
if (Number(result) !== 1) {
throw new Error('Opening promotion found a different Redis clock revision or generation.');
}
} finally {
await redis.disconnect().catch(() => undefined);
await postgres.disconnect().catch(() => undefined);
}
}
async reconcileNow(): Promise<void> {
if (this.stopping || this.reconcileInFlight) {
return;
@@ -1232,14 +1446,14 @@ export class GatewayOrchestrator implements GatewayOrchestratorHandle {
continue;
}
if (profile.currentScenario !== null && profile.buildStatus === 'SUCCEEDED' && profile.buildWorkspace) {
await this.repository.updateStatus(
profile.profileName,
resolveResetLifecycleStatus(now, preopenAt, openAt),
{
preopenAt: profile.preopenAt,
openAt: profile.openAt,
}
);
const nextStatus = resolveResetLifecycleStatus(now, preopenAt, openAt);
if (nextStatus === 'RUNNING') {
await this.promoteProfileOpening(profile);
}
await this.repository.updateStatus(profile.profileName, nextStatus, {
preopenAt: profile.preopenAt,
openAt: profile.openAt,
});
await this.repository.updateLastError(profile.profileName, null);
continue;
}
@@ -1255,6 +1469,7 @@ export class GatewayOrchestrator implements GatewayOrchestratorHandle {
const profiles = await this.repository.listProfiles();
for (const profile of profiles) {
if (profile.status === 'PREOPEN' && profile.openAt && new Date(profile.openAt) <= now) {
await this.promoteProfileOpening(profile);
await this.repository.updateStatus(profile.profileName, 'RUNNING', {
preopenAt: profile.preopenAt ?? null,
openAt: profile.openAt ?? null,
@@ -1301,16 +1516,17 @@ export class GatewayOrchestrator implements GatewayOrchestratorHandle {
error: null,
});
if (queued.status === 'RESERVED') {
await this.repository.updateStatus(
queued.profileName,
queued.openAt && new Date(queued.openAt) <= this.now() ? 'RUNNING' : 'PREOPEN',
{
preopenAt: queued.preopenAt ?? null,
openAt: queued.openAt ?? null,
}
);
const opensImmediately = Boolean(queued.openAt && new Date(queued.openAt) <= this.now());
if (opensImmediately) {
await this.promoteProfileOpening(queued);
}
await this.repository.updateStatus(queued.profileName, opensImmediately ? 'RUNNING' : 'PREOPEN', {
preopenAt: queued.preopenAt ?? null,
openAt: queued.openAt ?? null,
});
} else if (queued.status === 'PREOPEN' && queued.openAt) {
if (new Date(queued.openAt) <= this.now()) {
await this.promoteProfileOpening(queued);
await this.repository.updateStatus(queued.profileName, 'RUNNING', {
preopenAt: queued.preopenAt ?? null,
openAt: queued.openAt ?? null,
@@ -87,6 +87,7 @@ const buildCaller = async (
const updatedStatuses: GatewayProfileRecord['status'][] = [];
const updatedMetas: Record<string, unknown>[] = [];
const auditEvents: AdminAuditEventRecord[] = [];
const lifecycle: string[] = [];
let reconcileCount = 0;
let runtimeStateListCount = 0;
let storedNotice = options.initialNotice ?? '';
@@ -111,6 +112,7 @@ const buildCaller = async (
updateCurrentScenario: async () => profile,
updateStatus: async (_profileName, status) => {
updatedStatuses.push(status);
lifecycle.push(`status:${status}`);
return { ...profile, status };
},
updateBuildStatus: async () => profile,
@@ -288,6 +290,7 @@ const buildCaller = async (
stop: async () => {},
reconcileNow: async () => {
reconcileCount += 1;
lifecycle.push('runtime:reconcile');
},
runScheduleNow: async () => {},
runBuildQueueNow: async () => {},
@@ -297,6 +300,13 @@ const buildCaller = async (
}
},
cleanupStaleWorkspaces: async () => ({ removed: [], skipped: [] }),
transitionProfileClock: async (_profileName, action) => {
lifecycle.push(`clock:${action}`);
return {
phase: action === 'SUSPEND' ? 'SUSPENDED' : 'RUNNING',
revision: 1,
};
},
listRuntimeSettings: async () => [
{
profileName: 'che:2',
@@ -363,6 +373,7 @@ const buildCaller = async (
updatedStatuses,
updatedMetas,
auditEvents,
lifecycle,
getReconcileCount: () => reconcileCount,
getRuntimeStateListCount: () => runtimeStateListCount,
getStoredNotice: () => storedNotice,
@@ -1384,6 +1395,7 @@ describe('admin runtime clock action API', () => {
})
).resolves.toMatchObject({ ok: true });
expect(harness.updatedStatuses).toEqual(['RUNNING']);
expect(harness.lifecycle).toEqual(['clock:RESUME', 'status:RUNNING', 'runtime:reconcile']);
expect(harness.getReconcileCount()).toBe(1);
expect(harness.updatedMetas).toHaveLength(2);
expect(harness.updatedMetas.at(-1)).toMatchObject({
@@ -1410,6 +1422,11 @@ describe('admin runtime clock action API', () => {
});
expect(harness.updatedStatuses).toEqual([expectedStatus]);
expect(harness.lifecycle).toEqual(
action === 'STOP'
? ['clock:SUSPEND', `status:${expectedStatus}`, 'runtime:reconcile']
: [`status:${expectedStatus}`, 'runtime:reconcile']
);
expect(harness.getReconcileCount()).toBe(1);
expect(harness.updatedMetas).toHaveLength(2);
expect(harness.updatedMetas[0]).toMatchObject({
@@ -123,6 +123,10 @@ const createHarness = async (adminRoles = ['user', 'admin.users.manage', 'admin.
runBuildQueueNow: async () => {},
runOperationsNow: async () => {},
cleanupStaleWorkspaces: async () => ({ removed: [], skipped: [] }),
transitionProfileClock: async (_profileName, action) => ({
phase: action === 'SUSPEND' ? 'SUSPENDED' : 'RUNNING',
revision: 1,
}),
listRuntimeStates: async () => [],
},
profileStatus: new InMemoryProfileStatusService(),
@@ -183,9 +187,7 @@ describe('admin security over HTTP transport', () => {
},
});
const mutationInput = encodeURIComponent(
JSON.stringify({ json: { sessionToken: harness.adminSessionToken } })
);
const mutationInput = encodeURIComponent(JSON.stringify({ json: { sessionToken: harness.adminSessionToken } }));
const mutation = await fetch(`${harness.baseUrl}/trpc/auth.logout?input=${mutationInput}`);
expect(mutation.status).toBe(405);
expect(await mutation.json()).toMatchObject({
+4
View File
@@ -168,6 +168,10 @@ const buildCaller = (
removed: [],
skipped: [],
}),
transitionProfileClock: async (_profileName: string, action: 'SUSPEND' | 'RESUME') => ({
phase: action === 'SUSPEND' ? 'SUSPENDED' : 'RUNNING',
revision: 1,
}),
listRuntimeStates: async () => [],
};
const profileStatus = new InMemoryProfileStatusService(
@@ -65,6 +65,7 @@ const createHarness = (
frontendServeMode?: 'static';
frontendArtifactRoot?: string;
activeOperationProfileNames?: string[];
promoteProfileOpening?: GatewayOrchestratorOptions['promoteProfileOpening'];
} = {}
) => {
const harnessProfile = options.profile ?? profile;
@@ -77,6 +78,7 @@ const createHarness = (
const deleted: string[] = [];
const buildStatuses: string[] = [];
const logs: Array<{ phase: string; message: string; level: string }> = [];
const lifecycle: string[] = [];
const repository: GatewayProfileRepository = {
listProfiles: async () => options.profiles ?? [harnessProfile],
@@ -85,6 +87,7 @@ const createHarness = (
updateCurrentScenario: async () => harnessProfile,
updateStatus: async (_profileName, status) => {
statuses.push(status);
lifecycle.push(`status:${status}`);
return { ...harnessProfile, status };
},
updateBuildStatus: async (_profileName, status) => {
@@ -192,9 +195,26 @@ const createHarness = (
adminActionIntervalMs: 60_000,
now: options.now,
cancelGame: options.cancelGame,
promoteProfileOpening: options.promoteProfileOpening
? async (openingProfile) => {
lifecycle.push(`clock:${openingProfile.profileName}`);
await options.promoteProfileOpening?.(openingProfile);
}
: undefined,
});
return { orchestrator, statuses, buildStatuses, completions, completionFields, started, stopped, deleted, logs };
return {
orchestrator,
statuses,
buildStatuses,
completions,
completionFields,
started,
stopped,
deleted,
logs,
lifecycle,
};
};
describe('GatewayOrchestrator first-class operations', () => {
@@ -333,12 +353,14 @@ describe('GatewayOrchestrator first-class operations', () => {
profiles: [],
reservedToStart: [reservedProfile],
now: () => now,
promoteProfileOpening: async () => {},
});
await harness.orchestrator.runScheduleNow();
expect(harness.statuses).toEqual(['RUNNING']);
expect(harness.buildStatuses).toEqual([]);
expect(harness.lifecycle).toEqual([`clock:${reservedProfile.profileName}`, 'status:RUNNING']);
});
it('retains the legacy build queue for an unprepared reserved profile', async () => {
@@ -366,6 +388,26 @@ describe('GatewayOrchestrator first-class operations', () => {
expect(harness.buildStatuses).toEqual(['QUEUED']);
});
it('promotes the durable clock before a prepared preopen profile becomes running', async () => {
const now = new Date('2030-01-01T02:00:00.000Z');
const preopenProfile: GatewayProfileRecord = {
...profile,
status: 'PREOPEN',
openAt: now.toISOString(),
preopenAt: '2030-01-01T01:00:00.000Z',
};
const harness = createHarness(buildOperation('START'), false, false, false, false, undefined, undefined, {
profile: preopenProfile,
profiles: [preopenProfile],
now: () => now,
promoteProfileOpening: async () => {},
});
await harness.orchestrator.runScheduleNow();
expect(harness.lifecycle).toEqual([`clock:${preopenProfile.profileName}`, 'status:RUNNING']);
});
it('starts every profile process and records success', async () => {
const harness = createHarness(buildOperation('START'));
@@ -45,7 +45,7 @@ mean deployment or production validation.
- [x] All durable input events record accepted tick and accepted revision.
- [x] Processing converts accepted coordinates across revisions or fails closed.
- [ ] Gateway pause/resume/open orchestration writes the DB clock phase.
- [x] Gateway pause/resume/open orchestration writes the DB clock phase.
- [ ] Unification wait becomes a durable `UNIFICATION_WAIT` suspension.
- [ ] Alignment, optional rate change, invader IDs/RNG, creation, first schedule,
outbox, verification, and RUNNING transition form one retry-safe workflow.
@@ -101,3 +101,14 @@ mean deployment or production validation.
- The 24-hour/65m17.250s reconciliation suite passed 2/2 after the other DB
suites. Conditional files share a deliberately dedicated schema and are run
sequentially to prevent their fixture cleanup from racing another file.
### 2026-09-03 - Gateway lifecycle authority
- Runtime `PAUSE`/`STOP` starts a durable maintenance suspension before the
Gateway status and process reconciliation change. `RESUME` completes the DB
reconciliation and Redis outbox before the profile becomes `RUNNING`.
- Both an already-built overdue `RESERVED` profile and an overdue `PREOPEN`
profile promote the game DB from `PREOPEN@0` before the Gateway status changes
to `RUNNING`; the Redis clock phase is revision/generation fenced.
- `pnpm --filter @sammo-ts/gateway-api test` passed 313 tests with 35
environment-conditional skips. Gateway typecheck and target lint passed.