감사 정책 head 유실을 복구하고 데몬 준비 상태로 배포를 검증한다
This commit is contained in:
@@ -51,7 +51,7 @@ import { ReadModelOutboxWorker } from './realtime/outboxWorker.js';
|
||||
import { DeferredGeneralAccessWorker } from './services/deferredGeneralAccess.js';
|
||||
import { WebPushOutboxWorker } from './services/webPushOutboxWorker.js';
|
||||
import { scopeHttpIdempotencyKey } from './requestId.js';
|
||||
import { loadClockReadiness } from './services/clockReadiness.js';
|
||||
import { loadProfileReadiness } from './services/clockReadiness.js';
|
||||
|
||||
const extractBearerToken = (value: string | string[] | undefined): string | null => {
|
||||
if (!value) {
|
||||
@@ -436,14 +436,13 @@ export const createGameApiServer = async () => {
|
||||
});
|
||||
|
||||
app.get('/healthz', async (_request, reply) => {
|
||||
const clock = await loadClockReadiness(postgres.prisma);
|
||||
if (!clock.reconciliationComplete) reply.code(503);
|
||||
const readiness = await loadProfileReadiness(postgres.prisma, config.profileName);
|
||||
if (!readiness.ok) reply.code(503);
|
||||
return {
|
||||
ok: clock.reconciliationComplete,
|
||||
...readiness,
|
||||
profile: config.profileName,
|
||||
postgresPool: postgres.getPoolStats(),
|
||||
accountIconReconciliation: accountIconResetReconciler.getHealth(),
|
||||
clock,
|
||||
};
|
||||
});
|
||||
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { parseGameClockPhase } from '@sammo-ts/common';
|
||||
import { GamePrisma } from '@sammo-ts/infra';
|
||||
|
||||
import type { DatabaseClient } from '../context.js';
|
||||
|
||||
@@ -90,3 +91,20 @@ export const loadClockAdminStatus = async (db: DatabaseClient) => {
|
||||
},
|
||||
};
|
||||
};
|
||||
|
||||
/** API 기동과 별개로 데몬의 초기 감사 저장 및 clock 복구가 끝나야 배포 준비 완료다. */
|
||||
export const loadProfileReadiness = async (db: DatabaseClient, profileName: string) => {
|
||||
const [clock, leases] = await Promise.all([
|
||||
loadClockReadiness(db),
|
||||
db.$queryRaw<Array<{ ready: boolean }>>(GamePrisma.sql`
|
||||
SELECT EXISTS (
|
||||
SELECT 1 FROM turn_daemon_lease
|
||||
WHERE profile = ${profileName}
|
||||
AND clock_ready = TRUE
|
||||
AND lease_until > (clock_timestamp() AT TIME ZONE 'UTC')
|
||||
) AS ready
|
||||
`),
|
||||
]);
|
||||
const daemonReady = leases[0]?.ready === true;
|
||||
return { ok: clock.reconciliationComplete && daemonReady, clock, daemonReady };
|
||||
};
|
||||
|
||||
@@ -1,9 +1,24 @@
|
||||
import { describe, expect, it, vi } from 'vitest';
|
||||
|
||||
import type { DatabaseClient } from '../src/context.js';
|
||||
import { loadClockAdminStatus, loadClockReadiness } from '../src/services/clockReadiness.js';
|
||||
import { loadClockAdminStatus, loadClockReadiness, loadProfileReadiness } from '../src/services/clockReadiness.js';
|
||||
|
||||
describe('clock reconciliation readiness', () => {
|
||||
it.each([false, true])('requires the profile daemon lease to be ready: %s', async (ready) => {
|
||||
const db = {
|
||||
worldState: {
|
||||
findFirst: vi.fn(async () => ({ clockPhase: 'PREOPEN', clockRevision: 1n, deadlineGeneration: 0n })),
|
||||
},
|
||||
clockProjectionOutbox: { count: vi.fn(async () => 0) },
|
||||
$queryRaw: vi.fn(async () => [{ ready }]),
|
||||
} as unknown as DatabaseClient;
|
||||
await expect(loadProfileReadiness(db, 'che:default')).resolves.toMatchObject({
|
||||
ok: ready,
|
||||
daemonReady: ready,
|
||||
clock: { reconciliationComplete: true, gameplayEnabled: false },
|
||||
});
|
||||
});
|
||||
|
||||
it('fails closed when the reconciliation schema is not available', async () => {
|
||||
const db = {} as DatabaseClient;
|
||||
await expect(loadClockReadiness(db)).resolves.toEqual({
|
||||
|
||||
@@ -0,0 +1,62 @@
|
||||
import { afterAll, beforeAll, describe, expect, it } from 'vitest';
|
||||
import { createGamePostgresConnector, type GamePrismaClient } from '@sammo-ts/infra';
|
||||
import { loadProfileReadiness } from '../src/services/clockReadiness.js';
|
||||
|
||||
const databaseUrl = process.env.INPUT_EVENT_DATABASE_URL;
|
||||
const integration = describe.skipIf(!databaseUrl);
|
||||
integration('profile deployment readiness', () => {
|
||||
let db: GamePrismaClient;
|
||||
let close: () => Promise<void>;
|
||||
beforeAll(async () => {
|
||||
const connector = createGamePostgresConnector({ url: databaseUrl! });
|
||||
await connector.connect();
|
||||
db = connector.prisma;
|
||||
close = () => connector.disconnect();
|
||||
});
|
||||
afterAll(async () => {
|
||||
await close?.();
|
||||
});
|
||||
it('rejects absent, initializing, expired and other-profile leases; accepts ready paused/preopen runtime', async () => {
|
||||
const rollback = new Error('readiness fixture rollback');
|
||||
await expect(
|
||||
db.$transaction(async (tx) => {
|
||||
// Shared integration fixtures may leave projection work; rollback restores it.
|
||||
await tx.clockProjectionOutbox.deleteMany();
|
||||
await tx.worldState.create({
|
||||
data: {
|
||||
id: -998901,
|
||||
scenarioCode: 'readiness-fixture',
|
||||
currentYear: 190,
|
||||
currentMonth: 1,
|
||||
tickSeconds: 300,
|
||||
clockPhase: 'PREOPEN',
|
||||
config: {},
|
||||
meta: {},
|
||||
},
|
||||
});
|
||||
const profile = 'readiness:fixture';
|
||||
expect((await loadProfileReadiness(tx, profile)).ok).toBe(false);
|
||||
await tx.turnDaemonLease.create({
|
||||
data: {
|
||||
profile,
|
||||
ownerId: 'fixture',
|
||||
fencingEpoch: 1n,
|
||||
heartbeatAt: new Date(),
|
||||
leaseUntil: new Date(Date.now() + 60_000),
|
||||
clockReady: false,
|
||||
},
|
||||
});
|
||||
expect((await loadProfileReadiness(tx, profile)).ok).toBe(false);
|
||||
await tx.turnDaemonLease.update({ where: { profile }, data: { clockReady: true } });
|
||||
expect((await loadProfileReadiness(tx, profile)).ok).toBe(true);
|
||||
expect((await loadProfileReadiness(tx, 'readiness:other')).ok).toBe(false);
|
||||
await tx.turnDaemonLease.update({
|
||||
where: { profile },
|
||||
data: { leaseUntil: new Date(Date.now() - 60_000) },
|
||||
});
|
||||
expect((await loadProfileReadiness(tx, profile)).ok).toBe(false);
|
||||
throw rollback;
|
||||
})
|
||||
).rejects.toBe(rollback);
|
||||
});
|
||||
});
|
||||
@@ -1,5 +1,7 @@
|
||||
import { GamePrisma, type InputJsonValue } from '@sammo-ts/infra';
|
||||
import { auditPolicyHash, type PendingAuditPolicy } from './policy.js';
|
||||
import { asRecord } from '@sammo-ts/common';
|
||||
import { AUDIT_POLICY_AREAS, auditPolicyHash, type PendingAuditPolicy } from './policy.js';
|
||||
import type { InMemoryTurnWorld } from '../turn/inMemoryWorld.js';
|
||||
|
||||
export const persistAuditPolicies = async (
|
||||
tx: GamePrisma.TransactionClient,
|
||||
@@ -43,3 +45,48 @@ export const persistAuditPolicies = async (
|
||||
throw new Error('Play audit policy replay payload conflict');
|
||||
}
|
||||
};
|
||||
|
||||
/** 원장은 그대로 두고, 구버전 NPC 생성이 유실한 head projection만 복구한다. */
|
||||
export const restoreMissingAuditPolicyHeads = async (
|
||||
tx: GamePrisma.TransactionClient,
|
||||
world: InMemoryTurnWorld
|
||||
): Promise<boolean> => {
|
||||
const serverId = world.getState().meta.serverId;
|
||||
if (typeof serverId !== 'string' || !serverId.trim()) return false;
|
||||
const missing = world.listNations().flatMap((nation) => {
|
||||
const heads = asRecord(nation.meta._playAuditPolicy);
|
||||
return AUDIT_POLICY_AREAS.filter((area) => heads[area] === undefined).map(
|
||||
(area) => GamePrisma.sql`(nation_id = ${nation.id} AND area = ${area})`
|
||||
);
|
||||
});
|
||||
if (!missing.length) return false;
|
||||
// DISTINCT ON으로 각 국가/영역의 마지막 revision만 읽는다. 과거 기수는 복구하지 않는다.
|
||||
const rows = await tx.$queryRaw<
|
||||
{ id: string; nationId: number; area: string; revision: number; after: InputJsonValue }[]
|
||||
>(GamePrisma.sql`
|
||||
SELECT DISTINCT ON (nation_id, area)
|
||||
id, nation_id AS "nationId", area, revision, "after"
|
||||
FROM play_audit_policy
|
||||
WHERE server_id = ${serverId} AND (${GamePrisma.join(missing, ' OR ')})
|
||||
ORDER BY nation_id, area, revision DESC
|
||||
`);
|
||||
for (const row of rows) {
|
||||
if (row.revision < 1 || row.id !== auditPolicyHash([serverId, row.nationId, row.area, row.revision])) {
|
||||
throw new Error('Invalid persisted play audit policy head');
|
||||
}
|
||||
const nation = world.getNationById(row.nationId)!;
|
||||
const heads = nation.meta._playAuditPolicy;
|
||||
world.updateNation(nation.id, {
|
||||
meta: {
|
||||
...nation.meta,
|
||||
_playAuditPolicy: {
|
||||
...(heads && typeof heads === 'object' && !Array.isArray(heads) ? heads : {}),
|
||||
[row.area]: { id: row.id, revision: row.revision, hash: auditPolicyHash(row.after), serverId },
|
||||
},
|
||||
},
|
||||
});
|
||||
}
|
||||
// 실제 정책이 원장 after와 다르면 후속 initializeAuditPolicies가 OBSERVED_GAP을 남긴다.
|
||||
// 이 메타데이터와 새 이력은 기존 lease/fencing을 거친 startup flush에서 함께 commit된다.
|
||||
return rows.length > 0;
|
||||
};
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import { persistAuditDecisions } from '../playAudit/decisionPersistence.js';
|
||||
import { persistAuditDiplomacyEvents } from '@sammo-ts/infra';
|
||||
import { hasAuditDocumentBaseline, persistAuditDocumentBaseline } from '../playAudit/documentBaseline.js';
|
||||
import { persistAuditPolicies } from '../playAudit/policyPersistence.js';
|
||||
import { persistAuditPolicies, restoreMissingAuditPolicyHeads } from '../playAudit/policyPersistence.js';
|
||||
import { prunePreviousAuditBatch, type AuditRetentionResult } from '../playAudit/retention.js';
|
||||
import { persistAuditMonth } from '../playAudit/persistence.js';
|
||||
import { persistGeneralAccessScores, persistGeneralUpdates } from './generalBatchPersistence.js';
|
||||
@@ -78,6 +78,7 @@ import { prepareRealtimeRecovery } from './prepareRealtimeRecovery.js';
|
||||
export interface DatabaseTurnHooks {
|
||||
hooks: TurnDaemonHooks;
|
||||
flushChanges(): Promise<void>;
|
||||
restoreMissingAuditPolicyHeads(): Promise<boolean>;
|
||||
flushInitialAudit(observedAt: Date, force?: boolean): Promise<void>;
|
||||
takeCommittedReadModelChanges(): RealtimeReadModelChanges | null;
|
||||
takeCommittedReadModelChangeReceipt(): CommittedReadModelChangeReceipt | null;
|
||||
@@ -2164,6 +2165,7 @@ export const createDatabaseTurnHooks = async (
|
||||
hooks,
|
||||
flushChanges,
|
||||
flushInitialAudit,
|
||||
restoreMissingAuditPolicyHeads: () => restoreMissingAuditPolicyHeads(prisma, world),
|
||||
takeCommittedReadModelChanges: () => {
|
||||
return takeCommittedReceipt()?.changes ?? null;
|
||||
},
|
||||
|
||||
@@ -396,7 +396,8 @@ export const createRaiseInvaderHandler = (options: {
|
||||
}
|
||||
world.updateNation(nationId, {
|
||||
chiefGeneralId: ruler.id,
|
||||
meta: { ...nation.meta, gennum: npcEachCount },
|
||||
// addNation이 붙인 감사 이력 head를 오래된 생성 객체로 덮어쓰지 않는다.
|
||||
meta: { ...world.getNationById(nationId)!.meta, gennum: npcEachCount },
|
||||
});
|
||||
for (const officerLevel of [12, 11, 10, 9]) {
|
||||
options.reservedTurns.ensureNationTurns(nationId, officerLevel);
|
||||
|
||||
@@ -395,7 +395,8 @@ export const createRaiseNpcNationHandler = (options: {
|
||||
}
|
||||
world.updateNation(nationId, {
|
||||
chiefGeneralId: ruler.id,
|
||||
meta: { ...nation.meta, gennum: 1 + subordinateCandidates.length },
|
||||
// addNation이 붙인 감사 이력 head를 오래된 생성 객체로 덮어쓰지 않는다.
|
||||
meta: { ...world.getNationById(nationId)!.meta, gennum: 1 + subordinateCandidates.length },
|
||||
});
|
||||
options.reservedTurns.ensureNationTurns(nationId, 12);
|
||||
options.reservedTurns.ensureNationTurns(nationId, 11);
|
||||
|
||||
@@ -937,10 +937,11 @@ const createTurnDaemonRuntimeWithLease = async (
|
||||
await dbHooks.prepareRealtimeRecovery({ paused: await gatewayGate?.shouldPause() });
|
||||
// 복구된 clock에서 기준을 고정하고 readiness 공개 전에 원자적으로 저장한다.
|
||||
// 명령 없는 PREOPEN도 기록하며 input_event나 게임 RNG를 만들지 않는다.
|
||||
const policyHeadsRestored = await dbHooks.restoreMissingAuditPolicyHeads();
|
||||
initializeAuditPolicies(world);
|
||||
const diplomacyInitialized = initializeAuditDiplomacy(world, new Date(clock.nowMs()));
|
||||
initializeAuditCollection(world, new Date(clock.nowMs()));
|
||||
await dbHooks.flushInitialAudit(new Date(clock.nowMs()), diplomacyInitialized);
|
||||
await dbHooks.flushInitialAudit(new Date(clock.nowMs()), diplomacyInitialized || policyHeadsRestored);
|
||||
dbHooks.takeCommittedReadModelChangeReceipt();
|
||||
} catch (error) {
|
||||
await Promise.allSettled([
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { initializeAuditPolicies } from '../src/playAudit/policy.js';
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import type { City, Nation } from '@sammo-ts/logic';
|
||||
|
||||
@@ -186,6 +187,31 @@ describe('invader monthly actions', () => {
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
it('preserves new nation audit heads through ruler initialization and restart', async () => {
|
||||
const harness = buildHarness();
|
||||
const { world, environment } = harness;
|
||||
world.updateWorldMeta({ serverId: 'new-nation-audit' });
|
||||
initializeAuditPolicies(world);
|
||||
const handler = createRaiseInvaderHandler({
|
||||
getWorld: () => world,
|
||||
reservedTurns: harness.reservedTurns,
|
||||
env: buildCommandEnv(scenarioConfig),
|
||||
});
|
||||
await handler([10, 150, 100, 20], environment, event);
|
||||
const created = world.peekDirtyState().createdNations;
|
||||
expect(created.length).toBeGreaterThan(0);
|
||||
for (const nation of created) {
|
||||
expect(Object.keys(nation.meta._playAuditPolicy ?? {})).toHaveLength(4);
|
||||
}
|
||||
world.consumeDirtyState();
|
||||
initializeAuditPolicies(world);
|
||||
expect(
|
||||
world
|
||||
.peekDirtyState()
|
||||
.pendingAuditPolicies.filter((policy) => created.some((nation) => nation.id === policy.nationId))
|
||||
).toEqual([]);
|
||||
});
|
||||
|
||||
it('creates the invader nation, generals, diplomacy, follow-up events, and city state', async () => {
|
||||
const harness = buildHarness();
|
||||
const handler = createRaiseInvaderHandler({
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { initializeAuditPolicies } from '../src/playAudit/policy.js';
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import {
|
||||
parseScenarioGeneralPoolCandidate,
|
||||
@@ -228,6 +229,21 @@ describe('RaiseNPCNation monthly action', () => {
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
it('preserves new nation audit heads through ruler initialization and restart', async () => {
|
||||
const { world, handler, environment } = buildHarness();
|
||||
world.updateWorldMeta({ serverId: 'new-nation-audit' });
|
||||
initializeAuditPolicies(world);
|
||||
await handler([], environment, event);
|
||||
const created = world.peekDirtyState().createdNations;
|
||||
expect(created.length).toBeGreaterThan(0);
|
||||
for (const nation of created) {
|
||||
expect(Object.keys(nation.meta._playAuditPolicy ?? {})).toHaveLength(4);
|
||||
}
|
||||
world.consumeDirtyState();
|
||||
initializeAuditPolicies(world);
|
||||
expect(world.peekDirtyState().pendingAuditPolicies).toEqual([]);
|
||||
});
|
||||
|
||||
it('uses a U30 subordinate name/dex/special while preserving RaiseNPCNation random stats', async () => {
|
||||
const info = {
|
||||
generalName: '부장후보',
|
||||
@@ -337,6 +353,32 @@ describe('RaiseNPCNation monthly action', () => {
|
||||
"id": 2,
|
||||
"level": 2,
|
||||
"meta": {
|
||||
"_playAuditPolicy": {
|
||||
"DEFENCE": {
|
||||
"hash": "ffb1e0b6bbf1680af65800cfc1b166afbd68149217e2b039c95b4ce4db0d20aa",
|
||||
"id": "6329bea23c4db57a8c6bfb7d4c81e9f5a3e59171c883e9d22c5ba97dfb7d486c",
|
||||
"revision": 1,
|
||||
"serverId": "fixture-server",
|
||||
},
|
||||
"NPC_GENERAL_PRIORITY": {
|
||||
"hash": "0a6202f188859cb41dee0f08388e9da21babf6f4b569f7b18abf0453caaa1062",
|
||||
"id": "28bc6ff38cc5b1b250f6a738a6745b5d581badf34bbcaf191e5e8710c17263ec",
|
||||
"revision": 1,
|
||||
"serverId": "fixture-server",
|
||||
},
|
||||
"NPC_NATION_PRIORITY": {
|
||||
"hash": "0a6202f188859cb41dee0f08388e9da21babf6f4b569f7b18abf0453caaa1062",
|
||||
"id": "feefa645ba10e0fb8c2c78e0c189f9effe8a1c3d2adf4c0d866300d4123f3748",
|
||||
"revision": 1,
|
||||
"serverId": "fixture-server",
|
||||
},
|
||||
"NPC_VALUES": {
|
||||
"hash": "1906fb445a401a46f1279dbba3f3d572e9b213daf6c0f0d31cf8beeee3c77104",
|
||||
"id": "c6fe336ffe5870cc90e2abab797d050810400404f7ad51b3ceea987b21eb3baa",
|
||||
"revision": 1,
|
||||
"serverId": "fixture-server",
|
||||
},
|
||||
},
|
||||
"bill": 100,
|
||||
"can_국기변경": 1,
|
||||
"gennum": 1,
|
||||
|
||||
@@ -235,6 +235,64 @@ integration('initial audit durability before runtime readiness', () => {
|
||||
).rejects.toMatchObject({ code: 'P2002' });
|
||||
}, 30_000);
|
||||
|
||||
it('restores missing policy heads from immutable history before restart without rewriting history', async () => {
|
||||
await runtime?.close();
|
||||
runtime = undefined;
|
||||
const policies = await db.playAuditPolicy.findMany({ where: { serverId }, orderBy: { id: 'asc' } });
|
||||
const nation = await db.nation.findUniqueOrThrow({ where: { id: 91990 } });
|
||||
const { _playAuditPolicy: heads, ...meta } = asRecord(nation.meta);
|
||||
await db.nation.update({ where: { id: nation.id }, data: { meta: meta as GamePrisma.InputJsonObject } });
|
||||
runtime = await start();
|
||||
expect(
|
||||
asRecord((await db.nation.findUniqueOrThrow({ where: { id: nation.id } })).meta)._playAuditPolicy
|
||||
).toEqual(heads);
|
||||
expect(await db.playAuditPolicy.findMany({ where: { serverId }, orderBy: { id: 'asc' } })).toEqual(policies);
|
||||
expect((await db.turnDaemonLease.findUniqueOrThrow({ where: { profile } })).clockReady).toBe(true);
|
||||
await runtime.close();
|
||||
runtime = undefined;
|
||||
runtime = await start();
|
||||
expect(await db.playAuditPolicy.findMany({ where: { serverId }, orderBy: { id: 'asc' } })).toEqual(policies);
|
||||
}, 30_000);
|
||||
|
||||
it('links an observed policy gap to the recovered head and keeps it idempotent', async () => {
|
||||
await runtime?.close();
|
||||
runtime = undefined;
|
||||
const nation = await db.nation.findUniqueOrThrow({ where: { id: 91990 } });
|
||||
const meta = asRecord(nation.meta);
|
||||
const heads = asRecord(meta._playAuditPolicy);
|
||||
const previous = asRecord(heads.DEFENCE);
|
||||
const { DEFENCE: _lost, ...remainingHeads } = heads;
|
||||
await db.nation.update({
|
||||
where: { id: nation.id },
|
||||
data: {
|
||||
meta: { ...meta, scout: 1, _playAuditPolicy: remainingHeads } as GamePrisma.InputJsonObject,
|
||||
},
|
||||
});
|
||||
runtime = await start();
|
||||
const rows = await db.playAuditPolicy.findMany({
|
||||
where: { serverId, nationId: nation.id, area: 'DEFENCE' },
|
||||
orderBy: { revision: 'asc' },
|
||||
});
|
||||
expect(rows.at(-1)).toMatchObject({
|
||||
source: 'OBSERVED_GAP',
|
||||
previousId: previous.id,
|
||||
revision: Number(previous.revision) + 1,
|
||||
actor: null,
|
||||
before: null,
|
||||
after: { scout: 1 },
|
||||
});
|
||||
expect(asRecord((await db.nation.findUniqueOrThrow({ where: { id: nation.id } })).meta).scout).toBe(1);
|
||||
await runtime.close();
|
||||
runtime = undefined;
|
||||
runtime = await start();
|
||||
expect(
|
||||
await db.playAuditPolicy.findMany({
|
||||
where: { serverId, nationId: nation.id, area: 'DEFENCE' },
|
||||
orderBy: { revision: 'asc' },
|
||||
})
|
||||
).toEqual(rows);
|
||||
}, 30_000);
|
||||
|
||||
it('captures newly observed policies after durable clock recovery without replacing the initial sample', async () => {
|
||||
await runtime?.close();
|
||||
runtime = undefined;
|
||||
|
||||
Reference in New Issue
Block a user