감사 수집과 저장 장애를 게임 진행에서 분리한다

This commit is contained in:
2026-09-26 16:26:02 +00:00
parent daf7a87799
commit 6b65633324
22 changed files with 1108 additions and 550 deletions
+55 -11
View File
@@ -1,5 +1,6 @@
import { persistAuditDecisions } from '../playAudit/decisionPersistence.js';
import { persistAuditDiplomacyEvents } from '@sammo-ts/infra';
import { collectPlayAudit, markAuditGap } from '../playAudit/bestEffort.js';
import { persistAuditDiplomacyEvents, withPlayAuditSavepoint } from '@sammo-ts/infra';
import { hasAuditDocumentBaseline, persistAuditDocumentBaseline } from '../playAudit/documentBaseline.js';
import { persistAuditPolicies, restoreMissingAuditPolicyHeads } from '../playAudit/policyPersistence.js';
import { prunePreviousAuditBatch, type AuditRetentionResult } from '../playAudit/retention.js';
@@ -1236,6 +1237,7 @@ export const createDatabaseTurnHooks = async (
}
const persistedClock = await prisma.$queryRaw<
Array<{
audit_gap: unknown;
clock_phase: string;
clock_revision: bigint;
deadline_generation: bigint;
@@ -1243,7 +1245,7 @@ export const createDatabaseTurnHooks = async (
opening_reached: boolean;
}>
>(GamePrisma.sql`
SELECT clock_phase,
SELECT meta->'playAuditGap' AS audit_gap, clock_phase,
clock_revision,
deadline_generation,
clock_base_time IS NOT NULL
@@ -1259,6 +1261,21 @@ export const createDatabaseTurnHooks = async (
if (!durableClock) {
throw new Error(`world_state ${state.id} is missing during a fenced turn flush.`);
}
const durableGap = asRecord(durableClock.audit_gap);
if (durableGap.serverId && durableGap.serverId === state.meta.serverId) {
const localGap = asRecord(world.getState().meta.playAuditGap);
const gap =
localGap.serverId === durableGap.serverId
? {
...durableGap,
...localGap,
firstYear: durableGap.firstYear,
firstMonth: durableGap.firstMonth,
}
: durableGap;
world.updateWorldMeta({ playAuditGap: gap });
worldStateUpdate.meta = asJson(world.getState().meta);
}
const expectedPhase = state.clockPhase ?? (state.clockMode === 'realtime' ? 'RUNNING' : 'MANUAL');
const expectedRevision = BigInt(state.clockRevision ?? 1);
const expectedGeneration = BigInt(state.deadlineGeneration ?? 1);
@@ -1895,11 +1912,25 @@ export const createDatabaseTurnHooks = async (
data: pendingLogRows,
});
}
await persistAuditPolicies(prisma, pendingAuditPolicies, auditCommand);
await persistAuditDiplomacyEvents(prisma, pendingAuditDiplomacy);
await persistAuditDecisions(prisma, pendingAuditDecisions);
for (const snapshot of pendingAuditMonths) {
await persistAuditMonth(prisma, snapshot);
if (
pendingAuditPolicies.length ||
pendingAuditDiplomacy.length ||
pendingAuditDecisions.length ||
pendingAuditMonths.length
) {
const audit = await withPlayAuditSavepoint(prisma, async (auditDb) => {
await persistAuditPolicies(auditDb, pendingAuditPolicies, auditCommand);
await persistAuditDiplomacyEvents(auditDb, pendingAuditDiplomacy);
await persistAuditDecisions(auditDb, pendingAuditDecisions);
for (const snapshot of pendingAuditMonths) await persistAuditMonth(auditDb, snapshot);
});
if (!audit.ok) {
markAuditGap(world, 'persistence');
await prisma.worldState.update({
where: { id: state.id },
data: { meta: world.getState().meta as GamePrisma.InputJsonValue },
});
}
}
for (const snapshot of pendingYearbookSnapshots) {
await persistYearbookSnapshot(prisma, snapshot);
@@ -2094,8 +2125,8 @@ export const createDatabaseTurnHooks = async (
enqueueCommittedReceipt(committed.readModelChanges, committed.journalWrite);
};
const flushInitialAudit = async (observedAt: Date, force = false): Promise<void> => {
if (hasAuditDocumentBaseline(world)) {
if (force || world.hasPendingAuditRecords()) await flushChanges();
if (collectPlayAudit(world, 'document-boundary', () => hasAuditDocumentBaseline(world), false)) {
if (force || world.hasPendingAuditRecords() || world.getState().meta.playAuditGap) await flushChanges();
return;
}
const checkpoint = world.captureState();
@@ -2108,7 +2139,10 @@ export const createDatabaseTurnHooks = async (
await acquireGameSchemaAdvisoryXactLock(transaction, CLOCK_OPERATION_PERSISTENCE_LOCK);
await acquireGameSchemaAdvisoryXactLock(transaction, GENERAL_ACCESS_PERSISTENCE_LOCK);
await synchronizeRuntimeClockAuthorityUnderHeldLock(transaction, world);
await persistAuditDocumentBaseline(transaction, world, observedAt);
const audit = await withPlayAuditSavepoint(transaction, (auditDb) =>
persistAuditDocumentBaseline(auditDb, world, observedAt)
);
if (!audit.ok) markAuditGap(world, 'document-baseline');
return persistChanges(transaction);
}, transactionOptions);
} catch (error) {
@@ -2165,7 +2199,17 @@ export const createDatabaseTurnHooks = async (
hooks,
flushChanges,
flushInitialAudit,
restoreMissingAuditPolicyHeads: () => restoreMissingAuditPolicyHeads(prisma, world),
restoreMissingAuditPolicyHeads: async () => {
const result = await prisma.$transaction(
(tx) => withPlayAuditSavepoint(tx, (auditDb) => restoreMissingAuditPolicyHeads(auditDb, world)),
transactionOptions
);
if (!result.ok) {
markAuditGap(world, 'policy-head-recovery');
return false;
}
return result.value;
},
takeCommittedReadModelChanges: () => {
return takeCommittedReceipt()?.changes ?? null;
},
+33 -4
View File
@@ -1,3 +1,4 @@
import { collectPlayAudit } from '../playAudit/bestEffort.js';
import type { PendingAuditDecision } from '../playAudit/decision.js';
import {
recordTurnAuditDiplomacy,
@@ -1393,15 +1394,36 @@ export class InMemoryTurnWorld {
}
queueAuditDiplomacy(event: AuditDiplomacyEventDraft): void {
this.pendingAuditDiplomacy.push(structuredClone(event));
collectPlayAudit(
this,
'queueDiplomacy',
() => {
this.pendingAuditDiplomacy.push(structuredClone(event));
},
undefined
);
}
queueAuditDecision(decision: PendingAuditDecision): void {
this.pendingAuditDecisions.push(structuredClone(decision));
collectPlayAudit(
this,
'queueDecision',
() => {
this.pendingAuditDecisions.push(structuredClone(decision));
},
undefined
);
}
queueAuditPolicy(policy: PendingAuditPolicy): void {
this.pendingAuditPolicies.push(structuredClone(policy));
collectPlayAudit(
this,
'queuePolicy',
() => {
this.pendingAuditPolicies.push(structuredClone(policy));
},
undefined
);
}
hasPendingAuditRecords(): boolean {
@@ -1414,7 +1436,14 @@ export class InMemoryTurnWorld {
}
queueAuditMonth(snapshot: PendingAuditMonth): void {
this.pendingAuditMonths.push(structuredClone(snapshot));
collectPlayAudit(
this,
'queueMonth',
() => {
this.pendingAuditMonths.push(structuredClone(snapshot));
},
undefined
);
}
queueYearbookSnapshot(snapshot: PendingYearbookSnapshot): void {
+117 -93
View File
@@ -1,3 +1,4 @@
import { collectPlayAudit } from '../playAudit/bestEffort.js';
import { auditDecisionIdentity, normalizeAuditCodeVersion, type PendingAuditDecision } from '../playAudit/decision.js';
import { auditPolicyHash, AUDIT_POLICY_AREAS } from '../playAudit/policy.js';
import type { AiDecisionTraceEvent, AiExecutionAttempt, AiExecutionCheck } from './ai/generalAi/trace.js';
@@ -1085,46 +1086,61 @@ export const createReservedTurnHandler = async (options: {
const onDecisionTrace: AiDecisionTraceObserver | undefined =
collectDecisions || options.onDecisionTrace
? (event) => {
if (collectDecisions) {
if (event.kind === 'DECISION_START') {
decisionSteps.set(event.phase, []);
const heads = asRecord(currentNation?.meta._playAuditPolicy);
decisionPolicyRefs.set(
event.phase,
Object.fromEntries(
AUDIT_POLICY_AREAS.flatMap((area) => {
const head = asRecord(heads[area]);
return head.serverId === auditServerId && typeof head.id === 'string'
? [[area, head.id]]
: [];
})
)
);
}
decisionSteps
.get(event.phase)
?.push({ ...structuredClone(event), sequence: storedDecisionSequence++ });
}
if (collectDecisions && worldRef)
collectPlayAudit(
worldRef,
'decision-trace',
() => {
if (event.kind === 'DECISION_START') {
decisionSteps.set(event.phase, []);
const heads = asRecord(currentNation?.meta._playAuditPolicy);
decisionPolicyRefs.set(
event.phase,
Object.fromEntries(
AUDIT_POLICY_AREAS.flatMap((area) => {
const head = asRecord(heads[area]);
return head.serverId === auditServerId &&
typeof head.id === 'string'
? [[area, head.id]]
: [];
})
)
);
}
decisionSteps
.get(event.phase)
?.push({ ...structuredClone(event), sequence: storedDecisionSequence++ });
},
undefined
);
options.onDecisionTrace?.(event);
}
: undefined;
const recordExecution = (phase: 'general' | 'nation', attempt: AiExecutionAttempt): void => {
const steps = decisionSteps.get(phase);
const first = steps?.[0];
if (!collectDecisions || !first || !steps) return;
const { generalId, nationId, cityId, npcState, year, month, tick } = first;
steps.push({
...attempt,
sequence: storedDecisionSequence++,
phase,
generalId,
nationId,
cityId,
npcState,
year,
month,
tick,
});
if (!worldRef) return;
collectPlayAudit(
worldRef,
'decision-summary',
() => {
const steps = decisionSteps.get(phase);
const first = steps?.[0];
if (!collectDecisions || !first || !steps) return;
const { generalId, nationId, cityId, npcState, year, month, tick } = first;
steps.push({
...attempt,
sequence: storedDecisionSequence++,
phase,
generalId,
nationId,
cityId,
npcState,
year,
month,
tick,
});
},
undefined
);
};
const finishDecision = (
phase: 'general' | 'nation',
@@ -1135,64 +1151,72 @@ export const createReservedTurnHandler = async (options: {
blockedReason?: string;
}
): void => {
const steps = decisionSteps.get(phase);
const first = steps?.[0];
const last = steps?.find((step) => step.kind === 'DECISION_END');
if (
!collectDecisions ||
!auditServerId ||
!worldRef ||
!steps ||
first?.kind !== 'DECISION_START' ||
last?.kind !== 'DECISION_END'
)
return;
const tick = context.general.turnTick ?? worldRef.dateToGameTick(context.general.turnTime);
const revision = worldRef.getGameClockState().revision;
const executionId = auditDecisionIdentity(auditServerId, context.general.id, tick, revision);
const execution = steps.at(-1);
auditDecisions.push({
id: auditPolicyHash([executionId, phase]),
serverId: auditServerId,
executionId,
phase,
generalId: first.generalId,
nationId: first.nationId,
cityId: first.cityId,
npcState: first.npcState,
year: first.year,
month: first.month,
tick,
summary: {
schemaVersion: 1,
coverage: 'PROCEDURES',
executionCoverage: 'ATTEMPTS',
executionStatus:
execution?.kind === 'EXECUTION_ATTEMPT' && execution.preparation
? 'PREPARING'
: execution?.kind === 'EXECUTION_ATTEMPT' &&
execution.checks.some((check) => check.stage === 'BLOCK')
? 'BLOCKED'
: 'RESOLVED',
clockRevision: revision,
codeVersion: auditCodeVersion ?? null,
policyRefs: decisionPolicyRefs.get(phase) ?? {},
requestedAction: first.reservedAction,
selectedAction: last.action,
selectedReason: last.reason,
executedAction: outcome.actionKey,
completed: outcome.completed ?? null,
usedFallback:
outcome.usedFallback ||
steps.some(
(step) =>
step.kind === 'EXECUTION_ATTEMPT' &&
(step.usedFallback || step.alternativeAction !== null)
),
blockedReason: outcome.blockedReason ?? null,
if (!worldRef) return;
collectPlayAudit(
worldRef,
'decision-summary',
() => {
const steps = decisionSteps.get(phase);
const first = steps?.[0];
const last = steps?.find((step) => step.kind === 'DECISION_END');
if (
!collectDecisions ||
!auditServerId ||
!worldRef ||
!steps ||
first?.kind !== 'DECISION_START' ||
last?.kind !== 'DECISION_END'
)
return;
const tick = context.general.turnTick ?? worldRef.dateToGameTick(context.general.turnTime);
const revision = worldRef.getGameClockState().revision;
const executionId = auditDecisionIdentity(auditServerId, context.general.id, tick, revision);
const execution = steps.at(-1);
auditDecisions.push({
id: auditPolicyHash([executionId, phase]),
serverId: auditServerId,
executionId,
phase,
generalId: first.generalId,
nationId: first.nationId,
cityId: first.cityId,
npcState: first.npcState,
year: first.year,
month: first.month,
tick,
summary: {
schemaVersion: 1,
coverage: 'PROCEDURES',
executionCoverage: 'ATTEMPTS',
executionStatus:
execution?.kind === 'EXECUTION_ATTEMPT' && execution.preparation
? 'PREPARING'
: execution?.kind === 'EXECUTION_ATTEMPT' &&
execution.checks.some((check) => check.stage === 'BLOCK')
? 'BLOCKED'
: 'RESOLVED',
clockRevision: revision,
codeVersion: auditCodeVersion ?? null,
policyRefs: decisionPolicyRefs.get(phase) ?? {},
requestedAction: first.reservedAction,
selectedAction: last.action,
selectedReason: last.reason,
executedAction: outcome.actionKey,
completed: outcome.completed ?? null,
usedFallback:
outcome.usedFallback ||
steps.some(
(step) =>
step.kind === 'EXECUTION_ATTEMPT' &&
(step.usedFallback || step.alternativeAction !== null)
),
blockedReason: outcome.blockedReason ?? null,
},
steps,
});
},
steps,
});
undefined
);
};
// Ref는 장수와 첫 커맨드를 만들 때 getNationStaticInfo 캐시를 채운다.