diff --git a/app/game-engine/package.json b/app/game-engine/package.json index bf4694dc..f2ed4b63 100644 --- a/app/game-engine/package.json +++ b/app/game-engine/package.json @@ -107,6 +107,7 @@ "lint:fix": "eslint . --fix", "profile:npc-unification-memory": "node scripts/profile-npc-unification-memory.mjs", "profile:npc-unification-timing": "node scripts/profile-npc-unification-timing.mjs", + "profile:npc-lifecycle-memory": "node scripts/profile-npc-lifecycle-memory.mjs", "profile:npc-capacity-1200": "node scripts/profile-npc-capacity-1200.mjs", "test": "vitest run --config vitest.config.ts", "typecheck": "pnpm -w tsc7 -b app/game-engine/tsconfig.json" diff --git a/app/game-engine/scripts/profile-npc-lifecycle-memory.mjs b/app/game-engine/scripts/profile-npc-lifecycle-memory.mjs new file mode 100644 index 00000000..12414e00 --- /dev/null +++ b/app/game-engine/scripts/profile-npc-lifecycle-memory.mjs @@ -0,0 +1,120 @@ +import { spawn } from 'node:child_process'; +import { mkdtempSync, mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const packageRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..'); +const vitestPath = path.join(packageRoot, 'node_modules', 'vitest', 'vitest.mjs'); +const defaultRuns = [ + { scenario: 'steady-state', pruneDeletedQueues: true }, + { scenario: 'growth', pruneDeletedQueues: true }, + { scenario: 'death-drain', pruneDeletedQueues: false }, + { scenario: 'death-drain', pruneDeletedQueues: true }, + { scenario: 'balanced-churn', pruneDeletedQueues: false }, + { scenario: 'balanced-churn', pruneDeletedQueues: true }, + { scenario: 'rollback-churn', pruneDeletedQueues: true }, +]; + +const parseRuns = () => { + const raw = process.env.NPC_LIFECYCLE_MEMORY_SCENARIOS; + if (!raw) return defaultRuns; + return raw.split(',').map((entry) => { + const [scenario, variant = 'prune'] = entry.trim().split('@'); + if (!scenario) throw new Error(`invalid scenario entry: ${entry}`); + if (variant !== 'prune' && variant !== 'retain') { + throw new Error(`scenario variant must be prune or retain: ${entry}`); + } + return { scenario, pruneDeletedQueues: variant === 'prune' }; + }); +}; + +const runChild = (run, reportPath, repetition) => + new Promise((resolve, reject) => { + const child = spawn( + process.execPath, + [ + '--expose-gc', + vitestPath, + 'run', + '--config', + 'vitest.config.ts', + '--pool=threads', + '--maxWorkers=1', + 'test/npcLifecycleMemoryProfile.test.ts', + ], + { + cwd: packageRoot, + env: { + ...process.env, + NPC_LIFECYCLE_MEMORY_PROFILE: '1', + NPC_LIFECYCLE_MEMORY_SCENARIO: run.scenario, + NPC_LIFECYCLE_MEMORY_PRUNE_DELETED: run.pruneDeletedQueues ? '1' : '0', + NPC_LIFECYCLE_MEMORY_CHILD_REPORT_PATH: reportPath, + NPC_LIFECYCLE_MEMORY_REPETITION: String(repetition), + }, + stdio: 'inherit', + } + ); + child.once('error', reject); + child.once('exit', (code, signal) => { + if (signal) { + reject(new Error(`${run.scenario} terminated by ${signal}`)); + } else if (code !== 0) { + reject(new Error(`${run.scenario} exited with code ${code}`)); + } else { + resolve(); + } + }); + }); + +const runs = parseRuns(); +const repetitions = Number(process.env.NPC_LIFECYCLE_MEMORY_REPETITIONS ?? 1); +if (!Number.isSafeInteger(repetitions) || repetitions <= 0) { + throw new Error('NPC_LIFECYCLE_MEMORY_REPETITIONS must be a positive integer.'); +} +const reportPath = path.resolve( + packageRoot, + process.env.NPC_LIFECYCLE_MEMORY_REPORT_PATH ?? 'test-results/npc-lifecycle-memory.json' +); +const temporaryDirectory = mkdtempSync(path.join(tmpdir(), 'sammo-npc-lifecycle-memory-')); +const reports = []; +try { + for (let repetition = 1; repetition <= repetitions; repetition += 1) { + for (const [index, run] of runs.entries()) { + const childReportPath = path.join( + temporaryDirectory, + `${String(repetition).padStart(2, '0')}-${String(index).padStart(2, '0')}.json` + ); + await runChild(run, childReportPath, repetition); + reports.push({ + repetition, + ...JSON.parse(readFileSync(childReportPath, 'utf8')), + }); + } + } + mkdirSync(path.dirname(reportPath), { recursive: true }); + writeFileSync( + reportPath, + `${JSON.stringify( + { + schemaVersion: 1, + generatedAt: new Date().toISOString(), + inputs: { + cycles: Number(process.env.NPC_LIFECYCLE_MEMORY_CYCLES ?? 80), + batchSize: Number(process.env.NPC_LIFECYCLE_MEMORY_BATCH_SIZE ?? 100), + sampleEvery: Number(process.env.NPC_LIFECYCLE_MEMORY_SAMPLE_EVERY ?? 5), + baseGenerals: Number(process.env.NPC_LIFECYCLE_MEMORY_BASE_GENERALS ?? 1_200), + repetitions, + }, + reports, + }, + null, + 2 + )}\n`, + 'utf8' + ); + console.log(`[npc-lifecycle-memory] wrote ${reports.length} scenario reports to ${reportPath}`); +} finally { + rmSync(temporaryDirectory, { recursive: true, force: true }); +} diff --git a/app/game-engine/src/turn/cli.ts b/app/game-engine/src/turn/cli.ts index 9d1128a4..a00b68ca 100644 --- a/app/game-engine/src/turn/cli.ts +++ b/app/game-engine/src/turn/cli.ts @@ -96,10 +96,17 @@ export const runTurnDaemonCli = async (options: TurnDaemonCliOptions = {}): Prom intervalMs: memoryReportIntervalMs, getContext: () => { const state = runtime.world.getState(); + const queueCounts = runtime.reservedTurns?.getQueueCounts(); return { year: state.currentYear, month: state.currentMonth, ...runtime.world.getEntityCounts(), + ...(queueCounts + ? { + generalTurnQueues: queueCounts.generalQueues, + nationTurnQueues: queueCounts.nationQueues, + } + : {}), lifecycleState: runtime.lifecycle.getStatus().state, }; }, diff --git a/app/game-engine/src/turn/databaseHooks.ts b/app/game-engine/src/turn/databaseHooks.ts index 9480572f..4ad11556 100644 --- a/app/game-engine/src/turn/databaseHooks.ts +++ b/app/game-engine/src/turn/databaseHooks.ts @@ -1787,6 +1787,7 @@ export const createDatabaseTurnHooks = async ( world.acknowledgeDirtyState(changes); if (options?.reservedTurns && reservedTurnChanges) { options.reservedTurns.acknowledgeDirtyState(reservedTurnChanges); + options.reservedTurns.pruneDeletedEntityQueues(deletedGenerals, deletedNations); } applyRealtimeReadModelBaseline(readModelBaseline, changes); worldReadModelBaseline = persisted.worldReadModelSignature; diff --git a/app/game-engine/src/turn/reservedTurnStore.ts b/app/game-engine/src/turn/reservedTurnStore.ts index 0adfbaa6..b02f0fec 100644 --- a/app/game-engine/src/turn/reservedTurnStore.ts +++ b/app/game-engine/src/turn/reservedTurnStore.ts @@ -105,6 +105,17 @@ export interface InMemoryReservedTurnStateSnapshot { leasedNationKeys: string[]; } +export interface ReservedTurnQueueCounts { + generalQueues: number; + nationQueues: number; + dirtyGeneralQueues: number; + dirtyNationQueues: number; + pendingGeneralInitializations: number; + pendingNationInitializations: number; + leasedGeneralQueues: number; + leasedNationQueues: number; +} + export class ReservedTurnLeaseConflictError extends Error { constructor(readonly queueKey: string) { super(`Reserved turn queue lease conflict: ${queueKey}.`); @@ -189,6 +200,64 @@ export class InMemoryReservedTurnStore { return this.captureState(); } + getQueueCounts(): ReservedTurnQueueCounts { + return { + generalQueues: this.generalTurns.size, + nationQueues: this.nationTurns.size, + dirtyGeneralQueues: this.dirtyGeneralIds.size, + dirtyNationQueues: this.dirtyNationKeys.size, + pendingGeneralInitializations: this.pendingGeneralInitializationIds.size, + pendingNationInitializations: this.pendingNationInitializationKeys.size, + leasedGeneralQueues: this.leasedGeneralIds.size, + leasedNationQueues: this.leasedNationKeys.size, + }; + } + + /** + * Drops queues whose owning rows were deleted by a successful world flush. + * The daemon calls this inside EngineStateManager.transaction(), so a later + * failure still restores these maps and journals from the transaction savepoint. + */ + pruneDeletedEntityQueues( + generalIds: readonly number[], + nationIds: readonly number[] + ): { generalQueues: number; nationQueues: number } { + let generalQueues = 0; + for (const generalId of new Set(generalIds)) { + if (this.generalTurns.delete(generalId)) { + generalQueues += 1; + } + this.dirtyGeneralIds.delete(generalId); + this.pendingGeneralInitializationIds.delete(generalId); + this.leasedGeneralIds.delete(generalId); + } + + const deletedNations = new Set(nationIds); + let nationQueues = 0; + const pruneNationKeys = (keys: Iterable, remove: (key: string) => boolean | void): void => { + for (const key of keys) { + const nationId = Number(key.split(':', 1)[0]); + if (deletedNations.has(nationId) && remove(key) !== false) { + nationQueues += 1; + } + } + }; + pruneNationKeys(Array.from(this.nationTurns.keys()), (key) => this.nationTurns.delete(key)); + for (const keys of [ + this.dirtyNationKeys, + this.pendingNationInitializationKeys, + this.leasedNationKeys, + ]) { + for (const key of Array.from(keys)) { + const nationId = Number(key.split(':', 1)[0]); + if (deletedNations.has(nationId)) { + keys.delete(key); + } + } + } + return { generalQueues, nationQueues }; + } + inspectGeneralTurnActivity(): Array<[number, boolean]> { return Array.from(this.generalTurns, ([generalId, turns]) => [ generalId, diff --git a/app/game-engine/src/turn/turnDaemon.ts b/app/game-engine/src/turn/turnDaemon.ts index c6d3cc69..a05e43e3 100644 --- a/app/game-engine/src/turn/turnDaemon.ts +++ b/app/game-engine/src/turn/turnDaemon.ts @@ -41,7 +41,7 @@ import { } from './monthlyNationStatsHandler.js'; import { createFrontStateHandler } from './frontStateHandler.js'; import { createReservedTurnHandler } from './reservedTurnHandler.js'; -import { createReservedTurnStore } from './reservedTurnStore.js'; +import { createReservedTurnStore, type InMemoryReservedTurnStore } from './reservedTurnStore.js'; import { createTurnDaemonCommandHandler } from './worldCommandHandler.js'; import { loadTurnCommandProfile } from './turnCommandProfile.js'; import { loadTurnWorldFromDatabase } from './worldLoader.js'; @@ -132,6 +132,7 @@ export interface TurnDaemonRuntime { stateStore: InMemoryTurnStateStore; stateManager: EngineStateManager; processor: InMemoryTurnProcessor; + reservedTurns: InMemoryReservedTurnStore | null; hooks?: TurnDaemonHooks; close(): Promise; } @@ -1012,6 +1013,7 @@ const createTurnDaemonRuntimeWithLease = async ( stateStore, stateManager, processor, + reservedTurns: reservedTurnStoreHandle?.store ?? null, hooks, close, }; diff --git a/app/game-engine/src/turn/turnDaemonMemoryReporter.ts b/app/game-engine/src/turn/turnDaemonMemoryReporter.ts index 0662ee1f..e41b8d35 100644 --- a/app/game-engine/src/turn/turnDaemonMemoryReporter.ts +++ b/app/game-engine/src/turn/turnDaemonMemoryReporter.ts @@ -8,6 +8,8 @@ export interface TurnDaemonMemoryContext { nations: number; troops: number; events: number; + generalTurnQueues?: number; + nationTurnQueues?: number; lifecycleState: string; } @@ -48,6 +50,10 @@ export const buildTurnDaemonMemoryReport = ( `nations=${context.nations}`, `troops=${context.troops}`, `events=${context.events}`, + ...(context.generalTurnQueues === undefined + ? [] + : [`generalTurnQueues=${context.generalTurnQueues}`]), + ...(context.nationTurnQueues === undefined ? [] : [`nationTurnQueues=${context.nationTurnQueues}`]), `lifecycle=${context.lifecycleState}`, ].join(' '); return { message, warning: heapRatio >= HEAP_WARNING_RATIO }; diff --git a/app/game-engine/test/generalTurnLifecyclePersistence.integration.test.ts b/app/game-engine/test/generalTurnLifecyclePersistence.integration.test.ts index 8be37696..581f8db7 100644 --- a/app/game-engine/test/generalTurnLifecyclePersistence.integration.test.ts +++ b/app/game-engine/test/generalTurnLifecyclePersistence.integration.test.ts @@ -474,6 +474,12 @@ integration('general turn lifecycle persistence', () => { events: [], initialEvents: [], }; + const reservedTurns = new InMemoryReservedTurnStore(db, { + maxGeneralTurns: 3, + maxNationTurns: 3, + }); + reservedTurns.ensureGeneralTurns(general.id); + expect(reservedTurns.getQueueCounts().generalQueues).toBe(1); const world = new InMemoryTurnWorld(state, snapshot, { schedule: { entries: [{ startMinute: 0, tickMinutes: 10 }] }, generalTurnHandler: { @@ -517,7 +523,7 @@ integration('general turn lifecycle persistence', () => { }); world.executeGeneralTurn(world.getGeneralById(general.id)!); - const hooks = await createDatabaseTurnHooks(databaseUrl!, world); + const hooks = await createDatabaseTurnHooks(databaseUrl!, world, { reservedTurns }); try { await hooks.hooks.flushChanges?.({ lastTurnTime: state.lastTurnTime.toISOString(), @@ -529,6 +535,10 @@ integration('general turn lifecycle persistence', () => { } finally { await hooks.close(); } + expect(reservedTurns.getQueueCounts()).toMatchObject({ + generalQueues: 0, + pendingGeneralInitializations: 0, + }); const archived = await db.oldGeneral.findUniqueOrThrow({ where: { by_no: { serverId, generalNo: general.id } }, diff --git a/app/game-engine/test/helpers/npcLifecycleMemoryProfiler.ts b/app/game-engine/test/helpers/npcLifecycleMemoryProfiler.ts new file mode 100644 index 00000000..e693e819 --- /dev/null +++ b/app/game-engine/test/helpers/npcLifecycleMemoryProfiler.ts @@ -0,0 +1,210 @@ +import { performance } from 'node:perf_hooks'; +import { serialize } from 'node:v8'; + +import type { InMemoryTurnWorld } from '../../src/turn/inMemoryWorld.js'; +import type { + InMemoryReservedTurnStore, + ReservedTurnQueueCounts, +} from '../../src/turn/reservedTurnStore.js'; + +export type NpcLifecycleMemoryScenario = + | 'steady-state' + | 'growth' + | 'death-drain' + | 'balanced-churn' + | 'rollback-churn'; + +export interface ProcessMemorySnapshot { + rssBytes: number; + heapTotalBytes: number; + heapUsedBytes: number; + externalBytes: number; + arrayBuffersBytes: number; +} + +export interface NpcLifecycleMemorySample { + cycle: number; + phase: 'initial' | 'in-transaction' | 'post-flush'; + elapsedMs: number; + liveGeneralCount: number; + queueCounts: ReservedTurnQueueCounts; + process: ProcessMemorySnapshot; + pending?: { + createdGenerals: number; + deletedGenerals: number; + lifecycleEvents: number; + reservedGeneralQueues: number; + }; + snapshot?: { + worldBytes: number; + reservedTurnBytes: number; + totalBytes: number; + cloneAndSerializeMs: number; + heapUsedAfterReleaseBytes: number; + }; +} + +export const readProcessMemory = (): ProcessMemorySnapshot => { + const usage = process.memoryUsage(); + return { + rssBytes: usage.rss, + heapTotalBytes: usage.heapTotal, + heapUsedBytes: usage.heapUsed, + externalBytes: usage.external, + arrayBuffersBytes: usage.arrayBuffers, + }; +}; + +export const linearRegressionSlope = (points: ReadonlyArray<{ x: number; y: number }>): number => { + if (points.length < 2) { + return 0; + } + const meanX = points.reduce((sum, point) => sum + point.x, 0) / points.length; + const meanY = points.reduce((sum, point) => sum + point.y, 0) / points.length; + let numerator = 0; + let denominator = 0; + for (const point of points) { + const xDelta = point.x - meanX; + numerator += xDelta * (point.y - meanY); + denominator += xDelta * xDelta; + } + return denominator === 0 ? 0 : numerator / denominator; +}; + +export const captureLifecycleMemorySample = (input: { + world: InMemoryTurnWorld; + reservedTurns: InMemoryReservedTurnStore; + startedAtMs: number; + cycle: number; + phase: NpcLifecycleMemorySample['phase']; + includePending: boolean; + includeSnapshot: boolean; +}): NpcLifecycleMemorySample => { + globalThis.gc?.(); + const processSnapshot = readProcessMemory(); + const pending = input.includePending + ? (() => { + const worldChanges = input.world.peekDirtyState(); + const reservedChanges = input.reservedTurns.peekDirtyState(); + return { + createdGenerals: worldChanges.createdGenerals.length, + deletedGenerals: worldChanges.deletedGenerals.length, + lifecycleEvents: worldChanges.lifecycleEvents.length, + reservedGeneralQueues: reservedChanges.generalIds.length, + }; + })() + : undefined; + const sample: NpcLifecycleMemorySample = { + cycle: input.cycle, + phase: input.phase, + elapsedMs: performance.now() - input.startedAtMs, + liveGeneralCount: input.world.getEntityCounts().generals, + queueCounts: input.reservedTurns.getQueueCounts(), + process: processSnapshot, + ...(pending ? { pending } : {}), + }; + if (input.includeSnapshot) { + const snapshotMetrics = (() => { + const snapshotStartedAt = performance.now(); + const worldSnapshot = input.world.captureState(); + const reservedSnapshot = input.reservedTurns.captureTransactionState(); + const worldBytes = serialize(worldSnapshot).byteLength; + const reservedTurnBytes = serialize(reservedSnapshot).byteLength; + return { + worldBytes, + reservedTurnBytes, + cloneAndSerializeMs: performance.now() - snapshotStartedAt, + }; + })(); + globalThis.gc?.(); + sample.snapshot = { + ...snapshotMetrics, + totalBytes: snapshotMetrics.worldBytes + snapshotMetrics.reservedTurnBytes, + heapUsedAfterReleaseBytes: readProcessMemory().heapUsedBytes, + }; + } + return sample; +}; + +const maxValue = (values: readonly number[]): number => Math.max(0, ...values); + +export const buildNpcLifecycleMemoryReport = (input: { + scenario: NpcLifecycleMemoryScenario; + pruneDeletedQueues: boolean; + initialGeneralCount: number; + cycles: number; + batchSize: number; + sampleEvery: number; + createdTotal: number; + deletedTotal: number; + rolledBackCycles: number; + startedAtMs: number; + samples: NpcLifecycleMemorySample[]; +}) => { + const retained = input.samples.filter( + (sample) => sample.phase === 'initial' || sample.phase === 'post-flush' + ); + const warmSampleIndex = Math.floor(retained.length / 3); + const trendSamples = retained.slice(warmSampleIndex); + const first = retained[0]; + const final = retained.at(-1); + const heapSlope = linearRegressionSlope( + trendSamples.map((sample) => ({ x: sample.cycle, y: sample.process.heapUsedBytes })) + ); + const snapshotSlope = linearRegressionSlope( + trendSamples.flatMap((sample) => + sample.snapshot ? [{ x: sample.cycle, y: sample.snapshot.totalBytes }] : [] + ) + ); + const queueSlope = linearRegressionSlope( + trendSamples.map((sample) => ({ x: sample.cycle, y: sample.queueCounts.generalQueues })) + ); + const lifecycleOperations = input.createdTotal + input.deletedTotal; + + return { + schemaVersion: 1, + runtime: { + node: process.version, + platform: process.platform, + arch: process.arch, + explicitGc: typeof globalThis.gc === 'function', + }, + scenario: { + name: input.scenario, + pruneDeletedQueues: input.pruneDeletedQueues, + initialGeneralCount: input.initialGeneralCount, + cycles: input.cycles, + batchSize: input.batchSize, + sampleEvery: input.sampleEvery, + }, + result: { + createdTotal: input.createdTotal, + deletedTotal: input.deletedTotal, + rolledBackCycles: input.rolledBackCycles, + finalGeneralCount: final?.liveGeneralCount ?? 0, + finalGeneralQueueCount: final?.queueCounts.generalQueues ?? 0, + deadQueueRetentionCount: + (final?.queueCounts.generalQueues ?? 0) - (final?.liveGeneralCount ?? 0), + wallDurationMs: performance.now() - input.startedAtMs, + }, + memory: { + retainedHeapStartBytes: first?.process.heapUsedBytes ?? 0, + retainedHeapFinalBytes: final?.process.heapUsedBytes ?? 0, + retainedHeapDeltaBytes: + (final?.process.heapUsedBytes ?? 0) - (first?.process.heapUsedBytes ?? 0), + retainedHeapSlopeBytesPerCycle: heapSlope, + retainedHeapSlopeBytesPerLifecycleOperation: + lifecycleOperations === 0 ? 0 : (heapSlope * input.cycles) / lifecycleOperations, + retainedSnapshotStartBytes: first?.snapshot?.totalBytes ?? 0, + retainedSnapshotFinalBytes: final?.snapshot?.totalBytes ?? 0, + retainedSnapshotDeltaBytes: + (final?.snapshot?.totalBytes ?? 0) - (first?.snapshot?.totalBytes ?? 0), + retainedSnapshotSlopeBytesPerCycle: snapshotSlope, + generalQueueSlopePerCycle: queueSlope, + maxObservedHeapUsedBytes: maxValue(input.samples.map((sample) => sample.process.heapUsedBytes)), + maxObservedRssBytes: maxValue(input.samples.map((sample) => sample.process.rssBytes)), + processResourceMaxRssBytes: process.resourceUsage().maxRSS * 1024, + }, + samples: input.samples, + }; +}; diff --git a/app/game-engine/test/npcLifecycleMemoryProfile.test.ts b/app/game-engine/test/npcLifecycleMemoryProfile.test.ts new file mode 100644 index 00000000..36399831 --- /dev/null +++ b/app/game-engine/test/npcLifecycleMemoryProfile.test.ts @@ -0,0 +1,388 @@ +import { mkdirSync, writeFileSync } from 'node:fs'; +import { dirname, resolve } from 'node:path'; +import { performance } from 'node:perf_hooks'; + +import { buildScenarioBootstrap, type TurnSchedule } from '@sammo-ts/logic'; +import { describe, expect, it } from 'vitest'; + +import { loadMapDefinitionByName } from '../src/scenario/mapLoader.js'; +import { loadScenarioDefinitionById } from '../src/scenario/scenarioLoader.js'; +import { loadUnitSetDefinitionByName } from '../src/scenario/unitSetLoader.js'; +import { EngineStateManager } from '../src/turn/engineStateManager.js'; +import { InMemoryTurnWorld } from '../src/turn/inMemoryWorld.js'; +import { InMemoryReservedTurnStore } from '../src/turn/reservedTurnStore.js'; +import type { TurnGeneral, TurnWorldSnapshot, TurnWorldState } from '../src/turn/types.js'; +import { + buildNpcLifecycleMemoryReport, + captureLifecycleMemorySample, + type NpcLifecycleMemorySample, + type NpcLifecycleMemoryScenario, +} from './helpers/npcLifecycleMemoryProfiler.js'; + +const profileEnabled = process.env.NPC_LIFECYCLE_MEMORY_PROFILE === '1'; +const profileDescribe = describe.runIf(profileEnabled); +const SCENARIO_ID = 2601; +const HIDDEN_SEED = 'scenario-2601-npc-lifecycle-memory-v1'; +const ROLLBACK_SENTINEL = new Error('npc-lifecycle-memory-rollback'); +const VALID_SCENARIOS = new Set([ + 'steady-state', + 'growth', + 'death-drain', + 'balanced-churn', + 'rollback-churn', +]); + +const readPositiveInteger = (name: string, fallback: number): number => { + const raw = process.env[name]; + if (raw === undefined) { + return fallback; + } + const value = Number(raw); + if (!Number.isSafeInteger(value) || value <= 0) { + throw new Error(`${name} must be a positive integer: ${raw}`); + } + return value; +}; + +const createGameDate = (year: number, month: number): Date => { + const date = new Date(0); + date.setUTCFullYear(year, month - 1, 1); + date.setUTCHours(0, 0, 0, 0); + return date; +}; + +const buildTurnGeneral = ( + domainGeneral: ReturnType['snapshot']['generals'][number], + seedGeneral: ReturnType['seed']['generals'][number], + startTime: Date, + startYear: number, + startMonth: number +): TurnGeneral => { + const deathMonthRaw = seedGeneral.meta.deathMonth; + const deathMonth = + typeof deathMonthRaw === 'number' && Number.isInteger(deathMonthRaw) ? deathMonthRaw : startMonth; + const killturn = Math.max(0, (seedGeneral.deathYear - startYear) * 12 + deathMonth - startMonth); + return { + ...domainGeneral, + userId: null, + bornYear: seedGeneral.birthYear, + deadYear: seedGeneral.deathYear, + affinity: seedGeneral.affinity, + picture: seedGeneral.picture === null ? null : String(seedGeneral.picture), + startAge: 20, + turnTime: new Date(startTime), + recentWarTime: null, + lastTurn: { command: '휴식' }, + penalty: {}, + inheritancePoints: {}, + meta: { + ...domainGeneral.meta, + ...seedGeneral.meta, + killturn, + npcType: seedGeneral.npcType, + crewTypeId: seedGeneral.crewTypeId, + }, + }; +}; + +const cloneNpcGeneral = (source: TurnGeneral, id: number): TurnGeneral => { + const cloned = structuredClone(source); + return { + ...cloned, + id, + name: `${source.name}#M${id}`, + userId: null, + npcState: Math.max(2, source.npcState), + nationId: 0, + cityId: 0, + troopId: 0, + officerLevel: 0, + meta: { + ...cloned.meta, + lifecycleMemoryFixture: true, + }, + }; +}; + +const createProfileWorld = async (initialGeneralCount: number) => { + const scenario = await loadScenarioDefinitionById(SCENARIO_ID); + const map = await loadMapDefinitionByName(scenario.config.environment.mapName); + const unitSet = await loadUnitSetDefinitionByName(scenario.config.environment.unitSet); + const startYear = scenario.startYear ?? 180; + const startMonth = 1; + const startTime = createGameDate(startYear, startMonth); + const bootstrap = buildScenarioBootstrap({ + scenario, + map, + unitSet, + options: { + hiddenSeed: HIDDEN_SEED, + initialYear: startYear, + initialMonth: startMonth, + turnTermMinutes: 10, + includeNeutralNationInSeed: true, + }, + }); + if (bootstrap.warnings.length > 0) { + throw new Error(`scenario bootstrap warnings: ${bootstrap.warnings.join(', ')}`); + } + const domainGeneralById = new Map(bootstrap.snapshot.generals.map((general) => [general.id, general])); + const scenarioGenerals = bootstrap.seed.generals.map((seedGeneral) => { + const domainGeneral = domainGeneralById.get(seedGeneral.id); + if (!domainGeneral) { + throw new Error(`missing scenario general: ${seedGeneral.id}`); + } + return buildTurnGeneral(domainGeneral, seedGeneral, startTime, startYear, startMonth); + }); + const generals = Array.from({ length: initialGeneralCount }, (_, index) => + cloneNpcGeneral(scenarioGenerals[index % scenarioGenerals.length]!, index + 1) + ); + const snapshot: TurnWorldSnapshot = { + scenarioConfig: bootstrap.snapshot.scenarioConfig, + scenarioMeta: bootstrap.snapshot.scenarioMeta, + worldConfig: { + fiction: scenario.fiction, + npcMode: 2, + turnTermMinutes: 10, + tournamentTrig: false, + }, + map, + unitSet, + generals, + cities: bootstrap.snapshot.cities, + nations: bootstrap.snapshot.nations, + troops: bootstrap.snapshot.troops, + diplomacy: bootstrap.snapshot.diplomacy.map((entry) => ({ + fromNationId: entry.fromNationId, + toNationId: entry.toNationId, + state: entry.state, + term: entry.durationMonths, + dead: 0, + meta: {}, + })), + events: [], + initialEvents: [], + }; + const state: TurnWorldState = { + id: 1, + currentYear: startYear, + currentMonth: startMonth, + tickSeconds: 600, + lastTurnTime: startTime, + clockBaseTime: startTime, + clockTick: 0, + clockMode: 'manual', + clockWallAnchor: startTime, + lastTurnTick: 0, + meta: { + scenarioId: SCENARIO_ID, + hiddenSeed: HIDDEN_SEED, + killturn: 480, + lastGeneralId: initialGeneralCount, + serverId: 'npc-lifecycle-memory-profile', + }, + }; + const schedule: TurnSchedule = { entries: [{ startMinute: 0, tickMinutes: 10 }] }; + const world = new InMemoryTurnWorld(state, snapshot, { schedule }); + const reservedTurns = new InMemoryReservedTurnStore({} as never, { + maxGeneralTurns: 30, + maxNationTurns: 12, + leaseOwner: 'npc-lifecycle-memory-profile', + }); + for (const general of generals) { + reservedTurns.getGeneralTurns(general.id); + } + return { world, reservedTurns, templateGenerals: scenarioGenerals }; +}; + +profileDescribe('NPC 생성·사망 장기 구동 메모리 프로파일', () => { + it('격리 시나리오의 GC 안정 heap, rollback snapshot과 예약 큐 보유량을 기록한다', async () => { + expect(typeof globalThis.gc).toBe('function'); + const rawScenario = process.env.NPC_LIFECYCLE_MEMORY_SCENARIO ?? 'balanced-churn'; + if (!VALID_SCENARIOS.has(rawScenario as NpcLifecycleMemoryScenario)) { + throw new Error(`unknown NPC lifecycle memory scenario: ${rawScenario}`); + } + const scenario = rawScenario as NpcLifecycleMemoryScenario; + const cycles = readPositiveInteger('NPC_LIFECYCLE_MEMORY_CYCLES', 80); + const batchSize = readPositiveInteger('NPC_LIFECYCLE_MEMORY_BATCH_SIZE', 100); + const sampleEvery = readPositiveInteger('NPC_LIFECYCLE_MEMORY_SAMPLE_EVERY', 5); + const baseGeneralCount = readPositiveInteger('NPC_LIFECYCLE_MEMORY_BASE_GENERALS', 1_200); + const pruneDeletedQueues = process.env.NPC_LIFECYCLE_MEMORY_PRUNE_DELETED === '1'; + const initialGeneralCount = + scenario === 'death-drain' ? baseGeneralCount + cycles * batchSize : baseGeneralCount; + const { world, reservedTurns, templateGenerals } = await createProfileWorld(initialGeneralCount); + const stateManager = new EngineStateManager(); + stateManager.register('world', { + capture: () => world.captureState(), + restore: (snapshot) => world.restoreState(snapshot), + }); + stateManager.register('reservedTurns', { + capture: () => reservedTurns.captureTransactionState(), + restore: (snapshot) => reservedTurns.restoreState(snapshot), + }); + + const startedAtMs = performance.now(); + const samples: NpcLifecycleMemorySample[] = [ + captureLifecycleMemorySample({ + world, + reservedTurns, + startedAtMs, + cycle: 0, + phase: 'initial', + includePending: false, + includeSnapshot: true, + }), + ]; + const activeGeneralIds = world + .listGenerals() + .map((general) => general.id) + .sort((left, right) => left - right); + let nextGeneralId = Math.max(...activeGeneralIds) + 1; + let createdTotal = 0; + let deletedTotal = 0; + let rolledBackCycles = 0; + + const addGenerals = (count: number): void => { + for (let index = 0; index < count; index += 1) { + const generalId = nextGeneralId++; + const source = templateGenerals[(generalId - 1) % templateGenerals.length]!; + if (!world.addGeneral(cloneNpcGeneral(source, generalId))) { + throw new Error(`failed to add profile general ${generalId}`); + } + reservedTurns.ensureGeneralTurns(generalId); + activeGeneralIds.push(generalId); + createdTotal += 1; + } + }; + const deleteGenerals = (count: number): void => { + const targetIds = activeGeneralIds.splice(0, count); + for (const generalId of targetIds) { + if (!world.deleteGeneralWithLifecycle(generalId, 180, 1)) { + throw new Error(`failed to delete profile general ${generalId}`); + } + deletedTotal += 1; + } + }; + + for (let cycle = 1; cycle <= cycles; cycle += 1) { + const sampledCycle = cycle % sampleEvery === 0 || cycle === cycles; + try { + await stateManager.transaction(() => { + if (scenario === 'steady-state') { + for (let index = 0; index < batchSize; index += 1) { + const generalId = activeGeneralIds[((cycle - 1) * batchSize + index) % activeGeneralIds.length]!; + const current = world.getGeneralById(generalId); + if (!current) { + throw new Error(`missing steady-state general ${generalId}`); + } + world.updateGeneral(generalId, { experience: current.experience + 1 }); + reservedTurns.shiftGeneralTurns(generalId, -1); + } + } else if (scenario === 'growth') { + addGenerals(batchSize); + } else if (scenario === 'death-drain') { + deleteGenerals(batchSize); + } else if (scenario === 'balanced-churn') { + deleteGenerals(batchSize); + addGenerals(batchSize); + } else { + addGenerals(batchSize); + deleteGenerals(batchSize); + } + + if (sampledCycle) { + samples.push( + captureLifecycleMemorySample({ + world, + reservedTurns, + startedAtMs, + cycle, + phase: 'in-transaction', + includePending: true, + includeSnapshot: false, + }) + ); + } + if (scenario === 'rollback-churn') { + throw ROLLBACK_SENTINEL; + } + + const worldChanges = world.peekDirtyState(); + const reservedChanges = reservedTurns.peekDirtyState(); + world.acknowledgeDirtyState(worldChanges); + reservedTurns.acknowledgeDirtyState(reservedChanges); + if (pruneDeletedQueues) { + reservedTurns.pruneDeletedEntityQueues( + worldChanges.deletedGenerals, + worldChanges.deletedNations + ); + } + }); + } catch (error) { + if (scenario !== 'rollback-churn' || error !== ROLLBACK_SENTINEL) { + throw error; + } + rolledBackCycles += 1; + activeGeneralIds.splice(0, activeGeneralIds.length, ...world.listGenerals().map((general) => general.id)); + } + + if (sampledCycle) { + samples.push( + captureLifecycleMemorySample({ + world, + reservedTurns, + startedAtMs, + cycle, + phase: 'post-flush', + includePending: true, + includeSnapshot: true, + }) + ); + } + } + + const report = buildNpcLifecycleMemoryReport({ + scenario, + pruneDeletedQueues, + initialGeneralCount, + cycles, + batchSize, + sampleEvery, + createdTotal, + deletedTotal, + rolledBackCycles, + startedAtMs, + samples, + }); + const reportPath = resolve( + process.env.NPC_LIFECYCLE_MEMORY_CHILD_REPORT_PATH ?? + `test-results/npc-lifecycle-memory-${scenario}.json` + ); + mkdirSync(dirname(reportPath), { recursive: true }); + writeFileSync(reportPath, `${JSON.stringify(report, null, 2)}\n`, 'utf8'); + console.log( + `[NPC_LIFECYCLE_MEMORY_REPORT]${JSON.stringify({ + reportPath, + scenario: report.scenario, + result: report.result, + memory: report.memory, + })}` + ); + + const expectedFinalGeneralCount = + scenario === 'growth' + ? initialGeneralCount + cycles * batchSize + : scenario === 'death-drain' + ? baseGeneralCount + : initialGeneralCount; + expect(report.result.finalGeneralCount).toBe(expectedFinalGeneralCount); + expect(report.result.rolledBackCycles).toBe(scenario === 'rollback-churn' ? cycles : 0); + if (pruneDeletedQueues || !['death-drain', 'balanced-churn'].includes(scenario)) { + expect(report.result.deadQueueRetentionCount).toBe(0); + } else { + expect(report.result.deadQueueRetentionCount).toBe(cycles * batchSize); + } + expect(world.peekDirtyState().lifecycleEvents).toHaveLength(0); + expect(reservedTurns.peekDirtyState().generalIds).toHaveLength(0); + }, 600_000); +}); diff --git a/app/game-engine/test/npcLifecycleMemoryProfiler.test.ts b/app/game-engine/test/npcLifecycleMemoryProfiler.test.ts new file mode 100644 index 00000000..b2397ee0 --- /dev/null +++ b/app/game-engine/test/npcLifecycleMemoryProfiler.test.ts @@ -0,0 +1,20 @@ +import { describe, expect, it } from 'vitest'; + +import { linearRegressionSlope } from './helpers/npcLifecycleMemoryProfiler.js'; + +describe('NPC lifecycle memory profiler metrics', () => { + it('calculates the retained-byte slope from unevenly spaced samples', () => { + expect( + linearRegressionSlope([ + { x: 0, y: 100 }, + { x: 2, y: 140 }, + { x: 5, y: 200 }, + ]) + ).toBeCloseTo(20, 8); + }); + + it('returns zero when a trend cannot be established', () => { + expect(linearRegressionSlope([])).toBe(0); + expect(linearRegressionSlope([{ x: 1, y: 10 }])).toBe(0); + }); +}); diff --git a/app/game-engine/test/reservedTurnLease.test.ts b/app/game-engine/test/reservedTurnLease.test.ts index 981fa82c..a6eb98b3 100644 --- a/app/game-engine/test/reservedTurnLease.test.ts +++ b/app/game-engine/test/reservedTurnLease.test.ts @@ -169,6 +169,51 @@ const buildHarness = (initialRevision: RevisionRow | null = null) => { }; describe('reserved turn daemon lease', () => { + it('prunes deleted general and nation queues together with their journal state', () => { + const harness = buildHarness(); + harness.store.ensureGeneralTurns(7); + harness.store.ensureGeneralTurns(8); + harness.store.ensureNationTurns(3, 12); + harness.store.ensureNationTurns(4, 12); + + expect(harness.store.getQueueCounts()).toMatchObject({ + generalQueues: 2, + nationQueues: 2, + pendingGeneralInitializations: 2, + pendingNationInitializations: 2, + }); + expect(harness.store.pruneDeletedEntityQueues([7], [3])).toEqual({ + generalQueues: 1, + nationQueues: 1, + }); + expect(harness.store.getQueueCounts()).toMatchObject({ + generalQueues: 1, + nationQueues: 1, + pendingGeneralInitializations: 1, + pendingNationInitializations: 1, + }); + expect(harness.store.getGeneralTurns(8)).toHaveLength(2); + expect(harness.store.getNationTurns(4, 12)).toHaveLength(1); + }); + + it('restores pruned queues from the transaction savepoint', () => { + const harness = buildHarness(); + harness.store.ensureGeneralTurns(7); + harness.store.ensureNationTurns(3, 12); + const savepoint = harness.store.captureTransactionState(); + + harness.store.pruneDeletedEntityQueues([7], [3]); + expect(harness.store.getQueueCounts()).toMatchObject({ generalQueues: 0, nationQueues: 0 }); + + harness.store.restoreState(savepoint); + expect(harness.store.getQueueCounts()).toMatchObject({ + generalQueues: 1, + nationQueues: 1, + pendingGeneralInitializations: 1, + pendingNationInitializations: 1, + }); + }); + it('holds the queue lease from refresh through shift and releases it with the revision increment', async () => { const harness = buildHarness(); diff --git a/app/game-engine/test/turnDaemonMemoryReporter.test.ts b/app/game-engine/test/turnDaemonMemoryReporter.test.ts index 7d3ff937..a2f27105 100644 --- a/app/game-engine/test/turnDaemonMemoryReporter.test.ts +++ b/app/game-engine/test/turnDaemonMemoryReporter.test.ts @@ -10,6 +10,8 @@ const context = { nations: 3, troops: 15, events: 4, + generalTurnQueues: 2461, + nationTurnQueues: 18, lifecycleState: 'paused', }; @@ -33,6 +35,7 @@ describe('turn daemon memory reporting', () => { expect(result.message).toContain('profile=hwe reason=interval'); expect(result.message).toContain('heapLimitMiB=3072'); expect(result.message).toContain('year=214 month=12 generals=2461'); + expect(result.message).toContain('generalTurnQueues=2461 nationTurnQueues=18'); expect(result.message).toContain('lifecycle=paused'); }); diff --git a/docs/architecture/npc-lifecycle-memory-profile.md b/docs/architecture/npc-lifecycle-memory-profile.md new file mode 100644 index 00000000..601143ab --- /dev/null +++ b/docs/architecture/npc-lifecycle-memory-profile.md @@ -0,0 +1,106 @@ +# NPC 생명주기 메모리 프로파일 + +## 목적과 해석 경계 + +턴 데몬이 오래 실행될 때의 안정 상태와 NPC 생성·사망 churn을 서로 분리해 +측정한다. 각 case는 별도 Node/Vitest worker에서 실행하며 명시적 GC 뒤의 +retained heap, RSS, rollback participant의 V8 직렬화 크기, 실제 live 장수 수와 +예약 턴 큐 수를 기록한다. + +이 프로파일은 `scenario_2601`의 실제 장수 shape, `InMemoryTurnWorld`, +`InMemoryReservedTurnStore`, `EngineStateManager.transaction()`과 flush 뒤 +acknowledgement를 사용한다. PostgreSQL/Redis I/O, 실제 NPC AI·전투·월간 event, +다른 PM2 process와 allocator/cgroup 경합은 포함하지 않는다. 따라서 heap과 +snapshot은 엔진 상태 보유량 비교에 사용하고 RSS는 동일 host의 보조 high-water로만 +해석한다. + +## 시나리오 + +| 이름 | 고정 조건 | 확인하는 위험 | +| --- | --- | --- | +| `steady-state` | 1,200명, 매 cycle 100명 update/예약 턴 shift, 생성·사망 0 | 장기 transaction/flush 자체의 retained heap drift | +| `growth` | 1,200명에서 cycle마다 100명 생성 | live NPC 1명당 world+예약 큐 증가량 | +| `death-drain` | 종료 시 1,200명이 되도록 큰 roster에서 cycle마다 100명 사망 | 사망 뒤 world와 예약 큐가 실제로 줄어드는지 | +| `balanced-churn` | 1,200명을 유지하면서 cycle마다 100명 사망+100명 생성 | live 수가 일정해도 과거 ID/queue가 남는 누수 | +| `rollback-churn` | cycle마다 100명 생성+사망 후 강제 실패 | rollback snapshot 복원 뒤 retained state drift | + +`death-drain`과 `balanced-churn`은 `retain`과 `prune`을 같은 입력으로 A/B +실행할 수 있다. `retain`은 수정 전처럼 삭제된 예약 큐를 남기는 비교 모드이고, +`prune`은 PostgreSQL world flush 성공 뒤 삭제된 장수·국가 큐를 제거하는 현재 +제품 경계다. 큐 제거는 `EngineStateManager.transaction()` 안에서 수행되므로 이후 +오류가 발생하면 transaction savepoint가 큐와 journal set까지 복원한다. + +## 실행 + +기본 행렬은 1,200명, 80 cycle, cycle당 100명, 5 cycle 간격 sample이다. + +```sh +NPC_LIFECYCLE_MEMORY_REPORT_PATH=/dev/shm/npc-lifecycle-memory.json \ +pnpm --filter @sammo-ts/game-engine profile:npc-lifecycle-memory +``` + +시나리오와 규모를 좁힐 수 있다. + +```sh +NPC_LIFECYCLE_MEMORY_SCENARIOS=balanced-churn@retain,balanced-churn@prune \ +NPC_LIFECYCLE_MEMORY_CYCLES=240 \ +NPC_LIFECYCLE_MEMORY_BATCH_SIZE=100 \ +NPC_LIFECYCLE_MEMORY_SAMPLE_EVERY=15 \ +NPC_LIFECYCLE_MEMORY_BASE_GENERALS=1200 \ +NPC_LIFECYCLE_MEMORY_REPORT_PATH=/dev/shm/npc-lifecycle-memory-churn-240.json \ +pnpm --filter @sammo-ts/game-engine profile:npc-lifecycle-memory +``` + +지원 변수: + +- `NPC_LIFECYCLE_MEMORY_SCENARIOS`: 쉼표 구분 `name@prune|retain` +- `NPC_LIFECYCLE_MEMORY_CYCLES`: transaction/flush 반복 횟수 +- `NPC_LIFECYCLE_MEMORY_BATCH_SIZE`: cycle당 update/create/delete 수 +- `NPC_LIFECYCLE_MEMORY_SAMPLE_EVERY`: 명시적 GC와 snapshot sample 간격 +- `NPC_LIFECYCLE_MEMORY_BASE_GENERALS`: steady/growth/churn의 live 기준 수 +- `NPC_LIFECYCLE_MEMORY_REPETITIONS`: 각 독립 case 반복 횟수 +- `NPC_LIFECYCLE_MEMORY_REPORT_PATH`: aggregate JSON 경로 + +## 2026-08-24 기준 결과 + +Node v24.18.0, Linux x64 shared 개발 host에서 80×100 행렬을 독립적으로 두 번 +실행했다. 게임 결과에 영향을 주지 않는 profile 수치 중 retained heap delta는 +두 번의 차이가 14 KiB 이내였고 snapshot delta와 최종 queue 수는 정확히 +일치했다. + +| case | 최종 live/queue | retained heap delta (run 1/2) | snapshot delta | 해석 | +| --- | ---: | ---: | ---: | --- | +| steady 80 | 1,200 / 1,200 | +121,664 / +126,720 B | 0 B | transaction 반복 자체는 안정 | +| growth 8,000명 | 9,200 / 9,200 | +37,487,264 / +37,494,272 B | +13,807,868 B | live NPC 약 4.69 KiB heap, 1.73 KiB snapshot/명 | +| death retain | 1,200 / 9,200 | -11,407,296 / -11,393,936 B | -7,450,860 B | world는 줄지만 죽은 큐 8,000개 잔존 | +| death prune | 1,200 / 1,200 | -37,058,496 / -37,054,712 B | -13,802,736 B | live roster와 queue가 함께 감소 | +| balanced retain | 1,200 / 9,200 | +25,970,064 / +25,969,504 B | +6,357,008 B | live 수가 같아도 과거 큐가 선형 증가 | +| balanced prune | 1,200 / 1,200 | +314,464 / +305,984 B | +5,132 B | 8,000명 churn 뒤 dead queue 0 | +| rollback churn | 1,200 / 1,200 | +1,982,920 / +1,985,664 B | +72,000 B | 80회 강제 rollback 후 live/queue 복원 | + +장기 soak도 별도로 실행했다. + +- steady 720 cycle: live/queue `1,200/1,200`, heap `+217,192 B`, snapshot `0 B`, + 후반 slope 약 `182 B/cycle` +- balanced 24,000명 churn, retain: queue 25,200, heap `+77,121,104 B`, + snapshot `+19,063,275 B` +- 같은 churn, prune: queue 1,200, heap `+321,256 B`, snapshot `+7,400 B` + +수정 전 예약 큐 보유량은 churn 1명당 retained heap 약 3.2 KiB, 직렬화된 +transaction participant 약 794 B로 선형 증가했다. 정리 후 두 장기 case에서 +dead queue는 0이며 live 수가 일정한 snapshot은 ID 문자열 길이 차이 외에는 +거의 일정했다. + +`growth` 9,200명 sample에서 in-transaction heap은 같은 cycle의 flush 후보다 +최대 약 14.1 MiB 높았다. 현재 world rollback은 live state 전체를 deep clone하므로 +다음 절감 후보는 mutation별 copy-on-write/delta journal이다. 다만 이 변경은 +오류 rollback, RNG와 persistence 재시도 순서에 직접 닿으므로 이번 작업에서는 +예약 큐처럼 소유권이 명확한 항목만 정리하고 world snapshot 구조는 바꾸지 않는다. + +## 운영 관찰 + +턴 데몬 5분 telemetry는 world entity 수에 더해 `generalTurnQueues`와 +`nationTurnQueues`를 출력한다. 정상 flush가 이어지는 장기 서버에서는 +`generalTurnQueues - generals`가 사망 누적으로 계속 증가하지 않아야 한다. +heap/RSS만으로 queue leak을 추정하지 말고 두 count와 OOM/restart, 월 진행을 함께 +관찰한다. diff --git a/docs/architecture/runtime.md b/docs/architecture/runtime.md index 163f429b..6cecd93c 100644 --- a/docs/architecture/runtime.md +++ b/docs/architecture/runtime.md @@ -245,6 +245,13 @@ Transaction 실패 시 in-memory snapshot을 복원하고 event는 재시도 가 Checkpoint의 단일 소유자는 `InMemoryTurnWorld`이며 state store는 이를 위임 조회합니다. 별도 checkpoint 복사본이 snapshot보다 앞서 나가지 않습니다. +DB flush가 성공해 장수·국가 행과 예약 턴 행을 삭제한 뒤에는 같은 +`EngineStateManager` transaction 안에서 해당 in-memory 예약 큐도 제거합니다. +실패하면 savepoint가 큐를 복원합니다. 5분 memory telemetry의 +`generalTurnQueues`/`nationTurnQueues`와 world entity count로 장기 churn의 queue +잔존 여부를 확인할 수 있으며, 재현 행렬과 기준 수치는 +[`npc-lifecycle-memory-profile.md`](./npc-lifecycle-memory-profile.md)에 기록합니다. + 예약 턴은 revision/CAS와 lease를 사용합니다. API의 편집과 daemon의 실행이 경합해도 오래된 revision이 새 queue를 덮어쓰지 않게 합니다.