test(load): 1분 페이지 동시 부하 계측을 추가한다

국가 권한 사용자 10명과 800 NPC fixture를 만들고 Chromium 화면 순환, SSE, wall-clock 턴을 같은 시각에 측정한다.

tools/load-tests/src/cli.ts

@@ -10,6 +10,7 @@ import {
     seedCapacityFixture,
     verifyCapacityFixture,
 } from './fixture.js';
+import { measurePageNavigation } from './pageNavigation.js';
 import { describeDryRun, runLoadTest } from './runner.js';
 import { measureTurnFlush } from './turnFlush.js';
 
@@ -21,13 +22,15 @@ type Command =
     | 'seed'
     | 'verify-fixture'
     | 'activate-coverage'
+    | 'measure-page-navigation'
+    | 'measure-turn-cycle'
     | 'measure-turn-flush'
     | 'materialize-calibration'
     | 'cleanup';
 
 const usage = (): never => {
     process.stderr.write(
-        'usage: cli.ts <validate|dry-run|run|prepare|seed|verify-fixture|activate-coverage|measure-turn-flush|materialize-calibration|cleanup> --config <file> [--tokens <0600-gitignored-file>] [--output <new-json-file>] [--confirm <load_schema>]\n'
+        'usage: cli.ts <validate|dry-run|run|prepare|seed|verify-fixture|activate-coverage|measure-page-navigation|measure-turn-cycle|measure-turn-flush|materialize-calibration|cleanup> --config <file> [--tokens <0600-gitignored-file>] [--output <new-json-file>] [--confirm <load_schema>]\n'
     );
     process.exit(64);
 };
@@ -45,6 +48,8 @@ const parseArguments = (
             'seed',
             'verify-fixture',
             'activate-coverage',
+            'measure-page-navigation',
+            'measure-turn-cycle',
             'measure-turn-flush',
             'materialize-calibration',
             'cleanup',
@@ -73,7 +78,12 @@ const parseArguments = (
     )
         usage();
     if (
-        command === 'measure-turn-flush' &&
+        command === 'measure-page-navigation' &&
+        (!values.get('--tokens') || !values.get('--output') || values.has('--confirm'))
+    )
+        usage();
+    if (
+        (command === 'measure-turn-flush' || command === 'measure-turn-cycle') &&
         (!values.get('--confirm') || !values.get('--output') || values.has('--tokens'))
     )
         usage();
@@ -124,10 +134,14 @@ const main = async (): Promise<void> => {
         process.stdout.write(`${JSON.stringify(await activateCapacityCoverage(config, args.confirm!))}\n`);
         return;
     }
-    if (args.command === 'measure-turn-flush') {
+    if (args.command === 'measure-turn-flush' || args.command === 'measure-turn-cycle') {
         const output = path.resolve(args.output!);
         await mkdir(path.dirname(output), { recursive: true });
-        const result = await measureTurnFlush({ config, confirmation: args.confirm! });
+        const result = await measureTurnFlush({
+            config,
+            confirmation: args.confirm!,
+            paced: args.command === 'measure-turn-cycle',
+        });
         await writeFile(output, `${JSON.stringify(result, null, 2)}\n`, {
             encoding: 'utf8',
             flag: 'wx',
@@ -138,6 +152,21 @@ const main = async (): Promise<void> => {
         );
         return;
     }
+    if (args.command === 'measure-page-navigation') {
+        const tokens = await loadTokens(args.tokens!, workspaceRoot, config.capacity.authenticatedViewers);
+        const output = path.resolve(args.output!);
+        await mkdir(path.dirname(output), { recursive: true });
+        const result = await measurePageNavigation({ config, tokens, workspaceRoot });
+        await writeFile(output, `${JSON.stringify(result, null, 2)}\n`, {
+            encoding: 'utf8',
+            flag: 'wx',
+            mode: 0o600,
+        });
+        process.stdout.write(
+            `${JSON.stringify({ completed: true, viewers: result.fixture.viewers, outputWritten: true })}\n`
+        );
+        return;
+    }
     if (args.command === 'materialize-calibration') {
         process.stdout.write(
             `${JSON.stringify(

tools/load-tests/src/fixture.ts

@@ -182,9 +182,24 @@ const migrateDedicatedSchema = async (workspaceRoot: string, databaseUrl: string
 
 type GeneralRow = Awaited<ReturnType<GamePrismaClient['general']['findMany']>>[number];
 
+const PRIVILEGED_VIEWER_LEVELS = [12, 10, 8, 6, 11, 9, 7, 5] as const;
+
+export const privilegedViewerPlacement = (index: number, nationId: number, cityId: number) => ({
+    nationId,
+    cityId,
+    officerLevel: PRIVILEGED_VIEWER_LEVELS[index % PRIVILEGED_VIEWER_LEVELS.length]!,
+});
+
 const cloneGeneral = (
     source: GeneralRow,
-    input: { id: number; userId: string | null; npcState: number }
+    input: {
+        id: number;
+        userId: string | null;
+        npcState: number;
+        nationId?: number;
+        cityId?: number;
+        officerLevel?: number;
+    }
 ): GamePrisma.GeneralCreateManyInput =>
     ({
         ...source,
@@ -192,6 +207,9 @@ const cloneGeneral = (
         name: `${source.name}#L${input.id}`,
         userId: input.userId,
         npcState: input.npcState,
+        nationId: input.nationId ?? source.nationId,
+        cityId: input.cityId ?? source.cityId,
+        officerLevel: input.officerLevel ?? source.officerLevel,
         turnTime: new Date(source.turnTime),
         recentWarTime: source.recentWarTime ? new Date(source.recentWarTime) : null,
         createdAt: FIXED_NOW,
@@ -237,8 +255,17 @@ const projectFixtureState = async (db: GamePrismaClient) => {
 };
 
 const resizeSeededGenerals = async (db: GamePrismaClient, config: LoadConfig): Promise<void> => {
-    const source = await db.general.findMany({ orderBy: { id: 'asc' } });
+    const [source, nations, cities] = await Promise.all([
+        db.general.findMany({ orderBy: { id: 'asc' } }),
+        db.nation.findMany({ orderBy: { id: 'asc' } }),
+        db.city.findMany({ orderBy: { id: 'asc' } }),
+    ]);
     if (source.length === 0) throw new Error('scenario seed produced no generals');
+    const existingViewerNation = nations.find((nation) => nation.id > 0);
+    const viewerCity =
+        (existingViewerNation ? cities.find((city) => city.nationId === existingViewerNation.id) : null) ?? cities[0];
+    if (!viewerCity) throw new Error('scenario seed produced no city for privileged page load');
+    const viewerNationId = existingViewerNation?.id ?? Math.max(...nations.map((nation) => nation.id), 0) + 1;
     const expectedNpc = config.capacity.npcGenerals;
     const expectedHuman = config.capacity.humanGenerals;
     if (expectedHuman !== config.capacity.authenticatedViewers) {
@@ -246,20 +273,43 @@ const resizeSeededGenerals = async (db: GamePrismaClient, config: LoadConfig): P
     }
     await db.$transaction(async (transaction) => {
         await transaction.general.deleteMany();
+        if (!existingViewerNation) {
+            await transaction.nation.create({
+                data: {
+                    id: viewerNationId,
+                    name: '부하측정국',
+                    color: '#334466',
+                    capitalCityId: viewerCity.id,
+                    gold: 100_000,
+                    rice: 100_000,
+                    tech: 1_000,
+                    level: 1,
+                    typeCode: 'che_중립',
+                    meta: { cityIds: [viewerCity.id], infoText: null, secretlimit: 3 },
+                },
+            });
+        }
+        await transaction.city.update({ where: { id: viewerCity.id }, data: { nationId: viewerNationId } });
         const rows: GamePrisma.GeneralCreateManyInput[] = [];
         for (let index = 0; index < expectedNpc; index += 1) {
             rows.push(cloneGeneral(source[index % source.length]!, { id: index + 1, userId: null, npcState: 2 }));
         }
         for (let index = 0; index < expectedHuman; index += 1) {
+            const placement = privilegedViewerPlacement(index, viewerNationId, viewerCity.id);
             rows.push(
                 cloneGeneral(source[(expectedNpc + index) % source.length]!, {
                     id: expectedNpc + index + 1,
                     userId: `load-user-${String(index + 1).padStart(4, '0')}`,
                     npcState: 0,
+                    ...placement,
                 })
             );
         }
         await transaction.general.createMany({ data: rows });
+        await transaction.nation.update({
+            where: { id: viewerNationId },
+            data: { chiefGeneralId: expectedNpc + 1 },
+        });
         const world = await transaction.worldState.findFirstOrThrow({ select: { id: true, meta: true, config: true } });
         await transaction.worldState.update({
             where: { id: world.id },
@@ -435,10 +485,17 @@ export const verifyCapacityFixture = async (config: LoadConfig, env: NodeJS.Proc
             const redisVersion = /^redis_version:(.+)$/mu.exec(redisInfo)?.[1]?.trim() ?? 'unknown';
             const npcGenerals = state.generals.filter((general) => general.npcState >= 2).length;
             const humanGenerals = state.generals.filter((general) => general.npcState === 0 && general.userId).length;
+            const privilegedHumanGenerals = state.generals.filter(
+                (general) =>
+                    general.npcState === 0 && general.userId && general.nationId > 0 && general.officerLevel >= 5
+            );
+            const viewerNationIds = [...new Set(privilegedHumanGenerals.map((general) => general.nationId))];
             const valid =
                 state.generals.length === config.capacity.npcGenerals + config.capacity.humanGenerals &&
                 npcGenerals === config.capacity.npcGenerals &&
                 humanGenerals === config.capacity.humanGenerals &&
+                privilegedHumanGenerals.length === config.capacity.humanGenerals &&
+                viewerNationIds.length === 1 &&
                 accessTokens === config.capacity.authenticatedViewers &&
                 manifestFixtureSha256 === fixtureSha256;
             return {
@@ -447,6 +504,8 @@ export const verifyCapacityFixture = async (config: LoadConfig, env: NodeJS.Proc
                 generals: state.generals.length,
                 npcGenerals,
                 humanGenerals,
+                privilegedHumanGenerals: privilegedHumanGenerals.length,
+                viewerNations: viewerNationIds.length,
                 accessTokens,
                 redisManifestPresent: rawManifest !== null,
                 redisManifestMatches: manifestFixtureSha256 === fixtureSha256,

tools/load-tests/src/pageNavigation.ts

@@ -0,0 +1,380 @@
+import { execFile } from 'node:child_process';
+import { readFile, realpath } from 'node:fs/promises';
+import { promisify } from 'node:util';
+
+import { chromium, type Page } from '@playwright/test';
+
+import { isPrivateTargetHost, type LoadConfig } from './config.js';
+import { summarizeDistribution } from './metrics.js';
+import { PhaseMetrics } from './metrics.js';
+import { runSseConnection } from './sse.js';
+
+const execFileAsync = promisify(execFile);
+
+const PAGE_ROUTES = [
+    { name: 'nation-secret', path: 'nation/secret' },
+    { name: 'chief-center', path: 'chief-center' },
+    { name: 'current-city', path: 'current-city' },
+    { name: 'battle-center', path: 'battle-center' },
+    { name: 'nation-finance', path: 'nation/finance' },
+] as const;
+
+const wait = (milliseconds: number): Promise<void> => new Promise((resolve) => setTimeout(resolve, milliseconds));
+
+const increment = (target: Map<string, number>, key: string): void => {
+    target.set(key, (target.get(key) ?? 0) + 1);
+};
+
+const mapToObject = (target: ReadonlyMap<string, number>): Record<string, number> =>
+    Object.fromEntries([...target.entries()].sort(([left], [right]) => left.localeCompare(right)));
+
+export const parseTrpcProcedures = (requestUrl: string, trpcPath: string): string[] => {
+    const pathname = new URL(requestUrl).pathname;
+    const prefix = `${trpcPath.replace(/\/$/u, '')}/`;
+    if (!pathname.startsWith(prefix)) return [];
+    return decodeURIComponent(pathname.slice(prefix.length))
+        .split(',')
+        .filter((procedure) => /^[A-Za-z][A-Za-z0-9_.]+$/u.test(procedure));
+};
+
+type ProcessStat = { cpuTicks: number; rssPages: number; startTicks: number };
+
+const readProcessStat = async (pid: number): Promise<ProcessStat> => {
+    const raw = await readFile(`/proc/${pid}/stat`, 'utf8');
+    const commandEnd = raw.lastIndexOf(')');
+    if (commandEnd < 0) throw new Error('LOAD_TEST_API_PID has an invalid /proc stat record');
+    const fields = raw.slice(commandEnd + 2).trim().split(/\s+/u);
+    const cpuTicks = Number(fields[11]) + Number(fields[12]);
+    const startTicks = Number(fields[19]);
+    const rssPages = Number(fields[21]);
+    if (![cpuTicks, startTicks, rssPages].every(Number.isFinite)) {
+        throw new Error('LOAD_TEST_API_PID has an incomplete /proc stat record');
+    }
+    return { cpuTicks, startTicks, rssPages };
+};
+
+const startApiProcessSampler = async (pid: number, workspaceRoot: string) => {
+    const [commandLine, processCwd] = await Promise.all([
+        readFile(`/proc/${pid}/cmdline`, 'utf8').then((value) => value.replaceAll('\0', ' ')),
+        realpath(`/proc/${pid}/cwd`),
+    ]);
+    if (!commandLine.includes('game-api') || processCwd !== workspaceRoot) {
+        throw new Error('LOAD_TEST_API_PID must be the game-api process from this workspace');
+    }
+    const [{ stdout: clockText }, { stdout: pageText }, initial] = await Promise.all([
+        execFileAsync('getconf', ['CLK_TCK']),
+        execFileAsync('getconf', ['PAGESIZE']),
+        readProcessStat(pid),
+    ]);
+    const clockTicks = Number(clockText.trim());
+    const pageSize = Number(pageText.trim());
+    if (!Number.isFinite(clockTicks) || !Number.isFinite(pageSize)) {
+        throw new Error('could not determine process accounting units');
+    }
+    let sampling = true;
+    let maxRssPages = initial.rssPages;
+    let samples = 1;
+    const startedNs = process.hrtime.bigint();
+    const loop = (async () => {
+        while (sampling) {
+            await wait(250);
+            if (!sampling) break;
+            const sample = await readProcessStat(pid);
+            if (sample.startTicks !== initial.startTicks) throw new Error('game-api process changed during measurement');
+            maxRssPages = Math.max(maxRssPages, sample.rssPages);
+            samples += 1;
+        }
+    })();
+    return async () => {
+        sampling = false;
+        await loop;
+        const final = await readProcessStat(pid);
+        if (final.startTicks !== initial.startTicks) throw new Error('game-api process changed during measurement');
+        const elapsedSeconds = Number(process.hrtime.bigint() - startedNs) / 1_000_000_000;
+        const cpuSeconds = (final.cpuTicks - initial.cpuTicks) / clockTicks;
+        return {
+            pid,
+            samples,
+            elapsedMs: Math.round(elapsedSeconds * 1000),
+            cpuSeconds: Math.round(cpuSeconds * 1000) / 1000,
+            cpuPercentOfOneCore: Math.round((cpuSeconds / Math.max(elapsedSeconds, 0.001)) * 1000) / 10,
+            rssBytes: {
+                initial: initial.rssPages * pageSize,
+                final: final.rssPages * pageSize,
+                max: maxRssPages * pageSize,
+            },
+        };
+    };
+};
+
+const readHealth = async (baseUrl: string) => {
+    const response = await fetch(new URL('/healthz', baseUrl));
+    if (!response.ok) throw new Error(`game-api health returned HTTP ${response.status}`);
+    const body = (await response.json()) as Record<string, unknown>;
+    const pool = body.postgresPool;
+    return {
+        ok: body.ok === true,
+        postgresPool:
+            typeof pool === 'object' && pool !== null
+                ? Object.fromEntries(
+                      ['max', 'total', 'active', 'idle', 'waiting']
+                          .map((key) => [key, (pool as Record<string, unknown>)[key]])
+                          .filter(([, value]) => typeof value === 'number')
+                  )
+                : {},
+    };
+};
+
+const assertFrontendUrl = (value: string, config: LoadConfig): URL => {
+    const url = new URL(value);
+    if (!['http:', 'https:'].includes(url.protocol) || !isPrivateTargetHost(url.hostname)) {
+        throw new Error('LOAD_TEST_FRONTEND_URL must use HTTP(S) on a private or loopback host');
+    }
+    if (url.username || url.password || url.search || url.hash) {
+        throw new Error('LOAD_TEST_FRONTEND_URL must not contain credentials, query, or fragment');
+    }
+    const expectedSuffix = `/${config.isolation.postgresSchema.split('_')[2] ?? ''}/`;
+    if (!url.pathname.endsWith(expectedSuffix)) {
+        throw new Error(`LOAD_TEST_FRONTEND_URL must end with the configured profile path ${expectedSuffix}`);
+    }
+    return url;
+};
+
+export const measurePageNavigation = async (options: {
+    config: LoadConfig;
+    tokens: readonly string[];
+    workspaceRoot: string;
+    env?: NodeJS.ProcessEnv;
+}) => {
+    const env = options.env ?? process.env;
+    const frontendUrl = assertFrontendUrl(env.LOAD_TEST_FRONTEND_URL ?? '', options.config);
+    const apiPid = Number(env.LOAD_TEST_API_PID);
+    if (!Number.isSafeInteger(apiPid) || apiPid <= 1) throw new Error('LOAD_TEST_API_PID must be a positive process id');
+    if (options.tokens.length !== options.config.capacity.authenticatedViewers) {
+        throw new Error('page navigation requires exactly one token per authenticated viewer');
+    }
+    const dwellMs = Number(env.LOAD_TEST_PAGE_DWELL_MS ?? '1000');
+    if (!Number.isSafeInteger(dwellMs) || dwellMs < 250 || dwellMs > 10_000) {
+        throw new Error('LOAD_TEST_PAGE_DWELL_MS must be an integer from 250 to 10000');
+    }
+
+    const routeLatencyMs = new Map<string, number[]>();
+    const routeSuccess = new Map<string, number>();
+    const routeErrors = new Map<string, number>();
+    const procedureRequests = new Map<string, number>();
+    const procedureErrors = new Map<string, number>();
+    const procedureLatencyMs = new Map<string, number[]>();
+    let pageErrors = 0;
+    let permissionErrors = 0;
+    const pendingResponses = new Set<Promise<void>>();
+    const browser = await chromium.launch({ headless: true });
+    const viewers: Array<{ page: Page; close: () => Promise<void> }> = [];
+    const activeRouteByPage = new Map<Page, string>();
+    let activeSseController: AbortController | null = null;
+    let activeSseTasks: Promise<void>[] = [];
+    let activeSseSampleTimer: NodeJS.Timeout | null = null;
+    const profile = options.config.isolation.postgresSchema.split('_')[2] ?? '';
+    try {
+        for (const token of options.tokens) {
+            const context = await browser.newContext({ viewport: { width: 1280, height: 900 } });
+            await context.addInitScript(
+                ({ accessToken, profileName }) => {
+                    const storage = (
+                        globalThis as unknown as { localStorage: { setItem: (key: string, value: string) => void } }
+                    ).localStorage;
+                    storage.setItem('sammo-game-token', accessToken);
+                    storage.setItem('sammo-game-profile', profileName);
+                },
+                { accessToken: token, profileName: profile }
+            );
+            const page = await context.newPage();
+            page.setDefaultTimeout(20_000);
+            page.setDefaultNavigationTimeout(20_000);
+            await page.goto(new URL('current-city', frontendUrl).toString(), { waitUntil: 'networkidle' });
+            const warmupText = await page.locator('body').innerText();
+            if (warmupText.includes('권한이 부족합니다')) {
+                throw new Error('privileged viewer fixture failed the current-city warmup');
+            }
+            viewers.push({ page, close: () => context.close() });
+        }
+
+        const sseMetrics = new PhaseMetrics();
+        const sseController = new AbortController();
+        activeSseController = sseController;
+        let activeSse = 0;
+        sseMetrics.sseActiveConnections.push(0);
+        const sseUrl = new URL(options.config.target.ssePath, options.config.target.baseUrl).toString();
+        const sseTasks = options.tokens.map((token) =>
+            runSseConnection({
+                url: sseUrl,
+                token,
+                signal: sseController.signal,
+                metrics: sseMetrics,
+                onActiveChange: (delta) => {
+                    activeSse += delta;
+                    sseMetrics.sseActiveConnections.push(activeSse);
+                },
+            })
+        );
+        activeSseTasks = sseTasks;
+        const connectionDeadline = Date.now() + 10_000;
+        while (activeSse !== options.tokens.length && Date.now() < connectionDeadline) await wait(50);
+        if (activeSse !== options.tokens.length) {
+            sseController.abort();
+            await Promise.allSettled(sseTasks);
+            throw new Error(`only ${activeSse} of ${options.tokens.length} SSE connections became active`);
+        }
+
+        const healthBefore = await readHealth(options.config.target.baseUrl);
+        let scheduledStartAtEpochMs: number | null = null;
+        if (env.LOAD_TEST_START_AT_EPOCH_MS) {
+            scheduledStartAtEpochMs = Number(env.LOAD_TEST_START_AT_EPOCH_MS);
+            if (!Number.isSafeInteger(scheduledStartAtEpochMs)) {
+                throw new Error('LOAD_TEST_START_AT_EPOCH_MS must be an integer epoch in milliseconds');
+            }
+            const waitMs = scheduledStartAtEpochMs - Date.now();
+            if (waitMs > 120_000) throw new Error('LOAD_TEST_START_AT_EPOCH_MS must be within the next two minutes');
+            if (waitMs > 0) await wait(waitMs);
+        }
+        const stopApiSampler = await startApiProcessSampler(apiPid, options.workspaceRoot);
+        const startedAt = new Date().toISOString();
+        const started = performance.now();
+        const deadline = started + options.config.capacity.turnIntervalMs;
+        const activeSseDuringMeasurement = [activeSse];
+        activeSseSampleTimer = setInterval(() => activeSseDuringMeasurement.push(activeSse), 250);
+
+        for (const [viewerIndex, viewer] of viewers.entries()) {
+            activeRouteByPage.set(viewer.page, 'warmup');
+            viewer.page.on('pageerror', () => {
+                pageErrors += 1;
+            });
+            viewer.page.on('response', (response) => {
+                const procedures = parseTrpcProcedures(response.url(), options.config.target.trpcPath);
+                if (procedures.length === 0) return;
+                const task = (async () => {
+                    await response.finished().catch(() => null);
+                    const timing = response.request().timing();
+                    const latencyMs = timing.responseEnd;
+                    for (const procedure of procedures) {
+                        const key = `${activeRouteByPage.get(viewer.page) ?? 'unknown'}:${procedure}`;
+                        increment(procedureRequests, key);
+                        if (response.status() >= 400) increment(procedureErrors, `${key}:http-${response.status()}`);
+                        if (latencyMs >= 0) {
+                            const values = procedureLatencyMs.get(key) ?? [];
+                            values.push(latencyMs);
+                            procedureLatencyMs.set(key, values);
+                        }
+                    }
+                })().finally(() => pendingResponses.delete(task));
+                pendingResponses.add(task);
+            });
+            void viewerIndex;
+        }
+
+        await Promise.all(
+            viewers.map(async ({ page }, viewerIndex) => {
+                let iteration = 0;
+                while (performance.now() < deadline) {
+                    const route = PAGE_ROUTES[(viewerIndex + iteration) % PAGE_ROUTES.length]!;
+                    activeRouteByPage.set(page, route.name);
+                    const routeStarted = performance.now();
+                    try {
+                        await page.goto(new URL(route.path, frontendUrl).toString(), { waitUntil: 'networkidle' });
+                        const bodyText = await page.locator('body').innerText();
+                        if (bodyText.includes('권한이 부족합니다')) {
+                            permissionErrors += 1;
+                            increment(routeErrors, `${route.name}:permission`);
+                        } else {
+                            increment(routeSuccess, route.name);
+                        }
+                    } catch {
+                        increment(routeErrors, `${route.name}:navigation`);
+                    } finally {
+                        const values = routeLatencyMs.get(route.name) ?? [];
+                        values.push(performance.now() - routeStarted);
+                        routeLatencyMs.set(route.name, values);
+                    }
+                    iteration += 1;
+                    if (performance.now() < deadline) await wait(dwellMs);
+                }
+            })
+        );
+        clearInterval(activeSseSampleTimer);
+        activeSseSampleTimer = null;
+        activeSseDuringMeasurement.push(activeSse);
+        await Promise.allSettled([...pendingResponses]);
+        const apiProcess = await stopApiSampler();
+        const elapsedMs = Math.round(performance.now() - started);
+        const healthAfter = await readHealth(options.config.target.baseUrl);
+        sseController.abort();
+        await Promise.allSettled(sseTasks);
+        sseMetrics.sseActiveConnections.push(activeSse);
+
+        return {
+            formatVersion: 1,
+            startedAt,
+            finishedAt: new Date().toISOString(),
+            configuredDurationMs: options.config.capacity.turnIntervalMs,
+            elapsedMs,
+            scheduling: {
+                scheduledStartAtEpochMs,
+                actualStartAtEpochMs: Date.parse(startedAt),
+                startDelayMs:
+                    scheduledStartAtEpochMs === null ? null : Math.max(0, Date.parse(startedAt) - scheduledStartAtEpochMs),
+            },
+            fixture: {
+                name: options.config.name,
+                viewers: options.tokens.length,
+                npcGenerals: options.config.capacity.npcGenerals,
+                pageDwellMs: dwellMs,
+            },
+            browser: {
+                name: 'chromium',
+                contexts: viewers.length,
+                viewport: { width: 1280, height: 900 },
+                routes: Object.fromEntries(
+                    PAGE_ROUTES.map((route) => [
+                        route.name,
+                        {
+                            success: routeSuccess.get(route.name) ?? 0,
+                            latencyMs: summarizeDistribution(routeLatencyMs.get(route.name) ?? []),
+                        },
+                    ])
+                ),
+                routeErrors: mapToObject(routeErrors),
+                pageErrors,
+                permissionErrors,
+            },
+            trpc: {
+                requests: mapToObject(procedureRequests),
+                errors: mapToObject(procedureErrors),
+                latencyMs: Object.fromEntries(
+                    [...procedureLatencyMs.entries()]
+                        .sort(([left], [right]) => left.localeCompare(right))
+                        .map(([key, values]) => [key, summarizeDistribution(values)])
+                ),
+            },
+            sse: {
+                attempts: sseMetrics.sseAttempts,
+                opened: sseMetrics.sseOpened,
+                closed: sseMetrics.sseClosed,
+                reconnects: sseMetrics.sseReconnects,
+                failures: sseMetrics.sseFailures,
+                privacyViolations: sseMetrics.ssePrivacyViolations,
+                events: mapToObject(sseMetrics.sseEvents),
+                activeConnections: summarizeDistribution(activeSseDuringMeasurement),
+            },
+            server: {
+                healthBefore,
+                healthAfter,
+                apiProcess,
+            },
+        };
+    } finally {
+        if (activeSseSampleTimer) clearInterval(activeSseSampleTimer);
+        activeSseController?.abort();
+        await Promise.allSettled(activeSseTasks);
+        await Promise.allSettled(viewers.map((viewer) => viewer.close()));
+        await browser.close();
+    }
+};

tools/load-tests/src/turnFlush.ts

@@ -105,6 +105,7 @@ const includeSubMillisecondGameTick = (turnTime: Date): Date =>
 export const measureTurnFlush = async (options: {
     config: LoadConfig;
     confirmation: string;
+    paced?: boolean;
     env?: NodeJS.ProcessEnv;
 }) => {
     const env = options.env ?? process.env;
@@ -153,9 +154,8 @@ export const measureTurnFlush = async (options: {
     const activityPromise = sampleActivity();
 
     const histogram = monitorEventLoopDelay({ resolution: 20 });
-    histogram.enable();
-    const cpuStart = process.cpuUsage();
-    const wallStartNs = process.hrtime.bigint();
+    let cpuStart: NodeJS.CpuUsage | null = null;
+    let wallStartNs: bigint | null = null;
     let maxRssBytes = process.memoryUsage().rss;
     const generalTransactionMs: number[] = [];
     const monthlyTransactionMs: number[] = [];
@@ -167,6 +167,10 @@ export const measureTurnFlush = async (options: {
     let endYearMonth: string | null = null;
     let initialGeneralCount: number | null = null;
     let finalGeneralCount: number | null = null;
+    let scheduledStartAtEpochMs: number | null = null;
+    let actualStartAtEpochMs: number | null = null;
+    const scheduleLagMs: number[] = [];
+    let backdatedGeneralTurns = 0;
     let runError: unknown;
 
     try {
@@ -189,6 +193,21 @@ export const measureTurnFlush = async (options: {
         const boundary = getNextTickTime(initialState.lastTurnTime, tickMinutes);
         let checkpoint: TurnCheckpoint | undefined = await runtime.stateStore.loadCheckpoint();
 
+        if (env.LOAD_TEST_START_AT_EPOCH_MS) {
+            scheduledStartAtEpochMs = Number(env.LOAD_TEST_START_AT_EPOCH_MS);
+            if (!Number.isSafeInteger(scheduledStartAtEpochMs)) {
+                throw new Error('LOAD_TEST_START_AT_EPOCH_MS must be an integer epoch in milliseconds');
+            }
+            const waitMs = scheduledStartAtEpochMs - Date.now();
+            if (waitMs > 120_000) throw new Error('LOAD_TEST_START_AT_EPOCH_MS must be within the next two minutes');
+            if (waitMs > 0) await new Promise((resolve) => setTimeout(resolve, waitMs));
+        }
+        actualStartAtEpochMs = Date.now();
+        const pacingStarted = performance.now();
+        histogram.enable();
+        cpuStart = process.cpuUsage();
+        wallStartNs = process.hrtime.bigint();
+
         const execute = async (target: Date, maxGenerals: number): Promise<TurnRunResult> => {
             const started = performance.now();
             const result = await runtime!.stateManager.transaction(async () => {
@@ -219,6 +238,16 @@ export const measureTurnFlush = async (options: {
         while (true) {
             const nextGeneral = await runtime.stateStore.loadNextGeneralTurnTime();
             if (!nextGeneral || nextGeneral.getTime() >= boundary.getTime()) break;
+            if (options.paced) {
+                const targetElapsedMs = nextGeneral.getTime() - initialState.lastTurnTime.getTime();
+                if (targetElapsedMs < 0) {
+                    backdatedGeneralTurns += 1;
+                } else {
+                    const remainingMs = targetElapsedMs - (performance.now() - pacingStarted);
+                    if (remainingMs > 0) await new Promise((resolve) => setTimeout(resolve, remainingMs));
+                    scheduleLagMs.push(Math.max(0, performance.now() - pacingStarted - targetElapsedMs));
+                }
+            }
             const result = await execute(includeSubMillisecondGameTick(nextGeneral), 1);
             if (result.processedGenerals !== 1 || result.processedTurns !== 0) {
                 throw new Error(
@@ -227,6 +256,11 @@ export const measureTurnFlush = async (options: {
             }
         }
 
+        if (options.paced) {
+            const remainingMs =
+                options.config.capacity.turnIntervalMs - (performance.now() - pacingStarted);
+            if (remainingMs > 0) await new Promise((resolve) => setTimeout(resolve, remainingMs));
+        }
         const monthly = await execute(boundary, 200);
         if (monthly.processedTurns !== 1) {
             throw new Error('turn-flush measurement did not cross exactly one monthly boundary');
@@ -275,6 +309,9 @@ export const measureTurnFlush = async (options: {
     }
     await observer.disconnect();
     histogram.disable();
+    if (cpuStart === null || wallStartNs === null || actualStartAtEpochMs === null) {
+        throw new Error('turn-flush measurement did not start its measured interval');
+    }
     const elapsedMs = Number(process.hrtime.bigint() - wallStartNs) / 1_000_000;
     const cpu = process.cpuUsage(cpuStart);
     const cpuMs = (cpu.user + cpu.system) / 1_000;
@@ -287,7 +324,9 @@ export const measureTurnFlush = async (options: {
             fixtureSha256: fixture.fixtureSha256,
             capacity: options.config.capacity,
         },
-        mode: 'chronological-one-general-per-transaction-plus-month-boundary',
+        mode: options.paced
+            ? 'wall-clock-paced-one-general-per-transaction-plus-month-boundary'
+            : 'chronological-one-general-per-transaction-plus-month-boundary',
         startYearMonth,
         endYearMonth,
         elapsedMs: round(elapsedMs),
@@ -306,6 +345,16 @@ export const measureTurnFlush = async (options: {
             monthlyTransaction: summarizeDistribution(monthlyTransactionMs),
             redisPublication: summarizeDistribution(publicationMs),
         },
+        scheduling: {
+            paced: options.paced === true,
+            configuredTurnIntervalMs: options.config.capacity.turnIntervalMs,
+            scheduledStartAtEpochMs,
+            actualStartAtEpochMs,
+            startDelayMs:
+                scheduledStartAtEpochMs === null ? null : Math.max(0, actualStartAtEpochMs - scheduledStartAtEpochMs),
+            backdatedGeneralTurns,
+            generalScheduleLagMs: summarizeDistribution(scheduleLagMs),
+        },
         postgres: {
             statsScope: 'database-wide-including-observer-sampler',
             statsDelta: subtractDatabaseStats(beforeStats, afterStats),