From e67a40e51d15f671d425de6a2057d7970ede5936 Mon Sep 17 00:00:00 2001 From: hided62 Date: Fri, 21 Aug 2026 23:17:24 +0000 Subject: [PATCH] =?UTF-8?q?test(load):=201=EB=B6=84=20=ED=8E=98=EC=9D=B4?= =?UTF-8?q?=EC=A7=80=20=EB=8F=99=EC=8B=9C=20=EB=B6=80=ED=95=98=20=EA=B3=84?= =?UTF-8?q?=EC=B8=A1=EC=9D=84=20=EC=B6=94=EA=B0=80=ED=95=9C=EB=8B=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 국가 권한 사용자 10명과 800 NPC fixture를 만들고 Chromium 화면 순환, SSE, wall-clock 턴을 같은 시각에 측정한다. --- tools/load-tests/README.md | 44 ++- tools/load-tests/package.json | 2 + tools/load-tests/src/cli.ts | 37 +- tools/load-tests/src/fixture.ts | 63 ++- tools/load-tests/src/pageNavigation.ts | 380 +++++++++++++++++++ tools/load-tests/src/turnFlush.ts | 57 ++- tools/load-tests/test/fixture.test.ts | 14 +- tools/load-tests/test/pageNavigation.test.ts | 19 + 8 files changed, 601 insertions(+), 15 deletions(-) create mode 100644 tools/load-tests/src/pageNavigation.ts create mode 100644 tools/load-tests/test/pageNavigation.test.ts diff --git a/tools/load-tests/README.md b/tools/load-tests/README.md index 7e47387b..a224b002 100644 --- a/tools/load-tests/README.md +++ b/tools/load-tests/README.md @@ -62,7 +62,9 @@ pnpm --filter @sammo-ts/load-tests activate-coverage \ ``` `seed`는 해당 `load_` schema에 migration을 적용하고 scenario 2601을 고정 seed/time으로 설치한 뒤 정확히 -900 NPC + 300 synthetic 사용자 장수로 재구성한다. 각 사용자의 24시간 access token은 Redis 전용 DB와 +900 NPC + 300 synthetic 사용자 장수로 재구성한다. synthetic 사용자는 같은 fixture 국가와 도시에 배치하고 +관직 5 이상을 순환 배정하여 암행부·사령부·감찰부·내무부를 실제 권한으로 읽을 수 있게 한다. 시나리오에 +비중립 국가가 없으면 전용 fixture 안에만 측정 국가 하나를 만든다. 각 사용자의 24시간 access token은 Redis 전용 DB와 새 `0600` JSON에만 저장한다. stdout에는 token, user/general ID, DB/Redis URL을 내보내지 않고 count와 비밀값을 제외한 fixture SHA-256만 기록한다. token 파일이 이미 있으면 DB 작업 전에 실패한다. @@ -83,6 +85,7 @@ pnpm --filter @sammo-ts/game-api build systemd-run --user --unit=sammo-capacity-api --collect \ --property=MemoryMax=8G \ + --setenv=POSTGRES_POOL_MAX=4 \ --working-directory="$(pwd)" \ "$(pwd)/tools/load-tests/scripts/run-capacity-api.sh" ``` @@ -181,7 +184,40 @@ schema 간 row-lock 격리와 공유 CPU/I/O/connection 경합을 확인한다. - `tools/load-tests/config/nya-10-users-800-npcs-1m.json` - `tools/load-tests/config/pya-10-users-800-npcs-1m.json` -### 5. 명시적 cleanup +### 5. 1분 턴과 10-user 실제 화면 동시 측정 + +`measure-turn-cycle`은 fixture의 권위 `turn_time` 분포대로 장수 턴을 wall clock 1분에 pacing하고 월 경계를 +한 번 처리한다. `measure-page-navigation`은 실제 Chromium context 10개가 암행부, 사령부, 현재 도시, +감찰부, 내무부를 순환하며 페이지와 tRPC 지연을 기록한다. 화면 이동 중에도 사용자마다 별도 SSE 한 개를 +계속 유지한다. 이는 제품 UI가 메인 화면 밖에서 dashboard SSE를 닫는 동작과 “각 사용자가 실시간 알림을 +계속 받는다”는 부하 조건을 분리해 재현하기 위한 구성이다. + +두 명령에 같은 미래 `LOAD_TEST_START_AT_EPOCH_MS`를 주면 턴과 브라우저 측정 구간을 맞출 수 있다. +`LOAD_TEST_FRONTEND_URL`은 private/loopback URL과 정확한 profile suffix여야 하며, +`LOAD_TEST_API_PID`는 같은 workspace에서 실행 중인 game-api PID여야 한다. 결과에는 token, 응답 본문과 +사용자 식별자를 넣지 않는다. + +```sh +export LOAD_TEST_START_AT_EPOCH_MS=REPLACE_WITH_NEAR_FUTURE_EPOCH_MS +export LOAD_TEST_FRONTEND_URL=http://127.0.0.1:15000/pya/ +export LOAD_TEST_API_PID=REPLACE_WITH_LOCAL_GAME_API_PID + +pnpm --filter @sammo-ts/load-tests measure-page-navigation \ + --config tools/load-tests/config/pya-10-users-800-npcs-1m.json \ + --tokens tools/load-tests/secrets/game-tokens.json \ + --output tools/load-tests/results/page-navigation.json + +pnpm --filter @sammo-ts/load-tests measure-turn-cycle \ + --config tools/load-tests/config/pya-10-users-800-npcs-1m.json \ + --confirm load_capacity_pya_10_800_1m \ + --output tools/load-tests/results/turn-cycle.json +``` + +페이지 latency는 production frontend의 lazy chunk, 인증 초기화와 Playwright `networkidle` 500ms를 모두 +포함하므로 API procedure latency와 함께 해석한다. API CPU/RSS는 지정 PID만, PostgreSQL 지표는 같은 DB의 +observer를 포함한 database-wide delta만 나타낸다. 호스트의 다른 profile, Gateway와 worker 부하는 포함하지 않는다. + +### 6. 명시적 cleanup token 파일은 별도로 안전하게 삭제하고, fixture schema/Redis token은 schema명을 그대로 확인 인자로 주어 정리한다. named volume은 보존한다. 데이터 폐기가 필요하지 않으면 이 명령을 실행하지 않는다. @@ -195,8 +231,8 @@ docker compose -f tools/load-tests/compose.capacity.yml down ## 아직 남은 측정 경계 -- `measure-turn-flush`는 한 달을 wall-clock보다 빠르게 replay하는 처리량 시험이다. 실제 schedule lag, - 장시간 pool wait와 autovacuum/checkpoint 영향을 보려면 profile별 속도로 pacing한 soak가 별도로 필요하다. +- `measure-turn-flush`는 한 달을 wall-clock보다 빠르게 replay하는 처리량 시험이다. 실제 1분 schedule lag는 + `measure-turn-cycle`로 확인할 수 있지만 장시간 pool wait와 autovacuum/checkpoint는 더 긴 soak가 필요하다. - 월 경계 latency는 실행당 표본이 하나다. scenario 진행 시점과 월별 event 차이를 포괄하지 않는다. - 두 1분 profile 동시 실행은 최악 turn-rate 조합의 국소 증거이며, 여섯 profile의 전체 PM2 RSS, Gateway·worker connection과 운영 container cgroup을 재현하지 않는다. diff --git a/tools/load-tests/package.json b/tools/load-tests/package.json index 8c9c5d7f..ab8b32a4 100644 --- a/tools/load-tests/package.json +++ b/tools/load-tests/package.json @@ -11,6 +11,8 @@ "seed": "pnpm -w exec tsx tools/load-tests/src/cli.ts seed", "verify-fixture": "pnpm -w exec tsx tools/load-tests/src/cli.ts verify-fixture", "activate-coverage": "pnpm -w exec tsx tools/load-tests/src/cli.ts activate-coverage", + "measure-page-navigation": "pnpm -w exec tsx tools/load-tests/src/cli.ts measure-page-navigation", + "measure-turn-cycle": "pnpm -w exec tsx tools/load-tests/src/cli.ts measure-turn-cycle", "measure-turn-flush": "pnpm -w exec tsx tools/load-tests/src/cli.ts measure-turn-flush", "materialize-calibration": "pnpm -w exec tsx tools/load-tests/src/cli.ts materialize-calibration", "cleanup": "pnpm -w exec tsx tools/load-tests/src/cli.ts cleanup", diff --git a/tools/load-tests/src/cli.ts b/tools/load-tests/src/cli.ts index c6c9f5c5..455b4066 100644 --- a/tools/load-tests/src/cli.ts +++ b/tools/load-tests/src/cli.ts @@ -10,6 +10,7 @@ import { seedCapacityFixture, verifyCapacityFixture, } from './fixture.js'; +import { measurePageNavigation } from './pageNavigation.js'; import { describeDryRun, runLoadTest } from './runner.js'; import { measureTurnFlush } from './turnFlush.js'; @@ -21,13 +22,15 @@ type Command = | 'seed' | 'verify-fixture' | 'activate-coverage' + | 'measure-page-navigation' + | 'measure-turn-cycle' | 'measure-turn-flush' | 'materialize-calibration' | 'cleanup'; const usage = (): never => { process.stderr.write( - 'usage: cli.ts --config [--tokens <0600-gitignored-file>] [--output ] [--confirm ]\n' + 'usage: cli.ts --config [--tokens <0600-gitignored-file>] [--output ] [--confirm ]\n' ); process.exit(64); }; @@ -45,6 +48,8 @@ const parseArguments = ( 'seed', 'verify-fixture', 'activate-coverage', + 'measure-page-navigation', + 'measure-turn-cycle', 'measure-turn-flush', 'materialize-calibration', 'cleanup', @@ -73,7 +78,12 @@ const parseArguments = ( ) usage(); if ( - command === 'measure-turn-flush' && + command === 'measure-page-navigation' && + (!values.get('--tokens') || !values.get('--output') || values.has('--confirm')) + ) + usage(); + if ( + (command === 'measure-turn-flush' || command === 'measure-turn-cycle') && (!values.get('--confirm') || !values.get('--output') || values.has('--tokens')) ) usage(); @@ -124,10 +134,14 @@ const main = async (): Promise => { process.stdout.write(`${JSON.stringify(await activateCapacityCoverage(config, args.confirm!))}\n`); return; } - if (args.command === 'measure-turn-flush') { + if (args.command === 'measure-turn-flush' || args.command === 'measure-turn-cycle') { const output = path.resolve(args.output!); await mkdir(path.dirname(output), { recursive: true }); - const result = await measureTurnFlush({ config, confirmation: args.confirm! }); + const result = await measureTurnFlush({ + config, + confirmation: args.confirm!, + paced: args.command === 'measure-turn-cycle', + }); await writeFile(output, `${JSON.stringify(result, null, 2)}\n`, { encoding: 'utf8', flag: 'wx', @@ -138,6 +152,21 @@ const main = async (): Promise => { ); return; } + if (args.command === 'measure-page-navigation') { + const tokens = await loadTokens(args.tokens!, workspaceRoot, config.capacity.authenticatedViewers); + const output = path.resolve(args.output!); + await mkdir(path.dirname(output), { recursive: true }); + const result = await measurePageNavigation({ config, tokens, workspaceRoot }); + await writeFile(output, `${JSON.stringify(result, null, 2)}\n`, { + encoding: 'utf8', + flag: 'wx', + mode: 0o600, + }); + process.stdout.write( + `${JSON.stringify({ completed: true, viewers: result.fixture.viewers, outputWritten: true })}\n` + ); + return; + } if (args.command === 'materialize-calibration') { process.stdout.write( `${JSON.stringify( diff --git a/tools/load-tests/src/fixture.ts b/tools/load-tests/src/fixture.ts index 17626b34..d688c835 100644 --- a/tools/load-tests/src/fixture.ts +++ b/tools/load-tests/src/fixture.ts @@ -182,9 +182,24 @@ const migrateDedicatedSchema = async (workspaceRoot: string, databaseUrl: string type GeneralRow = Awaited>[number]; +const PRIVILEGED_VIEWER_LEVELS = [12, 10, 8, 6, 11, 9, 7, 5] as const; + +export const privilegedViewerPlacement = (index: number, nationId: number, cityId: number) => ({ + nationId, + cityId, + officerLevel: PRIVILEGED_VIEWER_LEVELS[index % PRIVILEGED_VIEWER_LEVELS.length]!, +}); + const cloneGeneral = ( source: GeneralRow, - input: { id: number; userId: string | null; npcState: number } + input: { + id: number; + userId: string | null; + npcState: number; + nationId?: number; + cityId?: number; + officerLevel?: number; + } ): GamePrisma.GeneralCreateManyInput => ({ ...source, @@ -192,6 +207,9 @@ const cloneGeneral = ( name: `${source.name}#L${input.id}`, userId: input.userId, npcState: input.npcState, + nationId: input.nationId ?? source.nationId, + cityId: input.cityId ?? source.cityId, + officerLevel: input.officerLevel ?? source.officerLevel, turnTime: new Date(source.turnTime), recentWarTime: source.recentWarTime ? new Date(source.recentWarTime) : null, createdAt: FIXED_NOW, @@ -237,8 +255,17 @@ const projectFixtureState = async (db: GamePrismaClient) => { }; const resizeSeededGenerals = async (db: GamePrismaClient, config: LoadConfig): Promise => { - const source = await db.general.findMany({ orderBy: { id: 'asc' } }); + const [source, nations, cities] = await Promise.all([ + db.general.findMany({ orderBy: { id: 'asc' } }), + db.nation.findMany({ orderBy: { id: 'asc' } }), + db.city.findMany({ orderBy: { id: 'asc' } }), + ]); if (source.length === 0) throw new Error('scenario seed produced no generals'); + const existingViewerNation = nations.find((nation) => nation.id > 0); + const viewerCity = + (existingViewerNation ? cities.find((city) => city.nationId === existingViewerNation.id) : null) ?? cities[0]; + if (!viewerCity) throw new Error('scenario seed produced no city for privileged page load'); + const viewerNationId = existingViewerNation?.id ?? Math.max(...nations.map((nation) => nation.id), 0) + 1; const expectedNpc = config.capacity.npcGenerals; const expectedHuman = config.capacity.humanGenerals; if (expectedHuman !== config.capacity.authenticatedViewers) { @@ -246,20 +273,43 @@ const resizeSeededGenerals = async (db: GamePrismaClient, config: LoadConfig): P } await db.$transaction(async (transaction) => { await transaction.general.deleteMany(); + if (!existingViewerNation) { + await transaction.nation.create({ + data: { + id: viewerNationId, + name: '부하측정국', + color: '#334466', + capitalCityId: viewerCity.id, + gold: 100_000, + rice: 100_000, + tech: 1_000, + level: 1, + typeCode: 'che_중립', + meta: { cityIds: [viewerCity.id], infoText: null, secretlimit: 3 }, + }, + }); + } + await transaction.city.update({ where: { id: viewerCity.id }, data: { nationId: viewerNationId } }); const rows: GamePrisma.GeneralCreateManyInput[] = []; for (let index = 0; index < expectedNpc; index += 1) { rows.push(cloneGeneral(source[index % source.length]!, { id: index + 1, userId: null, npcState: 2 })); } for (let index = 0; index < expectedHuman; index += 1) { + const placement = privilegedViewerPlacement(index, viewerNationId, viewerCity.id); rows.push( cloneGeneral(source[(expectedNpc + index) % source.length]!, { id: expectedNpc + index + 1, userId: `load-user-${String(index + 1).padStart(4, '0')}`, npcState: 0, + ...placement, }) ); } await transaction.general.createMany({ data: rows }); + await transaction.nation.update({ + where: { id: viewerNationId }, + data: { chiefGeneralId: expectedNpc + 1 }, + }); const world = await transaction.worldState.findFirstOrThrow({ select: { id: true, meta: true, config: true } }); await transaction.worldState.update({ where: { id: world.id }, @@ -435,10 +485,17 @@ export const verifyCapacityFixture = async (config: LoadConfig, env: NodeJS.Proc const redisVersion = /^redis_version:(.+)$/mu.exec(redisInfo)?.[1]?.trim() ?? 'unknown'; const npcGenerals = state.generals.filter((general) => general.npcState >= 2).length; const humanGenerals = state.generals.filter((general) => general.npcState === 0 && general.userId).length; + const privilegedHumanGenerals = state.generals.filter( + (general) => + general.npcState === 0 && general.userId && general.nationId > 0 && general.officerLevel >= 5 + ); + const viewerNationIds = [...new Set(privilegedHumanGenerals.map((general) => general.nationId))]; const valid = state.generals.length === config.capacity.npcGenerals + config.capacity.humanGenerals && npcGenerals === config.capacity.npcGenerals && humanGenerals === config.capacity.humanGenerals && + privilegedHumanGenerals.length === config.capacity.humanGenerals && + viewerNationIds.length === 1 && accessTokens === config.capacity.authenticatedViewers && manifestFixtureSha256 === fixtureSha256; return { @@ -447,6 +504,8 @@ export const verifyCapacityFixture = async (config: LoadConfig, env: NodeJS.Proc generals: state.generals.length, npcGenerals, humanGenerals, + privilegedHumanGenerals: privilegedHumanGenerals.length, + viewerNations: viewerNationIds.length, accessTokens, redisManifestPresent: rawManifest !== null, redisManifestMatches: manifestFixtureSha256 === fixtureSha256, diff --git a/tools/load-tests/src/pageNavigation.ts b/tools/load-tests/src/pageNavigation.ts new file mode 100644 index 00000000..c86ee631 --- /dev/null +++ b/tools/load-tests/src/pageNavigation.ts @@ -0,0 +1,380 @@ +import { execFile } from 'node:child_process'; +import { readFile, realpath } from 'node:fs/promises'; +import { promisify } from 'node:util'; + +import { chromium, type Page } from '@playwright/test'; + +import { isPrivateTargetHost, type LoadConfig } from './config.js'; +import { summarizeDistribution } from './metrics.js'; +import { PhaseMetrics } from './metrics.js'; +import { runSseConnection } from './sse.js'; + +const execFileAsync = promisify(execFile); + +const PAGE_ROUTES = [ + { name: 'nation-secret', path: 'nation/secret' }, + { name: 'chief-center', path: 'chief-center' }, + { name: 'current-city', path: 'current-city' }, + { name: 'battle-center', path: 'battle-center' }, + { name: 'nation-finance', path: 'nation/finance' }, +] as const; + +const wait = (milliseconds: number): Promise => new Promise((resolve) => setTimeout(resolve, milliseconds)); + +const increment = (target: Map, key: string): void => { + target.set(key, (target.get(key) ?? 0) + 1); +}; + +const mapToObject = (target: ReadonlyMap): Record => + Object.fromEntries([...target.entries()].sort(([left], [right]) => left.localeCompare(right))); + +export const parseTrpcProcedures = (requestUrl: string, trpcPath: string): string[] => { + const pathname = new URL(requestUrl).pathname; + const prefix = `${trpcPath.replace(/\/$/u, '')}/`; + if (!pathname.startsWith(prefix)) return []; + return decodeURIComponent(pathname.slice(prefix.length)) + .split(',') + .filter((procedure) => /^[A-Za-z][A-Za-z0-9_.]+$/u.test(procedure)); +}; + +type ProcessStat = { cpuTicks: number; rssPages: number; startTicks: number }; + +const readProcessStat = async (pid: number): Promise => { + const raw = await readFile(`/proc/${pid}/stat`, 'utf8'); + const commandEnd = raw.lastIndexOf(')'); + if (commandEnd < 0) throw new Error('LOAD_TEST_API_PID has an invalid /proc stat record'); + const fields = raw.slice(commandEnd + 2).trim().split(/\s+/u); + const cpuTicks = Number(fields[11]) + Number(fields[12]); + const startTicks = Number(fields[19]); + const rssPages = Number(fields[21]); + if (![cpuTicks, startTicks, rssPages].every(Number.isFinite)) { + throw new Error('LOAD_TEST_API_PID has an incomplete /proc stat record'); + } + return { cpuTicks, startTicks, rssPages }; +}; + +const startApiProcessSampler = async (pid: number, workspaceRoot: string) => { + const [commandLine, processCwd] = await Promise.all([ + readFile(`/proc/${pid}/cmdline`, 'utf8').then((value) => value.replaceAll('\0', ' ')), + realpath(`/proc/${pid}/cwd`), + ]); + if (!commandLine.includes('game-api') || processCwd !== workspaceRoot) { + throw new Error('LOAD_TEST_API_PID must be the game-api process from this workspace'); + } + const [{ stdout: clockText }, { stdout: pageText }, initial] = await Promise.all([ + execFileAsync('getconf', ['CLK_TCK']), + execFileAsync('getconf', ['PAGESIZE']), + readProcessStat(pid), + ]); + const clockTicks = Number(clockText.trim()); + const pageSize = Number(pageText.trim()); + if (!Number.isFinite(clockTicks) || !Number.isFinite(pageSize)) { + throw new Error('could not determine process accounting units'); + } + let sampling = true; + let maxRssPages = initial.rssPages; + let samples = 1; + const startedNs = process.hrtime.bigint(); + const loop = (async () => { + while (sampling) { + await wait(250); + if (!sampling) break; + const sample = await readProcessStat(pid); + if (sample.startTicks !== initial.startTicks) throw new Error('game-api process changed during measurement'); + maxRssPages = Math.max(maxRssPages, sample.rssPages); + samples += 1; + } + })(); + return async () => { + sampling = false; + await loop; + const final = await readProcessStat(pid); + if (final.startTicks !== initial.startTicks) throw new Error('game-api process changed during measurement'); + const elapsedSeconds = Number(process.hrtime.bigint() - startedNs) / 1_000_000_000; + const cpuSeconds = (final.cpuTicks - initial.cpuTicks) / clockTicks; + return { + pid, + samples, + elapsedMs: Math.round(elapsedSeconds * 1000), + cpuSeconds: Math.round(cpuSeconds * 1000) / 1000, + cpuPercentOfOneCore: Math.round((cpuSeconds / Math.max(elapsedSeconds, 0.001)) * 1000) / 10, + rssBytes: { + initial: initial.rssPages * pageSize, + final: final.rssPages * pageSize, + max: maxRssPages * pageSize, + }, + }; + }; +}; + +const readHealth = async (baseUrl: string) => { + const response = await fetch(new URL('/healthz', baseUrl)); + if (!response.ok) throw new Error(`game-api health returned HTTP ${response.status}`); + const body = (await response.json()) as Record; + const pool = body.postgresPool; + return { + ok: body.ok === true, + postgresPool: + typeof pool === 'object' && pool !== null + ? Object.fromEntries( + ['max', 'total', 'active', 'idle', 'waiting'] + .map((key) => [key, (pool as Record)[key]]) + .filter(([, value]) => typeof value === 'number') + ) + : {}, + }; +}; + +const assertFrontendUrl = (value: string, config: LoadConfig): URL => { + const url = new URL(value); + if (!['http:', 'https:'].includes(url.protocol) || !isPrivateTargetHost(url.hostname)) { + throw new Error('LOAD_TEST_FRONTEND_URL must use HTTP(S) on a private or loopback host'); + } + if (url.username || url.password || url.search || url.hash) { + throw new Error('LOAD_TEST_FRONTEND_URL must not contain credentials, query, or fragment'); + } + const expectedSuffix = `/${config.isolation.postgresSchema.split('_')[2] ?? ''}/`; + if (!url.pathname.endsWith(expectedSuffix)) { + throw new Error(`LOAD_TEST_FRONTEND_URL must end with the configured profile path ${expectedSuffix}`); + } + return url; +}; + +export const measurePageNavigation = async (options: { + config: LoadConfig; + tokens: readonly string[]; + workspaceRoot: string; + env?: NodeJS.ProcessEnv; +}) => { + const env = options.env ?? process.env; + const frontendUrl = assertFrontendUrl(env.LOAD_TEST_FRONTEND_URL ?? '', options.config); + const apiPid = Number(env.LOAD_TEST_API_PID); + if (!Number.isSafeInteger(apiPid) || apiPid <= 1) throw new Error('LOAD_TEST_API_PID must be a positive process id'); + if (options.tokens.length !== options.config.capacity.authenticatedViewers) { + throw new Error('page navigation requires exactly one token per authenticated viewer'); + } + const dwellMs = Number(env.LOAD_TEST_PAGE_DWELL_MS ?? '1000'); + if (!Number.isSafeInteger(dwellMs) || dwellMs < 250 || dwellMs > 10_000) { + throw new Error('LOAD_TEST_PAGE_DWELL_MS must be an integer from 250 to 10000'); + } + + const routeLatencyMs = new Map(); + const routeSuccess = new Map(); + const routeErrors = new Map(); + const procedureRequests = new Map(); + const procedureErrors = new Map(); + const procedureLatencyMs = new Map(); + let pageErrors = 0; + let permissionErrors = 0; + const pendingResponses = new Set>(); + const browser = await chromium.launch({ headless: true }); + const viewers: Array<{ page: Page; close: () => Promise }> = []; + const activeRouteByPage = new Map(); + let activeSseController: AbortController | null = null; + let activeSseTasks: Promise[] = []; + let activeSseSampleTimer: NodeJS.Timeout | null = null; + const profile = options.config.isolation.postgresSchema.split('_')[2] ?? ''; + try { + for (const token of options.tokens) { + const context = await browser.newContext({ viewport: { width: 1280, height: 900 } }); + await context.addInitScript( + ({ accessToken, profileName }) => { + const storage = ( + globalThis as unknown as { localStorage: { setItem: (key: string, value: string) => void } } + ).localStorage; + storage.setItem('sammo-game-token', accessToken); + storage.setItem('sammo-game-profile', profileName); + }, + { accessToken: token, profileName: profile } + ); + const page = await context.newPage(); + page.setDefaultTimeout(20_000); + page.setDefaultNavigationTimeout(20_000); + await page.goto(new URL('current-city', frontendUrl).toString(), { waitUntil: 'networkidle' }); + const warmupText = await page.locator('body').innerText(); + if (warmupText.includes('권한이 부족합니다')) { + throw new Error('privileged viewer fixture failed the current-city warmup'); + } + viewers.push({ page, close: () => context.close() }); + } + + const sseMetrics = new PhaseMetrics(); + const sseController = new AbortController(); + activeSseController = sseController; + let activeSse = 0; + sseMetrics.sseActiveConnections.push(0); + const sseUrl = new URL(options.config.target.ssePath, options.config.target.baseUrl).toString(); + const sseTasks = options.tokens.map((token) => + runSseConnection({ + url: sseUrl, + token, + signal: sseController.signal, + metrics: sseMetrics, + onActiveChange: (delta) => { + activeSse += delta; + sseMetrics.sseActiveConnections.push(activeSse); + }, + }) + ); + activeSseTasks = sseTasks; + const connectionDeadline = Date.now() + 10_000; + while (activeSse !== options.tokens.length && Date.now() < connectionDeadline) await wait(50); + if (activeSse !== options.tokens.length) { + sseController.abort(); + await Promise.allSettled(sseTasks); + throw new Error(`only ${activeSse} of ${options.tokens.length} SSE connections became active`); + } + + const healthBefore = await readHealth(options.config.target.baseUrl); + let scheduledStartAtEpochMs: number | null = null; + if (env.LOAD_TEST_START_AT_EPOCH_MS) { + scheduledStartAtEpochMs = Number(env.LOAD_TEST_START_AT_EPOCH_MS); + if (!Number.isSafeInteger(scheduledStartAtEpochMs)) { + throw new Error('LOAD_TEST_START_AT_EPOCH_MS must be an integer epoch in milliseconds'); + } + const waitMs = scheduledStartAtEpochMs - Date.now(); + if (waitMs > 120_000) throw new Error('LOAD_TEST_START_AT_EPOCH_MS must be within the next two minutes'); + if (waitMs > 0) await wait(waitMs); + } + const stopApiSampler = await startApiProcessSampler(apiPid, options.workspaceRoot); + const startedAt = new Date().toISOString(); + const started = performance.now(); + const deadline = started + options.config.capacity.turnIntervalMs; + const activeSseDuringMeasurement = [activeSse]; + activeSseSampleTimer = setInterval(() => activeSseDuringMeasurement.push(activeSse), 250); + + for (const [viewerIndex, viewer] of viewers.entries()) { + activeRouteByPage.set(viewer.page, 'warmup'); + viewer.page.on('pageerror', () => { + pageErrors += 1; + }); + viewer.page.on('response', (response) => { + const procedures = parseTrpcProcedures(response.url(), options.config.target.trpcPath); + if (procedures.length === 0) return; + const task = (async () => { + await response.finished().catch(() => null); + const timing = response.request().timing(); + const latencyMs = timing.responseEnd; + for (const procedure of procedures) { + const key = `${activeRouteByPage.get(viewer.page) ?? 'unknown'}:${procedure}`; + increment(procedureRequests, key); + if (response.status() >= 400) increment(procedureErrors, `${key}:http-${response.status()}`); + if (latencyMs >= 0) { + const values = procedureLatencyMs.get(key) ?? []; + values.push(latencyMs); + procedureLatencyMs.set(key, values); + } + } + })().finally(() => pendingResponses.delete(task)); + pendingResponses.add(task); + }); + void viewerIndex; + } + + await Promise.all( + viewers.map(async ({ page }, viewerIndex) => { + let iteration = 0; + while (performance.now() < deadline) { + const route = PAGE_ROUTES[(viewerIndex + iteration) % PAGE_ROUTES.length]!; + activeRouteByPage.set(page, route.name); + const routeStarted = performance.now(); + try { + await page.goto(new URL(route.path, frontendUrl).toString(), { waitUntil: 'networkidle' }); + const bodyText = await page.locator('body').innerText(); + if (bodyText.includes('권한이 부족합니다')) { + permissionErrors += 1; + increment(routeErrors, `${route.name}:permission`); + } else { + increment(routeSuccess, route.name); + } + } catch { + increment(routeErrors, `${route.name}:navigation`); + } finally { + const values = routeLatencyMs.get(route.name) ?? []; + values.push(performance.now() - routeStarted); + routeLatencyMs.set(route.name, values); + } + iteration += 1; + if (performance.now() < deadline) await wait(dwellMs); + } + }) + ); + clearInterval(activeSseSampleTimer); + activeSseSampleTimer = null; + activeSseDuringMeasurement.push(activeSse); + await Promise.allSettled([...pendingResponses]); + const apiProcess = await stopApiSampler(); + const elapsedMs = Math.round(performance.now() - started); + const healthAfter = await readHealth(options.config.target.baseUrl); + sseController.abort(); + await Promise.allSettled(sseTasks); + sseMetrics.sseActiveConnections.push(activeSse); + + return { + formatVersion: 1, + startedAt, + finishedAt: new Date().toISOString(), + configuredDurationMs: options.config.capacity.turnIntervalMs, + elapsedMs, + scheduling: { + scheduledStartAtEpochMs, + actualStartAtEpochMs: Date.parse(startedAt), + startDelayMs: + scheduledStartAtEpochMs === null ? null : Math.max(0, Date.parse(startedAt) - scheduledStartAtEpochMs), + }, + fixture: { + name: options.config.name, + viewers: options.tokens.length, + npcGenerals: options.config.capacity.npcGenerals, + pageDwellMs: dwellMs, + }, + browser: { + name: 'chromium', + contexts: viewers.length, + viewport: { width: 1280, height: 900 }, + routes: Object.fromEntries( + PAGE_ROUTES.map((route) => [ + route.name, + { + success: routeSuccess.get(route.name) ?? 0, + latencyMs: summarizeDistribution(routeLatencyMs.get(route.name) ?? []), + }, + ]) + ), + routeErrors: mapToObject(routeErrors), + pageErrors, + permissionErrors, + }, + trpc: { + requests: mapToObject(procedureRequests), + errors: mapToObject(procedureErrors), + latencyMs: Object.fromEntries( + [...procedureLatencyMs.entries()] + .sort(([left], [right]) => left.localeCompare(right)) + .map(([key, values]) => [key, summarizeDistribution(values)]) + ), + }, + sse: { + attempts: sseMetrics.sseAttempts, + opened: sseMetrics.sseOpened, + closed: sseMetrics.sseClosed, + reconnects: sseMetrics.sseReconnects, + failures: sseMetrics.sseFailures, + privacyViolations: sseMetrics.ssePrivacyViolations, + events: mapToObject(sseMetrics.sseEvents), + activeConnections: summarizeDistribution(activeSseDuringMeasurement), + }, + server: { + healthBefore, + healthAfter, + apiProcess, + }, + }; + } finally { + if (activeSseSampleTimer) clearInterval(activeSseSampleTimer); + activeSseController?.abort(); + await Promise.allSettled(activeSseTasks); + await Promise.allSettled(viewers.map((viewer) => viewer.close())); + await browser.close(); + } +}; diff --git a/tools/load-tests/src/turnFlush.ts b/tools/load-tests/src/turnFlush.ts index 74765594..258469b1 100644 --- a/tools/load-tests/src/turnFlush.ts +++ b/tools/load-tests/src/turnFlush.ts @@ -105,6 +105,7 @@ const includeSubMillisecondGameTick = (turnTime: Date): Date => export const measureTurnFlush = async (options: { config: LoadConfig; confirmation: string; + paced?: boolean; env?: NodeJS.ProcessEnv; }) => { const env = options.env ?? process.env; @@ -153,9 +154,8 @@ export const measureTurnFlush = async (options: { const activityPromise = sampleActivity(); const histogram = monitorEventLoopDelay({ resolution: 20 }); - histogram.enable(); - const cpuStart = process.cpuUsage(); - const wallStartNs = process.hrtime.bigint(); + let cpuStart: NodeJS.CpuUsage | null = null; + let wallStartNs: bigint | null = null; let maxRssBytes = process.memoryUsage().rss; const generalTransactionMs: number[] = []; const monthlyTransactionMs: number[] = []; @@ -167,6 +167,10 @@ export const measureTurnFlush = async (options: { let endYearMonth: string | null = null; let initialGeneralCount: number | null = null; let finalGeneralCount: number | null = null; + let scheduledStartAtEpochMs: number | null = null; + let actualStartAtEpochMs: number | null = null; + const scheduleLagMs: number[] = []; + let backdatedGeneralTurns = 0; let runError: unknown; try { @@ -189,6 +193,21 @@ export const measureTurnFlush = async (options: { const boundary = getNextTickTime(initialState.lastTurnTime, tickMinutes); let checkpoint: TurnCheckpoint | undefined = await runtime.stateStore.loadCheckpoint(); + if (env.LOAD_TEST_START_AT_EPOCH_MS) { + scheduledStartAtEpochMs = Number(env.LOAD_TEST_START_AT_EPOCH_MS); + if (!Number.isSafeInteger(scheduledStartAtEpochMs)) { + throw new Error('LOAD_TEST_START_AT_EPOCH_MS must be an integer epoch in milliseconds'); + } + const waitMs = scheduledStartAtEpochMs - Date.now(); + if (waitMs > 120_000) throw new Error('LOAD_TEST_START_AT_EPOCH_MS must be within the next two minutes'); + if (waitMs > 0) await new Promise((resolve) => setTimeout(resolve, waitMs)); + } + actualStartAtEpochMs = Date.now(); + const pacingStarted = performance.now(); + histogram.enable(); + cpuStart = process.cpuUsage(); + wallStartNs = process.hrtime.bigint(); + const execute = async (target: Date, maxGenerals: number): Promise => { const started = performance.now(); const result = await runtime!.stateManager.transaction(async () => { @@ -219,6 +238,16 @@ export const measureTurnFlush = async (options: { while (true) { const nextGeneral = await runtime.stateStore.loadNextGeneralTurnTime(); if (!nextGeneral || nextGeneral.getTime() >= boundary.getTime()) break; + if (options.paced) { + const targetElapsedMs = nextGeneral.getTime() - initialState.lastTurnTime.getTime(); + if (targetElapsedMs < 0) { + backdatedGeneralTurns += 1; + } else { + const remainingMs = targetElapsedMs - (performance.now() - pacingStarted); + if (remainingMs > 0) await new Promise((resolve) => setTimeout(resolve, remainingMs)); + scheduleLagMs.push(Math.max(0, performance.now() - pacingStarted - targetElapsedMs)); + } + } const result = await execute(includeSubMillisecondGameTick(nextGeneral), 1); if (result.processedGenerals !== 1 || result.processedTurns !== 0) { throw new Error( @@ -227,6 +256,11 @@ export const measureTurnFlush = async (options: { } } + if (options.paced) { + const remainingMs = + options.config.capacity.turnIntervalMs - (performance.now() - pacingStarted); + if (remainingMs > 0) await new Promise((resolve) => setTimeout(resolve, remainingMs)); + } const monthly = await execute(boundary, 200); if (monthly.processedTurns !== 1) { throw new Error('turn-flush measurement did not cross exactly one monthly boundary'); @@ -275,6 +309,9 @@ export const measureTurnFlush = async (options: { } await observer.disconnect(); histogram.disable(); + if (cpuStart === null || wallStartNs === null || actualStartAtEpochMs === null) { + throw new Error('turn-flush measurement did not start its measured interval'); + } const elapsedMs = Number(process.hrtime.bigint() - wallStartNs) / 1_000_000; const cpu = process.cpuUsage(cpuStart); const cpuMs = (cpu.user + cpu.system) / 1_000; @@ -287,7 +324,9 @@ export const measureTurnFlush = async (options: { fixtureSha256: fixture.fixtureSha256, capacity: options.config.capacity, }, - mode: 'chronological-one-general-per-transaction-plus-month-boundary', + mode: options.paced + ? 'wall-clock-paced-one-general-per-transaction-plus-month-boundary' + : 'chronological-one-general-per-transaction-plus-month-boundary', startYearMonth, endYearMonth, elapsedMs: round(elapsedMs), @@ -306,6 +345,16 @@ export const measureTurnFlush = async (options: { monthlyTransaction: summarizeDistribution(monthlyTransactionMs), redisPublication: summarizeDistribution(publicationMs), }, + scheduling: { + paced: options.paced === true, + configuredTurnIntervalMs: options.config.capacity.turnIntervalMs, + scheduledStartAtEpochMs, + actualStartAtEpochMs, + startDelayMs: + scheduledStartAtEpochMs === null ? null : Math.max(0, actualStartAtEpochMs - scheduledStartAtEpochMs), + backdatedGeneralTurns, + generalScheduleLagMs: summarizeDistribution(scheduleLagMs), + }, postgres: { statsScope: 'database-wide-including-observer-sampler', statsDelta: subtractDatabaseStats(beforeStats, afterStats), diff --git a/tools/load-tests/test/fixture.test.ts b/tools/load-tests/test/fixture.test.ts index 3a32b598..3d2a0766 100644 --- a/tools/load-tests/test/fixture.test.ts +++ b/tools/load-tests/test/fixture.test.ts @@ -5,10 +5,22 @@ import path from 'node:path'; import test from 'node:test'; import { validateLoadConfig } from '../src/config.js'; -import { activateCapacityCoverage, assertFixtureIsolation, prepareCapacitySecrets } from '../src/fixture.js'; +import { + activateCapacityCoverage, + assertFixtureIsolation, + prepareCapacitySecrets, + privilegedViewerPlacement, +} from '../src/fixture.js'; const samplePath = new URL('../config/300-users-900-npcs-5m.json', import.meta.url); +void test('viewer placement stays in one nation and covers every privileged chief level deterministically', () => { + assert.deepEqual( + Array.from({ length: 10 }, (_, index) => privilegedViewerPlacement(index, 3, 7)), + [12, 10, 8, 6, 11, 9, 7, 5, 12, 10].map((officerLevel) => ({ nationId: 3, cityId: 7, officerLevel })) + ); +}); + void test('fixture accepts only the configured private schema and dedicated Redis database', async () => { const config = validateLoadConfig(JSON.parse(await readFile(samplePath, 'utf8'))); assert.doesNotThrow(() => diff --git a/tools/load-tests/test/pageNavigation.test.ts b/tools/load-tests/test/pageNavigation.test.ts new file mode 100644 index 00000000..c0cbbd4a --- /dev/null +++ b/tools/load-tests/test/pageNavigation.test.ts @@ -0,0 +1,19 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; + +import { parseTrpcProcedures } from '../src/pageNavigation.js'; + +void test('tRPC page-load requests are attributed for single and batched procedures', () => { + assert.deepEqual( + parseTrpcProcedures('http://127.0.0.1:15001/api/trpc/auth.status?batch=1&input=%7B%7D', '/api/trpc'), + ['auth.status'] + ); + assert.deepEqual( + parseTrpcProcedures( + 'http://127.0.0.1:15001/api/trpc/world.getMap%2Cworld.getMapLayout?batch=1', + '/api/trpc' + ), + ['world.getMap', 'world.getMapLayout'] + ); + assert.deepEqual(parseTrpcProcedures('http://127.0.0.1:15001/healthz', '/api/trpc'), []); +});