From 54126cd67fb1e54462e7fc51cc48c579fccd293c Mon Sep 17 00:00:00 2001 From: hided62 Date: Sun, 16 Aug 2026 18:05:55 +0000 Subject: [PATCH] =?UTF-8?q?perf:=20300=EB=AA=85=20=EC=8B=A4=EC=8B=9C?= =?UTF-8?q?=EA=B0=84=20=EB=B6=80=ED=95=98=20=EC=B8=A1=EC=A0=95=20=EB=8F=84?= =?UTF-8?q?=EA=B5=AC=EB=A5=BC=20=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tools/load-tests/.gitignore | 4 + tools/load-tests/README.md | 75 ++++++ .../config/300-users-900-npcs-5m.json | 103 +++++++ tools/load-tests/config/load-test.schema.json | 96 +++++++ tools/load-tests/package.json | 13 + tools/load-tests/results/.gitkeep | 1 + tools/load-tests/secrets/.gitkeep | 1 + tools/load-tests/src/cli.ts | 63 +++++ tools/load-tests/src/config.ts | 251 ++++++++++++++++++ tools/load-tests/src/metrics.ts | 168 ++++++++++++ tools/load-tests/src/runner.ts | 201 ++++++++++++++ tools/load-tests/src/sse.ts | 134 ++++++++++ tools/load-tests/src/trpc.ts | 104 ++++++++ tools/load-tests/test/config.test.ts | 69 +++++ tools/load-tests/test/metrics.test.ts | 42 +++ tools/load-tests/test/sse.test.ts | 19 ++ tools/load-tests/test/trpc.test.ts | 35 +++ tools/load-tests/tsconfig.json | 8 + 18 files changed, 1387 insertions(+) create mode 100644 tools/load-tests/.gitignore create mode 100644 tools/load-tests/README.md create mode 100644 tools/load-tests/config/300-users-900-npcs-5m.json create mode 100644 tools/load-tests/config/load-test.schema.json create mode 100644 tools/load-tests/package.json create mode 100644 tools/load-tests/results/.gitkeep create mode 100644 tools/load-tests/secrets/.gitkeep create mode 100644 tools/load-tests/src/cli.ts create mode 100644 tools/load-tests/src/config.ts create mode 100644 tools/load-tests/src/metrics.ts create mode 100644 tools/load-tests/src/runner.ts create mode 100644 tools/load-tests/src/sse.ts create mode 100644 tools/load-tests/src/trpc.ts create mode 100644 tools/load-tests/test/config.test.ts create mode 100644 tools/load-tests/test/metrics.test.ts create mode 100644 tools/load-tests/test/sse.test.ts create mode 100644 tools/load-tests/test/trpc.test.ts create mode 100644 tools/load-tests/tsconfig.json diff --git a/tools/load-tests/.gitignore b/tools/load-tests/.gitignore new file mode 100644 index 00000000..5bd280dd --- /dev/null +++ b/tools/load-tests/.gitignore @@ -0,0 +1,4 @@ +secrets/*.json +results/*.json +!secrets/.gitkeep +!results/.gitkeep diff --git a/tools/load-tests/README.md b/tools/load-tests/README.md new file mode 100644 index 00000000..847902bf --- /dev/null +++ b/tools/load-tests/README.md @@ -0,0 +1,75 @@ +# 인증 HTTP/tRPC + SSE 부하 도구 + +이 package는 `docs/architecture/realtime-change-journal.md`의 A1/A2/A3/M1 viewer 부하를 +재현하기 위한 read-only driver다. 300개 game bearer token으로 SSE를 열고, idle/own/global/mixed +phase에서 실제 game API tRPC query를 실행한다. HTTP latency p50/p95/p99, 성공/오류, SSE +open/close/reconnect/event와 public payload 금지 field 수, driver CPU/RSS/event-loop lag를 raw JSON에 +남긴다. token 값, 사용자/장수/도시/국가 ID와 response/event payload는 출력하지 않는다. +dashboard query의 opaque revision은 viewer별 메모리에서만 다음 `known` 입력으로 이어서 +unchanged/snapshot/patch 경로를 구분하며 raw JSON에는 종류별 count만 남긴다. + +## 안전 경계 + +- 운영/public profile에 실행하지 않는다. config의 `publicProfile`은 반드시 `false`이고 target hostname은 + `allowedHosts`에 정확히 있어야 하며 loopback, RFC 1918/ULA 또는 `.local`/`.internal`이어야 한다. 이 + guard를 우회하는 CLI flag는 없다. +- 전용 PostgreSQL schema는 `load_`로, Redis prefix는 `load-tests:`로 시작해야 한다. fixture/runtime을 + 기동하는 외부 orchestration에도 같은 값을 주어 공유 개발·운영 profile과 분리한다. +- driver는 query만 허용한다. own/global phase 이름은 invalidation 뒤 viewer read fan-out을 뜻하며 mutation을 + 만들지 않는다. 실제 own/global change stimulus는 격리 runtime에서 별도 orchestration으로 발생시킨다. +- token 파일은 이 workspace 안의 Git ignored path여야 하고 정확히 `0600`이어야 한다. 권장 위치는 + `tools/load-tests/secrets/game-tokens.json`이며 JSON 형식은 `{"tokens":["...", "..."]}` 하나뿐이다. +- raw result는 새 파일로만 쓰고(`wx`) `0600`을 적용한다. 기본 ignored 위치는 + `tools/load-tests/results/`다. + +## 재현 명령 + +먼저 sample의 `runtimeMetadata` placeholder를 실제 fixture SHA-256, image digest, PostgreSQL/Redis +version으로 바꾼 복사본을 만든다. secret이나 ID를 config에 넣지 않는다. + +```sh +install -m 600 /dev/null tools/load-tests/secrets/game-tokens.json +# 편집기로 300개 synthetic game bearer token을 tokens 배열에 입력 + +pnpm --filter @sammo-ts/load-tests validate --config tools/load-tests/config/300-users-900-npcs-5m.json +pnpm --filter @sammo-ts/load-tests dry-run \ + --config tools/load-tests/config/300-users-900-npcs-5m.json \ + --tokens tools/load-tests/secrets/game-tokens.json +pnpm --filter @sammo-ts/load-tests run \ + --config tools/load-tests/config/300-users-900-npcs-5m.json \ + --tokens tools/load-tests/secrets/game-tokens.json \ + --output tools/load-tests/results/300-users-900-npcs-5m.json +``` + +`validate`는 config만 검사한다. `dry-run`은 config, host allowlist, token count/permission/Git-ignore와 phase +계획을 검사하지만 network connection을 열지 않는다. driver process는 가능하면 target runtime과 다른 +host/cgroup에서 실행하고 두 host의 CPU quota와 competing load를 별도로 기록한다. +`run`은 sample의 runtime metadata placeholder가 하나라도 남아 있으면 시작하지 않는다. + +## 결과와 해석 + +raw JSON은 Git commit/tree/dirty 상태, config/runtime/host hash, Node/V8, host CPU/memory와 cgroup limit, +fixture/image/PostgreSQL/Redis metadata, phase별 metric을 포함한다. 이 정보는 재현 조건이지 합격 판정 자체가 +아니다. `runtimeMetadata` placeholder가 남은 run과 외부 stimulus가 없던 own/global phase를 capacity pass로 +보고하지 않는다. + +E1의 DB-free 자연 통일 계산은 기존 실행 가능한 benchmark를 그대로 사용한다. + +```sh +NPC_UNIFICATION_BENCHMARK_CONVERGENCE_ASSIST=none \ +NPC_UNIFICATION_BENCHMARK_MAX_YEAR=300 \ +NPC_UNIFICATION_BENCHMARK_REPORT_PATH=/dev/shm/npc-unification.json \ +pnpm --filter @sammo-ts/game-engine profile:npc-unification-timing +``` + +현재 E1 command의 기본 fixture는 문서상 880 NPC/10분 턴이므로 900 NPC/5분 또는 총 1,200장수 E1이라고 +바꿔 부르지 않는다. E2는 실제 daemon fast-forward, PostgreSQL flush, Redis publish와 schedule-lag/DB +statement 계측을 한 lifecycle로 묶는 안전한 fixture API가 아직 없어 stub을 추가하지 않았다. 따라서 이 +package 단독 실행은 E2나 M1 전체 합격 근거가 아니다. + +## 도구 자체 검증 + +```sh +pnpm --filter @sammo-ts/load-tests test +pnpm --filter @sammo-ts/load-tests typecheck +``` diff --git a/tools/load-tests/config/300-users-900-npcs-5m.json b/tools/load-tests/config/300-users-900-npcs-5m.json new file mode 100644 index 00000000..92c18103 --- /dev/null +++ b/tools/load-tests/config/300-users-900-npcs-5m.json @@ -0,0 +1,103 @@ +{ + "$schema": "./load-test.schema.json", + "version": 1, + "name": "300-users-900-npcs-5m", + "target": { + "baseUrl": "http://127.0.0.1:15001", + "trpcPath": "/api/trpc", + "ssePath": "/events", + "publicProfile": false, + "allowedHosts": ["127.0.0.1", "localhost"] + }, + "isolation": { + "postgresSchema": "load_capacity_300_900_5m", + "redisPrefix": "load-tests:capacity-300-900-5m:" + }, + "capacity": { + "authenticatedViewers": 300, + "npcGenerals": 900, + "humanGenerals": 300, + "turnIntervalMs": 300000 + }, + "runtimeMetadata": { + "fixtureSha256": "sha256:0000000000000000000000000000000000000000000000000000000000000000", + "imageDigest": "replace-before-measurement", + "postgresVersion": "replace-before-measurement", + "redisVersion": "replace-before-measurement" + }, + "phases": [ + { + "name": "idle-30m", + "kind": "idle", + "durationMs": 1800000, + "sseConnections": 300, + "requestIntervalMs": null, + "operations": [] + }, + { + "name": "own-context-5m", + "kind": "own", + "durationMs": 300000, + "sseConnections": 300, + "requestIntervalMs": 1000, + "operations": [ + { + "name": "own-context", + "procedure": "dashboard.getContextBundleDelta", + "type": "query", + "weight": 1, + "input": { + "include": { "context": true, "commandTable": true, "boardAccess": true }, + "forceSnapshot": false + } + } + ] + }, + { + "name": "global-slices-10m", + "kind": "global", + "durationMs": 600000, + "sseConnections": 300, + "requestIntervalMs": 5000, + "operations": [ + { + "name": "global-records", + "procedure": "general.getRecentRecords", + "type": "query", + "weight": 1, + "input": { "lastGeneralRecordId": 0, "lastWorldHistoryId": 0 } + }, + { "name": "global-front", "procedure": "general.getFrontStatus", "type": "query", "weight": 1 }, + { "name": "global-lobby", "procedure": "lobby.info", "type": "query", "weight": 1 } + ] + }, + { + "name": "mixed-30m", + "kind": "mixed", + "durationMs": 1800000, + "sseConnections": 300, + "requestIntervalMs": 1000, + "operations": [ + { + "name": "own-context", + "procedure": "dashboard.getContextBundleDelta", + "type": "query", + "weight": 5, + "input": { + "include": { "context": true, "commandTable": true, "boardAccess": true }, + "forceSnapshot": false + } + }, + { + "name": "global-records", + "procedure": "general.getRecentRecords", + "type": "query", + "weight": 1, + "input": { "lastGeneralRecordId": 0, "lastWorldHistoryId": 0 } + }, + { "name": "global-front", "procedure": "general.getFrontStatus", "type": "query", "weight": 1 }, + { "name": "global-lobby", "procedure": "lobby.info", "type": "query", "weight": 1 } + ] + } + ] +} diff --git a/tools/load-tests/config/load-test.schema.json b/tools/load-tests/config/load-test.schema.json new file mode 100644 index 00000000..2ca4606b --- /dev/null +++ b/tools/load-tests/config/load-test.schema.json @@ -0,0 +1,96 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://sammo-ts.local/schemas/load-test.schema.json", + "title": "SAMMO authenticated API and SSE load configuration", + "type": "object", + "additionalProperties": false, + "required": ["version", "name", "target", "isolation", "capacity", "runtimeMetadata", "phases"], + "properties": { + "$schema": { "type": "string" }, + "version": { "const": 1 }, + "name": { "type": "string", "pattern": "^[a-z0-9][a-z0-9-]{0,63}$" }, + "target": { + "type": "object", + "additionalProperties": false, + "required": ["baseUrl", "trpcPath", "ssePath", "publicProfile", "allowedHosts"], + "properties": { + "baseUrl": { "type": "string", "format": "uri", "pattern": "^https?://" }, + "trpcPath": { "type": "string", "pattern": "^/" }, + "ssePath": { "type": "string", "pattern": "^/" }, + "publicProfile": { "const": false }, + "allowedHosts": { + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { "type": "string", "minLength": 1 } + } + } + }, + "isolation": { + "type": "object", + "additionalProperties": false, + "required": ["postgresSchema", "redisPrefix"], + "properties": { + "postgresSchema": { "type": "string", "pattern": "^load_[a-z0-9_]+$" }, + "redisPrefix": { "type": "string", "pattern": "^load-tests:[a-z0-9:_-]+:$" } + } + }, + "capacity": { + "type": "object", + "additionalProperties": false, + "required": ["authenticatedViewers", "npcGenerals", "humanGenerals", "turnIntervalMs"], + "properties": { + "authenticatedViewers": { "type": "integer", "minimum": 1 }, + "npcGenerals": { "type": "integer", "minimum": 0 }, + "humanGenerals": { "type": "integer", "minimum": 0 }, + "turnIntervalMs": { "type": "integer", "minimum": 1000 } + } + }, + "runtimeMetadata": { + "type": "object", + "additionalProperties": false, + "required": ["fixtureSha256", "imageDigest", "postgresVersion", "redisVersion"], + "properties": { + "fixtureSha256": { "type": "string", "pattern": "^sha256:[a-f0-9]{64}$" }, + "imageDigest": { "type": "string", "minLength": 1 }, + "postgresVersion": { "type": "string", "minLength": 1 }, + "redisVersion": { "type": "string", "minLength": 1 } + } + }, + "phases": { + "type": "array", + "minItems": 4, + "items": { "$ref": "#/$defs/phase" } + } + }, + "$defs": { + "phase": { + "type": "object", + "additionalProperties": false, + "required": ["name", "kind", "durationMs", "sseConnections", "requestIntervalMs", "operations"], + "properties": { + "name": { "type": "string", "pattern": "^[a-z0-9][a-z0-9-]{0,31}$" }, + "kind": { "enum": ["idle", "own", "global", "mixed"] }, + "durationMs": { "type": "integer", "minimum": 1000 }, + "sseConnections": { "type": "integer", "minimum": 0 }, + "requestIntervalMs": { "type": ["integer", "null"], "minimum": 50 }, + "operations": { + "type": "array", + "items": { "$ref": "#/$defs/operation" } + } + } + }, + "operation": { + "type": "object", + "additionalProperties": false, + "required": ["name", "procedure", "type", "weight"], + "properties": { + "name": { "type": "string", "pattern": "^[a-z0-9][a-z0-9-]{0,31}$" }, + "procedure": { "type": "string", "pattern": "^[A-Za-z][A-Za-z0-9_.]+$" }, + "type": { "const": "query" }, + "weight": { "type": "integer", "minimum": 1, "maximum": 100 }, + "input": {} + } + } + } +} diff --git a/tools/load-tests/package.json b/tools/load-tests/package.json new file mode 100644 index 00000000..02572403 --- /dev/null +++ b/tools/load-tests/package.json @@ -0,0 +1,13 @@ +{ + "name": "@sammo-ts/load-tests", + "private": true, + "version": "0.0.0", + "type": "module", + "scripts": { + "run": "pnpm -w exec tsx tools/load-tests/src/cli.ts run", + "dry-run": "pnpm -w exec tsx tools/load-tests/src/cli.ts dry-run", + "validate": "pnpm -w exec tsx tools/load-tests/src/cli.ts validate", + "test": "pnpm -w exec tsx --test tools/load-tests/test/*.test.ts", + "typecheck": "pnpm -w tsc7 -p tools/load-tests/tsconfig.json --noEmit" + } +} diff --git a/tools/load-tests/results/.gitkeep b/tools/load-tests/results/.gitkeep new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/tools/load-tests/results/.gitkeep @@ -0,0 +1 @@ + diff --git a/tools/load-tests/secrets/.gitkeep b/tools/load-tests/secrets/.gitkeep new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/tools/load-tests/secrets/.gitkeep @@ -0,0 +1 @@ + diff --git a/tools/load-tests/src/cli.ts b/tools/load-tests/src/cli.ts new file mode 100644 index 00000000..7a5d5bf2 --- /dev/null +++ b/tools/load-tests/src/cli.ts @@ -0,0 +1,63 @@ +import { mkdir, writeFile } from 'node:fs/promises'; +import path from 'node:path'; + +import { assertRuntimeMetadataFinalized, loadConfig, loadTokens } from './config.js'; +import { describeDryRun, runLoadTest } from './runner.js'; + +type Command = 'run' | 'dry-run' | 'validate'; + +const usage = (): never => { + process.stderr.write('usage: cli.ts --config [--tokens <0600-gitignored-file>] [--output ]\n'); + process.exit(64); +}; + +const parseArguments = (argv: readonly string[]): { command: Command; config: string; tokens?: string; output?: string } => { + const command = argv[0]; + if (!['run', 'dry-run', 'validate'].includes(command ?? '')) usage(); + const values = new Map(); + for (let index = 1; index < argv.length; index += 2) { + const flag = argv[index]; + const value = argv[index + 1]; + if (!flag || !['--config', '--tokens', '--output'].includes(flag) || !value) usage(); + values.set(flag, value); + } + const config = values.get('--config'); + if (!config) usage(); + if (command === 'run' && (!values.get('--tokens') || !values.get('--output'))) usage(); + if (command === 'validate' && (values.has('--tokens') || values.has('--output'))) usage(); + if (command === 'dry-run' && values.has('--output')) usage(); + return { + command: command as Command, + config: config!, + ...(values.get('--tokens') ? { tokens: values.get('--tokens')! } : {}), + ...(values.get('--output') ? { output: values.get('--output')! } : {}), + }; +}; + +const main = async (): Promise => { + const args = parseArguments(process.argv.slice(2)); + const workspaceRoot = path.resolve(import.meta.dirname, '../../..'); + const { config, sha256 } = await loadConfig(args.config); + if (args.command === 'validate') { + process.stdout.write(`${JSON.stringify({ valid: true, name: config.name, configSha256: sha256 })}\n`); + return; + } + const tokens = args.tokens ? await loadTokens(args.tokens, workspaceRoot, config.capacity.authenticatedViewers) : null; + if (args.command === 'dry-run') { + process.stdout.write(`${JSON.stringify({ valid: true, tokenFileValidated: tokens !== null, configSha256: sha256, plan: describeDryRun(config) }, null, 2)}\n`); + return; + } + assertRuntimeMetadataFinalized(config); + const output = path.resolve(args.output!); + await mkdir(path.dirname(output), { recursive: true }); + const result = await runLoadTest({ config, configSha256: sha256, tokens: tokens!, workspaceRoot }); + await writeFile(output, `${JSON.stringify(result, null, 2)}\n`, { encoding: 'utf8', flag: 'wx', mode: 0o600 }); + const failedRequests = result.phases.reduce((total, phase) => total + Object.values(phase.metrics.http.errors).reduce((sum, count) => sum + count, 0), 0); + process.stdout.write(`${JSON.stringify({ completed: true, phases: result.phases.length, failedRequests, outputWritten: true })}\n`); +}; + +main().catch((error: unknown) => { + const message = error instanceof Error ? error.message : 'unknown load-test error'; + process.stderr.write(`${message}\n`); + process.exitCode = 1; +}); diff --git a/tools/load-tests/src/config.ts b/tools/load-tests/src/config.ts new file mode 100644 index 00000000..bb4c8160 --- /dev/null +++ b/tools/load-tests/src/config.ts @@ -0,0 +1,251 @@ +import { createHash } from 'node:crypto'; +import { lstat, readFile, realpath, stat } from 'node:fs/promises'; +import { execFile } from 'node:child_process'; +import path from 'node:path'; +import { promisify } from 'node:util'; + +const execFileAsync = promisify(execFile); + +export type PhaseKind = 'idle' | 'own' | 'global' | 'mixed'; + +export interface LoadOperation { + name: string; + procedure: string; + type: 'query'; + weight: number; + input?: unknown; +} + +export interface LoadPhase { + name: string; + kind: PhaseKind; + durationMs: number; + sseConnections: number; + requestIntervalMs: number | null; + operations: LoadOperation[]; +} + +export interface LoadConfig { + $schema?: string; + version: 1; + name: string; + target: { + baseUrl: string; + trpcPath: string; + ssePath: string; + publicProfile: false; + allowedHosts: string[]; + }; + isolation: { + postgresSchema: string; + redisPrefix: string; + }; + capacity: { + authenticatedViewers: number; + npcGenerals: number; + humanGenerals: number; + turnIntervalMs: number; + }; + runtimeMetadata: { + fixtureSha256: string; + imageDigest: string; + postgresVersion: string; + redisVersion: string; + }; + phases: LoadPhase[]; +} + +const isRecord = (value: unknown): value is Record => + typeof value === 'object' && value !== null && !Array.isArray(value); + +const integerAtLeast = (value: unknown, minimum: number): boolean => + typeof value === 'number' && Number.isInteger(value) && value >= minimum; + +const hasOnlyKeys = (value: Record, allowed: readonly string[]): boolean => + Object.keys(value).every((key) => allowed.includes(key)); + +const isPrivateTargetHost = (hostname: string): boolean => { + const normalized = hostname.toLowerCase().replace(/^\[|\]$/gu, ''); + if (normalized === 'localhost' || normalized === '::1' || normalized.endsWith('.localhost')) return true; + if (normalized.endsWith('.internal') || normalized.endsWith('.local')) return true; + const parts = normalized.split('.').map(Number); + if (parts.length !== 4 || parts.some((part) => !Number.isInteger(part) || part < 0 || part > 255)) { + return normalized.startsWith('fc') || normalized.startsWith('fd') || normalized.startsWith('fe8') || normalized.startsWith('fe9') || normalized.startsWith('fea') || normalized.startsWith('feb'); + } + return parts[0] === 10 || parts[0] === 127 || (parts[0] === 192 && parts[1] === 168) || (parts[0] === 172 && parts[1]! >= 16 && parts[1]! <= 31); +}; + +export const validateLoadConfig = (raw: unknown): LoadConfig => { + const issues: string[] = []; + if (!isRecord(raw)) throw new Error('config must be a JSON object'); + if (!hasOnlyKeys(raw, ['$schema', 'version', 'name', 'target', 'isolation', 'capacity', 'runtimeMetadata', 'phases'])) { + issues.push('config contains unknown fields'); + } + if (raw.version !== 1) issues.push('version must be 1'); + if (typeof raw.name !== 'string' || !/^[a-z0-9][a-z0-9-]{0,63}$/u.test(raw.name)) issues.push('name is invalid'); + + const target = raw.target; + if (!isRecord(target)) { + issues.push('target must be an object'); + } else { + if (!hasOnlyKeys(target, ['baseUrl', 'trpcPath', 'ssePath', 'publicProfile', 'allowedHosts'])) { + issues.push('target contains unknown fields'); + } + let parsedUrl: URL | null = null; + try { + parsedUrl = new URL(typeof target.baseUrl === 'string' ? target.baseUrl : 'invalid:'); + } catch { + issues.push('target.baseUrl must be a URL'); + } + if (parsedUrl && !['http:', 'https:'].includes(parsedUrl.protocol)) issues.push('target.baseUrl must use HTTP(S)'); + if (parsedUrl && !isPrivateTargetHost(parsedUrl.hostname)) issues.push('target.baseUrl must use a loopback or private/internal hostname'); + if (parsedUrl && (parsedUrl.username || parsedUrl.password || parsedUrl.search || parsedUrl.hash)) { + issues.push('target.baseUrl must not contain credentials, query, or fragment'); + } + if (target.publicProfile !== false) issues.push('target.publicProfile must be false'); + if (!Array.isArray(target.allowedHosts) || target.allowedHosts.length === 0 || !target.allowedHosts.every((item) => typeof item === 'string' && item.length > 0)) { + issues.push('target.allowedHosts must contain explicit hostnames'); + } else if (parsedUrl && !target.allowedHosts.includes(parsedUrl.hostname)) { + issues.push('target hostname is not explicitly allowlisted'); + } + if (typeof target.trpcPath !== 'string' || !target.trpcPath.startsWith('/')) issues.push('target.trpcPath must be absolute'); + if (typeof target.ssePath !== 'string' || !target.ssePath.startsWith('/')) issues.push('target.ssePath must be absolute'); + } + + const isolation = raw.isolation; + if (!isRecord(isolation)) { + issues.push('isolation must be an object'); + } else { + if (!hasOnlyKeys(isolation, ['postgresSchema', 'redisPrefix'])) issues.push('isolation contains unknown fields'); + if (typeof isolation.postgresSchema !== 'string' || !/^load_[a-z0-9_]+$/u.test(isolation.postgresSchema)) { + issues.push('isolation.postgresSchema must start with load_'); + } + if (typeof isolation.redisPrefix !== 'string' || !/^load-tests:[a-z0-9:_-]+:$/u.test(isolation.redisPrefix)) { + issues.push('isolation.redisPrefix must be load-tests scoped and end with a colon'); + } + } + + const capacity = raw.capacity; + if (!isRecord(capacity)) { + issues.push('capacity must be an object'); + } else { + if (!hasOnlyKeys(capacity, ['authenticatedViewers', 'npcGenerals', 'humanGenerals', 'turnIntervalMs'])) issues.push('capacity contains unknown fields'); + if (!integerAtLeast(capacity.authenticatedViewers, 1)) issues.push('capacity.authenticatedViewers must be positive'); + if (!integerAtLeast(capacity.npcGenerals, 0)) issues.push('capacity.npcGenerals must be non-negative'); + if (!integerAtLeast(capacity.humanGenerals, 0)) issues.push('capacity.humanGenerals must be non-negative'); + if (!integerAtLeast(capacity.turnIntervalMs, 1000)) issues.push('capacity.turnIntervalMs must be at least 1000'); + } + + const metadata = raw.runtimeMetadata; + if (!isRecord(metadata)) { + issues.push('runtimeMetadata must be an object'); + } else { + if (!hasOnlyKeys(metadata, ['fixtureSha256', 'imageDigest', 'postgresVersion', 'redisVersion'])) issues.push('runtimeMetadata contains unknown fields'); + if (typeof metadata.fixtureSha256 !== 'string' || !/^sha256:[a-f0-9]{64}$/u.test(metadata.fixtureSha256)) issues.push('runtimeMetadata.fixtureSha256 must be a sha256 digest'); + for (const field of ['imageDigest', 'postgresVersion', 'redisVersion'] as const) { + if (typeof metadata[field] !== 'string' || metadata[field].length === 0) issues.push(`runtimeMetadata.${field} is required`); + } + } + + const phases = raw.phases; + if (!Array.isArray(phases)) { + issues.push('phases must be an array'); + } else { + const kinds = new Set(); + for (const [index, phase] of phases.entries()) { + if (!isRecord(phase)) { + issues.push(`phases[${index}] must be an object`); + continue; + } + if (!hasOnlyKeys(phase, ['name', 'kind', 'durationMs', 'sseConnections', 'requestIntervalMs', 'operations'])) issues.push(`phases[${index}] contains unknown fields`); + if (typeof phase.name !== 'string' || !/^[a-z0-9][a-z0-9-]{0,31}$/u.test(phase.name)) issues.push(`phases[${index}].name is invalid`); + if (!['idle', 'own', 'global', 'mixed'].includes(String(phase.kind))) issues.push(`phases[${index}].kind is invalid`); + else kinds.add(String(phase.kind)); + if (!integerAtLeast(phase.durationMs, 1000)) issues.push(`phases[${index}].durationMs must be at least 1000`); + if (!integerAtLeast(phase.sseConnections, 0)) issues.push(`phases[${index}].sseConnections must be non-negative`); + const operations = phase.operations; + if (!Array.isArray(operations)) { + issues.push(`phases[${index}].operations must be an array`); + continue; + } + if (phase.kind === 'idle') { + if (phase.requestIntervalMs !== null || operations.length !== 0) issues.push(`phases[${index}] idle phase must not issue HTTP requests`); + } else if (!integerAtLeast(phase.requestIntervalMs, 50) || operations.length === 0) { + issues.push(`phases[${index}] active phase requires an interval and operations`); + } + for (const [operationIndex, operation] of operations.entries()) { + if (!isRecord(operation)) { + issues.push(`phases[${index}].operations[${operationIndex}] must be an object`); + continue; + } + if (!hasOnlyKeys(operation, ['name', 'procedure', 'type', 'weight', 'input'])) issues.push(`phases[${index}].operations[${operationIndex}] contains unknown fields`); + if (typeof operation.name !== 'string' || !/^[a-z0-9][a-z0-9-]{0,31}$/u.test(operation.name)) issues.push(`phases[${index}].operations[${operationIndex}].name is invalid`); + if (typeof operation.procedure !== 'string' || !/^[A-Za-z][A-Za-z0-9_.]+$/u.test(operation.procedure)) issues.push(`phases[${index}].operations[${operationIndex}].procedure is invalid`); + if (operation.type !== 'query') issues.push(`phases[${index}].operations[${operationIndex}] must be a read-only query`); + if (!integerAtLeast(operation.weight, 1) || Number(operation.weight) > 100) issues.push(`phases[${index}].operations[${operationIndex}].weight must be 1..100`); + } + } + for (const required of ['idle', 'own', 'global', 'mixed']) { + if (!kinds.has(required)) issues.push(`phases must include ${required}`); + } + } + + if (isRecord(capacity) && Array.isArray(phases)) { + for (const [index, phase] of phases.entries()) { + if (isRecord(phase) && typeof phase.sseConnections === 'number' && typeof capacity.authenticatedViewers === 'number' && phase.sseConnections > capacity.authenticatedViewers) { + issues.push(`phases[${index}].sseConnections exceeds authenticatedViewers`); + } + } + } + if (issues.length > 0) throw new Error(`invalid load configuration:\n- ${issues.join('\n- ')}`); + return raw as unknown as LoadConfig; +}; + +export const canonicalJson = (value: unknown): string => { + if (Array.isArray(value)) return `[${value.map(canonicalJson).join(',')}]`; + if (isRecord(value)) return `{${Object.keys(value).sort().map((key) => `${JSON.stringify(key)}:${canonicalJson(value[key])}`).join(',')}}`; + return JSON.stringify(value); +}; + +export const sha256 = (value: string): string => createHash('sha256').update(value).digest('hex'); + +export const loadConfig = async (configPath: string): Promise<{ config: LoadConfig; sha256: string }> => { + const text = await readFile(configPath, 'utf8'); + const parsed: unknown = JSON.parse(text); + const config = validateLoadConfig(parsed); + return { config, sha256: sha256(canonicalJson(config)) }; +}; + +export const assertRuntimeMetadataFinalized = (config: LoadConfig): void => { + const placeholderFields = Object.entries(config.runtimeMetadata) + .filter(([, value]) => value.includes('replace-before-measurement') || /^sha256:0{64}$/u.test(value)) + .map(([key]) => key); + if (placeholderFields.length > 0) { + throw new Error(`runtime metadata placeholders must be replaced before run: ${placeholderFields.join(', ')}`); + } +}; + +export const loadTokens = async (tokenPath: string, workspaceRoot: string, requiredCount: number): Promise => { + const absolute = path.resolve(tokenPath); + const root = await realpath(path.resolve(workspaceRoot)); + const linkStat = await lstat(absolute); + if (linkStat.isSymbolicLink()) throw new Error('token file must not be a symbolic link'); + const resolved = await realpath(absolute); + if (resolved !== root && !resolved.startsWith(`${root}${path.sep}`)) throw new Error('token file must be inside the workspace and gitignored'); + const fileStat = await stat(absolute); + if ((fileStat.mode & 0o777) !== 0o600) throw new Error('token file mode must be exactly 0600'); + try { + await execFileAsync('git', ['check-ignore', '--quiet', '--', resolved], { cwd: root }); + } catch { + throw new Error('token file must be covered by .gitignore'); + } + const parsed: unknown = JSON.parse(await readFile(absolute, 'utf8')); + if (!isRecord(parsed) || !hasOnlyKeys(parsed, ['tokens']) || !Array.isArray(parsed.tokens)) throw new Error('token file must contain only a tokens array'); + if (!parsed.tokens.every((token) => typeof token === 'string' && token.length >= 16)) throw new Error('each bearer token must be a non-empty string of at least 16 characters'); + if (new Set(parsed.tokens).size !== parsed.tokens.length) throw new Error('token file contains duplicate tokens'); + if (parsed.tokens.length < requiredCount) throw new Error(`token file has fewer than ${requiredCount} entries`); + return parsed.tokens.slice(0, requiredCount); +}; + +export const expandWeightedOperations = (operations: readonly LoadOperation[]): LoadOperation[] => + operations.flatMap((operation) => Array.from({ length: operation.weight }, () => operation)); diff --git a/tools/load-tests/src/metrics.ts b/tools/load-tests/src/metrics.ts new file mode 100644 index 00000000..61d8037a --- /dev/null +++ b/tools/load-tests/src/metrics.ts @@ -0,0 +1,168 @@ +import { monitorEventLoopDelay } from 'node:perf_hooks'; + +export interface DistributionSummary { + count: number; + min: number | null; + max: number | null; + mean: number | null; + p50: number | null; + p95: number | null; + p99: number | null; +} + +const rounded = (value: number): number => Math.round(value * 1000) / 1000; + +export const percentile = (sorted: readonly number[], percentileValue: number): number | null => { + if (sorted.length === 0) return null; + if (percentileValue <= 0) return sorted[0] ?? null; + if (percentileValue >= 100) return sorted.at(-1) ?? null; + const rank = Math.ceil((percentileValue / 100) * sorted.length) - 1; + return sorted[Math.max(0, rank)] ?? null; +}; + +export const summarizeDistribution = (values: readonly number[]): DistributionSummary => { + if (values.length === 0) return { count: 0, min: null, max: null, mean: null, p50: null, p95: null, p99: null }; + const sorted = [...values].sort((left, right) => left - right); + const mean = sorted.reduce((total, value) => total + value, 0) / sorted.length; + return { + count: sorted.length, + min: rounded(sorted[0]!), + max: rounded(sorted.at(-1)!), + mean: rounded(mean), + p50: rounded(percentile(sorted, 50)!), + p95: rounded(percentile(sorted, 95)!), + p99: rounded(percentile(sorted, 99)!), + }; +}; + +export class PhaseMetrics { + readonly httpLatencyMs = new Map(); + readonly httpSuccess = new Map(); + readonly httpErrors = new Map(); + readonly httpResults = new Map(); + readonly sseEvents = new Map(); + readonly processRssBytes: number[] = []; + readonly sseActiveConnections: number[] = []; + sseActiveCurrent = 0; + sseAttempts = 0; + sseOpened = 0; + sseClosed = 0; + sseReconnects = 0; + sseFailures = 0; + ssePrivacyViolations = 0; + + recordHttp(name: string, latencyMs: number, outcome: string | null): void { + const values = this.httpLatencyMs.get(name) ?? []; + values.push(latencyMs); + this.httpLatencyMs.set(name, values); + const target = outcome === null ? this.httpSuccess : this.httpErrors; + const key = outcome === null ? name : `${name}:${outcome}`; + target.set(key, (target.get(key) ?? 0) + 1); + } + + recordSseEvent(name: string): void { + const safeName = /^[A-Za-z][A-Za-z0-9_-]{0,63}$/u.test(name) ? name : 'invalid-name'; + this.sseEvents.set(safeName, (this.sseEvents.get(safeName) ?? 0) + 1); + } + + recordHttpResult(name: string, result: string): void { + const safeResult = /^[a-z][a-z0-9-]{0,31}$/u.test(result) ? result : 'other'; + const key = `${name}:${safeResult}`; + this.httpResults.set(key, (this.httpResults.get(key) ?? 0) + 1); + } +} + +const mapToObject = (value: ReadonlyMap): Record => + Object.fromEntries([...value.entries()].sort(([left], [right]) => left.localeCompare(right))); + +export interface PhaseMetricSummary { + http: { + success: Record; + errors: Record; + results: Record; + latencyMs: Record; + }; + sse: { + attempts: number; + opened: number; + closed: number; + reconnects: number; + failures: number; + events: Record; + privacyViolations: number; + activeConnections: DistributionSummary; + }; + process: { + cpuPercentOfOneCore: number; + rssBytes: DistributionSummary; + eventLoopLagMs: Omit & { mean: number }; + }; +} + +export class ProcessSampler { + private readonly histogram = monitorEventLoopDelay({ resolution: 20 }); + private readonly startCpu = process.cpuUsage(); + private readonly startNs = process.hrtime.bigint(); + private timer: NodeJS.Timeout | null = null; + + constructor(private readonly metrics: PhaseMetrics) {} + + start(): void { + this.histogram.enable(); + this.sample(); + this.timer = setInterval(() => this.sample(), 1000); + this.timer.unref(); + } + + private sample(): void { + this.metrics.processRssBytes.push(process.memoryUsage().rss); + this.metrics.sseActiveConnections.push(this.metrics.sseActiveCurrent); + } + + stop(activeConnections: number): PhaseMetricSummary['process'] { + if (this.timer) clearInterval(this.timer); + this.sample(); + this.metrics.sseActiveConnections.push(activeConnections); + this.histogram.disable(); + const elapsedMs = Number(process.hrtime.bigint() - this.startNs) / 1_000_000; + const cpu = process.cpuUsage(this.startCpu); + const cpuMs = (cpu.user + cpu.system) / 1000; + const fromNs = (value: number): number => (Number.isFinite(value) ? rounded(value / 1_000_000) : 0); + return { + cpuPercentOfOneCore: rounded((cpuMs / Math.max(elapsedMs, 1)) * 100), + rssBytes: summarizeDistribution(this.metrics.processRssBytes), + eventLoopLagMs: { + min: fromNs(this.histogram.min), + max: fromNs(this.histogram.max), + mean: fromNs(this.histogram.mean), + p50: fromNs(this.histogram.percentile(50)), + p95: fromNs(this.histogram.percentile(95)), + p99: fromNs(this.histogram.percentile(99)), + }, + }; + } +} + +export const summarizePhaseMetrics = (metrics: PhaseMetrics, processSummary: PhaseMetricSummary['process']): PhaseMetricSummary => ({ + http: { + success: mapToObject(metrics.httpSuccess), + errors: mapToObject(metrics.httpErrors), + results: mapToObject(metrics.httpResults), + latencyMs: Object.fromEntries( + [...metrics.httpLatencyMs.entries()] + .sort(([left], [right]) => left.localeCompare(right)) + .map(([name, values]) => [name, summarizeDistribution(values)]) + ), + }, + sse: { + attempts: metrics.sseAttempts, + opened: metrics.sseOpened, + closed: metrics.sseClosed, + reconnects: metrics.sseReconnects, + failures: metrics.sseFailures, + events: mapToObject(metrics.sseEvents), + privacyViolations: metrics.ssePrivacyViolations, + activeConnections: summarizeDistribution(metrics.sseActiveConnections), + }, + process: processSummary, +}); diff --git a/tools/load-tests/src/runner.ts b/tools/load-tests/src/runner.ts new file mode 100644 index 00000000..b030a09a --- /dev/null +++ b/tools/load-tests/src/runner.ts @@ -0,0 +1,201 @@ +import { execFile } from 'node:child_process'; +import { setMaxListeners } from 'node:events'; +import { promisify } from 'node:util'; +import os from 'node:os'; +import { readFile as readTextFile } from 'node:fs/promises'; + +import { canonicalJson, expandWeightedOperations, sha256, type LoadConfig, type LoadPhase } from './config.js'; +import { PhaseMetrics, ProcessSampler, summarizePhaseMetrics, type PhaseMetricSummary } from './metrics.js'; +import { runSseConnection } from './sse.js'; +import { executeTrpcQuery } from './trpc.js'; +import type { DashboardRevisions } from './trpc.js'; + +const execFileAsync = promisify(execFile); + +const wait = (milliseconds: number, signal: AbortSignal): Promise => + new Promise((resolve) => { + if (signal.aborted) return resolve(); + const done = () => { + clearTimeout(timer); + signal.removeEventListener('abort', done); + resolve(); + }; + const timer = setTimeout(done, milliseconds); + signal.addEventListener('abort', done, { once: true }); + }); + +const loadText = async (file: string): Promise => { + try { + return (await readTextFile(file, 'utf8')).trim(); + } catch { + return null; + } +}; + +const runtimeAndHost = async () => { + const cpuQuota = await loadText('/sys/fs/cgroup/cpu.max'); + const memoryLimit = await loadText('/sys/fs/cgroup/memory.max'); + const runtime = { node: process.version, v8: process.versions.v8 }; + const cpus = os.cpus(); + const host = { + platform: os.platform(), + release: os.release(), + arch: os.arch(), + logicalCpuCount: cpus.length, + cpuModel: cpus[0]?.model ?? 'unknown', + totalMemoryBytes: os.totalmem(), + cgroupCpuMax: cpuQuota, + cgroupMemoryMax: memoryLimit, + }; + return { + runtime, + host, + runtimeSha256: sha256(canonicalJson(runtime)), + hostSha256: sha256(canonicalJson(host)), + }; +}; + +const gitMetadata = async (workspaceRoot: string) => { + const run = async (args: string[]): Promise => (await execFileAsync('git', args, { cwd: workspaceRoot })).stdout.trim(); + const [commit, tree, status] = await Promise.all([run(['rev-parse', 'HEAD']), run(['rev-parse', 'HEAD^{tree}']), run(['status', '--porcelain=v1'])]); + return { commit, tree, dirty: status.length > 0 }; +}; + +const runHttpViewers = async (options: { + config: LoadConfig; + phase: LoadPhase; + tokens: readonly string[]; + signal: AbortSignal; + metrics: PhaseMetrics; +}): Promise => { + if (options.phase.requestIntervalMs === null || options.phase.operations.length === 0) return; + const schedule = expandWeightedOperations(options.phase.operations); + const interval = options.phase.requestIntervalMs; + await Promise.all( + options.tokens.map(async (token, viewerIndex) => { + const stagger = Math.floor((viewerIndex / options.tokens.length) * interval); + await wait(stagger, options.signal); + let iteration = 0; + let dashboardRevisions: DashboardRevisions = {}; + while (!options.signal.aborted) { + const configuredOperation = schedule[(viewerIndex + iteration) % schedule.length]!; + const operation = + configuredOperation.procedure === 'dashboard.getContextBundleDelta' && Object.keys(dashboardRevisions).length > 0 + ? { + ...configuredOperation, + input: { + ...(typeof configuredOperation.input === 'object' && configuredOperation.input !== null + ? configuredOperation.input + : {}), + known: dashboardRevisions, + forceSnapshot: false, + }, + } + : configuredOperation; + const observedRevisions = await executeTrpcQuery({ + baseUrl: options.config.target.baseUrl, + trpcPath: options.config.target.trpcPath, + operation, + token, + signal: options.signal, + metrics: options.metrics, + }); + if (observedRevisions) dashboardRevisions = { ...dashboardRevisions, ...observedRevisions }; + iteration += 1; + await wait(interval, options.signal); + } + }) + ); +}; + +export interface PhaseResult { + name: string; + kind: LoadPhase['kind']; + configuredDurationMs: number; + elapsedMs: number; + metrics: PhaseMetricSummary; +} + +const runPhase = async (config: LoadConfig, phase: LoadPhase, tokens: readonly string[]): Promise => { + const metrics = new PhaseMetrics(); + const sampler = new ProcessSampler(metrics); + const controller = new AbortController(); + setMaxListeners(0, controller.signal); + let activeConnections = 0; + const started = performance.now(); + sampler.start(); + const timer = setTimeout(() => controller.abort(), phase.durationMs); + const sseUrl = new URL(config.target.ssePath, config.target.baseUrl).toString(); + const sseTasks = tokens.slice(0, phase.sseConnections).map((token) => + runSseConnection({ + url: sseUrl, + token, + signal: controller.signal, + metrics, + onActiveChange: (delta) => { + activeConnections += delta; + metrics.sseActiveCurrent = activeConnections; + }, + }) + ); + const httpTask = runHttpViewers({ config, phase, tokens, signal: controller.signal, metrics }); + const settled = await Promise.allSettled([...sseTasks, httpTask]); + clearTimeout(timer); + const rejected = settled.find((result): result is PromiseRejectedResult => result.status === 'rejected'); + if (rejected) { + sampler.stop(activeConnections); + throw rejected.reason; + } + const processSummary = sampler.stop(activeConnections); + return { + name: phase.name, + kind: phase.kind, + configuredDurationMs: phase.durationMs, + elapsedMs: Math.round(performance.now() - started), + metrics: summarizePhaseMetrics(metrics, processSummary), + }; +}; + +export const describeDryRun = (config: LoadConfig) => ({ + name: config.name, + targetHost: new URL(config.target.baseUrl).hostname, + isolation: config.isolation, + capacity: config.capacity, + phases: config.phases.map((phase) => ({ + name: phase.name, + kind: phase.kind, + durationMs: phase.durationMs, + sseConnections: phase.sseConnections, + requestIntervalMs: phase.requestIntervalMs, + operations: phase.operations.map((operation) => ({ name: operation.name, weight: operation.weight })), + })), +}); + +export const runLoadTest = async (options: { + config: LoadConfig; + configSha256: string; + tokens: readonly string[]; + workspaceRoot: string; +}) => { + const startedAt = new Date().toISOString(); + const [git, environment] = await Promise.all([gitMetadata(options.workspaceRoot), runtimeAndHost()]); + const phases: PhaseResult[] = []; + for (const phase of options.config.phases) phases.push(await runPhase(options.config, phase, options.tokens)); + return { + formatVersion: 1, + startedAt, + finishedAt: new Date().toISOString(), + config: { + name: options.config.name, + sha256: options.configSha256, + capacity: options.config.capacity, + isolation: options.config.isolation, + }, + git, + runtime: environment.runtime, + host: environment.host, + hashes: { configSha256: options.configSha256, runtimeSha256: environment.runtimeSha256, hostSha256: environment.hostSha256 }, + targetRuntime: options.config.runtimeMetadata, + phases, + }; +}; diff --git a/tools/load-tests/src/sse.ts b/tools/load-tests/src/sse.ts new file mode 100644 index 00000000..5338a783 --- /dev/null +++ b/tools/load-tests/src/sse.ts @@ -0,0 +1,134 @@ +import type { PhaseMetrics } from './metrics.js'; + +const forbiddenKeys = new Set([ + 'at', + 'lastTurnTime', + 'revision', + 'generalId', + 'cityId', + 'nationId', + 'entityId', + 'mailboxId', + 'messageId', + 'senderId', +]); + +export const containsForbiddenPublicField = (value: unknown): boolean => { + if (Array.isArray(value)) return value.some(containsForbiddenPublicField); + if (typeof value !== 'object' || value === null) return false; + return Object.entries(value).some(([key, item]) => forbiddenKeys.has(key) || containsForbiddenPublicField(item)); +}; + +export interface ParsedSseEvent { + event: string; + data: string; +} + +export class SseParser { + private buffer = ''; + private eventName = 'message'; + private data: string[] = []; + + constructor(private readonly onEvent: (event: ParsedSseEvent) => void) {} + + push(chunk: string): void { + this.buffer += chunk; + let newline = this.buffer.indexOf('\n'); + while (newline >= 0) { + const rawLine = this.buffer.slice(0, newline); + this.buffer = this.buffer.slice(newline + 1); + this.consumeLine(rawLine.endsWith('\r') ? rawLine.slice(0, -1) : rawLine); + newline = this.buffer.indexOf('\n'); + } + } + + finish(): void { + if (this.buffer.length > 0) this.consumeLine(this.buffer.endsWith('\r') ? this.buffer.slice(0, -1) : this.buffer); + this.buffer = ''; + this.dispatch(); + } + + private consumeLine(line: string): void { + if (line === '') { + this.dispatch(); + return; + } + if (line.startsWith(':')) return; + const separator = line.indexOf(':'); + const field = separator < 0 ? line : line.slice(0, separator); + let value = separator < 0 ? '' : line.slice(separator + 1); + if (value.startsWith(' ')) value = value.slice(1); + if (field === 'event') this.eventName = value; + if (field === 'data') this.data.push(value); + } + + private dispatch(): void { + if (this.data.length > 0) this.onEvent({ event: this.eventName, data: this.data.join('\n') }); + this.eventName = 'message'; + this.data = []; + } +} + +const wait = (milliseconds: number, signal: AbortSignal): Promise => + new Promise((resolve) => { + if (signal.aborted) return resolve(); + const done = () => { + clearTimeout(timer); + signal.removeEventListener('abort', done); + resolve(); + }; + const timer = setTimeout(done, milliseconds); + signal.addEventListener('abort', done, { once: true }); + }); + +export const runSseConnection = async (options: { + url: string; + token: string; + signal: AbortSignal; + metrics: PhaseMetrics; + onActiveChange: (delta: number) => void; + reconnectDelayMs?: number; +}): Promise => { + let priorAttempt = false; + while (!options.signal.aborted) { + if (priorAttempt) options.metrics.sseReconnects += 1; + priorAttempt = true; + options.metrics.sseAttempts += 1; + let active = false; + try { + const response = await fetch(options.url, { + headers: { accept: 'text/event-stream', authorization: `Bearer ${options.token}` }, + signal: options.signal, + }); + if (!response.ok || !response.body) { + options.metrics.sseFailures += 1; + await response.body?.cancel(); + } else { + options.metrics.sseOpened += 1; + active = true; + options.onActiveChange(1); + const parser = new SseParser(({ event, data }) => { + options.metrics.recordSseEvent(event); + try { + if (containsForbiddenPublicField(JSON.parse(data))) options.metrics.ssePrivacyViolations += 1; + } catch { + options.metrics.sseFailures += 1; + } + }); + const decoder = new TextDecoder(); + for await (const chunk of response.body) parser.push(decoder.decode(chunk, { stream: true })); + parser.push(decoder.decode()); + parser.finish(); + if (!options.signal.aborted) options.metrics.sseFailures += 1; + } + } catch (error) { + if (!options.signal.aborted) options.metrics.sseFailures += 1; + } finally { + if (active) { + options.metrics.sseClosed += 1; + options.onActiveChange(-1); + } + } + if (!options.signal.aborted) await wait(options.reconnectDelayMs ?? 1000, options.signal); + } +}; diff --git a/tools/load-tests/src/trpc.ts b/tools/load-tests/src/trpc.ts new file mode 100644 index 00000000..0461be49 --- /dev/null +++ b/tools/load-tests/src/trpc.ts @@ -0,0 +1,104 @@ +import type { LoadOperation } from './config.js'; +import type { PhaseMetrics } from './metrics.js'; + +export interface TrpcRequest { + url: string; + init: RequestInit; +} + +export const buildTrpcQuery = (baseUrl: string, trpcPath: string, operation: LoadOperation, token: string): TrpcRequest => { + const normalizedPath = trpcPath.endsWith('/') ? trpcPath.slice(0, -1) : trpcPath; + const url = new URL(`${normalizedPath}/${operation.procedure}`, baseUrl); + if (operation.input !== undefined) url.searchParams.set('input', JSON.stringify({ json: operation.input })); + return { + url: url.toString(), + init: { + method: 'GET', + headers: { accept: 'application/json', authorization: `Bearer ${token}` }, + }, + }; +}; + +const classifyTrpcPayload = (payload: unknown): string | null => { + if (typeof payload !== 'object' || payload === null) return 'invalid-payload'; + if ('error' in payload) { + const error = (payload as { error?: unknown }).error; + if (typeof error === 'object' && error !== null && 'data' in error) { + const data = (error as { data?: unknown }).data; + if (typeof data === 'object' && data !== null && 'code' in data && typeof (data as { code?: unknown }).code === 'string') { + const code = (data as { code: string }).code; + return `trpc-${/^[A-Z_]+$/u.test(code) ? code.toLowerCase() : 'error'}`; + } + } + return 'trpc-error'; + } + return null; +}; + +const asRecord = (value: unknown): Record | null => + typeof value === 'object' && value !== null && !Array.isArray(value) ? (value as Record) : null; + +const unwrapTrpcData = (payload: unknown): unknown => { + const result = asRecord(asRecord(payload)?.result); + const data = asRecord(result?.data); + return data && 'json' in data ? data.json : result?.data; +}; + +export interface DashboardRevisions { + context?: string; + commandTable?: string; + boardAccess?: string; +} + +export const extractDashboardRevisions = (payload: unknown): { revisions: DashboardRevisions; resultKinds: string[] } | null => { + const data = asRecord(unwrapTrpcData(payload)); + if (!data) return null; + const revisions: DashboardRevisions = {}; + const resultKinds: string[] = []; + for (const [wireName, outputName] of [ + ['context', 'context'], + ['commandTable', 'commandTable'], + ['boardAccess', 'boardAccess'], + ] as const) { + const slice = asRecord(data[wireName]); + if (!slice) continue; + if (typeof slice.revision === 'string' && /^[A-Za-z0-9_-]{22}$/u.test(slice.revision)) revisions[outputName] = slice.revision; + if (typeof slice.kind === 'string' && ['unchanged', 'snapshot', 'patch'].includes(slice.kind)) resultKinds.push(slice.kind); + else resultKinds.push('other'); + } + return { revisions, resultKinds }; +}; + +export const executeTrpcQuery = async (options: { + baseUrl: string; + trpcPath: string; + operation: LoadOperation; + token: string; + signal: AbortSignal; + metrics: PhaseMetrics; +}): Promise => { + const started = performance.now(); + let outcome: string | null; + try { + const request = buildTrpcQuery(options.baseUrl, options.trpcPath, options.operation, options.token); + const response = await fetch(request.url, { ...request.init, signal: options.signal }); + if (!response.ok) { + outcome = `http-${response.status}`; + await response.body?.cancel(); + } else { + const payload: unknown = await response.json(); + outcome = classifyTrpcPayload(payload); + if (outcome === null && options.operation.procedure === 'dashboard.getContextBundleDelta') { + const dashboard = extractDashboardRevisions(payload); + for (const kind of dashboard?.resultKinds ?? []) options.metrics.recordHttpResult(options.operation.name, kind); + options.metrics.recordHttp(options.operation.name, performance.now() - started, outcome); + return dashboard?.revisions; + } + } + } catch (error) { + if (options.signal.aborted) return; + outcome = error instanceof TypeError ? 'network' : 'client'; + } + options.metrics.recordHttp(options.operation.name, performance.now() - started, outcome); + return undefined; +}; diff --git a/tools/load-tests/test/config.test.ts b/tools/load-tests/test/config.test.ts new file mode 100644 index 00000000..06c8706c --- /dev/null +++ b/tools/load-tests/test/config.test.ts @@ -0,0 +1,69 @@ +import assert from 'node:assert/strict'; +import { chmod, readFile, symlink, unlink, writeFile } from 'node:fs/promises'; +import path from 'node:path'; +import test from 'node:test'; + +import { assertRuntimeMetadataFinalized, canonicalJson, expandWeightedOperations, loadTokens, validateLoadConfig } from '../src/config.js'; + +const samplePath = new URL('../config/300-users-900-npcs-5m.json', import.meta.url); + +void test('the 300 viewer, 900 NPC, five-minute sample validates', async () => { + const config = validateLoadConfig(JSON.parse(await readFile(samplePath, 'utf8'))); + assert.equal(config.capacity.authenticatedViewers, 300); + assert.equal(config.capacity.npcGenerals, 900); + assert.equal(config.capacity.turnIntervalMs, 300_000); + assert.deepEqual(new Set(config.phases.map((phase) => phase.kind)), new Set(['idle', 'own', 'global', 'mixed'])); +}); + +void test('validation rejects public, non-allowlisted, and mutating targets', async () => { + const raw = JSON.parse(await readFile(samplePath, 'utf8')) as Record; + raw.target.publicProfile = true; + raw.target.baseUrl = 'https://public.example.invalid'; + raw.phases[1].operations[0].type = 'mutation'; + assert.throws(() => validateLoadConfig(raw), /publicProfile must be false/u); + assert.throws(() => validateLoadConfig(raw), /hostname is not explicitly allowlisted/u); + assert.throws(() => validateLoadConfig(raw), /read-only query/u); +}); + +void test('a measurement run rejects sample metadata placeholders', async () => { + const config = validateLoadConfig(JSON.parse(await readFile(samplePath, 'utf8'))); + assert.throws(() => assertRuntimeMetadataFinalized(config), /fixtureSha256, imageDigest, postgresVersion, redisVersion/u); +}); + +void test('canonical JSON and weighted scheduling do not depend on object insertion order', () => { + assert.equal(canonicalJson({ b: 2, a: 1 }), canonicalJson({ a: 1, b: 2 })); + const expanded = expandWeightedOperations([ + { name: 'a', procedure: 'a.read', type: 'query', weight: 2 }, + { name: 'b', procedure: 'b.read', type: 'query', weight: 1 }, + ]); + assert.deepEqual(expanded.map((operation) => operation.name), ['a', 'a', 'b']); +}); + +void test('token loading requires an ignored 0600 file and returns no identity metadata', async () => { + const workspaceRoot = path.resolve(import.meta.dirname, '../../..'); + const tokenPath = path.join(workspaceRoot, 'tools/load-tests/secrets/unit-test-tokens.json'); + await writeFile(tokenPath, JSON.stringify({ tokens: ['ga_test_token_00000001'] }), { mode: 0o600 }); + await chmod(tokenPath, 0o600); + try { + assert.deepEqual(await loadTokens(tokenPath, workspaceRoot, 1), ['ga_test_token_00000001']); + await chmod(tokenPath, 0o644); + await assert.rejects(loadTokens(tokenPath, workspaceRoot, 1), /0600/u); + } finally { + await unlink(tokenPath).catch(() => undefined); + } +}); + +void test('token loading rejects a symlink even when its link path is ignored', async () => { + const workspaceRoot = path.resolve(import.meta.dirname, '../../..'); + const tokenPath = path.join(workspaceRoot, 'tools/load-tests/secrets/unit-test-token-target.json'); + const linkPath = path.join(workspaceRoot, 'tools/load-tests/secrets/unit-test-token-link.json'); + await writeFile(tokenPath, JSON.stringify({ tokens: ['ga_test_token_00000001'] }), { mode: 0o600 }); + await chmod(tokenPath, 0o600); + await symlink(tokenPath, linkPath); + try { + await assert.rejects(loadTokens(linkPath, workspaceRoot, 1), /symbolic link/u); + } finally { + await unlink(linkPath).catch(() => undefined); + await unlink(tokenPath).catch(() => undefined); + } +}); diff --git a/tools/load-tests/test/metrics.test.ts b/tools/load-tests/test/metrics.test.ts new file mode 100644 index 00000000..701cd105 --- /dev/null +++ b/tools/load-tests/test/metrics.test.ts @@ -0,0 +1,42 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; + +import { PhaseMetrics, percentile, summarizeDistribution, summarizePhaseMetrics } from '../src/metrics.js'; + +void test('nearest-rank percentiles and summaries are deterministic', () => { + const values = [100, 1, 5, 3, 2, 4]; + const sorted = [...values].sort((left, right) => left - right); + assert.equal(percentile(sorted, 50), 3); + assert.equal(percentile(sorted, 95), 100); + assert.deepEqual(summarizeDistribution(values), { + count: 6, + min: 1, + max: 100, + mean: 19.167, + p50: 3, + p95: 100, + p99: 100, + }); +}); + +void test('phase aggregation separates success, error, latency, and event counters', () => { + const metrics = new PhaseMetrics(); + metrics.recordHttp('own', 10, null); + metrics.recordHttp('own', 20, 'http-500'); + metrics.recordSseEvent('ready'); + metrics.recordSseEvent('ready'); + metrics.recordHttpResult('own', 'unchanged'); + metrics.processRssBytes.push(100, 200); + metrics.sseActiveConnections.push(0, 2); + const summary = summarizePhaseMetrics(metrics, { + cpuPercentOfOneCore: 5, + rssBytes: summarizeDistribution(metrics.processRssBytes), + eventLoopLagMs: { min: 1, max: 2, mean: 1.5, p50: 1, p95: 2, p99: 2 }, + }); + assert.deepEqual(summary.http.success, { own: 1 }); + assert.deepEqual(summary.http.errors, { 'own:http-500': 1 }); + assert.deepEqual(summary.http.results, { 'own:unchanged': 1 }); + assert.equal(summary.http.latencyMs.own?.p50, 10); + assert.deepEqual(summary.sse.events, { ready: 2 }); + assert.equal(summary.sse.activeConnections.max, 2); +}); diff --git a/tools/load-tests/test/sse.test.ts b/tools/load-tests/test/sse.test.ts new file mode 100644 index 00000000..42398b93 --- /dev/null +++ b/tools/load-tests/test/sse.test.ts @@ -0,0 +1,19 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; + +import { containsForbiddenPublicField, SseParser } from '../src/sse.js'; + +void test('SSE parser handles chunk boundaries, CRLF, comments, and multiline data', () => { + const events: Array<{ event: string; data: string }> = []; + const parser = new SseParser((event) => events.push(event)); + parser.push(': keepalive\r\nevent: rea'); + parser.push('dy\r\ndata: {"ok":\r\ndata: true}\r\n\r\n'); + parser.finish(); + assert.deepEqual(events, [{ event: 'ready', data: '{"ok":\ntrue}' }]); +}); + +void test('public payload privacy scan checks nested forbidden identifiers and timing fields', () => { + assert.equal(containsForbiddenPublicField({ type: 'readModelInvalidated', context: true }), false); + assert.equal(containsForbiddenPublicField({ nested: { generalId: 3 } }), true); + assert.equal(containsForbiddenPublicField([{ lastTurnTime: 'secret' }]), true); +}); diff --git a/tools/load-tests/test/trpc.test.ts b/tools/load-tests/test/trpc.test.ts new file mode 100644 index 00000000..ad912aad --- /dev/null +++ b/tools/load-tests/test/trpc.test.ts @@ -0,0 +1,35 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; + +import { buildTrpcQuery, extractDashboardRevisions } from '../src/trpc.js'; + +void test('tRPC query uses bearer auth without putting the token in the URL', () => { + const token = 'ga_example_secret_token'; + const request = buildTrpcQuery( + 'http://127.0.0.1:15001', + '/api/trpc', + { name: 'own', procedure: 'dashboard.getContextBundleDelta', type: 'query', weight: 1, input: { include: { context: true } } }, + token + ); + assert.equal(new Headers(request.init.headers).get('authorization'), `Bearer ${token}`); + assert.equal(request.url.includes(token), false); + assert.equal(new URL(request.url).pathname, '/api/trpc/dashboard.getContextBundleDelta'); + assert.deepEqual(JSON.parse(new URL(request.url).searchParams.get('input')!), { json: { include: { context: true } } }); +}); + +void test('dashboard observations retain only opaque revisions and aggregate-safe result kinds', () => { + const revision = 'Abcdefghijklmnopqrstuv'; + assert.deepEqual( + extractDashboardRevisions({ + result: { + data: { + json: { + context: { kind: 'unchanged', revision, data: { general: { id: 123 } } }, + commandTable: { kind: 'snapshot', revision }, + }, + }, + }, + }), + { revisions: { context: revision, commandTable: revision }, resultKinds: ['unchanged', 'snapshot'] } + ); +}); diff --git a/tools/load-tests/tsconfig.json b/tools/load-tests/tsconfig.json new file mode 100644 index 00000000..c0e398f2 --- /dev/null +++ b/tools/load-tests/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "noEmit": true, + "types": ["node"] + }, + "include": ["src/**/*.ts", "test/**/*.ts"] +}