diff --git a/app/game-api/src/router/archive/index.ts b/app/game-api/src/router/archive/index.ts index 49dab2b7..eeb7909b 100644 --- a/app/game-api/src/router/archive/index.ts +++ b/app/game-api/src/router/archive/index.ts @@ -18,6 +18,7 @@ import { import { findLegacyEmperors, findLegacyGeneral, + findLegacyGeneralBattleResult, findLegacyGeneralsByOwner, findLegacyGames, findLegacyNations, @@ -412,6 +413,8 @@ export const archiveRouter = router({ let entry: GeneralArchiveEntry | null = null; let nationRows: ArchiveNationEntry[] = []; let dynastyId: number | null = null; + let battleResultContent: string | null = null; + let battleResultAvailable = false; if (input.source === 'legacy') { if (!LEGACY_ARCHIVE_PROFILES.includes(sourceProfile as LegacyArchiveProfile)) { throw new TRPCError({ code: 'BAD_REQUEST', message: '지원하지 않는 이전 서버 프로필입니다.' }); @@ -435,9 +438,14 @@ export const archiveRouter = router({ snapshot: canonicalSnapshot(row.data, row.name), }; const keyInput = [{ sourceProfile: profile, serverId: input.serverId }]; - const [nations, emperors] = await Promise.all([ + const [nations, emperors, battleResult] = await Promise.all([ findLegacyNations(ctx.db, keyInput), findLegacyEmperors(ctx.db, keyInput), + findLegacyGeneralBattleResult(ctx.db, { + sourceProfile: profile, + serverId: input.serverId, + generalNo: input.generalNo, + }), ]); nationRows = nations.map((nation) => ({ source: 'legacy', @@ -448,6 +456,8 @@ export const archiveRouter = router({ data: asRecord(nation.data), })); dynastyId = Number(emperors[0]?.id ?? 0) || null; + battleResultContent = battleResult?.content ?? null; + battleResultAvailable = battleResult !== null; } } else { const row = await ctx.db.oldGeneral.findFirst({ @@ -494,13 +504,18 @@ export const archiveRouter = router({ const nation = resolveNation(nationMap, entry); const snapshot = entry.snapshot; const general = await buildGeneralDetail(entry, nation); + const battleResultEntries = (battleResultContent ?? '') + .split(/\r?\n/u) + .map((text, index) => ({ id: index + 1, text })) + .filter((item) => item.text.length > 0) + .reverse(); const logs = { generalHistory: { available: snapshot.availability.history, entries: snapshot.history.map((text, index) => ({ id: index + 1, text })), }, battleDetail: { available: snapshot.availability.battleDetailLogs, entries: [] }, - battleResult: { available: snapshot.availability.battleResultLogs, entries: [] }, + battleResult: { available: battleResultAvailable, entries: battleResultEntries }, generalAction: { available: false, entries: [] }, }; return { diff --git a/app/game-api/src/services/legacyArchiveStore.ts b/app/game-api/src/services/legacyArchiveStore.ts index b20ef0fa..f9b2d062 100644 --- a/app/game-api/src/services/legacyArchiveStore.ts +++ b/app/game-api/src/services/legacyArchiveStore.ts @@ -36,6 +36,12 @@ export interface LegacyGeneralRow { data: unknown; } +export interface LegacyGeneralBattleResultRow { + content: string; + lineCount: number; + contentHash: string; +} + export interface LegacyNationRow { sourceProfile: LegacyArchiveProfile; legacyId: number; @@ -120,6 +126,24 @@ export const findLegacyGeneral = async ( return rows[0] ?? null; }; +export const findLegacyGeneralBattleResult = async ( + db: LegacyArchiveDatabase, + input: { sourceProfile: LegacyArchiveProfile; serverId: string; generalNo: number } +): Promise => { + const rows = await db.$queryRaw(GamePrisma.sql` + SELECT + "content", + "line_count" AS "lineCount", + "content_hash" AS "contentHash" + FROM "legacy_archive"."general_battle_result" + WHERE "source_profile" = ${input.sourceProfile} + AND "server_id" = ${input.serverId} + AND "general_no" = ${input.generalNo} + LIMIT 1 + `); + return rows[0] ?? null; +}; + export const findLegacyGeneralsForServer = async ( db: LegacyArchiveDatabase, input: { sourceProfile: LegacyArchiveProfile; serverId: string; generalNos: number[] } diff --git a/app/game-api/test/archiveRouter.test.ts b/app/game-api/test/archiveRouter.test.ts index a0978383..572cbed1 100644 --- a/app/game-api/test/archiveRouter.test.ts +++ b/app/game-api/test/archiveRouter.test.ts @@ -30,6 +30,15 @@ const context = (session: GameSessionTokenPayload | null, includeLegacy = false) $queryRaw: async (query: { strings?: readonly string[] }) => { if (!includeLegacy) return []; const sql = query.strings?.join(' ') ?? ''; + if (sql.includes('legacy_archive"."general_battle_result')) { + return [ + { + content: '◆190년 1월:첫 전투\n◆190년 2월:둘째 전투\n', + lineCount: 2, + contentHash: 'a'.repeat(64), + }, + ]; + } if (sql.includes('legacy_archive"."general')) { return [ { @@ -385,6 +394,13 @@ describe('archive.myPastPlays', () => { logs: expect.objectContaining({ generalHistory: { available: true, entries: [{ id: 1, text: '이전 서버 열전' }] }, battleDetail: { available: false, entries: [] }, + battleResult: { + available: true, + entries: [ + { id: 2, text: '◆190년 2월:둘째 전투' }, + { id: 1, text: '◆190년 1월:첫 전투' }, + ], + }, }), }); expect(JSON.stringify(detail)).not.toContain('raw_data'); diff --git a/app/game-frontend/e2e/pastPlays.spec.ts b/app/game-frontend/e2e/pastPlays.spec.ts index d1965ee9..e1c0dab2 100644 --- a/app/game-frontend/e2e/pastPlays.spec.ts +++ b/app/game-frontend/e2e/pastPlays.spec.ts @@ -115,7 +115,16 @@ const installArchive = async (page: Page, options: { battleAvailable?: boolean } ], }, battleDetail: { available: false, entries: [] }, - battleResult: { available: false, entries: [] }, + battleResult: { + available: true, + entries: [ + { + id: 2, + text: '◆214년 3월:
관우 7000 ← 장비 0
', + }, + { id: 1, text: '◆214년 2월:관우 6500 → 여포 0' }, + ], + }, generalAction: { available: false, entries: [] }, }, }); @@ -177,9 +186,8 @@ test('past plays is available without a current general and preserves desktop in await expect(page.locator('[data-log-type="battleDetail"]')).toContainText( '이 기수에는 전투 기록이 보존되지 않았습니다.' ); - await expect(page.locator('[data-log-type="battleResult"]')).toContainText( - '이 기수에는 전투 결과가 보존되지 않았습니다.' - ); + await expect(page.locator('[data-log-type="battleResult"]')).toContainText('214년 3월:관우 7000 ← 장비 0'); + await expect(page.locator('[data-log-type="battleResult"]')).not.toContainText(' { await expect(readReleaseManifest(workspaceRoot)).resolves.toMatchObject({ controllerProtocol: RELEASE_CONTROLLER_PROTOCOL, gatewaySchemaHead: '20260818000000_add_legacy_import_checkpoints', - gameSchemaHead: '20260818000000_add_legacy_import_checkpoints', + gameSchemaHead: '20260818010000_add_legacy_battle_result_logs', }); }); diff --git a/docs/legacy-db-migration.md b/docs/legacy-db-migration.md index 2ad258c4..8bd59bd6 100644 --- a/docs/legacy-db-migration.md +++ b/docs/legacy-db-migration.md @@ -27,6 +27,10 @@ checks that the checkpoint migrations exist. Only after all stages pass does it run Gateway, then `che,kwe,pwe,twe,nya,pya,hwe` in that order, skipping disabled profiles. Dry-run and apply use the same transformations and return per-table `progress` with strategy, start cursor, end cursor and processed count. +`check-plan` additionally returns an `inventory` array for every stage. Each +entry names the source, target, full/incremental strategy and the user-visible or +archival information transferred, so the reviewed plan itself is the migration +manifest rather than an implicit table list. The initial apply uses `--mode full`. A later `--mode incremental` requires the same `sourceSet` and a completed checkpoint for every append-only table. @@ -92,19 +96,20 @@ The password is never accepted as an argument or printed. ### Game profiles -| Legacy table | Dedicated target | Policy | -| ------------------------------- | ----------------------------- | --------------------------------------------------------------------- | -| `ng_games` | `legacy_archive.game_history` | Preserve source profile, opening date, scenario and raw environment | -| `hall` | `legacy_archive.hall` | Preserve hall-of-fame rows without mixing current records | -| `ng_old_generals` | `legacy_archive.general` | Preserve canonical V1 plus private raw JSON and owner | -| `ng_old_nations` | `legacy_archive.nation` | Preserve all versions with profile and legacy primary key | -| `emperior` | `legacy_archive.emperor` | Preserve dynasty detail under a central archive ID | -| `inheritance_result` | `inheritance_result` | Preserve result JSON/string and legacy key | -| `user_record` | `inheritance_log` | Preserve complete long-lived user record | -| persistent `storage` namespaces | `legacy_game_storage` | Preserve raw `inheritance_*` and `user_*` rows before projection | -| `storage:inheritance_point` | `inheritance_point` | Project the numeric first tuple item; retain the tuple in raw storage | -| `storage:user` | `inheritance_user_state` | Project known current inheritance state; retain raw storage | -| `ng_history` | `legacy_archive.yearbook` | Preserve map, nation, global history and global action snapshots | +| Legacy table | Dedicated target | Policy | +| ---------------------------------- | -------------------------------------- | -------------------------------------------------------------------------- | +| `ng_games` | `legacy_archive.game_history` | Preserve source profile, opening date, scenario and raw environment | +| `hall` | `legacy_archive.hall` | Preserve hall-of-fame rows without mixing current records | +| `ng_old_generals` | `legacy_archive.general` | Preserve canonical V1 plus private raw JSON and owner | +| preserved `batres.txt` | `legacy_archive.general_battle_result` | Preserve exact old per-general battle-result summaries; exclude phase logs | +| `ng_old_nations` | `legacy_archive.nation` | Preserve all versions with profile and legacy primary key | +| `emperior` | `legacy_archive.emperor` | Preserve dynasty detail under a central archive ID | +| `inheritance_result` | `inheritance_result` | Preserve result JSON/string and legacy key | +| `user_record` | `inheritance_log` | Preserve complete long-lived user record | +| persistent `storage` namespaces | `legacy_game_storage` | Preserve raw `inheritance_*` and `user_*` rows before projection | +| `storage:inheritance_point` | `inheritance_point` | Project the numeric first tuple item; retain the tuple in raw storage | +| `storage:user` | `inheritance_user_state` | Project known current inheritance state; retain raw storage | +| `ng_history` | `legacy_archive.yearbook` | Preserve map, nation, global history and global action snapshots | The archive schema is shared by all game-profile schemas in the PostgreSQL database. Every natural key contains `source_profile`; the accepted profiles @@ -124,16 +129,31 @@ shape. Missing battle aggregates and logs are `null` plus explicit `availability`, never fabricated zeroes. The source JSON remains in `legacy_archive.general.raw_data` for recovery, but no API returns it. -Ref also retains per-season filesystem trees under `logs/preserved`. They are -not a MariaDB source: season reset moves the whole prior log directory there, -and the tree mixes legacy `gen*`, `batlog*`, `batres*`, tournament `fight*`, -SQLite API logs, and administrative/operational logs. The long-lived importer -therefore never reads a local or remote filesystem path. The required condensed -general history is already embedded by Ref in `ng_old_generals.data.history`, -and yearbook history comes from `ng_history`. Importing the other file artifacts -would require a separately reviewed archive format, ownership mapping, privacy -policy, and storage budget; their absence is reported as not preserved rather -than guessed from filenames. +Ref also retains heterogeneous per-season filesystem trees under +`logs/preserved`. When a profile plan supplies `battleResults.directory` and, +optionally, `sshHost`, the importer selects only immediate regular files matching +`_*/batres.txt`. The filename and directory provide the +same `(source_profile, server_id, general_no)` key as the archived general. +Exact UTF-8 text, line count, source bytes and SHA-256 are stored; the archive +API returns the lines newest-first like Ref and the frontend renders stripped +plain text, never trusted archived HTML. + +Every season gets a content manifest and checkpoint. Full apply can add or +atomically replace a changed season and is resumable at season boundaries. +Incremental apply requires a prior full checkpoint, accepts only new immutable +seasons and rejects a changed checkpointed season or changed filesystem source +identity. Both modes reject a disappeared checkpointed season instead of +silently retaining stale data. +`check-plan` reports season/file/byte totals without returning log contents. + +The importer deliberately excludes `batlog*` phase-by-phase battle detail, +`gen*` action logs, tournament `fight*`, SQLite API logs and administrative or +operational files. The production preserved trees contain battle-result files +only for the older filesystem-log era (observed seasons end around May 2020), +while later reset-time `general_record` rows were not retained in these trees. +Consequently this feature recovers meaningful old `batres` summaries but cannot +claim complete battle-result coverage for every historical season. Missing data +stays explicitly unavailable. The source contains legitimate duplicate `(server_id, nation)` old-nation rows and `(server_id, year, month)` history rows. `source_id` is consequently part of diff --git a/packages/infra/prisma/migrations/20260818010000_add_legacy_battle_result_logs/migration.sql b/packages/infra/prisma/migrations/20260818010000_add_legacy_battle_result_logs/migration.sql new file mode 100644 index 00000000..33eba150 --- /dev/null +++ b/packages/infra/prisma/migrations/20260818010000_add_legacy_battle_result_logs/migration.sql @@ -0,0 +1,63 @@ +CREATE TABLE "legacy_archive"."battle_result_import_run" ( + "id" BIGSERIAL PRIMARY KEY, + "source_profile" TEXT NOT NULL, + "source_key" TEXT NOT NULL, + "source_fingerprint" CHAR(64) NOT NULL, + "mode" TEXT NOT NULL, + "status" TEXT NOT NULL, + "started_at" TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + "finished_at" TIMESTAMPTZ, + "counts" JSONB NOT NULL DEFAULT '{}'::jsonb, + "progress" JSONB NOT NULL DEFAULT '{}'::jsonb, + "error" TEXT, + CONSTRAINT "legacy_archive_battle_result_run_profile_check" + CHECK ("source_profile" IN ('che', 'kwe', 'pwe', 'twe', 'nya', 'pya', 'hwe')), + CONSTRAINT "legacy_archive_battle_result_run_mode_check" CHECK ("mode" IN ('full', 'incremental')), + CONSTRAINT "legacy_archive_battle_result_run_status_check" + CHECK ("status" IN ('RUNNING', 'COMPLETED', 'FAILED')), + CONSTRAINT "legacy_archive_battle_result_run_fingerprint_check" + CHECK ("source_fingerprint" ~ '^[a-f0-9]{64}$') +); + +CREATE INDEX "legacy_archive_battle_result_run_source_started" + ON "legacy_archive"."battle_result_import_run" ("source_profile", "source_key", "started_at" DESC); + +CREATE TABLE "legacy_archive"."general_battle_result" ( + "source_profile" TEXT NOT NULL, + "server_id" TEXT NOT NULL, + "general_no" INTEGER NOT NULL, + "content" TEXT NOT NULL, + "line_count" INTEGER NOT NULL, + "source_bytes" BIGINT NOT NULL, + "content_hash" CHAR(64) NOT NULL, + "import_run_id" BIGINT NOT NULL REFERENCES "legacy_archive"."battle_result_import_run" ("id"), + "updated_at" TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY ("source_profile", "server_id", "general_no"), + CONSTRAINT "legacy_archive_general_battle_result_profile_check" + CHECK ("source_profile" IN ('che', 'kwe', 'pwe', 'twe', 'nya', 'pya', 'hwe')), + CONSTRAINT "legacy_archive_general_battle_result_general_no_check" CHECK ("general_no" >= 0), + CONSTRAINT "legacy_archive_general_battle_result_line_count_check" CHECK ("line_count" >= 0), + CONSTRAINT "legacy_archive_general_battle_result_source_bytes_check" CHECK ("source_bytes" >= 0), + CONSTRAINT "legacy_archive_general_battle_result_hash_check" CHECK ("content_hash" ~ '^[a-f0-9]{64}$') +); + +CREATE TABLE "legacy_archive"."battle_result_import_checkpoint" ( + "source_profile" TEXT NOT NULL, + "source_key" TEXT NOT NULL, + "source_fingerprint" CHAR(64) NOT NULL, + "server_id" TEXT NOT NULL, + "manifest_hash" CHAR(64) NOT NULL, + "file_count" INTEGER NOT NULL, + "total_bytes" BIGINT NOT NULL, + "import_run_id" BIGINT NOT NULL REFERENCES "legacy_archive"."battle_result_import_run" ("id"), + "updated_at" TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY ("source_profile", "source_key", "server_id"), + CONSTRAINT "legacy_archive_battle_result_checkpoint_profile_check" + CHECK ("source_profile" IN ('che', 'kwe', 'pwe', 'twe', 'nya', 'pya', 'hwe')), + CONSTRAINT "legacy_archive_battle_result_checkpoint_fingerprint_check" + CHECK ("source_fingerprint" ~ '^[a-f0-9]{64}$'), + CONSTRAINT "legacy_archive_battle_result_checkpoint_manifest_check" + CHECK ("manifest_hash" ~ '^[a-f0-9]{64}$'), + CONSTRAINT "legacy_archive_battle_result_checkpoint_file_count_check" CHECK ("file_count" >= 0), + CONSTRAINT "legacy_archive_battle_result_checkpoint_total_bytes_check" CHECK ("total_bytes" >= 0) +); diff --git a/release-manifest.json b/release-manifest.json index bc0038a1..7b831722 100644 --- a/release-manifest.json +++ b/release-manifest.json @@ -2,6 +2,6 @@ "formatVersion": 1, "controllerProtocol": 2, "gatewaySchemaHead": "20260818000000_add_legacy_import_checkpoints", - "gameSchemaHead": "20260818000000_add_legacy_import_checkpoints", + "gameSchemaHead": "20260818010000_add_legacy_battle_result_logs", "components": ["gateway-api", "gateway-frontend", "release-controller", "game-api", "game-engine", "game-frontend"] } diff --git a/tools/legacy-db-migration/README.md b/tools/legacy-db-migration/README.md index 00b9917b..1a65aeda 100644 --- a/tools/legacy-db-migration/README.md +++ b/tools/legacy-db-migration/README.md @@ -55,7 +55,22 @@ pnpm migrate:legacy -- run-plan \ --config tools/legacy-db-migration/migration-plan.json --mode full --apply ``` -`check-plan` opens every source and target without writing. `run-plan` also +For profiles whose old file archive is available, add a `battleResults` block. +`directory` may be a local absolute/plan-relative path, or `sshHost` may select +the host from which the directory is read. The SSH target must be a configured +host alias; do not put credentials in the plan. + +```json +"battleResults": { + "sshHost": "serv", + "directory": "/home/letrhee/web_symlinks/sam_hided_net/sam/che/logs/preserved" +} +``` + +`check-plan` opens every source and target without writing. Its stage JSON lists +every included item as `inventory`, including source, target, strategy and the +information transferred. A configured battle-result source also reports its +season/file/byte counts. `run-plan` also preflights every stage before the first import, is a dry-run without `--apply`, and stops at the first failed stage. Completed earlier stages remain committed; rerunning is safe because the Gateway and each profile have independent locks, @@ -77,24 +92,30 @@ apply. It also refuses a changed host/database/user identity or a source table whose maximum ID moved behind its checkpoint. Password rotation does not change the source fingerprint. -| Source data | Incremental policy | -| --------------------------------------------- | --------------------------------------------------------------------- | -| `member_log` | Read only IDs after the committed high-water mark. | -| game archive/event-history tables | Read only IDs after the profile checkpoint. | -| `member`, root/game `storage`, `system`, bans | Rescan and idempotently upsert because old rows are mutable. | -| `ng_games` | Rescan because a season row can gain its final winner after creation. | +| Source data | Incremental policy | +| --------------------------------------------- | ----------------------------------------------------------------------- | +| `member_log` | Read only IDs after the committed high-water mark. | +| game archive/event-history tables | Read only IDs after the profile checkpoint. | +| `member`, root/game `storage`, `system`, bans | Rescan and idempotently upsert because old rows are mutable. | +| `ng_games` | Rescan because a season row can gain its final winner after creation. | +| preserved `batres.txt` seasons | Hash each season; import new immutable seasons after a full checkpoint. | The append policy assumes Ref primary keys are never reused and completed archive rows are immutable. Incremental mode does not mirror source deletions. If either assumption is false, take a new reviewed backup and run full mode; do not edit checkpoint rows by hand. -The command reads MariaDB only. Ref `logs/preserved//` directories are -heterogeneous filesystem archives (old general/battle text, tournament text, -SQLite API logs, and operational logs), not an incremental database feed. -`ng_old_generals.data.history` and `ng_history` carry the supported long-lived -history. Do not point this CLI at, copy, or infer database rows from preserved -log files; a file-log archive needs a separate reviewed migration contract. +The optional file importer reads only immediate +`logs/preserved/_*/batres.txt` regular files. It maps the +profile and season directory to the archived general's `(source_profile, +server_id, general_no)` key, verifies file and season SHA-256 hashes, and stores +the exact text plus line/byte counts. It never follows symlinks and ignores +`batlog*` phase detail, `gen*`, `fight*`, SQLite and operational logs. Full mode +creates the season checkpoints. Incremental mode accepts new seasons but rejects +a changed checkpointed season or changed source identity. Every mode rejects a +disappeared checkpointed season. A reviewed full run atomically replaces a +changed season's archive rows. The preserved trees currently cover only the +old filesystem-log era, so an absent file remains explicitly unavailable. ## Commands diff --git a/tools/legacy-db-migration/migration-plan.example.json b/tools/legacy-db-migration/migration-plan.example.json index 5b1d90eb..c5bbd26f 100644 --- a/tools/legacy-db-migration/migration-plan.example.json +++ b/tools/legacy-db-migration/migration-plan.example.json @@ -23,6 +23,10 @@ "passwordFile": "./secrets/mysql-che-password", "tls": true }, + "battleResults": { + "sshHost": "serv", + "directory": "/home/letrhee/web_symlinks/sam_hided_net/sam/che/logs/preserved" + }, "targetUrlEnv": "CHE_GAME_DATABASE_URL" }, { diff --git a/tools/legacy-db-migration/src/battleResultSource.ts b/tools/legacy-db-migration/src/battleResultSource.ts new file mode 100644 index 00000000..34a17806 --- /dev/null +++ b/tools/legacy-db-migration/src/battleResultSource.ts @@ -0,0 +1,319 @@ +import { spawn } from 'node:child_process'; +import { createHash } from 'node:crypto'; +import { lstat, readFile, readdir, realpath } from 'node:fs/promises'; +import path from 'node:path'; +import readline from 'node:readline'; +import { TextDecoder } from 'node:util'; + +import type { LegacyArchiveProfile } from './game.js'; +import type { MigrationSourceIdentity } from './incremental.js'; + +const MAX_BATTLE_RESULT_FILE_BYTES = 4 * 1024 * 1024; +const SSH_HOST = /^(?:[a-zA-Z0-9._-]+@)?[a-zA-Z0-9._-]+$/u; +const HASH = /^[a-f0-9]{64}$/u; + +export interface BattleResultSourceConfig { + kind: 'local' | 'ssh'; + directory: string; + sshHost?: string; + identity: MigrationSourceIdentity; +} + +export interface BattleResultFileDescriptor { + serverId: string; + generalNo: number; + sourceBytes: number; + contentHash: string; +} + +export interface BattleResultSeasonManifest { + serverId: string; + files: BattleResultFileDescriptor[]; + fileCount: number; + totalBytes: number; + manifestHash: string; +} + +export interface BattleResultFile extends BattleResultFileDescriptor { + content: string; + lineCount: number; +} + +type RemoteDescriptor = { + serverId: string; + generalNo: number; + sourceBytes: number; + contentHash: string; + contentBase64?: string; +}; + +const REMOTE_READER = String.raw` +import base64, hashlib, json, os, re, sys + +MAX_BYTES = 4 * 1024 * 1024 +action, root_input, profile = sys.argv[1:4] +selected = set(json.loads(base64.urlsafe_b64decode(sys.argv[4]).decode('utf-8'))) if len(sys.argv) > 4 else set() +root = os.path.realpath(root_input) +season_re = re.compile(r'^' + re.escape(profile) + r'_[A-Za-z0-9_-]{1,96}$') +file_re = re.compile(r'^batres([0-9]+)\.txt$') + +if not os.path.isdir(root): + raise RuntimeError('preserved battle-result directory is not readable') + +for season in sorted(os.scandir(root), key=lambda item: item.name): + if not season.is_dir(follow_symlinks=False) or not season_re.fullmatch(season.name): + continue + if action == 'read' and season.name not in selected: + continue + for item in sorted(os.scandir(season.path), key=lambda entry: entry.name): + match = file_re.fullmatch(item.name) + if not match or not item.is_file(follow_symlinks=False): + continue + size = item.stat(follow_symlinks=False).st_size + if size > MAX_BYTES: + raise RuntimeError(f'battle-result file exceeds {MAX_BYTES} bytes: {season.name}/{item.name}') + digest = hashlib.sha256() + content = bytearray() + with open(item.path, 'rb') as handle: + while True: + chunk = handle.read(1024 * 1024) + if not chunk: + break + digest.update(chunk) + content.extend(chunk) + content.decode('utf-8') + if b'\0' in content: + raise RuntimeError(f'battle-result file contains NUL: {season.name}/{item.name}') + result = { + 'serverId': season.name, + 'generalNo': int(match.group(1)), + 'sourceBytes': size, + 'contentHash': digest.hexdigest(), + } + if action == 'read': + result['contentBase64'] = base64.b64encode(content).decode('ascii') + print(json.dumps(result, ensure_ascii=True), flush=True) +`; + +const safeSourceDirectory = (directory: string): string => { + if (!path.isAbsolute(directory) || directory.length > 4096 || /[\0\r\n]/u.test(directory)) { + throw new Error('Preserved battle-result directory must be a safe absolute path'); + } + return directory; +}; + +export const resolveBattleResultSourceConfig = async ( + input: { directory: string; sshHost?: string }, + sourceKey: string, + profile: LegacyArchiveProfile, + configDirectory: string +): Promise => { + const sshHost = input.sshHost?.trim(); + let directory: string; + let kind: BattleResultSourceConfig['kind']; + if (sshHost) { + if (!SSH_HOST.test(sshHost) || sshHost.startsWith('-')) { + throw new Error('battleResults.sshHost must be a safe SSH host or configured alias'); + } + directory = safeSourceDirectory(input.directory.trim()); + kind = 'ssh'; + } else { + directory = await realpath(path.resolve(configDirectory, input.directory)); + safeSourceDirectory(directory); + const info = await lstat(directory); + if (!info.isDirectory()) throw new Error('battleResults.directory must be a directory'); + kind = 'local'; + } + const fingerprint = createHash('sha256') + .update(JSON.stringify({ kind, directory, sshHost: sshHost ?? null, profile })) + .digest('hex'); + return { + kind, + directory, + ...(sshHost ? { sshHost } : {}), + identity: { key: `${sourceKey}:battle-results`, fingerprint }, + }; +}; + +const localDescriptors = async ( + source: BattleResultSourceConfig, + profile: LegacyArchiveProfile, + selected?: ReadonlySet +): Promise => { + const seasonPattern = new RegExp(`^${profile}_[A-Za-z0-9_-]{1,96}$`, 'u'); + const filePattern = /^batres([0-9]+)\.txt$/u; + const result: RemoteDescriptor[] = []; + for (const season of (await readdir(source.directory, { withFileTypes: true })).sort((a, b) => + a.name.localeCompare(b.name) + )) { + if (!season.isDirectory() || !seasonPattern.test(season.name) || (selected && !selected.has(season.name))) { + continue; + } + const seasonPath = path.join(source.directory, season.name); + for (const file of (await readdir(seasonPath, { withFileTypes: true })).sort((a, b) => + a.name.localeCompare(b.name) + )) { + const match = filePattern.exec(file.name); + if (!match || !file.isFile()) continue; + const filePath = path.join(seasonPath, file.name); + const info = await lstat(filePath); + if (!info.isFile() || info.isSymbolicLink()) continue; + if (info.size > MAX_BATTLE_RESULT_FILE_BYTES) { + throw new Error( + `Battle-result file exceeds ${MAX_BATTLE_RESULT_FILE_BYTES} bytes: ${season.name}/${file.name}` + ); + } + const content = await readFile(filePath); + const decoded = new TextDecoder('utf-8', { fatal: true }).decode(content); + if (decoded.includes('\0')) { + throw new Error(`Battle-result file contains NUL: ${season.name}/${file.name}`); + } + result.push({ + serverId: season.name, + generalNo: Number(match[1]), + sourceBytes: info.size, + contentHash: createHash('sha256').update(content).digest('hex'), + ...(selected ? { contentBase64: content.toString('base64') } : {}), + }); + } + } + return result; +}; + +const shellQuote = (value: string): string => `'${value.replaceAll("'", `'"'"'`)}'`; + +const remoteDescriptors = async ( + source: BattleResultSourceConfig, + profile: LegacyArchiveProfile, + selected?: ReadonlySet +): Promise => { + if (!source.sshHost) throw new Error('SSH battle-result source is missing its host'); + const action = selected ? 'read' : 'list'; + const encodedSelection = Buffer.from(JSON.stringify([...(selected ?? [])]), 'utf8').toString('base64'); + const remoteCommand = ['python3', '-c', REMOTE_READER, action, source.directory, profile, encodedSelection] + .map(shellQuote) + .join(' '); + const child = spawn('ssh', ['-C', '--', source.sshHost, remoteCommand], { + stdio: ['ignore', 'pipe', 'pipe'], + }); + let stderr = ''; + child.stderr.setEncoding('utf8'); + child.stderr.on('data', (chunk: string) => { + if (stderr.length < 8192) stderr += chunk.slice(0, 8192 - stderr.length); + }); + const exit = new Promise((resolve, reject) => { + child.once('error', reject); + child.once('close', (code) => resolve(code ?? 1)); + }); + const output: RemoteDescriptor[] = []; + const lines = readline.createInterface({ input: child.stdout, crlfDelay: Infinity }); + try { + for await (const line of lines) { + if (!line.trim()) continue; + const parsed = JSON.parse(line) as RemoteDescriptor; + output.push(parsed); + } + } catch (error) { + child.kill(); + throw new Error('Could not parse the preserved battle-result SSH stream', { cause: error }); + } + const code = await exit; + if (code !== 0) { + throw new Error(`Preserved battle-result SSH scan failed (${code}): ${stderr.trim() || 'no error text'}`); + } + return output; +}; + +const descriptors = ( + source: BattleResultSourceConfig, + profile: LegacyArchiveProfile, + selected?: ReadonlySet +): Promise => + source.kind === 'local' + ? localDescriptors(source, profile, selected) + : remoteDescriptors(source, profile, selected); + +const validateDescriptor = (descriptor: RemoteDescriptor, profile: LegacyArchiveProfile): void => { + if (!new RegExp(`^${profile}_[A-Za-z0-9_-]{1,96}$`, 'u').test(descriptor.serverId)) { + throw new Error(`Invalid battle-result server ID: ${descriptor.serverId}`); + } + if (!Number.isSafeInteger(descriptor.generalNo) || descriptor.generalNo < 0) { + throw new Error(`Invalid battle-result general number for ${descriptor.serverId}`); + } + if ( + !Number.isSafeInteger(descriptor.sourceBytes) || + descriptor.sourceBytes < 0 || + descriptor.sourceBytes > MAX_BATTLE_RESULT_FILE_BYTES || + !HASH.test(descriptor.contentHash) + ) { + throw new Error(`Invalid battle-result descriptor for ${descriptor.serverId}/${descriptor.generalNo}`); + } +}; + +const manifestFor = (serverId: string, files: BattleResultFileDescriptor[]): BattleResultSeasonManifest => { + files.sort((left, right) => left.generalNo - right.generalNo); + const manifest = createHash('sha256'); + for (const file of files) { + manifest.update(`${file.generalNo}\0${file.sourceBytes}\0${file.contentHash}\n`); + } + return { + serverId, + files, + fileCount: files.length, + totalBytes: files.reduce((sum, file) => sum + file.sourceBytes, 0), + manifestHash: manifest.digest('hex'), + }; +}; + +export const listBattleResultSeasons = async ( + source: BattleResultSourceConfig, + profile: LegacyArchiveProfile +): Promise => { + const grouped = new Map(); + for (const descriptor of await descriptors(source, profile)) { + validateDescriptor(descriptor, profile); + const files = grouped.get(descriptor.serverId) ?? []; + if (files.some((file) => file.generalNo === descriptor.generalNo)) { + throw new Error(`Duplicate battle-result file for ${descriptor.serverId}/${descriptor.generalNo}`); + } + files.push(descriptor); + grouped.set(descriptor.serverId, files); + } + return [...grouped.entries()] + .sort(([left], [right]) => left.localeCompare(right)) + .map(([serverId, files]) => manifestFor(serverId, files)); +}; + +export const readBattleResultSeason = async ( + source: BattleResultSourceConfig, + profile: LegacyArchiveProfile, + serverId: string +): Promise<{ manifest: BattleResultSeasonManifest; files: BattleResultFile[] }> => { + const files: BattleResultFile[] = []; + const decoder = new TextDecoder('utf-8', { fatal: true }); + for (const descriptor of await descriptors(source, profile, new Set([serverId]))) { + validateDescriptor(descriptor, profile); + if (descriptor.serverId !== serverId || typeof descriptor.contentBase64 !== 'string') { + throw new Error(`Unexpected battle-result file while reading ${serverId}`); + } + const bytes = Buffer.from(descriptor.contentBase64, 'base64'); + if ( + bytes.byteLength !== descriptor.sourceBytes || + createHash('sha256').update(bytes).digest('hex') !== descriptor.contentHash + ) { + throw new Error(`Battle-result content changed while reading ${serverId}/${descriptor.generalNo}`); + } + const content = decoder.decode(bytes); + if (content.includes('\0')) + throw new Error(`Battle-result file contains NUL: ${serverId}/${descriptor.generalNo}`); + files.push({ + serverId, + generalNo: descriptor.generalNo, + sourceBytes: descriptor.sourceBytes, + contentHash: descriptor.contentHash, + content, + lineCount: content.split(/\r?\n/u).filter((line) => line.length > 0).length, + }); + } + return { manifest: manifestFor(serverId, files), files }; +}; diff --git a/tools/legacy-db-migration/src/battleResults.ts b/tools/legacy-db-migration/src/battleResults.ts new file mode 100644 index 00000000..e5e9d1aa --- /dev/null +++ b/tools/legacy-db-migration/src/battleResults.ts @@ -0,0 +1,316 @@ +import type { Pool as PgPool, PoolClient } from 'pg'; + +import { + listBattleResultSeasons, + readBattleResultSeason, + type BattleResultSeasonManifest, + type BattleResultSourceConfig, +} from './battleResultSource.js'; +import { upsertRows, withMigrationLock, type TargetRow } from './db.js'; +import type { LegacyArchiveProfile } from './game.js'; +import { validateSourceIdentity, type MigrationExecutionOptions } from './incremental.js'; + +interface StoredSeasonCheckpoint { + sourceFingerprint: string; + serverId: string; + manifestHash: string; + fileCount: number; + totalBytes: number; +} + +export interface BattleResultMigrationSummary { + command: 'battle-results'; + apply: boolean; + mode: 'full' | 'incremental'; + sourceKey: string; + importRunId: string | null; + counts: { + discoveredSeasons: number; + discoveredFiles: number; + discoveredBytes: number; + unchangedSeasons: number; + pendingSeasons: number; + pendingFiles: number; + pendingBytes: number; + importedSeasons: number; + importedFiles: number; + importedLines: number; + importedBytes: number; + }; + progress: Record; +} + +const loadCheckpoints = async ( + client: PoolClient, + profile: LegacyArchiveProfile, + sourceKey: string +): Promise> => { + const result = await client.query<{ + source_fingerprint: string; + server_id: string; + manifest_hash: string; + file_count: number; + total_bytes: string; + }>( + `SELECT "source_fingerprint", "server_id", "manifest_hash", "file_count", "total_bytes" + FROM "legacy_archive"."battle_result_import_checkpoint" + WHERE "source_profile" = $1 AND "source_key" = $2`, + [profile, sourceKey] + ); + return new Map( + result.rows.map((row) => [ + row.server_id, + { + sourceFingerprint: row.source_fingerprint, + serverId: row.server_id, + manifestHash: row.manifest_hash, + fileCount: Number(row.file_count), + totalBytes: Number(row.total_bytes), + }, + ]) + ); +}; + +const sameManifest = (checkpoint: StoredSeasonCheckpoint, manifest: BattleResultSeasonManifest): boolean => + checkpoint.manifestHash === manifest.manifestHash && + checkpoint.fileCount === manifest.fileCount && + checkpoint.totalBytes === manifest.totalBytes; + +const upsertBattleResultRows = async ( + client: PoolClient, + profile: LegacyArchiveProfile, + manifest: BattleResultSeasonManifest, + source: BattleResultSourceConfig, + importRunId: string +): Promise<{ files: number; lines: number; bytes: number }> => { + const loaded = await readBattleResultSeason(source, profile, manifest.serverId); + if ( + loaded.manifest.manifestHash !== manifest.manifestHash || + loaded.manifest.fileCount !== manifest.fileCount || + loaded.manifest.totalBytes !== manifest.totalBytes + ) { + throw new Error(`Battle-result season changed after preflight: ${manifest.serverId}`); + } + + await client.query( + `DELETE FROM "legacy_archive"."general_battle_result" + WHERE "source_profile" = $1 AND "server_id" = $2`, + [profile, manifest.serverId] + ); + + let batch: TargetRow[] = []; + let batchBytes = 0; + let lines = 0; + const flush = async (): Promise => { + await upsertRows(client, 'legacy_archive.general_battle_result', batch, [ + 'source_profile', + 'server_id', + 'general_no', + ]); + batch = []; + batchBytes = 0; + }; + for (const file of loaded.files) { + if (batch.length >= 100 || batchBytes + file.sourceBytes > 4 * 1024 * 1024) await flush(); + batch.push({ + source_profile: profile, + server_id: file.serverId, + general_no: file.generalNo, + content: file.content, + line_count: file.lineCount, + source_bytes: file.sourceBytes, + content_hash: file.contentHash, + import_run_id: importRunId, + updated_at: new Date(), + }); + batchBytes += file.sourceBytes; + lines += file.lineCount; + } + await flush(); + return { files: loaded.files.length, lines, bytes: loaded.manifest.totalBytes }; +}; + +const saveCheckpoint = async ( + client: PoolClient, + profile: LegacyArchiveProfile, + source: BattleResultSourceConfig, + manifest: BattleResultSeasonManifest, + importRunId: string +): Promise => { + await client.query( + `INSERT INTO "legacy_archive"."battle_result_import_checkpoint" + ("source_profile", "source_key", "source_fingerprint", "server_id", "manifest_hash", + "file_count", "total_bytes", "import_run_id", "updated_at") + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, CURRENT_TIMESTAMP) + ON CONFLICT ("source_profile", "source_key", "server_id") DO UPDATE SET + "source_fingerprint" = EXCLUDED."source_fingerprint", + "manifest_hash" = EXCLUDED."manifest_hash", + "file_count" = EXCLUDED."file_count", + "total_bytes" = EXCLUDED."total_bytes", + "import_run_id" = EXCLUDED."import_run_id", + "updated_at" = CURRENT_TIMESTAMP`, + [ + profile, + source.identity.key, + source.identity.fingerprint, + manifest.serverId, + manifest.manifestHash, + manifest.fileCount, + manifest.totalBytes, + importRunId, + ] + ); +}; + +export const migrateBattleResults = async ( + targetPool: PgPool, + source: BattleResultSourceConfig, + apply: boolean, + profile: LegacyArchiveProfile, + execution: MigrationExecutionOptions, + prefetchedManifests?: readonly BattleResultSeasonManifest[] +): Promise => { + validateSourceIdentity(source.identity); + validateSourceIdentity(execution.source); + if (execution.source.key !== source.identity.key || execution.source.fingerprint !== source.identity.fingerprint) { + throw new Error('Preserved battle-result execution identity does not match its configured source'); + } + const manifests = prefetchedManifests ? [...prefetchedManifests] : await listBattleResultSeasons(source, profile); + const counts: BattleResultMigrationSummary['counts'] = { + discoveredSeasons: manifests.length, + discoveredFiles: manifests.reduce((sum, item) => sum + item.fileCount, 0), + discoveredBytes: manifests.reduce((sum, item) => sum + item.totalBytes, 0), + unchangedSeasons: 0, + pendingSeasons: 0, + pendingFiles: 0, + pendingBytes: 0, + importedSeasons: 0, + importedFiles: 0, + importedLines: 0, + importedBytes: 0, + }; + const progress: BattleResultMigrationSummary['progress'] = {}; + const client = await targetPool.connect(); + let importRunId: string | null = null; + try { + const checkpoints = await loadCheckpoints(client, profile, source.identity.key); + const currentServerIds = new Set(manifests.map((manifest) => manifest.serverId)); + const missing = [...checkpoints.keys()].filter((serverId) => !currentServerIds.has(serverId)); + if (missing.length) { + throw new Error(`Preserved battle-result seasons disappeared from the source: ${missing.join(', ')}`); + } + if (execution.mode === 'incremental') { + if (manifests.length > 0 && checkpoints.size === 0) { + throw new Error('Incremental preserved battle-result migration requires a completed full checkpoint'); + } + } + const pending: BattleResultSeasonManifest[] = []; + for (const manifest of manifests) { + const checkpoint = checkpoints.get(manifest.serverId); + if (checkpoint && checkpoint.sourceFingerprint !== source.identity.fingerprint) { + throw new Error(`Preserved battle-result source fingerprint changed for ${manifest.serverId}`); + } + if (checkpoint && sameManifest(checkpoint, manifest)) { + counts.unchangedSeasons += 1; + progress[manifest.serverId] = { + status: 'UNCHANGED', + files: manifest.fileCount, + bytes: manifest.totalBytes, + }; + continue; + } + if (checkpoint && execution.mode === 'incremental') { + throw new Error(`Preserved battle-result season changed after checkpoint: ${manifest.serverId}`); + } + pending.push(manifest); + counts.pendingSeasons += 1; + counts.pendingFiles += manifest.fileCount; + counts.pendingBytes += manifest.totalBytes; + progress[manifest.serverId] = { status: 'PENDING', files: manifest.fileCount, bytes: manifest.totalBytes }; + } + + if (!apply) { + return { + command: 'battle-results', + apply, + mode: execution.mode, + sourceKey: source.identity.key, + importRunId, + counts, + progress, + }; + } + + await withMigrationLock( + client, + `sammo-legacy-battle-results-v1:${profile}:${source.identity.key}`, + async () => { + const created = await client.query<{ id: string }>( + `INSERT INTO "legacy_archive"."battle_result_import_run" + ("source_profile", "source_key", "source_fingerprint", "mode", "status") + VALUES ($1, $2, $3, $4, 'RUNNING') RETURNING "id"`, + [profile, source.identity.key, source.identity.fingerprint, execution.mode] + ); + importRunId = created.rows[0]?.id ?? null; + if (!importRunId) throw new Error('Failed to create preserved battle-result import run'); + try { + for (const manifest of pending) { + await client.query('BEGIN'); + try { + const imported = await upsertBattleResultRows( + client, + profile, + manifest, + source, + importRunId + ); + await saveCheckpoint(client, profile, source, manifest, importRunId); + await client.query('COMMIT'); + counts.importedSeasons += 1; + counts.importedFiles += imported.files; + counts.importedLines += imported.lines; + counts.importedBytes += imported.bytes; + progress[manifest.serverId] = { + status: 'IMPORTED', + files: imported.files, + bytes: imported.bytes, + }; + } catch (error) { + await client.query('ROLLBACK'); + throw error; + } + } + await client.query( + `UPDATE "legacy_archive"."battle_result_import_run" + SET "status" = 'COMPLETED', "finished_at" = CURRENT_TIMESTAMP, + "counts" = $2::jsonb, "progress" = $3::jsonb + WHERE "id" = $1`, + [importRunId, JSON.stringify(counts), JSON.stringify(progress)] + ); + } catch (error) { + const message = + error instanceof Error ? error.message.slice(0, 2000) : String(error).slice(0, 2000); + await client.query( + `UPDATE "legacy_archive"."battle_result_import_run" + SET "status" = 'FAILED', "finished_at" = CURRENT_TIMESTAMP, + "counts" = $2::jsonb, "progress" = $3::jsonb, "error" = $4 + WHERE "id" = $1`, + [importRunId, JSON.stringify(counts), JSON.stringify(progress), message] + ); + throw error; + } + } + ); + return { + command: 'battle-results', + apply, + mode: execution.mode, + sourceKey: source.identity.key, + importRunId, + counts, + progress, + }; + } finally { + client.release(); + } +}; diff --git a/tools/legacy-db-migration/src/config.ts b/tools/legacy-db-migration/src/config.ts index 17bd5f3e..4d68144a 100644 --- a/tools/legacy-db-migration/src/config.ts +++ b/tools/legacy-db-migration/src/config.ts @@ -3,6 +3,7 @@ import { lstat, open } from 'node:fs/promises'; import { isIP } from 'node:net'; import path from 'node:path'; +import { resolveBattleResultSourceConfig, type BattleResultSourceConfig } from './battleResultSource.js'; import { isLegacyArchiveProfile, LEGACY_ARCHIVE_PROFILES, type LegacyArchiveProfile } from './game.js'; import { fingerprintMariaConnection, type MigrationSourceIdentity } from './incremental.js'; @@ -13,6 +14,7 @@ export interface ResolvedMigrationStage { sourceUrl: string; targetUrl: string; sourceIdentity: MigrationSourceIdentity; + battleResults?: BattleResultSourceConfig; } export interface ResolvedMigrationPlan { @@ -137,7 +139,7 @@ const resolveTargetUrl = (record: Record, label: string): strin const parseStage = (value: unknown, label: string): Record => { const record = asRecord(value, label); - rejectUnknownKeys(record, ['source', 'targetUrlEnv', 'profile', 'enabled'], label); + rejectUnknownKeys(record, ['source', 'targetUrlEnv', 'profile', 'enabled', 'battleResults'], label); if (!('source' in record)) throw new Error(`${label}.source is required`); return record; }; @@ -192,6 +194,22 @@ export const loadMigrationPlan = async (configPathInput: string): Promise/batres.txt', + target: 'legacy_archive.general_battle_result', + strategy: 'filesystem-season', + contents: '구형 기수의 장수별 전투 결과 요약; batlog 페이즈 상세는 제외', + }, + { + source: 'hall', + target: 'legacy_archive.hall', + strategy: 'append', + contents: '명예의 전당 순위와 점수', + }, + { + source: 'ng_old_nations', + target: 'legacy_archive.nation', + strategy: 'append', + contents: '지난 국가 구성, 장수 목록과 국가 연혁', + }, + { + source: 'emperior', + target: 'legacy_archive.emperor', + strategy: 'append', + contents: '왕조 일람, 통일 국가/황제/관직/국력/연혁', + }, + { + source: 'inheritance_result', + target: 'inheritance_result', + strategy: 'append', + contents: '유산 결과 원문과 점수', + }, + { + source: 'user_record', + target: 'inheritance_log', + strategy: 'append', + contents: '사용자별 유산 획득/사용 장기 기록', + }, + { + source: 'storage:inheritance_* / user_*', + target: 'legacy_game_storage + inheritance_point + inheritance_user_state', + strategy: 'rescan', + contents: '유산 포인트와 사용자 유산 상태 및 원본 tuple', + }, + { + source: 'ng_history', + target: 'legacy_archive.yearbook', + strategy: 'append', + contents: '월별 지도, 국가, 천하 동향과 전체 기록 연감', + }, +]; + +export const migrationInventoryForStage = (stage: ResolvedMigrationStage): readonly MigrationInventoryItem[] => + stage.kind === 'gateway' + ? GATEWAY_MIGRATION_INVENTORY + : GAME_MIGRATION_INVENTORY.filter( + (item) => item.strategy !== 'filesystem-season' || stage.battleResults !== undefined + ); diff --git a/tools/legacy-db-migration/src/plan.ts b/tools/legacy-db-migration/src/plan.ts index 0dd0d02b..775f85b4 100644 --- a/tools/legacy-db-migration/src/plan.ts +++ b/tools/legacy-db-migration/src/plan.ts @@ -1,18 +1,31 @@ +import { listBattleResultSeasons, type BattleResultSeasonManifest } from './battleResultSource.js'; +import { migrateBattleResults, type BattleResultMigrationSummary } from './battleResults.js'; import { createMariaPool, createPostgresPool, querySource } from './db.js'; import { migrateGame } from './game.js'; import { migrateGateway, type MigrationSummary } from './gateway.js'; import type { MigrationMode } from './incremental.js'; import type { ResolvedMigrationPlan, ResolvedMigrationStage } from './config.js'; +import { migrationInventoryForStage } from './inventory.js'; export interface PlanRunSummary { command: 'run-plan'; sourceSet: string; mode: MigrationMode; apply: boolean; - stages: Array<{ name: string; status: 'COMPLETED'; summary: MigrationSummary }>; + stages: Array<{ + name: string; + status: 'COMPLETED'; + summary: MigrationSummary; + battleResults?: BattleResultMigrationSummary; + }>; } -const preflightStage = async (stage: ResolvedMigrationStage): Promise => { +interface StagePreflight { + battleResults?: { seasons: number; files: number; bytes: number }; + battleResultManifests?: readonly BattleResultSeasonManifest[]; +} + +const preflightStage = async (stage: ResolvedMigrationStage): Promise => { const source = createMariaPool(stage.sourceUrl); const target = createPostgresPool(stage.targetUrl); try { @@ -63,6 +76,27 @@ const preflightStage = async (stage: ResolvedMigrationStage): Promise => { if (!targetReady.rows[0]?.table_name) { throw new Error(`Target migrations are not current for ${stage.name}; missing ${targetDataTable}`); } + if (stage.kind === 'game' && stage.battleResults) { + const battleResultReady = await target.query<{ table_name: string | null }>( + 'SELECT to_regclass($1) AS table_name', + ['legacy_archive.general_battle_result'] + ); + if (!battleResultReady.rows[0]?.table_name) { + throw new Error( + `Target migrations are not current for ${stage.name}; missing legacy_archive.general_battle_result` + ); + } + const seasons = await listBattleResultSeasons(stage.battleResults, stage.profile!); + return { + battleResults: { + seasons: seasons.length, + files: seasons.reduce((sum, season) => sum + season.fileCount, 0), + bytes: seasons.reduce((sum, season) => sum + season.totalBytes, 0), + }, + battleResultManifests: seasons, + }; + } + return {}; } finally { await source.end(); await target.end(); @@ -70,11 +104,21 @@ const preflightStage = async (stage: ResolvedMigrationStage): Promise => { }; export const checkMigrationPlan = async (plan: ResolvedMigrationPlan): Promise> => { - for (const stage of plan.stages) await preflightStage(stage); + const stages = []; + for (const stage of plan.stages) { + const preflight = await preflightStage(stage); + stages.push({ + name: stage.name, + kind: stage.kind, + status: 'READY', + inventory: migrationInventoryForStage(stage), + ...(preflight.battleResults ? { battleResults: preflight.battleResults } : {}), + }); + } return { command: 'check-plan', sourceSet: plan.sourceSet, - stages: plan.stages.map((stage) => ({ name: stage.name, kind: stage.kind, status: 'READY' })), + stages, }; }; @@ -84,7 +128,10 @@ export const runMigrationPlan = async ( apply: boolean, migratedAt = new Date() ): Promise => { - await checkMigrationPlan(plan); + const preflights = new Map(); + for (const stage of plan.stages) { + preflights.set(stage.name, await preflightStage(stage)); + } const stages: PlanRunSummary['stages'] = []; for (const stage of plan.stages) { const source = createMariaPool(stage.sourceUrl); @@ -95,7 +142,26 @@ export const runMigrationPlan = async ( stage.kind === 'gateway' ? await migrateGateway(source, target, apply, migratedAt, execution) : await migrateGame(source, target, apply, stage.profile!, execution); - stages.push({ name: stage.name, status: 'COMPLETED', summary }); + const battleResults = + stage.kind === 'game' && stage.battleResults + ? await migrateBattleResults( + target, + stage.battleResults, + apply, + stage.profile!, + { + mode, + source: stage.battleResults.identity, + }, + preflights.get(stage.name)?.battleResultManifests + ) + : undefined; + stages.push({ + name: stage.name, + status: 'COMPLETED', + summary, + ...(battleResults ? { battleResults } : {}), + }); } finally { await source.end(); await target.end(); diff --git a/tools/legacy-db-migration/test/battleResultSource.test.ts b/tools/legacy-db-migration/test/battleResultSource.test.ts new file mode 100644 index 00000000..f44870b6 --- /dev/null +++ b/tools/legacy-db-migration/test/battleResultSource.test.ts @@ -0,0 +1,69 @@ +import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; + +import { afterEach, describe, expect, it } from 'vitest'; + +import { + listBattleResultSeasons, + readBattleResultSeason, + resolveBattleResultSourceConfig, +} from '../src/battleResultSource.js'; + +const temporaryDirectories: string[] = []; + +afterEach(async () => { + await Promise.all( + temporaryDirectories.splice(0).map((directory) => rm(directory, { recursive: true, force: true })) + ); +}); + +const fixture = async () => { + const root = await mkdtemp(path.join(os.tmpdir(), 'sammo-battle-results-')); + temporaryDirectories.push(root); + const season = path.join(root, 'che_190815_w0sU'); + await mkdir(season); + await writeFile(path.join(season, 'batres17.txt'), '◆190년 1월:첫 전투\n◆190년 2월:둘째 전투\n'); + await writeFile(path.join(season, 'batlog17.txt'), '페이즈 상세는 제외\n'); + await mkdir(path.join(root, 'not-a-season')); + return { root, season }; +}; + +describe('preserved battle-result source', () => { + it('maps only batres files by profile/server/general and preserves source text', async () => { + const { root } = await fixture(); + const source = await resolveBattleResultSourceConfig({ directory: root }, 'fixture:che', 'che', process.cwd()); + const seasons = await listBattleResultSeasons(source, 'che'); + + expect(seasons).toHaveLength(1); + expect(seasons[0]).toMatchObject({ serverId: 'che_190815_w0sU', fileCount: 1 }); + expect(seasons[0]?.files[0]).toMatchObject({ generalNo: 17 }); + + const loaded = await readBattleResultSeason(source, 'che', 'che_190815_w0sU'); + expect(loaded.manifest.manifestHash).toBe(seasons[0]?.manifestHash); + expect(loaded.files).toHaveLength(1); + expect(loaded.files[0]).toMatchObject({ generalNo: 17, lineCount: 2 }); + expect(loaded.files[0]?.content).toContain('둘째 전투'); + }); + + it('builds a password-free source identity for the configured SSH location', async () => { + const source = await resolveBattleResultSourceConfig( + { directory: '/srv/sammo/che/logs/preserved', sshHost: 'serv' }, + 'cutover:che', + 'che', + process.cwd() + ); + + expect(source).toMatchObject({ kind: 'ssh', sshHost: 'serv' }); + expect(source.identity.key).toBe('cutover:che:battle-results'); + expect(source.identity.fingerprint).toMatch(/^[a-f0-9]{64}$/u); + }); + + it('rejects invalid UTF-8 during preflight instead of failing after target writes begin', async () => { + const { root, season } = await fixture(); + await writeFile(path.join(season, 'batres18.txt'), Buffer.from([0xff])); + const source = await resolveBattleResultSourceConfig({ directory: root }, 'fixture:che', 'che', process.cwd()); + + await expect(listBattleResultSeasons(source, 'che')).rejects.toThrow(); + }); +}); diff --git a/tools/legacy-db-migration/test/battleResults.test.ts b/tools/legacy-db-migration/test/battleResults.test.ts new file mode 100644 index 00000000..000fd04b --- /dev/null +++ b/tools/legacy-db-migration/test/battleResults.test.ts @@ -0,0 +1,177 @@ +import { mkdir, mkdtemp, rename, rm, writeFile } from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; + +import type { Pool as PgPool, PoolClient, QueryResult } from 'pg'; +import { afterEach, describe, expect, it, vi } from 'vitest'; + +import { listBattleResultSeasons, type BattleResultSourceConfig } from '../src/battleResultSource.js'; +import { migrateBattleResults } from '../src/battleResults.js'; + +const temporaryDirectories: string[] = []; + +afterEach(async () => { + await Promise.all( + temporaryDirectories.splice(0).map((directory) => rm(directory, { recursive: true, force: true })) + ); +}); + +const sourceFixture = async (): Promise => { + const root = await mkdtemp(path.join(os.tmpdir(), 'sammo-battle-results-migrate-')); + temporaryDirectories.push(root); + const season = path.join(root, 'che_190815_w0sU'); + await mkdir(season); + await writeFile(path.join(season, 'batres17.txt'), '첫 전투\n둘째 전투\n'); + return { + kind: 'local', + directory: root, + identity: { key: 'fixture:che:battle-results', fingerprint: 'b'.repeat(64) }, + }; +}; + +const targetPool = ( + checkpointRows: Array> = [], + failPattern?: string +): { pool: PgPool; queries: Array<{ sql: string; values: readonly unknown[] }> } => { + const queries: Array<{ sql: string; values: readonly unknown[] }> = []; + const query = vi.fn(async (sql: string, values: readonly unknown[] = []) => { + queries.push({ sql, values }); + if (failPattern && sql.includes(failPattern)) { + failPattern = undefined; + throw new Error('synthetic battle-result write failure'); + } + if (sql.includes('FROM "legacy_archive"."battle_result_import_checkpoint"')) { + return { rows: checkpointRows, rowCount: checkpointRows.length } as unknown as QueryResult; + } + if (sql.includes('INSERT INTO "legacy_archive"."battle_result_import_run"')) { + return { rows: [{ id: '91' }], rowCount: 1 } as QueryResult<{ id: string }>; + } + return { rows: [], rowCount: 0 } as unknown as QueryResult; + }); + const client = { query, release: vi.fn() } as unknown as PoolClient; + return { pool: { connect: vi.fn(async () => client) } as unknown as PgPool, queries }; +}; + +describe('preserved battle-result migration', () => { + it('rejects an execution identity different from the configured archive source', async () => { + const source = await sourceFixture(); + await expect( + migrateBattleResults(targetPool().pool, source, false, 'che', { + mode: 'full', + source: { key: 'different:source', fingerprint: 'c'.repeat(64) }, + }) + ).rejects.toThrow('execution identity does not match'); + }); + + it('imports one immutable season transactionally and checkpoints it', async () => { + const source = await sourceFixture(); + const target = targetPool(); + const summary = await migrateBattleResults(target.pool, source, true, 'che', { + mode: 'full', + source: source.identity, + }); + const sql = target.queries.map((entry) => entry.sql).join('\n'); + + expect(summary).toMatchObject({ + importRunId: '91', + counts: { discoveredSeasons: 1, importedSeasons: 1, importedFiles: 1, importedLines: 2 }, + }); + expect(sql).toContain('INSERT INTO "legacy_archive"."general_battle_result"'); + expect(sql).toContain('DELETE FROM "legacy_archive"."general_battle_result"'); + expect(sql).toContain('INSERT INTO "legacy_archive"."battle_result_import_checkpoint"'); + expect(target.queries.some((entry) => entry.sql === 'BEGIN')).toBe(true); + expect(target.queries.some((entry) => entry.sql === 'COMMIT')).toBe(true); + }); + + it('skips an unchanged checkpoint during incremental import', async () => { + const source = await sourceFixture(); + const [manifest] = await listBattleResultSeasons(source, 'che'); + const target = targetPool([ + { + source_fingerprint: source.identity.fingerprint, + server_id: manifest!.serverId, + manifest_hash: manifest!.manifestHash, + file_count: manifest!.fileCount, + total_bytes: String(manifest!.totalBytes), + }, + ]); + const summary = await migrateBattleResults(target.pool, source, false, 'che', { + mode: 'incremental', + source: source.identity, + }); + + expect(summary.counts).toMatchObject({ unchangedSeasons: 1, pendingSeasons: 0, importedFiles: 0 }); + expect(target.queries.some((entry) => entry.sql.includes('general_battle_result'))).toBe(false); + }); + + it('reuses manifests collected by plan preflight instead of scanning the source twice', async () => { + const source = await sourceFixture(); + const manifests = await listBattleResultSeasons(source, 'che'); + await rename(source.directory, `${source.directory}-moved`); + temporaryDirectories.push(`${source.directory}-moved`); + const target = targetPool(); + + const summary = await migrateBattleResults( + target.pool, + source, + false, + 'che', + { mode: 'full', source: source.identity }, + manifests + ); + + expect(summary.counts).toMatchObject({ discoveredSeasons: 1, pendingSeasons: 1 }); + }); + + it('rejects changed checkpointed content in incremental mode', async () => { + const source = await sourceFixture(); + const [manifest] = await listBattleResultSeasons(source, 'che'); + const target = targetPool([ + { + source_fingerprint: source.identity.fingerprint, + server_id: manifest!.serverId, + manifest_hash: 'c'.repeat(64), + file_count: manifest!.fileCount, + total_bytes: String(manifest!.totalBytes), + }, + ]); + + await expect( + migrateBattleResults(target.pool, source, false, 'che', { + mode: 'incremental', + source: source.identity, + }) + ).rejects.toThrow('changed after checkpoint'); + }); + + it('rejects a disappeared checkpointed season in full mode instead of leaving stale target rows', async () => { + const source = await sourceFixture(); + const target = targetPool([ + { + source_fingerprint: source.identity.fingerprint, + server_id: 'che_180101_missing', + manifest_hash: 'c'.repeat(64), + file_count: 1, + total_bytes: '10', + }, + ]); + + await expect( + migrateBattleResults(target.pool, source, false, 'che', { + mode: 'full', + source: source.identity, + }) + ).rejects.toThrow('seasons disappeared from the source'); + }); + + it('rolls back the current season and records a failed run', async () => { + const source = await sourceFixture(); + const target = targetPool([], 'general_battle_result'); + await expect( + migrateBattleResults(target.pool, source, true, 'che', { mode: 'full', source: source.identity }) + ).rejects.toThrow('synthetic battle-result write failure'); + + expect(target.queries.some((entry) => entry.sql === 'ROLLBACK')).toBe(true); + expect(target.queries.some((entry) => entry.sql.includes(`"status" = 'FAILED'`))).toBe(true); + }); +}); diff --git a/tools/legacy-db-migration/test/fixtures/preserved/che_190815_w0sU/batlog17.txt b/tools/legacy-db-migration/test/fixtures/preserved/che_190815_w0sU/batlog17.txt new file mode 100644 index 00000000..66b6f4df --- /dev/null +++ b/tools/legacy-db-migration/test/fixtures/preserved/che_190815_w0sU/batlog17.txt @@ -0,0 +1 @@ +이 페이즈 상세 로그는 이관하면 안 된다. diff --git a/tools/legacy-db-migration/test/fixtures/preserved/che_190815_w0sU/batres17.txt b/tools/legacy-db-migration/test/fixtures/preserved/che_190815_w0sU/batres17.txt new file mode 100644 index 00000000..dc914242 --- /dev/null +++ b/tools/legacy-db-migration/test/fixtures/preserved/che_190815_w0sU/batres17.txt @@ -0,0 +1,2 @@ +189년 4월: 테스트 장수 승리 +189년 3월: 테스트 장수 패배 diff --git a/tools/legacy-db-migration/test/inventory.test.ts b/tools/legacy-db-migration/test/inventory.test.ts new file mode 100644 index 00000000..6cb4db6d --- /dev/null +++ b/tools/legacy-db-migration/test/inventory.test.ts @@ -0,0 +1,57 @@ +import { describe, expect, it } from 'vitest'; + +import type { ResolvedMigrationStage } from '../src/config.js'; +import { migrationInventoryForStage } from '../src/inventory.js'; + +const gameStage = (withBattleResults: boolean): ResolvedMigrationStage => ({ + kind: 'game', + name: 'che', + profile: 'che', + sourceUrl: 'mariadb://source.invalid/che', + targetUrl: 'postgresql://target.invalid/game', + sourceIdentity: { key: 'fixture:che', fingerprint: 'a'.repeat(64) }, + ...(withBattleResults + ? { + battleResults: { + kind: 'ssh' as const, + directory: '/srv/sammo/che/logs/preserved', + sshHost: 'serv', + identity: { key: 'fixture:che:battle-results', fingerprint: 'b'.repeat(64) }, + }, + } + : {}), +}); + +describe('migration plan inventory', () => { + it('lists each gateway item with its transferred information', () => { + const inventory = migrationInventoryForStage({ + kind: 'gateway', + name: 'gateway', + sourceUrl: 'mariadb://source.invalid/root', + targetUrl: 'postgresql://target.invalid/gateway', + sourceIdentity: { key: 'fixture:gateway', fingerprint: 'a'.repeat(64) }, + }); + + expect(inventory.map((item) => item.source)).toEqual([ + 'member', + 'member_log', + 'banned_member', + 'storage', + 'system', + ]); + expect(inventory.every((item) => item.contents.length > 0)).toBe(true); + }); + + it('lists batres only when that filesystem source is configured', () => { + expect(migrationInventoryForStage(gameStage(false)).some((item) => item.strategy === 'filesystem-season')).toBe( + false + ); + expect(migrationInventoryForStage(gameStage(true))).toContainEqual( + expect.objectContaining({ + source: 'logs/preserved//batres.txt', + strategy: 'filesystem-season', + contents: expect.stringContaining('batlog 페이즈 상세는 제외'), + }) + ); + }); +});