From 0e3292f59150ff9972cbc8dae01bec1394a10310 Mon Sep 17 00:00:00 2001 From: hided62 Date: Sun, 23 Aug 2026 01:55:22 +0000 Subject: [PATCH] =?UTF-8?q?feat:=20=EC=98=A4=EB=9E=98=EB=90=9C=20=ED=94=84?= =?UTF-8?q?=EB=9F=B0=ED=8A=B8=EC=97=94=EB=93=9C=20=EB=B9=8C=EB=93=9C=20?= =?UTF-8?q?=EC=9E=90=EC=82=B0=EC=9D=84=20=EC=95=88=EC=A0=84=ED=95=98?= =?UTF-8?q?=EA=B2=8C=20=EC=A0=95=EB=A6=AC=ED=95=9C=EB=8B=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 현재·이전 release와 공유 asset 의존성, 진행 중 commit을 보호하고 24시간 유예 및 최신 2개 cache를 적용한다. Gateway와 profile daemon의 기존 24시간 관리 주기에 artifact cleanup을 포함하고 fail-closed 회귀 테스트와 운영 문서를 보강한다. --- .../orchestrator/frontendArtifactManager.ts | 319 +++++++++++++++++- .../src/orchestrator/gatewayOrchestrator.ts | 64 +++- .../test/frontendArtifactManager.test.ts | 159 +++++++++ .../test/orchestratorWorkspaceCleanup.test.ts | 63 +++- app/release-controller/src/index.ts | 8 +- .../src/releaseController.ts | 38 +++ .../test/releaseController.test.ts | 61 +++- docs/architecture/runtime.md | 7 + docs/release-operations.md | 30 ++ 9 files changed, 715 insertions(+), 34 deletions(-) diff --git a/app/gateway-api/src/orchestrator/frontendArtifactManager.ts b/app/gateway-api/src/orchestrator/frontendArtifactManager.ts index 1065bc6b..2a77d197 100644 --- a/app/gateway-api/src/orchestrator/frontendArtifactManager.ts +++ b/app/gateway-api/src/orchestrator/frontendArtifactManager.ts @@ -1,4 +1,5 @@ import { createHash, randomUUID } from 'node:crypto'; +import type { Dirent } from 'node:fs'; import fs from 'node:fs/promises'; import path from 'node:path'; @@ -11,6 +12,12 @@ export interface FrontendArtifactManifest { digest: string; releaseId: string; files: number; + dependencies?: FrontendArtifactDependency[]; +} + +export interface FrontendArtifactDependency { + frontendKey: string; + releaseId: string; } export interface StagedFrontendArtifact { @@ -19,6 +26,21 @@ export interface StagedFrontendArtifact { manifest: FrontendArtifactManifest; } +export interface FrontendArtifactCleanupResult { + removed: string[]; + retained: string[]; + skipped: string[]; +} + +export interface FrontendArtifactCleanupOptions { + frontendKeys: string[]; + protectedCommitShas?: Iterable; + retentionMs: number; + keepNewest: number; + now?: Date; + cleanupProfileWrapperStaging?: boolean; +} + export interface ProfileFrontendRuntimeConfig { version: 1; profile: string; @@ -34,11 +56,15 @@ export interface ProfileFrontendRuntimeConfig { export const SHARED_GAME_FRONTEND_KEY = 'game-assets'; export const GAME_FRONTEND_RUNTIME_CONFIG_ID = 'sammo-runtime-config'; +export const DEFAULT_FRONTEND_ARTIFACT_RETENTION_MS = 24 * 60 * 60 * 1_000; +export const DEFAULT_FRONTEND_ARTIFACT_KEEP_NEWEST = 2; const MANIFEST_FILE = '.sammo-artifact.json'; const FRONTEND_KEY = /^[a-z0-9][a-z0-9_-]{0,63}$/u; const COMMIT_SHA = /^[0-9a-f]{40,64}$/iu; const PUBLIC_ASSET_BASE = /^\/[0-9A-Za-z/_-]*$/u; +const RELEASE_ID = /^[0-9a-f]{40,64}-[0-9a-f]{16}$/iu; +const STAGING_DIRECTORY = /^\.staging-[0-9a-f-]+$/iu; export const resolveFrontendServeMode = (value: string | undefined): FrontendServeMode => { const normalized = value?.trim().toLowerCase(); @@ -55,6 +81,33 @@ const assertCommitSha = (value: string): void => { if (!COMMIT_SHA.test(value)) throw new Error('Frontend artifact commit SHA must be a full hexadecimal SHA.'); }; +const assertReleaseId = (value: string): void => { + if (!RELEASE_ID.test(value)) throw new Error(`Invalid frontend artifact release id: ${value}`); +}; + +const normalizeDependencies = ( + dependencies: FrontendArtifactDependency[] | undefined +): FrontendArtifactDependency[] => { + if (!dependencies) return []; + const normalized = dependencies.map((dependency) => { + assertFrontendKey(dependency.frontendKey); + assertReleaseId(dependency.releaseId); + return { + frontendKey: dependency.frontendKey, + releaseId: dependency.releaseId.toLowerCase(), + }; + }); + normalized.sort((left, right) => + `${left.frontendKey}/${left.releaseId}`.localeCompare(`${right.frontendKey}/${right.releaseId}`) + ); + return normalized.filter( + (dependency, index) => + index === 0 || + dependency.frontendKey !== normalized[index - 1].frontendKey || + dependency.releaseId !== normalized[index - 1].releaseId + ); +}; + const listSourceFiles = async (sourceRoot: string): Promise => { const rootStat = await fs.lstat(sourceRoot); if (!rootStat.isDirectory() || rootStat.isSymbolicLink()) { @@ -110,12 +163,14 @@ const readManifest = async (releasePath: string): Promise 0 ? { dependencies } : {}) }; }; const isMissing = (error: unknown): boolean => @@ -171,16 +226,77 @@ export class FrontendArtifactManager { } private releasePath(frontendKey: string, releaseId: string): string { - if (!/^[0-9a-f]{40,64}-[0-9a-f]{16}$/iu.test(releaseId)) { - throw new Error(`Invalid frontend artifact release id: ${releaseId}`); - } + assertReleaseId(releaseId); return path.join(this.frontendRoot(frontendKey), 'releases', releaseId); } + private async readPointerReleaseId(frontendKey: string, pointer: 'current' | 'previous'): Promise { + const frontendRoot = this.frontendRoot(frontendKey); + let target: string; + try { + target = await fs.readlink(path.join(frontendRoot, pointer)); + } catch (error) { + if (isMissing(error)) return null; + throw error; + } + const normalized = target.split(path.sep).join('/'); + const match = /^releases\/([0-9a-f]{40,64}-[0-9a-f]{16})$/iu.exec(normalized); + if (!match) throw new Error(`Invalid ${pointer} frontend artifact pointer for ${frontendKey}.`); + const manifest = await readManifest(this.releasePath(frontendKey, match[1])); + if (manifest.frontendKey !== frontendKey || manifest.releaseId !== match[1]) { + throw new Error(`Frontend artifact manifest does not match ${frontendKey}/${match[1]}.`); + } + return match[1]; + } + + private async readReleaseDependencies( + frontendKey: string, + releaseId: string + ): Promise { + const releasePath = this.releasePath(frontendKey, releaseId); + const manifest = await readManifest(releasePath); + if (manifest.dependencies?.length) return manifest.dependencies; + const indexHtml = await fs.readFile(path.join(releasePath, 'index.html'), 'utf8'); + if (!indexHtml.includes(GAME_FRONTEND_RUNTIME_CONFIG_ID)) return []; + const runtimeScript = + /]*\bid="sammo-runtime-config")(?=[^>]*\btype="application\/json")[^>]*>([\s\S]*?)<\/script>/iu.exec( + indexHtml + ); + if (!runtimeScript) { + throw new Error(`Invalid profile frontend runtime config script: ${releasePath}`); + } + const runtimeConfig = JSON.parse(runtimeScript[1]) as Partial; + if (typeof runtimeConfig.assetReleaseId !== 'string') { + throw new Error(`Profile frontend runtime config has no shared asset release: ${releasePath}`); + } + assertReleaseId(runtimeConfig.assetReleaseId); + return [{ frontendKey: SHARED_GAME_FRONTEND_KEY, releaseId: runtimeConfig.assetReleaseId.toLowerCase() }]; + } + + private async touchRelease(frontendKey: string, releaseId: string, at: Date): Promise { + const releasePath = this.releasePath(frontendKey, releaseId); + const dependencies = await this.readReleaseDependencies(frontendKey, releaseId); + for (const dependency of dependencies) { + const dependencyPath = this.releasePath(dependency.frontendKey, dependency.releaseId); + const dependencyManifest = await readManifest(dependencyPath); + if ( + dependencyManifest.frontendKey !== dependency.frontendKey || + dependencyManifest.releaseId !== dependency.releaseId + ) { + throw new Error( + `Frontend artifact dependency manifest does not match ${dependency.frontendKey}/${dependency.releaseId}.` + ); + } + await fs.utimes(dependencyPath, at, at); + } + await fs.utimes(releasePath, at, at); + } + async stage(options: { frontendKey: string; sourceRoot: string; commitSha: string; + dependencies?: FrontendArtifactDependency[]; }): Promise { assertFrontendKey(options.frontendKey); assertCommitSha(options.commitSha); @@ -190,6 +306,20 @@ export class FrontendArtifactManager { const digest = await buildDigest(sourceRoot, files); const releaseId = `${commitSha}-${digest.slice(0, 16)}`; const releasePath = this.releasePath(options.frontendKey, releaseId); + const dependencies = normalizeDependencies(options.dependencies); + for (const dependency of dependencies) { + const dependencyManifest = await readManifest( + this.releasePath(dependency.frontendKey, dependency.releaseId) + ); + if ( + dependencyManifest.frontendKey !== dependency.frontendKey || + dependencyManifest.releaseId !== dependency.releaseId + ) { + throw new Error( + `Frontend artifact dependency manifest does not match ${dependency.frontendKey}/${dependency.releaseId}.` + ); + } + } const manifest: FrontendArtifactManifest = { version: 1, frontendKey: options.frontendKey, @@ -197,6 +327,7 @@ export class FrontendArtifactManager { digest, releaseId, files: files.length, + ...(dependencies.length > 0 ? { dependencies } : {}), }; try { const existing = await readManifest(releasePath); @@ -283,6 +414,12 @@ export class FrontendArtifactManager { frontendKey: options.frontendKey, sourceRoot, commitSha: options.sharedArtifact.manifest.commitSha, + dependencies: [ + { + frontendKey: SHARED_GAME_FRONTEND_KEY, + releaseId: options.sharedArtifact.releaseId, + }, + ], }); } finally { await fs.rm(sourceRoot, { recursive: true, force: true }); @@ -290,18 +427,7 @@ export class FrontendArtifactManager { } async readCurrentReleaseId(frontendKey: string): Promise { - const frontendRoot = this.frontendRoot(frontendKey); - try { - const target = await fs.readlink(path.join(frontendRoot, 'current')); - const normalized = target.split(path.sep).join('/'); - const match = /^releases\/([0-9a-f]{40,64}-[0-9a-f]{16})$/iu.exec(normalized); - if (!match) throw new Error(`Invalid current frontend artifact pointer for ${frontendKey}.`); - await readManifest(this.releasePath(frontendKey, match[1])); - return match[1]; - } catch (error) { - if (isMissing(error)) return null; - throw error; - } + return this.readPointerReleaseId(frontendKey, 'current'); } async activate(frontendKey: string, releaseId: string): Promise { @@ -322,7 +448,10 @@ export class FrontendArtifactManager { await fs.rm(temporary, { force: true }); } }; + const activatedAt = new Date(); + await this.touchRelease(frontendKey, releaseId, activatedAt); if (previousReleaseId && previousReleaseId !== releaseId) { + await this.touchRelease(frontendKey, previousReleaseId, activatedAt); await replacePointer('previous', previousReleaseId); } await replacePointer('current', releaseId); @@ -344,4 +473,160 @@ export class FrontendArtifactManager { await fs.rm(path.join(this.frontendRoot(frontendKey), 'current'), { force: true }); return releaseId; } + + async cleanup(options: FrontendArtifactCleanupOptions): Promise { + if (!Number.isFinite(options.retentionMs) || options.retentionMs < 0) { + throw new Error('Frontend artifact retention must be a non-negative finite duration.'); + } + if (!Number.isInteger(options.keepNewest) || options.keepNewest < 0) { + throw new Error('Frontend artifact keepNewest must be a non-negative integer.'); + } + const frontendKeys = [...new Set(options.frontendKeys)]; + frontendKeys.forEach(assertFrontendKey); + const protectedCommitShas = new Set( + [...(options.protectedCommitShas ?? [])].map((commitSha) => { + assertCommitSha(commitSha); + return commitSha.toLowerCase(); + }) + ); + const result: FrontendArtifactCleanupResult = { removed: [], retained: [], skipped: [] }; + if (frontendKeys.length === 0) return result; + + const collectProtectedReleases = async (): Promise>> => { + const protectedReleases = new Map(frontendKeys.map((frontendKey) => [frontendKey, new Set()])); + for (const frontendKey of frontendKeys) { + for (const pointer of ['current', 'previous'] as const) { + const releaseId = await this.readPointerReleaseId(frontendKey, pointer); + if (!releaseId) continue; + protectedReleases.get(frontendKey)?.add(releaseId); + for (const dependency of await this.readReleaseDependencies(frontendKey, releaseId)) { + protectedReleases.get(dependency.frontendKey)?.add(dependency.releaseId); + } + } + } + return protectedReleases; + }; + + let protectedReleases: Map>; + try { + protectedReleases = await collectProtectedReleases(); + } catch { + for (const frontendKey of frontendKeys) result.skipped.push(this.frontendRoot(frontendKey)); + return result; + } + + const cutoffMs = (options.now ?? new Date()).getTime() - options.retentionMs; + const candidates: Array<{ frontendKey: string; releaseId: string; releasePath: string }> = []; + const staleStagingPaths: string[] = []; + for (const frontendKey of frontendKeys) { + const releasesRoot = path.join(this.frontendRoot(frontendKey), 'releases'); + let entries: Dirent[]; + try { + entries = await fs.readdir(releasesRoot, { withFileTypes: true }); + } catch (error) { + if (isMissing(error)) continue; + result.skipped.push(releasesRoot); + continue; + } + const unprotected: Array<{ releaseId: string; releasePath: string; mtimeMs: number }> = []; + for (const entry of entries) { + const entryPath = path.join(releasesRoot, entry.name); + if (STAGING_DIRECTORY.test(entry.name)) { + if (!entry.isDirectory()) { + result.skipped.push(entryPath); + continue; + } + const stat = await fs.lstat(entryPath); + if (stat.mtimeMs <= cutoffMs) staleStagingPaths.push(entryPath); + else result.retained.push(entryPath); + continue; + } + if (!RELEASE_ID.test(entry.name) || !entry.isDirectory()) { + result.skipped.push(entryPath); + continue; + } + try { + const manifest = await readManifest(entryPath); + if (manifest.frontendKey !== frontendKey || manifest.releaseId !== entry.name) { + result.skipped.push(entryPath); + continue; + } + const stat = await fs.lstat(entryPath); + if ( + protectedReleases.get(frontendKey)?.has(entry.name) || + protectedCommitShas.has(manifest.commitSha.toLowerCase()) + ) { + result.retained.push(entryPath); + continue; + } + unprotected.push({ releaseId: entry.name, releasePath: entryPath, mtimeMs: stat.mtimeMs }); + } catch { + result.skipped.push(entryPath); + } + } + unprotected.sort( + (left, right) => right.mtimeMs - left.mtimeMs || right.releaseId.localeCompare(left.releaseId) + ); + unprotected.forEach((release, index) => { + if (index < options.keepNewest || release.mtimeMs > cutoffMs) { + result.retained.push(release.releasePath); + } else { + candidates.push({ frontendKey, releaseId: release.releaseId, releasePath: release.releasePath }); + } + }); + } + + if (options.cleanupProfileWrapperStaging) { + let entries: Dirent[] = []; + try { + entries = await fs.readdir(this.root, { withFileTypes: true }); + } catch (error) { + if (!isMissing(error)) result.skipped.push(this.root); + } + for (const entry of entries) { + if (!entry.name.startsWith('.profile-wrapper-')) continue; + const entryPath = path.join(this.root, entry.name); + if (!entry.isDirectory()) { + result.skipped.push(entryPath); + continue; + } + const stat = await fs.lstat(entryPath); + if (stat.mtimeMs <= cutoffMs) staleStagingPaths.push(entryPath); + else result.retained.push(entryPath); + } + } + + for (const candidate of candidates) { + try { + protectedReleases = await collectProtectedReleases(); + if (protectedReleases.get(candidate.frontendKey)?.has(candidate.releaseId)) { + result.retained.push(candidate.releasePath); + continue; + } + const stat = await fs.lstat(candidate.releasePath); + if (!stat.isDirectory() || stat.isSymbolicLink()) { + result.skipped.push(candidate.releasePath); + continue; + } + await fs.rm(candidate.releasePath, { recursive: true }); + result.removed.push(candidate.releasePath); + } catch (error) { + if (!isMissing(error)) result.skipped.push(candidate.releasePath); + } + } + for (const stagingPath of staleStagingPaths) { + try { + const stat = await fs.lstat(stagingPath); + if (!stat.isDirectory() || stat.isSymbolicLink()) { + result.skipped.push(stagingPath); + continue; + } + await fs.rm(stagingPath, { recursive: true }); + result.removed.push(stagingPath); + } catch (error) { + if (!isMissing(error)) result.skipped.push(stagingPath); + } + } + return result; + } } diff --git a/app/gateway-api/src/orchestrator/gatewayOrchestrator.ts b/app/gateway-api/src/orchestrator/gatewayOrchestrator.ts index 3e78b2ad..c29fd7bc 100644 --- a/app/gateway-api/src/orchestrator/gatewayOrchestrator.ts +++ b/app/gateway-api/src/orchestrator/gatewayOrchestrator.ts @@ -55,9 +55,12 @@ import { import type { AdminSeedUser } from './seedProfileDatabase.js'; import { assertReleaseComponents, readReleaseManifest } from './releaseManifest.js'; import { + DEFAULT_FRONTEND_ARTIFACT_KEEP_NEWEST, + DEFAULT_FRONTEND_ARTIFACT_RETENTION_MS, FrontendArtifactManager, resolveFrontendServeMode, SHARED_GAME_FRONTEND_KEY, + type FrontendArtifactCleanupResult, type FrontendServeMode, type StagedFrontendArtifact, } from './frontendArtifactManager.js'; @@ -138,6 +141,11 @@ export interface GatewayOrchestratorHandle { listRuntimeSettings?(profileNames: string[]): Promise; } +export interface GatewayManagedCleanupResult { + workspaces: { removed: string[]; skipped: string[] }; + artifacts: FrontendArtifactCleanupResult; +} + const SENSITIVE_ENV_NAME = /(SECRET|TOKEN|PASSWORD|PASSWD|PRIVATE_KEY|CLIENT_SECRET|DATABASE_URL|REDIS_URL)/iu; const managedPostgresPoolMax = (env: Record, roleVariable: string, fallback: number): string => @@ -2318,15 +2326,21 @@ export class GatewayOrchestrator implements GatewayOrchestratorHandle { } } - async cleanupStaleWorkspaces(): Promise<{ removed: string[]; skipped: string[] }> { + async cleanupStaleResources(): Promise { if (this.buildInFlight || this.operationInFlight || this.workspaceCleanupInFlight) { const managedWorkspaces = await this.workspaceManager.listManagedWorkspaces(); - return { removed: [], skipped: managedWorkspaces.map((workspace) => workspace.root) }; + return { + workspaces: { removed: [], skipped: managedWorkspaces.map((workspace) => workspace.root) }, + artifacts: { removed: [], retained: [], skipped: [] }, + }; } this.workspaceCleanupInFlight = true; try { const managedWorkspaces = await this.workspaceManager.listManagedWorkspaces(); - const profiles = await this.repository.listProfiles(); + const [profiles, operations] = await Promise.all([ + this.repository.listProfiles(), + this.repository.listOperations({ limit: 100 }), + ]); const protectedWorkspaces = new Set(); for (const profile of profiles) { if (profile.buildWorkspace) { @@ -2353,22 +2367,56 @@ export class GatewayOrchestrator implements GatewayOrchestratorHandle { } } - return await this.workspaceManager.cleanup({ + const workspaces = await this.workspaceManager.cleanup({ protectedPaths: [...protectedWorkspaces], retentionMs: DEFAULT_MANAGED_WORKSPACE_RETENTION_MS, keepNewest: DEFAULT_MANAGED_WORKSPACE_KEEP_NEWEST, }); + const artifacts: FrontendArtifactCleanupResult = + this.frontendServeMode === 'static' + ? await this.artifactManager.cleanup({ + frontendKeys: [ + ...new Set(profiles.map((profile) => profile.profile)), + SHARED_GAME_FRONTEND_KEY, + ], + protectedCommitShas: [ + ...profiles + .filter( + (profile) => + profile.buildCommitSha && + (profile.buildStatus === 'QUEUED' || profile.buildStatus === 'RUNNING') + ) + .map((profile) => profile.buildCommitSha as string), + ...operations + .filter( + (operation) => + operation.resolvedCommitSha && + (operation.status === 'QUEUED' || operation.status === 'RUNNING') + ) + .map((operation) => operation.resolvedCommitSha as string), + ], + retentionMs: DEFAULT_FRONTEND_ARTIFACT_RETENTION_MS, + keepNewest: DEFAULT_FRONTEND_ARTIFACT_KEEP_NEWEST, + now: this.now(), + cleanupProfileWrapperStaging: true, + }) + : { removed: [], retained: [], skipped: [] }; + return { workspaces, artifacts }; } finally { this.workspaceCleanupInFlight = false; } } + async cleanupStaleWorkspaces(): Promise<{ removed: string[]; skipped: string[] }> { + return (await this.cleanupStaleResources()).workspaces; + } + private async cleanupWorkspacesScheduled(): Promise { if (this.stopping || this.buildInFlight || this.operationInFlight || this.workspaceCleanupInFlight) return; - const result = await this.cleanupStaleWorkspaces(); - if (result.removed.length > 0) { - console.info(`[gateway-orchestrator] removed ${result.removed.length} stale profile worktrees`); - } + const result = await this.cleanupStaleResources(); + console.info( + `[gateway-orchestrator] managed cleanup completed: removed ${result.workspaces.removed.length} profile worktrees and ${result.artifacts.removed.length} frontend artifacts; retained ${result.artifacts.retained.length}, skipped ${result.artifacts.skipped.length}` + ); } private async stageStaticProfileFrontend(profile: GatewayProfileRecord): Promise { diff --git a/app/gateway-api/test/frontendArtifactManager.test.ts b/app/gateway-api/test/frontendArtifactManager.test.ts index 2c221e4b..c5ab51bb 100644 --- a/app/gateway-api/test/frontendArtifactManager.test.ts +++ b/app/gateway-api/test/frontendArtifactManager.test.ts @@ -14,6 +14,7 @@ import { const roots: string[] = []; const sha = 'a'.repeat(40); +const cleanupNow = new Date('2026-08-23T00:00:00.000Z'); afterEach(async () => { await Promise.all(roots.splice(0).map((root) => fs.rm(root, { recursive: true, force: true }))); @@ -145,4 +146,162 @@ describe('FrontendArtifactManager', () => { expect(rendered).not.toContain('trpc?'); expect(rendered).toContain('\\u003c/script\\u003e'); }); + + it('removes only expired unreferenced releases while preserving pointers, active commits, grace, and caches', async () => { + const { source, artifacts } = await fixture(); + const manager = new FrontendArtifactManager(artifacts); + const stage = async (marker: string, commitMarker: string) => { + await fs.writeFile(path.join(source, 'index.html'), `
${marker}
`); + return manager.stage({ + frontendKey: 'gateway', + sourceRoot: source, + commitSha: commitMarker.repeat(40), + }); + }; + const current = await stage('current', '1'); + await manager.activate('gateway', current.releaseId); + const next = await stage('next', '2'); + await manager.activate('gateway', next.releaseId); + const pinned = await stage('pinned', '3'); + const recent = await stage('recent', '4'); + const cached = await stage('cached', '5'); + const stale = await stage('stale', '6'); + const releasesRoot = path.join(artifacts, 'gateway', 'releases'); + const staging = path.join(releasesRoot, '.staging-00000000-0000-0000-0000-000000000000'); + const unknownSymlink = path.join(releasesRoot, `${'7'.repeat(40)}-${'7'.repeat(16)}`); + await fs.mkdir(staging); + await fs.symlink(stale.releasePath, unknownSymlink); + const old = new Date(cleanupNow.getTime() - 72 * 60 * 60 * 1_000); + for (const artifact of [current, next, pinned, cached, stale]) { + await fs.utimes(artifact.releasePath, old, old); + } + await fs.utimes(cached.releasePath, new Date(old.getTime() + 1_000), new Date(old.getTime() + 1_000)); + await fs.utimes(staging, old, old); + const recentAt = new Date(cleanupNow.getTime() - 60 * 60 * 1_000); + await fs.utimes(recent.releasePath, recentAt, recentAt); + + const result = await manager.cleanup({ + frontendKeys: ['gateway'], + protectedCommitShas: [pinned.manifest.commitSha], + retentionMs: 24 * 60 * 60 * 1_000, + keepNewest: 2, + now: cleanupNow, + }); + + expect(result.removed.sort()).toEqual([staging, stale.releasePath].sort()); + expect(result.retained).toEqual( + expect.arrayContaining([ + current.releasePath, + next.releasePath, + pinned.releasePath, + recent.releasePath, + cached.releasePath, + ]) + ); + expect(result.skipped).toContain(unknownSymlink); + await expect(fs.access(stale.releasePath)).rejects.toMatchObject({ code: 'ENOENT' }); + await expect(fs.readFile(path.join(artifacts, 'gateway', 'current', 'index.html'), 'utf8')).resolves.toContain( + 'next' + ); + await expect(fs.readFile(path.join(artifacts, 'gateway', 'previous', 'index.html'), 'utf8')).resolves.toContain( + 'current' + ); + }); + + it('preserves shared assets referenced by current and previous profile wrappers, including old manifests', async () => { + const { source, artifacts } = await fixture(); + await fs.writeFile( + path.join(source, 'index.html'), + '' + ); + await fs.writeFile(path.join(source, 'deployment-version.json'), `${JSON.stringify({ commitSha: sha })}\n`); + const manager = new FrontendArtifactManager(artifacts); + const publish = async (commitMarker: string, script: string) => { + const commitSha = commitMarker.repeat(40); + await fs.writeFile(path.join(source, 'assets', 'app-deadbeef.js'), script); + await fs.writeFile(path.join(source, 'deployment-version.json'), `${JSON.stringify({ commitSha })}\n`); + const shared = await manager.stage({ + frontendKey: SHARED_GAME_FRONTEND_KEY, + sourceRoot: source, + commitSha, + }); + const wrapper = await manager.stageProfileWrapper({ + frontendKey: 'pya', + sharedArtifact: shared, + sharedAssetPublicBase: '/gateway/profile-assets', + runtimeConfig: { + version: 1, + profile: 'pya', + profileName: 'pya:default', + appBasePath: '/pya/', + gameApiUrl: '/pya/api/trpc', + gameSseUrl: '/pya/api/events', + gatewayApiUrl: '/gateway/api/trpc', + gatewayWebUrl: '/gateway/', + }, + }); + await manager.activate('pya', wrapper.releaseId); + return { shared, wrapper }; + }; + const first = await publish('1', 'console.log(1)'); + const second = await publish('2', 'console.log(2)'); + await fs.writeFile(path.join(source, 'assets', 'app-deadbeef.js'), 'console.log(3)'); + const unused = await manager.stage({ + frontendKey: SHARED_GAME_FRONTEND_KEY, + sourceRoot: source, + commitSha: '3'.repeat(40), + }); + const firstManifestPath = path.join(first.wrapper.releasePath, '.sammo-artifact.json'); + const firstManifest = JSON.parse(await fs.readFile(firstManifestPath, 'utf8')) as Record; + delete firstManifest.dependencies; + await fs.writeFile(firstManifestPath, `${JSON.stringify(firstManifest, null, 2)}\n`); + const old = new Date(cleanupNow.getTime() - 72 * 60 * 60 * 1_000); + for (const artifact of [first.shared, first.wrapper, second.shared, second.wrapper, unused]) { + await fs.utimes(artifact.releasePath, old, old); + } + + const result = await manager.cleanup({ + frontendKeys: ['pya', SHARED_GAME_FRONTEND_KEY], + retentionMs: 24 * 60 * 60 * 1_000, + keepNewest: 0, + now: cleanupNow, + }); + + expect(result.removed).toEqual([unused.releasePath]); + expect(result.retained).toEqual( + expect.arrayContaining([ + first.shared.releasePath, + first.wrapper.releasePath, + second.shared.releasePath, + second.wrapper.releasePath, + ]) + ); + await expect( + fs.readFile(path.join(first.shared.releasePath, 'assets', 'app-deadbeef.js'), 'utf8') + ).resolves.toBe('console.log(1)'); + await expect( + fs.readFile(path.join(second.shared.releasePath, 'assets', 'app-deadbeef.js'), 'utf8') + ).resolves.toBe('console.log(2)'); + }); + + it('fails closed when a live pointer cannot be validated', async () => { + const { source, artifacts } = await fixture(); + const manager = new FrontendArtifactManager(artifacts); + const stale = await manager.stage({ frontendKey: 'gateway', sourceRoot: source, commitSha: sha }); + const old = new Date(cleanupNow.getTime() - 72 * 60 * 60 * 1_000); + await fs.utimes(stale.releasePath, old, old); + await fs.mkdir(path.join(artifacts, 'gateway'), { recursive: true }); + await fs.symlink(`releases/${'f'.repeat(40)}-${'f'.repeat(16)}`, path.join(artifacts, 'gateway', 'current')); + + const result = await manager.cleanup({ + frontendKeys: ['gateway'], + retentionMs: 24 * 60 * 60 * 1_000, + keepNewest: 0, + now: cleanupNow, + }); + + expect(result.removed).toEqual([]); + expect(result.skipped).toEqual([path.join(artifacts, 'gateway')]); + await expect(fs.access(stale.releasePath)).resolves.toBeUndefined(); + }); }); diff --git a/app/gateway-api/test/orchestratorWorkspaceCleanup.test.ts b/app/gateway-api/test/orchestratorWorkspaceCleanup.test.ts index 8749f77c..6cd043be 100644 --- a/app/gateway-api/test/orchestratorWorkspaceCleanup.test.ts +++ b/app/gateway-api/test/orchestratorWorkspaceCleanup.test.ts @@ -1,8 +1,11 @@ +import fs from 'node:fs/promises'; +import os from 'node:os'; import path from 'node:path'; -import { describe, expect, it } from 'vitest'; +import { afterEach, describe, expect, it } from 'vitest'; import { GatewayOrchestrator } from '../src/orchestrator/gatewayOrchestrator.js'; +import { FrontendArtifactManager } from '../src/orchestrator/frontendArtifactManager.js'; import type { ProcessManager } from '../src/orchestrator/processManager.js'; import type { GatewayProfileRecord, GatewayProfileRepository } from '../src/orchestrator/profileRepository.js'; import { @@ -14,6 +17,11 @@ import { const COMMIT_SHA = '0123456789abcdef0123456789abcdef01234567'; const oldUsage = '2025-01-01T00:00:00.000Z'; +const temporaryDirectories: string[] = []; + +afterEach(async () => { + await Promise.all(temporaryDirectories.splice(0).map((directory) => fs.rm(directory, { recursive: true }))); +}); const makeProfile = ( profileName: string, @@ -40,10 +48,14 @@ const makeProfile = ( const createHarness = ( profiles: GatewayProfileRecord[], processes: Awaited>, - managedPaths: string[] + managedPaths: string[], + frontendArtifactRoot?: string ) => { const cleanupCalls: ManagedWorkspaceCleanupOptions[] = []; - const repository = { listProfiles: async () => profiles } as unknown as GatewayProfileRepository; + const repository = { + listProfiles: async () => profiles, + listOperations: async () => [], + } as unknown as GatewayProfileRepository; const processManager: ProcessManager = { list: async () => processes, start: async () => {}, @@ -73,11 +85,13 @@ const createHarness = ( redisKeyPrefix: 'sammo:test', gameTokenSecret: 'test-secret', gatewayInternalApiUrl: 'http://127.0.0.1:13000', + ...(frontendArtifactRoot ? { frontendServeMode: 'static' as const, frontendArtifactRoot } : {}), }, reconcileIntervalMs: 60_000, scheduleIntervalMs: 60_000, buildIntervalMs: 60_000, adminActionIntervalMs: 60_000, + now: () => new Date('2026-08-23T00:00:00.000Z'), }); return { orchestrator, cleanupCalls }; }; @@ -141,4 +155,47 @@ describe('GatewayOrchestrator workspace cleanup', () => { skipped: [], }); }); + + it('serializes profile worktree and frontend artifact cleanup under the same managed cycle', async () => { + const root = await fs.mkdtemp(path.join(os.tmpdir(), 'sammo-profile-artifact-cleanup-')); + temporaryDirectories.push(root); + const sourceRoot = path.join(root, 'dist'); + const artifactRoot = path.join(root, 'artifacts'); + await fs.mkdir(sourceRoot, { recursive: true }); + const manager = new FrontendArtifactManager(artifactRoot); + const stage = async (marker: string, commitMarker: string) => { + await fs.writeFile(path.join(sourceRoot, 'index.html'), `
${marker}
`); + return manager.stage({ + frontendKey: 'che', + sourceRoot, + commitSha: commitMarker.repeat(40), + }); + }; + const active = await stage('active', '1'); + await manager.activate('che', active.releaseId); + const cacheOne = await stage('cache-one', '2'); + const cacheTwo = await stage('cache-two', '3'); + const stale = await stage('stale', '4'); + const now = new Date('2026-08-23T00:00:00.000Z'); + const old = new Date(now.getTime() - 72 * 60 * 60 * 1_000); + for (const artifact of [active, cacheOne, cacheTwo, stale]) { + await fs.utimes(artifact.releasePath, old, old); + } + await fs.utimes(cacheTwo.releasePath, new Date(old.getTime() + 2_000), new Date(old.getTime() + 2_000)); + await fs.utimes(cacheOne.releasePath, new Date(old.getTime() + 1_000), new Date(old.getTime() + 1_000)); + const harness = createHarness( + [makeProfile('che:default', undefined, { buildCommitSha: active.manifest.commitSha })], + [], + [], + artifactRoot + ); + + const result = await harness.orchestrator.cleanupStaleResources(); + + expect(result.workspaces).toEqual({ removed: [], skipped: [] }); + expect(result.artifacts.removed).toEqual([stale.releasePath]); + expect(result.artifacts.retained).toEqual( + expect.arrayContaining([active.releasePath, cacheOne.releasePath, cacheTwo.releasePath]) + ); + }); }); diff --git a/app/release-controller/src/index.ts b/app/release-controller/src/index.ts index 3d132b27..235f5af2 100644 --- a/app/release-controller/src/index.ts +++ b/app/release-controller/src/index.ts @@ -89,10 +89,10 @@ const main = async (): Promise => { if (now >= nextWorkspaceCleanupAt) { nextWorkspaceCleanupAt = now + RELEASE_WORKSPACE_CLEANUP_INTERVAL_MS; try { - const result = await controller.cleanupStaleWorkspaces(); - if (result.removed.length > 0) { - console.info(`[release-controller] removed ${result.removed.length} stale Gateway worktrees`); - } + const result = await controller.cleanupStaleResources(); + console.info( + `[release-controller] managed cleanup completed: removed ${result.workspaces.removed.length} Gateway worktrees and ${result.artifacts.removed.length} frontend artifacts; retained ${result.artifacts.retained.length}, skipped ${result.artifacts.skipped.length}` + ); } catch (error) { console.error('[release-controller] workspace cleanup failed', error); } diff --git a/app/release-controller/src/releaseController.ts b/app/release-controller/src/releaseController.ts index 8f457609..3e3f1503 100644 --- a/app/release-controller/src/releaseController.ts +++ b/app/release-controller/src/releaseController.ts @@ -6,6 +6,8 @@ import { assertReleaseComponents, buildTurboReleaseCommand, buildTurboReleaseTaskCommand, + DEFAULT_FRONTEND_ARTIFACT_KEEP_NEWEST, + DEFAULT_FRONTEND_ARTIFACT_RETENTION_MS, DEFAULT_MANAGED_WORKSPACE_KEEP_NEWEST, DEFAULT_MANAGED_WORKSPACE_RETENTION_MS, type BuildCommand, @@ -14,6 +16,7 @@ import { type GatewayReleaseOperationRecord, type GatewayReleaseRepository, type GatewayReleaseStateRecord, + type FrontendArtifactCleanupResult, type GitWorkspaceManager, type ProcessDefinition, type ProcessManager, @@ -34,6 +37,11 @@ const MANAGED_PROCESS_NAMES = ['sammo:gateway-api', 'sammo:gateway-frontend', 's const SENSITIVE_ENV_NAME = /(SECRET|TOKEN|PASSWORD|PASSWD|PRIVATE_KEY|CLIENT_SECRET|DATABASE_URL|REDIS_URL)/iu; export const RELEASE_WORKSPACE_CLEANUP_INTERVAL_MS = 24 * 60 * 60 * 1_000; +export interface ReleaseManagedCleanupResult { + workspaces: { removed: string[]; skipped: string[] }; + artifacts: FrontendArtifactCleanupResult; +} + const isRuntimeProcessActive = (status: string): boolean => ['online', 'launching', 'stopping'].includes(status.toLowerCase()); @@ -183,6 +191,36 @@ export class GatewayReleaseController { }); } + async cleanupStaleResources(): Promise { + const workspaces = await this.cleanupStaleWorkspaces(); + let artifacts: FrontendArtifactCleanupResult = { removed: [], retained: [], skipped: [] }; + if (this.config.frontendServeMode === 'static') { + const [state, operations] = await Promise.all([ + this.repository.getState(), + this.repository.listOperations(100), + ]); + artifacts = await this.artifactManager.cleanup({ + frontendKeys: ['gateway'], + protectedCommitShas: [ + ...[state.activeCommitSha, state.previousCommitSha].filter((commitSha): commitSha is string => + Boolean(commitSha) + ), + ...operations + .filter( + (operation) => + operation.resolvedCommitSha && + (operation.status === 'QUEUED' || operation.status === 'RUNNING') + ) + .map((operation) => operation.resolvedCommitSha as string), + ], + retentionMs: DEFAULT_FRONTEND_ARTIFACT_RETENTION_MS, + keepNewest: DEFAULT_FRONTEND_ARTIFACT_KEEP_NEWEST, + now: this.now(), + }); + } + return { workspaces, artifacts }; + } + private sanitizeLogMessage(message: string): string { let sanitized = stripVTControlCharacters(message); const sensitiveValues = new Set([ diff --git a/app/release-controller/test/releaseController.test.ts b/app/release-controller/test/releaseController.test.ts index 396f68da..9dbb9804 100644 --- a/app/release-controller/test/releaseController.test.ts +++ b/app/release-controller/test/releaseController.test.ts @@ -3,8 +3,11 @@ import os from 'node:os'; import path from 'node:path'; import { + DEFAULT_FRONTEND_ARTIFACT_KEEP_NEWEST, + DEFAULT_FRONTEND_ARTIFACT_RETENTION_MS, DEFAULT_MANAGED_WORKSPACE_KEEP_NEWEST, DEFAULT_MANAGED_WORKSPACE_RETENTION_MS, + FrontendArtifactManager, type BuildRunner, type GatewayReleaseOperationRecord, type GatewayReleaseRepository, @@ -224,8 +227,7 @@ describe('GatewayReleaseController', () => { } as unknown as GitWorkspaceManager, { run: async () => ({ ok: true, exitCode: 0, output: '' }) }, { - list: async () => - [...running].map(([name, cwd]) => ({ name, cwd, status: 'online', restartCount: 0 })), + list: async () => [...running].map(([name, cwd]) => ({ name, cwd, status: 'online', restartCount: 0 })), start: async (definition) => { running.set(definition.name, definition.cwd); }, @@ -321,6 +323,61 @@ describe('GatewayReleaseController', () => { }); }); + it('cleans expired unreferenced Gateway frontend releases with the managed daily policy', async () => { + const workspace = await createReleaseWorkspace(); + const artifactRoot = path.join(workspace, 'artifact-cleanup-volume'); + const sourceRoot = path.join(workspace, 'gateway-cleanup-dist'); + await fs.mkdir(sourceRoot, { recursive: true }); + const manager = new FrontendArtifactManager(artifactRoot); + const stage = async (marker: string, commitSha: string) => { + await fs.writeFile(path.join(sourceRoot, 'index.html'), `
${marker}
`); + return manager.stage({ frontendKey: 'gateway', sourceRoot, commitSha }); + }; + const active = await stage('active', OLD_SHA); + await manager.activate('gateway', active.releaseId); + const cacheOne = await stage('cache-one', '3'.repeat(40)); + const cacheTwo = await stage('cache-two', '4'.repeat(40)); + const stale = await stage('stale', '5'.repeat(40)); + const now = new Date('2026-08-23T00:00:00.000Z'); + const old = new Date(now.getTime() - 72 * 60 * 60 * 1_000); + for (const artifact of [active, cacheOne, cacheTwo, stale]) { + await fs.utimes(artifact.releasePath, old, old); + } + await fs.utimes(cacheTwo.releasePath, new Date(old.getTime() + 2_000), new Date(old.getTime() + 2_000)); + await fs.utimes(cacheOne.releasePath, new Date(old.getTime() + 1_000), new Date(old.getTime() + 1_000)); + const harness = createRepository(); + const controller = new GatewayReleaseController( + harness.repository, + { + listManagedWorkspaces: async () => [], + cleanup: async () => ({ removed: [], skipped: [] }), + } as unknown as GitWorkspaceManager, + { run: async () => ({ ok: true, exitCode: 0, output: '' }) }, + { + list: async () => [], + start: async () => {}, + stop: async () => {}, + delete: async () => {}, + }, + { + ...config, + frontendServeMode: 'static', + frontendArtifactRoot: artifactRoot, + }, + () => now + ); + + const result = await controller.cleanupStaleResources(); + + expect(result.workspaces).toEqual({ removed: [], skipped: [] }); + expect(result.artifacts.removed).toEqual([stale.releasePath]); + expect(result.artifacts.retained).toEqual( + expect.arrayContaining([active.releasePath, cacheOne.releasePath, cacheTwo.releasePath]) + ); + expect(DEFAULT_FRONTEND_ARTIFACT_RETENTION_MS).toBe(24 * 60 * 60 * 1_000); + expect(DEFAULT_FRONTEND_ARTIFACT_KEEP_NEWEST).toBe(2); + }); + it('builds, migrates, switches all gateway roles, verifies readiness, and publishes atomically', async () => { const workspace = await createReleaseWorkspace(); const harness = createRepository(); diff --git a/docs/architecture/runtime.md b/docs/architecture/runtime.md index 9cc22d7e..163f429b 100644 --- a/docs/architecture/runtime.md +++ b/docs/architecture/runtime.md @@ -25,6 +25,13 @@ cluster `exec_mode`가 없으므로 모두 단일 fork입니다. Frontend는 Cad API는 하나의 Fastify process입니다. worker 역할 분리는 API event loop의 작업을 줄이지만 API replica나 장애 대체 backend를 제공하지는 않습니다. +정적 artifact 수명도 두 control-plane daemon이 소유합니다. Gateway orchestrator는 +profile wrapper와 commit 공용 `game-assets`, release-controller는 Gateway release를 +startup 직후와 24시간마다 operation/worktree 정리와 직렬화해 점검합니다. 현재·이전 +pointer, wrapper dependency, 진행 중 commit, 24시간 grace와 key별 최신 cache 2개를 +제외한 미참조 release만 제거합니다. 손상되거나 알 수 없는 pointer·manifest·symlink는 +삭제하지 않으며, sourcemap은 참조하는 bundle과 같은 release 단위로 유지됩니다. + Profile은 PostgreSQL schema와 Redis namespace를 분리하지만 같은 database, PostgreSQL instance, runtime cgroup을 공유합니다. 관리되는 PM2 정의는 game API 4, turn daemon 2, auction/battle/tournament worker 각 1, Gateway API 4, Gateway diff --git a/docs/release-operations.md b/docs/release-operations.md index ac76163f..1af71f89 100644 --- a/docs/release-operations.md +++ b/docs/release-operations.md @@ -105,6 +105,36 @@ Timeout은 scheduled task 오류로 끝나 정리 flag를 해제하고 다음 5 queue를 다시 claim하게 합니다. PM2 mutation이 timeout된 경우에는 같은 mutation을 즉시 직접 반복하지 않고 실제 process 목록과 operation terminal 상태를 먼저 재조회합니다. +## Frontend artifact 자동 정리 + +정적 운영의 `frontend-artifacts//releases`도 commit worktree와 같은 daemon 주기에 +정리합니다. Profile orchestrator는 profile wrapper와 `game-assets` 공용 bundle을, +release-controller는 Gateway bundle을 소유하며 startup 직후와 이후 24시간마다 한 번 +점검합니다. 별도 상시 polling process나 cron은 두지 않습니다. + +- `current`와 `previous` pointer가 가리키는 release는 기간과 무관하게 보호합니다. +- Profile wrapper manifest는 참조하는 `game-assets` release를 dependency로 기록합니다. + 전환 전에 생성된 manifest는 wrapper의 `sammo-runtime-config.assetReleaseId`를 읽어 같은 + 보호 집합을 복원하므로 기존 공용 bundle도 안전합니다. +- `QUEUED`/`RUNNING` operation과 profile build가 고정한 commit의 release도 보호합니다. +- 그 밖의 release는 마지막 stage/activation 이후 최소 24시간을 유예하고, key별 최신 + 2개를 재배포 cache로 추가 보존합니다. 두 조건 밖의 오래된 미참조 release만 + 재귀 삭제합니다. +- activation은 release와 dependency 디렉터리의 마지막 사용 시각을 갱신합니다. 오래된 + commit으로 rollback한 직후 cache가 즉시 만료되는 일을 막습니다. +- 24시간이 지난 crash 잔여 `.staging-*`와 profile wrapper 임시 디렉터리도 같은 소유 + daemon이 operation과 직렬화된 상태에서만 제거합니다. +- pointer, manifest, dependency 또는 symlink가 예상 형식이 아니면 해당 정리 주기는 + fail-closed로 건너뜁니다. 알 수 없는 파일과 디렉터리를 일반 release로 간주해 + 삭제하지 않습니다. + +이 정리는 Caddy가 읽는 재생성 가능한 frontend 산출물만 대상으로 하며 sourcemap도 +bundle과 함께 동일한 release 단위로 보존·정리합니다. PostgreSQL, Redis, `/image/*`, +사용자 업로드, 현재/이전 rollback pointer에는 접근하지 않습니다. Profile 관리자 API의 +`admin.profiles.cleanupWorkspaces`를 실행하면 같은 직렬화 경계에서 profile artifact도 +점검하지만 응답의 기존 worktree 목록 계약은 유지합니다. 실제 삭제 수·보존 수·skip +수는 orchestrator와 release-controller의 일일 managed cleanup 로그에서 확인합니다. + ## Profile 배포 버전 업데이트 화면에서 profile의 branch 또는 commit을 선택합니다. Branch는 worker가