diff --git a/app/gateway-api/src/orchestrator/gatewayOrchestrator.ts b/app/gateway-api/src/orchestrator/gatewayOrchestrator.ts index 8118680f..4cd21d8a 100644 --- a/app/gateway-api/src/orchestrator/gatewayOrchestrator.ts +++ b/app/gateway-api/src/orchestrator/gatewayOrchestrator.ts @@ -44,7 +44,11 @@ import { writeProfileReleaseSource, type ProfileReleaseSource, } from './profileReleaseSource.js'; -import type { GitWorkspaceManager } from './workspaceManager.js'; +import { + DEFAULT_MANAGED_WORKSPACE_KEEP_NEWEST, + DEFAULT_MANAGED_WORKSPACE_RETENTION_MS, + type GitWorkspaceManager, +} from './workspaceManager.js'; import type { AdminSeedUser } from './seedProfileDatabase.js'; import { assertReleaseComponents, readReleaseManifest } from './releaseManifest.js'; @@ -73,6 +77,8 @@ export interface GatewayOrchestratorOptions { cancelGame?: typeof defaultCancelGame; } +const WORKSPACE_CLEANUP_INTERVAL_MS = 24 * 60 * 60 * 1_000; + export interface ProfileRuntimeState { frontendRunning: boolean; apiRunning: boolean; @@ -629,11 +635,13 @@ export class GatewayOrchestrator implements GatewayOrchestratorHandle { private scheduleTimer?: NodeJS.Timeout; private buildTimer?: NodeJS.Timeout; private adminActionTimer?: NodeJS.Timeout; + private workspaceCleanupTimer?: NodeJS.Timeout; private reconcileInFlight = false; private scheduleInFlight = false; private buildInFlight = false; private adminActionInFlight = false; private operationInFlight = false; + private workspaceCleanupInFlight = false; private activeOperationAbortSignal?: AbortSignal; private readonly resetInFlight = new Set(); private readonly operationLeaseOwner = randomUUID(); @@ -714,7 +722,7 @@ export class GatewayOrchestrator implements GatewayOrchestratorHandle { start(): void { this.stopping = false; this.trackTask(this.reconcileNow()); - this.trackTask(this.runOperationsNow()); + this.trackTask(this.runOperationsNow().then(() => this.cleanupWorkspacesScheduled())); this.trackTask(this.runAdminActionsNow()); this.reconcileTimer = setInterval(() => this.trackTask(this.reconcileNow()), this.reconcileIntervalMs); this.scheduleTimer = setInterval(() => this.trackTask(this.runScheduleNow()), this.scheduleIntervalMs); @@ -723,6 +731,10 @@ export class GatewayOrchestrator implements GatewayOrchestratorHandle { this.trackTask(this.runOperationsNow()); this.trackTask(this.runAdminActionsNow()); }, this.adminActionIntervalMs); + this.workspaceCleanupTimer = setInterval( + () => this.trackTask(this.cleanupWorkspacesScheduled()), + WORKSPACE_CLEANUP_INTERVAL_MS + ); } async stop(): Promise { @@ -747,6 +759,9 @@ export class GatewayOrchestrator implements GatewayOrchestratorHandle { if (this.adminActionTimer) { clearInterval(this.adminActionTimer); } + if (this.workspaceCleanupTimer) { + clearInterval(this.workspaceCleanupTimer); + } await Promise.allSettled([...this.inFlightTasks]); } @@ -903,7 +918,7 @@ export class GatewayOrchestrator implements GatewayOrchestratorHandle { } async runBuildQueueNow(): Promise { - if (this.stopping || this.buildInFlight) { + if (this.stopping || this.buildInFlight || this.workspaceCleanupInFlight) { return; } this.buildInFlight = true; @@ -965,7 +980,7 @@ export class GatewayOrchestrator implements GatewayOrchestratorHandle { } async runOperationsNow(): Promise { - if (this.stopping || this.operationInFlight || this.buildInFlight) { + if (this.stopping || this.operationInFlight || this.buildInFlight || this.workspaceCleanupInFlight) { return; } this.operationInFlight = true; @@ -2160,83 +2175,56 @@ export class GatewayOrchestrator implements GatewayOrchestratorHandle { } async cleanupStaleWorkspaces(): Promise<{ removed: string[]; skipped: string[] }> { - const profiles = await this.repository.listProfiles(); - const cutoff = this.computeCutoffDate(6); - const workspaceMap = new Map(); - for (const profile of profiles) { - const workspace = profile.buildWorkspace; - if (!workspace) { - continue; - } - const entry = workspaceMap.get(workspace) ?? { - profileNames: [], - lastUsedAt: undefined, - hasActiveBuild: false, - }; - entry.profileNames.push(profile.profileName); - if (profile.buildLastUsedAt) { - const usedAt = new Date(profile.buildLastUsedAt); - if (!entry.lastUsedAt || usedAt > entry.lastUsedAt) { - entry.lastUsedAt = usedAt; + if (this.buildInFlight || this.operationInFlight || this.workspaceCleanupInFlight) { + const managedWorkspaces = await this.workspaceManager.listManagedWorkspaces(); + return { removed: [], skipped: managedWorkspaces.map((workspace) => workspace.root) }; + } + this.workspaceCleanupInFlight = true; + try { + const managedWorkspaces = await this.workspaceManager.listManagedWorkspaces(); + const profiles = await this.repository.listProfiles(); + const protectedWorkspaces = new Set(); + for (const profile of profiles) { + if (profile.buildWorkspace) { + protectedWorkspaces.add(path.resolve(profile.buildWorkspace)); + } + if (profile.buildCommitSha && (profile.buildStatus === 'RUNNING' || profile.buildStatus === 'QUEUED')) { + protectedWorkspaces.add( + path.resolve(this.workspaceManager.workspacePathForCommit(profile.buildCommitSha)) + ); } } - if (profile.buildStatus === 'RUNNING' || profile.buildStatus === 'QUEUED') { - entry.hasActiveBuild = true; - } - workspaceMap.set(workspace, entry); - } - const activeProcesses = (await this.processManager.list()).filter((process) => - isRuntimeProcessActive(process.status) - ); - const referencedWorkspaces = new Set(); - for (const [workspace, entry] of workspaceMap.entries()) { - const profileProcessNames = new Set( - entry.profileNames.flatMap((profileName) => [ - buildProcessName(profileName, 'frontend'), - buildProcessName(profileName, 'api'), - buildProcessName(profileName, 'daemon'), - buildProcessName(profileName, 'auction'), - buildProcessName(profileName, 'battle-sim'), - buildProcessName(profileName, 'tournament'), - ]) + const activeProcesses = (await this.processManager.list()).filter((process) => + isRuntimeProcessActive(process.status) ); - if ( - activeProcesses.some( - (process) => - profileProcessNames.has(process.name) || - isPathInside(process.cwd, workspace) || - isPathInside(process.script, workspace) - ) - ) { - referencedWorkspaces.add(workspace); + for (const workspace of managedWorkspaces) { + if ( + activeProcesses.some( + (process) => + isPathInside(process.cwd, workspace.root) || isPathInside(process.script, workspace.root) + ) + ) { + protectedWorkspaces.add(workspace.root); + } } - } - const removed: string[] = []; - const skipped: string[] = []; - for (const [workspace, entry] of workspaceMap.entries()) { - if (!entry.lastUsedAt || entry.hasActiveBuild || referencedWorkspaces.has(workspace)) { - skipped.push(workspace); - continue; - } - if (entry.lastUsedAt > cutoff) { - skipped.push(workspace); - continue; - } - await this.workspaceManager.remove(workspace); - await this.repository.clearWorkspaceUsage(entry.profileNames); - removed.push(workspace); + return await this.workspaceManager.cleanup({ + protectedPaths: [...protectedWorkspaces], + retentionMs: DEFAULT_MANAGED_WORKSPACE_RETENTION_MS, + keepNewest: DEFAULT_MANAGED_WORKSPACE_KEEP_NEWEST, + }); + } finally { + this.workspaceCleanupInFlight = false; } - - return { removed, skipped }; } - private computeCutoffDate(months: number): Date { - const date = this.now(); - const cutoff = new Date(date); - cutoff.setMonth(cutoff.getMonth() - months); - return cutoff; + private async cleanupWorkspacesScheduled(): Promise { + if (this.stopping || this.buildInFlight || this.operationInFlight || this.workspaceCleanupInFlight) return; + const result = await this.cleanupStaleWorkspaces(); + if (result.removed.length > 0) { + console.info(`[gateway-orchestrator] removed ${result.removed.length} stale profile worktrees`); + } } private async startProfile(profile: GatewayProfileRecord, assertLease?: () => Promise): Promise { diff --git a/app/gateway-api/src/orchestrator/workspaceManager.ts b/app/gateway-api/src/orchestrator/workspaceManager.ts index be1fc191..5257bcfe 100644 --- a/app/gateway-api/src/orchestrator/workspaceManager.ts +++ b/app/gateway-api/src/orchestrator/workspaceManager.ts @@ -6,6 +6,7 @@ export interface WorkspaceManagerOptions { repoRoot: string; worktreeRoot: string; baseEnv?: Record; + now?: () => Date; } export interface WorkspaceInfo { @@ -14,6 +15,26 @@ export interface WorkspaceInfo { needsInstall: boolean; } +export interface ManagedWorkspaceInfo { + root: string; + commitSha: string; + lastUsedAt: Date; +} + +export interface ManagedWorkspaceCleanupOptions { + protectedPaths?: readonly string[]; + retentionMs: number; + keepNewest: number; +} + +export interface ManagedWorkspaceCleanupResult { + removed: string[]; + skipped: string[]; +} + +export const DEFAULT_MANAGED_WORKSPACE_RETENTION_MS = 24 * 60 * 60 * 1_000; +export const DEFAULT_MANAGED_WORKSPACE_KEEP_NEWEST = 2; + const runGit = (args: string[], cwd: string, env?: Record): Promise<{ ok: boolean; output: string }> => new Promise((resolve) => { const child = spawn('git', args, { @@ -58,11 +79,13 @@ export class GitWorkspaceManager { private readonly repoRoot: string; private readonly worktreeRoot: string; private readonly baseEnv?: Record; + private readonly now: () => Date; constructor(options: WorkspaceManagerOptions) { this.repoRoot = options.repoRoot; this.worktreeRoot = options.worktreeRoot; this.baseEnv = options.baseEnv; + this.now = options.now ?? (() => new Date()); } async resolveCommit(sourceMode: 'BRANCH' | 'COMMIT', sourceRef: string): Promise { @@ -124,6 +147,8 @@ export class GitWorkspaceManager { } else { await this.assertReusableWorkspace(workspacePath, commitSha); } + const usedAt = this.now(); + fs.utimesSync(workspacePath, usedAt, usedAt); return { root: workspacePath, @@ -138,13 +163,100 @@ export class GitWorkspaceManager { return false; } await this.assertRegisteredWorkspace(resolved); + const status = await runGit(['status', '--porcelain'], resolved, this.baseEnv); + if (!status.ok) { + throw new Error(status.output || 'Failed to inspect managed workspace.'); + } + if (status.output.trim()) { + throw new Error('Managed workspace has uncommitted changes.'); + } const result = await runGit(['worktree', 'remove', '--force', resolved], this.repoRoot, this.baseEnv); if (!result.ok) { - fs.rmSync(resolved, { recursive: true, force: true }); + throw new Error(result.output || 'Failed to remove git worktree.'); } return true; } + workspacePathForCommit(commitSha: string): string { + if (!COMMIT_SHA_PATTERN.test(commitSha)) { + throw new Error('Invalid commit SHA.'); + } + return path.join(this.worktreeRoot, commitSha); + } + + async listManagedWorkspaces(): Promise { + const listed = await runGit(['worktree', 'list', '--porcelain'], this.repoRoot, this.baseEnv); + if (!listed.ok) { + throw new Error(listed.output || 'Failed to inspect git worktrees.'); + } + const workspaces: ManagedWorkspaceInfo[] = []; + for (const block of listed.output.split(/\n\n+/)) { + const lines = block.split('\n'); + const worktreeLine = lines.find((line) => line.startsWith('worktree ')); + const headLine = lines.find((line) => line.startsWith('HEAD ')); + if (!worktreeLine || !headLine) continue; + const workspacePath = path.resolve(worktreeLine.slice('worktree '.length)); + const commitSha = headLine.slice('HEAD '.length); + try { + this.assertManagedWorkspacePath(workspacePath); + } catch { + continue; + } + if (!COMMIT_SHA_PATTERN.test(commitSha) || !fs.existsSync(workspacePath)) continue; + workspaces.push({ + root: workspacePath, + commitSha, + lastUsedAt: fs.statSync(workspacePath).mtime, + }); + } + return workspaces; + } + + async cleanup(options: ManagedWorkspaceCleanupOptions): Promise { + if (!Number.isFinite(options.retentionMs) || options.retentionMs < 0) { + throw new Error('Workspace retention must be a non-negative duration.'); + } + if (!Number.isInteger(options.keepNewest) || options.keepNewest < 0) { + throw new Error('Workspace keepNewest must be a non-negative integer.'); + } + const protectedPaths = new Set((options.protectedPaths ?? []).map((item) => path.resolve(item))); + const workspaces = await this.listManagedWorkspaces(); + const unprotectedNewest = [...workspaces] + .filter((workspace) => !protectedPaths.has(workspace.root)) + .sort((left, right) => right.lastUsedAt.getTime() - left.lastUsedAt.getTime()) + .slice(0, options.keepNewest); + const retainedNewestPaths = new Set(unprotectedNewest.map((workspace) => workspace.root)); + const cutoff = this.now().getTime() - options.retentionMs; + const removed: string[] = []; + const skipped: string[] = []; + + for (const workspace of workspaces) { + if ( + protectedPaths.has(workspace.root) || + retainedNewestPaths.has(workspace.root) || + workspace.lastUsedAt.getTime() > cutoff + ) { + skipped.push(workspace.root); + continue; + } + try { + if (await this.remove(workspace.root)) { + removed.push(workspace.root); + } else { + skipped.push(workspace.root); + } + } catch { + skipped.push(workspace.root); + } + } + + const pruned = await runGit(['worktree', 'prune', '--expire', 'now'], this.repoRoot, this.baseEnv); + if (!pruned.ok) { + throw new Error(pruned.output || 'Failed to prune git worktree metadata.'); + } + return { removed, skipped }; + } + private assertManagedWorkspacePath(workspacePath: string): string { const resolved = path.resolve(workspacePath); const root = path.resolve(this.worktreeRoot); diff --git a/app/gateway-api/test/orchestratorWorkspaceCleanup.test.ts b/app/gateway-api/test/orchestratorWorkspaceCleanup.test.ts index b54a1a04..8749f77c 100644 --- a/app/gateway-api/test/orchestratorWorkspaceCleanup.test.ts +++ b/app/gateway-api/test/orchestratorWorkspaceCleanup.test.ts @@ -1,15 +1,23 @@ +import path from 'node:path'; + import { describe, expect, it } from 'vitest'; import { GatewayOrchestrator } from '../src/orchestrator/gatewayOrchestrator.js'; import type { ProcessManager } from '../src/orchestrator/processManager.js'; import type { GatewayProfileRecord, GatewayProfileRepository } from '../src/orchestrator/profileRepository.js'; -import type { GitWorkspaceManager } from '../src/orchestrator/workspaceManager.js'; +import { + DEFAULT_MANAGED_WORKSPACE_KEEP_NEWEST, + DEFAULT_MANAGED_WORKSPACE_RETENTION_MS, + type GitWorkspaceManager, + type ManagedWorkspaceCleanupOptions, +} from '../src/orchestrator/workspaceManager.js'; +const COMMIT_SHA = '0123456789abcdef0123456789abcdef01234567'; const oldUsage = '2025-01-01T00:00:00.000Z'; const makeProfile = ( profileName: string, - workspace: string, + workspace: string | undefined, overrides: Partial = {} ): GatewayProfileRecord => ({ profileName, @@ -20,7 +28,7 @@ const makeProfile = ( apiPort: 15_003, status: 'RUNNING', buildStatus: 'SUCCEEDED', - buildCommitSha: '0123456789abcdef0123456789abcdef01234567', + buildCommitSha: COMMIT_SHA, buildWorkspace: workspace, buildLastUsedAt: oldUsage, meta: {}, @@ -32,17 +40,10 @@ const makeProfile = ( const createHarness = ( profiles: GatewayProfileRecord[], processes: Awaited>, - workspaceExists = true + managedPaths: string[] ) => { - const removeCalls: string[] = []; - const clearedProfiles: string[][] = []; - - const repository = { - listProfiles: async () => profiles, - clearWorkspaceUsage: async (profileNames: string[]) => { - clearedProfiles.push(profileNames); - }, - } as unknown as GatewayProfileRepository; + const cleanupCalls: ManagedWorkspaceCleanupOptions[] = []; + const repository = { listProfiles: async () => profiles } as unknown as GatewayProfileRepository; const processManager: ProcessManager = { list: async () => processes, start: async () => {}, @@ -50,9 +51,16 @@ const createHarness = ( delete: async () => {}, }; const workspaceManager = { - remove: async (workspace: string) => { - removeCalls.push(workspace); - return workspaceExists; + listManagedWorkspaces: async () => + managedPaths.map((root) => ({ root, commitSha: path.basename(root), lastUsedAt: new Date(oldUsage) })), + workspacePathForCommit: (commitSha: string) => `/srv/sammo/worktrees/${commitSha}`, + cleanup: async (options: ManagedWorkspaceCleanupOptions) => { + cleanupCalls.push(options); + const protectedPaths = new Set(options.protectedPaths); + return { + removed: managedPaths.filter((workspace) => !protectedPaths.has(workspace)), + skipped: managedPaths.filter((workspace) => protectedPaths.has(workspace)), + }; }, } as unknown as GitWorkspaceManager; const orchestrator = new GatewayOrchestrator({ @@ -70,126 +78,67 @@ const createHarness = ( scheduleIntervalMs: 60_000, buildIntervalMs: 60_000, adminActionIntervalMs: 60_000, - now: () => new Date('2026-07-30T00:00:00.000Z'), }); - - return { orchestrator, removeCalls, clearedProfiles }; + return { orchestrator, cleanupCalls }; }; describe('GatewayOrchestrator workspace cleanup', () => { - it('skips a workspace referenced by any active process cwd', async () => { - const workspace = '/srv/sammo/worktrees/active'; + it('always protects every workspace currently selected by a profile', async () => { + const current = '/srv/sammo/worktrees/current'; + const stale = '/srv/sammo/worktrees/stale'; + const harness = createHarness([makeProfile('che:default', current)], [], [current, stale]); + + await expect(harness.orchestrator.cleanupStaleWorkspaces()).resolves.toEqual({ + removed: [stale], + skipped: [current], + }); + expect(harness.cleanupCalls[0]).toMatchObject({ + retentionMs: DEFAULT_MANAGED_WORKSPACE_RETENTION_MS, + keepNewest: DEFAULT_MANAGED_WORKSPACE_KEEP_NEWEST, + }); + }); + + it('protects the commit target of queued and running builds before the profile reference changes', async () => { + const target = `/srv/sammo/worktrees/${COMMIT_SHA}`; + const harness = createHarness([makeProfile('che:default', undefined, { buildStatus: 'QUEUED' })], [], [target]); + + await expect(harness.orchestrator.cleanupStaleWorkspaces()).resolves.toEqual({ + removed: [], + skipped: [target], + }); + }); + + it('protects an otherwise orphaned workspace referenced by any active process cwd or script', async () => { + const cwdWorkspace = '/srv/sammo/worktrees/cwd-orphan'; + const scriptWorkspace = '/srv/sammo/worktrees/script-orphan'; + const stale = '/srv/sammo/worktrees/stale'; const harness = createHarness( - [makeProfile('che:default', workspace)], + [], [ - { - name: 'sammo:che:default:frontend', - status: 'online', - cwd: `${workspace}/app/game-frontend`, - }, - ] + { name: 'custom-build', status: 'online', cwd: `${cwdWorkspace}/app/game-api` }, + { name: 'custom-worker', status: 'launching', script: `${scriptWorkspace}/dist/index.js` }, + { name: 'stopped-worker', status: 'stopped', cwd: `${stale}/app/game-api` }, + ], + [cwdWorkspace, scriptWorkspace, stale] ); await expect(harness.orchestrator.cleanupStaleWorkspaces()).resolves.toEqual({ - removed: [], - skipped: [workspace], + removed: [stale], + skipped: [cwdWorkspace, scriptWorkspace], }); - expect(harness.removeCalls).toEqual([]); - expect(harness.clearedProfiles).toEqual([]); }); - it('skips a workspace when only one profile process is active and cwd metadata is absent', async () => { - const workspace = '/srv/sammo/worktrees/partial'; - const harness = createHarness( - [makeProfile('che:default', workspace)], - [{ name: 'sammo:che:default:tournament-worker', status: 'launching' }] - ); - - await expect(harness.orchestrator.cleanupStaleWorkspaces()).resolves.toEqual({ - removed: [], - skipped: [workspace], - }); - expect(harness.removeCalls).toEqual([]); - }); - - it('skips a workspace referenced only by an active process script', async () => { - const workspace = '/srv/sammo/worktrees/script-reference'; - const harness = createHarness( - [makeProfile('che:default', workspace)], - [ - { - name: 'unregistered-worker-name', - status: 'online', - script: `${workspace}/app/game-api/dist/index.js`, - }, - ] - ); - - await expect(harness.orchestrator.cleanupStaleWorkspaces()).resolves.toEqual({ - removed: [], - skipped: [workspace], - }); - expect(harness.removeCalls).toEqual([]); - }); - - it('protects a shared workspace when a process for either profile is active', async () => { - const workspace = '/srv/sammo/worktrees/shared'; - const harness = createHarness( - [makeProfile('che:default', workspace), makeProfile('hwe:default', workspace)], - [{ name: 'sammo:hwe:default:game-api', status: 'stopping' }] - ); - - await expect(harness.orchestrator.cleanupStaleWorkspaces()).resolves.toEqual({ - removed: [], - skipped: [workspace], - }); - expect(harness.removeCalls).toEqual([]); - }); - - it('removes an old unreferenced workspace and clears every profile reference', async () => { - const workspace = '/srv/sammo/worktrees/stale'; - const harness = createHarness( - [makeProfile('che:default', workspace), makeProfile('hwe:default', workspace)], - [{ name: 'sammo:che:default:game-api', status: 'stopped', cwd: `${workspace}/app/game-api` }] - ); - - await expect(harness.orchestrator.cleanupStaleWorkspaces()).resolves.toEqual({ - removed: [workspace], - skipped: [], - }); - expect(harness.removeCalls).toEqual([workspace]); - expect(harness.clearedProfiles).toEqual([['che:default', 'hwe:default']]); - }); - - it('does not treat a sibling path with the same prefix as a workspace reference', async () => { + it('does not confuse sibling path prefixes with an active workspace reference', async () => { const workspace = '/srv/sammo/worktrees/commit-a'; const harness = createHarness( - [makeProfile('che:default', workspace)], - [ - { - name: 'unregistered-worker-name', - status: 'online', - cwd: '/srv/sammo/worktrees/commit-a-old/app/game-api', - }, - ] + [], + [{ name: 'custom-worker', status: 'online', cwd: `${workspace}-old/app/game-api` }], + [workspace] ); await expect(harness.orchestrator.cleanupStaleWorkspaces()).resolves.toEqual({ removed: [workspace], skipped: [], }); - expect(harness.removeCalls).toEqual([workspace]); - }); - - it('clears a stale database reference when the workspace is already missing', async () => { - const workspace = '/srv/sammo/worktrees/missing'; - const harness = createHarness([makeProfile('che:default', workspace)], [], false); - - await expect(harness.orchestrator.cleanupStaleWorkspaces()).resolves.toEqual({ - removed: [workspace], - skipped: [], - }); - expect(harness.removeCalls).toEqual([workspace]); - expect(harness.clearedProfiles).toEqual([['che:default']]); }); }); diff --git a/app/gateway-api/test/workspaceManager.test.ts b/app/gateway-api/test/workspaceManager.test.ts index bf0a50f2..2fa37eb9 100644 --- a/app/gateway-api/test/workspaceManager.test.ts +++ b/app/gateway-api/test/workspaceManager.test.ts @@ -163,4 +163,46 @@ describe('GitWorkspaceManager source resolution', () => { ); expect(fs.existsSync(unregistered)).toBe(true); }); + + it('cleans only expired unprotected worktrees beyond the newest cache and preserves dirty work', async () => { + const fixture = createRepositoryFixture(); + const now = new Date('2026-08-20T12:00:00.000Z'); + const manager = new GitWorkspaceManager({ + repoRoot: fixture.checkout, + worktreeRoot: fixture.worktrees, + now: () => now, + }); + const workspaces = [await manager.prepare(fixture.firstCommit)]; + for (let index = 2; index <= 5; index += 1) { + fs.writeFileSync(path.join(fixture.source, 'version.txt'), `version ${index}\n`); + git(fixture.source, 'add', 'version.txt'); + git(fixture.source, 'commit', '-m', `version ${index}`); + git(fixture.source, 'push', 'origin', 'main'); + const commit = await manager.resolveCommit('BRANCH', 'main'); + workspaces.push(await manager.prepare(commit)); + } + const expired = new Date('2026-08-01T00:00:00.000Z'); + for (const workspace of workspaces) fs.utimesSync(workspace.root, expired, expired); + fs.writeFileSync(path.join(workspaces[1]!.root, 'preserve-me.txt'), 'uncommitted\n'); + fs.utimesSync(workspaces[1]!.root, expired, expired); + const recent = new Date('2026-08-20T11:00:00.000Z'); + fs.utimesSync(workspaces[4]!.root, recent, recent); + + const result = await manager.cleanup({ + protectedPaths: [workspaces[0]!.root], + retentionMs: 24 * 60 * 60 * 1_000, + keepNewest: 1, + }); + expect(result.removed).toHaveLength(2); + expect(result.removed).toEqual(expect.arrayContaining([workspaces[2]!.root, workspaces[3]!.root])); + expect(result.skipped).toHaveLength(3); + expect(result.skipped).toEqual( + expect.arrayContaining([workspaces[0]!.root, workspaces[1]!.root, workspaces[4]!.root]) + ); + expect(fs.existsSync(workspaces[0]!.root)).toBe(true); + expect(fs.existsSync(workspaces[1]!.root)).toBe(true); + expect(fs.existsSync(workspaces[2]!.root)).toBe(false); + expect(fs.existsSync(workspaces[3]!.root)).toBe(false); + expect(fs.existsSync(workspaces[4]!.root)).toBe(true); + }); }); diff --git a/app/release-controller/README.md b/app/release-controller/README.md index 69a0a115..d319933e 100644 --- a/app/release-controller/README.md +++ b/app/release-controller/README.md @@ -70,6 +70,13 @@ pnpm --filter @sammo-ts/release-controller status pnpm --filter @sammo-ts/release-controller run-once ``` +Daemon은 시작 시와 이후 24시간마다 commit worktree를 자동 정리합니다. 현재·이전 +Gateway release와 활성 PM2 process가 사용하는 경로는 항상 보호하고, 나머지는 +마지막 사용 후 24시간과 최신 2개 cache를 보장한 뒤 제거합니다. 변경이 있거나 Git +제거가 실패한 worktree는 raw directory 삭제로 우회하지 않고 다음 주기까지 +보존합니다. Profile worktree는 Gateway orchestrator가 같은 정책으로 별도 +관리합니다. + ## Controller self-upgrade 이 명령은 현재 daemon과 별개의 CLI process에서 실행됩니다. 대상 worktree를 diff --git a/app/release-controller/src/index.ts b/app/release-controller/src/index.ts index 19b43276..c0810564 100644 --- a/app/release-controller/src/index.ts +++ b/app/release-controller/src/index.ts @@ -7,7 +7,7 @@ import { } from '@sammo-ts/gateway-api'; import { resolveReleaseControllerConfig } from './config.js'; -import { GatewayReleaseController } from './releaseController.js'; +import { GatewayReleaseController, RELEASE_WORKSPACE_CLEANUP_INTERVAL_MS } from './releaseController.js'; import { upgradeReleaseController } from './selfUpgrade.js'; export * from './config.js'; @@ -67,6 +67,7 @@ const main = async (): Promise => { } if (command !== 'daemon') throw new Error(`Unknown release-controller command: ${command}`); let stopping = false; + let nextWorkspaceCleanupAt = 0; const stop = async (): Promise => { if (stopping) return; stopping = true; @@ -75,6 +76,18 @@ const main = async (): Promise => { process.once('SIGINT', () => void stop()); process.once('SIGTERM', () => void stop()); while (!stopping) { + const now = Date.now(); + if (now >= nextWorkspaceCleanupAt) { + nextWorkspaceCleanupAt = now + RELEASE_WORKSPACE_CLEANUP_INTERVAL_MS; + try { + const result = await controller.cleanupStaleWorkspaces(); + if (result.removed.length > 0) { + console.info(`[release-controller] removed ${result.removed.length} stale Gateway worktrees`); + } + } catch (error) { + console.error('[release-controller] workspace cleanup failed', error); + } + } await controller.runOnce(); await new Promise((resolve) => setTimeout(resolve, config.pollIntervalMs)); } diff --git a/app/release-controller/src/releaseController.ts b/app/release-controller/src/releaseController.ts index 6237c163..4e8f5bf0 100644 --- a/app/release-controller/src/releaseController.ts +++ b/app/release-controller/src/releaseController.ts @@ -6,6 +6,8 @@ import { assertReleaseComponents, buildTurboReleaseCommand, buildTurboReleaseTaskCommand, + DEFAULT_MANAGED_WORKSPACE_KEEP_NEWEST, + DEFAULT_MANAGED_WORKSPACE_RETENTION_MS, type BuildCommand, type BuildProgressEvent, type BuildRunner, @@ -27,6 +29,16 @@ const HEARTBEAT_INTERVAL_MS = 60_000; const CANCELLATION_POLL_INTERVAL_MS = 500; const PROCESS_NAMES = ['sammo:gateway-api', 'sammo:gateway-frontend', 'sammo:gateway-orchestrator'] as const; const SENSITIVE_ENV_NAME = /(SECRET|TOKEN|PASSWORD|PASSWD|PRIVATE_KEY|CLIENT_SECRET|DATABASE_URL|REDIS_URL)/iu; +export const RELEASE_WORKSPACE_CLEANUP_INTERVAL_MS = 24 * 60 * 60 * 1_000; + +const isRuntimeProcessActive = (status: string): boolean => + ['online', 'launching', 'stopping'].includes(status.toLowerCase()); + +const isPathInside = (candidate: string | undefined, root: string): boolean => { + if (!candidate) return false; + const relative = path.relative(path.resolve(root), path.resolve(candidate)); + return relative === '' || (!relative.startsWith(`..${path.sep}`) && !path.isAbsolute(relative)); +}; const managedPostgresPoolMax = (env: Record, roleVariable: string, fallback: number): string => String(resolvePostgresPoolMax(env[roleVariable] ?? env.POSTGRES_POOL_MAX, fallback)); @@ -132,6 +144,33 @@ export class GatewayReleaseController { private readonly fetchImpl: typeof fetch = fetch ) {} + async cleanupStaleWorkspaces(): Promise<{ removed: string[]; skipped: string[] }> { + const [state, processes, workspaces] = await Promise.all([ + this.repository.getState(), + this.processManager.list(), + this.workspaceManager.listManagedWorkspaces(), + ]); + const protectedWorkspaces = new Set(); + if (state.activeWorkspace) protectedWorkspaces.add(path.resolve(state.activeWorkspace)); + if (state.previousWorkspace) protectedWorkspaces.add(path.resolve(state.previousWorkspace)); + const activeProcesses = processes.filter((process) => isRuntimeProcessActive(process.status)); + for (const workspace of workspaces) { + if ( + activeProcesses.some( + (process) => + isPathInside(process.cwd, workspace.root) || isPathInside(process.script, workspace.root) + ) + ) { + protectedWorkspaces.add(workspace.root); + } + } + return this.workspaceManager.cleanup({ + protectedPaths: [...protectedWorkspaces], + retentionMs: DEFAULT_MANAGED_WORKSPACE_RETENTION_MS, + keepNewest: DEFAULT_MANAGED_WORKSPACE_KEEP_NEWEST, + }); + } + private sanitizeLogMessage(message: string): string { let sanitized = stripVTControlCharacters(message); const sensitiveValues = new Set([ diff --git a/app/release-controller/test/releaseController.test.ts b/app/release-controller/test/releaseController.test.ts index aeb56f8e..a77e970b 100644 --- a/app/release-controller/test/releaseController.test.ts +++ b/app/release-controller/test/releaseController.test.ts @@ -2,14 +2,17 @@ import fs from 'node:fs/promises'; import os from 'node:os'; import path from 'node:path'; -import type { - BuildRunner, - GatewayReleaseOperationRecord, - GatewayReleaseRepository, - GatewayReleaseStateRecord, - GitWorkspaceManager, - ProcessDefinition, - ProcessManager, +import { + DEFAULT_MANAGED_WORKSPACE_KEEP_NEWEST, + DEFAULT_MANAGED_WORKSPACE_RETENTION_MS, + type BuildRunner, + type GatewayReleaseOperationRecord, + type GatewayReleaseRepository, + type GatewayReleaseStateRecord, + type GitWorkspaceManager, + type ManagedWorkspaceCleanupOptions, + type ProcessDefinition, + type ProcessManager, } from '@sammo-ts/gateway-api'; import { afterEach, describe, expect, it } from 'vitest'; @@ -178,6 +181,68 @@ it('rejects Gateway definitions before switching processes when Redis connection }); describe('GatewayReleaseController', () => { + it('protects active, rollback, and running-process worktrees while delegating bounded cleanup', async () => { + const active = '/srv/sammo/releases/active'; + const previous = '/srv/sammo/releases/previous'; + const controllerWorkspace = '/srv/sammo/releases/controller'; + const stale = '/srv/sammo/releases/stale'; + const managedPaths = [active, previous, controllerWorkspace, stale]; + const cleanupCalls: ManagedWorkspaceCleanupOptions[] = []; + const harness = createRepository(); + const workspaceManager = { + listManagedWorkspaces: async () => + managedPaths.map((root) => ({ + root, + commitSha: SHA, + lastUsedAt: new Date('2025-01-01T00:00:00.000Z'), + })), + cleanup: async (options: ManagedWorkspaceCleanupOptions) => { + cleanupCalls.push(options); + const protectedPaths = new Set(options.protectedPaths); + return { + removed: managedPaths.filter((workspace) => !protectedPaths.has(workspace)), + skipped: managedPaths.filter((workspace) => protectedPaths.has(workspace)), + }; + }, + } as unknown as GitWorkspaceManager; + const controller = new GatewayReleaseController( + { + ...harness.repository, + getState: async () => ({ + ...state, + activeWorkspace: active, + previousCommitSha: SHA, + previousWorkspace: previous, + }), + }, + workspaceManager, + { run: async () => ({ ok: true, exitCode: 0, output: '' }) }, + { + list: async () => [ + { + name: 'sammo:release-controller', + status: 'online', + cwd: `${controllerWorkspace}/app/release-controller`, + }, + { name: 'old-build', status: 'stopped', cwd: `${stale}/app/gateway-api` }, + ], + start: async () => {}, + stop: async () => {}, + delete: async () => {}, + }, + config + ); + + await expect(controller.cleanupStaleWorkspaces()).resolves.toEqual({ + removed: [stale], + skipped: [active, previous, controllerWorkspace], + }); + expect(cleanupCalls[0]).toMatchObject({ + retentionMs: DEFAULT_MANAGED_WORKSPACE_RETENTION_MS, + keepNewest: DEFAULT_MANAGED_WORKSPACE_KEEP_NEWEST, + }); + }); + it('builds, migrates, switches all gateway roles, verifies readiness, and publishes atomically', async () => { const workspace = await createReleaseWorkspace(); const harness = createRepository(); diff --git a/docs/release-operations.md b/docs/release-operations.md index 077b13a3..5e5fd86b 100644 --- a/docs/release-operations.md +++ b/docs/release-operations.md @@ -69,6 +69,29 @@ profile 범위 권한과 별개인 전역 `admin.releases.manage` 권한이 필 - migration 이후 이전 애플리케이션으로 돌아갈 때 schema 하위 호환성이 유지됩니다. +## Commit worktree 자동 정리 + +Profile orchestrator와 Gateway release-controller는 서로 다른 worktree root를 +사용하지만 같은 보존 정책을 적용합니다. 각 daemon은 시작 시 한 번, 이후 24시간마다 +자신이 소유한 commit worktree를 점검합니다. + +- `GatewayProfile.buildWorkspace`, `RUNNING`/`QUEUED` profile 빌드 대상, + `GatewayReleaseState`의 active/previous workspace는 기간과 무관하게 보호합니다. +- 활성 PM2 process의 cwd 또는 script 아래에 있는 worktree도 보호합니다. 여기에는 + self-upgrade된 release-controller worktree도 포함됩니다. +- 보호 대상이 아닌 worktree는 마지막 prepare 이후 최소 24시간을 유예하고, 그중 + 최신 2개는 재시도 cache로 더 남깁니다. 나머지는 Git worktree로 제거하고 + `git worktree prune --expire now`로 사라진 metadata를 정리합니다. +- tracked 또는 untracked 변경이 있으면 자동 삭제하지 않습니다. Git 제거 실패를 + raw directory 삭제로 우회하지 않으며 다음 주기까지 보존합니다. +- 정리는 commit checkout과 재생성 가능한 build artifact만 대상으로 합니다. + Gateway/profile PostgreSQL, Redis, image, runtime data volume에는 접근하지 않습니다. + +따라서 하루 안에 매우 많은 commit을 연속 배포하면 유예 구간만큼 일시적으로 늘 수 +있지만, active/rollback/current profile 경로 외의 장기 누적은 다음 정리 주기에 +제거됩니다. Profile 관리자 API의 `admin.profiles.cleanupWorkspaces`는 같은 보호 +규칙을 사용하므로 진행 중인 build/operation이 있으면 전체 정리를 보류합니다. + ## Profile 배포 버전 업데이트 화면에서 profile의 branch 또는 commit을 선택합니다. Branch는 worker가