diff --git a/app/gateway-api/src/orchestrator/gatewayOrchestrator.ts b/app/gateway-api/src/orchestrator/gatewayOrchestrator.ts index 7621d5f..64ca055 100644 --- a/app/gateway-api/src/orchestrator/gatewayOrchestrator.ts +++ b/app/gateway-api/src/orchestrator/gatewayOrchestrator.ts @@ -305,6 +305,19 @@ const buildProcessName = ( const isMissingProcessError = (error: unknown): boolean => error instanceof Error && /process or namespace not found/i.test(error.message); +const isRuntimeProcessActive = (status: string): boolean => { + const normalized = status.toLowerCase(); + return normalized === 'online' || normalized === 'launching' || normalized === 'stopping'; +}; + +const isPathInside = (candidate: string | undefined, root: string): boolean => { + if (!candidate) { + return false; + } + const relative = path.relative(path.resolve(root), path.resolve(candidate)); + return relative === '' || (!relative.startsWith('..') && !path.isAbsolute(relative)); +}; + export const buildProcessDefinitions = ( profile: GatewayProfileRecord, config: GatewayProcessConfig @@ -1003,10 +1016,36 @@ export class GatewayOrchestrator implements GatewayOrchestratorHandle { workspaceMap.set(workspace, entry); } + const activeProcesses = (await this.processManager.list()).filter((process) => + isRuntimeProcessActive(process.status) + ); + const referencedWorkspaces = new Set(); + for (const [workspace, entry] of workspaceMap.entries()) { + const profileProcessNames = new Set( + entry.profileNames.flatMap((profileName) => [ + buildProcessName(profileName, 'api'), + buildProcessName(profileName, 'daemon'), + buildProcessName(profileName, 'auction'), + buildProcessName(profileName, 'battle-sim'), + buildProcessName(profileName, 'tournament'), + ]) + ); + if ( + activeProcesses.some( + (process) => + profileProcessNames.has(process.name) || + isPathInside(process.cwd, workspace) || + isPathInside(process.script, workspace) + ) + ) { + referencedWorkspaces.add(workspace); + } + } + const removed: string[] = []; const skipped: string[] = []; for (const [workspace, entry] of workspaceMap.entries()) { - if (!entry.lastUsedAt || entry.hasActiveBuild) { + if (!entry.lastUsedAt || entry.hasActiveBuild || referencedWorkspaces.has(workspace)) { skipped.push(workspace); continue; } diff --git a/app/gateway-api/src/orchestrator/pm2ProcessManager.ts b/app/gateway-api/src/orchestrator/pm2ProcessManager.ts index d9bd60f..acd819e 100644 --- a/app/gateway-api/src/orchestrator/pm2ProcessManager.ts +++ b/app/gateway-api/src/orchestrator/pm2ProcessManager.ts @@ -42,6 +42,8 @@ export class Pm2ProcessManager implements ProcessManager { name: item.name ?? 'unknown', status: item.pm2_env?.status ?? 'unknown', pid: item.pid ?? undefined, + cwd: item.pm2_env?.pm_cwd ?? undefined, + script: item.pm2_env?.pm_exec_path ?? undefined, })) ?? []; resolve(normalized); }); diff --git a/app/gateway-api/src/orchestrator/processManager.ts b/app/gateway-api/src/orchestrator/processManager.ts index 2706c3c..4dfc10f 100644 --- a/app/gateway-api/src/orchestrator/processManager.ts +++ b/app/gateway-api/src/orchestrator/processManager.ts @@ -2,6 +2,8 @@ export interface ManagedProcessInfo { name: string; status: string; pid?: number; + cwd?: string; + script?: string; } export interface ProcessDefinition { diff --git a/app/gateway-api/test/orchestratorWorkspaceCleanup.test.ts b/app/gateway-api/test/orchestratorWorkspaceCleanup.test.ts new file mode 100644 index 0000000..9b92cc2 --- /dev/null +++ b/app/gateway-api/test/orchestratorWorkspaceCleanup.test.ts @@ -0,0 +1,192 @@ +import { describe, expect, it } from 'vitest'; + +import { GatewayOrchestrator } from '../src/orchestrator/gatewayOrchestrator.js'; +import type { ProcessManager } from '../src/orchestrator/processManager.js'; +import type { GatewayProfileRecord, GatewayProfileRepository } from '../src/orchestrator/profileRepository.js'; +import type { GitWorkspaceManager } from '../src/orchestrator/workspaceManager.js'; + +const oldUsage = '2025-01-01T00:00:00.000Z'; + +const makeProfile = ( + profileName: string, + workspace: string, + overrides: Partial = {} +): GatewayProfileRecord => ({ + profileName, + profile: profileName.split(':')[0] ?? 'che', + scenario: profileName.split(':')[1] ?? 'default', + apiPort: 15_003, + status: 'RUNNING', + buildStatus: 'SUCCEEDED', + buildCommitSha: '0123456789abcdef0123456789abcdef01234567', + buildWorkspace: workspace, + buildLastUsedAt: oldUsage, + meta: {}, + createdAt: oldUsage, + updatedAt: oldUsage, + ...overrides, +}); + +const createHarness = ( + profiles: GatewayProfileRecord[], + processes: Awaited>, + workspaceExists = true +) => { + const removeCalls: string[] = []; + const clearedProfiles: string[][] = []; + + const repository = { + listProfiles: async () => profiles, + clearWorkspaceUsage: async (profileNames: string[]) => { + clearedProfiles.push(profileNames); + }, + } as unknown as GatewayProfileRepository; + const processManager: ProcessManager = { + list: async () => processes, + start: async () => {}, + stop: async () => {}, + delete: async () => {}, + }; + const workspaceManager = { + remove: async (workspace: string) => { + removeCalls.push(workspace); + return workspaceExists; + }, + } as unknown as GitWorkspaceManager; + const orchestrator = new GatewayOrchestrator({ + repository, + processManager, + buildRunner: { run: async () => ({ ok: true, exitCode: 0, output: '' }) }, + workspaceManager, + processConfig: { + workspaceRoot: '/srv/sammo', + redisKeyPrefix: 'sammo:test', + gameTokenSecret: 'test-secret', + }, + reconcileIntervalMs: 60_000, + scheduleIntervalMs: 60_000, + buildIntervalMs: 60_000, + adminActionIntervalMs: 60_000, + now: () => new Date('2026-07-30T00:00:00.000Z'), + }); + + return { orchestrator, removeCalls, clearedProfiles }; +}; + +describe('GatewayOrchestrator workspace cleanup', () => { + it('skips a workspace referenced by any active process cwd', async () => { + const workspace = '/srv/sammo/worktrees/active'; + const harness = createHarness( + [makeProfile('che:default', workspace)], + [ + { + name: 'sammo:che:default:frontend', + status: 'online', + cwd: `${workspace}/app/game-frontend`, + }, + ] + ); + + await expect(harness.orchestrator.cleanupStaleWorkspaces()).resolves.toEqual({ + removed: [], + skipped: [workspace], + }); + expect(harness.removeCalls).toEqual([]); + expect(harness.clearedProfiles).toEqual([]); + }); + + it('skips a workspace when only one profile process is active and cwd metadata is absent', async () => { + const workspace = '/srv/sammo/worktrees/partial'; + const harness = createHarness( + [makeProfile('che:default', workspace)], + [{ name: 'sammo:che:default:tournament-worker', status: 'launching' }] + ); + + await expect(harness.orchestrator.cleanupStaleWorkspaces()).resolves.toEqual({ + removed: [], + skipped: [workspace], + }); + expect(harness.removeCalls).toEqual([]); + }); + + it('skips a workspace referenced only by an active process script', async () => { + const workspace = '/srv/sammo/worktrees/script-reference'; + const harness = createHarness( + [makeProfile('che:default', workspace)], + [ + { + name: 'unregistered-worker-name', + status: 'online', + script: `${workspace}/app/game-api/dist/index.js`, + }, + ] + ); + + await expect(harness.orchestrator.cleanupStaleWorkspaces()).resolves.toEqual({ + removed: [], + skipped: [workspace], + }); + expect(harness.removeCalls).toEqual([]); + }); + + it('protects a shared workspace when a process for either profile is active', async () => { + const workspace = '/srv/sammo/worktrees/shared'; + const harness = createHarness( + [makeProfile('che:default', workspace), makeProfile('hwe:default', workspace)], + [{ name: 'sammo:hwe:default:game-api', status: 'stopping' }] + ); + + await expect(harness.orchestrator.cleanupStaleWorkspaces()).resolves.toEqual({ + removed: [], + skipped: [workspace], + }); + expect(harness.removeCalls).toEqual([]); + }); + + it('removes an old unreferenced workspace and clears every profile reference', async () => { + const workspace = '/srv/sammo/worktrees/stale'; + const harness = createHarness( + [makeProfile('che:default', workspace), makeProfile('hwe:default', workspace)], + [{ name: 'sammo:che:default:game-api', status: 'stopped', cwd: `${workspace}/app/game-api` }] + ); + + await expect(harness.orchestrator.cleanupStaleWorkspaces()).resolves.toEqual({ + removed: [workspace], + skipped: [], + }); + expect(harness.removeCalls).toEqual([workspace]); + expect(harness.clearedProfiles).toEqual([['che:default', 'hwe:default']]); + }); + + it('does not treat a sibling path with the same prefix as a workspace reference', async () => { + const workspace = '/srv/sammo/worktrees/commit-a'; + const harness = createHarness( + [makeProfile('che:default', workspace)], + [ + { + name: 'unregistered-worker-name', + status: 'online', + cwd: '/srv/sammo/worktrees/commit-a-old/app/game-api', + }, + ] + ); + + await expect(harness.orchestrator.cleanupStaleWorkspaces()).resolves.toEqual({ + removed: [workspace], + skipped: [], + }); + expect(harness.removeCalls).toEqual([workspace]); + }); + + it('clears a stale database reference when the workspace is already missing', async () => { + const workspace = '/srv/sammo/worktrees/missing'; + const harness = createHarness([makeProfile('che:default', workspace)], [], false); + + await expect(harness.orchestrator.cleanupStaleWorkspaces()).resolves.toEqual({ + removed: [workspace], + skipped: [], + }); + expect(harness.removeCalls).toEqual([workspace]); + expect(harness.clearedProfiles).toEqual([['che:default']]); + }); +});