merge: push 직전 최신 main을 턴 실패 개인 기록 작업에 반영
This commit is contained in:
@@ -44,7 +44,11 @@ import {
|
||||
writeProfileReleaseSource,
|
||||
type ProfileReleaseSource,
|
||||
} from './profileReleaseSource.js';
|
||||
import type { GitWorkspaceManager } from './workspaceManager.js';
|
||||
import {
|
||||
DEFAULT_MANAGED_WORKSPACE_KEEP_NEWEST,
|
||||
DEFAULT_MANAGED_WORKSPACE_RETENTION_MS,
|
||||
type GitWorkspaceManager,
|
||||
} from './workspaceManager.js';
|
||||
import type { AdminSeedUser } from './seedProfileDatabase.js';
|
||||
import { assertReleaseComponents, readReleaseManifest } from './releaseManifest.js';
|
||||
|
||||
@@ -73,6 +77,8 @@ export interface GatewayOrchestratorOptions {
|
||||
cancelGame?: typeof defaultCancelGame;
|
||||
}
|
||||
|
||||
const WORKSPACE_CLEANUP_INTERVAL_MS = 24 * 60 * 60 * 1_000;
|
||||
|
||||
export interface ProfileRuntimeState {
|
||||
frontendRunning: boolean;
|
||||
apiRunning: boolean;
|
||||
@@ -629,11 +635,13 @@ export class GatewayOrchestrator implements GatewayOrchestratorHandle {
|
||||
private scheduleTimer?: NodeJS.Timeout;
|
||||
private buildTimer?: NodeJS.Timeout;
|
||||
private adminActionTimer?: NodeJS.Timeout;
|
||||
private workspaceCleanupTimer?: NodeJS.Timeout;
|
||||
private reconcileInFlight = false;
|
||||
private scheduleInFlight = false;
|
||||
private buildInFlight = false;
|
||||
private adminActionInFlight = false;
|
||||
private operationInFlight = false;
|
||||
private workspaceCleanupInFlight = false;
|
||||
private activeOperationAbortSignal?: AbortSignal;
|
||||
private readonly resetInFlight = new Set<string>();
|
||||
private readonly operationLeaseOwner = randomUUID();
|
||||
@@ -714,7 +722,7 @@ export class GatewayOrchestrator implements GatewayOrchestratorHandle {
|
||||
start(): void {
|
||||
this.stopping = false;
|
||||
this.trackTask(this.reconcileNow());
|
||||
this.trackTask(this.runOperationsNow());
|
||||
this.trackTask(this.runOperationsNow().then(() => this.cleanupWorkspacesScheduled()));
|
||||
this.trackTask(this.runAdminActionsNow());
|
||||
this.reconcileTimer = setInterval(() => this.trackTask(this.reconcileNow()), this.reconcileIntervalMs);
|
||||
this.scheduleTimer = setInterval(() => this.trackTask(this.runScheduleNow()), this.scheduleIntervalMs);
|
||||
@@ -723,6 +731,10 @@ export class GatewayOrchestrator implements GatewayOrchestratorHandle {
|
||||
this.trackTask(this.runOperationsNow());
|
||||
this.trackTask(this.runAdminActionsNow());
|
||||
}, this.adminActionIntervalMs);
|
||||
this.workspaceCleanupTimer = setInterval(
|
||||
() => this.trackTask(this.cleanupWorkspacesScheduled()),
|
||||
WORKSPACE_CLEANUP_INTERVAL_MS
|
||||
);
|
||||
}
|
||||
|
||||
async stop(): Promise<void> {
|
||||
@@ -747,6 +759,9 @@ export class GatewayOrchestrator implements GatewayOrchestratorHandle {
|
||||
if (this.adminActionTimer) {
|
||||
clearInterval(this.adminActionTimer);
|
||||
}
|
||||
if (this.workspaceCleanupTimer) {
|
||||
clearInterval(this.workspaceCleanupTimer);
|
||||
}
|
||||
await Promise.allSettled([...this.inFlightTasks]);
|
||||
}
|
||||
|
||||
@@ -903,7 +918,7 @@ export class GatewayOrchestrator implements GatewayOrchestratorHandle {
|
||||
}
|
||||
|
||||
async runBuildQueueNow(): Promise<void> {
|
||||
if (this.stopping || this.buildInFlight) {
|
||||
if (this.stopping || this.buildInFlight || this.workspaceCleanupInFlight) {
|
||||
return;
|
||||
}
|
||||
this.buildInFlight = true;
|
||||
@@ -965,7 +980,7 @@ export class GatewayOrchestrator implements GatewayOrchestratorHandle {
|
||||
}
|
||||
|
||||
async runOperationsNow(): Promise<void> {
|
||||
if (this.stopping || this.operationInFlight || this.buildInFlight) {
|
||||
if (this.stopping || this.operationInFlight || this.buildInFlight || this.workspaceCleanupInFlight) {
|
||||
return;
|
||||
}
|
||||
this.operationInFlight = true;
|
||||
@@ -2160,83 +2175,56 @@ export class GatewayOrchestrator implements GatewayOrchestratorHandle {
|
||||
}
|
||||
|
||||
async cleanupStaleWorkspaces(): Promise<{ removed: string[]; skipped: string[] }> {
|
||||
const profiles = await this.repository.listProfiles();
|
||||
const cutoff = this.computeCutoffDate(6);
|
||||
const workspaceMap = new Map<string, { profileNames: string[]; lastUsedAt?: Date; hasActiveBuild: boolean }>();
|
||||
for (const profile of profiles) {
|
||||
const workspace = profile.buildWorkspace;
|
||||
if (!workspace) {
|
||||
continue;
|
||||
}
|
||||
const entry = workspaceMap.get(workspace) ?? {
|
||||
profileNames: [],
|
||||
lastUsedAt: undefined,
|
||||
hasActiveBuild: false,
|
||||
};
|
||||
entry.profileNames.push(profile.profileName);
|
||||
if (profile.buildLastUsedAt) {
|
||||
const usedAt = new Date(profile.buildLastUsedAt);
|
||||
if (!entry.lastUsedAt || usedAt > entry.lastUsedAt) {
|
||||
entry.lastUsedAt = usedAt;
|
||||
if (this.buildInFlight || this.operationInFlight || this.workspaceCleanupInFlight) {
|
||||
const managedWorkspaces = await this.workspaceManager.listManagedWorkspaces();
|
||||
return { removed: [], skipped: managedWorkspaces.map((workspace) => workspace.root) };
|
||||
}
|
||||
this.workspaceCleanupInFlight = true;
|
||||
try {
|
||||
const managedWorkspaces = await this.workspaceManager.listManagedWorkspaces();
|
||||
const profiles = await this.repository.listProfiles();
|
||||
const protectedWorkspaces = new Set<string>();
|
||||
for (const profile of profiles) {
|
||||
if (profile.buildWorkspace) {
|
||||
protectedWorkspaces.add(path.resolve(profile.buildWorkspace));
|
||||
}
|
||||
if (profile.buildCommitSha && (profile.buildStatus === 'RUNNING' || profile.buildStatus === 'QUEUED')) {
|
||||
protectedWorkspaces.add(
|
||||
path.resolve(this.workspaceManager.workspacePathForCommit(profile.buildCommitSha))
|
||||
);
|
||||
}
|
||||
}
|
||||
if (profile.buildStatus === 'RUNNING' || profile.buildStatus === 'QUEUED') {
|
||||
entry.hasActiveBuild = true;
|
||||
}
|
||||
workspaceMap.set(workspace, entry);
|
||||
}
|
||||
|
||||
const activeProcesses = (await this.processManager.list()).filter((process) =>
|
||||
isRuntimeProcessActive(process.status)
|
||||
);
|
||||
const referencedWorkspaces = new Set<string>();
|
||||
for (const [workspace, entry] of workspaceMap.entries()) {
|
||||
const profileProcessNames = new Set(
|
||||
entry.profileNames.flatMap((profileName) => [
|
||||
buildProcessName(profileName, 'frontend'),
|
||||
buildProcessName(profileName, 'api'),
|
||||
buildProcessName(profileName, 'daemon'),
|
||||
buildProcessName(profileName, 'auction'),
|
||||
buildProcessName(profileName, 'battle-sim'),
|
||||
buildProcessName(profileName, 'tournament'),
|
||||
])
|
||||
const activeProcesses = (await this.processManager.list()).filter((process) =>
|
||||
isRuntimeProcessActive(process.status)
|
||||
);
|
||||
if (
|
||||
activeProcesses.some(
|
||||
(process) =>
|
||||
profileProcessNames.has(process.name) ||
|
||||
isPathInside(process.cwd, workspace) ||
|
||||
isPathInside(process.script, workspace)
|
||||
)
|
||||
) {
|
||||
referencedWorkspaces.add(workspace);
|
||||
for (const workspace of managedWorkspaces) {
|
||||
if (
|
||||
activeProcesses.some(
|
||||
(process) =>
|
||||
isPathInside(process.cwd, workspace.root) || isPathInside(process.script, workspace.root)
|
||||
)
|
||||
) {
|
||||
protectedWorkspaces.add(workspace.root);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const removed: string[] = [];
|
||||
const skipped: string[] = [];
|
||||
for (const [workspace, entry] of workspaceMap.entries()) {
|
||||
if (!entry.lastUsedAt || entry.hasActiveBuild || referencedWorkspaces.has(workspace)) {
|
||||
skipped.push(workspace);
|
||||
continue;
|
||||
}
|
||||
if (entry.lastUsedAt > cutoff) {
|
||||
skipped.push(workspace);
|
||||
continue;
|
||||
}
|
||||
await this.workspaceManager.remove(workspace);
|
||||
await this.repository.clearWorkspaceUsage(entry.profileNames);
|
||||
removed.push(workspace);
|
||||
return await this.workspaceManager.cleanup({
|
||||
protectedPaths: [...protectedWorkspaces],
|
||||
retentionMs: DEFAULT_MANAGED_WORKSPACE_RETENTION_MS,
|
||||
keepNewest: DEFAULT_MANAGED_WORKSPACE_KEEP_NEWEST,
|
||||
});
|
||||
} finally {
|
||||
this.workspaceCleanupInFlight = false;
|
||||
}
|
||||
|
||||
return { removed, skipped };
|
||||
}
|
||||
|
||||
private computeCutoffDate(months: number): Date {
|
||||
const date = this.now();
|
||||
const cutoff = new Date(date);
|
||||
cutoff.setMonth(cutoff.getMonth() - months);
|
||||
return cutoff;
|
||||
private async cleanupWorkspacesScheduled(): Promise<void> {
|
||||
if (this.stopping || this.buildInFlight || this.operationInFlight || this.workspaceCleanupInFlight) return;
|
||||
const result = await this.cleanupStaleWorkspaces();
|
||||
if (result.removed.length > 0) {
|
||||
console.info(`[gateway-orchestrator] removed ${result.removed.length} stale profile worktrees`);
|
||||
}
|
||||
}
|
||||
|
||||
private async startProfile(profile: GatewayProfileRecord, assertLease?: () => Promise<void>): Promise<boolean> {
|
||||
|
||||
@@ -6,6 +6,7 @@ export interface WorkspaceManagerOptions {
|
||||
repoRoot: string;
|
||||
worktreeRoot: string;
|
||||
baseEnv?: Record<string, string>;
|
||||
now?: () => Date;
|
||||
}
|
||||
|
||||
export interface WorkspaceInfo {
|
||||
@@ -14,6 +15,26 @@ export interface WorkspaceInfo {
|
||||
needsInstall: boolean;
|
||||
}
|
||||
|
||||
export interface ManagedWorkspaceInfo {
|
||||
root: string;
|
||||
commitSha: string;
|
||||
lastUsedAt: Date;
|
||||
}
|
||||
|
||||
export interface ManagedWorkspaceCleanupOptions {
|
||||
protectedPaths?: readonly string[];
|
||||
retentionMs: number;
|
||||
keepNewest: number;
|
||||
}
|
||||
|
||||
export interface ManagedWorkspaceCleanupResult {
|
||||
removed: string[];
|
||||
skipped: string[];
|
||||
}
|
||||
|
||||
export const DEFAULT_MANAGED_WORKSPACE_RETENTION_MS = 24 * 60 * 60 * 1_000;
|
||||
export const DEFAULT_MANAGED_WORKSPACE_KEEP_NEWEST = 2;
|
||||
|
||||
const runGit = (args: string[], cwd: string, env?: Record<string, string>): Promise<{ ok: boolean; output: string }> =>
|
||||
new Promise((resolve) => {
|
||||
const child = spawn('git', args, {
|
||||
@@ -58,11 +79,13 @@ export class GitWorkspaceManager {
|
||||
private readonly repoRoot: string;
|
||||
private readonly worktreeRoot: string;
|
||||
private readonly baseEnv?: Record<string, string>;
|
||||
private readonly now: () => Date;
|
||||
|
||||
constructor(options: WorkspaceManagerOptions) {
|
||||
this.repoRoot = options.repoRoot;
|
||||
this.worktreeRoot = options.worktreeRoot;
|
||||
this.baseEnv = options.baseEnv;
|
||||
this.now = options.now ?? (() => new Date());
|
||||
}
|
||||
|
||||
async resolveCommit(sourceMode: 'BRANCH' | 'COMMIT', sourceRef: string): Promise<string> {
|
||||
@@ -124,6 +147,8 @@ export class GitWorkspaceManager {
|
||||
} else {
|
||||
await this.assertReusableWorkspace(workspacePath, commitSha);
|
||||
}
|
||||
const usedAt = this.now();
|
||||
fs.utimesSync(workspacePath, usedAt, usedAt);
|
||||
|
||||
return {
|
||||
root: workspacePath,
|
||||
@@ -138,13 +163,100 @@ export class GitWorkspaceManager {
|
||||
return false;
|
||||
}
|
||||
await this.assertRegisteredWorkspace(resolved);
|
||||
const status = await runGit(['status', '--porcelain'], resolved, this.baseEnv);
|
||||
if (!status.ok) {
|
||||
throw new Error(status.output || 'Failed to inspect managed workspace.');
|
||||
}
|
||||
if (status.output.trim()) {
|
||||
throw new Error('Managed workspace has uncommitted changes.');
|
||||
}
|
||||
const result = await runGit(['worktree', 'remove', '--force', resolved], this.repoRoot, this.baseEnv);
|
||||
if (!result.ok) {
|
||||
fs.rmSync(resolved, { recursive: true, force: true });
|
||||
throw new Error(result.output || 'Failed to remove git worktree.');
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
workspacePathForCommit(commitSha: string): string {
|
||||
if (!COMMIT_SHA_PATTERN.test(commitSha)) {
|
||||
throw new Error('Invalid commit SHA.');
|
||||
}
|
||||
return path.join(this.worktreeRoot, commitSha);
|
||||
}
|
||||
|
||||
async listManagedWorkspaces(): Promise<ManagedWorkspaceInfo[]> {
|
||||
const listed = await runGit(['worktree', 'list', '--porcelain'], this.repoRoot, this.baseEnv);
|
||||
if (!listed.ok) {
|
||||
throw new Error(listed.output || 'Failed to inspect git worktrees.');
|
||||
}
|
||||
const workspaces: ManagedWorkspaceInfo[] = [];
|
||||
for (const block of listed.output.split(/\n\n+/)) {
|
||||
const lines = block.split('\n');
|
||||
const worktreeLine = lines.find((line) => line.startsWith('worktree '));
|
||||
const headLine = lines.find((line) => line.startsWith('HEAD '));
|
||||
if (!worktreeLine || !headLine) continue;
|
||||
const workspacePath = path.resolve(worktreeLine.slice('worktree '.length));
|
||||
const commitSha = headLine.slice('HEAD '.length);
|
||||
try {
|
||||
this.assertManagedWorkspacePath(workspacePath);
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
if (!COMMIT_SHA_PATTERN.test(commitSha) || !fs.existsSync(workspacePath)) continue;
|
||||
workspaces.push({
|
||||
root: workspacePath,
|
||||
commitSha,
|
||||
lastUsedAt: fs.statSync(workspacePath).mtime,
|
||||
});
|
||||
}
|
||||
return workspaces;
|
||||
}
|
||||
|
||||
async cleanup(options: ManagedWorkspaceCleanupOptions): Promise<ManagedWorkspaceCleanupResult> {
|
||||
if (!Number.isFinite(options.retentionMs) || options.retentionMs < 0) {
|
||||
throw new Error('Workspace retention must be a non-negative duration.');
|
||||
}
|
||||
if (!Number.isInteger(options.keepNewest) || options.keepNewest < 0) {
|
||||
throw new Error('Workspace keepNewest must be a non-negative integer.');
|
||||
}
|
||||
const protectedPaths = new Set((options.protectedPaths ?? []).map((item) => path.resolve(item)));
|
||||
const workspaces = await this.listManagedWorkspaces();
|
||||
const unprotectedNewest = [...workspaces]
|
||||
.filter((workspace) => !protectedPaths.has(workspace.root))
|
||||
.sort((left, right) => right.lastUsedAt.getTime() - left.lastUsedAt.getTime())
|
||||
.slice(0, options.keepNewest);
|
||||
const retainedNewestPaths = new Set(unprotectedNewest.map((workspace) => workspace.root));
|
||||
const cutoff = this.now().getTime() - options.retentionMs;
|
||||
const removed: string[] = [];
|
||||
const skipped: string[] = [];
|
||||
|
||||
for (const workspace of workspaces) {
|
||||
if (
|
||||
protectedPaths.has(workspace.root) ||
|
||||
retainedNewestPaths.has(workspace.root) ||
|
||||
workspace.lastUsedAt.getTime() > cutoff
|
||||
) {
|
||||
skipped.push(workspace.root);
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
if (await this.remove(workspace.root)) {
|
||||
removed.push(workspace.root);
|
||||
} else {
|
||||
skipped.push(workspace.root);
|
||||
}
|
||||
} catch {
|
||||
skipped.push(workspace.root);
|
||||
}
|
||||
}
|
||||
|
||||
const pruned = await runGit(['worktree', 'prune', '--expire', 'now'], this.repoRoot, this.baseEnv);
|
||||
if (!pruned.ok) {
|
||||
throw new Error(pruned.output || 'Failed to prune git worktree metadata.');
|
||||
}
|
||||
return { removed, skipped };
|
||||
}
|
||||
|
||||
private assertManagedWorkspacePath(workspacePath: string): string {
|
||||
const resolved = path.resolve(workspacePath);
|
||||
const root = path.resolve(this.worktreeRoot);
|
||||
|
||||
@@ -1,15 +1,23 @@
|
||||
import path from 'node:path';
|
||||
|
||||
import { describe, expect, it } from 'vitest';
|
||||
|
||||
import { GatewayOrchestrator } from '../src/orchestrator/gatewayOrchestrator.js';
|
||||
import type { ProcessManager } from '../src/orchestrator/processManager.js';
|
||||
import type { GatewayProfileRecord, GatewayProfileRepository } from '../src/orchestrator/profileRepository.js';
|
||||
import type { GitWorkspaceManager } from '../src/orchestrator/workspaceManager.js';
|
||||
import {
|
||||
DEFAULT_MANAGED_WORKSPACE_KEEP_NEWEST,
|
||||
DEFAULT_MANAGED_WORKSPACE_RETENTION_MS,
|
||||
type GitWorkspaceManager,
|
||||
type ManagedWorkspaceCleanupOptions,
|
||||
} from '../src/orchestrator/workspaceManager.js';
|
||||
|
||||
const COMMIT_SHA = '0123456789abcdef0123456789abcdef01234567';
|
||||
const oldUsage = '2025-01-01T00:00:00.000Z';
|
||||
|
||||
const makeProfile = (
|
||||
profileName: string,
|
||||
workspace: string,
|
||||
workspace: string | undefined,
|
||||
overrides: Partial<GatewayProfileRecord> = {}
|
||||
): GatewayProfileRecord => ({
|
||||
profileName,
|
||||
@@ -20,7 +28,7 @@ const makeProfile = (
|
||||
apiPort: 15_003,
|
||||
status: 'RUNNING',
|
||||
buildStatus: 'SUCCEEDED',
|
||||
buildCommitSha: '0123456789abcdef0123456789abcdef01234567',
|
||||
buildCommitSha: COMMIT_SHA,
|
||||
buildWorkspace: workspace,
|
||||
buildLastUsedAt: oldUsage,
|
||||
meta: {},
|
||||
@@ -32,17 +40,10 @@ const makeProfile = (
|
||||
const createHarness = (
|
||||
profiles: GatewayProfileRecord[],
|
||||
processes: Awaited<ReturnType<ProcessManager['list']>>,
|
||||
workspaceExists = true
|
||||
managedPaths: string[]
|
||||
) => {
|
||||
const removeCalls: string[] = [];
|
||||
const clearedProfiles: string[][] = [];
|
||||
|
||||
const repository = {
|
||||
listProfiles: async () => profiles,
|
||||
clearWorkspaceUsage: async (profileNames: string[]) => {
|
||||
clearedProfiles.push(profileNames);
|
||||
},
|
||||
} as unknown as GatewayProfileRepository;
|
||||
const cleanupCalls: ManagedWorkspaceCleanupOptions[] = [];
|
||||
const repository = { listProfiles: async () => profiles } as unknown as GatewayProfileRepository;
|
||||
const processManager: ProcessManager = {
|
||||
list: async () => processes,
|
||||
start: async () => {},
|
||||
@@ -50,9 +51,16 @@ const createHarness = (
|
||||
delete: async () => {},
|
||||
};
|
||||
const workspaceManager = {
|
||||
remove: async (workspace: string) => {
|
||||
removeCalls.push(workspace);
|
||||
return workspaceExists;
|
||||
listManagedWorkspaces: async () =>
|
||||
managedPaths.map((root) => ({ root, commitSha: path.basename(root), lastUsedAt: new Date(oldUsage) })),
|
||||
workspacePathForCommit: (commitSha: string) => `/srv/sammo/worktrees/${commitSha}`,
|
||||
cleanup: async (options: ManagedWorkspaceCleanupOptions) => {
|
||||
cleanupCalls.push(options);
|
||||
const protectedPaths = new Set(options.protectedPaths);
|
||||
return {
|
||||
removed: managedPaths.filter((workspace) => !protectedPaths.has(workspace)),
|
||||
skipped: managedPaths.filter((workspace) => protectedPaths.has(workspace)),
|
||||
};
|
||||
},
|
||||
} as unknown as GitWorkspaceManager;
|
||||
const orchestrator = new GatewayOrchestrator({
|
||||
@@ -70,126 +78,67 @@ const createHarness = (
|
||||
scheduleIntervalMs: 60_000,
|
||||
buildIntervalMs: 60_000,
|
||||
adminActionIntervalMs: 60_000,
|
||||
now: () => new Date('2026-07-30T00:00:00.000Z'),
|
||||
});
|
||||
|
||||
return { orchestrator, removeCalls, clearedProfiles };
|
||||
return { orchestrator, cleanupCalls };
|
||||
};
|
||||
|
||||
describe('GatewayOrchestrator workspace cleanup', () => {
|
||||
it('skips a workspace referenced by any active process cwd', async () => {
|
||||
const workspace = '/srv/sammo/worktrees/active';
|
||||
it('always protects every workspace currently selected by a profile', async () => {
|
||||
const current = '/srv/sammo/worktrees/current';
|
||||
const stale = '/srv/sammo/worktrees/stale';
|
||||
const harness = createHarness([makeProfile('che:default', current)], [], [current, stale]);
|
||||
|
||||
await expect(harness.orchestrator.cleanupStaleWorkspaces()).resolves.toEqual({
|
||||
removed: [stale],
|
||||
skipped: [current],
|
||||
});
|
||||
expect(harness.cleanupCalls[0]).toMatchObject({
|
||||
retentionMs: DEFAULT_MANAGED_WORKSPACE_RETENTION_MS,
|
||||
keepNewest: DEFAULT_MANAGED_WORKSPACE_KEEP_NEWEST,
|
||||
});
|
||||
});
|
||||
|
||||
it('protects the commit target of queued and running builds before the profile reference changes', async () => {
|
||||
const target = `/srv/sammo/worktrees/${COMMIT_SHA}`;
|
||||
const harness = createHarness([makeProfile('che:default', undefined, { buildStatus: 'QUEUED' })], [], [target]);
|
||||
|
||||
await expect(harness.orchestrator.cleanupStaleWorkspaces()).resolves.toEqual({
|
||||
removed: [],
|
||||
skipped: [target],
|
||||
});
|
||||
});
|
||||
|
||||
it('protects an otherwise orphaned workspace referenced by any active process cwd or script', async () => {
|
||||
const cwdWorkspace = '/srv/sammo/worktrees/cwd-orphan';
|
||||
const scriptWorkspace = '/srv/sammo/worktrees/script-orphan';
|
||||
const stale = '/srv/sammo/worktrees/stale';
|
||||
const harness = createHarness(
|
||||
[makeProfile('che:default', workspace)],
|
||||
[],
|
||||
[
|
||||
{
|
||||
name: 'sammo:che:default:frontend',
|
||||
status: 'online',
|
||||
cwd: `${workspace}/app/game-frontend`,
|
||||
},
|
||||
]
|
||||
{ name: 'custom-build', status: 'online', cwd: `${cwdWorkspace}/app/game-api` },
|
||||
{ name: 'custom-worker', status: 'launching', script: `${scriptWorkspace}/dist/index.js` },
|
||||
{ name: 'stopped-worker', status: 'stopped', cwd: `${stale}/app/game-api` },
|
||||
],
|
||||
[cwdWorkspace, scriptWorkspace, stale]
|
||||
);
|
||||
|
||||
await expect(harness.orchestrator.cleanupStaleWorkspaces()).resolves.toEqual({
|
||||
removed: [],
|
||||
skipped: [workspace],
|
||||
removed: [stale],
|
||||
skipped: [cwdWorkspace, scriptWorkspace],
|
||||
});
|
||||
expect(harness.removeCalls).toEqual([]);
|
||||
expect(harness.clearedProfiles).toEqual([]);
|
||||
});
|
||||
|
||||
it('skips a workspace when only one profile process is active and cwd metadata is absent', async () => {
|
||||
const workspace = '/srv/sammo/worktrees/partial';
|
||||
const harness = createHarness(
|
||||
[makeProfile('che:default', workspace)],
|
||||
[{ name: 'sammo:che:default:tournament-worker', status: 'launching' }]
|
||||
);
|
||||
|
||||
await expect(harness.orchestrator.cleanupStaleWorkspaces()).resolves.toEqual({
|
||||
removed: [],
|
||||
skipped: [workspace],
|
||||
});
|
||||
expect(harness.removeCalls).toEqual([]);
|
||||
});
|
||||
|
||||
it('skips a workspace referenced only by an active process script', async () => {
|
||||
const workspace = '/srv/sammo/worktrees/script-reference';
|
||||
const harness = createHarness(
|
||||
[makeProfile('che:default', workspace)],
|
||||
[
|
||||
{
|
||||
name: 'unregistered-worker-name',
|
||||
status: 'online',
|
||||
script: `${workspace}/app/game-api/dist/index.js`,
|
||||
},
|
||||
]
|
||||
);
|
||||
|
||||
await expect(harness.orchestrator.cleanupStaleWorkspaces()).resolves.toEqual({
|
||||
removed: [],
|
||||
skipped: [workspace],
|
||||
});
|
||||
expect(harness.removeCalls).toEqual([]);
|
||||
});
|
||||
|
||||
it('protects a shared workspace when a process for either profile is active', async () => {
|
||||
const workspace = '/srv/sammo/worktrees/shared';
|
||||
const harness = createHarness(
|
||||
[makeProfile('che:default', workspace), makeProfile('hwe:default', workspace)],
|
||||
[{ name: 'sammo:hwe:default:game-api', status: 'stopping' }]
|
||||
);
|
||||
|
||||
await expect(harness.orchestrator.cleanupStaleWorkspaces()).resolves.toEqual({
|
||||
removed: [],
|
||||
skipped: [workspace],
|
||||
});
|
||||
expect(harness.removeCalls).toEqual([]);
|
||||
});
|
||||
|
||||
it('removes an old unreferenced workspace and clears every profile reference', async () => {
|
||||
const workspace = '/srv/sammo/worktrees/stale';
|
||||
const harness = createHarness(
|
||||
[makeProfile('che:default', workspace), makeProfile('hwe:default', workspace)],
|
||||
[{ name: 'sammo:che:default:game-api', status: 'stopped', cwd: `${workspace}/app/game-api` }]
|
||||
);
|
||||
|
||||
await expect(harness.orchestrator.cleanupStaleWorkspaces()).resolves.toEqual({
|
||||
removed: [workspace],
|
||||
skipped: [],
|
||||
});
|
||||
expect(harness.removeCalls).toEqual([workspace]);
|
||||
expect(harness.clearedProfiles).toEqual([['che:default', 'hwe:default']]);
|
||||
});
|
||||
|
||||
it('does not treat a sibling path with the same prefix as a workspace reference', async () => {
|
||||
it('does not confuse sibling path prefixes with an active workspace reference', async () => {
|
||||
const workspace = '/srv/sammo/worktrees/commit-a';
|
||||
const harness = createHarness(
|
||||
[makeProfile('che:default', workspace)],
|
||||
[
|
||||
{
|
||||
name: 'unregistered-worker-name',
|
||||
status: 'online',
|
||||
cwd: '/srv/sammo/worktrees/commit-a-old/app/game-api',
|
||||
},
|
||||
]
|
||||
[],
|
||||
[{ name: 'custom-worker', status: 'online', cwd: `${workspace}-old/app/game-api` }],
|
||||
[workspace]
|
||||
);
|
||||
|
||||
await expect(harness.orchestrator.cleanupStaleWorkspaces()).resolves.toEqual({
|
||||
removed: [workspace],
|
||||
skipped: [],
|
||||
});
|
||||
expect(harness.removeCalls).toEqual([workspace]);
|
||||
});
|
||||
|
||||
it('clears a stale database reference when the workspace is already missing', async () => {
|
||||
const workspace = '/srv/sammo/worktrees/missing';
|
||||
const harness = createHarness([makeProfile('che:default', workspace)], [], false);
|
||||
|
||||
await expect(harness.orchestrator.cleanupStaleWorkspaces()).resolves.toEqual({
|
||||
removed: [workspace],
|
||||
skipped: [],
|
||||
});
|
||||
expect(harness.removeCalls).toEqual([workspace]);
|
||||
expect(harness.clearedProfiles).toEqual([['che:default']]);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -163,4 +163,46 @@ describe('GitWorkspaceManager source resolution', () => {
|
||||
);
|
||||
expect(fs.existsSync(unregistered)).toBe(true);
|
||||
});
|
||||
|
||||
it('cleans only expired unprotected worktrees beyond the newest cache and preserves dirty work', async () => {
|
||||
const fixture = createRepositoryFixture();
|
||||
const now = new Date('2026-08-20T12:00:00.000Z');
|
||||
const manager = new GitWorkspaceManager({
|
||||
repoRoot: fixture.checkout,
|
||||
worktreeRoot: fixture.worktrees,
|
||||
now: () => now,
|
||||
});
|
||||
const workspaces = [await manager.prepare(fixture.firstCommit)];
|
||||
for (let index = 2; index <= 5; index += 1) {
|
||||
fs.writeFileSync(path.join(fixture.source, 'version.txt'), `version ${index}\n`);
|
||||
git(fixture.source, 'add', 'version.txt');
|
||||
git(fixture.source, 'commit', '-m', `version ${index}`);
|
||||
git(fixture.source, 'push', 'origin', 'main');
|
||||
const commit = await manager.resolveCommit('BRANCH', 'main');
|
||||
workspaces.push(await manager.prepare(commit));
|
||||
}
|
||||
const expired = new Date('2026-08-01T00:00:00.000Z');
|
||||
for (const workspace of workspaces) fs.utimesSync(workspace.root, expired, expired);
|
||||
fs.writeFileSync(path.join(workspaces[1]!.root, 'preserve-me.txt'), 'uncommitted\n');
|
||||
fs.utimesSync(workspaces[1]!.root, expired, expired);
|
||||
const recent = new Date('2026-08-20T11:00:00.000Z');
|
||||
fs.utimesSync(workspaces[4]!.root, recent, recent);
|
||||
|
||||
const result = await manager.cleanup({
|
||||
protectedPaths: [workspaces[0]!.root],
|
||||
retentionMs: 24 * 60 * 60 * 1_000,
|
||||
keepNewest: 1,
|
||||
});
|
||||
expect(result.removed).toHaveLength(2);
|
||||
expect(result.removed).toEqual(expect.arrayContaining([workspaces[2]!.root, workspaces[3]!.root]));
|
||||
expect(result.skipped).toHaveLength(3);
|
||||
expect(result.skipped).toEqual(
|
||||
expect.arrayContaining([workspaces[0]!.root, workspaces[1]!.root, workspaces[4]!.root])
|
||||
);
|
||||
expect(fs.existsSync(workspaces[0]!.root)).toBe(true);
|
||||
expect(fs.existsSync(workspaces[1]!.root)).toBe(true);
|
||||
expect(fs.existsSync(workspaces[2]!.root)).toBe(false);
|
||||
expect(fs.existsSync(workspaces[3]!.root)).toBe(false);
|
||||
expect(fs.existsSync(workspaces[4]!.root)).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -70,6 +70,13 @@ pnpm --filter @sammo-ts/release-controller status
|
||||
pnpm --filter @sammo-ts/release-controller run-once
|
||||
```
|
||||
|
||||
Daemon은 시작 시와 이후 24시간마다 commit worktree를 자동 정리합니다. 현재·이전
|
||||
Gateway release와 활성 PM2 process가 사용하는 경로는 항상 보호하고, 나머지는
|
||||
마지막 사용 후 24시간과 최신 2개 cache를 보장한 뒤 제거합니다. 변경이 있거나 Git
|
||||
제거가 실패한 worktree는 raw directory 삭제로 우회하지 않고 다음 주기까지
|
||||
보존합니다. Profile worktree는 Gateway orchestrator가 같은 정책으로 별도
|
||||
관리합니다.
|
||||
|
||||
## Controller self-upgrade
|
||||
|
||||
이 명령은 현재 daemon과 별개의 CLI process에서 실행됩니다. 대상 worktree를
|
||||
|
||||
@@ -7,7 +7,7 @@ import {
|
||||
} from '@sammo-ts/gateway-api';
|
||||
|
||||
import { resolveReleaseControllerConfig } from './config.js';
|
||||
import { GatewayReleaseController } from './releaseController.js';
|
||||
import { GatewayReleaseController, RELEASE_WORKSPACE_CLEANUP_INTERVAL_MS } from './releaseController.js';
|
||||
import { upgradeReleaseController } from './selfUpgrade.js';
|
||||
|
||||
export * from './config.js';
|
||||
@@ -67,6 +67,7 @@ const main = async (): Promise<void> => {
|
||||
}
|
||||
if (command !== 'daemon') throw new Error(`Unknown release-controller command: ${command}`);
|
||||
let stopping = false;
|
||||
let nextWorkspaceCleanupAt = 0;
|
||||
const stop = async (): Promise<void> => {
|
||||
if (stopping) return;
|
||||
stopping = true;
|
||||
@@ -75,6 +76,18 @@ const main = async (): Promise<void> => {
|
||||
process.once('SIGINT', () => void stop());
|
||||
process.once('SIGTERM', () => void stop());
|
||||
while (!stopping) {
|
||||
const now = Date.now();
|
||||
if (now >= nextWorkspaceCleanupAt) {
|
||||
nextWorkspaceCleanupAt = now + RELEASE_WORKSPACE_CLEANUP_INTERVAL_MS;
|
||||
try {
|
||||
const result = await controller.cleanupStaleWorkspaces();
|
||||
if (result.removed.length > 0) {
|
||||
console.info(`[release-controller] removed ${result.removed.length} stale Gateway worktrees`);
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('[release-controller] workspace cleanup failed', error);
|
||||
}
|
||||
}
|
||||
await controller.runOnce();
|
||||
await new Promise<void>((resolve) => setTimeout(resolve, config.pollIntervalMs));
|
||||
}
|
||||
|
||||
@@ -6,6 +6,8 @@ import {
|
||||
assertReleaseComponents,
|
||||
buildTurboReleaseCommand,
|
||||
buildTurboReleaseTaskCommand,
|
||||
DEFAULT_MANAGED_WORKSPACE_KEEP_NEWEST,
|
||||
DEFAULT_MANAGED_WORKSPACE_RETENTION_MS,
|
||||
type BuildCommand,
|
||||
type BuildProgressEvent,
|
||||
type BuildRunner,
|
||||
@@ -27,6 +29,16 @@ const HEARTBEAT_INTERVAL_MS = 60_000;
|
||||
const CANCELLATION_POLL_INTERVAL_MS = 500;
|
||||
const PROCESS_NAMES = ['sammo:gateway-api', 'sammo:gateway-frontend', 'sammo:gateway-orchestrator'] as const;
|
||||
const SENSITIVE_ENV_NAME = /(SECRET|TOKEN|PASSWORD|PASSWD|PRIVATE_KEY|CLIENT_SECRET|DATABASE_URL|REDIS_URL)/iu;
|
||||
export const RELEASE_WORKSPACE_CLEANUP_INTERVAL_MS = 24 * 60 * 60 * 1_000;
|
||||
|
||||
const isRuntimeProcessActive = (status: string): boolean =>
|
||||
['online', 'launching', 'stopping'].includes(status.toLowerCase());
|
||||
|
||||
const isPathInside = (candidate: string | undefined, root: string): boolean => {
|
||||
if (!candidate) return false;
|
||||
const relative = path.relative(path.resolve(root), path.resolve(candidate));
|
||||
return relative === '' || (!relative.startsWith(`..${path.sep}`) && !path.isAbsolute(relative));
|
||||
};
|
||||
|
||||
const managedPostgresPoolMax = (env: Record<string, string>, roleVariable: string, fallback: number): string =>
|
||||
String(resolvePostgresPoolMax(env[roleVariable] ?? env.POSTGRES_POOL_MAX, fallback));
|
||||
@@ -132,6 +144,33 @@ export class GatewayReleaseController {
|
||||
private readonly fetchImpl: typeof fetch = fetch
|
||||
) {}
|
||||
|
||||
async cleanupStaleWorkspaces(): Promise<{ removed: string[]; skipped: string[] }> {
|
||||
const [state, processes, workspaces] = await Promise.all([
|
||||
this.repository.getState(),
|
||||
this.processManager.list(),
|
||||
this.workspaceManager.listManagedWorkspaces(),
|
||||
]);
|
||||
const protectedWorkspaces = new Set<string>();
|
||||
if (state.activeWorkspace) protectedWorkspaces.add(path.resolve(state.activeWorkspace));
|
||||
if (state.previousWorkspace) protectedWorkspaces.add(path.resolve(state.previousWorkspace));
|
||||
const activeProcesses = processes.filter((process) => isRuntimeProcessActive(process.status));
|
||||
for (const workspace of workspaces) {
|
||||
if (
|
||||
activeProcesses.some(
|
||||
(process) =>
|
||||
isPathInside(process.cwd, workspace.root) || isPathInside(process.script, workspace.root)
|
||||
)
|
||||
) {
|
||||
protectedWorkspaces.add(workspace.root);
|
||||
}
|
||||
}
|
||||
return this.workspaceManager.cleanup({
|
||||
protectedPaths: [...protectedWorkspaces],
|
||||
retentionMs: DEFAULT_MANAGED_WORKSPACE_RETENTION_MS,
|
||||
keepNewest: DEFAULT_MANAGED_WORKSPACE_KEEP_NEWEST,
|
||||
});
|
||||
}
|
||||
|
||||
private sanitizeLogMessage(message: string): string {
|
||||
let sanitized = stripVTControlCharacters(message);
|
||||
const sensitiveValues = new Set([
|
||||
|
||||
@@ -2,14 +2,17 @@ import fs from 'node:fs/promises';
|
||||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
|
||||
import type {
|
||||
BuildRunner,
|
||||
GatewayReleaseOperationRecord,
|
||||
GatewayReleaseRepository,
|
||||
GatewayReleaseStateRecord,
|
||||
GitWorkspaceManager,
|
||||
ProcessDefinition,
|
||||
ProcessManager,
|
||||
import {
|
||||
DEFAULT_MANAGED_WORKSPACE_KEEP_NEWEST,
|
||||
DEFAULT_MANAGED_WORKSPACE_RETENTION_MS,
|
||||
type BuildRunner,
|
||||
type GatewayReleaseOperationRecord,
|
||||
type GatewayReleaseRepository,
|
||||
type GatewayReleaseStateRecord,
|
||||
type GitWorkspaceManager,
|
||||
type ManagedWorkspaceCleanupOptions,
|
||||
type ProcessDefinition,
|
||||
type ProcessManager,
|
||||
} from '@sammo-ts/gateway-api';
|
||||
import { afterEach, describe, expect, it } from 'vitest';
|
||||
|
||||
@@ -178,6 +181,68 @@ it('rejects Gateway definitions before switching processes when Redis connection
|
||||
});
|
||||
|
||||
describe('GatewayReleaseController', () => {
|
||||
it('protects active, rollback, and running-process worktrees while delegating bounded cleanup', async () => {
|
||||
const active = '/srv/sammo/releases/active';
|
||||
const previous = '/srv/sammo/releases/previous';
|
||||
const controllerWorkspace = '/srv/sammo/releases/controller';
|
||||
const stale = '/srv/sammo/releases/stale';
|
||||
const managedPaths = [active, previous, controllerWorkspace, stale];
|
||||
const cleanupCalls: ManagedWorkspaceCleanupOptions[] = [];
|
||||
const harness = createRepository();
|
||||
const workspaceManager = {
|
||||
listManagedWorkspaces: async () =>
|
||||
managedPaths.map((root) => ({
|
||||
root,
|
||||
commitSha: SHA,
|
||||
lastUsedAt: new Date('2025-01-01T00:00:00.000Z'),
|
||||
})),
|
||||
cleanup: async (options: ManagedWorkspaceCleanupOptions) => {
|
||||
cleanupCalls.push(options);
|
||||
const protectedPaths = new Set(options.protectedPaths);
|
||||
return {
|
||||
removed: managedPaths.filter((workspace) => !protectedPaths.has(workspace)),
|
||||
skipped: managedPaths.filter((workspace) => protectedPaths.has(workspace)),
|
||||
};
|
||||
},
|
||||
} as unknown as GitWorkspaceManager;
|
||||
const controller = new GatewayReleaseController(
|
||||
{
|
||||
...harness.repository,
|
||||
getState: async () => ({
|
||||
...state,
|
||||
activeWorkspace: active,
|
||||
previousCommitSha: SHA,
|
||||
previousWorkspace: previous,
|
||||
}),
|
||||
},
|
||||
workspaceManager,
|
||||
{ run: async () => ({ ok: true, exitCode: 0, output: '' }) },
|
||||
{
|
||||
list: async () => [
|
||||
{
|
||||
name: 'sammo:release-controller',
|
||||
status: 'online',
|
||||
cwd: `${controllerWorkspace}/app/release-controller`,
|
||||
},
|
||||
{ name: 'old-build', status: 'stopped', cwd: `${stale}/app/gateway-api` },
|
||||
],
|
||||
start: async () => {},
|
||||
stop: async () => {},
|
||||
delete: async () => {},
|
||||
},
|
||||
config
|
||||
);
|
||||
|
||||
await expect(controller.cleanupStaleWorkspaces()).resolves.toEqual({
|
||||
removed: [stale],
|
||||
skipped: [active, previous, controllerWorkspace],
|
||||
});
|
||||
expect(cleanupCalls[0]).toMatchObject({
|
||||
retentionMs: DEFAULT_MANAGED_WORKSPACE_RETENTION_MS,
|
||||
keepNewest: DEFAULT_MANAGED_WORKSPACE_KEEP_NEWEST,
|
||||
});
|
||||
});
|
||||
|
||||
it('builds, migrates, switches all gateway roles, verifies readiness, and publishes atomically', async () => {
|
||||
const workspace = await createReleaseWorkspace();
|
||||
const harness = createRepository();
|
||||
|
||||
Reference in New Issue
Block a user