fix: 실행 중인 릴리스 빌드를 안전하게 중단

Gateway와 프로필 DEPLOY의 빌드 단계만 취소하고 lease와 phase 전환을 직렬화한다. 관리자 화면에 중단·재시도 절차와 회귀 검증을 추가한다.
This commit is contained in:
2026-08-20 02:27:52 +00:00
parent 1bacdaef5f
commit 466f030889
11 changed files with 554 additions and 48 deletions
@@ -24,6 +24,7 @@ import type { ReleaseControllerConfig } from './config.js';
const LEASE_DURATION_MS = 10 * 60_000;
const HEARTBEAT_INTERVAL_MS = 60_000;
const CANCELLATION_POLL_INTERVAL_MS = 500;
const PROCESS_NAMES = ['sammo:gateway-api', 'sammo:gateway-frontend', 'sammo:gateway-orchestrator'] as const;
const SENSITIVE_ENV_NAME = /(SECRET|TOKEN|PASSWORD|PASSWD|PRIVATE_KEY|CLIENT_SECRET|DATABASE_URL|REDIS_URL)/iu;
@@ -187,9 +188,25 @@ export class GatewayReleaseController {
});
if (!operation) return null;
await this.appendLog(operation.id, 'claim', `릴리스 작업을 시작합니다. 시도 ${operation.attempts}회차.`);
const abortController = new AbortController();
const heartbeat = setInterval(() => {
void this.repository.renewOperationLease(operation.id, this.ownerId, this.now(), LEASE_DURATION_MS);
void this.repository
.renewOperationLease(operation.id, this.ownerId, this.now(), LEASE_DURATION_MS)
.then((renewed) => {
if (!renewed) abortController.abort();
})
.catch(() => undefined);
}, HEARTBEAT_INTERVAL_MS);
const cancellationWatcher = setInterval(() => {
void this.repository
.getOperation(operation.id)
.then((current) => {
if (!current || current.status !== 'RUNNING' || current.leaseOwner !== this.ownerId) {
abortController.abort();
}
})
.catch(() => undefined);
}, CANCELLATION_POLL_INTERVAL_MS);
let resolvedCommitSha: string | undefined;
try {
await this.appendLog(operation.id, 'resolve', '현재 Gateway 릴리스 상태를 확인합니다.');
@@ -208,7 +225,7 @@ export class GatewayReleaseController {
throw new Error('Gateway release lease was lost while pinning the commit.');
}
await this.appendLog(operation.id, 'resolve', `대상 커밋을 ${resolvedCommitSha}로 고정했습니다.`);
await this.deploy(operation, deploymentState, resolvedCommitSha);
await this.deploy(operation, deploymentState, resolvedCommitSha, abortController.signal);
await this.appendLog(operation.id, 'complete', 'Gateway 릴리스가 완료되었습니다.');
return await this.repository.completeOperation(
operation.id,
@@ -217,6 +234,17 @@ export class GatewayReleaseController {
this.ownerId
);
} catch (error) {
const current = await this.repository.getOperation(operation.id);
if (
!current ||
current.status !== 'RUNNING' ||
(abortController.signal.aborted && current.leaseOwner !== this.ownerId)
) {
if (current?.status === 'CANCELLED') {
await this.appendLog(operation.id, 'cancel', '실행 중인 Gateway 빌드가 종료되었습니다.');
}
return current;
}
const detail = error instanceof Error ? error.message : String(error);
await this.appendLog(operation.id, 'failed', detail, 'ERROR');
await this.repository.recordStateError(detail);
@@ -228,13 +256,21 @@ export class GatewayReleaseController {
);
} finally {
clearInterval(heartbeat);
clearInterval(cancellationWatcher);
}
}
private async assertOperationLease(operationId: string): Promise<void> {
if (!(await this.repository.renewOperationLease(operationId, this.ownerId, this.now(), LEASE_DURATION_MS))) {
throw new Error(`Gateway release lease lost: ${operationId}`);
}
}
private async deploy(
operation: GatewayReleaseOperationRecord,
state: GatewayReleaseStateRecord,
commitSha: string
commitSha: string,
signal: AbortSignal
): Promise<void> {
await this.appendLog(operation.id, 'workspace', `커밋 ${commitSha}의 worktree를 준비합니다.`);
const workspace = await this.workspaceManager.prepare(commitSha);
@@ -244,13 +280,16 @@ export class GatewayReleaseController {
await this.appendLog(operation.id, 'build', 'Gateway 구성 요소를 빌드합니다.');
const build = await this.buildRunner.run(
buildGatewayReleaseCommands(workspace.root, workspace.needsInstall, this.config),
this.buildProgress(operation.id, 'build')
this.buildProgress(operation.id, 'build'),
{ signal }
);
if (!build.ok) throw new Error(`Gateway release build failed: ${build.output.slice(-4000)}`);
await this.appendLog(operation.id, 'migration', 'Gateway database migration을 적용합니다.');
await this.assertOperationLease(operation.id);
const migration = await this.buildRunner.run(
[buildGatewayMigrationCommand(workspace.root, this.config)],
this.buildProgress(operation.id, 'migration')
this.buildProgress(operation.id, 'migration'),
{ signal }
);
if (!migration.ok) throw new Error(`Gateway migration failed: ${migration.output.slice(-4000)}`);
await this.appendLog(operation.id, 'migration', 'Gateway database migration이 완료되었습니다.');
@@ -259,6 +298,7 @@ export class GatewayReleaseController {
? buildGatewayProcessDefinitions(state.activeWorkspace, this.config)
: [];
await this.appendLog(operation.id, 'switch', '기존 Gateway process를 정지합니다.');
await this.assertOperationLease(operation.id);
await this.stopManagedProcesses(operation.id);
try {
await this.startDefinitions(buildGatewayProcessDefinitions(workspace.root, this.config), operation.id);