fix: 실행 중인 릴리스 빌드를 안전하게 중단

Gateway와 프로필 DEPLOY의 빌드 단계만 취소하고 lease와 phase 전환을 직렬화한다. 관리자 화면에 중단·재시도 절차와 회귀 검증을 추가한다.
2026-08-20 02:27:52 +00:00
parent 1bacdaef5f
commit 466f030889
11 changed files with 554 additions and 48 deletions
@@ -1453,7 +1453,7 @@ export const adminRouter = router({
            if (!cancelled) {
                throw new TRPCError({
                    code: 'CONFLICT',
-                    message: 'Only queued operations can be cancelled.',
+                    message: 'Only queued operations or a DEPLOY that is still building can be cancelled.',
                });
            }
            return { ok: true };
@@ -1626,7 +1626,10 @@ export const adminRouter = router({
            }),
        cancel: releaseAdminProcedure.input(z.object({ id: z.string().uuid() })).mutation(async ({ ctx, input }) => {
            if (!(await ctx.releases.cancelOperation(input.id))) {
-                throw new TRPCError({ code: 'CONFLICT', message: 'Only queued releases can be cancelled.' });
+                throw new TRPCError({
+                    code: 'CONFLICT',
+                    message: 'Only queued releases or a release that is still building can be cancelled.',
+                });
            }
            return { ok: true };
        }),
@@ -12,6 +12,7 @@ export interface BuildResult {
    ok: boolean;
    exitCode: number | null;
    output: string;
+    aborted?: boolean;
 }

 export type BuildProgressEvent =
@@ -21,8 +22,13 @@ export type BuildProgressEvent =

 export type BuildProgressObserver = (event: BuildProgressEvent) => void | Promise<void>;

+export interface BuildRunOptions {
+    signal?: AbortSignal;
+    terminateGraceMs?: number;
+}
+
 export interface BuildRunner {
-    run(commands: BuildCommand[], onProgress?: BuildProgressObserver): Promise<BuildResult>;
+    run(commands: BuildCommand[], onProgress?: BuildProgressObserver, options?: BuildRunOptions): Promise<BuildResult>;
 }

 export const MAX_BUILD_OUTPUT_CHARS = 64 * 1024;
@@ -77,7 +83,28 @@ export const buildTurboReleaseTaskCommand = (
 const appendOutputTail = (current: string, chunk: unknown): string =>
    `${current}${String(chunk)}`.slice(-MAX_BUILD_OUTPUT_CHARS);

-const runCommand = (command: BuildCommand, onProgress?: BuildProgressObserver): Promise<BuildResult> =>
+const terminateChildProcess = (pid: number | undefined, signal: NodeJS.Signals): void => {
+    if (!pid) return;
+    try {
+        if (process.platform !== 'win32') {
+            process.kill(-pid, signal);
+            return;
+        }
+    } catch {
+        // Fall back to the direct child below when the process group already exited.
+    }
+    try {
+        process.kill(pid, signal);
+    } catch {
+        // The child already exited.
+    }
+};
+
+const runCommand = (
+    command: BuildCommand,
+    onProgress?: BuildProgressObserver,
+    options?: BuildRunOptions
+): Promise<BuildResult> =>
    new Promise((resolve) => {
        let progressQueue = Promise.resolve();
        const emit = (event: BuildProgressEvent) => {
@@ -89,9 +116,25 @@ const runCommand = (command: BuildCommand, onProgress?: BuildProgressObserver):
            cwd: command.cwd,
            env: command.env,
            stdio: ['ignore', 'pipe', 'pipe'],
+            detached: process.platform !== 'win32',
        });
        let output = '';
        let spawnFailed = false;
+        let aborted = false;
+        let killTimer: ReturnType<typeof setTimeout> | undefined;
+        const abort = () => {
+            if (aborted) return;
+            aborted = true;
+            output = appendOutputTail(output, '\nBuild cancelled by operator.');
+            terminateChildProcess(child.pid, 'SIGTERM');
+            killTimer = setTimeout(
+                () => terminateChildProcess(child.pid, 'SIGKILL'),
+                options?.terminateGraceMs ?? 5_000
+            );
+            killTimer.unref?.();
+        };
+        options?.signal?.addEventListener('abort', abort, { once: true });
+        if (options?.signal?.aborted) abort();
        const lineBuffers = { stdout: '', stderr: '' };
        const emitOutput = (stream: 'stdout' | 'stderr', chunk: unknown, flush = false) => {
            if (flush && !lineBuffers[stream]) return;
@@ -119,31 +162,47 @@ const runCommand = (command: BuildCommand, onProgress?: BuildProgressObserver):
            output = appendOutputTail(output, error.message);
        });
        child.on('close', (code) => {
+            options?.signal?.removeEventListener('abort', abort);
+            if (killTimer) clearTimeout(killTimer);
            emitOutput('stdout', '', true);
            emitOutput('stderr', '', true);
            const exitCode = spawnFailed ? null : code;
            emit({ type: 'COMMAND_END', command, exitCode });
            void progressQueue.then(() => {
                resolve({
-                    ok: !spawnFailed && code === 0,
+                    ok: !aborted && !spawnFailed && code === 0,
                    exitCode,
                    output,
+                    ...(aborted ? { aborted: true } : {}),
                });
            });
        });
    });

 export class PnpmBuildRunner implements BuildRunner {
-    async run(commands: BuildCommand[], onProgress?: BuildProgressObserver): Promise<BuildResult> {
+    async run(
+        commands: BuildCommand[],
+        onProgress?: BuildProgressObserver,
+        options?: BuildRunOptions
+    ): Promise<BuildResult> {
        let mergedOutput = '';
        for (const command of commands) {
-            const result = await runCommand(command, onProgress);
+            if (options?.signal?.aborted) {
+                return {
+                    ok: false,
+                    exitCode: null,
+                    output: appendOutputTail(mergedOutput, 'Build cancelled by operator.'),
+                    aborted: true,
+                };
+            }
+            const result = await runCommand(command, onProgress, options);
            mergedOutput = appendOutputTail(mergedOutput, result.output);
            if (!result.ok) {
                return {
                    ok: false,
                    exitCode: result.exitCode,
                    output: mergedOutput,
+                    ...(result.aborted ? { aborted: true } : {}),
                };
            }
        }
@@ -198,6 +198,7 @@ interface GatewayAdminActionResult {

 const OPERATION_LEASE_DURATION_MS = 10 * 60_000;
 const OPERATION_HEARTBEAT_INTERVAL_MS = 60_000;
+const OPERATION_CANCELLATION_POLL_INTERVAL_MS = 500;

 class OperationLeaseLostError extends Error {}

@@ -633,6 +634,7 @@ export class GatewayOrchestrator implements GatewayOrchestratorHandle {
    private buildInFlight = false;
    private adminActionInFlight = false;
    private operationInFlight = false;
+    private activeOperationAbortSignal?: AbortSignal;
    private readonly resetInFlight = new Set<string>();
    private readonly operationLeaseOwner = randomUUID();
    private readonly inFlightTasks = new Set<Promise<unknown>>();
@@ -975,6 +977,8 @@ export class GatewayOrchestrator implements GatewayOrchestratorHandle {
            if (!operation) {
                return;
            }
+            const abortController = new AbortController();
+            this.activeOperationAbortSignal = abortController.signal;
            const heartbeatTimer = this.repository.renewOperationLease
                ? setInterval(() => {
                      void this.repository
@@ -984,17 +988,36 @@ export class GatewayOrchestrator implements GatewayOrchestratorHandle {
                              this.now(),
                              OPERATION_LEASE_DURATION_MS
                          )
+                          .then((renewed) => {
+                              if (!renewed) abortController.abort();
+                          })
                          .catch((error) => {
                              console.error('[gateway-orchestrator] operation heartbeat failed', error);
                          });
                  }, OPERATION_HEARTBEAT_INTERVAL_MS)
                : undefined;
+            const cancellationTimer = setInterval(() => {
+                void this.repository
+                    .getOperation(operation.id)
+                    .then((current) => {
+                        if (
+                            !current ||
+                            current.status !== 'RUNNING' ||
+                            current.leaseOwner !== this.operationLeaseOwner
+                        ) {
+                            abortController.abort();
+                        }
+                    })
+                    .catch(() => undefined);
+            }, OPERATION_CANCELLATION_POLL_INTERVAL_MS);
            try {
                await this.handleOperation(operation);
            } finally {
+                clearInterval(cancellationTimer);
                if (heartbeatTimer) {
                    clearInterval(heartbeatTimer);
                }
+                this.activeOperationAbortSignal = undefined;
            }
        } finally {
            this.operationInFlight = false;
@@ -1442,9 +1465,11 @@ export class GatewayOrchestrator implements GatewayOrchestratorHandle {
                ),
            ];
            await this.appendOperationLog(operationId, 'build', `${profile.profileName} 구성 요소를 빌드합니다.`);
-            const result = await this.buildRunner.run(commands, this.buildProgress(operationId, 'build'));
-            await assertLease();
+            const result = await this.buildRunner.run(commands, this.buildProgress(operationId, 'build'), {
+                signal: this.activeOperationAbortSignal,
+            });
            if (!result.ok) {
+                await assertLease();
                const detail = result.output.slice(-4000) || 'selected workspace build failed';
                await updateClaimedProfile({
                    buildStatus: 'FAILED',
@@ -1455,6 +1480,7 @@ export class GatewayOrchestrator implements GatewayOrchestratorHandle {
            }

            await this.appendOperationLog(operationId, 'switch', '기존 profile process를 정지합니다.');
+            await assertLease();
            await this.stopProfile(profile, assertLease);
            oldRuntimeStopped = true;
            const profileDatabaseUrl = this.resolveProfileDatabaseUrl(profile);
@@ -2030,7 +2056,8 @@ export class GatewayOrchestrator implements GatewayOrchestratorHandle {
        return {
            result: await this.buildRunner.run(
                commands,
-                operationId ? this.buildProgress(operationId, 'build') : undefined
+                operationId ? this.buildProgress(operationId, 'build') : undefined,
+                { signal: this.activeOperationAbortSignal }
            ),
            workspace,
        };
@@ -2052,7 +2079,8 @@ export class GatewayOrchestrator implements GatewayOrchestratorHandle {
    ): Promise<Awaited<ReturnType<BuildRunner['run']>>> {
        return this.buildRunner.run(
            [buildProfileMigrationCommand(workspaceRoot, profileDatabaseUrl, this.processConfig.baseEnv)],
-            onProgress
+            onProgress,
+            { signal: this.activeOperationAbortSignal }
        );
    }

@@ -2106,7 +2134,8 @@ export class GatewayOrchestrator implements GatewayOrchestratorHandle {
                        },
                    },
                ],
-                onProgress
+                onProgress,
+                { signal: this.activeOperationAbortSignal }
            );
        } finally {
            await fs.rm(tempDirectory, { recursive: true, force: true });
@@ -208,13 +208,18 @@ export const createGatewayReleaseRepository = (prisma: GatewayPrismaClient): Gat
        return rows.map(mapLog);
    },
    async appendOperationLog(id, input) {
-        const row = await prisma.gatewayReleaseLog.create({
-            data: {
-                operationId: id,
-                level: input.level,
-                phase: input.phase.slice(0, 64),
-                message: input.message.slice(0, 4_000),
-            },
+        const row = await prisma.$transaction(async (tx) => {
+            await tx.$queryRaw<Array<{ id: string }>>`
+                SELECT "id" FROM "gateway_release_operation" WHERE "id" = ${id} FOR UPDATE
+            `;
+            return tx.gatewayReleaseLog.create({
+                data: {
+                    operationId: id,
+                    level: input.level,
+                    phase: input.phase.slice(0, 64),
+                    message: input.message.slice(0, 4_000),
+                },
+            });
        });
        return mapLog(row);
    },
@@ -367,11 +372,48 @@ export const createGatewayReleaseRepository = (prisma: GatewayPrismaClient): Gat
        });
    },
    async cancelOperation(id) {
-        const updated = await prisma.gatewayReleaseOperation.updateMany({
-            where: { id, status: 'QUEUED' },
-            data: { status: 'CANCELLED', completedAt: new Date() },
+        const count = await prisma.$transaction(async (tx) => {
+            const rows = await tx.$queryRaw<Array<{ status: GatewayOperationStatus }>>`
+                SELECT "status"
+                FROM "gateway_release_operation"
+                WHERE "id" = ${id}
+                FOR UPDATE
+            `;
+            const operation = rows[0];
+            if (!operation || (operation.status !== 'QUEUED' && operation.status !== 'RUNNING')) return 0;
+            if (operation.status === 'RUNNING') {
+                const latestLog = await tx.gatewayReleaseLog.findFirst({
+                    where: { operationId: id },
+                    orderBy: { id: 'desc' },
+                    select: { phase: true },
+                });
+                if (!latestLog || !['claim', 'resolve', 'workspace', 'build'].includes(latestLog.phase)) return 0;
+            }
+            const updated = await tx.gatewayReleaseOperation.updateMany({
+                where: { id, status: operation.status },
+                data: {
+                    status: 'CANCELLED',
+                    completedAt: new Date(),
+                    leaseOwner: null,
+                    leaseUntil: null,
+                    heartbeatAt: null,
+                },
+            });
+            if (updated.count !== 1) return 0;
+            await tx.gatewayReleaseLog.create({
+                data: {
+                    operationId: id,
+                    level: 'INFO',
+                    phase: 'cancel',
+                    message:
+                        operation.status === 'RUNNING'
+                            ? '실행 중인 Gateway 빌드를 중단했습니다. 현재 active release는 유지됩니다.'
+                            : '대기 중인 Gateway 릴리스를 취소했습니다.',
+                },
+            });
+            return 1;
        });
-        return updated.count === 1;
+        return count === 1;
    },
    async retryOperation(id, requestedBy) {
        const row = await prisma.$transaction(async (tx) => {
@@ -586,13 +586,18 @@ export const createGatewayProfileRepository = (prisma: GatewayPrismaClient): Gat
        return rows.map(mapOperationLog);
    },
    async appendOperationLog(id, input) {
-        const row = await prisma.gatewayOperationLog.create({
-            data: {
-                operationId: id,
-                level: input.level,
-                phase: input.phase.slice(0, 64),
-                message: input.message.slice(0, 4_000),
-            },
+        const row = await prisma.$transaction(async (tx) => {
+            await tx.$queryRaw<Array<{ id: string }>>`
+                SELECT "id" FROM "gateway_operation" WHERE "id" = ${id} FOR UPDATE
+            `;
+            return tx.gatewayOperationLog.create({
+                data: {
+                    operationId: id,
+                    level: input.level,
+                    phase: input.phase.slice(0, 64),
+                    message: input.message.slice(0, 4_000),
+                },
+            });
        });
        return mapOperationLog(row);
    },
@@ -805,21 +810,61 @@ export const createGatewayProfileRepository = (prisma: GatewayPrismaClient): Gat
    },
    async cancelOperation(id: string): Promise<boolean> {
        const count = await prisma.$transaction(async (tx) => {
+            const rows = await tx.$queryRaw<Array<{ status: GatewayOperationStatus; type: GatewayOperationType }>>`
+                SELECT "status", "type"
+                FROM "gateway_operation"
+                WHERE "id" = ${id}
+                FOR UPDATE
+            `;
+            const operation = rows[0];
+            if (!operation || (operation.status !== 'QUEUED' && operation.status !== 'RUNNING')) return 0;
+
+            let runningBuildCancelled = false;
+            if (operation.status === 'RUNNING') {
+                if (operation.type !== 'DEPLOY') return 0;
+                const latestLog = await tx.gatewayOperationLog.findFirst({
+                    where: { operationId: id },
+                    orderBy: { id: 'desc' },
+                    select: { phase: true },
+                });
+                if (!latestLog || !['claim', 'resolve', 'workspace', 'build'].includes(latestLog.phase)) return 0;
+                runningBuildCancelled = true;
+            }
+
            const result = await tx.gatewayOperation.updateMany({
-                where: { id, status: 'QUEUED' },
-                data: { status: 'CANCELLED', completedAt: new Date() },
+                where: { id, status: operation.status },
+                data: {
+                    status: 'CANCELLED',
+                    completedAt: new Date(),
+                    leaseOwner: null,
+                    leaseUntil: null,
+                    heartbeatAt: null,
+                },
            });
-            if (result.count === 1) {
-                await tx.gatewayOperationLog.create({
+            if (result.count !== 1) return 0;
+            if (runningBuildCancelled) {
+                await tx.gatewayProfile.updateMany({
+                    where: { operations: { some: { id } } },
                    data: {
-                        operationId: id,
-                        level: 'INFO',
-                        phase: 'cancel',
-                        message: '대기 중인 작업이 취소되었습니다.',
+                        buildStatus: 'SUCCEEDED',
+                        buildRequestedAt: null,
+                        buildStartedAt: null,
+                        buildCompletedAt: null,
+                        buildError: null,
                    },
                });
            }
-            return result.count;
+            await tx.gatewayOperationLog.create({
+                data: {
+                    operationId: id,
+                    level: 'INFO',
+                    phase: 'cancel',
+                    message: runningBuildCancelled
+                        ? '실행 중인 빌드를 중단했습니다. 기존 runtime과 DB는 유지됩니다.'
+                        : '대기 중인 작업이 취소되었습니다.',
+                },
+            });
+            return 1;
        });
        return count === 1;
    },