merge: 릴리스 빌드 안전 중단 통합
This commit is contained in:
@@ -1453,7 +1453,7 @@ export const adminRouter = router({
|
||||
if (!cancelled) {
|
||||
throw new TRPCError({
|
||||
code: 'CONFLICT',
|
||||
message: 'Only queued operations can be cancelled.',
|
||||
message: 'Only queued operations or a DEPLOY that is still building can be cancelled.',
|
||||
});
|
||||
}
|
||||
return { ok: true };
|
||||
@@ -1626,7 +1626,10 @@ export const adminRouter = router({
|
||||
}),
|
||||
cancel: releaseAdminProcedure.input(z.object({ id: z.string().uuid() })).mutation(async ({ ctx, input }) => {
|
||||
if (!(await ctx.releases.cancelOperation(input.id))) {
|
||||
throw new TRPCError({ code: 'CONFLICT', message: 'Only queued releases can be cancelled.' });
|
||||
throw new TRPCError({
|
||||
code: 'CONFLICT',
|
||||
message: 'Only queued releases or a release that is still building can be cancelled.',
|
||||
});
|
||||
}
|
||||
return { ok: true };
|
||||
}),
|
||||
|
||||
@@ -12,6 +12,7 @@ export interface BuildResult {
|
||||
ok: boolean;
|
||||
exitCode: number | null;
|
||||
output: string;
|
||||
aborted?: boolean;
|
||||
}
|
||||
|
||||
export type BuildProgressEvent =
|
||||
@@ -21,8 +22,13 @@ export type BuildProgressEvent =
|
||||
|
||||
export type BuildProgressObserver = (event: BuildProgressEvent) => void | Promise<void>;
|
||||
|
||||
export interface BuildRunOptions {
|
||||
signal?: AbortSignal;
|
||||
terminateGraceMs?: number;
|
||||
}
|
||||
|
||||
export interface BuildRunner {
|
||||
run(commands: BuildCommand[], onProgress?: BuildProgressObserver): Promise<BuildResult>;
|
||||
run(commands: BuildCommand[], onProgress?: BuildProgressObserver, options?: BuildRunOptions): Promise<BuildResult>;
|
||||
}
|
||||
|
||||
export const MAX_BUILD_OUTPUT_CHARS = 64 * 1024;
|
||||
@@ -77,7 +83,28 @@ export const buildTurboReleaseTaskCommand = (
|
||||
const appendOutputTail = (current: string, chunk: unknown): string =>
|
||||
`${current}${String(chunk)}`.slice(-MAX_BUILD_OUTPUT_CHARS);
|
||||
|
||||
const runCommand = (command: BuildCommand, onProgress?: BuildProgressObserver): Promise<BuildResult> =>
|
||||
const terminateChildProcess = (pid: number | undefined, signal: NodeJS.Signals): void => {
|
||||
if (!pid) return;
|
||||
try {
|
||||
if (process.platform !== 'win32') {
|
||||
process.kill(-pid, signal);
|
||||
return;
|
||||
}
|
||||
} catch {
|
||||
// Fall back to the direct child below when the process group already exited.
|
||||
}
|
||||
try {
|
||||
process.kill(pid, signal);
|
||||
} catch {
|
||||
// The child already exited.
|
||||
}
|
||||
};
|
||||
|
||||
const runCommand = (
|
||||
command: BuildCommand,
|
||||
onProgress?: BuildProgressObserver,
|
||||
options?: BuildRunOptions
|
||||
): Promise<BuildResult> =>
|
||||
new Promise((resolve) => {
|
||||
let progressQueue = Promise.resolve();
|
||||
const emit = (event: BuildProgressEvent) => {
|
||||
@@ -89,9 +116,25 @@ const runCommand = (command: BuildCommand, onProgress?: BuildProgressObserver):
|
||||
cwd: command.cwd,
|
||||
env: command.env,
|
||||
stdio: ['ignore', 'pipe', 'pipe'],
|
||||
detached: process.platform !== 'win32',
|
||||
});
|
||||
let output = '';
|
||||
let spawnFailed = false;
|
||||
let aborted = false;
|
||||
let killTimer: ReturnType<typeof setTimeout> | undefined;
|
||||
const abort = () => {
|
||||
if (aborted) return;
|
||||
aborted = true;
|
||||
output = appendOutputTail(output, '\nBuild cancelled by operator.');
|
||||
terminateChildProcess(child.pid, 'SIGTERM');
|
||||
killTimer = setTimeout(
|
||||
() => terminateChildProcess(child.pid, 'SIGKILL'),
|
||||
options?.terminateGraceMs ?? 5_000
|
||||
);
|
||||
killTimer.unref?.();
|
||||
};
|
||||
options?.signal?.addEventListener('abort', abort, { once: true });
|
||||
if (options?.signal?.aborted) abort();
|
||||
const lineBuffers = { stdout: '', stderr: '' };
|
||||
const emitOutput = (stream: 'stdout' | 'stderr', chunk: unknown, flush = false) => {
|
||||
if (flush && !lineBuffers[stream]) return;
|
||||
@@ -119,31 +162,47 @@ const runCommand = (command: BuildCommand, onProgress?: BuildProgressObserver):
|
||||
output = appendOutputTail(output, error.message);
|
||||
});
|
||||
child.on('close', (code) => {
|
||||
options?.signal?.removeEventListener('abort', abort);
|
||||
if (killTimer) clearTimeout(killTimer);
|
||||
emitOutput('stdout', '', true);
|
||||
emitOutput('stderr', '', true);
|
||||
const exitCode = spawnFailed ? null : code;
|
||||
emit({ type: 'COMMAND_END', command, exitCode });
|
||||
void progressQueue.then(() => {
|
||||
resolve({
|
||||
ok: !spawnFailed && code === 0,
|
||||
ok: !aborted && !spawnFailed && code === 0,
|
||||
exitCode,
|
||||
output,
|
||||
...(aborted ? { aborted: true } : {}),
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
export class PnpmBuildRunner implements BuildRunner {
|
||||
async run(commands: BuildCommand[], onProgress?: BuildProgressObserver): Promise<BuildResult> {
|
||||
async run(
|
||||
commands: BuildCommand[],
|
||||
onProgress?: BuildProgressObserver,
|
||||
options?: BuildRunOptions
|
||||
): Promise<BuildResult> {
|
||||
let mergedOutput = '';
|
||||
for (const command of commands) {
|
||||
const result = await runCommand(command, onProgress);
|
||||
if (options?.signal?.aborted) {
|
||||
return {
|
||||
ok: false,
|
||||
exitCode: null,
|
||||
output: appendOutputTail(mergedOutput, 'Build cancelled by operator.'),
|
||||
aborted: true,
|
||||
};
|
||||
}
|
||||
const result = await runCommand(command, onProgress, options);
|
||||
mergedOutput = appendOutputTail(mergedOutput, result.output);
|
||||
if (!result.ok) {
|
||||
return {
|
||||
ok: false,
|
||||
exitCode: result.exitCode,
|
||||
output: mergedOutput,
|
||||
...(result.aborted ? { aborted: true } : {}),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
@@ -198,6 +198,7 @@ interface GatewayAdminActionResult {
|
||||
|
||||
const OPERATION_LEASE_DURATION_MS = 10 * 60_000;
|
||||
const OPERATION_HEARTBEAT_INTERVAL_MS = 60_000;
|
||||
const OPERATION_CANCELLATION_POLL_INTERVAL_MS = 500;
|
||||
|
||||
class OperationLeaseLostError extends Error {}
|
||||
|
||||
@@ -633,6 +634,7 @@ export class GatewayOrchestrator implements GatewayOrchestratorHandle {
|
||||
private buildInFlight = false;
|
||||
private adminActionInFlight = false;
|
||||
private operationInFlight = false;
|
||||
private activeOperationAbortSignal?: AbortSignal;
|
||||
private readonly resetInFlight = new Set<string>();
|
||||
private readonly operationLeaseOwner = randomUUID();
|
||||
private readonly inFlightTasks = new Set<Promise<unknown>>();
|
||||
@@ -975,6 +977,8 @@ export class GatewayOrchestrator implements GatewayOrchestratorHandle {
|
||||
if (!operation) {
|
||||
return;
|
||||
}
|
||||
const abortController = new AbortController();
|
||||
this.activeOperationAbortSignal = abortController.signal;
|
||||
const heartbeatTimer = this.repository.renewOperationLease
|
||||
? setInterval(() => {
|
||||
void this.repository
|
||||
@@ -984,17 +988,36 @@ export class GatewayOrchestrator implements GatewayOrchestratorHandle {
|
||||
this.now(),
|
||||
OPERATION_LEASE_DURATION_MS
|
||||
)
|
||||
.then((renewed) => {
|
||||
if (!renewed) abortController.abort();
|
||||
})
|
||||
.catch((error) => {
|
||||
console.error('[gateway-orchestrator] operation heartbeat failed', error);
|
||||
});
|
||||
}, OPERATION_HEARTBEAT_INTERVAL_MS)
|
||||
: undefined;
|
||||
const cancellationTimer = setInterval(() => {
|
||||
void this.repository
|
||||
.getOperation(operation.id)
|
||||
.then((current) => {
|
||||
if (
|
||||
!current ||
|
||||
current.status !== 'RUNNING' ||
|
||||
current.leaseOwner !== this.operationLeaseOwner
|
||||
) {
|
||||
abortController.abort();
|
||||
}
|
||||
})
|
||||
.catch(() => undefined);
|
||||
}, OPERATION_CANCELLATION_POLL_INTERVAL_MS);
|
||||
try {
|
||||
await this.handleOperation(operation);
|
||||
} finally {
|
||||
clearInterval(cancellationTimer);
|
||||
if (heartbeatTimer) {
|
||||
clearInterval(heartbeatTimer);
|
||||
}
|
||||
this.activeOperationAbortSignal = undefined;
|
||||
}
|
||||
} finally {
|
||||
this.operationInFlight = false;
|
||||
@@ -1442,9 +1465,11 @@ export class GatewayOrchestrator implements GatewayOrchestratorHandle {
|
||||
),
|
||||
];
|
||||
await this.appendOperationLog(operationId, 'build', `${profile.profileName} 구성 요소를 빌드합니다.`);
|
||||
const result = await this.buildRunner.run(commands, this.buildProgress(operationId, 'build'));
|
||||
await assertLease();
|
||||
const result = await this.buildRunner.run(commands, this.buildProgress(operationId, 'build'), {
|
||||
signal: this.activeOperationAbortSignal,
|
||||
});
|
||||
if (!result.ok) {
|
||||
await assertLease();
|
||||
const detail = result.output.slice(-4000) || 'selected workspace build failed';
|
||||
await updateClaimedProfile({
|
||||
buildStatus: 'FAILED',
|
||||
@@ -1455,6 +1480,7 @@ export class GatewayOrchestrator implements GatewayOrchestratorHandle {
|
||||
}
|
||||
|
||||
await this.appendOperationLog(operationId, 'switch', '기존 profile process를 정지합니다.');
|
||||
await assertLease();
|
||||
await this.stopProfile(profile, assertLease);
|
||||
oldRuntimeStopped = true;
|
||||
const profileDatabaseUrl = this.resolveProfileDatabaseUrl(profile);
|
||||
@@ -2030,7 +2056,8 @@ export class GatewayOrchestrator implements GatewayOrchestratorHandle {
|
||||
return {
|
||||
result: await this.buildRunner.run(
|
||||
commands,
|
||||
operationId ? this.buildProgress(operationId, 'build') : undefined
|
||||
operationId ? this.buildProgress(operationId, 'build') : undefined,
|
||||
{ signal: this.activeOperationAbortSignal }
|
||||
),
|
||||
workspace,
|
||||
};
|
||||
@@ -2052,7 +2079,8 @@ export class GatewayOrchestrator implements GatewayOrchestratorHandle {
|
||||
): Promise<Awaited<ReturnType<BuildRunner['run']>>> {
|
||||
return this.buildRunner.run(
|
||||
[buildProfileMigrationCommand(workspaceRoot, profileDatabaseUrl, this.processConfig.baseEnv)],
|
||||
onProgress
|
||||
onProgress,
|
||||
{ signal: this.activeOperationAbortSignal }
|
||||
);
|
||||
}
|
||||
|
||||
@@ -2106,7 +2134,8 @@ export class GatewayOrchestrator implements GatewayOrchestratorHandle {
|
||||
},
|
||||
},
|
||||
],
|
||||
onProgress
|
||||
onProgress,
|
||||
{ signal: this.activeOperationAbortSignal }
|
||||
);
|
||||
} finally {
|
||||
await fs.rm(tempDirectory, { recursive: true, force: true });
|
||||
|
||||
@@ -208,13 +208,18 @@ export const createGatewayReleaseRepository = (prisma: GatewayPrismaClient): Gat
|
||||
return rows.map(mapLog);
|
||||
},
|
||||
async appendOperationLog(id, input) {
|
||||
const row = await prisma.gatewayReleaseLog.create({
|
||||
data: {
|
||||
operationId: id,
|
||||
level: input.level,
|
||||
phase: input.phase.slice(0, 64),
|
||||
message: input.message.slice(0, 4_000),
|
||||
},
|
||||
const row = await prisma.$transaction(async (tx) => {
|
||||
await tx.$queryRaw<Array<{ id: string }>>`
|
||||
SELECT "id" FROM "gateway_release_operation" WHERE "id" = ${id} FOR UPDATE
|
||||
`;
|
||||
return tx.gatewayReleaseLog.create({
|
||||
data: {
|
||||
operationId: id,
|
||||
level: input.level,
|
||||
phase: input.phase.slice(0, 64),
|
||||
message: input.message.slice(0, 4_000),
|
||||
},
|
||||
});
|
||||
});
|
||||
return mapLog(row);
|
||||
},
|
||||
@@ -367,11 +372,48 @@ export const createGatewayReleaseRepository = (prisma: GatewayPrismaClient): Gat
|
||||
});
|
||||
},
|
||||
async cancelOperation(id) {
|
||||
const updated = await prisma.gatewayReleaseOperation.updateMany({
|
||||
where: { id, status: 'QUEUED' },
|
||||
data: { status: 'CANCELLED', completedAt: new Date() },
|
||||
const count = await prisma.$transaction(async (tx) => {
|
||||
const rows = await tx.$queryRaw<Array<{ status: GatewayOperationStatus }>>`
|
||||
SELECT "status"
|
||||
FROM "gateway_release_operation"
|
||||
WHERE "id" = ${id}
|
||||
FOR UPDATE
|
||||
`;
|
||||
const operation = rows[0];
|
||||
if (!operation || (operation.status !== 'QUEUED' && operation.status !== 'RUNNING')) return 0;
|
||||
if (operation.status === 'RUNNING') {
|
||||
const latestLog = await tx.gatewayReleaseLog.findFirst({
|
||||
where: { operationId: id },
|
||||
orderBy: { id: 'desc' },
|
||||
select: { phase: true },
|
||||
});
|
||||
if (!latestLog || !['claim', 'resolve', 'workspace', 'build'].includes(latestLog.phase)) return 0;
|
||||
}
|
||||
const updated = await tx.gatewayReleaseOperation.updateMany({
|
||||
where: { id, status: operation.status },
|
||||
data: {
|
||||
status: 'CANCELLED',
|
||||
completedAt: new Date(),
|
||||
leaseOwner: null,
|
||||
leaseUntil: null,
|
||||
heartbeatAt: null,
|
||||
},
|
||||
});
|
||||
if (updated.count !== 1) return 0;
|
||||
await tx.gatewayReleaseLog.create({
|
||||
data: {
|
||||
operationId: id,
|
||||
level: 'INFO',
|
||||
phase: 'cancel',
|
||||
message:
|
||||
operation.status === 'RUNNING'
|
||||
? '실행 중인 Gateway 빌드를 중단했습니다. 현재 active release는 유지됩니다.'
|
||||
: '대기 중인 Gateway 릴리스를 취소했습니다.',
|
||||
},
|
||||
});
|
||||
return 1;
|
||||
});
|
||||
return updated.count === 1;
|
||||
return count === 1;
|
||||
},
|
||||
async retryOperation(id, requestedBy) {
|
||||
const row = await prisma.$transaction(async (tx) => {
|
||||
|
||||
@@ -586,13 +586,18 @@ export const createGatewayProfileRepository = (prisma: GatewayPrismaClient): Gat
|
||||
return rows.map(mapOperationLog);
|
||||
},
|
||||
async appendOperationLog(id, input) {
|
||||
const row = await prisma.gatewayOperationLog.create({
|
||||
data: {
|
||||
operationId: id,
|
||||
level: input.level,
|
||||
phase: input.phase.slice(0, 64),
|
||||
message: input.message.slice(0, 4_000),
|
||||
},
|
||||
const row = await prisma.$transaction(async (tx) => {
|
||||
await tx.$queryRaw<Array<{ id: string }>>`
|
||||
SELECT "id" FROM "gateway_operation" WHERE "id" = ${id} FOR UPDATE
|
||||
`;
|
||||
return tx.gatewayOperationLog.create({
|
||||
data: {
|
||||
operationId: id,
|
||||
level: input.level,
|
||||
phase: input.phase.slice(0, 64),
|
||||
message: input.message.slice(0, 4_000),
|
||||
},
|
||||
});
|
||||
});
|
||||
return mapOperationLog(row);
|
||||
},
|
||||
@@ -805,21 +810,61 @@ export const createGatewayProfileRepository = (prisma: GatewayPrismaClient): Gat
|
||||
},
|
||||
async cancelOperation(id: string): Promise<boolean> {
|
||||
const count = await prisma.$transaction(async (tx) => {
|
||||
const rows = await tx.$queryRaw<Array<{ status: GatewayOperationStatus; type: GatewayOperationType }>>`
|
||||
SELECT "status", "type"
|
||||
FROM "gateway_operation"
|
||||
WHERE "id" = ${id}
|
||||
FOR UPDATE
|
||||
`;
|
||||
const operation = rows[0];
|
||||
if (!operation || (operation.status !== 'QUEUED' && operation.status !== 'RUNNING')) return 0;
|
||||
|
||||
let runningBuildCancelled = false;
|
||||
if (operation.status === 'RUNNING') {
|
||||
if (operation.type !== 'DEPLOY') return 0;
|
||||
const latestLog = await tx.gatewayOperationLog.findFirst({
|
||||
where: { operationId: id },
|
||||
orderBy: { id: 'desc' },
|
||||
select: { phase: true },
|
||||
});
|
||||
if (!latestLog || !['claim', 'resolve', 'workspace', 'build'].includes(latestLog.phase)) return 0;
|
||||
runningBuildCancelled = true;
|
||||
}
|
||||
|
||||
const result = await tx.gatewayOperation.updateMany({
|
||||
where: { id, status: 'QUEUED' },
|
||||
data: { status: 'CANCELLED', completedAt: new Date() },
|
||||
where: { id, status: operation.status },
|
||||
data: {
|
||||
status: 'CANCELLED',
|
||||
completedAt: new Date(),
|
||||
leaseOwner: null,
|
||||
leaseUntil: null,
|
||||
heartbeatAt: null,
|
||||
},
|
||||
});
|
||||
if (result.count === 1) {
|
||||
await tx.gatewayOperationLog.create({
|
||||
if (result.count !== 1) return 0;
|
||||
if (runningBuildCancelled) {
|
||||
await tx.gatewayProfile.updateMany({
|
||||
where: { operations: { some: { id } } },
|
||||
data: {
|
||||
operationId: id,
|
||||
level: 'INFO',
|
||||
phase: 'cancel',
|
||||
message: '대기 중인 작업이 취소되었습니다.',
|
||||
buildStatus: 'SUCCEEDED',
|
||||
buildRequestedAt: null,
|
||||
buildStartedAt: null,
|
||||
buildCompletedAt: null,
|
||||
buildError: null,
|
||||
},
|
||||
});
|
||||
}
|
||||
return result.count;
|
||||
await tx.gatewayOperationLog.create({
|
||||
data: {
|
||||
operationId: id,
|
||||
level: 'INFO',
|
||||
phase: 'cancel',
|
||||
message: runningBuildCancelled
|
||||
? '실행 중인 빌드를 중단했습니다. 기존 runtime과 DB는 유지됩니다.'
|
||||
: '대기 중인 작업이 취소되었습니다.',
|
||||
},
|
||||
});
|
||||
return 1;
|
||||
});
|
||||
return count === 1;
|
||||
},
|
||||
|
||||
@@ -145,4 +145,28 @@ describe('PnpmBuildRunner', () => {
|
||||
{ type: 'COMMAND_END' },
|
||||
]);
|
||||
});
|
||||
|
||||
it('terminates a running build process group when the operation is cancelled', async () => {
|
||||
const runner = new PnpmBuildRunner();
|
||||
const abortController = new AbortController();
|
||||
const startedAt = Date.now();
|
||||
const timer = setTimeout(() => abortController.abort(), 50);
|
||||
|
||||
const result = await runner.run(
|
||||
[
|
||||
{
|
||||
command: process.execPath,
|
||||
args: ['-e', "setInterval(() => process.stdout.write('still-running\\n'), 25);"],
|
||||
cwd: process.cwd(),
|
||||
},
|
||||
],
|
||||
undefined,
|
||||
{ signal: abortController.signal, terminateGraceMs: 100 }
|
||||
);
|
||||
clearTimeout(timer);
|
||||
|
||||
expect(result).toMatchObject({ ok: false, aborted: true });
|
||||
expect(result.output).toContain('Build cancelled by operator.');
|
||||
expect(Date.now() - startedAt).toBeLessThan(2_000);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -249,6 +249,72 @@ describeDatabase('gateway operation lease and profile serialization', () => {
|
||||
).resolves.toMatchObject({ status: 'SUCCEEDED' });
|
||||
});
|
||||
|
||||
it('cancels only a running profile DEPLOY build and fences its worker lease', async () => {
|
||||
const operation = await repository.createOperation({
|
||||
profileName,
|
||||
type: 'DEPLOY',
|
||||
sourceMode: 'BRANCH',
|
||||
sourceRef: 'main',
|
||||
requestedBy: 'admin',
|
||||
});
|
||||
const now = new Date('2030-01-01T00:00:00.000Z');
|
||||
await repository.claimNextOperation(now, { ownerId: 'worker-a', durationMs: 10_000 });
|
||||
await repository.updateProfileForOperation?.(operation.id, 'worker-a', profileName, {
|
||||
buildStatus: 'RUNNING',
|
||||
buildError: 'temporary build output',
|
||||
});
|
||||
await repository.appendOperationLog(operation.id, {
|
||||
level: 'INFO',
|
||||
phase: 'build',
|
||||
message: 'building profile',
|
||||
});
|
||||
|
||||
await expect(repository.cancelOperation(operation.id)).resolves.toBe(true);
|
||||
await expect(repository.getOperation(operation.id)).resolves.toMatchObject({
|
||||
status: 'CANCELLED',
|
||||
leaseOwner: undefined,
|
||||
});
|
||||
await expect(repository.getProfile(profileName)).resolves.toMatchObject({
|
||||
buildStatus: 'SUCCEEDED',
|
||||
buildError: undefined,
|
||||
});
|
||||
await expect(repository.renewOperationLease?.(operation.id, 'worker-a', now, 10_000)).resolves.toBe(false);
|
||||
});
|
||||
|
||||
it('cancels a running Gateway build but rejects cancellation after migration starts', async () => {
|
||||
const buildOperation = await releaseRepository.createOperation({
|
||||
type: 'DEPLOY',
|
||||
sourceMode: 'BRANCH',
|
||||
sourceRef: 'main',
|
||||
requestedBy: 'admin',
|
||||
});
|
||||
const now = new Date('2030-01-01T00:00:00.000Z');
|
||||
await releaseRepository.claimNextOperation(now, { ownerId: 'release-worker', durationMs: 10_000 });
|
||||
await releaseRepository.appendOperationLog(buildOperation.id, {
|
||||
level: 'INFO',
|
||||
phase: 'build',
|
||||
message: 'building Gateway',
|
||||
});
|
||||
await expect(releaseRepository.cancelOperation(buildOperation.id)).resolves.toBe(true);
|
||||
await expect(
|
||||
releaseRepository.renewOperationLease(buildOperation.id, 'release-worker', now, 10_000)
|
||||
).resolves.toBe(false);
|
||||
|
||||
const migrationOperation = await releaseRepository.createOperation({
|
||||
type: 'DEPLOY',
|
||||
sourceMode: 'BRANCH',
|
||||
sourceRef: 'main',
|
||||
requestedBy: 'admin',
|
||||
});
|
||||
await releaseRepository.claimNextOperation(now, { ownerId: 'release-worker', durationMs: 10_000 });
|
||||
await releaseRepository.appendOperationLog(migrationOperation.id, {
|
||||
level: 'INFO',
|
||||
phase: 'migration',
|
||||
message: 'migrating Gateway',
|
||||
});
|
||||
await expect(releaseRepository.cancelOperation(migrationOperation.id)).resolves.toBe(false);
|
||||
});
|
||||
|
||||
it('pins retry to the first resolved commit and preserves its install generation', async () => {
|
||||
const operation = await repository.createOperation({
|
||||
profileName,
|
||||
|
||||
@@ -457,6 +457,33 @@ const installFixture = async (page: Page, state: FixtureState) => {
|
||||
state.operations = [operation, ...state.operations];
|
||||
return response(operation);
|
||||
}
|
||||
if (name === 'admin.operations.cancel') {
|
||||
const id = JSON.stringify(body).match(/[0-9a-f]{8}-[0-9a-f-]{27,}/u)?.[0];
|
||||
state.operations = state.operations.map((operation) =>
|
||||
operation.id === id ? { ...operation, status: 'CANCELLED' as const } : operation
|
||||
);
|
||||
return response({ ok: true });
|
||||
}
|
||||
if (name === 'admin.releases.cancel') {
|
||||
const id = JSON.stringify(body).match(/[0-9a-f]{8}-[0-9a-f-]{27,}/u)?.[0];
|
||||
state.gatewayOperations = state.gatewayOperations.map((operation) =>
|
||||
operation.id === id ? { ...operation, status: 'CANCELLED' as const } : operation
|
||||
);
|
||||
return response({ ok: true });
|
||||
}
|
||||
if (name === 'admin.releases.retry') {
|
||||
const previous = state.gatewayOperations[0];
|
||||
if (!previous) throw new Error('Release retry fixture is missing');
|
||||
const retried = {
|
||||
...previous,
|
||||
id: 'aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa',
|
||||
status: 'QUEUED' as const,
|
||||
sourceMode: 'COMMIT' as const,
|
||||
sourceRef: previous.resolvedCommitSha ?? previous.sourceRef,
|
||||
};
|
||||
state.gatewayOperations = [retried, ...state.gatewayOperations];
|
||||
return response(retried);
|
||||
}
|
||||
throw new Error(`Unhandled tRPC operation: ${name}`);
|
||||
});
|
||||
await route.fulfill({
|
||||
@@ -1203,6 +1230,77 @@ test('scenario-only operator resets the server-selected version without Git or G
|
||||
expect(JSON.stringify(request?.body)).not.toContain('"sourceRef"');
|
||||
});
|
||||
|
||||
test('stops a running profile build while keeping the existing runtime available', async ({ page }) => {
|
||||
const operation: Operation = {
|
||||
id: '12121212-1212-4212-8212-121212121212',
|
||||
profileName: 'che:default',
|
||||
type: 'DEPLOY',
|
||||
status: 'RUNNING',
|
||||
sourceMode: 'COMMIT',
|
||||
sourceRef: '0123456789abcdef0123456789abcdef01234567',
|
||||
payload: {},
|
||||
requestedBy: 'admin',
|
||||
createdAt: '2026-08-20T01:00:00.000Z',
|
||||
updatedAt: '2026-08-20T01:00:01.000Z',
|
||||
};
|
||||
const state: FixtureState = {
|
||||
operations: [operation],
|
||||
gatewayOperations: [],
|
||||
runtimeRunning: true,
|
||||
requestBodies: [],
|
||||
profileLogsEmpty: true,
|
||||
};
|
||||
await installFixture(page, state);
|
||||
page.on('dialog', (dialog) => dialog.accept());
|
||||
|
||||
await page.goto('admin/servers/che%3Adefault/version');
|
||||
await expect(page.getByTestId('profile-build-recovery-guide')).toContainText('기존 profile runtime과 게임 DB는');
|
||||
await page.getByRole('button', { name: '빌드 중단' }).click();
|
||||
|
||||
await expect(page.getByText('프로필 빌드를 중단했습니다.').first()).toBeVisible();
|
||||
await expect(page.getByTestId('operations-table').getByText('CANCELLED', { exact: true })).toBeVisible();
|
||||
expect(state.requestBodies.some((entry) => entry.operation === 'admin.operations.cancel')).toBe(true);
|
||||
});
|
||||
|
||||
test('stops and retries a running Gateway build from the release GUI', async ({ page }) => {
|
||||
const state: FixtureState = {
|
||||
operations: [],
|
||||
gatewayOperations: [
|
||||
{
|
||||
id: '34343434-3434-4434-8434-343434343434',
|
||||
type: 'DEPLOY',
|
||||
status: 'RUNNING',
|
||||
sourceMode: 'COMMIT',
|
||||
sourceRef: 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa',
|
||||
resolvedCommitSha: 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa',
|
||||
payload: {},
|
||||
requestedBy: 'admin',
|
||||
createdAt: '2026-08-20T01:00:00.000Z',
|
||||
updatedAt: '2026-08-20T01:00:01.000Z',
|
||||
},
|
||||
],
|
||||
runtimeRunning: true,
|
||||
requestBodies: [],
|
||||
gatewayLogsEmpty: true,
|
||||
};
|
||||
await installFixture(page, state);
|
||||
page.on('dialog', (dialog) => dialog.accept());
|
||||
|
||||
await page.goto('admin/releases');
|
||||
await expect(page.getByTestId('gateway-build-recovery-guide')).toContainText(
|
||||
'migration 또는 process 전환이 시작된 뒤에는'
|
||||
);
|
||||
await page.getByRole('button', { name: '빌드 중단' }).click();
|
||||
await expect(page.getByText('Gateway 빌드를 중단했습니다.').first()).toBeVisible();
|
||||
await expect(page.getByTestId('gateway-release-table').getByText('CANCELLED', { exact: true })).toBeVisible();
|
||||
|
||||
await page.getByRole('button', { name: '재시도' }).click();
|
||||
await expect(page.getByText('Gateway 릴리스 재시도 작업을 등록했습니다.').first()).toBeVisible();
|
||||
await expect(page.getByTestId('gateway-release-table').getByText('QUEUED', { exact: true })).toBeVisible();
|
||||
expect(state.requestBodies.some((entry) => entry.operation === 'admin.releases.cancel')).toBe(true);
|
||||
expect(state.requestBodies.some((entry) => entry.operation === 'admin.releases.retry')).toBe(true);
|
||||
});
|
||||
|
||||
test('controls gateway deployment and rollback through the external controller queue', async ({ page }, testInfo) => {
|
||||
const state: FixtureState = {
|
||||
operations: [],
|
||||
|
||||
@@ -546,6 +546,37 @@ const requestGatewayRollback = async () => {
|
||||
}
|
||||
};
|
||||
|
||||
const cancelGatewayRelease = async (operation: GatewayReleaseOperation) => {
|
||||
clearStatus();
|
||||
const prompt =
|
||||
operation.status === 'RUNNING'
|
||||
? '실행 중인 Gateway 빌드를 중단하시겠습니까? process 전환 또는 migration이 시작된 뒤에는 중단할 수 없습니다.'
|
||||
: '대기 중인 Gateway 릴리스를 취소하시겠습니까?';
|
||||
if (!window.confirm(prompt)) return;
|
||||
try {
|
||||
await adminClient.releases.cancel.mutate({ id: operation.id });
|
||||
selectedGatewayOperationId.value = operation.id;
|
||||
message.value =
|
||||
operation.status === 'RUNNING' ? 'Gateway 빌드를 중단했습니다.' : 'Gateway 릴리스를 취소했습니다.';
|
||||
await loadState(true);
|
||||
} catch (error) {
|
||||
errorMessage.value = error instanceof Error ? error.message : 'Gateway 릴리스 중단에 실패했습니다.';
|
||||
}
|
||||
};
|
||||
|
||||
const retryGatewayRelease = async (operation: GatewayReleaseOperation) => {
|
||||
clearStatus();
|
||||
if (!window.confirm('같은 고정 커밋으로 Gateway 릴리스를 다시 실행하시겠습니까?')) return;
|
||||
try {
|
||||
const retried = await adminClient.releases.retry.mutate({ id: operation.id });
|
||||
selectedGatewayOperationId.value = retried.id;
|
||||
message.value = 'Gateway 릴리스 재시도 작업을 등록했습니다.';
|
||||
await loadState(true);
|
||||
} catch (error) {
|
||||
errorMessage.value = error instanceof Error ? error.message : 'Gateway 릴리스 재시도에 실패했습니다.';
|
||||
}
|
||||
};
|
||||
|
||||
const loadScenarios = async () => {
|
||||
clearStatus();
|
||||
if (form.sourceMode !== 'CURRENT' && !form.sourceRef.trim()) {
|
||||
@@ -688,13 +719,17 @@ const requestGameCancellation = async () => {
|
||||
|
||||
const cancelOperation = async (operation: Operation) => {
|
||||
clearStatus();
|
||||
if (!window.confirm('대기 중인 작업을 취소하시겠습니까?')) {
|
||||
const prompt =
|
||||
operation.status === 'RUNNING'
|
||||
? '실행 중인 프로필 빌드를 중단하시겠습니까? 기존 runtime과 게임 DB는 유지됩니다.'
|
||||
: '대기 중인 작업을 취소하시겠습니까?';
|
||||
if (!window.confirm(prompt)) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
await adminClient.operations.cancel.mutate({ id: operation.id });
|
||||
selectedProfileOperationId.value = operation.id;
|
||||
message.value = '작업을 취소했습니다.';
|
||||
message.value = operation.status === 'RUNNING' ? '프로필 빌드를 중단했습니다.' : '작업을 취소했습니다.';
|
||||
await loadState(true);
|
||||
} catch (error) {
|
||||
errorMessage.value = error instanceof Error ? error.message : '작업 취소에 실패했습니다.';
|
||||
@@ -1191,6 +1226,14 @@ onBeforeUnmount(() => {
|
||||
class="rounded-lg border border-violet-800/70 bg-zinc-900 p-5 space-y-4"
|
||||
data-testid="gateway-release-panel"
|
||||
>
|
||||
<div
|
||||
class="rounded border border-amber-800/80 bg-amber-950/30 px-4 py-3 text-sm text-amber-100"
|
||||
data-testid="gateway-build-recovery-guide"
|
||||
>
|
||||
<strong>빌드가 멈춘 경우:</strong> 로그의 마지막 단계가 build일 때만
|
||||
<strong>빌드 중단</strong>을 누르고 CANCELLED를 확인한 뒤 <strong>재시도</strong>하세요.
|
||||
migration 또는 process 전환이 시작된 뒤에는 DB와 runtime 보호를 위해 중단할 수 없습니다.
|
||||
</div>
|
||||
<div class="flex flex-col gap-2 md:flex-row md:items-center md:justify-between">
|
||||
<div>
|
||||
<h3 class="text-lg font-semibold">Gateway 릴리스</h3>
|
||||
@@ -1388,6 +1431,24 @@ onBeforeUnmount(() => {
|
||||
: '오류 보기'
|
||||
}}
|
||||
</button>
|
||||
<button
|
||||
v-if="operation.status === 'QUEUED' || operation.status === 'RUNNING'"
|
||||
type="button"
|
||||
class="rounded border border-red-800 px-2 py-1 text-red-300 hover:bg-red-950 focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-red-400"
|
||||
@click="cancelGatewayRelease(operation)"
|
||||
>
|
||||
{{ operation.status === 'RUNNING' ? '빌드 중단' : '취소' }}
|
||||
</button>
|
||||
<button
|
||||
v-else-if="
|
||||
operation.status === 'FAILED' || operation.status === 'CANCELLED'
|
||||
"
|
||||
type="button"
|
||||
class="rounded border border-amber-700 px-2 py-1 text-amber-300 hover:bg-amber-950 focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-amber-400"
|
||||
@click="retryGatewayRelease(operation)"
|
||||
>
|
||||
재시도
|
||||
</button>
|
||||
</div>
|
||||
</td>
|
||||
</tr>
|
||||
@@ -1476,6 +1537,14 @@ onBeforeUnmount(() => {
|
||||
</section>
|
||||
|
||||
<section v-if="mode !== 'gateway'" class="rounded-lg border border-zinc-800 bg-zinc-900 p-5">
|
||||
<div
|
||||
class="mb-4 rounded border border-amber-800/80 bg-amber-950/30 px-4 py-3 text-sm text-amber-100"
|
||||
data-testid="profile-build-recovery-guide"
|
||||
>
|
||||
<strong>DB 보존 업데이트 빌드가 멈춘 경우:</strong> <strong>빌드 중단</strong>을 누르고
|
||||
CANCELLED를 확인한 뒤 <strong>재시도</strong>하세요. 기존 profile runtime과 게임 DB는
|
||||
유지됩니다. RESET·migration·process 전환 단계는 이 화면에서 강제 중단하지 않습니다.
|
||||
</div>
|
||||
<div class="mb-4 flex items-center justify-between">
|
||||
<h3 class="text-lg font-semibold">작업 이력</h3>
|
||||
<span class="text-xs text-zinc-500">3초마다 상태 갱신</span>
|
||||
@@ -1565,11 +1634,14 @@ onBeforeUnmount(() => {
|
||||
로그
|
||||
</button>
|
||||
<button
|
||||
v-if="operation.status === 'QUEUED'"
|
||||
v-if="
|
||||
operation.status === 'QUEUED' ||
|
||||
(operation.status === 'RUNNING' && operation.type === 'DEPLOY')
|
||||
"
|
||||
class="rounded border border-red-800 px-2 py-1 text-xs text-red-300 hover:bg-red-950"
|
||||
@click="cancelOperation(operation)"
|
||||
>
|
||||
취소
|
||||
{{ operation.status === 'RUNNING' ? '빌드 중단' : '취소' }}
|
||||
</button>
|
||||
<button
|
||||
v-else-if="
|
||||
|
||||
@@ -87,6 +87,34 @@ pnpm --filter @sammo-ts/release-controller self-upgrade COMMIT <full-sha>
|
||||
Database migration은 일반적으로 되돌리지 않습니다. 이전 애플리케이션으로
|
||||
rollback하려면 새 schema와의 하위 호환성을 릴리스 전에 확인해 주세요.
|
||||
|
||||
## 멈춘 빌드 복구
|
||||
|
||||
운영 container나 PM2 process를 먼저 종료하지 마세요. 관리자 화면의
|
||||
`Gateway 릴리스` 또는 profile `버전 업데이트` 작업 이력에서 로그의 마지막 단계와
|
||||
작업 상태를 확인합니다.
|
||||
|
||||
1. `RUNNING`이고 마지막 단계가 `claim`, `resolve`, `workspace`, `build` 중 하나이면
|
||||
`빌드 중단`을 누릅니다.
|
||||
2. 작업이 `CANCELLED`가 되고 로그에 빌드 종료가 기록될 때까지 기다립니다. Controller와
|
||||
orchestrator는 해당 process group에 SIGTERM을 보내고 제한 시간 뒤 SIGKILL로
|
||||
정리하며, 기존 active Gateway/profile runtime과 profile DB는 유지합니다.
|
||||
3. 같은 행의 `재시도`를 누르면 최초 작업이 고정한 commit으로 새 작업을 등록합니다.
|
||||
branch의 최신 commit을 새로 선택하려면 새 배포 작업을 등록합니다.
|
||||
|
||||
마지막 단계가 `migration`, `switch`, `readiness`이면 중단 요청을 거부합니다. 이 구간에서
|
||||
container restart, PM2 delete 또는 DB row 직접 변경으로 lease를 무효화하지 말고 작업 로그와
|
||||
controller/orchestrator 상태를 조사합니다. Profile 상태가 `PAUSED`이면 runtime 장애가 아니라
|
||||
turn gate가 닫힌 상태이므로 배포 완료 후 서버 관리 화면에서 `턴 재개`를 사용합니다.
|
||||
|
||||
호스트에서는 stack wrapper로 container와 로그를 읽기 전용 확인합니다. 운영 stack의 가까운
|
||||
README에 정의된 경로에서 다음 순서로 확인하며, `down --volumes`나 `RESET`은 빌드 복구에
|
||||
사용하지 않습니다.
|
||||
|
||||
```sh
|
||||
./scripts/stack.sh ps
|
||||
./scripts/stack.sh logs runtime
|
||||
```
|
||||
|
||||
`release-manifest.json`의 `controllerProtocol`이 올라간 릴리스는 controller를
|
||||
먼저 self-upgrade해야 합니다. Protocol 2는 `GatewayReleaseLog` 진행 로그 저장을
|
||||
요구합니다. 구형 controller로 새 Gateway만 배포하면 관리자 화면과 controller의
|
||||
|
||||
@@ -24,6 +24,7 @@ import type { ReleaseControllerConfig } from './config.js';
|
||||
|
||||
const LEASE_DURATION_MS = 10 * 60_000;
|
||||
const HEARTBEAT_INTERVAL_MS = 60_000;
|
||||
const CANCELLATION_POLL_INTERVAL_MS = 500;
|
||||
const PROCESS_NAMES = ['sammo:gateway-api', 'sammo:gateway-frontend', 'sammo:gateway-orchestrator'] as const;
|
||||
const SENSITIVE_ENV_NAME = /(SECRET|TOKEN|PASSWORD|PASSWD|PRIVATE_KEY|CLIENT_SECRET|DATABASE_URL|REDIS_URL)/iu;
|
||||
|
||||
@@ -187,9 +188,25 @@ export class GatewayReleaseController {
|
||||
});
|
||||
if (!operation) return null;
|
||||
await this.appendLog(operation.id, 'claim', `릴리스 작업을 시작합니다. 시도 ${operation.attempts}회차.`);
|
||||
const abortController = new AbortController();
|
||||
const heartbeat = setInterval(() => {
|
||||
void this.repository.renewOperationLease(operation.id, this.ownerId, this.now(), LEASE_DURATION_MS);
|
||||
void this.repository
|
||||
.renewOperationLease(operation.id, this.ownerId, this.now(), LEASE_DURATION_MS)
|
||||
.then((renewed) => {
|
||||
if (!renewed) abortController.abort();
|
||||
})
|
||||
.catch(() => undefined);
|
||||
}, HEARTBEAT_INTERVAL_MS);
|
||||
const cancellationWatcher = setInterval(() => {
|
||||
void this.repository
|
||||
.getOperation(operation.id)
|
||||
.then((current) => {
|
||||
if (!current || current.status !== 'RUNNING' || current.leaseOwner !== this.ownerId) {
|
||||
abortController.abort();
|
||||
}
|
||||
})
|
||||
.catch(() => undefined);
|
||||
}, CANCELLATION_POLL_INTERVAL_MS);
|
||||
let resolvedCommitSha: string | undefined;
|
||||
try {
|
||||
await this.appendLog(operation.id, 'resolve', '현재 Gateway 릴리스 상태를 확인합니다.');
|
||||
@@ -208,7 +225,7 @@ export class GatewayReleaseController {
|
||||
throw new Error('Gateway release lease was lost while pinning the commit.');
|
||||
}
|
||||
await this.appendLog(operation.id, 'resolve', `대상 커밋을 ${resolvedCommitSha}로 고정했습니다.`);
|
||||
await this.deploy(operation, deploymentState, resolvedCommitSha);
|
||||
await this.deploy(operation, deploymentState, resolvedCommitSha, abortController.signal);
|
||||
await this.appendLog(operation.id, 'complete', 'Gateway 릴리스가 완료되었습니다.');
|
||||
return await this.repository.completeOperation(
|
||||
operation.id,
|
||||
@@ -217,6 +234,17 @@ export class GatewayReleaseController {
|
||||
this.ownerId
|
||||
);
|
||||
} catch (error) {
|
||||
const current = await this.repository.getOperation(operation.id);
|
||||
if (
|
||||
!current ||
|
||||
current.status !== 'RUNNING' ||
|
||||
(abortController.signal.aborted && current.leaseOwner !== this.ownerId)
|
||||
) {
|
||||
if (current?.status === 'CANCELLED') {
|
||||
await this.appendLog(operation.id, 'cancel', '실행 중인 Gateway 빌드가 종료되었습니다.');
|
||||
}
|
||||
return current;
|
||||
}
|
||||
const detail = error instanceof Error ? error.message : String(error);
|
||||
await this.appendLog(operation.id, 'failed', detail, 'ERROR');
|
||||
await this.repository.recordStateError(detail);
|
||||
@@ -228,13 +256,21 @@ export class GatewayReleaseController {
|
||||
);
|
||||
} finally {
|
||||
clearInterval(heartbeat);
|
||||
clearInterval(cancellationWatcher);
|
||||
}
|
||||
}
|
||||
|
||||
private async assertOperationLease(operationId: string): Promise<void> {
|
||||
if (!(await this.repository.renewOperationLease(operationId, this.ownerId, this.now(), LEASE_DURATION_MS))) {
|
||||
throw new Error(`Gateway release lease lost: ${operationId}`);
|
||||
}
|
||||
}
|
||||
|
||||
private async deploy(
|
||||
operation: GatewayReleaseOperationRecord,
|
||||
state: GatewayReleaseStateRecord,
|
||||
commitSha: string
|
||||
commitSha: string,
|
||||
signal: AbortSignal
|
||||
): Promise<void> {
|
||||
await this.appendLog(operation.id, 'workspace', `커밋 ${commitSha}의 worktree를 준비합니다.`);
|
||||
const workspace = await this.workspaceManager.prepare(commitSha);
|
||||
@@ -244,13 +280,16 @@ export class GatewayReleaseController {
|
||||
await this.appendLog(operation.id, 'build', 'Gateway 구성 요소를 빌드합니다.');
|
||||
const build = await this.buildRunner.run(
|
||||
buildGatewayReleaseCommands(workspace.root, workspace.needsInstall, this.config),
|
||||
this.buildProgress(operation.id, 'build')
|
||||
this.buildProgress(operation.id, 'build'),
|
||||
{ signal }
|
||||
);
|
||||
if (!build.ok) throw new Error(`Gateway release build failed: ${build.output.slice(-4000)}`);
|
||||
await this.appendLog(operation.id, 'migration', 'Gateway database migration을 적용합니다.');
|
||||
await this.assertOperationLease(operation.id);
|
||||
const migration = await this.buildRunner.run(
|
||||
[buildGatewayMigrationCommand(workspace.root, this.config)],
|
||||
this.buildProgress(operation.id, 'migration')
|
||||
this.buildProgress(operation.id, 'migration'),
|
||||
{ signal }
|
||||
);
|
||||
if (!migration.ok) throw new Error(`Gateway migration failed: ${migration.output.slice(-4000)}`);
|
||||
await this.appendLog(operation.id, 'migration', 'Gateway database migration이 완료되었습니다.');
|
||||
@@ -259,6 +298,7 @@ export class GatewayReleaseController {
|
||||
? buildGatewayProcessDefinitions(state.activeWorkspace, this.config)
|
||||
: [];
|
||||
await this.appendLog(operation.id, 'switch', '기존 Gateway process를 정지합니다.');
|
||||
await this.assertOperationLease(operation.id);
|
||||
await this.stopManagedProcesses(operation.id);
|
||||
try {
|
||||
await this.startDefinitions(buildGatewayProcessDefinitions(workspace.root, this.config), operation.id);
|
||||
|
||||
Reference in New Issue
Block a user