feat(gateway): stream release progress logs
This commit is contained in:
@@ -1,9 +1,11 @@
|
||||
import path from 'node:path';
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import { stripVTControlCharacters } from 'node:util';
|
||||
|
||||
import {
|
||||
assertReleaseComponents,
|
||||
type BuildCommand,
|
||||
type BuildProgressEvent,
|
||||
type BuildRunner,
|
||||
type GatewayReleaseOperationRecord,
|
||||
type GatewayReleaseRepository,
|
||||
@@ -20,6 +22,7 @@ import type { ReleaseControllerConfig } from './config.js';
|
||||
const LEASE_DURATION_MS = 10 * 60_000;
|
||||
const HEARTBEAT_INTERVAL_MS = 60_000;
|
||||
const PROCESS_NAMES = ['sammo:gateway-api', 'sammo:gateway-frontend', 'sammo:gateway-orchestrator'] as const;
|
||||
const SENSITIVE_ENV_NAME = /(SECRET|TOKEN|PASSWORD|PASSWD|PRIVATE_KEY|CLIENT_SECRET|DATABASE_URL|REDIS_URL)/iu;
|
||||
|
||||
export const buildGatewayReleaseCommands = (
|
||||
workspaceRoot: string,
|
||||
@@ -108,17 +111,68 @@ export class GatewayReleaseController {
|
||||
private readonly fetchImpl: typeof fetch = fetch
|
||||
) {}
|
||||
|
||||
private sanitizeLogMessage(message: string): string {
|
||||
let sanitized = stripVTControlCharacters(message);
|
||||
const sensitiveValues = new Set([
|
||||
this.config.gatewayDatabaseUrl,
|
||||
...Object.entries(this.config.baseEnv)
|
||||
.filter(([name]) => SENSITIVE_ENV_NAME.test(name))
|
||||
.map(([, value]) => value),
|
||||
]);
|
||||
for (const secret of sensitiveValues) {
|
||||
if (secret && secret.length >= 4) sanitized = sanitized.replaceAll(secret, '[REDACTED]');
|
||||
}
|
||||
return sanitized.replace(/(:\/\/[^:\s/@]+:)[^@\s/]+@/gu, '$1[REDACTED]@').slice(0, 4_000);
|
||||
}
|
||||
|
||||
private async appendLog(
|
||||
operationId: string,
|
||||
phase: string,
|
||||
message: string,
|
||||
level: 'INFO' | 'OUTPUT' | 'ERROR' = 'INFO'
|
||||
): Promise<void> {
|
||||
try {
|
||||
await this.repository.appendOperationLog(operationId, {
|
||||
level,
|
||||
phase,
|
||||
message: this.sanitizeLogMessage(message),
|
||||
});
|
||||
} catch {
|
||||
// The first deployment that creates the log table must remain deployable.
|
||||
}
|
||||
}
|
||||
|
||||
private readonly buildProgress = (operationId: string, phase: string) => async (event: BuildProgressEvent) => {
|
||||
if (event.type === 'OUTPUT') {
|
||||
if (event.message) await this.appendLog(operationId, phase, event.message, 'OUTPUT');
|
||||
return;
|
||||
}
|
||||
const command = [event.command.command, ...event.command.args].join(' ');
|
||||
if (event.type === 'COMMAND_START') {
|
||||
await this.appendLog(operationId, phase, `$ ${command}`);
|
||||
return;
|
||||
}
|
||||
await this.appendLog(
|
||||
operationId,
|
||||
phase,
|
||||
`${command} 종료 (exit ${event.exitCode ?? 'unknown'})`,
|
||||
event.exitCode === 0 ? 'INFO' : 'ERROR'
|
||||
);
|
||||
};
|
||||
|
||||
async runOnce(): Promise<GatewayReleaseOperationRecord | null> {
|
||||
const operation = await this.repository.claimNextOperation(this.now(), {
|
||||
ownerId: this.ownerId,
|
||||
durationMs: LEASE_DURATION_MS,
|
||||
});
|
||||
if (!operation) return null;
|
||||
await this.appendLog(operation.id, 'claim', `릴리스 작업을 시작합니다. 시도 ${operation.attempts}회차.`);
|
||||
const heartbeat = setInterval(() => {
|
||||
void this.repository.renewOperationLease(operation.id, this.ownerId, this.now(), LEASE_DURATION_MS);
|
||||
}, HEARTBEAT_INTERVAL_MS);
|
||||
let resolvedCommitSha: string | undefined;
|
||||
try {
|
||||
await this.appendLog(operation.id, 'resolve', '현재 Gateway 릴리스 상태를 확인합니다.');
|
||||
const state = await this.repository.getState();
|
||||
const deploymentState: GatewayReleaseStateRecord = {
|
||||
...state,
|
||||
@@ -128,11 +182,14 @@ export class GatewayReleaseController {
|
||||
const sourceMode = operation.sourceMode ?? 'COMMIT';
|
||||
const sourceRef = operation.sourceRef ?? state.previousCommitSha;
|
||||
if (!sourceRef) throw new Error('Release source is missing.');
|
||||
await this.appendLog(operation.id, 'resolve', `${sourceMode} ${sourceRef} 커밋을 해석합니다.`);
|
||||
resolvedCommitSha = await this.workspaceManager.resolveCommit(sourceMode, sourceRef);
|
||||
if (!(await this.repository.pinOperationResolvedCommit(operation.id, this.ownerId, resolvedCommitSha))) {
|
||||
throw new Error('Gateway release lease was lost while pinning the commit.');
|
||||
}
|
||||
await this.appendLog(operation.id, 'resolve', `대상 커밋을 ${resolvedCommitSha}로 고정했습니다.`);
|
||||
await this.deploy(operation, deploymentState, resolvedCommitSha);
|
||||
await this.appendLog(operation.id, 'complete', 'Gateway 릴리스가 완료되었습니다.');
|
||||
return await this.repository.completeOperation(
|
||||
operation.id,
|
||||
'SUCCEEDED',
|
||||
@@ -141,6 +198,7 @@ export class GatewayReleaseController {
|
||||
);
|
||||
} catch (error) {
|
||||
const detail = error instanceof Error ? error.message : String(error);
|
||||
await this.appendLog(operation.id, 'failed', detail, 'ERROR');
|
||||
await this.repository.recordStateError(detail);
|
||||
return await this.repository.completeOperation(
|
||||
operation.id,
|
||||
@@ -158,31 +216,43 @@ export class GatewayReleaseController {
|
||||
state: GatewayReleaseStateRecord,
|
||||
commitSha: string
|
||||
): Promise<void> {
|
||||
await this.appendLog(operation.id, 'workspace', `커밋 ${commitSha}의 worktree를 준비합니다.`);
|
||||
const workspace = await this.workspaceManager.prepare(commitSha);
|
||||
await this.appendLog(operation.id, 'workspace', `worktree 준비 완료: ${workspace.root}`);
|
||||
const manifest = await readReleaseManifest(workspace.root);
|
||||
assertReleaseComponents(manifest, ['gateway-api', 'gateway-frontend']);
|
||||
await this.appendLog(operation.id, 'build', 'Gateway 구성 요소를 빌드합니다.');
|
||||
const build = await this.buildRunner.run(
|
||||
buildGatewayReleaseCommands(workspace.root, workspace.needsInstall, this.config)
|
||||
buildGatewayReleaseCommands(workspace.root, workspace.needsInstall, this.config),
|
||||
this.buildProgress(operation.id, 'build')
|
||||
);
|
||||
if (!build.ok) throw new Error(`Gateway release build failed: ${build.output.slice(-4000)}`);
|
||||
const migration = await this.buildRunner.run([buildGatewayMigrationCommand(workspace.root, this.config)]);
|
||||
await this.appendLog(operation.id, 'migration', 'Gateway database migration을 적용합니다.');
|
||||
const migration = await this.buildRunner.run(
|
||||
[buildGatewayMigrationCommand(workspace.root, this.config)],
|
||||
this.buildProgress(operation.id, 'migration')
|
||||
);
|
||||
if (!migration.ok) throw new Error(`Gateway migration failed: ${migration.output.slice(-4000)}`);
|
||||
await this.appendLog(operation.id, 'migration', 'Gateway database migration이 완료되었습니다.');
|
||||
|
||||
const previousDefinitions = state.activeWorkspace
|
||||
? buildGatewayProcessDefinitions(state.activeWorkspace, this.config)
|
||||
: [];
|
||||
await this.stopManagedProcesses();
|
||||
await this.appendLog(operation.id, 'switch', '기존 Gateway process를 정지합니다.');
|
||||
await this.stopManagedProcesses(operation.id);
|
||||
try {
|
||||
await this.startDefinitions(buildGatewayProcessDefinitions(workspace.root, this.config));
|
||||
await this.waitForReadiness();
|
||||
await this.startDefinitions(buildGatewayProcessDefinitions(workspace.root, this.config), operation.id);
|
||||
await this.waitForReadiness(operation.id);
|
||||
} catch (error) {
|
||||
await this.stopManagedProcesses();
|
||||
await this.appendLog(operation.id, 'rollback', '새 Gateway 시작에 실패하여 이전 process를 복구합니다.', 'ERROR');
|
||||
await this.stopManagedProcesses(operation.id);
|
||||
if (previousDefinitions.length) {
|
||||
await this.startDefinitions(previousDefinitions);
|
||||
await this.waitForReadiness();
|
||||
await this.startDefinitions(previousDefinitions, operation.id);
|
||||
await this.waitForReadiness(operation.id);
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
await this.appendLog(operation.id, 'publish', '검증된 Gateway 릴리스를 active 상태로 게시합니다.');
|
||||
await this.repository.publishRelease(operation.id, this.ownerId, {
|
||||
commitSha,
|
||||
workspace: workspace.root,
|
||||
@@ -191,10 +261,11 @@ export class GatewayReleaseController {
|
||||
});
|
||||
}
|
||||
|
||||
private async startDefinitions(definitions: ProcessDefinition[]): Promise<void> {
|
||||
private async startDefinitions(definitions: ProcessDefinition[], operationId: string): Promise<void> {
|
||||
const started: string[] = [];
|
||||
try {
|
||||
for (const definition of definitions) {
|
||||
await this.appendLog(operationId, 'switch', `${definition.name} process를 시작합니다.`);
|
||||
await this.processManager.start(definition);
|
||||
started.push(definition.name);
|
||||
}
|
||||
@@ -210,11 +281,12 @@ export class GatewayReleaseController {
|
||||
}
|
||||
}
|
||||
|
||||
private async stopManagedProcesses(): Promise<void> {
|
||||
private async stopManagedProcesses(operationId: string): Promise<void> {
|
||||
const existing = new Set((await this.processManager.list()).map((process) => process.name));
|
||||
const failures: string[] = [];
|
||||
for (const name of [...PROCESS_NAMES].reverse()) {
|
||||
if (!existing.has(name)) continue;
|
||||
await this.appendLog(operationId, 'switch', `${name} process를 정리합니다.`);
|
||||
try {
|
||||
await this.processManager.stop(name);
|
||||
} catch {
|
||||
@@ -229,7 +301,8 @@ export class GatewayReleaseController {
|
||||
if (failures.length) throw new Error(`Failed to stop gateway processes: ${failures.join('; ')}`);
|
||||
}
|
||||
|
||||
private async waitForReadiness(): Promise<void> {
|
||||
private async waitForReadiness(operationId: string): Promise<void> {
|
||||
await this.appendLog(operationId, 'readiness', 'Gateway API, frontend와 PM2 process readiness를 확인합니다.');
|
||||
const deadline = Date.now() + this.config.readinessTimeoutMs;
|
||||
const apiUrl = `http://127.0.0.1:${this.config.gatewayApiPort}/healthz`;
|
||||
const frontendUrl = `http://127.0.0.1:${this.config.gatewayFrontendPort}${this.config.gatewayBasePath}/`;
|
||||
@@ -250,6 +323,7 @@ export class GatewayReleaseController {
|
||||
safe.length === PROCESS_NAMES.length &&
|
||||
new Set(safe.map((process) => process.name)).size === PROCESS_NAMES.length
|
||||
) {
|
||||
await this.appendLog(operationId, 'readiness', 'Gateway readiness 확인을 통과했습니다.');
|
||||
return;
|
||||
}
|
||||
} catch {
|
||||
|
||||
Reference in New Issue
Block a user