Hide_D
c5a41b23a8
초기화 시작과 가오픈·정식 오픈 시각의 의미를 분리하고, 미래 가오픈 전에는 준비된 프로필을 RESERVED로 유지합니다.\n\n가오픈 시각에는 재빌드 없이 PREOPEN으로 전환하며 관리자 화면 도움말과 API·오케스트레이터·Chromium 회귀 검증을 보강합니다.
529 lines
20 KiB
TypeScript
529 lines
20 KiB
TypeScript
import { describe, expect, it } from 'vitest';
|
|
|
|
import { GatewayOrchestrator, type GatewayOrchestratorOptions } from '../src/orchestrator/gatewayOrchestrator.js';
|
|
import type { ProcessDefinition, ProcessManager } from '../src/orchestrator/processManager.js';
|
|
import type {
|
|
GatewayOperationRecord,
|
|
GatewayOperationStatus,
|
|
GatewayProfileRecord,
|
|
GatewayProfileRepository,
|
|
} from '../src/orchestrator/profileRepository.js';
|
|
import { GitWorkspaceManager } from '../src/orchestrator/workspaceManager.js';
|
|
|
|
const profile: GatewayProfileRecord = {
|
|
profileName: 'che:2',
|
|
profile: 'che',
|
|
instanceKey: '2',
|
|
currentScenario: '2',
|
|
scenario: '2',
|
|
apiPort: 15003,
|
|
status: 'STOPPED',
|
|
buildStatus: 'SUCCEEDED',
|
|
buildCommitSha: '0123456789abcdef0123456789abcdef01234567',
|
|
buildWorkspace: '/srv/sammo/worktrees/0123456789abcdef',
|
|
meta: {},
|
|
createdAt: '2026-07-25T00:00:00.000Z',
|
|
updatedAt: '2026-07-25T00:00:00.000Z',
|
|
};
|
|
|
|
const buildOperation = (type: 'START' | 'STOP'): GatewayOperationRecord => ({
|
|
id: '11111111-1111-4111-8111-111111111111',
|
|
profileName: profile.profileName,
|
|
type,
|
|
status: 'RUNNING',
|
|
payload: {},
|
|
requestedBy: 'admin',
|
|
createdAt: '2026-07-25T01:00:00.000Z',
|
|
startedAt: '2026-07-25T01:00:00.000Z',
|
|
updatedAt: '2026-07-25T01:00:00.000Z',
|
|
});
|
|
|
|
const createHarness = (
|
|
operation: GatewayOperationRecord,
|
|
failStart = false,
|
|
failStop = false,
|
|
processesPresent = true,
|
|
missingOnDelete = false,
|
|
workspaceManager?: GitWorkspaceManager,
|
|
startGate?: Promise<void>,
|
|
options: {
|
|
profile?: GatewayProfileRecord;
|
|
profiles?: GatewayProfileRecord[];
|
|
reservedToStart?: GatewayProfileRecord[];
|
|
now?: () => Date;
|
|
cancelGame?: GatewayOrchestratorOptions['cancelGame'];
|
|
} = {}
|
|
) => {
|
|
const harnessProfile = options.profile ?? profile;
|
|
let nextOperation: GatewayOperationRecord | null = operation;
|
|
const statuses: string[] = [];
|
|
const completions: GatewayOperationStatus[] = [];
|
|
const completionFields: Array<{ resolvedCommitSha?: string | null; error?: string | null } | undefined> = [];
|
|
const started: ProcessDefinition[] = [];
|
|
const stopped: string[] = [];
|
|
const deleted: string[] = [];
|
|
const buildStatuses: string[] = [];
|
|
const logs: Array<{ phase: string; message: string; level: string }> = [];
|
|
|
|
const repository: GatewayProfileRepository = {
|
|
listProfiles: async () => options.profiles ?? [harnessProfile],
|
|
getProfile: async () => harnessProfile,
|
|
upsertProfile: async () => harnessProfile,
|
|
updateCurrentScenario: async () => harnessProfile,
|
|
updateStatus: async (_profileName, status) => {
|
|
statuses.push(status);
|
|
return { ...harnessProfile, status };
|
|
},
|
|
updateBuildStatus: async (_profileName, status) => {
|
|
buildStatuses.push(status);
|
|
return { ...harnessProfile, buildStatus: status };
|
|
},
|
|
updateMeta: async () => harnessProfile,
|
|
listReservedToStart: async () => options.reservedToStart ?? [],
|
|
findQueuedBuild: async () => null,
|
|
updateLastError: async () => {},
|
|
updateWorkspaceUsage: async () => {},
|
|
clearWorkspaceUsage: async () => {},
|
|
listOperations: async () => [],
|
|
listActiveOperationProfileNames: async () => [harnessProfile.profileName],
|
|
getOperation: async () => operation,
|
|
listOperationLogs: async () => [],
|
|
appendOperationLog: async (operationId, input) => {
|
|
logs.push(input);
|
|
return {
|
|
cursor: String(logs.length),
|
|
operationId,
|
|
createdAt: '2026-08-11T00:00:00.000Z',
|
|
...input,
|
|
};
|
|
},
|
|
createOperation: async () => operation,
|
|
claimNextOperation: async () => {
|
|
const result = nextOperation;
|
|
nextOperation = null;
|
|
return result;
|
|
},
|
|
completeOperation: async (_id, status, fields) => {
|
|
completions.push(status);
|
|
completionFields.push(fields);
|
|
return { ...operation, status };
|
|
},
|
|
requeueOperation: async () => ({ ...operation, status: 'QUEUED' }),
|
|
cancelOperation: async () => false,
|
|
retryOperation: async () => null,
|
|
updateProfileForOperation: async (_id, _ownerId, _profileName, patch) => {
|
|
if (patch.status) statuses.push(patch.status);
|
|
return { ...harnessProfile, status: patch.status ?? harnessProfile.status };
|
|
},
|
|
};
|
|
const processManager: ProcessManager = {
|
|
list: async () =>
|
|
processesPresent
|
|
? [
|
|
{ name: 'sammo:che:2:game-frontend', status: 'online' },
|
|
{ name: 'sammo:che:2:game-api', status: 'online' },
|
|
{ name: 'sammo:che:2:turn-daemon', status: 'online' },
|
|
{ name: 'sammo:che:2:auction-worker', status: 'online' },
|
|
{ name: 'sammo:che:2:battle-sim-worker', status: 'online' },
|
|
{ name: 'sammo:che:2:tournament-worker', status: 'online' },
|
|
]
|
|
: [],
|
|
start: async (definition) => {
|
|
await startGate;
|
|
if (failStart && started.length === 2) {
|
|
throw new Error('pm2 unavailable');
|
|
}
|
|
started.push(definition);
|
|
},
|
|
stop: async (name) => {
|
|
stopped.push(name);
|
|
if (failStop) {
|
|
throw new Error('pm2 stop failed');
|
|
}
|
|
},
|
|
delete: async (name) => {
|
|
deleted.push(name);
|
|
if (missingOnDelete) {
|
|
throw new Error('process or namespace not found');
|
|
}
|
|
if (failStop) {
|
|
throw new Error('pm2 delete failed');
|
|
}
|
|
},
|
|
};
|
|
const orchestrator = new GatewayOrchestrator({
|
|
repository,
|
|
processManager,
|
|
buildRunner: {
|
|
run: async () => ({ ok: true, exitCode: 0, output: '' }),
|
|
},
|
|
workspaceManager:
|
|
workspaceManager ??
|
|
new GitWorkspaceManager({
|
|
repoRoot: '/tmp/not-used',
|
|
worktreeRoot: '/tmp/not-used-worktrees',
|
|
}),
|
|
processConfig: {
|
|
workspaceRoot: '/srv/sammo',
|
|
redisKeyPrefix: 'sammo:test',
|
|
gameTokenSecret: 'test-secret',
|
|
gatewayInternalApiUrl: 'http://127.0.0.1:13000',
|
|
baseEnv: { DATABASE_URL: 'postgresql://test:test@127.0.0.1:15432/test' },
|
|
},
|
|
reconcileIntervalMs: 60_000,
|
|
scheduleIntervalMs: 60_000,
|
|
buildIntervalMs: 60_000,
|
|
adminActionIntervalMs: 60_000,
|
|
now: options.now,
|
|
cancelGame: options.cancelGame,
|
|
});
|
|
|
|
return { orchestrator, statuses, buildStatuses, completions, completionFields, started, stopped, deleted, logs };
|
|
};
|
|
|
|
describe('GatewayOrchestrator first-class operations', () => {
|
|
it('stops runtime, settles once, and seals a cancelled profile', async () => {
|
|
const operation: GatewayOperationRecord = {
|
|
id: '88888888-8888-4888-8888-888888888888',
|
|
profileName: profile.profileName,
|
|
type: 'CANCEL_GAME',
|
|
status: 'RUNNING',
|
|
sourceMode: 'COMMIT',
|
|
sourceRef: profile.buildCommitSha,
|
|
resolvedCommitSha: profile.buildCommitSha,
|
|
payload: {
|
|
historyMode: 'RETAIN_ABANDONED',
|
|
generalMode: 'RETAIN',
|
|
earnedPointRetentionPercent: 40,
|
|
},
|
|
reason: '잘못 연 게임 취소',
|
|
requestedBy: 'admin',
|
|
createdAt: '2026-08-18T00:00:00.000Z',
|
|
startedAt: '2026-08-18T00:00:00.000Z',
|
|
updatedAt: '2026-08-18T00:00:00.000Z',
|
|
};
|
|
const workspaceManager = {
|
|
resolveCommit: async () => profile.buildCommitSha!,
|
|
prepare: async () => ({ root: process.cwd(), needsInstall: false }),
|
|
remove: async () => {},
|
|
} as unknown as GitWorkspaceManager;
|
|
const settlementRequests: Array<Record<string, unknown>> = [];
|
|
const cancellationProfile = { ...profile, status: 'RUNNING' as const };
|
|
const cancelGame: NonNullable<GatewayOrchestratorOptions['cancelGame']> = async (request) => {
|
|
settlementRequests.push(request as unknown as Record<string, unknown>);
|
|
return {
|
|
cancellationId: operation.id,
|
|
serverId: 'che_260818_fixture',
|
|
originalSeason: 7,
|
|
participantCount: 2,
|
|
preservedGeneralCount: 2,
|
|
historyMode: 'RETAIN_ABANDONED',
|
|
generalMode: 'RETAIN',
|
|
earnedPointRetentionPercent: 40,
|
|
alreadyApplied: false,
|
|
settlements: {},
|
|
};
|
|
};
|
|
const harness = createHarness(operation, false, false, true, false, workspaceManager, undefined, {
|
|
profile: cancellationProfile,
|
|
cancelGame,
|
|
});
|
|
|
|
await harness.orchestrator.runOperationsNow();
|
|
|
|
expect(settlementRequests).toHaveLength(1);
|
|
expect(settlementRequests[0]).toMatchObject({
|
|
cancellationId: operation.id,
|
|
cancelledBy: 'admin',
|
|
reason: '잘못 연 게임 취소',
|
|
historyMode: 'RETAIN_ABANDONED',
|
|
generalMode: 'RETAIN',
|
|
earnedPointRetentionPercent: 40,
|
|
});
|
|
expect(harness.statuses).toEqual(['STOPPED', 'CANCELLED']);
|
|
expect(harness.stopped).toHaveLength(6);
|
|
expect(harness.completions).toEqual(['SUCCEEDED']);
|
|
expect(harness.logs).toEqual(
|
|
expect.arrayContaining([
|
|
expect.objectContaining({ phase: 'settlement' }),
|
|
expect.objectContaining({ phase: 'publish', message: expect.stringContaining('CANCELLED') }),
|
|
])
|
|
);
|
|
});
|
|
|
|
it('does not let a stale START operation reopen a cancelled profile', async () => {
|
|
const cancelledProfile = { ...profile, status: 'CANCELLED' as const };
|
|
const operation = buildOperation('START');
|
|
const harness = createHarness(operation, false, false, true, false, undefined, undefined, {
|
|
profile: cancelledProfile,
|
|
});
|
|
|
|
await harness.orchestrator.runOperationsNow();
|
|
|
|
expect(harness.started).toEqual([]);
|
|
expect(harness.completions).toEqual(['FAILED']);
|
|
expect(harness.completionFields[0]?.error).toContain('CANCELLED');
|
|
});
|
|
|
|
it('does not reconcile a profile while a durable operation is active', async () => {
|
|
const harness = createHarness(buildOperation('START'));
|
|
|
|
await harness.orchestrator.reconcileNow();
|
|
|
|
expect(harness.started).toEqual([]);
|
|
expect(harness.stopped).toEqual([]);
|
|
expect(harness.deleted).toEqual([]);
|
|
});
|
|
|
|
it('opens a prepared reserved profile without rebuilding it again', async () => {
|
|
const now = new Date('2030-01-01T01:00:00.000Z');
|
|
const reservedProfile: GatewayProfileRecord = {
|
|
...profile,
|
|
status: 'RESERVED',
|
|
currentScenario: '1010',
|
|
scenario: '1010',
|
|
buildStatus: 'SUCCEEDED',
|
|
buildWorkspace: '/srv/sammo/worktrees/0123456789abcdef',
|
|
preopenAt: now.toISOString(),
|
|
openAt: '2030-01-01T02:00:00.000Z',
|
|
};
|
|
const harness = createHarness(buildOperation('START'), false, false, false, false, undefined, undefined, {
|
|
profile: reservedProfile,
|
|
profiles: [],
|
|
reservedToStart: [reservedProfile],
|
|
now: () => now,
|
|
});
|
|
|
|
await harness.orchestrator.runScheduleNow();
|
|
|
|
expect(harness.statuses).toEqual(['PREOPEN']);
|
|
expect(harness.buildStatuses).toEqual([]);
|
|
});
|
|
|
|
it('starts turns when a prepared reserved profile is handled after formal open', async () => {
|
|
const now = new Date('2030-01-01T02:00:00.000Z');
|
|
const reservedProfile: GatewayProfileRecord = {
|
|
...profile,
|
|
status: 'RESERVED',
|
|
currentScenario: '1010',
|
|
scenario: '1010',
|
|
buildStatus: 'SUCCEEDED',
|
|
buildWorkspace: '/srv/sammo/worktrees/0123456789abcdef',
|
|
preopenAt: '2030-01-01T01:00:00.000Z',
|
|
openAt: now.toISOString(),
|
|
};
|
|
const harness = createHarness(buildOperation('START'), false, false, false, false, undefined, undefined, {
|
|
profile: reservedProfile,
|
|
profiles: [],
|
|
reservedToStart: [reservedProfile],
|
|
now: () => now,
|
|
});
|
|
|
|
await harness.orchestrator.runScheduleNow();
|
|
|
|
expect(harness.statuses).toEqual(['RUNNING']);
|
|
expect(harness.buildStatuses).toEqual([]);
|
|
});
|
|
|
|
it('retains the legacy build queue for an unprepared reserved profile', async () => {
|
|
const now = new Date('2030-01-01T01:00:00.000Z');
|
|
const reservedProfile: GatewayProfileRecord = {
|
|
...profile,
|
|
status: 'RESERVED',
|
|
currentScenario: null,
|
|
scenario: 'default',
|
|
buildStatus: 'IDLE',
|
|
buildWorkspace: undefined,
|
|
preopenAt: now.toISOString(),
|
|
openAt: '2030-01-01T02:00:00.000Z',
|
|
};
|
|
const harness = createHarness(buildOperation('START'), false, false, false, false, undefined, undefined, {
|
|
profile: reservedProfile,
|
|
profiles: [],
|
|
reservedToStart: [reservedProfile],
|
|
now: () => now,
|
|
});
|
|
|
|
await harness.orchestrator.runScheduleNow();
|
|
|
|
expect(harness.statuses).toEqual([]);
|
|
expect(harness.buildStatuses).toEqual(['QUEUED']);
|
|
});
|
|
|
|
it('starts every profile process and records success', async () => {
|
|
const harness = createHarness(buildOperation('START'));
|
|
|
|
await harness.orchestrator.runOperationsNow();
|
|
|
|
expect(harness.statuses).toEqual(['RUNNING']);
|
|
expect(harness.started.map((definition) => definition.name)).toEqual([
|
|
'sammo:che:2:game-frontend',
|
|
'sammo:che:2:game-api',
|
|
'sammo:che:2:turn-daemon',
|
|
'sammo:che:2:auction-worker',
|
|
'sammo:che:2:battle-sim-worker',
|
|
'sammo:che:2:tournament-worker',
|
|
]);
|
|
expect(harness.deleted).toEqual([
|
|
'sammo:che:2:game-frontend',
|
|
'sammo:che:2:game-api',
|
|
'sammo:che:2:turn-daemon',
|
|
'sammo:che:2:auction-worker',
|
|
'sammo:che:2:battle-sim-worker',
|
|
'sammo:che:2:tournament-worker',
|
|
]);
|
|
expect(harness.completions).toEqual(['SUCCEEDED']);
|
|
});
|
|
|
|
it('stops every profile process and records success', async () => {
|
|
const harness = createHarness(buildOperation('STOP'));
|
|
|
|
await harness.orchestrator.runOperationsNow();
|
|
|
|
expect(harness.statuses).toEqual(['STOPPED']);
|
|
expect(harness.stopped).toEqual([
|
|
'sammo:che:2:game-frontend',
|
|
'sammo:che:2:game-api',
|
|
'sammo:che:2:turn-daemon',
|
|
'sammo:che:2:auction-worker',
|
|
'sammo:che:2:battle-sim-worker',
|
|
'sammo:che:2:tournament-worker',
|
|
]);
|
|
expect(harness.deleted).toEqual([
|
|
'sammo:che:2:game-frontend',
|
|
'sammo:che:2:game-api',
|
|
'sammo:che:2:turn-daemon',
|
|
'sammo:che:2:auction-worker',
|
|
'sammo:che:2:battle-sim-worker',
|
|
'sammo:che:2:tournament-worker',
|
|
]);
|
|
expect(harness.completions).toEqual(['SUCCEEDED']);
|
|
});
|
|
|
|
it('treats an already stopped profile as a successful idempotent stop', async () => {
|
|
const harness = createHarness(buildOperation('STOP'), false, false, false);
|
|
|
|
await harness.orchestrator.runOperationsNow();
|
|
|
|
expect(harness.statuses).toEqual(['STOPPED']);
|
|
expect(harness.stopped).toEqual([]);
|
|
expect(harness.deleted).toEqual([]);
|
|
expect(harness.completions).toEqual(['SUCCEEDED']);
|
|
});
|
|
|
|
it('treats a process removed concurrently as a successful stop', async () => {
|
|
const harness = createHarness(buildOperation('STOP'), false, false, true, true);
|
|
|
|
await harness.orchestrator.runOperationsNow();
|
|
|
|
expect(harness.deleted).toEqual([
|
|
'sammo:che:2:game-frontend',
|
|
'sammo:che:2:game-api',
|
|
'sammo:che:2:turn-daemon',
|
|
'sammo:che:2:auction-worker',
|
|
'sammo:che:2:battle-sim-worker',
|
|
'sammo:che:2:tournament-worker',
|
|
]);
|
|
expect(harness.completions).toEqual(['SUCCEEDED']);
|
|
});
|
|
|
|
it('records a failed start instead of reporting a false success', async () => {
|
|
const harness = createHarness(buildOperation('START'), true);
|
|
|
|
await harness.orchestrator.runOperationsNow();
|
|
|
|
expect(harness.completions).toEqual(['FAILED']);
|
|
expect(harness.deleted).toEqual([
|
|
'sammo:che:2:game-frontend',
|
|
'sammo:che:2:game-api',
|
|
'sammo:che:2:turn-daemon',
|
|
'sammo:che:2:auction-worker',
|
|
'sammo:che:2:battle-sim-worker',
|
|
'sammo:che:2:tournament-worker',
|
|
'sammo:che:2:turn-daemon',
|
|
'sammo:che:2:game-api',
|
|
'sammo:che:2:game-frontend',
|
|
]);
|
|
});
|
|
|
|
it('attempts to stop every role before reporting a partial PM2 failure', async () => {
|
|
const harness = createHarness(buildOperation('STOP'), false, true);
|
|
|
|
await harness.orchestrator.runOperationsNow();
|
|
|
|
expect(harness.stopped).toEqual([
|
|
'sammo:che:2:game-frontend',
|
|
'sammo:che:2:game-api',
|
|
'sammo:che:2:turn-daemon',
|
|
'sammo:che:2:auction-worker',
|
|
'sammo:che:2:battle-sim-worker',
|
|
'sammo:che:2:tournament-worker',
|
|
]);
|
|
expect(harness.deleted).toEqual([
|
|
'sammo:che:2:game-frontend',
|
|
'sammo:che:2:game-api',
|
|
'sammo:che:2:turn-daemon',
|
|
'sammo:che:2:auction-worker',
|
|
'sammo:che:2:battle-sim-worker',
|
|
'sammo:che:2:tournament-worker',
|
|
]);
|
|
expect(harness.completions).toEqual(['FAILED']);
|
|
});
|
|
|
|
it('records the resolved commit even when reset workspace preparation fails', async () => {
|
|
const resolvedCommitSha = 'abcdef0123456789abcdef0123456789abcdef01';
|
|
const resetOperation: GatewayOperationRecord = {
|
|
id: '33333333-3333-4333-8333-333333333333',
|
|
profileName: profile.profileName,
|
|
type: 'RESET',
|
|
status: 'RUNNING',
|
|
sourceMode: 'COMMIT',
|
|
sourceRef: 'requested-ref',
|
|
payload: { install: { scenarioId: 1010, turnTermMinutes: 60 } },
|
|
requestedBy: 'admin',
|
|
createdAt: '2026-07-31T00:00:00.000Z',
|
|
startedAt: '2026-07-31T00:00:00.000Z',
|
|
updatedAt: '2026-07-31T00:00:00.000Z',
|
|
};
|
|
const workspaceManager = {
|
|
resolveCommit: async () => resolvedCommitSha,
|
|
prepare: async () => {
|
|
throw new Error('injected workspace preparation failure');
|
|
},
|
|
remove: async () => {},
|
|
} as unknown as GitWorkspaceManager;
|
|
const harness = createHarness(resetOperation, false, false, true, false, workspaceManager);
|
|
|
|
await harness.orchestrator.runOperationsNow();
|
|
|
|
expect(harness.completions).toEqual(['FAILED']);
|
|
expect(harness.completionFields).toEqual([
|
|
{
|
|
resolvedCommitSha,
|
|
error: 'injected workspace preparation failure',
|
|
},
|
|
]);
|
|
});
|
|
|
|
it('drains an in-flight operation before shutdown completes', async () => {
|
|
let releaseStart: (() => void) | undefined;
|
|
const startGate = new Promise<void>((resolve) => {
|
|
releaseStart = resolve;
|
|
});
|
|
const harness = createHarness(buildOperation('START'), false, false, true, false, undefined, startGate);
|
|
harness.orchestrator.start();
|
|
await new Promise<void>((resolve) => setImmediate(resolve));
|
|
|
|
let stopped = false;
|
|
const stopPromise = harness.orchestrator.stop().then(() => {
|
|
stopped = true;
|
|
});
|
|
await new Promise<void>((resolve) => setImmediate(resolve));
|
|
expect(stopped).toBe(false);
|
|
|
|
releaseStart?.();
|
|
await stopPromise;
|
|
expect(harness.completions).toEqual(['SUCCEEDED']);
|
|
});
|
|
});
|