devsam/core2026
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: 일괄 릴리스 실행 계획을 추가한다

Browse Source 
Gateway와 권한 있는 프로필 작업을 하나의 고정 커밋과 순서로 영속화한다. 앞 대상 성공 뒤 다음 작업을 claim하고 실패 대상은 같은 작업 ID로 재시도한다.
This commit is contained in:
Hide_D
2026-08-25 09:19:12 +00:00
parent ac48f2ae5b
commit 9613b482c1
11 changed files with 609 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/gateway-api/src/adminCapabilities.ts
+7
View File
@@ -120,6 +120,13 @@ export const resolveAdminActionCapability = (path: string, rawInput?: unknown):
if (path.endsWith('.users.createLocal')) return 'admin.users.create';
if (path.includes('.users.')) return 'admin.users.manage';
if (path.includes('.system.')) return 'admin.notice.manage';
if (path.endsWith('.bulkReleases.request')) {
const includeGateway =
rawInput && typeof rawInput === 'object'
? (rawInput as { includeGateway?: unknown }).includeGateway
: false;
return includeGateway ? 'admin.releases.manage' : 'admin.profiles.deploy';
}
if (path.includes('.releases.')) return 'admin.releases.manage';
if (path.endsWith('.profiles.requestAction') && rawInput && typeof rawInput === 'object') {
const action = (rawInput as { action?: unknown }).action;
app/gateway-api/src/adminRouter.ts
+255 -1
View File
@@ -1,4 +1,4 @@
import { randomBytes } from 'node:crypto';
import { randomBytes, randomUUID } from 'node:crypto';
import { TRPCError } from '@trpc/server';
import { z } from 'zod';
@@ -1673,6 +1673,260 @@ export const adminRouter = router({
}
}),
}),
bulkReleases: router({
targets: adminProcedure.query(async ({ ctx }) => {
const adminAuth = requireAdminAuth(ctx);
const profiles = orderGatewayProfiles(await ctx.profiles.listProfiles()).filter((profile) =>
hasScopedPermission(adminAuth, ROLE_ADMIN_PROFILE_DEPLOY, profile.profileName)
);
const activeOperations = await ctx.profiles.listOperations({
statuses: ['QUEUED', 'RUNNING'],
limit: 200,
});
const now = Date.now();
const futureResetByProfile = new Map(
activeOperations
.filter(
(operation) =>
operation.type === 'RESET' &&
operation.status === 'QUEUED' &&
Boolean(operation.scheduledAt) &&
new Date(operation.scheduledAt ?? '').getTime() > now
)
.map((operation) => [operation.profileName, operation])
);
const activeByProfile = new Map(
activeOperations
.filter((operation) => operation.id !== futureResetByProfile.get(operation.profileName)?.id)
.map((operation) => [operation.profileName, operation])
);
return {
gateway: hasScopedPermission(adminAuth, ROLE_ADMIN_RELEASES),
profiles: profiles.map((profile) => ({
profileName: profile.profileName,
displayName: resolveGatewayProfileDisplayName(
profile.profile,
profile.instanceKey,
profile.meta.korName
),
status: profile.status,
currentScenario: profile.currentScenario,
buildCommitSha: profile.buildCommitSha,
activeOperation: activeByProfile.get(profile.profileName) ?? null,
scheduledResetAt: futureResetByProfile.get(profile.profileName)?.scheduledAt,
})),
};
}),
list: adminProcedure
.input(z.object({ limit: z.number().int().min(1).max(50).default(20) }).optional())
.query(async ({ ctx, input }) => {
const adminAuth = requireAdminAuth(ctx);
const profileRecords = await ctx.profiles.listProfiles();
const profileLabels = new Map(
profileRecords.map((profile) => [
profile.profileName,
resolveGatewayProfileDisplayName(profile.profile, profile.instanceKey, profile.meta.korName),
])
);
const batches = await ctx.prisma.gatewayBulkRelease.findMany({
orderBy: { createdAt: 'desc' },
take: input?.limit ?? 20,
include: {
gatewayOperations: true,
profileOperations: true,
},
});
return batches.flatMap((batch) => {
const targets = [
...batch.gatewayOperations
.filter(() => hasScopedPermission(adminAuth, ROLE_ADMIN_RELEASES))
.map((operation) => ({
kind: 'GATEWAY' as const,
order: operation.bulkOrder ?? 0,
label: 'Gateway',
operationId: operation.id,
status: operation.status,
error: operation.error ?? undefined,
startedAt: operation.startedAt?.toISOString(),
completedAt: operation.completedAt?.toISOString(),
})),
...batch.profileOperations
.filter((operation) =>
hasScopedPermission(adminAuth, ROLE_ADMIN_PROFILE_DEPLOY, operation.profileName)
)
.map((operation) => ({
kind: 'PROFILE' as const,
order: operation.bulkOrder ?? 0,
profileName: operation.profileName,
label: profileLabels.get(operation.profileName) ?? '삭제되었거나 접근할 수 없는 서버',
operationId: operation.id,
status: operation.status,
error: operation.error ?? undefined,
startedAt: operation.startedAt?.toISOString(),
completedAt: operation.completedAt?.toISOString(),
})),
].sort((left, right) => left.order - right.order);
if (!targets.length) return [];
const statuses = targets.map((target) => target.status);
const status = statuses.every((value) => value === 'SUCCEEDED')
? 'SUCCEEDED'
: statuses.some((value) => value === 'FAILED')
? 'FAILED'
: statuses.some((value) => value === 'CANCELLED')
? 'CANCELLED'
: statuses.some((value) => value === 'RUNNING')
? 'RUNNING'
: 'QUEUED';
return [
{
id: batch.id,
sourceMode: batch.sourceMode,
sourceRef: batch.sourceRef,
resolvedCommitSha: batch.resolvedCommitSha,
reason: batch.reason ?? undefined,
requestedBy: batch.requestedBy,
createdAt: batch.createdAt.toISOString(),
status,
targets,
},
];
});
}),
request: adminProcedure
.input(
z
.object({
includeGateway: z.boolean(),
profileNames: z.array(z.string().min(1).max(64)).max(50),
sourceMode: zSourceMode,
sourceRef: z.string().trim().min(1).max(128),
reason: z.string().trim().max(200).optional(),
})
.refine((input) => input.includeGateway || input.profileNames.length > 0, {
message: 'At least one update target is required.',
})
.refine((input) => new Set(input.profileNames).size === input.profileNames.length, {
message: 'Duplicate profile targets are not allowed.',
})
)
.mutation(async ({ ctx, input }) => {
const adminAuth = requireAdminAuth(ctx);
if (input.includeGateway) assertPermission(adminAuth, ROLE_ADMIN_RELEASES);
input.profileNames.forEach((profileName) =>
assertPermission(adminAuth, ROLE_ADMIN_PROFILE_DEPLOY, profileName)
);
const profiles = orderGatewayProfiles(
(
await Promise.all(input.profileNames.map((profileName) => ctx.profiles.getProfile(profileName)))
).filter((profile): profile is NonNullable<typeof profile> => profile !== null)
);
if (profiles.length !== input.profileNames.length) {
throw new TRPCError({ code: 'NOT_FOUND', message: '선택한 서버를 찾을 수 없습니다.' });
}
let resolvedCommitSha: string;
try {
resolvedCommitSha =
input.sourceMode === 'BRANCH'
? await resolveGitBranchCommitSha(input.sourceRef)
: await resolveGitCommitSha(input.sourceRef);
if (profiles.length) {
const scenarios = await listScenarioPreviews({ gitRef: resolvedCommitSha });
const incompatibleProfile = profiles.find(
(profile) =>
profile.currentScenario === null ||
!scenarios.some((scenario) => String(scenario.id) === profile.currentScenario)
);
if (incompatibleProfile) {
throw new TRPCError({
code: 'BAD_REQUEST',
message: `${resolveGatewayProfileDisplayName(incompatibleProfile.profile, incompatibleProfile.instanceKey, incompatibleProfile.meta.korName)}의 현재 시나리오가 대상 버전에 없습니다.`,
});
}
}
} catch (error) {
if (error instanceof TRPCError) throw error;
throw new TRPCError({ code: 'BAD_REQUEST', message: '일괄 업데이트 소스가 올바르지 않습니다.' });
}
try {
return await ctx.prisma.$transaction(async (tx) => {
const batchId = randomUUID();
const batch = await tx.gatewayBulkRelease.create({
data: {
id: batchId,
sourceMode: input.sourceMode,
sourceRef: input.sourceRef,
resolvedCommitSha,
reason: input.reason,
requestedBy: adminAuth.user.id,
},
});
let order = 0;
if (input.includeGateway) {
const operation = await tx.gatewayReleaseOperation.create({
data: {
type: 'DEPLOY',
sourceMode: 'COMMIT',
sourceRef: resolvedCommitSha,
payload: { bulkReleaseId: batchId },
reason: input.reason,
requestedBy: adminAuth.user.id,
bulkReleaseId: batchId,
bulkOrder: order++,
},
});
await tx.gatewayReleaseLog.create({
data: {
operationId: operation.id,
level: 'INFO',
phase: 'queue',
message: '일괄 업데이트의 Gateway 작업이 등록되었습니다.',
},
});
}
for (const profile of profiles) {
const operation = await tx.gatewayOperation.create({
data: {
profileName: profile.profileName,
type: 'DEPLOY',
sourceMode: 'COMMIT',
sourceRef: resolvedCommitSha,
payload: {
bulkReleaseId: batchId,
releaseSource: { mode: 'COMMIT', ref: resolvedCommitSha },
},
reason: input.reason,
requestedBy: adminAuth.user.id,
bulkReleaseId: batchId,
bulkOrder: order++,
},
});
await tx.gatewayOperationLog.create({
data: {
operationId: operation.id,
level: 'INFO',
phase: 'queue',
message: '일괄 업데이트의 DB 보존 버전 업데이트가 등록되었습니다.',
},
});
}
return {
id: batch.id,
resolvedCommitSha,
targetCount: order,
};
});
} catch (error) {
if (!isUniqueConstraintError(error)) throw error;
throw new TRPCError({
code: 'CONFLICT',
message: '선택한 대상 중 이미 대기 또는 실행 중인 릴리스 작업이 있습니다.',
});
}
}),
}),
releases: router({
gatewayState: releaseAdminProcedure.query(({ ctx }) => ctx.releases.getState()),
list: releaseAdminProcedure
app/gateway-api/src/orchestrator/gatewayReleaseRepository.ts
+70 -5
View File
@@ -36,6 +36,8 @@ export interface GatewayReleaseOperationRecord {
leaseUntil?: string;
heartbeatAt?: string;
attempts: number;
bulkReleaseId?: string;
bulkOrder?: number;
createdAt: string;
updatedAt: string;
}
@@ -135,6 +137,8 @@ const mapOperation = (row: {
leaseUntil: Date | null;
heartbeatAt: Date | null;
attempts: number;
bulkReleaseId: string | null;
bulkOrder: number | null;
createdAt: Date;
updatedAt: Date;
}): GatewayReleaseOperationRecord => ({
@@ -154,6 +158,8 @@ const mapOperation = (row: {
leaseUntil: toIso(row.leaseUntil),
heartbeatAt: toIso(row.heartbeatAt),
attempts: row.attempts,
bulkReleaseId: row.bulkReleaseId ?? undefined,
bulkOrder: row.bulkOrder ?? undefined,
createdAt: row.createdAt.toISOString(),
updatedAt: row.updatedAt.toISOString(),
});
@@ -263,12 +269,40 @@ export const createGatewayReleaseRepository = (prisma: GatewayPrismaClient): Gat
if (running && !runningIsStale) {
return null;
}
const candidate =
running ??
(await tx.gatewayReleaseOperation.findFirst({
let candidate = running;
if (!candidate) {
const queuedCandidates = await tx.gatewayReleaseOperation.findMany({
where: { status: 'QUEUED' },
orderBy: { createdAt: 'asc' },
}));
orderBy: [{ createdAt: 'asc' }, { bulkOrder: 'asc' }],
take: 200,
});
for (const queuedCandidate of queuedCandidates) {
if (!queuedCandidate.bulkReleaseId || queuedCandidate.bulkOrder === null) {
candidate = queuedCandidate;
break;
}
const [blockingGatewayTargets, blockingProfileTargets] = await Promise.all([
tx.gatewayReleaseOperation.count({
where: {
bulkReleaseId: queuedCandidate.bulkReleaseId,
bulkOrder: { lt: queuedCandidate.bulkOrder },
status: { not: 'SUCCEEDED' },
},
}),
tx.gatewayOperation.count({
where: {
bulkReleaseId: queuedCandidate.bulkReleaseId,
bulkOrder: { lt: queuedCandidate.bulkOrder },
status: { not: 'SUCCEEDED' },
},
}),
]);
if (blockingGatewayTargets + blockingProfileTargets === 0) {
candidate = queuedCandidate;
break;
}
}
}
if (!candidate) {
return null;
}
@@ -421,6 +455,37 @@ export const createGatewayReleaseRepository = (prisma: GatewayPrismaClient): Gat
if (!previous || (previous.status !== 'FAILED' && previous.status !== 'CANCELLED')) {
return null;
}
if (previous.bulkReleaseId) {
const batch = await tx.gatewayBulkRelease.findUniqueOrThrow({
where: { id: previous.bulkReleaseId },
select: { resolvedCommitSha: true },
});
const operation = await tx.gatewayReleaseOperation.update({
where: { id: previous.id },
data: {
status: 'QUEUED',
sourceMode: 'COMMIT',
sourceRef: batch.resolvedCommitSha,
resolvedCommitSha: null,
requestedBy,
startedAt: null,
completedAt: null,
error: null,
leaseOwner: null,
leaseUntil: null,
heartbeatAt: null,
},
});
await tx.gatewayReleaseLog.create({
data: {
operationId: operation.id,
level: 'INFO',
phase: 'queue',
message: '일괄 업데이트의 고정 커밋으로 재시도가 등록되었습니다.',
},
});
return operation;
}
return tx.gatewayReleaseOperation.create({
data: {
type: previous.type,
app/gateway-api/src/orchestrator/profileRepository.ts
+70 -5
View File
@@ -40,6 +40,8 @@ export interface GatewayOperationRecord {
leaseUntil?: string;
heartbeatAt?: string;
attempts?: number;
bulkReleaseId?: string;
bulkOrder?: number;
createdAt: string;
updatedAt: string;
}
@@ -270,6 +272,8 @@ type GatewayOperationRow = {
leaseUntil: Date | null;
heartbeatAt: Date | null;
attempts: number;
bulkReleaseId: string | null;
bulkOrder: number | null;
createdAt: Date;
updatedAt: Date;
};
@@ -337,6 +341,8 @@ const mapOperation = (row: GatewayOperationRow): GatewayOperationRecord => ({
leaseUntil: toIso(row.leaseUntil),
heartbeatAt: toIso(row.heartbeatAt),
attempts: row.attempts,
bulkReleaseId: row.bulkReleaseId ?? undefined,
bulkOrder: row.bulkOrder ?? undefined,
createdAt: row.createdAt.toISOString(),
updatedAt: row.updatedAt.toISOString(),
});
@@ -746,15 +752,43 @@ export const createGatewayProfileRepository = (prisma: GatewayPrismaClient): Gat
})),
});
}
const candidate =
running ??
(await tx.gatewayOperation.findFirst({
let candidate = running;
if (!candidate) {
const queuedCandidates = await tx.gatewayOperation.findMany({
where: {
status: 'QUEUED',
OR: [{ scheduledAt: null }, { scheduledAt: { lte: now } }],
},
orderBy: { createdAt: 'asc' },
}));
orderBy: [{ createdAt: 'asc' }, { bulkOrder: 'asc' }],
take: 200,
});
for (const queuedCandidate of queuedCandidates) {
if (!queuedCandidate.bulkReleaseId || queuedCandidate.bulkOrder === null) {
candidate = queuedCandidate;
break;
}
const [blockingGatewayTargets, blockingProfileTargets] = await Promise.all([
tx.gatewayReleaseOperation.count({
where: {
bulkReleaseId: queuedCandidate.bulkReleaseId,
bulkOrder: { lt: queuedCandidate.bulkOrder },
status: { not: 'SUCCEEDED' },
},
}),
tx.gatewayOperation.count({
where: {
bulkReleaseId: queuedCandidate.bulkReleaseId,
bulkOrder: { lt: queuedCandidate.bulkOrder },
status: { not: 'SUCCEEDED' },
},
}),
]);
if (blockingGatewayTargets + blockingProfileTargets === 0) {
candidate = queuedCandidate;
break;
}
}
}
if (!candidate) {
return null;
}
@@ -983,6 +1017,37 @@ export const createGatewayProfileRepository = (prisma: GatewayPrismaClient): Gat
) {
throw new GatewayProfileOperationConflictError();
}
if (previous.bulkReleaseId) {
const batch = await tx.gatewayBulkRelease.findUniqueOrThrow({
where: { id: previous.bulkReleaseId },
select: { resolvedCommitSha: true },
});
const operation = await tx.gatewayOperation.update({
where: { id: previous.id },
data: {
status: 'QUEUED',
sourceMode: 'COMMIT',
sourceRef: batch.resolvedCommitSha,
resolvedCommitSha: null,
requestedBy,
startedAt: null,
completedAt: null,
error: null,
leaseOwner: null,
leaseUntil: null,
heartbeatAt: null,
},
});
await tx.gatewayOperationLog.create({
data: {
operationId: operation.id,
level: 'INFO',
phase: 'queue',
message: '일괄 업데이트의 고정 커밋으로 재시도가 등록되었습니다.',
},
});
return operation;
}
const previousPayload = previous.payload as GatewayPrisma.JsonObject;
const retrySource = buildRetryOperationSource(previous);
const operation = await tx.gatewayOperation.create({
app/gateway-api/test/adminOperations.test.ts
+24
View File
@@ -1071,6 +1071,30 @@ describe('admin operation API', () => {
);
expect(capabilities).not.toContainEqual(expect.objectContaining({ permission: 'admin.profiles.manage' }));
});
it('lists only bulk-update targets covered by the authenticated release capabilities', async () => {
const profileOperator = await buildCaller(
async () => {
throw new Error('not used');
},
{ adminRoles: ['admin.profiles.deploy:che:2'], firstUserIsAdmin: false }
);
await expect(profileOperator.caller.admin.bulkReleases.targets()).resolves.toMatchObject({
gateway: false,
profiles: [{ profileName: 'che:2' }],
});
const gatewayOperator = await buildCaller(
async () => {
throw new Error('not used');
},
{ adminRoles: ['admin.releases.manage'], firstUserIsAdmin: false }
);
await expect(gatewayOperator.caller.admin.bulkReleases.targets()).resolves.toEqual({
gateway: true,
profiles: [],
});
});
});
describe('profile operation progress API', () => {
app/gateway-api/test/profileOperationLease.integration.test.ts
+93
View File
@@ -36,6 +36,7 @@ describeDatabase('gateway operation lease and profile serialization', () => {
await connector.prisma.gatewayOperation.deleteMany({
where: { profileName: { in: [profileName, secondProfileName] } },
});
await connector.prisma.gatewayBulkRelease.deleteMany();
await connector.prisma.gatewayProfile.updateMany({
where: { profileName: { in: [profileName, secondProfileName] } },
data: { buildStatus: 'IDLE', buildError: null },
@@ -46,6 +47,7 @@ describeDatabase('gateway operation lease and profile serialization', () => {
await connector.prisma.gatewayOperation.deleteMany({
where: { profileName: { in: [profileName, secondProfileName] } },
});
await connector.prisma.gatewayBulkRelease.deleteMany();
await connector.prisma.gatewayProfile.deleteMany({
where: { profileName: { in: [profileName, secondProfileName] } },
});
@@ -270,6 +272,97 @@ describeDatabase('gateway operation lease and profile serialization', () => {
).resolves.toMatchObject({ id: profileOperation.id });
});
it('runs a bulk release in Gateway-first order and pauses later profiles until a failed target retries', async () => {
const fixedCommit = 'd'.repeat(40);
const created = await connector.prisma.$transaction(async (tx) => {
const batch = await tx.gatewayBulkRelease.create({
data: {
sourceMode: 'BRANCH',
sourceRef: 'main',
resolvedCommitSha: fixedCommit,
requestedBy: 'bulk-admin',
},
});
const gateway = await tx.gatewayReleaseOperation.create({
data: {
type: 'DEPLOY',
sourceMode: 'COMMIT',
sourceRef: fixedCommit,
requestedBy: 'bulk-admin',
bulkReleaseId: batch.id,
bulkOrder: 0,
},
});
const firstProfile = await tx.gatewayOperation.create({
data: {
profileName,
type: 'DEPLOY',
sourceMode: 'COMMIT',
sourceRef: fixedCommit,
requestedBy: 'bulk-admin',
bulkReleaseId: batch.id,
bulkOrder: 1,
},
});
const secondProfile = await tx.gatewayOperation.create({
data: {
profileName: secondProfileName,
type: 'DEPLOY',
sourceMode: 'COMMIT',
sourceRef: fixedCommit,
requestedBy: 'bulk-admin',
bulkReleaseId: batch.id,
bulkOrder: 2,
},
});
return { gateway, firstProfile, secondProfile };
});
const now = new Date('2030-01-01T00:00:00.000Z');
await expect(
repository.claimNextOperation(now, { ownerId: 'profile-worker', durationMs: 10_000 })
).resolves.toBeNull();
await expect(
releaseRepository.claimNextOperation(now, { ownerId: 'release-worker', durationMs: 10_000 })
).resolves.toMatchObject({ id: created.gateway.id, sourceRef: fixedCommit });
await releaseRepository.completeOperation(
created.gateway.id,
'SUCCEEDED',
{ resolvedCommitSha: fixedCommit, error: null },
'release-worker'
);
await expect(
repository.claimNextOperation(now, { ownerId: 'profile-worker', durationMs: 10_000 })
).resolves.toMatchObject({ id: created.firstProfile.id, sourceRef: fixedCommit });
await repository.completeOperation(
created.firstProfile.id,
'FAILED',
{ resolvedCommitSha: fixedCommit, error: 'fixture failure' },
'profile-worker'
);
await expect(
repository.claimNextOperation(now, { ownerId: 'profile-worker', durationMs: 10_000 })
).resolves.toBeNull();
await expect(repository.retryOperation(created.firstProfile.id, 'retry-admin')).resolves.toMatchObject({
id: created.firstProfile.id,
status: 'QUEUED',
sourceMode: 'COMMIT',
sourceRef: fixedCommit,
});
await repository.claimNextOperation(now, { ownerId: 'profile-worker', durationMs: 10_000 });
await repository.completeOperation(
created.firstProfile.id,
'SUCCEEDED',
{ resolvedCommitSha: fixedCommit, error: null },
'profile-worker'
);
await expect(
repository.claimNextOperation(now, { ownerId: 'profile-worker', durationMs: 10_000 })
).resolves.toMatchObject({ id: created.secondProfile.id, sourceRef: fixedCommit });
});
it('does not let a future queued operation suppress runtime reconciliation early', async () => {
const now = new Date('2030-01-01T00:00:00.000Z');
await repository.createOperation({
app/gateway-api/test/releaseManifest.test.ts
+1 -1
View File
@@ -38,7 +38,7 @@ describe('readReleaseManifest', () => {
await expect(readReleaseManifest(workspaceRoot)).resolves.toMatchObject({
controllerProtocol: RELEASE_CONTROLLER_PROTOCOL,
gatewaySchemaHead: '20260824120000_add_account_identity_management',
gatewaySchemaHead: '20260825000000_add_bulk_release_batches',
gameSchemaHead: '20260824080000_vote_utc_wall_timestamps',
});
});