feat: 일괄 릴리스 실행 계획을 추가한다
Gateway와 권한 있는 프로필 작업을 하나의 고정 커밋과 순서로 영속화한다. 앞 대상 성공 뒤 다음 작업을 claim하고 실패 대상은 같은 작업 ID로 재시도한다.
This commit is contained in:
@@ -120,6 +120,13 @@ export const resolveAdminActionCapability = (path: string, rawInput?: unknown):
|
||||
if (path.endsWith('.users.createLocal')) return 'admin.users.create';
|
||||
if (path.includes('.users.')) return 'admin.users.manage';
|
||||
if (path.includes('.system.')) return 'admin.notice.manage';
|
||||
if (path.endsWith('.bulkReleases.request')) {
|
||||
const includeGateway =
|
||||
rawInput && typeof rawInput === 'object'
|
||||
? (rawInput as { includeGateway?: unknown }).includeGateway
|
||||
: false;
|
||||
return includeGateway ? 'admin.releases.manage' : 'admin.profiles.deploy';
|
||||
}
|
||||
if (path.includes('.releases.')) return 'admin.releases.manage';
|
||||
if (path.endsWith('.profiles.requestAction') && rawInput && typeof rawInput === 'object') {
|
||||
const action = (rawInput as { action?: unknown }).action;
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { randomBytes } from 'node:crypto';
|
||||
import { randomBytes, randomUUID } from 'node:crypto';
|
||||
|
||||
import { TRPCError } from '@trpc/server';
|
||||
import { z } from 'zod';
|
||||
@@ -1673,6 +1673,260 @@ export const adminRouter = router({
|
||||
}
|
||||
}),
|
||||
}),
|
||||
bulkReleases: router({
|
||||
targets: adminProcedure.query(async ({ ctx }) => {
|
||||
const adminAuth = requireAdminAuth(ctx);
|
||||
const profiles = orderGatewayProfiles(await ctx.profiles.listProfiles()).filter((profile) =>
|
||||
hasScopedPermission(adminAuth, ROLE_ADMIN_PROFILE_DEPLOY, profile.profileName)
|
||||
);
|
||||
const activeOperations = await ctx.profiles.listOperations({
|
||||
statuses: ['QUEUED', 'RUNNING'],
|
||||
limit: 200,
|
||||
});
|
||||
const now = Date.now();
|
||||
const futureResetByProfile = new Map(
|
||||
activeOperations
|
||||
.filter(
|
||||
(operation) =>
|
||||
operation.type === 'RESET' &&
|
||||
operation.status === 'QUEUED' &&
|
||||
Boolean(operation.scheduledAt) &&
|
||||
new Date(operation.scheduledAt ?? '').getTime() > now
|
||||
)
|
||||
.map((operation) => [operation.profileName, operation])
|
||||
);
|
||||
const activeByProfile = new Map(
|
||||
activeOperations
|
||||
.filter((operation) => operation.id !== futureResetByProfile.get(operation.profileName)?.id)
|
||||
.map((operation) => [operation.profileName, operation])
|
||||
);
|
||||
return {
|
||||
gateway: hasScopedPermission(adminAuth, ROLE_ADMIN_RELEASES),
|
||||
profiles: profiles.map((profile) => ({
|
||||
profileName: profile.profileName,
|
||||
displayName: resolveGatewayProfileDisplayName(
|
||||
profile.profile,
|
||||
profile.instanceKey,
|
||||
profile.meta.korName
|
||||
),
|
||||
status: profile.status,
|
||||
currentScenario: profile.currentScenario,
|
||||
buildCommitSha: profile.buildCommitSha,
|
||||
activeOperation: activeByProfile.get(profile.profileName) ?? null,
|
||||
scheduledResetAt: futureResetByProfile.get(profile.profileName)?.scheduledAt,
|
||||
})),
|
||||
};
|
||||
}),
|
||||
list: adminProcedure
|
||||
.input(z.object({ limit: z.number().int().min(1).max(50).default(20) }).optional())
|
||||
.query(async ({ ctx, input }) => {
|
||||
const adminAuth = requireAdminAuth(ctx);
|
||||
const profileRecords = await ctx.profiles.listProfiles();
|
||||
const profileLabels = new Map(
|
||||
profileRecords.map((profile) => [
|
||||
profile.profileName,
|
||||
resolveGatewayProfileDisplayName(profile.profile, profile.instanceKey, profile.meta.korName),
|
||||
])
|
||||
);
|
||||
const batches = await ctx.prisma.gatewayBulkRelease.findMany({
|
||||
orderBy: { createdAt: 'desc' },
|
||||
take: input?.limit ?? 20,
|
||||
include: {
|
||||
gatewayOperations: true,
|
||||
profileOperations: true,
|
||||
},
|
||||
});
|
||||
return batches.flatMap((batch) => {
|
||||
const targets = [
|
||||
...batch.gatewayOperations
|
||||
.filter(() => hasScopedPermission(adminAuth, ROLE_ADMIN_RELEASES))
|
||||
.map((operation) => ({
|
||||
kind: 'GATEWAY' as const,
|
||||
order: operation.bulkOrder ?? 0,
|
||||
label: 'Gateway',
|
||||
operationId: operation.id,
|
||||
status: operation.status,
|
||||
error: operation.error ?? undefined,
|
||||
startedAt: operation.startedAt?.toISOString(),
|
||||
completedAt: operation.completedAt?.toISOString(),
|
||||
})),
|
||||
...batch.profileOperations
|
||||
.filter((operation) =>
|
||||
hasScopedPermission(adminAuth, ROLE_ADMIN_PROFILE_DEPLOY, operation.profileName)
|
||||
)
|
||||
.map((operation) => ({
|
||||
kind: 'PROFILE' as const,
|
||||
order: operation.bulkOrder ?? 0,
|
||||
profileName: operation.profileName,
|
||||
label: profileLabels.get(operation.profileName) ?? '삭제되었거나 접근할 수 없는 서버',
|
||||
operationId: operation.id,
|
||||
status: operation.status,
|
||||
error: operation.error ?? undefined,
|
||||
startedAt: operation.startedAt?.toISOString(),
|
||||
completedAt: operation.completedAt?.toISOString(),
|
||||
})),
|
||||
].sort((left, right) => left.order - right.order);
|
||||
if (!targets.length) return [];
|
||||
const statuses = targets.map((target) => target.status);
|
||||
const status = statuses.every((value) => value === 'SUCCEEDED')
|
||||
? 'SUCCEEDED'
|
||||
: statuses.some((value) => value === 'FAILED')
|
||||
? 'FAILED'
|
||||
: statuses.some((value) => value === 'CANCELLED')
|
||||
? 'CANCELLED'
|
||||
: statuses.some((value) => value === 'RUNNING')
|
||||
? 'RUNNING'
|
||||
: 'QUEUED';
|
||||
return [
|
||||
{
|
||||
id: batch.id,
|
||||
sourceMode: batch.sourceMode,
|
||||
sourceRef: batch.sourceRef,
|
||||
resolvedCommitSha: batch.resolvedCommitSha,
|
||||
reason: batch.reason ?? undefined,
|
||||
requestedBy: batch.requestedBy,
|
||||
createdAt: batch.createdAt.toISOString(),
|
||||
status,
|
||||
targets,
|
||||
},
|
||||
];
|
||||
});
|
||||
}),
|
||||
request: adminProcedure
|
||||
.input(
|
||||
z
|
||||
.object({
|
||||
includeGateway: z.boolean(),
|
||||
profileNames: z.array(z.string().min(1).max(64)).max(50),
|
||||
sourceMode: zSourceMode,
|
||||
sourceRef: z.string().trim().min(1).max(128),
|
||||
reason: z.string().trim().max(200).optional(),
|
||||
})
|
||||
.refine((input) => input.includeGateway || input.profileNames.length > 0, {
|
||||
message: 'At least one update target is required.',
|
||||
})
|
||||
.refine((input) => new Set(input.profileNames).size === input.profileNames.length, {
|
||||
message: 'Duplicate profile targets are not allowed.',
|
||||
})
|
||||
)
|
||||
.mutation(async ({ ctx, input }) => {
|
||||
const adminAuth = requireAdminAuth(ctx);
|
||||
if (input.includeGateway) assertPermission(adminAuth, ROLE_ADMIN_RELEASES);
|
||||
input.profileNames.forEach((profileName) =>
|
||||
assertPermission(adminAuth, ROLE_ADMIN_PROFILE_DEPLOY, profileName)
|
||||
);
|
||||
|
||||
const profiles = orderGatewayProfiles(
|
||||
(
|
||||
await Promise.all(input.profileNames.map((profileName) => ctx.profiles.getProfile(profileName)))
|
||||
).filter((profile): profile is NonNullable<typeof profile> => profile !== null)
|
||||
);
|
||||
if (profiles.length !== input.profileNames.length) {
|
||||
throw new TRPCError({ code: 'NOT_FOUND', message: '선택한 서버를 찾을 수 없습니다.' });
|
||||
}
|
||||
|
||||
let resolvedCommitSha: string;
|
||||
try {
|
||||
resolvedCommitSha =
|
||||
input.sourceMode === 'BRANCH'
|
||||
? await resolveGitBranchCommitSha(input.sourceRef)
|
||||
: await resolveGitCommitSha(input.sourceRef);
|
||||
if (profiles.length) {
|
||||
const scenarios = await listScenarioPreviews({ gitRef: resolvedCommitSha });
|
||||
const incompatibleProfile = profiles.find(
|
||||
(profile) =>
|
||||
profile.currentScenario === null ||
|
||||
!scenarios.some((scenario) => String(scenario.id) === profile.currentScenario)
|
||||
);
|
||||
if (incompatibleProfile) {
|
||||
throw new TRPCError({
|
||||
code: 'BAD_REQUEST',
|
||||
message: `${resolveGatewayProfileDisplayName(incompatibleProfile.profile, incompatibleProfile.instanceKey, incompatibleProfile.meta.korName)}의 현재 시나리오가 대상 버전에 없습니다.`,
|
||||
});
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
if (error instanceof TRPCError) throw error;
|
||||
throw new TRPCError({ code: 'BAD_REQUEST', message: '일괄 업데이트 소스가 올바르지 않습니다.' });
|
||||
}
|
||||
|
||||
try {
|
||||
return await ctx.prisma.$transaction(async (tx) => {
|
||||
const batchId = randomUUID();
|
||||
const batch = await tx.gatewayBulkRelease.create({
|
||||
data: {
|
||||
id: batchId,
|
||||
sourceMode: input.sourceMode,
|
||||
sourceRef: input.sourceRef,
|
||||
resolvedCommitSha,
|
||||
reason: input.reason,
|
||||
requestedBy: adminAuth.user.id,
|
||||
},
|
||||
});
|
||||
let order = 0;
|
||||
if (input.includeGateway) {
|
||||
const operation = await tx.gatewayReleaseOperation.create({
|
||||
data: {
|
||||
type: 'DEPLOY',
|
||||
sourceMode: 'COMMIT',
|
||||
sourceRef: resolvedCommitSha,
|
||||
payload: { bulkReleaseId: batchId },
|
||||
reason: input.reason,
|
||||
requestedBy: adminAuth.user.id,
|
||||
bulkReleaseId: batchId,
|
||||
bulkOrder: order++,
|
||||
},
|
||||
});
|
||||
await tx.gatewayReleaseLog.create({
|
||||
data: {
|
||||
operationId: operation.id,
|
||||
level: 'INFO',
|
||||
phase: 'queue',
|
||||
message: '일괄 업데이트의 Gateway 작업이 등록되었습니다.',
|
||||
},
|
||||
});
|
||||
}
|
||||
for (const profile of profiles) {
|
||||
const operation = await tx.gatewayOperation.create({
|
||||
data: {
|
||||
profileName: profile.profileName,
|
||||
type: 'DEPLOY',
|
||||
sourceMode: 'COMMIT',
|
||||
sourceRef: resolvedCommitSha,
|
||||
payload: {
|
||||
bulkReleaseId: batchId,
|
||||
releaseSource: { mode: 'COMMIT', ref: resolvedCommitSha },
|
||||
},
|
||||
reason: input.reason,
|
||||
requestedBy: adminAuth.user.id,
|
||||
bulkReleaseId: batchId,
|
||||
bulkOrder: order++,
|
||||
},
|
||||
});
|
||||
await tx.gatewayOperationLog.create({
|
||||
data: {
|
||||
operationId: operation.id,
|
||||
level: 'INFO',
|
||||
phase: 'queue',
|
||||
message: '일괄 업데이트의 DB 보존 버전 업데이트가 등록되었습니다.',
|
||||
},
|
||||
});
|
||||
}
|
||||
return {
|
||||
id: batch.id,
|
||||
resolvedCommitSha,
|
||||
targetCount: order,
|
||||
};
|
||||
});
|
||||
} catch (error) {
|
||||
if (!isUniqueConstraintError(error)) throw error;
|
||||
throw new TRPCError({
|
||||
code: 'CONFLICT',
|
||||
message: '선택한 대상 중 이미 대기 또는 실행 중인 릴리스 작업이 있습니다.',
|
||||
});
|
||||
}
|
||||
}),
|
||||
}),
|
||||
releases: router({
|
||||
gatewayState: releaseAdminProcedure.query(({ ctx }) => ctx.releases.getState()),
|
||||
list: releaseAdminProcedure
|
||||
|
||||
@@ -36,6 +36,8 @@ export interface GatewayReleaseOperationRecord {
|
||||
leaseUntil?: string;
|
||||
heartbeatAt?: string;
|
||||
attempts: number;
|
||||
bulkReleaseId?: string;
|
||||
bulkOrder?: number;
|
||||
createdAt: string;
|
||||
updatedAt: string;
|
||||
}
|
||||
@@ -135,6 +137,8 @@ const mapOperation = (row: {
|
||||
leaseUntil: Date | null;
|
||||
heartbeatAt: Date | null;
|
||||
attempts: number;
|
||||
bulkReleaseId: string | null;
|
||||
bulkOrder: number | null;
|
||||
createdAt: Date;
|
||||
updatedAt: Date;
|
||||
}): GatewayReleaseOperationRecord => ({
|
||||
@@ -154,6 +158,8 @@ const mapOperation = (row: {
|
||||
leaseUntil: toIso(row.leaseUntil),
|
||||
heartbeatAt: toIso(row.heartbeatAt),
|
||||
attempts: row.attempts,
|
||||
bulkReleaseId: row.bulkReleaseId ?? undefined,
|
||||
bulkOrder: row.bulkOrder ?? undefined,
|
||||
createdAt: row.createdAt.toISOString(),
|
||||
updatedAt: row.updatedAt.toISOString(),
|
||||
});
|
||||
@@ -263,12 +269,40 @@ export const createGatewayReleaseRepository = (prisma: GatewayPrismaClient): Gat
|
||||
if (running && !runningIsStale) {
|
||||
return null;
|
||||
}
|
||||
const candidate =
|
||||
running ??
|
||||
(await tx.gatewayReleaseOperation.findFirst({
|
||||
let candidate = running;
|
||||
if (!candidate) {
|
||||
const queuedCandidates = await tx.gatewayReleaseOperation.findMany({
|
||||
where: { status: 'QUEUED' },
|
||||
orderBy: { createdAt: 'asc' },
|
||||
}));
|
||||
orderBy: [{ createdAt: 'asc' }, { bulkOrder: 'asc' }],
|
||||
take: 200,
|
||||
});
|
||||
for (const queuedCandidate of queuedCandidates) {
|
||||
if (!queuedCandidate.bulkReleaseId || queuedCandidate.bulkOrder === null) {
|
||||
candidate = queuedCandidate;
|
||||
break;
|
||||
}
|
||||
const [blockingGatewayTargets, blockingProfileTargets] = await Promise.all([
|
||||
tx.gatewayReleaseOperation.count({
|
||||
where: {
|
||||
bulkReleaseId: queuedCandidate.bulkReleaseId,
|
||||
bulkOrder: { lt: queuedCandidate.bulkOrder },
|
||||
status: { not: 'SUCCEEDED' },
|
||||
},
|
||||
}),
|
||||
tx.gatewayOperation.count({
|
||||
where: {
|
||||
bulkReleaseId: queuedCandidate.bulkReleaseId,
|
||||
bulkOrder: { lt: queuedCandidate.bulkOrder },
|
||||
status: { not: 'SUCCEEDED' },
|
||||
},
|
||||
}),
|
||||
]);
|
||||
if (blockingGatewayTargets + blockingProfileTargets === 0) {
|
||||
candidate = queuedCandidate;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (!candidate) {
|
||||
return null;
|
||||
}
|
||||
@@ -421,6 +455,37 @@ export const createGatewayReleaseRepository = (prisma: GatewayPrismaClient): Gat
|
||||
if (!previous || (previous.status !== 'FAILED' && previous.status !== 'CANCELLED')) {
|
||||
return null;
|
||||
}
|
||||
if (previous.bulkReleaseId) {
|
||||
const batch = await tx.gatewayBulkRelease.findUniqueOrThrow({
|
||||
where: { id: previous.bulkReleaseId },
|
||||
select: { resolvedCommitSha: true },
|
||||
});
|
||||
const operation = await tx.gatewayReleaseOperation.update({
|
||||
where: { id: previous.id },
|
||||
data: {
|
||||
status: 'QUEUED',
|
||||
sourceMode: 'COMMIT',
|
||||
sourceRef: batch.resolvedCommitSha,
|
||||
resolvedCommitSha: null,
|
||||
requestedBy,
|
||||
startedAt: null,
|
||||
completedAt: null,
|
||||
error: null,
|
||||
leaseOwner: null,
|
||||
leaseUntil: null,
|
||||
heartbeatAt: null,
|
||||
},
|
||||
});
|
||||
await tx.gatewayReleaseLog.create({
|
||||
data: {
|
||||
operationId: operation.id,
|
||||
level: 'INFO',
|
||||
phase: 'queue',
|
||||
message: '일괄 업데이트의 고정 커밋으로 재시도가 등록되었습니다.',
|
||||
},
|
||||
});
|
||||
return operation;
|
||||
}
|
||||
return tx.gatewayReleaseOperation.create({
|
||||
data: {
|
||||
type: previous.type,
|
||||
|
||||
@@ -40,6 +40,8 @@ export interface GatewayOperationRecord {
|
||||
leaseUntil?: string;
|
||||
heartbeatAt?: string;
|
||||
attempts?: number;
|
||||
bulkReleaseId?: string;
|
||||
bulkOrder?: number;
|
||||
createdAt: string;
|
||||
updatedAt: string;
|
||||
}
|
||||
@@ -270,6 +272,8 @@ type GatewayOperationRow = {
|
||||
leaseUntil: Date | null;
|
||||
heartbeatAt: Date | null;
|
||||
attempts: number;
|
||||
bulkReleaseId: string | null;
|
||||
bulkOrder: number | null;
|
||||
createdAt: Date;
|
||||
updatedAt: Date;
|
||||
};
|
||||
@@ -337,6 +341,8 @@ const mapOperation = (row: GatewayOperationRow): GatewayOperationRecord => ({
|
||||
leaseUntil: toIso(row.leaseUntil),
|
||||
heartbeatAt: toIso(row.heartbeatAt),
|
||||
attempts: row.attempts,
|
||||
bulkReleaseId: row.bulkReleaseId ?? undefined,
|
||||
bulkOrder: row.bulkOrder ?? undefined,
|
||||
createdAt: row.createdAt.toISOString(),
|
||||
updatedAt: row.updatedAt.toISOString(),
|
||||
});
|
||||
@@ -746,15 +752,43 @@ export const createGatewayProfileRepository = (prisma: GatewayPrismaClient): Gat
|
||||
})),
|
||||
});
|
||||
}
|
||||
const candidate =
|
||||
running ??
|
||||
(await tx.gatewayOperation.findFirst({
|
||||
let candidate = running;
|
||||
if (!candidate) {
|
||||
const queuedCandidates = await tx.gatewayOperation.findMany({
|
||||
where: {
|
||||
status: 'QUEUED',
|
||||
OR: [{ scheduledAt: null }, { scheduledAt: { lte: now } }],
|
||||
},
|
||||
orderBy: { createdAt: 'asc' },
|
||||
}));
|
||||
orderBy: [{ createdAt: 'asc' }, { bulkOrder: 'asc' }],
|
||||
take: 200,
|
||||
});
|
||||
for (const queuedCandidate of queuedCandidates) {
|
||||
if (!queuedCandidate.bulkReleaseId || queuedCandidate.bulkOrder === null) {
|
||||
candidate = queuedCandidate;
|
||||
break;
|
||||
}
|
||||
const [blockingGatewayTargets, blockingProfileTargets] = await Promise.all([
|
||||
tx.gatewayReleaseOperation.count({
|
||||
where: {
|
||||
bulkReleaseId: queuedCandidate.bulkReleaseId,
|
||||
bulkOrder: { lt: queuedCandidate.bulkOrder },
|
||||
status: { not: 'SUCCEEDED' },
|
||||
},
|
||||
}),
|
||||
tx.gatewayOperation.count({
|
||||
where: {
|
||||
bulkReleaseId: queuedCandidate.bulkReleaseId,
|
||||
bulkOrder: { lt: queuedCandidate.bulkOrder },
|
||||
status: { not: 'SUCCEEDED' },
|
||||
},
|
||||
}),
|
||||
]);
|
||||
if (blockingGatewayTargets + blockingProfileTargets === 0) {
|
||||
candidate = queuedCandidate;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (!candidate) {
|
||||
return null;
|
||||
}
|
||||
@@ -983,6 +1017,37 @@ export const createGatewayProfileRepository = (prisma: GatewayPrismaClient): Gat
|
||||
) {
|
||||
throw new GatewayProfileOperationConflictError();
|
||||
}
|
||||
if (previous.bulkReleaseId) {
|
||||
const batch = await tx.gatewayBulkRelease.findUniqueOrThrow({
|
||||
where: { id: previous.bulkReleaseId },
|
||||
select: { resolvedCommitSha: true },
|
||||
});
|
||||
const operation = await tx.gatewayOperation.update({
|
||||
where: { id: previous.id },
|
||||
data: {
|
||||
status: 'QUEUED',
|
||||
sourceMode: 'COMMIT',
|
||||
sourceRef: batch.resolvedCommitSha,
|
||||
resolvedCommitSha: null,
|
||||
requestedBy,
|
||||
startedAt: null,
|
||||
completedAt: null,
|
||||
error: null,
|
||||
leaseOwner: null,
|
||||
leaseUntil: null,
|
||||
heartbeatAt: null,
|
||||
},
|
||||
});
|
||||
await tx.gatewayOperationLog.create({
|
||||
data: {
|
||||
operationId: operation.id,
|
||||
level: 'INFO',
|
||||
phase: 'queue',
|
||||
message: '일괄 업데이트의 고정 커밋으로 재시도가 등록되었습니다.',
|
||||
},
|
||||
});
|
||||
return operation;
|
||||
}
|
||||
const previousPayload = previous.payload as GatewayPrisma.JsonObject;
|
||||
const retrySource = buildRetryOperationSource(previous);
|
||||
const operation = await tx.gatewayOperation.create({
|
||||
|
||||
@@ -1071,6 +1071,30 @@ describe('admin operation API', () => {
|
||||
);
|
||||
expect(capabilities).not.toContainEqual(expect.objectContaining({ permission: 'admin.profiles.manage' }));
|
||||
});
|
||||
|
||||
it('lists only bulk-update targets covered by the authenticated release capabilities', async () => {
|
||||
const profileOperator = await buildCaller(
|
||||
async () => {
|
||||
throw new Error('not used');
|
||||
},
|
||||
{ adminRoles: ['admin.profiles.deploy:che:2'], firstUserIsAdmin: false }
|
||||
);
|
||||
await expect(profileOperator.caller.admin.bulkReleases.targets()).resolves.toMatchObject({
|
||||
gateway: false,
|
||||
profiles: [{ profileName: 'che:2' }],
|
||||
});
|
||||
|
||||
const gatewayOperator = await buildCaller(
|
||||
async () => {
|
||||
throw new Error('not used');
|
||||
},
|
||||
{ adminRoles: ['admin.releases.manage'], firstUserIsAdmin: false }
|
||||
);
|
||||
await expect(gatewayOperator.caller.admin.bulkReleases.targets()).resolves.toEqual({
|
||||
gateway: true,
|
||||
profiles: [],
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('profile operation progress API', () => {
|
||||
|
||||
@@ -36,6 +36,7 @@ describeDatabase('gateway operation lease and profile serialization', () => {
|
||||
await connector.prisma.gatewayOperation.deleteMany({
|
||||
where: { profileName: { in: [profileName, secondProfileName] } },
|
||||
});
|
||||
await connector.prisma.gatewayBulkRelease.deleteMany();
|
||||
await connector.prisma.gatewayProfile.updateMany({
|
||||
where: { profileName: { in: [profileName, secondProfileName] } },
|
||||
data: { buildStatus: 'IDLE', buildError: null },
|
||||
@@ -46,6 +47,7 @@ describeDatabase('gateway operation lease and profile serialization', () => {
|
||||
await connector.prisma.gatewayOperation.deleteMany({
|
||||
where: { profileName: { in: [profileName, secondProfileName] } },
|
||||
});
|
||||
await connector.prisma.gatewayBulkRelease.deleteMany();
|
||||
await connector.prisma.gatewayProfile.deleteMany({
|
||||
where: { profileName: { in: [profileName, secondProfileName] } },
|
||||
});
|
||||
@@ -270,6 +272,97 @@ describeDatabase('gateway operation lease and profile serialization', () => {
|
||||
).resolves.toMatchObject({ id: profileOperation.id });
|
||||
});
|
||||
|
||||
it('runs a bulk release in Gateway-first order and pauses later profiles until a failed target retries', async () => {
|
||||
const fixedCommit = 'd'.repeat(40);
|
||||
const created = await connector.prisma.$transaction(async (tx) => {
|
||||
const batch = await tx.gatewayBulkRelease.create({
|
||||
data: {
|
||||
sourceMode: 'BRANCH',
|
||||
sourceRef: 'main',
|
||||
resolvedCommitSha: fixedCommit,
|
||||
requestedBy: 'bulk-admin',
|
||||
},
|
||||
});
|
||||
const gateway = await tx.gatewayReleaseOperation.create({
|
||||
data: {
|
||||
type: 'DEPLOY',
|
||||
sourceMode: 'COMMIT',
|
||||
sourceRef: fixedCommit,
|
||||
requestedBy: 'bulk-admin',
|
||||
bulkReleaseId: batch.id,
|
||||
bulkOrder: 0,
|
||||
},
|
||||
});
|
||||
const firstProfile = await tx.gatewayOperation.create({
|
||||
data: {
|
||||
profileName,
|
||||
type: 'DEPLOY',
|
||||
sourceMode: 'COMMIT',
|
||||
sourceRef: fixedCommit,
|
||||
requestedBy: 'bulk-admin',
|
||||
bulkReleaseId: batch.id,
|
||||
bulkOrder: 1,
|
||||
},
|
||||
});
|
||||
const secondProfile = await tx.gatewayOperation.create({
|
||||
data: {
|
||||
profileName: secondProfileName,
|
||||
type: 'DEPLOY',
|
||||
sourceMode: 'COMMIT',
|
||||
sourceRef: fixedCommit,
|
||||
requestedBy: 'bulk-admin',
|
||||
bulkReleaseId: batch.id,
|
||||
bulkOrder: 2,
|
||||
},
|
||||
});
|
||||
return { gateway, firstProfile, secondProfile };
|
||||
});
|
||||
const now = new Date('2030-01-01T00:00:00.000Z');
|
||||
|
||||
await expect(
|
||||
repository.claimNextOperation(now, { ownerId: 'profile-worker', durationMs: 10_000 })
|
||||
).resolves.toBeNull();
|
||||
await expect(
|
||||
releaseRepository.claimNextOperation(now, { ownerId: 'release-worker', durationMs: 10_000 })
|
||||
).resolves.toMatchObject({ id: created.gateway.id, sourceRef: fixedCommit });
|
||||
await releaseRepository.completeOperation(
|
||||
created.gateway.id,
|
||||
'SUCCEEDED',
|
||||
{ resolvedCommitSha: fixedCommit, error: null },
|
||||
'release-worker'
|
||||
);
|
||||
|
||||
await expect(
|
||||
repository.claimNextOperation(now, { ownerId: 'profile-worker', durationMs: 10_000 })
|
||||
).resolves.toMatchObject({ id: created.firstProfile.id, sourceRef: fixedCommit });
|
||||
await repository.completeOperation(
|
||||
created.firstProfile.id,
|
||||
'FAILED',
|
||||
{ resolvedCommitSha: fixedCommit, error: 'fixture failure' },
|
||||
'profile-worker'
|
||||
);
|
||||
await expect(
|
||||
repository.claimNextOperation(now, { ownerId: 'profile-worker', durationMs: 10_000 })
|
||||
).resolves.toBeNull();
|
||||
|
||||
await expect(repository.retryOperation(created.firstProfile.id, 'retry-admin')).resolves.toMatchObject({
|
||||
id: created.firstProfile.id,
|
||||
status: 'QUEUED',
|
||||
sourceMode: 'COMMIT',
|
||||
sourceRef: fixedCommit,
|
||||
});
|
||||
await repository.claimNextOperation(now, { ownerId: 'profile-worker', durationMs: 10_000 });
|
||||
await repository.completeOperation(
|
||||
created.firstProfile.id,
|
||||
'SUCCEEDED',
|
||||
{ resolvedCommitSha: fixedCommit, error: null },
|
||||
'profile-worker'
|
||||
);
|
||||
await expect(
|
||||
repository.claimNextOperation(now, { ownerId: 'profile-worker', durationMs: 10_000 })
|
||||
).resolves.toMatchObject({ id: created.secondProfile.id, sourceRef: fixedCommit });
|
||||
});
|
||||
|
||||
it('does not let a future queued operation suppress runtime reconciliation early', async () => {
|
||||
const now = new Date('2030-01-01T00:00:00.000Z');
|
||||
await repository.createOperation({
|
||||
|
||||
@@ -38,7 +38,7 @@ describe('readReleaseManifest', () => {
|
||||
|
||||
await expect(readReleaseManifest(workspaceRoot)).resolves.toMatchObject({
|
||||
controllerProtocol: RELEASE_CONTROLLER_PROTOCOL,
|
||||
gatewaySchemaHead: '20260824120000_add_account_identity_management',
|
||||
gatewaySchemaHead: '20260825000000_add_bulk_release_batches',
|
||||
gameSchemaHead: '20260824080000_vote_utc_wall_timestamps',
|
||||
});
|
||||
});
|
||||
|
||||
@@ -14,12 +14,37 @@ Gateway 전체는 별도 release-controller가 처리합니다.
|
||||
|
||||
Profile 화면은 `/gateway/admin/servers/:profileName/version`과
|
||||
`/gateway/admin/servers/:profileName/scenario`, Gateway 화면은
|
||||
`/gateway/admin/releases`입니다. 이전 `/gateway/admin/server-operations`는
|
||||
`/gateway/admin/releases`, 통합 화면은 `/gateway/admin/releases/batch`입니다.
|
||||
이전 `/gateway/admin/server-operations`는
|
||||
호환성을 위해 서버 목록으로 이동합니다. Profile 작업은 runtime/settings/deploy/reset
|
||||
capability로 분리되며 포괄 운영 권한은 사용하지 않습니다. Gateway 전체 릴리스에는
|
||||
profile 범위 권한과 별개인 전역 `admin.releases.manage` 권한이 필요합니다.
|
||||
일반 사용자와 권한이 없는 관리자는 Gateway 릴리스 영역을 사용할 수 없습니다.
|
||||
|
||||
### 일괄 업데이트
|
||||
|
||||
일괄 업데이트는 새 종류의 배포 엔진이 아니라 기존 Gateway release와 profile
|
||||
`DEPLOY` operation을 `GatewayBulkRelease`로 묶는 durable 실행 계획입니다.
|
||||
|
||||
1. 브랜치 또는 commit을 요청 시점에 하나의 full commit SHA로 고정합니다.
|
||||
2. 인증 session의 `admin.releases.manage`와 각
|
||||
`admin.profiles.deploy:<profileName>`을 대상별로 다시 검사합니다.
|
||||
3. 묶음, Gateway release operation과 profile `DEPLOY` operation을 한 Gateway DB
|
||||
transaction으로 등록합니다. 한 대상이라도 활성 작업과 충돌하면 아무것도
|
||||
등록하지 않습니다.
|
||||
4. Gateway가 포함되면 첫 순서로 실행하고, profile은 표시 순서대로 실행합니다.
|
||||
기존 전역 advisory lock은 그대로 사용하므로 동시에 여러 release build를
|
||||
실행하지 않습니다.
|
||||
5. 앞 대상이 `FAILED` 또는 `CANCELLED`이면 뒤 대상은 claim하지 않습니다. 실패
|
||||
대상을 재시도하면 새 branch head가 아니라 묶음의 고정 SHA로 같은 operation을
|
||||
다시 queue하고, 성공 후 다음 대상을 진행합니다.
|
||||
|
||||
일괄 업데이트는 여러 대상에 대한 원자적 runtime 전환이나 자동 rollback을 약속하지
|
||||
않습니다. 이미 성공한 profile의 forward migration을 자동으로 되돌리지 않으며,
|
||||
화면은 묶음 전체와 대상별 상태를 함께 표시합니다. 같은 commit의 후속 frontend와
|
||||
server build는 기존 Turbo cache를 재사용할 수 있지만, 자원 보호를 위한 기본 build
|
||||
동시성 1 계약은 변경하지 않습니다.
|
||||
|
||||
운영 전에 다음을 확인해 주세요.
|
||||
|
||||
- 대상 branch 또는 전체 commit SHA가 Core2026 저장소에 존재합니다.
|
||||
|
||||
+38
@@ -0,0 +1,38 @@
|
||||
CREATE TABLE "gateway_bulk_release" (
|
||||
"id" UUID NOT NULL,
|
||||
"source_mode" "GatewaySourceMode" NOT NULL,
|
||||
"source_ref" TEXT NOT NULL,
|
||||
"resolved_commit_sha" TEXT NOT NULL,
|
||||
"reason" TEXT,
|
||||
"requested_by" TEXT NOT NULL,
|
||||
"created_at" TIMESTAMPTZ(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
"updated_at" TIMESTAMPTZ(3) NOT NULL,
|
||||
CONSTRAINT "gateway_bulk_release_pkey" PRIMARY KEY ("id")
|
||||
);
|
||||
|
||||
ALTER TABLE "gateway_operation"
|
||||
ADD COLUMN "bulk_release_id" UUID,
|
||||
ADD COLUMN "bulk_order" INTEGER;
|
||||
|
||||
ALTER TABLE "gateway_release_operation"
|
||||
ADD COLUMN "bulk_release_id" UUID,
|
||||
ADD COLUMN "bulk_order" INTEGER;
|
||||
|
||||
ALTER TABLE "gateway_operation"
|
||||
ADD CONSTRAINT "gateway_operation_bulk_release_id_fkey"
|
||||
FOREIGN KEY ("bulk_release_id") REFERENCES "gateway_bulk_release"("id")
|
||||
ON DELETE SET NULL ON UPDATE CASCADE;
|
||||
|
||||
ALTER TABLE "gateway_release_operation"
|
||||
ADD CONSTRAINT "gateway_release_operation_bulk_release_id_fkey"
|
||||
FOREIGN KEY ("bulk_release_id") REFERENCES "gateway_bulk_release"("id")
|
||||
ON DELETE SET NULL ON UPDATE CASCADE;
|
||||
|
||||
CREATE INDEX "gateway_bulk_release_created_at_idx"
|
||||
ON "gateway_bulk_release"("created_at");
|
||||
|
||||
CREATE INDEX "gateway_operation_bulk_release_id_bulk_order_idx"
|
||||
ON "gateway_operation"("bulk_release_id", "bulk_order");
|
||||
|
||||
CREATE INDEX "gateway_release_operation_bulk_release_id_bulk_order_idx"
|
||||
ON "gateway_release_operation"("bulk_release_id", "bulk_order");
|
||||
@@ -402,6 +402,9 @@ model GatewayOperation {
|
||||
leaseUntil DateTime? @map("lease_until")
|
||||
heartbeatAt DateTime? @map("heartbeat_at")
|
||||
attempts Int @default(0)
|
||||
bulkReleaseId String? @map("bulk_release_id")
|
||||
bulkOrder Int? @map("bulk_order")
|
||||
bulkRelease GatewayBulkRelease? @relation(fields: [bulkReleaseId], references: [id], onDelete: SetNull)
|
||||
createdAt DateTime @default(now()) @map("created_at")
|
||||
updatedAt DateTime @updatedAt @map("updated_at")
|
||||
logs GatewayOperationLog[]
|
||||
@@ -409,6 +412,7 @@ model GatewayOperation {
|
||||
@@index([status, scheduledAt, createdAt])
|
||||
@@index([status, leaseUntil, createdAt])
|
||||
@@index([profileName, createdAt])
|
||||
@@index([bulkReleaseId, bulkOrder])
|
||||
@@map("gateway_operation")
|
||||
}
|
||||
|
||||
@@ -457,15 +461,35 @@ model GatewayReleaseOperation {
|
||||
leaseUntil DateTime? @map("lease_until") @db.Timestamptz(6)
|
||||
heartbeatAt DateTime? @map("heartbeat_at") @db.Timestamptz(6)
|
||||
attempts Int @default(0)
|
||||
bulkReleaseId String? @map("bulk_release_id")
|
||||
bulkOrder Int? @map("bulk_order")
|
||||
bulkRelease GatewayBulkRelease? @relation(fields: [bulkReleaseId], references: [id], onDelete: SetNull)
|
||||
createdAt DateTime @default(now()) @map("created_at") @db.Timestamptz(3)
|
||||
updatedAt DateTime @updatedAt @map("updated_at") @db.Timestamptz(3)
|
||||
logs GatewayReleaseLog[]
|
||||
|
||||
@@index([status, leaseUntil, createdAt])
|
||||
@@index([createdAt])
|
||||
@@index([bulkReleaseId, bulkOrder])
|
||||
@@map("gateway_release_operation")
|
||||
}
|
||||
|
||||
model GatewayBulkRelease {
|
||||
id String @id @default(uuid())
|
||||
sourceMode GatewaySourceMode @map("source_mode")
|
||||
sourceRef String @map("source_ref")
|
||||
resolvedCommitSha String @map("resolved_commit_sha")
|
||||
reason String?
|
||||
requestedBy String @map("requested_by")
|
||||
createdAt DateTime @default(now()) @map("created_at") @db.Timestamptz(3)
|
||||
updatedAt DateTime @updatedAt @map("updated_at") @db.Timestamptz(3)
|
||||
gatewayOperations GatewayReleaseOperation[]
|
||||
profileOperations GatewayOperation[]
|
||||
|
||||
@@index([createdAt])
|
||||
@@map("gateway_bulk_release")
|
||||
}
|
||||
|
||||
model GatewayReleaseLog {
|
||||
id BigInt @id @default(autoincrement())
|
||||
operationId String @map("operation_id")
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"formatVersion": 1,
|
||||
"controllerProtocol": 2,
|
||||
"gatewaySchemaHead": "20260824120000_add_account_identity_management",
|
||||
"gatewaySchemaHead": "20260825000000_add_bulk_release_batches",
|
||||
"gameSchemaHead": "20260824080000_vote_utc_wall_timestamps",
|
||||
"components": ["gateway-api", "gateway-frontend", "release-controller", "game-api", "game-engine", "game-frontend"]
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user