시간 도메인과 정지 중 메시지·베팅 경계 정리
This commit is contained in:
@@ -1260,7 +1260,7 @@ export const adminRouter = router({
|
||||
if (!canReadProfile(adminAuth, initialOperation.profileName)) {
|
||||
throw new TRPCError({ code: 'FORBIDDEN', message: 'Permission denied.' });
|
||||
}
|
||||
const deadline = Date.now() + input.timeoutMs;
|
||||
const deadline = performance.now() + input.timeoutMs;
|
||||
while (true) {
|
||||
const [operation, entries] = await Promise.all([
|
||||
ctx.profiles.getOperation(input.id),
|
||||
@@ -1270,7 +1270,7 @@ export const adminRouter = router({
|
||||
throw new TRPCError({ code: 'NOT_FOUND', message: 'Profile operation not found.' });
|
||||
}
|
||||
const terminal = ['SUCCEEDED', 'FAILED', 'CANCELLED'].includes(operation.status);
|
||||
if (entries.length || terminal || Date.now() >= deadline) {
|
||||
if (entries.length || terminal || performance.now() >= deadline) {
|
||||
return {
|
||||
operation,
|
||||
entries,
|
||||
@@ -1942,7 +1942,7 @@ export const adminRouter = router({
|
||||
})
|
||||
)
|
||||
.query(async ({ ctx, input }) => {
|
||||
const deadline = Date.now() + input.timeoutMs;
|
||||
const deadline = performance.now() + input.timeoutMs;
|
||||
while (true) {
|
||||
const [operation, entries] = await Promise.all([
|
||||
ctx.releases.getOperation(input.id),
|
||||
@@ -1952,7 +1952,7 @@ export const adminRouter = router({
|
||||
throw new TRPCError({ code: 'NOT_FOUND', message: 'Gateway release operation not found.' });
|
||||
}
|
||||
const terminal = ['SUCCEEDED', 'FAILED', 'CANCELLED'].includes(operation.status);
|
||||
if (entries.length || terminal || Date.now() >= deadline) {
|
||||
if (entries.length || terminal || performance.now() >= deadline) {
|
||||
return {
|
||||
operation,
|
||||
entries,
|
||||
@@ -2519,8 +2519,8 @@ export const adminRouter = router({
|
||||
reason: input.reason,
|
||||
requestedBy: adminAuth.user.id,
|
||||
});
|
||||
const deadline = Date.now() + 10 * 60_000;
|
||||
while (Date.now() < deadline) {
|
||||
const deadline = performance.now() + 10 * 60_000;
|
||||
while (performance.now() < deadline) {
|
||||
await ctx.orchestrator.runOperationsNow();
|
||||
const current = await ctx.profiles.getOperation(operation.id);
|
||||
if (current?.status === 'SUCCEEDED') {
|
||||
|
||||
@@ -2995,7 +2995,7 @@ export class GatewayOrchestrator implements GatewayOrchestratorHandle {
|
||||
profile: GatewayProfileRecord,
|
||||
assertLease?: () => Promise<void>
|
||||
): Promise<boolean> {
|
||||
const deadline = Date.now() + this.profileReadinessTimeoutMs;
|
||||
const deadline = performance.now() + this.profileReadinessTimeoutMs;
|
||||
const definitions = buildProcessDefinitions(profile, this.processConfig);
|
||||
const expectedNames = Object.entries(definitions)
|
||||
.filter(([role]) => this.frontendServeMode === 'preview' || role !== 'frontend')
|
||||
@@ -3008,7 +3008,7 @@ export class GatewayOrchestrator implements GatewayOrchestratorHandle {
|
||||
this.processConfig.frontendReadinessOrigin ?? 'http://caddy'
|
||||
).toString()
|
||||
: `http://127.0.0.1:${profile.apiPort - 1}/${profile.profile}/`;
|
||||
while (Date.now() < deadline) {
|
||||
while (performance.now() < deadline) {
|
||||
await assertLease?.();
|
||||
try {
|
||||
const [api, frontend, processes] = await Promise.all([
|
||||
|
||||
@@ -242,13 +242,19 @@ export const createGatewayReleaseRepository = (prisma: GatewayPrismaClient): Gat
|
||||
});
|
||||
return mapOperation(row);
|
||||
},
|
||||
async claimNextOperation(now, lease) {
|
||||
async claimNextOperation(_now, lease) {
|
||||
const row = await prisma.$transaction(async (tx) => {
|
||||
await tx.$queryRaw<Array<{ lock_result: string }>>`
|
||||
SELECT pg_advisory_xact_lock(
|
||||
hashtextextended(${CONTROL_PLANE_OPERATION_CLAIM_LOCK}, 0)
|
||||
)::text AS lock_result
|
||||
`;
|
||||
const [{ now }] = await tx.$queryRaw<Array<{ now: Date }>>`
|
||||
SELECT CURRENT_TIMESTAMP AS "now"
|
||||
`;
|
||||
if (!now) {
|
||||
throw new Error('Database wall clock is unavailable while claiming a Gateway release operation.');
|
||||
}
|
||||
const runningProfileOperation = await tx.gatewayOperation.findFirst({
|
||||
where: { status: 'RUNNING' },
|
||||
select: { id: true },
|
||||
@@ -326,13 +332,21 @@ export const createGatewayReleaseRepository = (prisma: GatewayPrismaClient): Gat
|
||||
});
|
||||
return row ? mapOperation(row) : null;
|
||||
},
|
||||
async renewOperationLease(id, ownerId, now, durationMs) {
|
||||
const updated = await prisma.gatewayReleaseOperation.updateMany({
|
||||
where: { id, status: 'RUNNING', leaseOwner: ownerId },
|
||||
data: {
|
||||
leaseUntil: new Date(now.getTime() + durationMs),
|
||||
heartbeatAt: now,
|
||||
},
|
||||
async renewOperationLease(id, ownerId, _now, durationMs) {
|
||||
const updated = await prisma.$transaction(async (tx) => {
|
||||
const [{ now }] = await tx.$queryRaw<Array<{ now: Date }>>`
|
||||
SELECT CURRENT_TIMESTAMP AS "now"
|
||||
`;
|
||||
if (!now) {
|
||||
throw new Error('Database wall clock is unavailable while renewing a Gateway release lease.');
|
||||
}
|
||||
return tx.gatewayReleaseOperation.updateMany({
|
||||
where: { id, status: 'RUNNING', leaseOwner: ownerId },
|
||||
data: {
|
||||
leaseUntil: new Date(now.getTime() + durationMs),
|
||||
heartbeatAt: now,
|
||||
},
|
||||
});
|
||||
});
|
||||
return updated.count === 1;
|
||||
},
|
||||
|
||||
@@ -691,7 +691,7 @@ export const createGatewayProfileRepository = (prisma: GatewayPrismaClient): Gat
|
||||
return mapOperation(row);
|
||||
},
|
||||
async claimNextOperation(
|
||||
now: Date,
|
||||
_now: Date,
|
||||
lease?: { ownerId: string; durationMs: number }
|
||||
): Promise<GatewayOperationRecord | null> {
|
||||
const row = await prisma.$transaction(async (tx) => {
|
||||
@@ -700,6 +700,12 @@ export const createGatewayProfileRepository = (prisma: GatewayPrismaClient): Gat
|
||||
hashtextextended(${CONTROL_PLANE_OPERATION_CLAIM_LOCK}, 0)
|
||||
)::text AS lock_result
|
||||
`;
|
||||
const [{ now }] = await tx.$queryRaw<Array<{ now: Date }>>`
|
||||
SELECT CURRENT_TIMESTAMP AS "now"
|
||||
`;
|
||||
if (!now) {
|
||||
throw new Error('Database wall clock is unavailable while claiming a Gateway operation.');
|
||||
}
|
||||
const runningRelease = await tx.gatewayReleaseOperation.findFirst({
|
||||
where: { status: 'RUNNING' },
|
||||
select: { id: true },
|
||||
@@ -815,13 +821,21 @@ export const createGatewayProfileRepository = (prisma: GatewayPrismaClient): Gat
|
||||
});
|
||||
return row ? mapOperation(row) : null;
|
||||
},
|
||||
async renewOperationLease(id: string, ownerId: string, now: Date, durationMs: number): Promise<boolean> {
|
||||
const renewed = await prisma.gatewayOperation.updateMany({
|
||||
where: { id, status: 'RUNNING', leaseOwner: ownerId },
|
||||
data: {
|
||||
leaseUntil: new Date(now.getTime() + durationMs),
|
||||
heartbeatAt: now,
|
||||
},
|
||||
async renewOperationLease(id: string, ownerId: string, _now: Date, durationMs: number): Promise<boolean> {
|
||||
const renewed = await prisma.$transaction(async (tx) => {
|
||||
const [{ now }] = await tx.$queryRaw<Array<{ now: Date }>>`
|
||||
SELECT CURRENT_TIMESTAMP AS "now"
|
||||
`;
|
||||
if (!now) {
|
||||
throw new Error('Database wall clock is unavailable while renewing a Gateway operation lease.');
|
||||
}
|
||||
return tx.gatewayOperation.updateMany({
|
||||
where: { id, status: 'RUNNING', leaseOwner: ownerId },
|
||||
data: {
|
||||
leaseUntil: new Date(now.getTime() + durationMs),
|
||||
heartbeatAt: now,
|
||||
},
|
||||
});
|
||||
});
|
||||
return renewed.count === 1;
|
||||
},
|
||||
|
||||
@@ -293,13 +293,13 @@ export const listScenarioPreviews = async (options?: { gitRef?: string | null })
|
||||
}
|
||||
if (!gitRef) {
|
||||
const cached = previewCache.get(DEFAULT_CACHE_KEY);
|
||||
if (cached && Date.now() - cached.loadedAt < CACHE_TTL_MS) {
|
||||
if (cached && performance.now() - cached.loadedAt < CACHE_TTL_MS) {
|
||||
return cached.data;
|
||||
}
|
||||
const ids = await listScenarioIds();
|
||||
const previews = await Promise.all(ids.map((id) => buildScenarioPreview(id)));
|
||||
previewCache.set(DEFAULT_CACHE_KEY, {
|
||||
loadedAt: Date.now(),
|
||||
loadedAt: performance.now(),
|
||||
data: previews,
|
||||
});
|
||||
return previews;
|
||||
@@ -308,7 +308,7 @@ export const listScenarioPreviews = async (options?: { gitRef?: string | null })
|
||||
const commitSha = await resolveGitCommitSha(gitRef);
|
||||
const cacheKey = commitSha;
|
||||
const cached = previewCache.get(cacheKey);
|
||||
if (cached && Date.now() - cached.loadedAt < CACHE_TTL_MS) {
|
||||
if (cached && performance.now() - cached.loadedAt < CACHE_TTL_MS) {
|
||||
return cached.data;
|
||||
}
|
||||
const ids = await listScenarioIdsFromGit(commitSha);
|
||||
@@ -317,7 +317,7 @@ export const listScenarioPreviews = async (options?: { gitRef?: string | null })
|
||||
previews.push(await buildScenarioPreviewFromGit(commitSha, id));
|
||||
}
|
||||
previewCache.set(cacheKey, {
|
||||
loadedAt: Date.now(),
|
||||
loadedAt: performance.now(),
|
||||
data: previews,
|
||||
});
|
||||
return previews;
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import { performance } from 'node:perf_hooks';
|
||||
|
||||
import {
|
||||
WEB_PUSH_EVENT_TYPES,
|
||||
@@ -409,10 +410,13 @@ export class WebPushCoordinator {
|
||||
`);
|
||||
if (rows.length === 0) return [];
|
||||
const ids = rows.map((row) => row.id);
|
||||
await tx.webPushDelivery.updateMany({
|
||||
where: { id: { in: ids } },
|
||||
data: { lockedAt: new Date(), lockOwner: this.owner, attempts: { increment: 1 } },
|
||||
});
|
||||
await tx.$executeRaw(GatewayPrisma.sql`
|
||||
UPDATE web_push_delivery
|
||||
SET locked_at = CURRENT_TIMESTAMP,
|
||||
lock_owner = ${this.owner},
|
||||
attempts = attempts + 1
|
||||
WHERE id IN (${GatewayPrisma.join(ids)})
|
||||
`);
|
||||
return tx.webPushDelivery.findMany({
|
||||
where: { id: { in: ids }, lockOwner: this.owner },
|
||||
include: { notification: true, subscription: true },
|
||||
@@ -421,22 +425,31 @@ export class WebPushCoordinator {
|
||||
});
|
||||
|
||||
for (const delivery of claimed) {
|
||||
if (delivery.subscription.expirationTime && delivery.subscription.expirationTime.getTime() <= Date.now()) {
|
||||
await this.prisma.$transaction(async (tx) => {
|
||||
await tx.webPushDelivery.updateMany({
|
||||
where: { id: delivery.id, lockOwner: this.owner },
|
||||
data: {
|
||||
status: 'FAILED',
|
||||
lockedAt: null,
|
||||
lockOwner: null,
|
||||
lastError: 'Push subscription expired.',
|
||||
},
|
||||
});
|
||||
await tx.webPushSubscription.update({
|
||||
where: { id: delivery.subscriptionId },
|
||||
data: { disabledAt: new Date() },
|
||||
});
|
||||
});
|
||||
const expired = await this.prisma.$transaction(async (tx) => {
|
||||
const count = await tx.$executeRaw(GatewayPrisma.sql`
|
||||
UPDATE web_push_delivery AS delivery
|
||||
SET status = 'FAILED'::"WebPushDeliveryStatus",
|
||||
locked_at = NULL,
|
||||
lock_owner = NULL,
|
||||
last_error = 'Push subscription expired.'
|
||||
FROM web_push_subscription AS subscription
|
||||
WHERE delivery.id = ${delivery.id}
|
||||
AND delivery.lock_owner = ${this.owner}
|
||||
AND subscription.id = delivery.subscription_id
|
||||
AND subscription.expiration_time IS NOT NULL
|
||||
AND subscription.expiration_time <= CURRENT_TIMESTAMP
|
||||
`);
|
||||
if (count > 0) {
|
||||
await tx.$executeRaw(GatewayPrisma.sql`
|
||||
UPDATE web_push_subscription
|
||||
SET disabled_at = CURRENT_TIMESTAMP,
|
||||
updated_at = CURRENT_TIMESTAMP
|
||||
WHERE id = ${delivery.subscriptionId}
|
||||
`);
|
||||
}
|
||||
return count > 0;
|
||||
});
|
||||
if (expired) {
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
@@ -453,16 +466,15 @@ export class WebPushCoordinator {
|
||||
}),
|
||||
{ TTL: 60 * 60 }
|
||||
);
|
||||
await this.prisma.webPushDelivery.updateMany({
|
||||
where: { id: delivery.id, lockOwner: this.owner },
|
||||
data: {
|
||||
status: 'DELIVERED',
|
||||
deliveredAt: new Date(),
|
||||
lockedAt: null,
|
||||
lockOwner: null,
|
||||
lastError: null,
|
||||
},
|
||||
});
|
||||
await this.prisma.$executeRaw(GatewayPrisma.sql`
|
||||
UPDATE web_push_delivery
|
||||
SET status = 'DELIVERED'::"WebPushDeliveryStatus",
|
||||
delivered_at = CURRENT_TIMESTAMP,
|
||||
locked_at = NULL,
|
||||
lock_owner = NULL,
|
||||
last_error = NULL
|
||||
WHERE id = ${delivery.id} AND lock_owner = ${this.owner}
|
||||
`);
|
||||
} catch (error) {
|
||||
const statusCode =
|
||||
typeof error === 'object' && error !== null && 'statusCode' in error
|
||||
@@ -478,28 +490,31 @@ export class WebPushCoordinator {
|
||||
const safeError =
|
||||
statusCode > 0 ? `Push service returned HTTP ${statusCode}.` : 'Push service request failed.';
|
||||
await this.prisma.$transaction(async (tx) => {
|
||||
await tx.webPushDelivery.updateMany({
|
||||
where: { id: delivery.id, lockOwner: this.owner },
|
||||
data: {
|
||||
status: terminal || exhausted ? 'FAILED' : 'PENDING',
|
||||
availableAt: new Date(Date.now() + delaySeconds * 1_000),
|
||||
lockedAt: null,
|
||||
lockOwner: null,
|
||||
lastError: safeError,
|
||||
},
|
||||
});
|
||||
const nextStatus = terminal || exhausted ? 'FAILED' : 'PENDING';
|
||||
await tx.$executeRaw(GatewayPrisma.sql`
|
||||
UPDATE web_push_delivery
|
||||
SET status = ${nextStatus}::"WebPushDeliveryStatus",
|
||||
available_at = CURRENT_TIMESTAMP
|
||||
+ ${delaySeconds * 1_000} * INTERVAL '1 millisecond',
|
||||
locked_at = NULL,
|
||||
lock_owner = NULL,
|
||||
last_error = ${safeError}
|
||||
WHERE id = ${delivery.id} AND lock_owner = ${this.owner}
|
||||
`);
|
||||
if (statusCode === 404 || statusCode === 410) {
|
||||
await tx.webPushSubscription.update({
|
||||
where: { id: delivery.subscriptionId },
|
||||
data: { disabledAt: new Date() },
|
||||
});
|
||||
await tx.$executeRaw(GatewayPrisma.sql`
|
||||
UPDATE web_push_subscription
|
||||
SET disabled_at = CURRENT_TIMESTAMP,
|
||||
updated_at = CURRENT_TIMESTAMP
|
||||
WHERE id = ${delivery.subscriptionId}
|
||||
`);
|
||||
}
|
||||
});
|
||||
if (!terminal) this.onError(new Error(safeError));
|
||||
}
|
||||
}
|
||||
if (Date.now() >= this.nextPruneAt) {
|
||||
this.nextPruneAt = Date.now() + 60_000;
|
||||
if (performance.now() >= this.nextPruneAt) {
|
||||
this.nextPruneAt = performance.now() + 60_000;
|
||||
await this.prisma.$transaction(async (tx) => {
|
||||
await tx.$executeRaw(GatewayPrisma.sql`
|
||||
WITH expired AS (
|
||||
@@ -534,7 +549,7 @@ export class WebPushCoordinator {
|
||||
|
||||
private run(): void {
|
||||
if (!this.configured || this.inFlight) return;
|
||||
const now = Date.now();
|
||||
const now = performance.now();
|
||||
const shouldReconcileProfiles = now >= this.nextProfileReconcileAt;
|
||||
if (shouldReconcileProfiles) this.nextProfileReconcileAt = now + 5_000;
|
||||
this.inFlight = (shouldReconcileProfiles ? this.reconcileProfiles() : Promise.resolve())
|
||||
|
||||
Reference in New Issue
Block a user