fix: 토너먼트 관리자 요청의 Redis와 DB 잠금 순서 통일

2026-09-06 23:32:24 +00:00
parent 27049dae1f
commit 9d5447dd3f
3 changed files with 273 additions and 9 deletions
@@ -8,7 +8,7 @@ import type { TournamentState } from '../../tournament/types.js';
 import { TournamentStore, type TournamentClockContext } from '../../tournament/store.js';
 import { buildTournamentKeys } from '../../tournament/keys.js';
 import { assignManualApplicantGroup } from '../../tournament/workerHelpers.js';
-import { accessAuthedProcedure, authedProcedure, engineAuthedProcedure, router } from '../../trpc.js';
+import { accessAuthedProcedure, authedProcedure, engineAuthedProcedure, procedure, router } from '../../trpc.js';
 import { getMyGeneral } from '../shared/general.js';
 import { loadCurrentGameTime } from '../../services/gameClock.js';
 import { ensureActiveRedisClockFence, ensureBettingRedisClockFence } from '../../services/redisClockFence.js';
@@ -38,19 +38,29 @@ const resolveCurrentDevelCost = (worldState: { config?: unknown; meta?: unknown
    return resolveNumber(asRecord(worldState?.meta), ['develcost', 'develCost', 'develrate'], configured);
 };

-const adminProcedure = authedProcedure.use(({ ctx, next }) => {
-    const roles = ctx.auth?.user.roles ?? [];
-    if (!hasAdminRole(roles, ctx.profile.name)) {
-        throw new TRPCError({ code: 'FORBIDDEN', message: 'Admin permission is required.' });
-    }
-    return next();
-});
+const adminProcedure = engineAuthedProcedure
+    .use(({ ctx, next }) => {
+        const roles = ctx.auth?.user.roles ?? [];
+        if (!hasAdminRole(roles, ctx.profile.name)) {
+            throw new TRPCError({ code: 'FORBIDDEN', message: 'Admin permission is required.' });
+        }
+        return next();
+    })
+    .use(async ({ ctx, type, next }) => {
+        if (type !== 'mutation') return next({ ctx: { tournamentMutationLockHeld: false } });
+        // 참가·베팅은 Redis lock 안에서 ENGINE의 DB commit을 기다린다.
+        // 관리자도 Redis를 먼저 잡아 DB clock fence → Redis 역순 대기를 막는다.
+        const store = new TournamentStore(ctx.redis, buildTournamentKeys(ctx.profile.name));
+        return store.withMutationLock(() => next({ ctx: { tournamentMutationLockHeld: true } }));
+    })
+    .concat(procedure);

 const withTournamentClockMutation = async <T>(
    ctx: {
        db: Parameters<typeof loadCurrentGameTime>[0];
        redis: Parameters<typeof ensureActiveRedisClockFence>[0];
        profile: { name: string };
+        tournamentMutationLockHeld?: boolean;
    },
    store: TournamentStore,
    operation: () => Promise<T>
@@ -69,7 +79,9 @@ const withTournamentClockMutation = async <T>(
        deadlineGeneration: fence.generation,
        dateToTick: gameTime.dateToTick,
    };
-    return store.withClockContext(clockContext, () => store.withMutationLock(operation));
+    return store.withClockContext(clockContext, () =>
+        ctx.tournamentMutationLockHeld ? operation() : store.withMutationLock(operation)
+    );
 };

 const withTournamentBetClockMutation = async <T>(
@@ -0,0 +1,232 @@
+import { it, expect } from 'vitest';
+import {
+    createGamePostgresConnector,
+    createRedisConnector,
+    acquireGameSchemaAdvisoryXactLock,
+    CLOCK_OPERATION_PERSISTENCE_LOCK,
+} from '@sammo-ts/infra';
+import { appRouter } from '../src/router.js';
+import { DatabaseTurnDaemonTransport } from '../src/daemon/databaseTransport.js';
+import { buildTournamentKeys } from '../src/tournament/keys.js';
+import type { GameApiContext } from '../src/context.js';
+
+const databaseUrl = process.env.INPUT_EVENT_DATABASE_URL;
+const integration = it.skipIf(!databaseUrl || !process.env.REDIS_URL);
+
+integration.each([
+    'setState',
+    'patchState',
+    'setParticipants',
+    'setMatches',
+    'setBettingEntries',
+    'seedParticipants',
+    'cancel',
+])(
+    'serializes %s behind a joining user without holding the ENGINE clock lock',
+    async (adminAction) => {
+        const url = databaseUrl!;
+        const connector = createGamePostgresConnector({ url });
+        const redisConnector = createRedisConnector({ url: process.env.REDIS_URL! });
+        await connector.connect();
+        await redisConnector.connect();
+        const db = connector.prisma;
+        const redis = redisConnector.client;
+        const profileName = 'che:tournament-lock-integration';
+        const keys = buildTournamentKeys(profileName);
+        const redisKeys = [...Object.values(keys), `${keys.stateKey}:mutation-lock`];
+        const prefix = 'integration:tournament-lock:';
+        const nextAt = new Date().toISOString();
+        let notifyJoin: () => void;
+        const joinAtDaemon = new Promise<void>((resolve) => {
+            notifyJoin = resolve;
+        });
+        let notifyAdmin: () => void;
+        const adminAtRedis = new Promise<void>((resolve) => {
+            notifyAdmin = resolve;
+        });
+        const transport = new DatabaseTurnDaemonTransport(db, 4_000);
+        try {
+            await redis.del(redisKeys);
+            await db.inputEvent.deleteMany({ where: { requestId: { startsWith: prefix } } });
+            await db.worldState.deleteMany({ where: { id: -991900 } });
+            await db.general.deleteMany({ where: { id: 991900 } });
+            await db.turnDaemonLease.deleteMany({ where: { profile: profileName } });
+            await db.turnDaemonLease.create({
+                data: {
+                    profile: profileName,
+                    ownerId: 'audit',
+                    leaseUntil: new Date(Date.now() + 60000),
+                    clockReady: true,
+                },
+            });
+            await db.worldState.create({
+                data: {
+                    id: -991900,
+                    scenarioCode: 'audit',
+                    currentYear: 200,
+                    currentMonth: 1,
+                    tickSeconds: 60,
+                    clockBaseTime: new Date(),
+                    clockTick: 0n,
+                    clockWallAnchor: new Date(),
+                    clockPhase: 'RUNNING',
+                    clockRevision: 1n,
+                    config: { const: { develCost: 10 } },
+                },
+            });
+            await db.general.create({
+                data: { id: 991900, userId: 'audit-user', name: 'audit', turnTime: new Date(), gold: 1000 },
+            });
+            await redis.set(
+                keys.stateKey,
+                JSON.stringify({
+                    stage: 1,
+                    phase: 0,
+                    type: 0,
+                    auto: false,
+                    openYear: 200,
+                    openMonth: 1,
+                    termSeconds: 60,
+                    nextAt,
+                })
+            );
+            const base: Partial<GameApiContext> = {
+                db,
+                redis,
+                profile: { id: 'che', scenario: 'tournament-lock-integration', name: profileName },
+                auth: {
+                    version: 1,
+                    profile: profileName,
+                    issuedAt: new Date().toISOString(),
+                    expiresAt: '2999-01-01T00:00:00Z',
+                    sessionId: 'audit',
+                    user: { id: 'audit-user', username: 'audit', displayName: 'audit', roles: [] },
+                    sanctions: {},
+                },
+                turnDaemon: {
+                    sendCommand: transport.sendCommand.bind(transport),
+                    requestStatus: transport.requestStatus.bind(transport),
+                    requestCommand: async (command) => {
+                        notifyJoin!();
+                        await adminAtRedis;
+                        const requestId = await transport.sendCommand(command);
+                        return db.$transaction(async (tx) => {
+                            // 이전 순서에서는 관리자가 DB lock을 보유하여 별도 ENGINE 연결이 막힌다.
+                            await tx.$executeRawUnsafe("SET LOCAL lock_timeout = '500ms'");
+                            await acquireGameSchemaAdvisoryXactLock(tx, CLOCK_OPERATION_PERSISTENCE_LOCK);
+                            if (command.type !== 'adjustGeneralResources') throw new Error('unexpected command');
+                            const result = {
+                                type: 'adjustGeneralResources' as const,
+                                ok: true as const,
+                                processed: 1,
+                                missing: 0,
+                                totalGoldDelta: -10,
+                                totalRiceDelta: 0,
+                            };
+                            await tx.inputEvent.update({ where: { requestId }, data: { status: 'SUCCEEDED', result } });
+                            return result;
+                        });
+                    },
+                },
+            };
+            const joinContext = { ...base, requestId: `${prefix}join` } as GameApiContext;
+            const adminRedis = new Proxy(redis, {
+                get(target, property) {
+                    if (property === 'set')
+                        return async (...args: Parameters<typeof redis.set>) => {
+                            if (args[0] === `${keys.stateKey}:mutation-lock`) notifyAdmin!();
+                            return redis.set(...args);
+                        };
+                    const value = Reflect.get(target, property);
+                    return typeof value === 'function' ? value.bind(target) : value;
+                },
+            });
+            const adminContext = {
+                ...base,
+                requestId: `${prefix}admin`,
+                redis: adminRedis,
+                auth: { ...base.auth!, user: { ...base.auth!.user, roles: ['admin'] } },
+            } as GameApiContext;
+            const callAdmin = async (context: GameApiContext) => {
+                const caller = appRouter.createCaller(context).tournament;
+                switch (adminAction) {
+                    case 'setState':
+                        return caller.setState({
+                            stage: 1,
+                            phase: 0,
+                            type: 0,
+                            auto: false,
+                            openYear: 200,
+                            openMonth: 1,
+                            termSeconds: 60,
+                            nextAt,
+                        });
+                    case 'patchState':
+                        return caller.patchState({ auto: true });
+                    case 'setParticipants':
+                        return caller.setParticipants([]);
+                    case 'setMatches':
+                        return caller.setMatches([]);
+                    case 'setBettingEntries':
+                        return caller.setBettingEntries([]);
+                    case 'seedParticipants':
+                        return caller.seedParticipants({ generalIds: [991900] });
+                    default:
+                        return caller.cancel();
+                }
+            };
+            const join = appRouter.createCaller(joinContext).tournament.join();
+            const joinOutcome = join.then(
+                (value) => ({ ok: true, value }),
+                (error) => ({ ok: false, error })
+            );
+            await joinAtDaemon;
+            const admin = callAdmin(adminContext);
+            const adminOutcome = admin.then(
+                (value) => ({ ok: true, value }),
+                (error) => ({ ok: false, error })
+            );
+            await adminAtRedis;
+            const [joined, managed] = await Promise.all([joinOutcome, adminOutcome]);
+            expect(joined).toMatchObject({ ok: true, value: { ok: true, count: 1 } });
+            expect(managed).toMatchObject({ ok: true, value: { ok: true } });
+            const revision = await redis.get(keys.sourceRevisionKey);
+            const engineCount = await db.inputEvent.count({
+                where: { requestId: { startsWith: prefix }, target: 'ENGINE' },
+            });
+            // API 입력 원장은 유지한다. 동일 요청 재실행은 Redis/환불 command를 다시 쓰지 않는다.
+            await expect(callAdmin(adminContext)).resolves.toMatchObject({ ok: true });
+            expect(await redis.get(keys.sourceRevisionKey)).toBe(revision);
+            expect(await db.inputEvent.count({ where: { requestId: { startsWith: prefix }, target: 'ENGINE' } })).toBe(
+                engineCount
+            );
+            const inputEvent = await db.inputEvent.findUniqueOrThrow({
+                where: { requestId: `${prefix}admin:tournament.${adminAction}` },
+            });
+            expect(inputEvent).toMatchObject({ target: 'API', status: 'SUCCEEDED', attempts: 1 });
+            expect(await redis.get(`${keys.stateKey}:mutation-lock`)).toBeNull();
+            const retryContext = { ...adminContext, requestId: `${prefix}retry` };
+            const beforeFailure = await redis.get(keys.sourceRevisionKey);
+            await db.worldState.update({ where: { id: -991900 }, data: { clockPhase: 'SUSPENDED' } });
+            await expect(callAdmin(retryContext)).rejects.toMatchObject({ code: 'PRECONDITION_FAILED' });
+            expect(await redis.get(`${keys.stateKey}:mutation-lock`)).toBeNull();
+            expect(await redis.get(keys.sourceRevisionKey)).toBe(beforeFailure);
+            await db.worldState.update({ where: { id: -991900 }, data: { clockPhase: 'RUNNING' } });
+            await expect(callAdmin(retryContext)).resolves.toMatchObject({ ok: true });
+            expect(
+                await db.inputEvent.findUniqueOrThrow({
+                    where: { requestId: `${prefix}retry:tournament.${adminAction}` },
+                })
+            ).toMatchObject({ status: 'SUCCEEDED', attempts: 2 });
+        } finally {
+            await redis.del(redisKeys);
+            await db.inputEvent.deleteMany({ where: { requestId: { startsWith: prefix } } });
+            await db.general.deleteMany({ where: { id: 991900 } });
+            await db.worldState.deleteMany({ where: { id: -991900 } });
+            await db.turnDaemonLease.deleteMany({ where: { profile: profileName } });
+            await redisConnector.disconnect();
+            await connector.disconnect();
+        }
+    },
+    15000
+);
@@ -59,6 +59,22 @@ selection-pool create/reselect는 client request ID가 있을 때

 합계 **3개 route**다.

+## 토너먼트 관리자 Redis/DB 잠금 순서
+
+`setState`, `patchState`, `setParticipants`, `setMatches`, `setBettingEntries`,
+`seedParticipants`, `cancel`은 위의 ENGINE 완료 대기 49개 route와 별개다.
+관리자는 API input-event transaction을 유지하되 인증/관리자 권한 검사 → Redis
+mutation lock → API input-event/DB clock fence → 상태 변경 → commit → Redis unlock
+순서로 실행한다. `adminProcedure`가 Redis lock을 먼저 획득한 뒤 `procedure`를
+연결하고, `withTournamentClockMutation`은 이미 보유한 lock을 다시 획득하지 않는다.
+조회는 mutation lock과 API transaction을 열지 않는다.
+
+기존 DB → Redis 순서는 Redis lock을 보유하고 ENGINE DB commit을 기다리는
+참가·베팅·worker와 역순 경합하여 요청을 실패시킬 수 있었다. 잠금 순서를 통일하면서
+관리자 API 원장의 중복 요청 결과 재사용, 실패 재시도와 child ENGINE identity를
+보존한다. Redis와 PostgreSQL 사이의 기존 부분 실패 경계는 그대로이며, 이 수정이
+두 저장소의 atomic commit이나 durable saga를 제공하지는 않는다.
+
 ## 기존에 API outer transaction이 없던 ENGINE route

 | route | 근거 |
@@ -96,6 +112,10 @@ fence를 가진 채 ENGINE 결과를 기다리던 교착을 제거했다. 투표
  daemon command와 Redis timer projection을 완료하는 계약을 검증한다.
 - `app/game-api/test/tournamentRouter.test.ts`: 참가·베팅이 API transaction을 열지 않고
  request-scoped ENGINE command ID와 Redis mutation lock을 사용하는지 검증한다.
+- `app/game-api/test/tournamentLockOrder.integration.test.ts`: 실제 PostgreSQL/Redis에서
+  참가 요청과 7개 관리자 명령의 경합, API replay와 실패 후 같은 요청 재시도를 검사한다.
+  ENGINE transport는 별도 DB transaction의 실제 clock fence/내구성 접수 경계로 제어하며
+  전체 daemon의 자원 차감·보상 실행을 대신하지 않는다.
 - `app/game-api/test/inheritRouter.test.ts`,
  `app/game-engine/test/inheritanceActionPersistence.integration.test.ts`: 인증 actor, point/log,
  general/message 변경이 `inheritanceAction` ENGINE transaction에 함께 있는지 검증한다.