From 9d5447dd3f363d64f31a2405ac9bae67af7ec415 Mon Sep 17 00:00:00 2001 From: hided62 Date: Sun, 6 Sep 2026 23:32:24 +0000 Subject: [PATCH] =?UTF-8?q?fix:=20=ED=86=A0=EB=84=88=EB=A8=BC=ED=8A=B8=20?= =?UTF-8?q?=EA=B4=80=EB=A6=AC=EC=9E=90=20=EC=9A=94=EC=B2=AD=EC=9D=98=20Red?= =?UTF-8?q?is=EC=99=80=20DB=20=EC=9E=A0=EA=B8=88=20=EC=88=9C=EC=84=9C=20?= =?UTF-8?q?=ED=86=B5=EC=9D=BC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/game-api/src/router/tournament/index.ts | 30 ++- .../tournamentLockOrder.integration.test.ts | 232 ++++++++++++++++++ .../game-api-daemon-procedure-inventory.md | 20 ++ 3 files changed, 273 insertions(+), 9 deletions(-) create mode 100644 app/game-api/test/tournamentLockOrder.integration.test.ts diff --git a/app/game-api/src/router/tournament/index.ts b/app/game-api/src/router/tournament/index.ts index 810a7daf..b8cc4e23 100644 --- a/app/game-api/src/router/tournament/index.ts +++ b/app/game-api/src/router/tournament/index.ts @@ -8,7 +8,7 @@ import type { TournamentState } from '../../tournament/types.js'; import { TournamentStore, type TournamentClockContext } from '../../tournament/store.js'; import { buildTournamentKeys } from '../../tournament/keys.js'; import { assignManualApplicantGroup } from '../../tournament/workerHelpers.js'; -import { accessAuthedProcedure, authedProcedure, engineAuthedProcedure, router } from '../../trpc.js'; +import { accessAuthedProcedure, authedProcedure, engineAuthedProcedure, procedure, router } from '../../trpc.js'; import { getMyGeneral } from '../shared/general.js'; import { loadCurrentGameTime } from '../../services/gameClock.js'; import { ensureActiveRedisClockFence, ensureBettingRedisClockFence } from '../../services/redisClockFence.js'; @@ -38,19 +38,29 @@ const resolveCurrentDevelCost = (worldState: { config?: unknown; meta?: unknown return resolveNumber(asRecord(worldState?.meta), ['develcost', 'develCost', 'develrate'], configured); }; -const adminProcedure = authedProcedure.use(({ ctx, next }) => { - const roles = ctx.auth?.user.roles ?? []; - if (!hasAdminRole(roles, ctx.profile.name)) { - throw new TRPCError({ code: 'FORBIDDEN', message: 'Admin permission is required.' }); - } - return next(); -}); +const adminProcedure = engineAuthedProcedure + .use(({ ctx, next }) => { + const roles = ctx.auth?.user.roles ?? []; + if (!hasAdminRole(roles, ctx.profile.name)) { + throw new TRPCError({ code: 'FORBIDDEN', message: 'Admin permission is required.' }); + } + return next(); + }) + .use(async ({ ctx, type, next }) => { + if (type !== 'mutation') return next({ ctx: { tournamentMutationLockHeld: false } }); + // 참가·베팅은 Redis lock 안에서 ENGINE의 DB commit을 기다린다. + // 관리자도 Redis를 먼저 잡아 DB clock fence → Redis 역순 대기를 막는다. + const store = new TournamentStore(ctx.redis, buildTournamentKeys(ctx.profile.name)); + return store.withMutationLock(() => next({ ctx: { tournamentMutationLockHeld: true } })); + }) + .concat(procedure); const withTournamentClockMutation = async ( ctx: { db: Parameters[0]; redis: Parameters[0]; profile: { name: string }; + tournamentMutationLockHeld?: boolean; }, store: TournamentStore, operation: () => Promise @@ -69,7 +79,9 @@ const withTournamentClockMutation = async ( deadlineGeneration: fence.generation, dateToTick: gameTime.dateToTick, }; - return store.withClockContext(clockContext, () => store.withMutationLock(operation)); + return store.withClockContext(clockContext, () => + ctx.tournamentMutationLockHeld ? operation() : store.withMutationLock(operation) + ); }; const withTournamentBetClockMutation = async ( diff --git a/app/game-api/test/tournamentLockOrder.integration.test.ts b/app/game-api/test/tournamentLockOrder.integration.test.ts new file mode 100644 index 00000000..b74fd9a5 --- /dev/null +++ b/app/game-api/test/tournamentLockOrder.integration.test.ts @@ -0,0 +1,232 @@ +import { it, expect } from 'vitest'; +import { + createGamePostgresConnector, + createRedisConnector, + acquireGameSchemaAdvisoryXactLock, + CLOCK_OPERATION_PERSISTENCE_LOCK, +} from '@sammo-ts/infra'; +import { appRouter } from '../src/router.js'; +import { DatabaseTurnDaemonTransport } from '../src/daemon/databaseTransport.js'; +import { buildTournamentKeys } from '../src/tournament/keys.js'; +import type { GameApiContext } from '../src/context.js'; + +const databaseUrl = process.env.INPUT_EVENT_DATABASE_URL; +const integration = it.skipIf(!databaseUrl || !process.env.REDIS_URL); + +integration.each([ + 'setState', + 'patchState', + 'setParticipants', + 'setMatches', + 'setBettingEntries', + 'seedParticipants', + 'cancel', +])( + 'serializes %s behind a joining user without holding the ENGINE clock lock', + async (adminAction) => { + const url = databaseUrl!; + const connector = createGamePostgresConnector({ url }); + const redisConnector = createRedisConnector({ url: process.env.REDIS_URL! }); + await connector.connect(); + await redisConnector.connect(); + const db = connector.prisma; + const redis = redisConnector.client; + const profileName = 'che:tournament-lock-integration'; + const keys = buildTournamentKeys(profileName); + const redisKeys = [...Object.values(keys), `${keys.stateKey}:mutation-lock`]; + const prefix = 'integration:tournament-lock:'; + const nextAt = new Date().toISOString(); + let notifyJoin: () => void; + const joinAtDaemon = new Promise((resolve) => { + notifyJoin = resolve; + }); + let notifyAdmin: () => void; + const adminAtRedis = new Promise((resolve) => { + notifyAdmin = resolve; + }); + const transport = new DatabaseTurnDaemonTransport(db, 4_000); + try { + await redis.del(redisKeys); + await db.inputEvent.deleteMany({ where: { requestId: { startsWith: prefix } } }); + await db.worldState.deleteMany({ where: { id: -991900 } }); + await db.general.deleteMany({ where: { id: 991900 } }); + await db.turnDaemonLease.deleteMany({ where: { profile: profileName } }); + await db.turnDaemonLease.create({ + data: { + profile: profileName, + ownerId: 'audit', + leaseUntil: new Date(Date.now() + 60000), + clockReady: true, + }, + }); + await db.worldState.create({ + data: { + id: -991900, + scenarioCode: 'audit', + currentYear: 200, + currentMonth: 1, + tickSeconds: 60, + clockBaseTime: new Date(), + clockTick: 0n, + clockWallAnchor: new Date(), + clockPhase: 'RUNNING', + clockRevision: 1n, + config: { const: { develCost: 10 } }, + }, + }); + await db.general.create({ + data: { id: 991900, userId: 'audit-user', name: 'audit', turnTime: new Date(), gold: 1000 }, + }); + await redis.set( + keys.stateKey, + JSON.stringify({ + stage: 1, + phase: 0, + type: 0, + auto: false, + openYear: 200, + openMonth: 1, + termSeconds: 60, + nextAt, + }) + ); + const base: Partial = { + db, + redis, + profile: { id: 'che', scenario: 'tournament-lock-integration', name: profileName }, + auth: { + version: 1, + profile: profileName, + issuedAt: new Date().toISOString(), + expiresAt: '2999-01-01T00:00:00Z', + sessionId: 'audit', + user: { id: 'audit-user', username: 'audit', displayName: 'audit', roles: [] }, + sanctions: {}, + }, + turnDaemon: { + sendCommand: transport.sendCommand.bind(transport), + requestStatus: transport.requestStatus.bind(transport), + requestCommand: async (command) => { + notifyJoin!(); + await adminAtRedis; + const requestId = await transport.sendCommand(command); + return db.$transaction(async (tx) => { + // 이전 순서에서는 관리자가 DB lock을 보유하여 별도 ENGINE 연결이 막힌다. + await tx.$executeRawUnsafe("SET LOCAL lock_timeout = '500ms'"); + await acquireGameSchemaAdvisoryXactLock(tx, CLOCK_OPERATION_PERSISTENCE_LOCK); + if (command.type !== 'adjustGeneralResources') throw new Error('unexpected command'); + const result = { + type: 'adjustGeneralResources' as const, + ok: true as const, + processed: 1, + missing: 0, + totalGoldDelta: -10, + totalRiceDelta: 0, + }; + await tx.inputEvent.update({ where: { requestId }, data: { status: 'SUCCEEDED', result } }); + return result; + }); + }, + }, + }; + const joinContext = { ...base, requestId: `${prefix}join` } as GameApiContext; + const adminRedis = new Proxy(redis, { + get(target, property) { + if (property === 'set') + return async (...args: Parameters) => { + if (args[0] === `${keys.stateKey}:mutation-lock`) notifyAdmin!(); + return redis.set(...args); + }; + const value = Reflect.get(target, property); + return typeof value === 'function' ? value.bind(target) : value; + }, + }); + const adminContext = { + ...base, + requestId: `${prefix}admin`, + redis: adminRedis, + auth: { ...base.auth!, user: { ...base.auth!.user, roles: ['admin'] } }, + } as GameApiContext; + const callAdmin = async (context: GameApiContext) => { + const caller = appRouter.createCaller(context).tournament; + switch (adminAction) { + case 'setState': + return caller.setState({ + stage: 1, + phase: 0, + type: 0, + auto: false, + openYear: 200, + openMonth: 1, + termSeconds: 60, + nextAt, + }); + case 'patchState': + return caller.patchState({ auto: true }); + case 'setParticipants': + return caller.setParticipants([]); + case 'setMatches': + return caller.setMatches([]); + case 'setBettingEntries': + return caller.setBettingEntries([]); + case 'seedParticipants': + return caller.seedParticipants({ generalIds: [991900] }); + default: + return caller.cancel(); + } + }; + const join = appRouter.createCaller(joinContext).tournament.join(); + const joinOutcome = join.then( + (value) => ({ ok: true, value }), + (error) => ({ ok: false, error }) + ); + await joinAtDaemon; + const admin = callAdmin(adminContext); + const adminOutcome = admin.then( + (value) => ({ ok: true, value }), + (error) => ({ ok: false, error }) + ); + await adminAtRedis; + const [joined, managed] = await Promise.all([joinOutcome, adminOutcome]); + expect(joined).toMatchObject({ ok: true, value: { ok: true, count: 1 } }); + expect(managed).toMatchObject({ ok: true, value: { ok: true } }); + const revision = await redis.get(keys.sourceRevisionKey); + const engineCount = await db.inputEvent.count({ + where: { requestId: { startsWith: prefix }, target: 'ENGINE' }, + }); + // API 입력 원장은 유지한다. 동일 요청 재실행은 Redis/환불 command를 다시 쓰지 않는다. + await expect(callAdmin(adminContext)).resolves.toMatchObject({ ok: true }); + expect(await redis.get(keys.sourceRevisionKey)).toBe(revision); + expect(await db.inputEvent.count({ where: { requestId: { startsWith: prefix }, target: 'ENGINE' } })).toBe( + engineCount + ); + const inputEvent = await db.inputEvent.findUniqueOrThrow({ + where: { requestId: `${prefix}admin:tournament.${adminAction}` }, + }); + expect(inputEvent).toMatchObject({ target: 'API', status: 'SUCCEEDED', attempts: 1 }); + expect(await redis.get(`${keys.stateKey}:mutation-lock`)).toBeNull(); + const retryContext = { ...adminContext, requestId: `${prefix}retry` }; + const beforeFailure = await redis.get(keys.sourceRevisionKey); + await db.worldState.update({ where: { id: -991900 }, data: { clockPhase: 'SUSPENDED' } }); + await expect(callAdmin(retryContext)).rejects.toMatchObject({ code: 'PRECONDITION_FAILED' }); + expect(await redis.get(`${keys.stateKey}:mutation-lock`)).toBeNull(); + expect(await redis.get(keys.sourceRevisionKey)).toBe(beforeFailure); + await db.worldState.update({ where: { id: -991900 }, data: { clockPhase: 'RUNNING' } }); + await expect(callAdmin(retryContext)).resolves.toMatchObject({ ok: true }); + expect( + await db.inputEvent.findUniqueOrThrow({ + where: { requestId: `${prefix}retry:tournament.${adminAction}` }, + }) + ).toMatchObject({ status: 'SUCCEEDED', attempts: 2 }); + } finally { + await redis.del(redisKeys); + await db.inputEvent.deleteMany({ where: { requestId: { startsWith: prefix } } }); + await db.general.deleteMany({ where: { id: 991900 } }); + await db.worldState.deleteMany({ where: { id: -991900 } }); + await db.turnDaemonLease.deleteMany({ where: { profile: profileName } }); + await redisConnector.disconnect(); + await connector.disconnect(); + } + }, + 15000 +); diff --git a/docs/architecture/game-api-daemon-procedure-inventory.md b/docs/architecture/game-api-daemon-procedure-inventory.md index 49495275..4fb8d1bd 100644 --- a/docs/architecture/game-api-daemon-procedure-inventory.md +++ b/docs/architecture/game-api-daemon-procedure-inventory.md @@ -59,6 +59,22 @@ selection-pool create/reselect는 client request ID가 있을 때 합계 **3개 route**다. +## 토너먼트 관리자 Redis/DB 잠금 순서 + +`setState`, `patchState`, `setParticipants`, `setMatches`, `setBettingEntries`, +`seedParticipants`, `cancel`은 위의 ENGINE 완료 대기 49개 route와 별개다. +관리자는 API input-event transaction을 유지하되 인증/관리자 권한 검사 → Redis +mutation lock → API input-event/DB clock fence → 상태 변경 → commit → Redis unlock +순서로 실행한다. `adminProcedure`가 Redis lock을 먼저 획득한 뒤 `procedure`를 +연결하고, `withTournamentClockMutation`은 이미 보유한 lock을 다시 획득하지 않는다. +조회는 mutation lock과 API transaction을 열지 않는다. + +기존 DB → Redis 순서는 Redis lock을 보유하고 ENGINE DB commit을 기다리는 +참가·베팅·worker와 역순 경합하여 요청을 실패시킬 수 있었다. 잠금 순서를 통일하면서 +관리자 API 원장의 중복 요청 결과 재사용, 실패 재시도와 child ENGINE identity를 +보존한다. Redis와 PostgreSQL 사이의 기존 부분 실패 경계는 그대로이며, 이 수정이 +두 저장소의 atomic commit이나 durable saga를 제공하지는 않는다. + ## 기존에 API outer transaction이 없던 ENGINE route | route | 근거 | @@ -96,6 +112,10 @@ fence를 가진 채 ENGINE 결과를 기다리던 교착을 제거했다. 투표 daemon command와 Redis timer projection을 완료하는 계약을 검증한다. - `app/game-api/test/tournamentRouter.test.ts`: 참가·베팅이 API transaction을 열지 않고 request-scoped ENGINE command ID와 Redis mutation lock을 사용하는지 검증한다. +- `app/game-api/test/tournamentLockOrder.integration.test.ts`: 실제 PostgreSQL/Redis에서 + 참가 요청과 7개 관리자 명령의 경합, API replay와 실패 후 같은 요청 재시도를 검사한다. + ENGINE transport는 별도 DB transaction의 실제 clock fence/내구성 접수 경계로 제어하며 + 전체 daemon의 자원 차감·보상 실행을 대신하지 않는다. - `app/game-api/test/inheritRouter.test.ts`, `app/game-engine/test/inheritanceActionPersistence.integration.test.ts`: 인증 actor, point/log, general/message 변경이 `inheritanceAction` ENGINE transaction에 함께 있는지 검증한다.