diff --git a/app/game-frontend/e2e/connectionRecovery.spec.ts b/app/game-frontend/e2e/connectionRecovery.spec.ts new file mode 100644 index 00000000..416e863d --- /dev/null +++ b/app/game-frontend/e2e/connectionRecovery.spec.ts @@ -0,0 +1,97 @@ +import { expect, test, type Page, type Route } from '@playwright/test'; +import { gameProfile, gameTrpcRoute } from './gameTestPaths.js'; + +const response = (data: unknown) => ({ result: { data } }); + +const operationNames = (route: Route): string[] => { + const url = new URL(route.request().url()); + return decodeURIComponent(url.pathname.slice(url.pathname.lastIndexOf('/trpc/') + 6)).split(','); +}; + +const installRecoveryFixture = async (page: Page) => { + let unavailable = true; + let lobbyRequests = 0; + await page.addInitScript( + ({ token, profile }) => { + window.localStorage.setItem('sammo-game-token', token); + window.localStorage.setItem('sammo-game-profile', profile); + window.addEventListener('sammo:game-server-reconnected', () => { + const state = window as typeof window & { __gameServerReconnects?: number }; + state.__gameServerReconnects = (state.__gameServerReconnects ?? 0) + 1; + }); + }, + { token: 'ga_recovery', profile: gameProfile } + ); + await page.route(gameTrpcRoute, async (route) => { + const operations = operationNames(route); + if (operations.includes('lobby.info')) { + lobbyRequests += 1; + if (unavailable) { + await route.fulfill({ + status: 503, + contentType: 'application/json', + body: JSON.stringify({ error: 'profile switch in progress' }), + }); + return; + } + } + await route.fulfill({ + status: 200, + contentType: 'application/json', + body: JSON.stringify( + operations.map((operation) => { + if (operation === 'auth.status') return response({ userId: 'recovery-user' }); + if (operation === 'lobby.info') return response({ myGeneral: null }); + if (operation === 'join.getConfig') return response({}); + return response(null); + }) + ), + }); + }); + + return { + recover: () => { + unavailable = false; + }, + lobbyRequests: () => lobbyRequests, + }; +}; + +for (const viewport of [ + { name: 'desktop', width: 1280, height: 800 }, + { name: 'mobile', width: 390, height: 844 }, +]) { + test(`keeps the current screen and reconnects after a transient profile switch on ${viewport.name}`, async ({ + page, + }) => { + await page.setViewportSize({ width: viewport.width, height: viewport.height }); + const fixture = await installRecoveryFixture(page); + await page.goto('select-general'); + + const heading = page.locator('.page-title'); + await expect(heading).toContainText('장 수 선 택'); + const notice = page.getByTestId('game-server-connection-notice'); + await expect(notice).toBeVisible(); + await expect(notice).toContainText('화면을 유지한 채 자동으로 다시 연결합니다'); + await expect(notice).toHaveCSS('position', 'fixed'); + const noticeBox = await notice.boundingBox(); + expect(noticeBox).not.toBeNull(); + expect(noticeBox!.x).toBeGreaterThanOrEqual(0); + expect(noticeBox!.x + noticeBox!.width).toBeLessThanOrEqual(viewport.width); + const documentWidthDuringReconnect = await page.evaluate(() => document.documentElement.scrollWidth); + await page.evaluate(() => { + Object.assign(window, { __connectionRecoveryPageMarker: 'kept' }); + }); + + fixture.recover(); + await page.evaluate(() => window.dispatchEvent(new Event('online'))); + + await expect(notice).toHaveCount(0); + await expect(heading).toContainText('장 수 선 택'); + expect(await page.evaluate(() => document.documentElement.scrollWidth)).toBe(documentWidthDuringReconnect); + expect(await page.evaluate(() => Reflect.get(window, '__connectionRecoveryPageMarker'))).toBe('kept'); + expect(await page.evaluate(() => Reflect.get(window, '__gameServerReconnects'))).toBe(1); + expect(fixture.lobbyRequests()).toBeGreaterThanOrEqual(2); + expect(await page.evaluate(() => window.localStorage.getItem('sammo-game-token'))).toBe('ga_recovery'); + }); +} diff --git a/app/game-frontend/e2e/playwright.config.mjs b/app/game-frontend/e2e/playwright.config.mjs index 5f422500..3d2b8fa4 100644 --- a/app/game-frontend/e2e/playwright.config.mjs +++ b/app/game-frontend/e2e/playwright.config.mjs @@ -43,6 +43,7 @@ export default defineConfig({ 'npcPossession.spec.ts', 'joinLayout.spec.ts', 'deploymentVersionNotice.spec.ts', + 'connectionRecovery.spec.ts', ], fullyParallel: false, workers: 1, diff --git a/app/game-frontend/e2e/session-auth.spec.ts b/app/game-frontend/e2e/session-auth.spec.ts index bb86a04d..2ffe278b 100644 --- a/app/game-frontend/e2e/session-auth.spec.ts +++ b/app/game-frontend/e2e/session-auth.spec.ts @@ -159,6 +159,6 @@ test('keeps a valid ga_ token when only lobby.info is unavailable', async ({ pag await expect(page.locator('.page-title')).toContainText('장 수 선 택'); expect(await page.evaluate(() => window.localStorage.getItem('sammo-game-token'))).toBe('ga_valid'); expect(statusRequests).toBe(1); - expect(lobbyRequests).toBe(1); + expect(lobbyRequests).toBeGreaterThanOrEqual(1); expect(gatewayRequests).toBe(0); }); diff --git a/app/game-frontend/src/App.vue b/app/game-frontend/src/App.vue index eae06717..488f243e 100644 --- a/app/game-frontend/src/App.vue +++ b/app/game-frontend/src/App.vue @@ -1,5 +1,6 @@ + + + + diff --git a/app/game-frontend/src/composables/useGameServerConnectionRecovery.ts b/app/game-frontend/src/composables/useGameServerConnectionRecovery.ts new file mode 100644 index 00000000..7d9389f6 --- /dev/null +++ b/app/game-frontend/src/composables/useGameServerConnectionRecovery.ts @@ -0,0 +1,75 @@ +import { computed, onBeforeUnmount, onMounted, watch } from 'vue'; +import { trpc } from '../utils/trpc'; +import { + GAME_SERVER_RECONNECTED_EVENT, + gameServerConnection, + retryDelayForFailure, +} from '../utils/gameServerConnection'; + +export const useGameServerConnectionRecovery = () => { + const reconnecting = computed(() => gameServerConnection.status.value === 'reconnecting'); + let retryTimer: ReturnType | null = null; + let failureCount = 0; + let mounted = false; + let wasReconnecting = false; + + const clearRetry = (): void => { + if (retryTimer) clearTimeout(retryTimer); + retryTimer = null; + }; + + const scheduleRetry = (): void => { + if (!mounted || !reconnecting.value || retryTimer) return; + failureCount += 1; + retryTimer = setTimeout(() => { + retryTimer = null; + void trpc.lobby.info + .query() + .catch(() => undefined) + .finally(() => scheduleRetry()); + }, retryDelayForFailure(failureCount)); + }; + + const retryNow = (): void => { + if (!reconnecting.value) return; + clearRetry(); + failureCount = 0; + void trpc.lobby.info + .query() + .catch(() => undefined) + .finally(() => scheduleRetry()); + }; + + const stopWatching = watch( + reconnecting, + (current) => { + if (current) { + wasReconnecting = true; + scheduleRetry(); + return; + } + clearRetry(); + failureCount = 0; + if (wasReconnecting && mounted) { + window.dispatchEvent(new Event(GAME_SERVER_RECONNECTED_EVENT)); + } + wasReconnecting = false; + }, + { immediate: true } + ); + + onMounted(() => { + mounted = true; + window.addEventListener('online', retryNow); + scheduleRetry(); + }); + + onBeforeUnmount(() => { + mounted = false; + clearRetry(); + stopWatching(); + window.removeEventListener('online', retryNow); + }); + + return { reconnecting }; +}; diff --git a/app/game-frontend/src/stores/mainDashboard.ts b/app/game-frontend/src/stores/mainDashboard.ts index ed15d8ca..43d406ca 100644 --- a/app/game-frontend/src/stores/mainDashboard.ts +++ b/app/game-frontend/src/stores/mainDashboard.ts @@ -22,6 +22,7 @@ import { createBroadcastTabCoordinator, type BroadcastTabCoordinator } from '../ import { resolveWithReadModelSnapshotFallback } from '../utils/readModelDeltaRecovery'; import { createRealtimeRequestOptions } from '../utils/realtimeAccessGrant'; import { markGameServerContact } from '../utils/gameServerActivity'; +import { GAME_SERVER_RECONNECTED_EVENT } from '../utils/gameServerConnection'; import { gameFrontendRuntimeConfig } from '../config/runtimeConfig'; const REALTIME_FULL_REFRESH_MIN_INTERVAL_MS = 5_000; @@ -1261,12 +1262,19 @@ export const useMainDashboardStore = defineStore('mainDashboard', () => { void refreshQueue.request().finally(() => reconcileRealtimeCoordinator()); }; + const handleGameServerReconnected = () => { + if (!realtimeActive.value || document.visibilityState === 'hidden') return; + realtimeRefreshQueue.beginCooldown(); + void refreshQueue.request().finally(() => reconcileRealtimeCoordinator()); + }; + const startRealtime = () => { if (typeof window === 'undefined' || realtimeActive.value) return; realtimeActive.value = true; realtimeRefreshQueue.beginCooldown(); if (!visibilityListenerInstalled) { document.addEventListener('visibilitychange', handleVisibilityChange); + window.addEventListener(GAME_SERVER_RECONNECTED_EVENT, handleGameServerReconnected); visibilityListenerInstalled = true; } reconcileRealtimeCoordinator(); @@ -1279,6 +1287,7 @@ export const useMainDashboardStore = defineStore('mainDashboard', () => { closeRealtimeCoordinator(); if (visibilityListenerInstalled) { document.removeEventListener('visibilitychange', handleVisibilityChange); + window.removeEventListener(GAME_SERVER_RECONNECTED_EVENT, handleGameServerReconnected); visibilityListenerInstalled = false; } realtimeStatus.value = realtimeEnabled.value ? 'idle' : 'paused'; diff --git a/app/game-frontend/src/utils/gameServerConnection.ts b/app/game-frontend/src/utils/gameServerConnection.ts new file mode 100644 index 00000000..340d59d9 --- /dev/null +++ b/app/game-frontend/src/utils/gameServerConnection.ts @@ -0,0 +1,54 @@ +import { readonly, ref, type Ref } from 'vue'; + +export const GAME_SERVER_RETRY_DELAYS_MS = [500, 1_000, 2_000, 4_000] as const; +export const GAME_SERVER_RECONNECTED_EVENT = 'sammo:game-server-reconnected'; + +export type GameServerConnectionStatus = 'connected' | 'reconnecting'; + +export type GameServerConnectionTracker = { + status: Readonly>; + markFailure: () => void; + markConnected: () => void; +}; + +export const isRetryableGameServerStatus = (status: number): boolean => + status === 502 || status === 503 || status === 504; + +export const canConfirmGameServerRecovery = (status: number): boolean => status < 500; + +export const isGameServerRecoveryRequest = (input: RequestInfo | URL): boolean => { + const rawUrl = typeof input === 'string' || input instanceof URL ? String(input) : input.url; + try { + const operationList = decodeURIComponent(new URL(rawUrl, 'http://game.local').pathname).split('/').at(-1); + return operationList?.split(',').includes('lobby.info') ?? false; + } catch { + return false; + } +}; + +export const isAbortedGameServerRequest = (error: unknown): boolean => + error instanceof DOMException && error.name === 'AbortError'; + +export const retryDelayForFailure = (failureCount: number): number => + GAME_SERVER_RETRY_DELAYS_MS[ + Math.min(Math.max(0, Math.trunc(failureCount) - 1), GAME_SERVER_RETRY_DELAYS_MS.length - 1) + ]; + +export const createGameServerConnectionTracker = (): GameServerConnectionTracker => { + const status = ref('connected'); + + return { + status: readonly(status), + markFailure() { + status.value = 'reconnecting'; + }, + markConnected() { + status.value = 'connected'; + }, + }; +}; + +export const gameServerConnection = createGameServerConnectionTracker(); + +export const markGameServerConnectionFailure = (): void => gameServerConnection.markFailure(); +export const markGameServerConnectionReady = (): void => gameServerConnection.markConnected(); diff --git a/app/game-frontend/src/utils/trpc.ts b/app/game-frontend/src/utils/trpc.ts index 1c4a9e45..d4c4549c 100644 --- a/app/game-frontend/src/utils/trpc.ts +++ b/app/game-frontend/src/utils/trpc.ts @@ -5,6 +5,15 @@ import type { AppRouter } from '@sammo-ts/game-api'; import { gameFrontendRuntimeConfig } from '../config/runtimeConfig'; import { resolveBatchRealtimeAccessGrant } from './realtimeAccessGrant'; import { markGameServerContact } from './gameServerActivity'; +import { + canConfirmGameServerRecovery, + gameServerConnection, + isAbortedGameServerRequest, + isGameServerRecoveryRequest, + isRetryableGameServerStatus, + markGameServerConnectionFailure, + markGameServerConnectionReady, +} from './gameServerConnection'; const getGameToken = (): string | null => { if (typeof window === 'undefined') { @@ -20,9 +29,24 @@ export const trpc = createTRPCProxyClient({ url: gameFrontendRuntimeConfig.gameApiUrl, ...trpcJsonBodyHttpClientOptions, async fetch(input, init) { - const result = await globalThis.fetch(input, init); - markGameServerContact(); - return result; + try { + const result = await globalThis.fetch(input, init); + if (isRetryableGameServerStatus(result.status)) { + markGameServerConnectionFailure(); + } else { + markGameServerContact(); + if ( + gameServerConnection.status.value === 'connected' || + (isGameServerRecoveryRequest(input) && canConfirmGameServerRecovery(result.status)) + ) { + markGameServerConnectionReady(); + } + } + return result; + } catch (error) { + if (!isAbortedGameServerRequest(error)) markGameServerConnectionFailure(); + throw error; + } }, headers({ opList }) { const token = getGameToken(); diff --git a/app/game-frontend/test/gameServerConnection.test.ts b/app/game-frontend/test/gameServerConnection.test.ts new file mode 100644 index 00000000..b1319a11 --- /dev/null +++ b/app/game-frontend/test/gameServerConnection.test.ts @@ -0,0 +1,50 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; + +import { + canConfirmGameServerRecovery, + createGameServerConnectionTracker, + isGameServerRecoveryRequest, + isRetryableGameServerStatus, + retryDelayForFailure, +} from '../src/utils/gameServerConnection.ts'; + +void test('classifies only transient deployment gateway responses as reconnectable', () => { + assert.equal(isRetryableGameServerStatus(502), true); + assert.equal(isRetryableGameServerStatus(503), true); + assert.equal(isRetryableGameServerStatus(504), true); + assert.equal(isRetryableGameServerStatus(401), false); + assert.equal(isRetryableGameServerStatus(403), false); + assert.equal(isRetryableGameServerStatus(500), false); +}); + +void test('does not treat a server error from the recovery probe as restored service', () => { + assert.equal(canConfirmGameServerRecovery(200), true); + assert.equal(canConfirmGameServerRecovery(401), true); + assert.equal(canConfirmGameServerRecovery(403), true); + assert.equal(canConfirmGameServerRecovery(500), false); + assert.equal(canConfirmGameServerRecovery(503), false); +}); + +void test('uses bounded reconnect delays', () => { + assert.equal(retryDelayForFailure(1), 500); + assert.equal(retryDelayForFailure(2), 1_000); + assert.equal(retryDelayForFailure(3), 2_000); + assert.equal(retryDelayForFailure(4), 4_000); + assert.equal(retryDelayForFailure(20), 4_000); +}); + +void test('recognizes only the read-only lobby probe as connection recovery evidence', () => { + assert.equal(isGameServerRecoveryRequest('/che/api/trpc/lobby.info'), true); + assert.equal(isGameServerRecoveryRequest('/che/api/trpc/auth.status,lobby.info?batch=1'), true); + assert.equal(isGameServerRecoveryRequest('/che/api/trpc/join.getConfig'), false); +}); + +void test('retains reconnecting state until the server responds again', () => { + const tracker = createGameServerConnectionTracker(); + tracker.markFailure(); + tracker.markFailure(); + assert.equal(tracker.status.value, 'reconnecting'); + tracker.markConnected(); + assert.equal(tracker.status.value, 'connected'); +}); diff --git a/app/gateway-api/src/orchestrator/gatewayOrchestrator.ts b/app/gateway-api/src/orchestrator/gatewayOrchestrator.ts index f3233b49..6abb1a28 100644 --- a/app/gateway-api/src/orchestrator/gatewayOrchestrator.ts +++ b/app/gateway-api/src/orchestrator/gatewayOrchestrator.ts @@ -1857,7 +1857,7 @@ export class GatewayOrchestrator implements GatewayOrchestratorHandle { await this.appendOperationLog(operationId, 'switch', '기존 profile process를 정지합니다.'); await assertLease(); - await this.stopProfile(profile, assertLease); + await this.stopProfile(profile, assertLease, { preserveStaticFrontend: true }); oldRuntimeStopped = true; const profileDatabaseUrl = this.resolveProfileDatabaseUrl(profile); await this.appendOperationLog(operationId, 'migration', '선택 버전의 game migration을 적용합니다.'); @@ -2821,7 +2821,11 @@ export class GatewayOrchestrator implements GatewayOrchestratorHandle { return false; } - private async stopProfile(profile: GatewayProfileRecord, assertLease?: () => Promise): Promise { + private async stopProfile( + profile: GatewayProfileRecord, + assertLease?: () => Promise, + options: { preserveStaticFrontend?: boolean } = {} + ): Promise { const frontendName = buildProcessName(profile.profileName, 'frontend'); const apiName = buildProcessName(profile.profileName, 'api'); const daemonName = buildProcessName(profile.profileName, 'daemon'); @@ -2829,7 +2833,9 @@ export class GatewayOrchestrator implements GatewayOrchestratorHandle { const battleSimName = buildProcessName(profile.profileName, 'battle-sim'); const tournamentName = buildProcessName(profile.profileName, 'tournament'); await assertLease?.(); - if (this.frontendServeMode === 'static') { + // A normal DEPLOY keeps the last immutable page available while its API is replaced. + // STOP/RESET still remove the pointer because those operations intentionally close the profile. + if (this.frontendServeMode === 'static' && !options.preserveStaticFrontend) { await this.artifactManager.deactivate(profile.profile); } await assertLease?.(); diff --git a/app/gateway-api/test/profileDeployOperation.test.ts b/app/gateway-api/test/profileDeployOperation.test.ts index f19848b4..24945af9 100644 --- a/app/gateway-api/test/profileDeployOperation.test.ts +++ b/app/gateway-api/test/profileDeployOperation.test.ts @@ -6,6 +6,7 @@ import { afterEach, describe, expect, it } from 'vitest'; import { GatewayOrchestrator } from '../src/orchestrator/gatewayOrchestrator.js'; import type { BuildCommand } from '../src/orchestrator/buildRunner.js'; +import { FrontendArtifactManager } from '../src/orchestrator/frontendArtifactManager.js'; import type { ProcessManager } from '../src/orchestrator/processManager.js'; import type { GatewayClaimedProfileUpdate, @@ -62,6 +63,17 @@ afterEach(async () => { describe('profile DEPLOY operation', () => { it('migrates and atomically switches a static frontend without executing the reset seed path', async () => { const workspace = await createReleaseWorkspace(); + const artifactRoot = path.join(workspace, 'artifact-volume'); + const oldFrontendRoot = path.join(workspace, 'old-static-frontend'); + await fs.mkdir(oldFrontendRoot, { recursive: true }); + await fs.writeFile(path.join(oldFrontendRoot, 'index.html'), 'old static profile'); + const artifactManager = new FrontendArtifactManager(artifactRoot); + const oldArtifact = await artifactManager.stage({ + frontendKey: 'che', + sourceRoot: oldFrontendRoot, + commitSha: '2222222222222222222222222222222222222222', + }); + await artifactManager.activate('che', oldArtifact.releaseId); const profile: GatewayProfileRecord = { profileName: 'che:1010', profile: 'che', @@ -149,13 +161,16 @@ describe('profile DEPLOY operation', () => { const backendProcessNames = processNames.filter((name) => !name.endsWith(':game-frontend')); const running = new Set(processNames); const startedDefinitions: Array[0]> = []; + const frontendDuringStop: string[] = []; const processManager: ProcessManager = { list: async () => [...running].map((name) => ({ name, status: 'online' })), start: async (definition) => { startedDefinitions.push(definition); running.add(definition.name); }, - stop: async () => {}, + stop: async () => { + frontendDuringStop.push(await fs.readFile(path.join(artifactRoot, 'che', 'current', 'index.html'), 'utf8')); + }, delete: async (name) => { running.delete(name); }, @@ -190,7 +205,7 @@ describe('profile DEPLOY operation', () => { gameTokenSecret: 'test-secret', gatewayInternalApiUrl: 'http://127.0.0.1:15001', frontendServeMode: 'static', - frontendArtifactRoot: path.join(workspace, 'artifact-volume'), + frontendArtifactRoot: artifactRoot, frontendReadinessOrigin: 'http://caddy', baseEnv: { GATEWAY_DATABASE_URL: @@ -271,8 +286,10 @@ describe('profile DEPLOY operation', () => { expect(logs.map((entry) => entry.message).join('\n')).not.toContain('pass@integration.invalid'); expect(logs.map((entry) => entry.message).join('\n')).toContain('[REDACTED]'); expect([...running].sort()).toEqual([...backendProcessNames].sort()); + expect(frontendDuringStop).toHaveLength(processNames.length); + expect(frontendDuringStop.every((html) => html.includes('old static profile'))).toBe(true); expect( - await fs.readFile(path.join(workspace, 'artifact-volume', 'che', 'current', 'index.html'), 'utf8') + await fs.readFile(path.join(artifactRoot, 'che', 'current', 'index.html'), 'utf8') ).toContain('/gateway/profile-assets/'); }); }); diff --git a/docs/release-operations.md b/docs/release-operations.md index f7b4958a..a811a0d3 100644 --- a/docs/release-operations.md +++ b/docs/release-operations.md @@ -185,9 +185,12 @@ Gateway process 전환이 진행 중인 profile migration·seed 실행자를 중 1. 대상 commit의 game API target과 그 transitive engine/worker artifact를 빌드한 뒤, profile frontend typecheck와 bundle을 공유 Turbo cache에서 복원하거나 생성합니다. -2. 기존 profile PM2 process를 정지합니다. +2. 정적 운영 모드에서는 현재 frontend artifact를 계속 게시한 채 기존 profile PM2 + process를 정지합니다. Preview 개발 모드와 명시적 STOP/RESET의 frontend 종료 계약은 + 바꾸지 않습니다. 3. profile game schema에 `prisma migrate deploy`를 실행합니다. -4. Scenario seed를 실행하지 않고 frontend, API, daemon과 worker를 시작합니다. +4. Scenario seed를 실행하지 않고 API, daemon과 worker를 시작하고 새 정적 frontend + artifact를 원자적으로 활성화합니다. 5. HTTP와 모든 PM2 role의 readiness가 확인된 뒤 build commit을 게시합니다. 이 모드는 현재 scenario, status와 인게임 DB를 유지합니다. Migration이 @@ -220,6 +223,14 @@ Profile process 전환 중에는 frontend/API port가 잠시 닫힐 수 있습 표시합니다. 정상 응답을 한 번 받은 profile은 실패한 지도·인증 재확인 중에도 마지막 상세를 유지합니다. +열린 game frontend도 502/503/504 또는 network failure를 인증 실패와 분리하여 현재 +화면을 유지하고 상단에 연결 복구 안내를 표시합니다. 0.5·1·2·4초 뒤 읽기 전용 +`lobby.info`만 재시도하며 이후에도 4초 상한을 유지합니다. 다른 조회의 우연한 성공은 +복구 완료로 간주하지 않고 이 probe가 성공해야 안내를 닫습니다. Mutation은 자동 +재전송하지 않으며, main dashboard가 열린 경우 복구 직후 기존 bounded refresh queue로 +전체 projection을 한 번 다시 읽습니다. 401/403과 일반 500 제품 오류는 이 배포 복구 +상태로 분류하지 않습니다. + ### 시나리오 초기화 시나리오 초기화는 새 시즌이나 새 scenario로 현 시즌 데이터를 교체할 때