감사 상세를 무손실 압축하고 이전 기수 정리 처리량을 개선한다
This commit is contained in:
1 parent
da79013226
commit
078b9aa205
23 files changed
+504
-95
No files matched your search
+18
@@ -0,0 +1,18 @@
|
||||
-- 대형 기존 기록을 migration 중 재작성/전체 검사하지 않는다. 이전 배열과 새 압축 형식을 함께 허용한다.
|
||||
BEGIN;
|
||||
SET LOCAL lock_timeout = '2s';
|
||||
ALTER TABLE "play_audit_decision_chunk" DROP CONSTRAINT "play_audit_decision_chunk_steps_check";
|
||||
ALTER TABLE "play_audit_decision_chunk" ADD CONSTRAINT "play_audit_decision_chunk_steps_check"
|
||||
CHECK (COALESCE(
|
||||
jsonb_typeof("steps") = 'array' OR (
|
||||
jsonb_typeof("steps") = 'object'
|
||||
AND "steps"->>'encoding' = 'gzip-json-v1'
|
||||
AND jsonb_typeof("steps"->'data') = 'string'
|
||||
), false
|
||||
)) NOT VALID;
|
||||
-- 기본 약 2KiB 문턱 아래의 반복 JSON도 압축하되 작은 snapshot은 TOAST 밖으로 쪼개지 않는다.
|
||||
ALTER TABLE "play_audit_general" ALTER COLUMN "data" SET STORAGE MAIN;
|
||||
ALTER TABLE "play_audit_general" SET (toast_tuple_target = 512);
|
||||
ALTER TABLE "play_audit_decision" ALTER COLUMN "summary" SET STORAGE MAIN;
|
||||
ALTER TABLE "play_audit_decision" SET (toast_tuple_target = 512);
|
||||
COMMIT;
|
||||
@@ -15,3 +15,4 @@ export * from './playAuditDiplomacy.js';
|
||||
export * from './messageEnvelope.js';
|
||||
export * from './webPushOutbox.js';
|
||||
export * from './playAuditBestEffort.js';
|
||||
export * from './playAuditDecisionCodec.js';
|
||||
@@ -0,0 +1,44 @@
|
||||
import { gzipSync, gunzipSync } from 'node:zlib';
|
||||
import type { InputJsonValue } from './turnEngineDb.js';
|
||||
|
||||
const ENCODING = 'gzip-json-v1';
|
||||
const MAX_STEPS = 128;
|
||||
const MAX_BYTES = 16 * 1024 * 1024;
|
||||
|
||||
/** 조회/재실행 hash에 쓰는 원문은 바꾸지 않고 저장할 때만 무손실 압축한다. */
|
||||
export const encodeAuditDecisionSteps = (steps: readonly InputJsonValue[]): InputJsonValue => {
|
||||
if (steps.length > MAX_STEPS) throw new Error('Audit decision chunk exceeds its step limit');
|
||||
const json = JSON.stringify(steps);
|
||||
// 드문 초대형 기존 payload도 버리거나 자르지 않는다. 압축 reader 상한 밖이면 원래 형식으로 저장한다.
|
||||
if (Buffer.byteLength(json) > MAX_BYTES) return steps.slice();
|
||||
const encoded = { encoding: ENCODING, data: gzipSync(json, { level: 3 }).toString('base64') };
|
||||
return Buffer.byteLength(JSON.stringify(encoded)) < Buffer.byteLength(json) ? encoded : steps.slice();
|
||||
};
|
||||
|
||||
/** 이전 JSONB 배열과 압축 형식을 함께 읽는다. 손상/알 수 없는 버전은 빈 기록으로 숨기지 않는다. */
|
||||
export const decodeAuditDecisionSteps = (value: unknown): unknown[] => {
|
||||
if (Array.isArray(value)) {
|
||||
if (value.length > MAX_STEPS) throw new Error('Audit decision chunk exceeds its step limit');
|
||||
return value;
|
||||
}
|
||||
if (!value || typeof value !== 'object' || !('encoding' in value) || !('data' in value))
|
||||
throw new Error('Invalid audit decision storage');
|
||||
if (
|
||||
value.encoding !== ENCODING ||
|
||||
typeof value.data !== 'string' ||
|
||||
value.data.length > Math.ceil(MAX_BYTES / 3) * 4 + 256 ||
|
||||
!/^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/.test(value.data)
|
||||
)
|
||||
throw new Error('Invalid audit decision encoding');
|
||||
let decoded: unknown;
|
||||
try {
|
||||
decoded = JSON.parse(
|
||||
gunzipSync(Buffer.from(value.data, 'base64'), { maxOutputLength: MAX_BYTES }).toString('utf8')
|
||||
);
|
||||
} catch {
|
||||
// DB 원문이나 압축 payload가 오류 응답/로그에 포함되지 않게 한다.
|
||||
throw new Error('Corrupt audit decision storage');
|
||||
}
|
||||
if (!Array.isArray(decoded) || decoded.length > MAX_STEPS) throw new Error('Invalid audit decision steps');
|
||||
return decoded;
|
||||
};
|
||||
@@ -195,8 +195,9 @@ export const pruneDeliveredReadModelOutbox = async (
|
||||
SELECT "id"
|
||||
FROM "read_model_outbox"
|
||||
WHERE "delivered_at" < ${input.deliveredBefore}
|
||||
ORDER BY "id"
|
||||
ORDER BY "delivered_at", "available_at", "id"
|
||||
LIMIT ${limit}
|
||||
FOR UPDATE SKIP LOCKED
|
||||
)
|
||||
RETURNING "id"
|
||||
`);
|
||||
|
||||
@@ -0,0 +1,54 @@
|
||||
import { gzipSync } from 'node:zlib';
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import { decodeAuditDecisionSteps, encodeAuditDecisionSteps } from '../src/playAuditDecisionCodec.js';
|
||||
|
||||
describe('lossless audit decision storage', () => {
|
||||
it('compresses repeated context while preserving every value and its order', () => {
|
||||
const steps = Array.from({ length: 128 }, (_, sequence) => ({
|
||||
sequence,
|
||||
phase: 'general',
|
||||
generalId: 51,
|
||||
nationId: 3,
|
||||
cityId: 12,
|
||||
year: 190,
|
||||
month: 7,
|
||||
kind: 'RNG',
|
||||
value: sequence / 128,
|
||||
text: '판단·줄바꿈\n따옴표 " 그대로',
|
||||
empty: null,
|
||||
}));
|
||||
const stored = encodeAuditDecisionSteps(steps);
|
||||
expect(Array.isArray(stored)).toBe(false);
|
||||
expect(Buffer.byteLength(JSON.stringify(stored))).toBeLessThan(Buffer.byteLength(JSON.stringify(steps)) / 3);
|
||||
expect(decodeAuditDecisionSteps(stored)).toEqual(steps);
|
||||
});
|
||||
it('keeps tiny and historical uncompressed arrays readable', () => {
|
||||
expect(encodeAuditDecisionSteps([])).toEqual([]);
|
||||
expect(decodeAuditDecisionSteps([{ marker: 'legacy', sequence: 0 }])).toEqual([
|
||||
{ marker: 'legacy', sequence: 0 },
|
||||
]);
|
||||
});
|
||||
it('rejects unknown versions, bad base64, truncated gzip and invalid contents', () => {
|
||||
for (const value of [
|
||||
null,
|
||||
{},
|
||||
{ encoding: 'gzip-json-v2', data: '' },
|
||||
{ encoding: 'gzip-json-v1', data: '%%%' },
|
||||
{ encoding: 'gzip-json-v1', data: 'YQ==' },
|
||||
{ encoding: 'gzip-json-v1', data: gzipSync('{}').toString('base64') },
|
||||
{ encoding: 'gzip-json-v1', data: gzipSync(JSON.stringify(Array(129).fill(null))).toString('base64') },
|
||||
])
|
||||
expect(() => decodeAuditDecisionSteps(value)).toThrow();
|
||||
expect(() => encodeAuditDecisionSteps(Array(129).fill(null))).toThrow();
|
||||
});
|
||||
it('bounds decompression and preserves oversized raw records rather than truncating them', () => {
|
||||
const steps = [{ text: 'x'.repeat(16 * 1024 * 1024) }];
|
||||
expect(encodeAuditDecisionSteps(steps)).toEqual(steps);
|
||||
expect(() =>
|
||||
decodeAuditDecisionSteps({
|
||||
encoding: 'gzip-json-v1',
|
||||
data: gzipSync(JSON.stringify(steps)).toString('base64'),
|
||||
})
|
||||
).toThrow('Corrupt');
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user