devsam/image
1
0
Fork 3
Code Issues 1 Pull Requests Releases 1 Wiki Activity

feat: manage uploaded image retention

Browse Source
This commit is contained in:
Hide_D
2026-08-07 07:14:14 +00:00
parent dc0e1a6da9
commit a935b23571
13 changed files with 966 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files
node-hook/src/admin-panel.mjs
+234
View File
File diff suppressed because one or more lines are too long
node-hook/src/asset-store.mjs
+355
View File
@@ -0,0 +1,355 @@
import { createHash } from 'node:crypto';
import { mkdir, readFile, readdir, rename, stat, unlink } from 'node:fs/promises';
import { dirname, join } from 'node:path';
import { DatabaseSync } from 'node:sqlite';
import { DeploymentError } from './git-service.mjs';
const CLIENT_PATTERN = /^[a-z0-9][a-z0-9_-]{1,31}$/;
const FILE_PATTERN = /^[a-f0-9]{32}\.(?:avif|webp|jpe?g|png|gif)$/;
const CONTENT_PATH_PATTERN = /^uploads\/([a-z0-9][a-z0-9_-]{1,31})\/([a-f0-9]{32}\.(?:avif|webp|jpe?g|png|gif))$/;
function publicPath(category, client, filename) {
return category === 'user-icons'
? `icons/users/${client}/${filename}`
: `uploads/${client}/${filename}`;
}
function fileLocation(root, category, client, filename) {
return join(root, category, client, filename);
}
function rowToAsset(row, now, retentionMs, quarantineMs) {
if (!row) return null;
return {
path: row.path,
category: row.category,
client: row.client,
filename: row.filename,
sizeBytes: row.size_bytes,
digest: row.digest,
createdAt: new Date(row.created_at).toISOString(),
lastSeenAt: new Date(row.last_seen_at).toISOString(),
state: row.state,
candidateAt: row.candidate_at === null ? null : new Date(row.candidate_at).toISOString(),
quarantinedAt: row.quarantined_at === null ? null : new Date(row.quarantined_at).toISOString(),
deletedAt: row.deleted_at === null ? null : new Date(row.deleted_at).toISOString(),
eligibleAt: row.category === 'content'
? new Date(row.last_seen_at + retentionMs).toISOString()
: null,
deleteAvailableAt: row.quarantined_at === null
? null
: new Date(row.quarantined_at + quarantineMs).toISOString(),
deleteAvailable: row.state === 'quarantined' && row.quarantined_at + quarantineMs <= now,
};
}
export class AssetStore {
constructor(config, options = {}) {
this.root = config.uploadRoot;
this.dbPath = config.assetDbPath;
this.retentionMs = config.contentRetentionMs;
this.quarantineMs = config.contentQuarantineMs;
this.maintenanceIntervalMs = config.assetMaintenanceIntervalMs;
this.touchFlushIntervalMs = config.assetTouchFlushIntervalMs;
this.now = options.now ?? (() => Date.now());
this.pendingTouches = new Set();
this.touchTimer = null;
this.maintenanceTimer = null;
this.db = null;
}
async initialize() {
await mkdir(this.root, { recursive: true });
await mkdir(dirname(this.dbPath), { recursive: true });
this.db = new DatabaseSync(this.dbPath);
this.db.exec(`
PRAGMA journal_mode = WAL;
PRAGMA synchronous = FULL;
PRAGMA busy_timeout = 5000;
CREATE TABLE IF NOT EXISTS asset (
path TEXT PRIMARY KEY,
category TEXT NOT NULL CHECK (category IN ('user-icons', 'content')),
client TEXT NOT NULL,
filename TEXT NOT NULL,
size_bytes INTEGER NOT NULL,
digest TEXT,
created_at INTEGER NOT NULL,
last_seen_at INTEGER NOT NULL,
state TEXT NOT NULL DEFAULT 'active'
CHECK (state IN ('active', 'candidate', 'quarantined', 'deleted')),
candidate_at INTEGER,
quarantined_at INTEGER,
deleted_at INTEGER
);
CREATE INDEX IF NOT EXISTS asset_state_seen_idx ON asset(category, state, last_seen_at);
CREATE INDEX IF NOT EXISTS asset_client_created_idx ON asset(client, created_at DESC);
`);
await this.#inventoryExisting();
this.markCandidates();
this.maintenanceTimer = setInterval(() => {
try {
this.flushTouches();
this.markCandidates();
} catch (error) {
console.error(JSON.stringify({ level: 'error', message: 'asset maintenance failed', reason: error.message }));
}
}, this.maintenanceIntervalMs);
this.maintenanceTimer.unref();
}
register({ category, client, filename, body, digest, createdAt = this.now() }) {
this.#assertOpen();
if (!['user-icons', 'content'].includes(category) || !CLIENT_PATTERN.test(client) || !FILE_PATTERN.test(filename)) {
throw new DeploymentError('Invalid asset path', 400);
}
const path = publicPath(category, client, filename);
const sizeBytes = body?.length ?? 0;
const sha256 = digest ?? (body ? createHash('sha256').update(body).digest('hex') : null);
this.db.prepare(`
INSERT INTO asset(path, category, client, filename, size_bytes, digest, created_at, last_seen_at, state)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, 'active')
ON CONFLICT(path) DO UPDATE SET
size_bytes = excluded.size_bytes,
digest = COALESCE(asset.digest, excluded.digest)
WHERE asset.state != 'deleted'
`).run(path, category, client, filename, sizeBytes, sha256, createdAt, createdAt);
return path;
}
touch(path) {
if (!CONTENT_PATH_PATTERN.test(path)) return false;
this.pendingTouches.add(path);
if (!this.touchTimer) {
this.touchTimer = setTimeout(() => {
this.touchTimer = null;
try {
this.flushTouches();
} catch (error) {
console.error(JSON.stringify({ level: 'error', message: 'asset access flush failed', reason: error.message }));
}
}, this.touchFlushIntervalMs);
this.touchTimer.unref();
}
return true;
}
flushTouches() {
this.#assertOpen();
if (this.pendingTouches.size === 0) return 0;
const paths = [...this.pendingTouches];
this.pendingTouches.clear();
const touchedAt = this.now();
const update = this.db.prepare(`
UPDATE asset
SET last_seen_at = ?,
state = CASE WHEN state = 'candidate' THEN 'active' ELSE state END,
candidate_at = CASE WHEN state = 'candidate' THEN NULL ELSE candidate_at END
WHERE path = ? AND category = 'content' AND state IN ('active', 'candidate')
`);
this.db.exec('BEGIN IMMEDIATE');
try {
let changed = 0;
for (const path of paths) changed += Number(update.run(touchedAt, path).changes);
this.db.exec('COMMIT');
return changed;
} catch (error) {
this.db.exec('ROLLBACK');
for (const path of paths) this.pendingTouches.add(path);
throw error;
}
}
markCandidates() {
this.#assertOpen();
const now = this.now();
const cutoff = now - this.retentionMs;
return Number(this.db.prepare(`
UPDATE asset
SET state = 'candidate', candidate_at = ?
WHERE category = 'content' AND state = 'active' AND last_seen_at <= ?
`).run(now, cutoff).changes);
}
summary() {
this.#assertOpen();
const grouped = this.db.prepare(`
SELECT category, state, COUNT(*) AS count, COALESCE(SUM(size_bytes), 0) AS bytes
FROM asset
GROUP BY category, state
`).all();
const result = { totalCount: 0, totalBytes: 0, groups: {} };
for (const row of grouped) {
const key = `${row.category}:${row.state}`;
result.groups[key] = { count: Number(row.count), bytes: Number(row.bytes) };
if (row.state !== 'deleted') {
result.totalCount += Number(row.count);
result.totalBytes += Number(row.bytes);
}
}
return result;
}
list({ category, state, client, search = '', limit = 100, offset = 0 } = {}) {
this.#assertOpen();
const where = [];
const values = [];
if (category) { where.push('category = ?'); values.push(category); }
if (state) { where.push('state = ?'); values.push(state); }
if (client) { where.push('client = ?'); values.push(client); }
if (search) { where.push('(path LIKE ? OR digest LIKE ?)'); values.push(`%${search}%`, `%${search}%`); }
const clause = where.length > 0 ? `WHERE ${where.join(' AND ')}` : '';
const boundedLimit = Math.max(1, Math.min(200, Number(limit) || 100));
const boundedOffset = Math.max(0, Number(offset) || 0);
const total = Number(this.db.prepare(`SELECT COUNT(*) AS count FROM asset ${clause}`).get(...values).count);
const rows = this.db.prepare(`
SELECT * FROM asset ${clause}
ORDER BY CASE state WHEN 'candidate' THEN 0 WHEN 'quarantined' THEN 1 ELSE 2 END,
last_seen_at ASC, path ASC
LIMIT ? OFFSET ?
`).all(...values, boundedLimit, boundedOffset);
const now = this.now();
return {
total,
limit: boundedLimit,
offset: boundedOffset,
assets: rows.map((row) => rowToAsset(row, now, this.retentionMs, this.quarantineMs)),
};
}
get(path) {
this.#assertOpen();
return rowToAsset(
this.db.prepare('SELECT * FROM asset WHERE path = ?').get(path),
this.now(),
this.retentionMs,
this.quarantineMs,
);
}
async preview(path) {
const asset = this.get(path);
if (!asset || !['active', 'candidate'].includes(asset.state)) {
throw new DeploymentError('Asset preview is not available', 404);
}
const contentTypeByExtension = {
avif: 'image/avif', webp: 'image/webp', jpg: 'image/jpeg', jpeg: 'image/jpeg', png: 'image/png', gif: 'image/gif',
};
const extension = asset.filename.slice(asset.filename.lastIndexOf('.') + 1);
return {
body: await readFile(fileLocation(this.root, asset.category, asset.client, asset.filename)),
contentType: contentTypeByExtension[extension],
};
}
async quarantine(path) {
this.flushTouches();
const asset = this.get(path);
if (!asset || asset.category !== 'content' || asset.state !== 'candidate') {
throw new DeploymentError('Only content deletion candidates can be quarantined', 409);
}
const source = fileLocation(this.root, 'content', asset.client, asset.filename);
const destination = fileLocation(this.root, '.trash/content', asset.client, asset.filename);
await mkdir(dirname(destination), { recursive: true });
await rename(source, destination);
const now = this.now();
try {
this.db.prepare(`
UPDATE asset SET state = 'quarantined', quarantined_at = ? WHERE path = ? AND state = 'candidate'
`).run(now, path);
} catch (error) {
await rename(destination, source).catch(() => undefined);
throw error;
}
return this.get(path);
}
async restore(path) {
const asset = this.get(path);
if (!asset || asset.category !== 'content' || asset.state !== 'quarantined') {
throw new DeploymentError('Only quarantined content can be restored', 409);
}
const source = fileLocation(this.root, '.trash/content', asset.client, asset.filename);
const destination = fileLocation(this.root, 'content', asset.client, asset.filename);
await mkdir(dirname(destination), { recursive: true });
await rename(source, destination);
const now = this.now();
try {
this.db.prepare(`
UPDATE asset
SET state = 'active', last_seen_at = ?, candidate_at = NULL, quarantined_at = NULL
WHERE path = ? AND state = 'quarantined'
`).run(now, path);
} catch (error) {
await rename(destination, source).catch(() => undefined);
throw error;
}
return this.get(path);
}
async delete(path) {
const asset = this.get(path);
if (!asset || asset.category !== 'content' || asset.state !== 'quarantined') {
throw new DeploymentError('Only quarantined content can be deleted', 409);
}
if (!asset.deleteAvailable) {
throw new DeploymentError('Quarantine grace period has not elapsed', 409);
}
await unlink(fileLocation(this.root, '.trash/content', asset.client, asset.filename));
const now = this.now();
this.db.prepare(`
UPDATE asset SET state = 'deleted', deleted_at = ? WHERE path = ? AND state = 'quarantined'
`).run(now, path);
return this.get(path);
}
close() {
if (this.touchTimer) clearTimeout(this.touchTimer);
if (this.maintenanceTimer) clearInterval(this.maintenanceTimer);
this.touchTimer = null;
this.maintenanceTimer = null;
if (this.db) {
this.flushTouches();
this.db.close();
this.db = null;
}
}
async #inventoryExisting() {
for (const category of ['user-icons', 'content']) {
const categoryRoot = join(this.root, category);
let clients;
try {
clients = await readdir(categoryRoot, { withFileTypes: true });
} catch (error) {
if (error.code === 'ENOENT') continue;
throw error;
}
for (const clientEntry of clients) {
if (!clientEntry.isDirectory() || !CLIENT_PATTERN.test(clientEntry.name)) continue;
const files = await readdir(join(categoryRoot, clientEntry.name), { withFileTypes: true });
for (const fileEntry of files) {
if (!fileEntry.isFile() || !FILE_PATTERN.test(fileEntry.name)) continue;
const info = await stat(join(categoryRoot, clientEntry.name, fileEntry.name));
const createdAt = Math.trunc(info.birthtimeMs > 0 ? Math.min(info.birthtimeMs, info.mtimeMs) : info.mtimeMs);
this.db.prepare(`
INSERT INTO asset(path, category, client, filename, size_bytes, digest, created_at, last_seen_at, state)
VALUES (?, ?, ?, ?, ?, NULL, ?, ?, 'active')
ON CONFLICT(path) DO NOTHING
`).run(
publicPath(category, clientEntry.name, fileEntry.name),
category,
clientEntry.name,
fileEntry.name,
info.size,
createdAt,
createdAt,
);
}
}
}
}
#assertOpen() {
if (!this.db) throw new Error('AssetStore is not initialized');
}
}
node-hook/src/config.mjs
+21 -2
View File
@@ -47,11 +47,22 @@ function clientSecrets(variableName) {
return result;
}
function positiveNumber(name, fallback) {
const value = Number(text(name, fallback));
if (!Number.isFinite(value) || value <= 0) {
throw new Error(`${name} must be a positive number`);
}
return value;
}
export function loadConfig() {
const webhookSecret = secret('GITEA_WEBHOOK_SECRET', 'GITEA_WEBHOOK_SECRET_FILE');
const adminSecret = secret('IMAGE_ADMIN_SECRET', 'IMAGE_ADMIN_SECRET_FILE');
if (webhookSecret.length < 32 || adminSecret.length < 32) {
throw new Error('Webhook and admin secrets must be at least 32 characters');
const adminPanelPassword = secret('IMAGE_ADMIN_PANEL_PASSWORD', 'IMAGE_ADMIN_PANEL_PASSWORD_FILE');
const adminPanelSessionSecret = secret('IMAGE_ADMIN_PANEL_SESSION_SECRET', 'IMAGE_ADMIN_PANEL_SESSION_SECRET_FILE');
if (webhookSecret.length < 32 || adminSecret.length < 32
|| adminPanelPassword.length < 16 || adminPanelSessionSecret.length < 32) {
throw new Error('Webhook, deployment, and session secrets need 32 characters; panel password needs 16');
}
const allowedBranches = text('IMAGE_ALLOWED_BRANCHES', 'master')
@@ -80,5 +91,13 @@ export function loadConfig() {
maxContentUploadBytes: Number(text('MAX_CONTENT_UPLOAD_BYTES', '1048576')),
uploadRoot: text('IMAGE_UPLOAD_ROOT', '/var/lib/image-hook/uploads'),
uploadStatePath: text('IMAGE_UPLOAD_STATE_PATH', '/var/lib/image-hook/upload-state.json'),
assetDbPath: text('IMAGE_ASSET_DB_PATH', '/var/lib/image-hook/image-assets.sqlite3'),
contentRetentionMs: positiveNumber('IMAGE_CONTENT_RETENTION_DAYS', '730') * 86_400_000,
contentQuarantineMs: positiveNumber('IMAGE_CONTENT_QUARANTINE_DAYS', '30') * 86_400_000,
assetMaintenanceIntervalMs: positiveNumber('IMAGE_ASSET_MAINTENANCE_SECONDS', '21600') * 1000,
assetTouchFlushIntervalMs: positiveNumber('IMAGE_ASSET_TOUCH_FLUSH_SECONDS', '60') * 1000,
adminPanelPassword,
adminPanelSessionSecret,
adminPanelSessionTtlMs: positiveNumber('IMAGE_ADMIN_PANEL_SESSION_HOURS', '8') * 3_600_000,
};
}
node-hook/src/server.mjs
+38 -2
View File
@@ -1,5 +1,7 @@
import { createServer } from 'node:http';
import { readFile } from 'node:fs/promises';
import { handleAdminPanel } from './admin-panel.mjs';
import { AssetStore } from './asset-store.mjs';
import { loadConfig } from './config.mjs';
import { verifyAdminSignature, verifyHexHmac, verifyUploadSignature } from './auth.mjs';
import { DeploymentError, GitService } from './git-service.mjs';
@@ -49,13 +51,18 @@ function hasImageSignature(body, extension) {
export async function createApp(config = loadConfig(), dependencies = {}) {
const service = dependencies.service ?? new GitService(config);
const uploadStore = dependencies.uploadStore ?? new UploadStore(config);
const assetStore = dependencies.assetStore ?? (dependencies.uploadStore ? {
async initialize() {}, touch() { return false; }, close() {},
} : new AssetStore(config));
const uploadStore = dependencies.uploadStore ?? new UploadStore(config, assetStore);
await service.initialize();
await assetStore.initialize();
await uploadStore.initialize();
const server = createServer(async (request, response) => {
const url = new URL(request.url, 'http://image-hook');
try {
if (await handleAdminPanel({ request, response, url, config, assetStore, readBody })) return;
if (request.method === 'GET' && url.pathname === '/healthz') {
return json(response, 200, { ok: true });
}
@@ -66,6 +73,23 @@ export async function createApp(config = loadConfig(), dependencies = {}) {
const inventory = JSON.parse(await readFile(`${config.repositoryPath}/hook/inventory.v2.json`, 'utf8'));
return json(response, 200, inventory);
}
if (['GET', 'HEAD'].includes(request.method) && url.pathname === '/v1/internal/content-access') {
const original = request.headers['x-image-path'];
let path;
try {
path = typeof original === 'string'
? new URL(original, 'http://image').pathname.replace(/^\/(?:image\/)?/, '')
: null;
} catch {
path = null;
}
if (!path || !assetStore.touch(path)) {
throw new DeploymentError('Invalid content image path', 400);
}
response.writeHead(204, { 'cache-control': 'no-store' });
response.end();
return;
}
if (request.method === 'POST' && url.pathname === '/v1/hooks/gitea') {
if (!request.headers['content-type']?.toLowerCase().startsWith('application/json')) {
throw new DeploymentError('Content-Type must be application/json', 415);
@@ -205,7 +229,9 @@ export async function createApp(config = loadConfig(), dependencies = {}) {
}
});
return { server, service, uploadStore };
server.once('close', () => assetStore.close?.());
return { server, service, uploadStore, assetStore };
}
if (process.argv[1] === new URL(import.meta.url).pathname) {
@@ -214,4 +240,14 @@ if (process.argv[1] === new URL(import.meta.url).pathname) {
server.listen(config.port, '0.0.0.0', () => {
console.log(JSON.stringify({ level: 'info', message: 'image hook listening', port: config.port }));
});
let stopping = false;
const shutdown = (signal) => {
if (stopping) return;
stopping = true;
console.log(JSON.stringify({ level: 'info', message: 'image hook stopping', signal }));
server.close(() => process.exit(0));
setTimeout(() => process.exit(1), 10_000).unref();
};
process.on('SIGTERM', () => shutdown('SIGTERM'));
process.on('SIGINT', () => shutdown('SIGINT'));
}
node-hook/src/upload-store.mjs
+4 -1
View File
@@ -4,9 +4,10 @@ import { dirname, join } from 'node:path';
import { DeploymentError } from './git-service.mjs';
export class UploadStore {
constructor(config) {
constructor(config, assetStore = null) {
this.root = config.uploadRoot;
this.statePath = config.uploadStatePath;
this.assetStore = assetStore;
this.queue = Promise.resolve();
this.uploads = [];
}
@@ -39,6 +40,7 @@ export class UploadStore {
if (previous.path !== path || previous.digest !== digest) {
throw new DeploymentError('Upload request ID was already used', 409);
}
this.assetStore?.register({ category, client, filename, body, digest });
return { duplicate: true, path: previous.path };
}
const destination = join(this.root, relativePath);
@@ -50,6 +52,7 @@ export class UploadStore {
throw new DeploymentError('Upload path already exists', 409);
}
}
this.assetStore?.register({ category, client, filename, body, digest });
this.uploads = [...this.uploads.slice(-999), { key: requestKey, path, digest }];
await this.#save();
return { duplicate: false, path };