From 2161970543b477960e878b0111b27b8567faea48 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E3=80=82?= Date: Wed, 8 Jul 2026 16:10:15 +0800 Subject: [PATCH 1/4] =?UTF-8?q?feat(ops):=20=E6=B7=BB=E5=8A=A0=E7=94=9F?= =?UTF-8?q?=E4=BA=A7=E5=A4=87=E4=BB=BD=E6=81=A2=E5=A4=8D=E8=84=9A=E6=9C=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 新增 PostgreSQL 备份、fresh DB 恢复和 server_data volume 备份脚本\n- 恢复默认拒绝覆盖，必须显式 --confirm-overwrite\n- 补充 package scripts、deploy verify 校验和部署文档\n\nCo-Authored-By: GPT-5 Codex --- .gitignore | 1 + docs/deployment.md | 58 ++++++++- package.json | 4 + scripts/backup-postgres.mjs | 139 ++++++++++++++++++++ scripts/backup-server-data.mjs | 99 +++++++++++++++ scripts/ops-scripts.test.mjs | 85 +++++++++++++ scripts/ops-utils.mjs | 116 +++++++++++++++++ scripts/restore-postgres.mjs | 181 +++++++++++++++++++++++++++ scripts/verify-production-deploy.mjs | 15 +++ 9 files changed, 697 insertions(+), 1 deletion(-) create mode 100644 scripts/backup-postgres.mjs create mode 100644 scripts/backup-server-data.mjs create mode 100644 scripts/ops-scripts.test.mjs create mode 100644 scripts/ops-utils.mjs create mode 100644 scripts/restore-postgres.mjs diff --git a/.gitignore b/.gitignore index 1eb6fa2..6793171 100644 --- a/.gitignore +++ b/.gitignore @@ -15,3 +15,4 @@ next-env.d.ts *.tsbuildinfo apps/server/data/ .worktrees/ +backups/ diff --git a/docs/deployment.md b/docs/deployment.md index 0b75669..830ac87 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -203,6 +203,61 @@ pnpm deploy:check-runtime http://localhost/api/v1/health/version ftb_pm_backup.sql +pnpm backup:postgres -- --env-file .env.production +pnpm backup:server-data -- --env-file .env.production ``` ## 常见排查 diff --git a/package.json b/package.json index 0cd5de7..0663a3e 100644 --- a/package.json +++ b/package.json @@ -7,8 +7,12 @@ "lint": "turbo lint", "type-check": "turbo type-check", "test": "turbo test", + "ops:test": "node --test scripts/*.test.mjs", "deploy:verify": "node scripts/verify-production-deploy.mjs", "deploy:check-runtime": "node scripts/check-runtime-version.mjs", + "backup:postgres": "node scripts/backup-postgres.mjs", + "restore:postgres": "node scripts/restore-postgres.mjs", + "backup:server-data": "node scripts/backup-server-data.mjs", "deploy:local:build": "docker compose --env-file .env.local-server -f docker-compose.local.yml build", "deploy:local:up": "docker compose --env-file .env.local-server -f docker-compose.local.yml up -d", "deploy:local:down": "docker compose --env-file .env.local-server -f docker-compose.local.yml down", diff --git a/scripts/backup-postgres.mjs b/scripts/backup-postgres.mjs new file mode 100644 index 0000000..08d134e --- /dev/null +++ b/scripts/backup-postgres.mjs @@ -0,0 +1,139 @@ +#!/usr/bin/env node +import { createWriteStream, unlinkSync } from 'node:fs'; +import { dirname, join, resolve } from 'node:path'; +import { pathToFileURL } from 'node:url'; +import { spawn } from 'node:child_process'; +import { + commandToString, + composePrefix, + ensureParentDir, + flag, + option, + parseArgs, + readEnvFile, + timestampSlug, +} from './ops-utils.mjs'; + +function usage() { + return `Usage: node scripts/backup-postgres.mjs [options] + +Options: + --env-file Compose env file (default: .env.production) + --compose-file Compose file (default: docker-compose.prod.yml) + --service Postgres service name (default: postgres) + --user Database user (default: POSTGRES_USER or postgres) + --db Database name (default: POSTGRES_DB or ftb_pm) + --backup-dir Directory for generated backups (default: backups/postgres) + --output Exact output dump path + --dry-run Print the pg_dump command without writing a file + --help Show this help +`; +} + +function buildOptions(argv) { + const args = parseArgs(argv); + if (flag(args, 'help')) return { help: true }; + + const envFile = option(args, 'env-file', '.env.production'); + const env = readEnvFile(envFile, { optional: flag(args, 'dry-run') }); + const db = option(args, 'db', env.POSTGRES_DB || 'ftb_pm'); + const user = option(args, 'user', env.POSTGRES_USER || 'postgres'); + const backupDir = option(args, 'backup-dir', 'backups/postgres'); + const output = option( + args, + 'output', + join(backupDir, `${db}-postgres-${timestampSlug()}.dump`), + ); + + return { + help: false, + dryRun: flag(args, 'dry-run'), + envFile, + composeFile: option(args, 'compose-file', 'docker-compose.prod.yml'), + service: option(args, 'service', 'postgres'), + db, + user, + output: resolve(output), + }; +} + +export function buildPgDumpCommand(options) { + return [ + ...composePrefix(options), + 'exec', + '-T', + options.service, + 'pg_dump', + '-U', + options.user, + '-d', + options.db, + '--format=custom', + '--no-owner', + '--no-acl', + ]; +} + +async function writeBackup(command, output) { + ensureParentDir(output); + await new Promise((resolveWrite, rejectWrite) => { + const file = createWriteStream(output, { flags: 'wx' }); + const child = spawn(command[0], command.slice(1), { + cwd: process.cwd(), + stdio: ['ignore', 'pipe', 'inherit'], + }); + + let settled = false; + const rejectOnce = (error) => { + if (settled) return; + settled = true; + try { + unlinkSync(output); + } catch { + // Best effort cleanup of a partial dump. + } + rejectWrite(error); + }; + + child.stdout.pipe(file); + child.on('error', rejectOnce); + file.on('error', rejectOnce); + child.on('close', (code) => { + file.end(() => { + if (settled) return; + settled = true; + if (code === 0) { + resolveWrite(); + } else { + rejectOnce(new Error(`pg_dump failed with exit code ${code}`)); + } + }); + }); + }); +} + +export async function main(argv = process.argv.slice(2)) { + const options = buildOptions(argv); + if (options.help) { + process.stdout.write(usage()); + return; + } + + const command = buildPgDumpCommand(options); + if (options.dryRun) { + process.stdout.write(`[dry-run] PostgreSQL backup would write: ${options.output}\n`); + process.stdout.write(`${commandToString(command, { stdout: options.output })}\n`); + return; + } + + await writeBackup(command, options.output); + process.stdout.write(`PostgreSQL backup written: ${options.output}\n`); +} + +if (import.meta.url === pathToFileURL(process.argv[1]).href) { + main().catch((error) => { + process.stderr.write(`${error.message}\n`); + process.exit(1); + }); +} + diff --git a/scripts/backup-server-data.mjs b/scripts/backup-server-data.mjs new file mode 100644 index 0000000..8e5bfc9 --- /dev/null +++ b/scripts/backup-server-data.mjs @@ -0,0 +1,99 @@ +#!/usr/bin/env node +import { basename, dirname, join, resolve } from 'node:path'; +import { pathToFileURL } from 'node:url'; +import { + commandToString, + ensureParentDir, + flag, + option, + parseArgs, + readEnvFile, + runCommand, + timestampSlug, +} from './ops-utils.mjs'; + +function usage() { + return `Usage: node scripts/backup-server-data.mjs [options] + +Backs up the server_data Docker volume that stores runtime AI provider config. + +Options: + --env-file Compose env file used to derive COMPOSE_PROJECT_NAME + (default: .env.production) + --volume Explicit Docker volume name + --backup-dir Directory for generated backups (default: backups/server-data) + --output Exact output .tgz path + --image Utility image (default: alpine:3.20) + --dry-run Print the docker run command without writing a file + --help Show this help +`; +} + +function buildOptions(argv) { + const args = parseArgs(argv); + if (flag(args, 'help')) return { help: true }; + + const envFile = option(args, 'env-file', '.env.production'); + const env = readEnvFile(envFile, { optional: flag(args, 'dry-run') }); + const projectName = env.COMPOSE_PROJECT_NAME || 'ftb_pm'; + const volume = option(args, 'volume', `${projectName}_server_data`); + const backupDir = option(args, 'backup-dir', 'backups/server-data'); + const output = option( + args, + 'output', + join(backupDir, `${volume}-${timestampSlug()}.tgz`), + ); + + return { + help: false, + dryRun: flag(args, 'dry-run'), + image: option(args, 'image', 'alpine:3.20'), + volume, + output: resolve(output), + }; +} + +export function buildServerDataBackupCommand(options) { + const outputDir = dirname(options.output); + const outputName = basename(options.output); + return [ + 'docker', + 'run', + '--rm', + '-v', + `${options.volume}:/data:ro`, + '-v', + `${outputDir}:/backup`, + options.image, + 'sh', + '-lc', + `tar -czf /backup/${outputName} -C /data .`, + ]; +} + +export async function main(argv = process.argv.slice(2)) { + const options = buildOptions(argv); + if (options.help) { + process.stdout.write(usage()); + return; + } + + const command = buildServerDataBackupCommand(options); + if (options.dryRun) { + process.stdout.write(`[dry-run] server_data backup would write: ${options.output}\n`); + process.stdout.write(`${commandToString(command)}\n`); + return; + } + + ensureParentDir(options.output); + await runCommand(command); + process.stdout.write(`server_data backup written: ${options.output}\n`); +} + +if (import.meta.url === pathToFileURL(process.argv[1]).href) { + main().catch((error) => { + process.stderr.write(`${error.message}\n`); + process.exit(1); + }); +} + diff --git a/scripts/ops-scripts.test.mjs b/scripts/ops-scripts.test.mjs new file mode 100644 index 0000000..e057418 --- /dev/null +++ b/scripts/ops-scripts.test.mjs @@ -0,0 +1,85 @@ +import { spawnSync } from 'node:child_process'; +import { mkdtempSync, writeFileSync, existsSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; + +const root = resolve(import.meta.dirname, '..'); + +function runScript(script, args) { + return spawnSync(process.execPath, [join(root, script), ...args], { + cwd: root, + encoding: 'utf8', + }); +} + +describe('production ops scripts', () => { + it('prints a pg_dump command in dry-run mode without writing the target file', () => { + const dir = mkdtempSync(join(tmpdir(), 'ftb-pg-backup-')); + const output = join(dir, 'backup.dump'); + + const result = runScript('scripts/backup-postgres.mjs', [ + '--dry-run', + '--env-file', + '.env.production.example', + '--output', + output, + ]); + + assert.equal(result.status, 0, result.stderr); + assert.match(result.stdout, /docker compose/); + assert.match(result.stdout, /pg_dump/); + assert.match(result.stdout, /backup\.dump/); + assert.equal(existsSync(output), false); + }); + + it('refuses restore by default unless --confirm-overwrite is supplied', () => { + const dir = mkdtempSync(join(tmpdir(), 'ftb-pg-restore-')); + const input = join(dir, 'backup.dump'); + writeFileSync(input, 'not-a-real-dump'); + + const result = runScript('scripts/restore-postgres.mjs', [ + '--env-file', + '.env.production.example', + '--input', + input, + ]); + + assert.notEqual(result.status, 0); + assert.match(result.stderr, /--confirm-overwrite/); + }); + + it('prints fresh database restore steps in dry-run mode after explicit overwrite confirmation', () => { + const dir = mkdtempSync(join(tmpdir(), 'ftb-pg-restore-dry-')); + const input = join(dir, 'backup.dump'); + writeFileSync(input, 'not-a-real-dump'); + + const result = runScript('scripts/restore-postgres.mjs', [ + '--dry-run', + '--confirm-overwrite', + '--env-file', + '.env.production.example', + '--input', + input, + ]); + + assert.equal(result.status, 0, result.stderr); + assert.match(result.stdout, /dropdb/); + assert.match(result.stdout, /createdb/); + assert.match(result.stdout, /pg_restore/); + }); + + it('prints a server_data volume tar backup command in dry-run mode', () => { + const result = runScript('scripts/backup-server-data.mjs', [ + '--dry-run', + '--env-file', + '.env.production.example', + ]); + + assert.equal(result.status, 0, result.stderr); + assert.match(result.stdout, /docker run/); + assert.match(result.stdout, /ftb_pm_server_data/); + assert.match(result.stdout, /tar -czf/); + }); +}); diff --git a/scripts/ops-utils.mjs b/scripts/ops-utils.mjs new file mode 100644 index 0000000..2d01f02 --- /dev/null +++ b/scripts/ops-utils.mjs @@ -0,0 +1,116 @@ +import { existsSync, mkdirSync, readFileSync } from 'node:fs'; +import { dirname, resolve } from 'node:path'; +import { spawn } from 'node:child_process'; + +export function parseArgs(argv) { + const parsed = { flags: new Set(), values: new Map(), positionals: [] }; + + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index]; + if (arg === '--') continue; + if (!arg.startsWith('--')) { + parsed.positionals.push(arg); + continue; + } + + const eq = arg.indexOf('='); + if (eq !== -1) { + parsed.values.set(arg.slice(2, eq), arg.slice(eq + 1)); + continue; + } + + const key = arg.slice(2); + const next = argv[index + 1]; + if (next && !next.startsWith('--')) { + parsed.values.set(key, next); + index += 1; + } else { + parsed.flags.add(key); + } + } + + return parsed; +} + +export function option(args, name, fallback = undefined) { + return args.values.has(name) ? args.values.get(name) : fallback; +} + +export function flag(args, name) { + return args.flags.has(name); +} + +export function readEnvFile(filePath, { optional = false } = {}) { + if (!existsSync(filePath)) { + if (optional) return {}; + throw new Error(`Env file not found: ${filePath}`); + } + + const env = {}; + const content = readFileSync(filePath, 'utf8'); + for (const rawLine of content.split(/\r?\n/)) { + const line = rawLine.trim(); + if (!line || line.startsWith('#')) continue; + const eq = line.indexOf('='); + if (eq === -1) continue; + const key = line.slice(0, eq).trim(); + let value = line.slice(eq + 1).trim(); + if ( + (value.startsWith('"') && value.endsWith('"')) || + (value.startsWith("'") && value.endsWith("'")) + ) { + value = value.slice(1, -1); + } + env[key] = value; + } + return env; +} + +export function ensureParentDir(filePath) { + mkdirSync(dirname(filePath), { recursive: true }); +} + +export function timestampSlug(date = new Date()) { + return date.toISOString().replace(/[-:]/g, '').replace(/\.\d{3}Z$/, 'Z'); +} + +export function shellQuote(value) { + if (/^[A-Za-z0-9_./:@%+=,-]+$/.test(value)) return value; + return `'${value.replace(/'/g, "'\\''")}'`; +} + +export function commandToString(command, { stdin, stdout } = {}) { + const rendered = command.map((part) => shellQuote(part)).join(' '); + const withStdin = stdin ? `${rendered} < ${shellQuote(stdin)}` : rendered; + return stdout ? `${withStdin} > ${shellQuote(stdout)}` : withStdin; +} + +export function composePrefix({ envFile, composeFile }) { + return ['docker', 'compose', '--env-file', envFile, '-f', composeFile]; +} + +export function resolvePath(path) { + return resolve(process.cwd(), path); +} + +export function runCommand(command, options = {}) { + return new Promise((resolveRun, rejectRun) => { + const child = spawn(command[0], command.slice(1), { + stdio: options.stdio ?? 'inherit', + cwd: options.cwd ?? process.cwd(), + }); + + child.on('error', rejectRun); + child.on('close', (code) => { + if (code === 0) { + resolveRun(); + } else { + rejectRun(new Error(`Command failed (${code}): ${commandToString(command)}`)); + } + }); + }); +} + +export function quotePgLiteral(value) { + return `'${value.replace(/'/g, "''")}'`; +} diff --git a/scripts/restore-postgres.mjs b/scripts/restore-postgres.mjs new file mode 100644 index 0000000..0322c9e --- /dev/null +++ b/scripts/restore-postgres.mjs @@ -0,0 +1,181 @@ +#!/usr/bin/env node +import { createReadStream, existsSync } from 'node:fs'; +import { resolve } from 'node:path'; +import { pathToFileURL } from 'node:url'; +import { spawn } from 'node:child_process'; +import { + commandToString, + composePrefix, + flag, + option, + parseArgs, + quotePgLiteral, + readEnvFile, + runCommand, +} from './ops-utils.mjs'; + +function usage() { + return `Usage: node scripts/restore-postgres.mjs --input --confirm-overwrite [options] + +This script restores into a freshly recreated PostgreSQL database. It refuses to +drop/recreate a database unless --confirm-overwrite is provided. + +Options: + --input pg_dump custom-format dump file + --confirm-overwrite Required safety flag for destructive restore + --env-file Compose env file (default: .env.production) + --compose-file Compose file (default: docker-compose.prod.yml) + --service Postgres service name (default: postgres) + --user Database user (default: POSTGRES_USER or postgres) + --target-db Target database (default: POSTGRES_DB or ftb_pm) + --maintenance-db Maintenance database (default: postgres) + --dry-run Print restore commands without changing data + --help Show this help +`; +} + +function buildOptions(argv) { + const args = parseArgs(argv); + if (flag(args, 'help')) return { help: true }; + + const envFile = option(args, 'env-file', '.env.production'); + const env = readEnvFile(envFile, { optional: flag(args, 'dry-run') }); + const input = option(args, 'input'); + if (!input) { + throw new Error('Missing required --input '); + } + + const inputPath = resolve(input); + if (!existsSync(inputPath)) { + throw new Error(`Restore input not found: ${inputPath}`); + } + + if (!flag(args, 'confirm-overwrite')) { + throw new Error('Refusing destructive restore. Re-run with --confirm-overwrite.'); + } + + return { + help: false, + dryRun: flag(args, 'dry-run'), + input: inputPath, + envFile, + composeFile: option(args, 'compose-file', 'docker-compose.prod.yml'), + service: option(args, 'service', 'postgres'), + user: option(args, 'user', env.POSTGRES_USER || 'postgres'), + targetDb: option(args, 'target-db', env.POSTGRES_DB || 'ftb_pm'), + maintenanceDb: option(args, 'maintenance-db', 'postgres'), + }; +} + +export function buildRestoreCommands(options) { + const prefix = composePrefix(options); + const terminateSql = [ + 'SELECT pg_terminate_backend(pid)', + 'FROM pg_stat_activity', + `WHERE datname = ${quotePgLiteral(options.targetDb)} AND pid <> pg_backend_pid();`, + ].join(' '); + + return [ + [ + ...prefix, + 'exec', + '-T', + options.service, + 'psql', + '-U', + options.user, + '-d', + options.maintenanceDb, + '-v', + 'ON_ERROR_STOP=1', + '-c', + terminateSql, + ], + [ + ...prefix, + 'exec', + '-T', + options.service, + 'dropdb', + '--if-exists', + '-U', + options.user, + options.targetDb, + ], + [ + ...prefix, + 'exec', + '-T', + options.service, + 'createdb', + '-U', + options.user, + options.targetDb, + ], + [ + ...prefix, + 'exec', + '-T', + options.service, + 'pg_restore', + '-U', + options.user, + '-d', + options.targetDb, + '--no-owner', + '--no-acl', + ], + ]; +} + +function runRestore(command, input) { + return new Promise((resolveRun, rejectRun) => { + const child = spawn(command[0], command.slice(1), { + cwd: process.cwd(), + stdio: ['pipe', 'inherit', 'inherit'], + }); + createReadStream(input).pipe(child.stdin); + child.on('error', rejectRun); + child.on('close', (code) => { + if (code === 0) { + resolveRun(); + } else { + rejectRun(new Error(`pg_restore failed with exit code ${code}`)); + } + }); + }); +} + +export async function main(argv = process.argv.slice(2)) { + const options = buildOptions(argv); + if (options.help) { + process.stdout.write(usage()); + return; + } + + const commands = buildRestoreCommands(options); + if (options.dryRun) { + process.stdout.write( + `[dry-run] PostgreSQL restore would recreate database: ${options.targetDb}\n`, + ); + for (const command of commands.slice(0, -1)) { + process.stdout.write(`${commandToString(command)}\n`); + } + process.stdout.write(`${commandToString(commands.at(-1), { stdin: options.input })}\n`); + return; + } + + for (const command of commands.slice(0, -1)) { + await runCommand(command); + } + await runRestore(commands.at(-1), options.input); + process.stdout.write(`PostgreSQL restore completed into fresh database: ${options.targetDb}\n`); +} + +if (import.meta.url === pathToFileURL(process.argv[1]).href) { + main().catch((error) => { + process.stderr.write(`${error.message}\n`); + process.exit(1); + }); +} + diff --git a/scripts/verify-production-deploy.mjs b/scripts/verify-production-deploy.mjs index bb238d3..510311e 100644 --- a/scripts/verify-production-deploy.mjs +++ b/scripts/verify-production-deploy.mjs @@ -66,6 +66,18 @@ const checks = [ file: 'scripts/check-runtime-version.mjs', snippets: ['Runtime version verified', '/api/v1/health/version', 'expectedVersion'], }, + { + file: 'scripts/backup-postgres.mjs', + snippets: ['pg_dump', '--format=custom', '--dry-run'], + }, + { + file: 'scripts/restore-postgres.mjs', + snippets: ['--confirm-overwrite', 'dropdb', 'pg_restore'], + }, + { + file: 'scripts/backup-server-data.mjs', + snippets: ['server_data', 'tar -czf', '--dry-run'], + }, { file: 'docker-compose.local.yml', snippets: [ @@ -118,6 +130,9 @@ const checks = [ 'docker-compose.local.yml', '本地服务器部署', 'pnpm deploy:verify', + 'pnpm backup:postgres', + 'pnpm restore:postgres', + 'pnpm backup:server-data', 'pnpm db:migrate', 'NEXT_PUBLIC_API_URL', 'Nginx', From eef09d5af49ca7f45613bb13671afeaf43aba7ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E3=80=82?= Date: Wed, 8 Jul 2026 16:14:48 +0800 Subject: [PATCH 2/4] =?UTF-8?q?feat(ops):=20=E6=B7=BB=E5=8A=A0=E5=8F=91?= =?UTF-8?q?=E5=B8=83=20smoke=20test?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 新增只读发布 smoke runner 和 dry-run 测试\n- 将生产部署 workflow 从单点版本检查升级为 smoke test\n- 补充 Docker web runtime 脚本复制、package script 和部署文档\n\nCo-Authored-By: GPT-5 Codex --- .github/workflows/deploy-production.yml | 19 +-- Dockerfile.web | 1 + docs/deployment.md | 5 +- package.json | 1 + scripts/ops-scripts.test.mjs | 15 +++ scripts/smoke-test-release.mjs | 160 ++++++++++++++++++++++++ scripts/verify-production-deploy.mjs | 12 +- 7 files changed, 193 insertions(+), 20 deletions(-) create mode 100644 scripts/smoke-test-release.mjs diff --git a/.github/workflows/deploy-production.yml b/.github/workflows/deploy-production.yml index 9fa0e97..a041693 100644 --- a/.github/workflows/deploy-production.yml +++ b/.github/workflows/deploy-production.yml @@ -123,26 +123,11 @@ jobs: docker compose --env-file .env.production -f docker-compose.prod.yml up -d --remove-orphans for attempt in $(seq 1 30); do - if docker compose --env-file .env.production -f docker-compose.prod.yml exec -T web node -e " - const expected = process.argv[1]; - fetch('http://nginx/api/v1/health/version') - .then(async (response) => { - if (!response.ok) throw new Error('HTTP ' + response.status); - const payload = await response.json(); - if (payload.version !== expected) { - throw new Error('Expected ' + expected + ', got ' + payload.version); - } - console.log('Runtime version verified: ' + payload.version); - }) - .catch((error) => { - console.error(error.message); - process.exit(1); - }); - " "${{ github.sha }}"; then + if docker compose --env-file .env.production -f docker-compose.prod.yml exec -T web node scripts/smoke-test-release.mjs --base-url http://nginx --expected-version "${{ github.sha }}"; then exit 0 fi sleep 2 done - echo "Runtime version check failed after retries" + echo "Release smoke check failed after retries" exit 1 diff --git a/Dockerfile.web b/Dockerfile.web index fe59920..c731c6b 100644 --- a/Dockerfile.web +++ b/Dockerfile.web @@ -57,6 +57,7 @@ COPY --from=builder /app/turbo.json ./turbo.json COPY --from=builder /app/node_modules ./node_modules COPY --from=builder /app/packages/shared ./packages/shared COPY --from=builder /app/apps/web ./apps/web +COPY scripts ./scripts RUN chown -R node:node /app USER node EXPOSE 3000 diff --git a/docs/deployment.md b/docs/deployment.md index 830ac87..ea3c4a0 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -175,12 +175,13 @@ cp .env.production.example .env.production 5. 执行 `docker compose --env-file .env.production -f docker-compose.prod.yml pull web server` 拉取本次 SHA 镜像。 6. 启动数据库与 Redis，执行 `pnpm --filter server db:deploy`。 7. 执行 `docker compose --env-file .env.production -f docker-compose.prod.yml up -d --remove-orphans` 重启服务。 -8. 通过 `/api/v1/health/version` 校验运行中的后端版本是否等于本次 commit SHA。 +8. 运行发布 smoke test：校验 `/api/v1/health/version`、前端首页、产品页、产品 API、V2.2 读路径和 AI 配置端点，并确认运行中的后端版本等于本次 commit SHA。 -如果最后一步失败，Actions 会红掉，说明“代码已合并”不等于“线上容器已更新”。本地或服务器也可以手工运行： +如果最后一步失败，Actions 会红掉，说明“代码已合并”不等于“线上容器已更新”或关键读路径不可用。本地或服务器也可以手工运行： ```bash pnpm deploy:check-runtime http://localhost/api/v1/health/version +pnpm deploy:smoke -- --base-url http://localhost --expected-version ``` ## Nginx 路由 diff --git a/package.json b/package.json index 0663a3e..213d37c 100644 --- a/package.json +++ b/package.json @@ -10,6 +10,7 @@ "ops:test": "node --test scripts/*.test.mjs", "deploy:verify": "node scripts/verify-production-deploy.mjs", "deploy:check-runtime": "node scripts/check-runtime-version.mjs", + "deploy:smoke": "node scripts/smoke-test-release.mjs", "backup:postgres": "node scripts/backup-postgres.mjs", "restore:postgres": "node scripts/restore-postgres.mjs", "backup:server-data": "node scripts/backup-server-data.mjs", diff --git a/scripts/ops-scripts.test.mjs b/scripts/ops-scripts.test.mjs index e057418..1b08db0 100644 --- a/scripts/ops-scripts.test.mjs +++ b/scripts/ops-scripts.test.mjs @@ -82,4 +82,19 @@ describe('production ops scripts', () => { assert.match(result.stdout, /ftb_pm_server_data/); assert.match(result.stdout, /tar -czf/); }); + + it('prints read-only release smoke checks in dry-run mode', () => { + const result = runScript('scripts/smoke-test-release.mjs', [ + '--dry-run', + '--base-url', + 'http://localhost', + ]); + + assert.equal(result.status, 0, result.stderr); + assert.match(result.stdout, /\/api\/v1\/health\/version/); + assert.match(result.stdout, /\/products/); + assert.match(result.stdout, /\/api\/v1\/products/); + assert.match(result.stdout, /\/api\/v1\/v2\.2\/requirements\?productId=__smoke__/); + assert.match(result.stdout, /\/api\/v1\/config\/ai/); + }); }); diff --git a/scripts/smoke-test-release.mjs b/scripts/smoke-test-release.mjs new file mode 100644 index 0000000..e199a4c --- /dev/null +++ b/scripts/smoke-test-release.mjs @@ -0,0 +1,160 @@ +#!/usr/bin/env node +import { pathToFileURL } from 'node:url'; +import { flag, option, parseArgs } from './ops-utils.mjs'; + +function usage() { + return `Usage: node scripts/smoke-test-release.mjs --base-url [options] + +Read-only release smoke checks: + - backend runtime version + - frontend root page + - frontend products route + - products API + - V2.2 requirement read path + - AI config public endpoint + +Options: + --base-url Deployment base URL (default: http://127.0.0.1) + --expected-version Expected /api/v1/health/version payload version + --timeout-ms Per-request timeout (default: 5000) + --dry-run Print planned checks without making requests + --help Show this help +`; +} + +function normalizeBaseUrl(value) { + const url = new URL(value || 'http://127.0.0.1'); + url.pathname = url.pathname.replace(/\/+$/, ''); + url.search = ''; + url.hash = ''; + return url.toString().replace(/\/$/, ''); +} + +export function buildSmokeChecks(baseUrl, expectedVersion = '') { + const checks = [ + { + name: 'backend runtime version', + path: '/api/v1/health/version', + kind: 'json', + validate(payload) { + if (payload?.service !== 'server' || typeof payload.version !== 'string') { + throw new Error(`unexpected version payload: ${JSON.stringify(payload)}`); + } + if (expectedVersion && payload.version !== expectedVersion) { + throw new Error(`expected version ${expectedVersion}, got ${payload.version}`); + } + }, + }, + { name: 'frontend root', path: '/', kind: 'text' }, + { name: 'frontend products route', path: '/products', kind: 'text' }, + { + name: 'products API', + path: '/api/v1/products', + kind: 'json', + validate(payload) { + if (!Array.isArray(payload)) { + throw new Error('products API did not return an array'); + } + }, + }, + { + name: 'V2.2 requirement scoped read', + path: '/api/v1/v2.2/requirements?productId=__smoke__&limit=1', + kind: 'json', + validate(payload) { + if (!payload || !Array.isArray(payload.items)) { + throw new Error('V2.2 requirements response missing items array'); + } + }, + }, + { + name: 'AI config public endpoint', + path: '/api/v1/config/ai', + kind: 'json', + validate(payload) { + if (!payload || !Array.isArray(payload.providers)) { + throw new Error('AI config response missing providers array'); + } + }, + }, + ]; + + return checks.map((check) => ({ + ...check, + url: new URL(check.path, `${baseUrl}/`).toString(), + })); +} + +async function fetchWithTimeout(url, timeoutMs) { + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), timeoutMs); + try { + return await fetch(url, { signal: controller.signal }); + } finally { + clearTimeout(timer); + } +} + +async function runCheck(check, timeoutMs) { + const response = await fetchWithTimeout(check.url, timeoutMs); + if (!response.ok) { + throw new Error(`HTTP ${response.status}`); + } + + if (check.kind === 'json') { + const payload = await response.json(); + check.validate?.(payload); + } else { + await response.text(); + } +} + +export async function main(argv = process.argv.slice(2)) { + const args = parseArgs(argv); + if (flag(args, 'help')) { + process.stdout.write(usage()); + return; + } + + const baseUrl = normalizeBaseUrl(option(args, 'base-url', 'http://127.0.0.1')); + const expectedVersion = option(args, 'expected-version', ''); + const timeoutMs = Number.parseInt(option(args, 'timeout-ms', '5000'), 10); + if (!Number.isFinite(timeoutMs) || timeoutMs <= 0) { + throw new Error('--timeout-ms must be a positive number'); + } + + const checks = buildSmokeChecks(baseUrl, expectedVersion); + if (flag(args, 'dry-run')) { + process.stdout.write(`[dry-run] Release smoke target: ${baseUrl}\n`); + for (const check of checks) { + process.stdout.write(`GET ${check.url} # ${check.name}\n`); + } + return; + } + + process.stdout.write(`Release smoke target: ${baseUrl}\n`); + const failures = []; + for (const check of checks) { + try { + await runCheck(check, timeoutMs); + process.stdout.write(`[pass] ${check.name} ${check.url}\n`); + } catch (error) { + failures.push(`[fail] ${check.name} ${check.url}: ${error.message}`); + process.stderr.write(`${failures.at(-1)}\n`); + } + } + + if (failures.length > 0) { + throw new Error(`Release smoke failed: ${failures.length} check(s) failed`); + } + + process.stdout.write('Release smoke test passed.\n'); +} + +if (import.meta.url === pathToFileURL(process.argv[1]).href) { + main().catch((error) => { + process.stderr.write(`${error.message}\n`); + process.exit(1); + }); +} + diff --git a/scripts/verify-production-deploy.mjs b/scripts/verify-production-deploy.mjs index 510311e..1ff3603 100644 --- a/scripts/verify-production-deploy.mjs +++ b/scripts/verify-production-deploy.mjs @@ -21,6 +21,7 @@ const checks = [ 'NEXT_PUBLIC_API_URL', 'NEXT_PUBLIC_APP_VERSION', 'ARG APP_VERSION=unknown', + 'COPY scripts ./scripts', 'CMD ["pnpm", "--filter", "web", "start"]', ], }, @@ -59,13 +60,22 @@ const checks = [ 'appleboy/ssh-action', 'docker compose --env-file .env.production -f docker-compose.prod.yml pull', 'pnpm --filter server db:deploy', - '/api/v1/health/version', + 'scripts/smoke-test-release.mjs', + '--expected-version', ], }, { file: 'scripts/check-runtime-version.mjs', snippets: ['Runtime version verified', '/api/v1/health/version', 'expectedVersion'], }, + { + file: 'scripts/smoke-test-release.mjs', + snippets: [ + 'Release smoke test passed', + '/api/v1/v2.2/requirements?productId=__smoke__', + '/api/v1/config/ai', + ], + }, { file: 'scripts/backup-postgres.mjs', snippets: ['pg_dump', '--format=custom', '--dry-run'], From 7837a809cad296a954c1e339eaca6ab7c1aeb100 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E3=80=82?= Date: Wed, 8 Jul 2026 16:21:07 +0800 Subject: [PATCH 3/4] =?UTF-8?q?feat(ops):=20=E6=B7=BB=E5=8A=A0=E7=94=9F?= =?UTF-8?q?=E4=BA=A7=E7=9B=91=E6=8E=A7=E5=91=8A=E8=AD=A6=E5=9F=BA=E7=BA=BF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 新增 Prometheus/Grafana/Loki/Promtail 监控 profile\n- 覆盖 DB、磁盘、慢 API、慢 Prisma、任务失败和小宝摘要 stale 告警\n- 补充 postgres-exporter 自定义查询、Dashboard、部署文档和校验\n\nCo-Authored-By: GPT-5 Codex --- .env.production.example | 7 ++ deploy/monitoring/README.md | 43 +++++++ deploy/monitoring/blackbox/config.yml | 10 ++ .../dashboards/ftb-production-overview.json | 78 +++++++++++++ .../provisioning/dashboards/dashboards.yml | 12 ++ .../provisioning/datasources/datasources.yml | 16 +++ deploy/monitoring/loki/config.yml | 32 +++++ .../monitoring/postgres/postgres-queries.yml | 12 ++ deploy/monitoring/prometheus/alert-rules.yml | 62 ++++++++++ deploy/monitoring/prometheus/prometheus.yml | 45 +++++++ deploy/monitoring/promtail/config.yml | 53 +++++++++ docker-compose.prod.yml | 110 ++++++++++++++++++ docs/deployment.md | 24 ++++ scripts/ops-scripts.test.mjs | 43 ++++++- scripts/verify-production-deploy.mjs | 41 +++++++ 15 files changed, 587 insertions(+), 1 deletion(-) create mode 100644 deploy/monitoring/README.md create mode 100644 deploy/monitoring/blackbox/config.yml create mode 100644 deploy/monitoring/grafana/dashboards/ftb-production-overview.json create mode 100644 deploy/monitoring/grafana/provisioning/dashboards/dashboards.yml create mode 100644 deploy/monitoring/grafana/provisioning/datasources/datasources.yml create mode 100644 deploy/monitoring/loki/config.yml create mode 100644 deploy/monitoring/postgres/postgres-queries.yml create mode 100644 deploy/monitoring/prometheus/alert-rules.yml create mode 100644 deploy/monitoring/prometheus/prometheus.yml create mode 100644 deploy/monitoring/promtail/config.yml diff --git a/.env.production.example b/.env.production.example index d8c423f..512d929 100644 --- a/.env.production.example +++ b/.env.production.example @@ -39,3 +39,10 @@ SMTP_HOST= SMTP_PORT=465 SMTP_USER= SMTP_PASS= + +# Optional monitoring profile. Do not commit real production passwords. +PROMETHEUS_PORT=9090 +PROMETHEUS_RETENTION=15d +GRAFANA_PORT=3002 +GRAFANA_ADMIN_USER=admin +GRAFANA_ADMIN_PASSWORD=change-me-monitoring-password diff --git a/deploy/monitoring/README.md b/deploy/monitoring/README.md new file mode 100644 index 0000000..e54dca6 --- /dev/null +++ b/deploy/monitoring/README.md @@ -0,0 +1,43 @@ +# FTB Production Monitoring Baseline + +This profile adds a deployable Prometheus/Grafana baseline for production operations. It is intentionally secret-free: no webhook URLs, API keys, SMTP passwords, or real alert receiver credentials are committed. + +## Start + +```bash +docker compose --env-file .env.production -f docker-compose.prod.yml --profile monitoring up -d +``` + +Default local ports: + +- Prometheus: `http://localhost:9090` +- Grafana: `http://localhost:3002` +- Loki: internal only + +Set `GRAFANA_ADMIN_USER` and `GRAFANA_ADMIN_PASSWORD` in `.env.production` before exposing Grafana beyond localhost. Keep real alert receivers in the server environment or an untracked Alertmanager file. + +## Coverage + +- DB availability: `pg_up` from postgres-exporter. +- Disk pressure: root filesystem availability from node-exporter. +- Slow API requests: Promtail turns `Slow API request` server logs into `ftb_slow_api_log_total`. +- Slow Prisma queries: Promtail turns `Slow Prisma query` server logs into `ftb_slow_prisma_log_total`. +- Job failures: Promtail turns `AppData relation sync failed` and AI call failure logs into `ftb_job_failure_log_total`. +- Xiaobao stale summaries: postgres-exporter custom query exposes `ftb_xiaobao_stale_summary_count` from `xiaobao_risk_summaries`. + +## Alerts + +Prometheus loads `prometheus/alert-rules.yml`. The rules evaluate locally and are visible in Prometheus/Grafana. To send notifications, add Alertmanager outside git or mount an environment-specific receiver file; do not commit webhook URLs or tokens. + +Baseline alert names: + +- `FtbPostgresDown` +- `FtbDiskPressure` +- `FtbSlowApiLogBurst` +- `FtbSlowPrismaLogBurst` +- `FtbJobFailureLogBurst` +- `FtbXiaobaoSummaryStale` + +## Log Search + +Promtail ships Docker logs to Loki with container labels. Grafana provisions both Prometheus and Loki data sources, so on-call checks can move from a firing alert to matching server logs without SSHing into the host. diff --git a/deploy/monitoring/blackbox/config.yml b/deploy/monitoring/blackbox/config.yml new file mode 100644 index 0000000..fa81bee --- /dev/null +++ b/deploy/monitoring/blackbox/config.yml @@ -0,0 +1,10 @@ +modules: + http_2xx: + prober: http + timeout: 5s + http: + valid_http_versions: ['HTTP/1.1', 'HTTP/2.0'] + valid_status_codes: [] + method: GET + preferred_ip_protocol: ip4 + diff --git a/deploy/monitoring/grafana/dashboards/ftb-production-overview.json b/deploy/monitoring/grafana/dashboards/ftb-production-overview.json new file mode 100644 index 0000000..0a753d3 --- /dev/null +++ b/deploy/monitoring/grafana/dashboards/ftb-production-overview.json @@ -0,0 +1,78 @@ +{ + "uid": "ftb-production-overview", + "title": "FTB Production Overview", + "schemaVersion": 39, + "version": 1, + "refresh": "30s", + "tags": ["ftb", "production", "v2.8"], + "time": { + "from": "now-6h", + "to": "now" + }, + "panels": [ + { + "id": 1, + "type": "stat", + "title": "DB Availability", + "gridPos": { "x": 0, "y": 0, "w": 6, "h": 4 }, + "targets": [ + { "datasource": { "type": "prometheus", "uid": "Prometheus" }, "expr": "pg_up", "refId": "A" } + ] + }, + { + "id": 2, + "type": "timeseries", + "title": "Slow API Logs", + "gridPos": { "x": 6, "y": 0, "w": 6, "h": 4 }, + "targets": [ + { "datasource": { "type": "prometheus", "uid": "Prometheus" }, "expr": "increase(ftb_slow_api_log_total[10m])", "refId": "A" } + ] + }, + { + "id": 3, + "type": "timeseries", + "title": "Slow Prisma Logs", + "gridPos": { "x": 12, "y": 0, "w": 6, "h": 4 }, + "targets": [ + { "datasource": { "type": "prometheus", "uid": "Prometheus" }, "expr": "increase(ftb_slow_prisma_log_total[10m])", "refId": "A" } + ] + }, + { + "id": 4, + "type": "stat", + "title": "Xiaobao Stale Summaries", + "gridPos": { "x": 18, "y": 0, "w": 6, "h": 4 }, + "targets": [ + { "datasource": { "type": "prometheus", "uid": "Prometheus" }, "expr": "ftb_xiaobao_stale_summary_count", "refId": "A" } + ] + }, + { + "id": 5, + "type": "timeseries", + "title": "Job Failure Logs", + "gridPos": { "x": 0, "y": 4, "w": 8, "h": 5 }, + "targets": [ + { "datasource": { "type": "prometheus", "uid": "Prometheus" }, "expr": "increase(ftb_job_failure_log_total[10m])", "refId": "A" } + ] + }, + { + "id": 6, + "type": "stat", + "title": "Root Disk Free %", + "gridPos": { "x": 8, "y": 4, "w": 8, "h": 5 }, + "targets": [ + { "datasource": { "type": "prometheus", "uid": "Prometheus" }, "expr": "100 * node_filesystem_avail_bytes{mountpoint=\"/\",fstype!~\"tmpfs|overlay\"} / node_filesystem_size_bytes{mountpoint=\"/\",fstype!~\"tmpfs|overlay\"}", "refId": "A" } + ] + }, + { + "id": 7, + "type": "logs", + "title": "Server Warning/Error Logs", + "gridPos": { "x": 16, "y": 4, "w": 8, "h": 5 }, + "targets": [ + { "datasource": { "type": "loki", "uid": "Loki" }, "expr": "{compose_service=\"server\"} |~ \"warn|error|失败|Slow\"", "refId": "A" } + ] + } + ] +} + diff --git a/deploy/monitoring/grafana/provisioning/dashboards/dashboards.yml b/deploy/monitoring/grafana/provisioning/dashboards/dashboards.yml new file mode 100644 index 0000000..adcba00 --- /dev/null +++ b/deploy/monitoring/grafana/provisioning/dashboards/dashboards.yml @@ -0,0 +1,12 @@ +apiVersion: 1 + +providers: + - name: FTB Production + orgId: 1 + folder: FTB + type: file + disableDeletion: false + updateIntervalSeconds: 30 + options: + path: /var/lib/grafana/dashboards + diff --git a/deploy/monitoring/grafana/provisioning/datasources/datasources.yml b/deploy/monitoring/grafana/provisioning/datasources/datasources.yml new file mode 100644 index 0000000..5ef7ccf --- /dev/null +++ b/deploy/monitoring/grafana/provisioning/datasources/datasources.yml @@ -0,0 +1,16 @@ +apiVersion: 1 + +datasources: + - name: Prometheus + type: prometheus + access: proxy + url: http://prometheus:9090 + isDefault: true + editable: false + + - name: Loki + type: loki + access: proxy + url: http://loki:3100 + editable: false + diff --git a/deploy/monitoring/loki/config.yml b/deploy/monitoring/loki/config.yml new file mode 100644 index 0000000..dad75da --- /dev/null +++ b/deploy/monitoring/loki/config.yml @@ -0,0 +1,32 @@ +auth_enabled: false + +server: + http_listen_port: 3100 + +common: + path_prefix: /loki + replication_factor: 1 + ring: + kvstore: + store: inmemory + +schema_config: + configs: + - from: 2026-01-01 + store: tsdb + object_store: filesystem + schema: v13 + index: + prefix: index_ + period: 24h + +storage_config: + tsdb_shipper: + active_index_directory: /loki/index + cache_location: /loki/index_cache + filesystem: + directory: /loki/chunks + +limits_config: + retention_period: 168h + diff --git a/deploy/monitoring/postgres/postgres-queries.yml b/deploy/monitoring/postgres/postgres-queries.yml new file mode 100644 index 0000000..8a19c94 --- /dev/null +++ b/deploy/monitoring/postgres/postgres-queries.yml @@ -0,0 +1,12 @@ +ftb_xiaobao: + query: | + SELECT + count(*)::float AS stale_summary_count + FROM xiaobao_risk_summaries + WHERE dirty = true + OR updated_at < now() - interval '6 hours'; + metrics: + - stale_summary_count: + usage: GAUGE + description: Xiaobao risk summaries that are dirty or older than 6 hours. + diff --git a/deploy/monitoring/prometheus/alert-rules.yml b/deploy/monitoring/prometheus/alert-rules.yml new file mode 100644 index 0000000..e3af268 --- /dev/null +++ b/deploy/monitoring/prometheus/alert-rules.yml @@ -0,0 +1,62 @@ +groups: + - name: ftb-production-alerts + rules: + - alert: FtbPostgresDown + expr: pg_up == 0 + for: 2m + labels: + severity: critical + annotations: + summary: PostgreSQL exporter cannot reach the FTB database. + runbook: docs/runbooks/migration-rollback.md + + - alert: FtbDiskPressure + expr: | + ( + node_filesystem_avail_bytes{mountpoint="/",fstype!~"tmpfs|overlay"} + / + node_filesystem_size_bytes{mountpoint="/",fstype!~"tmpfs|overlay"} + ) < 0.15 + for: 10m + labels: + severity: warning + annotations: + summary: Production host root filesystem has less than 15% free space. + runbook: docs/deployment.md + + - alert: FtbSlowApiLogBurst + expr: increase(ftb_slow_api_log_total[10m]) > 5 + for: 2m + labels: + severity: warning + annotations: + summary: Slow API request log volume crossed the V2.8 baseline threshold. + runbook: docs/deployment.md + + - alert: FtbSlowPrismaLogBurst + expr: increase(ftb_slow_prisma_log_total[10m]) > 3 + for: 2m + labels: + severity: warning + annotations: + summary: Slow Prisma query log volume crossed the V2.8 baseline threshold. + runbook: docs/deployment.md + + - alert: FtbJobFailureLogBurst + expr: increase(ftb_job_failure_log_total[10m]) > 0 + for: 1m + labels: + severity: warning + annotations: + summary: Background or compatibility job failure logs were detected. + runbook: docs/runbooks/xiaobao-background-jobs.md + + - alert: FtbXiaobaoSummaryStale + expr: ftb_xiaobao_stale_summary_count > 0 + for: 15m + labels: + severity: warning + annotations: + summary: Xiaobao warning summaries are dirty or stale. + runbook: docs/runbooks/xiaobao-background-jobs.md + diff --git a/deploy/monitoring/prometheus/prometheus.yml b/deploy/monitoring/prometheus/prometheus.yml new file mode 100644 index 0000000..b8b3b18 --- /dev/null +++ b/deploy/monitoring/prometheus/prometheus.yml @@ -0,0 +1,45 @@ +global: + scrape_interval: 15s + evaluation_interval: 15s + +rule_files: + - /etc/prometheus/alert-rules.yml + +scrape_configs: + - job_name: prometheus + static_configs: + - targets: ['prometheus:9090'] + + - job_name: postgres-exporter + static_configs: + - targets: ['postgres-exporter:9187'] + + - job_name: node-exporter + static_configs: + - targets: ['node-exporter:9100'] + + - job_name: cadvisor + static_configs: + - targets: ['cadvisor:8080'] + + - job_name: promtail + static_configs: + - targets: ['promtail:9080'] + + - job_name: blackbox-http + metrics_path: /probe + params: + module: [http_2xx] + static_configs: + - targets: + - http://nginx/api/v1/health/version + - http://nginx/api/v1/config/ai + - http://nginx/api/v1/v2.2/requirements?productId=__smoke__&limit=1 + relabel_configs: + - source_labels: [__address__] + target_label: __param_target + - source_labels: [__param_target] + target_label: instance + - target_label: __address__ + replacement: blackbox-exporter:9115 + diff --git a/deploy/monitoring/promtail/config.yml b/deploy/monitoring/promtail/config.yml new file mode 100644 index 0000000..6ebb2d7 --- /dev/null +++ b/deploy/monitoring/promtail/config.yml @@ -0,0 +1,53 @@ +server: + http_listen_port: 9080 + grpc_listen_port: 0 + +positions: + filename: /tmp/positions.yml + +clients: + - url: http://loki:3100/loki/api/v1/push + +scrape_configs: + - job_name: docker + docker_sd_configs: + - host: unix:///var/run/docker.sock + refresh_interval: 15s + relabel_configs: + - source_labels: ['__meta_docker_container_name'] + regex: '/(.*)' + target_label: container + - source_labels: ['__meta_docker_container_label_com_docker_compose_service'] + target_label: compose_service + - source_labels: ['__meta_docker_container_label_com_docker_compose_project'] + target_label: compose_project + pipeline_stages: + - docker: {} + - match: + selector: '{compose_service="server"} |= "Slow API request"' + stages: + - metrics: + ftb_slow_api_log_total: + type: Counter + description: Slow API request log entries emitted by the NestJS server. + config: + action: inc + - match: + selector: '{compose_service="server"} |= "Slow Prisma query"' + stages: + - metrics: + ftb_slow_prisma_log_total: + type: Counter + description: Slow Prisma query log entries emitted by the NestJS server. + config: + action: inc + - match: + selector: '{compose_service="server"} |~ "AppData relation sync failed|AI 调用失败|AI 风险解读调用失败"' + stages: + - metrics: + ftb_job_failure_log_total: + type: Counter + description: Compatibility sync, AI job, or background task failure logs. + config: + action: inc + diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index 3f85e01..a4a41c4 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -123,7 +123,117 @@ services: server: condition: service_healthy + prometheus: + image: prom/prometheus:v2.53.1 + profiles: ['monitoring'] + restart: unless-stopped + command: + - '--config.file=/etc/prometheus/prometheus.yml' + - '--storage.tsdb.path=/prometheus' + - '--storage.tsdb.retention.time=${PROMETHEUS_RETENTION:-15d}' + - '--web.enable-lifecycle' + ports: + - '${PROMETHEUS_PORT:-9090}:9090' + volumes: + - ./deploy/monitoring/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro + - ./deploy/monitoring/prometheus/alert-rules.yml:/etc/prometheus/alert-rules.yml:ro + - prometheus_data:/prometheus + depends_on: + - postgres-exporter + - node-exporter + - cadvisor + - promtail + - blackbox-exporter + + grafana: + image: grafana/grafana:11.1.0 + profiles: ['monitoring'] + restart: unless-stopped + ports: + - '${GRAFANA_PORT:-3002}:3000' + environment: + GF_SECURITY_ADMIN_USER: ${GRAFANA_ADMIN_USER:-admin} + GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD:-change-me-monitoring-password} + GF_USERS_ALLOW_SIGN_UP: 'false' + volumes: + - grafana_data:/var/lib/grafana + - ./deploy/monitoring/grafana/provisioning:/etc/grafana/provisioning:ro + - ./deploy/monitoring/grafana/dashboards:/var/lib/grafana/dashboards:ro + depends_on: + - prometheus + - loki + + loki: + image: grafana/loki:2.9.8 + profiles: ['monitoring'] + restart: unless-stopped + command: ['-config.file=/etc/loki/config.yml'] + volumes: + - ./deploy/monitoring/loki/config.yml:/etc/loki/config.yml:ro + - loki_data:/loki + + promtail: + image: grafana/promtail:2.9.8 + profiles: ['monitoring'] + restart: unless-stopped + command: ['-config.file=/etc/promtail/config.yml'] + volumes: + - ./deploy/monitoring/promtail/config.yml:/etc/promtail/config.yml:ro + - /var/lib/docker/containers:/var/lib/docker/containers:ro + - /var/run/docker.sock:/var/run/docker.sock:ro + depends_on: + - loki + + postgres-exporter: + image: quay.io/prometheuscommunity/postgres-exporter:v0.15.0 + profiles: ['monitoring'] + restart: unless-stopped + command: + - '--extend.query-path=/etc/postgres-exporter/postgres-queries.yml' + environment: + DATA_SOURCE_NAME: postgresql://${POSTGRES_USER:-postgres}:${POSTGRES_PASSWORD}@postgres:5432/${POSTGRES_DB:-ftb_pm}?sslmode=disable + volumes: + - ./deploy/monitoring/postgres/postgres-queries.yml:/etc/postgres-exporter/postgres-queries.yml:ro + depends_on: + postgres: + condition: service_healthy + + node-exporter: + image: prom/node-exporter:v1.8.2 + profiles: ['monitoring'] + restart: unless-stopped + command: + - '--path.rootfs=/host' + volumes: + - /:/host:ro,rslave + + cadvisor: + image: gcr.io/cadvisor/cadvisor:v0.49.1 + profiles: ['monitoring'] + restart: unless-stopped + privileged: true + devices: + - /dev/kmsg:/dev/kmsg + volumes: + - /:/rootfs:ro + - /var/run:/var/run:ro + - /sys:/sys:ro + - /var/lib/docker/:/var/lib/docker:ro + - /dev/disk/:/dev/disk:ro + + blackbox-exporter: + image: prom/blackbox-exporter:v0.25.0 + profiles: ['monitoring'] + restart: unless-stopped + command: + - '--config.file=/etc/blackbox/config.yml' + volumes: + - ./deploy/monitoring/blackbox/config.yml:/etc/blackbox/config.yml:ro + volumes: postgres_data: redis_data: server_data: + prometheus_data: + grafana_data: + loki_data: diff --git a/docs/deployment.md b/docs/deployment.md index ea3c4a0..ec1a67c 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -194,6 +194,30 @@ pnpm deploy:smoke -- --base-url http://localhost --expected-version { assert.match(result.stdout, /\/api\/v1\/v2\.2\/requirements\?productId=__smoke__/); assert.match(result.stdout, /\/api\/v1\/config\/ai/); }); + + it('declares the production monitoring baseline without real alert secrets', () => { + const alertRules = readFileSync( + join(root, 'deploy/monitoring/prometheus/alert-rules.yml'), + 'utf8', + ); + const postgresQueries = readFileSync( + join(root, 'deploy/monitoring/postgres/postgres-queries.yml'), + 'utf8', + ); + const promtailConfig = readFileSync( + join(root, 'deploy/monitoring/promtail/config.yml'), + 'utf8', + ); + const dashboard = readFileSync( + join(root, 'deploy/monitoring/grafana/dashboards/ftb-production-overview.json'), + 'utf8', + ); + + for (const alertName of [ + 'FtbPostgresDown', + 'FtbDiskPressure', + 'FtbSlowApiLogBurst', + 'FtbSlowPrismaLogBurst', + 'FtbJobFailureLogBurst', + 'FtbXiaobaoSummaryStale', + ]) { + assert.match(alertRules, new RegExp(alertName)); + } + + assert.match(postgresQueries, /xiaobao_risk_summaries/); + assert.match(postgresQueries, /dirty = true/); + assert.match(promtailConfig, /Slow API request/); + assert.match(promtailConfig, /Slow Prisma query/); + assert.match(promtailConfig, /AppData relation sync failed/); + assert.match(dashboard, /FTB Production Overview/); + + const combined = `${alertRules}\n${postgresQueries}\n${promtailConfig}\n${dashboard}`; + assert.doesNotMatch(combined, /sk-ant-[A-Za-z0-9]/); + assert.doesNotMatch(combined, /hooks\.slack\.com\/services\//); + }); }); diff --git a/scripts/verify-production-deploy.mjs b/scripts/verify-production-deploy.mjs index 1ff3603..9759a63 100644 --- a/scripts/verify-production-deploy.mjs +++ b/scripts/verify-production-deploy.mjs @@ -51,6 +51,11 @@ const checks = [ 'WEB_IMAGE', 'APP_VERSION', '/api/v1/health/version', + 'prometheus:', + "profiles: ['monitoring']", + 'postgres-exporter:', + 'blackbox-exporter:', + 'prometheus_data:', ], }, { @@ -110,6 +115,41 @@ const checks = [ 'X-Forwarded-Proto', ], }, + { + file: 'deploy/monitoring/README.md', + snippets: [ + '--profile monitoring', + 'FtbPostgresDown', + 'FtbXiaobaoSummaryStale', + 'no webhook URLs', + ], + }, + { + file: 'deploy/monitoring/prometheus/prometheus.yml', + snippets: ['postgres-exporter:9187', 'promtail:9080', 'blackbox-http'], + }, + { + file: 'deploy/monitoring/prometheus/alert-rules.yml', + snippets: [ + 'FtbPostgresDown', + 'FtbSlowApiLogBurst', + 'FtbSlowPrismaLogBurst', + 'FtbJobFailureLogBurst', + 'FtbXiaobaoSummaryStale', + ], + }, + { + file: 'deploy/monitoring/postgres/postgres-queries.yml', + snippets: ['xiaobao_risk_summaries', 'dirty = true', 'stale_summary_count'], + }, + { + file: 'deploy/monitoring/promtail/config.yml', + snippets: ['Slow API request', 'Slow Prisma query', 'AppData relation sync failed'], + }, + { + file: 'deploy/monitoring/grafana/dashboards/ftb-production-overview.json', + snippets: ['FTB Production Overview', 'ftb_xiaobao_stale_summary_count', 'Server Warning/Error Logs'], + }, { file: '.env.production.example', snippets: [ @@ -143,6 +183,7 @@ const checks = [ 'pnpm backup:postgres', 'pnpm restore:postgres', 'pnpm backup:server-data', + '--profile monitoring', 'pnpm db:migrate', 'NEXT_PUBLIC_API_URL', 'Nginx', From 72a59f125c04375cabe4ad1477e847457cdf00c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E3=80=82?= Date: Wed, 8 Jul 2026 16:28:02 +0800 Subject: [PATCH 4/4] =?UTF-8?q?docs(ops):=20=E8=A1=A5=E9=BD=90=E7=94=9F?= =?UTF-8?q?=E4=BA=A7=20runbook=20=E5=92=8C=20readiness=20=E6=B8=85?= =?UTF-8?q?=E5=8D=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 新增迁移回滚、AppData 退场、小宝后台任务 runbook\n- 新增生产 readiness 证据清单和 runbook placeholder 扫描\n- 更新部署文档与路线图到 V2.8 运维闭环阶段\n\nCo-Authored-By: GPT-5 Codex --- docs/deployment.md | 15 ++++ docs/production-readiness.md | 24 ++++++ docs/roadmap.md | 35 ++++---- docs/runbooks/appdata-retirement.md | 72 ++++++++++++++++ docs/runbooks/migration-rollback.md | 100 +++++++++++++++++++++++ docs/runbooks/xiaobao-background-jobs.md | 55 +++++++++++++ package.json | 1 + scripts/check-runbook-placeholders.mjs | 87 ++++++++++++++++++++ scripts/ops-scripts.test.mjs | 11 +++ scripts/verify-production-deploy.mjs | 35 ++++++++ 10 files changed, 420 insertions(+), 15 deletions(-) create mode 100644 docs/production-readiness.md create mode 100644 docs/runbooks/appdata-retirement.md create mode 100644 docs/runbooks/migration-rollback.md create mode 100644 docs/runbooks/xiaobao-background-jobs.md create mode 100644 scripts/check-runbook-placeholders.mjs diff --git a/docs/deployment.md b/docs/deployment.md index ec1a67c..7603237 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -283,6 +283,21 @@ pnpm restore:postgres -- \ --confirm-overwrite ``` +## 运维 Runbooks + +生产发布、迁移和异常处置优先使用这些手册： + +- `docs/runbooks/migration-rollback.md`：发布失败、迁移失败、数据恢复和镜像回滚。 +- `docs/runbooks/appdata-retirement.md`：AppData key 分阶段退场、双读核对、归档和回滚。 +- `docs/runbooks/xiaobao-background-jobs.md`：小宝摘要 stale 告警、手动刷新和未来后台任务规则。 +- `docs/production-readiness.md`：生产发布前后证据清单。 + +提交前运行 runbook 扫描： + +```bash +pnpm docs:check-runbooks +``` + ## 升级流程 ```bash diff --git a/docs/production-readiness.md b/docs/production-readiness.md new file mode 100644 index 0000000..afcc2ab --- /dev/null +++ b/docs/production-readiness.md @@ -0,0 +1,24 @@ +# Production Readiness Checklist + +Use this before a production release and again after the release smoke test. Each item requires evidence, not a verbal assertion. + +| Area | Gate | Evidence | +| --- | --- | --- | +| Backup and restore | PostgreSQL backup dry-run and server_data backup dry-run are reviewed. | `pnpm backup:postgres -- --dry-run --env-file .env.production` and `pnpm backup:server-data -- --dry-run --env-file .env.production` output saved in release notes. | +| Backup and restore | Restore command refuses overwrite without explicit confirmation. | `pnpm restore:postgres -- --env-file .env.production --input backups/postgres/ftb_pm-postgres-20260708T120000Z.dump` exits non-zero and names `--confirm-overwrite`. | +| Backup and restore | Fresh DB restore rehearsal completed before destructive restore. | Temporary database restore command and row-count comparison from `docs/runbooks/migration-rollback.md`. | +| Smoke tests | Release smoke is wired into GitHub Actions. | `.github/workflows/deploy-production.yml` runs `scripts/smoke-test-release.mjs` with `--expected-version`. | +| Smoke tests | Manual smoke can be run against the target. | `pnpm deploy:smoke -- --base-url http://localhost --expected-version "$APP_VERSION"` output. | +| Monitoring | Monitoring profile renders and can start without committed secrets. | `docker compose --env-file .env.production -f docker-compose.prod.yml --profile monitoring config` exits 0. | +| Monitoring | Required alert rules exist. | `FtbPostgresDown`, `FtbDiskPressure`, `FtbSlowApiLogBurst`, `FtbSlowPrismaLogBurst`, `FtbJobFailureLogBurst`, and `FtbXiaobaoSummaryStale` visible in Prometheus. | +| Audit | Domain write APIs record actor and resource scope where implemented. | API request sample or audit log sample for Product, Requirement, Project, Version, execution entities, and dictionary writes. | +| RBAC | Project and version operations enforce Owner, Admin, Member, Viewer boundaries where enabled. | Permission matrix test result or manual account walkthrough attached to release notes. | +| Consistency | AppData and relation tables are checked for migrated domains. | Count and missing-partition-key queries from `docs/runbooks/appdata-retirement.md`. | +| Performance | Slow API and slow Prisma thresholds are configured. | `API_SLOW_REQUEST_MS` and `PRISMA_SLOW_QUERY_MS` values recorded, plus Grafana slow-log panels checked. | +| Performance | V2.2 hot reads avoid full-table scans. | Requirement pool smoke uses `productId`; query plan or service test evidence attached for high-volume domains. | +| Post-release | Runtime version matches the release SHA. | `/api/v1/health/version` payload or `pnpm deploy:check-runtime` output. | +| Post-release | Product, requirement, workspace, Xiaobao, and AI config read paths respond. | `pnpm deploy:smoke` output attached. | +| Post-release | On-call rollback path is known. | `docs/runbooks/migration-rollback.md` link included in release notes. | + +Release owner signs off only after all required evidence is attached to the release notes or incident record. + diff --git a/docs/roadmap.md b/docs/roadmap.md index be4761c..f2d7a7c 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -1,17 +1,16 @@ # 开发路线图 -## 当前阶段：V2.4 — 领域 CRUD 主写迁移 +## 当前阶段：V2.8 — 生产硬化稳定版 + 运维闭环 -V2.4 的目标是把业务主数据源从 AppData JSONB 文档切换到 PostgreSQL 领域关系表。AppData 继续保留为迁移、回填、兼容读取和排查入口，但不再作为长期主写入源；新增业务能力必须优先设计关系表、领域 CRUD API、索引/分区键和权限边界。V2.4 做逐领域主写迁移，并随 CRUD 入口埋好基础权限、`actorId` 和审计事件骨架；完整 RBAC、审计覆盖和 AppData 退场收口放到 V2.5。 +V2.8 的目标是在既有生产 CI/CD 基线上补齐运维闭环：备份恢复演练、发布 smoke test、监控告警、日志检索、迁移回滚 runbook、AppData 退场 runbook、小宝后台化 runbook 和生产 readiness 证据清单。当前执行前提是 V2.4 领域 CRUD 主写迁移已由上游验收完成；本阶段不重新设计业务流程，专注把生产发布和故障处置做成可验证、可复盘、可回滚的标准流程。 ### 当前重点 -1. **领域写 API**：按模块补齐 Product/Project/Version/Requirement/VersionPlan/DevTask/TestCase/Bug/Member/TaskCategory/Worklog/Overtime 的关系表写入 API。 -2. **前端持久化切换**：Zustand store 保留状态管理，但保存入口从 `saveServerData(key)` 迁到领域 API；读取优先 V2.2/V2.3 关系表接口。 -3. **AppData 迁移工具化**：把 AppData → 关系表同步做成可重复运行、可计数校验、可回滚的运维脚本，不提交真实 `.env`。 -4. **主写切换闸门**：每个领域完成双读核对后，先停止该领域 JSON 主写入，再进入 V2.5 的 fallback 移除和归档退场。 -5. **数据一致性校验**：为每个迁移领域补 counts、抽样记录、孤儿引用、分区键完整性和唯一约束校验。 -6. **权限/审计骨架**：领域 API 必须携带当前用户、产品/项目/版本作用域和审计事件入口，避免 V2.5 做 RBAC 时返工。 +1. **备份恢复自动化**：PostgreSQL dump、`server_data` volume 备份、fresh DB restore dry-run 和显式覆盖确认。 +2. **发布 smoke test**：GitHub Actions 部署后自动校验 runtime version、前端根页、产品页、产品 API、V2.2 读路径和 AI 配置。 +3. **监控告警基线**：Prometheus/Grafana/Loki/Promtail 可选 profile，覆盖慢 API、慢 Prisma、任务失败、小宝摘要 stale、磁盘压力和 DB 可用性。 +4. **迁移和后台任务 runbook**：迁移回滚、AppData 退场、小宝后台化处置步骤、决策点和数据风险。 +5. **生产 readiness 清单**：backup/restore、smoke、monitoring、audit、RBAC、consistency、performance 和 post-release verification 都要有证据项。 ## V2 分阶段交付链路 @@ -34,12 +33,18 @@ V2.4 的目标是把业务主数据源从 AppData JSONB 文档切换到 PostgreS - 版本详情已有需求、调研、产品方案、UI、开发任务、测试用例、Bug、概览等核心 Tab；渲染重的路径优先接入 V2.2 关系表快读，并保留 AppData fallback。 - 后端已落地 Product、Requirement 领域 CRUD，DataModule AppData 乐观锁，V2.2 快读 API，V2.3 AppData 写后同步关系表，AI Provider 抽象和健康版本接口。 - Prisma schema 已包含 Product、Project、Version、Requirement、VersionPlan、DevTask、TestCase、Bug、WorkActivity、Xiaobao、AiLog、AppData 等关系模型；高增长表的分区 migration 已落地。 -- 主写入源仍处在兼容窗口：多数前端 store 继续通过 `apps/web/lib/server-data.ts` 的 `loadServerData` / `saveServerData` 写 AppData；`useProductStore` 仍以 `products-overview` 文档作为产品/项目/版本树主写入。 -- Project、Version、VersionPlan、DevTask、TestCase、Bug、Member、TaskCategory、TaskWorklog、Overtime 等领域写 API 尚未完整替代 AppData Store。若后续称为 V2.4，应理解为“领域 CRUD 迁移阶段”，不是 V2.3 已完成内容。 -- `packages/shared` 中仍保留早期枚举口径；切换领域 API 时需要统一为当前前端业务状态机。 +- 本 V2.8 执行线程以前提“V2.4 领域 CRUD 主写迁移已完成验收”推进；本阶段不重新逐项复核领域 API 清单。 +- V2.8 新增运维交付物集中在 `scripts/`、`.github/workflows/deploy-production.yml`、`deploy/monitoring/`、`docs/runbooks/`、`docs/deployment.md` 和 `docs/production-readiness.md`。 +- AppData 退场、RBAC/审计、性能和小宝后台化仍通过 production readiness 证据项追踪，避免把运维稳定版误当成业务治理已全部完成。 ### 已完成（按时间倒序） +**2026-07-08** +- V2.8 production ops closure started: added PostgreSQL backup, fresh DB restore with explicit overwrite confirmation, and `server_data` volume backup automation. +- Added release smoke suite and wired GitHub Actions deployment verification to runtime version, frontend root, products, V2.2 read path, and AI config checks. +- Added optional monitoring profile with Prometheus, Grafana, Loki, Promtail, postgres-exporter, node-exporter, cAdvisor, and blackbox-exporter. +- Added migration rollback, AppData retirement, Xiaobao background jobs runbooks, and production readiness evidence checklist. + **2026-07-06** - Added production CI/CD flow: GitHub Actions builds `web` and `server` Docker images, pushes immutable commit-SHA tags to GHCR, deploys by SSH, pulls images on the server, runs `pnpm --filter server db:deploy`, restarts Compose, and verifies `/api/v1/health/version`. - Added runtime version metadata: backend `GET /api/v1/health/version`, Docker build args/env, and a frontend refresh banner when browser assets are older than the server runtime. @@ -117,12 +122,12 @@ V2.4 的目标是把业务主数据源从 AppData JSONB 文档切换到 PostgreS ### 进行中 -- V2.4 领域 CRUD 主写迁移：从 AppData JSONB 主写入切换到关系表 API，并随 API 落基础权限、审计和查询性能边界。 +- V2.8 生产硬化稳定版：补齐备份恢复、发布 smoke、监控告警、日志检索、迁移回滚和运维证据闭环。 - 项目详情页 VersionCard 状态胶囊数据联动（部分已完成） ## V2 — 后端接入 -NestJS + Prisma + PostgreSQL 已接入到 V2.3。第一阶段用 `app_data` JSONB 文档表承接现有 store 数据形状，避免浏览器清站点数据导致业务数据丢失；第二阶段已建立分区关系表、V2.2 快读 API 和 V2.3 AppData 写后同步。当前 V2.4 才是逐领域启用写 API，让前端 store 从 AppData 主写入迁移到领域 CRUD；AppData 后续只保留为迁移兼容层。 +NestJS + Prisma + PostgreSQL 已完成 V2.1 至 V2.4 的后端迁移主线。第一阶段用 `app_data` JSONB 文档表承接现有 store 数据形状，避免浏览器清站点数据导致业务数据丢失；第二阶段建立分区关系表、V2.2 快读 API 和 V2.3 AppData 写后同步；V2.4 完成领域 CRUD 主写迁移验收。V2.8 不再新增业务主写迁移范围，而是把生产运维、备份恢复、监控告警和回滚手册补齐。 ### 关键任务 @@ -141,7 +146,7 @@ NestJS + Prisma + PostgreSQL 已接入到 V2.3。第一阶段用 `app_data` JSON 当前不做本地导入导出。清站点数据后浏览器旧数据无法恢复，后续新增数据直接写入 PostgreSQL。若以后需要迁移旧浏览器数据，再单独做管理员导入工具。 -## V2.4 — 领域 CRUD 迁移（当前阶段） +## V2.4 — 领域 CRUD 迁移（已完成前提） 目标是让关系表从“快读 + AppData 同步副本”逐步升级为主写入路径。迁移顺序应优先选择写入频率高、实体边界清晰、已经在 V2.2 mapper 中稳定的领域： @@ -221,7 +226,7 @@ V2.4 推进前必须先统一 `packages/shared` 的状态枚举与当前前端 |------|------| | V1 业务流程打磨 | 进行中 | | V1 朋友试用反馈 | 持续中 | -| V2 后端接入 | 进行中（V2.1/V2.2/V2.3 已完成，V2.4 主写迁移中） | +| V2 后端接入 | V2.1-V2.4 已完成；V2.8 运维闭环进行中 | | V3 AI 集成 | 等 V2 数据沉淀 | | 公开发布 | TBD | **2026-06-26** diff --git a/docs/runbooks/appdata-retirement.md b/docs/runbooks/appdata-retirement.md new file mode 100644 index 0000000..ec2d1ac --- /dev/null +++ b/docs/runbooks/appdata-retirement.md @@ -0,0 +1,72 @@ +# AppData Retirement Runbook + +Use this when retiring an AppData key after its domain writes have moved to relation-table APIs. The order is fixed: backup, measure, freeze writes, compare, remove fallback, archive. + +## Scope Gate + +Retire one AppData key family at a time. Good candidates have domain CRUD writes, read APIs, pagination boundaries, audit events, and a rollback path. + +## Preparation + +```bash +pnpm backup:postgres -- --env-file .env.production +pnpm backup:server-data -- --env-file .env.production +pnpm deploy:smoke -- --base-url http://localhost +``` + +Record the key family being retired, the owning domain API, and the relation tables that replace it. + +## Count And Consistency Checks + +Run counts before disabling writes. + +```bash +docker compose --env-file .env.production -f docker-compose.prod.yml exec postgres psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -c "select key, jsonb_array_length(value) as appdata_rows from app_data where key in ('products-overview','requirements','version-plans','dev-tasks','test-cases','bugs','members','task-categories','task-worklogs','overtime') order by key;" +docker compose --env-file .env.production -f docker-compose.prod.yml exec postgres psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -c "select 'requirements' as table_name, count(*) from requirements union all select 'version_plans', count(*) from version_plans union all select 'dev_tasks', count(*) from dev_tasks union all select 'test_cases', count(*) from test_cases union all select 'bugs', count(*) from bugs;" +``` + +For partitioned entities, also check missing partition keys. + +```bash +docker compose --env-file .env.production -f docker-compose.prod.yml exec postgres psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -c "select 'requirements_missing_product' as check_name, count(*) from requirements where product_id is null union all select 'dev_tasks_missing_version', count(*) from dev_tasks where version_id is null union all select 'test_cases_missing_version', count(*) from test_cases where version_id is null union all select 'bugs_missing_version', count(*) from bugs where version_id is null;" +``` + +## Disable AppData Writes + +1. Merge the domain-specific frontend store change that stops calling `saveServerData` for the retired key. +2. Keep AppData read fallback for one release while relation reads are verified. +3. Deploy and run smoke tests. + +```bash +pnpm deploy:smoke -- --base-url http://localhost --expected-version "$APP_VERSION" +``` + +## Remove Fallback + +Remove AppData read fallback only after one successful release where: + +- Domain writes went through relation APIs. +- AppData row counts did not grow for the retired key. +- V2.2 read paths and page workflows returned expected data. +- No `AppData relation sync failed` logs appeared during the observation window. + +## Archive + +Export retired keys before any later table cleanup. + +```bash +docker compose --env-file .env.production -f docker-compose.prod.yml exec postgres psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -c "copy (select key, value, updated_at from app_data where key in ('dev-tasks','test-cases','bugs')) to stdout with csv header" > backups/postgres/appdata-retired-executions-20260708.csv +``` + +## Rollback + +- If relation writes fail but AppData still has fresh data, roll back to the previous app image that still writes AppData. +- If AppData writes were already disabled and relation writes are bad, restore PostgreSQL from the backup made at the start of this runbook. +- If only read fallback removal caused the issue, roll back app images first and leave the database unchanged. + +## Data Risks + +- Removing fallback too early can hide valid historical JSON rows that were never mapped into relation tables. +- Re-enabling old AppData writes after relation writes have accepted new edits can overwrite newer relation state through compatibility sync. +- AppData exports can contain business-sensitive text; store backup CSV files in the same restricted location as database dumps. + diff --git a/docs/runbooks/migration-rollback.md b/docs/runbooks/migration-rollback.md new file mode 100644 index 0000000..2ffac4b --- /dev/null +++ b/docs/runbooks/migration-rollback.md @@ -0,0 +1,100 @@ +# Migration Rollback Runbook + +Use this when a production release, Prisma migration, or data migration causes failed smoke tests, missing data, bad query performance, or unsafe writes. + +## First Response + +1. Freeze new releases and ask product owners to pause bulk edits. +2. Capture current state before changing anything. + +```bash +date -u +git rev-parse HEAD +docker compose --env-file .env.production -f docker-compose.prod.yml ps +pnpm backup:postgres -- --env-file .env.production +pnpm backup:server-data -- --env-file .env.production +``` + +3. Run the read-only release smoke test. + +```bash +pnpm deploy:smoke -- --base-url http://localhost --expected-version "$APP_VERSION" +``` + +4. Check the database and latest server logs. + +```bash +docker compose --env-file .env.production -f docker-compose.prod.yml exec postgres pg_isready -U "$POSTGRES_USER" -d "$POSTGRES_DB" +docker compose --env-file .env.production -f docker-compose.prod.yml logs --tail=200 server +``` + +## Decision Points + +- App code bad, database healthy: roll back `WEB_IMAGE` and `SERVER_IMAGE` to the previous commit image tags, then restart Compose. +- Migration applied but only additive: roll back app images first; leave schema in place if old code remains compatible. +- Migration changed or removed data: restore a fresh database from the last known-good backup into a temporary database, compare row counts, then decide whether to restore production. +- AppData compatibility issue: keep production database intact, re-enable the previous app image that still reads the AppData fallback, and preserve the failing release backup for analysis. + +## App Image Rollback + +Set previous image tags in `.env.production`. Use the previous successful GitHub Actions run to identify the image tags. + +```bash +export PREVIOUS_WEB_IMAGE=ghcr.io/company/ftb-project-management/web:abc1234 +export PREVIOUS_SERVER_IMAGE=ghcr.io/company/ftb-project-management/server:abc1234 +sed -i "s|^WEB_IMAGE=.*|WEB_IMAGE=${PREVIOUS_WEB_IMAGE}|" .env.production +sed -i "s|^SERVER_IMAGE=.*|SERVER_IMAGE=${PREVIOUS_SERVER_IMAGE}|" .env.production +docker compose --env-file .env.production -f docker-compose.prod.yml pull web server +docker compose --env-file .env.production -f docker-compose.prod.yml up -d --remove-orphans +pnpm deploy:smoke -- --base-url http://localhost --expected-version abc1234 +``` + +## Fresh Database Restore + +Never restore over production until the backup has been rehearsed into a temporary database. + +```bash +export BACKUP_FILE=backups/postgres/ftb_pm-postgres-20260708T120000Z.dump +pnpm restore:postgres -- \ + --dry-run \ + --confirm-overwrite \ + --env-file .env.production \ + --input "$BACKUP_FILE" \ + --target-db ftb_pm_restore_check +pnpm restore:postgres -- \ + --confirm-overwrite \ + --env-file .env.production \ + --input "$BACKUP_FILE" \ + --target-db ftb_pm_restore_check +``` + +Compare critical row counts before touching production. + +```bash +docker compose --env-file .env.production -f docker-compose.prod.yml exec postgres psql -U "$POSTGRES_USER" -d ftb_pm_restore_check -c "select 'products' as table_name, count(*) from products union all select 'requirements', count(*) from requirements union all select 'versions', count(*) from versions union all select 'dev_tasks', count(*) from dev_tasks union all select 'test_cases', count(*) from test_cases union all select 'bugs', count(*) from bugs;" +``` + +If the temporary restore is healthy and production data is unsafe, restore production with explicit overwrite confirmation. + +```bash +pnpm restore:postgres -- \ + --confirm-overwrite \ + --env-file .env.production \ + --input "$BACKUP_FILE" +docker compose --env-file .env.production -f docker-compose.prod.yml up -d --remove-orphans +pnpm deploy:smoke -- --base-url http://localhost +``` + +## Data Risks + +- PostgreSQL restore is destructive for the target database because the script terminates connections, drops the target database, recreates it, and runs `pg_restore`. +- Restoring PostgreSQL does not restore `server_data`; keep AI provider config backup files with the same incident bundle. +- AppData and relation tables can diverge during compatibility windows. Before deleting or restoring, preserve both the failing production backup and the known-good backup. +- If users continued editing during the incident, record the time window and decide whether those edits must be replayed manually after restore. + +## Closeout + +1. Save the failed release SHA, rollback SHA, backup file names, smoke output, and row-count evidence in the incident notes. +2. Keep the failed backup until the next successful release has completed smoke tests and one business-day observation. +3. Add a regression test or runbook correction before re-attempting the migration. + diff --git a/docs/runbooks/xiaobao-background-jobs.md b/docs/runbooks/xiaobao-background-jobs.md new file mode 100644 index 0000000..c2fce9f --- /dev/null +++ b/docs/runbooks/xiaobao-background-jobs.md @@ -0,0 +1,55 @@ +# Xiaobao Background Jobs Runbook + +Current V2.8 production monitoring supports Xiaobao staleness detection. The first production implementation is still page-triggered: opening `/xiaobao-warning` computes risk, saves snapshots, and lets V2.3 sync refresh summaries. A future scheduler must preserve the same idempotent data contract. + +## Alert Triage + +When `FtbXiaobaoSummaryStale` fires: + +```bash +docker compose --env-file .env.production -f docker-compose.prod.yml exec postgres psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -c "select version_id, dirty, updated_at, recomputed_at, risk_level, risk_score from xiaobao_risk_summaries where dirty = true or updated_at < now() - interval '6 hours' order by updated_at asc limit 20;" +docker compose --env-file .env.production -f docker-compose.prod.yml logs --tail=200 server | grep -E "xiaobao|AppData relation sync failed|Slow Prisma query" +``` + +Decision points: + +- Rows are dirty after active version edits: ask a manager to open `/xiaobao-warning` once, then verify summaries refresh. +- Rows remain dirty and server logs show sync failures: treat as AppData relation sync incident and follow `migration-rollback.md`. +- Rows are stale but no product release is near: keep monitoring and schedule a manual refresh before the next release decision meeting. +- Rows are stale for a release due today: refresh manually and have the release owner review the resulting risk explanation before ship/no-ship decision. + +## Manual Refresh Path + +1. Log in as a user with `xiaobao.warning:manage`. +2. Open `/xiaobao-warning`. +3. Wait until AI interpretation status is no longer generating for high-risk versions. +4. Re-run the stale-summary query. +5. Run the release smoke test. + +```bash +pnpm deploy:smoke -- --base-url http://localhost +``` + +## Future Scheduler Rules + +When a background job is introduced, it must: + +- Read unfinished versions by relation-table scope, not by full AppData document scan. +- Use one idempotency key per `versionId + riskSignature + snapshotDate`. +- Write snapshots append-only and upsert summaries by `version_id`. +- Mark failures with structured logs containing `xiaobao background job failed`. +- Retry transient AI failures with backoff and keep rule-based risk output even when AI interpretation fails. +- Never mutate Version, Requirement, DevTask, TestCase, Bug, or Member data. + +## Monitoring Expectations + +- `FtbXiaobaoSummaryStale` alerts on dirty or older-than-6-hour summaries. +- `FtbJobFailureLogBurst` alerts when sync or future job failure log counters increase. +- Grafana dashboard shows the stale summary count and matching server warning/error logs. + +## Data Risks + +- Recomputing Xiaobao summaries can change release risk badges and manager decisions; record manual refresh time in release notes. +- AI interpretation cache is explanatory only. Do not restore or delete business entities to fix a bad explanation. +- If stale summaries are caused by relation sync failure, refreshing the page can mask the symptom without fixing the underlying sync path. Preserve logs before restarting services. + diff --git a/package.json b/package.json index 213d37c..57f7506 100644 --- a/package.json +++ b/package.json @@ -14,6 +14,7 @@ "backup:postgres": "node scripts/backup-postgres.mjs", "restore:postgres": "node scripts/restore-postgres.mjs", "backup:server-data": "node scripts/backup-server-data.mjs", + "docs:check-runbooks": "node scripts/check-runbook-placeholders.mjs --paths docs/runbooks docs/production-readiness.md", "deploy:local:build": "docker compose --env-file .env.local-server -f docker-compose.local.yml build", "deploy:local:up": "docker compose --env-file .env.local-server -f docker-compose.local.yml up -d", "deploy:local:down": "docker compose --env-file .env.local-server -f docker-compose.local.yml down", diff --git a/scripts/check-runbook-placeholders.mjs b/scripts/check-runbook-placeholders.mjs new file mode 100644 index 0000000..f99a3fe --- /dev/null +++ b/scripts/check-runbook-placeholders.mjs @@ -0,0 +1,87 @@ +#!/usr/bin/env node +import { existsSync, readFileSync, readdirSync, statSync } from 'node:fs'; +import { extname, join } from 'node:path'; +import { pathToFileURL } from 'node:url'; +import { option, parseArgs } from './ops-utils.mjs'; + +const forbiddenPatterns = [ + { name: 'TBD', pattern: /\bTBD\b/i }, + { name: 'TODO', pattern: /\bTODO\b/i }, + { name: 'fill-in', pattern: /fill in|fill-in/i }, + { name: 'angle-token', pattern: /<[^>\n]+>/ }, + { name: 'Chinese pending marker', pattern: /待定|占位/ }, +]; + +function usage() { + return `Usage: node scripts/check-runbook-placeholders.mjs --paths [path...] + +Scans Markdown runbooks for unresolved placeholder markers. +`; +} + +function collectMarkdownFiles(path) { + if (!existsSync(path)) { + throw new Error(`Path not found: ${path}`); + } + + const stat = statSync(path); + if (stat.isFile()) return extname(path) === '.md' ? [path] : []; + if (!stat.isDirectory()) return []; + + const files = []; + for (const entry of readdirSync(path)) { + files.push(...collectMarkdownFiles(join(path, entry))); + } + return files; +} + +function requestedPaths(argv) { + const args = parseArgs(argv); + if (args.flags.has('help')) return { help: true, paths: [] }; + const first = option(args, 'paths'); + const paths = [first, ...args.positionals].filter(Boolean); + if (paths.length === 0) { + throw new Error('Missing --paths [path...]'); + } + return { help: false, paths }; +} + +export function scanFiles(paths) { + const files = paths.flatMap(collectMarkdownFiles); + const findings = []; + for (const file of files) { + const lines = readFileSync(file, 'utf8').split(/\r?\n/); + lines.forEach((line, index) => { + for (const forbidden of forbiddenPatterns) { + if (forbidden.pattern.test(line)) { + findings.push(`${file}:${index + 1} ${forbidden.name}: ${line.trim()}`); + } + } + }); + } + return { files, findings }; +} + +export async function main(argv = process.argv.slice(2)) { + const { help, paths } = requestedPaths(argv); + if (help) { + process.stdout.write(usage()); + return; + } + + const { files, findings } = scanFiles(paths); + if (findings.length > 0) { + process.stderr.write(`${findings.join('\n')}\n`); + throw new Error(`Runbook placeholder scan failed: ${findings.length} finding(s)`); + } + + process.stdout.write(`Runbook placeholder scan passed (${files.length} file(s)).\n`); +} + +if (import.meta.url === pathToFileURL(process.argv[1]).href) { + main().catch((error) => { + process.stderr.write(`${error.message}\n`); + process.exit(1); + }); +} + diff --git a/scripts/ops-scripts.test.mjs b/scripts/ops-scripts.test.mjs index 3d21014..a70d0f8 100644 --- a/scripts/ops-scripts.test.mjs +++ b/scripts/ops-scripts.test.mjs @@ -138,4 +138,15 @@ describe('production ops scripts', () => { assert.doesNotMatch(combined, /sk-ant-[A-Za-z0-9]/); assert.doesNotMatch(combined, /hooks\.slack\.com\/services\//); }); + + it('passes the runbook placeholder scan', () => { + const result = runScript('scripts/check-runbook-placeholders.mjs', [ + '--paths', + 'docs/runbooks', + 'docs/production-readiness.md', + ]); + + assert.equal(result.status, 0, result.stderr); + assert.match(result.stdout, /Runbook placeholder scan passed/); + }); }); diff --git a/scripts/verify-production-deploy.mjs b/scripts/verify-production-deploy.mjs index 9759a63..957505f 100644 --- a/scripts/verify-production-deploy.mjs +++ b/scripts/verify-production-deploy.mjs @@ -93,6 +93,14 @@ const checks = [ file: 'scripts/backup-server-data.mjs', snippets: ['server_data', 'tar -czf', '--dry-run'], }, + { + file: 'scripts/check-runbook-placeholders.mjs', + snippets: ['Runbook placeholder scan passed', 'Runbook placeholder scan failed', '--paths'], + }, + { + file: 'package.json', + snippets: ['docs:check-runbooks', 'docs/runbooks', 'docs/production-readiness.md'], + }, { file: 'docker-compose.local.yml', snippets: [ @@ -138,6 +146,31 @@ const checks = [ 'FtbXiaobaoSummaryStale', ], }, + { + file: 'docs/runbooks/migration-rollback.md', + snippets: ['Fresh Database Restore', 'Data Risks', 'pnpm restore:postgres'], + }, + { + file: 'docs/runbooks/appdata-retirement.md', + snippets: ['Disable AppData Writes', 'Remove Fallback', 'Data Risks'], + }, + { + file: 'docs/runbooks/xiaobao-background-jobs.md', + snippets: ['FtbXiaobaoSummaryStale', 'Future Scheduler Rules', 'Data Risks'], + }, + { + file: 'docs/production-readiness.md', + snippets: [ + 'Backup and restore', + 'Smoke tests', + 'Monitoring', + 'Audit', + 'RBAC', + 'Consistency', + 'Performance', + 'Post-release', + ], + }, { file: 'deploy/monitoring/postgres/postgres-queries.yml', snippets: ['xiaobao_risk_summaries', 'dirty = true', 'stale_summary_count'], @@ -183,6 +216,8 @@ const checks = [ 'pnpm backup:postgres', 'pnpm restore:postgres', 'pnpm backup:server-data', + 'docs/runbooks/migration-rollback.md', + 'docs/production-readiness.md', '--profile monitoring', 'pnpm db:migrate', 'NEXT_PUBLIC_API_URL',