feat(appdata): 增加归档导出校验脚本

This commit is contained in:
2026-07-08 16:42:40 +08:00
parent 36202028d2
commit 869b1c1060
7 changed files with 389 additions and 0 deletions

1
.gitignore vendored
View File

@@ -15,3 +15,4 @@ next-env.d.ts
*.tsbuildinfo
apps/server/data/
.worktrees/
appdata-archive-*.json

View File

@@ -203,6 +203,24 @@ pnpm deploy:check-runtime http://localhost/api/v1/health/version <expected-commi
浏览器不再作为业务数据主存储。清浏览器缓存不会删除产品、项目、版本、任务、测试用例、Bug 等业务数据。
## AppData 归档与校验V2.5
V2.5 起,已迁移业务 key 的 `PUT /api/v1/data/:key` 会返回 `409 APP_DATA_WRITE_FROZEN`,读路径仍保留给历史核对和回滚。停用 AppData 写入前后都建议导出一份只读归档:
```bash
pnpm appdata:archive:export -- --out backups/appdata-archive-$(date +%Y%m%d%H%M%S).json
pnpm appdata:archive:verify -- --archive backups/appdata-archive-20260708120000.json
```
归档 JSON 包含:
- `metadata.appVersion` / `metadata.appBuildTime` / `metadata.sourceCommit`:导出时的应用版本信息。
- `keys`:本次导出的 AppData key 列表。
- `rows[].valueChecksum`:每个 key 的 JSON 内容 SHA-256。
- `metadata.payloadChecksum`:整份 key list + rows payload 的 SHA-256。
生产环境执行导出前需要确保 `DATABASE_URL` 指向当前 PostgreSQL。默认输出文件名为 `appdata-archive-<timestamp>.json`,该模式已加入 `.gitignore`;真实归档应放入服务器备份目录或对象存储,不提交到代码仓库。
## 升级流程
```bash

View File

@@ -9,6 +9,9 @@
"test": "turbo test",
"deploy:verify": "node scripts/verify-production-deploy.mjs",
"deploy:check-runtime": "node scripts/check-runtime-version.mjs",
"appdata:archive:export": "node scripts/export-appdata-archive.mjs",
"appdata:archive:verify": "node scripts/verify-appdata-archive.mjs",
"appdata:archive:test": "node --test scripts/appdata-archive.test.mjs",
"deploy:local:build": "docker compose --env-file .env.local-server -f docker-compose.local.yml build",
"deploy:local:up": "docker compose --env-file .env.local-server -f docker-compose.local.yml up -d",
"deploy:local:down": "docker compose --env-file .env.local-server -f docker-compose.local.yml down",

View File

@@ -0,0 +1,62 @@
import assert from 'node:assert/strict';
import test from 'node:test';
import {
buildAppDataArchive,
verifyAppDataArchive,
} from './lib/appdata-archive.mjs';
test('builds and verifies an AppData archive with deterministic checksums', () => {
const archive = buildAppDataArchive(
[
{
key: 'dev-tasks',
value: [{ id: 'dt-1', title: '开发任务' }],
updatedAt: new Date('2026-07-08T08:00:00.000Z'),
},
{
key: 'products-overview',
value: [{ id: 'p1', name: 'FTB' }],
updatedAt: new Date('2026-07-08T08:01:00.000Z'),
},
],
{
appVersion: 'test-version',
appBuildTime: '2026-07-08T08:02:00.000Z',
exportedAt: '2026-07-08T08:03:00.000Z',
sourceCommit: 'test-commit',
},
);
assert.deepEqual(archive.keys, ['dev-tasks', 'products-overview']);
assert.equal(archive.metadata.rowCount, 2);
assert.equal(archive.metadata.keyCount, 2);
assert.match(archive.metadata.payloadChecksum, /^[a-f0-9]{64}$/);
const verification = verifyAppDataArchive(archive);
assert.equal(verification.ok, true);
assert.deepEqual(verification.errors, []);
});
test('verification fails when archived row content is tampered', () => {
const archive = buildAppDataArchive(
[
{
key: 'dev-tasks',
value: [{ id: 'dt-1', title: '开发任务' }],
updatedAt: new Date('2026-07-08T08:00:00.000Z'),
},
],
{
appVersion: 'test-version',
appBuildTime: '2026-07-08T08:02:00.000Z',
exportedAt: '2026-07-08T08:03:00.000Z',
sourceCommit: 'test-commit',
},
);
archive.rows[0].value[0].title = '被篡改';
const verification = verifyAppDataArchive(archive);
assert.equal(verification.ok, false);
assert.match(verification.errors.join('\n'), /checksum/i);
});

View File

@@ -0,0 +1,87 @@
#!/usr/bin/env node
import { execFileSync } from 'node:child_process';
import { mkdirSync, readFileSync, writeFileSync } from 'node:fs';
import { createRequire } from 'node:module';
import { dirname, resolve } from 'node:path';
import { fileURLToPath } from 'node:url';
import { buildAppDataArchive } from './lib/appdata-archive.mjs';
const root = dirname(fileURLToPath(new URL('../package.json', import.meta.url)));
const args = parseArgs(process.argv.slice(2));
if (args.help) {
printHelp();
process.exit(0);
}
const outPath = resolve(root, args.out ?? defaultArchiveName());
const serverRequire = createRequire(resolve(root, 'apps/server/package.json'));
const { PrismaClient } = serverRequire('@prisma/client');
const prisma = new PrismaClient();
try {
const rows = await prisma.appData.findMany({
orderBy: { key: 'asc' },
select: { key: true, value: true, updatedAt: true },
});
const archive = buildAppDataArchive(rows, {
appVersion: process.env.APP_VERSION || readPackageVersion(),
appBuildTime: process.env.APP_BUILD_TIME || null,
sourceCommit: process.env.GITHUB_SHA || readGitCommit(),
});
mkdirSync(dirname(outPath), { recursive: true });
writeFileSync(outPath, `${JSON.stringify(archive, null, 2)}\n`);
console.log(`AppData archive exported: ${outPath}`);
console.log(`Rows: ${archive.metadata.rowCount}; keys: ${archive.keys.join(', ') || '(none)'}`);
console.log(`Payload checksum: ${archive.metadata.payloadChecksum}`);
} finally {
await prisma.$disconnect();
}
function parseArgs(argv) {
const parsed = { help: false, out: null };
for (let index = 0; index < argv.length; index += 1) {
const arg = argv[index];
if (arg === '--') continue;
else if (arg === '--help' || arg === '-h') parsed.help = true;
else if (arg === '--out') parsed.out = argv[++index];
else if (arg.startsWith('--out=')) parsed.out = arg.slice('--out='.length);
else throw new Error(`Unknown argument: ${arg}`);
}
return parsed;
}
function defaultArchiveName() {
const stamp = new Date().toISOString().replace(/[:.]/g, '-');
return `appdata-archive-${stamp}.json`;
}
function readPackageVersion() {
try {
const pkg = JSON.parse(readFileSync(resolve(root, 'apps/server/package.json'), 'utf8'));
return pkg.version || 'unknown';
} catch {
return 'unknown';
}
}
function readGitCommit() {
try {
return execFileSync('git', ['rev-parse', 'HEAD'], { cwd: root, encoding: 'utf8' }).trim();
} catch {
return 'unknown';
}
}
function printHelp() {
console.log(`Usage: pnpm appdata:archive:export -- --out backups/appdata-archive.json
Exports every row from app_data to a JSON archive with key list, app version metadata,
per-row SHA-256 checksums, and a payload checksum.
Options:
--out <path> Output archive path. Defaults to appdata-archive-<timestamp>.json.
-h, --help Show this help.
`);
}

View File

@@ -0,0 +1,161 @@
import { createHash } from 'node:crypto';
export const APPDATA_ARCHIVE_VERSION = 1;
export const APPDATA_ARCHIVE_CHECKSUM_ALGORITHM = 'sha256';
export function stableStringify(value) {
return JSON.stringify(sortJsonValue(value));
}
export function sha256Hex(value) {
return createHash('sha256').update(String(value)).digest('hex');
}
export function buildAppDataArchive(rows, options = {}) {
const normalizedRows = [...rows]
.sort((a, b) => String(a.key).localeCompare(String(b.key)))
.map((row) => {
const canonicalValue = stableStringify(row.value);
return {
key: String(row.key),
value: row.value,
version: toIsoString(row.updatedAt),
valueChecksum: sha256Hex(canonicalValue),
bytes: Buffer.byteLength(canonicalValue),
};
});
const keys = normalizedRows.map((row) => row.key);
const metadata = {
archiveVersion: APPDATA_ARCHIVE_VERSION,
exportedAt: options.exportedAt ?? new Date().toISOString(),
appVersion: options.appVersion ?? 'unknown',
appBuildTime: options.appBuildTime ?? null,
sourceCommit: options.sourceCommit ?? 'unknown',
rowCount: normalizedRows.length,
keyCount: keys.length,
checksumAlgorithm: APPDATA_ARCHIVE_CHECKSUM_ALGORITHM,
keyListChecksum: sha256Hex(stableStringify(keys)),
payloadChecksum: '',
};
const archive = { metadata, keys, rows: normalizedRows };
archive.metadata.payloadChecksum = checksumPayload(archive);
return archive;
}
export function verifyAppDataArchive(archive) {
const errors = [];
if (!archive || typeof archive !== 'object') {
return { ok: false, errors: ['Archive must be a JSON object'], summary: emptySummary() };
}
const metadata = archive.metadata;
const keys = Array.isArray(archive.keys) ? archive.keys : [];
const rows = Array.isArray(archive.rows) ? archive.rows : [];
if (!metadata || typeof metadata !== 'object') {
errors.push('Missing metadata object');
} else {
if (metadata.archiveVersion !== APPDATA_ARCHIVE_VERSION) {
errors.push(`Unsupported archiveVersion: ${metadata.archiveVersion}`);
}
if (metadata.checksumAlgorithm !== APPDATA_ARCHIVE_CHECKSUM_ALGORITHM) {
errors.push(`Unsupported checksumAlgorithm: ${metadata.checksumAlgorithm}`);
}
if (metadata.rowCount !== rows.length) {
errors.push(`rowCount mismatch: metadata=${metadata.rowCount}, actual=${rows.length}`);
}
if (metadata.keyCount !== keys.length) {
errors.push(`keyCount mismatch: metadata=${metadata.keyCount}, actual=${keys.length}`);
}
}
const rowKeys = rows.map((row) => row?.key);
if (stableStringify(keys) !== stableStringify(rowKeys)) {
errors.push('Key list does not match row keys');
}
const keyListChecksum = sha256Hex(stableStringify(keys));
if (metadata?.keyListChecksum !== keyListChecksum) {
errors.push('Key list checksum mismatch');
}
for (const row of rows) {
if (!row || typeof row !== 'object') {
errors.push('Archive contains a non-object row');
continue;
}
if (typeof row.key !== 'string') {
errors.push('Archive row is missing string key');
}
if (typeof row.version !== 'string') {
errors.push(`Archive row ${row.key ?? '<unknown>'} is missing string version`);
}
const canonicalValue = stableStringify(row.value);
const valueChecksum = sha256Hex(canonicalValue);
if (row.valueChecksum !== valueChecksum) {
errors.push(`Value checksum mismatch for ${row.key}`);
}
const bytes = Buffer.byteLength(canonicalValue);
if (row.bytes !== bytes) {
errors.push(`Byte size mismatch for ${row.key}`);
}
}
const payloadChecksum = checksumPayload({ keys, rows });
if (metadata?.payloadChecksum !== payloadChecksum) {
errors.push('Payload checksum mismatch');
}
return {
ok: errors.length === 0,
errors,
summary: {
archiveVersion: metadata?.archiveVersion ?? null,
exportedAt: metadata?.exportedAt ?? null,
appVersion: metadata?.appVersion ?? null,
sourceCommit: metadata?.sourceCommit ?? null,
rowCount: rows.length,
keyCount: keys.length,
keys,
payloadChecksum,
},
};
}
export function checksumPayload(archive) {
return sha256Hex(stableStringify({
keys: archive.keys,
rows: archive.rows,
}));
}
function sortJsonValue(value) {
if (Array.isArray(value)) return value.map(sortJsonValue);
if (!value || typeof value !== 'object') return value;
return Object.fromEntries(
Object.entries(value)
.filter(([, item]) => item !== undefined)
.sort(([a], [b]) => a.localeCompare(b))
.map(([key, item]) => [key, sortJsonValue(item)]),
);
}
function toIsoString(value) {
if (value instanceof Date) return value.toISOString();
const date = new Date(value);
if (Number.isFinite(date.getTime())) return date.toISOString();
throw new Error(`Invalid AppData updatedAt value: ${value}`);
}
function emptySummary() {
return {
archiveVersion: null,
exportedAt: null,
appVersion: null,
sourceCommit: null,
rowCount: 0,
keyCount: 0,
keys: [],
payloadChecksum: null,
};
}

View File

@@ -0,0 +1,57 @@
#!/usr/bin/env node
import { readFileSync } from 'node:fs';
import { resolve } from 'node:path';
import { verifyAppDataArchive } from './lib/appdata-archive.mjs';
const args = parseArgs(process.argv.slice(2));
if (args.help || !args.archive) {
printHelp();
process.exit(args.help ? 0 : 1);
}
const archivePath = resolve(process.cwd(), args.archive);
const archive = JSON.parse(readFileSync(archivePath, 'utf8'));
const verification = verifyAppDataArchive(archive);
if (args.json) {
console.log(JSON.stringify(verification, null, 2));
} else if (verification.ok) {
console.log(`AppData archive verified: ${archivePath}`);
console.log(`Rows: ${verification.summary.rowCount}; keys: ${verification.summary.keys.join(', ') || '(none)'}`);
console.log(`Payload checksum: ${verification.summary.payloadChecksum}`);
} else {
console.error(`AppData archive verification failed: ${archivePath}`);
for (const error of verification.errors) {
console.error(`- ${error}`);
}
}
process.exit(verification.ok ? 0 : 1);
function parseArgs(argv) {
const parsed = { help: false, json: false, archive: null };
for (let index = 0; index < argv.length; index += 1) {
const arg = argv[index];
if (arg === '--') continue;
else if (arg === '--help' || arg === '-h') parsed.help = true;
else if (arg === '--json') parsed.json = true;
else if (arg === '--archive') parsed.archive = argv[++index];
else if (arg.startsWith('--archive=')) parsed.archive = arg.slice('--archive='.length);
else if (!arg.startsWith('-') && !parsed.archive) parsed.archive = arg;
else throw new Error(`Unknown argument: ${arg}`);
}
return parsed;
}
function printHelp() {
console.log(`Usage: pnpm appdata:archive:verify -- --archive backups/appdata-archive.json
Verifies key list, per-row SHA-256 checksums, and the archive payload checksum.
Options:
--archive <path> Archive JSON path. A positional path is also accepted.
--json Print machine-readable verification output.
-h, --help Show this help.
`);
}