feat(appdata): 增加归档导出校验脚本
This commit is contained in:
1
.gitignore
vendored
1
.gitignore
vendored
@@ -15,3 +15,4 @@ next-env.d.ts
|
||||
*.tsbuildinfo
|
||||
apps/server/data/
|
||||
.worktrees/
|
||||
appdata-archive-*.json
|
||||
|
||||
@@ -203,6 +203,24 @@ pnpm deploy:check-runtime http://localhost/api/v1/health/version <expected-commi
|
||||
|
||||
浏览器不再作为业务数据主存储。清浏览器缓存不会删除产品、项目、版本、任务、测试用例、Bug 等业务数据。
|
||||
|
||||
## AppData 归档与校验(V2.5)
|
||||
|
||||
V2.5 起,已迁移业务 key 的 `PUT /api/v1/data/:key` 会返回 `409 APP_DATA_WRITE_FROZEN`,读路径仍保留给历史核对和回滚。停用 AppData 写入前后都建议导出一份只读归档:
|
||||
|
||||
```bash
|
||||
pnpm appdata:archive:export -- --out backups/appdata-archive-$(date +%Y%m%d%H%M%S).json
|
||||
pnpm appdata:archive:verify -- --archive backups/appdata-archive-20260708120000.json
|
||||
```
|
||||
|
||||
归档 JSON 包含:
|
||||
|
||||
- `metadata.appVersion` / `metadata.appBuildTime` / `metadata.sourceCommit`:导出时的应用版本信息。
|
||||
- `keys`:本次导出的 AppData key 列表。
|
||||
- `rows[].valueChecksum`:每个 key 的 JSON 内容 SHA-256。
|
||||
- `metadata.payloadChecksum`:整份 key list + rows payload 的 SHA-256。
|
||||
|
||||
生产环境执行导出前需要确保 `DATABASE_URL` 指向当前 PostgreSQL。默认输出文件名为 `appdata-archive-<timestamp>.json`,该模式已加入 `.gitignore`;真实归档应放入服务器备份目录或对象存储,不提交到代码仓库。
|
||||
|
||||
## 升级流程
|
||||
|
||||
```bash
|
||||
|
||||
@@ -9,6 +9,9 @@
|
||||
"test": "turbo test",
|
||||
"deploy:verify": "node scripts/verify-production-deploy.mjs",
|
||||
"deploy:check-runtime": "node scripts/check-runtime-version.mjs",
|
||||
"appdata:archive:export": "node scripts/export-appdata-archive.mjs",
|
||||
"appdata:archive:verify": "node scripts/verify-appdata-archive.mjs",
|
||||
"appdata:archive:test": "node --test scripts/appdata-archive.test.mjs",
|
||||
"deploy:local:build": "docker compose --env-file .env.local-server -f docker-compose.local.yml build",
|
||||
"deploy:local:up": "docker compose --env-file .env.local-server -f docker-compose.local.yml up -d",
|
||||
"deploy:local:down": "docker compose --env-file .env.local-server -f docker-compose.local.yml down",
|
||||
|
||||
62
scripts/appdata-archive.test.mjs
Normal file
62
scripts/appdata-archive.test.mjs
Normal file
@@ -0,0 +1,62 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import test from 'node:test';
|
||||
|
||||
import {
|
||||
buildAppDataArchive,
|
||||
verifyAppDataArchive,
|
||||
} from './lib/appdata-archive.mjs';
|
||||
|
||||
test('builds and verifies an AppData archive with deterministic checksums', () => {
|
||||
const archive = buildAppDataArchive(
|
||||
[
|
||||
{
|
||||
key: 'dev-tasks',
|
||||
value: [{ id: 'dt-1', title: '开发任务' }],
|
||||
updatedAt: new Date('2026-07-08T08:00:00.000Z'),
|
||||
},
|
||||
{
|
||||
key: 'products-overview',
|
||||
value: [{ id: 'p1', name: 'FTB' }],
|
||||
updatedAt: new Date('2026-07-08T08:01:00.000Z'),
|
||||
},
|
||||
],
|
||||
{
|
||||
appVersion: 'test-version',
|
||||
appBuildTime: '2026-07-08T08:02:00.000Z',
|
||||
exportedAt: '2026-07-08T08:03:00.000Z',
|
||||
sourceCommit: 'test-commit',
|
||||
},
|
||||
);
|
||||
|
||||
assert.deepEqual(archive.keys, ['dev-tasks', 'products-overview']);
|
||||
assert.equal(archive.metadata.rowCount, 2);
|
||||
assert.equal(archive.metadata.keyCount, 2);
|
||||
assert.match(archive.metadata.payloadChecksum, /^[a-f0-9]{64}$/);
|
||||
|
||||
const verification = verifyAppDataArchive(archive);
|
||||
assert.equal(verification.ok, true);
|
||||
assert.deepEqual(verification.errors, []);
|
||||
});
|
||||
|
||||
test('verification fails when archived row content is tampered', () => {
|
||||
const archive = buildAppDataArchive(
|
||||
[
|
||||
{
|
||||
key: 'dev-tasks',
|
||||
value: [{ id: 'dt-1', title: '开发任务' }],
|
||||
updatedAt: new Date('2026-07-08T08:00:00.000Z'),
|
||||
},
|
||||
],
|
||||
{
|
||||
appVersion: 'test-version',
|
||||
appBuildTime: '2026-07-08T08:02:00.000Z',
|
||||
exportedAt: '2026-07-08T08:03:00.000Z',
|
||||
sourceCommit: 'test-commit',
|
||||
},
|
||||
);
|
||||
archive.rows[0].value[0].title = '被篡改';
|
||||
|
||||
const verification = verifyAppDataArchive(archive);
|
||||
assert.equal(verification.ok, false);
|
||||
assert.match(verification.errors.join('\n'), /checksum/i);
|
||||
});
|
||||
87
scripts/export-appdata-archive.mjs
Normal file
87
scripts/export-appdata-archive.mjs
Normal file
@@ -0,0 +1,87 @@
|
||||
#!/usr/bin/env node
|
||||
import { execFileSync } from 'node:child_process';
|
||||
import { mkdirSync, readFileSync, writeFileSync } from 'node:fs';
|
||||
import { createRequire } from 'node:module';
|
||||
import { dirname, resolve } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { buildAppDataArchive } from './lib/appdata-archive.mjs';
|
||||
|
||||
const root = dirname(fileURLToPath(new URL('../package.json', import.meta.url)));
|
||||
const args = parseArgs(process.argv.slice(2));
|
||||
|
||||
if (args.help) {
|
||||
printHelp();
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
const outPath = resolve(root, args.out ?? defaultArchiveName());
|
||||
const serverRequire = createRequire(resolve(root, 'apps/server/package.json'));
|
||||
const { PrismaClient } = serverRequire('@prisma/client');
|
||||
const prisma = new PrismaClient();
|
||||
|
||||
try {
|
||||
const rows = await prisma.appData.findMany({
|
||||
orderBy: { key: 'asc' },
|
||||
select: { key: true, value: true, updatedAt: true },
|
||||
});
|
||||
const archive = buildAppDataArchive(rows, {
|
||||
appVersion: process.env.APP_VERSION || readPackageVersion(),
|
||||
appBuildTime: process.env.APP_BUILD_TIME || null,
|
||||
sourceCommit: process.env.GITHUB_SHA || readGitCommit(),
|
||||
});
|
||||
|
||||
mkdirSync(dirname(outPath), { recursive: true });
|
||||
writeFileSync(outPath, `${JSON.stringify(archive, null, 2)}\n`);
|
||||
console.log(`AppData archive exported: ${outPath}`);
|
||||
console.log(`Rows: ${archive.metadata.rowCount}; keys: ${archive.keys.join(', ') || '(none)'}`);
|
||||
console.log(`Payload checksum: ${archive.metadata.payloadChecksum}`);
|
||||
} finally {
|
||||
await prisma.$disconnect();
|
||||
}
|
||||
|
||||
function parseArgs(argv) {
|
||||
const parsed = { help: false, out: null };
|
||||
for (let index = 0; index < argv.length; index += 1) {
|
||||
const arg = argv[index];
|
||||
if (arg === '--') continue;
|
||||
else if (arg === '--help' || arg === '-h') parsed.help = true;
|
||||
else if (arg === '--out') parsed.out = argv[++index];
|
||||
else if (arg.startsWith('--out=')) parsed.out = arg.slice('--out='.length);
|
||||
else throw new Error(`Unknown argument: ${arg}`);
|
||||
}
|
||||
return parsed;
|
||||
}
|
||||
|
||||
function defaultArchiveName() {
|
||||
const stamp = new Date().toISOString().replace(/[:.]/g, '-');
|
||||
return `appdata-archive-${stamp}.json`;
|
||||
}
|
||||
|
||||
function readPackageVersion() {
|
||||
try {
|
||||
const pkg = JSON.parse(readFileSync(resolve(root, 'apps/server/package.json'), 'utf8'));
|
||||
return pkg.version || 'unknown';
|
||||
} catch {
|
||||
return 'unknown';
|
||||
}
|
||||
}
|
||||
|
||||
function readGitCommit() {
|
||||
try {
|
||||
return execFileSync('git', ['rev-parse', 'HEAD'], { cwd: root, encoding: 'utf8' }).trim();
|
||||
} catch {
|
||||
return 'unknown';
|
||||
}
|
||||
}
|
||||
|
||||
function printHelp() {
|
||||
console.log(`Usage: pnpm appdata:archive:export -- --out backups/appdata-archive.json
|
||||
|
||||
Exports every row from app_data to a JSON archive with key list, app version metadata,
|
||||
per-row SHA-256 checksums, and a payload checksum.
|
||||
|
||||
Options:
|
||||
--out <path> Output archive path. Defaults to appdata-archive-<timestamp>.json.
|
||||
-h, --help Show this help.
|
||||
`);
|
||||
}
|
||||
161
scripts/lib/appdata-archive.mjs
Normal file
161
scripts/lib/appdata-archive.mjs
Normal file
@@ -0,0 +1,161 @@
|
||||
import { createHash } from 'node:crypto';
|
||||
|
||||
export const APPDATA_ARCHIVE_VERSION = 1;
|
||||
export const APPDATA_ARCHIVE_CHECKSUM_ALGORITHM = 'sha256';
|
||||
|
||||
export function stableStringify(value) {
|
||||
return JSON.stringify(sortJsonValue(value));
|
||||
}
|
||||
|
||||
export function sha256Hex(value) {
|
||||
return createHash('sha256').update(String(value)).digest('hex');
|
||||
}
|
||||
|
||||
export function buildAppDataArchive(rows, options = {}) {
|
||||
const normalizedRows = [...rows]
|
||||
.sort((a, b) => String(a.key).localeCompare(String(b.key)))
|
||||
.map((row) => {
|
||||
const canonicalValue = stableStringify(row.value);
|
||||
return {
|
||||
key: String(row.key),
|
||||
value: row.value,
|
||||
version: toIsoString(row.updatedAt),
|
||||
valueChecksum: sha256Hex(canonicalValue),
|
||||
bytes: Buffer.byteLength(canonicalValue),
|
||||
};
|
||||
});
|
||||
const keys = normalizedRows.map((row) => row.key);
|
||||
const metadata = {
|
||||
archiveVersion: APPDATA_ARCHIVE_VERSION,
|
||||
exportedAt: options.exportedAt ?? new Date().toISOString(),
|
||||
appVersion: options.appVersion ?? 'unknown',
|
||||
appBuildTime: options.appBuildTime ?? null,
|
||||
sourceCommit: options.sourceCommit ?? 'unknown',
|
||||
rowCount: normalizedRows.length,
|
||||
keyCount: keys.length,
|
||||
checksumAlgorithm: APPDATA_ARCHIVE_CHECKSUM_ALGORITHM,
|
||||
keyListChecksum: sha256Hex(stableStringify(keys)),
|
||||
payloadChecksum: '',
|
||||
};
|
||||
const archive = { metadata, keys, rows: normalizedRows };
|
||||
archive.metadata.payloadChecksum = checksumPayload(archive);
|
||||
return archive;
|
||||
}
|
||||
|
||||
export function verifyAppDataArchive(archive) {
|
||||
const errors = [];
|
||||
if (!archive || typeof archive !== 'object') {
|
||||
return { ok: false, errors: ['Archive must be a JSON object'], summary: emptySummary() };
|
||||
}
|
||||
|
||||
const metadata = archive.metadata;
|
||||
const keys = Array.isArray(archive.keys) ? archive.keys : [];
|
||||
const rows = Array.isArray(archive.rows) ? archive.rows : [];
|
||||
|
||||
if (!metadata || typeof metadata !== 'object') {
|
||||
errors.push('Missing metadata object');
|
||||
} else {
|
||||
if (metadata.archiveVersion !== APPDATA_ARCHIVE_VERSION) {
|
||||
errors.push(`Unsupported archiveVersion: ${metadata.archiveVersion}`);
|
||||
}
|
||||
if (metadata.checksumAlgorithm !== APPDATA_ARCHIVE_CHECKSUM_ALGORITHM) {
|
||||
errors.push(`Unsupported checksumAlgorithm: ${metadata.checksumAlgorithm}`);
|
||||
}
|
||||
if (metadata.rowCount !== rows.length) {
|
||||
errors.push(`rowCount mismatch: metadata=${metadata.rowCount}, actual=${rows.length}`);
|
||||
}
|
||||
if (metadata.keyCount !== keys.length) {
|
||||
errors.push(`keyCount mismatch: metadata=${metadata.keyCount}, actual=${keys.length}`);
|
||||
}
|
||||
}
|
||||
|
||||
const rowKeys = rows.map((row) => row?.key);
|
||||
if (stableStringify(keys) !== stableStringify(rowKeys)) {
|
||||
errors.push('Key list does not match row keys');
|
||||
}
|
||||
|
||||
const keyListChecksum = sha256Hex(stableStringify(keys));
|
||||
if (metadata?.keyListChecksum !== keyListChecksum) {
|
||||
errors.push('Key list checksum mismatch');
|
||||
}
|
||||
|
||||
for (const row of rows) {
|
||||
if (!row || typeof row !== 'object') {
|
||||
errors.push('Archive contains a non-object row');
|
||||
continue;
|
||||
}
|
||||
if (typeof row.key !== 'string') {
|
||||
errors.push('Archive row is missing string key');
|
||||
}
|
||||
if (typeof row.version !== 'string') {
|
||||
errors.push(`Archive row ${row.key ?? '<unknown>'} is missing string version`);
|
||||
}
|
||||
const canonicalValue = stableStringify(row.value);
|
||||
const valueChecksum = sha256Hex(canonicalValue);
|
||||
if (row.valueChecksum !== valueChecksum) {
|
||||
errors.push(`Value checksum mismatch for ${row.key}`);
|
||||
}
|
||||
const bytes = Buffer.byteLength(canonicalValue);
|
||||
if (row.bytes !== bytes) {
|
||||
errors.push(`Byte size mismatch for ${row.key}`);
|
||||
}
|
||||
}
|
||||
|
||||
const payloadChecksum = checksumPayload({ keys, rows });
|
||||
if (metadata?.payloadChecksum !== payloadChecksum) {
|
||||
errors.push('Payload checksum mismatch');
|
||||
}
|
||||
|
||||
return {
|
||||
ok: errors.length === 0,
|
||||
errors,
|
||||
summary: {
|
||||
archiveVersion: metadata?.archiveVersion ?? null,
|
||||
exportedAt: metadata?.exportedAt ?? null,
|
||||
appVersion: metadata?.appVersion ?? null,
|
||||
sourceCommit: metadata?.sourceCommit ?? null,
|
||||
rowCount: rows.length,
|
||||
keyCount: keys.length,
|
||||
keys,
|
||||
payloadChecksum,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export function checksumPayload(archive) {
|
||||
return sha256Hex(stableStringify({
|
||||
keys: archive.keys,
|
||||
rows: archive.rows,
|
||||
}));
|
||||
}
|
||||
|
||||
function sortJsonValue(value) {
|
||||
if (Array.isArray(value)) return value.map(sortJsonValue);
|
||||
if (!value || typeof value !== 'object') return value;
|
||||
return Object.fromEntries(
|
||||
Object.entries(value)
|
||||
.filter(([, item]) => item !== undefined)
|
||||
.sort(([a], [b]) => a.localeCompare(b))
|
||||
.map(([key, item]) => [key, sortJsonValue(item)]),
|
||||
);
|
||||
}
|
||||
|
||||
function toIsoString(value) {
|
||||
if (value instanceof Date) return value.toISOString();
|
||||
const date = new Date(value);
|
||||
if (Number.isFinite(date.getTime())) return date.toISOString();
|
||||
throw new Error(`Invalid AppData updatedAt value: ${value}`);
|
||||
}
|
||||
|
||||
function emptySummary() {
|
||||
return {
|
||||
archiveVersion: null,
|
||||
exportedAt: null,
|
||||
appVersion: null,
|
||||
sourceCommit: null,
|
||||
rowCount: 0,
|
||||
keyCount: 0,
|
||||
keys: [],
|
||||
payloadChecksum: null,
|
||||
};
|
||||
}
|
||||
57
scripts/verify-appdata-archive.mjs
Normal file
57
scripts/verify-appdata-archive.mjs
Normal file
@@ -0,0 +1,57 @@
|
||||
#!/usr/bin/env node
|
||||
import { readFileSync } from 'node:fs';
|
||||
import { resolve } from 'node:path';
|
||||
import { verifyAppDataArchive } from './lib/appdata-archive.mjs';
|
||||
|
||||
const args = parseArgs(process.argv.slice(2));
|
||||
|
||||
if (args.help || !args.archive) {
|
||||
printHelp();
|
||||
process.exit(args.help ? 0 : 1);
|
||||
}
|
||||
|
||||
const archivePath = resolve(process.cwd(), args.archive);
|
||||
const archive = JSON.parse(readFileSync(archivePath, 'utf8'));
|
||||
const verification = verifyAppDataArchive(archive);
|
||||
|
||||
if (args.json) {
|
||||
console.log(JSON.stringify(verification, null, 2));
|
||||
} else if (verification.ok) {
|
||||
console.log(`AppData archive verified: ${archivePath}`);
|
||||
console.log(`Rows: ${verification.summary.rowCount}; keys: ${verification.summary.keys.join(', ') || '(none)'}`);
|
||||
console.log(`Payload checksum: ${verification.summary.payloadChecksum}`);
|
||||
} else {
|
||||
console.error(`AppData archive verification failed: ${archivePath}`);
|
||||
for (const error of verification.errors) {
|
||||
console.error(`- ${error}`);
|
||||
}
|
||||
}
|
||||
|
||||
process.exit(verification.ok ? 0 : 1);
|
||||
|
||||
function parseArgs(argv) {
|
||||
const parsed = { help: false, json: false, archive: null };
|
||||
for (let index = 0; index < argv.length; index += 1) {
|
||||
const arg = argv[index];
|
||||
if (arg === '--') continue;
|
||||
else if (arg === '--help' || arg === '-h') parsed.help = true;
|
||||
else if (arg === '--json') parsed.json = true;
|
||||
else if (arg === '--archive') parsed.archive = argv[++index];
|
||||
else if (arg.startsWith('--archive=')) parsed.archive = arg.slice('--archive='.length);
|
||||
else if (!arg.startsWith('-') && !parsed.archive) parsed.archive = arg;
|
||||
else throw new Error(`Unknown argument: ${arg}`);
|
||||
}
|
||||
return parsed;
|
||||
}
|
||||
|
||||
function printHelp() {
|
||||
console.log(`Usage: pnpm appdata:archive:verify -- --archive backups/appdata-archive.json
|
||||
|
||||
Verifies key list, per-row SHA-256 checksums, and the archive payload checksum.
|
||||
|
||||
Options:
|
||||
--archive <path> Archive JSON path. A positional path is also accepted.
|
||||
--json Print machine-readable verification output.
|
||||
-h, --help Show this help.
|
||||
`);
|
||||
}
|
||||
Reference in New Issue
Block a user