feat(ops): 添加生产监控告警基线

- 新增 Prometheus/Grafana/Loki/Promtail 监控 profile\n- 覆盖 DB、磁盘、慢 API、慢 Prisma、任务失败和小宝摘要 stale 告警\n- 补充 postgres-exporter 自定义查询、Dashboard、部署文档和校验\n\nCo-Authored-By: GPT-5 Codex <codex@openai.com>
This commit is contained in:
2026-07-08 16:21:07 +08:00
parent eef09d5af4
commit 7837a809ca
15 changed files with 587 additions and 1 deletions

View File

@@ -1,5 +1,5 @@
import { spawnSync } from 'node:child_process';
import { mkdtempSync, writeFileSync, existsSync } from 'node:fs';
import { mkdtempSync, writeFileSync, existsSync, readFileSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { join, resolve } from 'node:path';
import { describe, it } from 'node:test';
@@ -97,4 +97,45 @@ describe('production ops scripts', () => {
assert.match(result.stdout, /\/api\/v1\/v2\.2\/requirements\?productId=__smoke__/);
assert.match(result.stdout, /\/api\/v1\/config\/ai/);
});
it('declares the production monitoring baseline without real alert secrets', () => {
const alertRules = readFileSync(
join(root, 'deploy/monitoring/prometheus/alert-rules.yml'),
'utf8',
);
const postgresQueries = readFileSync(
join(root, 'deploy/monitoring/postgres/postgres-queries.yml'),
'utf8',
);
const promtailConfig = readFileSync(
join(root, 'deploy/monitoring/promtail/config.yml'),
'utf8',
);
const dashboard = readFileSync(
join(root, 'deploy/monitoring/grafana/dashboards/ftb-production-overview.json'),
'utf8',
);
for (const alertName of [
'FtbPostgresDown',
'FtbDiskPressure',
'FtbSlowApiLogBurst',
'FtbSlowPrismaLogBurst',
'FtbJobFailureLogBurst',
'FtbXiaobaoSummaryStale',
]) {
assert.match(alertRules, new RegExp(alertName));
}
assert.match(postgresQueries, /xiaobao_risk_summaries/);
assert.match(postgresQueries, /dirty = true/);
assert.match(promtailConfig, /Slow API request/);
assert.match(promtailConfig, /Slow Prisma query/);
assert.match(promtailConfig, /AppData relation sync failed/);
assert.match(dashboard, /FTB Production Overview/);
const combined = `${alertRules}\n${postgresQueries}\n${promtailConfig}\n${dashboard}`;
assert.doesNotMatch(combined, /sk-ant-[A-Za-z0-9]/);
assert.doesNotMatch(combined, /hooks\.slack\.com\/services\//);
});
});

View File

@@ -51,6 +51,11 @@ const checks = [
'WEB_IMAGE',
'APP_VERSION',
'/api/v1/health/version',
'prometheus:',
"profiles: ['monitoring']",
'postgres-exporter:',
'blackbox-exporter:',
'prometheus_data:',
],
},
{
@@ -110,6 +115,41 @@ const checks = [
'X-Forwarded-Proto',
],
},
{
file: 'deploy/monitoring/README.md',
snippets: [
'--profile monitoring',
'FtbPostgresDown',
'FtbXiaobaoSummaryStale',
'no webhook URLs',
],
},
{
file: 'deploy/monitoring/prometheus/prometheus.yml',
snippets: ['postgres-exporter:9187', 'promtail:9080', 'blackbox-http'],
},
{
file: 'deploy/monitoring/prometheus/alert-rules.yml',
snippets: [
'FtbPostgresDown',
'FtbSlowApiLogBurst',
'FtbSlowPrismaLogBurst',
'FtbJobFailureLogBurst',
'FtbXiaobaoSummaryStale',
],
},
{
file: 'deploy/monitoring/postgres/postgres-queries.yml',
snippets: ['xiaobao_risk_summaries', 'dirty = true', 'stale_summary_count'],
},
{
file: 'deploy/monitoring/promtail/config.yml',
snippets: ['Slow API request', 'Slow Prisma query', 'AppData relation sync failed'],
},
{
file: 'deploy/monitoring/grafana/dashboards/ftb-production-overview.json',
snippets: ['FTB Production Overview', 'ftb_xiaobao_stale_summary_count', 'Server Warning/Error Logs'],
},
{
file: '.env.production.example',
snippets: [
@@ -143,6 +183,7 @@ const checks = [
'pnpm backup:postgres',
'pnpm restore:postgres',
'pnpm backup:server-data',
'--profile monitoring',
'pnpm db:migrate',
'NEXT_PUBLIC_API_URL',
'Nginx',