feat(ops): 添加生产监控告警基线
- 新增 Prometheus/Grafana/Loki/Promtail 监控 profile\n- 覆盖 DB、磁盘、慢 API、慢 Prisma、任务失败和小宝摘要 stale 告警\n- 补充 postgres-exporter 自定义查询、Dashboard、部署文档和校验\n\nCo-Authored-By: GPT-5 Codex <codex@openai.com>
This commit is contained in:
@@ -1,5 +1,5 @@
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import { mkdtempSync, writeFileSync, existsSync } from 'node:fs';
|
||||
import { mkdtempSync, writeFileSync, existsSync, readFileSync } from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join, resolve } from 'node:path';
|
||||
import { describe, it } from 'node:test';
|
||||
@@ -97,4 +97,45 @@ describe('production ops scripts', () => {
|
||||
assert.match(result.stdout, /\/api\/v1\/v2\.2\/requirements\?productId=__smoke__/);
|
||||
assert.match(result.stdout, /\/api\/v1\/config\/ai/);
|
||||
});
|
||||
|
||||
it('declares the production monitoring baseline without real alert secrets', () => {
|
||||
const alertRules = readFileSync(
|
||||
join(root, 'deploy/monitoring/prometheus/alert-rules.yml'),
|
||||
'utf8',
|
||||
);
|
||||
const postgresQueries = readFileSync(
|
||||
join(root, 'deploy/monitoring/postgres/postgres-queries.yml'),
|
||||
'utf8',
|
||||
);
|
||||
const promtailConfig = readFileSync(
|
||||
join(root, 'deploy/monitoring/promtail/config.yml'),
|
||||
'utf8',
|
||||
);
|
||||
const dashboard = readFileSync(
|
||||
join(root, 'deploy/monitoring/grafana/dashboards/ftb-production-overview.json'),
|
||||
'utf8',
|
||||
);
|
||||
|
||||
for (const alertName of [
|
||||
'FtbPostgresDown',
|
||||
'FtbDiskPressure',
|
||||
'FtbSlowApiLogBurst',
|
||||
'FtbSlowPrismaLogBurst',
|
||||
'FtbJobFailureLogBurst',
|
||||
'FtbXiaobaoSummaryStale',
|
||||
]) {
|
||||
assert.match(alertRules, new RegExp(alertName));
|
||||
}
|
||||
|
||||
assert.match(postgresQueries, /xiaobao_risk_summaries/);
|
||||
assert.match(postgresQueries, /dirty = true/);
|
||||
assert.match(promtailConfig, /Slow API request/);
|
||||
assert.match(promtailConfig, /Slow Prisma query/);
|
||||
assert.match(promtailConfig, /AppData relation sync failed/);
|
||||
assert.match(dashboard, /FTB Production Overview/);
|
||||
|
||||
const combined = `${alertRules}\n${postgresQueries}\n${promtailConfig}\n${dashboard}`;
|
||||
assert.doesNotMatch(combined, /sk-ant-[A-Za-z0-9]/);
|
||||
assert.doesNotMatch(combined, /hooks\.slack\.com\/services\//);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -51,6 +51,11 @@ const checks = [
|
||||
'WEB_IMAGE',
|
||||
'APP_VERSION',
|
||||
'/api/v1/health/version',
|
||||
'prometheus:',
|
||||
"profiles: ['monitoring']",
|
||||
'postgres-exporter:',
|
||||
'blackbox-exporter:',
|
||||
'prometheus_data:',
|
||||
],
|
||||
},
|
||||
{
|
||||
@@ -110,6 +115,41 @@ const checks = [
|
||||
'X-Forwarded-Proto',
|
||||
],
|
||||
},
|
||||
{
|
||||
file: 'deploy/monitoring/README.md',
|
||||
snippets: [
|
||||
'--profile monitoring',
|
||||
'FtbPostgresDown',
|
||||
'FtbXiaobaoSummaryStale',
|
||||
'no webhook URLs',
|
||||
],
|
||||
},
|
||||
{
|
||||
file: 'deploy/monitoring/prometheus/prometheus.yml',
|
||||
snippets: ['postgres-exporter:9187', 'promtail:9080', 'blackbox-http'],
|
||||
},
|
||||
{
|
||||
file: 'deploy/monitoring/prometheus/alert-rules.yml',
|
||||
snippets: [
|
||||
'FtbPostgresDown',
|
||||
'FtbSlowApiLogBurst',
|
||||
'FtbSlowPrismaLogBurst',
|
||||
'FtbJobFailureLogBurst',
|
||||
'FtbXiaobaoSummaryStale',
|
||||
],
|
||||
},
|
||||
{
|
||||
file: 'deploy/monitoring/postgres/postgres-queries.yml',
|
||||
snippets: ['xiaobao_risk_summaries', 'dirty = true', 'stale_summary_count'],
|
||||
},
|
||||
{
|
||||
file: 'deploy/monitoring/promtail/config.yml',
|
||||
snippets: ['Slow API request', 'Slow Prisma query', 'AppData relation sync failed'],
|
||||
},
|
||||
{
|
||||
file: 'deploy/monitoring/grafana/dashboards/ftb-production-overview.json',
|
||||
snippets: ['FTB Production Overview', 'ftb_xiaobao_stale_summary_count', 'Server Warning/Error Logs'],
|
||||
},
|
||||
{
|
||||
file: '.env.production.example',
|
||||
snippets: [
|
||||
@@ -143,6 +183,7 @@ const checks = [
|
||||
'pnpm backup:postgres',
|
||||
'pnpm restore:postgres',
|
||||
'pnpm backup:server-data',
|
||||
'--profile monitoring',
|
||||
'pnpm db:migrate',
|
||||
'NEXT_PUBLIC_API_URL',
|
||||
'Nginx',
|
||||
|
||||
Reference in New Issue
Block a user