feat(ops): 添加生产监控告警基线

- 新增 Prometheus/Grafana/Loki/Promtail 监控 profile\n- 覆盖 DB、磁盘、慢 API、慢 Prisma、任务失败和小宝摘要 stale 告警\n- 补充 postgres-exporter 自定义查询、Dashboard、部署文档和校验\n\nCo-Authored-By: GPT-5 Codex <codex@openai.com>

.env.production.example

@@ -39,3 +39,10 @@ SMTP_HOST=
 SMTP_PORT=465
 SMTP_USER=
 SMTP_PASS=
+
+# Optional monitoring profile. Do not commit real production passwords.
+PROMETHEUS_PORT=9090
+PROMETHEUS_RETENTION=15d
+GRAFANA_PORT=3002
+GRAFANA_ADMIN_USER=admin
+GRAFANA_ADMIN_PASSWORD=change-me-monitoring-password

deploy/monitoring/README.md

@@ -0,0 +1,43 @@
+# FTB Production Monitoring Baseline
+
+This profile adds a deployable Prometheus/Grafana baseline for production operations. It is intentionally secret-free: no webhook URLs, API keys, SMTP passwords, or real alert receiver credentials are committed.
+
+## Start
+
+```bash
+docker compose --env-file .env.production -f docker-compose.prod.yml --profile monitoring up -d
+```
+
+Default local ports:
+
+- Prometheus: `http://localhost:9090`
+- Grafana: `http://localhost:3002`
+- Loki: internal only
+
+Set `GRAFANA_ADMIN_USER` and `GRAFANA_ADMIN_PASSWORD` in `.env.production` before exposing Grafana beyond localhost. Keep real alert receivers in the server environment or an untracked Alertmanager file.
+
+## Coverage
+
+- DB availability: `pg_up` from postgres-exporter.
+- Disk pressure: root filesystem availability from node-exporter.
+- Slow API requests: Promtail turns `Slow API request` server logs into `ftb_slow_api_log_total`.
+- Slow Prisma queries: Promtail turns `Slow Prisma query` server logs into `ftb_slow_prisma_log_total`.
+- Job failures: Promtail turns `AppData relation sync failed` and AI call failure logs into `ftb_job_failure_log_total`.
+- Xiaobao stale summaries: postgres-exporter custom query exposes `ftb_xiaobao_stale_summary_count` from `xiaobao_risk_summaries`.
+
+## Alerts
+
+Prometheus loads `prometheus/alert-rules.yml`. The rules evaluate locally and are visible in Prometheus/Grafana. To send notifications, add Alertmanager outside git or mount an environment-specific receiver file; do not commit webhook URLs or tokens.
+
+Baseline alert names:
+
+- `FtbPostgresDown`
+- `FtbDiskPressure`
+- `FtbSlowApiLogBurst`
+- `FtbSlowPrismaLogBurst`
+- `FtbJobFailureLogBurst`
+- `FtbXiaobaoSummaryStale`
+
+## Log Search
+
+Promtail ships Docker logs to Loki with container labels. Grafana provisions both Prometheus and Loki data sources, so on-call checks can move from a firing alert to matching server logs without SSHing into the host.

deploy/monitoring/blackbox/config.yml

@@ -0,0 +1,10 @@
+modules:
+  http_2xx:
+    prober: http
+    timeout: 5s
+    http:
+      valid_http_versions: ['HTTP/1.1', 'HTTP/2.0']
+      valid_status_codes: []
+      method: GET
+      preferred_ip_protocol: ip4
+

deploy/monitoring/grafana/dashboards/ftb-production-overview.json

@@ -0,0 +1,78 @@
+{
+  "uid": "ftb-production-overview",
+  "title": "FTB Production Overview",
+  "schemaVersion": 39,
+  "version": 1,
+  "refresh": "30s",
+  "tags": ["ftb", "production", "v2.8"],
+  "time": {
+    "from": "now-6h",
+    "to": "now"
+  },
+  "panels": [
+    {
+      "id": 1,
+      "type": "stat",
+      "title": "DB Availability",
+      "gridPos": { "x": 0, "y": 0, "w": 6, "h": 4 },
+      "targets": [
+        { "datasource": { "type": "prometheus", "uid": "Prometheus" }, "expr": "pg_up", "refId": "A" }
+      ]
+    },
+    {
+      "id": 2,
+      "type": "timeseries",
+      "title": "Slow API Logs",
+      "gridPos": { "x": 6, "y": 0, "w": 6, "h": 4 },
+      "targets": [
+        { "datasource": { "type": "prometheus", "uid": "Prometheus" }, "expr": "increase(ftb_slow_api_log_total[10m])", "refId": "A" }
+      ]
+    },
+    {
+      "id": 3,
+      "type": "timeseries",
+      "title": "Slow Prisma Logs",
+      "gridPos": { "x": 12, "y": 0, "w": 6, "h": 4 },
+      "targets": [
+        { "datasource": { "type": "prometheus", "uid": "Prometheus" }, "expr": "increase(ftb_slow_prisma_log_total[10m])", "refId": "A" }
+      ]
+    },
+    {
+      "id": 4,
+      "type": "stat",
+      "title": "Xiaobao Stale Summaries",
+      "gridPos": { "x": 18, "y": 0, "w": 6, "h": 4 },
+      "targets": [
+        { "datasource": { "type": "prometheus", "uid": "Prometheus" }, "expr": "ftb_xiaobao_stale_summary_count", "refId": "A" }
+      ]
+    },
+    {
+      "id": 5,
+      "type": "timeseries",
+      "title": "Job Failure Logs",
+      "gridPos": { "x": 0, "y": 4, "w": 8, "h": 5 },
+      "targets": [
+        { "datasource": { "type": "prometheus", "uid": "Prometheus" }, "expr": "increase(ftb_job_failure_log_total[10m])", "refId": "A" }
+      ]
+    },
+    {
+      "id": 6,
+      "type": "stat",
+      "title": "Root Disk Free %",
+      "gridPos": { "x": 8, "y": 4, "w": 8, "h": 5 },
+      "targets": [
+        { "datasource": { "type": "prometheus", "uid": "Prometheus" }, "expr": "100 * node_filesystem_avail_bytes{mountpoint=\"/\",fstype!~\"tmpfs|overlay\"} / node_filesystem_size_bytes{mountpoint=\"/\",fstype!~\"tmpfs|overlay\"}", "refId": "A" }
+      ]
+    },
+    {
+      "id": 7,
+      "type": "logs",
+      "title": "Server Warning/Error Logs",
+      "gridPos": { "x": 16, "y": 4, "w": 8, "h": 5 },
+      "targets": [
+        { "datasource": { "type": "loki", "uid": "Loki" }, "expr": "{compose_service=\"server\"} |~ \"warn|error|失败|Slow\"", "refId": "A" }
+      ]
+    }
+  ]
+}
+

deploy/monitoring/grafana/provisioning/dashboards/dashboards.yml

@@ -0,0 +1,12 @@
+apiVersion: 1
+
+providers:
+  - name: FTB Production
+    orgId: 1
+    folder: FTB
+    type: file
+    disableDeletion: false
+    updateIntervalSeconds: 30
+    options:
+      path: /var/lib/grafana/dashboards
+

deploy/monitoring/grafana/provisioning/datasources/datasources.yml

@@ -0,0 +1,16 @@
+apiVersion: 1
+
+datasources:
+  - name: Prometheus
+    type: prometheus
+    access: proxy
+    url: http://prometheus:9090
+    isDefault: true
+    editable: false
+
+  - name: Loki
+    type: loki
+    access: proxy
+    url: http://loki:3100
+    editable: false
+

deploy/monitoring/loki/config.yml

@@ -0,0 +1,32 @@
+auth_enabled: false
+
+server:
+  http_listen_port: 3100
+
+common:
+  path_prefix: /loki
+  replication_factor: 1
+  ring:
+    kvstore:
+      store: inmemory
+
+schema_config:
+  configs:
+    - from: 2026-01-01
+      store: tsdb
+      object_store: filesystem
+      schema: v13
+      index:
+        prefix: index_
+        period: 24h
+
+storage_config:
+  tsdb_shipper:
+    active_index_directory: /loki/index
+    cache_location: /loki/index_cache
+  filesystem:
+    directory: /loki/chunks
+
+limits_config:
+  retention_period: 168h
+

deploy/monitoring/postgres/postgres-queries.yml

@@ -0,0 +1,12 @@
+ftb_xiaobao:
+  query: |
+    SELECT
+      count(*)::float AS stale_summary_count
+    FROM xiaobao_risk_summaries
+    WHERE dirty = true
+       OR updated_at < now() - interval '6 hours';
+  metrics:
+    - stale_summary_count:
+        usage: GAUGE
+        description: Xiaobao risk summaries that are dirty or older than 6 hours.
+

deploy/monitoring/prometheus/alert-rules.yml

@@ -0,0 +1,62 @@
+groups:
+  - name: ftb-production-alerts
+    rules:
+      - alert: FtbPostgresDown
+        expr: pg_up == 0
+        for: 2m
+        labels:
+          severity: critical
+        annotations:
+          summary: PostgreSQL exporter cannot reach the FTB database.
+          runbook: docs/runbooks/migration-rollback.md
+
+      - alert: FtbDiskPressure
+        expr: |
+          (
+            node_filesystem_avail_bytes{mountpoint="/",fstype!~"tmpfs|overlay"}
+            /
+            node_filesystem_size_bytes{mountpoint="/",fstype!~"tmpfs|overlay"}
+          ) < 0.15
+        for: 10m
+        labels:
+          severity: warning
+        annotations:
+          summary: Production host root filesystem has less than 15% free space.
+          runbook: docs/deployment.md
+
+      - alert: FtbSlowApiLogBurst
+        expr: increase(ftb_slow_api_log_total[10m]) > 5
+        for: 2m
+        labels:
+          severity: warning
+        annotations:
+          summary: Slow API request log volume crossed the V2.8 baseline threshold.
+          runbook: docs/deployment.md
+
+      - alert: FtbSlowPrismaLogBurst
+        expr: increase(ftb_slow_prisma_log_total[10m]) > 3
+        for: 2m
+        labels:
+          severity: warning
+        annotations:
+          summary: Slow Prisma query log volume crossed the V2.8 baseline threshold.
+          runbook: docs/deployment.md
+
+      - alert: FtbJobFailureLogBurst
+        expr: increase(ftb_job_failure_log_total[10m]) > 0
+        for: 1m
+        labels:
+          severity: warning
+        annotations:
+          summary: Background or compatibility job failure logs were detected.
+          runbook: docs/runbooks/xiaobao-background-jobs.md
+
+      - alert: FtbXiaobaoSummaryStale
+        expr: ftb_xiaobao_stale_summary_count > 0
+        for: 15m
+        labels:
+          severity: warning
+        annotations:
+          summary: Xiaobao warning summaries are dirty or stale.
+          runbook: docs/runbooks/xiaobao-background-jobs.md
+

deploy/monitoring/prometheus/prometheus.yml

@@ -0,0 +1,45 @@
+global:
+  scrape_interval: 15s
+  evaluation_interval: 15s
+
+rule_files:
+  - /etc/prometheus/alert-rules.yml
+
+scrape_configs:
+  - job_name: prometheus
+    static_configs:
+      - targets: ['prometheus:9090']
+
+  - job_name: postgres-exporter
+    static_configs:
+      - targets: ['postgres-exporter:9187']
+
+  - job_name: node-exporter
+    static_configs:
+      - targets: ['node-exporter:9100']
+
+  - job_name: cadvisor
+    static_configs:
+      - targets: ['cadvisor:8080']
+
+  - job_name: promtail
+    static_configs:
+      - targets: ['promtail:9080']
+
+  - job_name: blackbox-http
+    metrics_path: /probe
+    params:
+      module: [http_2xx]
+    static_configs:
+      - targets:
+          - http://nginx/api/v1/health/version
+          - http://nginx/api/v1/config/ai
+          - http://nginx/api/v1/v2.2/requirements?productId=__smoke__&limit=1
+    relabel_configs:
+      - source_labels: [__address__]
+        target_label: __param_target
+      - source_labels: [__param_target]
+        target_label: instance
+      - target_label: __address__
+        replacement: blackbox-exporter:9115
+

deploy/monitoring/promtail/config.yml

@@ -0,0 +1,53 @@
+server:
+  http_listen_port: 9080
+  grpc_listen_port: 0
+
+positions:
+  filename: /tmp/positions.yml
+
+clients:
+  - url: http://loki:3100/loki/api/v1/push
+
+scrape_configs:
+  - job_name: docker
+    docker_sd_configs:
+      - host: unix:///var/run/docker.sock
+        refresh_interval: 15s
+    relabel_configs:
+      - source_labels: ['__meta_docker_container_name']
+        regex: '/(.*)'
+        target_label: container
+      - source_labels: ['__meta_docker_container_label_com_docker_compose_service']
+        target_label: compose_service
+      - source_labels: ['__meta_docker_container_label_com_docker_compose_project']
+        target_label: compose_project
+    pipeline_stages:
+      - docker: {}
+      - match:
+          selector: '{compose_service="server"} |= "Slow API request"'
+          stages:
+            - metrics:
+                ftb_slow_api_log_total:
+                  type: Counter
+                  description: Slow API request log entries emitted by the NestJS server.
+                  config:
+                    action: inc
+      - match:
+          selector: '{compose_service="server"} |= "Slow Prisma query"'
+          stages:
+            - metrics:
+                ftb_slow_prisma_log_total:
+                  type: Counter
+                  description: Slow Prisma query log entries emitted by the NestJS server.
+                  config:
+                    action: inc
+      - match:
+          selector: '{compose_service="server"} |~ "AppData relation sync failed|AI 调用失败|AI 风险解读调用失败"'
+          stages:
+            - metrics:
+                ftb_job_failure_log_total:
+                  type: Counter
+                  description: Compatibility sync, AI job, or background task failure logs.
+                  config:
+                    action: inc
+

docker-compose.prod.yml

@@ -123,7 +123,117 @@ services:
       server:
         condition: service_healthy
 
+  prometheus:
+    image: prom/prometheus:v2.53.1
+    profiles: ['monitoring']
+    restart: unless-stopped
+    command:
+      - '--config.file=/etc/prometheus/prometheus.yml'
+      - '--storage.tsdb.path=/prometheus'
+      - '--storage.tsdb.retention.time=${PROMETHEUS_RETENTION:-15d}'
+      - '--web.enable-lifecycle'
+    ports:
+      - '${PROMETHEUS_PORT:-9090}:9090'
+    volumes:
+      - ./deploy/monitoring/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
+      - ./deploy/monitoring/prometheus/alert-rules.yml:/etc/prometheus/alert-rules.yml:ro
+      - prometheus_data:/prometheus
+    depends_on:
+      - postgres-exporter
+      - node-exporter
+      - cadvisor
+      - promtail
+      - blackbox-exporter
+
+  grafana:
+    image: grafana/grafana:11.1.0
+    profiles: ['monitoring']
+    restart: unless-stopped
+    ports:
+      - '${GRAFANA_PORT:-3002}:3000'
+    environment:
+      GF_SECURITY_ADMIN_USER: ${GRAFANA_ADMIN_USER:-admin}
+      GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD:-change-me-monitoring-password}
+      GF_USERS_ALLOW_SIGN_UP: 'false'
+    volumes:
+      - grafana_data:/var/lib/grafana
+      - ./deploy/monitoring/grafana/provisioning:/etc/grafana/provisioning:ro
+      - ./deploy/monitoring/grafana/dashboards:/var/lib/grafana/dashboards:ro
+    depends_on:
+      - prometheus
+      - loki
+
+  loki:
+    image: grafana/loki:2.9.8
+    profiles: ['monitoring']
+    restart: unless-stopped
+    command: ['-config.file=/etc/loki/config.yml']
+    volumes:
+      - ./deploy/monitoring/loki/config.yml:/etc/loki/config.yml:ro
+      - loki_data:/loki
+
+  promtail:
+    image: grafana/promtail:2.9.8
+    profiles: ['monitoring']
+    restart: unless-stopped
+    command: ['-config.file=/etc/promtail/config.yml']
+    volumes:
+      - ./deploy/monitoring/promtail/config.yml:/etc/promtail/config.yml:ro
+      - /var/lib/docker/containers:/var/lib/docker/containers:ro
+      - /var/run/docker.sock:/var/run/docker.sock:ro
+    depends_on:
+      - loki
+
+  postgres-exporter:
+    image: quay.io/prometheuscommunity/postgres-exporter:v0.15.0
+    profiles: ['monitoring']
+    restart: unless-stopped
+    command:
+      - '--extend.query-path=/etc/postgres-exporter/postgres-queries.yml'
+    environment:
+      DATA_SOURCE_NAME: postgresql://${POSTGRES_USER:-postgres}:${POSTGRES_PASSWORD}@postgres:5432/${POSTGRES_DB:-ftb_pm}?sslmode=disable
+    volumes:
+      - ./deploy/monitoring/postgres/postgres-queries.yml:/etc/postgres-exporter/postgres-queries.yml:ro
+    depends_on:
+      postgres:
+        condition: service_healthy
+
+  node-exporter:
+    image: prom/node-exporter:v1.8.2
+    profiles: ['monitoring']
+    restart: unless-stopped
+    command:
+      - '--path.rootfs=/host'
+    volumes:
+      - /:/host:ro,rslave
+
+  cadvisor:
+    image: gcr.io/cadvisor/cadvisor:v0.49.1
+    profiles: ['monitoring']
+    restart: unless-stopped
+    privileged: true
+    devices:
+      - /dev/kmsg:/dev/kmsg
+    volumes:
+      - /:/rootfs:ro
+      - /var/run:/var/run:ro
+      - /sys:/sys:ro
+      - /var/lib/docker/:/var/lib/docker:ro
+      - /dev/disk/:/dev/disk:ro
+
+  blackbox-exporter:
+    image: prom/blackbox-exporter:v0.25.0
+    profiles: ['monitoring']
+    restart: unless-stopped
+    command:
+      - '--config.file=/etc/blackbox/config.yml'
+    volumes:
+      - ./deploy/monitoring/blackbox/config.yml:/etc/blackbox/config.yml:ro
+
 volumes:
   postgres_data:
   redis_data:
   server_data:
+  prometheus_data:
+  grafana_data:
+  loki_data:

docs/deployment.md

@@ -194,6 +194,30 @@ pnpm deploy:smoke -- --base-url http://localhost --expected-version <expected-co
 
 生产 Compose 默认只监听 HTTP 80。HTTPS 建议优先交给云负载均衡、CDN 或宿主机外层证书管理工具；如果要让本 Compose 内的 Nginx 直接处理 HTTPS，可以在后续增加证书 volume 和 443 server block。
 
+## 监控与告警基线
+
+V2.8 提供可选 `monitoring` profile，不影响默认生产启动。启用前先在 `.env.production` 设置 `GRAFANA_ADMIN_PASSWORD`，不要使用示例密码对外暴露 Grafana。
+
+```bash
+docker compose --env-file .env.production -f docker-compose.prod.yml --profile monitoring up -d
+```
+
+默认入口：
+
+- Prometheus: `http://localhost:9090`
+- Grafana: `http://localhost:3002`
+
+配置目录在 `deploy/monitoring/`。基线覆盖：
+
+- `FtbPostgresDown`：PostgreSQL 不可用。
+- `FtbDiskPressure`：宿主机根分区低于 15% 可用空间。
+- `FtbSlowApiLogBurst`：服务端慢 API 日志在 10 分钟内超过阈值。
+- `FtbSlowPrismaLogBurst`：慢 Prisma 查询日志在 10 分钟内超过阈值。
+- `FtbJobFailureLogBurst`：AppData 同步、AI 调用或后台任务失败日志出现。
+- `FtbXiaobaoSummaryStale`：`xiaobao_risk_summaries` 存在 dirty 或超过 6 小时未更新的摘要。
+
+Prometheus 只加载本地规则，不提交真实通知密钥。接入 Slack、企业微信、邮件等通知时，把 Alertmanager receiver 放在未跟踪的服务器文件或环境变量中。
+
 ## 数据持久化
 
 生产 Compose 使用三个命名 volume：

scripts/ops-scripts.test.mjs

@@ -1,5 +1,5 @@
 import { spawnSync } from 'node:child_process';
-import { mkdtempSync, writeFileSync, existsSync } from 'node:fs';
+import { mkdtempSync, writeFileSync, existsSync, readFileSync } from 'node:fs';
 import { tmpdir } from 'node:os';
 import { join, resolve } from 'node:path';
 import { describe, it } from 'node:test';
@@ -97,4 +97,45 @@ describe('production ops scripts', () => {
     assert.match(result.stdout, /\/api\/v1\/v2\.2\/requirements\?productId=__smoke__/);
     assert.match(result.stdout, /\/api\/v1\/config\/ai/);
   });
+
+  it('declares the production monitoring baseline without real alert secrets', () => {
+    const alertRules = readFileSync(
+      join(root, 'deploy/monitoring/prometheus/alert-rules.yml'),
+      'utf8',
+    );
+    const postgresQueries = readFileSync(
+      join(root, 'deploy/monitoring/postgres/postgres-queries.yml'),
+      'utf8',
+    );
+    const promtailConfig = readFileSync(
+      join(root, 'deploy/monitoring/promtail/config.yml'),
+      'utf8',
+    );
+    const dashboard = readFileSync(
+      join(root, 'deploy/monitoring/grafana/dashboards/ftb-production-overview.json'),
+      'utf8',
+    );
+
+    for (const alertName of [
+      'FtbPostgresDown',
+      'FtbDiskPressure',
+      'FtbSlowApiLogBurst',
+      'FtbSlowPrismaLogBurst',
+      'FtbJobFailureLogBurst',
+      'FtbXiaobaoSummaryStale',
+    ]) {
+      assert.match(alertRules, new RegExp(alertName));
+    }
+
+    assert.match(postgresQueries, /xiaobao_risk_summaries/);
+    assert.match(postgresQueries, /dirty = true/);
+    assert.match(promtailConfig, /Slow API request/);
+    assert.match(promtailConfig, /Slow Prisma query/);
+    assert.match(promtailConfig, /AppData relation sync failed/);
+    assert.match(dashboard, /FTB Production Overview/);
+
+    const combined = `${alertRules}\n${postgresQueries}\n${promtailConfig}\n${dashboard}`;
+    assert.doesNotMatch(combined, /sk-ant-[A-Za-z0-9]/);
+    assert.doesNotMatch(combined, /hooks\.slack\.com\/services\//);
+  });
 });

scripts/verify-production-deploy.mjs

@@ -51,6 +51,11 @@ const checks = [
       'WEB_IMAGE',
       'APP_VERSION',
       '/api/v1/health/version',
+      'prometheus:',
+      "profiles: ['monitoring']",
+      'postgres-exporter:',
+      'blackbox-exporter:',
+      'prometheus_data:',
     ],
   },
   {
@@ -110,6 +115,41 @@ const checks = [
       'X-Forwarded-Proto',
     ],
   },
+  {
+    file: 'deploy/monitoring/README.md',
+    snippets: [
+      '--profile monitoring',
+      'FtbPostgresDown',
+      'FtbXiaobaoSummaryStale',
+      'no webhook URLs',
+    ],
+  },
+  {
+    file: 'deploy/monitoring/prometheus/prometheus.yml',
+    snippets: ['postgres-exporter:9187', 'promtail:9080', 'blackbox-http'],
+  },
+  {
+    file: 'deploy/monitoring/prometheus/alert-rules.yml',
+    snippets: [
+      'FtbPostgresDown',
+      'FtbSlowApiLogBurst',
+      'FtbSlowPrismaLogBurst',
+      'FtbJobFailureLogBurst',
+      'FtbXiaobaoSummaryStale',
+    ],
+  },
+  {
+    file: 'deploy/monitoring/postgres/postgres-queries.yml',
+    snippets: ['xiaobao_risk_summaries', 'dirty = true', 'stale_summary_count'],
+  },
+  {
+    file: 'deploy/monitoring/promtail/config.yml',
+    snippets: ['Slow API request', 'Slow Prisma query', 'AppData relation sync failed'],
+  },
+  {
+    file: 'deploy/monitoring/grafana/dashboards/ftb-production-overview.json',
+    snippets: ['FTB Production Overview', 'ftb_xiaobao_stale_summary_count', 'Server Warning/Error Logs'],
+  },
   {
     file: '.env.production.example',
     snippets: [
@@ -143,6 +183,7 @@ const checks = [
       'pnpm backup:postgres',
       'pnpm restore:postgres',
       'pnpm backup:server-data',
+      '--profile monitoring',
       'pnpm db:migrate',
       'NEXT_PUBLIC_API_URL',
       'Nginx',