feat(ops): 添加生产监控告警基线

- 新增 Prometheus/Grafana/Loki/Promtail 监控 profile\n- 覆盖 DB、磁盘、慢 API、慢 Prisma、任务失败和小宝摘要 stale 告警\n- 补充 postgres-exporter 自定义查询、Dashboard、部署文档和校验\n\nCo-Authored-By: GPT-5 Codex <codex@openai.com>

deploy/monitoring/README.md

@@ -0,0 +1,43 @@
+# FTB Production Monitoring Baseline
+
+This profile adds a deployable Prometheus/Grafana baseline for production operations. It is intentionally secret-free: no webhook URLs, API keys, SMTP passwords, or real alert receiver credentials are committed.
+
+## Start
+
+```bash
+docker compose --env-file .env.production -f docker-compose.prod.yml --profile monitoring up -d
+```
+
+Default local ports:
+
+- Prometheus: `http://localhost:9090`
+- Grafana: `http://localhost:3002`
+- Loki: internal only
+
+Set `GRAFANA_ADMIN_USER` and `GRAFANA_ADMIN_PASSWORD` in `.env.production` before exposing Grafana beyond localhost. Keep real alert receivers in the server environment or an untracked Alertmanager file.
+
+## Coverage
+
+- DB availability: `pg_up` from postgres-exporter.
+- Disk pressure: root filesystem availability from node-exporter.
+- Slow API requests: Promtail turns `Slow API request` server logs into `ftb_slow_api_log_total`.
+- Slow Prisma queries: Promtail turns `Slow Prisma query` server logs into `ftb_slow_prisma_log_total`.
+- Job failures: Promtail turns `AppData relation sync failed` and AI call failure logs into `ftb_job_failure_log_total`.
+- Xiaobao stale summaries: postgres-exporter custom query exposes `ftb_xiaobao_stale_summary_count` from `xiaobao_risk_summaries`.
+
+## Alerts
+
+Prometheus loads `prometheus/alert-rules.yml`. The rules evaluate locally and are visible in Prometheus/Grafana. To send notifications, add Alertmanager outside git or mount an environment-specific receiver file; do not commit webhook URLs or tokens.
+
+Baseline alert names:
+
+- `FtbPostgresDown`
+- `FtbDiskPressure`
+- `FtbSlowApiLogBurst`
+- `FtbSlowPrismaLogBurst`
+- `FtbJobFailureLogBurst`
+- `FtbXiaobaoSummaryStale`
+
+## Log Search
+
+Promtail ships Docker logs to Loki with container labels. Grafana provisions both Prometheus and Loki data sources, so on-call checks can move from a firing alert to matching server logs without SSHing into the host.

deploy/monitoring/blackbox/config.yml

@@ -0,0 +1,10 @@
+modules:
+  http_2xx:
+    prober: http
+    timeout: 5s
+    http:
+      valid_http_versions: ['HTTP/1.1', 'HTTP/2.0']
+      valid_status_codes: []
+      method: GET
+      preferred_ip_protocol: ip4
+

deploy/monitoring/grafana/dashboards/ftb-production-overview.json

@@ -0,0 +1,78 @@
+{
+  "uid": "ftb-production-overview",
+  "title": "FTB Production Overview",
+  "schemaVersion": 39,
+  "version": 1,
+  "refresh": "30s",
+  "tags": ["ftb", "production", "v2.8"],
+  "time": {
+    "from": "now-6h",
+    "to": "now"
+  },
+  "panels": [
+    {
+      "id": 1,
+      "type": "stat",
+      "title": "DB Availability",
+      "gridPos": { "x": 0, "y": 0, "w": 6, "h": 4 },
+      "targets": [
+        { "datasource": { "type": "prometheus", "uid": "Prometheus" }, "expr": "pg_up", "refId": "A" }
+      ]
+    },
+    {
+      "id": 2,
+      "type": "timeseries",
+      "title": "Slow API Logs",
+      "gridPos": { "x": 6, "y": 0, "w": 6, "h": 4 },
+      "targets": [
+        { "datasource": { "type": "prometheus", "uid": "Prometheus" }, "expr": "increase(ftb_slow_api_log_total[10m])", "refId": "A" }
+      ]
+    },
+    {
+      "id": 3,
+      "type": "timeseries",
+      "title": "Slow Prisma Logs",
+      "gridPos": { "x": 12, "y": 0, "w": 6, "h": 4 },
+      "targets": [
+        { "datasource": { "type": "prometheus", "uid": "Prometheus" }, "expr": "increase(ftb_slow_prisma_log_total[10m])", "refId": "A" }
+      ]
+    },
+    {
+      "id": 4,
+      "type": "stat",
+      "title": "Xiaobao Stale Summaries",
+      "gridPos": { "x": 18, "y": 0, "w": 6, "h": 4 },
+      "targets": [
+        { "datasource": { "type": "prometheus", "uid": "Prometheus" }, "expr": "ftb_xiaobao_stale_summary_count", "refId": "A" }
+      ]
+    },
+    {
+      "id": 5,
+      "type": "timeseries",
+      "title": "Job Failure Logs",
+      "gridPos": { "x": 0, "y": 4, "w": 8, "h": 5 },
+      "targets": [
+        { "datasource": { "type": "prometheus", "uid": "Prometheus" }, "expr": "increase(ftb_job_failure_log_total[10m])", "refId": "A" }
+      ]
+    },
+    {
+      "id": 6,
+      "type": "stat",
+      "title": "Root Disk Free %",
+      "gridPos": { "x": 8, "y": 4, "w": 8, "h": 5 },
+      "targets": [
+        { "datasource": { "type": "prometheus", "uid": "Prometheus" }, "expr": "100 * node_filesystem_avail_bytes{mountpoint=\"/\",fstype!~\"tmpfs|overlay\"} / node_filesystem_size_bytes{mountpoint=\"/\",fstype!~\"tmpfs|overlay\"}", "refId": "A" }
+      ]
+    },
+    {
+      "id": 7,
+      "type": "logs",
+      "title": "Server Warning/Error Logs",
+      "gridPos": { "x": 16, "y": 4, "w": 8, "h": 5 },
+      "targets": [
+        { "datasource": { "type": "loki", "uid": "Loki" }, "expr": "{compose_service=\"server\"} |~ \"warn|error|失败|Slow\"", "refId": "A" }
+      ]
+    }
+  ]
+}
+

deploy/monitoring/grafana/provisioning/dashboards/dashboards.yml

@@ -0,0 +1,12 @@
+apiVersion: 1
+
+providers:
+  - name: FTB Production
+    orgId: 1
+    folder: FTB
+    type: file
+    disableDeletion: false
+    updateIntervalSeconds: 30
+    options:
+      path: /var/lib/grafana/dashboards
+

deploy/monitoring/grafana/provisioning/datasources/datasources.yml

@@ -0,0 +1,16 @@
+apiVersion: 1
+
+datasources:
+  - name: Prometheus
+    type: prometheus
+    access: proxy
+    url: http://prometheus:9090
+    isDefault: true
+    editable: false
+
+  - name: Loki
+    type: loki
+    access: proxy
+    url: http://loki:3100
+    editable: false
+

deploy/monitoring/loki/config.yml

@@ -0,0 +1,32 @@
+auth_enabled: false
+
+server:
+  http_listen_port: 3100
+
+common:
+  path_prefix: /loki
+  replication_factor: 1
+  ring:
+    kvstore:
+      store: inmemory
+
+schema_config:
+  configs:
+    - from: 2026-01-01
+      store: tsdb
+      object_store: filesystem
+      schema: v13
+      index:
+        prefix: index_
+        period: 24h
+
+storage_config:
+  tsdb_shipper:
+    active_index_directory: /loki/index
+    cache_location: /loki/index_cache
+  filesystem:
+    directory: /loki/chunks
+
+limits_config:
+  retention_period: 168h
+

deploy/monitoring/postgres/postgres-queries.yml

@@ -0,0 +1,12 @@
+ftb_xiaobao:
+  query: |
+    SELECT
+      count(*)::float AS stale_summary_count
+    FROM xiaobao_risk_summaries
+    WHERE dirty = true
+       OR updated_at < now() - interval '6 hours';
+  metrics:
+    - stale_summary_count:
+        usage: GAUGE
+        description: Xiaobao risk summaries that are dirty or older than 6 hours.
+

deploy/monitoring/prometheus/alert-rules.yml

@@ -0,0 +1,62 @@
+groups:
+  - name: ftb-production-alerts
+    rules:
+      - alert: FtbPostgresDown
+        expr: pg_up == 0
+        for: 2m
+        labels:
+          severity: critical
+        annotations:
+          summary: PostgreSQL exporter cannot reach the FTB database.
+          runbook: docs/runbooks/migration-rollback.md
+
+      - alert: FtbDiskPressure
+        expr: |
+          (
+            node_filesystem_avail_bytes{mountpoint="/",fstype!~"tmpfs|overlay"}
+            /
+            node_filesystem_size_bytes{mountpoint="/",fstype!~"tmpfs|overlay"}
+          ) < 0.15
+        for: 10m
+        labels:
+          severity: warning
+        annotations:
+          summary: Production host root filesystem has less than 15% free space.
+          runbook: docs/deployment.md
+
+      - alert: FtbSlowApiLogBurst
+        expr: increase(ftb_slow_api_log_total[10m]) > 5
+        for: 2m
+        labels:
+          severity: warning
+        annotations:
+          summary: Slow API request log volume crossed the V2.8 baseline threshold.
+          runbook: docs/deployment.md
+
+      - alert: FtbSlowPrismaLogBurst
+        expr: increase(ftb_slow_prisma_log_total[10m]) > 3
+        for: 2m
+        labels:
+          severity: warning
+        annotations:
+          summary: Slow Prisma query log volume crossed the V2.8 baseline threshold.
+          runbook: docs/deployment.md
+
+      - alert: FtbJobFailureLogBurst
+        expr: increase(ftb_job_failure_log_total[10m]) > 0
+        for: 1m
+        labels:
+          severity: warning
+        annotations:
+          summary: Background or compatibility job failure logs were detected.
+          runbook: docs/runbooks/xiaobao-background-jobs.md
+
+      - alert: FtbXiaobaoSummaryStale
+        expr: ftb_xiaobao_stale_summary_count > 0
+        for: 15m
+        labels:
+          severity: warning
+        annotations:
+          summary: Xiaobao warning summaries are dirty or stale.
+          runbook: docs/runbooks/xiaobao-background-jobs.md
+

deploy/monitoring/prometheus/prometheus.yml

@@ -0,0 +1,45 @@
+global:
+  scrape_interval: 15s
+  evaluation_interval: 15s
+
+rule_files:
+  - /etc/prometheus/alert-rules.yml
+
+scrape_configs:
+  - job_name: prometheus
+    static_configs:
+      - targets: ['prometheus:9090']
+
+  - job_name: postgres-exporter
+    static_configs:
+      - targets: ['postgres-exporter:9187']
+
+  - job_name: node-exporter
+    static_configs:
+      - targets: ['node-exporter:9100']
+
+  - job_name: cadvisor
+    static_configs:
+      - targets: ['cadvisor:8080']
+
+  - job_name: promtail
+    static_configs:
+      - targets: ['promtail:9080']
+
+  - job_name: blackbox-http
+    metrics_path: /probe
+    params:
+      module: [http_2xx]
+    static_configs:
+      - targets:
+          - http://nginx/api/v1/health/version
+          - http://nginx/api/v1/config/ai
+          - http://nginx/api/v1/v2.2/requirements?productId=__smoke__&limit=1
+    relabel_configs:
+      - source_labels: [__address__]
+        target_label: __param_target
+      - source_labels: [__param_target]
+        target_label: instance
+      - target_label: __address__
+        replacement: blackbox-exporter:9115
+

deploy/monitoring/promtail/config.yml

@@ -0,0 +1,53 @@
+server:
+  http_listen_port: 9080
+  grpc_listen_port: 0
+
+positions:
+  filename: /tmp/positions.yml
+
+clients:
+  - url: http://loki:3100/loki/api/v1/push
+
+scrape_configs:
+  - job_name: docker
+    docker_sd_configs:
+      - host: unix:///var/run/docker.sock
+        refresh_interval: 15s
+    relabel_configs:
+      - source_labels: ['__meta_docker_container_name']
+        regex: '/(.*)'
+        target_label: container
+      - source_labels: ['__meta_docker_container_label_com_docker_compose_service']
+        target_label: compose_service
+      - source_labels: ['__meta_docker_container_label_com_docker_compose_project']
+        target_label: compose_project
+    pipeline_stages:
+      - docker: {}
+      - match:
+          selector: '{compose_service="server"} |= "Slow API request"'
+          stages:
+            - metrics:
+                ftb_slow_api_log_total:
+                  type: Counter
+                  description: Slow API request log entries emitted by the NestJS server.
+                  config:
+                    action: inc
+      - match:
+          selector: '{compose_service="server"} |= "Slow Prisma query"'
+          stages:
+            - metrics:
+                ftb_slow_prisma_log_total:
+                  type: Counter
+                  description: Slow Prisma query log entries emitted by the NestJS server.
+                  config:
+                    action: inc
+      - match:
+          selector: '{compose_service="server"} |~ "AppData relation sync failed|AI 调用失败|AI 风险解读调用失败"'
+          stages:
+            - metrics:
+                ftb_job_failure_log_total:
+                  type: Counter
+                  description: Compatibility sync, AI job, or background task failure logs.
+                  config:
+                    action: inc
+