diff --git a/.env.production.example b/.env.production.example index d8c423f..512d929 100644 --- a/.env.production.example +++ b/.env.production.example @@ -39,3 +39,10 @@ SMTP_HOST= SMTP_PORT=465 SMTP_USER= SMTP_PASS= + +# Optional monitoring profile. Do not commit real production passwords. +PROMETHEUS_PORT=9090 +PROMETHEUS_RETENTION=15d +GRAFANA_PORT=3002 +GRAFANA_ADMIN_USER=admin +GRAFANA_ADMIN_PASSWORD=change-me-monitoring-password diff --git a/deploy/monitoring/README.md b/deploy/monitoring/README.md new file mode 100644 index 0000000..e54dca6 --- /dev/null +++ b/deploy/monitoring/README.md @@ -0,0 +1,43 @@ +# FTB Production Monitoring Baseline + +This profile adds a deployable Prometheus/Grafana baseline for production operations. It is intentionally secret-free: no webhook URLs, API keys, SMTP passwords, or real alert receiver credentials are committed. + +## Start + +```bash +docker compose --env-file .env.production -f docker-compose.prod.yml --profile monitoring up -d +``` + +Default local ports: + +- Prometheus: `http://localhost:9090` +- Grafana: `http://localhost:3002` +- Loki: internal only + +Set `GRAFANA_ADMIN_USER` and `GRAFANA_ADMIN_PASSWORD` in `.env.production` before exposing Grafana beyond localhost. Keep real alert receivers in the server environment or an untracked Alertmanager file. + +## Coverage + +- DB availability: `pg_up` from postgres-exporter. +- Disk pressure: root filesystem availability from node-exporter. +- Slow API requests: Promtail turns `Slow API request` server logs into `ftb_slow_api_log_total`. +- Slow Prisma queries: Promtail turns `Slow Prisma query` server logs into `ftb_slow_prisma_log_total`. +- Job failures: Promtail turns `AppData relation sync failed` and AI call failure logs into `ftb_job_failure_log_total`. +- Xiaobao stale summaries: postgres-exporter custom query exposes `ftb_xiaobao_stale_summary_count` from `xiaobao_risk_summaries`. + +## Alerts + +Prometheus loads `prometheus/alert-rules.yml`. The rules evaluate locally and are visible in Prometheus/Grafana. To send notifications, add Alertmanager outside git or mount an environment-specific receiver file; do not commit webhook URLs or tokens. + +Baseline alert names: + +- `FtbPostgresDown` +- `FtbDiskPressure` +- `FtbSlowApiLogBurst` +- `FtbSlowPrismaLogBurst` +- `FtbJobFailureLogBurst` +- `FtbXiaobaoSummaryStale` + +## Log Search + +Promtail ships Docker logs to Loki with container labels. Grafana provisions both Prometheus and Loki data sources, so on-call checks can move from a firing alert to matching server logs without SSHing into the host. diff --git a/deploy/monitoring/blackbox/config.yml b/deploy/monitoring/blackbox/config.yml new file mode 100644 index 0000000..fa81bee --- /dev/null +++ b/deploy/monitoring/blackbox/config.yml @@ -0,0 +1,10 @@ +modules: + http_2xx: + prober: http + timeout: 5s + http: + valid_http_versions: ['HTTP/1.1', 'HTTP/2.0'] + valid_status_codes: [] + method: GET + preferred_ip_protocol: ip4 + diff --git a/deploy/monitoring/grafana/dashboards/ftb-production-overview.json b/deploy/monitoring/grafana/dashboards/ftb-production-overview.json new file mode 100644 index 0000000..0a753d3 --- /dev/null +++ b/deploy/monitoring/grafana/dashboards/ftb-production-overview.json @@ -0,0 +1,78 @@ +{ + "uid": "ftb-production-overview", + "title": "FTB Production Overview", + "schemaVersion": 39, + "version": 1, + "refresh": "30s", + "tags": ["ftb", "production", "v2.8"], + "time": { + "from": "now-6h", + "to": "now" + }, + "panels": [ + { + "id": 1, + "type": "stat", + "title": "DB Availability", + "gridPos": { "x": 0, "y": 0, "w": 6, "h": 4 }, + "targets": [ + { "datasource": { "type": "prometheus", "uid": "Prometheus" }, "expr": "pg_up", "refId": "A" } + ] + }, + { + "id": 2, + "type": "timeseries", + "title": "Slow API Logs", + "gridPos": { "x": 6, "y": 0, "w": 6, "h": 4 }, + "targets": [ + { "datasource": { "type": "prometheus", "uid": "Prometheus" }, "expr": "increase(ftb_slow_api_log_total[10m])", "refId": "A" } + ] + }, + { + "id": 3, + "type": "timeseries", + "title": "Slow Prisma Logs", + "gridPos": { "x": 12, "y": 0, "w": 6, "h": 4 }, + "targets": [ + { "datasource": { "type": "prometheus", "uid": "Prometheus" }, "expr": "increase(ftb_slow_prisma_log_total[10m])", "refId": "A" } + ] + }, + { + "id": 4, + "type": "stat", + "title": "Xiaobao Stale Summaries", + "gridPos": { "x": 18, "y": 0, "w": 6, "h": 4 }, + "targets": [ + { "datasource": { "type": "prometheus", "uid": "Prometheus" }, "expr": "ftb_xiaobao_stale_summary_count", "refId": "A" } + ] + }, + { + "id": 5, + "type": "timeseries", + "title": "Job Failure Logs", + "gridPos": { "x": 0, "y": 4, "w": 8, "h": 5 }, + "targets": [ + { "datasource": { "type": "prometheus", "uid": "Prometheus" }, "expr": "increase(ftb_job_failure_log_total[10m])", "refId": "A" } + ] + }, + { + "id": 6, + "type": "stat", + "title": "Root Disk Free %", + "gridPos": { "x": 8, "y": 4, "w": 8, "h": 5 }, + "targets": [ + { "datasource": { "type": "prometheus", "uid": "Prometheus" }, "expr": "100 * node_filesystem_avail_bytes{mountpoint=\"/\",fstype!~\"tmpfs|overlay\"} / node_filesystem_size_bytes{mountpoint=\"/\",fstype!~\"tmpfs|overlay\"}", "refId": "A" } + ] + }, + { + "id": 7, + "type": "logs", + "title": "Server Warning/Error Logs", + "gridPos": { "x": 16, "y": 4, "w": 8, "h": 5 }, + "targets": [ + { "datasource": { "type": "loki", "uid": "Loki" }, "expr": "{compose_service=\"server\"} |~ \"warn|error|失败|Slow\"", "refId": "A" } + ] + } + ] +} + diff --git a/deploy/monitoring/grafana/provisioning/dashboards/dashboards.yml b/deploy/monitoring/grafana/provisioning/dashboards/dashboards.yml new file mode 100644 index 0000000..adcba00 --- /dev/null +++ b/deploy/monitoring/grafana/provisioning/dashboards/dashboards.yml @@ -0,0 +1,12 @@ +apiVersion: 1 + +providers: + - name: FTB Production + orgId: 1 + folder: FTB + type: file + disableDeletion: false + updateIntervalSeconds: 30 + options: + path: /var/lib/grafana/dashboards + diff --git a/deploy/monitoring/grafana/provisioning/datasources/datasources.yml b/deploy/monitoring/grafana/provisioning/datasources/datasources.yml new file mode 100644 index 0000000..5ef7ccf --- /dev/null +++ b/deploy/monitoring/grafana/provisioning/datasources/datasources.yml @@ -0,0 +1,16 @@ +apiVersion: 1 + +datasources: + - name: Prometheus + type: prometheus + access: proxy + url: http://prometheus:9090 + isDefault: true + editable: false + + - name: Loki + type: loki + access: proxy + url: http://loki:3100 + editable: false + diff --git a/deploy/monitoring/loki/config.yml b/deploy/monitoring/loki/config.yml new file mode 100644 index 0000000..dad75da --- /dev/null +++ b/deploy/monitoring/loki/config.yml @@ -0,0 +1,32 @@ +auth_enabled: false + +server: + http_listen_port: 3100 + +common: + path_prefix: /loki + replication_factor: 1 + ring: + kvstore: + store: inmemory + +schema_config: + configs: + - from: 2026-01-01 + store: tsdb + object_store: filesystem + schema: v13 + index: + prefix: index_ + period: 24h + +storage_config: + tsdb_shipper: + active_index_directory: /loki/index + cache_location: /loki/index_cache + filesystem: + directory: /loki/chunks + +limits_config: + retention_period: 168h + diff --git a/deploy/monitoring/postgres/postgres-queries.yml b/deploy/monitoring/postgres/postgres-queries.yml new file mode 100644 index 0000000..8a19c94 --- /dev/null +++ b/deploy/monitoring/postgres/postgres-queries.yml @@ -0,0 +1,12 @@ +ftb_xiaobao: + query: | + SELECT + count(*)::float AS stale_summary_count + FROM xiaobao_risk_summaries + WHERE dirty = true + OR updated_at < now() - interval '6 hours'; + metrics: + - stale_summary_count: + usage: GAUGE + description: Xiaobao risk summaries that are dirty or older than 6 hours. + diff --git a/deploy/monitoring/prometheus/alert-rules.yml b/deploy/monitoring/prometheus/alert-rules.yml new file mode 100644 index 0000000..e3af268 --- /dev/null +++ b/deploy/monitoring/prometheus/alert-rules.yml @@ -0,0 +1,62 @@ +groups: + - name: ftb-production-alerts + rules: + - alert: FtbPostgresDown + expr: pg_up == 0 + for: 2m + labels: + severity: critical + annotations: + summary: PostgreSQL exporter cannot reach the FTB database. + runbook: docs/runbooks/migration-rollback.md + + - alert: FtbDiskPressure + expr: | + ( + node_filesystem_avail_bytes{mountpoint="/",fstype!~"tmpfs|overlay"} + / + node_filesystem_size_bytes{mountpoint="/",fstype!~"tmpfs|overlay"} + ) < 0.15 + for: 10m + labels: + severity: warning + annotations: + summary: Production host root filesystem has less than 15% free space. + runbook: docs/deployment.md + + - alert: FtbSlowApiLogBurst + expr: increase(ftb_slow_api_log_total[10m]) > 5 + for: 2m + labels: + severity: warning + annotations: + summary: Slow API request log volume crossed the V2.8 baseline threshold. + runbook: docs/deployment.md + + - alert: FtbSlowPrismaLogBurst + expr: increase(ftb_slow_prisma_log_total[10m]) > 3 + for: 2m + labels: + severity: warning + annotations: + summary: Slow Prisma query log volume crossed the V2.8 baseline threshold. + runbook: docs/deployment.md + + - alert: FtbJobFailureLogBurst + expr: increase(ftb_job_failure_log_total[10m]) > 0 + for: 1m + labels: + severity: warning + annotations: + summary: Background or compatibility job failure logs were detected. + runbook: docs/runbooks/xiaobao-background-jobs.md + + - alert: FtbXiaobaoSummaryStale + expr: ftb_xiaobao_stale_summary_count > 0 + for: 15m + labels: + severity: warning + annotations: + summary: Xiaobao warning summaries are dirty or stale. + runbook: docs/runbooks/xiaobao-background-jobs.md + diff --git a/deploy/monitoring/prometheus/prometheus.yml b/deploy/monitoring/prometheus/prometheus.yml new file mode 100644 index 0000000..b8b3b18 --- /dev/null +++ b/deploy/monitoring/prometheus/prometheus.yml @@ -0,0 +1,45 @@ +global: + scrape_interval: 15s + evaluation_interval: 15s + +rule_files: + - /etc/prometheus/alert-rules.yml + +scrape_configs: + - job_name: prometheus + static_configs: + - targets: ['prometheus:9090'] + + - job_name: postgres-exporter + static_configs: + - targets: ['postgres-exporter:9187'] + + - job_name: node-exporter + static_configs: + - targets: ['node-exporter:9100'] + + - job_name: cadvisor + static_configs: + - targets: ['cadvisor:8080'] + + - job_name: promtail + static_configs: + - targets: ['promtail:9080'] + + - job_name: blackbox-http + metrics_path: /probe + params: + module: [http_2xx] + static_configs: + - targets: + - http://nginx/api/v1/health/version + - http://nginx/api/v1/config/ai + - http://nginx/api/v1/v2.2/requirements?productId=__smoke__&limit=1 + relabel_configs: + - source_labels: [__address__] + target_label: __param_target + - source_labels: [__param_target] + target_label: instance + - target_label: __address__ + replacement: blackbox-exporter:9115 + diff --git a/deploy/monitoring/promtail/config.yml b/deploy/monitoring/promtail/config.yml new file mode 100644 index 0000000..6ebb2d7 --- /dev/null +++ b/deploy/monitoring/promtail/config.yml @@ -0,0 +1,53 @@ +server: + http_listen_port: 9080 + grpc_listen_port: 0 + +positions: + filename: /tmp/positions.yml + +clients: + - url: http://loki:3100/loki/api/v1/push + +scrape_configs: + - job_name: docker + docker_sd_configs: + - host: unix:///var/run/docker.sock + refresh_interval: 15s + relabel_configs: + - source_labels: ['__meta_docker_container_name'] + regex: '/(.*)' + target_label: container + - source_labels: ['__meta_docker_container_label_com_docker_compose_service'] + target_label: compose_service + - source_labels: ['__meta_docker_container_label_com_docker_compose_project'] + target_label: compose_project + pipeline_stages: + - docker: {} + - match: + selector: '{compose_service="server"} |= "Slow API request"' + stages: + - metrics: + ftb_slow_api_log_total: + type: Counter + description: Slow API request log entries emitted by the NestJS server. + config: + action: inc + - match: + selector: '{compose_service="server"} |= "Slow Prisma query"' + stages: + - metrics: + ftb_slow_prisma_log_total: + type: Counter + description: Slow Prisma query log entries emitted by the NestJS server. + config: + action: inc + - match: + selector: '{compose_service="server"} |~ "AppData relation sync failed|AI 调用失败|AI 风险解读调用失败"' + stages: + - metrics: + ftb_job_failure_log_total: + type: Counter + description: Compatibility sync, AI job, or background task failure logs. + config: + action: inc + diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index 3f85e01..a4a41c4 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -123,7 +123,117 @@ services: server: condition: service_healthy + prometheus: + image: prom/prometheus:v2.53.1 + profiles: ['monitoring'] + restart: unless-stopped + command: + - '--config.file=/etc/prometheus/prometheus.yml' + - '--storage.tsdb.path=/prometheus' + - '--storage.tsdb.retention.time=${PROMETHEUS_RETENTION:-15d}' + - '--web.enable-lifecycle' + ports: + - '${PROMETHEUS_PORT:-9090}:9090' + volumes: + - ./deploy/monitoring/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro + - ./deploy/monitoring/prometheus/alert-rules.yml:/etc/prometheus/alert-rules.yml:ro + - prometheus_data:/prometheus + depends_on: + - postgres-exporter + - node-exporter + - cadvisor + - promtail + - blackbox-exporter + + grafana: + image: grafana/grafana:11.1.0 + profiles: ['monitoring'] + restart: unless-stopped + ports: + - '${GRAFANA_PORT:-3002}:3000' + environment: + GF_SECURITY_ADMIN_USER: ${GRAFANA_ADMIN_USER:-admin} + GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD:-change-me-monitoring-password} + GF_USERS_ALLOW_SIGN_UP: 'false' + volumes: + - grafana_data:/var/lib/grafana + - ./deploy/monitoring/grafana/provisioning:/etc/grafana/provisioning:ro + - ./deploy/monitoring/grafana/dashboards:/var/lib/grafana/dashboards:ro + depends_on: + - prometheus + - loki + + loki: + image: grafana/loki:2.9.8 + profiles: ['monitoring'] + restart: unless-stopped + command: ['-config.file=/etc/loki/config.yml'] + volumes: + - ./deploy/monitoring/loki/config.yml:/etc/loki/config.yml:ro + - loki_data:/loki + + promtail: + image: grafana/promtail:2.9.8 + profiles: ['monitoring'] + restart: unless-stopped + command: ['-config.file=/etc/promtail/config.yml'] + volumes: + - ./deploy/monitoring/promtail/config.yml:/etc/promtail/config.yml:ro + - /var/lib/docker/containers:/var/lib/docker/containers:ro + - /var/run/docker.sock:/var/run/docker.sock:ro + depends_on: + - loki + + postgres-exporter: + image: quay.io/prometheuscommunity/postgres-exporter:v0.15.0 + profiles: ['monitoring'] + restart: unless-stopped + command: + - '--extend.query-path=/etc/postgres-exporter/postgres-queries.yml' + environment: + DATA_SOURCE_NAME: postgresql://${POSTGRES_USER:-postgres}:${POSTGRES_PASSWORD}@postgres:5432/${POSTGRES_DB:-ftb_pm}?sslmode=disable + volumes: + - ./deploy/monitoring/postgres/postgres-queries.yml:/etc/postgres-exporter/postgres-queries.yml:ro + depends_on: + postgres: + condition: service_healthy + + node-exporter: + image: prom/node-exporter:v1.8.2 + profiles: ['monitoring'] + restart: unless-stopped + command: + - '--path.rootfs=/host' + volumes: + - /:/host:ro,rslave + + cadvisor: + image: gcr.io/cadvisor/cadvisor:v0.49.1 + profiles: ['monitoring'] + restart: unless-stopped + privileged: true + devices: + - /dev/kmsg:/dev/kmsg + volumes: + - /:/rootfs:ro + - /var/run:/var/run:ro + - /sys:/sys:ro + - /var/lib/docker/:/var/lib/docker:ro + - /dev/disk/:/dev/disk:ro + + blackbox-exporter: + image: prom/blackbox-exporter:v0.25.0 + profiles: ['monitoring'] + restart: unless-stopped + command: + - '--config.file=/etc/blackbox/config.yml' + volumes: + - ./deploy/monitoring/blackbox/config.yml:/etc/blackbox/config.yml:ro + volumes: postgres_data: redis_data: server_data: + prometheus_data: + grafana_data: + loki_data: diff --git a/docs/deployment.md b/docs/deployment.md index ea3c4a0..ec1a67c 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -194,6 +194,30 @@ pnpm deploy:smoke -- --base-url http://localhost --expected-version { assert.match(result.stdout, /\/api\/v1\/v2\.2\/requirements\?productId=__smoke__/); assert.match(result.stdout, /\/api\/v1\/config\/ai/); }); + + it('declares the production monitoring baseline without real alert secrets', () => { + const alertRules = readFileSync( + join(root, 'deploy/monitoring/prometheus/alert-rules.yml'), + 'utf8', + ); + const postgresQueries = readFileSync( + join(root, 'deploy/monitoring/postgres/postgres-queries.yml'), + 'utf8', + ); + const promtailConfig = readFileSync( + join(root, 'deploy/monitoring/promtail/config.yml'), + 'utf8', + ); + const dashboard = readFileSync( + join(root, 'deploy/monitoring/grafana/dashboards/ftb-production-overview.json'), + 'utf8', + ); + + for (const alertName of [ + 'FtbPostgresDown', + 'FtbDiskPressure', + 'FtbSlowApiLogBurst', + 'FtbSlowPrismaLogBurst', + 'FtbJobFailureLogBurst', + 'FtbXiaobaoSummaryStale', + ]) { + assert.match(alertRules, new RegExp(alertName)); + } + + assert.match(postgresQueries, /xiaobao_risk_summaries/); + assert.match(postgresQueries, /dirty = true/); + assert.match(promtailConfig, /Slow API request/); + assert.match(promtailConfig, /Slow Prisma query/); + assert.match(promtailConfig, /AppData relation sync failed/); + assert.match(dashboard, /FTB Production Overview/); + + const combined = `${alertRules}\n${postgresQueries}\n${promtailConfig}\n${dashboard}`; + assert.doesNotMatch(combined, /sk-ant-[A-Za-z0-9]/); + assert.doesNotMatch(combined, /hooks\.slack\.com\/services\//); + }); }); diff --git a/scripts/verify-production-deploy.mjs b/scripts/verify-production-deploy.mjs index 1ff3603..9759a63 100644 --- a/scripts/verify-production-deploy.mjs +++ b/scripts/verify-production-deploy.mjs @@ -51,6 +51,11 @@ const checks = [ 'WEB_IMAGE', 'APP_VERSION', '/api/v1/health/version', + 'prometheus:', + "profiles: ['monitoring']", + 'postgres-exporter:', + 'blackbox-exporter:', + 'prometheus_data:', ], }, { @@ -110,6 +115,41 @@ const checks = [ 'X-Forwarded-Proto', ], }, + { + file: 'deploy/monitoring/README.md', + snippets: [ + '--profile monitoring', + 'FtbPostgresDown', + 'FtbXiaobaoSummaryStale', + 'no webhook URLs', + ], + }, + { + file: 'deploy/monitoring/prometheus/prometheus.yml', + snippets: ['postgres-exporter:9187', 'promtail:9080', 'blackbox-http'], + }, + { + file: 'deploy/monitoring/prometheus/alert-rules.yml', + snippets: [ + 'FtbPostgresDown', + 'FtbSlowApiLogBurst', + 'FtbSlowPrismaLogBurst', + 'FtbJobFailureLogBurst', + 'FtbXiaobaoSummaryStale', + ], + }, + { + file: 'deploy/monitoring/postgres/postgres-queries.yml', + snippets: ['xiaobao_risk_summaries', 'dirty = true', 'stale_summary_count'], + }, + { + file: 'deploy/monitoring/promtail/config.yml', + snippets: ['Slow API request', 'Slow Prisma query', 'AppData relation sync failed'], + }, + { + file: 'deploy/monitoring/grafana/dashboards/ftb-production-overview.json', + snippets: ['FTB Production Overview', 'ftb_xiaobao_stale_summary_count', 'Server Warning/Error Logs'], + }, { file: '.env.production.example', snippets: [ @@ -143,6 +183,7 @@ const checks = [ 'pnpm backup:postgres', 'pnpm restore:postgres', 'pnpm backup:server-data', + '--profile monitoring', 'pnpm db:migrate', 'NEXT_PUBLIC_API_URL', 'Nginx',