feat(ops): 添加生产监控告警基线

- 新增 Prometheus/Grafana/Loki/Promtail 监控 profile\n- 覆盖 DB、磁盘、慢 API、慢 Prisma、任务失败和小宝摘要 stale 告警\n- 补充 postgres-exporter 自定义查询、Dashboard、部署文档和校验\n\nCo-Authored-By: GPT-5 Codex <codex@openai.com>
This commit is contained in:
2026-07-08 16:21:07 +08:00
parent eef09d5af4
commit 7837a809ca
15 changed files with 587 additions and 1 deletions

View File

@@ -0,0 +1,62 @@
groups:
- name: ftb-production-alerts
rules:
- alert: FtbPostgresDown
expr: pg_up == 0
for: 2m
labels:
severity: critical
annotations:
summary: PostgreSQL exporter cannot reach the FTB database.
runbook: docs/runbooks/migration-rollback.md
- alert: FtbDiskPressure
expr: |
(
node_filesystem_avail_bytes{mountpoint="/",fstype!~"tmpfs|overlay"}
/
node_filesystem_size_bytes{mountpoint="/",fstype!~"tmpfs|overlay"}
) < 0.15
for: 10m
labels:
severity: warning
annotations:
summary: Production host root filesystem has less than 15% free space.
runbook: docs/deployment.md
- alert: FtbSlowApiLogBurst
expr: increase(ftb_slow_api_log_total[10m]) > 5
for: 2m
labels:
severity: warning
annotations:
summary: Slow API request log volume crossed the V2.8 baseline threshold.
runbook: docs/deployment.md
- alert: FtbSlowPrismaLogBurst
expr: increase(ftb_slow_prisma_log_total[10m]) > 3
for: 2m
labels:
severity: warning
annotations:
summary: Slow Prisma query log volume crossed the V2.8 baseline threshold.
runbook: docs/deployment.md
- alert: FtbJobFailureLogBurst
expr: increase(ftb_job_failure_log_total[10m]) > 0
for: 1m
labels:
severity: warning
annotations:
summary: Background or compatibility job failure logs were detected.
runbook: docs/runbooks/xiaobao-background-jobs.md
- alert: FtbXiaobaoSummaryStale
expr: ftb_xiaobao_stale_summary_count > 0
for: 15m
labels:
severity: warning
annotations:
summary: Xiaobao warning summaries are dirty or stale.
runbook: docs/runbooks/xiaobao-background-jobs.md

View File

@@ -0,0 +1,45 @@
global:
scrape_interval: 15s
evaluation_interval: 15s
rule_files:
- /etc/prometheus/alert-rules.yml
scrape_configs:
- job_name: prometheus
static_configs:
- targets: ['prometheus:9090']
- job_name: postgres-exporter
static_configs:
- targets: ['postgres-exporter:9187']
- job_name: node-exporter
static_configs:
- targets: ['node-exporter:9100']
- job_name: cadvisor
static_configs:
- targets: ['cadvisor:8080']
- job_name: promtail
static_configs:
- targets: ['promtail:9080']
- job_name: blackbox-http
metrics_path: /probe
params:
module: [http_2xx]
static_configs:
- targets:
- http://nginx/api/v1/health/version
- http://nginx/api/v1/config/ai
- http://nginx/api/v1/v2.2/requirements?productId=__smoke__&limit=1
relabel_configs:
- source_labels: [__address__]
target_label: __param_target
- source_labels: [__param_target]
target_label: instance
- target_label: __address__
replacement: blackbox-exporter:9115