feat(ops): 添加生产监控告警基线
- 新增 Prometheus/Grafana/Loki/Promtail 监控 profile\n- 覆盖 DB、磁盘、慢 API、慢 Prisma、任务失败和小宝摘要 stale 告警\n- 补充 postgres-exporter 自定义查询、Dashboard、部署文档和校验\n\nCo-Authored-By: GPT-5 Codex <codex@openai.com>
This commit is contained in:
62
deploy/monitoring/prometheus/alert-rules.yml
Normal file
62
deploy/monitoring/prometheus/alert-rules.yml
Normal file
@@ -0,0 +1,62 @@
|
||||
groups:
|
||||
- name: ftb-production-alerts
|
||||
rules:
|
||||
- alert: FtbPostgresDown
|
||||
expr: pg_up == 0
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: PostgreSQL exporter cannot reach the FTB database.
|
||||
runbook: docs/runbooks/migration-rollback.md
|
||||
|
||||
- alert: FtbDiskPressure
|
||||
expr: |
|
||||
(
|
||||
node_filesystem_avail_bytes{mountpoint="/",fstype!~"tmpfs|overlay"}
|
||||
/
|
||||
node_filesystem_size_bytes{mountpoint="/",fstype!~"tmpfs|overlay"}
|
||||
) < 0.15
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: Production host root filesystem has less than 15% free space.
|
||||
runbook: docs/deployment.md
|
||||
|
||||
- alert: FtbSlowApiLogBurst
|
||||
expr: increase(ftb_slow_api_log_total[10m]) > 5
|
||||
for: 2m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: Slow API request log volume crossed the V2.8 baseline threshold.
|
||||
runbook: docs/deployment.md
|
||||
|
||||
- alert: FtbSlowPrismaLogBurst
|
||||
expr: increase(ftb_slow_prisma_log_total[10m]) > 3
|
||||
for: 2m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: Slow Prisma query log volume crossed the V2.8 baseline threshold.
|
||||
runbook: docs/deployment.md
|
||||
|
||||
- alert: FtbJobFailureLogBurst
|
||||
expr: increase(ftb_job_failure_log_total[10m]) > 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: Background or compatibility job failure logs were detected.
|
||||
runbook: docs/runbooks/xiaobao-background-jobs.md
|
||||
|
||||
- alert: FtbXiaobaoSummaryStale
|
||||
expr: ftb_xiaobao_stale_summary_count > 0
|
||||
for: 15m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: Xiaobao warning summaries are dirty or stale.
|
||||
runbook: docs/runbooks/xiaobao-background-jobs.md
|
||||
|
||||
45
deploy/monitoring/prometheus/prometheus.yml
Normal file
45
deploy/monitoring/prometheus/prometheus.yml
Normal file
@@ -0,0 +1,45 @@
|
||||
global:
|
||||
scrape_interval: 15s
|
||||
evaluation_interval: 15s
|
||||
|
||||
rule_files:
|
||||
- /etc/prometheus/alert-rules.yml
|
||||
|
||||
scrape_configs:
|
||||
- job_name: prometheus
|
||||
static_configs:
|
||||
- targets: ['prometheus:9090']
|
||||
|
||||
- job_name: postgres-exporter
|
||||
static_configs:
|
||||
- targets: ['postgres-exporter:9187']
|
||||
|
||||
- job_name: node-exporter
|
||||
static_configs:
|
||||
- targets: ['node-exporter:9100']
|
||||
|
||||
- job_name: cadvisor
|
||||
static_configs:
|
||||
- targets: ['cadvisor:8080']
|
||||
|
||||
- job_name: promtail
|
||||
static_configs:
|
||||
- targets: ['promtail:9080']
|
||||
|
||||
- job_name: blackbox-http
|
||||
metrics_path: /probe
|
||||
params:
|
||||
module: [http_2xx]
|
||||
static_configs:
|
||||
- targets:
|
||||
- http://nginx/api/v1/health/version
|
||||
- http://nginx/api/v1/config/ai
|
||||
- http://nginx/api/v1/v2.2/requirements?productId=__smoke__&limit=1
|
||||
relabel_configs:
|
||||
- source_labels: [__address__]
|
||||
target_label: __param_target
|
||||
- source_labels: [__param_target]
|
||||
target_label: instance
|
||||
- target_label: __address__
|
||||
replacement: blackbox-exporter:9115
|
||||
|
||||
Reference in New Issue
Block a user