feat(ops): 添加生产监控告警基线
- 新增 Prometheus/Grafana/Loki/Promtail 监控 profile\n- 覆盖 DB、磁盘、慢 API、慢 Prisma、任务失败和小宝摘要 stale 告警\n- 补充 postgres-exporter 自定义查询、Dashboard、部署文档和校验\n\nCo-Authored-By: GPT-5 Codex <codex@openai.com>
This commit is contained in:
@@ -39,3 +39,10 @@ SMTP_HOST=
|
||||
SMTP_PORT=465
|
||||
SMTP_USER=
|
||||
SMTP_PASS=
|
||||
|
||||
# Optional monitoring profile. Do not commit real production passwords.
|
||||
PROMETHEUS_PORT=9090
|
||||
PROMETHEUS_RETENTION=15d
|
||||
GRAFANA_PORT=3002
|
||||
GRAFANA_ADMIN_USER=admin
|
||||
GRAFANA_ADMIN_PASSWORD=change-me-monitoring-password
|
||||
|
||||
43
deploy/monitoring/README.md
Normal file
43
deploy/monitoring/README.md
Normal file
@@ -0,0 +1,43 @@
|
||||
# FTB Production Monitoring Baseline
|
||||
|
||||
This profile adds a deployable Prometheus/Grafana baseline for production operations. It is intentionally secret-free: no webhook URLs, API keys, SMTP passwords, or real alert receiver credentials are committed.
|
||||
|
||||
## Start
|
||||
|
||||
```bash
|
||||
docker compose --env-file .env.production -f docker-compose.prod.yml --profile monitoring up -d
|
||||
```
|
||||
|
||||
Default local ports:
|
||||
|
||||
- Prometheus: `http://localhost:9090`
|
||||
- Grafana: `http://localhost:3002`
|
||||
- Loki: internal only
|
||||
|
||||
Set `GRAFANA_ADMIN_USER` and `GRAFANA_ADMIN_PASSWORD` in `.env.production` before exposing Grafana beyond localhost. Keep real alert receivers in the server environment or an untracked Alertmanager file.
|
||||
|
||||
## Coverage
|
||||
|
||||
- DB availability: `pg_up` from postgres-exporter.
|
||||
- Disk pressure: root filesystem availability from node-exporter.
|
||||
- Slow API requests: Promtail turns `Slow API request` server logs into `ftb_slow_api_log_total`.
|
||||
- Slow Prisma queries: Promtail turns `Slow Prisma query` server logs into `ftb_slow_prisma_log_total`.
|
||||
- Job failures: Promtail turns `AppData relation sync failed` and AI call failure logs into `ftb_job_failure_log_total`.
|
||||
- Xiaobao stale summaries: postgres-exporter custom query exposes `ftb_xiaobao_stale_summary_count` from `xiaobao_risk_summaries`.
|
||||
|
||||
## Alerts
|
||||
|
||||
Prometheus loads `prometheus/alert-rules.yml`. The rules evaluate locally and are visible in Prometheus/Grafana. To send notifications, add Alertmanager outside git or mount an environment-specific receiver file; do not commit webhook URLs or tokens.
|
||||
|
||||
Baseline alert names:
|
||||
|
||||
- `FtbPostgresDown`
|
||||
- `FtbDiskPressure`
|
||||
- `FtbSlowApiLogBurst`
|
||||
- `FtbSlowPrismaLogBurst`
|
||||
- `FtbJobFailureLogBurst`
|
||||
- `FtbXiaobaoSummaryStale`
|
||||
|
||||
## Log Search
|
||||
|
||||
Promtail ships Docker logs to Loki with container labels. Grafana provisions both Prometheus and Loki data sources, so on-call checks can move from a firing alert to matching server logs without SSHing into the host.
|
||||
10
deploy/monitoring/blackbox/config.yml
Normal file
10
deploy/monitoring/blackbox/config.yml
Normal file
@@ -0,0 +1,10 @@
|
||||
modules:
|
||||
http_2xx:
|
||||
prober: http
|
||||
timeout: 5s
|
||||
http:
|
||||
valid_http_versions: ['HTTP/1.1', 'HTTP/2.0']
|
||||
valid_status_codes: []
|
||||
method: GET
|
||||
preferred_ip_protocol: ip4
|
||||
|
||||
@@ -0,0 +1,78 @@
|
||||
{
|
||||
"uid": "ftb-production-overview",
|
||||
"title": "FTB Production Overview",
|
||||
"schemaVersion": 39,
|
||||
"version": 1,
|
||||
"refresh": "30s",
|
||||
"tags": ["ftb", "production", "v2.8"],
|
||||
"time": {
|
||||
"from": "now-6h",
|
||||
"to": "now"
|
||||
},
|
||||
"panels": [
|
||||
{
|
||||
"id": 1,
|
||||
"type": "stat",
|
||||
"title": "DB Availability",
|
||||
"gridPos": { "x": 0, "y": 0, "w": 6, "h": 4 },
|
||||
"targets": [
|
||||
{ "datasource": { "type": "prometheus", "uid": "Prometheus" }, "expr": "pg_up", "refId": "A" }
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"type": "timeseries",
|
||||
"title": "Slow API Logs",
|
||||
"gridPos": { "x": 6, "y": 0, "w": 6, "h": 4 },
|
||||
"targets": [
|
||||
{ "datasource": { "type": "prometheus", "uid": "Prometheus" }, "expr": "increase(ftb_slow_api_log_total[10m])", "refId": "A" }
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"type": "timeseries",
|
||||
"title": "Slow Prisma Logs",
|
||||
"gridPos": { "x": 12, "y": 0, "w": 6, "h": 4 },
|
||||
"targets": [
|
||||
{ "datasource": { "type": "prometheus", "uid": "Prometheus" }, "expr": "increase(ftb_slow_prisma_log_total[10m])", "refId": "A" }
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 4,
|
||||
"type": "stat",
|
||||
"title": "Xiaobao Stale Summaries",
|
||||
"gridPos": { "x": 18, "y": 0, "w": 6, "h": 4 },
|
||||
"targets": [
|
||||
{ "datasource": { "type": "prometheus", "uid": "Prometheus" }, "expr": "ftb_xiaobao_stale_summary_count", "refId": "A" }
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 5,
|
||||
"type": "timeseries",
|
||||
"title": "Job Failure Logs",
|
||||
"gridPos": { "x": 0, "y": 4, "w": 8, "h": 5 },
|
||||
"targets": [
|
||||
{ "datasource": { "type": "prometheus", "uid": "Prometheus" }, "expr": "increase(ftb_job_failure_log_total[10m])", "refId": "A" }
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 6,
|
||||
"type": "stat",
|
||||
"title": "Root Disk Free %",
|
||||
"gridPos": { "x": 8, "y": 4, "w": 8, "h": 5 },
|
||||
"targets": [
|
||||
{ "datasource": { "type": "prometheus", "uid": "Prometheus" }, "expr": "100 * node_filesystem_avail_bytes{mountpoint=\"/\",fstype!~\"tmpfs|overlay\"} / node_filesystem_size_bytes{mountpoint=\"/\",fstype!~\"tmpfs|overlay\"}", "refId": "A" }
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 7,
|
||||
"type": "logs",
|
||||
"title": "Server Warning/Error Logs",
|
||||
"gridPos": { "x": 16, "y": 4, "w": 8, "h": 5 },
|
||||
"targets": [
|
||||
{ "datasource": { "type": "loki", "uid": "Loki" }, "expr": "{compose_service=\"server\"} |~ \"warn|error|失败|Slow\"", "refId": "A" }
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
@@ -0,0 +1,12 @@
|
||||
apiVersion: 1
|
||||
|
||||
providers:
|
||||
- name: FTB Production
|
||||
orgId: 1
|
||||
folder: FTB
|
||||
type: file
|
||||
disableDeletion: false
|
||||
updateIntervalSeconds: 30
|
||||
options:
|
||||
path: /var/lib/grafana/dashboards
|
||||
|
||||
@@ -0,0 +1,16 @@
|
||||
apiVersion: 1
|
||||
|
||||
datasources:
|
||||
- name: Prometheus
|
||||
type: prometheus
|
||||
access: proxy
|
||||
url: http://prometheus:9090
|
||||
isDefault: true
|
||||
editable: false
|
||||
|
||||
- name: Loki
|
||||
type: loki
|
||||
access: proxy
|
||||
url: http://loki:3100
|
||||
editable: false
|
||||
|
||||
32
deploy/monitoring/loki/config.yml
Normal file
32
deploy/monitoring/loki/config.yml
Normal file
@@ -0,0 +1,32 @@
|
||||
auth_enabled: false
|
||||
|
||||
server:
|
||||
http_listen_port: 3100
|
||||
|
||||
common:
|
||||
path_prefix: /loki
|
||||
replication_factor: 1
|
||||
ring:
|
||||
kvstore:
|
||||
store: inmemory
|
||||
|
||||
schema_config:
|
||||
configs:
|
||||
- from: 2026-01-01
|
||||
store: tsdb
|
||||
object_store: filesystem
|
||||
schema: v13
|
||||
index:
|
||||
prefix: index_
|
||||
period: 24h
|
||||
|
||||
storage_config:
|
||||
tsdb_shipper:
|
||||
active_index_directory: /loki/index
|
||||
cache_location: /loki/index_cache
|
||||
filesystem:
|
||||
directory: /loki/chunks
|
||||
|
||||
limits_config:
|
||||
retention_period: 168h
|
||||
|
||||
12
deploy/monitoring/postgres/postgres-queries.yml
Normal file
12
deploy/monitoring/postgres/postgres-queries.yml
Normal file
@@ -0,0 +1,12 @@
|
||||
ftb_xiaobao:
|
||||
query: |
|
||||
SELECT
|
||||
count(*)::float AS stale_summary_count
|
||||
FROM xiaobao_risk_summaries
|
||||
WHERE dirty = true
|
||||
OR updated_at < now() - interval '6 hours';
|
||||
metrics:
|
||||
- stale_summary_count:
|
||||
usage: GAUGE
|
||||
description: Xiaobao risk summaries that are dirty or older than 6 hours.
|
||||
|
||||
62
deploy/monitoring/prometheus/alert-rules.yml
Normal file
62
deploy/monitoring/prometheus/alert-rules.yml
Normal file
@@ -0,0 +1,62 @@
|
||||
groups:
|
||||
- name: ftb-production-alerts
|
||||
rules:
|
||||
- alert: FtbPostgresDown
|
||||
expr: pg_up == 0
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: PostgreSQL exporter cannot reach the FTB database.
|
||||
runbook: docs/runbooks/migration-rollback.md
|
||||
|
||||
- alert: FtbDiskPressure
|
||||
expr: |
|
||||
(
|
||||
node_filesystem_avail_bytes{mountpoint="/",fstype!~"tmpfs|overlay"}
|
||||
/
|
||||
node_filesystem_size_bytes{mountpoint="/",fstype!~"tmpfs|overlay"}
|
||||
) < 0.15
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: Production host root filesystem has less than 15% free space.
|
||||
runbook: docs/deployment.md
|
||||
|
||||
- alert: FtbSlowApiLogBurst
|
||||
expr: increase(ftb_slow_api_log_total[10m]) > 5
|
||||
for: 2m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: Slow API request log volume crossed the V2.8 baseline threshold.
|
||||
runbook: docs/deployment.md
|
||||
|
||||
- alert: FtbSlowPrismaLogBurst
|
||||
expr: increase(ftb_slow_prisma_log_total[10m]) > 3
|
||||
for: 2m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: Slow Prisma query log volume crossed the V2.8 baseline threshold.
|
||||
runbook: docs/deployment.md
|
||||
|
||||
- alert: FtbJobFailureLogBurst
|
||||
expr: increase(ftb_job_failure_log_total[10m]) > 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: Background or compatibility job failure logs were detected.
|
||||
runbook: docs/runbooks/xiaobao-background-jobs.md
|
||||
|
||||
- alert: FtbXiaobaoSummaryStale
|
||||
expr: ftb_xiaobao_stale_summary_count > 0
|
||||
for: 15m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: Xiaobao warning summaries are dirty or stale.
|
||||
runbook: docs/runbooks/xiaobao-background-jobs.md
|
||||
|
||||
45
deploy/monitoring/prometheus/prometheus.yml
Normal file
45
deploy/monitoring/prometheus/prometheus.yml
Normal file
@@ -0,0 +1,45 @@
|
||||
global:
|
||||
scrape_interval: 15s
|
||||
evaluation_interval: 15s
|
||||
|
||||
rule_files:
|
||||
- /etc/prometheus/alert-rules.yml
|
||||
|
||||
scrape_configs:
|
||||
- job_name: prometheus
|
||||
static_configs:
|
||||
- targets: ['prometheus:9090']
|
||||
|
||||
- job_name: postgres-exporter
|
||||
static_configs:
|
||||
- targets: ['postgres-exporter:9187']
|
||||
|
||||
- job_name: node-exporter
|
||||
static_configs:
|
||||
- targets: ['node-exporter:9100']
|
||||
|
||||
- job_name: cadvisor
|
||||
static_configs:
|
||||
- targets: ['cadvisor:8080']
|
||||
|
||||
- job_name: promtail
|
||||
static_configs:
|
||||
- targets: ['promtail:9080']
|
||||
|
||||
- job_name: blackbox-http
|
||||
metrics_path: /probe
|
||||
params:
|
||||
module: [http_2xx]
|
||||
static_configs:
|
||||
- targets:
|
||||
- http://nginx/api/v1/health/version
|
||||
- http://nginx/api/v1/config/ai
|
||||
- http://nginx/api/v1/v2.2/requirements?productId=__smoke__&limit=1
|
||||
relabel_configs:
|
||||
- source_labels: [__address__]
|
||||
target_label: __param_target
|
||||
- source_labels: [__param_target]
|
||||
target_label: instance
|
||||
- target_label: __address__
|
||||
replacement: blackbox-exporter:9115
|
||||
|
||||
53
deploy/monitoring/promtail/config.yml
Normal file
53
deploy/monitoring/promtail/config.yml
Normal file
@@ -0,0 +1,53 @@
|
||||
server:
|
||||
http_listen_port: 9080
|
||||
grpc_listen_port: 0
|
||||
|
||||
positions:
|
||||
filename: /tmp/positions.yml
|
||||
|
||||
clients:
|
||||
- url: http://loki:3100/loki/api/v1/push
|
||||
|
||||
scrape_configs:
|
||||
- job_name: docker
|
||||
docker_sd_configs:
|
||||
- host: unix:///var/run/docker.sock
|
||||
refresh_interval: 15s
|
||||
relabel_configs:
|
||||
- source_labels: ['__meta_docker_container_name']
|
||||
regex: '/(.*)'
|
||||
target_label: container
|
||||
- source_labels: ['__meta_docker_container_label_com_docker_compose_service']
|
||||
target_label: compose_service
|
||||
- source_labels: ['__meta_docker_container_label_com_docker_compose_project']
|
||||
target_label: compose_project
|
||||
pipeline_stages:
|
||||
- docker: {}
|
||||
- match:
|
||||
selector: '{compose_service="server"} |= "Slow API request"'
|
||||
stages:
|
||||
- metrics:
|
||||
ftb_slow_api_log_total:
|
||||
type: Counter
|
||||
description: Slow API request log entries emitted by the NestJS server.
|
||||
config:
|
||||
action: inc
|
||||
- match:
|
||||
selector: '{compose_service="server"} |= "Slow Prisma query"'
|
||||
stages:
|
||||
- metrics:
|
||||
ftb_slow_prisma_log_total:
|
||||
type: Counter
|
||||
description: Slow Prisma query log entries emitted by the NestJS server.
|
||||
config:
|
||||
action: inc
|
||||
- match:
|
||||
selector: '{compose_service="server"} |~ "AppData relation sync failed|AI 调用失败|AI 风险解读调用失败"'
|
||||
stages:
|
||||
- metrics:
|
||||
ftb_job_failure_log_total:
|
||||
type: Counter
|
||||
description: Compatibility sync, AI job, or background task failure logs.
|
||||
config:
|
||||
action: inc
|
||||
|
||||
@@ -123,7 +123,117 @@ services:
|
||||
server:
|
||||
condition: service_healthy
|
||||
|
||||
prometheus:
|
||||
image: prom/prometheus:v2.53.1
|
||||
profiles: ['monitoring']
|
||||
restart: unless-stopped
|
||||
command:
|
||||
- '--config.file=/etc/prometheus/prometheus.yml'
|
||||
- '--storage.tsdb.path=/prometheus'
|
||||
- '--storage.tsdb.retention.time=${PROMETHEUS_RETENTION:-15d}'
|
||||
- '--web.enable-lifecycle'
|
||||
ports:
|
||||
- '${PROMETHEUS_PORT:-9090}:9090'
|
||||
volumes:
|
||||
- ./deploy/monitoring/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
|
||||
- ./deploy/monitoring/prometheus/alert-rules.yml:/etc/prometheus/alert-rules.yml:ro
|
||||
- prometheus_data:/prometheus
|
||||
depends_on:
|
||||
- postgres-exporter
|
||||
- node-exporter
|
||||
- cadvisor
|
||||
- promtail
|
||||
- blackbox-exporter
|
||||
|
||||
grafana:
|
||||
image: grafana/grafana:11.1.0
|
||||
profiles: ['monitoring']
|
||||
restart: unless-stopped
|
||||
ports:
|
||||
- '${GRAFANA_PORT:-3002}:3000'
|
||||
environment:
|
||||
GF_SECURITY_ADMIN_USER: ${GRAFANA_ADMIN_USER:-admin}
|
||||
GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD:-change-me-monitoring-password}
|
||||
GF_USERS_ALLOW_SIGN_UP: 'false'
|
||||
volumes:
|
||||
- grafana_data:/var/lib/grafana
|
||||
- ./deploy/monitoring/grafana/provisioning:/etc/grafana/provisioning:ro
|
||||
- ./deploy/monitoring/grafana/dashboards:/var/lib/grafana/dashboards:ro
|
||||
depends_on:
|
||||
- prometheus
|
||||
- loki
|
||||
|
||||
loki:
|
||||
image: grafana/loki:2.9.8
|
||||
profiles: ['monitoring']
|
||||
restart: unless-stopped
|
||||
command: ['-config.file=/etc/loki/config.yml']
|
||||
volumes:
|
||||
- ./deploy/monitoring/loki/config.yml:/etc/loki/config.yml:ro
|
||||
- loki_data:/loki
|
||||
|
||||
promtail:
|
||||
image: grafana/promtail:2.9.8
|
||||
profiles: ['monitoring']
|
||||
restart: unless-stopped
|
||||
command: ['-config.file=/etc/promtail/config.yml']
|
||||
volumes:
|
||||
- ./deploy/monitoring/promtail/config.yml:/etc/promtail/config.yml:ro
|
||||
- /var/lib/docker/containers:/var/lib/docker/containers:ro
|
||||
- /var/run/docker.sock:/var/run/docker.sock:ro
|
||||
depends_on:
|
||||
- loki
|
||||
|
||||
postgres-exporter:
|
||||
image: quay.io/prometheuscommunity/postgres-exporter:v0.15.0
|
||||
profiles: ['monitoring']
|
||||
restart: unless-stopped
|
||||
command:
|
||||
- '--extend.query-path=/etc/postgres-exporter/postgres-queries.yml'
|
||||
environment:
|
||||
DATA_SOURCE_NAME: postgresql://${POSTGRES_USER:-postgres}:${POSTGRES_PASSWORD}@postgres:5432/${POSTGRES_DB:-ftb_pm}?sslmode=disable
|
||||
volumes:
|
||||
- ./deploy/monitoring/postgres/postgres-queries.yml:/etc/postgres-exporter/postgres-queries.yml:ro
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
|
||||
node-exporter:
|
||||
image: prom/node-exporter:v1.8.2
|
||||
profiles: ['monitoring']
|
||||
restart: unless-stopped
|
||||
command:
|
||||
- '--path.rootfs=/host'
|
||||
volumes:
|
||||
- /:/host:ro,rslave
|
||||
|
||||
cadvisor:
|
||||
image: gcr.io/cadvisor/cadvisor:v0.49.1
|
||||
profiles: ['monitoring']
|
||||
restart: unless-stopped
|
||||
privileged: true
|
||||
devices:
|
||||
- /dev/kmsg:/dev/kmsg
|
||||
volumes:
|
||||
- /:/rootfs:ro
|
||||
- /var/run:/var/run:ro
|
||||
- /sys:/sys:ro
|
||||
- /var/lib/docker/:/var/lib/docker:ro
|
||||
- /dev/disk/:/dev/disk:ro
|
||||
|
||||
blackbox-exporter:
|
||||
image: prom/blackbox-exporter:v0.25.0
|
||||
profiles: ['monitoring']
|
||||
restart: unless-stopped
|
||||
command:
|
||||
- '--config.file=/etc/blackbox/config.yml'
|
||||
volumes:
|
||||
- ./deploy/monitoring/blackbox/config.yml:/etc/blackbox/config.yml:ro
|
||||
|
||||
volumes:
|
||||
postgres_data:
|
||||
redis_data:
|
||||
server_data:
|
||||
prometheus_data:
|
||||
grafana_data:
|
||||
loki_data:
|
||||
|
||||
@@ -194,6 +194,30 @@ pnpm deploy:smoke -- --base-url http://localhost --expected-version <expected-co
|
||||
|
||||
生产 Compose 默认只监听 HTTP 80。HTTPS 建议优先交给云负载均衡、CDN 或宿主机外层证书管理工具;如果要让本 Compose 内的 Nginx 直接处理 HTTPS,可以在后续增加证书 volume 和 443 server block。
|
||||
|
||||
## 监控与告警基线
|
||||
|
||||
V2.8 提供可选 `monitoring` profile,不影响默认生产启动。启用前先在 `.env.production` 设置 `GRAFANA_ADMIN_PASSWORD`,不要使用示例密码对外暴露 Grafana。
|
||||
|
||||
```bash
|
||||
docker compose --env-file .env.production -f docker-compose.prod.yml --profile monitoring up -d
|
||||
```
|
||||
|
||||
默认入口:
|
||||
|
||||
- Prometheus: `http://localhost:9090`
|
||||
- Grafana: `http://localhost:3002`
|
||||
|
||||
配置目录在 `deploy/monitoring/`。基线覆盖:
|
||||
|
||||
- `FtbPostgresDown`:PostgreSQL 不可用。
|
||||
- `FtbDiskPressure`:宿主机根分区低于 15% 可用空间。
|
||||
- `FtbSlowApiLogBurst`:服务端慢 API 日志在 10 分钟内超过阈值。
|
||||
- `FtbSlowPrismaLogBurst`:慢 Prisma 查询日志在 10 分钟内超过阈值。
|
||||
- `FtbJobFailureLogBurst`:AppData 同步、AI 调用或后台任务失败日志出现。
|
||||
- `FtbXiaobaoSummaryStale`:`xiaobao_risk_summaries` 存在 dirty 或超过 6 小时未更新的摘要。
|
||||
|
||||
Prometheus 只加载本地规则,不提交真实通知密钥。接入 Slack、企业微信、邮件等通知时,把 Alertmanager receiver 放在未跟踪的服务器文件或环境变量中。
|
||||
|
||||
## 数据持久化
|
||||
|
||||
生产 Compose 使用三个命名 volume:
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import { mkdtempSync, writeFileSync, existsSync } from 'node:fs';
|
||||
import { mkdtempSync, writeFileSync, existsSync, readFileSync } from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join, resolve } from 'node:path';
|
||||
import { describe, it } from 'node:test';
|
||||
@@ -97,4 +97,45 @@ describe('production ops scripts', () => {
|
||||
assert.match(result.stdout, /\/api\/v1\/v2\.2\/requirements\?productId=__smoke__/);
|
||||
assert.match(result.stdout, /\/api\/v1\/config\/ai/);
|
||||
});
|
||||
|
||||
it('declares the production monitoring baseline without real alert secrets', () => {
|
||||
const alertRules = readFileSync(
|
||||
join(root, 'deploy/monitoring/prometheus/alert-rules.yml'),
|
||||
'utf8',
|
||||
);
|
||||
const postgresQueries = readFileSync(
|
||||
join(root, 'deploy/monitoring/postgres/postgres-queries.yml'),
|
||||
'utf8',
|
||||
);
|
||||
const promtailConfig = readFileSync(
|
||||
join(root, 'deploy/monitoring/promtail/config.yml'),
|
||||
'utf8',
|
||||
);
|
||||
const dashboard = readFileSync(
|
||||
join(root, 'deploy/monitoring/grafana/dashboards/ftb-production-overview.json'),
|
||||
'utf8',
|
||||
);
|
||||
|
||||
for (const alertName of [
|
||||
'FtbPostgresDown',
|
||||
'FtbDiskPressure',
|
||||
'FtbSlowApiLogBurst',
|
||||
'FtbSlowPrismaLogBurst',
|
||||
'FtbJobFailureLogBurst',
|
||||
'FtbXiaobaoSummaryStale',
|
||||
]) {
|
||||
assert.match(alertRules, new RegExp(alertName));
|
||||
}
|
||||
|
||||
assert.match(postgresQueries, /xiaobao_risk_summaries/);
|
||||
assert.match(postgresQueries, /dirty = true/);
|
||||
assert.match(promtailConfig, /Slow API request/);
|
||||
assert.match(promtailConfig, /Slow Prisma query/);
|
||||
assert.match(promtailConfig, /AppData relation sync failed/);
|
||||
assert.match(dashboard, /FTB Production Overview/);
|
||||
|
||||
const combined = `${alertRules}\n${postgresQueries}\n${promtailConfig}\n${dashboard}`;
|
||||
assert.doesNotMatch(combined, /sk-ant-[A-Za-z0-9]/);
|
||||
assert.doesNotMatch(combined, /hooks\.slack\.com\/services\//);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -51,6 +51,11 @@ const checks = [
|
||||
'WEB_IMAGE',
|
||||
'APP_VERSION',
|
||||
'/api/v1/health/version',
|
||||
'prometheus:',
|
||||
"profiles: ['monitoring']",
|
||||
'postgres-exporter:',
|
||||
'blackbox-exporter:',
|
||||
'prometheus_data:',
|
||||
],
|
||||
},
|
||||
{
|
||||
@@ -110,6 +115,41 @@ const checks = [
|
||||
'X-Forwarded-Proto',
|
||||
],
|
||||
},
|
||||
{
|
||||
file: 'deploy/monitoring/README.md',
|
||||
snippets: [
|
||||
'--profile monitoring',
|
||||
'FtbPostgresDown',
|
||||
'FtbXiaobaoSummaryStale',
|
||||
'no webhook URLs',
|
||||
],
|
||||
},
|
||||
{
|
||||
file: 'deploy/monitoring/prometheus/prometheus.yml',
|
||||
snippets: ['postgres-exporter:9187', 'promtail:9080', 'blackbox-http'],
|
||||
},
|
||||
{
|
||||
file: 'deploy/monitoring/prometheus/alert-rules.yml',
|
||||
snippets: [
|
||||
'FtbPostgresDown',
|
||||
'FtbSlowApiLogBurst',
|
||||
'FtbSlowPrismaLogBurst',
|
||||
'FtbJobFailureLogBurst',
|
||||
'FtbXiaobaoSummaryStale',
|
||||
],
|
||||
},
|
||||
{
|
||||
file: 'deploy/monitoring/postgres/postgres-queries.yml',
|
||||
snippets: ['xiaobao_risk_summaries', 'dirty = true', 'stale_summary_count'],
|
||||
},
|
||||
{
|
||||
file: 'deploy/monitoring/promtail/config.yml',
|
||||
snippets: ['Slow API request', 'Slow Prisma query', 'AppData relation sync failed'],
|
||||
},
|
||||
{
|
||||
file: 'deploy/monitoring/grafana/dashboards/ftb-production-overview.json',
|
||||
snippets: ['FTB Production Overview', 'ftb_xiaobao_stale_summary_count', 'Server Warning/Error Logs'],
|
||||
},
|
||||
{
|
||||
file: '.env.production.example',
|
||||
snippets: [
|
||||
@@ -143,6 +183,7 @@ const checks = [
|
||||
'pnpm backup:postgres',
|
||||
'pnpm restore:postgres',
|
||||
'pnpm backup:server-data',
|
||||
'--profile monitoring',
|
||||
'pnpm db:migrate',
|
||||
'NEXT_PUBLIC_API_URL',
|
||||
'Nginx',
|
||||
|
||||
Reference in New Issue
Block a user