diff --git a/.env.production.example b/.env.production.example index d8c423f..512d929 100644 --- a/.env.production.example +++ b/.env.production.example @@ -39,3 +39,10 @@ SMTP_HOST= SMTP_PORT=465 SMTP_USER= SMTP_PASS= + +# Optional monitoring profile. Do not commit real production passwords. +PROMETHEUS_PORT=9090 +PROMETHEUS_RETENTION=15d +GRAFANA_PORT=3002 +GRAFANA_ADMIN_USER=admin +GRAFANA_ADMIN_PASSWORD=change-me-monitoring-password diff --git a/.github/workflows/deploy-production.yml b/.github/workflows/deploy-production.yml index 9fa0e97..a041693 100644 --- a/.github/workflows/deploy-production.yml +++ b/.github/workflows/deploy-production.yml @@ -123,26 +123,11 @@ jobs: docker compose --env-file .env.production -f docker-compose.prod.yml up -d --remove-orphans for attempt in $(seq 1 30); do - if docker compose --env-file .env.production -f docker-compose.prod.yml exec -T web node -e " - const expected = process.argv[1]; - fetch('http://nginx/api/v1/health/version') - .then(async (response) => { - if (!response.ok) throw new Error('HTTP ' + response.status); - const payload = await response.json(); - if (payload.version !== expected) { - throw new Error('Expected ' + expected + ', got ' + payload.version); - } - console.log('Runtime version verified: ' + payload.version); - }) - .catch((error) => { - console.error(error.message); - process.exit(1); - }); - " "${{ github.sha }}"; then + if docker compose --env-file .env.production -f docker-compose.prod.yml exec -T web node scripts/smoke-test-release.mjs --base-url http://nginx --expected-version "${{ github.sha }}"; then exit 0 fi sleep 2 done - echo "Runtime version check failed after retries" + echo "Release smoke check failed after retries" exit 1 diff --git a/.gitignore b/.gitignore index a981be7..091f408 100644 --- a/.gitignore +++ b/.gitignore @@ -16,3 +16,4 @@ next-env.d.ts apps/server/data/ .worktrees/ appdata-archive-*.json +backups/ diff --git a/Dockerfile.web b/Dockerfile.web index fe59920..c731c6b 100644 --- a/Dockerfile.web +++ b/Dockerfile.web @@ -57,6 +57,7 @@ COPY --from=builder /app/turbo.json ./turbo.json COPY --from=builder /app/node_modules ./node_modules COPY --from=builder /app/packages/shared ./packages/shared COPY --from=builder /app/apps/web ./apps/web +COPY scripts ./scripts RUN chown -R node:node /app USER node EXPOSE 3000 diff --git a/deploy/monitoring/README.md b/deploy/monitoring/README.md new file mode 100644 index 0000000..e54dca6 --- /dev/null +++ b/deploy/monitoring/README.md @@ -0,0 +1,43 @@ +# FTB Production Monitoring Baseline + +This profile adds a deployable Prometheus/Grafana baseline for production operations. It is intentionally secret-free: no webhook URLs, API keys, SMTP passwords, or real alert receiver credentials are committed. + +## Start + +```bash +docker compose --env-file .env.production -f docker-compose.prod.yml --profile monitoring up -d +``` + +Default local ports: + +- Prometheus: `http://localhost:9090` +- Grafana: `http://localhost:3002` +- Loki: internal only + +Set `GRAFANA_ADMIN_USER` and `GRAFANA_ADMIN_PASSWORD` in `.env.production` before exposing Grafana beyond localhost. Keep real alert receivers in the server environment or an untracked Alertmanager file. + +## Coverage + +- DB availability: `pg_up` from postgres-exporter. +- Disk pressure: root filesystem availability from node-exporter. +- Slow API requests: Promtail turns `Slow API request` server logs into `ftb_slow_api_log_total`. +- Slow Prisma queries: Promtail turns `Slow Prisma query` server logs into `ftb_slow_prisma_log_total`. +- Job failures: Promtail turns `AppData relation sync failed` and AI call failure logs into `ftb_job_failure_log_total`. +- Xiaobao stale summaries: postgres-exporter custom query exposes `ftb_xiaobao_stale_summary_count` from `xiaobao_risk_summaries`. + +## Alerts + +Prometheus loads `prometheus/alert-rules.yml`. The rules evaluate locally and are visible in Prometheus/Grafana. To send notifications, add Alertmanager outside git or mount an environment-specific receiver file; do not commit webhook URLs or tokens. + +Baseline alert names: + +- `FtbPostgresDown` +- `FtbDiskPressure` +- `FtbSlowApiLogBurst` +- `FtbSlowPrismaLogBurst` +- `FtbJobFailureLogBurst` +- `FtbXiaobaoSummaryStale` + +## Log Search + +Promtail ships Docker logs to Loki with container labels. Grafana provisions both Prometheus and Loki data sources, so on-call checks can move from a firing alert to matching server logs without SSHing into the host. diff --git a/deploy/monitoring/blackbox/config.yml b/deploy/monitoring/blackbox/config.yml new file mode 100644 index 0000000..fa81bee --- /dev/null +++ b/deploy/monitoring/blackbox/config.yml @@ -0,0 +1,10 @@ +modules: + http_2xx: + prober: http + timeout: 5s + http: + valid_http_versions: ['HTTP/1.1', 'HTTP/2.0'] + valid_status_codes: [] + method: GET + preferred_ip_protocol: ip4 + diff --git a/deploy/monitoring/grafana/dashboards/ftb-production-overview.json b/deploy/monitoring/grafana/dashboards/ftb-production-overview.json new file mode 100644 index 0000000..0a753d3 --- /dev/null +++ b/deploy/monitoring/grafana/dashboards/ftb-production-overview.json @@ -0,0 +1,78 @@ +{ + "uid": "ftb-production-overview", + "title": "FTB Production Overview", + "schemaVersion": 39, + "version": 1, + "refresh": "30s", + "tags": ["ftb", "production", "v2.8"], + "time": { + "from": "now-6h", + "to": "now" + }, + "panels": [ + { + "id": 1, + "type": "stat", + "title": "DB Availability", + "gridPos": { "x": 0, "y": 0, "w": 6, "h": 4 }, + "targets": [ + { "datasource": { "type": "prometheus", "uid": "Prometheus" }, "expr": "pg_up", "refId": "A" } + ] + }, + { + "id": 2, + "type": "timeseries", + "title": "Slow API Logs", + "gridPos": { "x": 6, "y": 0, "w": 6, "h": 4 }, + "targets": [ + { "datasource": { "type": "prometheus", "uid": "Prometheus" }, "expr": "increase(ftb_slow_api_log_total[10m])", "refId": "A" } + ] + }, + { + "id": 3, + "type": "timeseries", + "title": "Slow Prisma Logs", + "gridPos": { "x": 12, "y": 0, "w": 6, "h": 4 }, + "targets": [ + { "datasource": { "type": "prometheus", "uid": "Prometheus" }, "expr": "increase(ftb_slow_prisma_log_total[10m])", "refId": "A" } + ] + }, + { + "id": 4, + "type": "stat", + "title": "Xiaobao Stale Summaries", + "gridPos": { "x": 18, "y": 0, "w": 6, "h": 4 }, + "targets": [ + { "datasource": { "type": "prometheus", "uid": "Prometheus" }, "expr": "ftb_xiaobao_stale_summary_count", "refId": "A" } + ] + }, + { + "id": 5, + "type": "timeseries", + "title": "Job Failure Logs", + "gridPos": { "x": 0, "y": 4, "w": 8, "h": 5 }, + "targets": [ + { "datasource": { "type": "prometheus", "uid": "Prometheus" }, "expr": "increase(ftb_job_failure_log_total[10m])", "refId": "A" } + ] + }, + { + "id": 6, + "type": "stat", + "title": "Root Disk Free %", + "gridPos": { "x": 8, "y": 4, "w": 8, "h": 5 }, + "targets": [ + { "datasource": { "type": "prometheus", "uid": "Prometheus" }, "expr": "100 * node_filesystem_avail_bytes{mountpoint=\"/\",fstype!~\"tmpfs|overlay\"} / node_filesystem_size_bytes{mountpoint=\"/\",fstype!~\"tmpfs|overlay\"}", "refId": "A" } + ] + }, + { + "id": 7, + "type": "logs", + "title": "Server Warning/Error Logs", + "gridPos": { "x": 16, "y": 4, "w": 8, "h": 5 }, + "targets": [ + { "datasource": { "type": "loki", "uid": "Loki" }, "expr": "{compose_service=\"server\"} |~ \"warn|error|失败|Slow\"", "refId": "A" } + ] + } + ] +} + diff --git a/deploy/monitoring/grafana/provisioning/dashboards/dashboards.yml b/deploy/monitoring/grafana/provisioning/dashboards/dashboards.yml new file mode 100644 index 0000000..adcba00 --- /dev/null +++ b/deploy/monitoring/grafana/provisioning/dashboards/dashboards.yml @@ -0,0 +1,12 @@ +apiVersion: 1 + +providers: + - name: FTB Production + orgId: 1 + folder: FTB + type: file + disableDeletion: false + updateIntervalSeconds: 30 + options: + path: /var/lib/grafana/dashboards + diff --git a/deploy/monitoring/grafana/provisioning/datasources/datasources.yml b/deploy/monitoring/grafana/provisioning/datasources/datasources.yml new file mode 100644 index 0000000..5ef7ccf --- /dev/null +++ b/deploy/monitoring/grafana/provisioning/datasources/datasources.yml @@ -0,0 +1,16 @@ +apiVersion: 1 + +datasources: + - name: Prometheus + type: prometheus + access: proxy + url: http://prometheus:9090 + isDefault: true + editable: false + + - name: Loki + type: loki + access: proxy + url: http://loki:3100 + editable: false + diff --git a/deploy/monitoring/loki/config.yml b/deploy/monitoring/loki/config.yml new file mode 100644 index 0000000..dad75da --- /dev/null +++ b/deploy/monitoring/loki/config.yml @@ -0,0 +1,32 @@ +auth_enabled: false + +server: + http_listen_port: 3100 + +common: + path_prefix: /loki + replication_factor: 1 + ring: + kvstore: + store: inmemory + +schema_config: + configs: + - from: 2026-01-01 + store: tsdb + object_store: filesystem + schema: v13 + index: + prefix: index_ + period: 24h + +storage_config: + tsdb_shipper: + active_index_directory: /loki/index + cache_location: /loki/index_cache + filesystem: + directory: /loki/chunks + +limits_config: + retention_period: 168h + diff --git a/deploy/monitoring/postgres/postgres-queries.yml b/deploy/monitoring/postgres/postgres-queries.yml new file mode 100644 index 0000000..8a19c94 --- /dev/null +++ b/deploy/monitoring/postgres/postgres-queries.yml @@ -0,0 +1,12 @@ +ftb_xiaobao: + query: | + SELECT + count(*)::float AS stale_summary_count + FROM xiaobao_risk_summaries + WHERE dirty = true + OR updated_at < now() - interval '6 hours'; + metrics: + - stale_summary_count: + usage: GAUGE + description: Xiaobao risk summaries that are dirty or older than 6 hours. + diff --git a/deploy/monitoring/prometheus/alert-rules.yml b/deploy/monitoring/prometheus/alert-rules.yml new file mode 100644 index 0000000..e3af268 --- /dev/null +++ b/deploy/monitoring/prometheus/alert-rules.yml @@ -0,0 +1,62 @@ +groups: + - name: ftb-production-alerts + rules: + - alert: FtbPostgresDown + expr: pg_up == 0 + for: 2m + labels: + severity: critical + annotations: + summary: PostgreSQL exporter cannot reach the FTB database. + runbook: docs/runbooks/migration-rollback.md + + - alert: FtbDiskPressure + expr: | + ( + node_filesystem_avail_bytes{mountpoint="/",fstype!~"tmpfs|overlay"} + / + node_filesystem_size_bytes{mountpoint="/",fstype!~"tmpfs|overlay"} + ) < 0.15 + for: 10m + labels: + severity: warning + annotations: + summary: Production host root filesystem has less than 15% free space. + runbook: docs/deployment.md + + - alert: FtbSlowApiLogBurst + expr: increase(ftb_slow_api_log_total[10m]) > 5 + for: 2m + labels: + severity: warning + annotations: + summary: Slow API request log volume crossed the V2.8 baseline threshold. + runbook: docs/deployment.md + + - alert: FtbSlowPrismaLogBurst + expr: increase(ftb_slow_prisma_log_total[10m]) > 3 + for: 2m + labels: + severity: warning + annotations: + summary: Slow Prisma query log volume crossed the V2.8 baseline threshold. + runbook: docs/deployment.md + + - alert: FtbJobFailureLogBurst + expr: increase(ftb_job_failure_log_total[10m]) > 0 + for: 1m + labels: + severity: warning + annotations: + summary: Background or compatibility job failure logs were detected. + runbook: docs/runbooks/xiaobao-background-jobs.md + + - alert: FtbXiaobaoSummaryStale + expr: ftb_xiaobao_stale_summary_count > 0 + for: 15m + labels: + severity: warning + annotations: + summary: Xiaobao warning summaries are dirty or stale. + runbook: docs/runbooks/xiaobao-background-jobs.md + diff --git a/deploy/monitoring/prometheus/prometheus.yml b/deploy/monitoring/prometheus/prometheus.yml new file mode 100644 index 0000000..b8b3b18 --- /dev/null +++ b/deploy/monitoring/prometheus/prometheus.yml @@ -0,0 +1,45 @@ +global: + scrape_interval: 15s + evaluation_interval: 15s + +rule_files: + - /etc/prometheus/alert-rules.yml + +scrape_configs: + - job_name: prometheus + static_configs: + - targets: ['prometheus:9090'] + + - job_name: postgres-exporter + static_configs: + - targets: ['postgres-exporter:9187'] + + - job_name: node-exporter + static_configs: + - targets: ['node-exporter:9100'] + + - job_name: cadvisor + static_configs: + - targets: ['cadvisor:8080'] + + - job_name: promtail + static_configs: + - targets: ['promtail:9080'] + + - job_name: blackbox-http + metrics_path: /probe + params: + module: [http_2xx] + static_configs: + - targets: + - http://nginx/api/v1/health/version + - http://nginx/api/v1/config/ai + - http://nginx/api/v1/v2.2/requirements?productId=__smoke__&limit=1 + relabel_configs: + - source_labels: [__address__] + target_label: __param_target + - source_labels: [__param_target] + target_label: instance + - target_label: __address__ + replacement: blackbox-exporter:9115 + diff --git a/deploy/monitoring/promtail/config.yml b/deploy/monitoring/promtail/config.yml new file mode 100644 index 0000000..6ebb2d7 --- /dev/null +++ b/deploy/monitoring/promtail/config.yml @@ -0,0 +1,53 @@ +server: + http_listen_port: 9080 + grpc_listen_port: 0 + +positions: + filename: /tmp/positions.yml + +clients: + - url: http://loki:3100/loki/api/v1/push + +scrape_configs: + - job_name: docker + docker_sd_configs: + - host: unix:///var/run/docker.sock + refresh_interval: 15s + relabel_configs: + - source_labels: ['__meta_docker_container_name'] + regex: '/(.*)' + target_label: container + - source_labels: ['__meta_docker_container_label_com_docker_compose_service'] + target_label: compose_service + - source_labels: ['__meta_docker_container_label_com_docker_compose_project'] + target_label: compose_project + pipeline_stages: + - docker: {} + - match: + selector: '{compose_service="server"} |= "Slow API request"' + stages: + - metrics: + ftb_slow_api_log_total: + type: Counter + description: Slow API request log entries emitted by the NestJS server. + config: + action: inc + - match: + selector: '{compose_service="server"} |= "Slow Prisma query"' + stages: + - metrics: + ftb_slow_prisma_log_total: + type: Counter + description: Slow Prisma query log entries emitted by the NestJS server. + config: + action: inc + - match: + selector: '{compose_service="server"} |~ "AppData relation sync failed|AI 调用失败|AI 风险解读调用失败"' + stages: + - metrics: + ftb_job_failure_log_total: + type: Counter + description: Compatibility sync, AI job, or background task failure logs. + config: + action: inc + diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index 3f85e01..a4a41c4 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -123,7 +123,117 @@ services: server: condition: service_healthy + prometheus: + image: prom/prometheus:v2.53.1 + profiles: ['monitoring'] + restart: unless-stopped + command: + - '--config.file=/etc/prometheus/prometheus.yml' + - '--storage.tsdb.path=/prometheus' + - '--storage.tsdb.retention.time=${PROMETHEUS_RETENTION:-15d}' + - '--web.enable-lifecycle' + ports: + - '${PROMETHEUS_PORT:-9090}:9090' + volumes: + - ./deploy/monitoring/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro + - ./deploy/monitoring/prometheus/alert-rules.yml:/etc/prometheus/alert-rules.yml:ro + - prometheus_data:/prometheus + depends_on: + - postgres-exporter + - node-exporter + - cadvisor + - promtail + - blackbox-exporter + + grafana: + image: grafana/grafana:11.1.0 + profiles: ['monitoring'] + restart: unless-stopped + ports: + - '${GRAFANA_PORT:-3002}:3000' + environment: + GF_SECURITY_ADMIN_USER: ${GRAFANA_ADMIN_USER:-admin} + GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD:-change-me-monitoring-password} + GF_USERS_ALLOW_SIGN_UP: 'false' + volumes: + - grafana_data:/var/lib/grafana + - ./deploy/monitoring/grafana/provisioning:/etc/grafana/provisioning:ro + - ./deploy/monitoring/grafana/dashboards:/var/lib/grafana/dashboards:ro + depends_on: + - prometheus + - loki + + loki: + image: grafana/loki:2.9.8 + profiles: ['monitoring'] + restart: unless-stopped + command: ['-config.file=/etc/loki/config.yml'] + volumes: + - ./deploy/monitoring/loki/config.yml:/etc/loki/config.yml:ro + - loki_data:/loki + + promtail: + image: grafana/promtail:2.9.8 + profiles: ['monitoring'] + restart: unless-stopped + command: ['-config.file=/etc/promtail/config.yml'] + volumes: + - ./deploy/monitoring/promtail/config.yml:/etc/promtail/config.yml:ro + - /var/lib/docker/containers:/var/lib/docker/containers:ro + - /var/run/docker.sock:/var/run/docker.sock:ro + depends_on: + - loki + + postgres-exporter: + image: quay.io/prometheuscommunity/postgres-exporter:v0.15.0 + profiles: ['monitoring'] + restart: unless-stopped + command: + - '--extend.query-path=/etc/postgres-exporter/postgres-queries.yml' + environment: + DATA_SOURCE_NAME: postgresql://${POSTGRES_USER:-postgres}:${POSTGRES_PASSWORD}@postgres:5432/${POSTGRES_DB:-ftb_pm}?sslmode=disable + volumes: + - ./deploy/monitoring/postgres/postgres-queries.yml:/etc/postgres-exporter/postgres-queries.yml:ro + depends_on: + postgres: + condition: service_healthy + + node-exporter: + image: prom/node-exporter:v1.8.2 + profiles: ['monitoring'] + restart: unless-stopped + command: + - '--path.rootfs=/host' + volumes: + - /:/host:ro,rslave + + cadvisor: + image: gcr.io/cadvisor/cadvisor:v0.49.1 + profiles: ['monitoring'] + restart: unless-stopped + privileged: true + devices: + - /dev/kmsg:/dev/kmsg + volumes: + - /:/rootfs:ro + - /var/run:/var/run:ro + - /sys:/sys:ro + - /var/lib/docker/:/var/lib/docker:ro + - /dev/disk/:/dev/disk:ro + + blackbox-exporter: + image: prom/blackbox-exporter:v0.25.0 + profiles: ['monitoring'] + restart: unless-stopped + command: + - '--config.file=/etc/blackbox/config.yml' + volumes: + - ./deploy/monitoring/blackbox/config.yml:/etc/blackbox/config.yml:ro + volumes: postgres_data: redis_data: server_data: + prometheus_data: + grafana_data: + loki_data: diff --git a/docs/deployment.md b/docs/deployment.md index 0b94fbb..3b0dd52 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -175,12 +175,13 @@ cp .env.production.example .env.production 5. 执行 `docker compose --env-file .env.production -f docker-compose.prod.yml pull web server` 拉取本次 SHA 镜像。 6. 启动数据库与 Redis，执行 `pnpm --filter server db:deploy`。 7. 执行 `docker compose --env-file .env.production -f docker-compose.prod.yml up -d --remove-orphans` 重启服务。 -8. 通过 `/api/v1/health/version` 校验运行中的后端版本是否等于本次 commit SHA。 +8. 运行发布 smoke test：校验 `/api/v1/health/version`、前端首页、产品页、产品 API、V2.2 读路径和 AI 配置端点，并确认运行中的后端版本等于本次 commit SHA。 -如果最后一步失败，Actions 会红掉，说明“代码已合并”不等于“线上容器已更新”。本地或服务器也可以手工运行： +如果最后一步失败，Actions 会红掉，说明“代码已合并”不等于“线上容器已更新”或关键读路径不可用。本地或服务器也可以手工运行： ```bash pnpm deploy:check-runtime http://localhost/api/v1/health/version +pnpm deploy:smoke -- --base-url http://localhost --expected-version ``` ## Nginx 路由 @@ -193,6 +194,30 @@ pnpm deploy:check-runtime http://localhost/api/v1/health/version ftb_pm_backup.sql +pnpm backup:postgres -- --env-file .env.production +pnpm backup:server-data -- --env-file .env.production ``` ## 常见排查 diff --git a/docs/production-readiness.md b/docs/production-readiness.md new file mode 100644 index 0000000..afcc2ab --- /dev/null +++ b/docs/production-readiness.md @@ -0,0 +1,24 @@ +# Production Readiness Checklist + +Use this before a production release and again after the release smoke test. Each item requires evidence, not a verbal assertion. + +| Area | Gate | Evidence | +| --- | --- | --- | +| Backup and restore | PostgreSQL backup dry-run and server_data backup dry-run are reviewed. | `pnpm backup:postgres -- --dry-run --env-file .env.production` and `pnpm backup:server-data -- --dry-run --env-file .env.production` output saved in release notes. | +| Backup and restore | Restore command refuses overwrite without explicit confirmation. | `pnpm restore:postgres -- --env-file .env.production --input backups/postgres/ftb_pm-postgres-20260708T120000Z.dump` exits non-zero and names `--confirm-overwrite`. | +| Backup and restore | Fresh DB restore rehearsal completed before destructive restore. | Temporary database restore command and row-count comparison from `docs/runbooks/migration-rollback.md`. | +| Smoke tests | Release smoke is wired into GitHub Actions. | `.github/workflows/deploy-production.yml` runs `scripts/smoke-test-release.mjs` with `--expected-version`. | +| Smoke tests | Manual smoke can be run against the target. | `pnpm deploy:smoke -- --base-url http://localhost --expected-version "$APP_VERSION"` output. | +| Monitoring | Monitoring profile renders and can start without committed secrets. | `docker compose --env-file .env.production -f docker-compose.prod.yml --profile monitoring config` exits 0. | +| Monitoring | Required alert rules exist. | `FtbPostgresDown`, `FtbDiskPressure`, `FtbSlowApiLogBurst`, `FtbSlowPrismaLogBurst`, `FtbJobFailureLogBurst`, and `FtbXiaobaoSummaryStale` visible in Prometheus. | +| Audit | Domain write APIs record actor and resource scope where implemented. | API request sample or audit log sample for Product, Requirement, Project, Version, execution entities, and dictionary writes. | +| RBAC | Project and version operations enforce Owner, Admin, Member, Viewer boundaries where enabled. | Permission matrix test result or manual account walkthrough attached to release notes. | +| Consistency | AppData and relation tables are checked for migrated domains. | Count and missing-partition-key queries from `docs/runbooks/appdata-retirement.md`. | +| Performance | Slow API and slow Prisma thresholds are configured. | `API_SLOW_REQUEST_MS` and `PRISMA_SLOW_QUERY_MS` values recorded, plus Grafana slow-log panels checked. | +| Performance | V2.2 hot reads avoid full-table scans. | Requirement pool smoke uses `productId`; query plan or service test evidence attached for high-volume domains. | +| Post-release | Runtime version matches the release SHA. | `/api/v1/health/version` payload or `pnpm deploy:check-runtime` output. | +| Post-release | Product, requirement, workspace, Xiaobao, and AI config read paths respond. | `pnpm deploy:smoke` output attached. | +| Post-release | On-call rollback path is known. | `docs/runbooks/migration-rollback.md` link included in release notes. | + +Release owner signs off only after all required evidence is attached to the release notes or incident record. + diff --git a/docs/roadmap.md b/docs/roadmap.md index db6bc1a..1c36ab6 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -1,26 +1,12 @@ # 开发路线图 -## 当前阶段：V2.7 已完成 — 下一阶段 V2.8 生产硬化与运维闭环集成 +## 当前阶段：V2.8 已完成 — 统一验证与生产交付收口 -V2.7 已在关系表主源方向上补齐企业协作和治理能力：通知、评论与提及、项目成员治理、管理驾驶舱、治理字典、以及统一 RBAC/audit 适配器。V2.7 不新增 AppData 主存储。 +V2.8 已在既有生产 CI/CD 基线上补齐运维闭环：备份恢复演练、发布 smoke test、监控告警、日志检索、迁移回滚 runbook、AppData 退场 runbook、小宝后台化 runbook 和生产 readiness 证据清单。本阶段不重新设计业务流程，专注把生产发布和故障处置做成可验证、可复盘、可回滚的标准流程。 -### 当前重点 +V2.4 已将高增长和核心业务领域从“AppData 主写 + 关系表同步副本”推进到“领域 CRUD 主写关系表 + AppData 兼容/迁移兜底”。V2.5 已收口后端权限、审计、AppData 禁写和一致性核对。V2.6 已完成大数据性能增强、小宝风险后台化、AI 解读队列和运行时 Ops 看板。V2.7 已补齐企业协作和治理能力，且不新增 AppData 主存储。 -1. **协作通知**：通知记录、已读状态、NotificationBell，并覆盖 assignment / mention / risk_alert / overdue_item 稳定事件类型。 -2. **通用评论**：DevTask/TestCase/Bug/Requirement/VersionPlan 统一评论面板，支持 `@成员名` 和显式成员选择，创建/删除写 audit。 -3. **项目成员治理**：Owner/Admin/Member/Viewer 服务端强校验，禁止移除最后 Owner，角色变更写 audit。 -4. **管理驾驶舱**：只读关系表和 summary，聚合活跃版本、逾期、阻塞、风险和成员负载。 -5. **治理设置**：集中维护 task category、requirement type/platform/source，使用中的字典不可硬删，支持导入导出。 - -V2.4 已将高增长和核心业务领域从“AppData 主写 + 关系表同步副本”推进到“领域 CRUD 主写关系表 + AppData 兼容/迁移兜底”。V2.2 快读 API 和 V2.3 AppData 写后同步继续保留，但它们现在是兼容基础设施，不再是已迁移领域的数据新鲜度主链路。 - -V2.5 的目标是正式收口后端权限、审计、AppData 禁写和一致性核对。AppData 不能直接删除，必须按“禁写 → 双读核对 → 移除 fallback → 只读归档/导出 → 后续删表”的顺序推进。 - -V2.6 的目标是在关系表主源稳定后完成大数据性能增强、小宝风险后台化、AI 解读队列和运行时 Ops 看板，让高增长热路径、后台任务和风险摘要不再依赖页面打开。 - -V2.8 的目标是在现有生产部署基线上补齐备份恢复演练、发布 smoke test、监控告警、日志检索、迁移回滚和运维手册，形成生产交付稳定版。 - -### V2.5-V2.7 完成范围 +### V2.5-V2.8 完成范围 1. **RBAC 收口**：领域 mutation API 已接入服务端权限校验、资源作用域和当前用户上下文。 2. **审计事件**：领域 mutation 通过 `audit_events` 写 append-only audit event，支持后台查询和敏感字段脱敏。 @@ -37,6 +23,11 @@ V2.8 的目标是在现有生产部署基线上补齐备份恢复演练、发布 13. **项目成员治理**：已补项目成员 Owner/Admin/Member/Viewer 服务端治理，禁止移除最后 Owner，角色变更写审计。 14. **管理驾驶舱**：已补只读关系表和 summary 的管理概览，聚合活跃版本、逾期、阻塞、风险和成员负载。 15. **治理设置**：已补 task category、requirement type/platform/source 等治理字典能力，使用中的字典不可硬删，支持导入导出。 +16. **备份恢复自动化**：已补 PostgreSQL dump、`server_data` volume 备份、fresh DB restore dry-run 和显式覆盖确认。 +17. **发布 smoke test**：已补部署后 runtime version、前端根页、产品页、产品 API、V2.2 读路径和 AI 配置校验。 +18. **监控告警基线**：已补 Prometheus/Grafana/Loki/Promtail 可选 profile，覆盖慢 API、慢 Prisma、任务失败、小宝摘要 stale、磁盘压力和 DB 可用性。 +19. **运维 runbook**：已补迁移回滚、AppData 退场、小宝后台化处置步骤、决策点和数据风险。 +20. **生产 readiness 清单**：已补 backup/restore、smoke、monitoring、audit、RBAC、consistency、performance 和 post-release verification 证据项。 ## V2 分阶段交付链路 @@ -58,7 +49,7 @@ V2.8 的目标是在现有生产部署基线上补齐备份恢复演练、发布 - 项目已经不是早期骨架。前端业务功能已覆盖产品、项目、版本详情、需求池、工作台、成员/角色/任务类型、加班、小宝预警和 AI 配置等主要管理端路由。 - 版本详情已有需求、调研、产品方案、UI、开发任务、测试用例、Bug、概览等核心 Tab；渲染重的路径优先接入关系表快读，并保留 AppData fallback。 - 后端已落地 Product、Project、Version、Requirement、VersionPlan、DevTask、TestCase、Bug、Member、TaskCategory、TaskWorklog、Overtime、WorkActivity 领域 CRUD/write API。 -- Prisma schema 已包含 Product、Project、Version、Requirement、VersionPlan、DevTask、TestCase、Bug、WorkActivity、TaskWorklog、Overtime、Xiaobao、AiLog、AppData 等关系模型；高增长表的分区 migration 已落地。 +- Prisma schema 已包含 Product、Project、Version、Requirement、VersionPlan、DevTask、TestCase、Bug、WorkActivity、TaskWorklog、Overtime、Xiaobao、AiLog、AppData、Notification、Comment、ProjectMember、GovernanceDictionary 等关系模型；高增长表的分区 migration 已落地。 - 主写入源已经切到领域 API：前端 store 优先调用 `apps/web/lib/domain-api.ts`，AppData 只保留兼容读取、历史核对和少量旧配置形状。 - V2.5 后端服务端权限、审计和一致性控制面已启用：领域 mutation 使用 `@ProtectedMutation()`，审计写 `audit_events`，后台查询需要 `audit:view` / `consistency:view`。 - 所有业务 AppData key 已明确冻结或只读归档；`PUT /api/v1/data/:key` 对这些 key 返回 `APP_DATA_WRITE_FROZEN`，`GET` 留作历史核对与归档。 @@ -76,10 +67,16 @@ V2.8 的目标是在现有生产部署基线上补齐备份恢复演练、发布 - V2.7.3 已新增项目成员治理 API 和项目页成员面板，服务端强校验 Owner/Admin/Member/Viewer 边界。 - V2.7.4 已新增管理驾驶舱和治理设置，聚合关系表指标并维护治理字典。 - V2.7.5 已新增协作治理 RBAC/audit adapter，避免新增模块绕开服务端权限和审计边界。 +- V2.8 新增运维交付物集中在 `scripts/`、`.github/workflows/deploy-production.yml`、`deploy/monitoring/`、`docs/runbooks/`、`docs/deployment.md` 和 `docs/production-readiness.md`。 +- AppData 退场、RBAC/审计、性能、小宝后台化、协作治理和生产发布都通过 production readiness 证据项追踪，避免把运维稳定版误当成一次性口头验收。 ### 已完成（按时间倒序） **2026-07-08** +- V2.8 added PostgreSQL backup, fresh DB restore with explicit overwrite confirmation, and `server_data` volume backup automation. +- V2.8 added release smoke suite and wired GitHub Actions deployment verification to runtime version, frontend root, products, V2.2 read path, and AI config checks. +- V2.8 added optional monitoring profile with Prometheus, Grafana, Loki, Promtail, postgres-exporter, node-exporter, cAdvisor, and blackbox-exporter. +- V2.8 added migration rollback, AppData retirement, Xiaobao background jobs runbooks, and production readiness evidence checklist. - V2.7.5 added shared collaboration/governance RBAC and audit adapters so notification, comment, project-member, management, and governance modules keep a single permission/audit boundary. - V2.7.4 added management and governance admin pages for relation-backed overview metrics and dictionary governance. - V2.7.3 added project-member governance APIs and project member panel with Owner/Admin/Member/Viewer safeguards. @@ -188,12 +185,12 @@ V2.8 的目标是在现有生产部署基线上补齐备份恢复演练、发布 ### 进行中 -- V2.8 生产硬化与运维闭环：备份恢复演练、发布 smoke test、监控告警、日志检索、迁移回滚和运维手册。 +- V2.8 统一验证与发布收口：合并后集中跑类型、测试、Prisma、部署、runbook、备份恢复和 smoke 验证。 - 项目详情页 VersionCard 状态胶囊数据联动（部分已完成） ## V2 — 后端接入 -NestJS + Prisma + PostgreSQL 已推进到 V2.7。第一阶段用 `app_data` JSONB 文档表承接现有 store 数据形状，避免浏览器清站点数据导致业务数据丢失；第二阶段建立分区关系表、V2.2 快读 API 和 V2.3 AppData 写后同步；第三阶段 V2.4 已逐领域启用写 API，让前端 store 从 AppData 主写入迁移到领域 CRUD 主写；第四阶段 V2.5 已冻结 AppData 业务写入并收口服务端 RBAC、审计和一致性校验；第五阶段 V2.6 已完成大数据性能和小宝后台化；第六阶段 V2.7 已补齐协作治理能力。 +NestJS + Prisma + PostgreSQL 已推进到 V2.8。第一阶段用 `app_data` JSONB 文档表承接现有 store 数据形状，避免浏览器清站点数据导致业务数据丢失；第二阶段建立分区关系表、V2.2 快读 API 和 V2.3 AppData 写后同步；第三阶段 V2.4 已逐领域启用写 API，让前端 store 从 AppData 主写入迁移到领域 CRUD 主写；第四阶段 V2.5 已冻结 AppData 业务写入并收口服务端 RBAC、审计和一致性校验；第五阶段 V2.6 已完成大数据性能和小宝后台化；第六阶段 V2.7 已补齐协作治理能力；第七阶段 V2.8 已补齐生产运维、备份恢复、监控告警和回滚手册。 ### 关键任务 @@ -306,6 +303,6 @@ V2.5 完成后的保留边界：`GET /api/v1/data/:key` 仍可读历史 JSON；X |------|------| | V1 业务流程打磨 | 进行中 | | V1 朋友试用反馈 | 持续中 | -| V2 后端接入 | 进行中（V2.7 已完成；V2.8 生产硬化与运维闭环待集成） | +| V2 后端接入 | V2.1-V2.8 已完成，等待统一验证与发布授权 | | V3 AI 集成 | 等 V2 数据沉淀 | | 公开发布 | TBD | diff --git a/docs/runbooks/appdata-retirement.md b/docs/runbooks/appdata-retirement.md new file mode 100644 index 0000000..ec2d1ac --- /dev/null +++ b/docs/runbooks/appdata-retirement.md @@ -0,0 +1,72 @@ +# AppData Retirement Runbook + +Use this when retiring an AppData key after its domain writes have moved to relation-table APIs. The order is fixed: backup, measure, freeze writes, compare, remove fallback, archive. + +## Scope Gate + +Retire one AppData key family at a time. Good candidates have domain CRUD writes, read APIs, pagination boundaries, audit events, and a rollback path. + +## Preparation + +```bash +pnpm backup:postgres -- --env-file .env.production +pnpm backup:server-data -- --env-file .env.production +pnpm deploy:smoke -- --base-url http://localhost +``` + +Record the key family being retired, the owning domain API, and the relation tables that replace it. + +## Count And Consistency Checks + +Run counts before disabling writes. + +```bash +docker compose --env-file .env.production -f docker-compose.prod.yml exec postgres psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -c "select key, jsonb_array_length(value) as appdata_rows from app_data where key in ('products-overview','requirements','version-plans','dev-tasks','test-cases','bugs','members','task-categories','task-worklogs','overtime') order by key;" +docker compose --env-file .env.production -f docker-compose.prod.yml exec postgres psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -c "select 'requirements' as table_name, count(*) from requirements union all select 'version_plans', count(*) from version_plans union all select 'dev_tasks', count(*) from dev_tasks union all select 'test_cases', count(*) from test_cases union all select 'bugs', count(*) from bugs;" +``` + +For partitioned entities, also check missing partition keys. + +```bash +docker compose --env-file .env.production -f docker-compose.prod.yml exec postgres psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -c "select 'requirements_missing_product' as check_name, count(*) from requirements where product_id is null union all select 'dev_tasks_missing_version', count(*) from dev_tasks where version_id is null union all select 'test_cases_missing_version', count(*) from test_cases where version_id is null union all select 'bugs_missing_version', count(*) from bugs where version_id is null;" +``` + +## Disable AppData Writes + +1. Merge the domain-specific frontend store change that stops calling `saveServerData` for the retired key. +2. Keep AppData read fallback for one release while relation reads are verified. +3. Deploy and run smoke tests. + +```bash +pnpm deploy:smoke -- --base-url http://localhost --expected-version "$APP_VERSION" +``` + +## Remove Fallback + +Remove AppData read fallback only after one successful release where: + +- Domain writes went through relation APIs. +- AppData row counts did not grow for the retired key. +- V2.2 read paths and page workflows returned expected data. +- No `AppData relation sync failed` logs appeared during the observation window. + +## Archive + +Export retired keys before any later table cleanup. + +```bash +docker compose --env-file .env.production -f docker-compose.prod.yml exec postgres psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -c "copy (select key, value, updated_at from app_data where key in ('dev-tasks','test-cases','bugs')) to stdout with csv header" > backups/postgres/appdata-retired-executions-20260708.csv +``` + +## Rollback + +- If relation writes fail but AppData still has fresh data, roll back to the previous app image that still writes AppData. +- If AppData writes were already disabled and relation writes are bad, restore PostgreSQL from the backup made at the start of this runbook. +- If only read fallback removal caused the issue, roll back app images first and leave the database unchanged. + +## Data Risks + +- Removing fallback too early can hide valid historical JSON rows that were never mapped into relation tables. +- Re-enabling old AppData writes after relation writes have accepted new edits can overwrite newer relation state through compatibility sync. +- AppData exports can contain business-sensitive text; store backup CSV files in the same restricted location as database dumps. + diff --git a/docs/runbooks/migration-rollback.md b/docs/runbooks/migration-rollback.md new file mode 100644 index 0000000..2ffac4b --- /dev/null +++ b/docs/runbooks/migration-rollback.md @@ -0,0 +1,100 @@ +# Migration Rollback Runbook + +Use this when a production release, Prisma migration, or data migration causes failed smoke tests, missing data, bad query performance, or unsafe writes. + +## First Response + +1. Freeze new releases and ask product owners to pause bulk edits. +2. Capture current state before changing anything. + +```bash +date -u +git rev-parse HEAD +docker compose --env-file .env.production -f docker-compose.prod.yml ps +pnpm backup:postgres -- --env-file .env.production +pnpm backup:server-data -- --env-file .env.production +``` + +3. Run the read-only release smoke test. + +```bash +pnpm deploy:smoke -- --base-url http://localhost --expected-version "$APP_VERSION" +``` + +4. Check the database and latest server logs. + +```bash +docker compose --env-file .env.production -f docker-compose.prod.yml exec postgres pg_isready -U "$POSTGRES_USER" -d "$POSTGRES_DB" +docker compose --env-file .env.production -f docker-compose.prod.yml logs --tail=200 server +``` + +## Decision Points + +- App code bad, database healthy: roll back `WEB_IMAGE` and `SERVER_IMAGE` to the previous commit image tags, then restart Compose. +- Migration applied but only additive: roll back app images first; leave schema in place if old code remains compatible. +- Migration changed or removed data: restore a fresh database from the last known-good backup into a temporary database, compare row counts, then decide whether to restore production. +- AppData compatibility issue: keep production database intact, re-enable the previous app image that still reads the AppData fallback, and preserve the failing release backup for analysis. + +## App Image Rollback + +Set previous image tags in `.env.production`. Use the previous successful GitHub Actions run to identify the image tags. + +```bash +export PREVIOUS_WEB_IMAGE=ghcr.io/company/ftb-project-management/web:abc1234 +export PREVIOUS_SERVER_IMAGE=ghcr.io/company/ftb-project-management/server:abc1234 +sed -i "s|^WEB_IMAGE=.*|WEB_IMAGE=${PREVIOUS_WEB_IMAGE}|" .env.production +sed -i "s|^SERVER_IMAGE=.*|SERVER_IMAGE=${PREVIOUS_SERVER_IMAGE}|" .env.production +docker compose --env-file .env.production -f docker-compose.prod.yml pull web server +docker compose --env-file .env.production -f docker-compose.prod.yml up -d --remove-orphans +pnpm deploy:smoke -- --base-url http://localhost --expected-version abc1234 +``` + +## Fresh Database Restore + +Never restore over production until the backup has been rehearsed into a temporary database. + +```bash +export BACKUP_FILE=backups/postgres/ftb_pm-postgres-20260708T120000Z.dump +pnpm restore:postgres -- \ + --dry-run \ + --confirm-overwrite \ + --env-file .env.production \ + --input "$BACKUP_FILE" \ + --target-db ftb_pm_restore_check +pnpm restore:postgres -- \ + --confirm-overwrite \ + --env-file .env.production \ + --input "$BACKUP_FILE" \ + --target-db ftb_pm_restore_check +``` + +Compare critical row counts before touching production. + +```bash +docker compose --env-file .env.production -f docker-compose.prod.yml exec postgres psql -U "$POSTGRES_USER" -d ftb_pm_restore_check -c "select 'products' as table_name, count(*) from products union all select 'requirements', count(*) from requirements union all select 'versions', count(*) from versions union all select 'dev_tasks', count(*) from dev_tasks union all select 'test_cases', count(*) from test_cases union all select 'bugs', count(*) from bugs;" +``` + +If the temporary restore is healthy and production data is unsafe, restore production with explicit overwrite confirmation. + +```bash +pnpm restore:postgres -- \ + --confirm-overwrite \ + --env-file .env.production \ + --input "$BACKUP_FILE" +docker compose --env-file .env.production -f docker-compose.prod.yml up -d --remove-orphans +pnpm deploy:smoke -- --base-url http://localhost +``` + +## Data Risks + +- PostgreSQL restore is destructive for the target database because the script terminates connections, drops the target database, recreates it, and runs `pg_restore`. +- Restoring PostgreSQL does not restore `server_data`; keep AI provider config backup files with the same incident bundle. +- AppData and relation tables can diverge during compatibility windows. Before deleting or restoring, preserve both the failing production backup and the known-good backup. +- If users continued editing during the incident, record the time window and decide whether those edits must be replayed manually after restore. + +## Closeout + +1. Save the failed release SHA, rollback SHA, backup file names, smoke output, and row-count evidence in the incident notes. +2. Keep the failed backup until the next successful release has completed smoke tests and one business-day observation. +3. Add a regression test or runbook correction before re-attempting the migration. + diff --git a/docs/runbooks/xiaobao-background-jobs.md b/docs/runbooks/xiaobao-background-jobs.md new file mode 100644 index 0000000..c2fce9f --- /dev/null +++ b/docs/runbooks/xiaobao-background-jobs.md @@ -0,0 +1,55 @@ +# Xiaobao Background Jobs Runbook + +Current V2.8 production monitoring supports Xiaobao staleness detection. The first production implementation is still page-triggered: opening `/xiaobao-warning` computes risk, saves snapshots, and lets V2.3 sync refresh summaries. A future scheduler must preserve the same idempotent data contract. + +## Alert Triage + +When `FtbXiaobaoSummaryStale` fires: + +```bash +docker compose --env-file .env.production -f docker-compose.prod.yml exec postgres psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -c "select version_id, dirty, updated_at, recomputed_at, risk_level, risk_score from xiaobao_risk_summaries where dirty = true or updated_at < now() - interval '6 hours' order by updated_at asc limit 20;" +docker compose --env-file .env.production -f docker-compose.prod.yml logs --tail=200 server | grep -E "xiaobao|AppData relation sync failed|Slow Prisma query" +``` + +Decision points: + +- Rows are dirty after active version edits: ask a manager to open `/xiaobao-warning` once, then verify summaries refresh. +- Rows remain dirty and server logs show sync failures: treat as AppData relation sync incident and follow `migration-rollback.md`. +- Rows are stale but no product release is near: keep monitoring and schedule a manual refresh before the next release decision meeting. +- Rows are stale for a release due today: refresh manually and have the release owner review the resulting risk explanation before ship/no-ship decision. + +## Manual Refresh Path + +1. Log in as a user with `xiaobao.warning:manage`. +2. Open `/xiaobao-warning`. +3. Wait until AI interpretation status is no longer generating for high-risk versions. +4. Re-run the stale-summary query. +5. Run the release smoke test. + +```bash +pnpm deploy:smoke -- --base-url http://localhost +``` + +## Future Scheduler Rules + +When a background job is introduced, it must: + +- Read unfinished versions by relation-table scope, not by full AppData document scan. +- Use one idempotency key per `versionId + riskSignature + snapshotDate`. +- Write snapshots append-only and upsert summaries by `version_id`. +- Mark failures with structured logs containing `xiaobao background job failed`. +- Retry transient AI failures with backoff and keep rule-based risk output even when AI interpretation fails. +- Never mutate Version, Requirement, DevTask, TestCase, Bug, or Member data. + +## Monitoring Expectations + +- `FtbXiaobaoSummaryStale` alerts on dirty or older-than-6-hour summaries. +- `FtbJobFailureLogBurst` alerts when sync or future job failure log counters increase. +- Grafana dashboard shows the stale summary count and matching server warning/error logs. + +## Data Risks + +- Recomputing Xiaobao summaries can change release risk badges and manager decisions; record manual refresh time in release notes. +- AI interpretation cache is explanatory only. Do not restore or delete business entities to fix a bad explanation. +- If stale summaries are caused by relation sync failure, refreshing the page can mask the symptom without fixing the underlying sync path. Preserve logs before restarting services. + diff --git a/package.json b/package.json index ac341f1..55398e6 100644 --- a/package.json +++ b/package.json @@ -10,8 +10,14 @@ "perf:seed": "node scripts/seed-large-dataset.mjs", "perf:check": "node scripts/perf-check.mjs", "perf:explain": "node scripts/explain-hot-queries.mjs", + "ops:test": "node --test scripts/*.test.mjs", "deploy:verify": "node scripts/verify-production-deploy.mjs", "deploy:check-runtime": "node scripts/check-runtime-version.mjs", + "deploy:smoke": "node scripts/smoke-test-release.mjs", + "backup:postgres": "node scripts/backup-postgres.mjs", + "restore:postgres": "node scripts/restore-postgres.mjs", + "backup:server-data": "node scripts/backup-server-data.mjs", + "docs:check-runbooks": "node scripts/check-runbook-placeholders.mjs --paths docs/runbooks docs/production-readiness.md", "appdata:archive:export": "node scripts/export-appdata-archive.mjs", "appdata:archive:verify": "node scripts/verify-appdata-archive.mjs", "appdata:archive:test": "node --test scripts/appdata-archive.test.mjs", diff --git a/scripts/backup-postgres.mjs b/scripts/backup-postgres.mjs new file mode 100644 index 0000000..08d134e --- /dev/null +++ b/scripts/backup-postgres.mjs @@ -0,0 +1,139 @@ +#!/usr/bin/env node +import { createWriteStream, unlinkSync } from 'node:fs'; +import { dirname, join, resolve } from 'node:path'; +import { pathToFileURL } from 'node:url'; +import { spawn } from 'node:child_process'; +import { + commandToString, + composePrefix, + ensureParentDir, + flag, + option, + parseArgs, + readEnvFile, + timestampSlug, +} from './ops-utils.mjs'; + +function usage() { + return `Usage: node scripts/backup-postgres.mjs [options] + +Options: + --env-file Compose env file (default: .env.production) + --compose-file Compose file (default: docker-compose.prod.yml) + --service Postgres service name (default: postgres) + --user Database user (default: POSTGRES_USER or postgres) + --db Database name (default: POSTGRES_DB or ftb_pm) + --backup-dir Directory for generated backups (default: backups/postgres) + --output Exact output dump path + --dry-run Print the pg_dump command without writing a file + --help Show this help +`; +} + +function buildOptions(argv) { + const args = parseArgs(argv); + if (flag(args, 'help')) return { help: true }; + + const envFile = option(args, 'env-file', '.env.production'); + const env = readEnvFile(envFile, { optional: flag(args, 'dry-run') }); + const db = option(args, 'db', env.POSTGRES_DB || 'ftb_pm'); + const user = option(args, 'user', env.POSTGRES_USER || 'postgres'); + const backupDir = option(args, 'backup-dir', 'backups/postgres'); + const output = option( + args, + 'output', + join(backupDir, `${db}-postgres-${timestampSlug()}.dump`), + ); + + return { + help: false, + dryRun: flag(args, 'dry-run'), + envFile, + composeFile: option(args, 'compose-file', 'docker-compose.prod.yml'), + service: option(args, 'service', 'postgres'), + db, + user, + output: resolve(output), + }; +} + +export function buildPgDumpCommand(options) { + return [ + ...composePrefix(options), + 'exec', + '-T', + options.service, + 'pg_dump', + '-U', + options.user, + '-d', + options.db, + '--format=custom', + '--no-owner', + '--no-acl', + ]; +} + +async function writeBackup(command, output) { + ensureParentDir(output); + await new Promise((resolveWrite, rejectWrite) => { + const file = createWriteStream(output, { flags: 'wx' }); + const child = spawn(command[0], command.slice(1), { + cwd: process.cwd(), + stdio: ['ignore', 'pipe', 'inherit'], + }); + + let settled = false; + const rejectOnce = (error) => { + if (settled) return; + settled = true; + try { + unlinkSync(output); + } catch { + // Best effort cleanup of a partial dump. + } + rejectWrite(error); + }; + + child.stdout.pipe(file); + child.on('error', rejectOnce); + file.on('error', rejectOnce); + child.on('close', (code) => { + file.end(() => { + if (settled) return; + settled = true; + if (code === 0) { + resolveWrite(); + } else { + rejectOnce(new Error(`pg_dump failed with exit code ${code}`)); + } + }); + }); + }); +} + +export async function main(argv = process.argv.slice(2)) { + const options = buildOptions(argv); + if (options.help) { + process.stdout.write(usage()); + return; + } + + const command = buildPgDumpCommand(options); + if (options.dryRun) { + process.stdout.write(`[dry-run] PostgreSQL backup would write: ${options.output}\n`); + process.stdout.write(`${commandToString(command, { stdout: options.output })}\n`); + return; + } + + await writeBackup(command, options.output); + process.stdout.write(`PostgreSQL backup written: ${options.output}\n`); +} + +if (import.meta.url === pathToFileURL(process.argv[1]).href) { + main().catch((error) => { + process.stderr.write(`${error.message}\n`); + process.exit(1); + }); +} + diff --git a/scripts/backup-server-data.mjs b/scripts/backup-server-data.mjs new file mode 100644 index 0000000..8e5bfc9 --- /dev/null +++ b/scripts/backup-server-data.mjs @@ -0,0 +1,99 @@ +#!/usr/bin/env node +import { basename, dirname, join, resolve } from 'node:path'; +import { pathToFileURL } from 'node:url'; +import { + commandToString, + ensureParentDir, + flag, + option, + parseArgs, + readEnvFile, + runCommand, + timestampSlug, +} from './ops-utils.mjs'; + +function usage() { + return `Usage: node scripts/backup-server-data.mjs [options] + +Backs up the server_data Docker volume that stores runtime AI provider config. + +Options: + --env-file Compose env file used to derive COMPOSE_PROJECT_NAME + (default: .env.production) + --volume Explicit Docker volume name + --backup-dir Directory for generated backups (default: backups/server-data) + --output Exact output .tgz path + --image Utility image (default: alpine:3.20) + --dry-run Print the docker run command without writing a file + --help Show this help +`; +} + +function buildOptions(argv) { + const args = parseArgs(argv); + if (flag(args, 'help')) return { help: true }; + + const envFile = option(args, 'env-file', '.env.production'); + const env = readEnvFile(envFile, { optional: flag(args, 'dry-run') }); + const projectName = env.COMPOSE_PROJECT_NAME || 'ftb_pm'; + const volume = option(args, 'volume', `${projectName}_server_data`); + const backupDir = option(args, 'backup-dir', 'backups/server-data'); + const output = option( + args, + 'output', + join(backupDir, `${volume}-${timestampSlug()}.tgz`), + ); + + return { + help: false, + dryRun: flag(args, 'dry-run'), + image: option(args, 'image', 'alpine:3.20'), + volume, + output: resolve(output), + }; +} + +export function buildServerDataBackupCommand(options) { + const outputDir = dirname(options.output); + const outputName = basename(options.output); + return [ + 'docker', + 'run', + '--rm', + '-v', + `${options.volume}:/data:ro`, + '-v', + `${outputDir}:/backup`, + options.image, + 'sh', + '-lc', + `tar -czf /backup/${outputName} -C /data .`, + ]; +} + +export async function main(argv = process.argv.slice(2)) { + const options = buildOptions(argv); + if (options.help) { + process.stdout.write(usage()); + return; + } + + const command = buildServerDataBackupCommand(options); + if (options.dryRun) { + process.stdout.write(`[dry-run] server_data backup would write: ${options.output}\n`); + process.stdout.write(`${commandToString(command)}\n`); + return; + } + + ensureParentDir(options.output); + await runCommand(command); + process.stdout.write(`server_data backup written: ${options.output}\n`); +} + +if (import.meta.url === pathToFileURL(process.argv[1]).href) { + main().catch((error) => { + process.stderr.write(`${error.message}\n`); + process.exit(1); + }); +} + diff --git a/scripts/check-runbook-placeholders.mjs b/scripts/check-runbook-placeholders.mjs new file mode 100644 index 0000000..f99a3fe --- /dev/null +++ b/scripts/check-runbook-placeholders.mjs @@ -0,0 +1,87 @@ +#!/usr/bin/env node +import { existsSync, readFileSync, readdirSync, statSync } from 'node:fs'; +import { extname, join } from 'node:path'; +import { pathToFileURL } from 'node:url'; +import { option, parseArgs } from './ops-utils.mjs'; + +const forbiddenPatterns = [ + { name: 'TBD', pattern: /\bTBD\b/i }, + { name: 'TODO', pattern: /\bTODO\b/i }, + { name: 'fill-in', pattern: /fill in|fill-in/i }, + { name: 'angle-token', pattern: /<[^>\n]+>/ }, + { name: 'Chinese pending marker', pattern: /待定|占位/ }, +]; + +function usage() { + return `Usage: node scripts/check-runbook-placeholders.mjs --paths [path...] + +Scans Markdown runbooks for unresolved placeholder markers. +`; +} + +function collectMarkdownFiles(path) { + if (!existsSync(path)) { + throw new Error(`Path not found: ${path}`); + } + + const stat = statSync(path); + if (stat.isFile()) return extname(path) === '.md' ? [path] : []; + if (!stat.isDirectory()) return []; + + const files = []; + for (const entry of readdirSync(path)) { + files.push(...collectMarkdownFiles(join(path, entry))); + } + return files; +} + +function requestedPaths(argv) { + const args = parseArgs(argv); + if (args.flags.has('help')) return { help: true, paths: [] }; + const first = option(args, 'paths'); + const paths = [first, ...args.positionals].filter(Boolean); + if (paths.length === 0) { + throw new Error('Missing --paths [path...]'); + } + return { help: false, paths }; +} + +export function scanFiles(paths) { + const files = paths.flatMap(collectMarkdownFiles); + const findings = []; + for (const file of files) { + const lines = readFileSync(file, 'utf8').split(/\r?\n/); + lines.forEach((line, index) => { + for (const forbidden of forbiddenPatterns) { + if (forbidden.pattern.test(line)) { + findings.push(`${file}:${index + 1} ${forbidden.name}: ${line.trim()}`); + } + } + }); + } + return { files, findings }; +} + +export async function main(argv = process.argv.slice(2)) { + const { help, paths } = requestedPaths(argv); + if (help) { + process.stdout.write(usage()); + return; + } + + const { files, findings } = scanFiles(paths); + if (findings.length > 0) { + process.stderr.write(`${findings.join('\n')}\n`); + throw new Error(`Runbook placeholder scan failed: ${findings.length} finding(s)`); + } + + process.stdout.write(`Runbook placeholder scan passed (${files.length} file(s)).\n`); +} + +if (import.meta.url === pathToFileURL(process.argv[1]).href) { + main().catch((error) => { + process.stderr.write(`${error.message}\n`); + process.exit(1); + }); +} + diff --git a/scripts/ops-scripts.test.mjs b/scripts/ops-scripts.test.mjs new file mode 100644 index 0000000..a70d0f8 --- /dev/null +++ b/scripts/ops-scripts.test.mjs @@ -0,0 +1,152 @@ +import { spawnSync } from 'node:child_process'; +import { mkdtempSync, writeFileSync, existsSync, readFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; + +const root = resolve(import.meta.dirname, '..'); + +function runScript(script, args) { + return spawnSync(process.execPath, [join(root, script), ...args], { + cwd: root, + encoding: 'utf8', + }); +} + +describe('production ops scripts', () => { + it('prints a pg_dump command in dry-run mode without writing the target file', () => { + const dir = mkdtempSync(join(tmpdir(), 'ftb-pg-backup-')); + const output = join(dir, 'backup.dump'); + + const result = runScript('scripts/backup-postgres.mjs', [ + '--dry-run', + '--env-file', + '.env.production.example', + '--output', + output, + ]); + + assert.equal(result.status, 0, result.stderr); + assert.match(result.stdout, /docker compose/); + assert.match(result.stdout, /pg_dump/); + assert.match(result.stdout, /backup\.dump/); + assert.equal(existsSync(output), false); + }); + + it('refuses restore by default unless --confirm-overwrite is supplied', () => { + const dir = mkdtempSync(join(tmpdir(), 'ftb-pg-restore-')); + const input = join(dir, 'backup.dump'); + writeFileSync(input, 'not-a-real-dump'); + + const result = runScript('scripts/restore-postgres.mjs', [ + '--env-file', + '.env.production.example', + '--input', + input, + ]); + + assert.notEqual(result.status, 0); + assert.match(result.stderr, /--confirm-overwrite/); + }); + + it('prints fresh database restore steps in dry-run mode after explicit overwrite confirmation', () => { + const dir = mkdtempSync(join(tmpdir(), 'ftb-pg-restore-dry-')); + const input = join(dir, 'backup.dump'); + writeFileSync(input, 'not-a-real-dump'); + + const result = runScript('scripts/restore-postgres.mjs', [ + '--dry-run', + '--confirm-overwrite', + '--env-file', + '.env.production.example', + '--input', + input, + ]); + + assert.equal(result.status, 0, result.stderr); + assert.match(result.stdout, /dropdb/); + assert.match(result.stdout, /createdb/); + assert.match(result.stdout, /pg_restore/); + }); + + it('prints a server_data volume tar backup command in dry-run mode', () => { + const result = runScript('scripts/backup-server-data.mjs', [ + '--dry-run', + '--env-file', + '.env.production.example', + ]); + + assert.equal(result.status, 0, result.stderr); + assert.match(result.stdout, /docker run/); + assert.match(result.stdout, /ftb_pm_server_data/); + assert.match(result.stdout, /tar -czf/); + }); + + it('prints read-only release smoke checks in dry-run mode', () => { + const result = runScript('scripts/smoke-test-release.mjs', [ + '--dry-run', + '--base-url', + 'http://localhost', + ]); + + assert.equal(result.status, 0, result.stderr); + assert.match(result.stdout, /\/api\/v1\/health\/version/); + assert.match(result.stdout, /\/products/); + assert.match(result.stdout, /\/api\/v1\/products/); + assert.match(result.stdout, /\/api\/v1\/v2\.2\/requirements\?productId=__smoke__/); + assert.match(result.stdout, /\/api\/v1\/config\/ai/); + }); + + it('declares the production monitoring baseline without real alert secrets', () => { + const alertRules = readFileSync( + join(root, 'deploy/monitoring/prometheus/alert-rules.yml'), + 'utf8', + ); + const postgresQueries = readFileSync( + join(root, 'deploy/monitoring/postgres/postgres-queries.yml'), + 'utf8', + ); + const promtailConfig = readFileSync( + join(root, 'deploy/monitoring/promtail/config.yml'), + 'utf8', + ); + const dashboard = readFileSync( + join(root, 'deploy/monitoring/grafana/dashboards/ftb-production-overview.json'), + 'utf8', + ); + + for (const alertName of [ + 'FtbPostgresDown', + 'FtbDiskPressure', + 'FtbSlowApiLogBurst', + 'FtbSlowPrismaLogBurst', + 'FtbJobFailureLogBurst', + 'FtbXiaobaoSummaryStale', + ]) { + assert.match(alertRules, new RegExp(alertName)); + } + + assert.match(postgresQueries, /xiaobao_risk_summaries/); + assert.match(postgresQueries, /dirty = true/); + assert.match(promtailConfig, /Slow API request/); + assert.match(promtailConfig, /Slow Prisma query/); + assert.match(promtailConfig, /AppData relation sync failed/); + assert.match(dashboard, /FTB Production Overview/); + + const combined = `${alertRules}\n${postgresQueries}\n${promtailConfig}\n${dashboard}`; + assert.doesNotMatch(combined, /sk-ant-[A-Za-z0-9]/); + assert.doesNotMatch(combined, /hooks\.slack\.com\/services\//); + }); + + it('passes the runbook placeholder scan', () => { + const result = runScript('scripts/check-runbook-placeholders.mjs', [ + '--paths', + 'docs/runbooks', + 'docs/production-readiness.md', + ]); + + assert.equal(result.status, 0, result.stderr); + assert.match(result.stdout, /Runbook placeholder scan passed/); + }); +}); diff --git a/scripts/ops-utils.mjs b/scripts/ops-utils.mjs new file mode 100644 index 0000000..2d01f02 --- /dev/null +++ b/scripts/ops-utils.mjs @@ -0,0 +1,116 @@ +import { existsSync, mkdirSync, readFileSync } from 'node:fs'; +import { dirname, resolve } from 'node:path'; +import { spawn } from 'node:child_process'; + +export function parseArgs(argv) { + const parsed = { flags: new Set(), values: new Map(), positionals: [] }; + + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index]; + if (arg === '--') continue; + if (!arg.startsWith('--')) { + parsed.positionals.push(arg); + continue; + } + + const eq = arg.indexOf('='); + if (eq !== -1) { + parsed.values.set(arg.slice(2, eq), arg.slice(eq + 1)); + continue; + } + + const key = arg.slice(2); + const next = argv[index + 1]; + if (next && !next.startsWith('--')) { + parsed.values.set(key, next); + index += 1; + } else { + parsed.flags.add(key); + } + } + + return parsed; +} + +export function option(args, name, fallback = undefined) { + return args.values.has(name) ? args.values.get(name) : fallback; +} + +export function flag(args, name) { + return args.flags.has(name); +} + +export function readEnvFile(filePath, { optional = false } = {}) { + if (!existsSync(filePath)) { + if (optional) return {}; + throw new Error(`Env file not found: ${filePath}`); + } + + const env = {}; + const content = readFileSync(filePath, 'utf8'); + for (const rawLine of content.split(/\r?\n/)) { + const line = rawLine.trim(); + if (!line || line.startsWith('#')) continue; + const eq = line.indexOf('='); + if (eq === -1) continue; + const key = line.slice(0, eq).trim(); + let value = line.slice(eq + 1).trim(); + if ( + (value.startsWith('"') && value.endsWith('"')) || + (value.startsWith("'") && value.endsWith("'")) + ) { + value = value.slice(1, -1); + } + env[key] = value; + } + return env; +} + +export function ensureParentDir(filePath) { + mkdirSync(dirname(filePath), { recursive: true }); +} + +export function timestampSlug(date = new Date()) { + return date.toISOString().replace(/[-:]/g, '').replace(/\.\d{3}Z$/, 'Z'); +} + +export function shellQuote(value) { + if (/^[A-Za-z0-9_./:@%+=,-]+$/.test(value)) return value; + return `'${value.replace(/'/g, "'\\''")}'`; +} + +export function commandToString(command, { stdin, stdout } = {}) { + const rendered = command.map((part) => shellQuote(part)).join(' '); + const withStdin = stdin ? `${rendered} < ${shellQuote(stdin)}` : rendered; + return stdout ? `${withStdin} > ${shellQuote(stdout)}` : withStdin; +} + +export function composePrefix({ envFile, composeFile }) { + return ['docker', 'compose', '--env-file', envFile, '-f', composeFile]; +} + +export function resolvePath(path) { + return resolve(process.cwd(), path); +} + +export function runCommand(command, options = {}) { + return new Promise((resolveRun, rejectRun) => { + const child = spawn(command[0], command.slice(1), { + stdio: options.stdio ?? 'inherit', + cwd: options.cwd ?? process.cwd(), + }); + + child.on('error', rejectRun); + child.on('close', (code) => { + if (code === 0) { + resolveRun(); + } else { + rejectRun(new Error(`Command failed (${code}): ${commandToString(command)}`)); + } + }); + }); +} + +export function quotePgLiteral(value) { + return `'${value.replace(/'/g, "''")}'`; +} diff --git a/scripts/restore-postgres.mjs b/scripts/restore-postgres.mjs new file mode 100644 index 0000000..0322c9e --- /dev/null +++ b/scripts/restore-postgres.mjs @@ -0,0 +1,181 @@ +#!/usr/bin/env node +import { createReadStream, existsSync } from 'node:fs'; +import { resolve } from 'node:path'; +import { pathToFileURL } from 'node:url'; +import { spawn } from 'node:child_process'; +import { + commandToString, + composePrefix, + flag, + option, + parseArgs, + quotePgLiteral, + readEnvFile, + runCommand, +} from './ops-utils.mjs'; + +function usage() { + return `Usage: node scripts/restore-postgres.mjs --input --confirm-overwrite [options] + +This script restores into a freshly recreated PostgreSQL database. It refuses to +drop/recreate a database unless --confirm-overwrite is provided. + +Options: + --input pg_dump custom-format dump file + --confirm-overwrite Required safety flag for destructive restore + --env-file Compose env file (default: .env.production) + --compose-file Compose file (default: docker-compose.prod.yml) + --service Postgres service name (default: postgres) + --user Database user (default: POSTGRES_USER or postgres) + --target-db Target database (default: POSTGRES_DB or ftb_pm) + --maintenance-db Maintenance database (default: postgres) + --dry-run Print restore commands without changing data + --help Show this help +`; +} + +function buildOptions(argv) { + const args = parseArgs(argv); + if (flag(args, 'help')) return { help: true }; + + const envFile = option(args, 'env-file', '.env.production'); + const env = readEnvFile(envFile, { optional: flag(args, 'dry-run') }); + const input = option(args, 'input'); + if (!input) { + throw new Error('Missing required --input '); + } + + const inputPath = resolve(input); + if (!existsSync(inputPath)) { + throw new Error(`Restore input not found: ${inputPath}`); + } + + if (!flag(args, 'confirm-overwrite')) { + throw new Error('Refusing destructive restore. Re-run with --confirm-overwrite.'); + } + + return { + help: false, + dryRun: flag(args, 'dry-run'), + input: inputPath, + envFile, + composeFile: option(args, 'compose-file', 'docker-compose.prod.yml'), + service: option(args, 'service', 'postgres'), + user: option(args, 'user', env.POSTGRES_USER || 'postgres'), + targetDb: option(args, 'target-db', env.POSTGRES_DB || 'ftb_pm'), + maintenanceDb: option(args, 'maintenance-db', 'postgres'), + }; +} + +export function buildRestoreCommands(options) { + const prefix = composePrefix(options); + const terminateSql = [ + 'SELECT pg_terminate_backend(pid)', + 'FROM pg_stat_activity', + `WHERE datname = ${quotePgLiteral(options.targetDb)} AND pid <> pg_backend_pid();`, + ].join(' '); + + return [ + [ + ...prefix, + 'exec', + '-T', + options.service, + 'psql', + '-U', + options.user, + '-d', + options.maintenanceDb, + '-v', + 'ON_ERROR_STOP=1', + '-c', + terminateSql, + ], + [ + ...prefix, + 'exec', + '-T', + options.service, + 'dropdb', + '--if-exists', + '-U', + options.user, + options.targetDb, + ], + [ + ...prefix, + 'exec', + '-T', + options.service, + 'createdb', + '-U', + options.user, + options.targetDb, + ], + [ + ...prefix, + 'exec', + '-T', + options.service, + 'pg_restore', + '-U', + options.user, + '-d', + options.targetDb, + '--no-owner', + '--no-acl', + ], + ]; +} + +function runRestore(command, input) { + return new Promise((resolveRun, rejectRun) => { + const child = spawn(command[0], command.slice(1), { + cwd: process.cwd(), + stdio: ['pipe', 'inherit', 'inherit'], + }); + createReadStream(input).pipe(child.stdin); + child.on('error', rejectRun); + child.on('close', (code) => { + if (code === 0) { + resolveRun(); + } else { + rejectRun(new Error(`pg_restore failed with exit code ${code}`)); + } + }); + }); +} + +export async function main(argv = process.argv.slice(2)) { + const options = buildOptions(argv); + if (options.help) { + process.stdout.write(usage()); + return; + } + + const commands = buildRestoreCommands(options); + if (options.dryRun) { + process.stdout.write( + `[dry-run] PostgreSQL restore would recreate database: ${options.targetDb}\n`, + ); + for (const command of commands.slice(0, -1)) { + process.stdout.write(`${commandToString(command)}\n`); + } + process.stdout.write(`${commandToString(commands.at(-1), { stdin: options.input })}\n`); + return; + } + + for (const command of commands.slice(0, -1)) { + await runCommand(command); + } + await runRestore(commands.at(-1), options.input); + process.stdout.write(`PostgreSQL restore completed into fresh database: ${options.targetDb}\n`); +} + +if (import.meta.url === pathToFileURL(process.argv[1]).href) { + main().catch((error) => { + process.stderr.write(`${error.message}\n`); + process.exit(1); + }); +} + diff --git a/scripts/smoke-test-release.mjs b/scripts/smoke-test-release.mjs new file mode 100644 index 0000000..e199a4c --- /dev/null +++ b/scripts/smoke-test-release.mjs @@ -0,0 +1,160 @@ +#!/usr/bin/env node +import { pathToFileURL } from 'node:url'; +import { flag, option, parseArgs } from './ops-utils.mjs'; + +function usage() { + return `Usage: node scripts/smoke-test-release.mjs --base-url [options] + +Read-only release smoke checks: + - backend runtime version + - frontend root page + - frontend products route + - products API + - V2.2 requirement read path + - AI config public endpoint + +Options: + --base-url Deployment base URL (default: http://127.0.0.1) + --expected-version Expected /api/v1/health/version payload version + --timeout-ms Per-request timeout (default: 5000) + --dry-run Print planned checks without making requests + --help Show this help +`; +} + +function normalizeBaseUrl(value) { + const url = new URL(value || 'http://127.0.0.1'); + url.pathname = url.pathname.replace(/\/+$/, ''); + url.search = ''; + url.hash = ''; + return url.toString().replace(/\/$/, ''); +} + +export function buildSmokeChecks(baseUrl, expectedVersion = '') { + const checks = [ + { + name: 'backend runtime version', + path: '/api/v1/health/version', + kind: 'json', + validate(payload) { + if (payload?.service !== 'server' || typeof payload.version !== 'string') { + throw new Error(`unexpected version payload: ${JSON.stringify(payload)}`); + } + if (expectedVersion && payload.version !== expectedVersion) { + throw new Error(`expected version ${expectedVersion}, got ${payload.version}`); + } + }, + }, + { name: 'frontend root', path: '/', kind: 'text' }, + { name: 'frontend products route', path: '/products', kind: 'text' }, + { + name: 'products API', + path: '/api/v1/products', + kind: 'json', + validate(payload) { + if (!Array.isArray(payload)) { + throw new Error('products API did not return an array'); + } + }, + }, + { + name: 'V2.2 requirement scoped read', + path: '/api/v1/v2.2/requirements?productId=__smoke__&limit=1', + kind: 'json', + validate(payload) { + if (!payload || !Array.isArray(payload.items)) { + throw new Error('V2.2 requirements response missing items array'); + } + }, + }, + { + name: 'AI config public endpoint', + path: '/api/v1/config/ai', + kind: 'json', + validate(payload) { + if (!payload || !Array.isArray(payload.providers)) { + throw new Error('AI config response missing providers array'); + } + }, + }, + ]; + + return checks.map((check) => ({ + ...check, + url: new URL(check.path, `${baseUrl}/`).toString(), + })); +} + +async function fetchWithTimeout(url, timeoutMs) { + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), timeoutMs); + try { + return await fetch(url, { signal: controller.signal }); + } finally { + clearTimeout(timer); + } +} + +async function runCheck(check, timeoutMs) { + const response = await fetchWithTimeout(check.url, timeoutMs); + if (!response.ok) { + throw new Error(`HTTP ${response.status}`); + } + + if (check.kind === 'json') { + const payload = await response.json(); + check.validate?.(payload); + } else { + await response.text(); + } +} + +export async function main(argv = process.argv.slice(2)) { + const args = parseArgs(argv); + if (flag(args, 'help')) { + process.stdout.write(usage()); + return; + } + + const baseUrl = normalizeBaseUrl(option(args, 'base-url', 'http://127.0.0.1')); + const expectedVersion = option(args, 'expected-version', ''); + const timeoutMs = Number.parseInt(option(args, 'timeout-ms', '5000'), 10); + if (!Number.isFinite(timeoutMs) || timeoutMs <= 0) { + throw new Error('--timeout-ms must be a positive number'); + } + + const checks = buildSmokeChecks(baseUrl, expectedVersion); + if (flag(args, 'dry-run')) { + process.stdout.write(`[dry-run] Release smoke target: ${baseUrl}\n`); + for (const check of checks) { + process.stdout.write(`GET ${check.url} # ${check.name}\n`); + } + return; + } + + process.stdout.write(`Release smoke target: ${baseUrl}\n`); + const failures = []; + for (const check of checks) { + try { + await runCheck(check, timeoutMs); + process.stdout.write(`[pass] ${check.name} ${check.url}\n`); + } catch (error) { + failures.push(`[fail] ${check.name} ${check.url}: ${error.message}`); + process.stderr.write(`${failures.at(-1)}\n`); + } + } + + if (failures.length > 0) { + throw new Error(`Release smoke failed: ${failures.length} check(s) failed`); + } + + process.stdout.write('Release smoke test passed.\n'); +} + +if (import.meta.url === pathToFileURL(process.argv[1]).href) { + main().catch((error) => { + process.stderr.write(`${error.message}\n`); + process.exit(1); + }); +} + diff --git a/scripts/verify-production-deploy.mjs b/scripts/verify-production-deploy.mjs index bb238d3..957505f 100644 --- a/scripts/verify-production-deploy.mjs +++ b/scripts/verify-production-deploy.mjs @@ -21,6 +21,7 @@ const checks = [ 'NEXT_PUBLIC_API_URL', 'NEXT_PUBLIC_APP_VERSION', 'ARG APP_VERSION=unknown', + 'COPY scripts ./scripts', 'CMD ["pnpm", "--filter", "web", "start"]', ], }, @@ -50,6 +51,11 @@ const checks = [ 'WEB_IMAGE', 'APP_VERSION', '/api/v1/health/version', + 'prometheus:', + "profiles: ['monitoring']", + 'postgres-exporter:', + 'blackbox-exporter:', + 'prometheus_data:', ], }, { @@ -59,13 +65,42 @@ const checks = [ 'appleboy/ssh-action', 'docker compose --env-file .env.production -f docker-compose.prod.yml pull', 'pnpm --filter server db:deploy', - '/api/v1/health/version', + 'scripts/smoke-test-release.mjs', + '--expected-version', ], }, { file: 'scripts/check-runtime-version.mjs', snippets: ['Runtime version verified', '/api/v1/health/version', 'expectedVersion'], }, + { + file: 'scripts/smoke-test-release.mjs', + snippets: [ + 'Release smoke test passed', + '/api/v1/v2.2/requirements?productId=__smoke__', + '/api/v1/config/ai', + ], + }, + { + file: 'scripts/backup-postgres.mjs', + snippets: ['pg_dump', '--format=custom', '--dry-run'], + }, + { + file: 'scripts/restore-postgres.mjs', + snippets: ['--confirm-overwrite', 'dropdb', 'pg_restore'], + }, + { + file: 'scripts/backup-server-data.mjs', + snippets: ['server_data', 'tar -czf', '--dry-run'], + }, + { + file: 'scripts/check-runbook-placeholders.mjs', + snippets: ['Runbook placeholder scan passed', 'Runbook placeholder scan failed', '--paths'], + }, + { + file: 'package.json', + snippets: ['docs:check-runbooks', 'docs/runbooks', 'docs/production-readiness.md'], + }, { file: 'docker-compose.local.yml', snippets: [ @@ -88,6 +123,66 @@ const checks = [ 'X-Forwarded-Proto', ], }, + { + file: 'deploy/monitoring/README.md', + snippets: [ + '--profile monitoring', + 'FtbPostgresDown', + 'FtbXiaobaoSummaryStale', + 'no webhook URLs', + ], + }, + { + file: 'deploy/monitoring/prometheus/prometheus.yml', + snippets: ['postgres-exporter:9187', 'promtail:9080', 'blackbox-http'], + }, + { + file: 'deploy/monitoring/prometheus/alert-rules.yml', + snippets: [ + 'FtbPostgresDown', + 'FtbSlowApiLogBurst', + 'FtbSlowPrismaLogBurst', + 'FtbJobFailureLogBurst', + 'FtbXiaobaoSummaryStale', + ], + }, + { + file: 'docs/runbooks/migration-rollback.md', + snippets: ['Fresh Database Restore', 'Data Risks', 'pnpm restore:postgres'], + }, + { + file: 'docs/runbooks/appdata-retirement.md', + snippets: ['Disable AppData Writes', 'Remove Fallback', 'Data Risks'], + }, + { + file: 'docs/runbooks/xiaobao-background-jobs.md', + snippets: ['FtbXiaobaoSummaryStale', 'Future Scheduler Rules', 'Data Risks'], + }, + { + file: 'docs/production-readiness.md', + snippets: [ + 'Backup and restore', + 'Smoke tests', + 'Monitoring', + 'Audit', + 'RBAC', + 'Consistency', + 'Performance', + 'Post-release', + ], + }, + { + file: 'deploy/monitoring/postgres/postgres-queries.yml', + snippets: ['xiaobao_risk_summaries', 'dirty = true', 'stale_summary_count'], + }, + { + file: 'deploy/monitoring/promtail/config.yml', + snippets: ['Slow API request', 'Slow Prisma query', 'AppData relation sync failed'], + }, + { + file: 'deploy/monitoring/grafana/dashboards/ftb-production-overview.json', + snippets: ['FTB Production Overview', 'ftb_xiaobao_stale_summary_count', 'Server Warning/Error Logs'], + }, { file: '.env.production.example', snippets: [ @@ -118,6 +213,12 @@ const checks = [ 'docker-compose.local.yml', '本地服务器部署', 'pnpm deploy:verify', + 'pnpm backup:postgres', + 'pnpm restore:postgres', + 'pnpm backup:server-data', + 'docs/runbooks/migration-rollback.md', + 'docs/production-readiness.md', + '--profile monitoring', 'pnpm db:migrate', 'NEXT_PUBLIC_API_URL', 'Nginx',