merge: 集成V2.8 生产硬化与运维闭环
# Conflicts: # .gitignore # docs/deployment.md # docs/roadmap.md # package.json
This commit is contained in:
。
@@ -39,3 +39,10 @@ SMTP_HOST=
|
|||||||
SMTP_PORT=465
|
SMTP_PORT=465
|
||||||
SMTP_USER=
|
SMTP_USER=
|
||||||
SMTP_PASS=
|
SMTP_PASS=
|
||||||
|
|
||||||
|
# Optional monitoring profile. Do not commit real production passwords.
|
||||||
|
PROMETHEUS_PORT=9090
|
||||||
|
PROMETHEUS_RETENTION=15d
|
||||||
|
GRAFANA_PORT=3002
|
||||||
|
GRAFANA_ADMIN_USER=admin
|
||||||
|
GRAFANA_ADMIN_PASSWORD=change-me-monitoring-password
|
||||||
|
|||||||
19
.github/workflows/deploy-production.yml
vendored
19
.github/workflows/deploy-production.yml
vendored
@@ -123,26 +123,11 @@ jobs:
|
|||||||
docker compose --env-file .env.production -f docker-compose.prod.yml up -d --remove-orphans
|
docker compose --env-file .env.production -f docker-compose.prod.yml up -d --remove-orphans
|
||||||
|
|
||||||
for attempt in $(seq 1 30); do
|
for attempt in $(seq 1 30); do
|
||||||
if docker compose --env-file .env.production -f docker-compose.prod.yml exec -T web node -e "
|
if docker compose --env-file .env.production -f docker-compose.prod.yml exec -T web node scripts/smoke-test-release.mjs --base-url http://nginx --expected-version "${{ github.sha }}"; then
|
||||||
const expected = process.argv[1];
|
|
||||||
fetch('http://nginx/api/v1/health/version')
|
|
||||||
.then(async (response) => {
|
|
||||||
if (!response.ok) throw new Error('HTTP ' + response.status);
|
|
||||||
const payload = await response.json();
|
|
||||||
if (payload.version !== expected) {
|
|
||||||
throw new Error('Expected ' + expected + ', got ' + payload.version);
|
|
||||||
}
|
|
||||||
console.log('Runtime version verified: ' + payload.version);
|
|
||||||
})
|
|
||||||
.catch((error) => {
|
|
||||||
console.error(error.message);
|
|
||||||
process.exit(1);
|
|
||||||
});
|
|
||||||
" "${{ github.sha }}"; then
|
|
||||||
exit 0
|
exit 0
|
||||||
fi
|
fi
|
||||||
sleep 2
|
sleep 2
|
||||||
done
|
done
|
||||||
|
|
||||||
echo "Runtime version check failed after retries"
|
echo "Release smoke check failed after retries"
|
||||||
exit 1
|
exit 1
|
||||||
|
|||||||
1
.gitignore
vendored
1
.gitignore
vendored
@@ -16,3 +16,4 @@ next-env.d.ts
|
|||||||
apps/server/data/
|
apps/server/data/
|
||||||
.worktrees/
|
.worktrees/
|
||||||
appdata-archive-*.json
|
appdata-archive-*.json
|
||||||
|
backups/
|
||||||
|
|||||||
@@ -57,6 +57,7 @@ COPY --from=builder /app/turbo.json ./turbo.json
|
|||||||
COPY --from=builder /app/node_modules ./node_modules
|
COPY --from=builder /app/node_modules ./node_modules
|
||||||
COPY --from=builder /app/packages/shared ./packages/shared
|
COPY --from=builder /app/packages/shared ./packages/shared
|
||||||
COPY --from=builder /app/apps/web ./apps/web
|
COPY --from=builder /app/apps/web ./apps/web
|
||||||
|
COPY scripts ./scripts
|
||||||
RUN chown -R node:node /app
|
RUN chown -R node:node /app
|
||||||
USER node
|
USER node
|
||||||
EXPOSE 3000
|
EXPOSE 3000
|
||||||
|
|||||||
43
deploy/monitoring/README.md
Normal file
43
deploy/monitoring/README.md
Normal file
@@ -0,0 +1,43 @@
|
|||||||
|
# FTB Production Monitoring Baseline
|
||||||
|
|
||||||
|
This profile adds a deployable Prometheus/Grafana baseline for production operations. It is intentionally secret-free: no webhook URLs, API keys, SMTP passwords, or real alert receiver credentials are committed.
|
||||||
|
|
||||||
|
## Start
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker compose --env-file .env.production -f docker-compose.prod.yml --profile monitoring up -d
|
||||||
|
```
|
||||||
|
|
||||||
|
Default local ports:
|
||||||
|
|
||||||
|
- Prometheus: `http://localhost:9090`
|
||||||
|
- Grafana: `http://localhost:3002`
|
||||||
|
- Loki: internal only
|
||||||
|
|
||||||
|
Set `GRAFANA_ADMIN_USER` and `GRAFANA_ADMIN_PASSWORD` in `.env.production` before exposing Grafana beyond localhost. Keep real alert receivers in the server environment or an untracked Alertmanager file.
|
||||||
|
|
||||||
|
## Coverage
|
||||||
|
|
||||||
|
- DB availability: `pg_up` from postgres-exporter.
|
||||||
|
- Disk pressure: root filesystem availability from node-exporter.
|
||||||
|
- Slow API requests: Promtail turns `Slow API request` server logs into `ftb_slow_api_log_total`.
|
||||||
|
- Slow Prisma queries: Promtail turns `Slow Prisma query` server logs into `ftb_slow_prisma_log_total`.
|
||||||
|
- Job failures: Promtail turns `AppData relation sync failed` and AI call failure logs into `ftb_job_failure_log_total`.
|
||||||
|
- Xiaobao stale summaries: postgres-exporter custom query exposes `ftb_xiaobao_stale_summary_count` from `xiaobao_risk_summaries`.
|
||||||
|
|
||||||
|
## Alerts
|
||||||
|
|
||||||
|
Prometheus loads `prometheus/alert-rules.yml`. The rules evaluate locally and are visible in Prometheus/Grafana. To send notifications, add Alertmanager outside git or mount an environment-specific receiver file; do not commit webhook URLs or tokens.
|
||||||
|
|
||||||
|
Baseline alert names:
|
||||||
|
|
||||||
|
- `FtbPostgresDown`
|
||||||
|
- `FtbDiskPressure`
|
||||||
|
- `FtbSlowApiLogBurst`
|
||||||
|
- `FtbSlowPrismaLogBurst`
|
||||||
|
- `FtbJobFailureLogBurst`
|
||||||
|
- `FtbXiaobaoSummaryStale`
|
||||||
|
|
||||||
|
## Log Search
|
||||||
|
|
||||||
|
Promtail ships Docker logs to Loki with container labels. Grafana provisions both Prometheus and Loki data sources, so on-call checks can move from a firing alert to matching server logs without SSHing into the host.
|
||||||
10
deploy/monitoring/blackbox/config.yml
Normal file
10
deploy/monitoring/blackbox/config.yml
Normal file
@@ -0,0 +1,10 @@
|
|||||||
|
modules:
|
||||||
|
http_2xx:
|
||||||
|
prober: http
|
||||||
|
timeout: 5s
|
||||||
|
http:
|
||||||
|
valid_http_versions: ['HTTP/1.1', 'HTTP/2.0']
|
||||||
|
valid_status_codes: []
|
||||||
|
method: GET
|
||||||
|
preferred_ip_protocol: ip4
|
||||||
|
|
||||||
@@ -0,0 +1,78 @@
|
|||||||
|
{
|
||||||
|
"uid": "ftb-production-overview",
|
||||||
|
"title": "FTB Production Overview",
|
||||||
|
"schemaVersion": 39,
|
||||||
|
"version": 1,
|
||||||
|
"refresh": "30s",
|
||||||
|
"tags": ["ftb", "production", "v2.8"],
|
||||||
|
"time": {
|
||||||
|
"from": "now-6h",
|
||||||
|
"to": "now"
|
||||||
|
},
|
||||||
|
"panels": [
|
||||||
|
{
|
||||||
|
"id": 1,
|
||||||
|
"type": "stat",
|
||||||
|
"title": "DB Availability",
|
||||||
|
"gridPos": { "x": 0, "y": 0, "w": 6, "h": 4 },
|
||||||
|
"targets": [
|
||||||
|
{ "datasource": { "type": "prometheus", "uid": "Prometheus" }, "expr": "pg_up", "refId": "A" }
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 2,
|
||||||
|
"type": "timeseries",
|
||||||
|
"title": "Slow API Logs",
|
||||||
|
"gridPos": { "x": 6, "y": 0, "w": 6, "h": 4 },
|
||||||
|
"targets": [
|
||||||
|
{ "datasource": { "type": "prometheus", "uid": "Prometheus" }, "expr": "increase(ftb_slow_api_log_total[10m])", "refId": "A" }
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 3,
|
||||||
|
"type": "timeseries",
|
||||||
|
"title": "Slow Prisma Logs",
|
||||||
|
"gridPos": { "x": 12, "y": 0, "w": 6, "h": 4 },
|
||||||
|
"targets": [
|
||||||
|
{ "datasource": { "type": "prometheus", "uid": "Prometheus" }, "expr": "increase(ftb_slow_prisma_log_total[10m])", "refId": "A" }
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 4,
|
||||||
|
"type": "stat",
|
||||||
|
"title": "Xiaobao Stale Summaries",
|
||||||
|
"gridPos": { "x": 18, "y": 0, "w": 6, "h": 4 },
|
||||||
|
"targets": [
|
||||||
|
{ "datasource": { "type": "prometheus", "uid": "Prometheus" }, "expr": "ftb_xiaobao_stale_summary_count", "refId": "A" }
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 5,
|
||||||
|
"type": "timeseries",
|
||||||
|
"title": "Job Failure Logs",
|
||||||
|
"gridPos": { "x": 0, "y": 4, "w": 8, "h": 5 },
|
||||||
|
"targets": [
|
||||||
|
{ "datasource": { "type": "prometheus", "uid": "Prometheus" }, "expr": "increase(ftb_job_failure_log_total[10m])", "refId": "A" }
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 6,
|
||||||
|
"type": "stat",
|
||||||
|
"title": "Root Disk Free %",
|
||||||
|
"gridPos": { "x": 8, "y": 4, "w": 8, "h": 5 },
|
||||||
|
"targets": [
|
||||||
|
{ "datasource": { "type": "prometheus", "uid": "Prometheus" }, "expr": "100 * node_filesystem_avail_bytes{mountpoint=\"/\",fstype!~\"tmpfs|overlay\"} / node_filesystem_size_bytes{mountpoint=\"/\",fstype!~\"tmpfs|overlay\"}", "refId": "A" }
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 7,
|
||||||
|
"type": "logs",
|
||||||
|
"title": "Server Warning/Error Logs",
|
||||||
|
"gridPos": { "x": 16, "y": 4, "w": 8, "h": 5 },
|
||||||
|
"targets": [
|
||||||
|
{ "datasource": { "type": "loki", "uid": "Loki" }, "expr": "{compose_service=\"server\"} |~ \"warn|error|失败|Slow\"", "refId": "A" }
|
||||||
|
]
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
|
||||||
@@ -0,0 +1,12 @@
|
|||||||
|
apiVersion: 1
|
||||||
|
|
||||||
|
providers:
|
||||||
|
- name: FTB Production
|
||||||
|
orgId: 1
|
||||||
|
folder: FTB
|
||||||
|
type: file
|
||||||
|
disableDeletion: false
|
||||||
|
updateIntervalSeconds: 30
|
||||||
|
options:
|
||||||
|
path: /var/lib/grafana/dashboards
|
||||||
|
|
||||||
@@ -0,0 +1,16 @@
|
|||||||
|
apiVersion: 1
|
||||||
|
|
||||||
|
datasources:
|
||||||
|
- name: Prometheus
|
||||||
|
type: prometheus
|
||||||
|
access: proxy
|
||||||
|
url: http://prometheus:9090
|
||||||
|
isDefault: true
|
||||||
|
editable: false
|
||||||
|
|
||||||
|
- name: Loki
|
||||||
|
type: loki
|
||||||
|
access: proxy
|
||||||
|
url: http://loki:3100
|
||||||
|
editable: false
|
||||||
|
|
||||||
32
deploy/monitoring/loki/config.yml
Normal file
32
deploy/monitoring/loki/config.yml
Normal file
@@ -0,0 +1,32 @@
|
|||||||
|
auth_enabled: false
|
||||||
|
|
||||||
|
server:
|
||||||
|
http_listen_port: 3100
|
||||||
|
|
||||||
|
common:
|
||||||
|
path_prefix: /loki
|
||||||
|
replication_factor: 1
|
||||||
|
ring:
|
||||||
|
kvstore:
|
||||||
|
store: inmemory
|
||||||
|
|
||||||
|
schema_config:
|
||||||
|
configs:
|
||||||
|
- from: 2026-01-01
|
||||||
|
store: tsdb
|
||||||
|
object_store: filesystem
|
||||||
|
schema: v13
|
||||||
|
index:
|
||||||
|
prefix: index_
|
||||||
|
period: 24h
|
||||||
|
|
||||||
|
storage_config:
|
||||||
|
tsdb_shipper:
|
||||||
|
active_index_directory: /loki/index
|
||||||
|
cache_location: /loki/index_cache
|
||||||
|
filesystem:
|
||||||
|
directory: /loki/chunks
|
||||||
|
|
||||||
|
limits_config:
|
||||||
|
retention_period: 168h
|
||||||
|
|
||||||
12
deploy/monitoring/postgres/postgres-queries.yml
Normal file
12
deploy/monitoring/postgres/postgres-queries.yml
Normal file
@@ -0,0 +1,12 @@
|
|||||||
|
ftb_xiaobao:
|
||||||
|
query: |
|
||||||
|
SELECT
|
||||||
|
count(*)::float AS stale_summary_count
|
||||||
|
FROM xiaobao_risk_summaries
|
||||||
|
WHERE dirty = true
|
||||||
|
OR updated_at < now() - interval '6 hours';
|
||||||
|
metrics:
|
||||||
|
- stale_summary_count:
|
||||||
|
usage: GAUGE
|
||||||
|
description: Xiaobao risk summaries that are dirty or older than 6 hours.
|
||||||
|
|
||||||
62
deploy/monitoring/prometheus/alert-rules.yml
Normal file
62
deploy/monitoring/prometheus/alert-rules.yml
Normal file
@@ -0,0 +1,62 @@
|
|||||||
|
groups:
|
||||||
|
- name: ftb-production-alerts
|
||||||
|
rules:
|
||||||
|
- alert: FtbPostgresDown
|
||||||
|
expr: pg_up == 0
|
||||||
|
for: 2m
|
||||||
|
labels:
|
||||||
|
severity: critical
|
||||||
|
annotations:
|
||||||
|
summary: PostgreSQL exporter cannot reach the FTB database.
|
||||||
|
runbook: docs/runbooks/migration-rollback.md
|
||||||
|
|
||||||
|
- alert: FtbDiskPressure
|
||||||
|
expr: |
|
||||||
|
(
|
||||||
|
node_filesystem_avail_bytes{mountpoint="/",fstype!~"tmpfs|overlay"}
|
||||||
|
/
|
||||||
|
node_filesystem_size_bytes{mountpoint="/",fstype!~"tmpfs|overlay"}
|
||||||
|
) < 0.15
|
||||||
|
for: 10m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
annotations:
|
||||||
|
summary: Production host root filesystem has less than 15% free space.
|
||||||
|
runbook: docs/deployment.md
|
||||||
|
|
||||||
|
- alert: FtbSlowApiLogBurst
|
||||||
|
expr: increase(ftb_slow_api_log_total[10m]) > 5
|
||||||
|
for: 2m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
annotations:
|
||||||
|
summary: Slow API request log volume crossed the V2.8 baseline threshold.
|
||||||
|
runbook: docs/deployment.md
|
||||||
|
|
||||||
|
- alert: FtbSlowPrismaLogBurst
|
||||||
|
expr: increase(ftb_slow_prisma_log_total[10m]) > 3
|
||||||
|
for: 2m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
annotations:
|
||||||
|
summary: Slow Prisma query log volume crossed the V2.8 baseline threshold.
|
||||||
|
runbook: docs/deployment.md
|
||||||
|
|
||||||
|
- alert: FtbJobFailureLogBurst
|
||||||
|
expr: increase(ftb_job_failure_log_total[10m]) > 0
|
||||||
|
for: 1m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
annotations:
|
||||||
|
summary: Background or compatibility job failure logs were detected.
|
||||||
|
runbook: docs/runbooks/xiaobao-background-jobs.md
|
||||||
|
|
||||||
|
- alert: FtbXiaobaoSummaryStale
|
||||||
|
expr: ftb_xiaobao_stale_summary_count > 0
|
||||||
|
for: 15m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
annotations:
|
||||||
|
summary: Xiaobao warning summaries are dirty or stale.
|
||||||
|
runbook: docs/runbooks/xiaobao-background-jobs.md
|
||||||
|
|
||||||
45
deploy/monitoring/prometheus/prometheus.yml
Normal file
45
deploy/monitoring/prometheus/prometheus.yml
Normal file
@@ -0,0 +1,45 @@
|
|||||||
|
global:
|
||||||
|
scrape_interval: 15s
|
||||||
|
evaluation_interval: 15s
|
||||||
|
|
||||||
|
rule_files:
|
||||||
|
- /etc/prometheus/alert-rules.yml
|
||||||
|
|
||||||
|
scrape_configs:
|
||||||
|
- job_name: prometheus
|
||||||
|
static_configs:
|
||||||
|
- targets: ['prometheus:9090']
|
||||||
|
|
||||||
|
- job_name: postgres-exporter
|
||||||
|
static_configs:
|
||||||
|
- targets: ['postgres-exporter:9187']
|
||||||
|
|
||||||
|
- job_name: node-exporter
|
||||||
|
static_configs:
|
||||||
|
- targets: ['node-exporter:9100']
|
||||||
|
|
||||||
|
- job_name: cadvisor
|
||||||
|
static_configs:
|
||||||
|
- targets: ['cadvisor:8080']
|
||||||
|
|
||||||
|
- job_name: promtail
|
||||||
|
static_configs:
|
||||||
|
- targets: ['promtail:9080']
|
||||||
|
|
||||||
|
- job_name: blackbox-http
|
||||||
|
metrics_path: /probe
|
||||||
|
params:
|
||||||
|
module: [http_2xx]
|
||||||
|
static_configs:
|
||||||
|
- targets:
|
||||||
|
- http://nginx/api/v1/health/version
|
||||||
|
- http://nginx/api/v1/config/ai
|
||||||
|
- http://nginx/api/v1/v2.2/requirements?productId=__smoke__&limit=1
|
||||||
|
relabel_configs:
|
||||||
|
- source_labels: [__address__]
|
||||||
|
target_label: __param_target
|
||||||
|
- source_labels: [__param_target]
|
||||||
|
target_label: instance
|
||||||
|
- target_label: __address__
|
||||||
|
replacement: blackbox-exporter:9115
|
||||||
|
|
||||||
53
deploy/monitoring/promtail/config.yml
Normal file
53
deploy/monitoring/promtail/config.yml
Normal file
@@ -0,0 +1,53 @@
|
|||||||
|
server:
|
||||||
|
http_listen_port: 9080
|
||||||
|
grpc_listen_port: 0
|
||||||
|
|
||||||
|
positions:
|
||||||
|
filename: /tmp/positions.yml
|
||||||
|
|
||||||
|
clients:
|
||||||
|
- url: http://loki:3100/loki/api/v1/push
|
||||||
|
|
||||||
|
scrape_configs:
|
||||||
|
- job_name: docker
|
||||||
|
docker_sd_configs:
|
||||||
|
- host: unix:///var/run/docker.sock
|
||||||
|
refresh_interval: 15s
|
||||||
|
relabel_configs:
|
||||||
|
- source_labels: ['__meta_docker_container_name']
|
||||||
|
regex: '/(.*)'
|
||||||
|
target_label: container
|
||||||
|
- source_labels: ['__meta_docker_container_label_com_docker_compose_service']
|
||||||
|
target_label: compose_service
|
||||||
|
- source_labels: ['__meta_docker_container_label_com_docker_compose_project']
|
||||||
|
target_label: compose_project
|
||||||
|
pipeline_stages:
|
||||||
|
- docker: {}
|
||||||
|
- match:
|
||||||
|
selector: '{compose_service="server"} |= "Slow API request"'
|
||||||
|
stages:
|
||||||
|
- metrics:
|
||||||
|
ftb_slow_api_log_total:
|
||||||
|
type: Counter
|
||||||
|
description: Slow API request log entries emitted by the NestJS server.
|
||||||
|
config:
|
||||||
|
action: inc
|
||||||
|
- match:
|
||||||
|
selector: '{compose_service="server"} |= "Slow Prisma query"'
|
||||||
|
stages:
|
||||||
|
- metrics:
|
||||||
|
ftb_slow_prisma_log_total:
|
||||||
|
type: Counter
|
||||||
|
description: Slow Prisma query log entries emitted by the NestJS server.
|
||||||
|
config:
|
||||||
|
action: inc
|
||||||
|
- match:
|
||||||
|
selector: '{compose_service="server"} |~ "AppData relation sync failed|AI 调用失败|AI 风险解读调用失败"'
|
||||||
|
stages:
|
||||||
|
- metrics:
|
||||||
|
ftb_job_failure_log_total:
|
||||||
|
type: Counter
|
||||||
|
description: Compatibility sync, AI job, or background task failure logs.
|
||||||
|
config:
|
||||||
|
action: inc
|
||||||
|
|
||||||
@@ -123,7 +123,117 @@ services:
|
|||||||
server:
|
server:
|
||||||
condition: service_healthy
|
condition: service_healthy
|
||||||
|
|
||||||
|
prometheus:
|
||||||
|
image: prom/prometheus:v2.53.1
|
||||||
|
profiles: ['monitoring']
|
||||||
|
restart: unless-stopped
|
||||||
|
command:
|
||||||
|
- '--config.file=/etc/prometheus/prometheus.yml'
|
||||||
|
- '--storage.tsdb.path=/prometheus'
|
||||||
|
- '--storage.tsdb.retention.time=${PROMETHEUS_RETENTION:-15d}'
|
||||||
|
- '--web.enable-lifecycle'
|
||||||
|
ports:
|
||||||
|
- '${PROMETHEUS_PORT:-9090}:9090'
|
||||||
|
volumes:
|
||||||
|
- ./deploy/monitoring/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
|
||||||
|
- ./deploy/monitoring/prometheus/alert-rules.yml:/etc/prometheus/alert-rules.yml:ro
|
||||||
|
- prometheus_data:/prometheus
|
||||||
|
depends_on:
|
||||||
|
- postgres-exporter
|
||||||
|
- node-exporter
|
||||||
|
- cadvisor
|
||||||
|
- promtail
|
||||||
|
- blackbox-exporter
|
||||||
|
|
||||||
|
grafana:
|
||||||
|
image: grafana/grafana:11.1.0
|
||||||
|
profiles: ['monitoring']
|
||||||
|
restart: unless-stopped
|
||||||
|
ports:
|
||||||
|
- '${GRAFANA_PORT:-3002}:3000'
|
||||||
|
environment:
|
||||||
|
GF_SECURITY_ADMIN_USER: ${GRAFANA_ADMIN_USER:-admin}
|
||||||
|
GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD:-change-me-monitoring-password}
|
||||||
|
GF_USERS_ALLOW_SIGN_UP: 'false'
|
||||||
|
volumes:
|
||||||
|
- grafana_data:/var/lib/grafana
|
||||||
|
- ./deploy/monitoring/grafana/provisioning:/etc/grafana/provisioning:ro
|
||||||
|
- ./deploy/monitoring/grafana/dashboards:/var/lib/grafana/dashboards:ro
|
||||||
|
depends_on:
|
||||||
|
- prometheus
|
||||||
|
- loki
|
||||||
|
|
||||||
|
loki:
|
||||||
|
image: grafana/loki:2.9.8
|
||||||
|
profiles: ['monitoring']
|
||||||
|
restart: unless-stopped
|
||||||
|
command: ['-config.file=/etc/loki/config.yml']
|
||||||
|
volumes:
|
||||||
|
- ./deploy/monitoring/loki/config.yml:/etc/loki/config.yml:ro
|
||||||
|
- loki_data:/loki
|
||||||
|
|
||||||
|
promtail:
|
||||||
|
image: grafana/promtail:2.9.8
|
||||||
|
profiles: ['monitoring']
|
||||||
|
restart: unless-stopped
|
||||||
|
command: ['-config.file=/etc/promtail/config.yml']
|
||||||
|
volumes:
|
||||||
|
- ./deploy/monitoring/promtail/config.yml:/etc/promtail/config.yml:ro
|
||||||
|
- /var/lib/docker/containers:/var/lib/docker/containers:ro
|
||||||
|
- /var/run/docker.sock:/var/run/docker.sock:ro
|
||||||
|
depends_on:
|
||||||
|
- loki
|
||||||
|
|
||||||
|
postgres-exporter:
|
||||||
|
image: quay.io/prometheuscommunity/postgres-exporter:v0.15.0
|
||||||
|
profiles: ['monitoring']
|
||||||
|
restart: unless-stopped
|
||||||
|
command:
|
||||||
|
- '--extend.query-path=/etc/postgres-exporter/postgres-queries.yml'
|
||||||
|
environment:
|
||||||
|
DATA_SOURCE_NAME: postgresql://${POSTGRES_USER:-postgres}:${POSTGRES_PASSWORD}@postgres:5432/${POSTGRES_DB:-ftb_pm}?sslmode=disable
|
||||||
|
volumes:
|
||||||
|
- ./deploy/monitoring/postgres/postgres-queries.yml:/etc/postgres-exporter/postgres-queries.yml:ro
|
||||||
|
depends_on:
|
||||||
|
postgres:
|
||||||
|
condition: service_healthy
|
||||||
|
|
||||||
|
node-exporter:
|
||||||
|
image: prom/node-exporter:v1.8.2
|
||||||
|
profiles: ['monitoring']
|
||||||
|
restart: unless-stopped
|
||||||
|
command:
|
||||||
|
- '--path.rootfs=/host'
|
||||||
|
volumes:
|
||||||
|
- /:/host:ro,rslave
|
||||||
|
|
||||||
|
cadvisor:
|
||||||
|
image: gcr.io/cadvisor/cadvisor:v0.49.1
|
||||||
|
profiles: ['monitoring']
|
||||||
|
restart: unless-stopped
|
||||||
|
privileged: true
|
||||||
|
devices:
|
||||||
|
- /dev/kmsg:/dev/kmsg
|
||||||
|
volumes:
|
||||||
|
- /:/rootfs:ro
|
||||||
|
- /var/run:/var/run:ro
|
||||||
|
- /sys:/sys:ro
|
||||||
|
- /var/lib/docker/:/var/lib/docker:ro
|
||||||
|
- /dev/disk/:/dev/disk:ro
|
||||||
|
|
||||||
|
blackbox-exporter:
|
||||||
|
image: prom/blackbox-exporter:v0.25.0
|
||||||
|
profiles: ['monitoring']
|
||||||
|
restart: unless-stopped
|
||||||
|
command:
|
||||||
|
- '--config.file=/etc/blackbox/config.yml'
|
||||||
|
volumes:
|
||||||
|
- ./deploy/monitoring/blackbox/config.yml:/etc/blackbox/config.yml:ro
|
||||||
|
|
||||||
volumes:
|
volumes:
|
||||||
postgres_data:
|
postgres_data:
|
||||||
redis_data:
|
redis_data:
|
||||||
server_data:
|
server_data:
|
||||||
|
prometheus_data:
|
||||||
|
grafana_data:
|
||||||
|
loki_data:
|
||||||
|
|||||||
@@ -175,12 +175,13 @@ cp .env.production.example .env.production
|
|||||||
5. 执行 `docker compose --env-file .env.production -f docker-compose.prod.yml pull web server` 拉取本次 SHA 镜像。
|
5. 执行 `docker compose --env-file .env.production -f docker-compose.prod.yml pull web server` 拉取本次 SHA 镜像。
|
||||||
6. 启动数据库与 Redis，执行 `pnpm --filter server db:deploy`。
|
6. 启动数据库与 Redis，执行 `pnpm --filter server db:deploy`。
|
||||||
7. 执行 `docker compose --env-file .env.production -f docker-compose.prod.yml up -d --remove-orphans` 重启服务。
|
7. 执行 `docker compose --env-file .env.production -f docker-compose.prod.yml up -d --remove-orphans` 重启服务。
|
||||||
8. 通过 `/api/v1/health/version` 校验运行中的后端版本是否等于本次 commit SHA。
|
8. 运行发布 smoke test：校验 `/api/v1/health/version`、前端首页、产品页、产品 API、V2.2 读路径和 AI 配置端点，并确认运行中的后端版本等于本次 commit SHA。
|
||||||
|
|
||||||
如果最后一步失败，Actions 会红掉，说明“代码已合并”不等于“线上容器已更新”。本地或服务器也可以手工运行：
|
如果最后一步失败，Actions 会红掉，说明“代码已合并”不等于“线上容器已更新”或关键读路径不可用。本地或服务器也可以手工运行：
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
pnpm deploy:check-runtime http://localhost/api/v1/health/version <expected-commit-sha>
|
pnpm deploy:check-runtime http://localhost/api/v1/health/version <expected-commit-sha>
|
||||||
|
pnpm deploy:smoke -- --base-url http://localhost --expected-version <expected-commit-sha>
|
||||||
```
|
```
|
||||||
|
|
||||||
## Nginx 路由
|
## Nginx 路由
|
||||||
@@ -193,6 +194,30 @@ pnpm deploy:check-runtime http://localhost/api/v1/health/version <expected-commi
|
|||||||
|
|
||||||
生产 Compose 默认只监听 HTTP 80。HTTPS 建议优先交给云负载均衡、CDN 或宿主机外层证书管理工具；如果要让本 Compose 内的 Nginx 直接处理 HTTPS，可以在后续增加证书 volume 和 443 server block。
|
生产 Compose 默认只监听 HTTP 80。HTTPS 建议优先交给云负载均衡、CDN 或宿主机外层证书管理工具；如果要让本 Compose 内的 Nginx 直接处理 HTTPS，可以在后续增加证书 volume 和 443 server block。
|
||||||
|
|
||||||
|
## 监控与告警基线
|
||||||
|
|
||||||
|
V2.8 提供可选 `monitoring` profile，不影响默认生产启动。启用前先在 `.env.production` 设置 `GRAFANA_ADMIN_PASSWORD`，不要使用示例密码对外暴露 Grafana。
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker compose --env-file .env.production -f docker-compose.prod.yml --profile monitoring up -d
|
||||||
|
```
|
||||||
|
|
||||||
|
默认入口：
|
||||||
|
|
||||||
|
- Prometheus: `http://localhost:9090`
|
||||||
|
- Grafana: `http://localhost:3002`
|
||||||
|
|
||||||
|
配置目录在 `deploy/monitoring/`。基线覆盖：
|
||||||
|
|
||||||
|
- `FtbPostgresDown`：PostgreSQL 不可用。
|
||||||
|
- `FtbDiskPressure`：宿主机根分区低于 15% 可用空间。
|
||||||
|
- `FtbSlowApiLogBurst`：服务端慢 API 日志在 10 分钟内超过阈值。
|
||||||
|
- `FtbSlowPrismaLogBurst`：慢 Prisma 查询日志在 10 分钟内超过阈值。
|
||||||
|
- `FtbJobFailureLogBurst`：AppData 同步、AI 调用或后台任务失败日志出现。
|
||||||
|
- `FtbXiaobaoSummaryStale`：`xiaobao_risk_summaries` 存在 dirty 或超过 6 小时未更新的摘要。
|
||||||
|
|
||||||
|
Prometheus 只加载本地规则，不提交真实通知密钥。接入 Slack、企业微信、邮件等通知时，把 Alertmanager receiver 放在未跟踪的服务器文件或环境变量中。
|
||||||
|
|
||||||
## 数据持久化
|
## 数据持久化
|
||||||
|
|
||||||
生产 Compose 使用三个命名 volume：
|
生产 Compose 使用三个命名 volume：
|
||||||
@@ -235,6 +260,76 @@ pnpm consistency:v25 -- --url http://localhost/api/v1/consistency
|
|||||||
- `warn` 需要记录原因；历史数据没有审计事件属于预期 warning，不阻断 V2.5。
|
- `warn` 需要记录原因；历史数据没有审计事件属于预期 warning，不阻断 V2.5。
|
||||||
- 后台页面 `/admin/consistency` 展示同一份报告，需要当前用户具备 `consistency:view`。
|
- 后台页面 `/admin/consistency` 展示同一份报告，需要当前用户具备 `consistency:view`。
|
||||||
|
|
||||||
|
## 备份与恢复
|
||||||
|
|
||||||
|
备份分两类：PostgreSQL 业务数据和 `server_data` volume 中的运行时配置。真实备份文件默认写到 `backups/`，该目录已加入 `.gitignore`，不要把 dump 或 tar 包提交到仓库。
|
||||||
|
|
||||||
|
先演练命令，不写文件：
|
||||||
|
|
||||||
|
```bash
|
||||||
|
pnpm backup:postgres -- --dry-run --env-file .env.production
|
||||||
|
pnpm backup:server-data -- --dry-run --env-file .env.production
|
||||||
|
```
|
||||||
|
|
||||||
|
执行正式备份：
|
||||||
|
|
||||||
|
```bash
|
||||||
|
pnpm backup:postgres -- --env-file .env.production
|
||||||
|
pnpm backup:server-data -- --env-file .env.production
|
||||||
|
```
|
||||||
|
|
||||||
|
PostgreSQL 备份使用 `pg_dump --format=custom --no-owner --no-acl`，便于跨环境恢复。`server_data` 备份使用只读 volume mount 和 `tar -czf`，覆盖 AI Provider 配置等后端运行时文件。
|
||||||
|
|
||||||
|
恢复数据库必须显式确认覆盖。默认命令会拒绝执行，防止误删现有库：
|
||||||
|
|
||||||
|
```bash
|
||||||
|
pnpm restore:postgres -- --env-file .env.production --input backups/postgres/ftb_pm-postgres-20260708T120000Z.dump
|
||||||
|
```
|
||||||
|
|
||||||
|
确认要把目标数据库重建为 fresh DB 后，再运行：
|
||||||
|
|
||||||
|
```bash
|
||||||
|
pnpm restore:postgres -- \
|
||||||
|
--env-file .env.production \
|
||||||
|
--input backups/postgres/ftb_pm-postgres-20260708T120000Z.dump \
|
||||||
|
--confirm-overwrite
|
||||||
|
```
|
||||||
|
|
||||||
|
恢复前建议先 dry-run 看清将执行的 `psql/dropdb/createdb/pg_restore` 步骤：
|
||||||
|
|
||||||
|
```bash
|
||||||
|
pnpm restore:postgres -- \
|
||||||
|
--dry-run \
|
||||||
|
--confirm-overwrite \
|
||||||
|
--env-file .env.production \
|
||||||
|
--input backups/postgres/ftb_pm-postgres-20260708T120000Z.dump
|
||||||
|
```
|
||||||
|
|
||||||
|
如果要恢复到临时库做校验，不覆盖当前生产库：
|
||||||
|
|
||||||
|
```bash
|
||||||
|
pnpm restore:postgres -- \
|
||||||
|
--env-file .env.production \
|
||||||
|
--input backups/postgres/ftb_pm-postgres-20260708T120000Z.dump \
|
||||||
|
--target-db ftb_pm_restore_check \
|
||||||
|
--confirm-overwrite
|
||||||
|
```
|
||||||
|
|
||||||
|
## 运维 Runbooks
|
||||||
|
|
||||||
|
生产发布、迁移和异常处置优先使用这些手册：
|
||||||
|
|
||||||
|
- `docs/runbooks/migration-rollback.md`：发布失败、迁移失败、数据恢复和镜像回滚。
|
||||||
|
- `docs/runbooks/appdata-retirement.md`：AppData key 分阶段退场、双读核对、归档和回滚。
|
||||||
|
- `docs/runbooks/xiaobao-background-jobs.md`：小宝摘要 stale 告警、手动刷新和未来后台任务规则。
|
||||||
|
- `docs/production-readiness.md`：生产发布前后证据清单。
|
||||||
|
|
||||||
|
提交前运行 runbook 扫描：
|
||||||
|
|
||||||
|
```bash
|
||||||
|
pnpm docs:check-runbooks
|
||||||
|
```
|
||||||
|
|
||||||
## 升级流程
|
## 升级流程
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
@@ -248,7 +343,8 @@ docker compose --env-file .env.production -f docker-compose.prod.yml exec server
|
|||||||
升级前建议先备份数据库：
|
升级前建议先备份数据库：
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
docker compose --env-file .env.production -f docker-compose.prod.yml exec postgres pg_dump -U "$POSTGRES_USER" "$POSTGRES_DB" > ftb_pm_backup.sql
|
pnpm backup:postgres -- --env-file .env.production
|
||||||
|
pnpm backup:server-data -- --env-file .env.production
|
||||||
```
|
```
|
||||||
|
|
||||||
## 常见排查
|
## 常见排查
|
||||||
|
|||||||
24
docs/production-readiness.md
Normal file
24
docs/production-readiness.md
Normal file
@@ -0,0 +1,24 @@
|
|||||||
|
# Production Readiness Checklist
|
||||||
|
|
||||||
|
Use this before a production release and again after the release smoke test. Each item requires evidence, not a verbal assertion.
|
||||||
|
|
||||||
|
| Area | Gate | Evidence |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| Backup and restore | PostgreSQL backup dry-run and server_data backup dry-run are reviewed. | `pnpm backup:postgres -- --dry-run --env-file .env.production` and `pnpm backup:server-data -- --dry-run --env-file .env.production` output saved in release notes. |
|
||||||
|
| Backup and restore | Restore command refuses overwrite without explicit confirmation. | `pnpm restore:postgres -- --env-file .env.production --input backups/postgres/ftb_pm-postgres-20260708T120000Z.dump` exits non-zero and names `--confirm-overwrite`. |
|
||||||
|
| Backup and restore | Fresh DB restore rehearsal completed before destructive restore. | Temporary database restore command and row-count comparison from `docs/runbooks/migration-rollback.md`. |
|
||||||
|
| Smoke tests | Release smoke is wired into GitHub Actions. | `.github/workflows/deploy-production.yml` runs `scripts/smoke-test-release.mjs` with `--expected-version`. |
|
||||||
|
| Smoke tests | Manual smoke can be run against the target. | `pnpm deploy:smoke -- --base-url http://localhost --expected-version "$APP_VERSION"` output. |
|
||||||
|
| Monitoring | Monitoring profile renders and can start without committed secrets. | `docker compose --env-file .env.production -f docker-compose.prod.yml --profile monitoring config` exits 0. |
|
||||||
|
| Monitoring | Required alert rules exist. | `FtbPostgresDown`, `FtbDiskPressure`, `FtbSlowApiLogBurst`, `FtbSlowPrismaLogBurst`, `FtbJobFailureLogBurst`, and `FtbXiaobaoSummaryStale` visible in Prometheus. |
|
||||||
|
| Audit | Domain write APIs record actor and resource scope where implemented. | API request sample or audit log sample for Product, Requirement, Project, Version, execution entities, and dictionary writes. |
|
||||||
|
| RBAC | Project and version operations enforce Owner, Admin, Member, Viewer boundaries where enabled. | Permission matrix test result or manual account walkthrough attached to release notes. |
|
||||||
|
| Consistency | AppData and relation tables are checked for migrated domains. | Count and missing-partition-key queries from `docs/runbooks/appdata-retirement.md`. |
|
||||||
|
| Performance | Slow API and slow Prisma thresholds are configured. | `API_SLOW_REQUEST_MS` and `PRISMA_SLOW_QUERY_MS` values recorded, plus Grafana slow-log panels checked. |
|
||||||
|
| Performance | V2.2 hot reads avoid full-table scans. | Requirement pool smoke uses `productId`; query plan or service test evidence attached for high-volume domains. |
|
||||||
|
| Post-release | Runtime version matches the release SHA. | `/api/v1/health/version` payload or `pnpm deploy:check-runtime` output. |
|
||||||
|
| Post-release | Product, requirement, workspace, Xiaobao, and AI config read paths respond. | `pnpm deploy:smoke` output attached. |
|
||||||
|
| Post-release | On-call rollback path is known. | `docs/runbooks/migration-rollback.md` link included in release notes. |
|
||||||
|
|
||||||
|
Release owner signs off only after all required evidence is attached to the release notes or incident record.
|
||||||
|
|
||||||
@@ -1,26 +1,12 @@
|
|||||||
# 开发路线图
|
# 开发路线图
|
||||||
|
|
||||||
## 当前阶段：V2.7 已完成 — 下一阶段 V2.8 生产硬化与运维闭环集成
|
## 当前阶段：V2.8 已完成 — 统一验证与生产交付收口
|
||||||
|
|
||||||
V2.7 已在关系表主源方向上补齐企业协作和治理能力：通知、评论与提及、项目成员治理、管理驾驶舱、治理字典、以及统一 RBAC/audit 适配器。V2.7 不新增 AppData 主存储。
|
V2.8 已在既有生产 CI/CD 基线上补齐运维闭环：备份恢复演练、发布 smoke test、监控告警、日志检索、迁移回滚 runbook、AppData 退场 runbook、小宝后台化 runbook 和生产 readiness 证据清单。本阶段不重新设计业务流程，专注把生产发布和故障处置做成可验证、可复盘、可回滚的标准流程。
|
||||||
|
|
||||||
### 当前重点
|
V2.4 已将高增长和核心业务领域从“AppData 主写 + 关系表同步副本”推进到“领域 CRUD 主写关系表 + AppData 兼容/迁移兜底”。V2.5 已收口后端权限、审计、AppData 禁写和一致性核对。V2.6 已完成大数据性能增强、小宝风险后台化、AI 解读队列和运行时 Ops 看板。V2.7 已补齐企业协作和治理能力，且不新增 AppData 主存储。
|
||||||
|
|
||||||
1. **协作通知**：通知记录、已读状态、NotificationBell，并覆盖 assignment / mention / risk_alert / overdue_item 稳定事件类型。
|
### V2.5-V2.8 完成范围
|
||||||
2. **通用评论**：DevTask/TestCase/Bug/Requirement/VersionPlan 统一评论面板，支持 `@成员名` 和显式成员选择，创建/删除写 audit。
|
|
||||||
3. **项目成员治理**：Owner/Admin/Member/Viewer 服务端强校验，禁止移除最后 Owner，角色变更写 audit。
|
|
||||||
4. **管理驾驶舱**：只读关系表和 summary，聚合活跃版本、逾期、阻塞、风险和成员负载。
|
|
||||||
5. **治理设置**：集中维护 task category、requirement type/platform/source，使用中的字典不可硬删，支持导入导出。
|
|
||||||
|
|
||||||
V2.4 已将高增长和核心业务领域从“AppData 主写 + 关系表同步副本”推进到“领域 CRUD 主写关系表 + AppData 兼容/迁移兜底”。V2.2 快读 API 和 V2.3 AppData 写后同步继续保留，但它们现在是兼容基础设施，不再是已迁移领域的数据新鲜度主链路。
|
|
||||||
|
|
||||||
V2.5 的目标是正式收口后端权限、审计、AppData 禁写和一致性核对。AppData 不能直接删除，必须按“禁写 → 双读核对 → 移除 fallback → 只读归档/导出 → 后续删表”的顺序推进。
|
|
||||||
|
|
||||||
V2.6 的目标是在关系表主源稳定后完成大数据性能增强、小宝风险后台化、AI 解读队列和运行时 Ops 看板，让高增长热路径、后台任务和风险摘要不再依赖页面打开。
|
|
||||||
|
|
||||||
V2.8 的目标是在现有生产部署基线上补齐备份恢复演练、发布 smoke test、监控告警、日志检索、迁移回滚和运维手册，形成生产交付稳定版。
|
|
||||||
|
|
||||||
### V2.5-V2.7 完成范围
|
|
||||||
|
|
||||||
1. **RBAC 收口**：领域 mutation API 已接入服务端权限校验、资源作用域和当前用户上下文。
|
1. **RBAC 收口**：领域 mutation API 已接入服务端权限校验、资源作用域和当前用户上下文。
|
||||||
2. **审计事件**：领域 mutation 通过 `audit_events` 写 append-only audit event，支持后台查询和敏感字段脱敏。
|
2. **审计事件**：领域 mutation 通过 `audit_events` 写 append-only audit event，支持后台查询和敏感字段脱敏。
|
||||||
@@ -37,6 +23,11 @@ V2.8 的目标是在现有生产部署基线上补齐备份恢复演练、发布
|
|||||||
13. **项目成员治理**：已补项目成员 Owner/Admin/Member/Viewer 服务端治理，禁止移除最后 Owner，角色变更写审计。
|
13. **项目成员治理**：已补项目成员 Owner/Admin/Member/Viewer 服务端治理，禁止移除最后 Owner，角色变更写审计。
|
||||||
14. **管理驾驶舱**：已补只读关系表和 summary 的管理概览，聚合活跃版本、逾期、阻塞、风险和成员负载。
|
14. **管理驾驶舱**：已补只读关系表和 summary 的管理概览，聚合活跃版本、逾期、阻塞、风险和成员负载。
|
||||||
15. **治理设置**：已补 task category、requirement type/platform/source 等治理字典能力，使用中的字典不可硬删，支持导入导出。
|
15. **治理设置**：已补 task category、requirement type/platform/source 等治理字典能力，使用中的字典不可硬删，支持导入导出。
|
||||||
|
16. **备份恢复自动化**：已补 PostgreSQL dump、`server_data` volume 备份、fresh DB restore dry-run 和显式覆盖确认。
|
||||||
|
17. **发布 smoke test**：已补部署后 runtime version、前端根页、产品页、产品 API、V2.2 读路径和 AI 配置校验。
|
||||||
|
18. **监控告警基线**：已补 Prometheus/Grafana/Loki/Promtail 可选 profile，覆盖慢 API、慢 Prisma、任务失败、小宝摘要 stale、磁盘压力和 DB 可用性。
|
||||||
|
19. **运维 runbook**：已补迁移回滚、AppData 退场、小宝后台化处置步骤、决策点和数据风险。
|
||||||
|
20. **生产 readiness 清单**：已补 backup/restore、smoke、monitoring、audit、RBAC、consistency、performance 和 post-release verification 证据项。
|
||||||
|
|
||||||
## V2 分阶段交付链路
|
## V2 分阶段交付链路
|
||||||
|
|
||||||
@@ -58,7 +49,7 @@ V2.8 的目标是在现有生产部署基线上补齐备份恢复演练、发布
|
|||||||
- 项目已经不是早期骨架。前端业务功能已覆盖产品、项目、版本详情、需求池、工作台、成员/角色/任务类型、加班、小宝预警和 AI 配置等主要管理端路由。
|
- 项目已经不是早期骨架。前端业务功能已覆盖产品、项目、版本详情、需求池、工作台、成员/角色/任务类型、加班、小宝预警和 AI 配置等主要管理端路由。
|
||||||
- 版本详情已有需求、调研、产品方案、UI、开发任务、测试用例、Bug、概览等核心 Tab；渲染重的路径优先接入关系表快读，并保留 AppData fallback。
|
- 版本详情已有需求、调研、产品方案、UI、开发任务、测试用例、Bug、概览等核心 Tab；渲染重的路径优先接入关系表快读，并保留 AppData fallback。
|
||||||
- 后端已落地 Product、Project、Version、Requirement、VersionPlan、DevTask、TestCase、Bug、Member、TaskCategory、TaskWorklog、Overtime、WorkActivity 领域 CRUD/write API。
|
- 后端已落地 Product、Project、Version、Requirement、VersionPlan、DevTask、TestCase、Bug、Member、TaskCategory、TaskWorklog、Overtime、WorkActivity 领域 CRUD/write API。
|
||||||
- Prisma schema 已包含 Product、Project、Version、Requirement、VersionPlan、DevTask、TestCase、Bug、WorkActivity、TaskWorklog、Overtime、Xiaobao、AiLog、AppData 等关系模型；高增长表的分区 migration 已落地。
|
- Prisma schema 已包含 Product、Project、Version、Requirement、VersionPlan、DevTask、TestCase、Bug、WorkActivity、TaskWorklog、Overtime、Xiaobao、AiLog、AppData、Notification、Comment、ProjectMember、GovernanceDictionary 等关系模型；高增长表的分区 migration 已落地。
|
||||||
- 主写入源已经切到领域 API：前端 store 优先调用 `apps/web/lib/domain-api.ts`，AppData 只保留兼容读取、历史核对和少量旧配置形状。
|
- 主写入源已经切到领域 API：前端 store 优先调用 `apps/web/lib/domain-api.ts`，AppData 只保留兼容读取、历史核对和少量旧配置形状。
|
||||||
- V2.5 后端服务端权限、审计和一致性控制面已启用：领域 mutation 使用 `@ProtectedMutation()`，审计写 `audit_events`，后台查询需要 `audit:view` / `consistency:view`。
|
- V2.5 后端服务端权限、审计和一致性控制面已启用：领域 mutation 使用 `@ProtectedMutation()`，审计写 `audit_events`，后台查询需要 `audit:view` / `consistency:view`。
|
||||||
- 所有业务 AppData key 已明确冻结或只读归档；`PUT /api/v1/data/:key` 对这些 key 返回 `APP_DATA_WRITE_FROZEN`，`GET` 留作历史核对与归档。
|
- 所有业务 AppData key 已明确冻结或只读归档；`PUT /api/v1/data/:key` 对这些 key 返回 `APP_DATA_WRITE_FROZEN`，`GET` 留作历史核对与归档。
|
||||||
@@ -76,10 +67,16 @@ V2.8 的目标是在现有生产部署基线上补齐备份恢复演练、发布
|
|||||||
- V2.7.3 已新增项目成员治理 API 和项目页成员面板，服务端强校验 Owner/Admin/Member/Viewer 边界。
|
- V2.7.3 已新增项目成员治理 API 和项目页成员面板，服务端强校验 Owner/Admin/Member/Viewer 边界。
|
||||||
- V2.7.4 已新增管理驾驶舱和治理设置，聚合关系表指标并维护治理字典。
|
- V2.7.4 已新增管理驾驶舱和治理设置，聚合关系表指标并维护治理字典。
|
||||||
- V2.7.5 已新增协作治理 RBAC/audit adapter，避免新增模块绕开服务端权限和审计边界。
|
- V2.7.5 已新增协作治理 RBAC/audit adapter，避免新增模块绕开服务端权限和审计边界。
|
||||||
|
- V2.8 新增运维交付物集中在 `scripts/`、`.github/workflows/deploy-production.yml`、`deploy/monitoring/`、`docs/runbooks/`、`docs/deployment.md` 和 `docs/production-readiness.md`。
|
||||||
|
- AppData 退场、RBAC/审计、性能、小宝后台化、协作治理和生产发布都通过 production readiness 证据项追踪，避免把运维稳定版误当成一次性口头验收。
|
||||||
|
|
||||||
### 已完成（按时间倒序）
|
### 已完成（按时间倒序）
|
||||||
|
|
||||||
**2026-07-08**
|
**2026-07-08**
|
||||||
|
- V2.8 added PostgreSQL backup, fresh DB restore with explicit overwrite confirmation, and `server_data` volume backup automation.
|
||||||
|
- V2.8 added release smoke suite and wired GitHub Actions deployment verification to runtime version, frontend root, products, V2.2 read path, and AI config checks.
|
||||||
|
- V2.8 added optional monitoring profile with Prometheus, Grafana, Loki, Promtail, postgres-exporter, node-exporter, cAdvisor, and blackbox-exporter.
|
||||||
|
- V2.8 added migration rollback, AppData retirement, Xiaobao background jobs runbooks, and production readiness evidence checklist.
|
||||||
- V2.7.5 added shared collaboration/governance RBAC and audit adapters so notification, comment, project-member, management, and governance modules keep a single permission/audit boundary.
|
- V2.7.5 added shared collaboration/governance RBAC and audit adapters so notification, comment, project-member, management, and governance modules keep a single permission/audit boundary.
|
||||||
- V2.7.4 added management and governance admin pages for relation-backed overview metrics and dictionary governance.
|
- V2.7.4 added management and governance admin pages for relation-backed overview metrics and dictionary governance.
|
||||||
- V2.7.3 added project-member governance APIs and project member panel with Owner/Admin/Member/Viewer safeguards.
|
- V2.7.3 added project-member governance APIs and project member panel with Owner/Admin/Member/Viewer safeguards.
|
||||||
@@ -188,12 +185,12 @@ V2.8 的目标是在现有生产部署基线上补齐备份恢复演练、发布
|
|||||||
|
|
||||||
### 进行中
|
### 进行中
|
||||||
|
|
||||||
- V2.8 生产硬化与运维闭环：备份恢复演练、发布 smoke test、监控告警、日志检索、迁移回滚和运维手册。
|
- V2.8 统一验证与发布收口：合并后集中跑类型、测试、Prisma、部署、runbook、备份恢复和 smoke 验证。
|
||||||
- 项目详情页 VersionCard 状态胶囊数据联动（部分已完成）
|
- 项目详情页 VersionCard 状态胶囊数据联动（部分已完成）
|
||||||
|
|
||||||
## V2 — 后端接入
|
## V2 — 后端接入
|
||||||
|
|
||||||
NestJS + Prisma + PostgreSQL 已推进到 V2.7。第一阶段用 `app_data` JSONB 文档表承接现有 store 数据形状，避免浏览器清站点数据导致业务数据丢失；第二阶段建立分区关系表、V2.2 快读 API 和 V2.3 AppData 写后同步；第三阶段 V2.4 已逐领域启用写 API，让前端 store 从 AppData 主写入迁移到领域 CRUD 主写；第四阶段 V2.5 已冻结 AppData 业务写入并收口服务端 RBAC、审计和一致性校验；第五阶段 V2.6 已完成大数据性能和小宝后台化；第六阶段 V2.7 已补齐协作治理能力。
|
NestJS + Prisma + PostgreSQL 已推进到 V2.8。第一阶段用 `app_data` JSONB 文档表承接现有 store 数据形状，避免浏览器清站点数据导致业务数据丢失；第二阶段建立分区关系表、V2.2 快读 API 和 V2.3 AppData 写后同步；第三阶段 V2.4 已逐领域启用写 API，让前端 store 从 AppData 主写入迁移到领域 CRUD 主写；第四阶段 V2.5 已冻结 AppData 业务写入并收口服务端 RBAC、审计和一致性校验；第五阶段 V2.6 已完成大数据性能和小宝后台化；第六阶段 V2.7 已补齐协作治理能力；第七阶段 V2.8 已补齐生产运维、备份恢复、监控告警和回滚手册。
|
||||||
|
|
||||||
### 关键任务
|
### 关键任务
|
||||||
|
|
||||||
@@ -306,6 +303,6 @@ V2.5 完成后的保留边界：`GET /api/v1/data/:key` 仍可读历史 JSON；X
|
|||||||
|------|------|
|
|------|------|
|
||||||
| V1 业务流程打磨 | 进行中 |
|
| V1 业务流程打磨 | 进行中 |
|
||||||
| V1 朋友试用反馈 | 持续中 |
|
| V1 朋友试用反馈 | 持续中 |
|
||||||
| V2 后端接入 | 进行中（V2.7 已完成；V2.8 生产硬化与运维闭环待集成） |
|
| V2 后端接入 | V2.1-V2.8 已完成，等待统一验证与发布授权 |
|
||||||
| V3 AI 集成 | 等 V2 数据沉淀 |
|
| V3 AI 集成 | 等 V2 数据沉淀 |
|
||||||
| 公开发布 | TBD |
|
| 公开发布 | TBD |
|
||||||
|
|||||||
72
docs/runbooks/appdata-retirement.md
Normal file
72
docs/runbooks/appdata-retirement.md
Normal file
@@ -0,0 +1,72 @@
|
|||||||
|
# AppData Retirement Runbook
|
||||||
|
|
||||||
|
Use this when retiring an AppData key after its domain writes have moved to relation-table APIs. The order is fixed: backup, measure, freeze writes, compare, remove fallback, archive.
|
||||||
|
|
||||||
|
## Scope Gate
|
||||||
|
|
||||||
|
Retire one AppData key family at a time. Good candidates have domain CRUD writes, read APIs, pagination boundaries, audit events, and a rollback path.
|
||||||
|
|
||||||
|
## Preparation
|
||||||
|
|
||||||
|
```bash
|
||||||
|
pnpm backup:postgres -- --env-file .env.production
|
||||||
|
pnpm backup:server-data -- --env-file .env.production
|
||||||
|
pnpm deploy:smoke -- --base-url http://localhost
|
||||||
|
```
|
||||||
|
|
||||||
|
Record the key family being retired, the owning domain API, and the relation tables that replace it.
|
||||||
|
|
||||||
|
## Count And Consistency Checks
|
||||||
|
|
||||||
|
Run counts before disabling writes.
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker compose --env-file .env.production -f docker-compose.prod.yml exec postgres psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -c "select key, jsonb_array_length(value) as appdata_rows from app_data where key in ('products-overview','requirements','version-plans','dev-tasks','test-cases','bugs','members','task-categories','task-worklogs','overtime') order by key;"
|
||||||
|
docker compose --env-file .env.production -f docker-compose.prod.yml exec postgres psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -c "select 'requirements' as table_name, count(*) from requirements union all select 'version_plans', count(*) from version_plans union all select 'dev_tasks', count(*) from dev_tasks union all select 'test_cases', count(*) from test_cases union all select 'bugs', count(*) from bugs;"
|
||||||
|
```
|
||||||
|
|
||||||
|
For partitioned entities, also check missing partition keys.
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker compose --env-file .env.production -f docker-compose.prod.yml exec postgres psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -c "select 'requirements_missing_product' as check_name, count(*) from requirements where product_id is null union all select 'dev_tasks_missing_version', count(*) from dev_tasks where version_id is null union all select 'test_cases_missing_version', count(*) from test_cases where version_id is null union all select 'bugs_missing_version', count(*) from bugs where version_id is null;"
|
||||||
|
```
|
||||||
|
|
||||||
|
## Disable AppData Writes
|
||||||
|
|
||||||
|
1. Merge the domain-specific frontend store change that stops calling `saveServerData` for the retired key.
|
||||||
|
2. Keep AppData read fallback for one release while relation reads are verified.
|
||||||
|
3. Deploy and run smoke tests.
|
||||||
|
|
||||||
|
```bash
|
||||||
|
pnpm deploy:smoke -- --base-url http://localhost --expected-version "$APP_VERSION"
|
||||||
|
```
|
||||||
|
|
||||||
|
## Remove Fallback
|
||||||
|
|
||||||
|
Remove AppData read fallback only after one successful release where:
|
||||||
|
|
||||||
|
- Domain writes went through relation APIs.
|
||||||
|
- AppData row counts did not grow for the retired key.
|
||||||
|
- V2.2 read paths and page workflows returned expected data.
|
||||||
|
- No `AppData relation sync failed` logs appeared during the observation window.
|
||||||
|
|
||||||
|
## Archive
|
||||||
|
|
||||||
|
Export retired keys before any later table cleanup.
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker compose --env-file .env.production -f docker-compose.prod.yml exec postgres psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -c "copy (select key, value, updated_at from app_data where key in ('dev-tasks','test-cases','bugs')) to stdout with csv header" > backups/postgres/appdata-retired-executions-20260708.csv
|
||||||
|
```
|
||||||
|
|
||||||
|
## Rollback
|
||||||
|
|
||||||
|
- If relation writes fail but AppData still has fresh data, roll back to the previous app image that still writes AppData.
|
||||||
|
- If AppData writes were already disabled and relation writes are bad, restore PostgreSQL from the backup made at the start of this runbook.
|
||||||
|
- If only read fallback removal caused the issue, roll back app images first and leave the database unchanged.
|
||||||
|
|
||||||
|
## Data Risks
|
||||||
|
|
||||||
|
- Removing fallback too early can hide valid historical JSON rows that were never mapped into relation tables.
|
||||||
|
- Re-enabling old AppData writes after relation writes have accepted new edits can overwrite newer relation state through compatibility sync.
|
||||||
|
- AppData exports can contain business-sensitive text; store backup CSV files in the same restricted location as database dumps.
|
||||||
|
|
||||||
100
docs/runbooks/migration-rollback.md
Normal file
100
docs/runbooks/migration-rollback.md
Normal file
@@ -0,0 +1,100 @@
|
|||||||
|
# Migration Rollback Runbook
|
||||||
|
|
||||||
|
Use this when a production release, Prisma migration, or data migration causes failed smoke tests, missing data, bad query performance, or unsafe writes.
|
||||||
|
|
||||||
|
## First Response
|
||||||
|
|
||||||
|
1. Freeze new releases and ask product owners to pause bulk edits.
|
||||||
|
2. Capture current state before changing anything.
|
||||||
|
|
||||||
|
```bash
|
||||||
|
date -u
|
||||||
|
git rev-parse HEAD
|
||||||
|
docker compose --env-file .env.production -f docker-compose.prod.yml ps
|
||||||
|
pnpm backup:postgres -- --env-file .env.production
|
||||||
|
pnpm backup:server-data -- --env-file .env.production
|
||||||
|
```
|
||||||
|
|
||||||
|
3. Run the read-only release smoke test.
|
||||||
|
|
||||||
|
```bash
|
||||||
|
pnpm deploy:smoke -- --base-url http://localhost --expected-version "$APP_VERSION"
|
||||||
|
```
|
||||||
|
|
||||||
|
4. Check the database and latest server logs.
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker compose --env-file .env.production -f docker-compose.prod.yml exec postgres pg_isready -U "$POSTGRES_USER" -d "$POSTGRES_DB"
|
||||||
|
docker compose --env-file .env.production -f docker-compose.prod.yml logs --tail=200 server
|
||||||
|
```
|
||||||
|
|
||||||
|
## Decision Points
|
||||||
|
|
||||||
|
- App code bad, database healthy: roll back `WEB_IMAGE` and `SERVER_IMAGE` to the previous commit image tags, then restart Compose.
|
||||||
|
- Migration applied but only additive: roll back app images first; leave schema in place if old code remains compatible.
|
||||||
|
- Migration changed or removed data: restore a fresh database from the last known-good backup into a temporary database, compare row counts, then decide whether to restore production.
|
||||||
|
- AppData compatibility issue: keep production database intact, re-enable the previous app image that still reads the AppData fallback, and preserve the failing release backup for analysis.
|
||||||
|
|
||||||
|
## App Image Rollback
|
||||||
|
|
||||||
|
Set previous image tags in `.env.production`. Use the previous successful GitHub Actions run to identify the image tags.
|
||||||
|
|
||||||
|
```bash
|
||||||
|
export PREVIOUS_WEB_IMAGE=ghcr.io/company/ftb-project-management/web:abc1234
|
||||||
|
export PREVIOUS_SERVER_IMAGE=ghcr.io/company/ftb-project-management/server:abc1234
|
||||||
|
sed -i "s|^WEB_IMAGE=.*|WEB_IMAGE=${PREVIOUS_WEB_IMAGE}|" .env.production
|
||||||
|
sed -i "s|^SERVER_IMAGE=.*|SERVER_IMAGE=${PREVIOUS_SERVER_IMAGE}|" .env.production
|
||||||
|
docker compose --env-file .env.production -f docker-compose.prod.yml pull web server
|
||||||
|
docker compose --env-file .env.production -f docker-compose.prod.yml up -d --remove-orphans
|
||||||
|
pnpm deploy:smoke -- --base-url http://localhost --expected-version abc1234
|
||||||
|
```
|
||||||
|
|
||||||
|
## Fresh Database Restore
|
||||||
|
|
||||||
|
Never restore over production until the backup has been rehearsed into a temporary database.
|
||||||
|
|
||||||
|
```bash
|
||||||
|
export BACKUP_FILE=backups/postgres/ftb_pm-postgres-20260708T120000Z.dump
|
||||||
|
pnpm restore:postgres -- \
|
||||||
|
--dry-run \
|
||||||
|
--confirm-overwrite \
|
||||||
|
--env-file .env.production \
|
||||||
|
--input "$BACKUP_FILE" \
|
||||||
|
--target-db ftb_pm_restore_check
|
||||||
|
pnpm restore:postgres -- \
|
||||||
|
--confirm-overwrite \
|
||||||
|
--env-file .env.production \
|
||||||
|
--input "$BACKUP_FILE" \
|
||||||
|
--target-db ftb_pm_restore_check
|
||||||
|
```
|
||||||
|
|
||||||
|
Compare critical row counts before touching production.
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker compose --env-file .env.production -f docker-compose.prod.yml exec postgres psql -U "$POSTGRES_USER" -d ftb_pm_restore_check -c "select 'products' as table_name, count(*) from products union all select 'requirements', count(*) from requirements union all select 'versions', count(*) from versions union all select 'dev_tasks', count(*) from dev_tasks union all select 'test_cases', count(*) from test_cases union all select 'bugs', count(*) from bugs;"
|
||||||
|
```
|
||||||
|
|
||||||
|
If the temporary restore is healthy and production data is unsafe, restore production with explicit overwrite confirmation.
|
||||||
|
|
||||||
|
```bash
|
||||||
|
pnpm restore:postgres -- \
|
||||||
|
--confirm-overwrite \
|
||||||
|
--env-file .env.production \
|
||||||
|
--input "$BACKUP_FILE"
|
||||||
|
docker compose --env-file .env.production -f docker-compose.prod.yml up -d --remove-orphans
|
||||||
|
pnpm deploy:smoke -- --base-url http://localhost
|
||||||
|
```
|
||||||
|
|
||||||
|
## Data Risks
|
||||||
|
|
||||||
|
- PostgreSQL restore is destructive for the target database because the script terminates connections, drops the target database, recreates it, and runs `pg_restore`.
|
||||||
|
- Restoring PostgreSQL does not restore `server_data`; keep AI provider config backup files with the same incident bundle.
|
||||||
|
- AppData and relation tables can diverge during compatibility windows. Before deleting or restoring, preserve both the failing production backup and the known-good backup.
|
||||||
|
- If users continued editing during the incident, record the time window and decide whether those edits must be replayed manually after restore.
|
||||||
|
|
||||||
|
## Closeout
|
||||||
|
|
||||||
|
1. Save the failed release SHA, rollback SHA, backup file names, smoke output, and row-count evidence in the incident notes.
|
||||||
|
2. Keep the failed backup until the next successful release has completed smoke tests and one business-day observation.
|
||||||
|
3. Add a regression test or runbook correction before re-attempting the migration.
|
||||||
|
|
||||||
55
docs/runbooks/xiaobao-background-jobs.md
Normal file
55
docs/runbooks/xiaobao-background-jobs.md
Normal file
@@ -0,0 +1,55 @@
|
|||||||
|
# Xiaobao Background Jobs Runbook
|
||||||
|
|
||||||
|
Current V2.8 production monitoring supports Xiaobao staleness detection. The first production implementation is still page-triggered: opening `/xiaobao-warning` computes risk, saves snapshots, and lets V2.3 sync refresh summaries. A future scheduler must preserve the same idempotent data contract.
|
||||||
|
|
||||||
|
## Alert Triage
|
||||||
|
|
||||||
|
When `FtbXiaobaoSummaryStale` fires:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker compose --env-file .env.production -f docker-compose.prod.yml exec postgres psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -c "select version_id, dirty, updated_at, recomputed_at, risk_level, risk_score from xiaobao_risk_summaries where dirty = true or updated_at < now() - interval '6 hours' order by updated_at asc limit 20;"
|
||||||
|
docker compose --env-file .env.production -f docker-compose.prod.yml logs --tail=200 server | grep -E "xiaobao|AppData relation sync failed|Slow Prisma query"
|
||||||
|
```
|
||||||
|
|
||||||
|
Decision points:
|
||||||
|
|
||||||
|
- Rows are dirty after active version edits: ask a manager to open `/xiaobao-warning` once, then verify summaries refresh.
|
||||||
|
- Rows remain dirty and server logs show sync failures: treat as AppData relation sync incident and follow `migration-rollback.md`.
|
||||||
|
- Rows are stale but no product release is near: keep monitoring and schedule a manual refresh before the next release decision meeting.
|
||||||
|
- Rows are stale for a release due today: refresh manually and have the release owner review the resulting risk explanation before ship/no-ship decision.
|
||||||
|
|
||||||
|
## Manual Refresh Path
|
||||||
|
|
||||||
|
1. Log in as a user with `xiaobao.warning:manage`.
|
||||||
|
2. Open `/xiaobao-warning`.
|
||||||
|
3. Wait until AI interpretation status is no longer generating for high-risk versions.
|
||||||
|
4. Re-run the stale-summary query.
|
||||||
|
5. Run the release smoke test.
|
||||||
|
|
||||||
|
```bash
|
||||||
|
pnpm deploy:smoke -- --base-url http://localhost
|
||||||
|
```
|
||||||
|
|
||||||
|
## Future Scheduler Rules
|
||||||
|
|
||||||
|
When a background job is introduced, it must:
|
||||||
|
|
||||||
|
- Read unfinished versions by relation-table scope, not by full AppData document scan.
|
||||||
|
- Use one idempotency key per `versionId + riskSignature + snapshotDate`.
|
||||||
|
- Write snapshots append-only and upsert summaries by `version_id`.
|
||||||
|
- Mark failures with structured logs containing `xiaobao background job failed`.
|
||||||
|
- Retry transient AI failures with backoff and keep rule-based risk output even when AI interpretation fails.
|
||||||
|
- Never mutate Version, Requirement, DevTask, TestCase, Bug, or Member data.
|
||||||
|
|
||||||
|
## Monitoring Expectations
|
||||||
|
|
||||||
|
- `FtbXiaobaoSummaryStale` alerts on dirty or older-than-6-hour summaries.
|
||||||
|
- `FtbJobFailureLogBurst` alerts when sync or future job failure log counters increase.
|
||||||
|
- Grafana dashboard shows the stale summary count and matching server warning/error logs.
|
||||||
|
|
||||||
|
## Data Risks
|
||||||
|
|
||||||
|
- Recomputing Xiaobao summaries can change release risk badges and manager decisions; record manual refresh time in release notes.
|
||||||
|
- AI interpretation cache is explanatory only. Do not restore or delete business entities to fix a bad explanation.
|
||||||
|
- If stale summaries are caused by relation sync failure, refreshing the page can mask the symptom without fixing the underlying sync path. Preserve logs before restarting services.
|
||||||
|
|
||||||
@@ -10,8 +10,14 @@
|
|||||||
"perf:seed": "node scripts/seed-large-dataset.mjs",
|
"perf:seed": "node scripts/seed-large-dataset.mjs",
|
||||||
"perf:check": "node scripts/perf-check.mjs",
|
"perf:check": "node scripts/perf-check.mjs",
|
||||||
"perf:explain": "node scripts/explain-hot-queries.mjs",
|
"perf:explain": "node scripts/explain-hot-queries.mjs",
|
||||||
|
"ops:test": "node --test scripts/*.test.mjs",
|
||||||
"deploy:verify": "node scripts/verify-production-deploy.mjs",
|
"deploy:verify": "node scripts/verify-production-deploy.mjs",
|
||||||
"deploy:check-runtime": "node scripts/check-runtime-version.mjs",
|
"deploy:check-runtime": "node scripts/check-runtime-version.mjs",
|
||||||
|
"deploy:smoke": "node scripts/smoke-test-release.mjs",
|
||||||
|
"backup:postgres": "node scripts/backup-postgres.mjs",
|
||||||
|
"restore:postgres": "node scripts/restore-postgres.mjs",
|
||||||
|
"backup:server-data": "node scripts/backup-server-data.mjs",
|
||||||
|
"docs:check-runbooks": "node scripts/check-runbook-placeholders.mjs --paths docs/runbooks docs/production-readiness.md",
|
||||||
"appdata:archive:export": "node scripts/export-appdata-archive.mjs",
|
"appdata:archive:export": "node scripts/export-appdata-archive.mjs",
|
||||||
"appdata:archive:verify": "node scripts/verify-appdata-archive.mjs",
|
"appdata:archive:verify": "node scripts/verify-appdata-archive.mjs",
|
||||||
"appdata:archive:test": "node --test scripts/appdata-archive.test.mjs",
|
"appdata:archive:test": "node --test scripts/appdata-archive.test.mjs",
|
||||||
|
|||||||
139
scripts/backup-postgres.mjs
Normal file
139
scripts/backup-postgres.mjs
Normal file
@@ -0,0 +1,139 @@
|
|||||||
|
#!/usr/bin/env node
|
||||||
|
import { createWriteStream, unlinkSync } from 'node:fs';
|
||||||
|
import { dirname, join, resolve } from 'node:path';
|
||||||
|
import { pathToFileURL } from 'node:url';
|
||||||
|
import { spawn } from 'node:child_process';
|
||||||
|
import {
|
||||||
|
commandToString,
|
||||||
|
composePrefix,
|
||||||
|
ensureParentDir,
|
||||||
|
flag,
|
||||||
|
option,
|
||||||
|
parseArgs,
|
||||||
|
readEnvFile,
|
||||||
|
timestampSlug,
|
||||||
|
} from './ops-utils.mjs';
|
||||||
|
|
||||||
|
function usage() {
|
||||||
|
return `Usage: node scripts/backup-postgres.mjs [options]
|
||||||
|
|
||||||
|
Options:
|
||||||
|
--env-file <path> Compose env file (default: .env.production)
|
||||||
|
--compose-file <path> Compose file (default: docker-compose.prod.yml)
|
||||||
|
--service <name> Postgres service name (default: postgres)
|
||||||
|
--user <name> Database user (default: POSTGRES_USER or postgres)
|
||||||
|
--db <name> Database name (default: POSTGRES_DB or ftb_pm)
|
||||||
|
--backup-dir <path> Directory for generated backups (default: backups/postgres)
|
||||||
|
--output <path> Exact output dump path
|
||||||
|
--dry-run Print the pg_dump command without writing a file
|
||||||
|
--help Show this help
|
||||||
|
`;
|
||||||
|
}
|
||||||
|
|
||||||
|
function buildOptions(argv) {
|
||||||
|
const args = parseArgs(argv);
|
||||||
|
if (flag(args, 'help')) return { help: true };
|
||||||
|
|
||||||
|
const envFile = option(args, 'env-file', '.env.production');
|
||||||
|
const env = readEnvFile(envFile, { optional: flag(args, 'dry-run') });
|
||||||
|
const db = option(args, 'db', env.POSTGRES_DB || 'ftb_pm');
|
||||||
|
const user = option(args, 'user', env.POSTGRES_USER || 'postgres');
|
||||||
|
const backupDir = option(args, 'backup-dir', 'backups/postgres');
|
||||||
|
const output = option(
|
||||||
|
args,
|
||||||
|
'output',
|
||||||
|
join(backupDir, `${db}-postgres-${timestampSlug()}.dump`),
|
||||||
|
);
|
||||||
|
|
||||||
|
return {
|
||||||
|
help: false,
|
||||||
|
dryRun: flag(args, 'dry-run'),
|
||||||
|
envFile,
|
||||||
|
composeFile: option(args, 'compose-file', 'docker-compose.prod.yml'),
|
||||||
|
service: option(args, 'service', 'postgres'),
|
||||||
|
db,
|
||||||
|
user,
|
||||||
|
output: resolve(output),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export function buildPgDumpCommand(options) {
|
||||||
|
return [
|
||||||
|
...composePrefix(options),
|
||||||
|
'exec',
|
||||||
|
'-T',
|
||||||
|
options.service,
|
||||||
|
'pg_dump',
|
||||||
|
'-U',
|
||||||
|
options.user,
|
||||||
|
'-d',
|
||||||
|
options.db,
|
||||||
|
'--format=custom',
|
||||||
|
'--no-owner',
|
||||||
|
'--no-acl',
|
||||||
|
];
|
||||||
|
}
|
||||||
|
|
||||||
|
async function writeBackup(command, output) {
|
||||||
|
ensureParentDir(output);
|
||||||
|
await new Promise((resolveWrite, rejectWrite) => {
|
||||||
|
const file = createWriteStream(output, { flags: 'wx' });
|
||||||
|
const child = spawn(command[0], command.slice(1), {
|
||||||
|
cwd: process.cwd(),
|
||||||
|
stdio: ['ignore', 'pipe', 'inherit'],
|
||||||
|
});
|
||||||
|
|
||||||
|
let settled = false;
|
||||||
|
const rejectOnce = (error) => {
|
||||||
|
if (settled) return;
|
||||||
|
settled = true;
|
||||||
|
try {
|
||||||
|
unlinkSync(output);
|
||||||
|
} catch {
|
||||||
|
// Best effort cleanup of a partial dump.
|
||||||
|
}
|
||||||
|
rejectWrite(error);
|
||||||
|
};
|
||||||
|
|
||||||
|
child.stdout.pipe(file);
|
||||||
|
child.on('error', rejectOnce);
|
||||||
|
file.on('error', rejectOnce);
|
||||||
|
child.on('close', (code) => {
|
||||||
|
file.end(() => {
|
||||||
|
if (settled) return;
|
||||||
|
settled = true;
|
||||||
|
if (code === 0) {
|
||||||
|
resolveWrite();
|
||||||
|
} else {
|
||||||
|
rejectOnce(new Error(`pg_dump failed with exit code ${code}`));
|
||||||
|
}
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function main(argv = process.argv.slice(2)) {
|
||||||
|
const options = buildOptions(argv);
|
||||||
|
if (options.help) {
|
||||||
|
process.stdout.write(usage());
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const command = buildPgDumpCommand(options);
|
||||||
|
if (options.dryRun) {
|
||||||
|
process.stdout.write(`[dry-run] PostgreSQL backup would write: ${options.output}\n`);
|
||||||
|
process.stdout.write(`${commandToString(command, { stdout: options.output })}\n`);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
await writeBackup(command, options.output);
|
||||||
|
process.stdout.write(`PostgreSQL backup written: ${options.output}\n`);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (import.meta.url === pathToFileURL(process.argv[1]).href) {
|
||||||
|
main().catch((error) => {
|
||||||
|
process.stderr.write(`${error.message}\n`);
|
||||||
|
process.exit(1);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
99
scripts/backup-server-data.mjs
Normal file
99
scripts/backup-server-data.mjs
Normal file
@@ -0,0 +1,99 @@
|
|||||||
|
#!/usr/bin/env node
|
||||||
|
import { basename, dirname, join, resolve } from 'node:path';
|
||||||
|
import { pathToFileURL } from 'node:url';
|
||||||
|
import {
|
||||||
|
commandToString,
|
||||||
|
ensureParentDir,
|
||||||
|
flag,
|
||||||
|
option,
|
||||||
|
parseArgs,
|
||||||
|
readEnvFile,
|
||||||
|
runCommand,
|
||||||
|
timestampSlug,
|
||||||
|
} from './ops-utils.mjs';
|
||||||
|
|
||||||
|
function usage() {
|
||||||
|
return `Usage: node scripts/backup-server-data.mjs [options]
|
||||||
|
|
||||||
|
Backs up the server_data Docker volume that stores runtime AI provider config.
|
||||||
|
|
||||||
|
Options:
|
||||||
|
--env-file <path> Compose env file used to derive COMPOSE_PROJECT_NAME
|
||||||
|
(default: .env.production)
|
||||||
|
--volume <name> Explicit Docker volume name
|
||||||
|
--backup-dir <path> Directory for generated backups (default: backups/server-data)
|
||||||
|
--output <path> Exact output .tgz path
|
||||||
|
--image <name> Utility image (default: alpine:3.20)
|
||||||
|
--dry-run Print the docker run command without writing a file
|
||||||
|
--help Show this help
|
||||||
|
`;
|
||||||
|
}
|
||||||
|
|
||||||
|
function buildOptions(argv) {
|
||||||
|
const args = parseArgs(argv);
|
||||||
|
if (flag(args, 'help')) return { help: true };
|
||||||
|
|
||||||
|
const envFile = option(args, 'env-file', '.env.production');
|
||||||
|
const env = readEnvFile(envFile, { optional: flag(args, 'dry-run') });
|
||||||
|
const projectName = env.COMPOSE_PROJECT_NAME || 'ftb_pm';
|
||||||
|
const volume = option(args, 'volume', `${projectName}_server_data`);
|
||||||
|
const backupDir = option(args, 'backup-dir', 'backups/server-data');
|
||||||
|
const output = option(
|
||||||
|
args,
|
||||||
|
'output',
|
||||||
|
join(backupDir, `${volume}-${timestampSlug()}.tgz`),
|
||||||
|
);
|
||||||
|
|
||||||
|
return {
|
||||||
|
help: false,
|
||||||
|
dryRun: flag(args, 'dry-run'),
|
||||||
|
image: option(args, 'image', 'alpine:3.20'),
|
||||||
|
volume,
|
||||||
|
output: resolve(output),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export function buildServerDataBackupCommand(options) {
|
||||||
|
const outputDir = dirname(options.output);
|
||||||
|
const outputName = basename(options.output);
|
||||||
|
return [
|
||||||
|
'docker',
|
||||||
|
'run',
|
||||||
|
'--rm',
|
||||||
|
'-v',
|
||||||
|
`${options.volume}:/data:ro`,
|
||||||
|
'-v',
|
||||||
|
`${outputDir}:/backup`,
|
||||||
|
options.image,
|
||||||
|
'sh',
|
||||||
|
'-lc',
|
||||||
|
`tar -czf /backup/${outputName} -C /data .`,
|
||||||
|
];
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function main(argv = process.argv.slice(2)) {
|
||||||
|
const options = buildOptions(argv);
|
||||||
|
if (options.help) {
|
||||||
|
process.stdout.write(usage());
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const command = buildServerDataBackupCommand(options);
|
||||||
|
if (options.dryRun) {
|
||||||
|
process.stdout.write(`[dry-run] server_data backup would write: ${options.output}\n`);
|
||||||
|
process.stdout.write(`${commandToString(command)}\n`);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
ensureParentDir(options.output);
|
||||||
|
await runCommand(command);
|
||||||
|
process.stdout.write(`server_data backup written: ${options.output}\n`);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (import.meta.url === pathToFileURL(process.argv[1]).href) {
|
||||||
|
main().catch((error) => {
|
||||||
|
process.stderr.write(`${error.message}\n`);
|
||||||
|
process.exit(1);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
87
scripts/check-runbook-placeholders.mjs
Normal file
87
scripts/check-runbook-placeholders.mjs
Normal file
@@ -0,0 +1,87 @@
|
|||||||
|
#!/usr/bin/env node
|
||||||
|
import { existsSync, readFileSync, readdirSync, statSync } from 'node:fs';
|
||||||
|
import { extname, join } from 'node:path';
|
||||||
|
import { pathToFileURL } from 'node:url';
|
||||||
|
import { option, parseArgs } from './ops-utils.mjs';
|
||||||
|
|
||||||
|
const forbiddenPatterns = [
|
||||||
|
{ name: 'TBD', pattern: /\bTBD\b/i },
|
||||||
|
{ name: 'TODO', pattern: /\bTODO\b/i },
|
||||||
|
{ name: 'fill-in', pattern: /fill in|fill-in/i },
|
||||||
|
{ name: 'angle-token', pattern: /<[^>\n]+>/ },
|
||||||
|
{ name: 'Chinese pending marker', pattern: /待定|占位/ },
|
||||||
|
];
|
||||||
|
|
||||||
|
function usage() {
|
||||||
|
return `Usage: node scripts/check-runbook-placeholders.mjs --paths <path> [path...]
|
||||||
|
|
||||||
|
Scans Markdown runbooks for unresolved placeholder markers.
|
||||||
|
`;
|
||||||
|
}
|
||||||
|
|
||||||
|
function collectMarkdownFiles(path) {
|
||||||
|
if (!existsSync(path)) {
|
||||||
|
throw new Error(`Path not found: ${path}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
const stat = statSync(path);
|
||||||
|
if (stat.isFile()) return extname(path) === '.md' ? [path] : [];
|
||||||
|
if (!stat.isDirectory()) return [];
|
||||||
|
|
||||||
|
const files = [];
|
||||||
|
for (const entry of readdirSync(path)) {
|
||||||
|
files.push(...collectMarkdownFiles(join(path, entry)));
|
||||||
|
}
|
||||||
|
return files;
|
||||||
|
}
|
||||||
|
|
||||||
|
function requestedPaths(argv) {
|
||||||
|
const args = parseArgs(argv);
|
||||||
|
if (args.flags.has('help')) return { help: true, paths: [] };
|
||||||
|
const first = option(args, 'paths');
|
||||||
|
const paths = [first, ...args.positionals].filter(Boolean);
|
||||||
|
if (paths.length === 0) {
|
||||||
|
throw new Error('Missing --paths <path> [path...]');
|
||||||
|
}
|
||||||
|
return { help: false, paths };
|
||||||
|
}
|
||||||
|
|
||||||
|
export function scanFiles(paths) {
|
||||||
|
const files = paths.flatMap(collectMarkdownFiles);
|
||||||
|
const findings = [];
|
||||||
|
for (const file of files) {
|
||||||
|
const lines = readFileSync(file, 'utf8').split(/\r?\n/);
|
||||||
|
lines.forEach((line, index) => {
|
||||||
|
for (const forbidden of forbiddenPatterns) {
|
||||||
|
if (forbidden.pattern.test(line)) {
|
||||||
|
findings.push(`${file}:${index + 1} ${forbidden.name}: ${line.trim()}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
return { files, findings };
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function main(argv = process.argv.slice(2)) {
|
||||||
|
const { help, paths } = requestedPaths(argv);
|
||||||
|
if (help) {
|
||||||
|
process.stdout.write(usage());
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const { files, findings } = scanFiles(paths);
|
||||||
|
if (findings.length > 0) {
|
||||||
|
process.stderr.write(`${findings.join('\n')}\n`);
|
||||||
|
throw new Error(`Runbook placeholder scan failed: ${findings.length} finding(s)`);
|
||||||
|
}
|
||||||
|
|
||||||
|
process.stdout.write(`Runbook placeholder scan passed (${files.length} file(s)).\n`);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (import.meta.url === pathToFileURL(process.argv[1]).href) {
|
||||||
|
main().catch((error) => {
|
||||||
|
process.stderr.write(`${error.message}\n`);
|
||||||
|
process.exit(1);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
152
scripts/ops-scripts.test.mjs
Normal file
152
scripts/ops-scripts.test.mjs
Normal file
@@ -0,0 +1,152 @@
|
|||||||
|
import { spawnSync } from 'node:child_process';
|
||||||
|
import { mkdtempSync, writeFileSync, existsSync, readFileSync } from 'node:fs';
|
||||||
|
import { tmpdir } from 'node:os';
|
||||||
|
import { join, resolve } from 'node:path';
|
||||||
|
import { describe, it } from 'node:test';
|
||||||
|
import assert from 'node:assert/strict';
|
||||||
|
|
||||||
|
const root = resolve(import.meta.dirname, '..');
|
||||||
|
|
||||||
|
function runScript(script, args) {
|
||||||
|
return spawnSync(process.execPath, [join(root, script), ...args], {
|
||||||
|
cwd: root,
|
||||||
|
encoding: 'utf8',
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
describe('production ops scripts', () => {
|
||||||
|
it('prints a pg_dump command in dry-run mode without writing the target file', () => {
|
||||||
|
const dir = mkdtempSync(join(tmpdir(), 'ftb-pg-backup-'));
|
||||||
|
const output = join(dir, 'backup.dump');
|
||||||
|
|
||||||
|
const result = runScript('scripts/backup-postgres.mjs', [
|
||||||
|
'--dry-run',
|
||||||
|
'--env-file',
|
||||||
|
'.env.production.example',
|
||||||
|
'--output',
|
||||||
|
output,
|
||||||
|
]);
|
||||||
|
|
||||||
|
assert.equal(result.status, 0, result.stderr);
|
||||||
|
assert.match(result.stdout, /docker compose/);
|
||||||
|
assert.match(result.stdout, /pg_dump/);
|
||||||
|
assert.match(result.stdout, /backup\.dump/);
|
||||||
|
assert.equal(existsSync(output), false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('refuses restore by default unless --confirm-overwrite is supplied', () => {
|
||||||
|
const dir = mkdtempSync(join(tmpdir(), 'ftb-pg-restore-'));
|
||||||
|
const input = join(dir, 'backup.dump');
|
||||||
|
writeFileSync(input, 'not-a-real-dump');
|
||||||
|
|
||||||
|
const result = runScript('scripts/restore-postgres.mjs', [
|
||||||
|
'--env-file',
|
||||||
|
'.env.production.example',
|
||||||
|
'--input',
|
||||||
|
input,
|
||||||
|
]);
|
||||||
|
|
||||||
|
assert.notEqual(result.status, 0);
|
||||||
|
assert.match(result.stderr, /--confirm-overwrite/);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('prints fresh database restore steps in dry-run mode after explicit overwrite confirmation', () => {
|
||||||
|
const dir = mkdtempSync(join(tmpdir(), 'ftb-pg-restore-dry-'));
|
||||||
|
const input = join(dir, 'backup.dump');
|
||||||
|
writeFileSync(input, 'not-a-real-dump');
|
||||||
|
|
||||||
|
const result = runScript('scripts/restore-postgres.mjs', [
|
||||||
|
'--dry-run',
|
||||||
|
'--confirm-overwrite',
|
||||||
|
'--env-file',
|
||||||
|
'.env.production.example',
|
||||||
|
'--input',
|
||||||
|
input,
|
||||||
|
]);
|
||||||
|
|
||||||
|
assert.equal(result.status, 0, result.stderr);
|
||||||
|
assert.match(result.stdout, /dropdb/);
|
||||||
|
assert.match(result.stdout, /createdb/);
|
||||||
|
assert.match(result.stdout, /pg_restore/);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('prints a server_data volume tar backup command in dry-run mode', () => {
|
||||||
|
const result = runScript('scripts/backup-server-data.mjs', [
|
||||||
|
'--dry-run',
|
||||||
|
'--env-file',
|
||||||
|
'.env.production.example',
|
||||||
|
]);
|
||||||
|
|
||||||
|
assert.equal(result.status, 0, result.stderr);
|
||||||
|
assert.match(result.stdout, /docker run/);
|
||||||
|
assert.match(result.stdout, /ftb_pm_server_data/);
|
||||||
|
assert.match(result.stdout, /tar -czf/);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('prints read-only release smoke checks in dry-run mode', () => {
|
||||||
|
const result = runScript('scripts/smoke-test-release.mjs', [
|
||||||
|
'--dry-run',
|
||||||
|
'--base-url',
|
||||||
|
'http://localhost',
|
||||||
|
]);
|
||||||
|
|
||||||
|
assert.equal(result.status, 0, result.stderr);
|
||||||
|
assert.match(result.stdout, /\/api\/v1\/health\/version/);
|
||||||
|
assert.match(result.stdout, /\/products/);
|
||||||
|
assert.match(result.stdout, /\/api\/v1\/products/);
|
||||||
|
assert.match(result.stdout, /\/api\/v1\/v2\.2\/requirements\?productId=__smoke__/);
|
||||||
|
assert.match(result.stdout, /\/api\/v1\/config\/ai/);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('declares the production monitoring baseline without real alert secrets', () => {
|
||||||
|
const alertRules = readFileSync(
|
||||||
|
join(root, 'deploy/monitoring/prometheus/alert-rules.yml'),
|
||||||
|
'utf8',
|
||||||
|
);
|
||||||
|
const postgresQueries = readFileSync(
|
||||||
|
join(root, 'deploy/monitoring/postgres/postgres-queries.yml'),
|
||||||
|
'utf8',
|
||||||
|
);
|
||||||
|
const promtailConfig = readFileSync(
|
||||||
|
join(root, 'deploy/monitoring/promtail/config.yml'),
|
||||||
|
'utf8',
|
||||||
|
);
|
||||||
|
const dashboard = readFileSync(
|
||||||
|
join(root, 'deploy/monitoring/grafana/dashboards/ftb-production-overview.json'),
|
||||||
|
'utf8',
|
||||||
|
);
|
||||||
|
|
||||||
|
for (const alertName of [
|
||||||
|
'FtbPostgresDown',
|
||||||
|
'FtbDiskPressure',
|
||||||
|
'FtbSlowApiLogBurst',
|
||||||
|
'FtbSlowPrismaLogBurst',
|
||||||
|
'FtbJobFailureLogBurst',
|
||||||
|
'FtbXiaobaoSummaryStale',
|
||||||
|
]) {
|
||||||
|
assert.match(alertRules, new RegExp(alertName));
|
||||||
|
}
|
||||||
|
|
||||||
|
assert.match(postgresQueries, /xiaobao_risk_summaries/);
|
||||||
|
assert.match(postgresQueries, /dirty = true/);
|
||||||
|
assert.match(promtailConfig, /Slow API request/);
|
||||||
|
assert.match(promtailConfig, /Slow Prisma query/);
|
||||||
|
assert.match(promtailConfig, /AppData relation sync failed/);
|
||||||
|
assert.match(dashboard, /FTB Production Overview/);
|
||||||
|
|
||||||
|
const combined = `${alertRules}\n${postgresQueries}\n${promtailConfig}\n${dashboard}`;
|
||||||
|
assert.doesNotMatch(combined, /sk-ant-[A-Za-z0-9]/);
|
||||||
|
assert.doesNotMatch(combined, /hooks\.slack\.com\/services\//);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('passes the runbook placeholder scan', () => {
|
||||||
|
const result = runScript('scripts/check-runbook-placeholders.mjs', [
|
||||||
|
'--paths',
|
||||||
|
'docs/runbooks',
|
||||||
|
'docs/production-readiness.md',
|
||||||
|
]);
|
||||||
|
|
||||||
|
assert.equal(result.status, 0, result.stderr);
|
||||||
|
assert.match(result.stdout, /Runbook placeholder scan passed/);
|
||||||
|
});
|
||||||
|
});
|
||||||
116
scripts/ops-utils.mjs
Normal file
116
scripts/ops-utils.mjs
Normal file
@@ -0,0 +1,116 @@
|
|||||||
|
import { existsSync, mkdirSync, readFileSync } from 'node:fs';
|
||||||
|
import { dirname, resolve } from 'node:path';
|
||||||
|
import { spawn } from 'node:child_process';
|
||||||
|
|
||||||
|
export function parseArgs(argv) {
|
||||||
|
const parsed = { flags: new Set(), values: new Map(), positionals: [] };
|
||||||
|
|
||||||
|
for (let index = 0; index < argv.length; index += 1) {
|
||||||
|
const arg = argv[index];
|
||||||
|
if (arg === '--') continue;
|
||||||
|
if (!arg.startsWith('--')) {
|
||||||
|
parsed.positionals.push(arg);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
const eq = arg.indexOf('=');
|
||||||
|
if (eq !== -1) {
|
||||||
|
parsed.values.set(arg.slice(2, eq), arg.slice(eq + 1));
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
const key = arg.slice(2);
|
||||||
|
const next = argv[index + 1];
|
||||||
|
if (next && !next.startsWith('--')) {
|
||||||
|
parsed.values.set(key, next);
|
||||||
|
index += 1;
|
||||||
|
} else {
|
||||||
|
parsed.flags.add(key);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return parsed;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function option(args, name, fallback = undefined) {
|
||||||
|
return args.values.has(name) ? args.values.get(name) : fallback;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function flag(args, name) {
|
||||||
|
return args.flags.has(name);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function readEnvFile(filePath, { optional = false } = {}) {
|
||||||
|
if (!existsSync(filePath)) {
|
||||||
|
if (optional) return {};
|
||||||
|
throw new Error(`Env file not found: ${filePath}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
const env = {};
|
||||||
|
const content = readFileSync(filePath, 'utf8');
|
||||||
|
for (const rawLine of content.split(/\r?\n/)) {
|
||||||
|
const line = rawLine.trim();
|
||||||
|
if (!line || line.startsWith('#')) continue;
|
||||||
|
const eq = line.indexOf('=');
|
||||||
|
if (eq === -1) continue;
|
||||||
|
const key = line.slice(0, eq).trim();
|
||||||
|
let value = line.slice(eq + 1).trim();
|
||||||
|
if (
|
||||||
|
(value.startsWith('"') && value.endsWith('"')) ||
|
||||||
|
(value.startsWith("'") && value.endsWith("'"))
|
||||||
|
) {
|
||||||
|
value = value.slice(1, -1);
|
||||||
|
}
|
||||||
|
env[key] = value;
|
||||||
|
}
|
||||||
|
return env;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function ensureParentDir(filePath) {
|
||||||
|
mkdirSync(dirname(filePath), { recursive: true });
|
||||||
|
}
|
||||||
|
|
||||||
|
export function timestampSlug(date = new Date()) {
|
||||||
|
return date.toISOString().replace(/[-:]/g, '').replace(/\.\d{3}Z$/, 'Z');
|
||||||
|
}
|
||||||
|
|
||||||
|
export function shellQuote(value) {
|
||||||
|
if (/^[A-Za-z0-9_./:@%+=,-]+$/.test(value)) return value;
|
||||||
|
return `'${value.replace(/'/g, "'\\''")}'`;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function commandToString(command, { stdin, stdout } = {}) {
|
||||||
|
const rendered = command.map((part) => shellQuote(part)).join(' ');
|
||||||
|
const withStdin = stdin ? `${rendered} < ${shellQuote(stdin)}` : rendered;
|
||||||
|
return stdout ? `${withStdin} > ${shellQuote(stdout)}` : withStdin;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function composePrefix({ envFile, composeFile }) {
|
||||||
|
return ['docker', 'compose', '--env-file', envFile, '-f', composeFile];
|
||||||
|
}
|
||||||
|
|
||||||
|
export function resolvePath(path) {
|
||||||
|
return resolve(process.cwd(), path);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function runCommand(command, options = {}) {
|
||||||
|
return new Promise((resolveRun, rejectRun) => {
|
||||||
|
const child = spawn(command[0], command.slice(1), {
|
||||||
|
stdio: options.stdio ?? 'inherit',
|
||||||
|
cwd: options.cwd ?? process.cwd(),
|
||||||
|
});
|
||||||
|
|
||||||
|
child.on('error', rejectRun);
|
||||||
|
child.on('close', (code) => {
|
||||||
|
if (code === 0) {
|
||||||
|
resolveRun();
|
||||||
|
} else {
|
||||||
|
rejectRun(new Error(`Command failed (${code}): ${commandToString(command)}`));
|
||||||
|
}
|
||||||
|
});
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
export function quotePgLiteral(value) {
|
||||||
|
return `'${value.replace(/'/g, "''")}'`;
|
||||||
|
}
|
||||||
181
scripts/restore-postgres.mjs
Normal file
181
scripts/restore-postgres.mjs
Normal file
@@ -0,0 +1,181 @@
|
|||||||
|
#!/usr/bin/env node
|
||||||
|
import { createReadStream, existsSync } from 'node:fs';
|
||||||
|
import { resolve } from 'node:path';
|
||||||
|
import { pathToFileURL } from 'node:url';
|
||||||
|
import { spawn } from 'node:child_process';
|
||||||
|
import {
|
||||||
|
commandToString,
|
||||||
|
composePrefix,
|
||||||
|
flag,
|
||||||
|
option,
|
||||||
|
parseArgs,
|
||||||
|
quotePgLiteral,
|
||||||
|
readEnvFile,
|
||||||
|
runCommand,
|
||||||
|
} from './ops-utils.mjs';
|
||||||
|
|
||||||
|
function usage() {
|
||||||
|
return `Usage: node scripts/restore-postgres.mjs --input <dump> --confirm-overwrite [options]
|
||||||
|
|
||||||
|
This script restores into a freshly recreated PostgreSQL database. It refuses to
|
||||||
|
drop/recreate a database unless --confirm-overwrite is provided.
|
||||||
|
|
||||||
|
Options:
|
||||||
|
--input <path> pg_dump custom-format dump file
|
||||||
|
--confirm-overwrite Required safety flag for destructive restore
|
||||||
|
--env-file <path> Compose env file (default: .env.production)
|
||||||
|
--compose-file <path> Compose file (default: docker-compose.prod.yml)
|
||||||
|
--service <name> Postgres service name (default: postgres)
|
||||||
|
--user <name> Database user (default: POSTGRES_USER or postgres)
|
||||||
|
--target-db <name> Target database (default: POSTGRES_DB or ftb_pm)
|
||||||
|
--maintenance-db <name> Maintenance database (default: postgres)
|
||||||
|
--dry-run Print restore commands without changing data
|
||||||
|
--help Show this help
|
||||||
|
`;
|
||||||
|
}
|
||||||
|
|
||||||
|
function buildOptions(argv) {
|
||||||
|
const args = parseArgs(argv);
|
||||||
|
if (flag(args, 'help')) return { help: true };
|
||||||
|
|
||||||
|
const envFile = option(args, 'env-file', '.env.production');
|
||||||
|
const env = readEnvFile(envFile, { optional: flag(args, 'dry-run') });
|
||||||
|
const input = option(args, 'input');
|
||||||
|
if (!input) {
|
||||||
|
throw new Error('Missing required --input <dump>');
|
||||||
|
}
|
||||||
|
|
||||||
|
const inputPath = resolve(input);
|
||||||
|
if (!existsSync(inputPath)) {
|
||||||
|
throw new Error(`Restore input not found: ${inputPath}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!flag(args, 'confirm-overwrite')) {
|
||||||
|
throw new Error('Refusing destructive restore. Re-run with --confirm-overwrite.');
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
help: false,
|
||||||
|
dryRun: flag(args, 'dry-run'),
|
||||||
|
input: inputPath,
|
||||||
|
envFile,
|
||||||
|
composeFile: option(args, 'compose-file', 'docker-compose.prod.yml'),
|
||||||
|
service: option(args, 'service', 'postgres'),
|
||||||
|
user: option(args, 'user', env.POSTGRES_USER || 'postgres'),
|
||||||
|
targetDb: option(args, 'target-db', env.POSTGRES_DB || 'ftb_pm'),
|
||||||
|
maintenanceDb: option(args, 'maintenance-db', 'postgres'),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export function buildRestoreCommands(options) {
|
||||||
|
const prefix = composePrefix(options);
|
||||||
|
const terminateSql = [
|
||||||
|
'SELECT pg_terminate_backend(pid)',
|
||||||
|
'FROM pg_stat_activity',
|
||||||
|
`WHERE datname = ${quotePgLiteral(options.targetDb)} AND pid <> pg_backend_pid();`,
|
||||||
|
].join(' ');
|
||||||
|
|
||||||
|
return [
|
||||||
|
[
|
||||||
|
...prefix,
|
||||||
|
'exec',
|
||||||
|
'-T',
|
||||||
|
options.service,
|
||||||
|
'psql',
|
||||||
|
'-U',
|
||||||
|
options.user,
|
||||||
|
'-d',
|
||||||
|
options.maintenanceDb,
|
||||||
|
'-v',
|
||||||
|
'ON_ERROR_STOP=1',
|
||||||
|
'-c',
|
||||||
|
terminateSql,
|
||||||
|
],
|
||||||
|
[
|
||||||
|
...prefix,
|
||||||
|
'exec',
|
||||||
|
'-T',
|
||||||
|
options.service,
|
||||||
|
'dropdb',
|
||||||
|
'--if-exists',
|
||||||
|
'-U',
|
||||||
|
options.user,
|
||||||
|
options.targetDb,
|
||||||
|
],
|
||||||
|
[
|
||||||
|
...prefix,
|
||||||
|
'exec',
|
||||||
|
'-T',
|
||||||
|
options.service,
|
||||||
|
'createdb',
|
||||||
|
'-U',
|
||||||
|
options.user,
|
||||||
|
options.targetDb,
|
||||||
|
],
|
||||||
|
[
|
||||||
|
...prefix,
|
||||||
|
'exec',
|
||||||
|
'-T',
|
||||||
|
options.service,
|
||||||
|
'pg_restore',
|
||||||
|
'-U',
|
||||||
|
options.user,
|
||||||
|
'-d',
|
||||||
|
options.targetDb,
|
||||||
|
'--no-owner',
|
||||||
|
'--no-acl',
|
||||||
|
],
|
||||||
|
];
|
||||||
|
}
|
||||||
|
|
||||||
|
function runRestore(command, input) {
|
||||||
|
return new Promise((resolveRun, rejectRun) => {
|
||||||
|
const child = spawn(command[0], command.slice(1), {
|
||||||
|
cwd: process.cwd(),
|
||||||
|
stdio: ['pipe', 'inherit', 'inherit'],
|
||||||
|
});
|
||||||
|
createReadStream(input).pipe(child.stdin);
|
||||||
|
child.on('error', rejectRun);
|
||||||
|
child.on('close', (code) => {
|
||||||
|
if (code === 0) {
|
||||||
|
resolveRun();
|
||||||
|
} else {
|
||||||
|
rejectRun(new Error(`pg_restore failed with exit code ${code}`));
|
||||||
|
}
|
||||||
|
});
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function main(argv = process.argv.slice(2)) {
|
||||||
|
const options = buildOptions(argv);
|
||||||
|
if (options.help) {
|
||||||
|
process.stdout.write(usage());
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const commands = buildRestoreCommands(options);
|
||||||
|
if (options.dryRun) {
|
||||||
|
process.stdout.write(
|
||||||
|
`[dry-run] PostgreSQL restore would recreate database: ${options.targetDb}\n`,
|
||||||
|
);
|
||||||
|
for (const command of commands.slice(0, -1)) {
|
||||||
|
process.stdout.write(`${commandToString(command)}\n`);
|
||||||
|
}
|
||||||
|
process.stdout.write(`${commandToString(commands.at(-1), { stdin: options.input })}\n`);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
for (const command of commands.slice(0, -1)) {
|
||||||
|
await runCommand(command);
|
||||||
|
}
|
||||||
|
await runRestore(commands.at(-1), options.input);
|
||||||
|
process.stdout.write(`PostgreSQL restore completed into fresh database: ${options.targetDb}\n`);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (import.meta.url === pathToFileURL(process.argv[1]).href) {
|
||||||
|
main().catch((error) => {
|
||||||
|
process.stderr.write(`${error.message}\n`);
|
||||||
|
process.exit(1);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
160
scripts/smoke-test-release.mjs
Normal file
160
scripts/smoke-test-release.mjs
Normal file
@@ -0,0 +1,160 @@
|
|||||||
|
#!/usr/bin/env node
|
||||||
|
import { pathToFileURL } from 'node:url';
|
||||||
|
import { flag, option, parseArgs } from './ops-utils.mjs';
|
||||||
|
|
||||||
|
function usage() {
|
||||||
|
return `Usage: node scripts/smoke-test-release.mjs --base-url <url> [options]
|
||||||
|
|
||||||
|
Read-only release smoke checks:
|
||||||
|
- backend runtime version
|
||||||
|
- frontend root page
|
||||||
|
- frontend products route
|
||||||
|
- products API
|
||||||
|
- V2.2 requirement read path
|
||||||
|
- AI config public endpoint
|
||||||
|
|
||||||
|
Options:
|
||||||
|
--base-url <url> Deployment base URL (default: http://127.0.0.1)
|
||||||
|
--expected-version <sha> Expected /api/v1/health/version payload version
|
||||||
|
--timeout-ms <number> Per-request timeout (default: 5000)
|
||||||
|
--dry-run Print planned checks without making requests
|
||||||
|
--help Show this help
|
||||||
|
`;
|
||||||
|
}
|
||||||
|
|
||||||
|
function normalizeBaseUrl(value) {
|
||||||
|
const url = new URL(value || 'http://127.0.0.1');
|
||||||
|
url.pathname = url.pathname.replace(/\/+$/, '');
|
||||||
|
url.search = '';
|
||||||
|
url.hash = '';
|
||||||
|
return url.toString().replace(/\/$/, '');
|
||||||
|
}
|
||||||
|
|
||||||
|
export function buildSmokeChecks(baseUrl, expectedVersion = '') {
|
||||||
|
const checks = [
|
||||||
|
{
|
||||||
|
name: 'backend runtime version',
|
||||||
|
path: '/api/v1/health/version',
|
||||||
|
kind: 'json',
|
||||||
|
validate(payload) {
|
||||||
|
if (payload?.service !== 'server' || typeof payload.version !== 'string') {
|
||||||
|
throw new Error(`unexpected version payload: ${JSON.stringify(payload)}`);
|
||||||
|
}
|
||||||
|
if (expectedVersion && payload.version !== expectedVersion) {
|
||||||
|
throw new Error(`expected version ${expectedVersion}, got ${payload.version}`);
|
||||||
|
}
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{ name: 'frontend root', path: '/', kind: 'text' },
|
||||||
|
{ name: 'frontend products route', path: '/products', kind: 'text' },
|
||||||
|
{
|
||||||
|
name: 'products API',
|
||||||
|
path: '/api/v1/products',
|
||||||
|
kind: 'json',
|
||||||
|
validate(payload) {
|
||||||
|
if (!Array.isArray(payload)) {
|
||||||
|
throw new Error('products API did not return an array');
|
||||||
|
}
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: 'V2.2 requirement scoped read',
|
||||||
|
path: '/api/v1/v2.2/requirements?productId=__smoke__&limit=1',
|
||||||
|
kind: 'json',
|
||||||
|
validate(payload) {
|
||||||
|
if (!payload || !Array.isArray(payload.items)) {
|
||||||
|
throw new Error('V2.2 requirements response missing items array');
|
||||||
|
}
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: 'AI config public endpoint',
|
||||||
|
path: '/api/v1/config/ai',
|
||||||
|
kind: 'json',
|
||||||
|
validate(payload) {
|
||||||
|
if (!payload || !Array.isArray(payload.providers)) {
|
||||||
|
throw new Error('AI config response missing providers array');
|
||||||
|
}
|
||||||
|
},
|
||||||
|
},
|
||||||
|
];
|
||||||
|
|
||||||
|
return checks.map((check) => ({
|
||||||
|
...check,
|
||||||
|
url: new URL(check.path, `${baseUrl}/`).toString(),
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
|
||||||
|
async function fetchWithTimeout(url, timeoutMs) {
|
||||||
|
const controller = new AbortController();
|
||||||
|
const timer = setTimeout(() => controller.abort(), timeoutMs);
|
||||||
|
try {
|
||||||
|
return await fetch(url, { signal: controller.signal });
|
||||||
|
} finally {
|
||||||
|
clearTimeout(timer);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function runCheck(check, timeoutMs) {
|
||||||
|
const response = await fetchWithTimeout(check.url, timeoutMs);
|
||||||
|
if (!response.ok) {
|
||||||
|
throw new Error(`HTTP ${response.status}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (check.kind === 'json') {
|
||||||
|
const payload = await response.json();
|
||||||
|
check.validate?.(payload);
|
||||||
|
} else {
|
||||||
|
await response.text();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function main(argv = process.argv.slice(2)) {
|
||||||
|
const args = parseArgs(argv);
|
||||||
|
if (flag(args, 'help')) {
|
||||||
|
process.stdout.write(usage());
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const baseUrl = normalizeBaseUrl(option(args, 'base-url', 'http://127.0.0.1'));
|
||||||
|
const expectedVersion = option(args, 'expected-version', '');
|
||||||
|
const timeoutMs = Number.parseInt(option(args, 'timeout-ms', '5000'), 10);
|
||||||
|
if (!Number.isFinite(timeoutMs) || timeoutMs <= 0) {
|
||||||
|
throw new Error('--timeout-ms must be a positive number');
|
||||||
|
}
|
||||||
|
|
||||||
|
const checks = buildSmokeChecks(baseUrl, expectedVersion);
|
||||||
|
if (flag(args, 'dry-run')) {
|
||||||
|
process.stdout.write(`[dry-run] Release smoke target: ${baseUrl}\n`);
|
||||||
|
for (const check of checks) {
|
||||||
|
process.stdout.write(`GET ${check.url} # ${check.name}\n`);
|
||||||
|
}
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
process.stdout.write(`Release smoke target: ${baseUrl}\n`);
|
||||||
|
const failures = [];
|
||||||
|
for (const check of checks) {
|
||||||
|
try {
|
||||||
|
await runCheck(check, timeoutMs);
|
||||||
|
process.stdout.write(`[pass] ${check.name} ${check.url}\n`);
|
||||||
|
} catch (error) {
|
||||||
|
failures.push(`[fail] ${check.name} ${check.url}: ${error.message}`);
|
||||||
|
process.stderr.write(`${failures.at(-1)}\n`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (failures.length > 0) {
|
||||||
|
throw new Error(`Release smoke failed: ${failures.length} check(s) failed`);
|
||||||
|
}
|
||||||
|
|
||||||
|
process.stdout.write('Release smoke test passed.\n');
|
||||||
|
}
|
||||||
|
|
||||||
|
if (import.meta.url === pathToFileURL(process.argv[1]).href) {
|
||||||
|
main().catch((error) => {
|
||||||
|
process.stderr.write(`${error.message}\n`);
|
||||||
|
process.exit(1);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
@@ -21,6 +21,7 @@ const checks = [
|
|||||||
'NEXT_PUBLIC_API_URL',
|
'NEXT_PUBLIC_API_URL',
|
||||||
'NEXT_PUBLIC_APP_VERSION',
|
'NEXT_PUBLIC_APP_VERSION',
|
||||||
'ARG APP_VERSION=unknown',
|
'ARG APP_VERSION=unknown',
|
||||||
|
'COPY scripts ./scripts',
|
||||||
'CMD ["pnpm", "--filter", "web", "start"]',
|
'CMD ["pnpm", "--filter", "web", "start"]',
|
||||||
],
|
],
|
||||||
},
|
},
|
||||||
@@ -50,6 +51,11 @@ const checks = [
|
|||||||
'WEB_IMAGE',
|
'WEB_IMAGE',
|
||||||
'APP_VERSION',
|
'APP_VERSION',
|
||||||
'/api/v1/health/version',
|
'/api/v1/health/version',
|
||||||
|
'prometheus:',
|
||||||
|
"profiles: ['monitoring']",
|
||||||
|
'postgres-exporter:',
|
||||||
|
'blackbox-exporter:',
|
||||||
|
'prometheus_data:',
|
||||||
],
|
],
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -59,13 +65,42 @@ const checks = [
|
|||||||
'appleboy/ssh-action',
|
'appleboy/ssh-action',
|
||||||
'docker compose --env-file .env.production -f docker-compose.prod.yml pull',
|
'docker compose --env-file .env.production -f docker-compose.prod.yml pull',
|
||||||
'pnpm --filter server db:deploy',
|
'pnpm --filter server db:deploy',
|
||||||
'/api/v1/health/version',
|
'scripts/smoke-test-release.mjs',
|
||||||
|
'--expected-version',
|
||||||
],
|
],
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
file: 'scripts/check-runtime-version.mjs',
|
file: 'scripts/check-runtime-version.mjs',
|
||||||
snippets: ['Runtime version verified', '/api/v1/health/version', 'expectedVersion'],
|
snippets: ['Runtime version verified', '/api/v1/health/version', 'expectedVersion'],
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
file: 'scripts/smoke-test-release.mjs',
|
||||||
|
snippets: [
|
||||||
|
'Release smoke test passed',
|
||||||
|
'/api/v1/v2.2/requirements?productId=__smoke__',
|
||||||
|
'/api/v1/config/ai',
|
||||||
|
],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
file: 'scripts/backup-postgres.mjs',
|
||||||
|
snippets: ['pg_dump', '--format=custom', '--dry-run'],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
file: 'scripts/restore-postgres.mjs',
|
||||||
|
snippets: ['--confirm-overwrite', 'dropdb', 'pg_restore'],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
file: 'scripts/backup-server-data.mjs',
|
||||||
|
snippets: ['server_data', 'tar -czf', '--dry-run'],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
file: 'scripts/check-runbook-placeholders.mjs',
|
||||||
|
snippets: ['Runbook placeholder scan passed', 'Runbook placeholder scan failed', '--paths'],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
file: 'package.json',
|
||||||
|
snippets: ['docs:check-runbooks', 'docs/runbooks', 'docs/production-readiness.md'],
|
||||||
|
},
|
||||||
{
|
{
|
||||||
file: 'docker-compose.local.yml',
|
file: 'docker-compose.local.yml',
|
||||||
snippets: [
|
snippets: [
|
||||||
@@ -88,6 +123,66 @@ const checks = [
|
|||||||
'X-Forwarded-Proto',
|
'X-Forwarded-Proto',
|
||||||
],
|
],
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
file: 'deploy/monitoring/README.md',
|
||||||
|
snippets: [
|
||||||
|
'--profile monitoring',
|
||||||
|
'FtbPostgresDown',
|
||||||
|
'FtbXiaobaoSummaryStale',
|
||||||
|
'no webhook URLs',
|
||||||
|
],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
file: 'deploy/monitoring/prometheus/prometheus.yml',
|
||||||
|
snippets: ['postgres-exporter:9187', 'promtail:9080', 'blackbox-http'],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
file: 'deploy/monitoring/prometheus/alert-rules.yml',
|
||||||
|
snippets: [
|
||||||
|
'FtbPostgresDown',
|
||||||
|
'FtbSlowApiLogBurst',
|
||||||
|
'FtbSlowPrismaLogBurst',
|
||||||
|
'FtbJobFailureLogBurst',
|
||||||
|
'FtbXiaobaoSummaryStale',
|
||||||
|
],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
file: 'docs/runbooks/migration-rollback.md',
|
||||||
|
snippets: ['Fresh Database Restore', 'Data Risks', 'pnpm restore:postgres'],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
file: 'docs/runbooks/appdata-retirement.md',
|
||||||
|
snippets: ['Disable AppData Writes', 'Remove Fallback', 'Data Risks'],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
file: 'docs/runbooks/xiaobao-background-jobs.md',
|
||||||
|
snippets: ['FtbXiaobaoSummaryStale', 'Future Scheduler Rules', 'Data Risks'],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
file: 'docs/production-readiness.md',
|
||||||
|
snippets: [
|
||||||
|
'Backup and restore',
|
||||||
|
'Smoke tests',
|
||||||
|
'Monitoring',
|
||||||
|
'Audit',
|
||||||
|
'RBAC',
|
||||||
|
'Consistency',
|
||||||
|
'Performance',
|
||||||
|
'Post-release',
|
||||||
|
],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
file: 'deploy/monitoring/postgres/postgres-queries.yml',
|
||||||
|
snippets: ['xiaobao_risk_summaries', 'dirty = true', 'stale_summary_count'],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
file: 'deploy/monitoring/promtail/config.yml',
|
||||||
|
snippets: ['Slow API request', 'Slow Prisma query', 'AppData relation sync failed'],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
file: 'deploy/monitoring/grafana/dashboards/ftb-production-overview.json',
|
||||||
|
snippets: ['FTB Production Overview', 'ftb_xiaobao_stale_summary_count', 'Server Warning/Error Logs'],
|
||||||
|
},
|
||||||
{
|
{
|
||||||
file: '.env.production.example',
|
file: '.env.production.example',
|
||||||
snippets: [
|
snippets: [
|
||||||
@@ -118,6 +213,12 @@ const checks = [
|
|||||||
'docker-compose.local.yml',
|
'docker-compose.local.yml',
|
||||||
'本地服务器部署',
|
'本地服务器部署',
|
||||||
'pnpm deploy:verify',
|
'pnpm deploy:verify',
|
||||||
|
'pnpm backup:postgres',
|
||||||
|
'pnpm restore:postgres',
|
||||||
|
'pnpm backup:server-data',
|
||||||
|
'docs/runbooks/migration-rollback.md',
|
||||||
|
'docs/production-readiness.md',
|
||||||
|
'--profile monitoring',
|
||||||
'pnpm db:migrate',
|
'pnpm db:migrate',
|
||||||
'NEXT_PUBLIC_API_URL',
|
'NEXT_PUBLIC_API_URL',
|
||||||
'Nginx',
|
'Nginx',
|
||||||
|
|||||||
Reference in New Issue
Block a user