feat: 完善服务监控与下游重投

tools/deploy/production-bootstrap.sh

@@ -6,6 +6,9 @@ REPO_URL="${REPO_URL:-http://175.27.255.91:3000/hectorzhao/lislgosms.git}"
 BRANCH="${BRANCH:-main}"
 PUBLIC_HTTP_PORT="${PUBLIC_HTTP_PORT:-12026}"
 API_PORT="${API_PORT:-3000}"
+API_HOST="${API_HOST:-127.0.0.1}"
+API_METRICS_HOST="${API_METRICS_HOST:-127.0.0.1}"
+API_METRICS_PORT="${API_METRICS_PORT:-9464}"
 API_ENABLE_SEND_WORKER="${API_ENABLE_SEND_WORKER:-true}"
 API_SEND_WORKER_CONCURRENCY="${API_SEND_WORKER_CONCURRENCY:-50}"
 GATEWAY_CONTROL_ADDR="${GATEWAY_CONTROL_ADDR:-127.0.0.1:8090}"
@@ -183,6 +186,9 @@ write_env() {
   cat >/etc/cmpp-platform/cmpp-platform.env <<EOF
 NODE_ENV=production
 API_PORT=${API_PORT}
+API_HOST=${API_HOST}
+API_METRICS_HOST=${API_METRICS_HOST}
+API_METRICS_PORT=${API_METRICS_PORT}
 API_ENABLE_SEND_WORKER=${API_ENABLE_SEND_WORKER}
 API_SEND_WORKER_CONCURRENCY=${API_SEND_WORKER_CONCURRENCY}
 DATABASE_URL=postgresql://${DB_USER}:${DB_PASSWORD}@127.0.0.1:5432/${DB_NAME}?schema=public

tools/deploy/production-deploy.sh

@@ -42,6 +42,7 @@ npm --prefix api ci --include=dev
 
 echo "[deploy] Verifying dependency security mitigations"
 npm run security:verify
+npm run deploy:verify
 
 echo "[deploy] Generating Prisma client and applying migrations"
 npm --prefix api run prisma:generate
@@ -69,13 +70,20 @@ echo "[deploy] Installing restricted security boundary"
 bash "$APP_DIR/tools/security/install-security-agent.sh"
 
 echo "[deploy] Ensuring HTTP response compression"
-cat >/etc/nginx/conf.d/cmpp-compression.conf <<'EOF'
+compression_config=/etc/nginx/conf.d/cmpp-compression.conf
+: >"$compression_config"
+if grep -RqsE --exclude='cmpp-compression.conf' '^[[:space:]]*gzip[[:space:]]+on;' \
+  /etc/nginx/nginx.conf /etc/nginx/conf.d /etc/nginx/sites-enabled 2>/dev/null; then
+  echo "[deploy] Reusing existing Nginx gzip configuration"
+else
+cat >"$compression_config" <<'EOF'
 gzip on;
 gzip_vary on;
 gzip_min_length 1024;
 gzip_comp_level 5;
 gzip_types application/json application/javascript text/javascript text/css text/plain text/csv image/svg+xml;
 EOF
+fi
 nginx -t
 
 echo "[deploy] Restarting services"

tools/deploy/verify-production-deployment.mjs

@@ -0,0 +1,29 @@
+import { readFileSync } from 'node:fs';
+import { resolve } from 'node:path';
+
+const deploy = readFileSync(resolve(import.meta.dirname, 'production-deploy.sh'), 'utf8');
+const bootstrap = readFileSync(resolve(import.meta.dirname, 'production-bootstrap.sh'), 'utf8');
+const apiMain = readFileSync(resolve(import.meta.dirname, '../../api/src/main.ts'), 'utf8');
+const required = [
+  'compression_config=/etc/nginx/conf.d/cmpp-compression.conf',
+  ': >"$compression_config"',
+  "--exclude='cmpp-compression.conf'",
+  "'^[[:space:]]*gzip[[:space:]]+on;'",
+  'Reusing existing Nginx gzip configuration',
+  'cat >"$compression_config"',
+];
+for (const marker of required) {
+  if (!deploy.includes(marker)) throw new Error(`production deploy is missing the idempotent Nginx compression guard: ${marker}`);
+}
+if (deploy.includes("cat >/etc/nginx/conf.d/cmpp-compression.conf <<'EOF'")) {
+  throw new Error('production deploy still writes a duplicate global gzip directive unconditionally');
+}
+
+for (const marker of ['API_HOST="${API_HOST:-127.0.0.1}"', 'API_HOST=${API_HOST}']) {
+  if (!bootstrap.includes(marker)) throw new Error(`production bootstrap is missing the loopback API binding: ${marker}`);
+}
+if (!apiMain.includes("process.env.API_HOST?.trim() || '127.0.0.1'") || !apiMain.includes('app.listen(port, host)')) {
+  throw new Error('NestJS API must bind to API_HOST and default to loopback');
+}
+
+console.log('Production deployment verified: Nginx compression is idempotent and NestJS defaults to loopback.');

tools/monitoring/README.md

@@ -7,9 +7,10 @@
 ```bash
 cd /opt/cmpp-platform
 bash tools/monitoring/install-prometheus-monitoring.sh
+bash tools/monitoring/install-service-exporters.sh
 ```
 
-脚本会安装Prometheus与Node Exporter、备份已有Prometheus配置及本脚本曾写入的systemd override、校验规则、写入新override，并仅重启这两个监控服务。备份目录会在脚本结束时打印。它不会重启API、Gateway、数据库、Redis、MinIO或Nginx。9090和9100固定监听`127.0.0.1`。
+第一个脚本安装Prometheus与Node Exporter，备份已有配置并校验规则；第二个脚本安装PostgreSQL、Redis和Nginx Exporter，开启MinIO回环原生指标。API指标仅监听`127.0.0.1:9464`，Gateway指标复用回环控制端口`8090`。9090、9100、9187、9121、9113和9464均不得对公网开放。两个脚本均会打印恢复资产路径。
 
 安装后将下列配置写入`/etc/cmpp-platform/cmpp-platform.env`，再按正常发布窗口重启API：
 
@@ -26,6 +27,8 @@ promtool check rules /etc/prometheus/cmpp-alerts.yml
 curl -fsS http://127.0.0.1:9090/-/ready
 curl -fsS 'http://127.0.0.1:9090/api/v1/query?query=up'
 ss -lnt | grep -E ':(9090|9100)'
+curl -fsS http://127.0.0.1:9464/metrics
+curl -fsS http://127.0.0.1:8090/metrics
 ```
 
 完整架构、PromQL口径、故障语义和验收标准见`docs/prometheus-system-monitoring-design-20260814.md`。

tools/monitoring/cmpp-alerts.yml

@@ -1,4 +1,50 @@
 groups:
+  - name: cmpp-service-recording
+    interval: 15s
+    rules:
+      - record: cmpp:service_api:requests_per_second
+        expr: sum(rate(cmpp_api_http_requests_total[5m]))
+      - record: cmpp:service_api:error_percent
+        expr: 100 * sum(rate(cmpp_api_http_requests_total{status=~"5.."}[5m])) / clamp_min(sum(rate(cmpp_api_http_requests_total[5m])), 0.001)
+      - record: cmpp:service_api:latency_p95_seconds
+        expr: histogram_quantile(0.95, sum by (le) (rate(cmpp_api_http_request_duration_seconds_bucket[5m])))
+      - record: cmpp:service_api:event_loop_p99_seconds
+        expr: cmpp_api_nodejs_event_loop_lag_p99_seconds
+      - record: cmpp:service_gateway:submits_per_second
+        expr: sum(rate(cmpp_gateway_submit_total[5m]))
+      - record: cmpp:service_gateway:failure_percent
+        expr: 100 * sum(rate(cmpp_gateway_submit_total{result="failed"}[5m])) / clamp_min(sum(rate(cmpp_gateway_submit_total[5m])), 0.001)
+      - record: cmpp:service_gateway:queue_pending
+        expr: cmpp_gateway_submit_queue_pending
+      - record: cmpp:service_gateway:queue_lag
+        expr: cmpp_gateway_submit_queue_lag
+      - record: cmpp:service_gateway:queue_oldest_seconds
+        expr: cmpp_gateway_submit_queue_oldest_pending_age_seconds
+      - record: cmpp:service_postgresql:connection_percent
+        expr: 100 * sum(pg_stat_activity_count) / clamp_min(max(pg_settings_max_connections), 1)
+      - record: cmpp:service_postgresql:deadlocks_15m
+        expr: sum(increase(pg_stat_database_deadlocks[15m]))
+      - record: cmpp:service_redis:memory_percent
+        expr: (100 * redis_memory_used_bytes / redis_memory_max_bytes) and on(instance) (redis_memory_max_bytes > 0)
+      - record: cmpp:service_redis:memory_used_bytes
+        expr: redis_memory_used_bytes
+      - record: cmpp:service_redis:evictions_5m
+        expr: increase(redis_evicted_keys_total[5m])
+      - record: cmpp:service_redis:connected_clients
+        expr: redis_connected_clients
+      - record: cmpp:service_nginx:connections_active
+        expr: nginx_connections_active
+      - record: cmpp:service_nginx:requests_per_second
+        expr: rate(nginx_http_requests_total[5m])
+      - record: cmpp:service_minio:capacity_percent
+        expr: 100 * (1 - minio_cluster_capacity_usable_free_bytes / minio_cluster_capacity_usable_total_bytes)
+      - record: cmpp:service_minio:usage_bytes
+        expr: minio_cluster_usage_total_bytes
+      - record: cmpp:service_minio:objects
+        expr: minio_cluster_usage_object_total
+      - record: cmpp:service_minio:drives_offline
+        expr: minio_cluster_drive_offline_total
+
   - name: cmpp-host-resources
     rules:
       - alert: NodeExporterDown
@@ -14,28 +60,28 @@ groups:
           threshold: "up = 1"
 
       - alert: HostCpuUsageWarning
-        expr: (100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 85) and (100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) <= 95)
+        expr: (100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 80) and (100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) <= 90)
         for: 10m
         labels:
           severity: warning
           service: host
         annotations:
           summary: CPU使用率持续偏高
-          description: 主机CPU使用率连续10分钟高于85%。
+          description: 主机CPU使用率连续10分钟高于80%。
           currentValue: "{{ printf \"%.1f\" $value }}%"
-          threshold: "85%"
+          threshold: "80%"
 
       - alert: HostCpuUsageCritical
-        expr: 100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 95
+        expr: 100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 90
         for: 5m
         labels:
           severity: critical
           service: host
         annotations:
           summary: CPU使用率严重超限
-          description: 主机CPU使用率连续5分钟高于95%。
+          description: 主机CPU使用率连续5分钟高于90%。
           currentValue: "{{ printf \"%.1f\" $value }}%"
-          threshold: "95%"
+          threshold: "90%"
 
       - alert: HostMemoryUsageWarning
         expr: ((1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100 > 85) and ((1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100 <= 95)
@@ -146,3 +192,158 @@ groups:
           description: "systemd服务 {{ $labels.name }} 连续2分钟未处于active状态。"
           currentValue: "{{ $value }}"
           threshold: "active = 1"
+
+  - name: cmpp-api-runtime
+    rules:
+      - alert: CmppApiMetricsDown
+        expr: up{job="cmpp-api"} == 0
+        for: 2m
+        labels: { severity: critical, service: api }
+        annotations: { summary: "API指标采集不可用", description: "Prometheus连续2分钟无法读取API内部指标端点。", currentValue: "{{ $value }}", threshold: "up = 1" }
+      - alert: CmppApiHttpErrorRateWarning
+        expr: (sum(rate(cmpp_api_http_requests_total{status=~"5.."}[5m])) / clamp_min(sum(rate(cmpp_api_http_requests_total[5m])), 0.001) > 0.01) and (sum(rate(cmpp_api_http_requests_total{status=~"5.."}[5m])) / clamp_min(sum(rate(cmpp_api_http_requests_total[5m])), 0.001) <= 0.05) and sum(increase(cmpp_api_http_requests_total{status=~"5.."}[5m])) >= 5
+        for: 5m
+        labels: { severity: warning, service: api }
+        annotations: { summary: "API 5xx错误率偏高", description: "API 5xx错误率连续5分钟高于1%，且窗口内至少5次错误。", currentValue: "{{ printf \"%.2f\" $value }}", threshold: "1%" }
+      - alert: CmppApiHttpErrorRateCritical
+        expr: (sum(rate(cmpp_api_http_requests_total{status=~"5.."}[5m])) / clamp_min(sum(rate(cmpp_api_http_requests_total[5m])), 0.001) > 0.05) and sum(increase(cmpp_api_http_requests_total{status=~"5.."}[5m])) >= 5
+        for: 5m
+        labels: { severity: critical, service: api }
+        annotations: { summary: "API 5xx错误率严重超限", description: "API 5xx错误率连续5分钟高于5%。", currentValue: "{{ printf \"%.2f\" $value }}", threshold: "5%" }
+      - alert: CmppApiLatencyWarning
+        expr: (histogram_quantile(0.95, sum by (le) (rate(cmpp_api_http_request_duration_seconds_bucket[10m]))) > 1) and (histogram_quantile(0.95, sum by (le) (rate(cmpp_api_http_request_duration_seconds_bucket[10m]))) <= 3)
+        for: 10m
+        labels: { severity: warning, service: api }
+        annotations: { summary: "API P95响应偏慢", description: "API P95响应时间连续10分钟超过1秒。", currentValue: "{{ printf \"%.3f\" $value }}s", threshold: "1s" }
+      - alert: CmppApiLatencyCritical
+        expr: histogram_quantile(0.95, sum by (le) (rate(cmpp_api_http_request_duration_seconds_bucket[5m]))) > 3
+        for: 5m
+        labels: { severity: critical, service: api }
+        annotations: { summary: "API P95响应严重超时", description: "API P95响应时间连续5分钟超过3秒。", currentValue: "{{ printf \"%.3f\" $value }}s", threshold: "3s" }
+      - alert: CmppApiEventLoopLagWarning
+        expr: (cmpp_api_nodejs_event_loop_lag_p99_seconds > 0.2) and (cmpp_api_nodejs_event_loop_lag_p99_seconds <= 1)
+        for: 10m
+        labels: { severity: warning, service: api }
+        annotations: { summary: "API事件循环延迟偏高", description: "Node.js事件循环P99延迟连续10分钟超过200ms。", currentValue: "{{ printf \"%.3f\" $value }}s", threshold: "0.2s" }
+      - alert: CmppApiEventLoopLagCritical
+        expr: cmpp_api_nodejs_event_loop_lag_p99_seconds > 1
+        for: 5m
+        labels: { severity: critical, service: api }
+        annotations: { summary: "API事件循环严重阻塞", description: "Node.js事件循环P99延迟连续5分钟超过1秒。", currentValue: "{{ printf \"%.3f\" $value }}s", threshold: "1s" }
+
+  - name: cmpp-gateway-runtime
+    rules:
+      - alert: CmppGatewayMetricsDown
+        expr: up{job="cmpp-gateway"} == 0
+        for: 2m
+        labels: { severity: critical, service: gateway }
+        annotations: { summary: "Gateway指标采集不可用", description: "Prometheus连续2分钟无法读取Gateway指标。", currentValue: "{{ $value }}", threshold: "up = 1" }
+      - alert: CmppGatewaySubmitWorkerDown
+        expr: cmpp_gateway_submit_worker_up == 0
+        for: 1m
+        labels: { severity: critical, service: gateway }
+        annotations: { summary: "Gateway提交消费者未运行", description: "Gateway进程存活，但提交消费者未成功初始化。", currentValue: "{{ $value }}", threshold: "1" }
+      - alert: CmppGatewayUpstreamConnectionShortage
+        expr: cmpp_gateway_upstream_connections{state="connected"} < cmpp_gateway_upstream_connections{state="desired"}
+        for: 2m
+        labels: { severity: critical, service: gateway }
+        annotations: { summary: "Gateway上游连接不足", description: "实际上游CMPP连接数连续2分钟低于期望数。", currentValue: "{{ $value }}", threshold: "connected = desired" }
+      - alert: CmppGatewayQueueDelayedWarning
+        expr: (cmpp_gateway_submit_queue_oldest_pending_age_seconds > 30) and (cmpp_gateway_submit_queue_oldest_pending_age_seconds <= 120)
+        for: 2m
+        labels: { severity: warning, service: gateway }
+        annotations: { summary: "Gateway提交队列开始延迟", description: "Redis Stream最旧pending消息等待超过30秒。", currentValue: "{{ printf \"%.0f\" $value }}s", threshold: "30s" }
+      - alert: CmppGatewayQueueDelayedCritical
+        expr: cmpp_gateway_submit_queue_oldest_pending_age_seconds > 120
+        for: 2m
+        labels: { severity: critical, service: gateway }
+        annotations: { summary: "Gateway提交队列严重延迟", description: "Redis Stream最旧pending消息等待超过120秒。", currentValue: "{{ printf \"%.0f\" $value }}s", threshold: "120s" }
+
+  - name: cmpp-data-services
+    rules:
+      - alert: PostgresExporterDown
+        expr: up{job="postgresql"} == 0 or pg_up == 0
+        for: 2m
+        labels: { severity: critical, service: postgresql }
+        annotations: { summary: "PostgreSQL指标或数据库不可用", description: "PostgreSQL Exporter或其数据库连接连续2分钟不可用。", currentValue: "{{ $value }}", threshold: "up = 1" }
+      - alert: PostgresConnectionsWarning
+        expr: (sum(pg_stat_activity_count) / clamp_min(max(pg_settings_max_connections), 1) > 0.70) and (sum(pg_stat_activity_count) / clamp_min(max(pg_settings_max_connections), 1) <= 0.85)
+        for: 10m
+        labels: { severity: warning, service: postgresql }
+        annotations: { summary: "PostgreSQL连接使用率偏高", description: "数据库连接数连续10分钟超过上限的70%。", currentValue: "{{ printf \"%.2f\" $value }}", threshold: "70%" }
+      - alert: PostgresConnectionsCritical
+        expr: sum(pg_stat_activity_count) / clamp_min(max(pg_settings_max_connections), 1) > 0.85
+        for: 5m
+        labels: { severity: critical, service: postgresql }
+        annotations: { summary: "PostgreSQL连接即将耗尽", description: "数据库连接数连续5分钟超过上限的85%。", currentValue: "{{ printf \"%.2f\" $value }}", threshold: "85%" }
+      - alert: PostgresDeadlocksDetected
+        expr: sum(increase(pg_stat_database_deadlocks[15m])) > 0
+        for: 1m
+        labels: { severity: warning, service: postgresql }
+        annotations: { summary: "PostgreSQL发生死锁", description: "15分钟窗口内检测到数据库死锁。", currentValue: "{{ $value }}", threshold: "0" }
+      - alert: RedisExporterDown
+        expr: up{job="redis"} == 0 or redis_up == 0
+        for: 2m
+        labels: { severity: critical, service: redis }
+        annotations: { summary: "Redis指标或服务不可用", description: "Redis Exporter或Redis连接连续2分钟不可用。", currentValue: "{{ $value }}", threshold: "up = 1" }
+      - alert: RedisMemoryWarning
+        expr: (redis_memory_max_bytes > 0) and (redis_memory_used_bytes / redis_memory_max_bytes > 0.70) and (redis_memory_used_bytes / redis_memory_max_bytes <= 0.85)
+        for: 10m
+        labels: { severity: warning, service: redis }
+        annotations: { summary: "Redis内存使用率偏高", description: "Redis内存连续10分钟超过maxmemory的70%。", currentValue: "{{ printf \"%.2f\" $value }}", threshold: "70%" }
+      - alert: RedisMemoryCritical
+        expr: (redis_memory_max_bytes > 0) and (redis_memory_used_bytes / redis_memory_max_bytes > 0.85)
+        for: 5m
+        labels: { severity: critical, service: redis }
+        annotations: { summary: "Redis内存即将耗尽", description: "Redis内存连续5分钟超过maxmemory的85%。", currentValue: "{{ printf \"%.2f\" $value }}", threshold: "85%" }
+      - alert: RedisUnexpectedEvictions
+        expr: increase(redis_evicted_keys_total[5m]) > 0
+        for: 1m
+        labels: { severity: critical, service: redis }
+        annotations: { summary: "Redis发生Key淘汰", description: "Redis承载队列和运行状态，5分钟内不应出现淘汰。", currentValue: "{{ $value }}", threshold: "0" }
+      - alert: RedisRejectedConnections
+        expr: increase(redis_rejected_connections_total[5m]) > 0
+        for: 1m
+        labels: { severity: critical, service: redis }
+        annotations: { summary: "Redis拒绝连接", description: "5分钟内Redis出现被拒绝连接。", currentValue: "{{ $value }}", threshold: "0" }
+
+  - name: cmpp-storage-and-edge
+    rules:
+      - alert: MinioMetricsDown
+        expr: up{job="minio"} == 0
+        for: 2m
+        labels: { severity: critical, service: minio }
+        annotations: { summary: "MinIO指标采集不可用", description: "Prometheus连续2分钟无法读取MinIO原生指标。", currentValue: "{{ $value }}", threshold: "up = 1" }
+      - alert: MinioCapacityWarning
+        expr: (100 * (1 - minio_cluster_capacity_usable_free_bytes / minio_cluster_capacity_usable_total_bytes) > 80) and (100 * (1 - minio_cluster_capacity_usable_free_bytes / minio_cluster_capacity_usable_total_bytes) <= 90)
+        for: 15m
+        labels: { severity: warning, service: minio }
+        annotations: { summary: "MinIO存储容量偏高", description: "MinIO可用容量使用率连续15分钟超过80%。", currentValue: "{{ printf \"%.1f\" $value }}%", threshold: "80%" }
+      - alert: MinioCapacityCritical
+        expr: 100 * (1 - minio_cluster_capacity_usable_free_bytes / minio_cluster_capacity_usable_total_bytes) > 90
+        for: 5m
+        labels: { severity: critical, service: minio }
+        annotations: { summary: "MinIO存储容量即将耗尽", description: "MinIO可用容量使用率连续5分钟超过90%。", currentValue: "{{ printf \"%.1f\" $value }}%", threshold: "90%" }
+      - alert: MinioDriveOffline
+        expr: minio_cluster_drive_offline_total > 0
+        for: 1m
+        labels: { severity: critical, service: minio }
+        annotations: { summary: "MinIO存储盘离线", description: "MinIO检测到离线存储盘。", currentValue: "{{ $value }}", threshold: "0" }
+      - alert: NginxExporterDown
+        expr: up{job="nginx"} == 0 or nginx_up == 0
+        for: 2m
+        labels: { severity: critical, service: nginx }
+        annotations: { summary: "Nginx指标或状态页不可用", description: "Nginx Exporter或回环stub_status连续2分钟不可用。", currentValue: "{{ $value }}", threshold: "up = 1" }
+
+  - name: cmpp-monitoring-self
+    rules:
+      - alert: PrometheusScrapeSlow
+        expr: scrape_duration_seconds / scrape_interval_seconds > 0.8
+        for: 5m
+        labels: { severity: warning, service: prometheus }
+        annotations: { summary: "Prometheus采集接近超时", description: "采集耗时连续5分钟超过采集周期的80%。", currentValue: "{{ printf \"%.2f\" $value }}", threshold: "80%" }
+      - alert: PrometheusRuleEvaluationFailures
+        expr: increase(prometheus_rule_evaluation_failures_total[5m]) > 0
+        for: 1m
+        labels: { severity: critical, service: prometheus }
+        annotations: { summary: "Prometheus告警规则计算失败", description: "5分钟内出现告警规则计算失败。", currentValue: "{{ $value }}", threshold: "0" }

tools/monitoring/install-service-exporters.sh

@@ -0,0 +1,92 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+if [[ "$(id -u)" -ne 0 ]]; then echo "Run as root." >&2; exit 1; fi
+for command_name in apt-get systemctl nginx curl; do command -v "$command_name" >/dev/null || { echo "Missing command: $command_name" >&2; exit 1; }; done
+
+log() { printf '\n[%s] %s\n' "$(date '+%F %T')" "$*"; }
+backup_dir="/etc/prometheus/cmpp-backups/$(date '+%Y%m%d-%H%M%S')-service-exporters"
+mkdir -p "$backup_dir"
+
+for config_file in /etc/cmpp-platform/minio.env /etc/nginx/conf.d/cmpp-monitoring-status.conf /etc/cmpp-platform/monitoring-exporters.env; do
+  [[ -f "$config_file" ]] && cp --preserve=mode,timestamps "$config_file" "$backup_dir/$(basename "$config_file")"
+done
+
+log "Installing bounded service exporters"
+apt-get update
+DEBIAN_FRONTEND=noninteractive apt-get install -y prometheus-postgres-exporter prometheus-redis-exporter prometheus-nginx-exporter
+
+[[ -f /etc/cmpp-platform/cmpp-platform.env ]] || { echo "Missing platform environment file." >&2; exit 1; }
+set -a
+# shellcheck disable=SC1091
+. /etc/cmpp-platform/cmpp-platform.env
+set +a
+database_base="${DATABASE_URL%%\?*}"
+cat >/etc/cmpp-platform/monitoring-exporters.env <<EOF
+DATA_SOURCE_NAME=${database_base}?sslmode=disable
+REDIS_ADDR=redis://${REDIS_HOST:-127.0.0.1}:${REDIS_PORT:-6379}
+EOF
+chmod 0600 /etc/cmpp-platform/monitoring-exporters.env
+
+write_override() {
+  local unit="$1" executable="$2" arguments="$3"
+  local directory="/etc/systemd/system/${unit}.service.d"
+  mkdir -p "$directory"
+  [[ -f "$directory/cmpp-monitoring.conf" ]] && cp --preserve=mode,timestamps "$directory/cmpp-monitoring.conf" "$backup_dir/${unit}-override.conf"
+  cat >"$directory/cmpp-monitoring.conf" <<EOF
+[Service]
+EnvironmentFile=/etc/cmpp-platform/monitoring-exporters.env
+ExecStart=
+ExecStart=${executable} ${arguments}
+EOF
+}
+
+postgres_exporter="$(command -v prometheus-postgres-exporter)"
+redis_exporter="$(command -v prometheus-redis-exporter)"
+nginx_exporter="$(command -v prometheus-nginx-exporter)"
+write_override prometheus-postgres-exporter "$postgres_exporter" '--web.listen-address=127.0.0.1:9187'
+write_override prometheus-redis-exporter "$redis_exporter" '--web.listen-address=127.0.0.1:9121'
+
+cat >/etc/nginx/conf.d/cmpp-monitoring-status.conf <<'EOF'
+server {
+  listen 127.0.0.1:8088;
+  server_name localhost;
+  access_log off;
+  location = /stub_status {
+    stub_status;
+    allow 127.0.0.1;
+    allow ::1;
+    deny all;
+  }
+}
+EOF
+write_override prometheus-nginx-exporter "$nginx_exporter" '--web.listen-address=127.0.0.1:9113 --nginx.scrape-uri=http://127.0.0.1:8088/stub_status'
+
+# MinIO exposes only operational aggregates and listens on loopback; public auth here does not expose objects or credentials.
+grep -q '^MINIO_PROMETHEUS_AUTH_TYPE=' /etc/cmpp-platform/minio.env \
+  && sed -i 's/^MINIO_PROMETHEUS_AUTH_TYPE=.*/MINIO_PROMETHEUS_AUTH_TYPE=public/' /etc/cmpp-platform/minio.env \
+  || printf '\nMINIO_PROMETHEUS_AUTH_TYPE=public\n' >>/etc/cmpp-platform/minio.env
+
+nginx -t
+systemctl daemon-reload
+systemctl enable prometheus-postgres-exporter prometheus-redis-exporter prometheus-nginx-exporter
+systemctl restart prometheus-postgres-exporter prometheus-redis-exporter prometheus-nginx-exporter
+systemctl restart cmpp-minio
+systemctl reload nginx
+
+wait_for_http() {
+  local endpoint="$1" attempt
+  for attempt in $(seq 1 30); do
+    curl -fsS "$endpoint" >/dev/null 2>&1 && return 0
+    sleep 1
+  done
+  echo "Monitoring endpoint did not become ready: $endpoint" >&2
+  return 1
+}
+for endpoint in 127.0.0.1:9187 127.0.0.1:9121 127.0.0.1:9113; do wait_for_http "http://${endpoint}/metrics"; done
+wait_for_http http://127.0.0.1:9000/minio/v2/metrics/cluster
+if ss -lnt | grep -Eq '(^|[[:space:]])(0\.0\.0\.0|\[::\]):(9187|9121|9113)([[:space:]]|$)'; then
+  echo "A service exporter unexpectedly listens on a wildcard address." >&2
+  exit 1
+fi
+log "Service exporters are ready on loopback only; backup: $backup_dir"

tools/monitoring/prometheus.yml

@@ -18,3 +18,29 @@ scrape_configs:
       - targets: [127.0.0.1:9100]
         labels:
           host: cmpp-primary
+
+  - job_name: cmpp-api
+    static_configs:
+      - targets: [127.0.0.1:9464]
+
+  - job_name: cmpp-gateway
+    metrics_path: /metrics
+    static_configs:
+      - targets: [127.0.0.1:8090]
+
+  - job_name: postgresql
+    static_configs:
+      - targets: [127.0.0.1:9187]
+
+  - job_name: redis
+    static_configs:
+      - targets: [127.0.0.1:9121]
+
+  - job_name: minio
+    metrics_path: /minio/v2/metrics/cluster
+    static_configs:
+      - targets: [127.0.0.1:9000]
+
+  - job_name: nginx
+    static_configs:
+      - targets: [127.0.0.1:9113]

tools/security/install-security-agent.sh

@@ -4,7 +4,8 @@ set -Eeuo pipefail
 APP_DIR="${APP_DIR:-/opt/cmpp-platform}"
 if [[ "$(id -u)" -ne 0 ]]; then echo "Run as root." >&2; exit 1; fi
 for command_name in fail2ban-client nft nginx systemctl; do command -v "$command_name" >/dev/null || { echo "Missing command: $command_name" >&2; exit 1; }; done
-[[ -x "$APP_DIR/dist/cmpp-security-agent" ]] || { echo "Missing built security agent" >&2; exit 1; }
+agent_binary="$APP_DIR/dist/cmpp-security-agent"
+[[ -x "$agent_binary" ]] || { echo "Missing built security agent: $agent_binary" >&2; exit 1; }
 
 getent group cmpp-security >/dev/null || groupadd --system cmpp-security
 id cmpp-api >/dev/null 2>&1 || useradd --system --home-dir /nonexistent --shell /usr/sbin/nologin cmpp-api
@@ -14,7 +15,8 @@ install -d -o root -g cmpp-security -m 0750 /var/lib/cmpp-security-agent
 install -d -o cmpp-api -g cmpp-security -m 0750 "$APP_DIR/logs/api"
 [[ -d /var/lib/cmpp-platform/object-storage ]] && chown -R cmpp-api:cmpp-security /var/lib/cmpp-platform/object-storage
 
-install -m 0640 "$APP_DIR/deploy/security/cmpp-report-only.conf" /etc/fail2ban/action.d/cmpp-report-only.conf
+sed "s#@CMPP_SECURITY_AGENT_BIN@#$agent_binary#g" "$APP_DIR/deploy/security/cmpp-report-only.conf" >/etc/fail2ban/action.d/cmpp-report-only.conf
+chmod 0640 /etc/fail2ban/action.d/cmpp-report-only.conf
 install -m 0640 "$APP_DIR/deploy/security/cmpp-http-scan.conf" /etc/fail2ban/filter.d/cmpp-http-scan.conf
 install -d -m 0750 /etc/nginx/snippets /etc/nftables.d
 touch /etc/nginx/snippets/cmpp-security-deny.conf
@@ -31,7 +33,11 @@ grep -q 'cmpp-security.nft' /etc/nftables.conf || printf '\ninclude "/etc/nftabl
 nft -c -f /etc/nftables.conf
 nft list table inet cmpp_security >/dev/null 2>&1 || nft -f /etc/nftables.d/cmpp-security.nft
 
-sed "s#/opt/cmpp-platform/current#$APP_DIR#g" "$APP_DIR/deploy/security/cmpp-security-agent.service" >/etc/systemd/system/cmpp-security-agent.service
+sed "s#@CMPP_SECURITY_AGENT_BIN@#$agent_binary#g" "$APP_DIR/deploy/security/cmpp-security-agent.service" >/etc/systemd/system/cmpp-security-agent.service
+if grep -Rqs '@CMPP_SECURITY_AGENT_BIN@' /etc/systemd/system/cmpp-security-agent.service /etc/fail2ban/action.d/cmpp-report-only.conf; then
+  echo "Security agent executable placeholder was not rendered." >&2
+  exit 1
+fi
 install -d -m 0755 /etc/systemd/system/cmpp-api.service.d
 cat >/etc/systemd/system/cmpp-api.service.d/security-boundary.conf <<EOF
 [Service]

tools/security/verify-security-deployment.mjs

@@ -0,0 +1,30 @@
+import { readFileSync } from 'node:fs';
+import { resolve } from 'node:path';
+
+const root = resolve(import.meta.dirname, '..', '..');
+const read = (relativePath) => readFileSync(resolve(root, relativePath), 'utf8');
+const installer = read('tools/security/install-security-agent.sh');
+const service = read('deploy/security/cmpp-security-agent.service');
+const action = read('deploy/security/cmpp-report-only.conf');
+const placeholder = '@CMPP_SECURITY_AGENT_BIN@';
+
+for (const [label, source] of [['systemd service', service], ['Fail2ban action', action]]) {
+  if (!source.includes(placeholder)) throw new Error(`${label} is missing the security-agent executable placeholder`);
+  if (source.includes('/opt/cmpp-platform/current/bin/cmpp-security-agent')) {
+    throw new Error(`${label} still references the removed current/bin deployment layout`);
+  }
+}
+
+if (!installer.includes('agent_binary="$APP_DIR/dist/cmpp-security-agent"')) {
+  throw new Error('installer does not bind the security agent to the built dist executable');
+}
+for (const target of ['cmpp-security-agent.service', 'cmpp-report-only.conf']) {
+  if (!installer.includes(`sed "s#${placeholder}#$agent_binary#g"`) || !installer.includes(target)) {
+    throw new Error(`installer does not render ${target} with the built security-agent executable`);
+  }
+}
+if (!installer.includes("grep -Rqs '@CMPP_SECURITY_AGENT_BIN@'")) {
+  throw new Error('installer does not fail closed when an executable placeholder remains');
+}
+
+console.log('Security deployment verified: systemd and Fail2ban use the built dist security-agent executable.');