feat: 完善服务监控与下游重投

tools/monitoring/README.md

@@ -7,9 +7,10 @@
 ```bash
 cd /opt/cmpp-platform
 bash tools/monitoring/install-prometheus-monitoring.sh
+bash tools/monitoring/install-service-exporters.sh
 ```
 
-脚本会安装Prometheus与Node Exporter、备份已有Prometheus配置及本脚本曾写入的systemd override、校验规则、写入新override，并仅重启这两个监控服务。备份目录会在脚本结束时打印。它不会重启API、Gateway、数据库、Redis、MinIO或Nginx。9090和9100固定监听`127.0.0.1`。
+第一个脚本安装Prometheus与Node Exporter，备份已有配置并校验规则；第二个脚本安装PostgreSQL、Redis和Nginx Exporter，开启MinIO回环原生指标。API指标仅监听`127.0.0.1:9464`，Gateway指标复用回环控制端口`8090`。9090、9100、9187、9121、9113和9464均不得对公网开放。两个脚本均会打印恢复资产路径。
 
 安装后将下列配置写入`/etc/cmpp-platform/cmpp-platform.env`，再按正常发布窗口重启API：
 
@@ -26,6 +27,8 @@ promtool check rules /etc/prometheus/cmpp-alerts.yml
 curl -fsS http://127.0.0.1:9090/-/ready
 curl -fsS 'http://127.0.0.1:9090/api/v1/query?query=up'
 ss -lnt | grep -E ':(9090|9100)'
+curl -fsS http://127.0.0.1:9464/metrics
+curl -fsS http://127.0.0.1:8090/metrics
 ```
 
 完整架构、PromQL口径、故障语义和验收标准见`docs/prometheus-system-monitoring-design-20260814.md`。

tools/monitoring/cmpp-alerts.yml

@@ -1,4 +1,50 @@
 groups:
+  - name: cmpp-service-recording
+    interval: 15s
+    rules:
+      - record: cmpp:service_api:requests_per_second
+        expr: sum(rate(cmpp_api_http_requests_total[5m]))
+      - record: cmpp:service_api:error_percent
+        expr: 100 * sum(rate(cmpp_api_http_requests_total{status=~"5.."}[5m])) / clamp_min(sum(rate(cmpp_api_http_requests_total[5m])), 0.001)
+      - record: cmpp:service_api:latency_p95_seconds
+        expr: histogram_quantile(0.95, sum by (le) (rate(cmpp_api_http_request_duration_seconds_bucket[5m])))
+      - record: cmpp:service_api:event_loop_p99_seconds
+        expr: cmpp_api_nodejs_event_loop_lag_p99_seconds
+      - record: cmpp:service_gateway:submits_per_second
+        expr: sum(rate(cmpp_gateway_submit_total[5m]))
+      - record: cmpp:service_gateway:failure_percent
+        expr: 100 * sum(rate(cmpp_gateway_submit_total{result="failed"}[5m])) / clamp_min(sum(rate(cmpp_gateway_submit_total[5m])), 0.001)
+      - record: cmpp:service_gateway:queue_pending
+        expr: cmpp_gateway_submit_queue_pending
+      - record: cmpp:service_gateway:queue_lag
+        expr: cmpp_gateway_submit_queue_lag
+      - record: cmpp:service_gateway:queue_oldest_seconds
+        expr: cmpp_gateway_submit_queue_oldest_pending_age_seconds
+      - record: cmpp:service_postgresql:connection_percent
+        expr: 100 * sum(pg_stat_activity_count) / clamp_min(max(pg_settings_max_connections), 1)
+      - record: cmpp:service_postgresql:deadlocks_15m
+        expr: sum(increase(pg_stat_database_deadlocks[15m]))
+      - record: cmpp:service_redis:memory_percent
+        expr: (100 * redis_memory_used_bytes / redis_memory_max_bytes) and on(instance) (redis_memory_max_bytes > 0)
+      - record: cmpp:service_redis:memory_used_bytes
+        expr: redis_memory_used_bytes
+      - record: cmpp:service_redis:evictions_5m
+        expr: increase(redis_evicted_keys_total[5m])
+      - record: cmpp:service_redis:connected_clients
+        expr: redis_connected_clients
+      - record: cmpp:service_nginx:connections_active
+        expr: nginx_connections_active
+      - record: cmpp:service_nginx:requests_per_second
+        expr: rate(nginx_http_requests_total[5m])
+      - record: cmpp:service_minio:capacity_percent
+        expr: 100 * (1 - minio_cluster_capacity_usable_free_bytes / minio_cluster_capacity_usable_total_bytes)
+      - record: cmpp:service_minio:usage_bytes
+        expr: minio_cluster_usage_total_bytes
+      - record: cmpp:service_minio:objects
+        expr: minio_cluster_usage_object_total
+      - record: cmpp:service_minio:drives_offline
+        expr: minio_cluster_drive_offline_total
+
   - name: cmpp-host-resources
     rules:
       - alert: NodeExporterDown
@@ -14,28 +60,28 @@ groups:
           threshold: "up = 1"
 
       - alert: HostCpuUsageWarning
-        expr: (100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 85) and (100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) <= 95)
+        expr: (100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 80) and (100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) <= 90)
         for: 10m
         labels:
           severity: warning
           service: host
         annotations:
           summary: CPU使用率持续偏高
-          description: 主机CPU使用率连续10分钟高于85%。
+          description: 主机CPU使用率连续10分钟高于80%。
           currentValue: "{{ printf \"%.1f\" $value }}%"
-          threshold: "85%"
+          threshold: "80%"
 
       - alert: HostCpuUsageCritical
-        expr: 100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 95
+        expr: 100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 90
         for: 5m
         labels:
           severity: critical
           service: host
         annotations:
           summary: CPU使用率严重超限
-          description: 主机CPU使用率连续5分钟高于95%。
+          description: 主机CPU使用率连续5分钟高于90%。
           currentValue: "{{ printf \"%.1f\" $value }}%"
-          threshold: "95%"
+          threshold: "90%"
 
       - alert: HostMemoryUsageWarning
         expr: ((1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100 > 85) and ((1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100 <= 95)
@@ -146,3 +192,158 @@ groups:
           description: "systemd服务 {{ $labels.name }} 连续2分钟未处于active状态。"
           currentValue: "{{ $value }}"
           threshold: "active = 1"
+
+  - name: cmpp-api-runtime
+    rules:
+      - alert: CmppApiMetricsDown
+        expr: up{job="cmpp-api"} == 0
+        for: 2m
+        labels: { severity: critical, service: api }
+        annotations: { summary: "API指标采集不可用", description: "Prometheus连续2分钟无法读取API内部指标端点。", currentValue: "{{ $value }}", threshold: "up = 1" }
+      - alert: CmppApiHttpErrorRateWarning
+        expr: (sum(rate(cmpp_api_http_requests_total{status=~"5.."}[5m])) / clamp_min(sum(rate(cmpp_api_http_requests_total[5m])), 0.001) > 0.01) and (sum(rate(cmpp_api_http_requests_total{status=~"5.."}[5m])) / clamp_min(sum(rate(cmpp_api_http_requests_total[5m])), 0.001) <= 0.05) and sum(increase(cmpp_api_http_requests_total{status=~"5.."}[5m])) >= 5
+        for: 5m
+        labels: { severity: warning, service: api }
+        annotations: { summary: "API 5xx错误率偏高", description: "API 5xx错误率连续5分钟高于1%，且窗口内至少5次错误。", currentValue: "{{ printf \"%.2f\" $value }}", threshold: "1%" }
+      - alert: CmppApiHttpErrorRateCritical
+        expr: (sum(rate(cmpp_api_http_requests_total{status=~"5.."}[5m])) / clamp_min(sum(rate(cmpp_api_http_requests_total[5m])), 0.001) > 0.05) and sum(increase(cmpp_api_http_requests_total{status=~"5.."}[5m])) >= 5
+        for: 5m
+        labels: { severity: critical, service: api }
+        annotations: { summary: "API 5xx错误率严重超限", description: "API 5xx错误率连续5分钟高于5%。", currentValue: "{{ printf \"%.2f\" $value }}", threshold: "5%" }
+      - alert: CmppApiLatencyWarning
+        expr: (histogram_quantile(0.95, sum by (le) (rate(cmpp_api_http_request_duration_seconds_bucket[10m]))) > 1) and (histogram_quantile(0.95, sum by (le) (rate(cmpp_api_http_request_duration_seconds_bucket[10m]))) <= 3)
+        for: 10m
+        labels: { severity: warning, service: api }
+        annotations: { summary: "API P95响应偏慢", description: "API P95响应时间连续10分钟超过1秒。", currentValue: "{{ printf \"%.3f\" $value }}s", threshold: "1s" }
+      - alert: CmppApiLatencyCritical
+        expr: histogram_quantile(0.95, sum by (le) (rate(cmpp_api_http_request_duration_seconds_bucket[5m]))) > 3
+        for: 5m
+        labels: { severity: critical, service: api }
+        annotations: { summary: "API P95响应严重超时", description: "API P95响应时间连续5分钟超过3秒。", currentValue: "{{ printf \"%.3f\" $value }}s", threshold: "3s" }
+      - alert: CmppApiEventLoopLagWarning
+        expr: (cmpp_api_nodejs_event_loop_lag_p99_seconds > 0.2) and (cmpp_api_nodejs_event_loop_lag_p99_seconds <= 1)
+        for: 10m
+        labels: { severity: warning, service: api }
+        annotations: { summary: "API事件循环延迟偏高", description: "Node.js事件循环P99延迟连续10分钟超过200ms。", currentValue: "{{ printf \"%.3f\" $value }}s", threshold: "0.2s" }
+      - alert: CmppApiEventLoopLagCritical
+        expr: cmpp_api_nodejs_event_loop_lag_p99_seconds > 1
+        for: 5m
+        labels: { severity: critical, service: api }
+        annotations: { summary: "API事件循环严重阻塞", description: "Node.js事件循环P99延迟连续5分钟超过1秒。", currentValue: "{{ printf \"%.3f\" $value }}s", threshold: "1s" }
+
+  - name: cmpp-gateway-runtime
+    rules:
+      - alert: CmppGatewayMetricsDown
+        expr: up{job="cmpp-gateway"} == 0
+        for: 2m
+        labels: { severity: critical, service: gateway }
+        annotations: { summary: "Gateway指标采集不可用", description: "Prometheus连续2分钟无法读取Gateway指标。", currentValue: "{{ $value }}", threshold: "up = 1" }
+      - alert: CmppGatewaySubmitWorkerDown
+        expr: cmpp_gateway_submit_worker_up == 0
+        for: 1m
+        labels: { severity: critical, service: gateway }
+        annotations: { summary: "Gateway提交消费者未运行", description: "Gateway进程存活，但提交消费者未成功初始化。", currentValue: "{{ $value }}", threshold: "1" }
+      - alert: CmppGatewayUpstreamConnectionShortage
+        expr: cmpp_gateway_upstream_connections{state="connected"} < cmpp_gateway_upstream_connections{state="desired"}
+        for: 2m
+        labels: { severity: critical, service: gateway }
+        annotations: { summary: "Gateway上游连接不足", description: "实际上游CMPP连接数连续2分钟低于期望数。", currentValue: "{{ $value }}", threshold: "connected = desired" }
+      - alert: CmppGatewayQueueDelayedWarning
+        expr: (cmpp_gateway_submit_queue_oldest_pending_age_seconds > 30) and (cmpp_gateway_submit_queue_oldest_pending_age_seconds <= 120)
+        for: 2m
+        labels: { severity: warning, service: gateway }
+        annotations: { summary: "Gateway提交队列开始延迟", description: "Redis Stream最旧pending消息等待超过30秒。", currentValue: "{{ printf \"%.0f\" $value }}s", threshold: "30s" }
+      - alert: CmppGatewayQueueDelayedCritical
+        expr: cmpp_gateway_submit_queue_oldest_pending_age_seconds > 120
+        for: 2m
+        labels: { severity: critical, service: gateway }
+        annotations: { summary: "Gateway提交队列严重延迟", description: "Redis Stream最旧pending消息等待超过120秒。", currentValue: "{{ printf \"%.0f\" $value }}s", threshold: "120s" }
+
+  - name: cmpp-data-services
+    rules:
+      - alert: PostgresExporterDown
+        expr: up{job="postgresql"} == 0 or pg_up == 0
+        for: 2m
+        labels: { severity: critical, service: postgresql }
+        annotations: { summary: "PostgreSQL指标或数据库不可用", description: "PostgreSQL Exporter或其数据库连接连续2分钟不可用。", currentValue: "{{ $value }}", threshold: "up = 1" }
+      - alert: PostgresConnectionsWarning
+        expr: (sum(pg_stat_activity_count) / clamp_min(max(pg_settings_max_connections), 1) > 0.70) and (sum(pg_stat_activity_count) / clamp_min(max(pg_settings_max_connections), 1) <= 0.85)
+        for: 10m
+        labels: { severity: warning, service: postgresql }
+        annotations: { summary: "PostgreSQL连接使用率偏高", description: "数据库连接数连续10分钟超过上限的70%。", currentValue: "{{ printf \"%.2f\" $value }}", threshold: "70%" }
+      - alert: PostgresConnectionsCritical
+        expr: sum(pg_stat_activity_count) / clamp_min(max(pg_settings_max_connections), 1) > 0.85
+        for: 5m
+        labels: { severity: critical, service: postgresql }
+        annotations: { summary: "PostgreSQL连接即将耗尽", description: "数据库连接数连续5分钟超过上限的85%。", currentValue: "{{ printf \"%.2f\" $value }}", threshold: "85%" }
+      - alert: PostgresDeadlocksDetected
+        expr: sum(increase(pg_stat_database_deadlocks[15m])) > 0
+        for: 1m
+        labels: { severity: warning, service: postgresql }
+        annotations: { summary: "PostgreSQL发生死锁", description: "15分钟窗口内检测到数据库死锁。", currentValue: "{{ $value }}", threshold: "0" }
+      - alert: RedisExporterDown
+        expr: up{job="redis"} == 0 or redis_up == 0
+        for: 2m
+        labels: { severity: critical, service: redis }
+        annotations: { summary: "Redis指标或服务不可用", description: "Redis Exporter或Redis连接连续2分钟不可用。", currentValue: "{{ $value }}", threshold: "up = 1" }
+      - alert: RedisMemoryWarning
+        expr: (redis_memory_max_bytes > 0) and (redis_memory_used_bytes / redis_memory_max_bytes > 0.70) and (redis_memory_used_bytes / redis_memory_max_bytes <= 0.85)
+        for: 10m
+        labels: { severity: warning, service: redis }
+        annotations: { summary: "Redis内存使用率偏高", description: "Redis内存连续10分钟超过maxmemory的70%。", currentValue: "{{ printf \"%.2f\" $value }}", threshold: "70%" }
+      - alert: RedisMemoryCritical
+        expr: (redis_memory_max_bytes > 0) and (redis_memory_used_bytes / redis_memory_max_bytes > 0.85)
+        for: 5m
+        labels: { severity: critical, service: redis }
+        annotations: { summary: "Redis内存即将耗尽", description: "Redis内存连续5分钟超过maxmemory的85%。", currentValue: "{{ printf \"%.2f\" $value }}", threshold: "85%" }
+      - alert: RedisUnexpectedEvictions
+        expr: increase(redis_evicted_keys_total[5m]) > 0
+        for: 1m
+        labels: { severity: critical, service: redis }
+        annotations: { summary: "Redis发生Key淘汰", description: "Redis承载队列和运行状态，5分钟内不应出现淘汰。", currentValue: "{{ $value }}", threshold: "0" }
+      - alert: RedisRejectedConnections
+        expr: increase(redis_rejected_connections_total[5m]) > 0
+        for: 1m
+        labels: { severity: critical, service: redis }
+        annotations: { summary: "Redis拒绝连接", description: "5分钟内Redis出现被拒绝连接。", currentValue: "{{ $value }}", threshold: "0" }
+
+  - name: cmpp-storage-and-edge
+    rules:
+      - alert: MinioMetricsDown
+        expr: up{job="minio"} == 0
+        for: 2m
+        labels: { severity: critical, service: minio }
+        annotations: { summary: "MinIO指标采集不可用", description: "Prometheus连续2分钟无法读取MinIO原生指标。", currentValue: "{{ $value }}", threshold: "up = 1" }
+      - alert: MinioCapacityWarning
+        expr: (100 * (1 - minio_cluster_capacity_usable_free_bytes / minio_cluster_capacity_usable_total_bytes) > 80) and (100 * (1 - minio_cluster_capacity_usable_free_bytes / minio_cluster_capacity_usable_total_bytes) <= 90)
+        for: 15m
+        labels: { severity: warning, service: minio }
+        annotations: { summary: "MinIO存储容量偏高", description: "MinIO可用容量使用率连续15分钟超过80%。", currentValue: "{{ printf \"%.1f\" $value }}%", threshold: "80%" }
+      - alert: MinioCapacityCritical
+        expr: 100 * (1 - minio_cluster_capacity_usable_free_bytes / minio_cluster_capacity_usable_total_bytes) > 90
+        for: 5m
+        labels: { severity: critical, service: minio }
+        annotations: { summary: "MinIO存储容量即将耗尽", description: "MinIO可用容量使用率连续5分钟超过90%。", currentValue: "{{ printf \"%.1f\" $value }}%", threshold: "90%" }
+      - alert: MinioDriveOffline
+        expr: minio_cluster_drive_offline_total > 0
+        for: 1m
+        labels: { severity: critical, service: minio }
+        annotations: { summary: "MinIO存储盘离线", description: "MinIO检测到离线存储盘。", currentValue: "{{ $value }}", threshold: "0" }
+      - alert: NginxExporterDown
+        expr: up{job="nginx"} == 0 or nginx_up == 0
+        for: 2m
+        labels: { severity: critical, service: nginx }
+        annotations: { summary: "Nginx指标或状态页不可用", description: "Nginx Exporter或回环stub_status连续2分钟不可用。", currentValue: "{{ $value }}", threshold: "up = 1" }
+
+  - name: cmpp-monitoring-self
+    rules:
+      - alert: PrometheusScrapeSlow
+        expr: scrape_duration_seconds / scrape_interval_seconds > 0.8
+        for: 5m
+        labels: { severity: warning, service: prometheus }
+        annotations: { summary: "Prometheus采集接近超时", description: "采集耗时连续5分钟超过采集周期的80%。", currentValue: "{{ printf \"%.2f\" $value }}", threshold: "80%" }
+      - alert: PrometheusRuleEvaluationFailures
+        expr: increase(prometheus_rule_evaluation_failures_total[5m]) > 0
+        for: 1m
+        labels: { severity: critical, service: prometheus }
+        annotations: { summary: "Prometheus告警规则计算失败", description: "5分钟内出现告警规则计算失败。", currentValue: "{{ $value }}", threshold: "0" }

tools/monitoring/install-service-exporters.sh

@@ -0,0 +1,92 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+if [[ "$(id -u)" -ne 0 ]]; then echo "Run as root." >&2; exit 1; fi
+for command_name in apt-get systemctl nginx curl; do command -v "$command_name" >/dev/null || { echo "Missing command: $command_name" >&2; exit 1; }; done
+
+log() { printf '\n[%s] %s\n' "$(date '+%F %T')" "$*"; }
+backup_dir="/etc/prometheus/cmpp-backups/$(date '+%Y%m%d-%H%M%S')-service-exporters"
+mkdir -p "$backup_dir"
+
+for config_file in /etc/cmpp-platform/minio.env /etc/nginx/conf.d/cmpp-monitoring-status.conf /etc/cmpp-platform/monitoring-exporters.env; do
+  [[ -f "$config_file" ]] && cp --preserve=mode,timestamps "$config_file" "$backup_dir/$(basename "$config_file")"
+done
+
+log "Installing bounded service exporters"
+apt-get update
+DEBIAN_FRONTEND=noninteractive apt-get install -y prometheus-postgres-exporter prometheus-redis-exporter prometheus-nginx-exporter
+
+[[ -f /etc/cmpp-platform/cmpp-platform.env ]] || { echo "Missing platform environment file." >&2; exit 1; }
+set -a
+# shellcheck disable=SC1091
+. /etc/cmpp-platform/cmpp-platform.env
+set +a
+database_base="${DATABASE_URL%%\?*}"
+cat >/etc/cmpp-platform/monitoring-exporters.env <<EOF
+DATA_SOURCE_NAME=${database_base}?sslmode=disable
+REDIS_ADDR=redis://${REDIS_HOST:-127.0.0.1}:${REDIS_PORT:-6379}
+EOF
+chmod 0600 /etc/cmpp-platform/monitoring-exporters.env
+
+write_override() {
+  local unit="$1" executable="$2" arguments="$3"
+  local directory="/etc/systemd/system/${unit}.service.d"
+  mkdir -p "$directory"
+  [[ -f "$directory/cmpp-monitoring.conf" ]] && cp --preserve=mode,timestamps "$directory/cmpp-monitoring.conf" "$backup_dir/${unit}-override.conf"
+  cat >"$directory/cmpp-monitoring.conf" <<EOF
+[Service]
+EnvironmentFile=/etc/cmpp-platform/monitoring-exporters.env
+ExecStart=
+ExecStart=${executable} ${arguments}
+EOF
+}
+
+postgres_exporter="$(command -v prometheus-postgres-exporter)"
+redis_exporter="$(command -v prometheus-redis-exporter)"
+nginx_exporter="$(command -v prometheus-nginx-exporter)"
+write_override prometheus-postgres-exporter "$postgres_exporter" '--web.listen-address=127.0.0.1:9187'
+write_override prometheus-redis-exporter "$redis_exporter" '--web.listen-address=127.0.0.1:9121'
+
+cat >/etc/nginx/conf.d/cmpp-monitoring-status.conf <<'EOF'
+server {
+  listen 127.0.0.1:8088;
+  server_name localhost;
+  access_log off;
+  location = /stub_status {
+    stub_status;
+    allow 127.0.0.1;
+    allow ::1;
+    deny all;
+  }
+}
+EOF
+write_override prometheus-nginx-exporter "$nginx_exporter" '--web.listen-address=127.0.0.1:9113 --nginx.scrape-uri=http://127.0.0.1:8088/stub_status'
+
+# MinIO exposes only operational aggregates and listens on loopback; public auth here does not expose objects or credentials.
+grep -q '^MINIO_PROMETHEUS_AUTH_TYPE=' /etc/cmpp-platform/minio.env \
+  && sed -i 's/^MINIO_PROMETHEUS_AUTH_TYPE=.*/MINIO_PROMETHEUS_AUTH_TYPE=public/' /etc/cmpp-platform/minio.env \
+  || printf '\nMINIO_PROMETHEUS_AUTH_TYPE=public\n' >>/etc/cmpp-platform/minio.env
+
+nginx -t
+systemctl daemon-reload
+systemctl enable prometheus-postgres-exporter prometheus-redis-exporter prometheus-nginx-exporter
+systemctl restart prometheus-postgres-exporter prometheus-redis-exporter prometheus-nginx-exporter
+systemctl restart cmpp-minio
+systemctl reload nginx
+
+wait_for_http() {
+  local endpoint="$1" attempt
+  for attempt in $(seq 1 30); do
+    curl -fsS "$endpoint" >/dev/null 2>&1 && return 0
+    sleep 1
+  done
+  echo "Monitoring endpoint did not become ready: $endpoint" >&2
+  return 1
+}
+for endpoint in 127.0.0.1:9187 127.0.0.1:9121 127.0.0.1:9113; do wait_for_http "http://${endpoint}/metrics"; done
+wait_for_http http://127.0.0.1:9000/minio/v2/metrics/cluster
+if ss -lnt | grep -Eq '(^|[[:space:]])(0\.0\.0\.0|\[::\]):(9187|9121|9113)([[:space:]]|$)'; then
+  echo "A service exporter unexpectedly listens on a wildcard address." >&2
+  exit 1
+fi
+log "Service exporters are ready on loopback only; backup: $backup_dir"

tools/monitoring/prometheus.yml

@@ -18,3 +18,29 @@ scrape_configs:
       - targets: [127.0.0.1:9100]
         labels:
           host: cmpp-primary
+
+  - job_name: cmpp-api
+    static_configs:
+      - targets: [127.0.0.1:9464]
+
+  - job_name: cmpp-gateway
+    metrics_path: /metrics
+    static_configs:
+      - targets: [127.0.0.1:8090]
+
+  - job_name: postgresql
+    static_configs:
+      - targets: [127.0.0.1:9187]
+
+  - job_name: redis
+    static_configs:
+      - targets: [127.0.0.1:9121]
+
+  - job_name: minio
+    metrics_path: /minio/v2/metrics/cluster
+    static_configs:
+      - targets: [127.0.0.1:9000]
+
+  - job_name: nginx
+    static_configs:
+      - targets: [127.0.0.1:9113]