feat: add Prometheus system monitoring

2026-08-14 10:10:28 +08:00
parent 96e475d60d
commit b78faa1aa2
25 changed files with 1675 additions and 0 deletions
@@ -0,0 +1,31 @@
+# CMPP Prometheus 监控配置
+
+本目录提供系统监控第一版所需的真实采集配置。运营端不嵌入Prometheus或Grafana页面；NestJS仅从本机Prometheus读取固定指标，再由平台原生UI展示。
+
+在目标Debian/Ubuntu测试服务器上，以root执行：
+
+```bash
+cd /opt/cmpp-platform
+bash tools/monitoring/install-prometheus-monitoring.sh
+```
+
+脚本会安装Prometheus与Node Exporter、备份已有Prometheus配置及本脚本曾写入的systemd override、校验规则、写入新override，并仅重启这两个监控服务。备份目录会在脚本结束时打印。它不会重启API、Gateway、数据库、Redis、MinIO或Nginx。9090和9100固定监听`127.0.0.1`。
+
+安装后将下列配置写入`/etc/cmpp-platform/cmpp-platform.env`，再按正常发布窗口重启API：
+
+```dotenv
+PROMETHEUS_URL=http://127.0.0.1:9090
+PROMETHEUS_QUERY_TIMEOUT_MS=5000
+```
+
+校验命令：
+
+```bash
+promtool check config /etc/prometheus/prometheus.yml
+promtool check rules /etc/prometheus/cmpp-alerts.yml
+curl -fsS http://127.0.0.1:9090/-/ready
+curl -fsS 'http://127.0.0.1:9090/api/v1/query?query=up'
+ss -lnt | grep -E ':(9090|9100)'
+```
+
+完整架构、PromQL口径、故障语义和验收标准见`docs/prometheus-system-monitoring-design-20260814.md`。
@@ -0,0 +1,148 @@
+groups:
+  - name: cmpp-host-resources
+    rules:
+      - alert: NodeExporterDown
+        expr: up{job="node"} == 0
+        for: 2m
+        labels:
+          severity: critical
+          service: node-exporter
+        annotations:
+          summary: 主机指标采集不可用
+          description: Prometheus连续2分钟无法采集Node Exporter。
+          currentValue: "{{ $value }}"
+          threshold: "up = 1"
+
+      - alert: HostCpuUsageWarning
+        expr: (100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 85) and (100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) <= 95)
+        for: 10m
+        labels:
+          severity: warning
+          service: host
+        annotations:
+          summary: CPU使用率持续偏高
+          description: 主机CPU使用率连续10分钟高于85%。
+          currentValue: "{{ printf \"%.1f\" $value }}%"
+          threshold: "85%"
+
+      - alert: HostCpuUsageCritical
+        expr: 100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 95
+        for: 5m
+        labels:
+          severity: critical
+          service: host
+        annotations:
+          summary: CPU使用率严重超限
+          description: 主机CPU使用率连续5分钟高于95%。
+          currentValue: "{{ printf \"%.1f\" $value }}%"
+          threshold: "95%"
+
+      - alert: HostMemoryUsageWarning
+        expr: ((1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100 > 85) and ((1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100 <= 95)
+        for: 10m
+        labels:
+          severity: warning
+          service: host
+        annotations:
+          summary: 内存使用率持续偏高
+          description: 主机可用内存连续10分钟低于15%。
+          currentValue: "{{ printf \"%.1f\" $value }}%"
+          threshold: "85%"
+
+      - alert: HostMemoryUsageCritical
+        expr: (1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100 > 95
+        for: 5m
+        labels:
+          severity: critical
+          service: host
+        annotations:
+          summary: 内存使用率严重超限
+          description: 主机可用内存连续5分钟低于5%。
+          currentValue: "{{ printf \"%.1f\" $value }}%"
+          threshold: "95%"
+
+      - alert: HostRootDiskUsageWarning
+        expr: ((1 - node_filesystem_avail_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"} / node_filesystem_size_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"}) * 100 > 80) and ((1 - node_filesystem_avail_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"} / node_filesystem_size_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"}) * 100 <= 90)
+        for: 15m
+        labels:
+          severity: warning
+          service: host
+        annotations:
+          summary: 根文件系统空间不足
+          description: 根文件系统使用率连续15分钟高于80%。
+          currentValue: "{{ printf \"%.1f\" $value }}%"
+          threshold: "80%"
+
+      - alert: HostRootDiskUsageCritical
+        expr: (1 - node_filesystem_avail_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"} / node_filesystem_size_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"}) * 100 > 90
+        for: 5m
+        labels:
+          severity: critical
+          service: host
+        annotations:
+          summary: 根文件系统空间严重不足
+          description: 根文件系统使用率连续5分钟高于90%。
+          currentValue: "{{ printf \"%.1f\" $value }}%"
+          threshold: "90%"
+
+      - alert: HostRootInodeUsageWarning
+        expr: ((1 - node_filesystem_files_free{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"} / node_filesystem_files{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"}) * 100 > 80) and ((1 - node_filesystem_files_free{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"} / node_filesystem_files{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"}) * 100 <= 90)
+        for: 15m
+        labels:
+          severity: warning
+          service: host
+        annotations:
+          summary: 根文件系统inode余量偏低
+          description: 根文件系统inode使用率连续15分钟高于80%。
+          currentValue: "{{ printf \"%.1f\" $value }}%"
+          threshold: "80%"
+
+      - alert: HostRootInodeUsageCritical
+        expr: (1 - node_filesystem_files_free{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"} / node_filesystem_files{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"}) * 100 > 90
+        for: 5m
+        labels:
+          severity: critical
+          service: host
+        annotations:
+          summary: 根文件系统inode严重不足
+          description: 根文件系统inode使用率连续5分钟高于90%。
+          currentValue: "{{ printf \"%.1f\" $value }}%"
+          threshold: "90%"
+
+      - alert: HostCpuIowaitWarning
+        expr: (avg by (instance) (rate(node_cpu_seconds_total{mode="iowait"}[5m])) * 100 > 20) and (avg by (instance) (rate(node_cpu_seconds_total{mode="iowait"}[5m])) * 100 <= 35)
+        for: 10m
+        labels:
+          severity: warning
+          service: host
+        annotations:
+          summary: CPU iowait持续偏高
+          description: 主机CPU iowait连续10分钟高于20%，请检查磁盘I/O。
+          currentValue: "{{ printf \"%.1f\" $value }}%"
+          threshold: "20%"
+
+      - alert: HostCpuIowaitCritical
+        expr: avg by (instance) (rate(node_cpu_seconds_total{mode="iowait"}[5m])) * 100 > 35
+        for: 10m
+        labels:
+          severity: critical
+          service: host
+        annotations:
+          summary: CPU iowait严重超限
+          description: 主机CPU iowait连续10分钟高于35%，磁盘I/O可能已成为瓶颈。
+          currentValue: "{{ printf \"%.1f\" $value }}%"
+          threshold: "35%"
+
+  - name: cmpp-core-services
+    rules:
+      - alert: CmppCoreServiceInactive
+        expr: node_systemd_unit_state{name=~"cmpp-api\\.service|cmpp-gateway\\.service|postgresql\\.service|redis(-server)?\\.service|cmpp-minio\\.service|nginx\\.service",state="active"} == 0
+        for: 2m
+        labels:
+          severity: critical
+          service: "{{ $labels.name }}"
+        annotations:
+          summary: CMPP核心服务未处于active状态
+          description: "systemd服务 {{ $labels.name }} 连续2分钟未处于active状态。"
+          currentValue: "{{ $value }}"
+          threshold: "active = 1"
@@ -0,0 +1,82 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+PROMETHEUS_RETENTION_TIME="${PROMETHEUS_RETENTION_TIME:-30d}"
+PROMETHEUS_RETENTION_SIZE="${PROMETHEUS_RETENTION_SIZE:-8GB}"
+
+if [[ "$(id -u)" -ne 0 ]]; then
+  echo "Run as root." >&2
+  exit 1
+fi
+if ! command -v apt-get >/dev/null 2>&1; then
+  echo "This installer currently supports Debian/Ubuntu apt packages only." >&2
+  exit 1
+fi
+
+log() { printf '\n[%s] %s\n' "$(date '+%F %T')" "$*"; }
+
+log "Installing Prometheus and Node Exporter packages"
+apt-get update
+DEBIAN_FRONTEND=noninteractive apt-get install -y prometheus prometheus-node-exporter curl iproute2
+
+prometheus_bin="$(command -v prometheus)"
+node_exporter_bin="$(command -v prometheus-node-exporter)"
+promtool_bin="$(command -v promtool)"
+backup_dir="/etc/prometheus/cmpp-backups/$(date '+%Y%m%d-%H%M%S')"
+mkdir -p "$backup_dir" /etc/systemd/system/prometheus.service.d /etc/systemd/system/prometheus-node-exporter.service.d
+
+for config_file in /etc/prometheus/prometheus.yml /etc/prometheus/cmpp-alerts.yml; do
+  if [[ -f "$config_file" ]]; then
+    cp --preserve=mode,timestamps "$config_file" "$backup_dir/$(basename "$config_file")"
+  fi
+done
+if [[ -f /etc/systemd/system/prometheus.service.d/cmpp-monitoring.conf ]]; then
+  cp --preserve=mode,timestamps /etc/systemd/system/prometheus.service.d/cmpp-monitoring.conf "$backup_dir/prometheus-service-override.conf"
+fi
+if [[ -f /etc/systemd/system/prometheus-node-exporter.service.d/cmpp-monitoring.conf ]]; then
+  cp --preserve=mode,timestamps /etc/systemd/system/prometheus-node-exporter.service.d/cmpp-monitoring.conf "$backup_dir/node-exporter-service-override.conf"
+fi
+
+log "Installing platform-owned scrape and alert configuration"
+install -o root -g root -m 0644 "$SCRIPT_DIR/prometheus.yml" /etc/prometheus/prometheus.yml
+install -o root -g root -m 0644 "$SCRIPT_DIR/cmpp-alerts.yml" /etc/prometheus/cmpp-alerts.yml
+
+cat >/etc/systemd/system/prometheus.service.d/cmpp-monitoring.conf <<EOF
+[Service]
+ExecStart=
+ExecStart=${prometheus_bin} --config.file=/etc/prometheus/prometheus.yml --storage.tsdb.path=/var/lib/prometheus/metrics2 --storage.tsdb.retention.time=${PROMETHEUS_RETENTION_TIME} --storage.tsdb.retention.size=${PROMETHEUS_RETENTION_SIZE} --web.listen-address=127.0.0.1:9090
+EOF
+
+cat >/etc/systemd/system/prometheus-node-exporter.service.d/cmpp-monitoring.conf <<EOF
+[Service]
+ExecStart=
+ExecStart=${node_exporter_bin} --web.listen-address=127.0.0.1:9100 --collector.systemd --collector.systemd.unit-include='cmpp-api\\.service|cmpp-gateway\\.service|postgresql\\.service|redis(-server)?\\.service|cmpp-minio\\.service|nginx\\.service' --collector.filesystem.mount-points-exclude='^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/)'
+EOF
+
+log "Validating Prometheus configuration before restart"
+"$promtool_bin" check rules /etc/prometheus/cmpp-alerts.yml
+"$promtool_bin" check config /etc/prometheus/prometheus.yml
+
+systemctl daemon-reload
+systemctl enable prometheus prometheus-node-exporter
+systemctl restart prometheus-node-exporter
+systemctl restart prometheus
+
+for _ in $(seq 1 30); do
+  if curl -fsS http://127.0.0.1:9090/-/ready >/dev/null; then
+    break
+  fi
+  sleep 1
+done
+curl -fsS http://127.0.0.1:9090/-/ready >/dev/null
+curl -fsS http://127.0.0.1:9100/metrics >/dev/null
+
+if ss -lnt | grep -Eq '(^|[[:space:]])(0\.0\.0\.0|\[::\]):(9090|9100)([[:space:]]|$)'; then
+  echo "Prometheus monitoring ports unexpectedly listen on a wildcard address." >&2
+  exit 1
+fi
+
+log "Prometheus monitoring is ready on loopback only"
+echo "Configuration backup: $backup_dir"
+echo "The CMPP API and Gateway were not restarted."
@@ -0,0 +1,20 @@
+global:
+  scrape_interval: 15s
+  evaluation_interval: 15s
+  external_labels:
+    platform: cmpp
+    environment: preproduction
+
+rule_files:
+  - /etc/prometheus/cmpp-alerts.yml
+
+scrape_configs:
+  - job_name: prometheus
+    static_configs:
+      - targets: [127.0.0.1:9090]
+
+  - job_name: node
+    static_configs:
+      - targets: [127.0.0.1:9100]
+        labels:
+          host: cmpp-primary