feat: add Prometheus system monitoring

2026-08-14 10:10:28 +08:00
parent 96e475d60d
commit b78faa1aa2
25 changed files with 1675 additions and 0 deletions
@@ -0,0 +1,82 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+PROMETHEUS_RETENTION_TIME="${PROMETHEUS_RETENTION_TIME:-30d}"
+PROMETHEUS_RETENTION_SIZE="${PROMETHEUS_RETENTION_SIZE:-8GB}"
+
+if [[ "$(id -u)" -ne 0 ]]; then
+  echo "Run as root." >&2
+  exit 1
+fi
+if ! command -v apt-get >/dev/null 2>&1; then
+  echo "This installer currently supports Debian/Ubuntu apt packages only." >&2
+  exit 1
+fi
+
+log() { printf '\n[%s] %s\n' "$(date '+%F %T')" "$*"; }
+
+log "Installing Prometheus and Node Exporter packages"
+apt-get update
+DEBIAN_FRONTEND=noninteractive apt-get install -y prometheus prometheus-node-exporter curl iproute2
+
+prometheus_bin="$(command -v prometheus)"
+node_exporter_bin="$(command -v prometheus-node-exporter)"
+promtool_bin="$(command -v promtool)"
+backup_dir="/etc/prometheus/cmpp-backups/$(date '+%Y%m%d-%H%M%S')"
+mkdir -p "$backup_dir" /etc/systemd/system/prometheus.service.d /etc/systemd/system/prometheus-node-exporter.service.d
+
+for config_file in /etc/prometheus/prometheus.yml /etc/prometheus/cmpp-alerts.yml; do
+  if [[ -f "$config_file" ]]; then
+    cp --preserve=mode,timestamps "$config_file" "$backup_dir/$(basename "$config_file")"
+  fi
+done
+if [[ -f /etc/systemd/system/prometheus.service.d/cmpp-monitoring.conf ]]; then
+  cp --preserve=mode,timestamps /etc/systemd/system/prometheus.service.d/cmpp-monitoring.conf "$backup_dir/prometheus-service-override.conf"
+fi
+if [[ -f /etc/systemd/system/prometheus-node-exporter.service.d/cmpp-monitoring.conf ]]; then
+  cp --preserve=mode,timestamps /etc/systemd/system/prometheus-node-exporter.service.d/cmpp-monitoring.conf "$backup_dir/node-exporter-service-override.conf"
+fi
+
+log "Installing platform-owned scrape and alert configuration"
+install -o root -g root -m 0644 "$SCRIPT_DIR/prometheus.yml" /etc/prometheus/prometheus.yml
+install -o root -g root -m 0644 "$SCRIPT_DIR/cmpp-alerts.yml" /etc/prometheus/cmpp-alerts.yml
+
+cat >/etc/systemd/system/prometheus.service.d/cmpp-monitoring.conf <<EOF
+[Service]
+ExecStart=
+ExecStart=${prometheus_bin} --config.file=/etc/prometheus/prometheus.yml --storage.tsdb.path=/var/lib/prometheus/metrics2 --storage.tsdb.retention.time=${PROMETHEUS_RETENTION_TIME} --storage.tsdb.retention.size=${PROMETHEUS_RETENTION_SIZE} --web.listen-address=127.0.0.1:9090
+EOF
+
+cat >/etc/systemd/system/prometheus-node-exporter.service.d/cmpp-monitoring.conf <<EOF
+[Service]
+ExecStart=
+ExecStart=${node_exporter_bin} --web.listen-address=127.0.0.1:9100 --collector.systemd --collector.systemd.unit-include='cmpp-api\\.service|cmpp-gateway\\.service|postgresql\\.service|redis(-server)?\\.service|cmpp-minio\\.service|nginx\\.service' --collector.filesystem.mount-points-exclude='^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/)'
+EOF
+
+log "Validating Prometheus configuration before restart"
+"$promtool_bin" check rules /etc/prometheus/cmpp-alerts.yml
+"$promtool_bin" check config /etc/prometheus/prometheus.yml
+
+systemctl daemon-reload
+systemctl enable prometheus prometheus-node-exporter
+systemctl restart prometheus-node-exporter
+systemctl restart prometheus
+
+for _ in $(seq 1 30); do
+  if curl -fsS http://127.0.0.1:9090/-/ready >/dev/null; then
+    break
+  fi
+  sleep 1
+done
+curl -fsS http://127.0.0.1:9090/-/ready >/dev/null
+curl -fsS http://127.0.0.1:9100/metrics >/dev/null
+
+if ss -lnt | grep -Eq '(^|[[:space:]])(0\.0\.0\.0|\[::\]):(9090|9100)([[:space:]]|$)'; then
+  echo "Prometheus monitoring ports unexpectedly listen on a wildcard address." >&2
+  exit 1
+fi
+
+log "Prometheus monitoring is ready on loopback only"
+echo "Configuration backup: $backup_dir"
+echo "The CMPP API and Gateway were not restarted."