feat: 完善服务监控与下游重投

tools/monitoring/install-service-exporters.sh

@@ -0,0 +1,92 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+if [[ "$(id -u)" -ne 0 ]]; then echo "Run as root." >&2; exit 1; fi
+for command_name in apt-get systemctl nginx curl; do command -v "$command_name" >/dev/null || { echo "Missing command: $command_name" >&2; exit 1; }; done
+
+log() { printf '\n[%s] %s\n' "$(date '+%F %T')" "$*"; }
+backup_dir="/etc/prometheus/cmpp-backups/$(date '+%Y%m%d-%H%M%S')-service-exporters"
+mkdir -p "$backup_dir"
+
+for config_file in /etc/cmpp-platform/minio.env /etc/nginx/conf.d/cmpp-monitoring-status.conf /etc/cmpp-platform/monitoring-exporters.env; do
+  [[ -f "$config_file" ]] && cp --preserve=mode,timestamps "$config_file" "$backup_dir/$(basename "$config_file")"
+done
+
+log "Installing bounded service exporters"
+apt-get update
+DEBIAN_FRONTEND=noninteractive apt-get install -y prometheus-postgres-exporter prometheus-redis-exporter prometheus-nginx-exporter
+
+[[ -f /etc/cmpp-platform/cmpp-platform.env ]] || { echo "Missing platform environment file." >&2; exit 1; }
+set -a
+# shellcheck disable=SC1091
+. /etc/cmpp-platform/cmpp-platform.env
+set +a
+database_base="${DATABASE_URL%%\?*}"
+cat >/etc/cmpp-platform/monitoring-exporters.env <<EOF
+DATA_SOURCE_NAME=${database_base}?sslmode=disable
+REDIS_ADDR=redis://${REDIS_HOST:-127.0.0.1}:${REDIS_PORT:-6379}
+EOF
+chmod 0600 /etc/cmpp-platform/monitoring-exporters.env
+
+write_override() {
+  local unit="$1" executable="$2" arguments="$3"
+  local directory="/etc/systemd/system/${unit}.service.d"
+  mkdir -p "$directory"
+  [[ -f "$directory/cmpp-monitoring.conf" ]] && cp --preserve=mode,timestamps "$directory/cmpp-monitoring.conf" "$backup_dir/${unit}-override.conf"
+  cat >"$directory/cmpp-monitoring.conf" <<EOF
+[Service]
+EnvironmentFile=/etc/cmpp-platform/monitoring-exporters.env
+ExecStart=
+ExecStart=${executable} ${arguments}
+EOF
+}
+
+postgres_exporter="$(command -v prometheus-postgres-exporter)"
+redis_exporter="$(command -v prometheus-redis-exporter)"
+nginx_exporter="$(command -v prometheus-nginx-exporter)"
+write_override prometheus-postgres-exporter "$postgres_exporter" '--web.listen-address=127.0.0.1:9187'
+write_override prometheus-redis-exporter "$redis_exporter" '--web.listen-address=127.0.0.1:9121'
+
+cat >/etc/nginx/conf.d/cmpp-monitoring-status.conf <<'EOF'
+server {
+  listen 127.0.0.1:8088;
+  server_name localhost;
+  access_log off;
+  location = /stub_status {
+    stub_status;
+    allow 127.0.0.1;
+    allow ::1;
+    deny all;
+  }
+}
+EOF
+write_override prometheus-nginx-exporter "$nginx_exporter" '--web.listen-address=127.0.0.1:9113 --nginx.scrape-uri=http://127.0.0.1:8088/stub_status'
+
+# MinIO exposes only operational aggregates and listens on loopback; public auth here does not expose objects or credentials.
+grep -q '^MINIO_PROMETHEUS_AUTH_TYPE=' /etc/cmpp-platform/minio.env \
+  && sed -i 's/^MINIO_PROMETHEUS_AUTH_TYPE=.*/MINIO_PROMETHEUS_AUTH_TYPE=public/' /etc/cmpp-platform/minio.env \
+  || printf '\nMINIO_PROMETHEUS_AUTH_TYPE=public\n' >>/etc/cmpp-platform/minio.env
+
+nginx -t
+systemctl daemon-reload
+systemctl enable prometheus-postgres-exporter prometheus-redis-exporter prometheus-nginx-exporter
+systemctl restart prometheus-postgres-exporter prometheus-redis-exporter prometheus-nginx-exporter
+systemctl restart cmpp-minio
+systemctl reload nginx
+
+wait_for_http() {
+  local endpoint="$1" attempt
+  for attempt in $(seq 1 30); do
+    curl -fsS "$endpoint" >/dev/null 2>&1 && return 0
+    sleep 1
+  done
+  echo "Monitoring endpoint did not become ready: $endpoint" >&2
+  return 1
+}
+for endpoint in 127.0.0.1:9187 127.0.0.1:9121 127.0.0.1:9113; do wait_for_http "http://${endpoint}/metrics"; done
+wait_for_http http://127.0.0.1:9000/minio/v2/metrics/cluster
+if ss -lnt | grep -Eq '(^|[[:space:]])(0\.0\.0\.0|\[::\]):(9187|9121|9113)([[:space:]]|$)'; then
+  echo "A service exporter unexpectedly listens on a wildcard address." >&2
+  exit 1
+fi
+log "Service exporters are ready on loopback only; backup: $backup_dir"