From b0bdb608c13d2e4debdd446af777ebaddc7c5d74 Mon Sep 17 00:00:00 2001 From: hectorzhao Date: Wed, 26 Aug 2026 14:25:31 +0800 Subject: [PATCH] fix: support preproduction monitoring on arm64 --- docs/production-deployment.md | 2 +- tools/monitoring/README.md | 4 +- .../install-prometheus-monitoring.sh | 117 ++++++++++++++++-- tools/monitoring/install-service-exporters.sh | 99 +++++++++++++-- 4 files changed, 200 insertions(+), 22 deletions(-) diff --git a/docs/production-deployment.md b/docs/production-deployment.md index 45021ab..a573916 100644 --- a/docs/production-deployment.md +++ b/docs/production-deployment.md @@ -75,7 +75,7 @@ PROD_ADMIN_USERNAME=prod_admin PROD_ADMIN_PASSWORD='change-me' ``` -系统监控需要在发布前单独安装，配置和规则位于`tools/monitoring/`。在Debian/Ubuntu服务器依次执行`bash tools/monitoring/install-prometheus-monitoring.sh`和`bash tools/monitoring/install-service-exporters.sh`；前者安装Prometheus/Node Exporter，后者安装PostgreSQL/Redis/Nginx Exporter并开启MinIO回环原生指标。脚本必须先备份现有配置并运行`promtool`校验；9090、9100、9187、9121、9113、9464和API/Gateway控制端口必须只监听`127.0.0.1`，不得加入Nginx公网反向代理或安全组放行。安装完成且确认全部target为up后，才可完成发布；详细口径见`docs/prometheus-system-monitoring-design-20260814.md`。 +系统监控需要在发布前单独安装，配置和规则位于`tools/monitoring/`。在 Debian/Ubuntu（apt）或 Alibaba Cloud Linux 4 ARM64 服务器依次执行`bash tools/monitoring/install-prometheus-monitoring.sh`和`bash tools/monitoring/install-service-exporters.sh`；apt 环境使用系统包，Alibaba Cloud Linux 4 ARM64 使用脚本内锁定版本与 SHA-256 的官方 release 二进制。前者安装 Prometheus/Node Exporter，后者安装 PostgreSQL/Redis/Nginx Exporter 并开启 MinIO 回环原生指标。脚本必须先备份已有配置并运行`promtool`校验；9090、9100、9187、9121、9113、9464和API/Gateway控制端口必须只监听`127.0.0.1`，不得加入Nginx公网反向代理或安全组放行。安装完成且确认全部 target 为 up 后，才可完成发布；详细口径见`docs/prometheus-system-monitoring-design-20260814.md`。 安全会话使用 HttpOnly Cookie，正式生产必须先为页面和管理 API 配置 HTTPS，并保持 `SESSION_COOKIE_SECURE=true`；纯 HTTP 的 `IP:12026` 不作为受支持的登录入口，即使切换期仍保留其监听，也只允许用于非登录的兼容检查并应尽快下线。`sms.lisglo.com` 只允许 Cloudflare 回源，`api.lisglo.com` 通过独立 Nginx SNI 虚拟主机只开放客户接口、客户 Swagger 和健康检查；Let’s Encrypt 使用 DNS-01 自动续期，不依赖开放 80 端口。 diff --git a/tools/monitoring/README.md b/tools/monitoring/README.md index fcb59be..479c0af 100644 --- a/tools/monitoring/README.md +++ b/tools/monitoring/README.md @@ -2,7 +2,7 @@ 本目录提供系统监控第一版所需的真实采集配置。运营端不嵌入Prometheus或Grafana页面；NestJS仅从本机Prometheus读取固定指标，再由平台原生UI展示。 -在目标Debian/Ubuntu测试服务器上，以root执行： +在目标 Debian/Ubuntu（apt）或 Alibaba Cloud Linux 4 ARM64 测试服务器上，以 root 执行： ```bash cd /opt/cmpp-platform @@ -10,7 +10,7 @@ bash tools/monitoring/install-prometheus-monitoring.sh bash tools/monitoring/install-service-exporters.sh ``` -第一个脚本安装Prometheus与Node Exporter，备份已有配置并校验规则；第二个脚本安装PostgreSQL、Redis和Nginx Exporter，开启MinIO回环原生指标。API指标仅监听`127.0.0.1:9464`，Gateway指标复用回环控制端口`8090`。9090、9100、9187、9121、9113和9464均不得对公网开放。两个脚本均会打印恢复资产路径。 +Debian/Ubuntu 使用系统包；Alibaba Cloud Linux 4 ARM64 使用脚本中锁定版本和 SHA-256 的官方 release 二进制。第一个脚本安装 Prometheus 与 Node Exporter，备份已有配置并校验规则；第二个脚本安装 PostgreSQL、Redis 和 Nginx Exporter，开启 MinIO 回环原生指标。API 指标仅监听 `127.0.0.1:9464`，Gateway 指标复用回环控制端口 `8090`。9090、9100、9187、9121、9113 和 9464 均不得对公网开放。两个脚本均会打印恢复资产路径。 安装后将下列配置写入`/etc/cmpp-platform/cmpp-platform.env`，再按正常发布窗口重启API： diff --git a/tools/monitoring/install-prometheus-monitoring.sh b/tools/monitoring/install-prometheus-monitoring.sh index d0a20bd..b471e59 100644 --- a/tools/monitoring/install-prometheus-monitoring.sh +++ b/tools/monitoring/install-prometheus-monitoring.sh @@ -4,27 +4,113 @@ set -Eeuo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" PROMETHEUS_RETENTION_TIME="${PROMETHEUS_RETENTION_TIME:-30d}" PROMETHEUS_RETENTION_SIZE="${PROMETHEUS_RETENTION_SIZE:-8GB}" +PROMETHEUS_VERSION="3.14.0" +PROMETHEUS_SHA256_ARM64="077f3781ab7245dc04c9a3c9b78ba120fc8e41aa0dc97489b0af67247e50ba83" +NODE_EXPORTER_VERSION="1.12.1" +NODE_EXPORTER_SHA256_ARM64="ad35b605f9954b9f1ffddf5ba054bdc5a98d790b9eae5291e1eeb83f1ecbd0e7" if [[ "$(id -u)" -ne 0 ]]; then echo "Run as root." >&2 exit 1 fi -if ! command -v apt-get >/dev/null 2>&1; then - echo "This installer currently supports Debian/Ubuntu apt packages only." >&2 - exit 1 -fi - log() { printf '\n[%s] %s\n' "$(date '+%F %T')" "$*"; } -log "Installing Prometheus and Node Exporter packages" -apt-get update -DEBIAN_FRONTEND=noninteractive apt-get install -y prometheus prometheus-node-exporter curl iproute2 +install_release_binary() { + local url="$1" expected_sha256="$2" archive_member="$3" destination="$4" + local work_dir archive + work_dir="$(mktemp -d /tmp/cmpp-monitoring.XXXXXX)" + archive="$work_dir/release.tar.gz" + curl -fL --retry 5 --retry-delay 2 --connect-timeout 15 -o "$archive" "$url" + printf '%s %s\n' "$expected_sha256" "$archive" | sha256sum -c - + tar -xzf "$archive" -C "$work_dir" + install -o root -g root -m 0755 "$work_dir/$archive_member" "$destination" + rm -rf -- "$work_dir" +} + +install_official_arm64_releases() { + [[ "$(uname -m)" == "aarch64" ]] || { + echo "Official-binary installation currently supports Linux aarch64 only." >&2 + exit 1 + } + for command_name in curl sha256sum tar; do + command -v "$command_name" >/dev/null || { echo "Missing command: $command_name" >&2; exit 1; } + done + + log "Installing checksum-pinned official Prometheus ARM64 releases" + install_release_binary \ + "https://github.com/prometheus/prometheus/releases/download/v${PROMETHEUS_VERSION}/prometheus-${PROMETHEUS_VERSION}.linux-arm64.tar.gz" \ + "$PROMETHEUS_SHA256_ARM64" \ + "prometheus-${PROMETHEUS_VERSION}.linux-arm64/prometheus" \ + /usr/local/bin/prometheus + install_release_binary \ + "https://github.com/prometheus/prometheus/releases/download/v${PROMETHEUS_VERSION}/prometheus-${PROMETHEUS_VERSION}.linux-arm64.tar.gz" \ + "$PROMETHEUS_SHA256_ARM64" \ + "prometheus-${PROMETHEUS_VERSION}.linux-arm64/promtool" \ + /usr/local/bin/promtool + install_release_binary \ + "https://github.com/prometheus/node_exporter/releases/download/v${NODE_EXPORTER_VERSION}/node_exporter-${NODE_EXPORTER_VERSION}.linux-arm64.tar.gz" \ + "$NODE_EXPORTER_SHA256_ARM64" \ + "node_exporter-${NODE_EXPORTER_VERSION}.linux-arm64/node_exporter" \ + /usr/local/bin/prometheus-node-exporter + + getent group prometheus >/dev/null || groupadd --system prometheus + id prometheus >/dev/null 2>&1 || useradd --system --gid prometheus --home-dir /var/lib/prometheus --shell /sbin/nologin prometheus + install -d -o root -g root -m 0755 /etc/prometheus + install -d -o prometheus -g prometheus -m 0750 /var/lib/prometheus/metrics2 + + cat >/etc/systemd/system/prometheus.service <<'EOF' +[Unit] +Description=Prometheus monitoring server +Wants=network-online.target +After=network-online.target + +[Service] +Type=simple +User=prometheus +Group=prometheus +Restart=on-failure +RestartSec=5s +ExecStart=/usr/local/bin/prometheus + +[Install] +WantedBy=multi-user.target +EOF + cat >/etc/systemd/system/prometheus-node-exporter.service <<'EOF' +[Unit] +Description=Prometheus Node Exporter +Wants=network-online.target +After=network-online.target + +[Service] +Type=simple +User=prometheus +Group=prometheus +Restart=on-failure +RestartSec=5s +ExecStart=/usr/local/bin/prometheus-node-exporter + +[Install] +WantedBy=multi-user.target +EOF +} + +for command_name in systemctl install awk curl ss; do + command -v "$command_name" >/dev/null || { echo "Missing command: $command_name" >&2; exit 1; } +done + +if command -v apt-get >/dev/null 2>&1; then + log "Installing Prometheus and Node Exporter packages" + apt-get update + DEBIAN_FRONTEND=noninteractive apt-get install -y prometheus prometheus-node-exporter curl iproute2 +else + install_official_arm64_releases +fi prometheus_bin="$(command -v prometheus)" node_exporter_bin="$(command -v prometheus-node-exporter)" promtool_bin="$(command -v promtool)" backup_dir="/etc/prometheus/cmpp-backups/$(date '+%Y%m%d-%H%M%S')" -mkdir -p "$backup_dir" /etc/systemd/system/prometheus.service.d /etc/systemd/system/prometheus-node-exporter.service.d +mkdir -p "$backup_dir" /etc/systemd/system/prometheus.service.d /etc/systemd/system/prometheus-node-exporter.service.d /etc/systemd/system/cmpp-api.service.d for config_file in /etc/prometheus/prometheus.yml /etc/prometheus/cmpp-alerts.yml /etc/prometheus/cmpp-alerts-source.yml; do if [[ -f "$config_file" ]]; then @@ -37,6 +123,9 @@ fi if [[ -f /etc/systemd/system/prometheus-node-exporter.service.d/cmpp-monitoring.conf ]]; then cp --preserve=mode,timestamps /etc/systemd/system/prometheus-node-exporter.service.d/cmpp-monitoring.conf "$backup_dir/node-exporter-service-override.conf" fi +if [[ -f /etc/systemd/system/cmpp-api.service.d/monitoring-storage.conf ]]; then + cp --preserve=mode,timestamps /etc/systemd/system/cmpp-api.service.d/monitoring-storage.conf "$backup_dir/cmpp-api-monitoring-storage.conf" +fi log "Installing platform-owned scrape and alert configuration" install -o root -g root -m 0644 "$SCRIPT_DIR/prometheus.yml" /etc/prometheus/prometheus.yml @@ -58,6 +147,14 @@ install -d -o cmpp-api -g prometheus -m 2750 /var/lib/cmpp-platform/monitoring if [[ ! -f /var/lib/cmpp-platform/monitoring/cmpp-managed-alerts.yml ]]; then install -o cmpp-api -g prometheus -m 0640 "$SCRIPT_DIR/cmpp-managed-alerts.yml" /var/lib/cmpp-platform/monitoring/cmpp-managed-alerts.yml fi +chown cmpp-api:prometheus /var/lib/cmpp-platform/monitoring/cmpp-managed-alerts.yml +chmod 0640 /var/lib/cmpp-platform/monitoring/cmpp-managed-alerts.yml + +# The API writes managed rules atomically. Its hardened unit otherwise sees /var/lib as read-only. +cat >/etc/systemd/system/cmpp-api.service.d/monitoring-storage.conf <<'EOF' +[Service] +ReadWritePaths=/var/lib/cmpp-platform/monitoring +EOF cat >/etc/systemd/system/prometheus.service.d/cmpp-monitoring.conf <&2; exit 1; fi -for command_name in apt-get systemctl nginx curl; do command -v "$command_name" >/dev/null || { echo "Missing command: $command_name" >&2; exit 1; }; done +for command_name in systemctl nginx curl install ss; do command -v "$command_name" >/dev/null || { echo "Missing command: $command_name" >&2; exit 1; }; done log() { printf '\n[%s] %s\n' "$(date '+%F %T')" "$*"; } + +install_release_binary() { + local url="$1" expected_sha256="$2" archive_member="$3" destination="$4" + local work_dir archive + work_dir="$(mktemp -d /tmp/cmpp-monitoring.XXXXXX)" + archive="$work_dir/release.tar.gz" + curl -fL --retry 5 --retry-delay 2 --connect-timeout 15 -o "$archive" "$url" + printf '%s %s\n' "$expected_sha256" "$archive" | sha256sum -c - + tar -xzf "$archive" -C "$work_dir" + install -o root -g root -m 0755 "$work_dir/$archive_member" "$destination" + rm -rf -- "$work_dir" +} + +write_base_unit() { + local unit="$1" description="$2" executable="$3" + cat >"/etc/systemd/system/${unit}.service" <&2 + exit 1 + } + for command_name in sha256sum tar; do + command -v "$command_name" >/dev/null || { echo "Missing command: $command_name" >&2; exit 1; } + done + getent group prometheus >/dev/null || groupadd --system prometheus + id prometheus >/dev/null 2>&1 || useradd --system --gid prometheus --home-dir /var/lib/prometheus --shell /sbin/nologin prometheus + + log "Installing checksum-pinned official service exporter ARM64 releases" + install_release_binary \ + "https://github.com/prometheus-community/postgres_exporter/releases/download/v${POSTGRES_EXPORTER_VERSION}/postgres_exporter-${POSTGRES_EXPORTER_VERSION}.linux-arm64.tar.gz" \ + "$POSTGRES_EXPORTER_SHA256_ARM64" \ + "postgres_exporter-${POSTGRES_EXPORTER_VERSION}.linux-arm64/postgres_exporter" \ + /usr/local/bin/prometheus-postgres-exporter + install_release_binary \ + "https://github.com/oliver006/redis_exporter/releases/download/v${REDIS_EXPORTER_VERSION}/redis_exporter-v${REDIS_EXPORTER_VERSION}.linux-arm64.tar.gz" \ + "$REDIS_EXPORTER_SHA256_ARM64" \ + "redis_exporter-v${REDIS_EXPORTER_VERSION}.linux-arm64/redis_exporter" \ + /usr/local/bin/prometheus-redis-exporter + install_release_binary \ + "https://github.com/nginx/nginx-prometheus-exporter/releases/download/v${NGINX_EXPORTER_VERSION}/nginx-prometheus-exporter_${NGINX_EXPORTER_VERSION}_linux_arm64.tar.gz" \ + "$NGINX_EXPORTER_SHA256_ARM64" \ + nginx-prometheus-exporter \ + /usr/local/bin/prometheus-nginx-exporter + + write_base_unit prometheus-postgres-exporter "Prometheus PostgreSQL Exporter" /usr/local/bin/prometheus-postgres-exporter + write_base_unit prometheus-redis-exporter "Prometheus Redis Exporter" /usr/local/bin/prometheus-redis-exporter + write_base_unit prometheus-nginx-exporter "Prometheus Nginx Exporter" /usr/local/bin/prometheus-nginx-exporter +} + backup_dir="/etc/prometheus/cmpp-backups/$(date '+%Y%m%d-%H%M%S')-service-exporters" mkdir -p "$backup_dir" @@ -12,9 +86,13 @@ for config_file in /etc/cmpp-platform/minio.env /etc/nginx/conf.d/cmpp-monitorin [[ -f "$config_file" ]] && cp --preserve=mode,timestamps "$config_file" "$backup_dir/$(basename "$config_file")" done -log "Installing bounded service exporters" -apt-get update -DEBIAN_FRONTEND=noninteractive apt-get install -y prometheus-postgres-exporter prometheus-redis-exporter prometheus-nginx-exporter +if command -v apt-get >/dev/null 2>&1; then + log "Installing bounded service exporter packages" + apt-get update + DEBIAN_FRONTEND=noninteractive apt-get install -y prometheus-postgres-exporter prometheus-redis-exporter prometheus-nginx-exporter +else + install_official_arm64_releases +fi [[ -f /etc/cmpp-platform/cmpp-platform.env ]] || { echo "Missing platform environment file." >&2; exit 1; } set -a @@ -26,26 +104,29 @@ cat >/etc/cmpp-platform/monitoring-exporters.env <"$directory/cmpp-monitoring.conf" </etc/nginx/conf.d/cmpp-monitoring-status.conf <<'EOF' server {