hectorzhao/lislgosms
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

fix: support preproduction monitoring on arm64

Browse Source
This commit is contained in:
hectorzhao
2026-08-26 14:25:31 +08:00
parent b7ae1aa41a
commit b0bdb608c1
4 changed files with 200 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/monitoring/README.md
+2 -2
View File
@@ -2,7 +2,7 @@
本目录提供系统监控第一版所需的真实采集配置。运营端不嵌入Prometheus或Grafana页面；NestJS仅从本机Prometheus读取固定指标，再由平台原生UI展示。
在目标Debian/Ubuntu测试服务器上，以root执行：
在目标 Debian/Ubuntu（apt）或 Alibaba Cloud Linux 4 ARM64 测试服务器上，以 root 执行：
```bash
cd /opt/cmpp-platform
@@ -10,7 +10,7 @@ bash tools/monitoring/install-prometheus-monitoring.sh
bash tools/monitoring/install-service-exporters.sh
```
第一个脚本安装Prometheus与Node Exporter，备份已有配置并校验规则；第二个脚本安装PostgreSQL、Redis和Nginx Exporter，开启MinIO回环原生指标。API指标仅监听`127.0.0.1:9464`，Gateway指标复用回环控制端口`8090`。9090、9100、9187、9121、9113和9464均不得对公网开放。两个脚本均会打印恢复资产路径。
Debian/Ubuntu 使用系统包；Alibaba Cloud Linux 4 ARM64 使用脚本中锁定版本和 SHA-256 的官方 release 二进制。第一个脚本安装 Prometheus 与 Node Exporter，备份已有配置并校验规则；第二个脚本安装 PostgreSQL、Redis 和 Nginx Exporter，开启 MinIO 回环原生指标。API 指标仅监听 `127.0.0.1:9464`，Gateway 指标复用回环控制端口 `8090`。9090、9100、9187、9121、9113 和 9464 均不得对公网开放。两个脚本均会打印恢复资产路径。
安装后将下列配置写入`/etc/cmpp-platform/cmpp-platform.env`，再按正常发布窗口重启API：
tools/monitoring/install-prometheus-monitoring.sh
+107 -10
View File
@@ -4,27 +4,113 @@ set -Eeuo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROMETHEUS_RETENTION_TIME="${PROMETHEUS_RETENTION_TIME:-30d}"
PROMETHEUS_RETENTION_SIZE="${PROMETHEUS_RETENTION_SIZE:-8GB}"
PROMETHEUS_VERSION="3.14.0"
PROMETHEUS_SHA256_ARM64="077f3781ab7245dc04c9a3c9b78ba120fc8e41aa0dc97489b0af67247e50ba83"
NODE_EXPORTER_VERSION="1.12.1"
NODE_EXPORTER_SHA256_ARM64="ad35b605f9954b9f1ffddf5ba054bdc5a98d790b9eae5291e1eeb83f1ecbd0e7"
if [[ "$(id -u)" -ne 0 ]]; then
echo "Run as root." >&2
exit 1
fi
if ! command -v apt-get >/dev/null 2>&1; then
echo "This installer currently supports Debian/Ubuntu apt packages only." >&2
exit 1
fi
log() { printf '\n[%s] %s\n' "$(date '+%F %T')" "$*"; }
log "Installing Prometheus and Node Exporter packages"
apt-get update
DEBIAN_FRONTEND=noninteractive apt-get install -y prometheus prometheus-node-exporter curl iproute2
install_release_binary() {
local url="$1" expected_sha256="$2" archive_member="$3" destination="$4"
local work_dir archive
work_dir="$(mktemp -d /tmp/cmpp-monitoring.XXXXXX)"
archive="$work_dir/release.tar.gz"
curl -fL --retry 5 --retry-delay 2 --connect-timeout 15 -o "$archive" "$url"
printf '%s %s\n' "$expected_sha256" "$archive" | sha256sum -c -
tar -xzf "$archive" -C "$work_dir"
install -o root -g root -m 0755 "$work_dir/$archive_member" "$destination"
rm -rf -- "$work_dir"
}
install_official_arm64_releases() {
[[ "$(uname -m)" == "aarch64" ]] || {
echo "Official-binary installation currently supports Linux aarch64 only." >&2
exit 1
}
for command_name in curl sha256sum tar; do
command -v "$command_name" >/dev/null || { echo "Missing command: $command_name" >&2; exit 1; }
done
log "Installing checksum-pinned official Prometheus ARM64 releases"
install_release_binary \
"https://github.com/prometheus/prometheus/releases/download/v${PROMETHEUS_VERSION}/prometheus-${PROMETHEUS_VERSION}.linux-arm64.tar.gz" \
"$PROMETHEUS_SHA256_ARM64" \
"prometheus-${PROMETHEUS_VERSION}.linux-arm64/prometheus" \
/usr/local/bin/prometheus
install_release_binary \
"https://github.com/prometheus/prometheus/releases/download/v${PROMETHEUS_VERSION}/prometheus-${PROMETHEUS_VERSION}.linux-arm64.tar.gz" \
"$PROMETHEUS_SHA256_ARM64" \
"prometheus-${PROMETHEUS_VERSION}.linux-arm64/promtool" \
/usr/local/bin/promtool
install_release_binary \
"https://github.com/prometheus/node_exporter/releases/download/v${NODE_EXPORTER_VERSION}/node_exporter-${NODE_EXPORTER_VERSION}.linux-arm64.tar.gz" \
"$NODE_EXPORTER_SHA256_ARM64" \
"node_exporter-${NODE_EXPORTER_VERSION}.linux-arm64/node_exporter" \
/usr/local/bin/prometheus-node-exporter
getent group prometheus >/dev/null || groupadd --system prometheus
id prometheus >/dev/null 2>&1 || useradd --system --gid prometheus --home-dir /var/lib/prometheus --shell /sbin/nologin prometheus
install -d -o root -g root -m 0755 /etc/prometheus
install -d -o prometheus -g prometheus -m 0750 /var/lib/prometheus/metrics2
cat >/etc/systemd/system/prometheus.service <<'EOF'
[Unit]
Description=Prometheus monitoring server
Wants=network-online.target
After=network-online.target
[Service]
Type=simple
User=prometheus
Group=prometheus
Restart=on-failure
RestartSec=5s
ExecStart=/usr/local/bin/prometheus
[Install]
WantedBy=multi-user.target
EOF
cat >/etc/systemd/system/prometheus-node-exporter.service <<'EOF'
[Unit]
Description=Prometheus Node Exporter
Wants=network-online.target
After=network-online.target
[Service]
Type=simple
User=prometheus
Group=prometheus
Restart=on-failure
RestartSec=5s
ExecStart=/usr/local/bin/prometheus-node-exporter
[Install]
WantedBy=multi-user.target
EOF
}
for command_name in systemctl install awk curl ss; do
command -v "$command_name" >/dev/null || { echo "Missing command: $command_name" >&2; exit 1; }
done
if command -v apt-get >/dev/null 2>&1; then
log "Installing Prometheus and Node Exporter packages"
apt-get update
DEBIAN_FRONTEND=noninteractive apt-get install -y prometheus prometheus-node-exporter curl iproute2
else
install_official_arm64_releases
fi
prometheus_bin="$(command -v prometheus)"
node_exporter_bin="$(command -v prometheus-node-exporter)"
promtool_bin="$(command -v promtool)"
backup_dir="/etc/prometheus/cmpp-backups/$(date '+%Y%m%d-%H%M%S')"
mkdir -p "$backup_dir" /etc/systemd/system/prometheus.service.d /etc/systemd/system/prometheus-node-exporter.service.d
mkdir -p "$backup_dir" /etc/systemd/system/prometheus.service.d /etc/systemd/system/prometheus-node-exporter.service.d /etc/systemd/system/cmpp-api.service.d
for config_file in /etc/prometheus/prometheus.yml /etc/prometheus/cmpp-alerts.yml /etc/prometheus/cmpp-alerts-source.yml; do
if [[ -f "$config_file" ]]; then
@@ -37,6 +123,9 @@ fi
if [[ -f /etc/systemd/system/prometheus-node-exporter.service.d/cmpp-monitoring.conf ]]; then
cp --preserve=mode,timestamps /etc/systemd/system/prometheus-node-exporter.service.d/cmpp-monitoring.conf "$backup_dir/node-exporter-service-override.conf"
fi
if [[ -f /etc/systemd/system/cmpp-api.service.d/monitoring-storage.conf ]]; then
cp --preserve=mode,timestamps /etc/systemd/system/cmpp-api.service.d/monitoring-storage.conf "$backup_dir/cmpp-api-monitoring-storage.conf"
fi
log "Installing platform-owned scrape and alert configuration"
install -o root -g root -m 0644 "$SCRIPT_DIR/prometheus.yml" /etc/prometheus/prometheus.yml
@@ -58,6 +147,14 @@ install -d -o cmpp-api -g prometheus -m 2750 /var/lib/cmpp-platform/monitoring
if [[ ! -f /var/lib/cmpp-platform/monitoring/cmpp-managed-alerts.yml ]]; then
install -o cmpp-api -g prometheus -m 0640 "$SCRIPT_DIR/cmpp-managed-alerts.yml" /var/lib/cmpp-platform/monitoring/cmpp-managed-alerts.yml
fi
chown cmpp-api:prometheus /var/lib/cmpp-platform/monitoring/cmpp-managed-alerts.yml
chmod 0640 /var/lib/cmpp-platform/monitoring/cmpp-managed-alerts.yml
# The API writes managed rules atomically. Its hardened unit otherwise sees /var/lib as read-only.
cat >/etc/systemd/system/cmpp-api.service.d/monitoring-storage.conf <<'EOF'
[Service]
ReadWritePaths=/var/lib/cmpp-platform/monitoring
EOF
cat >/etc/systemd/system/prometheus.service.d/cmpp-monitoring.conf <<EOF
[Service]
@@ -97,4 +194,4 @@ fi
log "Prometheus monitoring is ready on loopback only"
echo "Configuration backup: $backup_dir"
echo "The CMPP API and Gateway were not restarted."
echo "The CMPP API and Gateway were not restarted. Restart the API after setting PROMETHEUS_URL."
tools/monitoring/install-service-exporters.sh
+90 -9
View File
@@ -1,10 +1,84 @@
#!/usr/bin/env bash
set -Eeuo pipefail
POSTGRES_EXPORTER_VERSION="0.20.1"
POSTGRES_EXPORTER_SHA256_ARM64="d5d86fb98bb1f26b088d1a6fda07fd6b6f035cb5d40492f75ec3bfebb5ddfe9d"
REDIS_EXPORTER_VERSION="1.89.0"
REDIS_EXPORTER_SHA256_ARM64="1a70d42cd1d96a9388f8b528f55ad5bdc2c3cbd911c1dff3875fcb9ac01fa28a"
NGINX_EXPORTER_VERSION="1.5.3"
NGINX_EXPORTER_SHA256_ARM64="a0b1a5f5bba09483bd2e04c759c1a75fffe46ca72c0314ee2bb925d1742ff23b"
if [[ "$(id -u)" -ne 0 ]]; then echo "Run as root." >&2; exit 1; fi
for command_name in apt-get systemctl nginx curl; do command -v "$command_name" >/dev/null || { echo "Missing command: $command_name" >&2; exit 1; }; done
for command_name in systemctl nginx curl install ss; do command -v "$command_name" >/dev/null || { echo "Missing command: $command_name" >&2; exit 1; }; done
log() { printf '\n[%s] %s\n' "$(date '+%F %T')" "$*"; }
install_release_binary() {
local url="$1" expected_sha256="$2" archive_member="$3" destination="$4"
local work_dir archive
work_dir="$(mktemp -d /tmp/cmpp-monitoring.XXXXXX)"
archive="$work_dir/release.tar.gz"
curl -fL --retry 5 --retry-delay 2 --connect-timeout 15 -o "$archive" "$url"
printf '%s %s\n' "$expected_sha256" "$archive" | sha256sum -c -
tar -xzf "$archive" -C "$work_dir"
install -o root -g root -m 0755 "$work_dir/$archive_member" "$destination"
rm -rf -- "$work_dir"
}
write_base_unit() {
local unit="$1" description="$2" executable="$3"
cat >"/etc/systemd/system/${unit}.service" <<EOF
[Unit]
Description=${description}
Wants=network-online.target
After=network-online.target
[Service]
Type=simple
User=prometheus
Group=prometheus
Restart=on-failure
RestartSec=5s
ExecStart=${executable}
[Install]
WantedBy=multi-user.target
EOF
}
install_official_arm64_releases() {
[[ "$(uname -m)" == "aarch64" ]] || {
echo "Official-binary installation currently supports Linux aarch64 only." >&2
exit 1
}
for command_name in sha256sum tar; do
command -v "$command_name" >/dev/null || { echo "Missing command: $command_name" >&2; exit 1; }
done
getent group prometheus >/dev/null || groupadd --system prometheus
id prometheus >/dev/null 2>&1 || useradd --system --gid prometheus --home-dir /var/lib/prometheus --shell /sbin/nologin prometheus
log "Installing checksum-pinned official service exporter ARM64 releases"
install_release_binary \
"https://github.com/prometheus-community/postgres_exporter/releases/download/v${POSTGRES_EXPORTER_VERSION}/postgres_exporter-${POSTGRES_EXPORTER_VERSION}.linux-arm64.tar.gz" \
"$POSTGRES_EXPORTER_SHA256_ARM64" \
"postgres_exporter-${POSTGRES_EXPORTER_VERSION}.linux-arm64/postgres_exporter" \
/usr/local/bin/prometheus-postgres-exporter
install_release_binary \
"https://github.com/oliver006/redis_exporter/releases/download/v${REDIS_EXPORTER_VERSION}/redis_exporter-v${REDIS_EXPORTER_VERSION}.linux-arm64.tar.gz" \
"$REDIS_EXPORTER_SHA256_ARM64" \
"redis_exporter-v${REDIS_EXPORTER_VERSION}.linux-arm64/redis_exporter" \
/usr/local/bin/prometheus-redis-exporter
install_release_binary \
"https://github.com/nginx/nginx-prometheus-exporter/releases/download/v${NGINX_EXPORTER_VERSION}/nginx-prometheus-exporter_${NGINX_EXPORTER_VERSION}_linux_arm64.tar.gz" \
"$NGINX_EXPORTER_SHA256_ARM64" \
nginx-prometheus-exporter \
/usr/local/bin/prometheus-nginx-exporter
write_base_unit prometheus-postgres-exporter "Prometheus PostgreSQL Exporter" /usr/local/bin/prometheus-postgres-exporter
write_base_unit prometheus-redis-exporter "Prometheus Redis Exporter" /usr/local/bin/prometheus-redis-exporter
write_base_unit prometheus-nginx-exporter "Prometheus Nginx Exporter" /usr/local/bin/prometheus-nginx-exporter
}
backup_dir="/etc/prometheus/cmpp-backups/$(date '+%Y%m%d-%H%M%S')-service-exporters"
mkdir -p "$backup_dir"
@@ -12,9 +86,13 @@ for config_file in /etc/cmpp-platform/minio.env /etc/nginx/conf.d/cmpp-monitorin
[[ -f "$config_file" ]] && cp --preserve=mode,timestamps "$config_file" "$backup_dir/$(basename "$config_file")"
done
log "Installing bounded service exporters"
apt-get update
DEBIAN_FRONTEND=noninteractive apt-get install -y prometheus-postgres-exporter prometheus-redis-exporter prometheus-nginx-exporter
if command -v apt-get >/dev/null 2>&1; then
log "Installing bounded service exporter packages"
apt-get update
DEBIAN_FRONTEND=noninteractive apt-get install -y prometheus-postgres-exporter prometheus-redis-exporter prometheus-nginx-exporter
else
install_official_arm64_releases
fi
[[ -f /etc/cmpp-platform/cmpp-platform.env ]] || { echo "Missing platform environment file." >&2; exit 1; }
set -a
@@ -26,26 +104,29 @@ cat >/etc/cmpp-platform/monitoring-exporters.env <<EOF
DATA_SOURCE_NAME=${database_base}?sslmode=disable
REDIS_ADDR=redis://${REDIS_HOST:-127.0.0.1}:${REDIS_PORT:-6379}
EOF
chmod 0600 /etc/cmpp-platform/monitoring-exporters.env
chown root:prometheus /etc/cmpp-platform/monitoring-exporters.env
chmod 0640 /etc/cmpp-platform/monitoring-exporters.env
write_override() {
local unit="$1" executable="$2" arguments="$3"
local unit="$1" executable="$2" arguments="$3" environment_file="${4:-}"
local directory="/etc/systemd/system/${unit}.service.d"
mkdir -p "$directory"
[[ -f "$directory/cmpp-monitoring.conf" ]] && cp --preserve=mode,timestamps "$directory/cmpp-monitoring.conf" "$backup_dir/${unit}-override.conf"
cat >"$directory/cmpp-monitoring.conf" <<EOF
[Service]
EnvironmentFile=/etc/cmpp-platform/monitoring-exporters.env
ExecStart=
ExecStart=${executable} ${arguments}
EOF
if [[ -n "$environment_file" ]]; then
sed -i "/^\[Service\]$/a EnvironmentFile=${environment_file}" "$directory/cmpp-monitoring.conf"
fi
}
postgres_exporter="$(command -v prometheus-postgres-exporter)"
redis_exporter="$(command -v prometheus-redis-exporter)"
nginx_exporter="$(command -v prometheus-nginx-exporter)"
write_override prometheus-postgres-exporter "$postgres_exporter" '--web.listen-address=127.0.0.1:9187'
write_override prometheus-redis-exporter "$redis_exporter" '--web.listen-address=127.0.0.1:9121'
write_override prometheus-postgres-exporter "$postgres_exporter" '--web.listen-address=127.0.0.1:9187' /etc/cmpp-platform/monitoring-exporters.env
write_override prometheus-redis-exporter "$redis_exporter" '--web.listen-address=127.0.0.1:9121' /etc/cmpp-platform/monitoring-exporters.env
cat >/etc/nginx/conf.d/cmpp-monitoring-status.conf <<'EOF'
server {