204 lines
9.2 KiB
Bash
204 lines
9.2 KiB
Bash
#!/usr/bin/env bash
|
|
set -Eeuo pipefail
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
PROMETHEUS_RETENTION_TIME="${PROMETHEUS_RETENTION_TIME:-30d}"
|
|
PROMETHEUS_RETENTION_SIZE="${PROMETHEUS_RETENTION_SIZE:-8GB}"
|
|
PROMETHEUS_VERSION="3.14.0"
|
|
PROMETHEUS_SHA256_ARM64="077f3781ab7245dc04c9a3c9b78ba120fc8e41aa0dc97489b0af67247e50ba83"
|
|
NODE_EXPORTER_VERSION="1.12.1"
|
|
NODE_EXPORTER_SHA256_ARM64="ad35b605f9954b9f1ffddf5ba054bdc5a98d790b9eae5291e1eeb83f1ecbd0e7"
|
|
MONITORING_RELEASE_CACHE_DIR="${MONITORING_RELEASE_CACHE_DIR:-}"
|
|
|
|
if [[ "$(id -u)" -ne 0 ]]; then
|
|
echo "Run as root." >&2
|
|
exit 1
|
|
fi
|
|
log() { printf '\n[%s] %s\n' "$(date '+%F %T')" "$*"; }
|
|
|
|
install_release_binary() {
|
|
local url="$1" expected_sha256="$2" archive_member="$3" destination="$4"
|
|
local work_dir archive asset_name
|
|
work_dir="$(mktemp -d /tmp/cmpp-monitoring.XXXXXX)"
|
|
archive="$work_dir/release.tar.gz"
|
|
asset_name="${url##*/}"
|
|
if [[ -n "$MONITORING_RELEASE_CACHE_DIR" && -f "$MONITORING_RELEASE_CACHE_DIR/$asset_name" ]]; then
|
|
cp "$MONITORING_RELEASE_CACHE_DIR/$asset_name" "$archive"
|
|
else
|
|
curl -fL --retry 5 --retry-delay 2 --connect-timeout 15 -o "$archive" "$url"
|
|
fi
|
|
printf '%s %s\n' "$expected_sha256" "$archive" | sha256sum -c -
|
|
tar -xzf "$archive" -C "$work_dir"
|
|
install -o root -g root -m 0755 "$work_dir/$archive_member" "$destination"
|
|
rm -rf -- "$work_dir"
|
|
}
|
|
|
|
install_official_arm64_releases() {
|
|
[[ "$(uname -m)" == "aarch64" ]] || {
|
|
echo "Official-binary installation currently supports Linux aarch64 only." >&2
|
|
exit 1
|
|
}
|
|
for command_name in curl sha256sum tar; do
|
|
command -v "$command_name" >/dev/null || { echo "Missing command: $command_name" >&2; exit 1; }
|
|
done
|
|
|
|
log "Installing checksum-pinned official Prometheus ARM64 releases"
|
|
install_release_binary \
|
|
"https://github.com/prometheus/prometheus/releases/download/v${PROMETHEUS_VERSION}/prometheus-${PROMETHEUS_VERSION}.linux-arm64.tar.gz" \
|
|
"$PROMETHEUS_SHA256_ARM64" \
|
|
"prometheus-${PROMETHEUS_VERSION}.linux-arm64/prometheus" \
|
|
/usr/local/bin/prometheus
|
|
install_release_binary \
|
|
"https://github.com/prometheus/prometheus/releases/download/v${PROMETHEUS_VERSION}/prometheus-${PROMETHEUS_VERSION}.linux-arm64.tar.gz" \
|
|
"$PROMETHEUS_SHA256_ARM64" \
|
|
"prometheus-${PROMETHEUS_VERSION}.linux-arm64/promtool" \
|
|
/usr/local/bin/promtool
|
|
install_release_binary \
|
|
"https://github.com/prometheus/node_exporter/releases/download/v${NODE_EXPORTER_VERSION}/node_exporter-${NODE_EXPORTER_VERSION}.linux-arm64.tar.gz" \
|
|
"$NODE_EXPORTER_SHA256_ARM64" \
|
|
"node_exporter-${NODE_EXPORTER_VERSION}.linux-arm64/node_exporter" \
|
|
/usr/local/bin/prometheus-node-exporter
|
|
|
|
getent group prometheus >/dev/null || groupadd --system prometheus
|
|
id prometheus >/dev/null 2>&1 || useradd --system --gid prometheus --home-dir /var/lib/prometheus --shell /sbin/nologin prometheus
|
|
install -d -o root -g root -m 0755 /etc/prometheus
|
|
install -d -o prometheus -g prometheus -m 0750 /var/lib/prometheus/metrics2
|
|
|
|
cat >/etc/systemd/system/prometheus.service <<'EOF'
|
|
[Unit]
|
|
Description=Prometheus monitoring server
|
|
Wants=network-online.target
|
|
After=network-online.target
|
|
|
|
[Service]
|
|
Type=simple
|
|
User=prometheus
|
|
Group=prometheus
|
|
Restart=on-failure
|
|
RestartSec=5s
|
|
ExecStart=/usr/local/bin/prometheus
|
|
|
|
[Install]
|
|
WantedBy=multi-user.target
|
|
EOF
|
|
cat >/etc/systemd/system/prometheus-node-exporter.service <<'EOF'
|
|
[Unit]
|
|
Description=Prometheus Node Exporter
|
|
Wants=network-online.target
|
|
After=network-online.target
|
|
|
|
[Service]
|
|
Type=simple
|
|
User=prometheus
|
|
Group=prometheus
|
|
Restart=on-failure
|
|
RestartSec=5s
|
|
ExecStart=/usr/local/bin/prometheus-node-exporter
|
|
|
|
[Install]
|
|
WantedBy=multi-user.target
|
|
EOF
|
|
}
|
|
|
|
for command_name in systemctl install awk curl ss; do
|
|
command -v "$command_name" >/dev/null || { echo "Missing command: $command_name" >&2; exit 1; }
|
|
done
|
|
|
|
if command -v apt-get >/dev/null 2>&1; then
|
|
log "Installing Prometheus and Node Exporter packages"
|
|
apt-get update
|
|
DEBIAN_FRONTEND=noninteractive apt-get install -y prometheus prometheus-node-exporter curl iproute2
|
|
else
|
|
install_official_arm64_releases
|
|
fi
|
|
|
|
prometheus_bin="$(command -v prometheus)"
|
|
node_exporter_bin="$(command -v prometheus-node-exporter)"
|
|
promtool_bin="$(command -v promtool)"
|
|
backup_dir="/etc/prometheus/cmpp-backups/$(date '+%Y%m%d-%H%M%S')"
|
|
mkdir -p "$backup_dir" /etc/systemd/system/prometheus.service.d /etc/systemd/system/prometheus-node-exporter.service.d /etc/systemd/system/cmpp-api.service.d
|
|
|
|
for config_file in /etc/prometheus/prometheus.yml /etc/prometheus/cmpp-alerts.yml /etc/prometheus/cmpp-alerts-source.yml; do
|
|
if [[ -f "$config_file" ]]; then
|
|
cp --preserve=mode,timestamps "$config_file" "$backup_dir/$(basename "$config_file")"
|
|
fi
|
|
done
|
|
if [[ -f /etc/systemd/system/prometheus.service.d/cmpp-monitoring.conf ]]; then
|
|
cp --preserve=mode,timestamps /etc/systemd/system/prometheus.service.d/cmpp-monitoring.conf "$backup_dir/prometheus-service-override.conf"
|
|
fi
|
|
if [[ -f /etc/systemd/system/prometheus-node-exporter.service.d/cmpp-monitoring.conf ]]; then
|
|
cp --preserve=mode,timestamps /etc/systemd/system/prometheus-node-exporter.service.d/cmpp-monitoring.conf "$backup_dir/node-exporter-service-override.conf"
|
|
fi
|
|
if [[ -f /etc/systemd/system/cmpp-api.service.d/monitoring-storage.conf ]]; then
|
|
cp --preserve=mode,timestamps /etc/systemd/system/cmpp-api.service.d/monitoring-storage.conf "$backup_dir/cmpp-api-monitoring-storage.conf"
|
|
fi
|
|
|
|
log "Installing platform-owned scrape and alert configuration"
|
|
install -o root -g root -m 0644 "$SCRIPT_DIR/prometheus.yml" /etc/prometheus/prometheus.yml
|
|
install -o root -g root -m 0644 "$SCRIPT_DIR/cmpp-alerts.yml" /etc/prometheus/cmpp-alerts-source.yml
|
|
# 可配置规则由 API 管理；基础规则必须排除同名项，否则 Prometheus 会同时计算旧阈值和新阈值。
|
|
awk '
|
|
BEGIN {
|
|
split("HostCpuUsageWarning HostCpuUsageCritical HostMemoryUsageWarning HostMemoryUsageCritical HostRootDiskUsageWarning HostRootDiskUsageCritical CmppApiHttpErrorRateWarning CmppApiHttpErrorRateCritical CmppApiLatencyWarning CmppApiLatencyCritical CmppApiEventLoopLagWarning CmppApiEventLoopLagCritical CmppGatewayQueueDelayedWarning CmppGatewayQueueDelayedCritical PostgresConnectionsWarning PostgresConnectionsCritical RedisMemoryWarning RedisMemoryCritical MinioCapacityWarning MinioCapacityCritical", names, " ")
|
|
for (i in names) dropped[names[i]] = 1
|
|
}
|
|
/^ - name:/ { skip = 0 }
|
|
/^ - alert:/ { skip = ($3 in dropped) }
|
|
!skip { print }
|
|
' "$SCRIPT_DIR/cmpp-alerts.yml" > /etc/prometheus/cmpp-alerts.yml
|
|
chown root:root /etc/prometheus/cmpp-alerts.yml
|
|
chmod 0644 /etc/prometheus/cmpp-alerts.yml
|
|
# SGID ensures API原子rename生成的新规则继续继承prometheus组，否则reload会因不可读返回500。
|
|
install -d -o cmpp-api -g prometheus -m 2750 /var/lib/cmpp-platform/monitoring
|
|
if [[ ! -f /var/lib/cmpp-platform/monitoring/cmpp-managed-alerts.yml ]]; then
|
|
install -o cmpp-api -g prometheus -m 0640 "$SCRIPT_DIR/cmpp-managed-alerts.yml" /var/lib/cmpp-platform/monitoring/cmpp-managed-alerts.yml
|
|
fi
|
|
chown cmpp-api:prometheus /var/lib/cmpp-platform/monitoring/cmpp-managed-alerts.yml
|
|
chmod 0640 /var/lib/cmpp-platform/monitoring/cmpp-managed-alerts.yml
|
|
|
|
# The API writes managed rules atomically. Its hardened unit otherwise sees /var/lib as read-only.
|
|
cat >/etc/systemd/system/cmpp-api.service.d/monitoring-storage.conf <<'EOF'
|
|
[Service]
|
|
ReadWritePaths=/var/lib/cmpp-platform/monitoring
|
|
EOF
|
|
|
|
cat >/etc/systemd/system/prometheus.service.d/cmpp-monitoring.conf <<EOF
|
|
[Service]
|
|
ExecStart=
|
|
ExecStart=${prometheus_bin} --config.file=/etc/prometheus/prometheus.yml --storage.tsdb.path=/var/lib/prometheus/metrics2 --storage.tsdb.retention.time=${PROMETHEUS_RETENTION_TIME} --storage.tsdb.retention.size=${PROMETHEUS_RETENTION_SIZE} --web.listen-address=127.0.0.1:9090 --web.enable-lifecycle
|
|
EOF
|
|
|
|
cat >/etc/systemd/system/prometheus-node-exporter.service.d/cmpp-monitoring.conf <<EOF
|
|
[Service]
|
|
ExecStart=
|
|
ExecStart=${node_exporter_bin} --web.listen-address=127.0.0.1:9100 --collector.systemd --collector.systemd.unit-include='cmpp-api\\\\.service|cmpp-send-worker\\\\.service|cmpp-gateway\\\\.service|postgresql\\\\.service|redis(-server)?\\\\.service|cmpp-minio\\\\.service|nginx\\\\.service' --collector.filesystem.mount-points-exclude='^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/)'
|
|
EOF
|
|
|
|
log "Validating Prometheus configuration before restart"
|
|
"$promtool_bin" check rules /etc/prometheus/cmpp-alerts.yml
|
|
"$promtool_bin" check rules /var/lib/cmpp-platform/monitoring/cmpp-managed-alerts.yml
|
|
"$promtool_bin" check config /etc/prometheus/prometheus.yml
|
|
|
|
systemctl daemon-reload
|
|
systemctl enable prometheus prometheus-node-exporter
|
|
systemctl restart prometheus-node-exporter
|
|
systemctl restart prometheus
|
|
|
|
for _ in $(seq 1 30); do
|
|
if curl -fsS http://127.0.0.1:9090/-/ready >/dev/null; then
|
|
break
|
|
fi
|
|
sleep 1
|
|
done
|
|
curl -fsS http://127.0.0.1:9090/-/ready >/dev/null
|
|
curl -fsS http://127.0.0.1:9100/metrics >/dev/null
|
|
|
|
if ss -lnt | grep -Eq '(^|[[:space:]])(0\.0\.0\.0|\[::\]):(9090|9100)([[:space:]]|$)'; then
|
|
echo "Prometheus monitoring ports unexpectedly listen on a wildcard address." >&2
|
|
exit 1
|
|
fi
|
|
|
|
log "Prometheus monitoring is ready on loopback only"
|
|
echo "Configuration backup: $backup_dir"
|
|
echo "The CMPP API and Gateway were not restarted. Restart the API after setting PROMETHEUS_URL."
|