174 lines
7.4 KiB
Bash
174 lines
7.4 KiB
Bash
#!/usr/bin/env bash
|
|
set -Eeuo pipefail
|
|
|
|
POSTGRES_EXPORTER_VERSION="0.20.1"
|
|
POSTGRES_EXPORTER_SHA256_ARM64="d5d86fb98bb1f26b088d1a6fda07fd6b6f035cb5d40492f75ec3bfebb5ddfe9d"
|
|
REDIS_EXPORTER_VERSION="1.89.0"
|
|
REDIS_EXPORTER_SHA256_ARM64="1a70d42cd1d96a9388f8b528f55ad5bdc2c3cbd911c1dff3875fcb9ac01fa28a"
|
|
NGINX_EXPORTER_VERSION="1.5.3"
|
|
NGINX_EXPORTER_SHA256_ARM64="a0b1a5f5bba09483bd2e04c759c1a75fffe46ca72c0314ee2bb925d1742ff23b"
|
|
|
|
if [[ "$(id -u)" -ne 0 ]]; then echo "Run as root." >&2; exit 1; fi
|
|
for command_name in systemctl nginx curl install ss; do command -v "$command_name" >/dev/null || { echo "Missing command: $command_name" >&2; exit 1; }; done
|
|
|
|
log() { printf '\n[%s] %s\n' "$(date '+%F %T')" "$*"; }
|
|
|
|
install_release_binary() {
|
|
local url="$1" expected_sha256="$2" archive_member="$3" destination="$4"
|
|
local work_dir archive
|
|
work_dir="$(mktemp -d /tmp/cmpp-monitoring.XXXXXX)"
|
|
archive="$work_dir/release.tar.gz"
|
|
curl -fL --retry 5 --retry-delay 2 --connect-timeout 15 -o "$archive" "$url"
|
|
printf '%s %s\n' "$expected_sha256" "$archive" | sha256sum -c -
|
|
tar -xzf "$archive" -C "$work_dir"
|
|
install -o root -g root -m 0755 "$work_dir/$archive_member" "$destination"
|
|
rm -rf -- "$work_dir"
|
|
}
|
|
|
|
write_base_unit() {
|
|
local unit="$1" description="$2" executable="$3"
|
|
cat >"/etc/systemd/system/${unit}.service" <<EOF
|
|
[Unit]
|
|
Description=${description}
|
|
Wants=network-online.target
|
|
After=network-online.target
|
|
|
|
[Service]
|
|
Type=simple
|
|
User=prometheus
|
|
Group=prometheus
|
|
Restart=on-failure
|
|
RestartSec=5s
|
|
ExecStart=${executable}
|
|
|
|
[Install]
|
|
WantedBy=multi-user.target
|
|
EOF
|
|
}
|
|
|
|
install_official_arm64_releases() {
|
|
[[ "$(uname -m)" == "aarch64" ]] || {
|
|
echo "Official-binary installation currently supports Linux aarch64 only." >&2
|
|
exit 1
|
|
}
|
|
for command_name in sha256sum tar; do
|
|
command -v "$command_name" >/dev/null || { echo "Missing command: $command_name" >&2; exit 1; }
|
|
done
|
|
getent group prometheus >/dev/null || groupadd --system prometheus
|
|
id prometheus >/dev/null 2>&1 || useradd --system --gid prometheus --home-dir /var/lib/prometheus --shell /sbin/nologin prometheus
|
|
|
|
log "Installing checksum-pinned official service exporter ARM64 releases"
|
|
install_release_binary \
|
|
"https://github.com/prometheus-community/postgres_exporter/releases/download/v${POSTGRES_EXPORTER_VERSION}/postgres_exporter-${POSTGRES_EXPORTER_VERSION}.linux-arm64.tar.gz" \
|
|
"$POSTGRES_EXPORTER_SHA256_ARM64" \
|
|
"postgres_exporter-${POSTGRES_EXPORTER_VERSION}.linux-arm64/postgres_exporter" \
|
|
/usr/local/bin/prometheus-postgres-exporter
|
|
install_release_binary \
|
|
"https://github.com/oliver006/redis_exporter/releases/download/v${REDIS_EXPORTER_VERSION}/redis_exporter-v${REDIS_EXPORTER_VERSION}.linux-arm64.tar.gz" \
|
|
"$REDIS_EXPORTER_SHA256_ARM64" \
|
|
"redis_exporter-v${REDIS_EXPORTER_VERSION}.linux-arm64/redis_exporter" \
|
|
/usr/local/bin/prometheus-redis-exporter
|
|
install_release_binary \
|
|
"https://github.com/nginx/nginx-prometheus-exporter/releases/download/v${NGINX_EXPORTER_VERSION}/nginx-prometheus-exporter_${NGINX_EXPORTER_VERSION}_linux_arm64.tar.gz" \
|
|
"$NGINX_EXPORTER_SHA256_ARM64" \
|
|
nginx-prometheus-exporter \
|
|
/usr/local/bin/prometheus-nginx-exporter
|
|
|
|
write_base_unit prometheus-postgres-exporter "Prometheus PostgreSQL Exporter" /usr/local/bin/prometheus-postgres-exporter
|
|
write_base_unit prometheus-redis-exporter "Prometheus Redis Exporter" /usr/local/bin/prometheus-redis-exporter
|
|
write_base_unit prometheus-nginx-exporter "Prometheus Nginx Exporter" /usr/local/bin/prometheus-nginx-exporter
|
|
}
|
|
|
|
backup_dir="/etc/prometheus/cmpp-backups/$(date '+%Y%m%d-%H%M%S')-service-exporters"
|
|
mkdir -p "$backup_dir"
|
|
|
|
for config_file in /etc/cmpp-platform/minio.env /etc/nginx/conf.d/cmpp-monitoring-status.conf /etc/cmpp-platform/monitoring-exporters.env; do
|
|
[[ -f "$config_file" ]] && cp --preserve=mode,timestamps "$config_file" "$backup_dir/$(basename "$config_file")"
|
|
done
|
|
|
|
if command -v apt-get >/dev/null 2>&1; then
|
|
log "Installing bounded service exporter packages"
|
|
apt-get update
|
|
DEBIAN_FRONTEND=noninteractive apt-get install -y prometheus-postgres-exporter prometheus-redis-exporter prometheus-nginx-exporter
|
|
else
|
|
install_official_arm64_releases
|
|
fi
|
|
|
|
[[ -f /etc/cmpp-platform/cmpp-platform.env ]] || { echo "Missing platform environment file." >&2; exit 1; }
|
|
set -a
|
|
# shellcheck disable=SC1091
|
|
. /etc/cmpp-platform/cmpp-platform.env
|
|
set +a
|
|
database_base="${DATABASE_URL%%\?*}"
|
|
cat >/etc/cmpp-platform/monitoring-exporters.env <<EOF
|
|
DATA_SOURCE_NAME=${database_base}?sslmode=disable
|
|
REDIS_ADDR=redis://${REDIS_HOST:-127.0.0.1}:${REDIS_PORT:-6379}
|
|
EOF
|
|
chown root:prometheus /etc/cmpp-platform/monitoring-exporters.env
|
|
chmod 0640 /etc/cmpp-platform/monitoring-exporters.env
|
|
|
|
write_override() {
|
|
local unit="$1" executable="$2" arguments="$3" environment_file="${4:-}"
|
|
local directory="/etc/systemd/system/${unit}.service.d"
|
|
mkdir -p "$directory"
|
|
[[ -f "$directory/cmpp-monitoring.conf" ]] && cp --preserve=mode,timestamps "$directory/cmpp-monitoring.conf" "$backup_dir/${unit}-override.conf"
|
|
cat >"$directory/cmpp-monitoring.conf" <<EOF
|
|
[Service]
|
|
ExecStart=
|
|
ExecStart=${executable} ${arguments}
|
|
EOF
|
|
if [[ -n "$environment_file" ]]; then
|
|
sed -i "/^\[Service\]$/a EnvironmentFile=${environment_file}" "$directory/cmpp-monitoring.conf"
|
|
fi
|
|
}
|
|
|
|
postgres_exporter="$(command -v prometheus-postgres-exporter)"
|
|
redis_exporter="$(command -v prometheus-redis-exporter)"
|
|
nginx_exporter="$(command -v prometheus-nginx-exporter)"
|
|
write_override prometheus-postgres-exporter "$postgres_exporter" '--web.listen-address=127.0.0.1:9187' /etc/cmpp-platform/monitoring-exporters.env
|
|
write_override prometheus-redis-exporter "$redis_exporter" '--web.listen-address=127.0.0.1:9121' /etc/cmpp-platform/monitoring-exporters.env
|
|
|
|
cat >/etc/nginx/conf.d/cmpp-monitoring-status.conf <<'EOF'
|
|
server {
|
|
listen 127.0.0.1:8088;
|
|
server_name localhost;
|
|
access_log off;
|
|
location = /stub_status {
|
|
stub_status;
|
|
allow 127.0.0.1;
|
|
allow ::1;
|
|
deny all;
|
|
}
|
|
}
|
|
EOF
|
|
write_override prometheus-nginx-exporter "$nginx_exporter" '--web.listen-address=127.0.0.1:9113 --nginx.scrape-uri=http://127.0.0.1:8088/stub_status'
|
|
|
|
# MinIO exposes only operational aggregates and listens on loopback; public auth here does not expose objects or credentials.
|
|
grep -q '^MINIO_PROMETHEUS_AUTH_TYPE=' /etc/cmpp-platform/minio.env \
|
|
&& sed -i 's/^MINIO_PROMETHEUS_AUTH_TYPE=.*/MINIO_PROMETHEUS_AUTH_TYPE=public/' /etc/cmpp-platform/minio.env \
|
|
|| printf '\nMINIO_PROMETHEUS_AUTH_TYPE=public\n' >>/etc/cmpp-platform/minio.env
|
|
|
|
nginx -t
|
|
systemctl daemon-reload
|
|
systemctl enable prometheus-postgres-exporter prometheus-redis-exporter prometheus-nginx-exporter
|
|
systemctl restart prometheus-postgres-exporter prometheus-redis-exporter prometheus-nginx-exporter
|
|
systemctl restart cmpp-minio
|
|
systemctl reload nginx
|
|
|
|
wait_for_http() {
|
|
local endpoint="$1" attempt
|
|
for attempt in $(seq 1 30); do
|
|
curl -fsS "$endpoint" >/dev/null 2>&1 && return 0
|
|
sleep 1
|
|
done
|
|
echo "Monitoring endpoint did not become ready: $endpoint" >&2
|
|
return 1
|
|
}
|
|
for endpoint in 127.0.0.1:9187 127.0.0.1:9121 127.0.0.1:9113; do wait_for_http "http://${endpoint}/metrics"; done
|
|
wait_for_http http://127.0.0.1:9000/minio/v2/metrics/cluster
|
|
if ss -lnt | grep -Eq '(^|[[:space:]])(0\.0\.0\.0|\[::\]):(9187|9121|9113)([[:space:]]|$)'; then
|
|
echo "A service exporter unexpectedly listens on a wildcard address." >&2
|
|
exit 1
|
|
fi
|
|
log "Service exporters are ready on loopback only; backup: $backup_dir"
|