hectorzhao/lislgosms
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add configurable infrastructure alerts

Browse Source
This commit is contained in:
hectorzhao
2026-08-14 17:51:42 +08:00
parent 1ef4380422
commit 6ccc102830
23 changed files with 506 additions and 39 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/monitoring/install-prometheus-monitoring.sh
+20 -3
View File
@@ -26,7 +26,7 @@ promtool_bin="$(command -v promtool)"
backup_dir="/etc/prometheus/cmpp-backups/$(date '+%Y%m%d-%H%M%S')"
mkdir -p "$backup_dir" /etc/systemd/system/prometheus.service.d /etc/systemd/system/prometheus-node-exporter.service.d
for config_file in /etc/prometheus/prometheus.yml /etc/prometheus/cmpp-alerts.yml; do
for config_file in /etc/prometheus/prometheus.yml /etc/prometheus/cmpp-alerts.yml /etc/prometheus/cmpp-alerts-source.yml; do
if [[ -f "$config_file" ]]; then
cp --preserve=mode,timestamps "$config_file" "$backup_dir/$(basename "$config_file")"
fi
@@ -40,12 +40,28 @@ fi
log "Installing platform-owned scrape and alert configuration"
install -o root -g root -m 0644 "$SCRIPT_DIR/prometheus.yml" /etc/prometheus/prometheus.yml
install -o root -g root -m 0644 "$SCRIPT_DIR/cmpp-alerts.yml" /etc/prometheus/cmpp-alerts.yml
install -o root -g root -m 0644 "$SCRIPT_DIR/cmpp-alerts.yml" /etc/prometheus/cmpp-alerts-source.yml
# 可配置规则由 API 管理；基础规则必须排除同名项，否则 Prometheus 会同时计算旧阈值和新阈值。
awk '
BEGIN {
split("HostCpuUsageWarning HostCpuUsageCritical HostMemoryUsageWarning HostMemoryUsageCritical HostRootDiskUsageWarning HostRootDiskUsageCritical CmppApiHttpErrorRateWarning CmppApiHttpErrorRateCritical CmppApiLatencyWarning CmppApiLatencyCritical CmppApiEventLoopLagWarning CmppApiEventLoopLagCritical CmppGatewayQueueDelayedWarning CmppGatewayQueueDelayedCritical PostgresConnectionsWarning PostgresConnectionsCritical RedisMemoryWarning RedisMemoryCritical MinioCapacityWarning MinioCapacityCritical", names, " ")
for (i in names) dropped[names[i]] = 1
}
/^ - name:/ { skip = 0 }
/^ - alert:/ { skip = ($3 in dropped) }
!skip { print }
' "$SCRIPT_DIR/cmpp-alerts.yml" > /etc/prometheus/cmpp-alerts.yml
chown root:root /etc/prometheus/cmpp-alerts.yml
chmod 0644 /etc/prometheus/cmpp-alerts.yml
install -d -o cmpp-api -g prometheus -m 0750 /var/lib/cmpp-platform/monitoring
if [[ ! -f /var/lib/cmpp-platform/monitoring/cmpp-managed-alerts.yml ]]; then
install -o cmpp-api -g prometheus -m 0640 "$SCRIPT_DIR/cmpp-managed-alerts.yml" /var/lib/cmpp-platform/monitoring/cmpp-managed-alerts.yml
fi
cat >/etc/systemd/system/prometheus.service.d/cmpp-monitoring.conf <<EOF
[Service]
ExecStart=
ExecStart=${prometheus_bin} --config.file=/etc/prometheus/prometheus.yml --storage.tsdb.path=/var/lib/prometheus/metrics2 --storage.tsdb.retention.time=${PROMETHEUS_RETENTION_TIME} --storage.tsdb.retention.size=${PROMETHEUS_RETENTION_SIZE} --web.listen-address=127.0.0.1:9090
ExecStart=${prometheus_bin} --config.file=/etc/prometheus/prometheus.yml --storage.tsdb.path=/var/lib/prometheus/metrics2 --storage.tsdb.retention.time=${PROMETHEUS_RETENTION_TIME} --storage.tsdb.retention.size=${PROMETHEUS_RETENTION_SIZE} --web.listen-address=127.0.0.1:9090 --web.enable-lifecycle
EOF
cat >/etc/systemd/system/prometheus-node-exporter.service.d/cmpp-monitoring.conf <<EOF
@@ -56,6 +72,7 @@ EOF
log "Validating Prometheus configuration before restart"
"$promtool_bin" check rules /etc/prometheus/cmpp-alerts.yml
"$promtool_bin" check rules /var/lib/cmpp-platform/monitoring/cmpp-managed-alerts.yml
"$promtool_bin" check config /etc/prometheus/prometheus.yml
systemctl daemon-reload