feat: add configurable infrastructure alerts

tools/monitoring/cmpp-managed-alerts.yml

@@ -0,0 +1,103 @@
+groups:
+  - name: cmpp-managed-thresholds
+    rules:
+      - alert: HostCpuUsageWarning
+        expr: (100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 80) and (100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) <= 90)
+        for: 10m
+        labels: { severity: warning, service: host }
+        annotations: { summary: "主机 CPU 使用率达到警告阈值", description: "主机 CPU 使用率持续超过80%。", currentValue: "{{ $value }}", threshold: "80%" }
+      - alert: HostCpuUsageCritical
+        expr: 100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 90
+        for: 5m
+        labels: { severity: critical, service: host }
+        annotations: { summary: "主机 CPU 使用率达到严重阈值", description: "主机 CPU 使用率持续超过90%。", currentValue: "{{ $value }}", threshold: "90%" }
+      - alert: HostMemoryUsageWarning
+        expr: ((1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100 > 85) and ((1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100 <= 95)
+        for: 10m
+        labels: { severity: warning, service: host }
+        annotations: { summary: "主机内存使用率达到警告阈值", description: "主机内存使用率持续超过85%。", currentValue: "{{ $value }}", threshold: "85%" }
+      - alert: HostMemoryUsageCritical
+        expr: (1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100 > 95
+        for: 5m
+        labels: { severity: critical, service: host }
+        annotations: { summary: "主机内存使用率达到严重阈值", description: "主机内存使用率持续超过95%。", currentValue: "{{ $value }}", threshold: "95%" }
+      - alert: HostRootDiskUsageWarning
+        expr: ((1 - node_filesystem_avail_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"} / node_filesystem_size_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"}) * 100 > 80) and ((1 - node_filesystem_avail_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"} / node_filesystem_size_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"}) * 100 <= 90)
+        for: 15m
+        labels: { severity: warning, service: host }
+        annotations: { summary: "根磁盘使用率达到警告阈值", description: "根磁盘使用率持续超过80%。", currentValue: "{{ $value }}", threshold: "80%" }
+      - alert: HostRootDiskUsageCritical
+        expr: (1 - node_filesystem_avail_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"} / node_filesystem_size_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"}) * 100 > 90
+        for: 5m
+        labels: { severity: critical, service: host }
+        annotations: { summary: "根磁盘使用率达到严重阈值", description: "根磁盘使用率持续超过90%。", currentValue: "{{ $value }}", threshold: "90%" }
+      - alert: CmppApiHttpErrorRateWarning
+        expr: (100 * sum(rate(cmpp_api_http_requests_total{status=~"5.."}[5m])) / clamp_min(sum(rate(cmpp_api_http_requests_total[5m])), 0.001) > 1) and (100 * sum(rate(cmpp_api_http_requests_total{status=~"5.."}[5m])) / clamp_min(sum(rate(cmpp_api_http_requests_total[5m])), 0.001) <= 5)
+        for: 5m
+        labels: { severity: warning, service: api }
+        annotations: { summary: "API 5xx 错误率达到警告阈值", description: "API 5xx 错误率持续超过1%。", currentValue: "{{ $value }}", threshold: "1%" }
+      - alert: CmppApiHttpErrorRateCritical
+        expr: 100 * sum(rate(cmpp_api_http_requests_total{status=~"5.."}[5m])) / clamp_min(sum(rate(cmpp_api_http_requests_total[5m])), 0.001) > 5
+        for: 5m
+        labels: { severity: critical, service: api }
+        annotations: { summary: "API 5xx 错误率达到严重阈值", description: "API 5xx 错误率持续超过5%。", currentValue: "{{ $value }}", threshold: "5%" }
+      - alert: CmppApiLatencyWarning
+        expr: (histogram_quantile(0.95, sum by (le) (rate(cmpp_api_http_request_duration_seconds_bucket[10m]))) > 1) and (histogram_quantile(0.95, sum by (le) (rate(cmpp_api_http_request_duration_seconds_bucket[10m]))) <= 3)
+        for: 10m
+        labels: { severity: warning, service: api }
+        annotations: { summary: "API P95 响应时间达到警告阈值", description: "API P95 响应时间持续超过1秒。", currentValue: "{{ $value }}", threshold: "1秒" }
+      - alert: CmppApiLatencyCritical
+        expr: histogram_quantile(0.95, sum by (le) (rate(cmpp_api_http_request_duration_seconds_bucket[10m]))) > 3
+        for: 5m
+        labels: { severity: critical, service: api }
+        annotations: { summary: "API P95 响应时间达到严重阈值", description: "API P95 响应时间持续超过3秒。", currentValue: "{{ $value }}", threshold: "3秒" }
+      - alert: CmppApiEventLoopLagWarning
+        expr: (cmpp_api_nodejs_event_loop_lag_p99_seconds > 0.2) and (cmpp_api_nodejs_event_loop_lag_p99_seconds <= 1)
+        for: 10m
+        labels: { severity: warning, service: api }
+        annotations: { summary: "API 事件循环 P99 达到警告阈值", description: "API 事件循环 P99 持续超过0.2秒。", currentValue: "{{ $value }}", threshold: "0.2秒" }
+      - alert: CmppApiEventLoopLagCritical
+        expr: cmpp_api_nodejs_event_loop_lag_p99_seconds > 1
+        for: 5m
+        labels: { severity: critical, service: api }
+        annotations: { summary: "API 事件循环 P99 达到严重阈值", description: "API 事件循环 P99 持续超过1秒。", currentValue: "{{ $value }}", threshold: "1秒" }
+      - alert: CmppGatewayQueueDelayedWarning
+        expr: (cmpp_gateway_submit_queue_oldest_pending_age_seconds > 30) and (cmpp_gateway_submit_queue_oldest_pending_age_seconds <= 120)
+        for: 2m
+        labels: { severity: warning, service: gateway }
+        annotations: { summary: "Gateway 最旧 pending 达到警告阈值", description: "Gateway 最旧 pending 持续超过30秒。", currentValue: "{{ $value }}", threshold: "30秒" }
+      - alert: CmppGatewayQueueDelayedCritical
+        expr: cmpp_gateway_submit_queue_oldest_pending_age_seconds > 120
+        for: 2m
+        labels: { severity: critical, service: gateway }
+        annotations: { summary: "Gateway 最旧 pending 达到严重阈值", description: "Gateway 最旧 pending 持续超过120秒。", currentValue: "{{ $value }}", threshold: "120秒" }
+      - alert: PostgresConnectionsWarning
+        expr: (100 * sum(pg_stat_activity_count) / clamp_min(max(pg_settings_max_connections), 1) > 70) and (100 * sum(pg_stat_activity_count) / clamp_min(max(pg_settings_max_connections), 1) <= 85)
+        for: 10m
+        labels: { severity: warning, service: postgresql }
+        annotations: { summary: "PostgreSQL 连接使用率达到警告阈值", description: "PostgreSQL 连接使用率持续超过70%。", currentValue: "{{ $value }}", threshold: "70%" }
+      - alert: PostgresConnectionsCritical
+        expr: 100 * sum(pg_stat_activity_count) / clamp_min(max(pg_settings_max_connections), 1) > 85
+        for: 5m
+        labels: { severity: critical, service: postgresql }
+        annotations: { summary: "PostgreSQL 连接使用率达到严重阈值", description: "PostgreSQL 连接使用率持续超过85%。", currentValue: "{{ $value }}", threshold: "85%" }
+      - alert: RedisMemoryWarning
+        expr: (100 * redis_memory_used_bytes / redis_memory_max_bytes > 70) and (100 * redis_memory_used_bytes / redis_memory_max_bytes <= 85)
+        for: 10m
+        labels: { severity: warning, service: redis }
+        annotations: { summary: "Redis 内存使用率达到警告阈值", description: "Redis 内存使用率持续超过70%。", currentValue: "{{ $value }}", threshold: "70%" }
+      - alert: RedisMemoryCritical
+        expr: 100 * redis_memory_used_bytes / redis_memory_max_bytes > 85
+        for: 5m
+        labels: { severity: critical, service: redis }
+        annotations: { summary: "Redis 内存使用率达到严重阈值", description: "Redis 内存使用率持续超过85%。", currentValue: "{{ $value }}", threshold: "85%" }
+      - alert: MinioCapacityWarning
+        expr: (100 * (1 - minio_cluster_capacity_usable_free_bytes / minio_cluster_capacity_usable_total_bytes) > 80) and (100 * (1 - minio_cluster_capacity_usable_free_bytes / minio_cluster_capacity_usable_total_bytes) <= 90)
+        for: 15m
+        labels: { severity: warning, service: minio }
+        annotations: { summary: "MinIO 容量使用率达到警告阈值", description: "MinIO 容量使用率持续超过80%。", currentValue: "{{ $value }}", threshold: "80%" }
+      - alert: MinioCapacityCritical
+        expr: 100 * (1 - minio_cluster_capacity_usable_free_bytes / minio_cluster_capacity_usable_total_bytes) > 90
+        for: 5m
+        labels: { severity: critical, service: minio }
+        annotations: { summary: "MinIO 容量使用率达到严重阈值", description: "MinIO 容量使用率持续超过90%。", currentValue: "{{ $value }}", threshold: "90%" }

tools/monitoring/install-prometheus-monitoring.sh

@@ -26,7 +26,7 @@ promtool_bin="$(command -v promtool)"
 backup_dir="/etc/prometheus/cmpp-backups/$(date '+%Y%m%d-%H%M%S')"
 mkdir -p "$backup_dir" /etc/systemd/system/prometheus.service.d /etc/systemd/system/prometheus-node-exporter.service.d
 
-for config_file in /etc/prometheus/prometheus.yml /etc/prometheus/cmpp-alerts.yml; do
+for config_file in /etc/prometheus/prometheus.yml /etc/prometheus/cmpp-alerts.yml /etc/prometheus/cmpp-alerts-source.yml; do
   if [[ -f "$config_file" ]]; then
     cp --preserve=mode,timestamps "$config_file" "$backup_dir/$(basename "$config_file")"
   fi
@@ -40,12 +40,28 @@ fi
 
 log "Installing platform-owned scrape and alert configuration"
 install -o root -g root -m 0644 "$SCRIPT_DIR/prometheus.yml" /etc/prometheus/prometheus.yml
-install -o root -g root -m 0644 "$SCRIPT_DIR/cmpp-alerts.yml" /etc/prometheus/cmpp-alerts.yml
+install -o root -g root -m 0644 "$SCRIPT_DIR/cmpp-alerts.yml" /etc/prometheus/cmpp-alerts-source.yml
+# 可配置规则由 API 管理；基础规则必须排除同名项，否则 Prometheus 会同时计算旧阈值和新阈值。
+awk '
+BEGIN {
+  split("HostCpuUsageWarning HostCpuUsageCritical HostMemoryUsageWarning HostMemoryUsageCritical HostRootDiskUsageWarning HostRootDiskUsageCritical CmppApiHttpErrorRateWarning CmppApiHttpErrorRateCritical CmppApiLatencyWarning CmppApiLatencyCritical CmppApiEventLoopLagWarning CmppApiEventLoopLagCritical CmppGatewayQueueDelayedWarning CmppGatewayQueueDelayedCritical PostgresConnectionsWarning PostgresConnectionsCritical RedisMemoryWarning RedisMemoryCritical MinioCapacityWarning MinioCapacityCritical", names, " ")
+  for (i in names) dropped[names[i]] = 1
+}
+/^  - name:/ { skip = 0 }
+/^      - alert:/ { skip = ($3 in dropped) }
+!skip { print }
+' "$SCRIPT_DIR/cmpp-alerts.yml" > /etc/prometheus/cmpp-alerts.yml
+chown root:root /etc/prometheus/cmpp-alerts.yml
+chmod 0644 /etc/prometheus/cmpp-alerts.yml
+install -d -o cmpp-api -g prometheus -m 0750 /var/lib/cmpp-platform/monitoring
+if [[ ! -f /var/lib/cmpp-platform/monitoring/cmpp-managed-alerts.yml ]]; then
+  install -o cmpp-api -g prometheus -m 0640 "$SCRIPT_DIR/cmpp-managed-alerts.yml" /var/lib/cmpp-platform/monitoring/cmpp-managed-alerts.yml
+fi
 
 cat >/etc/systemd/system/prometheus.service.d/cmpp-monitoring.conf <<EOF
 [Service]
 ExecStart=
-ExecStart=${prometheus_bin} --config.file=/etc/prometheus/prometheus.yml --storage.tsdb.path=/var/lib/prometheus/metrics2 --storage.tsdb.retention.time=${PROMETHEUS_RETENTION_TIME} --storage.tsdb.retention.size=${PROMETHEUS_RETENTION_SIZE} --web.listen-address=127.0.0.1:9090
+ExecStart=${prometheus_bin} --config.file=/etc/prometheus/prometheus.yml --storage.tsdb.path=/var/lib/prometheus/metrics2 --storage.tsdb.retention.time=${PROMETHEUS_RETENTION_TIME} --storage.tsdb.retention.size=${PROMETHEUS_RETENTION_SIZE} --web.listen-address=127.0.0.1:9090 --web.enable-lifecycle
 EOF
 
 cat >/etc/systemd/system/prometheus-node-exporter.service.d/cmpp-monitoring.conf <<EOF
@@ -56,6 +72,7 @@ EOF
 
 log "Validating Prometheus configuration before restart"
 "$promtool_bin" check rules /etc/prometheus/cmpp-alerts.yml
+"$promtool_bin" check rules /var/lib/cmpp-platform/monitoring/cmpp-managed-alerts.yml
 "$promtool_bin" check config /etc/prometheus/prometheus.yml
 
 systemctl daemon-reload

tools/monitoring/prometheus.yml

@@ -7,6 +7,7 @@ global:
 
 rule_files:
   - /etc/prometheus/cmpp-alerts.yml
+  - /var/lib/cmpp-platform/monitoring/*.yml
 
 scrape_configs:
   - job_name: prometheus