360 lines
22 KiB
YAML
360 lines
22 KiB
YAML
groups:
|
|
- name: cmpp-service-recording
|
|
interval: 15s
|
|
rules:
|
|
- record: cmpp:service_api:requests_per_second
|
|
expr: sum(rate(cmpp_api_http_requests_total[5m]))
|
|
- record: cmpp:service_api:error_percent
|
|
expr: 100 * sum(rate(cmpp_api_http_requests_total{status=~"5.."}[5m])) / clamp_min(sum(rate(cmpp_api_http_requests_total[5m])), 0.001)
|
|
- record: cmpp:service_api:latency_p95_seconds
|
|
expr: histogram_quantile(0.95, sum by (le) (rate(cmpp_api_http_request_duration_seconds_bucket[5m])))
|
|
- record: cmpp:service_api:event_loop_p99_seconds
|
|
expr: cmpp_api_nodejs_event_loop_lag_p99_seconds
|
|
- record: cmpp:service_gateway:submits_per_second
|
|
expr: sum(rate(cmpp_gateway_submit_total[5m]))
|
|
- record: cmpp:service_gateway:failure_percent
|
|
expr: 100 * sum(rate(cmpp_gateway_submit_total{result="failed"}[5m])) / clamp_min(sum(rate(cmpp_gateway_submit_total[5m])), 0.001)
|
|
- record: cmpp:service_gateway:queue_pending
|
|
expr: cmpp_gateway_submit_queue_pending
|
|
- record: cmpp:service_gateway:queue_lag
|
|
expr: cmpp_gateway_submit_queue_lag
|
|
- record: cmpp:service_gateway:queue_oldest_seconds
|
|
expr: cmpp_gateway_submit_queue_oldest_pending_age_seconds
|
|
- record: cmpp:service_postgresql:connection_percent
|
|
expr: 100 * sum(pg_stat_activity_count) / clamp_min(max(pg_settings_max_connections), 1)
|
|
- record: cmpp:service_postgresql:deadlocks_15m
|
|
expr: sum(increase(pg_stat_database_deadlocks[15m]))
|
|
- record: cmpp:service_redis:memory_percent
|
|
expr: (100 * redis_memory_used_bytes / redis_memory_max_bytes) and on(instance) (redis_memory_max_bytes > 0)
|
|
- record: cmpp:service_redis:memory_used_bytes
|
|
expr: redis_memory_used_bytes
|
|
- record: cmpp:service_redis:evictions_5m
|
|
expr: increase(redis_evicted_keys_total[5m])
|
|
- record: cmpp:service_redis:connected_clients
|
|
expr: redis_connected_clients
|
|
- record: cmpp:service_nginx:connections_active
|
|
expr: nginx_connections_active
|
|
- record: cmpp:service_nginx:requests_per_second
|
|
expr: rate(nginx_http_requests_total[5m])
|
|
- record: cmpp:service_minio:capacity_percent
|
|
expr: 100 * (1 - minio_cluster_capacity_usable_free_bytes / minio_cluster_capacity_usable_total_bytes)
|
|
- record: cmpp:service_minio:usage_bytes
|
|
expr: minio_cluster_usage_total_bytes
|
|
- record: cmpp:service_minio:objects
|
|
expr: minio_cluster_usage_object_total
|
|
- record: cmpp:service_minio:drives_offline
|
|
expr: minio_cluster_drive_offline_total
|
|
|
|
- name: cmpp-host-resources
|
|
rules:
|
|
- alert: NodeExporterDown
|
|
expr: up{job="node"} == 0
|
|
for: 2m
|
|
labels:
|
|
severity: critical
|
|
service: node-exporter
|
|
annotations:
|
|
summary: 主机指标采集不可用
|
|
description: Prometheus连续2分钟无法采集Node Exporter。
|
|
currentValue: "{{ $value }}"
|
|
threshold: "up = 1"
|
|
|
|
- alert: HostCpuUsageWarning
|
|
expr: (100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 80) and (100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) <= 90)
|
|
for: 10m
|
|
labels:
|
|
severity: warning
|
|
service: host
|
|
annotations:
|
|
summary: CPU使用率持续偏高
|
|
description: 主机CPU使用率连续10分钟高于80%。
|
|
currentValue: "{{ printf \"%.1f\" $value }}%"
|
|
threshold: "80%"
|
|
|
|
- alert: HostCpuUsageCritical
|
|
expr: 100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 90
|
|
for: 5m
|
|
labels:
|
|
severity: critical
|
|
service: host
|
|
annotations:
|
|
summary: CPU使用率严重超限
|
|
description: 主机CPU使用率连续5分钟高于90%。
|
|
currentValue: "{{ printf \"%.1f\" $value }}%"
|
|
threshold: "90%"
|
|
|
|
- alert: HostMemoryUsageWarning
|
|
expr: ((1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100 > 85) and ((1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100 <= 95)
|
|
for: 10m
|
|
labels:
|
|
severity: warning
|
|
service: host
|
|
annotations:
|
|
summary: 内存使用率持续偏高
|
|
description: 主机可用内存连续10分钟低于15%。
|
|
currentValue: "{{ printf \"%.1f\" $value }}%"
|
|
threshold: "85%"
|
|
|
|
- alert: HostMemoryUsageCritical
|
|
expr: (1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100 > 95
|
|
for: 5m
|
|
labels:
|
|
severity: critical
|
|
service: host
|
|
annotations:
|
|
summary: 内存使用率严重超限
|
|
description: 主机可用内存连续5分钟低于5%。
|
|
currentValue: "{{ printf \"%.1f\" $value }}%"
|
|
threshold: "95%"
|
|
|
|
- alert: HostRootDiskUsageWarning
|
|
expr: ((1 - node_filesystem_avail_bytes{device=~"/dev/.+",fstype!~"tmpfs|devtmpfs|overlay|squashfs|ramfs"} / node_filesystem_size_bytes{device=~"/dev/.+",fstype!~"tmpfs|devtmpfs|overlay|squashfs|ramfs"}) * 100 > 80) and ((1 - node_filesystem_avail_bytes{device=~"/dev/.+",fstype!~"tmpfs|devtmpfs|overlay|squashfs|ramfs"} / node_filesystem_size_bytes{device=~"/dev/.+",fstype!~"tmpfs|devtmpfs|overlay|squashfs|ramfs"}) * 100 <= 90)
|
|
for: 15m
|
|
labels:
|
|
severity: warning
|
|
service: host
|
|
annotations:
|
|
summary: 磁盘文件系统空间不足
|
|
description: "磁盘文件系统使用率连续15分钟高于80%。 挂载点：{{ $labels.mountpoint }}；设备：{{ $labels.device }}。"
|
|
currentValue: "{{ printf \"%.1f\" $value }}%"
|
|
threshold: "80%"
|
|
|
|
- alert: HostRootDiskUsageCritical
|
|
expr: (1 - node_filesystem_avail_bytes{device=~"/dev/.+",fstype!~"tmpfs|devtmpfs|overlay|squashfs|ramfs"} / node_filesystem_size_bytes{device=~"/dev/.+",fstype!~"tmpfs|devtmpfs|overlay|squashfs|ramfs"}) * 100 > 90
|
|
for: 5m
|
|
labels:
|
|
severity: critical
|
|
service: host
|
|
annotations:
|
|
summary: 磁盘文件系统空间严重不足
|
|
description: "磁盘文件系统使用率连续5分钟高于90%。 挂载点：{{ $labels.mountpoint }}；设备：{{ $labels.device }}。"
|
|
currentValue: "{{ printf \"%.1f\" $value }}%"
|
|
threshold: "90%"
|
|
|
|
- alert: HostRootInodeUsageWarning
|
|
expr: ((1 - node_filesystem_files_free{device=~"/dev/.+",fstype!~"tmpfs|devtmpfs|overlay|squashfs|ramfs"} / node_filesystem_files{device=~"/dev/.+",fstype!~"tmpfs|devtmpfs|overlay|squashfs|ramfs"}) * 100 > 80) and ((1 - node_filesystem_files_free{device=~"/dev/.+",fstype!~"tmpfs|devtmpfs|overlay|squashfs|ramfs"} / node_filesystem_files{device=~"/dev/.+",fstype!~"tmpfs|devtmpfs|overlay|squashfs|ramfs"}) * 100 <= 90)
|
|
for: 15m
|
|
labels:
|
|
severity: warning
|
|
service: host
|
|
annotations:
|
|
summary: 磁盘文件系统inode余量偏低
|
|
description: "磁盘文件系统inode使用率连续15分钟高于80%。 挂载点：{{ $labels.mountpoint }}；设备：{{ $labels.device }}。"
|
|
currentValue: "{{ printf \"%.1f\" $value }}%"
|
|
threshold: "80%"
|
|
|
|
- alert: HostRootInodeUsageCritical
|
|
expr: (1 - node_filesystem_files_free{device=~"/dev/.+",fstype!~"tmpfs|devtmpfs|overlay|squashfs|ramfs"} / node_filesystem_files{device=~"/dev/.+",fstype!~"tmpfs|devtmpfs|overlay|squashfs|ramfs"}) * 100 > 90
|
|
for: 5m
|
|
labels:
|
|
severity: critical
|
|
service: host
|
|
annotations:
|
|
summary: 磁盘文件系统inode严重不足
|
|
description: "磁盘文件系统inode使用率连续5分钟高于90%。 挂载点：{{ $labels.mountpoint }}；设备：{{ $labels.device }}。"
|
|
currentValue: "{{ printf \"%.1f\" $value }}%"
|
|
threshold: "90%"
|
|
|
|
- alert: HostCpuIowaitWarning
|
|
expr: (avg by (instance) (rate(node_cpu_seconds_total{mode="iowait"}[5m])) * 100 > 20) and (avg by (instance) (rate(node_cpu_seconds_total{mode="iowait"}[5m])) * 100 <= 35)
|
|
for: 10m
|
|
labels:
|
|
severity: warning
|
|
service: host
|
|
annotations:
|
|
summary: CPU iowait持续偏高
|
|
description: 主机CPU iowait连续10分钟高于20%，请检查磁盘I/O。
|
|
currentValue: "{{ printf \"%.1f\" $value }}%"
|
|
threshold: "20%"
|
|
|
|
- alert: HostCpuIowaitCritical
|
|
expr: avg by (instance) (rate(node_cpu_seconds_total{mode="iowait"}[5m])) * 100 > 35
|
|
for: 10m
|
|
labels:
|
|
severity: critical
|
|
service: host
|
|
annotations:
|
|
summary: CPU iowait严重超限
|
|
description: 主机CPU iowait连续10分钟高于35%，磁盘I/O可能已成为瓶颈。
|
|
currentValue: "{{ printf \"%.1f\" $value }}%"
|
|
threshold: "35%"
|
|
|
|
- name: cmpp-core-services
|
|
rules:
|
|
- alert: CmppCoreServiceInactive
|
|
expr: node_systemd_unit_state{name=~"cmpp-api\\.service|cmpp-send-worker\\.service|cmpp-gateway\\.service|postgresql\\.service|redis(-server)?\\.service|cmpp-minio\\.service|nginx\\.service",state="active"} == 0
|
|
for: 2m
|
|
labels:
|
|
severity: critical
|
|
service: "{{ $labels.name }}"
|
|
annotations:
|
|
summary: CMPP核心服务未处于active状态
|
|
description: "systemd服务 {{ $labels.name }} 连续2分钟未处于active状态。"
|
|
currentValue: "{{ $value }}"
|
|
threshold: "active = 1"
|
|
|
|
- name: cmpp-api-runtime
|
|
rules:
|
|
- alert: CmppApiMetricsDown
|
|
expr: up{job="cmpp-api"} == 0
|
|
for: 2m
|
|
labels: { severity: critical, service: api }
|
|
annotations: { summary: "API指标采集不可用", description: "Prometheus连续2分钟无法读取API内部指标端点。", currentValue: "{{ $value }}", threshold: "up = 1" }
|
|
- alert: CmppSendWorkerMetricsDown
|
|
expr: up{job="cmpp-send-worker"} == 0
|
|
for: 2m
|
|
labels: { severity: critical, service: send-worker }
|
|
annotations: { summary: "发送Worker指标采集不可用", description: "Prometheus连续2分钟无法读取独立发送Worker指标端点。", currentValue: "{{ $value }}", threshold: "up = 1" }
|
|
- alert: CmppInboundWorkflowBacklogCritical
|
|
expr: cmpp_worker_inbound_workflow_oldest_pending_age_seconds > 120
|
|
for: 2m
|
|
labels: { severity: critical, service: send-worker }
|
|
annotations: { summary: "CMPP耐久Inbox严重积压", description: "最旧待处理CMPP Inbox连续2分钟超过120秒。", currentValue: "{{ printf \"%.0f\" $value }}s", threshold: "120s" }
|
|
- alert: CmppApiHttpErrorRateWarning
|
|
expr: (sum(rate(cmpp_api_http_requests_total{status=~"5.."}[5m])) / clamp_min(sum(rate(cmpp_api_http_requests_total[5m])), 0.001) > 0.01) and (sum(rate(cmpp_api_http_requests_total{status=~"5.."}[5m])) / clamp_min(sum(rate(cmpp_api_http_requests_total[5m])), 0.001) <= 0.05) and sum(increase(cmpp_api_http_requests_total{status=~"5.."}[5m])) >= 5
|
|
for: 5m
|
|
labels: { severity: warning, service: api }
|
|
annotations: { summary: "API 5xx错误率偏高", description: "API 5xx错误率连续5分钟高于1%，且窗口内至少5次错误。", currentValue: "{{ printf \"%.2f\" $value }}", threshold: "1%" }
|
|
- alert: CmppApiHttpErrorRateCritical
|
|
expr: (sum(rate(cmpp_api_http_requests_total{status=~"5.."}[5m])) / clamp_min(sum(rate(cmpp_api_http_requests_total[5m])), 0.001) > 0.05) and sum(increase(cmpp_api_http_requests_total{status=~"5.."}[5m])) >= 5
|
|
for: 5m
|
|
labels: { severity: critical, service: api }
|
|
annotations: { summary: "API 5xx错误率严重超限", description: "API 5xx错误率连续5分钟高于5%。", currentValue: "{{ printf \"%.2f\" $value }}", threshold: "5%" }
|
|
- alert: CmppApiLatencyWarning
|
|
expr: (histogram_quantile(0.95, sum by (le) (rate(cmpp_api_http_request_duration_seconds_bucket[10m]))) > 1) and (histogram_quantile(0.95, sum by (le) (rate(cmpp_api_http_request_duration_seconds_bucket[10m]))) <= 3)
|
|
for: 10m
|
|
labels: { severity: warning, service: api }
|
|
annotations: { summary: "API P95响应偏慢", description: "API P95响应时间连续10分钟超过1秒。", currentValue: "{{ printf \"%.3f\" $value }}s", threshold: "1s" }
|
|
- alert: CmppApiLatencyCritical
|
|
expr: histogram_quantile(0.95, sum by (le) (rate(cmpp_api_http_request_duration_seconds_bucket[5m]))) > 3
|
|
for: 5m
|
|
labels: { severity: critical, service: api }
|
|
annotations: { summary: "API P95响应严重超时", description: "API P95响应时间连续5分钟超过3秒。", currentValue: "{{ printf \"%.3f\" $value }}s", threshold: "3s" }
|
|
- alert: CmppApiEventLoopLagWarning
|
|
expr: (cmpp_api_nodejs_event_loop_lag_p99_seconds > 0.2) and (cmpp_api_nodejs_event_loop_lag_p99_seconds <= 1)
|
|
for: 10m
|
|
labels: { severity: warning, service: api }
|
|
annotations: { summary: "API事件循环延迟偏高", description: "Node.js事件循环P99延迟连续10分钟超过200ms。", currentValue: "{{ printf \"%.3f\" $value }}s", threshold: "0.2s" }
|
|
- alert: CmppApiEventLoopLagCritical
|
|
expr: cmpp_api_nodejs_event_loop_lag_p99_seconds > 1
|
|
for: 5m
|
|
labels: { severity: critical, service: api }
|
|
annotations: { summary: "API事件循环严重阻塞", description: "Node.js事件循环P99延迟连续5分钟超过1秒。", currentValue: "{{ printf \"%.3f\" $value }}s", threshold: "1s" }
|
|
|
|
- name: cmpp-gateway-runtime
|
|
rules:
|
|
- alert: CmppGatewayMetricsDown
|
|
expr: up{job="cmpp-gateway"} == 0
|
|
for: 2m
|
|
labels: { severity: critical, service: gateway }
|
|
annotations: { summary: "Gateway指标采集不可用", description: "Prometheus连续2分钟无法读取Gateway指标。", currentValue: "{{ $value }}", threshold: "up = 1" }
|
|
- alert: CmppGatewaySubmitWorkerDown
|
|
expr: cmpp_gateway_submit_worker_up == 0
|
|
for: 1m
|
|
labels: { severity: critical, service: gateway }
|
|
annotations: { summary: "Gateway提交消费者未运行", description: "Gateway进程存活，但提交消费者未成功初始化。", currentValue: "{{ $value }}", threshold: "1" }
|
|
- alert: CmppGatewayUpstreamConnectionShortage
|
|
expr: cmpp_gateway_upstream_connections{state="connected"} < cmpp_gateway_upstream_connections{state="desired"}
|
|
for: 2m
|
|
labels: { severity: critical, service: gateway }
|
|
annotations: { summary: "Gateway上游连接不足", description: "实际上游CMPP连接数连续2分钟低于期望数。", currentValue: "{{ $value }}", threshold: "connected = desired" }
|
|
- alert: CmppGatewayQueueDelayedWarning
|
|
expr: (cmpp_gateway_submit_queue_oldest_pending_age_seconds > 30) and (cmpp_gateway_submit_queue_oldest_pending_age_seconds <= 120)
|
|
for: 2m
|
|
labels: { severity: warning, service: gateway }
|
|
annotations: { summary: "Gateway提交队列开始延迟", description: "Redis Stream最旧pending消息等待超过30秒。", currentValue: "{{ printf \"%.0f\" $value }}s", threshold: "30s" }
|
|
- alert: CmppGatewayQueueDelayedCritical
|
|
expr: cmpp_gateway_submit_queue_oldest_pending_age_seconds > 120
|
|
for: 2m
|
|
labels: { severity: critical, service: gateway }
|
|
annotations: { summary: "Gateway提交队列严重延迟", description: "Redis Stream最旧pending消息等待超过120秒。", currentValue: "{{ printf \"%.0f\" $value }}s", threshold: "120s" }
|
|
|
|
- name: cmpp-data-services
|
|
rules:
|
|
- alert: PostgresExporterDown
|
|
expr: up{job="postgresql"} == 0 or pg_up == 0
|
|
for: 2m
|
|
labels: { severity: critical, service: postgresql }
|
|
annotations: { summary: "PostgreSQL指标或数据库不可用", description: "PostgreSQL Exporter或其数据库连接连续2分钟不可用。", currentValue: "{{ $value }}", threshold: "up = 1" }
|
|
- alert: PostgresConnectionsWarning
|
|
expr: (sum(pg_stat_activity_count) / clamp_min(max(pg_settings_max_connections), 1) > 0.70) and (sum(pg_stat_activity_count) / clamp_min(max(pg_settings_max_connections), 1) <= 0.85)
|
|
for: 10m
|
|
labels: { severity: warning, service: postgresql }
|
|
annotations: { summary: "PostgreSQL连接使用率偏高", description: "数据库连接数连续10分钟超过上限的70%。", currentValue: "{{ printf \"%.2f\" $value }}", threshold: "70%" }
|
|
- alert: PostgresConnectionsCritical
|
|
expr: sum(pg_stat_activity_count) / clamp_min(max(pg_settings_max_connections), 1) > 0.85
|
|
for: 5m
|
|
labels: { severity: critical, service: postgresql }
|
|
annotations: { summary: "PostgreSQL连接即将耗尽", description: "数据库连接数连续5分钟超过上限的85%。", currentValue: "{{ printf \"%.2f\" $value }}", threshold: "85%" }
|
|
- alert: PostgresDeadlocksDetected
|
|
expr: sum(increase(pg_stat_database_deadlocks[15m])) > 0
|
|
for: 1m
|
|
labels: { severity: warning, service: postgresql }
|
|
annotations: { summary: "PostgreSQL发生死锁", description: "15分钟窗口内检测到数据库死锁。", currentValue: "{{ $value }}", threshold: "0" }
|
|
- alert: RedisExporterDown
|
|
expr: up{job="redis"} == 0 or redis_up == 0
|
|
for: 2m
|
|
labels: { severity: critical, service: redis }
|
|
annotations: { summary: "Redis指标或服务不可用", description: "Redis Exporter或Redis连接连续2分钟不可用。", currentValue: "{{ $value }}", threshold: "up = 1" }
|
|
- alert: RedisMemoryWarning
|
|
expr: (redis_memory_max_bytes > 0) and (redis_memory_used_bytes / redis_memory_max_bytes > 0.70) and (redis_memory_used_bytes / redis_memory_max_bytes <= 0.85)
|
|
for: 10m
|
|
labels: { severity: warning, service: redis }
|
|
annotations: { summary: "Redis内存使用率偏高", description: "Redis内存连续10分钟超过maxmemory的70%。", currentValue: "{{ printf \"%.2f\" $value }}", threshold: "70%" }
|
|
- alert: RedisMemoryCritical
|
|
expr: (redis_memory_max_bytes > 0) and (redis_memory_used_bytes / redis_memory_max_bytes > 0.85)
|
|
for: 5m
|
|
labels: { severity: critical, service: redis }
|
|
annotations: { summary: "Redis内存即将耗尽", description: "Redis内存连续5分钟超过maxmemory的85%。", currentValue: "{{ printf \"%.2f\" $value }}", threshold: "85%" }
|
|
- alert: RedisUnexpectedEvictions
|
|
expr: increase(redis_evicted_keys_total[5m]) > 0
|
|
for: 1m
|
|
labels: { severity: critical, service: redis }
|
|
annotations: { summary: "Redis发生Key淘汰", description: "Redis承载队列和运行状态，5分钟内不应出现淘汰。", currentValue: "{{ $value }}", threshold: "0" }
|
|
- alert: RedisRejectedConnections
|
|
expr: increase(redis_rejected_connections_total[5m]) > 0
|
|
for: 1m
|
|
labels: { severity: critical, service: redis }
|
|
annotations: { summary: "Redis拒绝连接", description: "5分钟内Redis出现被拒绝连接。", currentValue: "{{ $value }}", threshold: "0" }
|
|
|
|
- name: cmpp-storage-and-edge
|
|
rules:
|
|
- alert: MinioMetricsDown
|
|
expr: up{job="minio"} == 0
|
|
for: 2m
|
|
labels: { severity: critical, service: minio }
|
|
annotations: { summary: "MinIO指标采集不可用", description: "Prometheus连续2分钟无法读取MinIO原生指标。", currentValue: "{{ $value }}", threshold: "up = 1" }
|
|
- alert: MinioCapacityWarning
|
|
expr: (100 * (1 - minio_cluster_capacity_usable_free_bytes / minio_cluster_capacity_usable_total_bytes) > 80) and (100 * (1 - minio_cluster_capacity_usable_free_bytes / minio_cluster_capacity_usable_total_bytes) <= 90)
|
|
for: 15m
|
|
labels: { severity: warning, service: minio }
|
|
annotations: { summary: "MinIO存储容量偏高", description: "MinIO可用容量使用率连续15分钟超过80%。", currentValue: "{{ printf \"%.1f\" $value }}%", threshold: "80%" }
|
|
- alert: MinioCapacityCritical
|
|
expr: 100 * (1 - minio_cluster_capacity_usable_free_bytes / minio_cluster_capacity_usable_total_bytes) > 90
|
|
for: 5m
|
|
labels: { severity: critical, service: minio }
|
|
annotations: { summary: "MinIO存储容量即将耗尽", description: "MinIO可用容量使用率连续5分钟超过90%。", currentValue: "{{ printf \"%.1f\" $value }}%", threshold: "90%" }
|
|
- alert: MinioDriveOffline
|
|
expr: minio_cluster_drive_offline_total > 0
|
|
for: 1m
|
|
labels: { severity: critical, service: minio }
|
|
annotations: { summary: "MinIO存储盘离线", description: "MinIO检测到离线存储盘。", currentValue: "{{ $value }}", threshold: "0" }
|
|
- alert: NginxExporterDown
|
|
expr: up{job="nginx"} == 0 or nginx_up == 0
|
|
for: 2m
|
|
labels: { severity: critical, service: nginx }
|
|
annotations: { summary: "Nginx指标或状态页不可用", description: "Nginx Exporter或回环stub_status连续2分钟不可用。", currentValue: "{{ $value }}", threshold: "up = 1" }
|
|
|
|
- name: cmpp-monitoring-self
|
|
rules:
|
|
- alert: PrometheusScrapeSlow
|
|
expr: scrape_duration_seconds / scrape_interval_seconds > 0.8
|
|
for: 5m
|
|
labels: { severity: warning, service: prometheus }
|
|
annotations: { summary: "Prometheus采集接近超时", description: "采集耗时连续5分钟超过采集周期的80%。", currentValue: "{{ printf \"%.2f\" $value }}", threshold: "80%" }
|
|
- alert: PrometheusRuleEvaluationFailures
|
|
expr: increase(prometheus_rule_evaluation_failures_total[5m]) > 0
|
|
for: 1m
|
|
labels: { severity: critical, service: prometheus }
|
|
annotations: { summary: "Prometheus告警规则计算失败", description: "5分钟内出现告警规则计算失败。", currentValue: "{{ $value }}", threshold: "0" }
|