groups: - name: cmpp-host-resources rules: - alert: NodeExporterDown expr: up{job="node"} == 0 for: 2m labels: severity: critical service: node-exporter annotations: summary: 主机指标采集不可用 description: Prometheus连续2分钟无法采集Node Exporter。 currentValue: "{{ $value }}" threshold: "up = 1" - alert: HostCpuUsageWarning expr: (100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 85) and (100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) <= 95) for: 10m labels: severity: warning service: host annotations: summary: CPU使用率持续偏高 description: 主机CPU使用率连续10分钟高于85%。 currentValue: "{{ printf \"%.1f\" $value }}%" threshold: "85%" - alert: HostCpuUsageCritical expr: 100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 95 for: 5m labels: severity: critical service: host annotations: summary: CPU使用率严重超限 description: 主机CPU使用率连续5分钟高于95%。 currentValue: "{{ printf \"%.1f\" $value }}%" threshold: "95%" - alert: HostMemoryUsageWarning expr: ((1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100 > 85) and ((1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100 <= 95) for: 10m labels: severity: warning service: host annotations: summary: 内存使用率持续偏高 description: 主机可用内存连续10分钟低于15%。 currentValue: "{{ printf \"%.1f\" $value }}%" threshold: "85%" - alert: HostMemoryUsageCritical expr: (1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100 > 95 for: 5m labels: severity: critical service: host annotations: summary: 内存使用率严重超限 description: 主机可用内存连续5分钟低于5%。 currentValue: "{{ printf \"%.1f\" $value }}%" threshold: "95%" - alert: HostRootDiskUsageWarning expr: ((1 - node_filesystem_avail_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"} / node_filesystem_size_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"}) * 100 > 80) and ((1 - node_filesystem_avail_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"} / node_filesystem_size_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"}) * 100 <= 90) for: 15m labels: severity: warning service: host annotations: summary: 根文件系统空间不足 description: 根文件系统使用率连续15分钟高于80%。 currentValue: "{{ printf \"%.1f\" $value }}%" threshold: "80%" - alert: HostRootDiskUsageCritical expr: (1 - node_filesystem_avail_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"} / node_filesystem_size_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"}) * 100 > 90 for: 5m labels: severity: critical service: host annotations: summary: 根文件系统空间严重不足 description: 根文件系统使用率连续5分钟高于90%。 currentValue: "{{ printf \"%.1f\" $value }}%" threshold: "90%" - alert: HostRootInodeUsageWarning expr: ((1 - node_filesystem_files_free{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"} / node_filesystem_files{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"}) * 100 > 80) and ((1 - node_filesystem_files_free{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"} / node_filesystem_files{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"}) * 100 <= 90) for: 15m labels: severity: warning service: host annotations: summary: 根文件系统inode余量偏低 description: 根文件系统inode使用率连续15分钟高于80%。 currentValue: "{{ printf \"%.1f\" $value }}%" threshold: "80%" - alert: HostRootInodeUsageCritical expr: (1 - node_filesystem_files_free{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"} / node_filesystem_files{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"}) * 100 > 90 for: 5m labels: severity: critical service: host annotations: summary: 根文件系统inode严重不足 description: 根文件系统inode使用率连续5分钟高于90%。 currentValue: "{{ printf \"%.1f\" $value }}%" threshold: "90%" - alert: HostCpuIowaitWarning expr: (avg by (instance) (rate(node_cpu_seconds_total{mode="iowait"}[5m])) * 100 > 20) and (avg by (instance) (rate(node_cpu_seconds_total{mode="iowait"}[5m])) * 100 <= 35) for: 10m labels: severity: warning service: host annotations: summary: CPU iowait持续偏高 description: 主机CPU iowait连续10分钟高于20%,请检查磁盘I/O。 currentValue: "{{ printf \"%.1f\" $value }}%" threshold: "20%" - alert: HostCpuIowaitCritical expr: avg by (instance) (rate(node_cpu_seconds_total{mode="iowait"}[5m])) * 100 > 35 for: 10m labels: severity: critical service: host annotations: summary: CPU iowait严重超限 description: 主机CPU iowait连续10分钟高于35%,磁盘I/O可能已成为瓶颈。 currentValue: "{{ printf \"%.1f\" $value }}%" threshold: "35%" - name: cmpp-core-services rules: - alert: CmppCoreServiceInactive expr: node_systemd_unit_state{name=~"cmpp-api\\.service|cmpp-gateway\\.service|postgresql\\.service|redis(-server)?\\.service|cmpp-minio\\.service|nginx\\.service",state="active"} == 0 for: 2m labels: severity: critical service: "{{ $labels.name }}" annotations: summary: CMPP核心服务未处于active状态 description: "systemd服务 {{ $labels.name }} 连续2分钟未处于active状态。" currentValue: "{{ $value }}" threshold: "active = 1"