feat: add Prometheus system monitoring
This commit is contained in:
hectorzhao
@@ -0,0 +1,148 @@
|
||||
groups:
|
||||
- name: cmpp-host-resources
|
||||
rules:
|
||||
- alert: NodeExporterDown
|
||||
expr: up{job="node"} == 0
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
service: node-exporter
|
||||
annotations:
|
||||
summary: 主机指标采集不可用
|
||||
description: Prometheus连续2分钟无法采集Node Exporter。
|
||||
currentValue: "{{ $value }}"
|
||||
threshold: "up = 1"
|
||||
|
||||
- alert: HostCpuUsageWarning
|
||||
expr: (100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 85) and (100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) <= 95)
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
service: host
|
||||
annotations:
|
||||
summary: CPU使用率持续偏高
|
||||
description: 主机CPU使用率连续10分钟高于85%。
|
||||
currentValue: "{{ printf \"%.1f\" $value }}%"
|
||||
threshold: "85%"
|
||||
|
||||
- alert: HostCpuUsageCritical
|
||||
expr: 100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 95
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
service: host
|
||||
annotations:
|
||||
summary: CPU使用率严重超限
|
||||
description: 主机CPU使用率连续5分钟高于95%。
|
||||
currentValue: "{{ printf \"%.1f\" $value }}%"
|
||||
threshold: "95%"
|
||||
|
||||
- alert: HostMemoryUsageWarning
|
||||
expr: ((1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100 > 85) and ((1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100 <= 95)
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
service: host
|
||||
annotations:
|
||||
summary: 内存使用率持续偏高
|
||||
description: 主机可用内存连续10分钟低于15%。
|
||||
currentValue: "{{ printf \"%.1f\" $value }}%"
|
||||
threshold: "85%"
|
||||
|
||||
- alert: HostMemoryUsageCritical
|
||||
expr: (1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100 > 95
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
service: host
|
||||
annotations:
|
||||
summary: 内存使用率严重超限
|
||||
description: 主机可用内存连续5分钟低于5%。
|
||||
currentValue: "{{ printf \"%.1f\" $value }}%"
|
||||
threshold: "95%"
|
||||
|
||||
- alert: HostRootDiskUsageWarning
|
||||
expr: ((1 - node_filesystem_avail_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"} / node_filesystem_size_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"}) * 100 > 80) and ((1 - node_filesystem_avail_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"} / node_filesystem_size_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"}) * 100 <= 90)
|
||||
for: 15m
|
||||
labels:
|
||||
severity: warning
|
||||
service: host
|
||||
annotations:
|
||||
summary: 根文件系统空间不足
|
||||
description: 根文件系统使用率连续15分钟高于80%。
|
||||
currentValue: "{{ printf \"%.1f\" $value }}%"
|
||||
threshold: "80%"
|
||||
|
||||
- alert: HostRootDiskUsageCritical
|
||||
expr: (1 - node_filesystem_avail_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"} / node_filesystem_size_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"}) * 100 > 90
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
service: host
|
||||
annotations:
|
||||
summary: 根文件系统空间严重不足
|
||||
description: 根文件系统使用率连续5分钟高于90%。
|
||||
currentValue: "{{ printf \"%.1f\" $value }}%"
|
||||
threshold: "90%"
|
||||
|
||||
- alert: HostRootInodeUsageWarning
|
||||
expr: ((1 - node_filesystem_files_free{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"} / node_filesystem_files{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"}) * 100 > 80) and ((1 - node_filesystem_files_free{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"} / node_filesystem_files{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"}) * 100 <= 90)
|
||||
for: 15m
|
||||
labels:
|
||||
severity: warning
|
||||
service: host
|
||||
annotations:
|
||||
summary: 根文件系统inode余量偏低
|
||||
description: 根文件系统inode使用率连续15分钟高于80%。
|
||||
currentValue: "{{ printf \"%.1f\" $value }}%"
|
||||
threshold: "80%"
|
||||
|
||||
- alert: HostRootInodeUsageCritical
|
||||
expr: (1 - node_filesystem_files_free{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"} / node_filesystem_files{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"}) * 100 > 90
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
service: host
|
||||
annotations:
|
||||
summary: 根文件系统inode严重不足
|
||||
description: 根文件系统inode使用率连续5分钟高于90%。
|
||||
currentValue: "{{ printf \"%.1f\" $value }}%"
|
||||
threshold: "90%"
|
||||
|
||||
- alert: HostCpuIowaitWarning
|
||||
expr: (avg by (instance) (rate(node_cpu_seconds_total{mode="iowait"}[5m])) * 100 > 20) and (avg by (instance) (rate(node_cpu_seconds_total{mode="iowait"}[5m])) * 100 <= 35)
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
service: host
|
||||
annotations:
|
||||
summary: CPU iowait持续偏高
|
||||
description: 主机CPU iowait连续10分钟高于20%，请检查磁盘I/O。
|
||||
currentValue: "{{ printf \"%.1f\" $value }}%"
|
||||
threshold: "20%"
|
||||
|
||||
- alert: HostCpuIowaitCritical
|
||||
expr: avg by (instance) (rate(node_cpu_seconds_total{mode="iowait"}[5m])) * 100 > 35
|
||||
for: 10m
|
||||
labels:
|
||||
severity: critical
|
||||
service: host
|
||||
annotations:
|
||||
summary: CPU iowait严重超限
|
||||
description: 主机CPU iowait连续10分钟高于35%，磁盘I/O可能已成为瓶颈。
|
||||
currentValue: "{{ printf \"%.1f\" $value }}%"
|
||||
threshold: "35%"
|
||||
|
||||
- name: cmpp-core-services
|
||||
rules:
|
||||
- alert: CmppCoreServiceInactive
|
||||
expr: node_systemd_unit_state{name=~"cmpp-api\\.service|cmpp-gateway\\.service|postgresql\\.service|redis(-server)?\\.service|cmpp-minio\\.service|nginx\\.service",state="active"} == 0
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
service: "{{ $labels.name }}"
|
||||
annotations:
|
||||
summary: CMPP核心服务未处于active状态
|
||||
description: "systemd服务 {{ $labels.name }} 连续2分钟未处于active状态。"
|
||||
currentValue: "{{ $value }}"
|
||||
threshold: "active = 1"
|
||||
Reference in New Issue
Block a user