hectorzhao/lislgosms
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add Prometheus system monitoring

Browse Source
This commit is contained in:
hectorzhao
2026-08-14 10:10:28 +08:00
parent 96e475d60d
commit b78faa1aa2
25 changed files with 1675 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/deploy/production-bootstrap.sh
+4
View File
@@ -31,6 +31,8 @@ OPERATION_LOG_ARCHIVE_INTERVAL_MS="${OPERATION_LOG_ARCHIVE_INTERVAL_MS:-86400000
SMS_RECEIPT_TIMEOUT_SCAN_ENABLED="${SMS_RECEIPT_TIMEOUT_SCAN_ENABLED:-true}"
SMS_RECEIPT_TIMEOUT_HOURS="${SMS_RECEIPT_TIMEOUT_HOURS:-72}"
SMS_RECEIPT_TIMEOUT_SCAN_INTERVAL_MS="${SMS_RECEIPT_TIMEOUT_SCAN_INTERVAL_MS:-300000}"
PROMETHEUS_URL="${PROMETHEUS_URL:-http://127.0.0.1:9090}"
PROMETHEUS_QUERY_TIMEOUT_MS="${PROMETHEUS_QUERY_TIMEOUT_MS:-5000}"
if [[ "$(id -u)" -ne 0 ]]; then
echo "Run as root." >&2
@@ -192,6 +194,8 @@ OPERATION_LOG_ARCHIVE_INTERVAL_MS=${OPERATION_LOG_ARCHIVE_INTERVAL_MS}
SMS_RECEIPT_TIMEOUT_SCAN_ENABLED=${SMS_RECEIPT_TIMEOUT_SCAN_ENABLED}
SMS_RECEIPT_TIMEOUT_HOURS=${SMS_RECEIPT_TIMEOUT_HOURS}
SMS_RECEIPT_TIMEOUT_SCAN_INTERVAL_MS=${SMS_RECEIPT_TIMEOUT_SCAN_INTERVAL_MS}
PROMETHEUS_URL=${PROMETHEUS_URL}
PROMETHEUS_QUERY_TIMEOUT_MS=${PROMETHEUS_QUERY_TIMEOUT_MS}
MINIO_ENDPOINT=127.0.0.1:9000
MINIO_ACCESS_KEY=${MINIO_ROOT_USER}
MINIO_SECRET_KEY=${MINIO_ROOT_PASSWORD}
tools/monitoring/README.md
+31
View File
@@ -0,0 +1,31 @@
# CMPP Prometheus 监控配置
本目录提供系统监控第一版所需的真实采集配置。运营端不嵌入Prometheus或Grafana页面；NestJS仅从本机Prometheus读取固定指标，再由平台原生UI展示。
在目标Debian/Ubuntu测试服务器上，以root执行：
```bash
cd /opt/cmpp-platform
bash tools/monitoring/install-prometheus-monitoring.sh
```
脚本会安装Prometheus与Node Exporter、备份已有Prometheus配置及本脚本曾写入的systemd override、校验规则、写入新override，并仅重启这两个监控服务。备份目录会在脚本结束时打印。它不会重启API、Gateway、数据库、Redis、MinIO或Nginx。9090和9100固定监听`127.0.0.1`。
安装后将下列配置写入`/etc/cmpp-platform/cmpp-platform.env`，再按正常发布窗口重启API：
```dotenv
PROMETHEUS_URL=http://127.0.0.1:9090
PROMETHEUS_QUERY_TIMEOUT_MS=5000
```
校验命令：
```bash
promtool check config /etc/prometheus/prometheus.yml
promtool check rules /etc/prometheus/cmpp-alerts.yml
curl -fsS http://127.0.0.1:9090/-/ready
curl -fsS 'http://127.0.0.1:9090/api/v1/query?query=up'
ss -lnt | grep -E ':(9090|9100)'
```
完整架构、PromQL口径、故障语义和验收标准见`docs/prometheus-system-monitoring-design-20260814.md`。
tools/monitoring/cmpp-alerts.yml
+148
View File
@@ -0,0 +1,148 @@
groups:
- name: cmpp-host-resources
rules:
- alert: NodeExporterDown
expr: up{job="node"} == 0
for: 2m
labels:
severity: critical
service: node-exporter
annotations:
summary: 主机指标采集不可用
description: Prometheus连续2分钟无法采集Node Exporter。
currentValue: "{{ $value }}"
threshold: "up = 1"
- alert: HostCpuUsageWarning
expr: (100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 85) and (100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) <= 95)
for: 10m
labels:
severity: warning
service: host
annotations:
summary: CPU使用率持续偏高
description: 主机CPU使用率连续10分钟高于85%。
currentValue: "{{ printf \"%.1f\" $value }}%"
threshold: "85%"
- alert: HostCpuUsageCritical
expr: 100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 95
for: 5m
labels:
severity: critical
service: host
annotations:
summary: CPU使用率严重超限
description: 主机CPU使用率连续5分钟高于95%。
currentValue: "{{ printf \"%.1f\" $value }}%"
threshold: "95%"
- alert: HostMemoryUsageWarning
expr: ((1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100 > 85) and ((1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100 <= 95)
for: 10m
labels:
severity: warning
service: host
annotations:
summary: 内存使用率持续偏高
description: 主机可用内存连续10分钟低于15%。
currentValue: "{{ printf \"%.1f\" $value }}%"
threshold: "85%"
- alert: HostMemoryUsageCritical
expr: (1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100 > 95
for: 5m
labels:
severity: critical
service: host
annotations:
summary: 内存使用率严重超限
description: 主机可用内存连续5分钟低于5%。
currentValue: "{{ printf \"%.1f\" $value }}%"
threshold: "95%"
- alert: HostRootDiskUsageWarning
expr: ((1 - node_filesystem_avail_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"} / node_filesystem_size_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"}) * 100 > 80) and ((1 - node_filesystem_avail_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"} / node_filesystem_size_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"}) * 100 <= 90)
for: 15m
labels:
severity: warning
service: host
annotations:
summary: 根文件系统空间不足
description: 根文件系统使用率连续15分钟高于80%。
currentValue: "{{ printf \"%.1f\" $value }}%"
threshold: "80%"
- alert: HostRootDiskUsageCritical
expr: (1 - node_filesystem_avail_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"} / node_filesystem_size_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"}) * 100 > 90
for: 5m
labels:
severity: critical
service: host
annotations:
summary: 根文件系统空间严重不足
description: 根文件系统使用率连续5分钟高于90%。
currentValue: "{{ printf \"%.1f\" $value }}%"
threshold: "90%"
- alert: HostRootInodeUsageWarning
expr: ((1 - node_filesystem_files_free{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"} / node_filesystem_files{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"}) * 100 > 80) and ((1 - node_filesystem_files_free{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"} / node_filesystem_files{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"}) * 100 <= 90)
for: 15m
labels:
severity: warning
service: host
annotations:
summary: 根文件系统inode余量偏低
description: 根文件系统inode使用率连续15分钟高于80%。
currentValue: "{{ printf \"%.1f\" $value }}%"
threshold: "80%"
- alert: HostRootInodeUsageCritical
expr: (1 - node_filesystem_files_free{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"} / node_filesystem_files{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"}) * 100 > 90
for: 5m
labels:
severity: critical
service: host
annotations:
summary: 根文件系统inode严重不足
description: 根文件系统inode使用率连续5分钟高于90%。
currentValue: "{{ printf \"%.1f\" $value }}%"
threshold: "90%"
- alert: HostCpuIowaitWarning
expr: (avg by (instance) (rate(node_cpu_seconds_total{mode="iowait"}[5m])) * 100 > 20) and (avg by (instance) (rate(node_cpu_seconds_total{mode="iowait"}[5m])) * 100 <= 35)
for: 10m
labels:
severity: warning
service: host
annotations:
summary: CPU iowait持续偏高
description: 主机CPU iowait连续10分钟高于20%，请检查磁盘I/O。
currentValue: "{{ printf \"%.1f\" $value }}%"
threshold: "20%"
- alert: HostCpuIowaitCritical
expr: avg by (instance) (rate(node_cpu_seconds_total{mode="iowait"}[5m])) * 100 > 35
for: 10m
labels:
severity: critical
service: host
annotations:
summary: CPU iowait严重超限
description: 主机CPU iowait连续10分钟高于35%，磁盘I/O可能已成为瓶颈。
currentValue: "{{ printf \"%.1f\" $value }}%"
threshold: "35%"
- name: cmpp-core-services
rules:
- alert: CmppCoreServiceInactive
expr: node_systemd_unit_state{name=~"cmpp-api\\.service|cmpp-gateway\\.service|postgresql\\.service|redis(-server)?\\.service|cmpp-minio\\.service|nginx\\.service",state="active"} == 0
for: 2m
labels:
severity: critical
service: "{{ $labels.name }}"
annotations:
summary: CMPP核心服务未处于active状态
description: "systemd服务 {{ $labels.name }} 连续2分钟未处于active状态。"
currentValue: "{{ $value }}"
threshold: "active = 1"
tools/monitoring/install-prometheus-monitoring.sh
+82
View File
@@ -0,0 +1,82 @@
#!/usr/bin/env bash
set -Eeuo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROMETHEUS_RETENTION_TIME="${PROMETHEUS_RETENTION_TIME:-30d}"
PROMETHEUS_RETENTION_SIZE="${PROMETHEUS_RETENTION_SIZE:-8GB}"
if [[ "$(id -u)" -ne 0 ]]; then
echo "Run as root." >&2
exit 1
fi
if ! command -v apt-get >/dev/null 2>&1; then
echo "This installer currently supports Debian/Ubuntu apt packages only." >&2
exit 1
fi
log() { printf '\n[%s] %s\n' "$(date '+%F %T')" "$*"; }
log "Installing Prometheus and Node Exporter packages"
apt-get update
DEBIAN_FRONTEND=noninteractive apt-get install -y prometheus prometheus-node-exporter curl iproute2
prometheus_bin="$(command -v prometheus)"
node_exporter_bin="$(command -v prometheus-node-exporter)"
promtool_bin="$(command -v promtool)"
backup_dir="/etc/prometheus/cmpp-backups/$(date '+%Y%m%d-%H%M%S')"
mkdir -p "$backup_dir" /etc/systemd/system/prometheus.service.d /etc/systemd/system/prometheus-node-exporter.service.d
for config_file in /etc/prometheus/prometheus.yml /etc/prometheus/cmpp-alerts.yml; do
if [[ -f "$config_file" ]]; then
cp --preserve=mode,timestamps "$config_file" "$backup_dir/$(basename "$config_file")"
fi
done
if [[ -f /etc/systemd/system/prometheus.service.d/cmpp-monitoring.conf ]]; then
cp --preserve=mode,timestamps /etc/systemd/system/prometheus.service.d/cmpp-monitoring.conf "$backup_dir/prometheus-service-override.conf"
fi
if [[ -f /etc/systemd/system/prometheus-node-exporter.service.d/cmpp-monitoring.conf ]]; then
cp --preserve=mode,timestamps /etc/systemd/system/prometheus-node-exporter.service.d/cmpp-monitoring.conf "$backup_dir/node-exporter-service-override.conf"
fi
log "Installing platform-owned scrape and alert configuration"
install -o root -g root -m 0644 "$SCRIPT_DIR/prometheus.yml" /etc/prometheus/prometheus.yml
install -o root -g root -m 0644 "$SCRIPT_DIR/cmpp-alerts.yml" /etc/prometheus/cmpp-alerts.yml
cat >/etc/systemd/system/prometheus.service.d/cmpp-monitoring.conf <<EOF
[Service]
ExecStart=
ExecStart=${prometheus_bin} --config.file=/etc/prometheus/prometheus.yml --storage.tsdb.path=/var/lib/prometheus/metrics2 --storage.tsdb.retention.time=${PROMETHEUS_RETENTION_TIME} --storage.tsdb.retention.size=${PROMETHEUS_RETENTION_SIZE} --web.listen-address=127.0.0.1:9090
EOF
cat >/etc/systemd/system/prometheus-node-exporter.service.d/cmpp-monitoring.conf <<EOF
[Service]
ExecStart=
ExecStart=${node_exporter_bin} --web.listen-address=127.0.0.1:9100 --collector.systemd --collector.systemd.unit-include='cmpp-api\\.service|cmpp-gateway\\.service|postgresql\\.service|redis(-server)?\\.service|cmpp-minio\\.service|nginx\\.service' --collector.filesystem.mount-points-exclude='^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/)'
EOF
log "Validating Prometheus configuration before restart"
"$promtool_bin" check rules /etc/prometheus/cmpp-alerts.yml
"$promtool_bin" check config /etc/prometheus/prometheus.yml
systemctl daemon-reload
systemctl enable prometheus prometheus-node-exporter
systemctl restart prometheus-node-exporter
systemctl restart prometheus
for _ in $(seq 1 30); do
if curl -fsS http://127.0.0.1:9090/-/ready >/dev/null; then
break
fi
sleep 1
done
curl -fsS http://127.0.0.1:9090/-/ready >/dev/null
curl -fsS http://127.0.0.1:9100/metrics >/dev/null
if ss -lnt | grep -Eq '(^|[[:space:]])(0\.0\.0\.0|\[::\]):(9090|9100)([[:space:]]|$)'; then
echo "Prometheus monitoring ports unexpectedly listen on a wildcard address." >&2
exit 1
fi
log "Prometheus monitoring is ready on loopback only"
echo "Configuration backup: $backup_dir"
echo "The CMPP API and Gateway were not restarted."
tools/monitoring/prometheus.yml
+20
View File
@@ -0,0 +1,20 @@
global:
scrape_interval: 15s
evaluation_interval: 15s
external_labels:
platform: cmpp
environment: preproduction
rule_files:
- /etc/prometheus/cmpp-alerts.yml
scrape_configs:
- job_name: prometheus
static_configs:
- targets: [127.0.0.1:9090]
- job_name: node
static_configs:
- targets: [127.0.0.1:9100]
labels:
host: cmpp-primary