hectorzhao/lislgosms
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

perf(cmpp): add durable inbound fast path

Browse Source
This commit is contained in:
hectorzhao
2026-08-20 17:34:45 +08:00
parent 26ef67fb6a
commit 0b63bcd74e
29 changed files with 1136 additions and 87 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/monitoring/cmpp-alerts.yml
+11 -1
View File
@@ -182,7 +182,7 @@ groups:
- name: cmpp-core-services
rules:
- alert: CmppCoreServiceInactive
expr: node_systemd_unit_state{name=~"cmpp-api\\.service|cmpp-gateway\\.service|postgresql\\.service|redis(-server)?\\.service|cmpp-minio\\.service|nginx\\.service",state="active"} == 0
expr: node_systemd_unit_state{name=~"cmpp-api\\.service|cmpp-send-worker\\.service|cmpp-gateway\\.service|postgresql\\.service|redis(-server)?\\.service|cmpp-minio\\.service|nginx\\.service",state="active"} == 0
for: 2m
labels:
severity: critical
@@ -200,6 +200,16 @@ groups:
for: 2m
labels: { severity: critical, service: api }
annotations: { summary: "API指标采集不可用", description: "Prometheus连续2分钟无法读取API内部指标端点。", currentValue: "{{ $value }}", threshold: "up = 1" }
- alert: CmppSendWorkerMetricsDown
expr: up{job="cmpp-send-worker"} == 0
for: 2m
labels: { severity: critical, service: send-worker }
annotations: { summary: "发送Worker指标采集不可用", description: "Prometheus连续2分钟无法读取独立发送Worker指标端点。", currentValue: "{{ $value }}", threshold: "up = 1" }
- alert: CmppInboundWorkflowBacklogCritical
expr: cmpp_worker_inbound_workflow_oldest_pending_age_seconds > 120
for: 2m
labels: { severity: critical, service: send-worker }
annotations: { summary: "CMPP耐久Inbox严重积压", description: "最旧待处理CMPP Inbox连续2分钟超过120秒。", currentValue: "{{ printf \"%.0f\" $value }}s", threshold: "120s" }
- alert: CmppApiHttpErrorRateWarning
expr: (sum(rate(cmpp_api_http_requests_total{status=~"5.."}[5m])) / clamp_min(sum(rate(cmpp_api_http_requests_total[5m])), 0.001) > 0.01) and (sum(rate(cmpp_api_http_requests_total{status=~"5.."}[5m])) / clamp_min(sum(rate(cmpp_api_http_requests_total[5m])), 0.001) <= 0.05) and sum(increase(cmpp_api_http_requests_total{status=~"5.."}[5m])) >= 5
for: 5m
tools/monitoring/install-prometheus-monitoring.sh
+1 -1
View File
@@ -68,7 +68,7 @@ EOF
cat >/etc/systemd/system/prometheus-node-exporter.service.d/cmpp-monitoring.conf <<EOF
[Service]
ExecStart=
ExecStart=${node_exporter_bin} --web.listen-address=127.0.0.1:9100 --collector.systemd --collector.systemd.unit-include='cmpp-api\\.service|cmpp-gateway\\.service|postgresql\\.service|redis(-server)?\\.service|cmpp-minio\\.service|nginx\\.service' --collector.filesystem.mount-points-exclude='^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/)'
ExecStart=${node_exporter_bin} --web.listen-address=127.0.0.1:9100 --collector.systemd --collector.systemd.unit-include='cmpp-api\\.service|cmpp-send-worker\\.service|cmpp-gateway\\.service|postgresql\\.service|redis(-server)?\\.service|cmpp-minio\\.service|nginx\\.service' --collector.filesystem.mount-points-exclude='^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/)'
EOF
log "Validating Prometheus configuration before restart"
tools/monitoring/prometheus.yml
+4
View File
@@ -24,6 +24,10 @@ scrape_configs:
static_configs:
- targets: [127.0.0.1:9464]
- job_name: cmpp-send-worker
static_configs:
- targets: [127.0.0.1:9465]
- job_name: cmpp-gateway
metrics_path: /metrics
static_configs: