diff --git a/api/src/infrastructure-monitoring/infrastructure-alert-settings.service.spec.ts b/api/src/infrastructure-monitoring/infrastructure-alert-settings.service.spec.ts index 6539285..bf70dd1 100644 --- a/api/src/infrastructure-monitoring/infrastructure-alert-settings.service.spec.ts +++ b/api/src/infrastructure-monitoring/infrastructure-alert-settings.service.spec.ts @@ -10,6 +10,8 @@ describe('InfrastructureAlertSettingsService', () => { expect(rules).toContain('HostCpuUsageWarning'); expect(rules).toContain('CmppGatewayQueueDelayedCritical'); expect(rules).toContain('threshold: "120秒"'); + expect(rules).toContain('redis_memory_max_bytes > 0'); + expect(rules).toContain('sum(increase(cmpp_api_http_requests_total'); }); it('rejects unknown keys and warning thresholds that are not below critical', () => { diff --git a/api/src/infrastructure-monitoring/infrastructure-alert-settings.service.ts b/api/src/infrastructure-monitoring/infrastructure-alert-settings.service.ts index 62ab7c7..4601666 100644 --- a/api/src/infrastructure-monitoring/infrastructure-alert-settings.service.ts +++ b/api/src/infrastructure-monitoring/infrastructure-alert-settings.service.ts @@ -14,12 +14,12 @@ export const ALERT_THRESHOLD_DEFINITIONS = [ { key: 'hostCpu', label: '主机 CPU 使用率', unit: '%', min: 1, max: 100, step: 1, warning: 80, critical: 90, expr: '100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100)', names: ['HostCpuUsageWarning', 'HostCpuUsageCritical'], service: 'host', durations: ['10m', '5m'] }, { key: 'hostMemory', label: '主机内存使用率', unit: '%', min: 1, max: 100, step: 1, warning: 85, critical: 95, expr: '(1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100', names: ['HostMemoryUsageWarning', 'HostMemoryUsageCritical'], service: 'host', durations: ['10m', '5m'] }, { key: 'hostDisk', label: '根磁盘使用率', unit: '%', min: 1, max: 100, step: 1, warning: 80, critical: 90, expr: '(1 - node_filesystem_avail_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"} / node_filesystem_size_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"}) * 100', names: ['HostRootDiskUsageWarning', 'HostRootDiskUsageCritical'], service: 'host', durations: ['15m', '5m'] }, - { key: 'apiError', label: 'API 5xx 错误率', unit: '%', min: 0.1, max: 100, step: 0.1, warning: 1, critical: 5, expr: '100 * sum(rate(cmpp_api_http_requests_total{status=~"5.."}[5m])) / clamp_min(sum(rate(cmpp_api_http_requests_total[5m])), 0.001)', names: ['CmppApiHttpErrorRateWarning', 'CmppApiHttpErrorRateCritical'], service: 'api', durations: ['5m', '5m'] }, + { key: 'apiError', label: 'API 5xx 错误率', unit: '%', min: 0.1, max: 100, step: 0.1, warning: 1, critical: 5, expr: '100 * sum(rate(cmpp_api_http_requests_total{status=~"5.."}[5m])) / clamp_min(sum(rate(cmpp_api_http_requests_total[5m])), 0.001)', guard: 'sum(increase(cmpp_api_http_requests_total{status=~"5.."}[5m])) >= 5', names: ['CmppApiHttpErrorRateWarning', 'CmppApiHttpErrorRateCritical'], service: 'api', durations: ['5m', '5m'] }, { key: 'apiLatency', label: 'API P95 响应时间', unit: '秒', min: 0.1, max: 60, step: 0.1, warning: 1, critical: 3, expr: 'histogram_quantile(0.95, sum by (le) (rate(cmpp_api_http_request_duration_seconds_bucket[10m])))', names: ['CmppApiLatencyWarning', 'CmppApiLatencyCritical'], service: 'api', durations: ['10m', '5m'] }, { key: 'apiEventLoop', label: 'API 事件循环 P99', unit: '秒', min: 0.01, max: 10, step: 0.01, warning: 0.2, critical: 1, expr: 'cmpp_api_nodejs_event_loop_lag_p99_seconds', names: ['CmppApiEventLoopLagWarning', 'CmppApiEventLoopLagCritical'], service: 'api', durations: ['10m', '5m'] }, { key: 'gatewayQueue', label: 'Gateway 最旧 pending', unit: '秒', min: 1, max: 3600, step: 1, warning: 30, critical: 120, expr: 'cmpp_gateway_submit_queue_oldest_pending_age_seconds', names: ['CmppGatewayQueueDelayedWarning', 'CmppGatewayQueueDelayedCritical'], service: 'gateway', durations: ['2m', '2m'] }, { key: 'postgresConnections', label: 'PostgreSQL 连接使用率', unit: '%', min: 1, max: 100, step: 1, warning: 70, critical: 85, expr: '100 * sum(pg_stat_activity_count) / clamp_min(max(pg_settings_max_connections), 1)', names: ['PostgresConnectionsWarning', 'PostgresConnectionsCritical'], service: 'postgresql', durations: ['10m', '5m'] }, - { key: 'redisMemory', label: 'Redis 内存使用率', unit: '%', min: 1, max: 100, step: 1, warning: 70, critical: 85, expr: '100 * redis_memory_used_bytes / redis_memory_max_bytes', names: ['RedisMemoryWarning', 'RedisMemoryCritical'], service: 'redis', durations: ['10m', '5m'] }, + { key: 'redisMemory', label: 'Redis 内存使用率', unit: '%', min: 1, max: 100, step: 1, warning: 70, critical: 85, expr: '100 * redis_memory_used_bytes / redis_memory_max_bytes', guard: 'redis_memory_max_bytes > 0', names: ['RedisMemoryWarning', 'RedisMemoryCritical'], service: 'redis', durations: ['10m', '5m'] }, { key: 'minioCapacity', label: 'MinIO 容量使用率', unit: '%', min: 1, max: 100, step: 1, warning: 80, critical: 90, expr: '100 * (1 - minio_cluster_capacity_usable_free_bytes / minio_cluster_capacity_usable_total_bytes)', names: ['MinioCapacityWarning', 'MinioCapacityCritical'], service: 'minio', durations: ['15m', '5m'] }, ] as const; @@ -110,7 +110,9 @@ export class InfrastructureAlertSettingsService { const values = [pair.warning, pair.critical]; for (let index = 0; index < 2; index += 1) { const isWarning = index === 0; - const expr = isWarning ? `(${definition.expr} > ${values[0]}) and (${definition.expr} <= ${values[1]})` : `${definition.expr} > ${values[1]}`; + // 低样本量与“未配置容量上限”必须继续作为固定保护条件,避免单次错误或除零结果触发伪告警。 + const guard = 'guard' in definition ? ` and (${definition.guard})` : ''; + const expr = isWarning ? `(${definition.expr} > ${values[0]}) and (${definition.expr} <= ${values[1]})${guard}` : `(${definition.expr} > ${values[1]})${guard}`; lines.push(` - alert: ${definition.names[index]}`, ` expr: ${expr}`, ` for: ${definition.durations[index]}`, ' labels:', ` severity: ${isWarning ? 'warning' : 'critical'}`, ` service: ${definition.service}`, ' annotations:', ` summary: "${definition.label}${isWarning ? '达到警告阈值' : '达到严重阈值'}"`, ` description: "${definition.label}持续超过${values[index]}${definition.unit}。"`, ' currentValue: "{{ $value }}"', ` threshold: "${values[index]}${definition.unit}"`); } } diff --git a/docs/testing-progress.md b/docs/testing-progress.md index 1fe2f97..15e864a 100644 --- a/docs/testing-progress.md +++ b/docs/testing-progress.md @@ -3664,3 +3664,4 @@ git diff --check - 系统监控将“服务关键指标”移至活动告警正上方,标题说明明确使用 Prometheus;新增十组固定指标警告/严重阈值设置。配置真实写入 PostgreSQL,使用版本条件并发认领,promtool 校验、原子替换和回环 reload 成功后才推进生效版本,失败保留旧生效值。 - 右上角预警中心新增“系统监控告警”及数量/严重数,读取独立 Prometheus 活动告警汇总并跳转活动告警锚点;继续使用 `Promise.allSettled` 隔离签名、安全与监控域故障。安全检测页移除重复大号标题,说明明确使用 Fail2ban。 - 新增 migration `20260814173000_add_infrastructure_alert_settings`。本机 Prisma Client 生成、前后端 TypeScript 与 API 正式编译已通过;测试机恢复资产、migration、Prometheus规则校验、真实告警数据、浏览器验收和提交/部署结果待本轮后续补记。 +- 首次测试机规则验收发现 Redis 未配置 `maxmemory` 时除零结果会误触发 Critical pending,立即补回 `redis_memory_max_bytes > 0` 固定保护,并补回 API 5xx 窗口至少 5 次错误的最低样本门槛;该真实验收缺陷未按通过处理。 diff --git a/tools/monitoring/cmpp-managed-alerts.yml b/tools/monitoring/cmpp-managed-alerts.yml index e992164..3a206ba 100644 --- a/tools/monitoring/cmpp-managed-alerts.yml +++ b/tools/monitoring/cmpp-managed-alerts.yml @@ -32,12 +32,12 @@ groups: labels: { severity: critical, service: host } annotations: { summary: "根磁盘使用率达到严重阈值", description: "根磁盘使用率持续超过90%。", currentValue: "{{ $value }}", threshold: "90%" } - alert: CmppApiHttpErrorRateWarning - expr: (100 * sum(rate(cmpp_api_http_requests_total{status=~"5.."}[5m])) / clamp_min(sum(rate(cmpp_api_http_requests_total[5m])), 0.001) > 1) and (100 * sum(rate(cmpp_api_http_requests_total{status=~"5.."}[5m])) / clamp_min(sum(rate(cmpp_api_http_requests_total[5m])), 0.001) <= 5) + expr: (100 * sum(rate(cmpp_api_http_requests_total{status=~"5.."}[5m])) / clamp_min(sum(rate(cmpp_api_http_requests_total[5m])), 0.001) > 1) and (100 * sum(rate(cmpp_api_http_requests_total{status=~"5.."}[5m])) / clamp_min(sum(rate(cmpp_api_http_requests_total[5m])), 0.001) <= 5) and (sum(increase(cmpp_api_http_requests_total{status=~"5.."}[5m])) >= 5) for: 5m labels: { severity: warning, service: api } annotations: { summary: "API 5xx 错误率达到警告阈值", description: "API 5xx 错误率持续超过1%。", currentValue: "{{ $value }}", threshold: "1%" } - alert: CmppApiHttpErrorRateCritical - expr: 100 * sum(rate(cmpp_api_http_requests_total{status=~"5.."}[5m])) / clamp_min(sum(rate(cmpp_api_http_requests_total[5m])), 0.001) > 5 + expr: (100 * sum(rate(cmpp_api_http_requests_total{status=~"5.."}[5m])) / clamp_min(sum(rate(cmpp_api_http_requests_total[5m])), 0.001) > 5) and (sum(increase(cmpp_api_http_requests_total{status=~"5.."}[5m])) >= 5) for: 5m labels: { severity: critical, service: api } annotations: { summary: "API 5xx 错误率达到严重阈值", description: "API 5xx 错误率持续超过5%。", currentValue: "{{ $value }}", threshold: "5%" } @@ -82,12 +82,12 @@ groups: labels: { severity: critical, service: postgresql } annotations: { summary: "PostgreSQL 连接使用率达到严重阈值", description: "PostgreSQL 连接使用率持续超过85%。", currentValue: "{{ $value }}", threshold: "85%" } - alert: RedisMemoryWarning - expr: (100 * redis_memory_used_bytes / redis_memory_max_bytes > 70) and (100 * redis_memory_used_bytes / redis_memory_max_bytes <= 85) + expr: (100 * redis_memory_used_bytes / redis_memory_max_bytes > 70) and (100 * redis_memory_used_bytes / redis_memory_max_bytes <= 85) and (redis_memory_max_bytes > 0) for: 10m labels: { severity: warning, service: redis } annotations: { summary: "Redis 内存使用率达到警告阈值", description: "Redis 内存使用率持续超过70%。", currentValue: "{{ $value }}", threshold: "70%" } - alert: RedisMemoryCritical - expr: 100 * redis_memory_used_bytes / redis_memory_max_bytes > 85 + expr: (100 * redis_memory_used_bytes / redis_memory_max_bytes > 85) and (redis_memory_max_bytes > 0) for: 5m labels: { severity: critical, service: redis } annotations: { summary: "Redis 内存使用率达到严重阈值", description: "Redis 内存使用率持续超过85%。", currentValue: "{{ $value }}", threshold: "85%" }