From 6ccc102830aae68e1b5d4dc78ce1a35272b5dc52 Mon Sep 17 00:00:00 2001 From: hectorzhao Date: Fri, 14 Aug 2026 17:51:42 +0800 Subject: [PATCH] feat: add configurable infrastructure alerts --- .../migration.sql | 23 +++ api/prisma/schema.prisma | 14 ++ ...rastructure-alert-settings.service.spec.ts | 19 +++ .../infrastructure-alert-settings.service.ts | 142 ++++++++++++++++++ .../infrastructure-monitoring.contracts.ts | 13 ++ .../infrastructure-monitoring.controller.ts | 19 ++- .../infrastructure-monitoring.module.ts | 3 +- .../infrastructure-monitoring.service.ts | 12 +- docs/codebase-modularization-roadmap.md | 2 +- .../first-version-development-requirements.md | 3 + ...theus-system-monitoring-design-20260814.md | 6 + docs/system-functional-test-cases.md | 13 ++ docs/testing-progress.md | 6 + .../admin/infrastructure-monitoring.api.ts | 5 +- src/api/types/infrastructure-monitoring.ts | 13 ++ .../AdminSecurityDetectionPage.css | 1 + .../AdminSecurityDetectionPage.tsx | 2 +- .../AdminSystemMonitoringPage.css | 13 ++ .../AdminSystemMonitoringPage.tsx | 99 ++++++++---- src/layouts/AdminLayout.tsx | 10 +- tools/monitoring/cmpp-managed-alerts.yml | 103 +++++++++++++ .../install-prometheus-monitoring.sh | 23 ++- tools/monitoring/prometheus.yml | 1 + 23 files changed, 506 insertions(+), 39 deletions(-) create mode 100644 api/prisma/migrations/20260814173000_add_infrastructure_alert_settings/migration.sql create mode 100644 api/src/infrastructure-monitoring/infrastructure-alert-settings.service.spec.ts create mode 100644 api/src/infrastructure-monitoring/infrastructure-alert-settings.service.ts create mode 100644 tools/monitoring/cmpp-managed-alerts.yml diff --git a/api/prisma/migrations/20260814173000_add_infrastructure_alert_settings/migration.sql b/api/prisma/migrations/20260814173000_add_infrastructure_alert_settings/migration.sql new file mode 100644 index 0000000..791237a --- /dev/null +++ b/api/prisma/migrations/20260814173000_add_infrastructure_alert_settings/migration.sql @@ -0,0 +1,23 @@ +CREATE TABLE "InfrastructureAlertSetting" ( + "id" TEXT NOT NULL DEFAULT 'global', + "configVersion" INTEGER NOT NULL DEFAULT 1, + "effectiveVersion" INTEGER NOT NULL DEFAULT 1, + "thresholds" JSONB NOT NULL, + "effectiveThresholds" JSONB NOT NULL, + "applyStatus" TEXT NOT NULL DEFAULT 'effective', + "lastError" TEXT, + "updatedById" TEXT, + "appliedAt" TIMESTAMP(3), + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + CONSTRAINT "InfrastructureAlertSetting_pkey" PRIMARY KEY ("id") +); + +INSERT INTO "InfrastructureAlertSetting" ( + "id", "thresholds", "effectiveThresholds", "appliedAt" +) VALUES ( + 'global', + '{"hostCpu":{"warning":80,"critical":90},"hostMemory":{"warning":85,"critical":95},"hostDisk":{"warning":80,"critical":90},"apiError":{"warning":1,"critical":5},"apiLatency":{"warning":1,"critical":3},"apiEventLoop":{"warning":0.2,"critical":1},"gatewayQueue":{"warning":30,"critical":120},"postgresConnections":{"warning":70,"critical":85},"redisMemory":{"warning":70,"critical":85},"minioCapacity":{"warning":80,"critical":90}}'::jsonb, + '{"hostCpu":{"warning":80,"critical":90},"hostMemory":{"warning":85,"critical":95},"hostDisk":{"warning":80,"critical":90},"apiError":{"warning":1,"critical":5},"apiLatency":{"warning":1,"critical":3},"apiEventLoop":{"warning":0.2,"critical":1},"gatewayQueue":{"warning":30,"critical":120},"postgresConnections":{"warning":70,"critical":85},"redisMemory":{"warning":70,"critical":85},"minioCapacity":{"warning":80,"critical":90}}'::jsonb, + CURRENT_TIMESTAMP +); diff --git a/api/prisma/schema.prisma b/api/prisma/schema.prisma index 086e13a..738e90d 100644 --- a/api/prisma/schema.prisma +++ b/api/prisma/schema.prisma @@ -2432,3 +2432,17 @@ model SecurityProtectedNetwork { @@index([enabled, createdAt]) } + +model InfrastructureAlertSetting { + id String @id @default("global") + configVersion Int @default(1) + effectiveVersion Int @default(1) + thresholds Json + effectiveThresholds Json + applyStatus String @default("effective") + lastError String? + updatedById String? + appliedAt DateTime? + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt +} diff --git a/api/src/infrastructure-monitoring/infrastructure-alert-settings.service.spec.ts b/api/src/infrastructure-monitoring/infrastructure-alert-settings.service.spec.ts new file mode 100644 index 0000000..6539285 --- /dev/null +++ b/api/src/infrastructure-monitoring/infrastructure-alert-settings.service.spec.ts @@ -0,0 +1,19 @@ +import { BadRequestException } from '@nestjs/common'; +import { DEFAULT_ALERT_THRESHOLDS, InfrastructureAlertSettingsService } from './infrastructure-alert-settings.service'; + +describe('InfrastructureAlertSettingsService', () => { + const service = new InfrastructureAlertSettingsService({} as never, { get: () => undefined } as never); + + it('accepts the fixed threshold whitelist and renders managed rules', () => { + const validated = (service as unknown as { validate(value: unknown): unknown }).validate(DEFAULT_ALERT_THRESHOLDS); + const rules = (service as unknown as { renderRules(value: unknown): string }).renderRules(validated); + expect(rules).toContain('HostCpuUsageWarning'); + expect(rules).toContain('CmppGatewayQueueDelayedCritical'); + expect(rules).toContain('threshold: "120秒"'); + }); + + it('rejects unknown keys and warning thresholds that are not below critical', () => { + expect(() => (service as unknown as { validate(value: unknown): unknown }).validate({ ...DEFAULT_ALERT_THRESHOLDS, promql: { warning: 1, critical: 2 } })).toThrow(BadRequestException); + expect(() => (service as unknown as { validate(value: unknown): unknown }).validate({ ...DEFAULT_ALERT_THRESHOLDS, hostCpu: { warning: 90, critical: 90 } })).toThrow(BadRequestException); + }); +}); diff --git a/api/src/infrastructure-monitoring/infrastructure-alert-settings.service.ts b/api/src/infrastructure-monitoring/infrastructure-alert-settings.service.ts new file mode 100644 index 0000000..62ab7c7 --- /dev/null +++ b/api/src/infrastructure-monitoring/infrastructure-alert-settings.service.ts @@ -0,0 +1,142 @@ +import { BadRequestException, ConflictException, Injectable, Logger, ServiceUnavailableException } from '@nestjs/common'; +import { ConfigService } from '@nestjs/config'; +import { Prisma } from '@prisma/client'; +import { execFile } from 'node:child_process'; +import { mkdir, readFile, rename, rm, writeFile } from 'node:fs/promises'; +import { dirname } from 'node:path'; +import { promisify } from 'node:util'; +import { PrismaService } from '../prisma/prisma.service'; +import type { InfrastructureAlertSettings, InfrastructureAlertThresholds } from './infrastructure-monitoring.contracts'; + +const execFileAsync = promisify(execFile); + +export const ALERT_THRESHOLD_DEFINITIONS = [ + { key: 'hostCpu', label: '主机 CPU 使用率', unit: '%', min: 1, max: 100, step: 1, warning: 80, critical: 90, expr: '100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100)', names: ['HostCpuUsageWarning', 'HostCpuUsageCritical'], service: 'host', durations: ['10m', '5m'] }, + { key: 'hostMemory', label: '主机内存使用率', unit: '%', min: 1, max: 100, step: 1, warning: 85, critical: 95, expr: '(1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100', names: ['HostMemoryUsageWarning', 'HostMemoryUsageCritical'], service: 'host', durations: ['10m', '5m'] }, + { key: 'hostDisk', label: '根磁盘使用率', unit: '%', min: 1, max: 100, step: 1, warning: 80, critical: 90, expr: '(1 - node_filesystem_avail_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"} / node_filesystem_size_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"}) * 100', names: ['HostRootDiskUsageWarning', 'HostRootDiskUsageCritical'], service: 'host', durations: ['15m', '5m'] }, + { key: 'apiError', label: 'API 5xx 错误率', unit: '%', min: 0.1, max: 100, step: 0.1, warning: 1, critical: 5, expr: '100 * sum(rate(cmpp_api_http_requests_total{status=~"5.."}[5m])) / clamp_min(sum(rate(cmpp_api_http_requests_total[5m])), 0.001)', names: ['CmppApiHttpErrorRateWarning', 'CmppApiHttpErrorRateCritical'], service: 'api', durations: ['5m', '5m'] }, + { key: 'apiLatency', label: 'API P95 响应时间', unit: '秒', min: 0.1, max: 60, step: 0.1, warning: 1, critical: 3, expr: 'histogram_quantile(0.95, sum by (le) (rate(cmpp_api_http_request_duration_seconds_bucket[10m])))', names: ['CmppApiLatencyWarning', 'CmppApiLatencyCritical'], service: 'api', durations: ['10m', '5m'] }, + { key: 'apiEventLoop', label: 'API 事件循环 P99', unit: '秒', min: 0.01, max: 10, step: 0.01, warning: 0.2, critical: 1, expr: 'cmpp_api_nodejs_event_loop_lag_p99_seconds', names: ['CmppApiEventLoopLagWarning', 'CmppApiEventLoopLagCritical'], service: 'api', durations: ['10m', '5m'] }, + { key: 'gatewayQueue', label: 'Gateway 最旧 pending', unit: '秒', min: 1, max: 3600, step: 1, warning: 30, critical: 120, expr: 'cmpp_gateway_submit_queue_oldest_pending_age_seconds', names: ['CmppGatewayQueueDelayedWarning', 'CmppGatewayQueueDelayedCritical'], service: 'gateway', durations: ['2m', '2m'] }, + { key: 'postgresConnections', label: 'PostgreSQL 连接使用率', unit: '%', min: 1, max: 100, step: 1, warning: 70, critical: 85, expr: '100 * sum(pg_stat_activity_count) / clamp_min(max(pg_settings_max_connections), 1)', names: ['PostgresConnectionsWarning', 'PostgresConnectionsCritical'], service: 'postgresql', durations: ['10m', '5m'] }, + { key: 'redisMemory', label: 'Redis 内存使用率', unit: '%', min: 1, max: 100, step: 1, warning: 70, critical: 85, expr: '100 * redis_memory_used_bytes / redis_memory_max_bytes', names: ['RedisMemoryWarning', 'RedisMemoryCritical'], service: 'redis', durations: ['10m', '5m'] }, + { key: 'minioCapacity', label: 'MinIO 容量使用率', unit: '%', min: 1, max: 100, step: 1, warning: 80, critical: 90, expr: '100 * (1 - minio_cluster_capacity_usable_free_bytes / minio_cluster_capacity_usable_total_bytes)', names: ['MinioCapacityWarning', 'MinioCapacityCritical'], service: 'minio', durations: ['15m', '5m'] }, +] as const; + +export const DEFAULT_ALERT_THRESHOLDS: InfrastructureAlertThresholds = Object.fromEntries( + ALERT_THRESHOLD_DEFINITIONS.map((item) => [item.key, { warning: item.warning, critical: item.critical }]), +); + +@Injectable() +export class InfrastructureAlertSettingsService { + private readonly logger = new Logger(InfrastructureAlertSettingsService.name); + private readonly rulesPath: string; + private readonly promtoolPath: string; + private readonly reloadUrl: string; + + constructor(private readonly prisma: PrismaService, config: ConfigService) { + this.rulesPath = String(config.get('PROMETHEUS_MANAGED_RULES_PATH') ?? '/var/lib/cmpp-platform/monitoring/cmpp-managed-alerts.yml'); + this.promtoolPath = String(config.get('PROMTOOL_PATH') ?? '/usr/bin/promtool'); + this.reloadUrl = String(config.get('PROMETHEUS_RELOAD_URL') ?? 'http://127.0.0.1:9090/-/reload'); + } + + async get(): Promise { + const row = await this.prisma.infrastructureAlertSetting.findUnique({ where: { id: 'global' } }); + const thresholds = this.asThresholds(row?.thresholds) ?? DEFAULT_ALERT_THRESHOLDS; + const effective = this.asThresholds(row?.effectiveThresholds) ?? thresholds; + return { + configVersion: row?.configVersion ?? 1, + effectiveVersion: row?.effectiveVersion ?? 1, + applyStatus: (row?.applyStatus as InfrastructureAlertSettings['applyStatus']) ?? 'effective', + lastError: row?.lastError ?? null, + appliedAt: row?.appliedAt?.toISOString() ?? null, + thresholds, + effectiveThresholds: effective, + definitions: ALERT_THRESHOLD_DEFINITIONS.map(({ key, label, unit, min, max, step }) => ({ key, label, unit, min, max, step })), + }; + } + + async update(body: { configVersion?: number; thresholds?: unknown }, operatorId?: string) { + const expectedVersion = Number(body.configVersion); + if (!Number.isInteger(expectedVersion) || expectedVersion < 1) throw new BadRequestException('配置版本无效'); + const thresholds = this.validate(body.thresholds); + const claimed = await this.prisma.infrastructureAlertSetting.updateMany({ + where: { id: 'global', configVersion: expectedVersion }, + data: { configVersion: { increment: 1 }, thresholds: thresholds as Prisma.InputJsonValue, applyStatus: 'applying', lastError: null, updatedById: operatorId }, + }); + // 版本条件更新是跨进程的写锁；避免两个 API 实例同时覆盖规则文件并把旧配置误标成已生效。 + if (claimed.count !== 1) throw new ConflictException('告警阈值已被其他管理员修改，请刷新后重试'); + const nextVersion = expectedVersion + 1; + try { + await this.applyRules(thresholds); + await this.prisma.$transaction([ + this.prisma.infrastructureAlertSetting.update({ where: { id: 'global' }, data: { effectiveVersion: nextVersion, effectiveThresholds: thresholds as Prisma.InputJsonValue, applyStatus: 'effective', lastError: null, appliedAt: new Date() } }), + this.prisma.operationLog.create({ data: { userId: operatorId, action: 'monitoring.alert_thresholds_updated', resource: 'infrastructure_alert_setting', resourceId: 'global', detail: { configVersion: nextVersion, thresholds } } }), + ]); + } catch (error) { + const message = error instanceof Error ? error.message.slice(0, 500) : 'unknown error'; + await this.prisma.infrastructureAlertSetting.update({ where: { id: 'global' }, data: { applyStatus: 'failed', lastError: message } }); + this.logger.error(`Prometheus managed rules apply failed: ${message}`); + throw new ServiceUnavailableException('阈值已保存但 Prometheus 应用失败，原生效规则已保留'); + } + return this.get(); + } + + private validate(value: unknown): InfrastructureAlertThresholds { + if (!value || typeof value !== 'object' || Array.isArray(value)) throw new BadRequestException('告警阈值格式无效'); + const input = value as Record; + if (Object.keys(input).some((key) => !ALERT_THRESHOLD_DEFINITIONS.some((item) => item.key === key))) throw new BadRequestException('存在不允许配置的告警指标'); + const result: InfrastructureAlertThresholds = {}; + for (const definition of ALERT_THRESHOLD_DEFINITIONS) { + const pair = input[definition.key] as { warning?: unknown; critical?: unknown } | undefined; + const warning = Number(pair?.warning); + const critical = Number(pair?.critical); + if (!Number.isFinite(warning) || !Number.isFinite(critical) || warning < definition.min || critical > definition.max || warning >= critical) { + throw new BadRequestException(`${definition.label}必须满足最小值 ≤ 警告阈值 < 严重阈值 ≤ 最大值`); + } + result[definition.key] = { warning, critical }; + } + return result; + } + + private asThresholds(value: unknown) { + try { return this.validate(value); } catch { return null; } + } + + private renderRules(thresholds: InfrastructureAlertThresholds) { + const lines = ['groups:', ' - name: cmpp-managed-thresholds', ' rules:']; + for (const definition of ALERT_THRESHOLD_DEFINITIONS) { + const pair = thresholds[definition.key]; + const values = [pair.warning, pair.critical]; + for (let index = 0; index < 2; index += 1) { + const isWarning = index === 0; + const expr = isWarning ? `(${definition.expr} > ${values[0]}) and (${definition.expr} <= ${values[1]})` : `${definition.expr} > ${values[1]}`; + lines.push(` - alert: ${definition.names[index]}`, ` expr: ${expr}`, ` for: ${definition.durations[index]}`, ' labels:', ` severity: ${isWarning ? 'warning' : 'critical'}`, ` service: ${definition.service}`, ' annotations:', ` summary: "${definition.label}${isWarning ? '达到警告阈值' : '达到严重阈值'}"`, ` description: "${definition.label}持续超过${values[index]}${definition.unit}。"`, ' currentValue: "{{ $value }}"', ` threshold: "${values[index]}${definition.unit}"`); + } + } + return `${lines.join('\n')}\n`; + } + + private async applyRules(thresholds: InfrastructureAlertThresholds) { + const directory = dirname(this.rulesPath); + const temporary = `${this.rulesPath}.${process.pid}.${Date.now()}.tmp`; + await mkdir(directory, { recursive: true }); + const previous = await readFile(this.rulesPath).catch(() => null); + try { + await writeFile(temporary, this.renderRules(thresholds), { mode: 0o640 }); + await execFileAsync(this.promtoolPath, ['check', 'rules', temporary], { timeout: 10_000 }); + await rename(temporary, this.rulesPath); + const response = await fetch(this.reloadUrl, { method: 'POST', signal: AbortSignal.timeout(5_000) }); + if (!response.ok) throw new Error(`Prometheus reload HTTP ${response.status}`); + } catch (error) { + await rm(temporary, { force: true }); + // 规则替换和 reload 不是一个事务，失败时必须恢复旧文件并再次 reload，避免数据库状态与实际告警漂移。 + if (previous) { + await writeFile(temporary, previous, { mode: 0o640 }); + await rename(temporary, this.rulesPath); + await fetch(this.reloadUrl, { method: 'POST', signal: AbortSignal.timeout(5_000) }).catch(() => undefined); + } + throw error; + } + } +} diff --git a/api/src/infrastructure-monitoring/infrastructure-monitoring.contracts.ts b/api/src/infrastructure-monitoring/infrastructure-monitoring.contracts.ts index 2f926cc..e6c20bb 100644 --- a/api/src/infrastructure-monitoring/infrastructure-monitoring.contracts.ts +++ b/api/src/infrastructure-monitoring/infrastructure-monitoring.contracts.ts @@ -76,3 +76,16 @@ export type InfrastructureMonitoringOverview = { serviceMetrics: InfrastructureServiceMetricGroup[]; alerts: InfrastructureAlert[]; }; + +export type InfrastructureAlertThresholds = Record; + +export type InfrastructureAlertSettings = { + configVersion: number; + effectiveVersion: number; + applyStatus: 'effective' | 'applying' | 'failed'; + lastError: string | null; + appliedAt: string | null; + thresholds: InfrastructureAlertThresholds; + effectiveThresholds: InfrastructureAlertThresholds; + definitions: Array<{ key: string; label: string; unit: string; min: number; max: number; step: number }>; +}; diff --git a/api/src/infrastructure-monitoring/infrastructure-monitoring.controller.ts b/api/src/infrastructure-monitoring/infrastructure-monitoring.controller.ts index c4e588a..dbc7e44 100644 --- a/api/src/infrastructure-monitoring/infrastructure-monitoring.controller.ts +++ b/api/src/infrastructure-monitoring/infrastructure-monitoring.controller.ts @@ -1,14 +1,29 @@ -import { Controller, Get, Query } from '@nestjs/common'; +import { Body, Controller, Get, Put, Query } from '@nestjs/common'; import { ApiTags } from '@nestjs/swagger'; +import { CurrentSessionUserId } from '../auth/current-session-user.decorator'; +import { RequireRecentAuthentication } from '../auth/require-recent-authentication.decorator'; +import { InfrastructureAlertSettingsService } from './infrastructure-alert-settings.service'; import { InfrastructureMonitoringService } from './infrastructure-monitoring.service'; @ApiTags('infrastructure-monitoring') @Controller('admin/infrastructure-monitoring') export class InfrastructureMonitoringController { - constructor(private readonly monitoring: InfrastructureMonitoringService) {} + constructor(private readonly monitoring: InfrastructureMonitoringService, private readonly settings: InfrastructureAlertSettingsService) {} @Get('overview') overview(@Query('range') range?: string) { return this.monitoring.overview(range); } + + @Get('notification-summary') + notificationSummary() { return this.monitoring.notificationSummary(); } + + @Get('alert-thresholds') + alertThresholds() { return this.settings.get(); } + + @Put('alert-thresholds') + @RequireRecentAuthentication() + updateAlertThresholds(@Body() body: { configVersion?: number; thresholds?: unknown }, @CurrentSessionUserId() operatorId?: string) { + return this.settings.update(body, operatorId); + } } diff --git a/api/src/infrastructure-monitoring/infrastructure-monitoring.module.ts b/api/src/infrastructure-monitoring/infrastructure-monitoring.module.ts index 243ec14..1fecd49 100644 --- a/api/src/infrastructure-monitoring/infrastructure-monitoring.module.ts +++ b/api/src/infrastructure-monitoring/infrastructure-monitoring.module.ts @@ -1,9 +1,10 @@ import { Module } from '@nestjs/common'; import { InfrastructureMonitoringController } from './infrastructure-monitoring.controller'; import { InfrastructureMonitoringService } from './infrastructure-monitoring.service'; +import { InfrastructureAlertSettingsService } from './infrastructure-alert-settings.service'; @Module({ controllers: [InfrastructureMonitoringController], - providers: [InfrastructureMonitoringService], + providers: [InfrastructureMonitoringService, InfrastructureAlertSettingsService], }) export class InfrastructureMonitoringModule {} diff --git a/api/src/infrastructure-monitoring/infrastructure-monitoring.service.ts b/api/src/infrastructure-monitoring/infrastructure-monitoring.service.ts index 9e22b82..f8d3879 100644 --- a/api/src/infrastructure-monitoring/infrastructure-monitoring.service.ts +++ b/api/src/infrastructure-monitoring/infrastructure-monitoring.service.ts @@ -1,4 +1,4 @@ -import { BadRequestException, Injectable, Logger } from '@nestjs/common'; +import { BadRequestException, Injectable, Logger, ServiceUnavailableException } from '@nestjs/common'; import { ConfigService } from '@nestjs/config'; import { createHash } from 'node:crypto'; import type { @@ -210,6 +210,16 @@ export class InfrastructureMonitoringService { } } + async notificationSummary() { + try { + const alerts = this.parseAlerts(await this.getJson('/api/v1/alerts')); + return { count: alerts.length, criticalCount: alerts.filter((item) => item.severity === 'critical').length }; + } catch (error) { + this.logger.warn(`Prometheus notification summary unavailable: ${error instanceof Error ? error.message : 'unknown error'}`); + throw new ServiceUnavailableException('Prometheus活动告警当前不可用'); + } + } + private parseRange(value?: string): InfrastructureMonitoringRange { const range = value || '24h'; if (!(range in RANGE_CONFIG)) throw new BadRequestException('监控时间范围只支持1h、24h或7d'); diff --git a/docs/codebase-modularization-roadmap.md b/docs/codebase-modularization-roadmap.md index 187765e..875854b 100644 --- a/docs/codebase-modularization-roadmap.md +++ b/docs/codebase-modularization-roadmap.md @@ -1150,4 +1150,4 @@ global，不能错误归入client。`client-signature-*`、发送页、企业认 - `api/src/metrics/`只负责API进程和HTTP路由模板的低基数计数/直方图，不依赖Prisma、Redis或业务Service；独立回环监听由`api/src/main.ts`组装。 - `gateway/internal/metrics/`只负责线程安全聚合和Prometheus文本暴露；上游连接池、下游Session和Submit Worker仅提供总数或有界结果，不把实体ID或凭据交给监控模块。 -- `api/src/infrastructure-monitoring/`仍是运营端只读聚合门面，只消费固定PromQL和Recording Rules；Exporter安装、端口隔离和阈值归`tools/monitoring/`治理。 +- `api/src/infrastructure-monitoring/`是运营端监控聚合与固定阈值应用边界：只消费代码白名单 PromQL，并通过版本化 PostgreSQL 单例、promtool 校验和原子规则热加载管理数值阈值；Exporter安装、端口隔离、固定规则模板和权限仍归`tools/monitoring/`治理。 diff --git a/docs/first-version-development-requirements.md b/docs/first-version-development-requirements.md index b211d29..5a54284 100644 --- a/docs/first-version-development-requirements.md +++ b/docs/first-version-development-requirements.md @@ -2096,3 +2096,6 @@ - 运营端展示API请求/错误/延迟/事件循环、Gateway Submit/队列/连接、PostgreSQL连接/死锁、Redis内存/连接/淘汰、Nginx连接/请求和MinIO可用性；指标缺失显示“待采集”，不以0伪装。 - 告警必须使用持续窗口和最低样本量；默认阈值、收敛关系、标签禁止项和性能预算以`docs/prometheus-system-monitoring-design-20260814.md`第9节为准。 - 指标不得包含手机号、短信正文、短信/CMPP/任务ID、密钥、完整URL或SQL原文；不得把时序指标高频写入业务PostgreSQL。 +- 系统监控标题说明必须明确标注数据来自 Prometheus；“服务关键指标”位于趋势/核心服务区域之后、活动告警之前，并提供统一的“告警阈值设置”入口。安全检测页不重复渲染大号标题，说明文字必须明确标注使用 Fail2ban。 +- 告警阈值仅开放固定指标的警告/严重数值，不允许前端提交 PromQL、标签、文件路径或持续时间；必须满足警告值小于严重值。配置以 PostgreSQL 保存版本、期望值、生效值和应用状态，经 `promtool check rules` 校验、同目录原子替换和 Prometheus 热加载成功后才标记生效，失败保留上一生效规则并展示原因。 +- 右上角预警中心增加“系统监控告警”，通过独立轻量接口统计 Prometheus 当前 firing/pending 告警及严重数，跳转系统监控活动告警区；任一预警域失败不得清空其他域。 diff --git a/docs/prometheus-system-monitoring-design-20260814.md b/docs/prometheus-system-monitoring-design-20260814.md index cc418a9..3db24f0 100644 --- a/docs/prometheus-system-monitoring-design-20260814.md +++ b/docs/prometheus-system-monitoring-design-20260814.md @@ -237,3 +237,9 @@ type InfrastructureOverview = { - API和Gateway请求路径只做内存计数、有界直方图和原子计数，不在业务请求中写PostgreSQL或Redis。 - PostgreSQL Exporter只使用发行版默认低代价查询，不采集SQL原文或扫描业务大表。 - 时序仍受30天和8GB双重上限约束；时序增长时先缩短实际保留期，不允许无界占满业务盘。 + +## 10. 阈值配置与全局预警入口（2026-08-14 增补） + +- 可配置范围固定为主机 CPU/内存/根磁盘、API 5xx/P95/事件循环、Gateway 最旧 pending、PostgreSQL 连接、Redis 内存和 MinIO 容量十组警告/严重数值。PromQL、持续窗口、标签和规则文件路径仍由代码固定，浏览器无权提交。 +- PostgreSQL 单例记录同时保存期望阈值、生效阈值、配置版本、生效版本和 `applying/effective/failed` 状态。更新使用版本条件认领，防止多个 API 实例并发覆盖；规则先经 promtool 校验，再在同一目录原子替换并调用仅回环开放的 `/-/reload`。失败恢复旧文件并保留旧生效版本。 +- 安装器把可配置规则从基础规则中剥离，托管文件归 `cmpp-api:prometheus` 且权限为 0640；Prometheus 仍只监听回环。右上角铃铛只轮询轻量活动告警汇总接口，不重复加载趋势或服务指标。 diff --git a/docs/system-functional-test-cases.md b/docs/system-functional-test-cases.md index 629c747..b2f0b76 100644 --- a/docs/system-functional-test-cases.md +++ b/docs/system-functional-test-cases.md @@ -4675,3 +4675,16 @@ npm run verify:phase8 - 集成验收必须在隔离测试节点或网络 namespace 使用文档保留 IP；不得封禁预生产运维出口、Cloudflare 节点或真实客户 IP。未安装真实 Fail2ban/nftables/Nginx 资产时，只能把相关用例标记阻塞，不得用 Mock 通过代替。 - UI 验收覆盖桌面与窄屏的总览、告警、规则、封禁记录、保护名单、加载、空数据、失败和规则未生效状态；所有数字与操作结果必须能从 API、数据库、agent 与执行器证据交叉验证。 - `TC-F2B-OPS-008`：在非默认`APP_DIR`构建安全代理后执行安装器，核对systemd `ExecStart`与Fail2ban `actionban`均指向同一个真实可执行的`$APP_DIR/dist/cmpp-security-agent`；任一文件残留占位符、旧`current/bin`路径或目标不可执行时，安装/发布必须失败。 + +## 系统监控阈值与预警中心增量用例（2026-08-14） + +| 用例ID | 场景 | 步骤 | 预期 | +| --- | --- | --- | --- | +| TC-INFRA-MON-029 | 模块顺序与技术说明 | 打开系统监控并检查标题和模块顺序 | 说明明确写明 Prometheus；服务关键指标紧邻活动告警上方，活动告警锚点可定位 | +| TC-INFRA-MON-030 | 阈值真实读取 | 打开阈值设置并核对 API 与数据库 | 十组固定指标来自 `InfrastructureAlertSetting`，不使用 Mock/localStorage，不允许编辑 PromQL | +| TC-INFRA-MON-031 | 阈值边界 | 提交警告≥严重、越界、缺项和未知指标 | API 返回 400，数据库版本和 Prometheus 规则均不改变 | +| TC-INFRA-MON-032 | 并发版本 | 两个会话以同一版本先后保存 | 仅第一个原子认领成功，后者返回 409 并提示刷新 | +| TC-INFRA-MON-033 | 规则校验与热加载 | 保存合法阈值，检查 promtool、规则文件、reload 与数据库 | 先校验再同目录原子替换，reload 成功后生效版本前进且写操作日志 | +| TC-INFRA-MON-034 | 应用失败回滚 | 令 promtool 或 reload 失败后保存 | 状态为 failed、展示原因，旧规则文件与旧生效阈值保留，不误报已生效 | +| TC-GLOBAL-ALERT-004 | 系统监控预警入口 | 准备隔离 QA Prometheus firing 告警并点击铃铛 | 第三项显示真实总数/严重数，角标计入三域总和，点击跳转系统监控活动告警区 | +| TC-SECURITY-UI-001 | Fail2ban 标识与标题规范 | 打开安全检测与封禁 | 不出现重复大号页面标题，说明明确写明使用 Fail2ban，字号遵循通用菜单标题 | diff --git a/docs/testing-progress.md b/docs/testing-progress.md index 7be52ba..1fe2f97 100644 --- a/docs/testing-progress.md +++ b/docs/testing-progress.md @@ -3658,3 +3658,9 @@ git diff --check - 测试机静态包只读核对显示当前已部署“服务关键指标”，但尚未包含“重复投递风险”和已确认重投的新表单文案，证明服务指标会话使用定向发布，没有把尚未授权部署的下游重投改动意外带入测试机；两项发布记录保持一致。 - 合并后统一回归通过：API全量41套/477项、前后端TypeScript、API正式编译、Vite 8.1.5生产构建（2549 modules）、Gateway全量`go test ./... -count=1`与`go vet ./...`、SendChain R10、生产部署和安全代理静态契约、5个Shell脚本语法、真实本地PostgreSQL 88/88 migration状态及`git diff --check`。Vite仅保留既有约2.11MiB单chunk提示；Operations R2仍因本轮开始前已有的`sendQuality`查询哈希漂移失败，与本次合并文件无交集。 - 本次合并没有发送、补发或重投短信，没有创建后台重投任务，没有修改通道账号、密码、启停状态、企业余额或客户连接；提交范围继续排除`api/tsconfig.build.tsbuildinfo`、`tsconfig.tsbuildinfo`、`outputs/`和空文件`=`。 + +# 2026-08-14 系统监控阈值与预警入口优化（进行中） + +- 系统监控将“服务关键指标”移至活动告警正上方，标题说明明确使用 Prometheus；新增十组固定指标警告/严重阈值设置。配置真实写入 PostgreSQL，使用版本条件并发认领，promtool 校验、原子替换和回环 reload 成功后才推进生效版本，失败保留旧生效值。 +- 右上角预警中心新增“系统监控告警”及数量/严重数，读取独立 Prometheus 活动告警汇总并跳转活动告警锚点；继续使用 `Promise.allSettled` 隔离签名、安全与监控域故障。安全检测页移除重复大号标题，说明明确使用 Fail2ban。 +- 新增 migration `20260814173000_add_infrastructure_alert_settings`。本机 Prisma Client 生成、前后端 TypeScript 与 API 正式编译已通过；测试机恢复资产、migration、Prometheus规则校验、真实告警数据、浏览器验收和提交/部署结果待本轮后续补记。 diff --git a/src/api/admin/infrastructure-monitoring.api.ts b/src/api/admin/infrastructure-monitoring.api.ts index 92dab94..7e0ecc5 100644 --- a/src/api/admin/infrastructure-monitoring.api.ts +++ b/src/api/admin/infrastructure-monitoring.api.ts @@ -1,7 +1,10 @@ import { request, withQuery } from '../core/httpClient'; -import type { InfrastructureMonitoringOverview, InfrastructureMonitoringRange } from '../types'; +import type { InfrastructureAlertSettings, InfrastructureAlertThresholds, InfrastructureMonitoringOverview, InfrastructureMonitoringRange } from '../types'; export const adminInfrastructureMonitoringApi = { getInfrastructureMonitoringOverview: (range: InfrastructureMonitoringRange) => request(withQuery('/admin/infrastructure-monitoring/overview', { range })), + getInfrastructureMonitoringNotificationSummary: () => request<{ count: number; criticalCount: number }>('/admin/infrastructure-monitoring/notification-summary'), + getInfrastructureAlertThresholds: () => request('/admin/infrastructure-monitoring/alert-thresholds'), + updateInfrastructureAlertThresholds: (body: { configVersion: number; thresholds: InfrastructureAlertThresholds }) => request('/admin/infrastructure-monitoring/alert-thresholds', { method: 'PUT', body: JSON.stringify(body) }), }; diff --git a/src/api/types/infrastructure-monitoring.ts b/src/api/types/infrastructure-monitoring.ts index 2f926cc..e6c20bb 100644 --- a/src/api/types/infrastructure-monitoring.ts +++ b/src/api/types/infrastructure-monitoring.ts @@ -76,3 +76,16 @@ export type InfrastructureMonitoringOverview = { serviceMetrics: InfrastructureServiceMetricGroup[]; alerts: InfrastructureAlert[]; }; + +export type InfrastructureAlertThresholds = Record; + +export type InfrastructureAlertSettings = { + configVersion: number; + effectiveVersion: number; + applyStatus: 'effective' | 'applying' | 'failed'; + lastError: string | null; + appliedAt: string | null; + thresholds: InfrastructureAlertThresholds; + effectiveThresholds: InfrastructureAlertThresholds; + definitions: Array<{ key: string; label: string; unit: string; min: number; max: number; step: number }>; +}; diff --git a/src/apps/admin/security-detection/AdminSecurityDetectionPage.css b/src/apps/admin/security-detection/AdminSecurityDetectionPage.css index fa07c6f..5289943 100644 --- a/src/apps/admin/security-detection/AdminSecurityDetectionPage.css +++ b/src/apps/admin/security-detection/AdminSecurityDetectionPage.css @@ -1 +1,2 @@ .admin-security-page{gap:20px}.security-heading{align-items:flex-end}.security-heading h1{margin:10px 0 4px;font-size:26px}.security-heading p{margin:0;color:#64748b}.security-kpis{display:grid;grid-template-columns:repeat(4,minmax(0,1fr));gap:14px}.security-kpi{display:grid;grid-template-columns:42px 1fr;grid-template-rows:auto auto;gap:3px 12px;padding:18px}.security-kpi>div{grid-row:1/3;width:42px;height:42px;border-radius:12px;display:grid;place-items:center;background:#e8f0ff;color:#2563eb}.security-kpi>div svg{width:20px}.security-kpi span{font-size:13px;color:#64748b}.security-kpi strong{font-size:24px;line-height:1.1}.security-kpi.is-danger>div{background:#fef2f2;color:#dc2626}.security-overview-grid{display:grid;grid-template-columns:minmax(0,1fr) minmax(360px,.8fr);gap:16px}.security-chart,.security-latest,.security-table-card{padding:18px}.security-chart header,.security-latest header{display:flex;justify-content:space-between}.security-chart header span,.security-latest header span{font-size:12px;color:#94a3b8}.security-latest-row{display:grid;grid-template-columns:10px 1fr auto;gap:10px;align-items:center;padding:13px 0;border-bottom:1px solid #eef2f7}.security-latest-row div{display:grid;gap:3px}.security-latest-row small,.security-latest-row time{color:#64748b;font-size:12px}.severity-dot{width:8px;height:8px;border-radius:99px;background:#3b82f6}.severity-dot.is-high{background:#f59e0b}.severity-dot.is-critical{background:#ef4444}.security-cell{display:grid;gap:3px}.security-cell span{font-size:11px;color:#94a3b8}.security-actions{display:flex;gap:6px}.security-note,.security-section-toolbar{display:flex;align-items:center;gap:9px;margin-bottom:16px;padding:12px 14px;border-radius:10px;background:#f8fafc;color:#475569;font-size:13px}.security-section-toolbar{justify-content:space-between}.security-error{display:flex;gap:8px;align-items:center;padding:12px 14px;border:1px solid #fecaca;border-radius:10px;background:#fef2f2;color:#b91c1c}.security-empty{height:265px;display:grid;place-items:center;color:#94a3b8}.security-dialog{display:grid;gap:16px}.security-target{display:grid;gap:4px;padding:14px;border-radius:10px;background:#f8fafc}.security-target span,.security-target small{color:#64748b;font-size:12px}.security-target strong{font-family:ui-monospace,monospace;font-size:18px}.security-form-grid{display:grid;grid-template-columns:1fr 1fr;gap:16px}.security-form-grid .security-error{grid-column:1/-1}.admin-security-page code{font-size:12px;color:#334155}@media(max-width:1100px){.security-kpis{grid-template-columns:repeat(2,1fr)}.security-overview-grid{grid-template-columns:1fr}}@media(max-width:640px){.security-kpis{grid-template-columns:1fr}.security-form-grid{grid-template-columns:1fr}} +.security-heading p{margin:10px 0 0} diff --git a/src/apps/admin/security-detection/AdminSecurityDetectionPage.tsx b/src/apps/admin/security-detection/AdminSecurityDetectionPage.tsx index c472792..a3b777a 100644 --- a/src/apps/admin/security-detection/AdminSecurityDetectionPage.tsx +++ b/src/apps/admin/security-detection/AdminSecurityDetectionPage.tsx @@ -76,7 +76,7 @@ export function AdminSecurityDetectionPage() { { value: 'protected', label: '保护名单', content:
受保护的运维出口、内网和可信代理永远不能从面板封禁。
}, ]; - return

安全检测与封禁

Fail2ban 与业务事件只负责检测，由运营人员复核后人工处置

+ return

使用 Fail2ban 与平台业务安全事件进行检测，由运营人员复核后人工处置

{error ?
{error}
: null}
} label="24小时检测事件" value={overview?.totalEvents ?? 0} />} label="待处置告警" value={overview?.activeAlerts ?? 0} danger={Boolean(overview?.criticalAlerts)} />} label="生效封禁" value={overview?.activeBlocks ?? 0} /> : } label="安全代理" value={overview?.health.agent === 'healthy' ? '在线' : '不可用'} danger={overview?.health.agent !== 'healthy'} />
diff --git a/src/apps/admin/system-monitoring/AdminSystemMonitoringPage.css b/src/apps/admin/system-monitoring/AdminSystemMonitoringPage.css index 3ab4b14..013b7b3 100644 --- a/src/apps/admin/system-monitoring/AdminSystemMonitoringPage.css +++ b/src/apps/admin/system-monitoring/AdminSystemMonitoringPage.css @@ -321,3 +321,16 @@ @media (prefers-reduced-motion: reduce) { .is-spinning { animation: none; } } + +.system-monitoring-service-actions { align-items: center; display: flex; gap: 12px; } +.system-monitoring-threshold-dialog { display: grid; gap: 12px; max-height: 62vh; overflow: auto; padding-right: 4px; } +.system-monitoring-threshold-note { align-items: flex-start; background: #eff6ff; border-radius: 10px; color: #475569; display: flex; font-size: 13px; gap: 9px; padding: 12px 14px; } +.system-monitoring-threshold-row { align-items: end; border-bottom: 1px solid #eef2f7; display: grid; gap: 14px; grid-template-columns: minmax(180px, 1fr) 150px 150px; padding: 12px 0; } +.system-monitoring-threshold-row > div:first-child { align-self: center; display: grid; gap: 4px; } +.system-monitoring-threshold-row small { color: #64748b; } + +@media (max-width: 760px) { + .system-monitoring-service-actions { align-items: flex-start; flex-direction: column; } + .system-monitoring-threshold-row { grid-template-columns: 1fr 1fr; } + .system-monitoring-threshold-row > div:first-child { grid-column: 1 / -1; } +} diff --git a/src/apps/admin/system-monitoring/AdminSystemMonitoringPage.tsx b/src/apps/admin/system-monitoring/AdminSystemMonitoringPage.tsx index 24ab184..40af230 100644 --- a/src/apps/admin/system-monitoring/AdminSystemMonitoringPage.tsx +++ b/src/apps/admin/system-monitoring/AdminSystemMonitoringPage.tsx @@ -12,16 +12,19 @@ import { Network, RefreshCw, Server, + Settings2, ShieldAlert, } from 'lucide-react'; import { adminApi, type InfrastructureAlert, + type InfrastructureAlertSettings, + type InfrastructureAlertThresholds, type InfrastructureMetricPoint, type InfrastructureMonitoringOverview, type InfrastructureMonitoringRange, } from '@/api/adminApi'; -import { Breadcrumb, Button, Chart, Table, Tag, type TableColumn } from '@/components/ui'; +import { Breadcrumb, Button, Chart, Input, Modal, Table, Tag, type TableColumn } from '@/components/ui'; import './AdminSystemMonitoringPage.css'; const RANGE_OPTIONS: Array<{ value: InfrastructureMonitoringRange; label: string }> = [ @@ -166,6 +169,11 @@ export function AdminSystemMonitoringPage() { const [overview, setOverview] = useState(null); const [loading, setLoading] = useState(true); const [error, setError] = useState(''); + const [settings, setSettings] = useState(null); + const [draftThresholds, setDraftThresholds] = useState({}); + const [showSettings, setShowSettings] = useState(false); + const [settingsError, setSettingsError] = useState(''); + const [savingSettings, setSavingSettings] = useState(false); const requestSequence = useRef(0); const pendingRequests = useRef(0); @@ -189,8 +197,38 @@ export function AdminSystemMonitoringPage() { } }, [range]); + const loadSettings = useCallback(async () => { + try { + const result = await adminApi.getInfrastructureAlertThresholds(); + setSettings(result); + setDraftThresholds(result.thresholds); + setSettingsError(''); + } catch (reason) { + setSettingsError(reason instanceof Error ? reason.message : '告警阈值加载失败'); + } + }, []); + + const saveSettings = useCallback(async () => { + if (!settings) return; + setSavingSettings(true); + setSettingsError(''); + try { + const result = await adminApi.updateInfrastructureAlertThresholds({ configVersion: settings.configVersion, thresholds: draftThresholds }); + setSettings(result); + setDraftThresholds(result.thresholds); + setShowSettings(false); + window.dispatchEvent(new Event('cmpp-infrastructure-alert-count-refresh')); + await loadData(true); + } catch (reason) { + setSettingsError(reason instanceof Error ? reason.message : '告警阈值保存失败'); + } finally { + setSavingSettings(false); + } + }, [draftThresholds, loadData, settings]); + useEffect(() => { void loadData(true); + void loadSettings(); const intervalId = window.setInterval(() => { if (document.visibilityState === 'visible') void loadData(); }, 30_000); @@ -203,7 +241,7 @@ export function AdminSystemMonitoringPage() { window.clearInterval(intervalId); document.removeEventListener('visibilitychange', handleVisibility); }; - }, [loadData]); + }, [loadData, loadSettings]); const status = STATUS_COPY[overview?.summary.overallStatus ?? 'unknown']; const cpuOption = useMemo(() => makeTrendOption({ @@ -232,7 +270,7 @@ export function AdminSystemMonitoringPage() {
-

服务器资源、核心服务与活动告警

+

服务器资源、核心服务与活动告警，数据由 Prometheus 采集与计算

{status.label}
@@ -283,29 +321,6 @@ export function AdminSystemMonitoringPage() {
网络吞吐{formatRate(totalNetworkRate(metrics?.networkReceiveBytesPerSecond, metrics?.networkTransmitBytesPerSecond))}接收 {formatRate(metrics?.networkReceiveBytesPerSecond ?? null)} · 发送 {formatRate(metrics?.networkTransmitBytesPerSecond ?? null)}
-
-
-
服务关键指标
- 固定低基数聚合，不含手机号、短信ID或SQL文本 -
-
- {(overview?.serviceMetrics ?? []).map((group) => ( -
-
- {group.name} - {group.available ? '已采集' : '待采集'} -
- {group.metrics.length ? group.metrics.map((metric) => ( -
- {metric.label} - {formatServiceMetric(metric.value, metric.unit)} -
- )) :
已监控服务可用性，待原生容量指标接入
} -
- ))} -
-
-
CPU 趋势
{formatPercent(metrics?.cpuUsagePercent ?? null)}
{overview?.trends.cpuUsagePercent.length ? : }
@@ -332,10 +347,40 @@ export function AdminSystemMonitoringPage() {
-
+
+
+
服务关键指标
+
固定低基数聚合，不含手机号、短信ID或SQL文本
+
+
+ {(overview?.serviceMetrics ?? []).map((group) => ( +
+
{group.name}{group.available ? '已采集' : '待采集'}
+ {group.metrics.length ? group.metrics.map((metric) =>
{metric.label}{formatServiceMetric(metric.value, metric.unit)}
) :
已监控服务可用性，待原生容量指标接入
} +
+ ))} +
+
+ +
活动告警{overview?.summary.activeAlerts ?? 0}
刷新于 {formatTime(overview?.collectedAt ?? null)}
+ + } onClose={() => setShowSettings(false)} open={showSettings} title="Prometheus 告警阈值设置"> +
+
仅允许修改固定指标的数值阈值。保存时先由 promtool 校验，再原子替换规则并热加载 Prometheus。
+ {settings?.definitions.map((definition) => ( +
+
{definition.label}单位：{definition.unit}
+ setDraftThresholds((current) => ({ ...current, [definition.key]: { ...current[definition.key], warning: Number(event.target.value) } }))} step={definition.step} type="number" value={draftThresholds[definition.key]?.warning ?? ''} /> + setDraftThresholds((current) => ({ ...current, [definition.key]: { ...current[definition.key], critical: Number(event.target.value) } }))} step={definition.step} type="number" value={draftThresholds[definition.key]?.critical ?? ''} /> +
+ ))} + {settings?.applyStatus === 'failed' ?
上次应用失败{settings.lastError}
: null} + {settingsError ?
阈值配置不可用{settingsError}
: null} +
+ ); } diff --git a/src/layouts/AdminLayout.tsx b/src/layouts/AdminLayout.tsx index 57c9631..ce83ee0 100644 --- a/src/layouts/AdminLayout.tsx +++ b/src/layouts/AdminLayout.tsx @@ -49,6 +49,7 @@ function AdminAuthenticatedLayout({ session }: { session: LoginSession }) { const [pendingAudits, setPendingAudits] = useState(EMPTY_PENDING_AUDITS); const [retirementUnreadCount, setRetirementUnreadCount] = useState(0); const [securityAlertSummary, setSecurityAlertSummary] = useState({ count: 0, criticalCount: 0 }); + const [infrastructureAlertSummary, setInfrastructureAlertSummary] = useState({ count: 0, criticalCount: 0 }); const [sessionLocked, setSessionLocked] = useState(Boolean(session.locked)); const loadPendingAuditCount = useCallback(() => { const currentSession = readSession('admin'); @@ -57,16 +58,18 @@ function AdminAuthenticatedLayout({ session }: { session: LoginSession }) { return; } // This runs globally and on a timer, so it must not fan out through the full dashboard aggregation. - Promise.allSettled([adminApi.getPendingAudits(), adminApi.getSignatureRetirementUnreadCount(), adminApi.getSecurityNotificationSummary()]) - .then(([audits, retirement, security]) => { + Promise.allSettled([adminApi.getPendingAudits(), adminApi.getSignatureRetirementUnreadCount(), adminApi.getSecurityNotificationSummary(), adminApi.getInfrastructureMonitoringNotificationSummary()]) + .then(([audits, retirement, security, infrastructure]) => { setPendingAudits(audits.status === 'fulfilled' ? audits.value : EMPTY_PENDING_AUDITS); setRetirementUnreadCount(retirement.status === 'fulfilled' ? retirement.value.count : 0); setSecurityAlertSummary(security.status === 'fulfilled' ? security.value : { count: 0, criticalCount: 0 }); + setInfrastructureAlertSummary(infrastructure.status === 'fulfilled' ? infrastructure.value : { count: 0, criticalCount: 0 }); }) .catch(() => { setPendingAudits(EMPTY_PENDING_AUDITS); setRetirementUnreadCount(0); setSecurityAlertSummary({ count: 0, criticalCount: 0 }); + setInfrastructureAlertSummary({ count: 0, criticalCount: 0 }); }); }, []); @@ -82,12 +85,14 @@ function AdminAuthenticatedLayout({ session }: { session: LoginSession }) { window.addEventListener('cmpp-audit-count-refresh', onAuditRefresh); window.addEventListener('cmpp-retirement-count-refresh', onAuditRefresh); window.addEventListener('cmpp-security-alert-count-refresh', onAuditRefresh); + window.addEventListener('cmpp-infrastructure-alert-count-refresh', onAuditRefresh); return () => { window.clearInterval(timer); window.removeEventListener('focus', onFocus); window.removeEventListener('cmpp-audit-count-refresh', onAuditRefresh); window.removeEventListener('cmpp-retirement-count-refresh', onAuditRefresh); window.removeEventListener('cmpp-security-alert-count-refresh', onAuditRefresh); + window.removeEventListener('cmpp-infrastructure-alert-count-refresh', onAuditRefresh); }; }, [loadPendingAuditCount, session.portal, sessionLocked]); @@ -104,6 +109,7 @@ function AdminAuthenticatedLayout({ session }: { session: LoginSession }) { alertNotifications={[ { label: '签名清退预警', count: retirementUnreadCount, description: '今日未读且未抑制', to: '/admin/signature-retirement' }, { label: '安全检测与封禁', count: securityAlertSummary.count, description: securityAlertSummary.criticalCount > 0 ? `${securityAlertSummary.criticalCount} 条严重告警待处置` : '待处置安全告警', to: '/admin/security-detection' }, + { label: '系统监控告警', count: infrastructureAlertSummary.count, description: infrastructureAlertSummary.criticalCount > 0 ? `${infrastructureAlertSummary.criticalCount} 条 Prometheus 严重告警` : 'Prometheus 活动告警', to: '/admin/system-monitoring#active-alerts' }, ]} auditNotifications={[ { label: '企业认证待审', count: pendingAudits.enterpriseCertifications, to: '/admin/enterprise-audit' }, diff --git a/tools/monitoring/cmpp-managed-alerts.yml b/tools/monitoring/cmpp-managed-alerts.yml new file mode 100644 index 0000000..e992164 --- /dev/null +++ b/tools/monitoring/cmpp-managed-alerts.yml @@ -0,0 +1,103 @@ +groups: + - name: cmpp-managed-thresholds + rules: + - alert: HostCpuUsageWarning + expr: (100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 80) and (100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) <= 90) + for: 10m + labels: { severity: warning, service: host } + annotations: { summary: "主机 CPU 使用率达到警告阈值", description: "主机 CPU 使用率持续超过80%。", currentValue: "{{ $value }}", threshold: "80%" } + - alert: HostCpuUsageCritical + expr: 100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 90 + for: 5m + labels: { severity: critical, service: host } + annotations: { summary: "主机 CPU 使用率达到严重阈值", description: "主机 CPU 使用率持续超过90%。", currentValue: "{{ $value }}", threshold: "90%" } + - alert: HostMemoryUsageWarning + expr: ((1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100 > 85) and ((1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100 <= 95) + for: 10m + labels: { severity: warning, service: host } + annotations: { summary: "主机内存使用率达到警告阈值", description: "主机内存使用率持续超过85%。", currentValue: "{{ $value }}", threshold: "85%" } + - alert: HostMemoryUsageCritical + expr: (1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100 > 95 + for: 5m + labels: { severity: critical, service: host } + annotations: { summary: "主机内存使用率达到严重阈值", description: "主机内存使用率持续超过95%。", currentValue: "{{ $value }}", threshold: "95%" } + - alert: HostRootDiskUsageWarning + expr: ((1 - node_filesystem_avail_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"} / node_filesystem_size_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"}) * 100 > 80) and ((1 - node_filesystem_avail_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"} / node_filesystem_size_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"}) * 100 <= 90) + for: 15m + labels: { severity: warning, service: host } + annotations: { summary: "根磁盘使用率达到警告阈值", description: "根磁盘使用率持续超过80%。", currentValue: "{{ $value }}", threshold: "80%" } + - alert: HostRootDiskUsageCritical + expr: (1 - node_filesystem_avail_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"} / node_filesystem_size_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"}) * 100 > 90 + for: 5m + labels: { severity: critical, service: host } + annotations: { summary: "根磁盘使用率达到严重阈值", description: "根磁盘使用率持续超过90%。", currentValue: "{{ $value }}", threshold: "90%" } + - alert: CmppApiHttpErrorRateWarning + expr: (100 * sum(rate(cmpp_api_http_requests_total{status=~"5.."}[5m])) / clamp_min(sum(rate(cmpp_api_http_requests_total[5m])), 0.001) > 1) and (100 * sum(rate(cmpp_api_http_requests_total{status=~"5.."}[5m])) / clamp_min(sum(rate(cmpp_api_http_requests_total[5m])), 0.001) <= 5) + for: 5m + labels: { severity: warning, service: api } + annotations: { summary: "API 5xx 错误率达到警告阈值", description: "API 5xx 错误率持续超过1%。", currentValue: "{{ $value }}", threshold: "1%" } + - alert: CmppApiHttpErrorRateCritical + expr: 100 * sum(rate(cmpp_api_http_requests_total{status=~"5.."}[5m])) / clamp_min(sum(rate(cmpp_api_http_requests_total[5m])), 0.001) > 5 + for: 5m + labels: { severity: critical, service: api } + annotations: { summary: "API 5xx 错误率达到严重阈值", description: "API 5xx 错误率持续超过5%。", currentValue: "{{ $value }}", threshold: "5%" } + - alert: CmppApiLatencyWarning + expr: (histogram_quantile(0.95, sum by (le) (rate(cmpp_api_http_request_duration_seconds_bucket[10m]))) > 1) and (histogram_quantile(0.95, sum by (le) (rate(cmpp_api_http_request_duration_seconds_bucket[10m]))) <= 3) + for: 10m + labels: { severity: warning, service: api } + annotations: { summary: "API P95 响应时间达到警告阈值", description: "API P95 响应时间持续超过1秒。", currentValue: "{{ $value }}", threshold: "1秒" } + - alert: CmppApiLatencyCritical + expr: histogram_quantile(0.95, sum by (le) (rate(cmpp_api_http_request_duration_seconds_bucket[10m]))) > 3 + for: 5m + labels: { severity: critical, service: api } + annotations: { summary: "API P95 响应时间达到严重阈值", description: "API P95 响应时间持续超过3秒。", currentValue: "{{ $value }}", threshold: "3秒" } + - alert: CmppApiEventLoopLagWarning + expr: (cmpp_api_nodejs_event_loop_lag_p99_seconds > 0.2) and (cmpp_api_nodejs_event_loop_lag_p99_seconds <= 1) + for: 10m + labels: { severity: warning, service: api } + annotations: { summary: "API 事件循环 P99 达到警告阈值", description: "API 事件循环 P99 持续超过0.2秒。", currentValue: "{{ $value }}", threshold: "0.2秒" } + - alert: CmppApiEventLoopLagCritical + expr: cmpp_api_nodejs_event_loop_lag_p99_seconds > 1 + for: 5m + labels: { severity: critical, service: api } + annotations: { summary: "API 事件循环 P99 达到严重阈值", description: "API 事件循环 P99 持续超过1秒。", currentValue: "{{ $value }}", threshold: "1秒" } + - alert: CmppGatewayQueueDelayedWarning + expr: (cmpp_gateway_submit_queue_oldest_pending_age_seconds > 30) and (cmpp_gateway_submit_queue_oldest_pending_age_seconds <= 120) + for: 2m + labels: { severity: warning, service: gateway } + annotations: { summary: "Gateway 最旧 pending 达到警告阈值", description: "Gateway 最旧 pending 持续超过30秒。", currentValue: "{{ $value }}", threshold: "30秒" } + - alert: CmppGatewayQueueDelayedCritical + expr: cmpp_gateway_submit_queue_oldest_pending_age_seconds > 120 + for: 2m + labels: { severity: critical, service: gateway } + annotations: { summary: "Gateway 最旧 pending 达到严重阈值", description: "Gateway 最旧 pending 持续超过120秒。", currentValue: "{{ $value }}", threshold: "120秒" } + - alert: PostgresConnectionsWarning + expr: (100 * sum(pg_stat_activity_count) / clamp_min(max(pg_settings_max_connections), 1) > 70) and (100 * sum(pg_stat_activity_count) / clamp_min(max(pg_settings_max_connections), 1) <= 85) + for: 10m + labels: { severity: warning, service: postgresql } + annotations: { summary: "PostgreSQL 连接使用率达到警告阈值", description: "PostgreSQL 连接使用率持续超过70%。", currentValue: "{{ $value }}", threshold: "70%" } + - alert: PostgresConnectionsCritical + expr: 100 * sum(pg_stat_activity_count) / clamp_min(max(pg_settings_max_connections), 1) > 85 + for: 5m + labels: { severity: critical, service: postgresql } + annotations: { summary: "PostgreSQL 连接使用率达到严重阈值", description: "PostgreSQL 连接使用率持续超过85%。", currentValue: "{{ $value }}", threshold: "85%" } + - alert: RedisMemoryWarning + expr: (100 * redis_memory_used_bytes / redis_memory_max_bytes > 70) and (100 * redis_memory_used_bytes / redis_memory_max_bytes <= 85) + for: 10m + labels: { severity: warning, service: redis } + annotations: { summary: "Redis 内存使用率达到警告阈值", description: "Redis 内存使用率持续超过70%。", currentValue: "{{ $value }}", threshold: "70%" } + - alert: RedisMemoryCritical + expr: 100 * redis_memory_used_bytes / redis_memory_max_bytes > 85 + for: 5m + labels: { severity: critical, service: redis } + annotations: { summary: "Redis 内存使用率达到严重阈值", description: "Redis 内存使用率持续超过85%。", currentValue: "{{ $value }}", threshold: "85%" } + - alert: MinioCapacityWarning + expr: (100 * (1 - minio_cluster_capacity_usable_free_bytes / minio_cluster_capacity_usable_total_bytes) > 80) and (100 * (1 - minio_cluster_capacity_usable_free_bytes / minio_cluster_capacity_usable_total_bytes) <= 90) + for: 15m + labels: { severity: warning, service: minio } + annotations: { summary: "MinIO 容量使用率达到警告阈值", description: "MinIO 容量使用率持续超过80%。", currentValue: "{{ $value }}", threshold: "80%" } + - alert: MinioCapacityCritical + expr: 100 * (1 - minio_cluster_capacity_usable_free_bytes / minio_cluster_capacity_usable_total_bytes) > 90 + for: 5m + labels: { severity: critical, service: minio } + annotations: { summary: "MinIO 容量使用率达到严重阈值", description: "MinIO 容量使用率持续超过90%。", currentValue: "{{ $value }}", threshold: "90%" } diff --git a/tools/monitoring/install-prometheus-monitoring.sh b/tools/monitoring/install-prometheus-monitoring.sh index 75264aa..c2d56f4 100644 --- a/tools/monitoring/install-prometheus-monitoring.sh +++ b/tools/monitoring/install-prometheus-monitoring.sh @@ -26,7 +26,7 @@ promtool_bin="$(command -v promtool)" backup_dir="/etc/prometheus/cmpp-backups/$(date '+%Y%m%d-%H%M%S')" mkdir -p "$backup_dir" /etc/systemd/system/prometheus.service.d /etc/systemd/system/prometheus-node-exporter.service.d -for config_file in /etc/prometheus/prometheus.yml /etc/prometheus/cmpp-alerts.yml; do +for config_file in /etc/prometheus/prometheus.yml /etc/prometheus/cmpp-alerts.yml /etc/prometheus/cmpp-alerts-source.yml; do if [[ -f "$config_file" ]]; then cp --preserve=mode,timestamps "$config_file" "$backup_dir/$(basename "$config_file")" fi @@ -40,12 +40,28 @@ fi log "Installing platform-owned scrape and alert configuration" install -o root -g root -m 0644 "$SCRIPT_DIR/prometheus.yml" /etc/prometheus/prometheus.yml -install -o root -g root -m 0644 "$SCRIPT_DIR/cmpp-alerts.yml" /etc/prometheus/cmpp-alerts.yml +install -o root -g root -m 0644 "$SCRIPT_DIR/cmpp-alerts.yml" /etc/prometheus/cmpp-alerts-source.yml +# 可配置规则由 API 管理；基础规则必须排除同名项，否则 Prometheus 会同时计算旧阈值和新阈值。 +awk ' +BEGIN { + split("HostCpuUsageWarning HostCpuUsageCritical HostMemoryUsageWarning HostMemoryUsageCritical HostRootDiskUsageWarning HostRootDiskUsageCritical CmppApiHttpErrorRateWarning CmppApiHttpErrorRateCritical CmppApiLatencyWarning CmppApiLatencyCritical CmppApiEventLoopLagWarning CmppApiEventLoopLagCritical CmppGatewayQueueDelayedWarning CmppGatewayQueueDelayedCritical PostgresConnectionsWarning PostgresConnectionsCritical RedisMemoryWarning RedisMemoryCritical MinioCapacityWarning MinioCapacityCritical", names, " ") + for (i in names) dropped[names[i]] = 1 +} +/^ - name:/ { skip = 0 } +/^ - alert:/ { skip = ($3 in dropped) } +!skip { print } +' "$SCRIPT_DIR/cmpp-alerts.yml" > /etc/prometheus/cmpp-alerts.yml +chown root:root /etc/prometheus/cmpp-alerts.yml +chmod 0644 /etc/prometheus/cmpp-alerts.yml +install -d -o cmpp-api -g prometheus -m 0750 /var/lib/cmpp-platform/monitoring +if [[ ! -f /var/lib/cmpp-platform/monitoring/cmpp-managed-alerts.yml ]]; then + install -o cmpp-api -g prometheus -m 0640 "$SCRIPT_DIR/cmpp-managed-alerts.yml" /var/lib/cmpp-platform/monitoring/cmpp-managed-alerts.yml +fi cat >/etc/systemd/system/prometheus.service.d/cmpp-monitoring.conf </etc/systemd/system/prometheus-node-exporter.service.d/cmpp-monitoring.conf <