feat: add configurable infrastructure alerts
This commit is contained in:
hectorzhao
@@ -0,0 +1,23 @@
|
||||
CREATE TABLE "InfrastructureAlertSetting" (
|
||||
"id" TEXT NOT NULL DEFAULT 'global',
|
||||
"configVersion" INTEGER NOT NULL DEFAULT 1,
|
||||
"effectiveVersion" INTEGER NOT NULL DEFAULT 1,
|
||||
"thresholds" JSONB NOT NULL,
|
||||
"effectiveThresholds" JSONB NOT NULL,
|
||||
"applyStatus" TEXT NOT NULL DEFAULT 'effective',
|
||||
"lastError" TEXT,
|
||||
"updatedById" TEXT,
|
||||
"appliedAt" TIMESTAMP(3),
|
||||
"createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
"updatedAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
CONSTRAINT "InfrastructureAlertSetting_pkey" PRIMARY KEY ("id")
|
||||
);
|
||||
|
||||
INSERT INTO "InfrastructureAlertSetting" (
|
||||
"id", "thresholds", "effectiveThresholds", "appliedAt"
|
||||
) VALUES (
|
||||
'global',
|
||||
'{"hostCpu":{"warning":80,"critical":90},"hostMemory":{"warning":85,"critical":95},"hostDisk":{"warning":80,"critical":90},"apiError":{"warning":1,"critical":5},"apiLatency":{"warning":1,"critical":3},"apiEventLoop":{"warning":0.2,"critical":1},"gatewayQueue":{"warning":30,"critical":120},"postgresConnections":{"warning":70,"critical":85},"redisMemory":{"warning":70,"critical":85},"minioCapacity":{"warning":80,"critical":90}}'::jsonb,
|
||||
'{"hostCpu":{"warning":80,"critical":90},"hostMemory":{"warning":85,"critical":95},"hostDisk":{"warning":80,"critical":90},"apiError":{"warning":1,"critical":5},"apiLatency":{"warning":1,"critical":3},"apiEventLoop":{"warning":0.2,"critical":1},"gatewayQueue":{"warning":30,"critical":120},"postgresConnections":{"warning":70,"critical":85},"redisMemory":{"warning":70,"critical":85},"minioCapacity":{"warning":80,"critical":90}}'::jsonb,
|
||||
CURRENT_TIMESTAMP
|
||||
);
|
||||
@@ -2432,3 +2432,17 @@ model SecurityProtectedNetwork {
|
||||
|
||||
@@index([enabled, createdAt])
|
||||
}
|
||||
|
||||
model InfrastructureAlertSetting {
|
||||
id String @id @default("global")
|
||||
configVersion Int @default(1)
|
||||
effectiveVersion Int @default(1)
|
||||
thresholds Json
|
||||
effectiveThresholds Json
|
||||
applyStatus String @default("effective")
|
||||
lastError String?
|
||||
updatedById String?
|
||||
appliedAt DateTime?
|
||||
createdAt DateTime @default(now())
|
||||
updatedAt DateTime @updatedAt
|
||||
}
|
||||
|
||||
@@ -0,0 +1,19 @@
|
||||
import { BadRequestException } from '@nestjs/common';
|
||||
import { DEFAULT_ALERT_THRESHOLDS, InfrastructureAlertSettingsService } from './infrastructure-alert-settings.service';
|
||||
|
||||
describe('InfrastructureAlertSettingsService', () => {
|
||||
const service = new InfrastructureAlertSettingsService({} as never, { get: () => undefined } as never);
|
||||
|
||||
it('accepts the fixed threshold whitelist and renders managed rules', () => {
|
||||
const validated = (service as unknown as { validate(value: unknown): unknown }).validate(DEFAULT_ALERT_THRESHOLDS);
|
||||
const rules = (service as unknown as { renderRules(value: unknown): string }).renderRules(validated);
|
||||
expect(rules).toContain('HostCpuUsageWarning');
|
||||
expect(rules).toContain('CmppGatewayQueueDelayedCritical');
|
||||
expect(rules).toContain('threshold: "120秒"');
|
||||
});
|
||||
|
||||
it('rejects unknown keys and warning thresholds that are not below critical', () => {
|
||||
expect(() => (service as unknown as { validate(value: unknown): unknown }).validate({ ...DEFAULT_ALERT_THRESHOLDS, promql: { warning: 1, critical: 2 } })).toThrow(BadRequestException);
|
||||
expect(() => (service as unknown as { validate(value: unknown): unknown }).validate({ ...DEFAULT_ALERT_THRESHOLDS, hostCpu: { warning: 90, critical: 90 } })).toThrow(BadRequestException);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,142 @@
|
||||
import { BadRequestException, ConflictException, Injectable, Logger, ServiceUnavailableException } from '@nestjs/common';
|
||||
import { ConfigService } from '@nestjs/config';
|
||||
import { Prisma } from '@prisma/client';
|
||||
import { execFile } from 'node:child_process';
|
||||
import { mkdir, readFile, rename, rm, writeFile } from 'node:fs/promises';
|
||||
import { dirname } from 'node:path';
|
||||
import { promisify } from 'node:util';
|
||||
import { PrismaService } from '../prisma/prisma.service';
|
||||
import type { InfrastructureAlertSettings, InfrastructureAlertThresholds } from './infrastructure-monitoring.contracts';
|
||||
|
||||
const execFileAsync = promisify(execFile);
|
||||
|
||||
export const ALERT_THRESHOLD_DEFINITIONS = [
|
||||
{ key: 'hostCpu', label: '主机 CPU 使用率', unit: '%', min: 1, max: 100, step: 1, warning: 80, critical: 90, expr: '100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100)', names: ['HostCpuUsageWarning', 'HostCpuUsageCritical'], service: 'host', durations: ['10m', '5m'] },
|
||||
{ key: 'hostMemory', label: '主机内存使用率', unit: '%', min: 1, max: 100, step: 1, warning: 85, critical: 95, expr: '(1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100', names: ['HostMemoryUsageWarning', 'HostMemoryUsageCritical'], service: 'host', durations: ['10m', '5m'] },
|
||||
{ key: 'hostDisk', label: '根磁盘使用率', unit: '%', min: 1, max: 100, step: 1, warning: 80, critical: 90, expr: '(1 - node_filesystem_avail_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"} / node_filesystem_size_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"}) * 100', names: ['HostRootDiskUsageWarning', 'HostRootDiskUsageCritical'], service: 'host', durations: ['15m', '5m'] },
|
||||
{ key: 'apiError', label: 'API 5xx 错误率', unit: '%', min: 0.1, max: 100, step: 0.1, warning: 1, critical: 5, expr: '100 * sum(rate(cmpp_api_http_requests_total{status=~"5.."}[5m])) / clamp_min(sum(rate(cmpp_api_http_requests_total[5m])), 0.001)', names: ['CmppApiHttpErrorRateWarning', 'CmppApiHttpErrorRateCritical'], service: 'api', durations: ['5m', '5m'] },
|
||||
{ key: 'apiLatency', label: 'API P95 响应时间', unit: '秒', min: 0.1, max: 60, step: 0.1, warning: 1, critical: 3, expr: 'histogram_quantile(0.95, sum by (le) (rate(cmpp_api_http_request_duration_seconds_bucket[10m])))', names: ['CmppApiLatencyWarning', 'CmppApiLatencyCritical'], service: 'api', durations: ['10m', '5m'] },
|
||||
{ key: 'apiEventLoop', label: 'API 事件循环 P99', unit: '秒', min: 0.01, max: 10, step: 0.01, warning: 0.2, critical: 1, expr: 'cmpp_api_nodejs_event_loop_lag_p99_seconds', names: ['CmppApiEventLoopLagWarning', 'CmppApiEventLoopLagCritical'], service: 'api', durations: ['10m', '5m'] },
|
||||
{ key: 'gatewayQueue', label: 'Gateway 最旧 pending', unit: '秒', min: 1, max: 3600, step: 1, warning: 30, critical: 120, expr: 'cmpp_gateway_submit_queue_oldest_pending_age_seconds', names: ['CmppGatewayQueueDelayedWarning', 'CmppGatewayQueueDelayedCritical'], service: 'gateway', durations: ['2m', '2m'] },
|
||||
{ key: 'postgresConnections', label: 'PostgreSQL 连接使用率', unit: '%', min: 1, max: 100, step: 1, warning: 70, critical: 85, expr: '100 * sum(pg_stat_activity_count) / clamp_min(max(pg_settings_max_connections), 1)', names: ['PostgresConnectionsWarning', 'PostgresConnectionsCritical'], service: 'postgresql', durations: ['10m', '5m'] },
|
||||
{ key: 'redisMemory', label: 'Redis 内存使用率', unit: '%', min: 1, max: 100, step: 1, warning: 70, critical: 85, expr: '100 * redis_memory_used_bytes / redis_memory_max_bytes', names: ['RedisMemoryWarning', 'RedisMemoryCritical'], service: 'redis', durations: ['10m', '5m'] },
|
||||
{ key: 'minioCapacity', label: 'MinIO 容量使用率', unit: '%', min: 1, max: 100, step: 1, warning: 80, critical: 90, expr: '100 * (1 - minio_cluster_capacity_usable_free_bytes / minio_cluster_capacity_usable_total_bytes)', names: ['MinioCapacityWarning', 'MinioCapacityCritical'], service: 'minio', durations: ['15m', '5m'] },
|
||||
] as const;
|
||||
|
||||
export const DEFAULT_ALERT_THRESHOLDS: InfrastructureAlertThresholds = Object.fromEntries(
|
||||
ALERT_THRESHOLD_DEFINITIONS.map((item) => [item.key, { warning: item.warning, critical: item.critical }]),
|
||||
);
|
||||
|
||||
@Injectable()
|
||||
export class InfrastructureAlertSettingsService {
|
||||
private readonly logger = new Logger(InfrastructureAlertSettingsService.name);
|
||||
private readonly rulesPath: string;
|
||||
private readonly promtoolPath: string;
|
||||
private readonly reloadUrl: string;
|
||||
|
||||
constructor(private readonly prisma: PrismaService, config: ConfigService) {
|
||||
this.rulesPath = String(config.get('PROMETHEUS_MANAGED_RULES_PATH') ?? '/var/lib/cmpp-platform/monitoring/cmpp-managed-alerts.yml');
|
||||
this.promtoolPath = String(config.get('PROMTOOL_PATH') ?? '/usr/bin/promtool');
|
||||
this.reloadUrl = String(config.get('PROMETHEUS_RELOAD_URL') ?? 'http://127.0.0.1:9090/-/reload');
|
||||
}
|
||||
|
||||
async get(): Promise<InfrastructureAlertSettings> {
|
||||
const row = await this.prisma.infrastructureAlertSetting.findUnique({ where: { id: 'global' } });
|
||||
const thresholds = this.asThresholds(row?.thresholds) ?? DEFAULT_ALERT_THRESHOLDS;
|
||||
const effective = this.asThresholds(row?.effectiveThresholds) ?? thresholds;
|
||||
return {
|
||||
configVersion: row?.configVersion ?? 1,
|
||||
effectiveVersion: row?.effectiveVersion ?? 1,
|
||||
applyStatus: (row?.applyStatus as InfrastructureAlertSettings['applyStatus']) ?? 'effective',
|
||||
lastError: row?.lastError ?? null,
|
||||
appliedAt: row?.appliedAt?.toISOString() ?? null,
|
||||
thresholds,
|
||||
effectiveThresholds: effective,
|
||||
definitions: ALERT_THRESHOLD_DEFINITIONS.map(({ key, label, unit, min, max, step }) => ({ key, label, unit, min, max, step })),
|
||||
};
|
||||
}
|
||||
|
||||
async update(body: { configVersion?: number; thresholds?: unknown }, operatorId?: string) {
|
||||
const expectedVersion = Number(body.configVersion);
|
||||
if (!Number.isInteger(expectedVersion) || expectedVersion < 1) throw new BadRequestException('配置版本无效');
|
||||
const thresholds = this.validate(body.thresholds);
|
||||
const claimed = await this.prisma.infrastructureAlertSetting.updateMany({
|
||||
where: { id: 'global', configVersion: expectedVersion },
|
||||
data: { configVersion: { increment: 1 }, thresholds: thresholds as Prisma.InputJsonValue, applyStatus: 'applying', lastError: null, updatedById: operatorId },
|
||||
});
|
||||
// 版本条件更新是跨进程的写锁；避免两个 API 实例同时覆盖规则文件并把旧配置误标成已生效。
|
||||
if (claimed.count !== 1) throw new ConflictException('告警阈值已被其他管理员修改，请刷新后重试');
|
||||
const nextVersion = expectedVersion + 1;
|
||||
try {
|
||||
await this.applyRules(thresholds);
|
||||
await this.prisma.$transaction([
|
||||
this.prisma.infrastructureAlertSetting.update({ where: { id: 'global' }, data: { effectiveVersion: nextVersion, effectiveThresholds: thresholds as Prisma.InputJsonValue, applyStatus: 'effective', lastError: null, appliedAt: new Date() } }),
|
||||
this.prisma.operationLog.create({ data: { userId: operatorId, action: 'monitoring.alert_thresholds_updated', resource: 'infrastructure_alert_setting', resourceId: 'global', detail: { configVersion: nextVersion, thresholds } } }),
|
||||
]);
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message.slice(0, 500) : 'unknown error';
|
||||
await this.prisma.infrastructureAlertSetting.update({ where: { id: 'global' }, data: { applyStatus: 'failed', lastError: message } });
|
||||
this.logger.error(`Prometheus managed rules apply failed: ${message}`);
|
||||
throw new ServiceUnavailableException('阈值已保存但 Prometheus 应用失败，原生效规则已保留');
|
||||
}
|
||||
return this.get();
|
||||
}
|
||||
|
||||
private validate(value: unknown): InfrastructureAlertThresholds {
|
||||
if (!value || typeof value !== 'object' || Array.isArray(value)) throw new BadRequestException('告警阈值格式无效');
|
||||
const input = value as Record<string, unknown>;
|
||||
if (Object.keys(input).some((key) => !ALERT_THRESHOLD_DEFINITIONS.some((item) => item.key === key))) throw new BadRequestException('存在不允许配置的告警指标');
|
||||
const result: InfrastructureAlertThresholds = {};
|
||||
for (const definition of ALERT_THRESHOLD_DEFINITIONS) {
|
||||
const pair = input[definition.key] as { warning?: unknown; critical?: unknown } | undefined;
|
||||
const warning = Number(pair?.warning);
|
||||
const critical = Number(pair?.critical);
|
||||
if (!Number.isFinite(warning) || !Number.isFinite(critical) || warning < definition.min || critical > definition.max || warning >= critical) {
|
||||
throw new BadRequestException(`${definition.label}必须满足最小值 ≤ 警告阈值 < 严重阈值 ≤ 最大值`);
|
||||
}
|
||||
result[definition.key] = { warning, critical };
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
private asThresholds(value: unknown) {
|
||||
try { return this.validate(value); } catch { return null; }
|
||||
}
|
||||
|
||||
private renderRules(thresholds: InfrastructureAlertThresholds) {
|
||||
const lines = ['groups:', ' - name: cmpp-managed-thresholds', ' rules:'];
|
||||
for (const definition of ALERT_THRESHOLD_DEFINITIONS) {
|
||||
const pair = thresholds[definition.key];
|
||||
const values = [pair.warning, pair.critical];
|
||||
for (let index = 0; index < 2; index += 1) {
|
||||
const isWarning = index === 0;
|
||||
const expr = isWarning ? `(${definition.expr} > ${values[0]}) and (${definition.expr} <= ${values[1]})` : `${definition.expr} > ${values[1]}`;
|
||||
lines.push(` - alert: ${definition.names[index]}`, ` expr: ${expr}`, ` for: ${definition.durations[index]}`, ' labels:', ` severity: ${isWarning ? 'warning' : 'critical'}`, ` service: ${definition.service}`, ' annotations:', ` summary: "${definition.label}${isWarning ? '达到警告阈值' : '达到严重阈值'}"`, ` description: "${definition.label}持续超过${values[index]}${definition.unit}。"`, ' currentValue: "{{ $value }}"', ` threshold: "${values[index]}${definition.unit}"`);
|
||||
}
|
||||
}
|
||||
return `${lines.join('\n')}\n`;
|
||||
}
|
||||
|
||||
private async applyRules(thresholds: InfrastructureAlertThresholds) {
|
||||
const directory = dirname(this.rulesPath);
|
||||
const temporary = `${this.rulesPath}.${process.pid}.${Date.now()}.tmp`;
|
||||
await mkdir(directory, { recursive: true });
|
||||
const previous = await readFile(this.rulesPath).catch(() => null);
|
||||
try {
|
||||
await writeFile(temporary, this.renderRules(thresholds), { mode: 0o640 });
|
||||
await execFileAsync(this.promtoolPath, ['check', 'rules', temporary], { timeout: 10_000 });
|
||||
await rename(temporary, this.rulesPath);
|
||||
const response = await fetch(this.reloadUrl, { method: 'POST', signal: AbortSignal.timeout(5_000) });
|
||||
if (!response.ok) throw new Error(`Prometheus reload HTTP ${response.status}`);
|
||||
} catch (error) {
|
||||
await rm(temporary, { force: true });
|
||||
// 规则替换和 reload 不是一个事务，失败时必须恢复旧文件并再次 reload，避免数据库状态与实际告警漂移。
|
||||
if (previous) {
|
||||
await writeFile(temporary, previous, { mode: 0o640 });
|
||||
await rename(temporary, this.rulesPath);
|
||||
await fetch(this.reloadUrl, { method: 'POST', signal: AbortSignal.timeout(5_000) }).catch(() => undefined);
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -76,3 +76,16 @@ export type InfrastructureMonitoringOverview = {
|
||||
serviceMetrics: InfrastructureServiceMetricGroup[];
|
||||
alerts: InfrastructureAlert[];
|
||||
};
|
||||
|
||||
export type InfrastructureAlertThresholds = Record<string, { warning: number; critical: number }>;
|
||||
|
||||
export type InfrastructureAlertSettings = {
|
||||
configVersion: number;
|
||||
effectiveVersion: number;
|
||||
applyStatus: 'effective' | 'applying' | 'failed';
|
||||
lastError: string | null;
|
||||
appliedAt: string | null;
|
||||
thresholds: InfrastructureAlertThresholds;
|
||||
effectiveThresholds: InfrastructureAlertThresholds;
|
||||
definitions: Array<{ key: string; label: string; unit: string; min: number; max: number; step: number }>;
|
||||
};
|
||||
|
||||
@@ -1,14 +1,29 @@
|
||||
import { Controller, Get, Query } from '@nestjs/common';
|
||||
import { Body, Controller, Get, Put, Query } from '@nestjs/common';
|
||||
import { ApiTags } from '@nestjs/swagger';
|
||||
import { CurrentSessionUserId } from '../auth/current-session-user.decorator';
|
||||
import { RequireRecentAuthentication } from '../auth/require-recent-authentication.decorator';
|
||||
import { InfrastructureAlertSettingsService } from './infrastructure-alert-settings.service';
|
||||
import { InfrastructureMonitoringService } from './infrastructure-monitoring.service';
|
||||
|
||||
@ApiTags('infrastructure-monitoring')
|
||||
@Controller('admin/infrastructure-monitoring')
|
||||
export class InfrastructureMonitoringController {
|
||||
constructor(private readonly monitoring: InfrastructureMonitoringService) {}
|
||||
constructor(private readonly monitoring: InfrastructureMonitoringService, private readonly settings: InfrastructureAlertSettingsService) {}
|
||||
|
||||
@Get('overview')
|
||||
overview(@Query('range') range?: string) {
|
||||
return this.monitoring.overview(range);
|
||||
}
|
||||
|
||||
@Get('notification-summary')
|
||||
notificationSummary() { return this.monitoring.notificationSummary(); }
|
||||
|
||||
@Get('alert-thresholds')
|
||||
alertThresholds() { return this.settings.get(); }
|
||||
|
||||
@Put('alert-thresholds')
|
||||
@RequireRecentAuthentication()
|
||||
updateAlertThresholds(@Body() body: { configVersion?: number; thresholds?: unknown }, @CurrentSessionUserId() operatorId?: string) {
|
||||
return this.settings.update(body, operatorId);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,9 +1,10 @@
|
||||
import { Module } from '@nestjs/common';
|
||||
import { InfrastructureMonitoringController } from './infrastructure-monitoring.controller';
|
||||
import { InfrastructureMonitoringService } from './infrastructure-monitoring.service';
|
||||
import { InfrastructureAlertSettingsService } from './infrastructure-alert-settings.service';
|
||||
|
||||
@Module({
|
||||
controllers: [InfrastructureMonitoringController],
|
||||
providers: [InfrastructureMonitoringService],
|
||||
providers: [InfrastructureMonitoringService, InfrastructureAlertSettingsService],
|
||||
})
|
||||
export class InfrastructureMonitoringModule {}
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { BadRequestException, Injectable, Logger } from '@nestjs/common';
|
||||
import { BadRequestException, Injectable, Logger, ServiceUnavailableException } from '@nestjs/common';
|
||||
import { ConfigService } from '@nestjs/config';
|
||||
import { createHash } from 'node:crypto';
|
||||
import type {
|
||||
@@ -210,6 +210,16 @@ export class InfrastructureMonitoringService {
|
||||
}
|
||||
}
|
||||
|
||||
async notificationSummary() {
|
||||
try {
|
||||
const alerts = this.parseAlerts(await this.getJson<PrometheusAlertResponse>('/api/v1/alerts'));
|
||||
return { count: alerts.length, criticalCount: alerts.filter((item) => item.severity === 'critical').length };
|
||||
} catch (error) {
|
||||
this.logger.warn(`Prometheus notification summary unavailable: ${error instanceof Error ? error.message : 'unknown error'}`);
|
||||
throw new ServiceUnavailableException('Prometheus活动告警当前不可用');
|
||||
}
|
||||
}
|
||||
|
||||
private parseRange(value?: string): InfrastructureMonitoringRange {
|
||||
const range = value || '24h';
|
||||
if (!(range in RANGE_CONFIG)) throw new BadRequestException('监控时间范围只支持1h、24h或7d');
|
||||
|
||||
@@ -1150,4 +1150,4 @@ global，不能错误归入client。`client-signature-*`、发送页、企业认
|
||||
|
||||
- `api/src/metrics/`只负责API进程和HTTP路由模板的低基数计数/直方图，不依赖Prisma、Redis或业务Service；独立回环监听由`api/src/main.ts`组装。
|
||||
- `gateway/internal/metrics/`只负责线程安全聚合和Prometheus文本暴露；上游连接池、下游Session和Submit Worker仅提供总数或有界结果，不把实体ID或凭据交给监控模块。
|
||||
- `api/src/infrastructure-monitoring/`仍是运营端只读聚合门面，只消费固定PromQL和Recording Rules；Exporter安装、端口隔离和阈值归`tools/monitoring/`治理。
|
||||
- `api/src/infrastructure-monitoring/`是运营端监控聚合与固定阈值应用边界：只消费代码白名单 PromQL，并通过版本化 PostgreSQL 单例、promtool 校验和原子规则热加载管理数值阈值；Exporter安装、端口隔离、固定规则模板和权限仍归`tools/monitoring/`治理。
|
||||
|
||||
@@ -2096,3 +2096,6 @@
|
||||
- 运营端展示API请求/错误/延迟/事件循环、Gateway Submit/队列/连接、PostgreSQL连接/死锁、Redis内存/连接/淘汰、Nginx连接/请求和MinIO可用性；指标缺失显示“待采集”，不以0伪装。
|
||||
- 告警必须使用持续窗口和最低样本量；默认阈值、收敛关系、标签禁止项和性能预算以`docs/prometheus-system-monitoring-design-20260814.md`第9节为准。
|
||||
- 指标不得包含手机号、短信正文、短信/CMPP/任务ID、密钥、完整URL或SQL原文；不得把时序指标高频写入业务PostgreSQL。
|
||||
- 系统监控标题说明必须明确标注数据来自 Prometheus；“服务关键指标”位于趋势/核心服务区域之后、活动告警之前，并提供统一的“告警阈值设置”入口。安全检测页不重复渲染大号标题，说明文字必须明确标注使用 Fail2ban。
|
||||
- 告警阈值仅开放固定指标的警告/严重数值，不允许前端提交 PromQL、标签、文件路径或持续时间；必须满足警告值小于严重值。配置以 PostgreSQL 保存版本、期望值、生效值和应用状态，经 `promtool check rules` 校验、同目录原子替换和 Prometheus 热加载成功后才标记生效，失败保留上一生效规则并展示原因。
|
||||
- 右上角预警中心增加“系统监控告警”，通过独立轻量接口统计 Prometheus 当前 firing/pending 告警及严重数，跳转系统监控活动告警区；任一预警域失败不得清空其他域。
|
||||
|
||||
@@ -237,3 +237,9 @@ type InfrastructureOverview = {
|
||||
- API和Gateway请求路径只做内存计数、有界直方图和原子计数，不在业务请求中写PostgreSQL或Redis。
|
||||
- PostgreSQL Exporter只使用发行版默认低代价查询，不采集SQL原文或扫描业务大表。
|
||||
- 时序仍受30天和8GB双重上限约束；时序增长时先缩短实际保留期，不允许无界占满业务盘。
|
||||
|
||||
## 10. 阈值配置与全局预警入口（2026-08-14 增补）
|
||||
|
||||
- 可配置范围固定为主机 CPU/内存/根磁盘、API 5xx/P95/事件循环、Gateway 最旧 pending、PostgreSQL 连接、Redis 内存和 MinIO 容量十组警告/严重数值。PromQL、持续窗口、标签和规则文件路径仍由代码固定，浏览器无权提交。
|
||||
- PostgreSQL 单例记录同时保存期望阈值、生效阈值、配置版本、生效版本和 `applying/effective/failed` 状态。更新使用版本条件认领，防止多个 API 实例并发覆盖；规则先经 promtool 校验，再在同一目录原子替换并调用仅回环开放的 `/-/reload`。失败恢复旧文件并保留旧生效版本。
|
||||
- 安装器把可配置规则从基础规则中剥离，托管文件归 `cmpp-api:prometheus` 且权限为 0640；Prometheus 仍只监听回环。右上角铃铛只轮询轻量活动告警汇总接口，不重复加载趋势或服务指标。
|
||||
|
||||
@@ -4675,3 +4675,16 @@ npm run verify:phase8
|
||||
- 集成验收必须在隔离测试节点或网络 namespace 使用文档保留 IP；不得封禁预生产运维出口、Cloudflare 节点或真实客户 IP。未安装真实 Fail2ban/nftables/Nginx 资产时，只能把相关用例标记阻塞，不得用 Mock 通过代替。
|
||||
- UI 验收覆盖桌面与窄屏的总览、告警、规则、封禁记录、保护名单、加载、空数据、失败和规则未生效状态；所有数字与操作结果必须能从 API、数据库、agent 与执行器证据交叉验证。
|
||||
- `TC-F2B-OPS-008`：在非默认`APP_DIR`构建安全代理后执行安装器，核对systemd `ExecStart`与Fail2ban `actionban`均指向同一个真实可执行的`$APP_DIR/dist/cmpp-security-agent`；任一文件残留占位符、旧`current/bin`路径或目标不可执行时，安装/发布必须失败。
|
||||
|
||||
## 系统监控阈值与预警中心增量用例（2026-08-14）
|
||||
|
||||
| 用例ID | 场景 | 步骤 | 预期 |
|
||||
| --- | --- | --- | --- |
|
||||
| TC-INFRA-MON-029 | 模块顺序与技术说明 | 打开系统监控并检查标题和模块顺序 | 说明明确写明 Prometheus；服务关键指标紧邻活动告警上方，活动告警锚点可定位 |
|
||||
| TC-INFRA-MON-030 | 阈值真实读取 | 打开阈值设置并核对 API 与数据库 | 十组固定指标来自 `InfrastructureAlertSetting`，不使用 Mock/localStorage，不允许编辑 PromQL |
|
||||
| TC-INFRA-MON-031 | 阈值边界 | 提交警告≥严重、越界、缺项和未知指标 | API 返回 400，数据库版本和 Prometheus 规则均不改变 |
|
||||
| TC-INFRA-MON-032 | 并发版本 | 两个会话以同一版本先后保存 | 仅第一个原子认领成功，后者返回 409 并提示刷新 |
|
||||
| TC-INFRA-MON-033 | 规则校验与热加载 | 保存合法阈值，检查 promtool、规则文件、reload 与数据库 | 先校验再同目录原子替换，reload 成功后生效版本前进且写操作日志 |
|
||||
| TC-INFRA-MON-034 | 应用失败回滚 | 令 promtool 或 reload 失败后保存 | 状态为 failed、展示原因，旧规则文件与旧生效阈值保留，不误报已生效 |
|
||||
| TC-GLOBAL-ALERT-004 | 系统监控预警入口 | 准备隔离 QA Prometheus firing 告警并点击铃铛 | 第三项显示真实总数/严重数，角标计入三域总和，点击跳转系统监控活动告警区 |
|
||||
| TC-SECURITY-UI-001 | Fail2ban 标识与标题规范 | 打开安全检测与封禁 | 不出现重复大号页面标题，说明明确写明使用 Fail2ban，字号遵循通用菜单标题 |
|
||||
|
||||
@@ -3658,3 +3658,9 @@ git diff --check
|
||||
- 测试机静态包只读核对显示当前已部署“服务关键指标”，但尚未包含“重复投递风险”和已确认重投的新表单文案，证明服务指标会话使用定向发布，没有把尚未授权部署的下游重投改动意外带入测试机；两项发布记录保持一致。
|
||||
- 合并后统一回归通过：API全量41套/477项、前后端TypeScript、API正式编译、Vite 8.1.5生产构建（2549 modules）、Gateway全量`go test ./... -count=1`与`go vet ./...`、SendChain R10、生产部署和安全代理静态契约、5个Shell脚本语法、真实本地PostgreSQL 88/88 migration状态及`git diff --check`。Vite仅保留既有约2.11MiB单chunk提示；Operations R2仍因本轮开始前已有的`sendQuality`查询哈希漂移失败，与本次合并文件无交集。
|
||||
- 本次合并没有发送、补发或重投短信，没有创建后台重投任务，没有修改通道账号、密码、启停状态、企业余额或客户连接；提交范围继续排除`api/tsconfig.build.tsbuildinfo`、`tsconfig.tsbuildinfo`、`outputs/`和空文件`=`。
|
||||
|
||||
# 2026-08-14 系统监控阈值与预警入口优化（进行中）
|
||||
|
||||
- 系统监控将“服务关键指标”移至活动告警正上方，标题说明明确使用 Prometheus；新增十组固定指标警告/严重阈值设置。配置真实写入 PostgreSQL，使用版本条件并发认领，promtool 校验、原子替换和回环 reload 成功后才推进生效版本，失败保留旧生效值。
|
||||
- 右上角预警中心新增“系统监控告警”及数量/严重数，读取独立 Prometheus 活动告警汇总并跳转活动告警锚点；继续使用 `Promise.allSettled` 隔离签名、安全与监控域故障。安全检测页移除重复大号标题，说明明确使用 Fail2ban。
|
||||
- 新增 migration `20260814173000_add_infrastructure_alert_settings`。本机 Prisma Client 生成、前后端 TypeScript 与 API 正式编译已通过；测试机恢复资产、migration、Prometheus规则校验、真实告警数据、浏览器验收和提交/部署结果待本轮后续补记。
|
||||
|
||||
@@ -1,7 +1,10 @@
|
||||
import { request, withQuery } from '../core/httpClient';
|
||||
import type { InfrastructureMonitoringOverview, InfrastructureMonitoringRange } from '../types';
|
||||
import type { InfrastructureAlertSettings, InfrastructureAlertThresholds, InfrastructureMonitoringOverview, InfrastructureMonitoringRange } from '../types';
|
||||
|
||||
export const adminInfrastructureMonitoringApi = {
|
||||
getInfrastructureMonitoringOverview: (range: InfrastructureMonitoringRange) =>
|
||||
request<InfrastructureMonitoringOverview>(withQuery('/admin/infrastructure-monitoring/overview', { range })),
|
||||
getInfrastructureMonitoringNotificationSummary: () => request<{ count: number; criticalCount: number }>('/admin/infrastructure-monitoring/notification-summary'),
|
||||
getInfrastructureAlertThresholds: () => request<InfrastructureAlertSettings>('/admin/infrastructure-monitoring/alert-thresholds'),
|
||||
updateInfrastructureAlertThresholds: (body: { configVersion: number; thresholds: InfrastructureAlertThresholds }) => request<InfrastructureAlertSettings>('/admin/infrastructure-monitoring/alert-thresholds', { method: 'PUT', body: JSON.stringify(body) }),
|
||||
};
|
||||
|
||||
@@ -76,3 +76,16 @@ export type InfrastructureMonitoringOverview = {
|
||||
serviceMetrics: InfrastructureServiceMetricGroup[];
|
||||
alerts: InfrastructureAlert[];
|
||||
};
|
||||
|
||||
export type InfrastructureAlertThresholds = Record<string, { warning: number; critical: number }>;
|
||||
|
||||
export type InfrastructureAlertSettings = {
|
||||
configVersion: number;
|
||||
effectiveVersion: number;
|
||||
applyStatus: 'effective' | 'applying' | 'failed';
|
||||
lastError: string | null;
|
||||
appliedAt: string | null;
|
||||
thresholds: InfrastructureAlertThresholds;
|
||||
effectiveThresholds: InfrastructureAlertThresholds;
|
||||
definitions: Array<{ key: string; label: string; unit: string; min: number; max: number; step: number }>;
|
||||
};
|
||||
|
||||
@@ -1 +1,2 @@
|
||||
.admin-security-page{gap:20px}.security-heading{align-items:flex-end}.security-heading h1{margin:10px 0 4px;font-size:26px}.security-heading p{margin:0;color:#64748b}.security-kpis{display:grid;grid-template-columns:repeat(4,minmax(0,1fr));gap:14px}.security-kpi{display:grid;grid-template-columns:42px 1fr;grid-template-rows:auto auto;gap:3px 12px;padding:18px}.security-kpi>div{grid-row:1/3;width:42px;height:42px;border-radius:12px;display:grid;place-items:center;background:#e8f0ff;color:#2563eb}.security-kpi>div svg{width:20px}.security-kpi span{font-size:13px;color:#64748b}.security-kpi strong{font-size:24px;line-height:1.1}.security-kpi.is-danger>div{background:#fef2f2;color:#dc2626}.security-overview-grid{display:grid;grid-template-columns:minmax(0,1fr) minmax(360px,.8fr);gap:16px}.security-chart,.security-latest,.security-table-card{padding:18px}.security-chart header,.security-latest header{display:flex;justify-content:space-between}.security-chart header span,.security-latest header span{font-size:12px;color:#94a3b8}.security-latest-row{display:grid;grid-template-columns:10px 1fr auto;gap:10px;align-items:center;padding:13px 0;border-bottom:1px solid #eef2f7}.security-latest-row div{display:grid;gap:3px}.security-latest-row small,.security-latest-row time{color:#64748b;font-size:12px}.severity-dot{width:8px;height:8px;border-radius:99px;background:#3b82f6}.severity-dot.is-high{background:#f59e0b}.severity-dot.is-critical{background:#ef4444}.security-cell{display:grid;gap:3px}.security-cell span{font-size:11px;color:#94a3b8}.security-actions{display:flex;gap:6px}.security-note,.security-section-toolbar{display:flex;align-items:center;gap:9px;margin-bottom:16px;padding:12px 14px;border-radius:10px;background:#f8fafc;color:#475569;font-size:13px}.security-section-toolbar{justify-content:space-between}.security-error{display:flex;gap:8px;align-items:center;padding:12px 14px;border:1px solid #fecaca;border-radius:10px;background:#fef2f2;color:#b91c1c}.security-empty{height:265px;display:grid;place-items:center;color:#94a3b8}.security-dialog{display:grid;gap:16px}.security-target{display:grid;gap:4px;padding:14px;border-radius:10px;background:#f8fafc}.security-target span,.security-target small{color:#64748b;font-size:12px}.security-target strong{font-family:ui-monospace,monospace;font-size:18px}.security-form-grid{display:grid;grid-template-columns:1fr 1fr;gap:16px}.security-form-grid .security-error{grid-column:1/-1}.admin-security-page code{font-size:12px;color:#334155}@media(max-width:1100px){.security-kpis{grid-template-columns:repeat(2,1fr)}.security-overview-grid{grid-template-columns:1fr}}@media(max-width:640px){.security-kpis{grid-template-columns:1fr}.security-form-grid{grid-template-columns:1fr}}
|
||||
.security-heading p{margin:10px 0 0}
|
||||
|
||||
@@ -76,7 +76,7 @@ export function AdminSecurityDetectionPage() {
|
||||
{ value: 'protected', label: '保护名单', content: <section className="surface security-table-card"><div className="security-section-toolbar"><span>受保护的运维出口、内网和可信代理永远不能从面板封禁。</span><Button onClick={() => setShowNetwork(true)}>新增保护网段</Button></div><Table columns={networkColumns} data={networks} emptyText="暂无保护网段" rowKey="id" /></section> },
|
||||
];
|
||||
|
||||
return <section className="page-stack admin-security-page"><div className="page-heading security-heading"><div><Breadcrumb items={['安全控制', '安全检测与封禁']} /><h1>安全检测与封禁</h1><p>Fail2ban 与业务事件只负责检测，由运营人员复核后人工处置</p></div><Button disabled={loading} icon={<RefreshCw className={loading ? 'is-spinning' : ''} size={16} />} onClick={() => void load()} variant="ghost">刷新</Button></div>
|
||||
return <section className="page-stack admin-security-page"><div className="page-heading security-heading"><div><Breadcrumb items={['安全控制', '安全检测与封禁']} /><p>使用 Fail2ban 与平台业务安全事件进行检测，由运营人员复核后人工处置</p></div><Button disabled={loading} icon={<RefreshCw className={loading ? 'is-spinning' : ''} size={16} />} onClick={() => void load()} variant="ghost">刷新</Button></div>
|
||||
{error ? <div className="security-error" role="alert"><AlertTriangle size={18} />{error}</div> : null}
|
||||
<div className="security-kpis"><Kpi icon={<BellRing />} label="24小时检测事件" value={overview?.totalEvents ?? 0} /><Kpi icon={<AlertTriangle />} label="待处置告警" value={overview?.activeAlerts ?? 0} danger={Boolean(overview?.criticalAlerts)} /><Kpi icon={<Ban />} label="生效封禁" value={overview?.activeBlocks ?? 0} /><Kpi icon={overview?.health.agent === 'healthy' ? <ShieldCheck /> : <ShieldOff />} label="安全代理" value={overview?.health.agent === 'healthy' ? '在线' : '不可用'} danger={overview?.health.agent !== 'healthy'} /></div>
|
||||
<Tabs items={tabs} />
|
||||
|
||||
@@ -321,3 +321,16 @@
|
||||
@media (prefers-reduced-motion: reduce) {
|
||||
.is-spinning { animation: none; }
|
||||
}
|
||||
|
||||
.system-monitoring-service-actions { align-items: center; display: flex; gap: 12px; }
|
||||
.system-monitoring-threshold-dialog { display: grid; gap: 12px; max-height: 62vh; overflow: auto; padding-right: 4px; }
|
||||
.system-monitoring-threshold-note { align-items: flex-start; background: #eff6ff; border-radius: 10px; color: #475569; display: flex; font-size: 13px; gap: 9px; padding: 12px 14px; }
|
||||
.system-monitoring-threshold-row { align-items: end; border-bottom: 1px solid #eef2f7; display: grid; gap: 14px; grid-template-columns: minmax(180px, 1fr) 150px 150px; padding: 12px 0; }
|
||||
.system-monitoring-threshold-row > div:first-child { align-self: center; display: grid; gap: 4px; }
|
||||
.system-monitoring-threshold-row small { color: #64748b; }
|
||||
|
||||
@media (max-width: 760px) {
|
||||
.system-monitoring-service-actions { align-items: flex-start; flex-direction: column; }
|
||||
.system-monitoring-threshold-row { grid-template-columns: 1fr 1fr; }
|
||||
.system-monitoring-threshold-row > div:first-child { grid-column: 1 / -1; }
|
||||
}
|
||||
|
||||
@@ -12,16 +12,19 @@ import {
|
||||
Network,
|
||||
RefreshCw,
|
||||
Server,
|
||||
Settings2,
|
||||
ShieldAlert,
|
||||
} from 'lucide-react';
|
||||
import {
|
||||
adminApi,
|
||||
type InfrastructureAlert,
|
||||
type InfrastructureAlertSettings,
|
||||
type InfrastructureAlertThresholds,
|
||||
type InfrastructureMetricPoint,
|
||||
type InfrastructureMonitoringOverview,
|
||||
type InfrastructureMonitoringRange,
|
||||
} from '@/api/adminApi';
|
||||
import { Breadcrumb, Button, Chart, Table, Tag, type TableColumn } from '@/components/ui';
|
||||
import { Breadcrumb, Button, Chart, Input, Modal, Table, Tag, type TableColumn } from '@/components/ui';
|
||||
import './AdminSystemMonitoringPage.css';
|
||||
|
||||
const RANGE_OPTIONS: Array<{ value: InfrastructureMonitoringRange; label: string }> = [
|
||||
@@ -166,6 +169,11 @@ export function AdminSystemMonitoringPage() {
|
||||
const [overview, setOverview] = useState<InfrastructureMonitoringOverview | null>(null);
|
||||
const [loading, setLoading] = useState(true);
|
||||
const [error, setError] = useState('');
|
||||
const [settings, setSettings] = useState<InfrastructureAlertSettings | null>(null);
|
||||
const [draftThresholds, setDraftThresholds] = useState<InfrastructureAlertThresholds>({});
|
||||
const [showSettings, setShowSettings] = useState(false);
|
||||
const [settingsError, setSettingsError] = useState('');
|
||||
const [savingSettings, setSavingSettings] = useState(false);
|
||||
const requestSequence = useRef(0);
|
||||
const pendingRequests = useRef(0);
|
||||
|
||||
@@ -189,8 +197,38 @@ export function AdminSystemMonitoringPage() {
|
||||
}
|
||||
}, [range]);
|
||||
|
||||
const loadSettings = useCallback(async () => {
|
||||
try {
|
||||
const result = await adminApi.getInfrastructureAlertThresholds();
|
||||
setSettings(result);
|
||||
setDraftThresholds(result.thresholds);
|
||||
setSettingsError('');
|
||||
} catch (reason) {
|
||||
setSettingsError(reason instanceof Error ? reason.message : '告警阈值加载失败');
|
||||
}
|
||||
}, []);
|
||||
|
||||
const saveSettings = useCallback(async () => {
|
||||
if (!settings) return;
|
||||
setSavingSettings(true);
|
||||
setSettingsError('');
|
||||
try {
|
||||
const result = await adminApi.updateInfrastructureAlertThresholds({ configVersion: settings.configVersion, thresholds: draftThresholds });
|
||||
setSettings(result);
|
||||
setDraftThresholds(result.thresholds);
|
||||
setShowSettings(false);
|
||||
window.dispatchEvent(new Event('cmpp-infrastructure-alert-count-refresh'));
|
||||
await loadData(true);
|
||||
} catch (reason) {
|
||||
setSettingsError(reason instanceof Error ? reason.message : '告警阈值保存失败');
|
||||
} finally {
|
||||
setSavingSettings(false);
|
||||
}
|
||||
}, [draftThresholds, loadData, settings]);
|
||||
|
||||
useEffect(() => {
|
||||
void loadData(true);
|
||||
void loadSettings();
|
||||
const intervalId = window.setInterval(() => {
|
||||
if (document.visibilityState === 'visible') void loadData();
|
||||
}, 30_000);
|
||||
@@ -203,7 +241,7 @@ export function AdminSystemMonitoringPage() {
|
||||
window.clearInterval(intervalId);
|
||||
document.removeEventListener('visibilitychange', handleVisibility);
|
||||
};
|
||||
}, [loadData]);
|
||||
}, [loadData, loadSettings]);
|
||||
|
||||
const status = STATUS_COPY[overview?.summary.overallStatus ?? 'unknown'];
|
||||
const cpuOption = useMemo(() => makeTrendOption({
|
||||
@@ -232,7 +270,7 @@ export function AdminSystemMonitoringPage() {
|
||||
<div>
|
||||
<Breadcrumb items={['系统管理', '系统监控']} />
|
||||
<div className="system-monitoring-title-row">
|
||||
<p>服务器资源、核心服务与活动告警</p>
|
||||
<p>服务器资源、核心服务与活动告警，数据由 Prometheus 采集与计算</p>
|
||||
<Tag tone={status.tone}>{status.label}</Tag>
|
||||
</div>
|
||||
</div>
|
||||
@@ -283,29 +321,6 @@ export function AdminSystemMonitoringPage() {
|
||||
<article className="surface system-monitoring-metric"><div className="system-monitoring-metric__icon is-green"><Network size={19} /></div><div><span>网络吞吐</span><strong>{formatRate(totalNetworkRate(metrics?.networkReceiveBytesPerSecond, metrics?.networkTransmitBytesPerSecond))}</strong><small>接收 {formatRate(metrics?.networkReceiveBytesPerSecond ?? null)} · 发送 {formatRate(metrics?.networkTransmitBytesPerSecond ?? null)}</small></div></article>
|
||||
</div>
|
||||
|
||||
<section className="surface system-monitoring-service-metrics">
|
||||
<header>
|
||||
<div><Database size={18} /><strong>服务关键指标</strong></div>
|
||||
<span>固定低基数聚合，不含手机号、短信ID或SQL文本</span>
|
||||
</header>
|
||||
<div className="system-monitoring-service-metric-grid">
|
||||
{(overview?.serviceMetrics ?? []).map((group) => (
|
||||
<article key={group.key}>
|
||||
<div className="system-monitoring-service-metric-title">
|
||||
<strong>{group.name}</strong>
|
||||
<Tag tone={group.available ? 'success' : 'neutral'}>{group.available ? '已采集' : '待采集'}</Tag>
|
||||
</div>
|
||||
{group.metrics.length ? group.metrics.map((metric) => (
|
||||
<div className="system-monitoring-service-metric-row" key={metric.key}>
|
||||
<span>{metric.label}</span>
|
||||
<strong>{formatServiceMetric(metric.value, metric.unit)}</strong>
|
||||
</div>
|
||||
)) : <div className="system-monitoring-service-metric-empty">已监控服务可用性，待原生容量指标接入</div>}
|
||||
</article>
|
||||
))}
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<div className="system-monitoring-main-grid">
|
||||
<div className="system-monitoring-chart-stack">
|
||||
<article className="surface system-monitoring-chart-card"><header><div><Cpu size={17} /><strong>CPU 趋势</strong></div><span>{formatPercent(metrics?.cpuUsagePercent ?? null)}</span></header>{overview?.trends.cpuUsagePercent.length ? <Chart height={230} option={cpuOption} /> : <EmptyChart />}</article>
|
||||
@@ -332,10 +347,40 @@ export function AdminSystemMonitoringPage() {
|
||||
</aside>
|
||||
</div>
|
||||
|
||||
<section className="surface system-monitoring-alerts">
|
||||
<section className="surface system-monitoring-service-metrics">
|
||||
<header>
|
||||
<div><Database size={18} /><strong>服务关键指标</strong></div>
|
||||
<div className="system-monitoring-service-actions"><span>固定低基数聚合，不含手机号、短信ID或SQL文本</span><Button icon={<Settings2 size={15} />} onClick={() => setShowSettings(true)} variant="ghost">告警阈值设置</Button></div>
|
||||
</header>
|
||||
<div className="system-monitoring-service-metric-grid">
|
||||
{(overview?.serviceMetrics ?? []).map((group) => (
|
||||
<article key={group.key}>
|
||||
<div className="system-monitoring-service-metric-title"><strong>{group.name}</strong><Tag tone={group.available ? 'success' : 'neutral'}>{group.available ? '已采集' : '待采集'}</Tag></div>
|
||||
{group.metrics.length ? group.metrics.map((metric) => <div className="system-monitoring-service-metric-row" key={metric.key}><span>{metric.label}</span><strong>{formatServiceMetric(metric.value, metric.unit)}</strong></div>) : <div className="system-monitoring-service-metric-empty">已监控服务可用性，待原生容量指标接入</div>}
|
||||
</article>
|
||||
))}
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section className="surface system-monitoring-alerts" id="active-alerts">
|
||||
<header><div><AlertTriangle size={18} /><strong>活动告警</strong><Tag tone={overview?.summary.activeAlerts ? 'warning' : 'success'}>{overview?.summary.activeAlerts ?? 0}</Tag></div><span><Clock3 size={14} /> 刷新于 {formatTime(overview?.collectedAt ?? null)}</span></header>
|
||||
<Table columns={alertColumns} data={overview?.alerts ?? []} emptyText={overview?.available ? '当前没有活动告警' : '监控不可用，无法读取活动告警'} pagination={false} rowKey="fingerprint" />
|
||||
</section>
|
||||
|
||||
<Modal footer={<><Button onClick={() => setShowSettings(false)} variant="ghost">取消</Button><Button disabled={savingSettings || !settings} onClick={() => void saveSettings()}>{savingSettings ? '验证并应用中' : '保存并应用'}</Button></>} onClose={() => setShowSettings(false)} open={showSettings} title="Prometheus 告警阈值设置">
|
||||
<div className="system-monitoring-threshold-dialog">
|
||||
<div className="system-monitoring-threshold-note"><ShieldAlert size={17} /><span>仅允许修改固定指标的数值阈值。保存时先由 promtool 校验，再原子替换规则并热加载 Prometheus。</span></div>
|
||||
{settings?.definitions.map((definition) => (
|
||||
<div className="system-monitoring-threshold-row" key={definition.key}>
|
||||
<div><strong>{definition.label}</strong><small>单位：{definition.unit}</small></div>
|
||||
<Input label="警告阈值" max={definition.max} min={definition.min} onChange={(event) => setDraftThresholds((current) => ({ ...current, [definition.key]: { ...current[definition.key], warning: Number(event.target.value) } }))} step={definition.step} type="number" value={draftThresholds[definition.key]?.warning ?? ''} />
|
||||
<Input label="严重阈值" max={definition.max} min={definition.min} onChange={(event) => setDraftThresholds((current) => ({ ...current, [definition.key]: { ...current[definition.key], critical: Number(event.target.value) } }))} step={definition.step} type="number" value={draftThresholds[definition.key]?.critical ?? ''} />
|
||||
</div>
|
||||
))}
|
||||
{settings?.applyStatus === 'failed' ? <div className="system-monitoring-unavailable"><AlertTriangle size={18} /><div><strong>上次应用失败</strong><span>{settings.lastError}</span></div></div> : null}
|
||||
{settingsError ? <div className="system-monitoring-unavailable"><AlertTriangle size={18} /><div><strong>阈值配置不可用</strong><span>{settingsError}</span></div></div> : null}
|
||||
</div>
|
||||
</Modal>
|
||||
</section>
|
||||
);
|
||||
}
|
||||
|
||||
@@ -49,6 +49,7 @@ function AdminAuthenticatedLayout({ session }: { session: LoginSession }) {
|
||||
const [pendingAudits, setPendingAudits] = useState(EMPTY_PENDING_AUDITS);
|
||||
const [retirementUnreadCount, setRetirementUnreadCount] = useState(0);
|
||||
const [securityAlertSummary, setSecurityAlertSummary] = useState({ count: 0, criticalCount: 0 });
|
||||
const [infrastructureAlertSummary, setInfrastructureAlertSummary] = useState({ count: 0, criticalCount: 0 });
|
||||
const [sessionLocked, setSessionLocked] = useState(Boolean(session.locked));
|
||||
const loadPendingAuditCount = useCallback(() => {
|
||||
const currentSession = readSession('admin');
|
||||
@@ -57,16 +58,18 @@ function AdminAuthenticatedLayout({ session }: { session: LoginSession }) {
|
||||
return;
|
||||
}
|
||||
// This runs globally and on a timer, so it must not fan out through the full dashboard aggregation.
|
||||
Promise.allSettled([adminApi.getPendingAudits(), adminApi.getSignatureRetirementUnreadCount(), adminApi.getSecurityNotificationSummary()])
|
||||
.then(([audits, retirement, security]) => {
|
||||
Promise.allSettled([adminApi.getPendingAudits(), adminApi.getSignatureRetirementUnreadCount(), adminApi.getSecurityNotificationSummary(), adminApi.getInfrastructureMonitoringNotificationSummary()])
|
||||
.then(([audits, retirement, security, infrastructure]) => {
|
||||
setPendingAudits(audits.status === 'fulfilled' ? audits.value : EMPTY_PENDING_AUDITS);
|
||||
setRetirementUnreadCount(retirement.status === 'fulfilled' ? retirement.value.count : 0);
|
||||
setSecurityAlertSummary(security.status === 'fulfilled' ? security.value : { count: 0, criticalCount: 0 });
|
||||
setInfrastructureAlertSummary(infrastructure.status === 'fulfilled' ? infrastructure.value : { count: 0, criticalCount: 0 });
|
||||
})
|
||||
.catch(() => {
|
||||
setPendingAudits(EMPTY_PENDING_AUDITS);
|
||||
setRetirementUnreadCount(0);
|
||||
setSecurityAlertSummary({ count: 0, criticalCount: 0 });
|
||||
setInfrastructureAlertSummary({ count: 0, criticalCount: 0 });
|
||||
});
|
||||
}, []);
|
||||
|
||||
@@ -82,12 +85,14 @@ function AdminAuthenticatedLayout({ session }: { session: LoginSession }) {
|
||||
window.addEventListener('cmpp-audit-count-refresh', onAuditRefresh);
|
||||
window.addEventListener('cmpp-retirement-count-refresh', onAuditRefresh);
|
||||
window.addEventListener('cmpp-security-alert-count-refresh', onAuditRefresh);
|
||||
window.addEventListener('cmpp-infrastructure-alert-count-refresh', onAuditRefresh);
|
||||
return () => {
|
||||
window.clearInterval(timer);
|
||||
window.removeEventListener('focus', onFocus);
|
||||
window.removeEventListener('cmpp-audit-count-refresh', onAuditRefresh);
|
||||
window.removeEventListener('cmpp-retirement-count-refresh', onAuditRefresh);
|
||||
window.removeEventListener('cmpp-security-alert-count-refresh', onAuditRefresh);
|
||||
window.removeEventListener('cmpp-infrastructure-alert-count-refresh', onAuditRefresh);
|
||||
};
|
||||
}, [loadPendingAuditCount, session.portal, sessionLocked]);
|
||||
|
||||
@@ -104,6 +109,7 @@ function AdminAuthenticatedLayout({ session }: { session: LoginSession }) {
|
||||
alertNotifications={[
|
||||
{ label: '签名清退预警', count: retirementUnreadCount, description: '今日未读且未抑制', to: '/admin/signature-retirement' },
|
||||
{ label: '安全检测与封禁', count: securityAlertSummary.count, description: securityAlertSummary.criticalCount > 0 ? `${securityAlertSummary.criticalCount} 条严重告警待处置` : '待处置安全告警', to: '/admin/security-detection' },
|
||||
{ label: '系统监控告警', count: infrastructureAlertSummary.count, description: infrastructureAlertSummary.criticalCount > 0 ? `${infrastructureAlertSummary.criticalCount} 条 Prometheus 严重告警` : 'Prometheus 活动告警', to: '/admin/system-monitoring#active-alerts' },
|
||||
]}
|
||||
auditNotifications={[
|
||||
{ label: '企业认证待审', count: pendingAudits.enterpriseCertifications, to: '/admin/enterprise-audit' },
|
||||
|
||||
@@ -0,0 +1,103 @@
|
||||
groups:
|
||||
- name: cmpp-managed-thresholds
|
||||
rules:
|
||||
- alert: HostCpuUsageWarning
|
||||
expr: (100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 80) and (100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) <= 90)
|
||||
for: 10m
|
||||
labels: { severity: warning, service: host }
|
||||
annotations: { summary: "主机 CPU 使用率达到警告阈值", description: "主机 CPU 使用率持续超过80%。", currentValue: "{{ $value }}", threshold: "80%" }
|
||||
- alert: HostCpuUsageCritical
|
||||
expr: 100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 90
|
||||
for: 5m
|
||||
labels: { severity: critical, service: host }
|
||||
annotations: { summary: "主机 CPU 使用率达到严重阈值", description: "主机 CPU 使用率持续超过90%。", currentValue: "{{ $value }}", threshold: "90%" }
|
||||
- alert: HostMemoryUsageWarning
|
||||
expr: ((1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100 > 85) and ((1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100 <= 95)
|
||||
for: 10m
|
||||
labels: { severity: warning, service: host }
|
||||
annotations: { summary: "主机内存使用率达到警告阈值", description: "主机内存使用率持续超过85%。", currentValue: "{{ $value }}", threshold: "85%" }
|
||||
- alert: HostMemoryUsageCritical
|
||||
expr: (1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100 > 95
|
||||
for: 5m
|
||||
labels: { severity: critical, service: host }
|
||||
annotations: { summary: "主机内存使用率达到严重阈值", description: "主机内存使用率持续超过95%。", currentValue: "{{ $value }}", threshold: "95%" }
|
||||
- alert: HostRootDiskUsageWarning
|
||||
expr: ((1 - node_filesystem_avail_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"} / node_filesystem_size_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"}) * 100 > 80) and ((1 - node_filesystem_avail_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"} / node_filesystem_size_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"}) * 100 <= 90)
|
||||
for: 15m
|
||||
labels: { severity: warning, service: host }
|
||||
annotations: { summary: "根磁盘使用率达到警告阈值", description: "根磁盘使用率持续超过80%。", currentValue: "{{ $value }}", threshold: "80%" }
|
||||
- alert: HostRootDiskUsageCritical
|
||||
expr: (1 - node_filesystem_avail_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"} / node_filesystem_size_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"}) * 100 > 90
|
||||
for: 5m
|
||||
labels: { severity: critical, service: host }
|
||||
annotations: { summary: "根磁盘使用率达到严重阈值", description: "根磁盘使用率持续超过90%。", currentValue: "{{ $value }}", threshold: "90%" }
|
||||
- alert: CmppApiHttpErrorRateWarning
|
||||
expr: (100 * sum(rate(cmpp_api_http_requests_total{status=~"5.."}[5m])) / clamp_min(sum(rate(cmpp_api_http_requests_total[5m])), 0.001) > 1) and (100 * sum(rate(cmpp_api_http_requests_total{status=~"5.."}[5m])) / clamp_min(sum(rate(cmpp_api_http_requests_total[5m])), 0.001) <= 5)
|
||||
for: 5m
|
||||
labels: { severity: warning, service: api }
|
||||
annotations: { summary: "API 5xx 错误率达到警告阈值", description: "API 5xx 错误率持续超过1%。", currentValue: "{{ $value }}", threshold: "1%" }
|
||||
- alert: CmppApiHttpErrorRateCritical
|
||||
expr: 100 * sum(rate(cmpp_api_http_requests_total{status=~"5.."}[5m])) / clamp_min(sum(rate(cmpp_api_http_requests_total[5m])), 0.001) > 5
|
||||
for: 5m
|
||||
labels: { severity: critical, service: api }
|
||||
annotations: { summary: "API 5xx 错误率达到严重阈值", description: "API 5xx 错误率持续超过5%。", currentValue: "{{ $value }}", threshold: "5%" }
|
||||
- alert: CmppApiLatencyWarning
|
||||
expr: (histogram_quantile(0.95, sum by (le) (rate(cmpp_api_http_request_duration_seconds_bucket[10m]))) > 1) and (histogram_quantile(0.95, sum by (le) (rate(cmpp_api_http_request_duration_seconds_bucket[10m]))) <= 3)
|
||||
for: 10m
|
||||
labels: { severity: warning, service: api }
|
||||
annotations: { summary: "API P95 响应时间达到警告阈值", description: "API P95 响应时间持续超过1秒。", currentValue: "{{ $value }}", threshold: "1秒" }
|
||||
- alert: CmppApiLatencyCritical
|
||||
expr: histogram_quantile(0.95, sum by (le) (rate(cmpp_api_http_request_duration_seconds_bucket[10m]))) > 3
|
||||
for: 5m
|
||||
labels: { severity: critical, service: api }
|
||||
annotations: { summary: "API P95 响应时间达到严重阈值", description: "API P95 响应时间持续超过3秒。", currentValue: "{{ $value }}", threshold: "3秒" }
|
||||
- alert: CmppApiEventLoopLagWarning
|
||||
expr: (cmpp_api_nodejs_event_loop_lag_p99_seconds > 0.2) and (cmpp_api_nodejs_event_loop_lag_p99_seconds <= 1)
|
||||
for: 10m
|
||||
labels: { severity: warning, service: api }
|
||||
annotations: { summary: "API 事件循环 P99 达到警告阈值", description: "API 事件循环 P99 持续超过0.2秒。", currentValue: "{{ $value }}", threshold: "0.2秒" }
|
||||
- alert: CmppApiEventLoopLagCritical
|
||||
expr: cmpp_api_nodejs_event_loop_lag_p99_seconds > 1
|
||||
for: 5m
|
||||
labels: { severity: critical, service: api }
|
||||
annotations: { summary: "API 事件循环 P99 达到严重阈值", description: "API 事件循环 P99 持续超过1秒。", currentValue: "{{ $value }}", threshold: "1秒" }
|
||||
- alert: CmppGatewayQueueDelayedWarning
|
||||
expr: (cmpp_gateway_submit_queue_oldest_pending_age_seconds > 30) and (cmpp_gateway_submit_queue_oldest_pending_age_seconds <= 120)
|
||||
for: 2m
|
||||
labels: { severity: warning, service: gateway }
|
||||
annotations: { summary: "Gateway 最旧 pending 达到警告阈值", description: "Gateway 最旧 pending 持续超过30秒。", currentValue: "{{ $value }}", threshold: "30秒" }
|
||||
- alert: CmppGatewayQueueDelayedCritical
|
||||
expr: cmpp_gateway_submit_queue_oldest_pending_age_seconds > 120
|
||||
for: 2m
|
||||
labels: { severity: critical, service: gateway }
|
||||
annotations: { summary: "Gateway 最旧 pending 达到严重阈值", description: "Gateway 最旧 pending 持续超过120秒。", currentValue: "{{ $value }}", threshold: "120秒" }
|
||||
- alert: PostgresConnectionsWarning
|
||||
expr: (100 * sum(pg_stat_activity_count) / clamp_min(max(pg_settings_max_connections), 1) > 70) and (100 * sum(pg_stat_activity_count) / clamp_min(max(pg_settings_max_connections), 1) <= 85)
|
||||
for: 10m
|
||||
labels: { severity: warning, service: postgresql }
|
||||
annotations: { summary: "PostgreSQL 连接使用率达到警告阈值", description: "PostgreSQL 连接使用率持续超过70%。", currentValue: "{{ $value }}", threshold: "70%" }
|
||||
- alert: PostgresConnectionsCritical
|
||||
expr: 100 * sum(pg_stat_activity_count) / clamp_min(max(pg_settings_max_connections), 1) > 85
|
||||
for: 5m
|
||||
labels: { severity: critical, service: postgresql }
|
||||
annotations: { summary: "PostgreSQL 连接使用率达到严重阈值", description: "PostgreSQL 连接使用率持续超过85%。", currentValue: "{{ $value }}", threshold: "85%" }
|
||||
- alert: RedisMemoryWarning
|
||||
expr: (100 * redis_memory_used_bytes / redis_memory_max_bytes > 70) and (100 * redis_memory_used_bytes / redis_memory_max_bytes <= 85)
|
||||
for: 10m
|
||||
labels: { severity: warning, service: redis }
|
||||
annotations: { summary: "Redis 内存使用率达到警告阈值", description: "Redis 内存使用率持续超过70%。", currentValue: "{{ $value }}", threshold: "70%" }
|
||||
- alert: RedisMemoryCritical
|
||||
expr: 100 * redis_memory_used_bytes / redis_memory_max_bytes > 85
|
||||
for: 5m
|
||||
labels: { severity: critical, service: redis }
|
||||
annotations: { summary: "Redis 内存使用率达到严重阈值", description: "Redis 内存使用率持续超过85%。", currentValue: "{{ $value }}", threshold: "85%" }
|
||||
- alert: MinioCapacityWarning
|
||||
expr: (100 * (1 - minio_cluster_capacity_usable_free_bytes / minio_cluster_capacity_usable_total_bytes) > 80) and (100 * (1 - minio_cluster_capacity_usable_free_bytes / minio_cluster_capacity_usable_total_bytes) <= 90)
|
||||
for: 15m
|
||||
labels: { severity: warning, service: minio }
|
||||
annotations: { summary: "MinIO 容量使用率达到警告阈值", description: "MinIO 容量使用率持续超过80%。", currentValue: "{{ $value }}", threshold: "80%" }
|
||||
- alert: MinioCapacityCritical
|
||||
expr: 100 * (1 - minio_cluster_capacity_usable_free_bytes / minio_cluster_capacity_usable_total_bytes) > 90
|
||||
for: 5m
|
||||
labels: { severity: critical, service: minio }
|
||||
annotations: { summary: "MinIO 容量使用率达到严重阈值", description: "MinIO 容量使用率持续超过90%。", currentValue: "{{ $value }}", threshold: "90%" }
|
||||
@@ -26,7 +26,7 @@ promtool_bin="$(command -v promtool)"
|
||||
backup_dir="/etc/prometheus/cmpp-backups/$(date '+%Y%m%d-%H%M%S')"
|
||||
mkdir -p "$backup_dir" /etc/systemd/system/prometheus.service.d /etc/systemd/system/prometheus-node-exporter.service.d
|
||||
|
||||
for config_file in /etc/prometheus/prometheus.yml /etc/prometheus/cmpp-alerts.yml; do
|
||||
for config_file in /etc/prometheus/prometheus.yml /etc/prometheus/cmpp-alerts.yml /etc/prometheus/cmpp-alerts-source.yml; do
|
||||
if [[ -f "$config_file" ]]; then
|
||||
cp --preserve=mode,timestamps "$config_file" "$backup_dir/$(basename "$config_file")"
|
||||
fi
|
||||
@@ -40,12 +40,28 @@ fi
|
||||
|
||||
log "Installing platform-owned scrape and alert configuration"
|
||||
install -o root -g root -m 0644 "$SCRIPT_DIR/prometheus.yml" /etc/prometheus/prometheus.yml
|
||||
install -o root -g root -m 0644 "$SCRIPT_DIR/cmpp-alerts.yml" /etc/prometheus/cmpp-alerts.yml
|
||||
install -o root -g root -m 0644 "$SCRIPT_DIR/cmpp-alerts.yml" /etc/prometheus/cmpp-alerts-source.yml
|
||||
# 可配置规则由 API 管理；基础规则必须排除同名项，否则 Prometheus 会同时计算旧阈值和新阈值。
|
||||
awk '
|
||||
BEGIN {
|
||||
split("HostCpuUsageWarning HostCpuUsageCritical HostMemoryUsageWarning HostMemoryUsageCritical HostRootDiskUsageWarning HostRootDiskUsageCritical CmppApiHttpErrorRateWarning CmppApiHttpErrorRateCritical CmppApiLatencyWarning CmppApiLatencyCritical CmppApiEventLoopLagWarning CmppApiEventLoopLagCritical CmppGatewayQueueDelayedWarning CmppGatewayQueueDelayedCritical PostgresConnectionsWarning PostgresConnectionsCritical RedisMemoryWarning RedisMemoryCritical MinioCapacityWarning MinioCapacityCritical", names, " ")
|
||||
for (i in names) dropped[names[i]] = 1
|
||||
}
|
||||
/^ - name:/ { skip = 0 }
|
||||
/^ - alert:/ { skip = ($3 in dropped) }
|
||||
!skip { print }
|
||||
' "$SCRIPT_DIR/cmpp-alerts.yml" > /etc/prometheus/cmpp-alerts.yml
|
||||
chown root:root /etc/prometheus/cmpp-alerts.yml
|
||||
chmod 0644 /etc/prometheus/cmpp-alerts.yml
|
||||
install -d -o cmpp-api -g prometheus -m 0750 /var/lib/cmpp-platform/monitoring
|
||||
if [[ ! -f /var/lib/cmpp-platform/monitoring/cmpp-managed-alerts.yml ]]; then
|
||||
install -o cmpp-api -g prometheus -m 0640 "$SCRIPT_DIR/cmpp-managed-alerts.yml" /var/lib/cmpp-platform/monitoring/cmpp-managed-alerts.yml
|
||||
fi
|
||||
|
||||
cat >/etc/systemd/system/prometheus.service.d/cmpp-monitoring.conf <<EOF
|
||||
[Service]
|
||||
ExecStart=
|
||||
ExecStart=${prometheus_bin} --config.file=/etc/prometheus/prometheus.yml --storage.tsdb.path=/var/lib/prometheus/metrics2 --storage.tsdb.retention.time=${PROMETHEUS_RETENTION_TIME} --storage.tsdb.retention.size=${PROMETHEUS_RETENTION_SIZE} --web.listen-address=127.0.0.1:9090
|
||||
ExecStart=${prometheus_bin} --config.file=/etc/prometheus/prometheus.yml --storage.tsdb.path=/var/lib/prometheus/metrics2 --storage.tsdb.retention.time=${PROMETHEUS_RETENTION_TIME} --storage.tsdb.retention.size=${PROMETHEUS_RETENTION_SIZE} --web.listen-address=127.0.0.1:9090 --web.enable-lifecycle
|
||||
EOF
|
||||
|
||||
cat >/etc/systemd/system/prometheus-node-exporter.service.d/cmpp-monitoring.conf <<EOF
|
||||
@@ -56,6 +72,7 @@ EOF
|
||||
|
||||
log "Validating Prometheus configuration before restart"
|
||||
"$promtool_bin" check rules /etc/prometheus/cmpp-alerts.yml
|
||||
"$promtool_bin" check rules /var/lib/cmpp-platform/monitoring/cmpp-managed-alerts.yml
|
||||
"$promtool_bin" check config /etc/prometheus/prometheus.yml
|
||||
|
||||
systemctl daemon-reload
|
||||
|
||||
@@ -7,6 +7,7 @@ global:
|
||||
|
||||
rule_files:
|
||||
- /etc/prometheus/cmpp-alerts.yml
|
||||
- /var/lib/cmpp-platform/monitoring/*.yml
|
||||
|
||||
scrape_configs:
|
||||
- job_name: prometheus
|
||||
|
||||
Reference in New Issue
Block a user