feat: add configurable infrastructure alerts
This commit is contained in:
@@ -0,0 +1,23 @@
|
|||||||
|
CREATE TABLE "InfrastructureAlertSetting" (
|
||||||
|
"id" TEXT NOT NULL DEFAULT 'global',
|
||||||
|
"configVersion" INTEGER NOT NULL DEFAULT 1,
|
||||||
|
"effectiveVersion" INTEGER NOT NULL DEFAULT 1,
|
||||||
|
"thresholds" JSONB NOT NULL,
|
||||||
|
"effectiveThresholds" JSONB NOT NULL,
|
||||||
|
"applyStatus" TEXT NOT NULL DEFAULT 'effective',
|
||||||
|
"lastError" TEXT,
|
||||||
|
"updatedById" TEXT,
|
||||||
|
"appliedAt" TIMESTAMP(3),
|
||||||
|
"createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||||
|
"updatedAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||||
|
CONSTRAINT "InfrastructureAlertSetting_pkey" PRIMARY KEY ("id")
|
||||||
|
);
|
||||||
|
|
||||||
|
INSERT INTO "InfrastructureAlertSetting" (
|
||||||
|
"id", "thresholds", "effectiveThresholds", "appliedAt"
|
||||||
|
) VALUES (
|
||||||
|
'global',
|
||||||
|
'{"hostCpu":{"warning":80,"critical":90},"hostMemory":{"warning":85,"critical":95},"hostDisk":{"warning":80,"critical":90},"apiError":{"warning":1,"critical":5},"apiLatency":{"warning":1,"critical":3},"apiEventLoop":{"warning":0.2,"critical":1},"gatewayQueue":{"warning":30,"critical":120},"postgresConnections":{"warning":70,"critical":85},"redisMemory":{"warning":70,"critical":85},"minioCapacity":{"warning":80,"critical":90}}'::jsonb,
|
||||||
|
'{"hostCpu":{"warning":80,"critical":90},"hostMemory":{"warning":85,"critical":95},"hostDisk":{"warning":80,"critical":90},"apiError":{"warning":1,"critical":5},"apiLatency":{"warning":1,"critical":3},"apiEventLoop":{"warning":0.2,"critical":1},"gatewayQueue":{"warning":30,"critical":120},"postgresConnections":{"warning":70,"critical":85},"redisMemory":{"warning":70,"critical":85},"minioCapacity":{"warning":80,"critical":90}}'::jsonb,
|
||||||
|
CURRENT_TIMESTAMP
|
||||||
|
);
|
||||||
@@ -2432,3 +2432,17 @@ model SecurityProtectedNetwork {
|
|||||||
|
|
||||||
@@index([enabled, createdAt])
|
@@index([enabled, createdAt])
|
||||||
}
|
}
|
||||||
|
|
||||||
|
model InfrastructureAlertSetting {
|
||||||
|
id String @id @default("global")
|
||||||
|
configVersion Int @default(1)
|
||||||
|
effectiveVersion Int @default(1)
|
||||||
|
thresholds Json
|
||||||
|
effectiveThresholds Json
|
||||||
|
applyStatus String @default("effective")
|
||||||
|
lastError String?
|
||||||
|
updatedById String?
|
||||||
|
appliedAt DateTime?
|
||||||
|
createdAt DateTime @default(now())
|
||||||
|
updatedAt DateTime @updatedAt
|
||||||
|
}
|
||||||
|
|||||||
@@ -0,0 +1,19 @@
|
|||||||
|
import { BadRequestException } from '@nestjs/common';
|
||||||
|
import { DEFAULT_ALERT_THRESHOLDS, InfrastructureAlertSettingsService } from './infrastructure-alert-settings.service';
|
||||||
|
|
||||||
|
describe('InfrastructureAlertSettingsService', () => {
|
||||||
|
const service = new InfrastructureAlertSettingsService({} as never, { get: () => undefined } as never);
|
||||||
|
|
||||||
|
it('accepts the fixed threshold whitelist and renders managed rules', () => {
|
||||||
|
const validated = (service as unknown as { validate(value: unknown): unknown }).validate(DEFAULT_ALERT_THRESHOLDS);
|
||||||
|
const rules = (service as unknown as { renderRules(value: unknown): string }).renderRules(validated);
|
||||||
|
expect(rules).toContain('HostCpuUsageWarning');
|
||||||
|
expect(rules).toContain('CmppGatewayQueueDelayedCritical');
|
||||||
|
expect(rules).toContain('threshold: "120秒"');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('rejects unknown keys and warning thresholds that are not below critical', () => {
|
||||||
|
expect(() => (service as unknown as { validate(value: unknown): unknown }).validate({ ...DEFAULT_ALERT_THRESHOLDS, promql: { warning: 1, critical: 2 } })).toThrow(BadRequestException);
|
||||||
|
expect(() => (service as unknown as { validate(value: unknown): unknown }).validate({ ...DEFAULT_ALERT_THRESHOLDS, hostCpu: { warning: 90, critical: 90 } })).toThrow(BadRequestException);
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -0,0 +1,142 @@
|
|||||||
|
import { BadRequestException, ConflictException, Injectable, Logger, ServiceUnavailableException } from '@nestjs/common';
|
||||||
|
import { ConfigService } from '@nestjs/config';
|
||||||
|
import { Prisma } from '@prisma/client';
|
||||||
|
import { execFile } from 'node:child_process';
|
||||||
|
import { mkdir, readFile, rename, rm, writeFile } from 'node:fs/promises';
|
||||||
|
import { dirname } from 'node:path';
|
||||||
|
import { promisify } from 'node:util';
|
||||||
|
import { PrismaService } from '../prisma/prisma.service';
|
||||||
|
import type { InfrastructureAlertSettings, InfrastructureAlertThresholds } from './infrastructure-monitoring.contracts';
|
||||||
|
|
||||||
|
const execFileAsync = promisify(execFile);
|
||||||
|
|
||||||
|
export const ALERT_THRESHOLD_DEFINITIONS = [
|
||||||
|
{ key: 'hostCpu', label: '主机 CPU 使用率', unit: '%', min: 1, max: 100, step: 1, warning: 80, critical: 90, expr: '100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100)', names: ['HostCpuUsageWarning', 'HostCpuUsageCritical'], service: 'host', durations: ['10m', '5m'] },
|
||||||
|
{ key: 'hostMemory', label: '主机内存使用率', unit: '%', min: 1, max: 100, step: 1, warning: 85, critical: 95, expr: '(1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100', names: ['HostMemoryUsageWarning', 'HostMemoryUsageCritical'], service: 'host', durations: ['10m', '5m'] },
|
||||||
|
{ key: 'hostDisk', label: '根磁盘使用率', unit: '%', min: 1, max: 100, step: 1, warning: 80, critical: 90, expr: '(1 - node_filesystem_avail_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"} / node_filesystem_size_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"}) * 100', names: ['HostRootDiskUsageWarning', 'HostRootDiskUsageCritical'], service: 'host', durations: ['15m', '5m'] },
|
||||||
|
{ key: 'apiError', label: 'API 5xx 错误率', unit: '%', min: 0.1, max: 100, step: 0.1, warning: 1, critical: 5, expr: '100 * sum(rate(cmpp_api_http_requests_total{status=~"5.."}[5m])) / clamp_min(sum(rate(cmpp_api_http_requests_total[5m])), 0.001)', names: ['CmppApiHttpErrorRateWarning', 'CmppApiHttpErrorRateCritical'], service: 'api', durations: ['5m', '5m'] },
|
||||||
|
{ key: 'apiLatency', label: 'API P95 响应时间', unit: '秒', min: 0.1, max: 60, step: 0.1, warning: 1, critical: 3, expr: 'histogram_quantile(0.95, sum by (le) (rate(cmpp_api_http_request_duration_seconds_bucket[10m])))', names: ['CmppApiLatencyWarning', 'CmppApiLatencyCritical'], service: 'api', durations: ['10m', '5m'] },
|
||||||
|
{ key: 'apiEventLoop', label: 'API 事件循环 P99', unit: '秒', min: 0.01, max: 10, step: 0.01, warning: 0.2, critical: 1, expr: 'cmpp_api_nodejs_event_loop_lag_p99_seconds', names: ['CmppApiEventLoopLagWarning', 'CmppApiEventLoopLagCritical'], service: 'api', durations: ['10m', '5m'] },
|
||||||
|
{ key: 'gatewayQueue', label: 'Gateway 最旧 pending', unit: '秒', min: 1, max: 3600, step: 1, warning: 30, critical: 120, expr: 'cmpp_gateway_submit_queue_oldest_pending_age_seconds', names: ['CmppGatewayQueueDelayedWarning', 'CmppGatewayQueueDelayedCritical'], service: 'gateway', durations: ['2m', '2m'] },
|
||||||
|
{ key: 'postgresConnections', label: 'PostgreSQL 连接使用率', unit: '%', min: 1, max: 100, step: 1, warning: 70, critical: 85, expr: '100 * sum(pg_stat_activity_count) / clamp_min(max(pg_settings_max_connections), 1)', names: ['PostgresConnectionsWarning', 'PostgresConnectionsCritical'], service: 'postgresql', durations: ['10m', '5m'] },
|
||||||
|
{ key: 'redisMemory', label: 'Redis 内存使用率', unit: '%', min: 1, max: 100, step: 1, warning: 70, critical: 85, expr: '100 * redis_memory_used_bytes / redis_memory_max_bytes', names: ['RedisMemoryWarning', 'RedisMemoryCritical'], service: 'redis', durations: ['10m', '5m'] },
|
||||||
|
{ key: 'minioCapacity', label: 'MinIO 容量使用率', unit: '%', min: 1, max: 100, step: 1, warning: 80, critical: 90, expr: '100 * (1 - minio_cluster_capacity_usable_free_bytes / minio_cluster_capacity_usable_total_bytes)', names: ['MinioCapacityWarning', 'MinioCapacityCritical'], service: 'minio', durations: ['15m', '5m'] },
|
||||||
|
] as const;
|
||||||
|
|
||||||
|
export const DEFAULT_ALERT_THRESHOLDS: InfrastructureAlertThresholds = Object.fromEntries(
|
||||||
|
ALERT_THRESHOLD_DEFINITIONS.map((item) => [item.key, { warning: item.warning, critical: item.critical }]),
|
||||||
|
);
|
||||||
|
|
||||||
|
@Injectable()
|
||||||
|
export class InfrastructureAlertSettingsService {
|
||||||
|
private readonly logger = new Logger(InfrastructureAlertSettingsService.name);
|
||||||
|
private readonly rulesPath: string;
|
||||||
|
private readonly promtoolPath: string;
|
||||||
|
private readonly reloadUrl: string;
|
||||||
|
|
||||||
|
constructor(private readonly prisma: PrismaService, config: ConfigService) {
|
||||||
|
this.rulesPath = String(config.get('PROMETHEUS_MANAGED_RULES_PATH') ?? '/var/lib/cmpp-platform/monitoring/cmpp-managed-alerts.yml');
|
||||||
|
this.promtoolPath = String(config.get('PROMTOOL_PATH') ?? '/usr/bin/promtool');
|
||||||
|
this.reloadUrl = String(config.get('PROMETHEUS_RELOAD_URL') ?? 'http://127.0.0.1:9090/-/reload');
|
||||||
|
}
|
||||||
|
|
||||||
|
async get(): Promise<InfrastructureAlertSettings> {
|
||||||
|
const row = await this.prisma.infrastructureAlertSetting.findUnique({ where: { id: 'global' } });
|
||||||
|
const thresholds = this.asThresholds(row?.thresholds) ?? DEFAULT_ALERT_THRESHOLDS;
|
||||||
|
const effective = this.asThresholds(row?.effectiveThresholds) ?? thresholds;
|
||||||
|
return {
|
||||||
|
configVersion: row?.configVersion ?? 1,
|
||||||
|
effectiveVersion: row?.effectiveVersion ?? 1,
|
||||||
|
applyStatus: (row?.applyStatus as InfrastructureAlertSettings['applyStatus']) ?? 'effective',
|
||||||
|
lastError: row?.lastError ?? null,
|
||||||
|
appliedAt: row?.appliedAt?.toISOString() ?? null,
|
||||||
|
thresholds,
|
||||||
|
effectiveThresholds: effective,
|
||||||
|
definitions: ALERT_THRESHOLD_DEFINITIONS.map(({ key, label, unit, min, max, step }) => ({ key, label, unit, min, max, step })),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
async update(body: { configVersion?: number; thresholds?: unknown }, operatorId?: string) {
|
||||||
|
const expectedVersion = Number(body.configVersion);
|
||||||
|
if (!Number.isInteger(expectedVersion) || expectedVersion < 1) throw new BadRequestException('配置版本无效');
|
||||||
|
const thresholds = this.validate(body.thresholds);
|
||||||
|
const claimed = await this.prisma.infrastructureAlertSetting.updateMany({
|
||||||
|
where: { id: 'global', configVersion: expectedVersion },
|
||||||
|
data: { configVersion: { increment: 1 }, thresholds: thresholds as Prisma.InputJsonValue, applyStatus: 'applying', lastError: null, updatedById: operatorId },
|
||||||
|
});
|
||||||
|
// 版本条件更新是跨进程的写锁;避免两个 API 实例同时覆盖规则文件并把旧配置误标成已生效。
|
||||||
|
if (claimed.count !== 1) throw new ConflictException('告警阈值已被其他管理员修改,请刷新后重试');
|
||||||
|
const nextVersion = expectedVersion + 1;
|
||||||
|
try {
|
||||||
|
await this.applyRules(thresholds);
|
||||||
|
await this.prisma.$transaction([
|
||||||
|
this.prisma.infrastructureAlertSetting.update({ where: { id: 'global' }, data: { effectiveVersion: nextVersion, effectiveThresholds: thresholds as Prisma.InputJsonValue, applyStatus: 'effective', lastError: null, appliedAt: new Date() } }),
|
||||||
|
this.prisma.operationLog.create({ data: { userId: operatorId, action: 'monitoring.alert_thresholds_updated', resource: 'infrastructure_alert_setting', resourceId: 'global', detail: { configVersion: nextVersion, thresholds } } }),
|
||||||
|
]);
|
||||||
|
} catch (error) {
|
||||||
|
const message = error instanceof Error ? error.message.slice(0, 500) : 'unknown error';
|
||||||
|
await this.prisma.infrastructureAlertSetting.update({ where: { id: 'global' }, data: { applyStatus: 'failed', lastError: message } });
|
||||||
|
this.logger.error(`Prometheus managed rules apply failed: ${message}`);
|
||||||
|
throw new ServiceUnavailableException('阈值已保存但 Prometheus 应用失败,原生效规则已保留');
|
||||||
|
}
|
||||||
|
return this.get();
|
||||||
|
}
|
||||||
|
|
||||||
|
private validate(value: unknown): InfrastructureAlertThresholds {
|
||||||
|
if (!value || typeof value !== 'object' || Array.isArray(value)) throw new BadRequestException('告警阈值格式无效');
|
||||||
|
const input = value as Record<string, unknown>;
|
||||||
|
if (Object.keys(input).some((key) => !ALERT_THRESHOLD_DEFINITIONS.some((item) => item.key === key))) throw new BadRequestException('存在不允许配置的告警指标');
|
||||||
|
const result: InfrastructureAlertThresholds = {};
|
||||||
|
for (const definition of ALERT_THRESHOLD_DEFINITIONS) {
|
||||||
|
const pair = input[definition.key] as { warning?: unknown; critical?: unknown } | undefined;
|
||||||
|
const warning = Number(pair?.warning);
|
||||||
|
const critical = Number(pair?.critical);
|
||||||
|
if (!Number.isFinite(warning) || !Number.isFinite(critical) || warning < definition.min || critical > definition.max || warning >= critical) {
|
||||||
|
throw new BadRequestException(`${definition.label}必须满足最小值 ≤ 警告阈值 < 严重阈值 ≤ 最大值`);
|
||||||
|
}
|
||||||
|
result[definition.key] = { warning, critical };
|
||||||
|
}
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
private asThresholds(value: unknown) {
|
||||||
|
try { return this.validate(value); } catch { return null; }
|
||||||
|
}
|
||||||
|
|
||||||
|
private renderRules(thresholds: InfrastructureAlertThresholds) {
|
||||||
|
const lines = ['groups:', ' - name: cmpp-managed-thresholds', ' rules:'];
|
||||||
|
for (const definition of ALERT_THRESHOLD_DEFINITIONS) {
|
||||||
|
const pair = thresholds[definition.key];
|
||||||
|
const values = [pair.warning, pair.critical];
|
||||||
|
for (let index = 0; index < 2; index += 1) {
|
||||||
|
const isWarning = index === 0;
|
||||||
|
const expr = isWarning ? `(${definition.expr} > ${values[0]}) and (${definition.expr} <= ${values[1]})` : `${definition.expr} > ${values[1]}`;
|
||||||
|
lines.push(` - alert: ${definition.names[index]}`, ` expr: ${expr}`, ` for: ${definition.durations[index]}`, ' labels:', ` severity: ${isWarning ? 'warning' : 'critical'}`, ` service: ${definition.service}`, ' annotations:', ` summary: "${definition.label}${isWarning ? '达到警告阈值' : '达到严重阈值'}"`, ` description: "${definition.label}持续超过${values[index]}${definition.unit}。"`, ' currentValue: "{{ $value }}"', ` threshold: "${values[index]}${definition.unit}"`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return `${lines.join('\n')}\n`;
|
||||||
|
}
|
||||||
|
|
||||||
|
private async applyRules(thresholds: InfrastructureAlertThresholds) {
|
||||||
|
const directory = dirname(this.rulesPath);
|
||||||
|
const temporary = `${this.rulesPath}.${process.pid}.${Date.now()}.tmp`;
|
||||||
|
await mkdir(directory, { recursive: true });
|
||||||
|
const previous = await readFile(this.rulesPath).catch(() => null);
|
||||||
|
try {
|
||||||
|
await writeFile(temporary, this.renderRules(thresholds), { mode: 0o640 });
|
||||||
|
await execFileAsync(this.promtoolPath, ['check', 'rules', temporary], { timeout: 10_000 });
|
||||||
|
await rename(temporary, this.rulesPath);
|
||||||
|
const response = await fetch(this.reloadUrl, { method: 'POST', signal: AbortSignal.timeout(5_000) });
|
||||||
|
if (!response.ok) throw new Error(`Prometheus reload HTTP ${response.status}`);
|
||||||
|
} catch (error) {
|
||||||
|
await rm(temporary, { force: true });
|
||||||
|
// 规则替换和 reload 不是一个事务,失败时必须恢复旧文件并再次 reload,避免数据库状态与实际告警漂移。
|
||||||
|
if (previous) {
|
||||||
|
await writeFile(temporary, previous, { mode: 0o640 });
|
||||||
|
await rename(temporary, this.rulesPath);
|
||||||
|
await fetch(this.reloadUrl, { method: 'POST', signal: AbortSignal.timeout(5_000) }).catch(() => undefined);
|
||||||
|
}
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -76,3 +76,16 @@ export type InfrastructureMonitoringOverview = {
|
|||||||
serviceMetrics: InfrastructureServiceMetricGroup[];
|
serviceMetrics: InfrastructureServiceMetricGroup[];
|
||||||
alerts: InfrastructureAlert[];
|
alerts: InfrastructureAlert[];
|
||||||
};
|
};
|
||||||
|
|
||||||
|
export type InfrastructureAlertThresholds = Record<string, { warning: number; critical: number }>;
|
||||||
|
|
||||||
|
export type InfrastructureAlertSettings = {
|
||||||
|
configVersion: number;
|
||||||
|
effectiveVersion: number;
|
||||||
|
applyStatus: 'effective' | 'applying' | 'failed';
|
||||||
|
lastError: string | null;
|
||||||
|
appliedAt: string | null;
|
||||||
|
thresholds: InfrastructureAlertThresholds;
|
||||||
|
effectiveThresholds: InfrastructureAlertThresholds;
|
||||||
|
definitions: Array<{ key: string; label: string; unit: string; min: number; max: number; step: number }>;
|
||||||
|
};
|
||||||
|
|||||||
@@ -1,14 +1,29 @@
|
|||||||
import { Controller, Get, Query } from '@nestjs/common';
|
import { Body, Controller, Get, Put, Query } from '@nestjs/common';
|
||||||
import { ApiTags } from '@nestjs/swagger';
|
import { ApiTags } from '@nestjs/swagger';
|
||||||
|
import { CurrentSessionUserId } from '../auth/current-session-user.decorator';
|
||||||
|
import { RequireRecentAuthentication } from '../auth/require-recent-authentication.decorator';
|
||||||
|
import { InfrastructureAlertSettingsService } from './infrastructure-alert-settings.service';
|
||||||
import { InfrastructureMonitoringService } from './infrastructure-monitoring.service';
|
import { InfrastructureMonitoringService } from './infrastructure-monitoring.service';
|
||||||
|
|
||||||
@ApiTags('infrastructure-monitoring')
|
@ApiTags('infrastructure-monitoring')
|
||||||
@Controller('admin/infrastructure-monitoring')
|
@Controller('admin/infrastructure-monitoring')
|
||||||
export class InfrastructureMonitoringController {
|
export class InfrastructureMonitoringController {
|
||||||
constructor(private readonly monitoring: InfrastructureMonitoringService) {}
|
constructor(private readonly monitoring: InfrastructureMonitoringService, private readonly settings: InfrastructureAlertSettingsService) {}
|
||||||
|
|
||||||
@Get('overview')
|
@Get('overview')
|
||||||
overview(@Query('range') range?: string) {
|
overview(@Query('range') range?: string) {
|
||||||
return this.monitoring.overview(range);
|
return this.monitoring.overview(range);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Get('notification-summary')
|
||||||
|
notificationSummary() { return this.monitoring.notificationSummary(); }
|
||||||
|
|
||||||
|
@Get('alert-thresholds')
|
||||||
|
alertThresholds() { return this.settings.get(); }
|
||||||
|
|
||||||
|
@Put('alert-thresholds')
|
||||||
|
@RequireRecentAuthentication()
|
||||||
|
updateAlertThresholds(@Body() body: { configVersion?: number; thresholds?: unknown }, @CurrentSessionUserId() operatorId?: string) {
|
||||||
|
return this.settings.update(body, operatorId);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,9 +1,10 @@
|
|||||||
import { Module } from '@nestjs/common';
|
import { Module } from '@nestjs/common';
|
||||||
import { InfrastructureMonitoringController } from './infrastructure-monitoring.controller';
|
import { InfrastructureMonitoringController } from './infrastructure-monitoring.controller';
|
||||||
import { InfrastructureMonitoringService } from './infrastructure-monitoring.service';
|
import { InfrastructureMonitoringService } from './infrastructure-monitoring.service';
|
||||||
|
import { InfrastructureAlertSettingsService } from './infrastructure-alert-settings.service';
|
||||||
|
|
||||||
@Module({
|
@Module({
|
||||||
controllers: [InfrastructureMonitoringController],
|
controllers: [InfrastructureMonitoringController],
|
||||||
providers: [InfrastructureMonitoringService],
|
providers: [InfrastructureMonitoringService, InfrastructureAlertSettingsService],
|
||||||
})
|
})
|
||||||
export class InfrastructureMonitoringModule {}
|
export class InfrastructureMonitoringModule {}
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
import { BadRequestException, Injectable, Logger } from '@nestjs/common';
|
import { BadRequestException, Injectable, Logger, ServiceUnavailableException } from '@nestjs/common';
|
||||||
import { ConfigService } from '@nestjs/config';
|
import { ConfigService } from '@nestjs/config';
|
||||||
import { createHash } from 'node:crypto';
|
import { createHash } from 'node:crypto';
|
||||||
import type {
|
import type {
|
||||||
@@ -210,6 +210,16 @@ export class InfrastructureMonitoringService {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async notificationSummary() {
|
||||||
|
try {
|
||||||
|
const alerts = this.parseAlerts(await this.getJson<PrometheusAlertResponse>('/api/v1/alerts'));
|
||||||
|
return { count: alerts.length, criticalCount: alerts.filter((item) => item.severity === 'critical').length };
|
||||||
|
} catch (error) {
|
||||||
|
this.logger.warn(`Prometheus notification summary unavailable: ${error instanceof Error ? error.message : 'unknown error'}`);
|
||||||
|
throw new ServiceUnavailableException('Prometheus活动告警当前不可用');
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
private parseRange(value?: string): InfrastructureMonitoringRange {
|
private parseRange(value?: string): InfrastructureMonitoringRange {
|
||||||
const range = value || '24h';
|
const range = value || '24h';
|
||||||
if (!(range in RANGE_CONFIG)) throw new BadRequestException('监控时间范围只支持1h、24h或7d');
|
if (!(range in RANGE_CONFIG)) throw new BadRequestException('监控时间范围只支持1h、24h或7d');
|
||||||
|
|||||||
@@ -1150,4 +1150,4 @@ global,不能错误归入client。`client-signature-*`、发送页、企业认
|
|||||||
|
|
||||||
- `api/src/metrics/`只负责API进程和HTTP路由模板的低基数计数/直方图,不依赖Prisma、Redis或业务Service;独立回环监听由`api/src/main.ts`组装。
|
- `api/src/metrics/`只负责API进程和HTTP路由模板的低基数计数/直方图,不依赖Prisma、Redis或业务Service;独立回环监听由`api/src/main.ts`组装。
|
||||||
- `gateway/internal/metrics/`只负责线程安全聚合和Prometheus文本暴露;上游连接池、下游Session和Submit Worker仅提供总数或有界结果,不把实体ID或凭据交给监控模块。
|
- `gateway/internal/metrics/`只负责线程安全聚合和Prometheus文本暴露;上游连接池、下游Session和Submit Worker仅提供总数或有界结果,不把实体ID或凭据交给监控模块。
|
||||||
- `api/src/infrastructure-monitoring/`仍是运营端只读聚合门面,只消费固定PromQL和Recording Rules;Exporter安装、端口隔离和阈值归`tools/monitoring/`治理。
|
- `api/src/infrastructure-monitoring/`是运营端监控聚合与固定阈值应用边界:只消费代码白名单 PromQL,并通过版本化 PostgreSQL 单例、promtool 校验和原子规则热加载管理数值阈值;Exporter安装、端口隔离、固定规则模板和权限仍归`tools/monitoring/`治理。
|
||||||
|
|||||||
@@ -2096,3 +2096,6 @@
|
|||||||
- 运营端展示API请求/错误/延迟/事件循环、Gateway Submit/队列/连接、PostgreSQL连接/死锁、Redis内存/连接/淘汰、Nginx连接/请求和MinIO可用性;指标缺失显示“待采集”,不以0伪装。
|
- 运营端展示API请求/错误/延迟/事件循环、Gateway Submit/队列/连接、PostgreSQL连接/死锁、Redis内存/连接/淘汰、Nginx连接/请求和MinIO可用性;指标缺失显示“待采集”,不以0伪装。
|
||||||
- 告警必须使用持续窗口和最低样本量;默认阈值、收敛关系、标签禁止项和性能预算以`docs/prometheus-system-monitoring-design-20260814.md`第9节为准。
|
- 告警必须使用持续窗口和最低样本量;默认阈值、收敛关系、标签禁止项和性能预算以`docs/prometheus-system-monitoring-design-20260814.md`第9节为准。
|
||||||
- 指标不得包含手机号、短信正文、短信/CMPP/任务ID、密钥、完整URL或SQL原文;不得把时序指标高频写入业务PostgreSQL。
|
- 指标不得包含手机号、短信正文、短信/CMPP/任务ID、密钥、完整URL或SQL原文;不得把时序指标高频写入业务PostgreSQL。
|
||||||
|
- 系统监控标题说明必须明确标注数据来自 Prometheus;“服务关键指标”位于趋势/核心服务区域之后、活动告警之前,并提供统一的“告警阈值设置”入口。安全检测页不重复渲染大号标题,说明文字必须明确标注使用 Fail2ban。
|
||||||
|
- 告警阈值仅开放固定指标的警告/严重数值,不允许前端提交 PromQL、标签、文件路径或持续时间;必须满足警告值小于严重值。配置以 PostgreSQL 保存版本、期望值、生效值和应用状态,经 `promtool check rules` 校验、同目录原子替换和 Prometheus 热加载成功后才标记生效,失败保留上一生效规则并展示原因。
|
||||||
|
- 右上角预警中心增加“系统监控告警”,通过独立轻量接口统计 Prometheus 当前 firing/pending 告警及严重数,跳转系统监控活动告警区;任一预警域失败不得清空其他域。
|
||||||
|
|||||||
@@ -237,3 +237,9 @@ type InfrastructureOverview = {
|
|||||||
- API和Gateway请求路径只做内存计数、有界直方图和原子计数,不在业务请求中写PostgreSQL或Redis。
|
- API和Gateway请求路径只做内存计数、有界直方图和原子计数,不在业务请求中写PostgreSQL或Redis。
|
||||||
- PostgreSQL Exporter只使用发行版默认低代价查询,不采集SQL原文或扫描业务大表。
|
- PostgreSQL Exporter只使用发行版默认低代价查询,不采集SQL原文或扫描业务大表。
|
||||||
- 时序仍受30天和8GB双重上限约束;时序增长时先缩短实际保留期,不允许无界占满业务盘。
|
- 时序仍受30天和8GB双重上限约束;时序增长时先缩短实际保留期,不允许无界占满业务盘。
|
||||||
|
|
||||||
|
## 10. 阈值配置与全局预警入口(2026-08-14 增补)
|
||||||
|
|
||||||
|
- 可配置范围固定为主机 CPU/内存/根磁盘、API 5xx/P95/事件循环、Gateway 最旧 pending、PostgreSQL 连接、Redis 内存和 MinIO 容量十组警告/严重数值。PromQL、持续窗口、标签和规则文件路径仍由代码固定,浏览器无权提交。
|
||||||
|
- PostgreSQL 单例记录同时保存期望阈值、生效阈值、配置版本、生效版本和 `applying/effective/failed` 状态。更新使用版本条件认领,防止多个 API 实例并发覆盖;规则先经 promtool 校验,再在同一目录原子替换并调用仅回环开放的 `/-/reload`。失败恢复旧文件并保留旧生效版本。
|
||||||
|
- 安装器把可配置规则从基础规则中剥离,托管文件归 `cmpp-api:prometheus` 且权限为 0640;Prometheus 仍只监听回环。右上角铃铛只轮询轻量活动告警汇总接口,不重复加载趋势或服务指标。
|
||||||
|
|||||||
@@ -4675,3 +4675,16 @@ npm run verify:phase8
|
|||||||
- 集成验收必须在隔离测试节点或网络 namespace 使用文档保留 IP;不得封禁预生产运维出口、Cloudflare 节点或真实客户 IP。未安装真实 Fail2ban/nftables/Nginx 资产时,只能把相关用例标记阻塞,不得用 Mock 通过代替。
|
- 集成验收必须在隔离测试节点或网络 namespace 使用文档保留 IP;不得封禁预生产运维出口、Cloudflare 节点或真实客户 IP。未安装真实 Fail2ban/nftables/Nginx 资产时,只能把相关用例标记阻塞,不得用 Mock 通过代替。
|
||||||
- UI 验收覆盖桌面与窄屏的总览、告警、规则、封禁记录、保护名单、加载、空数据、失败和规则未生效状态;所有数字与操作结果必须能从 API、数据库、agent 与执行器证据交叉验证。
|
- UI 验收覆盖桌面与窄屏的总览、告警、规则、封禁记录、保护名单、加载、空数据、失败和规则未生效状态;所有数字与操作结果必须能从 API、数据库、agent 与执行器证据交叉验证。
|
||||||
- `TC-F2B-OPS-008`:在非默认`APP_DIR`构建安全代理后执行安装器,核对systemd `ExecStart`与Fail2ban `actionban`均指向同一个真实可执行的`$APP_DIR/dist/cmpp-security-agent`;任一文件残留占位符、旧`current/bin`路径或目标不可执行时,安装/发布必须失败。
|
- `TC-F2B-OPS-008`:在非默认`APP_DIR`构建安全代理后执行安装器,核对systemd `ExecStart`与Fail2ban `actionban`均指向同一个真实可执行的`$APP_DIR/dist/cmpp-security-agent`;任一文件残留占位符、旧`current/bin`路径或目标不可执行时,安装/发布必须失败。
|
||||||
|
|
||||||
|
## 系统监控阈值与预警中心增量用例(2026-08-14)
|
||||||
|
|
||||||
|
| 用例ID | 场景 | 步骤 | 预期 |
|
||||||
|
| --- | --- | --- | --- |
|
||||||
|
| TC-INFRA-MON-029 | 模块顺序与技术说明 | 打开系统监控并检查标题和模块顺序 | 说明明确写明 Prometheus;服务关键指标紧邻活动告警上方,活动告警锚点可定位 |
|
||||||
|
| TC-INFRA-MON-030 | 阈值真实读取 | 打开阈值设置并核对 API 与数据库 | 十组固定指标来自 `InfrastructureAlertSetting`,不使用 Mock/localStorage,不允许编辑 PromQL |
|
||||||
|
| TC-INFRA-MON-031 | 阈值边界 | 提交警告≥严重、越界、缺项和未知指标 | API 返回 400,数据库版本和 Prometheus 规则均不改变 |
|
||||||
|
| TC-INFRA-MON-032 | 并发版本 | 两个会话以同一版本先后保存 | 仅第一个原子认领成功,后者返回 409 并提示刷新 |
|
||||||
|
| TC-INFRA-MON-033 | 规则校验与热加载 | 保存合法阈值,检查 promtool、规则文件、reload 与数据库 | 先校验再同目录原子替换,reload 成功后生效版本前进且写操作日志 |
|
||||||
|
| TC-INFRA-MON-034 | 应用失败回滚 | 令 promtool 或 reload 失败后保存 | 状态为 failed、展示原因,旧规则文件与旧生效阈值保留,不误报已生效 |
|
||||||
|
| TC-GLOBAL-ALERT-004 | 系统监控预警入口 | 准备隔离 QA Prometheus firing 告警并点击铃铛 | 第三项显示真实总数/严重数,角标计入三域总和,点击跳转系统监控活动告警区 |
|
||||||
|
| TC-SECURITY-UI-001 | Fail2ban 标识与标题规范 | 打开安全检测与封禁 | 不出现重复大号页面标题,说明明确写明使用 Fail2ban,字号遵循通用菜单标题 |
|
||||||
|
|||||||
@@ -3658,3 +3658,9 @@ git diff --check
|
|||||||
- 测试机静态包只读核对显示当前已部署“服务关键指标”,但尚未包含“重复投递风险”和已确认重投的新表单文案,证明服务指标会话使用定向发布,没有把尚未授权部署的下游重投改动意外带入测试机;两项发布记录保持一致。
|
- 测试机静态包只读核对显示当前已部署“服务关键指标”,但尚未包含“重复投递风险”和已确认重投的新表单文案,证明服务指标会话使用定向发布,没有把尚未授权部署的下游重投改动意外带入测试机;两项发布记录保持一致。
|
||||||
- 合并后统一回归通过:API全量41套/477项、前后端TypeScript、API正式编译、Vite 8.1.5生产构建(2549 modules)、Gateway全量`go test ./... -count=1`与`go vet ./...`、SendChain R10、生产部署和安全代理静态契约、5个Shell脚本语法、真实本地PostgreSQL 88/88 migration状态及`git diff --check`。Vite仅保留既有约2.11MiB单chunk提示;Operations R2仍因本轮开始前已有的`sendQuality`查询哈希漂移失败,与本次合并文件无交集。
|
- 合并后统一回归通过:API全量41套/477项、前后端TypeScript、API正式编译、Vite 8.1.5生产构建(2549 modules)、Gateway全量`go test ./... -count=1`与`go vet ./...`、SendChain R10、生产部署和安全代理静态契约、5个Shell脚本语法、真实本地PostgreSQL 88/88 migration状态及`git diff --check`。Vite仅保留既有约2.11MiB单chunk提示;Operations R2仍因本轮开始前已有的`sendQuality`查询哈希漂移失败,与本次合并文件无交集。
|
||||||
- 本次合并没有发送、补发或重投短信,没有创建后台重投任务,没有修改通道账号、密码、启停状态、企业余额或客户连接;提交范围继续排除`api/tsconfig.build.tsbuildinfo`、`tsconfig.tsbuildinfo`、`outputs/`和空文件`=`。
|
- 本次合并没有发送、补发或重投短信,没有创建后台重投任务,没有修改通道账号、密码、启停状态、企业余额或客户连接;提交范围继续排除`api/tsconfig.build.tsbuildinfo`、`tsconfig.tsbuildinfo`、`outputs/`和空文件`=`。
|
||||||
|
|
||||||
|
# 2026-08-14 系统监控阈值与预警入口优化(进行中)
|
||||||
|
|
||||||
|
- 系统监控将“服务关键指标”移至活动告警正上方,标题说明明确使用 Prometheus;新增十组固定指标警告/严重阈值设置。配置真实写入 PostgreSQL,使用版本条件并发认领,promtool 校验、原子替换和回环 reload 成功后才推进生效版本,失败保留旧生效值。
|
||||||
|
- 右上角预警中心新增“系统监控告警”及数量/严重数,读取独立 Prometheus 活动告警汇总并跳转活动告警锚点;继续使用 `Promise.allSettled` 隔离签名、安全与监控域故障。安全检测页移除重复大号标题,说明明确使用 Fail2ban。
|
||||||
|
- 新增 migration `20260814173000_add_infrastructure_alert_settings`。本机 Prisma Client 生成、前后端 TypeScript 与 API 正式编译已通过;测试机恢复资产、migration、Prometheus规则校验、真实告警数据、浏览器验收和提交/部署结果待本轮后续补记。
|
||||||
|
|||||||
@@ -1,7 +1,10 @@
|
|||||||
import { request, withQuery } from '../core/httpClient';
|
import { request, withQuery } from '../core/httpClient';
|
||||||
import type { InfrastructureMonitoringOverview, InfrastructureMonitoringRange } from '../types';
|
import type { InfrastructureAlertSettings, InfrastructureAlertThresholds, InfrastructureMonitoringOverview, InfrastructureMonitoringRange } from '../types';
|
||||||
|
|
||||||
export const adminInfrastructureMonitoringApi = {
|
export const adminInfrastructureMonitoringApi = {
|
||||||
getInfrastructureMonitoringOverview: (range: InfrastructureMonitoringRange) =>
|
getInfrastructureMonitoringOverview: (range: InfrastructureMonitoringRange) =>
|
||||||
request<InfrastructureMonitoringOverview>(withQuery('/admin/infrastructure-monitoring/overview', { range })),
|
request<InfrastructureMonitoringOverview>(withQuery('/admin/infrastructure-monitoring/overview', { range })),
|
||||||
|
getInfrastructureMonitoringNotificationSummary: () => request<{ count: number; criticalCount: number }>('/admin/infrastructure-monitoring/notification-summary'),
|
||||||
|
getInfrastructureAlertThresholds: () => request<InfrastructureAlertSettings>('/admin/infrastructure-monitoring/alert-thresholds'),
|
||||||
|
updateInfrastructureAlertThresholds: (body: { configVersion: number; thresholds: InfrastructureAlertThresholds }) => request<InfrastructureAlertSettings>('/admin/infrastructure-monitoring/alert-thresholds', { method: 'PUT', body: JSON.stringify(body) }),
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -76,3 +76,16 @@ export type InfrastructureMonitoringOverview = {
|
|||||||
serviceMetrics: InfrastructureServiceMetricGroup[];
|
serviceMetrics: InfrastructureServiceMetricGroup[];
|
||||||
alerts: InfrastructureAlert[];
|
alerts: InfrastructureAlert[];
|
||||||
};
|
};
|
||||||
|
|
||||||
|
export type InfrastructureAlertThresholds = Record<string, { warning: number; critical: number }>;
|
||||||
|
|
||||||
|
export type InfrastructureAlertSettings = {
|
||||||
|
configVersion: number;
|
||||||
|
effectiveVersion: number;
|
||||||
|
applyStatus: 'effective' | 'applying' | 'failed';
|
||||||
|
lastError: string | null;
|
||||||
|
appliedAt: string | null;
|
||||||
|
thresholds: InfrastructureAlertThresholds;
|
||||||
|
effectiveThresholds: InfrastructureAlertThresholds;
|
||||||
|
definitions: Array<{ key: string; label: string; unit: string; min: number; max: number; step: number }>;
|
||||||
|
};
|
||||||
|
|||||||
@@ -1 +1,2 @@
|
|||||||
.admin-security-page{gap:20px}.security-heading{align-items:flex-end}.security-heading h1{margin:10px 0 4px;font-size:26px}.security-heading p{margin:0;color:#64748b}.security-kpis{display:grid;grid-template-columns:repeat(4,minmax(0,1fr));gap:14px}.security-kpi{display:grid;grid-template-columns:42px 1fr;grid-template-rows:auto auto;gap:3px 12px;padding:18px}.security-kpi>div{grid-row:1/3;width:42px;height:42px;border-radius:12px;display:grid;place-items:center;background:#e8f0ff;color:#2563eb}.security-kpi>div svg{width:20px}.security-kpi span{font-size:13px;color:#64748b}.security-kpi strong{font-size:24px;line-height:1.1}.security-kpi.is-danger>div{background:#fef2f2;color:#dc2626}.security-overview-grid{display:grid;grid-template-columns:minmax(0,1fr) minmax(360px,.8fr);gap:16px}.security-chart,.security-latest,.security-table-card{padding:18px}.security-chart header,.security-latest header{display:flex;justify-content:space-between}.security-chart header span,.security-latest header span{font-size:12px;color:#94a3b8}.security-latest-row{display:grid;grid-template-columns:10px 1fr auto;gap:10px;align-items:center;padding:13px 0;border-bottom:1px solid #eef2f7}.security-latest-row div{display:grid;gap:3px}.security-latest-row small,.security-latest-row time{color:#64748b;font-size:12px}.severity-dot{width:8px;height:8px;border-radius:99px;background:#3b82f6}.severity-dot.is-high{background:#f59e0b}.severity-dot.is-critical{background:#ef4444}.security-cell{display:grid;gap:3px}.security-cell span{font-size:11px;color:#94a3b8}.security-actions{display:flex;gap:6px}.security-note,.security-section-toolbar{display:flex;align-items:center;gap:9px;margin-bottom:16px;padding:12px 14px;border-radius:10px;background:#f8fafc;color:#475569;font-size:13px}.security-section-toolbar{justify-content:space-between}.security-error{display:flex;gap:8px;align-items:center;padding:12px 14px;border:1px solid #fecaca;border-radius:10px;background:#fef2f2;color:#b91c1c}.security-empty{height:265px;display:grid;place-items:center;color:#94a3b8}.security-dialog{display:grid;gap:16px}.security-target{display:grid;gap:4px;padding:14px;border-radius:10px;background:#f8fafc}.security-target span,.security-target small{color:#64748b;font-size:12px}.security-target strong{font-family:ui-monospace,monospace;font-size:18px}.security-form-grid{display:grid;grid-template-columns:1fr 1fr;gap:16px}.security-form-grid .security-error{grid-column:1/-1}.admin-security-page code{font-size:12px;color:#334155}@media(max-width:1100px){.security-kpis{grid-template-columns:repeat(2,1fr)}.security-overview-grid{grid-template-columns:1fr}}@media(max-width:640px){.security-kpis{grid-template-columns:1fr}.security-form-grid{grid-template-columns:1fr}}
|
.admin-security-page{gap:20px}.security-heading{align-items:flex-end}.security-heading h1{margin:10px 0 4px;font-size:26px}.security-heading p{margin:0;color:#64748b}.security-kpis{display:grid;grid-template-columns:repeat(4,minmax(0,1fr));gap:14px}.security-kpi{display:grid;grid-template-columns:42px 1fr;grid-template-rows:auto auto;gap:3px 12px;padding:18px}.security-kpi>div{grid-row:1/3;width:42px;height:42px;border-radius:12px;display:grid;place-items:center;background:#e8f0ff;color:#2563eb}.security-kpi>div svg{width:20px}.security-kpi span{font-size:13px;color:#64748b}.security-kpi strong{font-size:24px;line-height:1.1}.security-kpi.is-danger>div{background:#fef2f2;color:#dc2626}.security-overview-grid{display:grid;grid-template-columns:minmax(0,1fr) minmax(360px,.8fr);gap:16px}.security-chart,.security-latest,.security-table-card{padding:18px}.security-chart header,.security-latest header{display:flex;justify-content:space-between}.security-chart header span,.security-latest header span{font-size:12px;color:#94a3b8}.security-latest-row{display:grid;grid-template-columns:10px 1fr auto;gap:10px;align-items:center;padding:13px 0;border-bottom:1px solid #eef2f7}.security-latest-row div{display:grid;gap:3px}.security-latest-row small,.security-latest-row time{color:#64748b;font-size:12px}.severity-dot{width:8px;height:8px;border-radius:99px;background:#3b82f6}.severity-dot.is-high{background:#f59e0b}.severity-dot.is-critical{background:#ef4444}.security-cell{display:grid;gap:3px}.security-cell span{font-size:11px;color:#94a3b8}.security-actions{display:flex;gap:6px}.security-note,.security-section-toolbar{display:flex;align-items:center;gap:9px;margin-bottom:16px;padding:12px 14px;border-radius:10px;background:#f8fafc;color:#475569;font-size:13px}.security-section-toolbar{justify-content:space-between}.security-error{display:flex;gap:8px;align-items:center;padding:12px 14px;border:1px solid #fecaca;border-radius:10px;background:#fef2f2;color:#b91c1c}.security-empty{height:265px;display:grid;place-items:center;color:#94a3b8}.security-dialog{display:grid;gap:16px}.security-target{display:grid;gap:4px;padding:14px;border-radius:10px;background:#f8fafc}.security-target span,.security-target small{color:#64748b;font-size:12px}.security-target strong{font-family:ui-monospace,monospace;font-size:18px}.security-form-grid{display:grid;grid-template-columns:1fr 1fr;gap:16px}.security-form-grid .security-error{grid-column:1/-1}.admin-security-page code{font-size:12px;color:#334155}@media(max-width:1100px){.security-kpis{grid-template-columns:repeat(2,1fr)}.security-overview-grid{grid-template-columns:1fr}}@media(max-width:640px){.security-kpis{grid-template-columns:1fr}.security-form-grid{grid-template-columns:1fr}}
|
||||||
|
.security-heading p{margin:10px 0 0}
|
||||||
|
|||||||
@@ -76,7 +76,7 @@ export function AdminSecurityDetectionPage() {
|
|||||||
{ value: 'protected', label: '保护名单', content: <section className="surface security-table-card"><div className="security-section-toolbar"><span>受保护的运维出口、内网和可信代理永远不能从面板封禁。</span><Button onClick={() => setShowNetwork(true)}>新增保护网段</Button></div><Table columns={networkColumns} data={networks} emptyText="暂无保护网段" rowKey="id" /></section> },
|
{ value: 'protected', label: '保护名单', content: <section className="surface security-table-card"><div className="security-section-toolbar"><span>受保护的运维出口、内网和可信代理永远不能从面板封禁。</span><Button onClick={() => setShowNetwork(true)}>新增保护网段</Button></div><Table columns={networkColumns} data={networks} emptyText="暂无保护网段" rowKey="id" /></section> },
|
||||||
];
|
];
|
||||||
|
|
||||||
return <section className="page-stack admin-security-page"><div className="page-heading security-heading"><div><Breadcrumb items={['安全控制', '安全检测与封禁']} /><h1>安全检测与封禁</h1><p>Fail2ban 与业务事件只负责检测,由运营人员复核后人工处置</p></div><Button disabled={loading} icon={<RefreshCw className={loading ? 'is-spinning' : ''} size={16} />} onClick={() => void load()} variant="ghost">刷新</Button></div>
|
return <section className="page-stack admin-security-page"><div className="page-heading security-heading"><div><Breadcrumb items={['安全控制', '安全检测与封禁']} /><p>使用 Fail2ban 与平台业务安全事件进行检测,由运营人员复核后人工处置</p></div><Button disabled={loading} icon={<RefreshCw className={loading ? 'is-spinning' : ''} size={16} />} onClick={() => void load()} variant="ghost">刷新</Button></div>
|
||||||
{error ? <div className="security-error" role="alert"><AlertTriangle size={18} />{error}</div> : null}
|
{error ? <div className="security-error" role="alert"><AlertTriangle size={18} />{error}</div> : null}
|
||||||
<div className="security-kpis"><Kpi icon={<BellRing />} label="24小时检测事件" value={overview?.totalEvents ?? 0} /><Kpi icon={<AlertTriangle />} label="待处置告警" value={overview?.activeAlerts ?? 0} danger={Boolean(overview?.criticalAlerts)} /><Kpi icon={<Ban />} label="生效封禁" value={overview?.activeBlocks ?? 0} /><Kpi icon={overview?.health.agent === 'healthy' ? <ShieldCheck /> : <ShieldOff />} label="安全代理" value={overview?.health.agent === 'healthy' ? '在线' : '不可用'} danger={overview?.health.agent !== 'healthy'} /></div>
|
<div className="security-kpis"><Kpi icon={<BellRing />} label="24小时检测事件" value={overview?.totalEvents ?? 0} /><Kpi icon={<AlertTriangle />} label="待处置告警" value={overview?.activeAlerts ?? 0} danger={Boolean(overview?.criticalAlerts)} /><Kpi icon={<Ban />} label="生效封禁" value={overview?.activeBlocks ?? 0} /><Kpi icon={overview?.health.agent === 'healthy' ? <ShieldCheck /> : <ShieldOff />} label="安全代理" value={overview?.health.agent === 'healthy' ? '在线' : '不可用'} danger={overview?.health.agent !== 'healthy'} /></div>
|
||||||
<Tabs items={tabs} />
|
<Tabs items={tabs} />
|
||||||
|
|||||||
@@ -321,3 +321,16 @@
|
|||||||
@media (prefers-reduced-motion: reduce) {
|
@media (prefers-reduced-motion: reduce) {
|
||||||
.is-spinning { animation: none; }
|
.is-spinning { animation: none; }
|
||||||
}
|
}
|
||||||
|
|
||||||
|
.system-monitoring-service-actions { align-items: center; display: flex; gap: 12px; }
|
||||||
|
.system-monitoring-threshold-dialog { display: grid; gap: 12px; max-height: 62vh; overflow: auto; padding-right: 4px; }
|
||||||
|
.system-monitoring-threshold-note { align-items: flex-start; background: #eff6ff; border-radius: 10px; color: #475569; display: flex; font-size: 13px; gap: 9px; padding: 12px 14px; }
|
||||||
|
.system-monitoring-threshold-row { align-items: end; border-bottom: 1px solid #eef2f7; display: grid; gap: 14px; grid-template-columns: minmax(180px, 1fr) 150px 150px; padding: 12px 0; }
|
||||||
|
.system-monitoring-threshold-row > div:first-child { align-self: center; display: grid; gap: 4px; }
|
||||||
|
.system-monitoring-threshold-row small { color: #64748b; }
|
||||||
|
|
||||||
|
@media (max-width: 760px) {
|
||||||
|
.system-monitoring-service-actions { align-items: flex-start; flex-direction: column; }
|
||||||
|
.system-monitoring-threshold-row { grid-template-columns: 1fr 1fr; }
|
||||||
|
.system-monitoring-threshold-row > div:first-child { grid-column: 1 / -1; }
|
||||||
|
}
|
||||||
|
|||||||
@@ -12,16 +12,19 @@ import {
|
|||||||
Network,
|
Network,
|
||||||
RefreshCw,
|
RefreshCw,
|
||||||
Server,
|
Server,
|
||||||
|
Settings2,
|
||||||
ShieldAlert,
|
ShieldAlert,
|
||||||
} from 'lucide-react';
|
} from 'lucide-react';
|
||||||
import {
|
import {
|
||||||
adminApi,
|
adminApi,
|
||||||
type InfrastructureAlert,
|
type InfrastructureAlert,
|
||||||
|
type InfrastructureAlertSettings,
|
||||||
|
type InfrastructureAlertThresholds,
|
||||||
type InfrastructureMetricPoint,
|
type InfrastructureMetricPoint,
|
||||||
type InfrastructureMonitoringOverview,
|
type InfrastructureMonitoringOverview,
|
||||||
type InfrastructureMonitoringRange,
|
type InfrastructureMonitoringRange,
|
||||||
} from '@/api/adminApi';
|
} from '@/api/adminApi';
|
||||||
import { Breadcrumb, Button, Chart, Table, Tag, type TableColumn } from '@/components/ui';
|
import { Breadcrumb, Button, Chart, Input, Modal, Table, Tag, type TableColumn } from '@/components/ui';
|
||||||
import './AdminSystemMonitoringPage.css';
|
import './AdminSystemMonitoringPage.css';
|
||||||
|
|
||||||
const RANGE_OPTIONS: Array<{ value: InfrastructureMonitoringRange; label: string }> = [
|
const RANGE_OPTIONS: Array<{ value: InfrastructureMonitoringRange; label: string }> = [
|
||||||
@@ -166,6 +169,11 @@ export function AdminSystemMonitoringPage() {
|
|||||||
const [overview, setOverview] = useState<InfrastructureMonitoringOverview | null>(null);
|
const [overview, setOverview] = useState<InfrastructureMonitoringOverview | null>(null);
|
||||||
const [loading, setLoading] = useState(true);
|
const [loading, setLoading] = useState(true);
|
||||||
const [error, setError] = useState('');
|
const [error, setError] = useState('');
|
||||||
|
const [settings, setSettings] = useState<InfrastructureAlertSettings | null>(null);
|
||||||
|
const [draftThresholds, setDraftThresholds] = useState<InfrastructureAlertThresholds>({});
|
||||||
|
const [showSettings, setShowSettings] = useState(false);
|
||||||
|
const [settingsError, setSettingsError] = useState('');
|
||||||
|
const [savingSettings, setSavingSettings] = useState(false);
|
||||||
const requestSequence = useRef(0);
|
const requestSequence = useRef(0);
|
||||||
const pendingRequests = useRef(0);
|
const pendingRequests = useRef(0);
|
||||||
|
|
||||||
@@ -189,8 +197,38 @@ export function AdminSystemMonitoringPage() {
|
|||||||
}
|
}
|
||||||
}, [range]);
|
}, [range]);
|
||||||
|
|
||||||
|
const loadSettings = useCallback(async () => {
|
||||||
|
try {
|
||||||
|
const result = await adminApi.getInfrastructureAlertThresholds();
|
||||||
|
setSettings(result);
|
||||||
|
setDraftThresholds(result.thresholds);
|
||||||
|
setSettingsError('');
|
||||||
|
} catch (reason) {
|
||||||
|
setSettingsError(reason instanceof Error ? reason.message : '告警阈值加载失败');
|
||||||
|
}
|
||||||
|
}, []);
|
||||||
|
|
||||||
|
const saveSettings = useCallback(async () => {
|
||||||
|
if (!settings) return;
|
||||||
|
setSavingSettings(true);
|
||||||
|
setSettingsError('');
|
||||||
|
try {
|
||||||
|
const result = await adminApi.updateInfrastructureAlertThresholds({ configVersion: settings.configVersion, thresholds: draftThresholds });
|
||||||
|
setSettings(result);
|
||||||
|
setDraftThresholds(result.thresholds);
|
||||||
|
setShowSettings(false);
|
||||||
|
window.dispatchEvent(new Event('cmpp-infrastructure-alert-count-refresh'));
|
||||||
|
await loadData(true);
|
||||||
|
} catch (reason) {
|
||||||
|
setSettingsError(reason instanceof Error ? reason.message : '告警阈值保存失败');
|
||||||
|
} finally {
|
||||||
|
setSavingSettings(false);
|
||||||
|
}
|
||||||
|
}, [draftThresholds, loadData, settings]);
|
||||||
|
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
void loadData(true);
|
void loadData(true);
|
||||||
|
void loadSettings();
|
||||||
const intervalId = window.setInterval(() => {
|
const intervalId = window.setInterval(() => {
|
||||||
if (document.visibilityState === 'visible') void loadData();
|
if (document.visibilityState === 'visible') void loadData();
|
||||||
}, 30_000);
|
}, 30_000);
|
||||||
@@ -203,7 +241,7 @@ export function AdminSystemMonitoringPage() {
|
|||||||
window.clearInterval(intervalId);
|
window.clearInterval(intervalId);
|
||||||
document.removeEventListener('visibilitychange', handleVisibility);
|
document.removeEventListener('visibilitychange', handleVisibility);
|
||||||
};
|
};
|
||||||
}, [loadData]);
|
}, [loadData, loadSettings]);
|
||||||
|
|
||||||
const status = STATUS_COPY[overview?.summary.overallStatus ?? 'unknown'];
|
const status = STATUS_COPY[overview?.summary.overallStatus ?? 'unknown'];
|
||||||
const cpuOption = useMemo(() => makeTrendOption({
|
const cpuOption = useMemo(() => makeTrendOption({
|
||||||
@@ -232,7 +270,7 @@ export function AdminSystemMonitoringPage() {
|
|||||||
<div>
|
<div>
|
||||||
<Breadcrumb items={['系统管理', '系统监控']} />
|
<Breadcrumb items={['系统管理', '系统监控']} />
|
||||||
<div className="system-monitoring-title-row">
|
<div className="system-monitoring-title-row">
|
||||||
<p>服务器资源、核心服务与活动告警</p>
|
<p>服务器资源、核心服务与活动告警,数据由 Prometheus 采集与计算</p>
|
||||||
<Tag tone={status.tone}>{status.label}</Tag>
|
<Tag tone={status.tone}>{status.label}</Tag>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
@@ -283,29 +321,6 @@ export function AdminSystemMonitoringPage() {
|
|||||||
<article className="surface system-monitoring-metric"><div className="system-monitoring-metric__icon is-green"><Network size={19} /></div><div><span>网络吞吐</span><strong>{formatRate(totalNetworkRate(metrics?.networkReceiveBytesPerSecond, metrics?.networkTransmitBytesPerSecond))}</strong><small>接收 {formatRate(metrics?.networkReceiveBytesPerSecond ?? null)} · 发送 {formatRate(metrics?.networkTransmitBytesPerSecond ?? null)}</small></div></article>
|
<article className="surface system-monitoring-metric"><div className="system-monitoring-metric__icon is-green"><Network size={19} /></div><div><span>网络吞吐</span><strong>{formatRate(totalNetworkRate(metrics?.networkReceiveBytesPerSecond, metrics?.networkTransmitBytesPerSecond))}</strong><small>接收 {formatRate(metrics?.networkReceiveBytesPerSecond ?? null)} · 发送 {formatRate(metrics?.networkTransmitBytesPerSecond ?? null)}</small></div></article>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<section className="surface system-monitoring-service-metrics">
|
|
||||||
<header>
|
|
||||||
<div><Database size={18} /><strong>服务关键指标</strong></div>
|
|
||||||
<span>固定低基数聚合,不含手机号、短信ID或SQL文本</span>
|
|
||||||
</header>
|
|
||||||
<div className="system-monitoring-service-metric-grid">
|
|
||||||
{(overview?.serviceMetrics ?? []).map((group) => (
|
|
||||||
<article key={group.key}>
|
|
||||||
<div className="system-monitoring-service-metric-title">
|
|
||||||
<strong>{group.name}</strong>
|
|
||||||
<Tag tone={group.available ? 'success' : 'neutral'}>{group.available ? '已采集' : '待采集'}</Tag>
|
|
||||||
</div>
|
|
||||||
{group.metrics.length ? group.metrics.map((metric) => (
|
|
||||||
<div className="system-monitoring-service-metric-row" key={metric.key}>
|
|
||||||
<span>{metric.label}</span>
|
|
||||||
<strong>{formatServiceMetric(metric.value, metric.unit)}</strong>
|
|
||||||
</div>
|
|
||||||
)) : <div className="system-monitoring-service-metric-empty">已监控服务可用性,待原生容量指标接入</div>}
|
|
||||||
</article>
|
|
||||||
))}
|
|
||||||
</div>
|
|
||||||
</section>
|
|
||||||
|
|
||||||
<div className="system-monitoring-main-grid">
|
<div className="system-monitoring-main-grid">
|
||||||
<div className="system-monitoring-chart-stack">
|
<div className="system-monitoring-chart-stack">
|
||||||
<article className="surface system-monitoring-chart-card"><header><div><Cpu size={17} /><strong>CPU 趋势</strong></div><span>{formatPercent(metrics?.cpuUsagePercent ?? null)}</span></header>{overview?.trends.cpuUsagePercent.length ? <Chart height={230} option={cpuOption} /> : <EmptyChart />}</article>
|
<article className="surface system-monitoring-chart-card"><header><div><Cpu size={17} /><strong>CPU 趋势</strong></div><span>{formatPercent(metrics?.cpuUsagePercent ?? null)}</span></header>{overview?.trends.cpuUsagePercent.length ? <Chart height={230} option={cpuOption} /> : <EmptyChart />}</article>
|
||||||
@@ -332,10 +347,40 @@ export function AdminSystemMonitoringPage() {
|
|||||||
</aside>
|
</aside>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<section className="surface system-monitoring-alerts">
|
<section className="surface system-monitoring-service-metrics">
|
||||||
|
<header>
|
||||||
|
<div><Database size={18} /><strong>服务关键指标</strong></div>
|
||||||
|
<div className="system-monitoring-service-actions"><span>固定低基数聚合,不含手机号、短信ID或SQL文本</span><Button icon={<Settings2 size={15} />} onClick={() => setShowSettings(true)} variant="ghost">告警阈值设置</Button></div>
|
||||||
|
</header>
|
||||||
|
<div className="system-monitoring-service-metric-grid">
|
||||||
|
{(overview?.serviceMetrics ?? []).map((group) => (
|
||||||
|
<article key={group.key}>
|
||||||
|
<div className="system-monitoring-service-metric-title"><strong>{group.name}</strong><Tag tone={group.available ? 'success' : 'neutral'}>{group.available ? '已采集' : '待采集'}</Tag></div>
|
||||||
|
{group.metrics.length ? group.metrics.map((metric) => <div className="system-monitoring-service-metric-row" key={metric.key}><span>{metric.label}</span><strong>{formatServiceMetric(metric.value, metric.unit)}</strong></div>) : <div className="system-monitoring-service-metric-empty">已监控服务可用性,待原生容量指标接入</div>}
|
||||||
|
</article>
|
||||||
|
))}
|
||||||
|
</div>
|
||||||
|
</section>
|
||||||
|
|
||||||
|
<section className="surface system-monitoring-alerts" id="active-alerts">
|
||||||
<header><div><AlertTriangle size={18} /><strong>活动告警</strong><Tag tone={overview?.summary.activeAlerts ? 'warning' : 'success'}>{overview?.summary.activeAlerts ?? 0}</Tag></div><span><Clock3 size={14} /> 刷新于 {formatTime(overview?.collectedAt ?? null)}</span></header>
|
<header><div><AlertTriangle size={18} /><strong>活动告警</strong><Tag tone={overview?.summary.activeAlerts ? 'warning' : 'success'}>{overview?.summary.activeAlerts ?? 0}</Tag></div><span><Clock3 size={14} /> 刷新于 {formatTime(overview?.collectedAt ?? null)}</span></header>
|
||||||
<Table columns={alertColumns} data={overview?.alerts ?? []} emptyText={overview?.available ? '当前没有活动告警' : '监控不可用,无法读取活动告警'} pagination={false} rowKey="fingerprint" />
|
<Table columns={alertColumns} data={overview?.alerts ?? []} emptyText={overview?.available ? '当前没有活动告警' : '监控不可用,无法读取活动告警'} pagination={false} rowKey="fingerprint" />
|
||||||
</section>
|
</section>
|
||||||
|
|
||||||
|
<Modal footer={<><Button onClick={() => setShowSettings(false)} variant="ghost">取消</Button><Button disabled={savingSettings || !settings} onClick={() => void saveSettings()}>{savingSettings ? '验证并应用中' : '保存并应用'}</Button></>} onClose={() => setShowSettings(false)} open={showSettings} title="Prometheus 告警阈值设置">
|
||||||
|
<div className="system-monitoring-threshold-dialog">
|
||||||
|
<div className="system-monitoring-threshold-note"><ShieldAlert size={17} /><span>仅允许修改固定指标的数值阈值。保存时先由 promtool 校验,再原子替换规则并热加载 Prometheus。</span></div>
|
||||||
|
{settings?.definitions.map((definition) => (
|
||||||
|
<div className="system-monitoring-threshold-row" key={definition.key}>
|
||||||
|
<div><strong>{definition.label}</strong><small>单位:{definition.unit}</small></div>
|
||||||
|
<Input label="警告阈值" max={definition.max} min={definition.min} onChange={(event) => setDraftThresholds((current) => ({ ...current, [definition.key]: { ...current[definition.key], warning: Number(event.target.value) } }))} step={definition.step} type="number" value={draftThresholds[definition.key]?.warning ?? ''} />
|
||||||
|
<Input label="严重阈值" max={definition.max} min={definition.min} onChange={(event) => setDraftThresholds((current) => ({ ...current, [definition.key]: { ...current[definition.key], critical: Number(event.target.value) } }))} step={definition.step} type="number" value={draftThresholds[definition.key]?.critical ?? ''} />
|
||||||
|
</div>
|
||||||
|
))}
|
||||||
|
{settings?.applyStatus === 'failed' ? <div className="system-monitoring-unavailable"><AlertTriangle size={18} /><div><strong>上次应用失败</strong><span>{settings.lastError}</span></div></div> : null}
|
||||||
|
{settingsError ? <div className="system-monitoring-unavailable"><AlertTriangle size={18} /><div><strong>阈值配置不可用</strong><span>{settingsError}</span></div></div> : null}
|
||||||
|
</div>
|
||||||
|
</Modal>
|
||||||
</section>
|
</section>
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -49,6 +49,7 @@ function AdminAuthenticatedLayout({ session }: { session: LoginSession }) {
|
|||||||
const [pendingAudits, setPendingAudits] = useState(EMPTY_PENDING_AUDITS);
|
const [pendingAudits, setPendingAudits] = useState(EMPTY_PENDING_AUDITS);
|
||||||
const [retirementUnreadCount, setRetirementUnreadCount] = useState(0);
|
const [retirementUnreadCount, setRetirementUnreadCount] = useState(0);
|
||||||
const [securityAlertSummary, setSecurityAlertSummary] = useState({ count: 0, criticalCount: 0 });
|
const [securityAlertSummary, setSecurityAlertSummary] = useState({ count: 0, criticalCount: 0 });
|
||||||
|
const [infrastructureAlertSummary, setInfrastructureAlertSummary] = useState({ count: 0, criticalCount: 0 });
|
||||||
const [sessionLocked, setSessionLocked] = useState(Boolean(session.locked));
|
const [sessionLocked, setSessionLocked] = useState(Boolean(session.locked));
|
||||||
const loadPendingAuditCount = useCallback(() => {
|
const loadPendingAuditCount = useCallback(() => {
|
||||||
const currentSession = readSession('admin');
|
const currentSession = readSession('admin');
|
||||||
@@ -57,16 +58,18 @@ function AdminAuthenticatedLayout({ session }: { session: LoginSession }) {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
// This runs globally and on a timer, so it must not fan out through the full dashboard aggregation.
|
// This runs globally and on a timer, so it must not fan out through the full dashboard aggregation.
|
||||||
Promise.allSettled([adminApi.getPendingAudits(), adminApi.getSignatureRetirementUnreadCount(), adminApi.getSecurityNotificationSummary()])
|
Promise.allSettled([adminApi.getPendingAudits(), adminApi.getSignatureRetirementUnreadCount(), adminApi.getSecurityNotificationSummary(), adminApi.getInfrastructureMonitoringNotificationSummary()])
|
||||||
.then(([audits, retirement, security]) => {
|
.then(([audits, retirement, security, infrastructure]) => {
|
||||||
setPendingAudits(audits.status === 'fulfilled' ? audits.value : EMPTY_PENDING_AUDITS);
|
setPendingAudits(audits.status === 'fulfilled' ? audits.value : EMPTY_PENDING_AUDITS);
|
||||||
setRetirementUnreadCount(retirement.status === 'fulfilled' ? retirement.value.count : 0);
|
setRetirementUnreadCount(retirement.status === 'fulfilled' ? retirement.value.count : 0);
|
||||||
setSecurityAlertSummary(security.status === 'fulfilled' ? security.value : { count: 0, criticalCount: 0 });
|
setSecurityAlertSummary(security.status === 'fulfilled' ? security.value : { count: 0, criticalCount: 0 });
|
||||||
|
setInfrastructureAlertSummary(infrastructure.status === 'fulfilled' ? infrastructure.value : { count: 0, criticalCount: 0 });
|
||||||
})
|
})
|
||||||
.catch(() => {
|
.catch(() => {
|
||||||
setPendingAudits(EMPTY_PENDING_AUDITS);
|
setPendingAudits(EMPTY_PENDING_AUDITS);
|
||||||
setRetirementUnreadCount(0);
|
setRetirementUnreadCount(0);
|
||||||
setSecurityAlertSummary({ count: 0, criticalCount: 0 });
|
setSecurityAlertSummary({ count: 0, criticalCount: 0 });
|
||||||
|
setInfrastructureAlertSummary({ count: 0, criticalCount: 0 });
|
||||||
});
|
});
|
||||||
}, []);
|
}, []);
|
||||||
|
|
||||||
@@ -82,12 +85,14 @@ function AdminAuthenticatedLayout({ session }: { session: LoginSession }) {
|
|||||||
window.addEventListener('cmpp-audit-count-refresh', onAuditRefresh);
|
window.addEventListener('cmpp-audit-count-refresh', onAuditRefresh);
|
||||||
window.addEventListener('cmpp-retirement-count-refresh', onAuditRefresh);
|
window.addEventListener('cmpp-retirement-count-refresh', onAuditRefresh);
|
||||||
window.addEventListener('cmpp-security-alert-count-refresh', onAuditRefresh);
|
window.addEventListener('cmpp-security-alert-count-refresh', onAuditRefresh);
|
||||||
|
window.addEventListener('cmpp-infrastructure-alert-count-refresh', onAuditRefresh);
|
||||||
return () => {
|
return () => {
|
||||||
window.clearInterval(timer);
|
window.clearInterval(timer);
|
||||||
window.removeEventListener('focus', onFocus);
|
window.removeEventListener('focus', onFocus);
|
||||||
window.removeEventListener('cmpp-audit-count-refresh', onAuditRefresh);
|
window.removeEventListener('cmpp-audit-count-refresh', onAuditRefresh);
|
||||||
window.removeEventListener('cmpp-retirement-count-refresh', onAuditRefresh);
|
window.removeEventListener('cmpp-retirement-count-refresh', onAuditRefresh);
|
||||||
window.removeEventListener('cmpp-security-alert-count-refresh', onAuditRefresh);
|
window.removeEventListener('cmpp-security-alert-count-refresh', onAuditRefresh);
|
||||||
|
window.removeEventListener('cmpp-infrastructure-alert-count-refresh', onAuditRefresh);
|
||||||
};
|
};
|
||||||
}, [loadPendingAuditCount, session.portal, sessionLocked]);
|
}, [loadPendingAuditCount, session.portal, sessionLocked]);
|
||||||
|
|
||||||
@@ -104,6 +109,7 @@ function AdminAuthenticatedLayout({ session }: { session: LoginSession }) {
|
|||||||
alertNotifications={[
|
alertNotifications={[
|
||||||
{ label: '签名清退预警', count: retirementUnreadCount, description: '今日未读且未抑制', to: '/admin/signature-retirement' },
|
{ label: '签名清退预警', count: retirementUnreadCount, description: '今日未读且未抑制', to: '/admin/signature-retirement' },
|
||||||
{ label: '安全检测与封禁', count: securityAlertSummary.count, description: securityAlertSummary.criticalCount > 0 ? `${securityAlertSummary.criticalCount} 条严重告警待处置` : '待处置安全告警', to: '/admin/security-detection' },
|
{ label: '安全检测与封禁', count: securityAlertSummary.count, description: securityAlertSummary.criticalCount > 0 ? `${securityAlertSummary.criticalCount} 条严重告警待处置` : '待处置安全告警', to: '/admin/security-detection' },
|
||||||
|
{ label: '系统监控告警', count: infrastructureAlertSummary.count, description: infrastructureAlertSummary.criticalCount > 0 ? `${infrastructureAlertSummary.criticalCount} 条 Prometheus 严重告警` : 'Prometheus 活动告警', to: '/admin/system-monitoring#active-alerts' },
|
||||||
]}
|
]}
|
||||||
auditNotifications={[
|
auditNotifications={[
|
||||||
{ label: '企业认证待审', count: pendingAudits.enterpriseCertifications, to: '/admin/enterprise-audit' },
|
{ label: '企业认证待审', count: pendingAudits.enterpriseCertifications, to: '/admin/enterprise-audit' },
|
||||||
|
|||||||
@@ -0,0 +1,103 @@
|
|||||||
|
groups:
|
||||||
|
- name: cmpp-managed-thresholds
|
||||||
|
rules:
|
||||||
|
- alert: HostCpuUsageWarning
|
||||||
|
expr: (100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 80) and (100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) <= 90)
|
||||||
|
for: 10m
|
||||||
|
labels: { severity: warning, service: host }
|
||||||
|
annotations: { summary: "主机 CPU 使用率达到警告阈值", description: "主机 CPU 使用率持续超过80%。", currentValue: "{{ $value }}", threshold: "80%" }
|
||||||
|
- alert: HostCpuUsageCritical
|
||||||
|
expr: 100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 90
|
||||||
|
for: 5m
|
||||||
|
labels: { severity: critical, service: host }
|
||||||
|
annotations: { summary: "主机 CPU 使用率达到严重阈值", description: "主机 CPU 使用率持续超过90%。", currentValue: "{{ $value }}", threshold: "90%" }
|
||||||
|
- alert: HostMemoryUsageWarning
|
||||||
|
expr: ((1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100 > 85) and ((1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100 <= 95)
|
||||||
|
for: 10m
|
||||||
|
labels: { severity: warning, service: host }
|
||||||
|
annotations: { summary: "主机内存使用率达到警告阈值", description: "主机内存使用率持续超过85%。", currentValue: "{{ $value }}", threshold: "85%" }
|
||||||
|
- alert: HostMemoryUsageCritical
|
||||||
|
expr: (1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100 > 95
|
||||||
|
for: 5m
|
||||||
|
labels: { severity: critical, service: host }
|
||||||
|
annotations: { summary: "主机内存使用率达到严重阈值", description: "主机内存使用率持续超过95%。", currentValue: "{{ $value }}", threshold: "95%" }
|
||||||
|
- alert: HostRootDiskUsageWarning
|
||||||
|
expr: ((1 - node_filesystem_avail_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"} / node_filesystem_size_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"}) * 100 > 80) and ((1 - node_filesystem_avail_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"} / node_filesystem_size_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"}) * 100 <= 90)
|
||||||
|
for: 15m
|
||||||
|
labels: { severity: warning, service: host }
|
||||||
|
annotations: { summary: "根磁盘使用率达到警告阈值", description: "根磁盘使用率持续超过80%。", currentValue: "{{ $value }}", threshold: "80%" }
|
||||||
|
- alert: HostRootDiskUsageCritical
|
||||||
|
expr: (1 - node_filesystem_avail_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"} / node_filesystem_size_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"}) * 100 > 90
|
||||||
|
for: 5m
|
||||||
|
labels: { severity: critical, service: host }
|
||||||
|
annotations: { summary: "根磁盘使用率达到严重阈值", description: "根磁盘使用率持续超过90%。", currentValue: "{{ $value }}", threshold: "90%" }
|
||||||
|
- alert: CmppApiHttpErrorRateWarning
|
||||||
|
expr: (100 * sum(rate(cmpp_api_http_requests_total{status=~"5.."}[5m])) / clamp_min(sum(rate(cmpp_api_http_requests_total[5m])), 0.001) > 1) and (100 * sum(rate(cmpp_api_http_requests_total{status=~"5.."}[5m])) / clamp_min(sum(rate(cmpp_api_http_requests_total[5m])), 0.001) <= 5)
|
||||||
|
for: 5m
|
||||||
|
labels: { severity: warning, service: api }
|
||||||
|
annotations: { summary: "API 5xx 错误率达到警告阈值", description: "API 5xx 错误率持续超过1%。", currentValue: "{{ $value }}", threshold: "1%" }
|
||||||
|
- alert: CmppApiHttpErrorRateCritical
|
||||||
|
expr: 100 * sum(rate(cmpp_api_http_requests_total{status=~"5.."}[5m])) / clamp_min(sum(rate(cmpp_api_http_requests_total[5m])), 0.001) > 5
|
||||||
|
for: 5m
|
||||||
|
labels: { severity: critical, service: api }
|
||||||
|
annotations: { summary: "API 5xx 错误率达到严重阈值", description: "API 5xx 错误率持续超过5%。", currentValue: "{{ $value }}", threshold: "5%" }
|
||||||
|
- alert: CmppApiLatencyWarning
|
||||||
|
expr: (histogram_quantile(0.95, sum by (le) (rate(cmpp_api_http_request_duration_seconds_bucket[10m]))) > 1) and (histogram_quantile(0.95, sum by (le) (rate(cmpp_api_http_request_duration_seconds_bucket[10m]))) <= 3)
|
||||||
|
for: 10m
|
||||||
|
labels: { severity: warning, service: api }
|
||||||
|
annotations: { summary: "API P95 响应时间达到警告阈值", description: "API P95 响应时间持续超过1秒。", currentValue: "{{ $value }}", threshold: "1秒" }
|
||||||
|
- alert: CmppApiLatencyCritical
|
||||||
|
expr: histogram_quantile(0.95, sum by (le) (rate(cmpp_api_http_request_duration_seconds_bucket[10m]))) > 3
|
||||||
|
for: 5m
|
||||||
|
labels: { severity: critical, service: api }
|
||||||
|
annotations: { summary: "API P95 响应时间达到严重阈值", description: "API P95 响应时间持续超过3秒。", currentValue: "{{ $value }}", threshold: "3秒" }
|
||||||
|
- alert: CmppApiEventLoopLagWarning
|
||||||
|
expr: (cmpp_api_nodejs_event_loop_lag_p99_seconds > 0.2) and (cmpp_api_nodejs_event_loop_lag_p99_seconds <= 1)
|
||||||
|
for: 10m
|
||||||
|
labels: { severity: warning, service: api }
|
||||||
|
annotations: { summary: "API 事件循环 P99 达到警告阈值", description: "API 事件循环 P99 持续超过0.2秒。", currentValue: "{{ $value }}", threshold: "0.2秒" }
|
||||||
|
- alert: CmppApiEventLoopLagCritical
|
||||||
|
expr: cmpp_api_nodejs_event_loop_lag_p99_seconds > 1
|
||||||
|
for: 5m
|
||||||
|
labels: { severity: critical, service: api }
|
||||||
|
annotations: { summary: "API 事件循环 P99 达到严重阈值", description: "API 事件循环 P99 持续超过1秒。", currentValue: "{{ $value }}", threshold: "1秒" }
|
||||||
|
- alert: CmppGatewayQueueDelayedWarning
|
||||||
|
expr: (cmpp_gateway_submit_queue_oldest_pending_age_seconds > 30) and (cmpp_gateway_submit_queue_oldest_pending_age_seconds <= 120)
|
||||||
|
for: 2m
|
||||||
|
labels: { severity: warning, service: gateway }
|
||||||
|
annotations: { summary: "Gateway 最旧 pending 达到警告阈值", description: "Gateway 最旧 pending 持续超过30秒。", currentValue: "{{ $value }}", threshold: "30秒" }
|
||||||
|
- alert: CmppGatewayQueueDelayedCritical
|
||||||
|
expr: cmpp_gateway_submit_queue_oldest_pending_age_seconds > 120
|
||||||
|
for: 2m
|
||||||
|
labels: { severity: critical, service: gateway }
|
||||||
|
annotations: { summary: "Gateway 最旧 pending 达到严重阈值", description: "Gateway 最旧 pending 持续超过120秒。", currentValue: "{{ $value }}", threshold: "120秒" }
|
||||||
|
- alert: PostgresConnectionsWarning
|
||||||
|
expr: (100 * sum(pg_stat_activity_count) / clamp_min(max(pg_settings_max_connections), 1) > 70) and (100 * sum(pg_stat_activity_count) / clamp_min(max(pg_settings_max_connections), 1) <= 85)
|
||||||
|
for: 10m
|
||||||
|
labels: { severity: warning, service: postgresql }
|
||||||
|
annotations: { summary: "PostgreSQL 连接使用率达到警告阈值", description: "PostgreSQL 连接使用率持续超过70%。", currentValue: "{{ $value }}", threshold: "70%" }
|
||||||
|
- alert: PostgresConnectionsCritical
|
||||||
|
expr: 100 * sum(pg_stat_activity_count) / clamp_min(max(pg_settings_max_connections), 1) > 85
|
||||||
|
for: 5m
|
||||||
|
labels: { severity: critical, service: postgresql }
|
||||||
|
annotations: { summary: "PostgreSQL 连接使用率达到严重阈值", description: "PostgreSQL 连接使用率持续超过85%。", currentValue: "{{ $value }}", threshold: "85%" }
|
||||||
|
- alert: RedisMemoryWarning
|
||||||
|
expr: (100 * redis_memory_used_bytes / redis_memory_max_bytes > 70) and (100 * redis_memory_used_bytes / redis_memory_max_bytes <= 85)
|
||||||
|
for: 10m
|
||||||
|
labels: { severity: warning, service: redis }
|
||||||
|
annotations: { summary: "Redis 内存使用率达到警告阈值", description: "Redis 内存使用率持续超过70%。", currentValue: "{{ $value }}", threshold: "70%" }
|
||||||
|
- alert: RedisMemoryCritical
|
||||||
|
expr: 100 * redis_memory_used_bytes / redis_memory_max_bytes > 85
|
||||||
|
for: 5m
|
||||||
|
labels: { severity: critical, service: redis }
|
||||||
|
annotations: { summary: "Redis 内存使用率达到严重阈值", description: "Redis 内存使用率持续超过85%。", currentValue: "{{ $value }}", threshold: "85%" }
|
||||||
|
- alert: MinioCapacityWarning
|
||||||
|
expr: (100 * (1 - minio_cluster_capacity_usable_free_bytes / minio_cluster_capacity_usable_total_bytes) > 80) and (100 * (1 - minio_cluster_capacity_usable_free_bytes / minio_cluster_capacity_usable_total_bytes) <= 90)
|
||||||
|
for: 15m
|
||||||
|
labels: { severity: warning, service: minio }
|
||||||
|
annotations: { summary: "MinIO 容量使用率达到警告阈值", description: "MinIO 容量使用率持续超过80%。", currentValue: "{{ $value }}", threshold: "80%" }
|
||||||
|
- alert: MinioCapacityCritical
|
||||||
|
expr: 100 * (1 - minio_cluster_capacity_usable_free_bytes / minio_cluster_capacity_usable_total_bytes) > 90
|
||||||
|
for: 5m
|
||||||
|
labels: { severity: critical, service: minio }
|
||||||
|
annotations: { summary: "MinIO 容量使用率达到严重阈值", description: "MinIO 容量使用率持续超过90%。", currentValue: "{{ $value }}", threshold: "90%" }
|
||||||
@@ -26,7 +26,7 @@ promtool_bin="$(command -v promtool)"
|
|||||||
backup_dir="/etc/prometheus/cmpp-backups/$(date '+%Y%m%d-%H%M%S')"
|
backup_dir="/etc/prometheus/cmpp-backups/$(date '+%Y%m%d-%H%M%S')"
|
||||||
mkdir -p "$backup_dir" /etc/systemd/system/prometheus.service.d /etc/systemd/system/prometheus-node-exporter.service.d
|
mkdir -p "$backup_dir" /etc/systemd/system/prometheus.service.d /etc/systemd/system/prometheus-node-exporter.service.d
|
||||||
|
|
||||||
for config_file in /etc/prometheus/prometheus.yml /etc/prometheus/cmpp-alerts.yml; do
|
for config_file in /etc/prometheus/prometheus.yml /etc/prometheus/cmpp-alerts.yml /etc/prometheus/cmpp-alerts-source.yml; do
|
||||||
if [[ -f "$config_file" ]]; then
|
if [[ -f "$config_file" ]]; then
|
||||||
cp --preserve=mode,timestamps "$config_file" "$backup_dir/$(basename "$config_file")"
|
cp --preserve=mode,timestamps "$config_file" "$backup_dir/$(basename "$config_file")"
|
||||||
fi
|
fi
|
||||||
@@ -40,12 +40,28 @@ fi
|
|||||||
|
|
||||||
log "Installing platform-owned scrape and alert configuration"
|
log "Installing platform-owned scrape and alert configuration"
|
||||||
install -o root -g root -m 0644 "$SCRIPT_DIR/prometheus.yml" /etc/prometheus/prometheus.yml
|
install -o root -g root -m 0644 "$SCRIPT_DIR/prometheus.yml" /etc/prometheus/prometheus.yml
|
||||||
install -o root -g root -m 0644 "$SCRIPT_DIR/cmpp-alerts.yml" /etc/prometheus/cmpp-alerts.yml
|
install -o root -g root -m 0644 "$SCRIPT_DIR/cmpp-alerts.yml" /etc/prometheus/cmpp-alerts-source.yml
|
||||||
|
# 可配置规则由 API 管理;基础规则必须排除同名项,否则 Prometheus 会同时计算旧阈值和新阈值。
|
||||||
|
awk '
|
||||||
|
BEGIN {
|
||||||
|
split("HostCpuUsageWarning HostCpuUsageCritical HostMemoryUsageWarning HostMemoryUsageCritical HostRootDiskUsageWarning HostRootDiskUsageCritical CmppApiHttpErrorRateWarning CmppApiHttpErrorRateCritical CmppApiLatencyWarning CmppApiLatencyCritical CmppApiEventLoopLagWarning CmppApiEventLoopLagCritical CmppGatewayQueueDelayedWarning CmppGatewayQueueDelayedCritical PostgresConnectionsWarning PostgresConnectionsCritical RedisMemoryWarning RedisMemoryCritical MinioCapacityWarning MinioCapacityCritical", names, " ")
|
||||||
|
for (i in names) dropped[names[i]] = 1
|
||||||
|
}
|
||||||
|
/^ - name:/ { skip = 0 }
|
||||||
|
/^ - alert:/ { skip = ($3 in dropped) }
|
||||||
|
!skip { print }
|
||||||
|
' "$SCRIPT_DIR/cmpp-alerts.yml" > /etc/prometheus/cmpp-alerts.yml
|
||||||
|
chown root:root /etc/prometheus/cmpp-alerts.yml
|
||||||
|
chmod 0644 /etc/prometheus/cmpp-alerts.yml
|
||||||
|
install -d -o cmpp-api -g prometheus -m 0750 /var/lib/cmpp-platform/monitoring
|
||||||
|
if [[ ! -f /var/lib/cmpp-platform/monitoring/cmpp-managed-alerts.yml ]]; then
|
||||||
|
install -o cmpp-api -g prometheus -m 0640 "$SCRIPT_DIR/cmpp-managed-alerts.yml" /var/lib/cmpp-platform/monitoring/cmpp-managed-alerts.yml
|
||||||
|
fi
|
||||||
|
|
||||||
cat >/etc/systemd/system/prometheus.service.d/cmpp-monitoring.conf <<EOF
|
cat >/etc/systemd/system/prometheus.service.d/cmpp-monitoring.conf <<EOF
|
||||||
[Service]
|
[Service]
|
||||||
ExecStart=
|
ExecStart=
|
||||||
ExecStart=${prometheus_bin} --config.file=/etc/prometheus/prometheus.yml --storage.tsdb.path=/var/lib/prometheus/metrics2 --storage.tsdb.retention.time=${PROMETHEUS_RETENTION_TIME} --storage.tsdb.retention.size=${PROMETHEUS_RETENTION_SIZE} --web.listen-address=127.0.0.1:9090
|
ExecStart=${prometheus_bin} --config.file=/etc/prometheus/prometheus.yml --storage.tsdb.path=/var/lib/prometheus/metrics2 --storage.tsdb.retention.time=${PROMETHEUS_RETENTION_TIME} --storage.tsdb.retention.size=${PROMETHEUS_RETENTION_SIZE} --web.listen-address=127.0.0.1:9090 --web.enable-lifecycle
|
||||||
EOF
|
EOF
|
||||||
|
|
||||||
cat >/etc/systemd/system/prometheus-node-exporter.service.d/cmpp-monitoring.conf <<EOF
|
cat >/etc/systemd/system/prometheus-node-exporter.service.d/cmpp-monitoring.conf <<EOF
|
||||||
@@ -56,6 +72,7 @@ EOF
|
|||||||
|
|
||||||
log "Validating Prometheus configuration before restart"
|
log "Validating Prometheus configuration before restart"
|
||||||
"$promtool_bin" check rules /etc/prometheus/cmpp-alerts.yml
|
"$promtool_bin" check rules /etc/prometheus/cmpp-alerts.yml
|
||||||
|
"$promtool_bin" check rules /var/lib/cmpp-platform/monitoring/cmpp-managed-alerts.yml
|
||||||
"$promtool_bin" check config /etc/prometheus/prometheus.yml
|
"$promtool_bin" check config /etc/prometheus/prometheus.yml
|
||||||
|
|
||||||
systemctl daemon-reload
|
systemctl daemon-reload
|
||||||
|
|||||||
@@ -7,6 +7,7 @@ global:
|
|||||||
|
|
||||||
rule_files:
|
rule_files:
|
||||||
- /etc/prometheus/cmpp-alerts.yml
|
- /etc/prometheus/cmpp-alerts.yml
|
||||||
|
- /var/lib/cmpp-platform/monitoring/*.yml
|
||||||
|
|
||||||
scrape_configs:
|
scrape_configs:
|
||||||
- job_name: prometheus
|
- job_name: prometheus
|
||||||
|
|||||||
Reference in New Issue
Block a user