feat: add configurable infrastructure alerts
This commit is contained in:
hectorzhao
@@ -0,0 +1,23 @@
|
||||
CREATE TABLE "InfrastructureAlertSetting" (
|
||||
"id" TEXT NOT NULL DEFAULT 'global',
|
||||
"configVersion" INTEGER NOT NULL DEFAULT 1,
|
||||
"effectiveVersion" INTEGER NOT NULL DEFAULT 1,
|
||||
"thresholds" JSONB NOT NULL,
|
||||
"effectiveThresholds" JSONB NOT NULL,
|
||||
"applyStatus" TEXT NOT NULL DEFAULT 'effective',
|
||||
"lastError" TEXT,
|
||||
"updatedById" TEXT,
|
||||
"appliedAt" TIMESTAMP(3),
|
||||
"createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
"updatedAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
CONSTRAINT "InfrastructureAlertSetting_pkey" PRIMARY KEY ("id")
|
||||
);
|
||||
|
||||
INSERT INTO "InfrastructureAlertSetting" (
|
||||
"id", "thresholds", "effectiveThresholds", "appliedAt"
|
||||
) VALUES (
|
||||
'global',
|
||||
'{"hostCpu":{"warning":80,"critical":90},"hostMemory":{"warning":85,"critical":95},"hostDisk":{"warning":80,"critical":90},"apiError":{"warning":1,"critical":5},"apiLatency":{"warning":1,"critical":3},"apiEventLoop":{"warning":0.2,"critical":1},"gatewayQueue":{"warning":30,"critical":120},"postgresConnections":{"warning":70,"critical":85},"redisMemory":{"warning":70,"critical":85},"minioCapacity":{"warning":80,"critical":90}}'::jsonb,
|
||||
'{"hostCpu":{"warning":80,"critical":90},"hostMemory":{"warning":85,"critical":95},"hostDisk":{"warning":80,"critical":90},"apiError":{"warning":1,"critical":5},"apiLatency":{"warning":1,"critical":3},"apiEventLoop":{"warning":0.2,"critical":1},"gatewayQueue":{"warning":30,"critical":120},"postgresConnections":{"warning":70,"critical":85},"redisMemory":{"warning":70,"critical":85},"minioCapacity":{"warning":80,"critical":90}}'::jsonb,
|
||||
CURRENT_TIMESTAMP
|
||||
);
|
||||
@@ -2432,3 +2432,17 @@ model SecurityProtectedNetwork {
|
||||
|
||||
@@index([enabled, createdAt])
|
||||
}
|
||||
|
||||
model InfrastructureAlertSetting {
|
||||
id String @id @default("global")
|
||||
configVersion Int @default(1)
|
||||
effectiveVersion Int @default(1)
|
||||
thresholds Json
|
||||
effectiveThresholds Json
|
||||
applyStatus String @default("effective")
|
||||
lastError String?
|
||||
updatedById String?
|
||||
appliedAt DateTime?
|
||||
createdAt DateTime @default(now())
|
||||
updatedAt DateTime @updatedAt
|
||||
}
|
||||
|
||||
@@ -0,0 +1,19 @@
|
||||
import { BadRequestException } from '@nestjs/common';
|
||||
import { DEFAULT_ALERT_THRESHOLDS, InfrastructureAlertSettingsService } from './infrastructure-alert-settings.service';
|
||||
|
||||
describe('InfrastructureAlertSettingsService', () => {
|
||||
const service = new InfrastructureAlertSettingsService({} as never, { get: () => undefined } as never);
|
||||
|
||||
it('accepts the fixed threshold whitelist and renders managed rules', () => {
|
||||
const validated = (service as unknown as { validate(value: unknown): unknown }).validate(DEFAULT_ALERT_THRESHOLDS);
|
||||
const rules = (service as unknown as { renderRules(value: unknown): string }).renderRules(validated);
|
||||
expect(rules).toContain('HostCpuUsageWarning');
|
||||
expect(rules).toContain('CmppGatewayQueueDelayedCritical');
|
||||
expect(rules).toContain('threshold: "120秒"');
|
||||
});
|
||||
|
||||
it('rejects unknown keys and warning thresholds that are not below critical', () => {
|
||||
expect(() => (service as unknown as { validate(value: unknown): unknown }).validate({ ...DEFAULT_ALERT_THRESHOLDS, promql: { warning: 1, critical: 2 } })).toThrow(BadRequestException);
|
||||
expect(() => (service as unknown as { validate(value: unknown): unknown }).validate({ ...DEFAULT_ALERT_THRESHOLDS, hostCpu: { warning: 90, critical: 90 } })).toThrow(BadRequestException);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,142 @@
|
||||
import { BadRequestException, ConflictException, Injectable, Logger, ServiceUnavailableException } from '@nestjs/common';
|
||||
import { ConfigService } from '@nestjs/config';
|
||||
import { Prisma } from '@prisma/client';
|
||||
import { execFile } from 'node:child_process';
|
||||
import { mkdir, readFile, rename, rm, writeFile } from 'node:fs/promises';
|
||||
import { dirname } from 'node:path';
|
||||
import { promisify } from 'node:util';
|
||||
import { PrismaService } from '../prisma/prisma.service';
|
||||
import type { InfrastructureAlertSettings, InfrastructureAlertThresholds } from './infrastructure-monitoring.contracts';
|
||||
|
||||
const execFileAsync = promisify(execFile);
|
||||
|
||||
export const ALERT_THRESHOLD_DEFINITIONS = [
|
||||
{ key: 'hostCpu', label: '主机 CPU 使用率', unit: '%', min: 1, max: 100, step: 1, warning: 80, critical: 90, expr: '100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100)', names: ['HostCpuUsageWarning', 'HostCpuUsageCritical'], service: 'host', durations: ['10m', '5m'] },
|
||||
{ key: 'hostMemory', label: '主机内存使用率', unit: '%', min: 1, max: 100, step: 1, warning: 85, critical: 95, expr: '(1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100', names: ['HostMemoryUsageWarning', 'HostMemoryUsageCritical'], service: 'host', durations: ['10m', '5m'] },
|
||||
{ key: 'hostDisk', label: '根磁盘使用率', unit: '%', min: 1, max: 100, step: 1, warning: 80, critical: 90, expr: '(1 - node_filesystem_avail_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"} / node_filesystem_size_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"}) * 100', names: ['HostRootDiskUsageWarning', 'HostRootDiskUsageCritical'], service: 'host', durations: ['15m', '5m'] },
|
||||
{ key: 'apiError', label: 'API 5xx 错误率', unit: '%', min: 0.1, max: 100, step: 0.1, warning: 1, critical: 5, expr: '100 * sum(rate(cmpp_api_http_requests_total{status=~"5.."}[5m])) / clamp_min(sum(rate(cmpp_api_http_requests_total[5m])), 0.001)', names: ['CmppApiHttpErrorRateWarning', 'CmppApiHttpErrorRateCritical'], service: 'api', durations: ['5m', '5m'] },
|
||||
{ key: 'apiLatency', label: 'API P95 响应时间', unit: '秒', min: 0.1, max: 60, step: 0.1, warning: 1, critical: 3, expr: 'histogram_quantile(0.95, sum by (le) (rate(cmpp_api_http_request_duration_seconds_bucket[10m])))', names: ['CmppApiLatencyWarning', 'CmppApiLatencyCritical'], service: 'api', durations: ['10m', '5m'] },
|
||||
{ key: 'apiEventLoop', label: 'API 事件循环 P99', unit: '秒', min: 0.01, max: 10, step: 0.01, warning: 0.2, critical: 1, expr: 'cmpp_api_nodejs_event_loop_lag_p99_seconds', names: ['CmppApiEventLoopLagWarning', 'CmppApiEventLoopLagCritical'], service: 'api', durations: ['10m', '5m'] },
|
||||
{ key: 'gatewayQueue', label: 'Gateway 最旧 pending', unit: '秒', min: 1, max: 3600, step: 1, warning: 30, critical: 120, expr: 'cmpp_gateway_submit_queue_oldest_pending_age_seconds', names: ['CmppGatewayQueueDelayedWarning', 'CmppGatewayQueueDelayedCritical'], service: 'gateway', durations: ['2m', '2m'] },
|
||||
{ key: 'postgresConnections', label: 'PostgreSQL 连接使用率', unit: '%', min: 1, max: 100, step: 1, warning: 70, critical: 85, expr: '100 * sum(pg_stat_activity_count) / clamp_min(max(pg_settings_max_connections), 1)', names: ['PostgresConnectionsWarning', 'PostgresConnectionsCritical'], service: 'postgresql', durations: ['10m', '5m'] },
|
||||
{ key: 'redisMemory', label: 'Redis 内存使用率', unit: '%', min: 1, max: 100, step: 1, warning: 70, critical: 85, expr: '100 * redis_memory_used_bytes / redis_memory_max_bytes', names: ['RedisMemoryWarning', 'RedisMemoryCritical'], service: 'redis', durations: ['10m', '5m'] },
|
||||
{ key: 'minioCapacity', label: 'MinIO 容量使用率', unit: '%', min: 1, max: 100, step: 1, warning: 80, critical: 90, expr: '100 * (1 - minio_cluster_capacity_usable_free_bytes / minio_cluster_capacity_usable_total_bytes)', names: ['MinioCapacityWarning', 'MinioCapacityCritical'], service: 'minio', durations: ['15m', '5m'] },
|
||||
] as const;
|
||||
|
||||
export const DEFAULT_ALERT_THRESHOLDS: InfrastructureAlertThresholds = Object.fromEntries(
|
||||
ALERT_THRESHOLD_DEFINITIONS.map((item) => [item.key, { warning: item.warning, critical: item.critical }]),
|
||||
);
|
||||
|
||||
@Injectable()
|
||||
export class InfrastructureAlertSettingsService {
|
||||
private readonly logger = new Logger(InfrastructureAlertSettingsService.name);
|
||||
private readonly rulesPath: string;
|
||||
private readonly promtoolPath: string;
|
||||
private readonly reloadUrl: string;
|
||||
|
||||
constructor(private readonly prisma: PrismaService, config: ConfigService) {
|
||||
this.rulesPath = String(config.get('PROMETHEUS_MANAGED_RULES_PATH') ?? '/var/lib/cmpp-platform/monitoring/cmpp-managed-alerts.yml');
|
||||
this.promtoolPath = String(config.get('PROMTOOL_PATH') ?? '/usr/bin/promtool');
|
||||
this.reloadUrl = String(config.get('PROMETHEUS_RELOAD_URL') ?? 'http://127.0.0.1:9090/-/reload');
|
||||
}
|
||||
|
||||
async get(): Promise<InfrastructureAlertSettings> {
|
||||
const row = await this.prisma.infrastructureAlertSetting.findUnique({ where: { id: 'global' } });
|
||||
const thresholds = this.asThresholds(row?.thresholds) ?? DEFAULT_ALERT_THRESHOLDS;
|
||||
const effective = this.asThresholds(row?.effectiveThresholds) ?? thresholds;
|
||||
return {
|
||||
configVersion: row?.configVersion ?? 1,
|
||||
effectiveVersion: row?.effectiveVersion ?? 1,
|
||||
applyStatus: (row?.applyStatus as InfrastructureAlertSettings['applyStatus']) ?? 'effective',
|
||||
lastError: row?.lastError ?? null,
|
||||
appliedAt: row?.appliedAt?.toISOString() ?? null,
|
||||
thresholds,
|
||||
effectiveThresholds: effective,
|
||||
definitions: ALERT_THRESHOLD_DEFINITIONS.map(({ key, label, unit, min, max, step }) => ({ key, label, unit, min, max, step })),
|
||||
};
|
||||
}
|
||||
|
||||
async update(body: { configVersion?: number; thresholds?: unknown }, operatorId?: string) {
|
||||
const expectedVersion = Number(body.configVersion);
|
||||
if (!Number.isInteger(expectedVersion) || expectedVersion < 1) throw new BadRequestException('配置版本无效');
|
||||
const thresholds = this.validate(body.thresholds);
|
||||
const claimed = await this.prisma.infrastructureAlertSetting.updateMany({
|
||||
where: { id: 'global', configVersion: expectedVersion },
|
||||
data: { configVersion: { increment: 1 }, thresholds: thresholds as Prisma.InputJsonValue, applyStatus: 'applying', lastError: null, updatedById: operatorId },
|
||||
});
|
||||
// 版本条件更新是跨进程的写锁；避免两个 API 实例同时覆盖规则文件并把旧配置误标成已生效。
|
||||
if (claimed.count !== 1) throw new ConflictException('告警阈值已被其他管理员修改，请刷新后重试');
|
||||
const nextVersion = expectedVersion + 1;
|
||||
try {
|
||||
await this.applyRules(thresholds);
|
||||
await this.prisma.$transaction([
|
||||
this.prisma.infrastructureAlertSetting.update({ where: { id: 'global' }, data: { effectiveVersion: nextVersion, effectiveThresholds: thresholds as Prisma.InputJsonValue, applyStatus: 'effective', lastError: null, appliedAt: new Date() } }),
|
||||
this.prisma.operationLog.create({ data: { userId: operatorId, action: 'monitoring.alert_thresholds_updated', resource: 'infrastructure_alert_setting', resourceId: 'global', detail: { configVersion: nextVersion, thresholds } } }),
|
||||
]);
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message.slice(0, 500) : 'unknown error';
|
||||
await this.prisma.infrastructureAlertSetting.update({ where: { id: 'global' }, data: { applyStatus: 'failed', lastError: message } });
|
||||
this.logger.error(`Prometheus managed rules apply failed: ${message}`);
|
||||
throw new ServiceUnavailableException('阈值已保存但 Prometheus 应用失败，原生效规则已保留');
|
||||
}
|
||||
return this.get();
|
||||
}
|
||||
|
||||
private validate(value: unknown): InfrastructureAlertThresholds {
|
||||
if (!value || typeof value !== 'object' || Array.isArray(value)) throw new BadRequestException('告警阈值格式无效');
|
||||
const input = value as Record<string, unknown>;
|
||||
if (Object.keys(input).some((key) => !ALERT_THRESHOLD_DEFINITIONS.some((item) => item.key === key))) throw new BadRequestException('存在不允许配置的告警指标');
|
||||
const result: InfrastructureAlertThresholds = {};
|
||||
for (const definition of ALERT_THRESHOLD_DEFINITIONS) {
|
||||
const pair = input[definition.key] as { warning?: unknown; critical?: unknown } | undefined;
|
||||
const warning = Number(pair?.warning);
|
||||
const critical = Number(pair?.critical);
|
||||
if (!Number.isFinite(warning) || !Number.isFinite(critical) || warning < definition.min || critical > definition.max || warning >= critical) {
|
||||
throw new BadRequestException(`${definition.label}必须满足最小值 ≤ 警告阈值 < 严重阈值 ≤ 最大值`);
|
||||
}
|
||||
result[definition.key] = { warning, critical };
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
private asThresholds(value: unknown) {
|
||||
try { return this.validate(value); } catch { return null; }
|
||||
}
|
||||
|
||||
private renderRules(thresholds: InfrastructureAlertThresholds) {
|
||||
const lines = ['groups:', ' - name: cmpp-managed-thresholds', ' rules:'];
|
||||
for (const definition of ALERT_THRESHOLD_DEFINITIONS) {
|
||||
const pair = thresholds[definition.key];
|
||||
const values = [pair.warning, pair.critical];
|
||||
for (let index = 0; index < 2; index += 1) {
|
||||
const isWarning = index === 0;
|
||||
const expr = isWarning ? `(${definition.expr} > ${values[0]}) and (${definition.expr} <= ${values[1]})` : `${definition.expr} > ${values[1]}`;
|
||||
lines.push(` - alert: ${definition.names[index]}`, ` expr: ${expr}`, ` for: ${definition.durations[index]}`, ' labels:', ` severity: ${isWarning ? 'warning' : 'critical'}`, ` service: ${definition.service}`, ' annotations:', ` summary: "${definition.label}${isWarning ? '达到警告阈值' : '达到严重阈值'}"`, ` description: "${definition.label}持续超过${values[index]}${definition.unit}。"`, ' currentValue: "{{ $value }}"', ` threshold: "${values[index]}${definition.unit}"`);
|
||||
}
|
||||
}
|
||||
return `${lines.join('\n')}\n`;
|
||||
}
|
||||
|
||||
private async applyRules(thresholds: InfrastructureAlertThresholds) {
|
||||
const directory = dirname(this.rulesPath);
|
||||
const temporary = `${this.rulesPath}.${process.pid}.${Date.now()}.tmp`;
|
||||
await mkdir(directory, { recursive: true });
|
||||
const previous = await readFile(this.rulesPath).catch(() => null);
|
||||
try {
|
||||
await writeFile(temporary, this.renderRules(thresholds), { mode: 0o640 });
|
||||
await execFileAsync(this.promtoolPath, ['check', 'rules', temporary], { timeout: 10_000 });
|
||||
await rename(temporary, this.rulesPath);
|
||||
const response = await fetch(this.reloadUrl, { method: 'POST', signal: AbortSignal.timeout(5_000) });
|
||||
if (!response.ok) throw new Error(`Prometheus reload HTTP ${response.status}`);
|
||||
} catch (error) {
|
||||
await rm(temporary, { force: true });
|
||||
// 规则替换和 reload 不是一个事务，失败时必须恢复旧文件并再次 reload，避免数据库状态与实际告警漂移。
|
||||
if (previous) {
|
||||
await writeFile(temporary, previous, { mode: 0o640 });
|
||||
await rename(temporary, this.rulesPath);
|
||||
await fetch(this.reloadUrl, { method: 'POST', signal: AbortSignal.timeout(5_000) }).catch(() => undefined);
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -76,3 +76,16 @@ export type InfrastructureMonitoringOverview = {
|
||||
serviceMetrics: InfrastructureServiceMetricGroup[];
|
||||
alerts: InfrastructureAlert[];
|
||||
};
|
||||
|
||||
export type InfrastructureAlertThresholds = Record<string, { warning: number; critical: number }>;
|
||||
|
||||
export type InfrastructureAlertSettings = {
|
||||
configVersion: number;
|
||||
effectiveVersion: number;
|
||||
applyStatus: 'effective' | 'applying' | 'failed';
|
||||
lastError: string | null;
|
||||
appliedAt: string | null;
|
||||
thresholds: InfrastructureAlertThresholds;
|
||||
effectiveThresholds: InfrastructureAlertThresholds;
|
||||
definitions: Array<{ key: string; label: string; unit: string; min: number; max: number; step: number }>;
|
||||
};
|
||||
|
||||
@@ -1,14 +1,29 @@
|
||||
import { Controller, Get, Query } from '@nestjs/common';
|
||||
import { Body, Controller, Get, Put, Query } from '@nestjs/common';
|
||||
import { ApiTags } from '@nestjs/swagger';
|
||||
import { CurrentSessionUserId } from '../auth/current-session-user.decorator';
|
||||
import { RequireRecentAuthentication } from '../auth/require-recent-authentication.decorator';
|
||||
import { InfrastructureAlertSettingsService } from './infrastructure-alert-settings.service';
|
||||
import { InfrastructureMonitoringService } from './infrastructure-monitoring.service';
|
||||
|
||||
@ApiTags('infrastructure-monitoring')
|
||||
@Controller('admin/infrastructure-monitoring')
|
||||
export class InfrastructureMonitoringController {
|
||||
constructor(private readonly monitoring: InfrastructureMonitoringService) {}
|
||||
constructor(private readonly monitoring: InfrastructureMonitoringService, private readonly settings: InfrastructureAlertSettingsService) {}
|
||||
|
||||
@Get('overview')
|
||||
overview(@Query('range') range?: string) {
|
||||
return this.monitoring.overview(range);
|
||||
}
|
||||
|
||||
@Get('notification-summary')
|
||||
notificationSummary() { return this.monitoring.notificationSummary(); }
|
||||
|
||||
@Get('alert-thresholds')
|
||||
alertThresholds() { return this.settings.get(); }
|
||||
|
||||
@Put('alert-thresholds')
|
||||
@RequireRecentAuthentication()
|
||||
updateAlertThresholds(@Body() body: { configVersion?: number; thresholds?: unknown }, @CurrentSessionUserId() operatorId?: string) {
|
||||
return this.settings.update(body, operatorId);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,9 +1,10 @@
|
||||
import { Module } from '@nestjs/common';
|
||||
import { InfrastructureMonitoringController } from './infrastructure-monitoring.controller';
|
||||
import { InfrastructureMonitoringService } from './infrastructure-monitoring.service';
|
||||
import { InfrastructureAlertSettingsService } from './infrastructure-alert-settings.service';
|
||||
|
||||
@Module({
|
||||
controllers: [InfrastructureMonitoringController],
|
||||
providers: [InfrastructureMonitoringService],
|
||||
providers: [InfrastructureMonitoringService, InfrastructureAlertSettingsService],
|
||||
})
|
||||
export class InfrastructureMonitoringModule {}
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { BadRequestException, Injectable, Logger } from '@nestjs/common';
|
||||
import { BadRequestException, Injectable, Logger, ServiceUnavailableException } from '@nestjs/common';
|
||||
import { ConfigService } from '@nestjs/config';
|
||||
import { createHash } from 'node:crypto';
|
||||
import type {
|
||||
@@ -210,6 +210,16 @@ export class InfrastructureMonitoringService {
|
||||
}
|
||||
}
|
||||
|
||||
async notificationSummary() {
|
||||
try {
|
||||
const alerts = this.parseAlerts(await this.getJson<PrometheusAlertResponse>('/api/v1/alerts'));
|
||||
return { count: alerts.length, criticalCount: alerts.filter((item) => item.severity === 'critical').length };
|
||||
} catch (error) {
|
||||
this.logger.warn(`Prometheus notification summary unavailable: ${error instanceof Error ? error.message : 'unknown error'}`);
|
||||
throw new ServiceUnavailableException('Prometheus活动告警当前不可用');
|
||||
}
|
||||
}
|
||||
|
||||
private parseRange(value?: string): InfrastructureMonitoringRange {
|
||||
const range = value || '24h';
|
||||
if (!(range in RANGE_CONFIG)) throw new BadRequestException('监控时间范围只支持1h、24h或7d');
|
||||
|
||||
Reference in New Issue
Block a user