feat: add configurable infrastructure alerts
This commit is contained in:
hectorzhao
@@ -0,0 +1,142 @@
|
||||
import { BadRequestException, ConflictException, Injectable, Logger, ServiceUnavailableException } from '@nestjs/common';
|
||||
import { ConfigService } from '@nestjs/config';
|
||||
import { Prisma } from '@prisma/client';
|
||||
import { execFile } from 'node:child_process';
|
||||
import { mkdir, readFile, rename, rm, writeFile } from 'node:fs/promises';
|
||||
import { dirname } from 'node:path';
|
||||
import { promisify } from 'node:util';
|
||||
import { PrismaService } from '../prisma/prisma.service';
|
||||
import type { InfrastructureAlertSettings, InfrastructureAlertThresholds } from './infrastructure-monitoring.contracts';
|
||||
|
||||
const execFileAsync = promisify(execFile);
|
||||
|
||||
export const ALERT_THRESHOLD_DEFINITIONS = [
|
||||
{ key: 'hostCpu', label: '主机 CPU 使用率', unit: '%', min: 1, max: 100, step: 1, warning: 80, critical: 90, expr: '100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100)', names: ['HostCpuUsageWarning', 'HostCpuUsageCritical'], service: 'host', durations: ['10m', '5m'] },
|
||||
{ key: 'hostMemory', label: '主机内存使用率', unit: '%', min: 1, max: 100, step: 1, warning: 85, critical: 95, expr: '(1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100', names: ['HostMemoryUsageWarning', 'HostMemoryUsageCritical'], service: 'host', durations: ['10m', '5m'] },
|
||||
{ key: 'hostDisk', label: '根磁盘使用率', unit: '%', min: 1, max: 100, step: 1, warning: 80, critical: 90, expr: '(1 - node_filesystem_avail_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"} / node_filesystem_size_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"}) * 100', names: ['HostRootDiskUsageWarning', 'HostRootDiskUsageCritical'], service: 'host', durations: ['15m', '5m'] },
|
||||
{ key: 'apiError', label: 'API 5xx 错误率', unit: '%', min: 0.1, max: 100, step: 0.1, warning: 1, critical: 5, expr: '100 * sum(rate(cmpp_api_http_requests_total{status=~"5.."}[5m])) / clamp_min(sum(rate(cmpp_api_http_requests_total[5m])), 0.001)', names: ['CmppApiHttpErrorRateWarning', 'CmppApiHttpErrorRateCritical'], service: 'api', durations: ['5m', '5m'] },
|
||||
{ key: 'apiLatency', label: 'API P95 响应时间', unit: '秒', min: 0.1, max: 60, step: 0.1, warning: 1, critical: 3, expr: 'histogram_quantile(0.95, sum by (le) (rate(cmpp_api_http_request_duration_seconds_bucket[10m])))', names: ['CmppApiLatencyWarning', 'CmppApiLatencyCritical'], service: 'api', durations: ['10m', '5m'] },
|
||||
{ key: 'apiEventLoop', label: 'API 事件循环 P99', unit: '秒', min: 0.01, max: 10, step: 0.01, warning: 0.2, critical: 1, expr: 'cmpp_api_nodejs_event_loop_lag_p99_seconds', names: ['CmppApiEventLoopLagWarning', 'CmppApiEventLoopLagCritical'], service: 'api', durations: ['10m', '5m'] },
|
||||
{ key: 'gatewayQueue', label: 'Gateway 最旧 pending', unit: '秒', min: 1, max: 3600, step: 1, warning: 30, critical: 120, expr: 'cmpp_gateway_submit_queue_oldest_pending_age_seconds', names: ['CmppGatewayQueueDelayedWarning', 'CmppGatewayQueueDelayedCritical'], service: 'gateway', durations: ['2m', '2m'] },
|
||||
{ key: 'postgresConnections', label: 'PostgreSQL 连接使用率', unit: '%', min: 1, max: 100, step: 1, warning: 70, critical: 85, expr: '100 * sum(pg_stat_activity_count) / clamp_min(max(pg_settings_max_connections), 1)', names: ['PostgresConnectionsWarning', 'PostgresConnectionsCritical'], service: 'postgresql', durations: ['10m', '5m'] },
|
||||
{ key: 'redisMemory', label: 'Redis 内存使用率', unit: '%', min: 1, max: 100, step: 1, warning: 70, critical: 85, expr: '100 * redis_memory_used_bytes / redis_memory_max_bytes', names: ['RedisMemoryWarning', 'RedisMemoryCritical'], service: 'redis', durations: ['10m', '5m'] },
|
||||
{ key: 'minioCapacity', label: 'MinIO 容量使用率', unit: '%', min: 1, max: 100, step: 1, warning: 80, critical: 90, expr: '100 * (1 - minio_cluster_capacity_usable_free_bytes / minio_cluster_capacity_usable_total_bytes)', names: ['MinioCapacityWarning', 'MinioCapacityCritical'], service: 'minio', durations: ['15m', '5m'] },
|
||||
] as const;
|
||||
|
||||
export const DEFAULT_ALERT_THRESHOLDS: InfrastructureAlertThresholds = Object.fromEntries(
|
||||
ALERT_THRESHOLD_DEFINITIONS.map((item) => [item.key, { warning: item.warning, critical: item.critical }]),
|
||||
);
|
||||
|
||||
@Injectable()
|
||||
export class InfrastructureAlertSettingsService {
|
||||
private readonly logger = new Logger(InfrastructureAlertSettingsService.name);
|
||||
private readonly rulesPath: string;
|
||||
private readonly promtoolPath: string;
|
||||
private readonly reloadUrl: string;
|
||||
|
||||
constructor(private readonly prisma: PrismaService, config: ConfigService) {
|
||||
this.rulesPath = String(config.get('PROMETHEUS_MANAGED_RULES_PATH') ?? '/var/lib/cmpp-platform/monitoring/cmpp-managed-alerts.yml');
|
||||
this.promtoolPath = String(config.get('PROMTOOL_PATH') ?? '/usr/bin/promtool');
|
||||
this.reloadUrl = String(config.get('PROMETHEUS_RELOAD_URL') ?? 'http://127.0.0.1:9090/-/reload');
|
||||
}
|
||||
|
||||
async get(): Promise<InfrastructureAlertSettings> {
|
||||
const row = await this.prisma.infrastructureAlertSetting.findUnique({ where: { id: 'global' } });
|
||||
const thresholds = this.asThresholds(row?.thresholds) ?? DEFAULT_ALERT_THRESHOLDS;
|
||||
const effective = this.asThresholds(row?.effectiveThresholds) ?? thresholds;
|
||||
return {
|
||||
configVersion: row?.configVersion ?? 1,
|
||||
effectiveVersion: row?.effectiveVersion ?? 1,
|
||||
applyStatus: (row?.applyStatus as InfrastructureAlertSettings['applyStatus']) ?? 'effective',
|
||||
lastError: row?.lastError ?? null,
|
||||
appliedAt: row?.appliedAt?.toISOString() ?? null,
|
||||
thresholds,
|
||||
effectiveThresholds: effective,
|
||||
definitions: ALERT_THRESHOLD_DEFINITIONS.map(({ key, label, unit, min, max, step }) => ({ key, label, unit, min, max, step })),
|
||||
};
|
||||
}
|
||||
|
||||
async update(body: { configVersion?: number; thresholds?: unknown }, operatorId?: string) {
|
||||
const expectedVersion = Number(body.configVersion);
|
||||
if (!Number.isInteger(expectedVersion) || expectedVersion < 1) throw new BadRequestException('配置版本无效');
|
||||
const thresholds = this.validate(body.thresholds);
|
||||
const claimed = await this.prisma.infrastructureAlertSetting.updateMany({
|
||||
where: { id: 'global', configVersion: expectedVersion },
|
||||
data: { configVersion: { increment: 1 }, thresholds: thresholds as Prisma.InputJsonValue, applyStatus: 'applying', lastError: null, updatedById: operatorId },
|
||||
});
|
||||
// 版本条件更新是跨进程的写锁；避免两个 API 实例同时覆盖规则文件并把旧配置误标成已生效。
|
||||
if (claimed.count !== 1) throw new ConflictException('告警阈值已被其他管理员修改，请刷新后重试');
|
||||
const nextVersion = expectedVersion + 1;
|
||||
try {
|
||||
await this.applyRules(thresholds);
|
||||
await this.prisma.$transaction([
|
||||
this.prisma.infrastructureAlertSetting.update({ where: { id: 'global' }, data: { effectiveVersion: nextVersion, effectiveThresholds: thresholds as Prisma.InputJsonValue, applyStatus: 'effective', lastError: null, appliedAt: new Date() } }),
|
||||
this.prisma.operationLog.create({ data: { userId: operatorId, action: 'monitoring.alert_thresholds_updated', resource: 'infrastructure_alert_setting', resourceId: 'global', detail: { configVersion: nextVersion, thresholds } } }),
|
||||
]);
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message.slice(0, 500) : 'unknown error';
|
||||
await this.prisma.infrastructureAlertSetting.update({ where: { id: 'global' }, data: { applyStatus: 'failed', lastError: message } });
|
||||
this.logger.error(`Prometheus managed rules apply failed: ${message}`);
|
||||
throw new ServiceUnavailableException('阈值已保存但 Prometheus 应用失败，原生效规则已保留');
|
||||
}
|
||||
return this.get();
|
||||
}
|
||||
|
||||
private validate(value: unknown): InfrastructureAlertThresholds {
|
||||
if (!value || typeof value !== 'object' || Array.isArray(value)) throw new BadRequestException('告警阈值格式无效');
|
||||
const input = value as Record<string, unknown>;
|
||||
if (Object.keys(input).some((key) => !ALERT_THRESHOLD_DEFINITIONS.some((item) => item.key === key))) throw new BadRequestException('存在不允许配置的告警指标');
|
||||
const result: InfrastructureAlertThresholds = {};
|
||||
for (const definition of ALERT_THRESHOLD_DEFINITIONS) {
|
||||
const pair = input[definition.key] as { warning?: unknown; critical?: unknown } | undefined;
|
||||
const warning = Number(pair?.warning);
|
||||
const critical = Number(pair?.critical);
|
||||
if (!Number.isFinite(warning) || !Number.isFinite(critical) || warning < definition.min || critical > definition.max || warning >= critical) {
|
||||
throw new BadRequestException(`${definition.label}必须满足最小值 ≤ 警告阈值 < 严重阈值 ≤ 最大值`);
|
||||
}
|
||||
result[definition.key] = { warning, critical };
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
private asThresholds(value: unknown) {
|
||||
try { return this.validate(value); } catch { return null; }
|
||||
}
|
||||
|
||||
private renderRules(thresholds: InfrastructureAlertThresholds) {
|
||||
const lines = ['groups:', ' - name: cmpp-managed-thresholds', ' rules:'];
|
||||
for (const definition of ALERT_THRESHOLD_DEFINITIONS) {
|
||||
const pair = thresholds[definition.key];
|
||||
const values = [pair.warning, pair.critical];
|
||||
for (let index = 0; index < 2; index += 1) {
|
||||
const isWarning = index === 0;
|
||||
const expr = isWarning ? `(${definition.expr} > ${values[0]}) and (${definition.expr} <= ${values[1]})` : `${definition.expr} > ${values[1]}`;
|
||||
lines.push(` - alert: ${definition.names[index]}`, ` expr: ${expr}`, ` for: ${definition.durations[index]}`, ' labels:', ` severity: ${isWarning ? 'warning' : 'critical'}`, ` service: ${definition.service}`, ' annotations:', ` summary: "${definition.label}${isWarning ? '达到警告阈值' : '达到严重阈值'}"`, ` description: "${definition.label}持续超过${values[index]}${definition.unit}。"`, ' currentValue: "{{ $value }}"', ` threshold: "${values[index]}${definition.unit}"`);
|
||||
}
|
||||
}
|
||||
return `${lines.join('\n')}\n`;
|
||||
}
|
||||
|
||||
private async applyRules(thresholds: InfrastructureAlertThresholds) {
|
||||
const directory = dirname(this.rulesPath);
|
||||
const temporary = `${this.rulesPath}.${process.pid}.${Date.now()}.tmp`;
|
||||
await mkdir(directory, { recursive: true });
|
||||
const previous = await readFile(this.rulesPath).catch(() => null);
|
||||
try {
|
||||
await writeFile(temporary, this.renderRules(thresholds), { mode: 0o640 });
|
||||
await execFileAsync(this.promtoolPath, ['check', 'rules', temporary], { timeout: 10_000 });
|
||||
await rename(temporary, this.rulesPath);
|
||||
const response = await fetch(this.reloadUrl, { method: 'POST', signal: AbortSignal.timeout(5_000) });
|
||||
if (!response.ok) throw new Error(`Prometheus reload HTTP ${response.status}`);
|
||||
} catch (error) {
|
||||
await rm(temporary, { force: true });
|
||||
// 规则替换和 reload 不是一个事务，失败时必须恢复旧文件并再次 reload，避免数据库状态与实际告警漂移。
|
||||
if (previous) {
|
||||
await writeFile(temporary, previous, { mode: 0o640 });
|
||||
await rename(temporary, this.rulesPath);
|
||||
await fetch(this.reloadUrl, { method: 'POST', signal: AbortSignal.timeout(5_000) }).catch(() => undefined);
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user