import { BadRequestException, ConflictException, Injectable, Logger, ServiceUnavailableException } from '@nestjs/common'; import { ConfigService } from '@nestjs/config'; import { Prisma } from '@prisma/client'; import { execFile } from 'node:child_process'; import { mkdir, readFile, rename, rm, writeFile } from 'node:fs/promises'; import { dirname } from 'node:path'; import { promisify } from 'node:util'; import { PrismaService } from '../prisma/prisma.service'; import { FILESYSTEM_USAGE_PERCENT } from './filesystem-metrics'; import type { InfrastructureAlertSettings, InfrastructureAlertThresholds } from './infrastructure-monitoring.contracts'; const execFileAsync = promisify(execFile); export const ALERT_THRESHOLD_DEFINITIONS = [ { key: 'hostCpu', label: '主机 CPU 使用率', unit: '%', min: 1, max: 100, step: 1, warning: 80, critical: 90, expr: '100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100)', names: ['HostCpuUsageWarning', 'HostCpuUsageCritical'], service: 'host', durations: ['10m', '5m'] }, { key: 'hostMemory', label: '主机内存使用率', unit: '%', min: 1, max: 100, step: 1, warning: 85, critical: 95, expr: '(1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100', names: ['HostMemoryUsageWarning', 'HostMemoryUsageCritical'], service: 'host', durations: ['10m', '5m'] }, { key: 'hostDisk', label: '磁盘(独立文件系统)使用率', unit: '%', min: 1, max: 100, step: 1, warning: 80, critical: 90, expr: FILESYSTEM_USAGE_PERCENT, names: ['HostRootDiskUsageWarning', 'HostRootDiskUsageCritical'], service: 'host', durations: ['15m', '5m'] }, { key: 'apiError', label: 'API 5xx 错误率', unit: '%', min: 0.1, max: 100, step: 0.1, warning: 1, critical: 5, expr: '100 * sum(rate(cmpp_api_http_requests_total{status=~"5.."}[5m])) / clamp_min(sum(rate(cmpp_api_http_requests_total[5m])), 0.001)', guard: 'sum(increase(cmpp_api_http_requests_total{status=~"5.."}[5m])) >= 5', names: ['CmppApiHttpErrorRateWarning', 'CmppApiHttpErrorRateCritical'], service: 'api', durations: ['5m', '5m'] }, { key: 'apiLatency', label: 'API P95 响应时间', unit: '秒', min: 0.1, max: 60, step: 0.1, warning: 1, critical: 3, expr: 'histogram_quantile(0.95, sum by (le) (rate(cmpp_api_http_request_duration_seconds_bucket[10m])))', names: ['CmppApiLatencyWarning', 'CmppApiLatencyCritical'], service: 'api', durations: ['10m', '5m'] }, { key: 'apiEventLoop', label: 'API 事件循环 P99', unit: '秒', min: 0.01, max: 10, step: 0.01, warning: 0.2, critical: 1, expr: 'cmpp_api_nodejs_event_loop_lag_p99_seconds', names: ['CmppApiEventLoopLagWarning', 'CmppApiEventLoopLagCritical'], service: 'api', durations: ['10m', '5m'] }, { key: 'gatewayQueue', label: 'Gateway 最旧 pending', unit: '秒', min: 1, max: 3600, step: 1, warning: 30, critical: 120, expr: 'cmpp_gateway_submit_queue_oldest_pending_age_seconds', names: ['CmppGatewayQueueDelayedWarning', 'CmppGatewayQueueDelayedCritical'], service: 'gateway', durations: ['2m', '2m'] }, { key: 'postgresConnections', label: 'PostgreSQL 连接使用率', unit: '%', min: 1, max: 100, step: 1, warning: 70, critical: 85, expr: '100 * sum(pg_stat_activity_count) / clamp_min(max(pg_settings_max_connections), 1)', names: ['PostgresConnectionsWarning', 'PostgresConnectionsCritical'], service: 'postgresql', durations: ['10m', '5m'] }, { key: 'redisMemory', label: 'Redis 内存使用率', unit: '%', min: 1, max: 100, step: 1, warning: 70, critical: 85, expr: '100 * redis_memory_used_bytes / redis_memory_max_bytes', guard: 'redis_memory_max_bytes > 0', names: ['RedisMemoryWarning', 'RedisMemoryCritical'], service: 'redis', durations: ['10m', '5m'] }, { key: 'minioCapacity', label: 'MinIO 容量使用率', unit: '%', min: 1, max: 100, step: 1, warning: 80, critical: 90, expr: '100 * (1 - minio_cluster_capacity_usable_free_bytes / minio_cluster_capacity_usable_total_bytes)', names: ['MinioCapacityWarning', 'MinioCapacityCritical'], service: 'minio', durations: ['15m', '5m'] }, ] as const; export const DEFAULT_ALERT_THRESHOLDS: InfrastructureAlertThresholds = Object.fromEntries( ALERT_THRESHOLD_DEFINITIONS.map((item) => [item.key, { warning: item.warning, critical: item.critical }]), ); @Injectable() export class InfrastructureAlertSettingsService { private readonly logger = new Logger(InfrastructureAlertSettingsService.name); private readonly rulesPath: string; private readonly promtoolPath: string; private readonly reloadUrl: string; constructor(private readonly prisma: PrismaService, config: ConfigService) { this.rulesPath = String(config.get('PROMETHEUS_MANAGED_RULES_PATH') ?? '/var/lib/cmpp-platform/monitoring/cmpp-managed-alerts.yml'); this.promtoolPath = String(config.get('PROMTOOL_PATH') ?? '/usr/bin/promtool'); this.reloadUrl = String(config.get('PROMETHEUS_RELOAD_URL') ?? 'http://127.0.0.1:9090/-/reload'); } async get(): Promise { const row = await this.prisma.infrastructureAlertSetting.findUnique({ where: { id: 'global' } }); const thresholds = this.asThresholds(row?.thresholds) ?? DEFAULT_ALERT_THRESHOLDS; const effective = this.asThresholds(row?.effectiveThresholds) ?? thresholds; return { configVersion: row?.configVersion ?? 1, effectiveVersion: row?.effectiveVersion ?? 1, applyStatus: (row?.applyStatus as InfrastructureAlertSettings['applyStatus']) ?? 'effective', lastError: row?.lastError ?? null, appliedAt: row?.appliedAt?.toISOString() ?? null, thresholds, effectiveThresholds: effective, definitions: ALERT_THRESHOLD_DEFINITIONS.map(({ key, label, unit, min, max, step }) => ({ key, label, unit, min, max, step })), }; } async update(body: { configVersion?: number; thresholds?: unknown }, operatorId?: string) { const expectedVersion = Number(body.configVersion); if (!Number.isInteger(expectedVersion) || expectedVersion < 1) throw new BadRequestException('配置版本无效'); const thresholds = this.validate(body.thresholds); const claimed = await this.prisma.infrastructureAlertSetting.updateMany({ where: { id: 'global', configVersion: expectedVersion }, data: { configVersion: { increment: 1 }, thresholds: thresholds as Prisma.InputJsonValue, applyStatus: 'applying', lastError: null, updatedById: operatorId }, }); // 版本条件更新是跨进程的写锁;避免两个 API 实例同时覆盖规则文件并把旧配置误标成已生效。 if (claimed.count !== 1) throw new ConflictException('告警阈值已被其他管理员修改,请刷新后重试'); const nextVersion = expectedVersion + 1; try { await this.applyRules(thresholds); await this.prisma.$transaction([ this.prisma.infrastructureAlertSetting.update({ where: { id: 'global' }, data: { effectiveVersion: nextVersion, effectiveThresholds: thresholds as Prisma.InputJsonValue, applyStatus: 'effective', lastError: null, appliedAt: new Date() } }), this.prisma.operationLog.create({ data: { userId: operatorId, action: 'monitoring.alert_thresholds_updated', resource: 'infrastructure_alert_setting', resourceId: 'global', detail: { configVersion: nextVersion, thresholds } } }), ]); } catch (error) { const message = error instanceof Error ? error.message.slice(0, 500) : 'unknown error'; await this.prisma.infrastructureAlertSetting.update({ where: { id: 'global' }, data: { applyStatus: 'failed', lastError: message } }); this.logger.error(`Prometheus managed rules apply failed: ${message}`); throw new ServiceUnavailableException('阈值已保存但 Prometheus 应用失败,原生效规则已保留'); } return this.get(); } private validate(value: unknown): InfrastructureAlertThresholds { if (!value || typeof value !== 'object' || Array.isArray(value)) throw new BadRequestException('告警阈值格式无效'); const input = value as Record; if (Object.keys(input).some((key) => !ALERT_THRESHOLD_DEFINITIONS.some((item) => item.key === key))) throw new BadRequestException('存在不允许配置的告警指标'); const result: InfrastructureAlertThresholds = {}; for (const definition of ALERT_THRESHOLD_DEFINITIONS) { const pair = input[definition.key] as { warning?: unknown; critical?: unknown } | undefined; const warning = Number(pair?.warning); const critical = Number(pair?.critical); if (!Number.isFinite(warning) || !Number.isFinite(critical) || warning < definition.min || critical > definition.max || warning >= critical) { throw new BadRequestException(`${definition.label}必须满足最小值 ≤ 警告阈值 < 严重阈值 ≤ 最大值`); } result[definition.key] = { warning, critical }; } return result; } private asThresholds(value: unknown) { try { return this.validate(value); } catch { return null; } } private renderRules(thresholds: InfrastructureAlertThresholds) { const lines = ['groups:', ' - name: cmpp-managed-thresholds', ' rules:']; for (const definition of ALERT_THRESHOLD_DEFINITIONS) { const pair = thresholds[definition.key]; const values = [pair.warning, pair.critical]; for (let index = 0; index < 2; index += 1) { const isWarning = index === 0; // 低样本量与“未配置容量上限”必须继续作为固定保护条件,避免单次错误或除零结果触发伪告警。 const guard = 'guard' in definition ? ` and (${definition.guard})` : ''; const expr = isWarning ? `(${definition.expr} > ${values[0]}) and (${definition.expr} <= ${values[1]})${guard}` : `(${definition.expr} > ${values[1]})${guard}`; const diskLocation = definition.key === 'hostDisk' ? ' 设备:{{ $labels.device }};文件系统:{{ $labels.fstype }}(绑定挂载已合并)。' : ''; lines.push(` - alert: ${definition.names[index]}`, ` expr: ${expr}`, ` for: ${definition.durations[index]}`, ' labels:', ` severity: ${isWarning ? 'warning' : 'critical'}`, ` service: ${definition.service}`, ' annotations:', ` summary: "${definition.label}${isWarning ? '达到警告阈值' : '达到严重阈值'}"`, ` description: "${definition.label}持续超过${values[index]}${definition.unit}。${diskLocation}"`, ' currentValue: "{{ $value }}"', ` threshold: "${values[index]}${definition.unit}"`); } } return `${lines.join('\n')}\n`; } private async applyRules(thresholds: InfrastructureAlertThresholds) { const directory = dirname(this.rulesPath); const temporary = `${this.rulesPath}.${process.pid}.${Date.now()}.tmp`; await mkdir(directory, { recursive: true }); const previous = await readFile(this.rulesPath).catch(() => null); try { await writeFile(temporary, this.renderRules(thresholds), { mode: 0o640 }); await execFileAsync(this.promtoolPath, ['check', 'rules', temporary], { timeout: 10_000 }); await rename(temporary, this.rulesPath); const response = await fetch(this.reloadUrl, { method: 'POST', signal: AbortSignal.timeout(5_000) }); if (!response.ok) throw new Error(`Prometheus reload HTTP ${response.status}`); } catch (error) { await rm(temporary, { force: true }); // 规则替换和 reload 不是一个事务,失败时必须恢复旧文件并再次 reload,避免数据库状态与实际告警漂移。 if (previous) { await writeFile(temporary, previous, { mode: 0o640 }); await rename(temporary, this.rulesPath); await fetch(this.reloadUrl, { method: 'POST', signal: AbortSignal.timeout(5_000) }).catch(() => undefined); } throw error; } } }