368 lines
13 KiB
TypeScript
368 lines
13 KiB
TypeScript
import {
|
|
BadRequestException,
|
|
ConflictException,
|
|
Injectable,
|
|
Logger,
|
|
ServiceUnavailableException,
|
|
} from '@nestjs/common';
|
|
import { ConfigService } from '@nestjs/config';
|
|
import { Prisma } from '@prisma/client';
|
|
import { execFile } from 'node:child_process';
|
|
import { access, mkdir, readFile, rename, rm, writeFile } from 'node:fs/promises';
|
|
import { constants } from 'node:fs';
|
|
import { dirname } from 'node:path';
|
|
import { promisify } from 'node:util';
|
|
import { PrismaService } from '../prisma/prisma.service';
|
|
import { FILESYSTEM_USAGE_PERCENT } from './filesystem-metrics';
|
|
import type { InfrastructureAlertSettings, InfrastructureAlertThresholds } from './infrastructure-monitoring.contracts';
|
|
|
|
const execFileAsync = promisify(execFile);
|
|
|
|
export const ALERT_THRESHOLD_DEFINITIONS = [
|
|
{
|
|
key: 'hostCpu',
|
|
label: '主机 CPU 使用率',
|
|
unit: '%',
|
|
min: 1,
|
|
max: 100,
|
|
step: 1,
|
|
warning: 80,
|
|
critical: 90,
|
|
expr: '100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100)',
|
|
names: ['HostCpuUsageWarning', 'HostCpuUsageCritical'],
|
|
service: 'host',
|
|
durations: ['10m', '5m'],
|
|
},
|
|
{
|
|
key: 'hostMemory',
|
|
label: '主机内存使用率',
|
|
unit: '%',
|
|
min: 1,
|
|
max: 100,
|
|
step: 1,
|
|
warning: 85,
|
|
critical: 95,
|
|
expr: '(1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100',
|
|
names: ['HostMemoryUsageWarning', 'HostMemoryUsageCritical'],
|
|
service: 'host',
|
|
durations: ['10m', '5m'],
|
|
},
|
|
{
|
|
key: 'hostDisk',
|
|
label: '磁盘（独立文件系统）使用率',
|
|
unit: '%',
|
|
min: 1,
|
|
max: 100,
|
|
step: 1,
|
|
warning: 80,
|
|
critical: 90,
|
|
expr: FILESYSTEM_USAGE_PERCENT,
|
|
names: ['HostRootDiskUsageWarning', 'HostRootDiskUsageCritical'],
|
|
service: 'host',
|
|
durations: ['15m', '5m'],
|
|
},
|
|
{
|
|
key: 'apiError',
|
|
label: 'API 5xx 错误率',
|
|
unit: '%',
|
|
min: 0.1,
|
|
max: 100,
|
|
step: 0.1,
|
|
warning: 1,
|
|
critical: 5,
|
|
expr: '100 * sum(rate(cmpp_api_http_requests_total{status=~"5.."}[5m])) / clamp_min(sum(rate(cmpp_api_http_requests_total[5m])), 0.001)',
|
|
guard: 'sum(increase(cmpp_api_http_requests_total{status=~"5.."}[5m])) >= 5',
|
|
names: ['CmppApiHttpErrorRateWarning', 'CmppApiHttpErrorRateCritical'],
|
|
service: 'api',
|
|
durations: ['5m', '5m'],
|
|
},
|
|
{
|
|
key: 'apiLatency',
|
|
label: 'API P95 响应时间',
|
|
unit: '秒',
|
|
min: 0.1,
|
|
max: 60,
|
|
step: 0.1,
|
|
warning: 1,
|
|
critical: 3,
|
|
expr: 'histogram_quantile(0.95, sum by (le) (rate(cmpp_api_http_request_duration_seconds_bucket[10m])))',
|
|
names: ['CmppApiLatencyWarning', 'CmppApiLatencyCritical'],
|
|
service: 'api',
|
|
durations: ['10m', '5m'],
|
|
},
|
|
{
|
|
key: 'apiEventLoop',
|
|
label: 'API 事件循环 P99',
|
|
unit: '秒',
|
|
min: 0.01,
|
|
max: 10,
|
|
step: 0.01,
|
|
warning: 0.2,
|
|
critical: 1,
|
|
expr: 'cmpp_api_nodejs_event_loop_lag_p99_seconds',
|
|
names: ['CmppApiEventLoopLagWarning', 'CmppApiEventLoopLagCritical'],
|
|
service: 'api',
|
|
durations: ['10m', '5m'],
|
|
},
|
|
{
|
|
key: 'gatewayQueue',
|
|
label: 'Gateway 最旧 pending',
|
|
unit: '秒',
|
|
min: 1,
|
|
max: 3600,
|
|
step: 1,
|
|
warning: 30,
|
|
critical: 120,
|
|
expr: 'cmpp_gateway_submit_queue_oldest_pending_age_seconds',
|
|
names: ['CmppGatewayQueueDelayedWarning', 'CmppGatewayQueueDelayedCritical'],
|
|
service: 'gateway',
|
|
durations: ['2m', '2m'],
|
|
},
|
|
{
|
|
key: 'postgresConnections',
|
|
label: 'PostgreSQL 连接使用率',
|
|
unit: '%',
|
|
min: 1,
|
|
max: 100,
|
|
step: 1,
|
|
warning: 70,
|
|
critical: 85,
|
|
expr: '100 * sum(pg_stat_activity_count) / clamp_min(max(pg_settings_max_connections), 1)',
|
|
names: ['PostgresConnectionsWarning', 'PostgresConnectionsCritical'],
|
|
service: 'postgresql',
|
|
durations: ['10m', '5m'],
|
|
},
|
|
{
|
|
key: 'redisMemory',
|
|
label: 'Redis 内存使用率',
|
|
unit: '%',
|
|
min: 1,
|
|
max: 100,
|
|
step: 1,
|
|
warning: 70,
|
|
critical: 85,
|
|
expr: '100 * redis_memory_used_bytes / redis_memory_max_bytes',
|
|
guard: 'redis_memory_max_bytes > 0',
|
|
names: ['RedisMemoryWarning', 'RedisMemoryCritical'],
|
|
service: 'redis',
|
|
durations: ['10m', '5m'],
|
|
},
|
|
{
|
|
key: 'minioCapacity',
|
|
label: 'MinIO 容量使用率',
|
|
unit: '%',
|
|
min: 1,
|
|
max: 100,
|
|
step: 1,
|
|
warning: 80,
|
|
critical: 90,
|
|
expr: '100 * (1 - minio_cluster_capacity_usable_free_bytes / minio_cluster_capacity_usable_total_bytes)',
|
|
names: ['MinioCapacityWarning', 'MinioCapacityCritical'],
|
|
service: 'minio',
|
|
durations: ['15m', '5m'],
|
|
},
|
|
] as const;
|
|
|
|
export const DEFAULT_ALERT_THRESHOLDS: InfrastructureAlertThresholds = Object.fromEntries(
|
|
ALERT_THRESHOLD_DEFINITIONS.map((item) => [item.key, { warning: item.warning, critical: item.critical }]),
|
|
);
|
|
|
|
@Injectable()
|
|
export class InfrastructureAlertSettingsService {
|
|
private readonly logger = new Logger(InfrastructureAlertSettingsService.name);
|
|
private readonly rulesPath: string;
|
|
private readonly promtoolPath: string | undefined;
|
|
private readonly reloadUrl: string;
|
|
|
|
constructor(
|
|
private readonly prisma: PrismaService,
|
|
config: ConfigService,
|
|
) {
|
|
this.rulesPath = String(
|
|
config.get('PROMETHEUS_MANAGED_RULES_PATH') ?? '/var/lib/cmpp-platform/monitoring/cmpp-managed-alerts.yml',
|
|
);
|
|
this.promtoolPath = config.get<string>('PROMTOOL_PATH');
|
|
this.reloadUrl = String(config.get('PROMETHEUS_RELOAD_URL') ?? 'http://127.0.0.1:9090/-/reload');
|
|
}
|
|
|
|
async get(): Promise<InfrastructureAlertSettings> {
|
|
const row = await this.prisma.infrastructureAlertSetting.findUnique({ where: { id: 'global' } });
|
|
const thresholds = this.asThresholds(row?.thresholds) ?? DEFAULT_ALERT_THRESHOLDS;
|
|
const effective = this.asThresholds(row?.effectiveThresholds) ?? thresholds;
|
|
return {
|
|
configVersion: row?.configVersion ?? 1,
|
|
effectiveVersion: row?.effectiveVersion ?? 1,
|
|
applyStatus: (row?.applyStatus as InfrastructureAlertSettings['applyStatus']) ?? 'effective',
|
|
lastError: row?.lastError ?? null,
|
|
appliedAt: row?.appliedAt?.toISOString() ?? null,
|
|
thresholds,
|
|
effectiveThresholds: effective,
|
|
definitions: ALERT_THRESHOLD_DEFINITIONS.map(({ key, label, unit, min, max, step }) => ({
|
|
key,
|
|
label,
|
|
unit,
|
|
min,
|
|
max,
|
|
step,
|
|
})),
|
|
};
|
|
}
|
|
|
|
async update(body: { configVersion?: number; thresholds?: unknown }, operatorId?: string) {
|
|
const expectedVersion = Number(body.configVersion);
|
|
if (!Number.isInteger(expectedVersion) || expectedVersion < 1) throw new BadRequestException('配置版本无效');
|
|
const thresholds = this.validate(body.thresholds);
|
|
const claimed = await this.prisma.infrastructureAlertSetting.updateMany({
|
|
where: { id: 'global', configVersion: expectedVersion },
|
|
data: {
|
|
configVersion: { increment: 1 },
|
|
thresholds: thresholds as Prisma.InputJsonValue,
|
|
applyStatus: 'applying',
|
|
lastError: null,
|
|
updatedById: operatorId,
|
|
},
|
|
});
|
|
// 版本条件更新是跨进程的写锁；避免两个 API 实例同时覆盖规则文件并把旧配置误标成已生效。
|
|
if (claimed.count !== 1) throw new ConflictException('告警阈值已被其他管理员修改，请刷新后重试');
|
|
const nextVersion = expectedVersion + 1;
|
|
try {
|
|
await this.applyRules(thresholds);
|
|
await this.prisma.$transaction([
|
|
this.prisma.infrastructureAlertSetting.update({
|
|
where: { id: 'global' },
|
|
data: {
|
|
effectiveVersion: nextVersion,
|
|
effectiveThresholds: thresholds as Prisma.InputJsonValue,
|
|
applyStatus: 'effective',
|
|
lastError: null,
|
|
appliedAt: new Date(),
|
|
},
|
|
}),
|
|
this.prisma.operationLog.create({
|
|
data: {
|
|
userId: operatorId,
|
|
action: 'monitoring.alert_thresholds_updated',
|
|
resource: 'infrastructure_alert_setting',
|
|
resourceId: 'global',
|
|
detail: { configVersion: nextVersion, thresholds },
|
|
},
|
|
}),
|
|
]);
|
|
} catch (error) {
|
|
const message = error instanceof Error ? error.message.slice(0, 500) : 'unknown error';
|
|
await this.prisma.infrastructureAlertSetting.update({
|
|
where: { id: 'global' },
|
|
data: { applyStatus: 'failed', lastError: message },
|
|
});
|
|
this.logger.error(`Prometheus managed rules apply failed: ${message}`);
|
|
throw new ServiceUnavailableException('阈值已保存但 Prometheus 应用失败，原生效规则已保留');
|
|
}
|
|
return this.get();
|
|
}
|
|
|
|
private validate(value: unknown): InfrastructureAlertThresholds {
|
|
if (!value || typeof value !== 'object' || Array.isArray(value)) throw new BadRequestException('告警阈值格式无效');
|
|
const input = value as Record<string, unknown>;
|
|
if (Object.keys(input).some((key) => !ALERT_THRESHOLD_DEFINITIONS.some((item) => item.key === key)))
|
|
throw new BadRequestException('存在不允许配置的告警指标');
|
|
const result: InfrastructureAlertThresholds = {};
|
|
for (const definition of ALERT_THRESHOLD_DEFINITIONS) {
|
|
const pair = input[definition.key] as { warning?: unknown; critical?: unknown } | undefined;
|
|
const warning = Number(pair?.warning);
|
|
const critical = Number(pair?.critical);
|
|
if (
|
|
!Number.isFinite(warning) ||
|
|
!Number.isFinite(critical) ||
|
|
warning < definition.min ||
|
|
critical > definition.max ||
|
|
warning >= critical
|
|
) {
|
|
throw new BadRequestException(`${definition.label}必须满足最小值 ≤ 警告阈值 < 严重阈值 ≤ 最大值`);
|
|
}
|
|
result[definition.key] = { warning, critical };
|
|
}
|
|
return result;
|
|
}
|
|
|
|
private asThresholds(value: unknown) {
|
|
try {
|
|
return this.validate(value);
|
|
} catch {
|
|
return null;
|
|
}
|
|
}
|
|
|
|
private renderRules(thresholds: InfrastructureAlertThresholds) {
|
|
const lines = ['groups:', ' - name: cmpp-managed-thresholds', ' rules:'];
|
|
for (const definition of ALERT_THRESHOLD_DEFINITIONS) {
|
|
const pair = thresholds[definition.key];
|
|
const values = [pair.warning, pair.critical];
|
|
for (let index = 0; index < 2; index += 1) {
|
|
const isWarning = index === 0;
|
|
// 低样本量与“未配置容量上限”必须继续作为固定保护条件，避免单次错误或除零结果触发伪告警。
|
|
const guard = 'guard' in definition ? ` and (${definition.guard})` : '';
|
|
const expr = isWarning
|
|
? `(${definition.expr} > ${values[0]}) and (${definition.expr} <= ${values[1]})${guard}`
|
|
: `(${definition.expr} > ${values[1]})${guard}`;
|
|
const diskLocation =
|
|
definition.key === 'hostDisk'
|
|
? ' 设备：{{ $labels.device }}；文件系统：{{ $labels.fstype }}（绑定挂载已合并）。'
|
|
: '';
|
|
lines.push(
|
|
` - alert: ${definition.names[index]}`,
|
|
` expr: ${expr}`,
|
|
` for: ${definition.durations[index]}`,
|
|
' labels:',
|
|
` severity: ${isWarning ? 'warning' : 'critical'}`,
|
|
` service: ${definition.service}`,
|
|
' annotations:',
|
|
` summary: "${definition.label}${isWarning ? '达到警告阈值' : '达到严重阈值'}"`,
|
|
` description: "${definition.label}持续超过${values[index]}${definition.unit}。${diskLocation}"`,
|
|
' currentValue: "{{ $value }}"',
|
|
` threshold: "${values[index]}${definition.unit}"`,
|
|
);
|
|
}
|
|
}
|
|
return `${lines.join('\n')}\n`;
|
|
}
|
|
|
|
private async applyRules(thresholds: InfrastructureAlertThresholds) {
|
|
const directory = dirname(this.rulesPath);
|
|
const temporary = `${this.rulesPath}.${process.pid}.${Date.now()}.tmp`;
|
|
await mkdir(directory, { recursive: true });
|
|
const previous = await readFile(this.rulesPath).catch(() => null);
|
|
try {
|
|
await writeFile(temporary, this.renderRules(thresholds), { mode: 0o640 });
|
|
await execFileAsync(await resolvePromtoolPath(this.promtoolPath), ['check', 'rules', temporary], {
|
|
timeout: 10_000,
|
|
});
|
|
await rename(temporary, this.rulesPath);
|
|
const response = await fetch(this.reloadUrl, { method: 'POST', signal: AbortSignal.timeout(5_000) });
|
|
if (!response.ok) throw new Error(`Prometheus reload HTTP ${response.status}`);
|
|
} catch (error) {
|
|
await rm(temporary, { force: true });
|
|
// 规则替换和 reload 不是一个事务，失败时必须恢复旧文件并再次 reload，避免数据库状态与实际告警漂移。
|
|
if (previous) {
|
|
await writeFile(temporary, previous, { mode: 0o640 });
|
|
await rename(temporary, this.rulesPath);
|
|
await fetch(this.reloadUrl, { method: 'POST', signal: AbortSignal.timeout(5_000) }).catch(() => undefined);
|
|
}
|
|
throw error;
|
|
}
|
|
}
|
|
}
|
|
|
|
// Explicit configuration is authoritative; never silently replace a broken configured binary.
|
|
export async function resolvePromtoolPath(configured?: string) {
|
|
const candidates = configured ? [configured] : ['/usr/local/bin/promtool', '/usr/bin/promtool'];
|
|
for (const candidate of candidates) {
|
|
try {
|
|
await access(candidate, constants.X_OK);
|
|
return candidate;
|
|
} catch {
|
|
/* Try the next standard install location. */
|
|
}
|
|
}
|
|
throw new Error('promtool不可执行，请检查PROMTOOL_PATH或标准安装目录');
|
|
}
|