hectorzhao/lislgosms
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: bound formatter memory and improve operations workflows

Browse Source
This commit is contained in:
hectorzhao
2026-09-08 12:46:15 +08:00
parent 50ae37242b
commit 2c228a94e1
27 changed files with 4013 additions and 1230 deletions
Download Patch File Download Diff File Expand all files Collapse all files
api/src/infrastructure-monitoring/infrastructure-alert-settings.service.ts
+249 -28
View File
@@ -1,8 +1,15 @@
import { BadRequestException, ConflictException, Injectable, Logger, ServiceUnavailableException } from '@nestjs/common';
import {
BadRequestException,
ConflictException,
Injectable,
Logger,
ServiceUnavailableException,
} from '@nestjs/common';
import { ConfigService } from '@nestjs/config';
import { Prisma } from '@prisma/client';
import { execFile } from 'node:child_process';
import { mkdir, readFile, rename, rm, writeFile } from 'node:fs/promises';
import { access, mkdir, readFile, rename, rm, writeFile } from 'node:fs/promises';
import { constants } from 'node:fs';
import { dirname } from 'node:path';
import { promisify } from 'node:util';
import { PrismaService } from '../prisma/prisma.service';
@@ -12,16 +19,148 @@ import type { InfrastructureAlertSettings, InfrastructureAlertThresholds } from
const execFileAsync = promisify(execFile);
export const ALERT_THRESHOLD_DEFINITIONS = [
{ key: 'hostCpu', label: '主机 CPU 使用率', unit: '%', min: 1, max: 100, step: 1, warning: 80, critical: 90, expr: '100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100)', names: ['HostCpuUsageWarning', 'HostCpuUsageCritical'], service: 'host', durations: ['10m', '5m'] },
{ key: 'hostMemory', label: '主机内存使用率', unit: '%', min: 1, max: 100, step: 1, warning: 85, critical: 95, expr: '(1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100', names: ['HostMemoryUsageWarning', 'HostMemoryUsageCritical'], service: 'host', durations: ['10m', '5m'] },
{ key: 'hostDisk', label: '磁盘（独立文件系统）使用率', unit: '%', min: 1, max: 100, step: 1, warning: 80, critical: 90, expr: FILESYSTEM_USAGE_PERCENT, names: ['HostRootDiskUsageWarning', 'HostRootDiskUsageCritical'], service: 'host', durations: ['15m', '5m'] },
{ key: 'apiError', label: 'API 5xx 错误率', unit: '%', min: 0.1, max: 100, step: 0.1, warning: 1, critical: 5, expr: '100 * sum(rate(cmpp_api_http_requests_total{status=~"5.."}[5m])) / clamp_min(sum(rate(cmpp_api_http_requests_total[5m])), 0.001)', guard: 'sum(increase(cmpp_api_http_requests_total{status=~"5.."}[5m])) >= 5', names: ['CmppApiHttpErrorRateWarning', 'CmppApiHttpErrorRateCritical'], service: 'api', durations: ['5m', '5m'] },
{ key: 'apiLatency', label: 'API P95 响应时间', unit: '秒', min: 0.1, max: 60, step: 0.1, warning: 1, critical: 3, expr: 'histogram_quantile(0.95, sum by (le) (rate(cmpp_api_http_request_duration_seconds_bucket[10m])))', names: ['CmppApiLatencyWarning', 'CmppApiLatencyCritical'], service: 'api', durations: ['10m', '5m'] },
{ key: 'apiEventLoop', label: 'API 事件循环 P99', unit: '秒', min: 0.01, max: 10, step: 0.01, warning: 0.2, critical: 1, expr: 'cmpp_api_nodejs_event_loop_lag_p99_seconds', names: ['CmppApiEventLoopLagWarning', 'CmppApiEventLoopLagCritical'], service: 'api', durations: ['10m', '5m'] },
{ key: 'gatewayQueue', label: 'Gateway 最旧 pending', unit: '秒', min: 1, max: 3600, step: 1, warning: 30, critical: 120, expr: 'cmpp_gateway_submit_queue_oldest_pending_age_seconds', names: ['CmppGatewayQueueDelayedWarning', 'CmppGatewayQueueDelayedCritical'], service: 'gateway', durations: ['2m', '2m'] },
{ key: 'postgresConnections', label: 'PostgreSQL 连接使用率', unit: '%', min: 1, max: 100, step: 1, warning: 70, critical: 85, expr: '100 * sum(pg_stat_activity_count) / clamp_min(max(pg_settings_max_connections), 1)', names: ['PostgresConnectionsWarning', 'PostgresConnectionsCritical'], service: 'postgresql', durations: ['10m', '5m'] },
{ key: 'redisMemory', label: 'Redis 内存使用率', unit: '%', min: 1, max: 100, step: 1, warning: 70, critical: 85, expr: '100 * redis_memory_used_bytes / redis_memory_max_bytes', guard: 'redis_memory_max_bytes > 0', names: ['RedisMemoryWarning', 'RedisMemoryCritical'], service: 'redis', durations: ['10m', '5m'] },
{ key: 'minioCapacity', label: 'MinIO 容量使用率', unit: '%', min: 1, max: 100, step: 1, warning: 80, critical: 90, expr: '100 * (1 - minio_cluster_capacity_usable_free_bytes / minio_cluster_capacity_usable_total_bytes)', names: ['MinioCapacityWarning', 'MinioCapacityCritical'], service: 'minio', durations: ['15m', '5m'] },
{
key: 'hostCpu',
label: '主机 CPU 使用率',
unit: '%',
min: 1,
max: 100,
step: 1,
warning: 80,
critical: 90,
expr: '100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100)',
names: ['HostCpuUsageWarning', 'HostCpuUsageCritical'],
service: 'host',
durations: ['10m', '5m'],
},
{
key: 'hostMemory',
label: '主机内存使用率',
unit: '%',
min: 1,
max: 100,
step: 1,
warning: 85,
critical: 95,
expr: '(1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100',
names: ['HostMemoryUsageWarning', 'HostMemoryUsageCritical'],
service: 'host',
durations: ['10m', '5m'],
},
{
key: 'hostDisk',
label: '磁盘（独立文件系统）使用率',
unit: '%',
min: 1,
max: 100,
step: 1,
warning: 80,
critical: 90,
expr: FILESYSTEM_USAGE_PERCENT,
names: ['HostRootDiskUsageWarning', 'HostRootDiskUsageCritical'],
service: 'host',
durations: ['15m', '5m'],
},
{
key: 'apiError',
label: 'API 5xx 错误率',
unit: '%',
min: 0.1,
max: 100,
step: 0.1,
warning: 1,
critical: 5,
expr: '100 * sum(rate(cmpp_api_http_requests_total{status=~"5.."}[5m])) / clamp_min(sum(rate(cmpp_api_http_requests_total[5m])), 0.001)',
guard: 'sum(increase(cmpp_api_http_requests_total{status=~"5.."}[5m])) >= 5',
names: ['CmppApiHttpErrorRateWarning', 'CmppApiHttpErrorRateCritical'],
service: 'api',
durations: ['5m', '5m'],
},
{
key: 'apiLatency',
label: 'API P95 响应时间',
unit: '秒',
min: 0.1,
max: 60,
step: 0.1,
warning: 1,
critical: 3,
expr: 'histogram_quantile(0.95, sum by (le) (rate(cmpp_api_http_request_duration_seconds_bucket[10m])))',
names: ['CmppApiLatencyWarning', 'CmppApiLatencyCritical'],
service: 'api',
durations: ['10m', '5m'],
},
{
key: 'apiEventLoop',
label: 'API 事件循环 P99',
unit: '秒',
min: 0.01,
max: 10,
step: 0.01,
warning: 0.2,
critical: 1,
expr: 'cmpp_api_nodejs_event_loop_lag_p99_seconds',
names: ['CmppApiEventLoopLagWarning', 'CmppApiEventLoopLagCritical'],
service: 'api',
durations: ['10m', '5m'],
},
{
key: 'gatewayQueue',
label: 'Gateway 最旧 pending',
unit: '秒',
min: 1,
max: 3600,
step: 1,
warning: 30,
critical: 120,
expr: 'cmpp_gateway_submit_queue_oldest_pending_age_seconds',
names: ['CmppGatewayQueueDelayedWarning', 'CmppGatewayQueueDelayedCritical'],
service: 'gateway',
durations: ['2m', '2m'],
},
{
key: 'postgresConnections',
label: 'PostgreSQL 连接使用率',
unit: '%',
min: 1,
max: 100,
step: 1,
warning: 70,
critical: 85,
expr: '100 * sum(pg_stat_activity_count) / clamp_min(max(pg_settings_max_connections), 1)',
names: ['PostgresConnectionsWarning', 'PostgresConnectionsCritical'],
service: 'postgresql',
durations: ['10m', '5m'],
},
{
key: 'redisMemory',
label: 'Redis 内存使用率',
unit: '%',
min: 1,
max: 100,
step: 1,
warning: 70,
critical: 85,
expr: '100 * redis_memory_used_bytes / redis_memory_max_bytes',
guard: 'redis_memory_max_bytes > 0',
names: ['RedisMemoryWarning', 'RedisMemoryCritical'],
service: 'redis',
durations: ['10m', '5m'],
},
{
key: 'minioCapacity',
label: 'MinIO 容量使用率',
unit: '%',
min: 1,
max: 100,
step: 1,
warning: 80,
critical: 90,
expr: '100 * (1 - minio_cluster_capacity_usable_free_bytes / minio_cluster_capacity_usable_total_bytes)',
names: ['MinioCapacityWarning', 'MinioCapacityCritical'],
service: 'minio',
durations: ['15m', '5m'],
},
] as const;
export const DEFAULT_ALERT_THRESHOLDS: InfrastructureAlertThresholds = Object.fromEntries(
@@ -32,12 +171,17 @@ export const DEFAULT_ALERT_THRESHOLDS: InfrastructureAlertThresholds = Object.fr
export class InfrastructureAlertSettingsService {
private readonly logger = new Logger(InfrastructureAlertSettingsService.name);
private readonly rulesPath: string;
private readonly promtoolPath: string;
private readonly promtoolPath: string | undefined;
private readonly reloadUrl: string;
constructor(private readonly prisma: PrismaService, config: ConfigService) {
this.rulesPath = String(config.get('PROMETHEUS_MANAGED_RULES_PATH') ?? '/var/lib/cmpp-platform/monitoring/cmpp-managed-alerts.yml');
this.promtoolPath = String(config.get('PROMTOOL_PATH') ?? '/usr/bin/promtool');
constructor(
private readonly prisma: PrismaService,
config: ConfigService,
) {
this.rulesPath = String(
config.get('PROMETHEUS_MANAGED_RULES_PATH') ?? '/var/lib/cmpp-platform/monitoring/cmpp-managed-alerts.yml',
);
this.promtoolPath = config.get<string>('PROMTOOL_PATH');
this.reloadUrl = String(config.get('PROMETHEUS_RELOAD_URL') ?? 'http://127.0.0.1:9090/-/reload');
}
@@ -53,7 +197,14 @@ export class InfrastructureAlertSettingsService {
appliedAt: row?.appliedAt?.toISOString() ?? null,
thresholds,
effectiveThresholds: effective,
definitions: ALERT_THRESHOLD_DEFINITIONS.map(({ key, label, unit, min, max, step }) => ({ key, label, unit, min, max, step })),
definitions: ALERT_THRESHOLD_DEFINITIONS.map(({ key, label, unit, min, max, step }) => ({
key,
label,
unit,
min,
max,
step,
})),
};
}
@@ -63,7 +214,13 @@ export class InfrastructureAlertSettingsService {
const thresholds = this.validate(body.thresholds);
const claimed = await this.prisma.infrastructureAlertSetting.updateMany({
where: { id: 'global', configVersion: expectedVersion },
data: { configVersion: { increment: 1 }, thresholds: thresholds as Prisma.InputJsonValue, applyStatus: 'applying', lastError: null, updatedById: operatorId },
data: {
configVersion: { increment: 1 },
thresholds: thresholds as Prisma.InputJsonValue,
applyStatus: 'applying',
lastError: null,
updatedById: operatorId,
},
});
// 版本条件更新是跨进程的写锁；避免两个 API 实例同时覆盖规则文件并把旧配置误标成已生效。
if (claimed.count !== 1) throw new ConflictException('告警阈值已被其他管理员修改，请刷新后重试');
@@ -71,12 +228,32 @@ export class InfrastructureAlertSettingsService {
try {
await this.applyRules(thresholds);
await this.prisma.$transaction([
this.prisma.infrastructureAlertSetting.update({ where: { id: 'global' }, data: { effectiveVersion: nextVersion, effectiveThresholds: thresholds as Prisma.InputJsonValue, applyStatus: 'effective', lastError: null, appliedAt: new Date() } }),
this.prisma.operationLog.create({ data: { userId: operatorId, action: 'monitoring.alert_thresholds_updated', resource: 'infrastructure_alert_setting', resourceId: 'global', detail: { configVersion: nextVersion, thresholds } } }),
this.prisma.infrastructureAlertSetting.update({
where: { id: 'global' },
data: {
effectiveVersion: nextVersion,
effectiveThresholds: thresholds as Prisma.InputJsonValue,
applyStatus: 'effective',
lastError: null,
appliedAt: new Date(),
},
}),
this.prisma.operationLog.create({
data: {
userId: operatorId,
action: 'monitoring.alert_thresholds_updated',
resource: 'infrastructure_alert_setting',
resourceId: 'global',
detail: { configVersion: nextVersion, thresholds },
},
}),
]);
} catch (error) {
const message = error instanceof Error ? error.message.slice(0, 500) : 'unknown error';
await this.prisma.infrastructureAlertSetting.update({ where: { id: 'global' }, data: { applyStatus: 'failed', lastError: message } });
await this.prisma.infrastructureAlertSetting.update({
where: { id: 'global' },
data: { applyStatus: 'failed', lastError: message },
});
this.logger.error(`Prometheus managed rules apply failed: ${message}`);
throw new ServiceUnavailableException('阈值已保存但 Prometheus 应用失败，原生效规则已保留');
}
@@ -86,13 +263,20 @@ export class InfrastructureAlertSettingsService {
private validate(value: unknown): InfrastructureAlertThresholds {
if (!value || typeof value !== 'object' || Array.isArray(value)) throw new BadRequestException('告警阈值格式无效');
const input = value as Record<string, unknown>;
if (Object.keys(input).some((key) => !ALERT_THRESHOLD_DEFINITIONS.some((item) => item.key === key))) throw new BadRequestException('存在不允许配置的告警指标');
if (Object.keys(input).some((key) => !ALERT_THRESHOLD_DEFINITIONS.some((item) => item.key === key)))
throw new BadRequestException('存在不允许配置的告警指标');
const result: InfrastructureAlertThresholds = {};
for (const definition of ALERT_THRESHOLD_DEFINITIONS) {
const pair = input[definition.key] as { warning?: unknown; critical?: unknown } | undefined;
const warning = Number(pair?.warning);
const critical = Number(pair?.critical);
if (!Number.isFinite(warning) || !Number.isFinite(critical) || warning < definition.min || critical > definition.max || warning >= critical) {
if (
!Number.isFinite(warning) ||
!Number.isFinite(critical) ||
warning < definition.min ||
critical > definition.max ||
warning >= critical
) {
throw new BadRequestException(`${definition.label}必须满足最小值 ≤ 警告阈值 < 严重阈值 ≤ 最大值`);
}
result[definition.key] = { warning, critical };
@@ -101,7 +285,11 @@ export class InfrastructureAlertSettingsService {
}
private asThresholds(value: unknown) {
try { return this.validate(value); } catch { return null; }
try {
return this.validate(value);
} catch {
return null;
}
}
private renderRules(thresholds: InfrastructureAlertThresholds) {
@@ -113,9 +301,26 @@ export class InfrastructureAlertSettingsService {
const isWarning = index === 0;
// 低样本量与“未配置容量上限”必须继续作为固定保护条件，避免单次错误或除零结果触发伪告警。
const guard = 'guard' in definition ? ` and (${definition.guard})` : '';
const expr = isWarning ? `(${definition.expr} > ${values[0]}) and (${definition.expr} <= ${values[1]})${guard}` : `(${definition.expr} > ${values[1]})${guard}`;
const diskLocation = definition.key === 'hostDisk' ? ' 设备：{{ $labels.device }}；文件系统：{{ $labels.fstype }}（绑定挂载已合并）。' : '';
lines.push(` - alert: ${definition.names[index]}`, ` expr: ${expr}`, ` for: ${definition.durations[index]}`, ' labels:', ` severity: ${isWarning ? 'warning' : 'critical'}`, ` service: ${definition.service}`, ' annotations:', ` summary: "${definition.label}${isWarning ? '达到警告阈值' : '达到严重阈值'}"`, ` description: "${definition.label}持续超过${values[index]}${definition.unit}。${diskLocation}"`, ' currentValue: "{{ $value }}"', ` threshold: "${values[index]}${definition.unit}"`);
const expr = isWarning
? `(${definition.expr} > ${values[0]}) and (${definition.expr} <= ${values[1]})${guard}`
: `(${definition.expr} > ${values[1]})${guard}`;
const diskLocation =
definition.key === 'hostDisk'
? ' 设备：{{ $labels.device }}；文件系统：{{ $labels.fstype }}（绑定挂载已合并）。'
: '';
lines.push(
` - alert: ${definition.names[index]}`,
` expr: ${expr}`,
` for: ${definition.durations[index]}`,
' labels:',
` severity: ${isWarning ? 'warning' : 'critical'}`,
` service: ${definition.service}`,
' annotations:',
` summary: "${definition.label}${isWarning ? '达到警告阈值' : '达到严重阈值'}"`,
` description: "${definition.label}持续超过${values[index]}${definition.unit}。${diskLocation}"`,
' currentValue: "{{ $value }}"',
` threshold: "${values[index]}${definition.unit}"`,
);
}
}
return `${lines.join('\n')}\n`;
@@ -128,7 +333,9 @@ export class InfrastructureAlertSettingsService {
const previous = await readFile(this.rulesPath).catch(() => null);
try {
await writeFile(temporary, this.renderRules(thresholds), { mode: 0o640 });
await execFileAsync(this.promtoolPath, ['check', 'rules', temporary], { timeout: 10_000 });
await execFileAsync(await resolvePromtoolPath(this.promtoolPath), ['check', 'rules', temporary], {
timeout: 10_000,
});
await rename(temporary, this.rulesPath);
const response = await fetch(this.reloadUrl, { method: 'POST', signal: AbortSignal.timeout(5_000) });
if (!response.ok) throw new Error(`Prometheus reload HTTP ${response.status}`);
@@ -144,3 +351,17 @@ export class InfrastructureAlertSettingsService {
}
}
}
// Explicit configuration is authoritative; never silently replace a broken configured binary.
export async function resolvePromtoolPath(configured?: string) {
const candidates = configured ? [configured] : ['/usr/local/bin/promtool', '/usr/bin/promtool'];
for (const candidate of candidates) {
try {
await access(candidate, constants.X_OK);
return candidate;
} catch {
/* Try the next standard install location. */
}
}
throw new Error('promtool不可执行，请检查PROMTOOL_PATH或标准安装目录');
}