hectorzhao/lislgosms
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: correct operational statistics and form interactions

Browse Source
This commit is contained in:
hectorzhao
2026-09-09 23:15:42 +08:00
parent 6d63eb5452
commit 5bcdbb2a03
33 changed files with 2920 additions and 829 deletions
Download Patch File Download Diff File Expand all files Collapse all files
api/src/infrastructure-monitoring/infrastructure-monitoring.service.ts
+255 -83
View File
@@ -1,9 +1,22 @@
import { BadRequestException, Injectable, Logger, NotFoundException, ServiceUnavailableException } from '@nestjs/common';
import {
BadRequestException,
Injectable,
Logger,
NotFoundException,
ServiceUnavailableException,
} from '@nestjs/common';
import { ConfigService } from '@nestjs/config';
import { Prisma } from '@prisma/client';
import { createHash } from 'node:crypto';
import { PrismaService } from '../prisma/prisma.service';
import { compareMountpoints, FILESYSTEM_LABELS, FILESYSTEM_SELECTOR, FILESYSTEM_USAGE_PERCENT, filesystemIdentity } from './filesystem-metrics';
import { alertHistoryRange, mergeAlertHistory, type AlertHistoryItem } from './alert-history';
import {
compareMountpoints,
FILESYSTEM_LABELS,
FILESYSTEM_SELECTOR,
FILESYSTEM_USAGE_PERCENT,
filesystemIdentity,
} from './filesystem-metrics';
import type {
InfrastructureAlert,
InfrastructureMetricPoint,
@@ -57,7 +70,8 @@ const QUERIES = {
uptimeSeconds: 'time() - node_boot_time_seconds',
lastSampleAt: 'max(timestamp(node_uname_info))',
// PromQL字符串本身需要两个反斜杠才能把正则的“\.”传给RE2；TypeScript字面量因此需要写四个。
services: 'max by (name) (node_systemd_unit_state{name=~"cmpp-api\\\\.service|cmpp-gateway\\\\.service|postgresql\\\\.service|redis(-server)?\\\\.service|cmpp-minio\\\\.service|nginx\\\\.service",state="active"})',
services:
'max by (name) (node_systemd_unit_state{name=~"cmpp-api\\\\.service|cmpp-gateway\\\\.service|postgresql\\\\.service|redis(-server)?\\\\.service|cmpp-minio\\\\.service|nginx\\\\.service",state="active"})',
} as const;
const SERVICE_DEFINITIONS = [
@@ -70,38 +84,62 @@ const SERVICE_DEFINITIONS = [
] as const;
const SERVICE_METRIC_DEFINITIONS = [
{ key: 'api', name: 'API服务', metrics: [
['requestsPerSecond', '请求速率', 'cmpp:service_api:requests_per_second', 'per_second'],
['errorPercent', '5xx错误率', 'cmpp:service_api:error_percent', 'percent'],
['latencyP95', 'P95响应', 'cmpp:service_api:latency_p95_seconds', 'seconds'],
['eventLoopP99', '事件循环P99', 'cmpp:service_api:event_loop_p99_seconds', 'seconds'],
] },
{ key: 'gateway', name: 'Gateway服务', metrics: [
['submitsPerSecond', '提交速率', 'cmpp:service_gateway:submits_per_second', 'per_second'],
['failurePercent', '提交失败率', 'cmpp:service_gateway:failure_percent', 'percent'],
['queuePending', 'Stream pending', 'cmpp:service_gateway:queue_pending', 'count'],
['queueOldestSeconds', '最旧pending', 'cmpp:service_gateway:queue_oldest_seconds', 'seconds'],
] },
{ key: 'postgresql', name: 'PostgreSQL', metrics: [
['connectionPercent', '连接使用率', 'cmpp:service_postgresql:connection_percent', 'percent'],
['deadlocks15m', '15分钟死锁', 'cmpp:service_postgresql:deadlocks_15m', 'count'],
] },
{ key: 'redis', name: 'Redis', metrics: [
['memoryPercent', '内存使用率', 'cmpp:service_redis:memory_percent', 'percent'],
['memoryUsedBytes', '已用内存', 'cmpp:service_redis:memory_used_bytes', 'bytes'],
['connectedClients', '客户端连接', 'cmpp:service_redis:connected_clients', 'count'],
['evictions5m', '5分钟淘汰', 'cmpp:service_redis:evictions_5m', 'count'],
] },
{ key: 'minio', name: 'MinIO', metrics: [
['capacityPercent', '存储容量使用率', 'cmpp:service_minio:capacity_percent', 'percent'],
['usageBytes', '对象数据量', 'cmpp:service_minio:usage_bytes', 'bytes'],
['objects', '对象数', 'cmpp:service_minio:objects', 'count'],
['drivesOffline', '离线存储盘', 'cmpp:service_minio:drives_offline', 'count'],
] },
{ key: 'nginx', name: 'Nginx', metrics: [
['connectionsActive', '活跃连接', 'cmpp:service_nginx:connections_active', 'count'],
['requestsPerSecond', '请求速率', 'cmpp:service_nginx:requests_per_second', 'per_second'],
] },
{
key: 'api',
name: 'API服务',
metrics: [
['requestsPerSecond', '请求速率', 'cmpp:service_api:requests_per_second', 'per_second'],
['errorPercent', '5xx错误率', 'cmpp:service_api:error_percent', 'percent'],
['latencyP95', 'P95响应', 'cmpp:service_api:latency_p95_seconds', 'seconds'],
['eventLoopP99', '事件循环P99', 'cmpp:service_api:event_loop_p99_seconds', 'seconds'],
],
},
{
key: 'gateway',
name: 'Gateway服务',
metrics: [
['submitsPerSecond', '提交速率', 'cmpp:service_gateway:submits_per_second', 'per_second'],
['failurePercent', '提交失败率', 'cmpp:service_gateway:failure_percent', 'percent'],
['queuePending', 'Stream pending', 'cmpp:service_gateway:queue_pending', 'count'],
['queueOldestSeconds', '最旧pending', 'cmpp:service_gateway:queue_oldest_seconds', 'seconds'],
],
},
{
key: 'postgresql',
name: 'PostgreSQL',
metrics: [
['connectionPercent', '连接使用率', 'cmpp:service_postgresql:connection_percent', 'percent'],
['deadlocks15m', '15分钟死锁', 'cmpp:service_postgresql:deadlocks_15m', 'count'],
],
},
{
key: 'redis',
name: 'Redis',
metrics: [
['memoryPercent', '内存使用率', 'cmpp:service_redis:memory_percent', 'percent'],
['memoryUsedBytes', '已用内存', 'cmpp:service_redis:memory_used_bytes', 'bytes'],
['connectedClients', '客户端连接', 'cmpp:service_redis:connected_clients', 'count'],
['evictions5m', '5分钟淘汰', 'cmpp:service_redis:evictions_5m', 'count'],
],
},
{
key: 'minio',
name: 'MinIO',
metrics: [
['capacityPercent', '存储容量使用率', 'cmpp:service_minio:capacity_percent', 'percent'],
['usageBytes', '对象数据量', 'cmpp:service_minio:usage_bytes', 'bytes'],
['objects', '对象数', 'cmpp:service_minio:objects', 'count'],
['drivesOffline', '离线存储盘', 'cmpp:service_minio:drives_offline', 'count'],
],
},
{
key: 'nginx',
name: 'Nginx',
metrics: [
['connectionsActive', '活跃连接', 'cmpp:service_nginx:connections_active', 'count'],
['requestsPerSecond', '请求速率', 'cmpp:service_nginx:requests_per_second', 'per_second'],
],
},
] as const;
const SERVICE_METRICS_QUERY = '{__name__=~"cmpp:service_.*"}';
@@ -165,7 +203,10 @@ export class InfrastructureMonitoringService {
private readonly prometheusUrl: string;
private readonly queryTimeoutMs: number;
constructor(config: ConfigService, private readonly prisma: PrismaService) {
constructor(
config: ConfigService,
private readonly prisma: PrismaService,
) {
this.prometheusUrl = normalizePrometheusUrl(config.get('PROMETHEUS_URL'));
this.queryTimeoutMs = Math.min(15_000, Math.max(1_000, Number(config.get('PROMETHEUS_QUERY_TIMEOUT_MS') ?? 5_000)));
}
@@ -202,7 +243,7 @@ export class InfrastructureMonitoringService {
activeAlerts: alerts.length,
},
metrics: instant.metrics,
trends: { ...trends.metrics, diskUsagePercent: rootDisk ? trends.disks.get(rootDisk.id) ?? [] : [] },
trends: { ...trends.metrics, diskUsagePercent: rootDisk ? (trends.disks.get(rootDisk.id) ?? []) : [] },
disks: instant.disks.map((disk) => ({ ...disk, trend: trends.disks.get(disk.id) ?? [] })),
services,
serviceMetrics,
@@ -210,18 +251,28 @@ export class InfrastructureMonitoringService {
};
} catch (error) {
// 页面必须整体清空陈旧指标，但服务端仍需留下不含PromQL/地址/凭据的根因摘要便于运维诊断。
this.logger.warn(`Prometheus monitoring overview unavailable: ${error instanceof Error ? error.message : 'unknown error'}`);
this.logger.warn(
`Prometheus monitoring overview unavailable: ${error instanceof Error ? error.message : 'unknown error'}`,
);
return this.unavailable(range, collectedAt);
}
}
async notificationSummary(userId?: string) {
try {
const alerts = await this.attachReadState(this.parseAlerts(await this.getJson<PrometheusAlertResponse>('/api/v1/alerts')), userId);
const alerts = await this.attachReadState(
this.parseAlerts(await this.getJson<PrometheusAlertResponse>('/api/v1/alerts')),
userId,
);
const unreadAlerts = alerts.filter((item) => !item.acknowledged);
return { count: unreadAlerts.length, criticalCount: unreadAlerts.filter((item) => item.severity === 'critical').length };
return {
count: unreadAlerts.length,
criticalCount: unreadAlerts.filter((item) => item.severity === 'critical').length,
};
} catch (error) {
this.logger.warn(`Prometheus notification summary unavailable: ${error instanceof Error ? error.message : 'unknown error'}`);
this.logger.warn(
`Prometheus notification summary unavailable: ${error instanceof Error ? error.message : 'unknown error'}`,
);
throw new ServiceUnavailableException('Prometheus活动告警当前不可用');
}
}
@@ -231,12 +282,21 @@ export class InfrastructureMonitoringService {
const activeAt = new Date(String(rawActiveAt ?? ''));
if (!Number.isFinite(activeAt.getTime())) throw new BadRequestException('告警开始时间无效');
const activeAlerts = this.parseAlerts(await this.getJson<PrometheusAlertResponse>('/api/v1/alerts'));
const current = activeAlerts.find((item) => item.fingerprint === fingerprint && Date.parse(item.startedAt) === activeAt.getTime());
const current = activeAlerts.find(
(item) => item.fingerprint === fingerprint && Date.parse(item.startedAt) === activeAt.getTime(),
);
if (!current) throw new NotFoundException('该次活动告警已结束或已重新触发，请刷新后重试');
const readAt = new Date();
const log = () => this.prisma.operationLog.create({
data: { userId, action: 'monitoring.alert_marked_read', resource: 'infrastructure_alert', resourceId: fingerprint, detail: { activeAt: activeAt.toISOString(), alertName: current.name, severity: current.severity } },
});
const log = () =>
this.prisma.operationLog.create({
data: {
userId,
action: 'monitoring.alert_marked_read',
resource: 'infrastructure_alert',
resourceId: fingerprint,
detail: { activeAt: activeAt.toISOString(), alertName: current.name, severity: current.severity },
},
});
let read;
try {
[read] = await this.prisma.$transaction([
@@ -245,15 +305,57 @@ export class InfrastructureMonitoringService {
]);
} catch (error) {
if (!(error instanceof Prisma.PrismaClientKnownRequestError) || error.code !== 'P2002') throw error;
const existing = await this.prisma.infrastructureAlertRead.findUniqueOrThrow({ where: { fingerprint_userId: { fingerprint, userId } } });
const existing = await this.prisma.infrastructureAlertRead.findUniqueOrThrow({
where: { fingerprint_userId: { fingerprint, userId } },
});
// 同一次触发重复点击不更新readAt也不重复写日志；activeAt变化才代表同指纹的新触发周期。
if (existing.activeAt.getTime() === activeAt.getTime()) read = existing;
else [read] = await this.prisma.$transaction([
this.prisma.infrastructureAlertRead.update({ where: { fingerprint_userId: { fingerprint, userId } }, data: { activeAt, readAt } }),
log(),
]);
else
[read] = await this.prisma.$transaction([
this.prisma.infrastructureAlertRead.update({
where: { fingerprint_userId: { fingerprint, userId } },
data: { activeAt, readAt },
}),
log(),
]);
}
return { fingerprint, activeAt: read.activeAt.toISOString(), acknowledged: true, acknowledgedAt: read.readAt.toISOString() };
return {
fingerprint,
activeAt: read.activeAt.toISOString(),
acknowledged: true,
acknowledgedAt: read.readAt.toISOString(),
};
}
async alertHistory(from?: string, to?: string, rawPage?: string) {
const range = alertHistoryRange(from, to);
const page = rawPage === undefined ? 1 : Number(rawPage);
if (!Number.isSafeInteger(page) || page < 1) throw new BadRequestException('告警页码无效');
const history = new Map<string, AlertHistoryItem>();
try {
// Daily raw range vectors retain short events that a coarse query_range step would miss.
for (let start = range.start; start < range.end; start += 86400) {
const end = Math.min(start + 86400, range.end);
const response = await this.getJson<PrometheusQueryResponse>('/api/v1/query', {
query: `ALERTS_FOR_STATE[${Math.ceil(end - start)}s]`,
time: String(end),
});
mergeAlertHistory(history, response.data?.result ?? [], range.start, range.end);
}
} catch {
throw new ServiceUnavailableException('历史告警查询失败，请稍后重试');
}
const items = [...history.values()].sort(
(a, b) => b.startedAt.localeCompare(a.startedAt) || a.id.localeCompare(b.id),
);
return {
items: items.slice((page - 1) * 25, page * 25),
total: items.length,
page,
pageSize: 25,
startDate: range.startDate,
endDate: range.endDate,
};
}
private parseRange(value?: string): InfrastructureMonitoringRange {
@@ -264,12 +366,21 @@ export class InfrastructureMonitoringService {
private async loadInstantMetrics() {
const keys = Object.keys(emptyMetrics()) as Array<keyof InfrastructureMonitoringOverview['metrics']>;
const responses = await Promise.all([...keys.map((key) => this.query(QUERIES[key])), this.query(QUERIES.lastSampleAt)]);
const responses = await Promise.all([
...keys.map((key) => this.query(QUERIES[key])),
this.query(QUERIES.lastSampleAt),
]);
const metrics = emptyMetrics();
keys.forEach((key, index) => { if (!key.startsWith('disk')) metrics[key] = vectorValue(responses[index]); });
keys.forEach((key, index) => {
if (!key.startsWith('disk')) metrics[key] = vectorValue(responses[index]);
});
const diskSamples = (key: keyof typeof metrics) => responses[keys.indexOf(key)].data?.result ?? [];
const usage = new Map(diskSamples('diskUsagePercent').map((item) => [filesystemIdentity(item.metric), finiteNumber(item.value?.[1])]));
const available = new Map(diskSamples('diskAvailableBytes').map((item) => [filesystemIdentity(item.metric), finiteNumber(item.value?.[1])]));
const usage = new Map(
diskSamples('diskUsagePercent').map((item) => [filesystemIdentity(item.metric), finiteNumber(item.value?.[1])]),
);
const available = new Map(
diskSamples('diskAvailableBytes').map((item) => [filesystemIdentity(item.metric), finiteNumber(item.value?.[1])]),
);
const groups = new Map<string, PrometheusSeries[]>();
for (const item of diskSamples('diskTotalBytes')) {
if (!item.metric.device || !item.metric.mountpoint || (finiteNumber(item.value?.[1]) ?? 0) <= 0) continue;
@@ -278,18 +389,32 @@ export class InfrastructureMonitoringService {
group.push(item);
groups.set(id, group);
}
const disks = [...groups].map(([id, items]) => {
const mountpoints = [...new Set(items.map((item) => item.metric.mountpoint))].sort(compareMountpoints);
const metric = items[0].metric;
return {
id, instance: metric.instance ?? '', device: metric.device, filesystem: metric.fstype ?? '',
mountpoint: mountpoints[0], mountpoints,
// Never sum aliases. Max/min also tolerate slight sampling differences.
totalBytes: Math.max(...items.map((item) => finiteNumber(item.value?.[1])!)),
availableBytes: available.get(id) ?? null, usagePercent: usage.get(id) ?? null,
};
})
.sort((left, right) => left.instance.localeCompare(right.instance) || (left.mountpoint === '/' ? -1 : right.mountpoint === '/' ? 1 : left.mountpoint.localeCompare(right.mountpoint)));
const disks = [...groups]
.map(([id, items]) => {
const mountpoints = [...new Set(items.map((item) => item.metric.mountpoint))].sort(compareMountpoints);
const metric = items[0].metric;
return {
id,
instance: metric.instance ?? '',
device: metric.device,
filesystem: metric.fstype ?? '',
mountpoint: mountpoints[0],
mountpoints,
// Never sum aliases. Max/min also tolerate slight sampling differences.
totalBytes: Math.max(...items.map((item) => finiteNumber(item.value?.[1])!)),
availableBytes: available.get(id) ?? null,
usagePercent: usage.get(id) ?? null,
};
})
.sort(
(left, right) =>
left.instance.localeCompare(right.instance) ||
(left.mountpoint === '/'
? -1
: right.mountpoint === '/'
? 1
: left.mountpoint.localeCompare(right.mountpoint)),
);
const rootDisk = disks.find((disk) => disk.mountpoints.includes('/'));
metrics.diskUsagePercent = rootDisk?.usagePercent ?? null;
metrics.diskTotalBytes = rootDisk?.totalBytes ?? null;
@@ -304,21 +429,32 @@ export class InfrastructureMonitoringService {
const keys = Object.keys(emptyTrends()) as Array<keyof InfrastructureMonitoringOverview['trends']>;
const responses = await Promise.all(keys.map((key) => this.queryRange(QUERIES[key], start, end, config.step)));
return {
metrics: Object.fromEntries(keys.map((key, index) => [key, key === 'diskUsagePercent' ? [] : matrixValues(responses[index])])) as InfrastructureMonitoringOverview['trends'],
disks: new Map((responses[keys.indexOf('diskUsagePercent')].data?.result ?? []).map((item) => [
filesystemIdentity(item.metric), matrixValues({ status: 'success', data: { result: [item] } }),
])),
metrics: Object.fromEntries(
keys.map((key, index) => [key, key === 'diskUsagePercent' ? [] : matrixValues(responses[index])]),
) as InfrastructureMonitoringOverview['trends'],
disks: new Map(
(responses[keys.indexOf('diskUsagePercent')].data?.result ?? []).map((item) => [
filesystemIdentity(item.metric),
matrixValues({ status: 'success', data: { result: [item] } }),
]),
),
};
}
private parseServices(response: PrometheusQueryResponse): InfrastructureServiceStatus[] {
const values = new Map<string, number>();
for (const item of response.data?.result ?? []) {
if (item.metric.name) values.set(item.metric.name, vectorValue({ status: 'success', data: { result: [item] } }) ?? 0);
if (item.metric.name)
values.set(item.metric.name, vectorValue({ status: 'success', data: { result: [item] } }) ?? 0);
}
return SERVICE_DEFINITIONS.map((definition) => {
const present = definition.units.filter((unit) => values.has(unit));
const status = present.length === 0 ? 'unknown' : present.some((unit) => (values.get(unit) ?? 0) >= 1) ? 'healthy' : 'unhealthy';
const status =
present.length === 0
? 'unknown'
: present.some((unit) => (values.get(unit) ?? 0) >= 1)
? 'healthy'
: 'unhealthy';
return { key: definition.key, name: definition.name, unit: present[0] ?? definition.units[0], status };
});
}
@@ -329,7 +465,8 @@ export class InfrastructureMonitoringService {
.map<InfrastructureAlert>((item) => {
const labels = item.labels ?? {};
const annotations = item.annotations ?? {};
const severity: InfrastructureAlert['severity'] = labels.severity === 'critical' ? 'critical' : labels.severity === 'warning' ? 'warning' : 'info';
const severity: InfrastructureAlert['severity'] =
labels.severity === 'critical' ? 'critical' : labels.severity === 'warning' ? 'warning' : 'info';
const identity = JSON.stringify(Object.entries(labels).sort(([left], [right]) => left.localeCompare(right)));
return {
fingerprint: createHash('sha256').update(identity).digest('hex').slice(0, 24),
@@ -348,7 +485,9 @@ export class InfrastructureMonitoringService {
})
.sort((left, right) => {
const priority: Record<InfrastructureAlert['severity'], number> = { critical: 0, warning: 1, info: 2 };
return priority[left.severity] - priority[right.severity] || Date.parse(left.startedAt) - Date.parse(right.startedAt);
return (
priority[left.severity] - priority[right.severity] || Date.parse(left.startedAt) - Date.parse(right.startedAt)
);
});
}
@@ -377,24 +516,46 @@ export class InfrastructureMonitoringService {
key: group.key,
name: group.name,
available: group.metrics.some((metric) => values.has(metric[2])),
metrics: group.metrics.map(([key, label, metricName, unit]) => ({ key, label, value: values.get(metricName) ?? null, unit })),
metrics: group.metrics.map(([key, label, metricName, unit]) => ({
key,
label,
value: values.get(metricName) ?? null,
unit,
})),
}));
}
private unavailable(range: InfrastructureMonitoringRange, collectedAt: string): InfrastructureMonitoringOverview {
const services = SERVICE_DEFINITIONS.map((item) => ({ key: item.key, name: item.name, unit: item.units[0], status: 'unknown' as const }));
const services = SERVICE_DEFINITIONS.map((item) => ({
key: item.key,
name: item.name,
unit: item.units[0],
status: 'unknown' as const,
}));
return {
available: false,
range,
collectedAt,
lastSampleAt: null,
error: 'Prometheus监控数据当前不可用，请检查采集与服务状态',
summary: { overallStatus: 'unknown', serviceTotal: services.length, serviceHealthy: 0, warningAlerts: 0, criticalAlerts: 0, activeAlerts: 0 },
summary: {
overallStatus: 'unknown',
serviceTotal: services.length,
serviceHealthy: 0,
warningAlerts: 0,
criticalAlerts: 0,
activeAlerts: 0,
},
metrics: emptyMetrics(),
disks: [],
trends: emptyTrends(),
services,
serviceMetrics: SERVICE_METRIC_DEFINITIONS.map((group) => ({ key: group.key, name: group.name, available: false, metrics: [] })),
serviceMetrics: SERVICE_METRIC_DEFINITIONS.map((group) => ({
key: group.key,
name: group.name,
available: false,
metrics: [],
})),
alerts: [],
};
}
@@ -404,15 +565,26 @@ export class InfrastructureMonitoringService {
}
private queryRange(query: string, start: number, end: number, step: number) {
return this.getJson<PrometheusQueryResponse>('/api/v1/query_range', { query, start: String(start), end: String(end), step: String(step) });
return this.getJson<PrometheusQueryResponse>('/api/v1/query_range', {
query,
start: String(start),
end: String(end),
step: String(step),
});
}
private async getJson<T extends { status: 'success' | 'error'; error?: string }>(path: string, params: Record<string, string> = {}): Promise<T> {
private async getJson<T extends { status: 'success' | 'error'; error?: string }>(
path: string,
params: Record<string, string> = {},
): Promise<T> {
const url = new URL(`${this.prometheusUrl}${path}`);
Object.entries(params).forEach(([key, value]) => url.searchParams.set(key, value));
const response = await fetch(url, { headers: { Accept: 'application/json' }, signal: AbortSignal.timeout(this.queryTimeoutMs) });
const response = await fetch(url, {
headers: { Accept: 'application/json' },
signal: AbortSignal.timeout(this.queryTimeoutMs),
});
if (!response.ok) throw new Error(`Prometheus HTTP ${response.status}`);
const result = await response.json() as T;
const result = (await response.json()) as T;
if (result.status !== 'success') throw new Error('Prometheus query failed');
return result;
}