|
|
|
@@ -1,9 +1,22 @@
|
|
|
|
|
import { BadRequestException, Injectable, Logger, NotFoundException, ServiceUnavailableException } from '@nestjs/common';
|
|
|
|
|
import {
|
|
|
|
|
BadRequestException,
|
|
|
|
|
Injectable,
|
|
|
|
|
Logger,
|
|
|
|
|
NotFoundException,
|
|
|
|
|
ServiceUnavailableException,
|
|
|
|
|
} from '@nestjs/common';
|
|
|
|
|
import { ConfigService } from '@nestjs/config';
|
|
|
|
|
import { Prisma } from '@prisma/client';
|
|
|
|
|
import { createHash } from 'node:crypto';
|
|
|
|
|
import { PrismaService } from '../prisma/prisma.service';
|
|
|
|
|
import { compareMountpoints, FILESYSTEM_LABELS, FILESYSTEM_SELECTOR, FILESYSTEM_USAGE_PERCENT, filesystemIdentity } from './filesystem-metrics';
|
|
|
|
|
import { alertHistoryRange, mergeAlertHistory, type AlertHistoryItem } from './alert-history';
|
|
|
|
|
import {
|
|
|
|
|
compareMountpoints,
|
|
|
|
|
FILESYSTEM_LABELS,
|
|
|
|
|
FILESYSTEM_SELECTOR,
|
|
|
|
|
FILESYSTEM_USAGE_PERCENT,
|
|
|
|
|
filesystemIdentity,
|
|
|
|
|
} from './filesystem-metrics';
|
|
|
|
|
import type {
|
|
|
|
|
InfrastructureAlert,
|
|
|
|
|
InfrastructureMetricPoint,
|
|
|
|
@@ -57,7 +70,8 @@ const QUERIES = {
|
|
|
|
|
uptimeSeconds: 'time() - node_boot_time_seconds',
|
|
|
|
|
lastSampleAt: 'max(timestamp(node_uname_info))',
|
|
|
|
|
// PromQL字符串本身需要两个反斜杠才能把正则的“\.”传给RE2;TypeScript字面量因此需要写四个。
|
|
|
|
|
services: 'max by (name) (node_systemd_unit_state{name=~"cmpp-api\\\\.service|cmpp-gateway\\\\.service|postgresql\\\\.service|redis(-server)?\\\\.service|cmpp-minio\\\\.service|nginx\\\\.service",state="active"})',
|
|
|
|
|
services:
|
|
|
|
|
'max by (name) (node_systemd_unit_state{name=~"cmpp-api\\\\.service|cmpp-gateway\\\\.service|postgresql\\\\.service|redis(-server)?\\\\.service|cmpp-minio\\\\.service|nginx\\\\.service",state="active"})',
|
|
|
|
|
} as const;
|
|
|
|
|
|
|
|
|
|
const SERVICE_DEFINITIONS = [
|
|
|
|
@@ -70,38 +84,62 @@ const SERVICE_DEFINITIONS = [
|
|
|
|
|
] as const;
|
|
|
|
|
|
|
|
|
|
const SERVICE_METRIC_DEFINITIONS = [
|
|
|
|
|
{ key: 'api', name: 'API服务', metrics: [
|
|
|
|
|
['requestsPerSecond', '请求速率', 'cmpp:service_api:requests_per_second', 'per_second'],
|
|
|
|
|
['errorPercent', '5xx错误率', 'cmpp:service_api:error_percent', 'percent'],
|
|
|
|
|
['latencyP95', 'P95响应', 'cmpp:service_api:latency_p95_seconds', 'seconds'],
|
|
|
|
|
['eventLoopP99', '事件循环P99', 'cmpp:service_api:event_loop_p99_seconds', 'seconds'],
|
|
|
|
|
] },
|
|
|
|
|
{ key: 'gateway', name: 'Gateway服务', metrics: [
|
|
|
|
|
['submitsPerSecond', '提交速率', 'cmpp:service_gateway:submits_per_second', 'per_second'],
|
|
|
|
|
['failurePercent', '提交失败率', 'cmpp:service_gateway:failure_percent', 'percent'],
|
|
|
|
|
['queuePending', 'Stream pending', 'cmpp:service_gateway:queue_pending', 'count'],
|
|
|
|
|
['queueOldestSeconds', '最旧pending', 'cmpp:service_gateway:queue_oldest_seconds', 'seconds'],
|
|
|
|
|
] },
|
|
|
|
|
{ key: 'postgresql', name: 'PostgreSQL', metrics: [
|
|
|
|
|
['connectionPercent', '连接使用率', 'cmpp:service_postgresql:connection_percent', 'percent'],
|
|
|
|
|
['deadlocks15m', '15分钟死锁', 'cmpp:service_postgresql:deadlocks_15m', 'count'],
|
|
|
|
|
] },
|
|
|
|
|
{ key: 'redis', name: 'Redis', metrics: [
|
|
|
|
|
['memoryPercent', '内存使用率', 'cmpp:service_redis:memory_percent', 'percent'],
|
|
|
|
|
['memoryUsedBytes', '已用内存', 'cmpp:service_redis:memory_used_bytes', 'bytes'],
|
|
|
|
|
['connectedClients', '客户端连接', 'cmpp:service_redis:connected_clients', 'count'],
|
|
|
|
|
['evictions5m', '5分钟淘汰', 'cmpp:service_redis:evictions_5m', 'count'],
|
|
|
|
|
] },
|
|
|
|
|
{ key: 'minio', name: 'MinIO', metrics: [
|
|
|
|
|
['capacityPercent', '存储容量使用率', 'cmpp:service_minio:capacity_percent', 'percent'],
|
|
|
|
|
['usageBytes', '对象数据量', 'cmpp:service_minio:usage_bytes', 'bytes'],
|
|
|
|
|
['objects', '对象数', 'cmpp:service_minio:objects', 'count'],
|
|
|
|
|
['drivesOffline', '离线存储盘', 'cmpp:service_minio:drives_offline', 'count'],
|
|
|
|
|
] },
|
|
|
|
|
{ key: 'nginx', name: 'Nginx', metrics: [
|
|
|
|
|
['connectionsActive', '活跃连接', 'cmpp:service_nginx:connections_active', 'count'],
|
|
|
|
|
['requestsPerSecond', '请求速率', 'cmpp:service_nginx:requests_per_second', 'per_second'],
|
|
|
|
|
] },
|
|
|
|
|
{
|
|
|
|
|
key: 'api',
|
|
|
|
|
name: 'API服务',
|
|
|
|
|
metrics: [
|
|
|
|
|
['requestsPerSecond', '请求速率', 'cmpp:service_api:requests_per_second', 'per_second'],
|
|
|
|
|
['errorPercent', '5xx错误率', 'cmpp:service_api:error_percent', 'percent'],
|
|
|
|
|
['latencyP95', 'P95响应', 'cmpp:service_api:latency_p95_seconds', 'seconds'],
|
|
|
|
|
['eventLoopP99', '事件循环P99', 'cmpp:service_api:event_loop_p99_seconds', 'seconds'],
|
|
|
|
|
],
|
|
|
|
|
},
|
|
|
|
|
{
|
|
|
|
|
key: 'gateway',
|
|
|
|
|
name: 'Gateway服务',
|
|
|
|
|
metrics: [
|
|
|
|
|
['submitsPerSecond', '提交速率', 'cmpp:service_gateway:submits_per_second', 'per_second'],
|
|
|
|
|
['failurePercent', '提交失败率', 'cmpp:service_gateway:failure_percent', 'percent'],
|
|
|
|
|
['queuePending', 'Stream pending', 'cmpp:service_gateway:queue_pending', 'count'],
|
|
|
|
|
['queueOldestSeconds', '最旧pending', 'cmpp:service_gateway:queue_oldest_seconds', 'seconds'],
|
|
|
|
|
],
|
|
|
|
|
},
|
|
|
|
|
{
|
|
|
|
|
key: 'postgresql',
|
|
|
|
|
name: 'PostgreSQL',
|
|
|
|
|
metrics: [
|
|
|
|
|
['connectionPercent', '连接使用率', 'cmpp:service_postgresql:connection_percent', 'percent'],
|
|
|
|
|
['deadlocks15m', '15分钟死锁', 'cmpp:service_postgresql:deadlocks_15m', 'count'],
|
|
|
|
|
],
|
|
|
|
|
},
|
|
|
|
|
{
|
|
|
|
|
key: 'redis',
|
|
|
|
|
name: 'Redis',
|
|
|
|
|
metrics: [
|
|
|
|
|
['memoryPercent', '内存使用率', 'cmpp:service_redis:memory_percent', 'percent'],
|
|
|
|
|
['memoryUsedBytes', '已用内存', 'cmpp:service_redis:memory_used_bytes', 'bytes'],
|
|
|
|
|
['connectedClients', '客户端连接', 'cmpp:service_redis:connected_clients', 'count'],
|
|
|
|
|
['evictions5m', '5分钟淘汰', 'cmpp:service_redis:evictions_5m', 'count'],
|
|
|
|
|
],
|
|
|
|
|
},
|
|
|
|
|
{
|
|
|
|
|
key: 'minio',
|
|
|
|
|
name: 'MinIO',
|
|
|
|
|
metrics: [
|
|
|
|
|
['capacityPercent', '存储容量使用率', 'cmpp:service_minio:capacity_percent', 'percent'],
|
|
|
|
|
['usageBytes', '对象数据量', 'cmpp:service_minio:usage_bytes', 'bytes'],
|
|
|
|
|
['objects', '对象数', 'cmpp:service_minio:objects', 'count'],
|
|
|
|
|
['drivesOffline', '离线存储盘', 'cmpp:service_minio:drives_offline', 'count'],
|
|
|
|
|
],
|
|
|
|
|
},
|
|
|
|
|
{
|
|
|
|
|
key: 'nginx',
|
|
|
|
|
name: 'Nginx',
|
|
|
|
|
metrics: [
|
|
|
|
|
['connectionsActive', '活跃连接', 'cmpp:service_nginx:connections_active', 'count'],
|
|
|
|
|
['requestsPerSecond', '请求速率', 'cmpp:service_nginx:requests_per_second', 'per_second'],
|
|
|
|
|
],
|
|
|
|
|
},
|
|
|
|
|
] as const;
|
|
|
|
|
|
|
|
|
|
const SERVICE_METRICS_QUERY = '{__name__=~"cmpp:service_.*"}';
|
|
|
|
@@ -165,7 +203,10 @@ export class InfrastructureMonitoringService {
|
|
|
|
|
private readonly prometheusUrl: string;
|
|
|
|
|
private readonly queryTimeoutMs: number;
|
|
|
|
|
|
|
|
|
|
constructor(config: ConfigService, private readonly prisma: PrismaService) {
|
|
|
|
|
constructor(
|
|
|
|
|
config: ConfigService,
|
|
|
|
|
private readonly prisma: PrismaService,
|
|
|
|
|
) {
|
|
|
|
|
this.prometheusUrl = normalizePrometheusUrl(config.get('PROMETHEUS_URL'));
|
|
|
|
|
this.queryTimeoutMs = Math.min(15_000, Math.max(1_000, Number(config.get('PROMETHEUS_QUERY_TIMEOUT_MS') ?? 5_000)));
|
|
|
|
|
}
|
|
|
|
@@ -202,7 +243,7 @@ export class InfrastructureMonitoringService {
|
|
|
|
|
activeAlerts: alerts.length,
|
|
|
|
|
},
|
|
|
|
|
metrics: instant.metrics,
|
|
|
|
|
trends: { ...trends.metrics, diskUsagePercent: rootDisk ? trends.disks.get(rootDisk.id) ?? [] : [] },
|
|
|
|
|
trends: { ...trends.metrics, diskUsagePercent: rootDisk ? (trends.disks.get(rootDisk.id) ?? []) : [] },
|
|
|
|
|
disks: instant.disks.map((disk) => ({ ...disk, trend: trends.disks.get(disk.id) ?? [] })),
|
|
|
|
|
services,
|
|
|
|
|
serviceMetrics,
|
|
|
|
@@ -210,18 +251,28 @@ export class InfrastructureMonitoringService {
|
|
|
|
|
};
|
|
|
|
|
} catch (error) {
|
|
|
|
|
// 页面必须整体清空陈旧指标,但服务端仍需留下不含PromQL/地址/凭据的根因摘要便于运维诊断。
|
|
|
|
|
this.logger.warn(`Prometheus monitoring overview unavailable: ${error instanceof Error ? error.message : 'unknown error'}`);
|
|
|
|
|
this.logger.warn(
|
|
|
|
|
`Prometheus monitoring overview unavailable: ${error instanceof Error ? error.message : 'unknown error'}`,
|
|
|
|
|
);
|
|
|
|
|
return this.unavailable(range, collectedAt);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
async notificationSummary(userId?: string) {
|
|
|
|
|
try {
|
|
|
|
|
const alerts = await this.attachReadState(this.parseAlerts(await this.getJson<PrometheusAlertResponse>('/api/v1/alerts')), userId);
|
|
|
|
|
const alerts = await this.attachReadState(
|
|
|
|
|
this.parseAlerts(await this.getJson<PrometheusAlertResponse>('/api/v1/alerts')),
|
|
|
|
|
userId,
|
|
|
|
|
);
|
|
|
|
|
const unreadAlerts = alerts.filter((item) => !item.acknowledged);
|
|
|
|
|
return { count: unreadAlerts.length, criticalCount: unreadAlerts.filter((item) => item.severity === 'critical').length };
|
|
|
|
|
return {
|
|
|
|
|
count: unreadAlerts.length,
|
|
|
|
|
criticalCount: unreadAlerts.filter((item) => item.severity === 'critical').length,
|
|
|
|
|
};
|
|
|
|
|
} catch (error) {
|
|
|
|
|
this.logger.warn(`Prometheus notification summary unavailable: ${error instanceof Error ? error.message : 'unknown error'}`);
|
|
|
|
|
this.logger.warn(
|
|
|
|
|
`Prometheus notification summary unavailable: ${error instanceof Error ? error.message : 'unknown error'}`,
|
|
|
|
|
);
|
|
|
|
|
throw new ServiceUnavailableException('Prometheus活动告警当前不可用');
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
@@ -231,12 +282,21 @@ export class InfrastructureMonitoringService {
|
|
|
|
|
const activeAt = new Date(String(rawActiveAt ?? ''));
|
|
|
|
|
if (!Number.isFinite(activeAt.getTime())) throw new BadRequestException('告警开始时间无效');
|
|
|
|
|
const activeAlerts = this.parseAlerts(await this.getJson<PrometheusAlertResponse>('/api/v1/alerts'));
|
|
|
|
|
const current = activeAlerts.find((item) => item.fingerprint === fingerprint && Date.parse(item.startedAt) === activeAt.getTime());
|
|
|
|
|
const current = activeAlerts.find(
|
|
|
|
|
(item) => item.fingerprint === fingerprint && Date.parse(item.startedAt) === activeAt.getTime(),
|
|
|
|
|
);
|
|
|
|
|
if (!current) throw new NotFoundException('该次活动告警已结束或已重新触发,请刷新后重试');
|
|
|
|
|
const readAt = new Date();
|
|
|
|
|
const log = () => this.prisma.operationLog.create({
|
|
|
|
|
data: { userId, action: 'monitoring.alert_marked_read', resource: 'infrastructure_alert', resourceId: fingerprint, detail: { activeAt: activeAt.toISOString(), alertName: current.name, severity: current.severity } },
|
|
|
|
|
});
|
|
|
|
|
const log = () =>
|
|
|
|
|
this.prisma.operationLog.create({
|
|
|
|
|
data: {
|
|
|
|
|
userId,
|
|
|
|
|
action: 'monitoring.alert_marked_read',
|
|
|
|
|
resource: 'infrastructure_alert',
|
|
|
|
|
resourceId: fingerprint,
|
|
|
|
|
detail: { activeAt: activeAt.toISOString(), alertName: current.name, severity: current.severity },
|
|
|
|
|
},
|
|
|
|
|
});
|
|
|
|
|
let read;
|
|
|
|
|
try {
|
|
|
|
|
[read] = await this.prisma.$transaction([
|
|
|
|
@@ -245,15 +305,57 @@ export class InfrastructureMonitoringService {
|
|
|
|
|
]);
|
|
|
|
|
} catch (error) {
|
|
|
|
|
if (!(error instanceof Prisma.PrismaClientKnownRequestError) || error.code !== 'P2002') throw error;
|
|
|
|
|
const existing = await this.prisma.infrastructureAlertRead.findUniqueOrThrow({ where: { fingerprint_userId: { fingerprint, userId } } });
|
|
|
|
|
const existing = await this.prisma.infrastructureAlertRead.findUniqueOrThrow({
|
|
|
|
|
where: { fingerprint_userId: { fingerprint, userId } },
|
|
|
|
|
});
|
|
|
|
|
// 同一次触发重复点击不更新readAt也不重复写日志;activeAt变化才代表同指纹的新触发周期。
|
|
|
|
|
if (existing.activeAt.getTime() === activeAt.getTime()) read = existing;
|
|
|
|
|
else [read] = await this.prisma.$transaction([
|
|
|
|
|
this.prisma.infrastructureAlertRead.update({ where: { fingerprint_userId: { fingerprint, userId } }, data: { activeAt, readAt } }),
|
|
|
|
|
log(),
|
|
|
|
|
]);
|
|
|
|
|
else
|
|
|
|
|
[read] = await this.prisma.$transaction([
|
|
|
|
|
this.prisma.infrastructureAlertRead.update({
|
|
|
|
|
where: { fingerprint_userId: { fingerprint, userId } },
|
|
|
|
|
data: { activeAt, readAt },
|
|
|
|
|
}),
|
|
|
|
|
log(),
|
|
|
|
|
]);
|
|
|
|
|
}
|
|
|
|
|
return { fingerprint, activeAt: read.activeAt.toISOString(), acknowledged: true, acknowledgedAt: read.readAt.toISOString() };
|
|
|
|
|
return {
|
|
|
|
|
fingerprint,
|
|
|
|
|
activeAt: read.activeAt.toISOString(),
|
|
|
|
|
acknowledged: true,
|
|
|
|
|
acknowledgedAt: read.readAt.toISOString(),
|
|
|
|
|
};
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
async alertHistory(from?: string, to?: string, rawPage?: string) {
|
|
|
|
|
const range = alertHistoryRange(from, to);
|
|
|
|
|
const page = rawPage === undefined ? 1 : Number(rawPage);
|
|
|
|
|
if (!Number.isSafeInteger(page) || page < 1) throw new BadRequestException('告警页码无效');
|
|
|
|
|
const history = new Map<string, AlertHistoryItem>();
|
|
|
|
|
try {
|
|
|
|
|
// Daily raw range vectors retain short events that a coarse query_range step would miss.
|
|
|
|
|
for (let start = range.start; start < range.end; start += 86400) {
|
|
|
|
|
const end = Math.min(start + 86400, range.end);
|
|
|
|
|
const response = await this.getJson<PrometheusQueryResponse>('/api/v1/query', {
|
|
|
|
|
query: `ALERTS_FOR_STATE[${Math.ceil(end - start)}s]`,
|
|
|
|
|
time: String(end),
|
|
|
|
|
});
|
|
|
|
|
mergeAlertHistory(history, response.data?.result ?? [], range.start, range.end);
|
|
|
|
|
}
|
|
|
|
|
} catch {
|
|
|
|
|
throw new ServiceUnavailableException('历史告警查询失败,请稍后重试');
|
|
|
|
|
}
|
|
|
|
|
const items = [...history.values()].sort(
|
|
|
|
|
(a, b) => b.startedAt.localeCompare(a.startedAt) || a.id.localeCompare(b.id),
|
|
|
|
|
);
|
|
|
|
|
return {
|
|
|
|
|
items: items.slice((page - 1) * 25, page * 25),
|
|
|
|
|
total: items.length,
|
|
|
|
|
page,
|
|
|
|
|
pageSize: 25,
|
|
|
|
|
startDate: range.startDate,
|
|
|
|
|
endDate: range.endDate,
|
|
|
|
|
};
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private parseRange(value?: string): InfrastructureMonitoringRange {
|
|
|
|
@@ -264,12 +366,21 @@ export class InfrastructureMonitoringService {
|
|
|
|
|
|
|
|
|
|
private async loadInstantMetrics() {
|
|
|
|
|
const keys = Object.keys(emptyMetrics()) as Array<keyof InfrastructureMonitoringOverview['metrics']>;
|
|
|
|
|
const responses = await Promise.all([...keys.map((key) => this.query(QUERIES[key])), this.query(QUERIES.lastSampleAt)]);
|
|
|
|
|
const responses = await Promise.all([
|
|
|
|
|
...keys.map((key) => this.query(QUERIES[key])),
|
|
|
|
|
this.query(QUERIES.lastSampleAt),
|
|
|
|
|
]);
|
|
|
|
|
const metrics = emptyMetrics();
|
|
|
|
|
keys.forEach((key, index) => { if (!key.startsWith('disk')) metrics[key] = vectorValue(responses[index]); });
|
|
|
|
|
keys.forEach((key, index) => {
|
|
|
|
|
if (!key.startsWith('disk')) metrics[key] = vectorValue(responses[index]);
|
|
|
|
|
});
|
|
|
|
|
const diskSamples = (key: keyof typeof metrics) => responses[keys.indexOf(key)].data?.result ?? [];
|
|
|
|
|
const usage = new Map(diskSamples('diskUsagePercent').map((item) => [filesystemIdentity(item.metric), finiteNumber(item.value?.[1])]));
|
|
|
|
|
const available = new Map(diskSamples('diskAvailableBytes').map((item) => [filesystemIdentity(item.metric), finiteNumber(item.value?.[1])]));
|
|
|
|
|
const usage = new Map(
|
|
|
|
|
diskSamples('diskUsagePercent').map((item) => [filesystemIdentity(item.metric), finiteNumber(item.value?.[1])]),
|
|
|
|
|
);
|
|
|
|
|
const available = new Map(
|
|
|
|
|
diskSamples('diskAvailableBytes').map((item) => [filesystemIdentity(item.metric), finiteNumber(item.value?.[1])]),
|
|
|
|
|
);
|
|
|
|
|
const groups = new Map<string, PrometheusSeries[]>();
|
|
|
|
|
for (const item of diskSamples('diskTotalBytes')) {
|
|
|
|
|
if (!item.metric.device || !item.metric.mountpoint || (finiteNumber(item.value?.[1]) ?? 0) <= 0) continue;
|
|
|
|
@@ -278,18 +389,32 @@ export class InfrastructureMonitoringService {
|
|
|
|
|
group.push(item);
|
|
|
|
|
groups.set(id, group);
|
|
|
|
|
}
|
|
|
|
|
const disks = [...groups].map(([id, items]) => {
|
|
|
|
|
const mountpoints = [...new Set(items.map((item) => item.metric.mountpoint))].sort(compareMountpoints);
|
|
|
|
|
const metric = items[0].metric;
|
|
|
|
|
return {
|
|
|
|
|
id, instance: metric.instance ?? '', device: metric.device, filesystem: metric.fstype ?? '',
|
|
|
|
|
mountpoint: mountpoints[0], mountpoints,
|
|
|
|
|
// Never sum aliases. Max/min also tolerate slight sampling differences.
|
|
|
|
|
totalBytes: Math.max(...items.map((item) => finiteNumber(item.value?.[1])!)),
|
|
|
|
|
availableBytes: available.get(id) ?? null, usagePercent: usage.get(id) ?? null,
|
|
|
|
|
};
|
|
|
|
|
})
|
|
|
|
|
.sort((left, right) => left.instance.localeCompare(right.instance) || (left.mountpoint === '/' ? -1 : right.mountpoint === '/' ? 1 : left.mountpoint.localeCompare(right.mountpoint)));
|
|
|
|
|
const disks = [...groups]
|
|
|
|
|
.map(([id, items]) => {
|
|
|
|
|
const mountpoints = [...new Set(items.map((item) => item.metric.mountpoint))].sort(compareMountpoints);
|
|
|
|
|
const metric = items[0].metric;
|
|
|
|
|
return {
|
|
|
|
|
id,
|
|
|
|
|
instance: metric.instance ?? '',
|
|
|
|
|
device: metric.device,
|
|
|
|
|
filesystem: metric.fstype ?? '',
|
|
|
|
|
mountpoint: mountpoints[0],
|
|
|
|
|
mountpoints,
|
|
|
|
|
// Never sum aliases. Max/min also tolerate slight sampling differences.
|
|
|
|
|
totalBytes: Math.max(...items.map((item) => finiteNumber(item.value?.[1])!)),
|
|
|
|
|
availableBytes: available.get(id) ?? null,
|
|
|
|
|
usagePercent: usage.get(id) ?? null,
|
|
|
|
|
};
|
|
|
|
|
})
|
|
|
|
|
.sort(
|
|
|
|
|
(left, right) =>
|
|
|
|
|
left.instance.localeCompare(right.instance) ||
|
|
|
|
|
(left.mountpoint === '/'
|
|
|
|
|
? -1
|
|
|
|
|
: right.mountpoint === '/'
|
|
|
|
|
? 1
|
|
|
|
|
: left.mountpoint.localeCompare(right.mountpoint)),
|
|
|
|
|
);
|
|
|
|
|
const rootDisk = disks.find((disk) => disk.mountpoints.includes('/'));
|
|
|
|
|
metrics.diskUsagePercent = rootDisk?.usagePercent ?? null;
|
|
|
|
|
metrics.diskTotalBytes = rootDisk?.totalBytes ?? null;
|
|
|
|
@@ -304,21 +429,32 @@ export class InfrastructureMonitoringService {
|
|
|
|
|
const keys = Object.keys(emptyTrends()) as Array<keyof InfrastructureMonitoringOverview['trends']>;
|
|
|
|
|
const responses = await Promise.all(keys.map((key) => this.queryRange(QUERIES[key], start, end, config.step)));
|
|
|
|
|
return {
|
|
|
|
|
metrics: Object.fromEntries(keys.map((key, index) => [key, key === 'diskUsagePercent' ? [] : matrixValues(responses[index])])) as InfrastructureMonitoringOverview['trends'],
|
|
|
|
|
disks: new Map((responses[keys.indexOf('diskUsagePercent')].data?.result ?? []).map((item) => [
|
|
|
|
|
filesystemIdentity(item.metric), matrixValues({ status: 'success', data: { result: [item] } }),
|
|
|
|
|
])),
|
|
|
|
|
metrics: Object.fromEntries(
|
|
|
|
|
keys.map((key, index) => [key, key === 'diskUsagePercent' ? [] : matrixValues(responses[index])]),
|
|
|
|
|
) as InfrastructureMonitoringOverview['trends'],
|
|
|
|
|
disks: new Map(
|
|
|
|
|
(responses[keys.indexOf('diskUsagePercent')].data?.result ?? []).map((item) => [
|
|
|
|
|
filesystemIdentity(item.metric),
|
|
|
|
|
matrixValues({ status: 'success', data: { result: [item] } }),
|
|
|
|
|
]),
|
|
|
|
|
),
|
|
|
|
|
};
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private parseServices(response: PrometheusQueryResponse): InfrastructureServiceStatus[] {
|
|
|
|
|
const values = new Map<string, number>();
|
|
|
|
|
for (const item of response.data?.result ?? []) {
|
|
|
|
|
if (item.metric.name) values.set(item.metric.name, vectorValue({ status: 'success', data: { result: [item] } }) ?? 0);
|
|
|
|
|
if (item.metric.name)
|
|
|
|
|
values.set(item.metric.name, vectorValue({ status: 'success', data: { result: [item] } }) ?? 0);
|
|
|
|
|
}
|
|
|
|
|
return SERVICE_DEFINITIONS.map((definition) => {
|
|
|
|
|
const present = definition.units.filter((unit) => values.has(unit));
|
|
|
|
|
const status = present.length === 0 ? 'unknown' : present.some((unit) => (values.get(unit) ?? 0) >= 1) ? 'healthy' : 'unhealthy';
|
|
|
|
|
const status =
|
|
|
|
|
present.length === 0
|
|
|
|
|
? 'unknown'
|
|
|
|
|
: present.some((unit) => (values.get(unit) ?? 0) >= 1)
|
|
|
|
|
? 'healthy'
|
|
|
|
|
: 'unhealthy';
|
|
|
|
|
return { key: definition.key, name: definition.name, unit: present[0] ?? definition.units[0], status };
|
|
|
|
|
});
|
|
|
|
|
}
|
|
|
|
@@ -329,7 +465,8 @@ export class InfrastructureMonitoringService {
|
|
|
|
|
.map<InfrastructureAlert>((item) => {
|
|
|
|
|
const labels = item.labels ?? {};
|
|
|
|
|
const annotations = item.annotations ?? {};
|
|
|
|
|
const severity: InfrastructureAlert['severity'] = labels.severity === 'critical' ? 'critical' : labels.severity === 'warning' ? 'warning' : 'info';
|
|
|
|
|
const severity: InfrastructureAlert['severity'] =
|
|
|
|
|
labels.severity === 'critical' ? 'critical' : labels.severity === 'warning' ? 'warning' : 'info';
|
|
|
|
|
const identity = JSON.stringify(Object.entries(labels).sort(([left], [right]) => left.localeCompare(right)));
|
|
|
|
|
return {
|
|
|
|
|
fingerprint: createHash('sha256').update(identity).digest('hex').slice(0, 24),
|
|
|
|
@@ -348,7 +485,9 @@ export class InfrastructureMonitoringService {
|
|
|
|
|
})
|
|
|
|
|
.sort((left, right) => {
|
|
|
|
|
const priority: Record<InfrastructureAlert['severity'], number> = { critical: 0, warning: 1, info: 2 };
|
|
|
|
|
return priority[left.severity] - priority[right.severity] || Date.parse(left.startedAt) - Date.parse(right.startedAt);
|
|
|
|
|
return (
|
|
|
|
|
priority[left.severity] - priority[right.severity] || Date.parse(left.startedAt) - Date.parse(right.startedAt)
|
|
|
|
|
);
|
|
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
@@ -377,24 +516,46 @@ export class InfrastructureMonitoringService {
|
|
|
|
|
key: group.key,
|
|
|
|
|
name: group.name,
|
|
|
|
|
available: group.metrics.some((metric) => values.has(metric[2])),
|
|
|
|
|
metrics: group.metrics.map(([key, label, metricName, unit]) => ({ key, label, value: values.get(metricName) ?? null, unit })),
|
|
|
|
|
metrics: group.metrics.map(([key, label, metricName, unit]) => ({
|
|
|
|
|
key,
|
|
|
|
|
label,
|
|
|
|
|
value: values.get(metricName) ?? null,
|
|
|
|
|
unit,
|
|
|
|
|
})),
|
|
|
|
|
}));
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private unavailable(range: InfrastructureMonitoringRange, collectedAt: string): InfrastructureMonitoringOverview {
|
|
|
|
|
const services = SERVICE_DEFINITIONS.map((item) => ({ key: item.key, name: item.name, unit: item.units[0], status: 'unknown' as const }));
|
|
|
|
|
const services = SERVICE_DEFINITIONS.map((item) => ({
|
|
|
|
|
key: item.key,
|
|
|
|
|
name: item.name,
|
|
|
|
|
unit: item.units[0],
|
|
|
|
|
status: 'unknown' as const,
|
|
|
|
|
}));
|
|
|
|
|
return {
|
|
|
|
|
available: false,
|
|
|
|
|
range,
|
|
|
|
|
collectedAt,
|
|
|
|
|
lastSampleAt: null,
|
|
|
|
|
error: 'Prometheus监控数据当前不可用,请检查采集与服务状态',
|
|
|
|
|
summary: { overallStatus: 'unknown', serviceTotal: services.length, serviceHealthy: 0, warningAlerts: 0, criticalAlerts: 0, activeAlerts: 0 },
|
|
|
|
|
summary: {
|
|
|
|
|
overallStatus: 'unknown',
|
|
|
|
|
serviceTotal: services.length,
|
|
|
|
|
serviceHealthy: 0,
|
|
|
|
|
warningAlerts: 0,
|
|
|
|
|
criticalAlerts: 0,
|
|
|
|
|
activeAlerts: 0,
|
|
|
|
|
},
|
|
|
|
|
metrics: emptyMetrics(),
|
|
|
|
|
disks: [],
|
|
|
|
|
trends: emptyTrends(),
|
|
|
|
|
services,
|
|
|
|
|
serviceMetrics: SERVICE_METRIC_DEFINITIONS.map((group) => ({ key: group.key, name: group.name, available: false, metrics: [] })),
|
|
|
|
|
serviceMetrics: SERVICE_METRIC_DEFINITIONS.map((group) => ({
|
|
|
|
|
key: group.key,
|
|
|
|
|
name: group.name,
|
|
|
|
|
available: false,
|
|
|
|
|
metrics: [],
|
|
|
|
|
})),
|
|
|
|
|
alerts: [],
|
|
|
|
|
};
|
|
|
|
|
}
|
|
|
|
@@ -404,15 +565,26 @@ export class InfrastructureMonitoringService {
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private queryRange(query: string, start: number, end: number, step: number) {
|
|
|
|
|
return this.getJson<PrometheusQueryResponse>('/api/v1/query_range', { query, start: String(start), end: String(end), step: String(step) });
|
|
|
|
|
return this.getJson<PrometheusQueryResponse>('/api/v1/query_range', {
|
|
|
|
|
query,
|
|
|
|
|
start: String(start),
|
|
|
|
|
end: String(end),
|
|
|
|
|
step: String(step),
|
|
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private async getJson<T extends { status: 'success' | 'error'; error?: string }>(path: string, params: Record<string, string> = {}): Promise<T> {
|
|
|
|
|
private async getJson<T extends { status: 'success' | 'error'; error?: string }>(
|
|
|
|
|
path: string,
|
|
|
|
|
params: Record<string, string> = {},
|
|
|
|
|
): Promise<T> {
|
|
|
|
|
const url = new URL(`${this.prometheusUrl}${path}`);
|
|
|
|
|
Object.entries(params).forEach(([key, value]) => url.searchParams.set(key, value));
|
|
|
|
|
const response = await fetch(url, { headers: { Accept: 'application/json' }, signal: AbortSignal.timeout(this.queryTimeoutMs) });
|
|
|
|
|
const response = await fetch(url, {
|
|
|
|
|
headers: { Accept: 'application/json' },
|
|
|
|
|
signal: AbortSignal.timeout(this.queryTimeoutMs),
|
|
|
|
|
});
|
|
|
|
|
if (!response.ok) throw new Error(`Prometheus HTTP ${response.status}`);
|
|
|
|
|
const result = await response.json() as T;
|
|
|
|
|
const result = (await response.json()) as T;
|
|
|
|
|
if (result.status !== 'success') throw new Error('Prometheus query failed');
|
|
|
|
|
return result;
|
|
|
|
|
}
|
|
|
|
|