hectorzhao/lislgosms
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: align reporting fields queries and disk monitoring

Browse Source
This commit is contained in:
hectorzhao
2026-08-31 12:19:39 +08:00
parent 119d57772e
commit b0deef5e6e
37 changed files with 596 additions and 110 deletions
Download Patch File Download Diff File Expand all files Collapse all files
api/src/infrastructure-monitoring/infrastructure-alert-settings.service.spec.ts
+4
View File
@@ -12,6 +12,10 @@ describe('InfrastructureAlertSettingsService', () => {
expect(rules).toContain('threshold: "120秒"');
expect(rules).toContain('redis_memory_max_bytes > 0');
expect(rules).toContain('sum(increase(cmpp_api_http_requests_total');
expect(rules).not.toContain('mountpoint="/"');
expect(rules).toContain('device=~"/dev/.+"');
expect(rules).toContain('{{ $labels.mountpoint }}');
expect(rules).toContain('{{ $labels.device }}');
});
it('rejects unknown keys and warning thresholds that are not below critical', () => {
api/src/infrastructure-monitoring/infrastructure-alert-settings.service.ts
+3 -2
View File
@@ -13,7 +13,7 @@ const execFileAsync = promisify(execFile);
export const ALERT_THRESHOLD_DEFINITIONS = [
{ key: 'hostCpu', label: '主机 CPU 使用率', unit: '%', min: 1, max: 100, step: 1, warning: 80, critical: 90, expr: '100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100)', names: ['HostCpuUsageWarning', 'HostCpuUsageCritical'], service: 'host', durations: ['10m', '5m'] },
{ key: 'hostMemory', label: '主机内存使用率', unit: '%', min: 1, max: 100, step: 1, warning: 85, critical: 95, expr: '(1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100', names: ['HostMemoryUsageWarning', 'HostMemoryUsageCritical'], service: 'host', durations: ['10m', '5m'] },
{ key: 'hostDisk', label: '根磁盘使用率', unit: '%', min: 1, max: 100, step: 1, warning: 80, critical: 90, expr: '(1 - node_filesystem_avail_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"} / node_filesystem_size_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"}) * 100', names: ['HostRootDiskUsageWarning', 'HostRootDiskUsageCritical'], service: 'host', durations: ['15m', '5m'] },
{ key: 'hostDisk', label: '磁盘（所有挂载点）使用率', unit: '%', min: 1, max: 100, step: 1, warning: 80, critical: 90, expr: '(1 - node_filesystem_avail_bytes{device=~"/dev/.+",fstype!~"tmpfs|devtmpfs|overlay|squashfs|ramfs"} / node_filesystem_size_bytes{device=~"/dev/.+",fstype!~"tmpfs|devtmpfs|overlay|squashfs|ramfs"}) * 100', names: ['HostRootDiskUsageWarning', 'HostRootDiskUsageCritical'], service: 'host', durations: ['15m', '5m'] },
{ key: 'apiError', label: 'API 5xx 错误率', unit: '%', min: 0.1, max: 100, step: 0.1, warning: 1, critical: 5, expr: '100 * sum(rate(cmpp_api_http_requests_total{status=~"5.."}[5m])) / clamp_min(sum(rate(cmpp_api_http_requests_total[5m])), 0.001)', guard: 'sum(increase(cmpp_api_http_requests_total{status=~"5.."}[5m])) >= 5', names: ['CmppApiHttpErrorRateWarning', 'CmppApiHttpErrorRateCritical'], service: 'api', durations: ['5m', '5m'] },
{ key: 'apiLatency', label: 'API P95 响应时间', unit: '秒', min: 0.1, max: 60, step: 0.1, warning: 1, critical: 3, expr: 'histogram_quantile(0.95, sum by (le) (rate(cmpp_api_http_request_duration_seconds_bucket[10m])))', names: ['CmppApiLatencyWarning', 'CmppApiLatencyCritical'], service: 'api', durations: ['10m', '5m'] },
{ key: 'apiEventLoop', label: 'API 事件循环 P99', unit: '秒', min: 0.01, max: 10, step: 0.01, warning: 0.2, critical: 1, expr: 'cmpp_api_nodejs_event_loop_lag_p99_seconds', names: ['CmppApiEventLoopLagWarning', 'CmppApiEventLoopLagCritical'], service: 'api', durations: ['10m', '5m'] },
@@ -113,7 +113,8 @@ export class InfrastructureAlertSettingsService {
// 低样本量与“未配置容量上限”必须继续作为固定保护条件，避免单次错误或除零结果触发伪告警。
const guard = 'guard' in definition ? ` and (${definition.guard})` : '';
const expr = isWarning ? `(${definition.expr} > ${values[0]}) and (${definition.expr} <= ${values[1]})${guard}` : `(${definition.expr} > ${values[1]})${guard}`;
lines.push(` - alert: ${definition.names[index]}`, ` expr: ${expr}`, ` for: ${definition.durations[index]}`, ' labels:', ` severity: ${isWarning ? 'warning' : 'critical'}`, ` service: ${definition.service}`, ' annotations:', ` summary: "${definition.label}${isWarning ? '达到警告阈值' : '达到严重阈值'}"`, ` description: "${definition.label}持续超过${values[index]}${definition.unit}。"`, ' currentValue: "{{ $value }}"', ` threshold: "${values[index]}${definition.unit}"`);
const diskLocation = definition.key === 'hostDisk' ? ' 挂载点：{{ $labels.mountpoint }}；设备：{{ $labels.device }}。' : '';
lines.push(` - alert: ${definition.names[index]}`, ` expr: ${expr}`, ` for: ${definition.durations[index]}`, ' labels:', ` severity: ${isWarning ? 'warning' : 'critical'}`, ` service: ${definition.service}`, ' annotations:', ` summary: "${definition.label}${isWarning ? '达到警告阈值' : '达到严重阈值'}"`, ` description: "${definition.label}持续超过${values[index]}${definition.unit}。${diskLocation}"`, ' currentValue: "{{ $value }}"', ` threshold: "${values[index]}${definition.unit}"`);
}
}
return `${lines.join('\n')}\n`;
api/src/infrastructure-monitoring/infrastructure-monitoring.contracts.ts
+11
View File
@@ -41,6 +41,17 @@ export type InfrastructureAlert = {
};
export type InfrastructureMonitoringOverview = {
disks: Array<{
id: string;
instance: string;
device: string;
mountpoint: string;
filesystem: string;
usagePercent: number | null;
totalBytes: number | null;
availableBytes: number | null;
trend: InfrastructureMetricPoint[];
}>;
available: boolean;
range: InfrastructureMonitoringRange;
collectedAt: string;
api/src/infrastructure-monitoring/infrastructure-monitoring.service.spec.ts
+24
View File
@@ -108,6 +108,30 @@ describe('InfrastructureMonitoringService', () => {
expect(result.trends.cpuUsagePercent).toEqual([]);
expect(result.serviceMetrics.every((item) => item.available === false)).toBe(true);
expect(result.error).not.toContain('ECONNREFUSED');
expect(result.disks).toEqual([]);
});
it('keeps system, data and additional disks distinct regardless of Prometheus series order', async () => {
const metrics = [
{ instance: 'host:9100', device: '/dev/sdb1', mountpoint: '/data', fstype: 'ext4' },
{ instance: 'host:9100', device: '/dev/sda2', mountpoint: '/', fstype: 'ext4' },
{ instance: 'host:9100', device: '/dev/nvme1n1p1', mountpoint: '/archive', fstype: 'xfs' },
];
jest.spyOn(global, 'fetch').mockImplementation(async (input) => {
const url = new URL(String(input));
const query = url.searchParams.get('query') ?? '';
if (url.pathname.endsWith('/alerts')) return success({ alerts: [] });
if (!query.includes('node_filesystem_')) return success({ result: [] });
expect(query).not.toContain('mountpoint="/"');
if (url.pathname.endsWith('/query_range')) return success({ result: [...metrics].reverse().map((metric) => ({ metric, values: [[1765000060, metric.mountpoint === '/' ? '91' : '12']] })) });
return success({ result: metrics.map((metric) => ({ metric, value: [1765000060, query.startsWith('(1') ? (metric.mountpoint === '/' ? '91' : '12') : query.includes('avail') ? '9' : '100'] })) });
});
const result = await new InfrastructureMonitoringService(new ConfigService(), prisma as never).overview('1h');
expect(result.disks.map((disk) => disk.mountpoint)).toEqual(['/', '/archive', '/data']);
expect(result.disks[0]).toMatchObject({ usagePercent: 91, totalBytes: 100, availableBytes: 9, trend: [{ timestamp: new Date(1765000060000).toISOString(), value: 91 }] });
expect(result.disks[2].trend[0].value).toBe(12);
expect(result.metrics.diskUsagePercent).toBe(91);
expect(result.trends.diskUsagePercent[0].value).toBe(91);
});
it('excludes only the current alert occurrence after the current administrator marks it read', async () => {
api/src/infrastructure-monitoring/infrastructure-monitoring.service.ts
+36 -7
View File
@@ -47,9 +47,9 @@ const QUERIES = {
memoryUsagePercent: '(1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) * 100',
memoryTotalBytes: 'node_memory_MemTotal_bytes',
memoryAvailableBytes: 'node_memory_MemAvailable_bytes',
diskUsagePercent: '(1 - (node_filesystem_avail_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"} / node_filesystem_size_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"})) * 100',
diskTotalBytes: 'node_filesystem_size_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"}',
diskAvailableBytes: 'node_filesystem_avail_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"}',
diskUsagePercent: '(1 - (node_filesystem_avail_bytes{device=~"/dev/.+",fstype!~"tmpfs|devtmpfs|overlay|squashfs|ramfs"} / node_filesystem_size_bytes{device=~"/dev/.+",fstype!~"tmpfs|devtmpfs|overlay|squashfs|ramfs"})) * 100',
diskTotalBytes: 'node_filesystem_size_bytes{device=~"/dev/.+",fstype!~"tmpfs|devtmpfs|overlay|squashfs|ramfs"}',
diskAvailableBytes: 'node_filesystem_avail_bytes{device=~"/dev/.+",fstype!~"tmpfs|devtmpfs|overlay|squashfs|ramfs"}',
networkReceiveBytesPerSecond: 'sum(rate(node_network_receive_bytes_total{device!~"lo"}[5m]))',
networkTransmitBytesPerSecond: 'sum(rate(node_network_transmit_bytes_total{device!~"lo"}[5m]))',
load1: 'node_load1',
@@ -132,6 +132,15 @@ function matrixValues(response: PrometheusQueryResponse): InfrastructureMetricPo
});
}
function diskIdentity(metric: Record<string, string>) {
return JSON.stringify([metric.instance ?? '', metric.device ?? '', metric.mountpoint ?? '', metric.fstype ?? '']);
}
// Preserve the legacy scalar fields as root-only; never silently pick the first disk.
function rootSeries(response: PrometheusQueryResponse): PrometheusQueryResponse {
return { ...response, data: { result: (response.data?.result ?? []).filter((item) => item.metric.mountpoint === '/') } };
}
function emptyMetrics(): InfrastructureMonitoringOverview['metrics'] {
return {
cpuUsagePercent: null,
@@ -200,7 +209,8 @@ export class InfrastructureMonitoringService {
activeAlerts: alerts.length,
},
metrics: instant.metrics,
trends,
trends: trends.metrics,
disks: instant.disks.map((disk) => ({ ...disk, trend: trends.disks.get(disk.id) ?? [] })),
services,
serviceMetrics,
alerts,
@@ -263,8 +273,21 @@ export class InfrastructureMonitoringService {
const keys = Object.keys(emptyMetrics()) as Array<keyof InfrastructureMonitoringOverview['metrics']>;
const responses = await Promise.all([...keys.map((key) => this.query(QUERIES[key])), this.query(QUERIES.lastSampleAt)]);
const metrics = emptyMetrics();
keys.forEach((key, index) => { metrics[key] = vectorValue(responses[index]); });
return { metrics, lastSampleAt: vectorValue(responses[responses.length - 1]) };
keys.forEach((key, index) => { metrics[key] = vectorValue(key.startsWith('disk') ? rootSeries(responses[index]) : responses[index]); });
const diskSamples = (key: keyof typeof metrics) => responses[keys.indexOf(key)].data?.result ?? [];
const usage = new Map(diskSamples('diskUsagePercent').map((item) => [diskIdentity(item.metric), finiteNumber(item.value?.[1])]));
const available = new Map(diskSamples('diskAvailableBytes').map((item) => [diskIdentity(item.metric), finiteNumber(item.value?.[1])]));
const disks = diskSamples('diskTotalBytes')
.filter((item) => item.metric.mountpoint && (finiteNumber(item.value?.[1]) ?? 0) > 0)
.map((item) => ({
id: diskIdentity(item.metric), instance: item.metric.instance ?? '', device: item.metric.device ?? '',
mountpoint: item.metric.mountpoint, filesystem: item.metric.fstype ?? '',
totalBytes: finiteNumber(item.value?.[1]),
availableBytes: available.get(diskIdentity(item.metric)) ?? null,
usagePercent: usage.get(diskIdentity(item.metric)) ?? null,
}))
.sort((left, right) => left.instance.localeCompare(right.instance) || (left.mountpoint === '/' ? -1 : right.mountpoint === '/' ? 1 : left.mountpoint.localeCompare(right.mountpoint)));
return { metrics, disks, lastSampleAt: vectorValue(responses[responses.length - 1]) };
}
private async loadTrends(range: InfrastructureMonitoringRange) {
@@ -273,7 +296,12 @@ export class InfrastructureMonitoringService {
const start = end - config.seconds;
const keys = Object.keys(emptyTrends()) as Array<keyof InfrastructureMonitoringOverview['trends']>;
const responses = await Promise.all(keys.map((key) => this.queryRange(QUERIES[key], start, end, config.step)));
return Object.fromEntries(keys.map((key, index) => [key, matrixValues(responses[index])])) as InfrastructureMonitoringOverview['trends'];
return {
metrics: Object.fromEntries(keys.map((key, index) => [key, matrixValues(key === 'diskUsagePercent' ? rootSeries(responses[index]) : responses[index])])) as InfrastructureMonitoringOverview['trends'],
disks: new Map((responses[keys.indexOf('diskUsagePercent')].data?.result ?? []).map((item) => [
diskIdentity(item.metric), matrixValues({ status: 'success', data: { result: [item] } }),
])),
};
}
private parseServices(response: PrometheusQueryResponse): InfrastructureServiceStatus[] {
@@ -356,6 +384,7 @@ export class InfrastructureMonitoringService {
error: 'Prometheus监控数据当前不可用，请检查采集与服务状态',
summary: { overallStatus: 'unknown', serviceTotal: services.length, serviceHealthy: 0, warningAlerts: 0, criticalAlerts: 0, activeAlerts: 0 },
metrics: emptyMetrics(),
disks: [],
trends: emptyTrends(),
services,
serviceMetrics: SERVICE_METRIC_DEFINITIONS.map((group) => ({ key: group.key, name: group.name, available: false, metrics: [] })),