fix: deduplicate filesystem mounts in monitoring

This commit is contained in:
hectorzhao
2026-08-31 23:39:24 +08:00
parent 12668cee87
commit 1f2dfb5caf
14 changed files with 246 additions and 51 deletions
@@ -0,0 +1,15 @@
// A bind mount is another path to the same filesystem, not another disk.
export const FILESYSTEM_LABELS = 'instance, device, fstype';
export const FILESYSTEM_SELECTOR = '{device=~"/dev/.+",fstype!~"tmpfs|devtmpfs|overlay|squashfs|ramfs"}';
export const FILESYSTEM_USAGE_PERCENT = `max by (${FILESYSTEM_LABELS}) ((1 - node_filesystem_avail_bytes${FILESYSTEM_SELECTOR} / node_filesystem_size_bytes${FILESYSTEM_SELECTOR}) * 100)`;
export const FILESYSTEM_INODE_USAGE_PERCENT = `max by (${FILESYSTEM_LABELS}) ((1 - node_filesystem_files_free${FILESYSTEM_SELECTOR} / node_filesystem_files${FILESYSTEM_SELECTOR}) * 100)`;
export function filesystemIdentity(metric: Record<string, string>) {
return JSON.stringify([metric.instance ?? '', metric.device ?? '', metric.fstype ?? '']);
}
// Prefer the filesystem's shallowest visible path; tie-breaking is deterministic.
export function compareMountpoints(left: string, right: string) {
return left.split('/').filter(Boolean).length - right.split('/').filter(Boolean).length
|| left.length - right.length || left.localeCompare(right);
}
@@ -1,5 +1,8 @@
import { BadRequestException } from '@nestjs/common';
import { DEFAULT_ALERT_THRESHOLDS, InfrastructureAlertSettingsService } from './infrastructure-alert-settings.service';
import { readFileSync } from 'node:fs';
import { resolve } from 'node:path';
import { FILESYSTEM_INODE_USAGE_PERCENT, FILESYSTEM_USAGE_PERCENT } from './filesystem-metrics';
describe('InfrastructureAlertSettingsService', () => {
const service = new InfrastructureAlertSettingsService({} as never, { get: () => undefined } as never);
@@ -14,8 +17,25 @@ describe('InfrastructureAlertSettingsService', () => {
expect(rules).toContain('sum(increase(cmpp_api_http_requests_total');
expect(rules).not.toContain('mountpoint="/"');
expect(rules).toContain('device=~"/dev/.+"');
expect(rules).toContain('{{ $labels.mountpoint }}');
expect(rules).not.toContain('{{ $labels.mountpoint }}');
expect(rules).toContain('{{ $labels.device }}');
expect(rules).toContain('{{ $labels.fstype }}');
expect(rules).toContain(FILESYSTEM_USAGE_PERCENT);
});
it('keeps static capacity and inode rules aligned with filesystem-level deduplication', () => {
for (const file of ['cmpp-alerts.yml', 'cmpp-managed-alerts.yml']) {
const content = readFileSync(resolve(__dirname, '../../../tools/monitoring', file), 'utf8');
const blocks = [...content.matchAll(/ - alert: (HostRoot(?:Disk|Inode)Usage(?:Warning|Critical))\r?\n([\s\S]*?)(?=\r?\n - alert:|$)/g)];
expect(blocks).toHaveLength(file === 'cmpp-alerts.yml' ? 4 : 2);
for (const [, name, body] of blocks) {
const expression = body.match(/^\s+expr: (.+)$/m)![1].trim();
const base = name.includes('Inode') ? FILESYSTEM_INODE_USAGE_PERCENT : FILESYSTEM_USAGE_PERCENT;
expect(expression).toBe(name.endsWith('Warning') ? `(${base} > 80) and (${base} <= 90)` : `${base} > 90`);
expect(body).not.toContain('$labels.mountpoint');
expect(body).toContain('$labels.device');
}
}
});
it('rejects unknown keys and warning thresholds that are not below critical', () => {
@@ -6,6 +6,7 @@ import { mkdir, readFile, rename, rm, writeFile } from 'node:fs/promises';
import { dirname } from 'node:path';
import { promisify } from 'node:util';
import { PrismaService } from '../prisma/prisma.service';
import { FILESYSTEM_USAGE_PERCENT } from './filesystem-metrics';
import type { InfrastructureAlertSettings, InfrastructureAlertThresholds } from './infrastructure-monitoring.contracts';
const execFileAsync = promisify(execFile);
@@ -13,7 +14,7 @@ const execFileAsync = promisify(execFile);
export const ALERT_THRESHOLD_DEFINITIONS = [
{ key: 'hostCpu', label: '主机 CPU 使用率', unit: '%', min: 1, max: 100, step: 1, warning: 80, critical: 90, expr: '100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100)', names: ['HostCpuUsageWarning', 'HostCpuUsageCritical'], service: 'host', durations: ['10m', '5m'] },
{ key: 'hostMemory', label: '主机内存使用率', unit: '%', min: 1, max: 100, step: 1, warning: 85, critical: 95, expr: '(1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100', names: ['HostMemoryUsageWarning', 'HostMemoryUsageCritical'], service: 'host', durations: ['10m', '5m'] },
{ key: 'hostDisk', label: '磁盘(所有挂载点)使用率', unit: '%', min: 1, max: 100, step: 1, warning: 80, critical: 90, expr: '(1 - node_filesystem_avail_bytes{device=~"/dev/.+",fstype!~"tmpfs|devtmpfs|overlay|squashfs|ramfs"} / node_filesystem_size_bytes{device=~"/dev/.+",fstype!~"tmpfs|devtmpfs|overlay|squashfs|ramfs"}) * 100', names: ['HostRootDiskUsageWarning', 'HostRootDiskUsageCritical'], service: 'host', durations: ['15m', '5m'] },
{ key: 'hostDisk', label: '磁盘(独立文件系统)使用率', unit: '%', min: 1, max: 100, step: 1, warning: 80, critical: 90, expr: FILESYSTEM_USAGE_PERCENT, names: ['HostRootDiskUsageWarning', 'HostRootDiskUsageCritical'], service: 'host', durations: ['15m', '5m'] },
{ key: 'apiError', label: 'API 5xx 错误率', unit: '%', min: 0.1, max: 100, step: 0.1, warning: 1, critical: 5, expr: '100 * sum(rate(cmpp_api_http_requests_total{status=~"5.."}[5m])) / clamp_min(sum(rate(cmpp_api_http_requests_total[5m])), 0.001)', guard: 'sum(increase(cmpp_api_http_requests_total{status=~"5.."}[5m])) >= 5', names: ['CmppApiHttpErrorRateWarning', 'CmppApiHttpErrorRateCritical'], service: 'api', durations: ['5m', '5m'] },
{ key: 'apiLatency', label: 'API P95 响应时间', unit: '秒', min: 0.1, max: 60, step: 0.1, warning: 1, critical: 3, expr: 'histogram_quantile(0.95, sum by (le) (rate(cmpp_api_http_request_duration_seconds_bucket[10m])))', names: ['CmppApiLatencyWarning', 'CmppApiLatencyCritical'], service: 'api', durations: ['10m', '5m'] },
{ key: 'apiEventLoop', label: 'API 事件循环 P99', unit: '秒', min: 0.01, max: 10, step: 0.01, warning: 0.2, critical: 1, expr: 'cmpp_api_nodejs_event_loop_lag_p99_seconds', names: ['CmppApiEventLoopLagWarning', 'CmppApiEventLoopLagCritical'], service: 'api', durations: ['10m', '5m'] },
@@ -113,7 +114,7 @@ export class InfrastructureAlertSettingsService {
// 低样本量与“未配置容量上限”必须继续作为固定保护条件,避免单次错误或除零结果触发伪告警。
const guard = 'guard' in definition ? ` and (${definition.guard})` : '';
const expr = isWarning ? `(${definition.expr} > ${values[0]}) and (${definition.expr} <= ${values[1]})${guard}` : `(${definition.expr} > ${values[1]})${guard}`;
const diskLocation = definition.key === 'hostDisk' ? ' 挂载点{{ $labels.mountpoint }};设备:{{ $labels.device }}。' : '';
const diskLocation = definition.key === 'hostDisk' ? ' 设备{{ $labels.device }};文件系统:{{ $labels.fstype }}(绑定挂载已合并)。' : '';
lines.push(` - alert: ${definition.names[index]}`, ` expr: ${expr}`, ` for: ${definition.durations[index]}`, ' labels:', ` severity: ${isWarning ? 'warning' : 'critical'}`, ` service: ${definition.service}`, ' annotations:', ` summary: "${definition.label}${isWarning ? '达到警告阈值' : '达到严重阈值'}"`, ` description: "${definition.label}持续超过${values[index]}${definition.unit}${diskLocation}"`, ' currentValue: "{{ $value }}"', ` threshold: "${values[index]}${definition.unit}"`);
}
}
@@ -46,6 +46,7 @@ export type InfrastructureMonitoringOverview = {
instance: string;
device: string;
mountpoint: string;
mountpoints: string[];
filesystem: string;
usagePercent: number | null;
totalBytes: number | null;
@@ -2,6 +2,7 @@ import { BadRequestException } from '@nestjs/common';
import { ConfigService } from '@nestjs/config';
import { createHash } from 'node:crypto';
import { InfrastructureMonitoringService } from './infrastructure-monitoring.service';
import { FILESYSTEM_USAGE_PERCENT, filesystemIdentity } from './filesystem-metrics';
function success(data: unknown) {
return {
@@ -124,7 +125,7 @@ describe('InfrastructureMonitoringService', () => {
if (!query.includes('node_filesystem_')) return success({ result: [] });
expect(query).not.toContain('mountpoint="/"');
if (url.pathname.endsWith('/query_range')) return success({ result: [...metrics].reverse().map((metric) => ({ metric, values: [[1765000060, metric.mountpoint === '/' ? '91' : '12']] })) });
return success({ result: metrics.map((metric) => ({ metric, value: [1765000060, query.startsWith('(1') ? (metric.mountpoint === '/' ? '91' : '12') : query.includes('avail') ? '9' : '100'] })) });
return success({ result: metrics.map((metric) => ({ metric, value: [1765000060, query === FILESYSTEM_USAGE_PERCENT ? (metric.mountpoint === '/' ? '91' : '12') : query.includes('avail') ? '9' : '100'] })) });
});
const result = await new InfrastructureMonitoringService(new ConfigService(), prisma as never).overview('1h');
expect(result.disks.map((disk) => disk.mountpoint)).toEqual(['/', '/archive', '/data']);
@@ -134,6 +135,74 @@ describe('InfrastructureMonitoringService', () => {
expect(result.trends.diskUsagePercent[0].value).toBe(91);
});
it('merges bind mounts without summing capacity, keeps a stable identity and uses one aggregated trend', async () => {
const data = { instance: 'host:9100', device: '/dev/sdb1', fstype: 'ext4' };
const root = { instance: 'host:9100', device: '/dev/sda2', fstype: 'ext4' };
let mounts = ['/var/lib/redis', '/var/lib/pgsql', '/data', '/var/lib/minio'];
jest.spyOn(global, 'fetch').mockImplementation(async (input) => {
const url = new URL(String(input));
const query = url.searchParams.get('query') ?? '';
if (url.pathname.endsWith('/alerts')) return success({ alerts: [] });
if (!query.includes('node_filesystem_')) return success({ result: [] });
if (url.pathname.endsWith('/query_range')) {
expect(query).toBe(FILESYSTEM_USAGE_PERCENT);
return success({ result: [
{ metric: data, values: [[1765000000, '82'], [1765000060, 'NaN'], [1765000120, '83.5']] },
{ metric: root, values: [[1765000000, '91']] },
] });
}
if (query.startsWith('node_filesystem_size_bytes')) return success({ result: [
...mounts.map((mountpoint) => ({ metric: { ...data, mountpoint }, value: [1765000120, '100'] })),
...['/var/root-bind', '/'].map((mountpoint) => ({ metric: { ...root, mountpoint }, value: [1765000120, '200'] })),
] });
if (query === FILESYSTEM_USAGE_PERCENT) return success({ result: [
{ metric: data, value: [1765000120, '83.5'] }, { metric: root, value: [1765000120, '91'] },
] });
expect(query).toContain('min by (instance, device, fstype)');
return success({ result: [{ metric: data, value: [1765000120, '16.5'] }, { metric: root, value: [1765000120, '18'] }] });
});
const service = new InfrastructureMonitoringService(new ConfigService(), prisma as never);
const result = await service.overview('1h');
expect(result.available).toBe(true);
expect(result.disks).toHaveLength(2);
expect(result.disks[0]).toMatchObject({ mountpoint: '/', mountpoints: ['/', '/var/root-bind'], totalBytes: 200 });
expect(result.metrics.diskUsagePercent).toBe(91);
expect(result.trends.diskUsagePercent[0].value).toBe(91);
const disk = result.disks[1];
expect(disk).toMatchObject({ id: filesystemIdentity(data), mountpoint: '/data', totalBytes: 100, availableBytes: 16.5, usagePercent: 83.5 });
expect(disk.mountpoints).toEqual(['/data', '/var/lib/minio', '/var/lib/pgsql', '/var/lib/redis']);
expect(disk.trend.map((point) => point.value)).toEqual([82, 83.5]);
mounts.reverse();
expect((await service.overview('1h')).disks).toEqual(result.disks);
mounts = ['/var/lib/redis'];
const aliasOnly = (await service.overview('1h')).disks.find((item) => item.id === disk.id)!;
expect(aliasOnly.mountpoint).toBe('/var/lib/redis');
expect(aliasOnly.trend).toEqual(disk.trend);
});
it('does not merge different hosts, devices or filesystem types with identical capacity', async () => {
const metrics = [
{ instance: 'a:9100', device: '/dev/sdb1', fstype: 'ext4', mountpoint: '/data' },
{ instance: 'b:9100', device: '/dev/sdb1', fstype: 'ext4', mountpoint: '/data' },
{ instance: 'a:9100', device: '/dev/sdc1', fstype: 'ext4', mountpoint: '/archive' },
{ instance: 'a:9100', device: '/dev/sdb1', fstype: 'xfs', mountpoint: '/other' },
];
jest.spyOn(global, 'fetch').mockImplementation(async (input) => {
const url = new URL(String(input));
if (url.pathname.endsWith('/alerts')) return success({ alerts: [] });
if (url.searchParams.get('query')?.startsWith('node_filesystem_size_bytes')) {
return success({ result: metrics.map((metric) => ({ metric, value: [1765000060, '100'] })) });
}
return success({ result: [] });
});
const result = await new InfrastructureMonitoringService(new ConfigService(), prisma as never).overview('1h');
expect(result.disks).toHaveLength(4);
expect(new Set(result.disks.map((disk) => disk.id)).size).toBe(4);
expect(result.disks.every((disk) => disk.usagePercent === null && disk.availableBytes === null && disk.trend.length === 0)).toBe(true);
expect(result.metrics.diskUsagePercent).toBeNull();
expect(result.trends.diskUsagePercent).toEqual([]);
});
it('excludes only the current alert occurrence after the current administrator marks it read', async () => {
const labels = { alertname: 'QaWarning', severity: 'warning', service: 'qa-preview' };
const fingerprint = createHash('sha256').update(JSON.stringify(Object.entries(labels).sort(([left], [right]) => left.localeCompare(right)))).digest('hex').slice(0, 24);
@@ -3,6 +3,7 @@ import { ConfigService } from '@nestjs/config';
import { Prisma } from '@prisma/client';
import { createHash } from 'node:crypto';
import { PrismaService } from '../prisma/prisma.service';
import { compareMountpoints, FILESYSTEM_LABELS, FILESYSTEM_SELECTOR, FILESYSTEM_USAGE_PERCENT, filesystemIdentity } from './filesystem-metrics';
import type {
InfrastructureAlert,
InfrastructureMetricPoint,
@@ -47,9 +48,9 @@ const QUERIES = {
memoryUsagePercent: '(1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) * 100',
memoryTotalBytes: 'node_memory_MemTotal_bytes',
memoryAvailableBytes: 'node_memory_MemAvailable_bytes',
diskUsagePercent: '(1 - (node_filesystem_avail_bytes{device=~"/dev/.+",fstype!~"tmpfs|devtmpfs|overlay|squashfs|ramfs"} / node_filesystem_size_bytes{device=~"/dev/.+",fstype!~"tmpfs|devtmpfs|overlay|squashfs|ramfs"})) * 100',
diskTotalBytes: 'node_filesystem_size_bytes{device=~"/dev/.+",fstype!~"tmpfs|devtmpfs|overlay|squashfs|ramfs"}',
diskAvailableBytes: 'node_filesystem_avail_bytes{device=~"/dev/.+",fstype!~"tmpfs|devtmpfs|overlay|squashfs|ramfs"}',
diskUsagePercent: FILESYSTEM_USAGE_PERCENT,
diskTotalBytes: `node_filesystem_size_bytes${FILESYSTEM_SELECTOR}`,
diskAvailableBytes: `min by (${FILESYSTEM_LABELS}) (node_filesystem_avail_bytes${FILESYSTEM_SELECTOR})`,
networkReceiveBytesPerSecond: 'sum(rate(node_network_receive_bytes_total{device!~"lo"}[5m]))',
networkTransmitBytesPerSecond: 'sum(rate(node_network_transmit_bytes_total{device!~"lo"}[5m]))',
load1: 'node_load1',
@@ -132,15 +133,6 @@ function matrixValues(response: PrometheusQueryResponse): InfrastructureMetricPo
});
}
function diskIdentity(metric: Record<string, string>) {
return JSON.stringify([metric.instance ?? '', metric.device ?? '', metric.mountpoint ?? '', metric.fstype ?? '']);
}
// Preserve the legacy scalar fields as root-only; never silently pick the first disk.
function rootSeries(response: PrometheusQueryResponse): PrometheusQueryResponse {
return { ...response, data: { result: (response.data?.result ?? []).filter((item) => item.metric.mountpoint === '/') } };
}
function emptyMetrics(): InfrastructureMonitoringOverview['metrics'] {
return {
cpuUsagePercent: null,
@@ -195,6 +187,7 @@ export class InfrastructureMonitoringService {
const warningAlerts = alerts.filter((item) => item.severity === 'warning').length;
const criticalAlerts = alerts.filter((item) => item.severity === 'critical').length;
const overallStatus = criticalAlerts > 0 ? 'critical' : warningAlerts > 0 ? 'warning' : 'healthy';
const rootDisk = instant.disks.find((disk) => disk.mountpoints.includes('/'));
return {
available: true,
range,
@@ -209,7 +202,7 @@ export class InfrastructureMonitoringService {
activeAlerts: alerts.length,
},
metrics: instant.metrics,
trends: trends.metrics,
trends: { ...trends.metrics, diskUsagePercent: rootDisk ? trends.disks.get(rootDisk.id) ?? [] : [] },
disks: instant.disks.map((disk) => ({ ...disk, trend: trends.disks.get(disk.id) ?? [] })),
services,
serviceMetrics,
@@ -273,20 +266,34 @@ export class InfrastructureMonitoringService {
const keys = Object.keys(emptyMetrics()) as Array<keyof InfrastructureMonitoringOverview['metrics']>;
const responses = await Promise.all([...keys.map((key) => this.query(QUERIES[key])), this.query(QUERIES.lastSampleAt)]);
const metrics = emptyMetrics();
keys.forEach((key, index) => { metrics[key] = vectorValue(key.startsWith('disk') ? rootSeries(responses[index]) : responses[index]); });
keys.forEach((key, index) => { if (!key.startsWith('disk')) metrics[key] = vectorValue(responses[index]); });
const diskSamples = (key: keyof typeof metrics) => responses[keys.indexOf(key)].data?.result ?? [];
const usage = new Map(diskSamples('diskUsagePercent').map((item) => [diskIdentity(item.metric), finiteNumber(item.value?.[1])]));
const available = new Map(diskSamples('diskAvailableBytes').map((item) => [diskIdentity(item.metric), finiteNumber(item.value?.[1])]));
const disks = diskSamples('diskTotalBytes')
.filter((item) => item.metric.mountpoint && (finiteNumber(item.value?.[1]) ?? 0) > 0)
.map((item) => ({
id: diskIdentity(item.metric), instance: item.metric.instance ?? '', device: item.metric.device ?? '',
mountpoint: item.metric.mountpoint, filesystem: item.metric.fstype ?? '',
totalBytes: finiteNumber(item.value?.[1]),
availableBytes: available.get(diskIdentity(item.metric)) ?? null,
usagePercent: usage.get(diskIdentity(item.metric)) ?? null,
}))
const usage = new Map(diskSamples('diskUsagePercent').map((item) => [filesystemIdentity(item.metric), finiteNumber(item.value?.[1])]));
const available = new Map(diskSamples('diskAvailableBytes').map((item) => [filesystemIdentity(item.metric), finiteNumber(item.value?.[1])]));
const groups = new Map<string, PrometheusSeries[]>();
for (const item of diskSamples('diskTotalBytes')) {
if (!item.metric.device || !item.metric.mountpoint || (finiteNumber(item.value?.[1]) ?? 0) <= 0) continue;
const id = filesystemIdentity(item.metric);
const group = groups.get(id) ?? [];
group.push(item);
groups.set(id, group);
}
const disks = [...groups].map(([id, items]) => {
const mountpoints = [...new Set(items.map((item) => item.metric.mountpoint))].sort(compareMountpoints);
const metric = items[0].metric;
return {
id, instance: metric.instance ?? '', device: metric.device, filesystem: metric.fstype ?? '',
mountpoint: mountpoints[0], mountpoints,
// Never sum aliases. Max/min also tolerate slight sampling differences.
totalBytes: Math.max(...items.map((item) => finiteNumber(item.value?.[1])!)),
availableBytes: available.get(id) ?? null, usagePercent: usage.get(id) ?? null,
};
})
.sort((left, right) => left.instance.localeCompare(right.instance) || (left.mountpoint === '/' ? -1 : right.mountpoint === '/' ? 1 : left.mountpoint.localeCompare(right.mountpoint)));
const rootDisk = disks.find((disk) => disk.mountpoints.includes('/'));
metrics.diskUsagePercent = rootDisk?.usagePercent ?? null;
metrics.diskTotalBytes = rootDisk?.totalBytes ?? null;
metrics.diskAvailableBytes = rootDisk?.availableBytes ?? null;
return { metrics, disks, lastSampleAt: vectorValue(responses[responses.length - 1]) };
}
@@ -297,9 +304,9 @@ export class InfrastructureMonitoringService {
const keys = Object.keys(emptyTrends()) as Array<keyof InfrastructureMonitoringOverview['trends']>;
const responses = await Promise.all(keys.map((key) => this.queryRange(QUERIES[key], start, end, config.step)));
return {
metrics: Object.fromEntries(keys.map((key, index) => [key, matrixValues(key === 'diskUsagePercent' ? rootSeries(responses[index]) : responses[index])])) as InfrastructureMonitoringOverview['trends'],
metrics: Object.fromEntries(keys.map((key, index) => [key, key === 'diskUsagePercent' ? [] : matrixValues(responses[index])])) as InfrastructureMonitoringOverview['trends'],
disks: new Map((responses[keys.indexOf('diskUsagePercent')].data?.result ?? []).map((item) => [
diskIdentity(item.metric), matrixValues({ status: 'success', data: { result: [item] } }),
filesystemIdentity(item.metric), matrixValues({ status: 'success', data: { result: [item] } }),
])),
};
}