From 1f2dfb5caf1d56d6f0a5324e1ba01a17f9ce67de Mon Sep 17 00:00:00 2001 From: hectorzhao Date: Mon, 31 Aug 2026 23:39:24 +0800 Subject: [PATCH] fix: deduplicate filesystem mounts in monitoring --- .../filesystem-metrics.ts | 15 ++++ ...rastructure-alert-settings.service.spec.ts | 22 +++++- .../infrastructure-alert-settings.service.ts | 5 +- .../infrastructure-monitoring.contracts.ts | 1 + .../infrastructure-monitoring.service.spec.ts | 71 ++++++++++++++++++- .../infrastructure-monitoring.service.ts | 61 +++++++++------- docs/system-functional-test-cases.md | 13 +++- docs/testing-progress.md | 11 +++ src/api/types/infrastructure-monitoring.ts | 1 + .../AdminSystemMonitoringPage.css | 3 + .../AdminSystemMonitoringPage.tsx | 28 ++++++-- .../DiskMetricCards.test.tsx | 42 +++++++++++ tools/monitoring/cmpp-alerts.yml | 16 ++--- tools/monitoring/cmpp-managed-alerts.yml | 8 +-- 14 files changed, 246 insertions(+), 51 deletions(-) create mode 100644 api/src/infrastructure-monitoring/filesystem-metrics.ts create mode 100644 src/apps/admin/system-monitoring/DiskMetricCards.test.tsx diff --git a/api/src/infrastructure-monitoring/filesystem-metrics.ts b/api/src/infrastructure-monitoring/filesystem-metrics.ts new file mode 100644 index 0000000..f1d5d35 --- /dev/null +++ b/api/src/infrastructure-monitoring/filesystem-metrics.ts @@ -0,0 +1,15 @@ +// A bind mount is another path to the same filesystem, not another disk. +export const FILESYSTEM_LABELS = 'instance, device, fstype'; +export const FILESYSTEM_SELECTOR = '{device=~"/dev/.+",fstype!~"tmpfs|devtmpfs|overlay|squashfs|ramfs"}'; +export const FILESYSTEM_USAGE_PERCENT = `max by (${FILESYSTEM_LABELS}) ((1 - node_filesystem_avail_bytes${FILESYSTEM_SELECTOR} / node_filesystem_size_bytes${FILESYSTEM_SELECTOR}) * 100)`; +export const FILESYSTEM_INODE_USAGE_PERCENT = `max by (${FILESYSTEM_LABELS}) ((1 - node_filesystem_files_free${FILESYSTEM_SELECTOR} / node_filesystem_files${FILESYSTEM_SELECTOR}) * 100)`; + +export function filesystemIdentity(metric: Record) { + return JSON.stringify([metric.instance ?? '', metric.device ?? '', metric.fstype ?? '']); +} + +// Prefer the filesystem's shallowest visible path; tie-breaking is deterministic. +export function compareMountpoints(left: string, right: string) { + return left.split('/').filter(Boolean).length - right.split('/').filter(Boolean).length + || left.length - right.length || left.localeCompare(right); +} diff --git a/api/src/infrastructure-monitoring/infrastructure-alert-settings.service.spec.ts b/api/src/infrastructure-monitoring/infrastructure-alert-settings.service.spec.ts index c027b84..3352ed0 100644 --- a/api/src/infrastructure-monitoring/infrastructure-alert-settings.service.spec.ts +++ b/api/src/infrastructure-monitoring/infrastructure-alert-settings.service.spec.ts @@ -1,5 +1,8 @@ import { BadRequestException } from '@nestjs/common'; import { DEFAULT_ALERT_THRESHOLDS, InfrastructureAlertSettingsService } from './infrastructure-alert-settings.service'; +import { readFileSync } from 'node:fs'; +import { resolve } from 'node:path'; +import { FILESYSTEM_INODE_USAGE_PERCENT, FILESYSTEM_USAGE_PERCENT } from './filesystem-metrics'; describe('InfrastructureAlertSettingsService', () => { const service = new InfrastructureAlertSettingsService({} as never, { get: () => undefined } as never); @@ -14,8 +17,25 @@ describe('InfrastructureAlertSettingsService', () => { expect(rules).toContain('sum(increase(cmpp_api_http_requests_total'); expect(rules).not.toContain('mountpoint="/"'); expect(rules).toContain('device=~"/dev/.+"'); - expect(rules).toContain('{{ $labels.mountpoint }}'); + expect(rules).not.toContain('{{ $labels.mountpoint }}'); expect(rules).toContain('{{ $labels.device }}'); + expect(rules).toContain('{{ $labels.fstype }}'); + expect(rules).toContain(FILESYSTEM_USAGE_PERCENT); + }); + + it('keeps static capacity and inode rules aligned with filesystem-level deduplication', () => { + for (const file of ['cmpp-alerts.yml', 'cmpp-managed-alerts.yml']) { + const content = readFileSync(resolve(__dirname, '../../../tools/monitoring', file), 'utf8'); + const blocks = [...content.matchAll(/ - alert: (HostRoot(?:Disk|Inode)Usage(?:Warning|Critical))\r?\n([\s\S]*?)(?=\r?\n - alert:|$)/g)]; + expect(blocks).toHaveLength(file === 'cmpp-alerts.yml' ? 4 : 2); + for (const [, name, body] of blocks) { + const expression = body.match(/^\s+expr: (.+)$/m)![1].trim(); + const base = name.includes('Inode') ? FILESYSTEM_INODE_USAGE_PERCENT : FILESYSTEM_USAGE_PERCENT; + expect(expression).toBe(name.endsWith('Warning') ? `(${base} > 80) and (${base} <= 90)` : `${base} > 90`); + expect(body).not.toContain('$labels.mountpoint'); + expect(body).toContain('$labels.device'); + } + } }); it('rejects unknown keys and warning thresholds that are not below critical', () => { diff --git a/api/src/infrastructure-monitoring/infrastructure-alert-settings.service.ts b/api/src/infrastructure-monitoring/infrastructure-alert-settings.service.ts index f3ff23a..60cacc1 100644 --- a/api/src/infrastructure-monitoring/infrastructure-alert-settings.service.ts +++ b/api/src/infrastructure-monitoring/infrastructure-alert-settings.service.ts @@ -6,6 +6,7 @@ import { mkdir, readFile, rename, rm, writeFile } from 'node:fs/promises'; import { dirname } from 'node:path'; import { promisify } from 'node:util'; import { PrismaService } from '../prisma/prisma.service'; +import { FILESYSTEM_USAGE_PERCENT } from './filesystem-metrics'; import type { InfrastructureAlertSettings, InfrastructureAlertThresholds } from './infrastructure-monitoring.contracts'; const execFileAsync = promisify(execFile); @@ -13,7 +14,7 @@ const execFileAsync = promisify(execFile); export const ALERT_THRESHOLD_DEFINITIONS = [ { key: 'hostCpu', label: '主机 CPU 使用率', unit: '%', min: 1, max: 100, step: 1, warning: 80, critical: 90, expr: '100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100)', names: ['HostCpuUsageWarning', 'HostCpuUsageCritical'], service: 'host', durations: ['10m', '5m'] }, { key: 'hostMemory', label: '主机内存使用率', unit: '%', min: 1, max: 100, step: 1, warning: 85, critical: 95, expr: '(1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100', names: ['HostMemoryUsageWarning', 'HostMemoryUsageCritical'], service: 'host', durations: ['10m', '5m'] }, - { key: 'hostDisk', label: '磁盘（所有挂载点）使用率', unit: '%', min: 1, max: 100, step: 1, warning: 80, critical: 90, expr: '(1 - node_filesystem_avail_bytes{device=~"/dev/.+",fstype!~"tmpfs|devtmpfs|overlay|squashfs|ramfs"} / node_filesystem_size_bytes{device=~"/dev/.+",fstype!~"tmpfs|devtmpfs|overlay|squashfs|ramfs"}) * 100', names: ['HostRootDiskUsageWarning', 'HostRootDiskUsageCritical'], service: 'host', durations: ['15m', '5m'] }, + { key: 'hostDisk', label: '磁盘（独立文件系统）使用率', unit: '%', min: 1, max: 100, step: 1, warning: 80, critical: 90, expr: FILESYSTEM_USAGE_PERCENT, names: ['HostRootDiskUsageWarning', 'HostRootDiskUsageCritical'], service: 'host', durations: ['15m', '5m'] }, { key: 'apiError', label: 'API 5xx 错误率', unit: '%', min: 0.1, max: 100, step: 0.1, warning: 1, critical: 5, expr: '100 * sum(rate(cmpp_api_http_requests_total{status=~"5.."}[5m])) / clamp_min(sum(rate(cmpp_api_http_requests_total[5m])), 0.001)', guard: 'sum(increase(cmpp_api_http_requests_total{status=~"5.."}[5m])) >= 5', names: ['CmppApiHttpErrorRateWarning', 'CmppApiHttpErrorRateCritical'], service: 'api', durations: ['5m', '5m'] }, { key: 'apiLatency', label: 'API P95 响应时间', unit: '秒', min: 0.1, max: 60, step: 0.1, warning: 1, critical: 3, expr: 'histogram_quantile(0.95, sum by (le) (rate(cmpp_api_http_request_duration_seconds_bucket[10m])))', names: ['CmppApiLatencyWarning', 'CmppApiLatencyCritical'], service: 'api', durations: ['10m', '5m'] }, { key: 'apiEventLoop', label: 'API 事件循环 P99', unit: '秒', min: 0.01, max: 10, step: 0.01, warning: 0.2, critical: 1, expr: 'cmpp_api_nodejs_event_loop_lag_p99_seconds', names: ['CmppApiEventLoopLagWarning', 'CmppApiEventLoopLagCritical'], service: 'api', durations: ['10m', '5m'] }, @@ -113,7 +114,7 @@ export class InfrastructureAlertSettingsService { // 低样本量与“未配置容量上限”必须继续作为固定保护条件，避免单次错误或除零结果触发伪告警。 const guard = 'guard' in definition ? ` and (${definition.guard})` : ''; const expr = isWarning ? `(${definition.expr} > ${values[0]}) and (${definition.expr} <= ${values[1]})${guard}` : `(${definition.expr} > ${values[1]})${guard}`; - const diskLocation = definition.key === 'hostDisk' ? ' 挂载点：{{ $labels.mountpoint }}；设备：{{ $labels.device }}。' : ''; + const diskLocation = definition.key === 'hostDisk' ? ' 设备：{{ $labels.device }}；文件系统：{{ $labels.fstype }}（绑定挂载已合并）。' : ''; lines.push(` - alert: ${definition.names[index]}`, ` expr: ${expr}`, ` for: ${definition.durations[index]}`, ' labels:', ` severity: ${isWarning ? 'warning' : 'critical'}`, ` service: ${definition.service}`, ' annotations:', ` summary: "${definition.label}${isWarning ? '达到警告阈值' : '达到严重阈值'}"`, ` description: "${definition.label}持续超过${values[index]}${definition.unit}。${diskLocation}"`, ' currentValue: "{{ $value }}"', ` threshold: "${values[index]}${definition.unit}"`); } } diff --git a/api/src/infrastructure-monitoring/infrastructure-monitoring.contracts.ts b/api/src/infrastructure-monitoring/infrastructure-monitoring.contracts.ts index 56a15d3..cc4deef 100644 --- a/api/src/infrastructure-monitoring/infrastructure-monitoring.contracts.ts +++ b/api/src/infrastructure-monitoring/infrastructure-monitoring.contracts.ts @@ -46,6 +46,7 @@ export type InfrastructureMonitoringOverview = { instance: string; device: string; mountpoint: string; + mountpoints: string[]; filesystem: string; usagePercent: number | null; totalBytes: number | null; diff --git a/api/src/infrastructure-monitoring/infrastructure-monitoring.service.spec.ts b/api/src/infrastructure-monitoring/infrastructure-monitoring.service.spec.ts index 7932095..8c6fe12 100644 --- a/api/src/infrastructure-monitoring/infrastructure-monitoring.service.spec.ts +++ b/api/src/infrastructure-monitoring/infrastructure-monitoring.service.spec.ts @@ -2,6 +2,7 @@ import { BadRequestException } from '@nestjs/common'; import { ConfigService } from '@nestjs/config'; import { createHash } from 'node:crypto'; import { InfrastructureMonitoringService } from './infrastructure-monitoring.service'; +import { FILESYSTEM_USAGE_PERCENT, filesystemIdentity } from './filesystem-metrics'; function success(data: unknown) { return { @@ -124,7 +125,7 @@ describe('InfrastructureMonitoringService', () => { if (!query.includes('node_filesystem_')) return success({ result: [] }); expect(query).not.toContain('mountpoint="/"'); if (url.pathname.endsWith('/query_range')) return success({ result: [...metrics].reverse().map((metric) => ({ metric, values: [[1765000060, metric.mountpoint === '/' ? '91' : '12']] })) }); - return success({ result: metrics.map((metric) => ({ metric, value: [1765000060, query.startsWith('(1') ? (metric.mountpoint === '/' ? '91' : '12') : query.includes('avail') ? '9' : '100'] })) }); + return success({ result: metrics.map((metric) => ({ metric, value: [1765000060, query === FILESYSTEM_USAGE_PERCENT ? (metric.mountpoint === '/' ? '91' : '12') : query.includes('avail') ? '9' : '100'] })) }); }); const result = await new InfrastructureMonitoringService(new ConfigService(), prisma as never).overview('1h'); expect(result.disks.map((disk) => disk.mountpoint)).toEqual(['/', '/archive', '/data']); @@ -134,6 +135,74 @@ describe('InfrastructureMonitoringService', () => { expect(result.trends.diskUsagePercent[0].value).toBe(91); }); + it('merges bind mounts without summing capacity, keeps a stable identity and uses one aggregated trend', async () => { + const data = { instance: 'host:9100', device: '/dev/sdb1', fstype: 'ext4' }; + const root = { instance: 'host:9100', device: '/dev/sda2', fstype: 'ext4' }; + let mounts = ['/var/lib/redis', '/var/lib/pgsql', '/data', '/var/lib/minio']; + jest.spyOn(global, 'fetch').mockImplementation(async (input) => { + const url = new URL(String(input)); + const query = url.searchParams.get('query') ?? ''; + if (url.pathname.endsWith('/alerts')) return success({ alerts: [] }); + if (!query.includes('node_filesystem_')) return success({ result: [] }); + if (url.pathname.endsWith('/query_range')) { + expect(query).toBe(FILESYSTEM_USAGE_PERCENT); + return success({ result: [ + { metric: data, values: [[1765000000, '82'], [1765000060, 'NaN'], [1765000120, '83.5']] }, + { metric: root, values: [[1765000000, '91']] }, + ] }); + } + if (query.startsWith('node_filesystem_size_bytes')) return success({ result: [ + ...mounts.map((mountpoint) => ({ metric: { ...data, mountpoint }, value: [1765000120, '100'] })), + ...['/var/root-bind', '/'].map((mountpoint) => ({ metric: { ...root, mountpoint }, value: [1765000120, '200'] })), + ] }); + if (query === FILESYSTEM_USAGE_PERCENT) return success({ result: [ + { metric: data, value: [1765000120, '83.5'] }, { metric: root, value: [1765000120, '91'] }, + ] }); + expect(query).toContain('min by (instance, device, fstype)'); + return success({ result: [{ metric: data, value: [1765000120, '16.5'] }, { metric: root, value: [1765000120, '18'] }] }); + }); + const service = new InfrastructureMonitoringService(new ConfigService(), prisma as never); + const result = await service.overview('1h'); + expect(result.available).toBe(true); + expect(result.disks).toHaveLength(2); + expect(result.disks[0]).toMatchObject({ mountpoint: '/', mountpoints: ['/', '/var/root-bind'], totalBytes: 200 }); + expect(result.metrics.diskUsagePercent).toBe(91); + expect(result.trends.diskUsagePercent[0].value).toBe(91); + const disk = result.disks[1]; + expect(disk).toMatchObject({ id: filesystemIdentity(data), mountpoint: '/data', totalBytes: 100, availableBytes: 16.5, usagePercent: 83.5 }); + expect(disk.mountpoints).toEqual(['/data', '/var/lib/minio', '/var/lib/pgsql', '/var/lib/redis']); + expect(disk.trend.map((point) => point.value)).toEqual([82, 83.5]); + mounts.reverse(); + expect((await service.overview('1h')).disks).toEqual(result.disks); + mounts = ['/var/lib/redis']; + const aliasOnly = (await service.overview('1h')).disks.find((item) => item.id === disk.id)!; + expect(aliasOnly.mountpoint).toBe('/var/lib/redis'); + expect(aliasOnly.trend).toEqual(disk.trend); + }); + + it('does not merge different hosts, devices or filesystem types with identical capacity', async () => { + const metrics = [ + { instance: 'a:9100', device: '/dev/sdb1', fstype: 'ext4', mountpoint: '/data' }, + { instance: 'b:9100', device: '/dev/sdb1', fstype: 'ext4', mountpoint: '/data' }, + { instance: 'a:9100', device: '/dev/sdc1', fstype: 'ext4', mountpoint: '/archive' }, + { instance: 'a:9100', device: '/dev/sdb1', fstype: 'xfs', mountpoint: '/other' }, + ]; + jest.spyOn(global, 'fetch').mockImplementation(async (input) => { + const url = new URL(String(input)); + if (url.pathname.endsWith('/alerts')) return success({ alerts: [] }); + if (url.searchParams.get('query')?.startsWith('node_filesystem_size_bytes')) { + return success({ result: metrics.map((metric) => ({ metric, value: [1765000060, '100'] })) }); + } + return success({ result: [] }); + }); + const result = await new InfrastructureMonitoringService(new ConfigService(), prisma as never).overview('1h'); + expect(result.disks).toHaveLength(4); + expect(new Set(result.disks.map((disk) => disk.id)).size).toBe(4); + expect(result.disks.every((disk) => disk.usagePercent === null && disk.availableBytes === null && disk.trend.length === 0)).toBe(true); + expect(result.metrics.diskUsagePercent).toBeNull(); + expect(result.trends.diskUsagePercent).toEqual([]); + }); + it('excludes only the current alert occurrence after the current administrator marks it read', async () => { const labels = { alertname: 'QaWarning', severity: 'warning', service: 'qa-preview' }; const fingerprint = createHash('sha256').update(JSON.stringify(Object.entries(labels).sort(([left], [right]) => left.localeCompare(right)))).digest('hex').slice(0, 24); diff --git a/api/src/infrastructure-monitoring/infrastructure-monitoring.service.ts b/api/src/infrastructure-monitoring/infrastructure-monitoring.service.ts index d82e1ed..1ef753a 100644 --- a/api/src/infrastructure-monitoring/infrastructure-monitoring.service.ts +++ b/api/src/infrastructure-monitoring/infrastructure-monitoring.service.ts @@ -3,6 +3,7 @@ import { ConfigService } from '@nestjs/config'; import { Prisma } from '@prisma/client'; import { createHash } from 'node:crypto'; import { PrismaService } from '../prisma/prisma.service'; +import { compareMountpoints, FILESYSTEM_LABELS, FILESYSTEM_SELECTOR, FILESYSTEM_USAGE_PERCENT, filesystemIdentity } from './filesystem-metrics'; import type { InfrastructureAlert, InfrastructureMetricPoint, @@ -47,9 +48,9 @@ const QUERIES = { memoryUsagePercent: '(1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) * 100', memoryTotalBytes: 'node_memory_MemTotal_bytes', memoryAvailableBytes: 'node_memory_MemAvailable_bytes', - diskUsagePercent: '(1 - (node_filesystem_avail_bytes{device=~"/dev/.+",fstype!~"tmpfs|devtmpfs|overlay|squashfs|ramfs"} / node_filesystem_size_bytes{device=~"/dev/.+",fstype!~"tmpfs|devtmpfs|overlay|squashfs|ramfs"})) * 100', - diskTotalBytes: 'node_filesystem_size_bytes{device=~"/dev/.+",fstype!~"tmpfs|devtmpfs|overlay|squashfs|ramfs"}', - diskAvailableBytes: 'node_filesystem_avail_bytes{device=~"/dev/.+",fstype!~"tmpfs|devtmpfs|overlay|squashfs|ramfs"}', + diskUsagePercent: FILESYSTEM_USAGE_PERCENT, + diskTotalBytes: `node_filesystem_size_bytes${FILESYSTEM_SELECTOR}`, + diskAvailableBytes: `min by (${FILESYSTEM_LABELS}) (node_filesystem_avail_bytes${FILESYSTEM_SELECTOR})`, networkReceiveBytesPerSecond: 'sum(rate(node_network_receive_bytes_total{device!~"lo"}[5m]))', networkTransmitBytesPerSecond: 'sum(rate(node_network_transmit_bytes_total{device!~"lo"}[5m]))', load1: 'node_load1', @@ -132,15 +133,6 @@ function matrixValues(response: PrometheusQueryResponse): InfrastructureMetricPo }); } -function diskIdentity(metric: Record) { - return JSON.stringify([metric.instance ?? '', metric.device ?? '', metric.mountpoint ?? '', metric.fstype ?? '']); -} - -// Preserve the legacy scalar fields as root-only; never silently pick the first disk. -function rootSeries(response: PrometheusQueryResponse): PrometheusQueryResponse { - return { ...response, data: { result: (response.data?.result ?? []).filter((item) => item.metric.mountpoint === '/') } }; -} - function emptyMetrics(): InfrastructureMonitoringOverview['metrics'] { return { cpuUsagePercent: null, @@ -195,6 +187,7 @@ export class InfrastructureMonitoringService { const warningAlerts = alerts.filter((item) => item.severity === 'warning').length; const criticalAlerts = alerts.filter((item) => item.severity === 'critical').length; const overallStatus = criticalAlerts > 0 ? 'critical' : warningAlerts > 0 ? 'warning' : 'healthy'; + const rootDisk = instant.disks.find((disk) => disk.mountpoints.includes('/')); return { available: true, range, @@ -209,7 +202,7 @@ export class InfrastructureMonitoringService { activeAlerts: alerts.length, }, metrics: instant.metrics, - trends: trends.metrics, + trends: { ...trends.metrics, diskUsagePercent: rootDisk ? trends.disks.get(rootDisk.id) ?? [] : [] }, disks: instant.disks.map((disk) => ({ ...disk, trend: trends.disks.get(disk.id) ?? [] })), services, serviceMetrics, @@ -273,20 +266,34 @@ export class InfrastructureMonitoringService { const keys = Object.keys(emptyMetrics()) as Array; const responses = await Promise.all([...keys.map((key) => this.query(QUERIES[key])), this.query(QUERIES.lastSampleAt)]); const metrics = emptyMetrics(); - keys.forEach((key, index) => { metrics[key] = vectorValue(key.startsWith('disk') ? rootSeries(responses[index]) : responses[index]); }); + keys.forEach((key, index) => { if (!key.startsWith('disk')) metrics[key] = vectorValue(responses[index]); }); const diskSamples = (key: keyof typeof metrics) => responses[keys.indexOf(key)].data?.result ?? []; - const usage = new Map(diskSamples('diskUsagePercent').map((item) => [diskIdentity(item.metric), finiteNumber(item.value?.[1])])); - const available = new Map(diskSamples('diskAvailableBytes').map((item) => [diskIdentity(item.metric), finiteNumber(item.value?.[1])])); - const disks = diskSamples('diskTotalBytes') - .filter((item) => item.metric.mountpoint && (finiteNumber(item.value?.[1]) ?? 0) > 0) - .map((item) => ({ - id: diskIdentity(item.metric), instance: item.metric.instance ?? '', device: item.metric.device ?? '', - mountpoint: item.metric.mountpoint, filesystem: item.metric.fstype ?? '', - totalBytes: finiteNumber(item.value?.[1]), - availableBytes: available.get(diskIdentity(item.metric)) ?? null, - usagePercent: usage.get(diskIdentity(item.metric)) ?? null, - })) + const usage = new Map(diskSamples('diskUsagePercent').map((item) => [filesystemIdentity(item.metric), finiteNumber(item.value?.[1])])); + const available = new Map(diskSamples('diskAvailableBytes').map((item) => [filesystemIdentity(item.metric), finiteNumber(item.value?.[1])])); + const groups = new Map(); + for (const item of diskSamples('diskTotalBytes')) { + if (!item.metric.device || !item.metric.mountpoint || (finiteNumber(item.value?.[1]) ?? 0) <= 0) continue; + const id = filesystemIdentity(item.metric); + const group = groups.get(id) ?? []; + group.push(item); + groups.set(id, group); + } + const disks = [...groups].map(([id, items]) => { + const mountpoints = [...new Set(items.map((item) => item.metric.mountpoint))].sort(compareMountpoints); + const metric = items[0].metric; + return { + id, instance: metric.instance ?? '', device: metric.device, filesystem: metric.fstype ?? '', + mountpoint: mountpoints[0], mountpoints, + // Never sum aliases. Max/min also tolerate slight sampling differences. + totalBytes: Math.max(...items.map((item) => finiteNumber(item.value?.[1])!)), + availableBytes: available.get(id) ?? null, usagePercent: usage.get(id) ?? null, + }; + }) .sort((left, right) => left.instance.localeCompare(right.instance) || (left.mountpoint === '/' ? -1 : right.mountpoint === '/' ? 1 : left.mountpoint.localeCompare(right.mountpoint))); + const rootDisk = disks.find((disk) => disk.mountpoints.includes('/')); + metrics.diskUsagePercent = rootDisk?.usagePercent ?? null; + metrics.diskTotalBytes = rootDisk?.totalBytes ?? null; + metrics.diskAvailableBytes = rootDisk?.availableBytes ?? null; return { metrics, disks, lastSampleAt: vectorValue(responses[responses.length - 1]) }; } @@ -297,9 +304,9 @@ export class InfrastructureMonitoringService { const keys = Object.keys(emptyTrends()) as Array; const responses = await Promise.all(keys.map((key) => this.queryRange(QUERIES[key], start, end, config.step))); return { - metrics: Object.fromEntries(keys.map((key, index) => [key, matrixValues(key === 'diskUsagePercent' ? rootSeries(responses[index]) : responses[index])])) as InfrastructureMonitoringOverview['trends'], + metrics: Object.fromEntries(keys.map((key, index) => [key, key === 'diskUsagePercent' ? [] : matrixValues(responses[index])])) as InfrastructureMonitoringOverview['trends'], disks: new Map((responses[keys.indexOf('diskUsagePercent')].data?.result ?? []).map((item) => [ - diskIdentity(item.metric), matrixValues({ status: 'success', data: { result: [item] } }), + filesystemIdentity(item.metric), matrixValues({ status: 'success', data: { result: [item] } }), ])), }; } diff --git a/docs/system-functional-test-cases.md b/docs/system-functional-test-cases.md index c98b575..c2b0550 100644 --- a/docs/system-functional-test-cases.md +++ b/docs/system-functional-test-cases.md @@ -4977,8 +4977,17 @@ npm run verify:phase8 | TC-PORTAL-005 | 含引流、不含引流、未检测三类历史记录 | 列表仅含引流记录在发送状态下显示标签，无负向标签；详情明确三种状态，按真实后端位置高亮URL/号码 | | TC-PORTAL-006 | 查看最终已回执及未回执短信 | 列表无回执时间列；详情“最终回执时间”来自消息最终结果，无值显示“-”；各通道回执时间仍保留 | | TC-PORTAL-007 | 三个客户端查询页修改输入、日期、下拉后等待并翻页；查询/重置 | 输入不发搜索请求；翻页保持已应用条件；查询/重置回到第一页，只发一次新查询；上行返回第一页重新加载 | -| TC-PORTAL-008 | Prometheus同时返回系统盘、数据盘、第三块磁盘，顺序打乱或有缺失点 | 全部挂载点各有容量卡片及独立趋势；按设备/挂载点匹配，不串盘，不以0填缺失点；采集失败清空指标 | -| TC-PORTAL-009 | 系统盘或任一数据盘分别超过容量阈值 | 使用原有效阈值逐盘告警，信息含挂载点与设备；基础和托管规则无同名重复；tmpfs/overlay等虚拟盘不参与 | +| TC-PORTAL-008 | Prometheus同时返回系统盘、数据盘、第三块磁盘，顺序打乱或有缺失点 | 全部独立文件系统各有容量卡片及独立趋势；按instance/device/fstype匹配，绑定挂载合并，不串盘，不以0填缺失点；采集失败清空指标 | +| TC-PORTAL-009 | 系统盘或任一数据盘分别超过容量阈值 | 使用原有效阈值按独立文件系统告警，信息含设备及文件系统类型，同一文件系统绑定路径不重复告警；实际加载的基础和托管规则无同名重复；tmpfs/overlay等虚拟盘不参与 | | TC-PORTAL-010 | 测试发布与安全边界 | 新独立恢复资产的custom dump、运行tar、配置tar、原标记和SHA全部验证后才能发布；查服务、health、Stream、窗口日志与资源哈希；不发/补发/重投短信，不修改客户/余额/通道配置 | | TC-STORAGE-DEPLOY-001 | 数据盘迁移后安装/发布前置检查；在隔离挂载命名空间隐藏数据盘或绑定挂载 | 两个入口均在初始化/发布写入前拒绝继续；验证UUID、源/目标inode、读写属性及既有存储标记；宿主挂载及服务PID不变 | | TC-STORAGE-DEPLOY-002 | 数据盘迁移后发布业务代码 | 新恢复资产在数据盘上，custom dump/tar/SHA验证通过；存储服务PID、绑定挂载、fstab、环境及systemd保护保持不变；代码回退不恢复旧系统盘业务数据；系统盘和数据盘均有真实监控指标 | + +## TC-DISK-DEDUP-20260831 绑定挂载磁盘去重 + +| 用例ID | 场景 | 预期 | +| --- | --- | --- | +| TC-DISK-DEDUP-001 | 同一设备同时采集/data、/var/lib/pgsql、/var/lib/redis、/var/lib/minio | API仅返回一个数据盘，容量不累加，mountpoints保留四个路径；/data优先展示，展开“其他挂载点（3）”可查看绑定路径；系统盘和EFI独立保留 | +| TC-DISK-DEDUP-002 | 打乱采集顺序、根目录有绑定路径、主路径停止采集、趋势缺失或NaN | 文件系统ID保持稳定；根盘兼容指标及趋势正确；主路径按层级/长度/字典序确定；同一文件系统只有一条趋势，不补零，不因路径改变丢失历史 | +| TC-DISK-DEDUP-003 | 不同instance/device/fstype的样本容量相同，或指标不可用 | 不同文件系统不误合并；缺失指标为null/空趋势，页面显示暂无数据而非旧卡片；不得仅凭容量相同合并 | +| TC-DISK-DEDUP-004 | 核对API查询、基础规则和按当前有效阈值生成的托管规则 | 容量和inode均max by(instance,device,fstype)，可用容量min聚合，不按mountpoint重复告警；阈值不变；promtool校验通过，未来发布后再核对实际加载规则、告警及真实页面；告警标签变化会影响指纹和for计时，不虚报已保留 | diff --git a/docs/testing-progress.md b/docs/testing-progress.md index 3561622..fd183fc 100644 --- a/docs/testing-progress.md +++ b/docs/testing-progress.md @@ -4227,3 +4227,14 @@ git diff --check - 外部首页、两端登录入口及API健康均HTTP 200；主JS/CSS和8个改动页面JS实际HTTP下载SHA全部与服务器产物一致。主资源 `index-BaM_U9uq.js` SHA为 `b14c2ee21f80d718e6f78289cc6e676f215c2e16bd24712202df731aeadb3d6f`，`index-D8PbKTXI.css` SHA为 `86b17c3845da62022861df97536db00df9d241ceac9677434d7028286e0ac0ca`。 - 本轮API51套581项通过；前端初次fork工作进程启动超时，改threads/单worker重跑8文件45项通过；前后端TypeScript、Vite、依赖安全、部署契约、包体积、bash语法及隔离缺挂载负向检查通过。browser技能路径中Chrome标签发现可用，但旧标签DOM读取及新标签导航均超时，真实页面身份/非空/错误遮罩/控制台/截图/交互六项均未完成；没有切换未授权的浏览器控制替代方式。 - 线上证据：`/tmp/cmpp-preprod-sixfix-activate.log`、`/tmp/cmpp-preprod-sixfix-backend-after.json`、`/tmp/cmpp-preprod-sixfix-postcheck.json`（保留首次日志失败）和 `/tmp/cmpp-preprod-sixfix-stability.json`。除明确接续的三份迁移保护脚本外，原有4份修改文档及4项未跟踪文件仍未提交；发布记录采用独立追加暂存，没有夹带另一会话的迁移文档。 + +## 2026-08-31 磁盘绑定挂载去重修复（仅本地提交，待授权发布） + +- 用户要求修复并提交，下个版本发布须另等指令。本轮基于`main / 12668cee87400fdacf2d5609e2f18a75caa71cbf`，重新核验Git和最近进度；保留另一会话4份修改文档及4项未跟踪文件，不推送、不部署、不重载规则、不重启服务。 +- 根因：前次“全部磁盘”按`instance/device/mountpoint/fstype`区分，把同一数据盘的`/data`及三个存储绑定路径当成四块盘。现改为`instance/device/fstype`独立文件系统标识，卡片、瞬时使用率、趋势和容量/inode规则一致去重；总容量取最大值、可用容量取最小值，不累加别名容量，不按容量相同误合并不同设备/主机/文件系统。 +- API新增`mountpoints`保留所有路径，主路径按层级、长度及字典序稳定选择；前端一张卡片展示主路径，原生可展开区域展示其他挂载点，趋势计数改为文件系统。保留系统盘及EFI等独立分区、根盘兼容字段，采集缺失不补零。未变更业务数据、数据库结构、迁盘绑定和存储保护配置。 +- 23:35（北京时间）对预生产Prometheus执行只读即时/范围查询：原始容量6条，聚合后3个文件系统（根、EFI、数据盘），每个趋势61点；数据盘容量`105087164416`字节，四个路径只产生一条使用率/可用容量及趋势。此证据验证新查询，不代表线上API或页面已升级。 +- 候选基础规则63条、托管规则20条通过线上现有`promtool check rules /dev/stdin`只读语法校验；未写入线上规则文件。补充生成规则与基础容量/inode规则表达式一致性测试；发布时仍须按当前有效阈值生成托管规则、移除实际加载的同名基础规则并核验health。聚合会改变告警标签、指纹及for计时，不能承诺保留原告警已读状态或连续计时。 +- 本地全量API 51套584项、前端9文件48项通过；前后端TypeScript检查、Vite构建、依赖安全、部署契约、包体积门禁及`git diff --check`通过。Vite仍提示既有Chart分块超过500kB，但入口gzip约107.39KiB，符合250KiB预算。新增回归覆盖绑定路径、乱序、主路径缺失、不同主机/设备/类型、缺失样本及展开交互；更新`TC-PORTAL-008/009`，新增`TC-DISK-DEDUP-001..004`。 +- 本轮仅本地单元/组件回归及真实采集查询核验，没有部署新API/页面；登录浏览器视觉、控制台、真实页面展开及告警加载验收留待下一次明确授权发布，不以测试替身或构建通过冒充真实页面验收。既存日报5秒事务超时不在本轮修复范围。 +- 提交仅包含本轮监控源码、规则、测试用例和本节进度追加；其他会话的迁盘文档及进度追加不纳入。本轮没有发送、补发、重投短信，也没有修改余额、通道或客户配置；待下一版本指令后再决定推送和目标环境发布。 diff --git a/src/api/types/infrastructure-monitoring.ts b/src/api/types/infrastructure-monitoring.ts index 56a15d3..cc4deef 100644 --- a/src/api/types/infrastructure-monitoring.ts +++ b/src/api/types/infrastructure-monitoring.ts @@ -46,6 +46,7 @@ export type InfrastructureMonitoringOverview = { instance: string; device: string; mountpoint: string; + mountpoints: string[]; filesystem: string; usagePercent: number | null; totalBytes: number | null; diff --git a/src/apps/admin/system-monitoring/AdminSystemMonitoringPage.css b/src/apps/admin/system-monitoring/AdminSystemMonitoringPage.css index 3f9aa19..1e88686 100644 --- a/src/apps/admin/system-monitoring/AdminSystemMonitoringPage.css +++ b/src/apps/admin/system-monitoring/AdminSystemMonitoringPage.css @@ -176,6 +176,9 @@ .system-monitoring-metric small { color: var(--color-text-muted); font-size: 12px; } .system-monitoring-metric strong { color: var(--color-text-strong); font-size: 23px; line-height: 1.25; } .system-monitoring-metric small { overflow: hidden; text-overflow: ellipsis; white-space: nowrap; } +.system-monitoring-metric__mounts { color: var(--color-text-muted); font-size: 11px; margin-top: 3px; } +.system-monitoring-metric__mounts summary { cursor: pointer; } +.system-monitoring-metric__mounts ul { margin: 5px 0 0; padding-left: 16px; overflow-wrap: anywhere; } .system-monitoring-main-grid { align-items: start; diff --git a/src/apps/admin/system-monitoring/AdminSystemMonitoringPage.tsx b/src/apps/admin/system-monitoring/AdminSystemMonitoringPage.tsx index 7183ee3..05574ab 100644 --- a/src/apps/admin/system-monitoring/AdminSystemMonitoringPage.tsx +++ b/src/apps/admin/system-monitoring/AdminSystemMonitoringPage.tsx @@ -61,6 +61,26 @@ function formatRate(value: number | null) { return value === null ? '—' : `${formatBytes(value)}/s`; } +export function DiskMetricCards({ disks }: { disks: InfrastructureMonitoringOverview['disks'] }) { + if (!disks.length) return
磁盘暂无数据
; + return <>{disks.map((disk) => { + const aliases = (disk.mountpoints ?? [disk.mountpoint]).filter((path) => path !== disk.mountpoint); + return
+
+
+ {disk.mountpoint === '/' ? '系统盘' : '磁盘'} {disk.mountpoint} + {formatPercent(disk.usagePercent)} + {disk.device} · {disk.filesystem} + {formatBytes(disk.availableBytes)} 可用 / {formatBytes(disk.totalBytes)} + {aliases.length > 0 ?
+ 其他挂载点（{aliases.length}） +
    {aliases.map((path) =>
  • {path}
    • )}
+
: null} +
+
; + })}; +} + function totalNetworkRate(receive: number | null | undefined, transmit: number | null | undefined) { if (receive === null || receive === undefined || transmit === null || transmit === undefined) return null; return receive + transmit; @@ -352,11 +372,7 @@ export function AdminSystemMonitoringPage() {
CPU 使用率{formatPercent(metrics?.cpuUsagePercent ?? null)}5分钟平均
内存使用率{formatPercent(metrics?.memoryUsagePercent ?? null)}{formatBytes(metrics?.memoryAvailableBytes ?? null)} 可用 / {formatBytes(metrics?.memoryTotalBytes ?? null)}
- {(overview?.disks ?? []).map((disk) =>
-
-
{disk.mountpoint === '/' ? '系统盘' : '磁盘'} {disk.mountpoint}{formatPercent(disk.usagePercent)}{disk.device} · {disk.filesystem}{formatBytes(disk.availableBytes)} 可用 / {formatBytes(disk.totalBytes)}
-
)} - {!overview?.disks?.length ?
磁盘暂无数据
: null} +
网络吞吐{formatRate(totalNetworkRate(metrics?.networkReceiveBytesPerSecond, metrics?.networkTransmitBytesPerSecond))}接收 {formatRate(metrics?.networkReceiveBytesPerSecond ?? null)} · 发送 {formatRate(metrics?.networkTransmitBytesPerSecond ?? null)}
@@ -364,7 +380,7 @@ export function AdminSystemMonitoringPage() {
CPU 趋势
{formatPercent(metrics?.cpuUsagePercent ?? null)}
{overview?.trends.cpuUsagePercent.length ? : }
内存趋势
{formatPercent(metrics?.memoryUsagePercent ?? null)}
{overview?.trends.memoryUsagePercent.length ? : }
-
全部磁盘趋势
{overview?.disks?.length ?? 0} 个挂载点
{overview?.disks?.some((disk) => disk.trend.length) ? : }
+
全部磁盘趋势
{overview?.disks?.length ?? 0} 个文件系统
{overview?.disks?.some((disk) => disk.trend.length) ? : }
网络趋势
{formatRate(totalNetworkRate(metrics?.networkReceiveBytesPerSecond, metrics?.networkTransmitBytesPerSecond))}
{overview?.trends.networkReceiveBytesPerSecond.length ? : }
diff --git a/src/apps/admin/system-monitoring/DiskMetricCards.test.tsx b/src/apps/admin/system-monitoring/DiskMetricCards.test.tsx new file mode 100644 index 0000000..33c0f3a --- /dev/null +++ b/src/apps/admin/system-monitoring/DiskMetricCards.test.tsx @@ -0,0 +1,42 @@ +import { render, screen } from '@testing-library/react'; +import userEvent from '@testing-library/user-event'; +import { describe, expect, it, vi } from 'vitest'; +import type { InfrastructureMonitoringOverview } from '@/api/adminApi'; +import { DiskMetricCards } from './AdminSystemMonitoringPage'; + +vi.mock('@/components/ui/Chart', () => ({ Chart: () => null })); + +const disk: InfrastructureMonitoringOverview['disks'][number] = { + id: '["host:9100","/dev/sdb1","ext4"]', instance: 'host:9100', device: '/dev/sdb1', + filesystem: 'ext4', mountpoint: '/data', mountpoints: ['/data', '/var/lib/minio', '/var/lib/pgsql', '/var/lib/redis'], + totalBytes: 100 * 1024 ** 3, availableBytes: 90 * 1024 ** 3, usagePercent: 10, trend: [], +}; + +describe('filesystem metric cards', () => { + it('shows one capacity card and expandable alias paths for a bind-mounted filesystem', async () => { + render(); + expect(screen.getAllByRole('article')).toHaveLength(1); + expect(screen.getByText('磁盘 /data')).toBeInTheDocument(); + expect(screen.getByText('90.0 GB 可用 / 100.0 GB')).toBeInTheDocument(); + const summary = screen.getByText('其他挂载点（3）'); + expect(summary.closest('details')).not.toHaveAttribute('open'); + await userEvent.click(summary); + expect(summary.closest('details')).toHaveAttribute('open'); + for (const path of disk.mountpoints.slice(1)) expect(screen.getByText(path)).toBeVisible(); + }); + + it('keeps the root filesystem distinct and does not invent aliases or missing metrics', () => { + render(); + expect(screen.getAllByRole('article')).toHaveLength(2); + expect(screen.getByText('系统盘 /')).toBeInTheDocument(); + expect(screen.getAllByText('其他挂载点（3）')).toHaveLength(1); + expect(screen.getByText('—')).toBeInTheDocument(); + }); + + it('shows an explicit unavailable state without stale disk cards', () => { + const { rerender } = render(); + rerender(); + expect(screen.getByText('暂无数据')).toBeInTheDocument(); + expect(screen.queryByText('磁盘 /data')).not.toBeInTheDocument(); + }); +}); diff --git a/tools/monitoring/cmpp-alerts.yml b/tools/monitoring/cmpp-alerts.yml index ea4e930..1c5148d 100644 --- a/tools/monitoring/cmpp-alerts.yml +++ b/tools/monitoring/cmpp-alerts.yml @@ -108,50 +108,50 @@ groups: threshold: "95%" - alert: HostRootDiskUsageWarning - expr: ((1 - node_filesystem_avail_bytes{device=~"/dev/.+",fstype!~"tmpfs|devtmpfs|overlay|squashfs|ramfs"} / node_filesystem_size_bytes{device=~"/dev/.+",fstype!~"tmpfs|devtmpfs|overlay|squashfs|ramfs"}) * 100 > 80) and ((1 - node_filesystem_avail_bytes{device=~"/dev/.+",fstype!~"tmpfs|devtmpfs|overlay|squashfs|ramfs"} / node_filesystem_size_bytes{device=~"/dev/.+",fstype!~"tmpfs|devtmpfs|overlay|squashfs|ramfs"}) * 100 <= 90) + expr: (max by (instance, device, fstype) ((1 - node_filesystem_avail_bytes{device=~"/dev/.+",fstype!~"tmpfs|devtmpfs|overlay|squashfs|ramfs"} / node_filesystem_size_bytes{device=~"/dev/.+",fstype!~"tmpfs|devtmpfs|overlay|squashfs|ramfs"}) * 100) > 80) and (max by (instance, device, fstype) ((1 - node_filesystem_avail_bytes{device=~"/dev/.+",fstype!~"tmpfs|devtmpfs|overlay|squashfs|ramfs"} / node_filesystem_size_bytes{device=~"/dev/.+",fstype!~"tmpfs|devtmpfs|overlay|squashfs|ramfs"}) * 100) <= 90) for: 15m labels: severity: warning service: host annotations: summary: 磁盘文件系统空间不足 - description: "磁盘文件系统使用率连续15分钟高于80%。 挂载点：{{ $labels.mountpoint }}；设备：{{ $labels.device }}。" + description: "磁盘文件系统使用率连续15分钟高于80%。 设备：{{ $labels.device }}；文件系统：{{ $labels.fstype }}（绑定挂载已合并）。" currentValue: "{{ printf \"%.1f\" $value }}%" threshold: "80%" - alert: HostRootDiskUsageCritical - expr: (1 - node_filesystem_avail_bytes{device=~"/dev/.+",fstype!~"tmpfs|devtmpfs|overlay|squashfs|ramfs"} / node_filesystem_size_bytes{device=~"/dev/.+",fstype!~"tmpfs|devtmpfs|overlay|squashfs|ramfs"}) * 100 > 90 + expr: max by (instance, device, fstype) ((1 - node_filesystem_avail_bytes{device=~"/dev/.+",fstype!~"tmpfs|devtmpfs|overlay|squashfs|ramfs"} / node_filesystem_size_bytes{device=~"/dev/.+",fstype!~"tmpfs|devtmpfs|overlay|squashfs|ramfs"}) * 100) > 90 for: 5m labels: severity: critical service: host annotations: summary: 磁盘文件系统空间严重不足 - description: "磁盘文件系统使用率连续5分钟高于90%。 挂载点：{{ $labels.mountpoint }}；设备：{{ $labels.device }}。" + description: "磁盘文件系统使用率连续5分钟高于90%。 设备：{{ $labels.device }}；文件系统：{{ $labels.fstype }}（绑定挂载已合并）。" currentValue: "{{ printf \"%.1f\" $value }}%" threshold: "90%" - alert: HostRootInodeUsageWarning - expr: ((1 - node_filesystem_files_free{device=~"/dev/.+",fstype!~"tmpfs|devtmpfs|overlay|squashfs|ramfs"} / node_filesystem_files{device=~"/dev/.+",fstype!~"tmpfs|devtmpfs|overlay|squashfs|ramfs"}) * 100 > 80) and ((1 - node_filesystem_files_free{device=~"/dev/.+",fstype!~"tmpfs|devtmpfs|overlay|squashfs|ramfs"} / node_filesystem_files{device=~"/dev/.+",fstype!~"tmpfs|devtmpfs|overlay|squashfs|ramfs"}) * 100 <= 90) + expr: (max by (instance, device, fstype) ((1 - node_filesystem_files_free{device=~"/dev/.+",fstype!~"tmpfs|devtmpfs|overlay|squashfs|ramfs"} / node_filesystem_files{device=~"/dev/.+",fstype!~"tmpfs|devtmpfs|overlay|squashfs|ramfs"}) * 100) > 80) and (max by (instance, device, fstype) ((1 - node_filesystem_files_free{device=~"/dev/.+",fstype!~"tmpfs|devtmpfs|overlay|squashfs|ramfs"} / node_filesystem_files{device=~"/dev/.+",fstype!~"tmpfs|devtmpfs|overlay|squashfs|ramfs"}) * 100) <= 90) for: 15m labels: severity: warning service: host annotations: summary: 磁盘文件系统inode余量偏低 - description: "磁盘文件系统inode使用率连续15分钟高于80%。 挂载点：{{ $labels.mountpoint }}；设备：{{ $labels.device }}。" + description: "磁盘文件系统inode使用率连续15分钟高于80%。 设备：{{ $labels.device }}；文件系统：{{ $labels.fstype }}（绑定挂载已合并）。" currentValue: "{{ printf \"%.1f\" $value }}%" threshold: "80%" - alert: HostRootInodeUsageCritical - expr: (1 - node_filesystem_files_free{device=~"/dev/.+",fstype!~"tmpfs|devtmpfs|overlay|squashfs|ramfs"} / node_filesystem_files{device=~"/dev/.+",fstype!~"tmpfs|devtmpfs|overlay|squashfs|ramfs"}) * 100 > 90 + expr: max by (instance, device, fstype) ((1 - node_filesystem_files_free{device=~"/dev/.+",fstype!~"tmpfs|devtmpfs|overlay|squashfs|ramfs"} / node_filesystem_files{device=~"/dev/.+",fstype!~"tmpfs|devtmpfs|overlay|squashfs|ramfs"}) * 100) > 90 for: 5m labels: severity: critical service: host annotations: summary: 磁盘文件系统inode严重不足 - description: "磁盘文件系统inode使用率连续5分钟高于90%。 挂载点：{{ $labels.mountpoint }}；设备：{{ $labels.device }}。" + description: "磁盘文件系统inode使用率连续5分钟高于90%。 设备：{{ $labels.device }}；文件系统：{{ $labels.fstype }}（绑定挂载已合并）。" currentValue: "{{ printf \"%.1f\" $value }}%" threshold: "90%" diff --git a/tools/monitoring/cmpp-managed-alerts.yml b/tools/monitoring/cmpp-managed-alerts.yml index fb90169..0b82946 100644 --- a/tools/monitoring/cmpp-managed-alerts.yml +++ b/tools/monitoring/cmpp-managed-alerts.yml @@ -22,15 +22,15 @@ groups: labels: { severity: critical, service: host } annotations: { summary: "主机内存使用率达到严重阈值", description: "主机内存使用率持续超过95%。", currentValue: "{{ $value }}", threshold: "95%" } - alert: HostRootDiskUsageWarning - expr: ((1 - node_filesystem_avail_bytes{device=~"/dev/.+",fstype!~"tmpfs|devtmpfs|overlay|squashfs|ramfs"} / node_filesystem_size_bytes{device=~"/dev/.+",fstype!~"tmpfs|devtmpfs|overlay|squashfs|ramfs"}) * 100 > 80) and ((1 - node_filesystem_avail_bytes{device=~"/dev/.+",fstype!~"tmpfs|devtmpfs|overlay|squashfs|ramfs"} / node_filesystem_size_bytes{device=~"/dev/.+",fstype!~"tmpfs|devtmpfs|overlay|squashfs|ramfs"}) * 100 <= 90) + expr: (max by (instance, device, fstype) ((1 - node_filesystem_avail_bytes{device=~"/dev/.+",fstype!~"tmpfs|devtmpfs|overlay|squashfs|ramfs"} / node_filesystem_size_bytes{device=~"/dev/.+",fstype!~"tmpfs|devtmpfs|overlay|squashfs|ramfs"}) * 100) > 80) and (max by (instance, device, fstype) ((1 - node_filesystem_avail_bytes{device=~"/dev/.+",fstype!~"tmpfs|devtmpfs|overlay|squashfs|ramfs"} / node_filesystem_size_bytes{device=~"/dev/.+",fstype!~"tmpfs|devtmpfs|overlay|squashfs|ramfs"}) * 100) <= 90) for: 15m labels: { severity: warning, service: host } - annotations: { summary: "磁盘（所有挂载点）使用率达到警告阈值", description: "磁盘（所有挂载点）使用率持续超过80%。", currentValue: "{{ $value }}", threshold: "80%" } + annotations: { summary: "磁盘（独立文件系统）使用率达到警告阈值", description: "磁盘（独立文件系统）使用率持续超过80%。 设备：{{ $labels.device }}；文件系统：{{ $labels.fstype }}（绑定挂载已合并）。", currentValue: "{{ $value }}", threshold: "80%" } - alert: HostRootDiskUsageCritical - expr: (1 - node_filesystem_avail_bytes{device=~"/dev/.+",fstype!~"tmpfs|devtmpfs|overlay|squashfs|ramfs"} / node_filesystem_size_bytes{device=~"/dev/.+",fstype!~"tmpfs|devtmpfs|overlay|squashfs|ramfs"}) * 100 > 90 + expr: max by (instance, device, fstype) ((1 - node_filesystem_avail_bytes{device=~"/dev/.+",fstype!~"tmpfs|devtmpfs|overlay|squashfs|ramfs"} / node_filesystem_size_bytes{device=~"/dev/.+",fstype!~"tmpfs|devtmpfs|overlay|squashfs|ramfs"}) * 100) > 90 for: 5m labels: { severity: critical, service: host } - annotations: { summary: "磁盘（所有挂载点）使用率达到严重阈值", description: "磁盘（所有挂载点）使用率持续超过90%。", currentValue: "{{ $value }}", threshold: "90%" } + annotations: { summary: "磁盘（独立文件系统）使用率达到严重阈值", description: "磁盘（独立文件系统）使用率持续超过90%。 设备：{{ $labels.device }}；文件系统：{{ $labels.fstype }}（绑定挂载已合并）。", currentValue: "{{ $value }}", threshold: "90%" } - alert: CmppApiHttpErrorRateWarning expr: (100 * sum(rate(cmpp_api_http_requests_total{status=~"5.."}[5m])) / clamp_min(sum(rate(cmpp_api_http_requests_total[5m])), 0.001) > 1) and (100 * sum(rate(cmpp_api_http_requests_total{status=~"5.."}[5m])) / clamp_min(sum(rate(cmpp_api_http_requests_total[5m])), 0.001) <= 5) and (sum(increase(cmpp_api_http_requests_total{status=~"5.."}[5m])) >= 5) for: 5m