fix: deduplicate filesystem mounts in monitoring
This commit is contained in:
hectorzhao
@@ -0,0 +1,15 @@
|
||||
// A bind mount is another path to the same filesystem, not another disk.
|
||||
export const FILESYSTEM_LABELS = 'instance, device, fstype';
|
||||
export const FILESYSTEM_SELECTOR = '{device=~"/dev/.+",fstype!~"tmpfs|devtmpfs|overlay|squashfs|ramfs"}';
|
||||
export const FILESYSTEM_USAGE_PERCENT = `max by (${FILESYSTEM_LABELS}) ((1 - node_filesystem_avail_bytes${FILESYSTEM_SELECTOR} / node_filesystem_size_bytes${FILESYSTEM_SELECTOR}) * 100)`;
|
||||
export const FILESYSTEM_INODE_USAGE_PERCENT = `max by (${FILESYSTEM_LABELS}) ((1 - node_filesystem_files_free${FILESYSTEM_SELECTOR} / node_filesystem_files${FILESYSTEM_SELECTOR}) * 100)`;
|
||||
|
||||
export function filesystemIdentity(metric: Record<string, string>) {
|
||||
return JSON.stringify([metric.instance ?? '', metric.device ?? '', metric.fstype ?? '']);
|
||||
}
|
||||
|
||||
// Prefer the filesystem's shallowest visible path; tie-breaking is deterministic.
|
||||
export function compareMountpoints(left: string, right: string) {
|
||||
return left.split('/').filter(Boolean).length - right.split('/').filter(Boolean).length
|
||||
|| left.length - right.length || left.localeCompare(right);
|
||||
}
|
||||
@@ -1,5 +1,8 @@
|
||||
import { BadRequestException } from '@nestjs/common';
|
||||
import { DEFAULT_ALERT_THRESHOLDS, InfrastructureAlertSettingsService } from './infrastructure-alert-settings.service';
|
||||
import { readFileSync } from 'node:fs';
|
||||
import { resolve } from 'node:path';
|
||||
import { FILESYSTEM_INODE_USAGE_PERCENT, FILESYSTEM_USAGE_PERCENT } from './filesystem-metrics';
|
||||
|
||||
describe('InfrastructureAlertSettingsService', () => {
|
||||
const service = new InfrastructureAlertSettingsService({} as never, { get: () => undefined } as never);
|
||||
@@ -14,8 +17,25 @@ describe('InfrastructureAlertSettingsService', () => {
|
||||
expect(rules).toContain('sum(increase(cmpp_api_http_requests_total');
|
||||
expect(rules).not.toContain('mountpoint="/"');
|
||||
expect(rules).toContain('device=~"/dev/.+"');
|
||||
expect(rules).toContain('{{ $labels.mountpoint }}');
|
||||
expect(rules).not.toContain('{{ $labels.mountpoint }}');
|
||||
expect(rules).toContain('{{ $labels.device }}');
|
||||
expect(rules).toContain('{{ $labels.fstype }}');
|
||||
expect(rules).toContain(FILESYSTEM_USAGE_PERCENT);
|
||||
});
|
||||
|
||||
it('keeps static capacity and inode rules aligned with filesystem-level deduplication', () => {
|
||||
for (const file of ['cmpp-alerts.yml', 'cmpp-managed-alerts.yml']) {
|
||||
const content = readFileSync(resolve(__dirname, '../../../tools/monitoring', file), 'utf8');
|
||||
const blocks = [...content.matchAll(/ - alert: (HostRoot(?:Disk|Inode)Usage(?:Warning|Critical))\r?\n([\s\S]*?)(?=\r?\n - alert:|$)/g)];
|
||||
expect(blocks).toHaveLength(file === 'cmpp-alerts.yml' ? 4 : 2);
|
||||
for (const [, name, body] of blocks) {
|
||||
const expression = body.match(/^\s+expr: (.+)$/m)![1].trim();
|
||||
const base = name.includes('Inode') ? FILESYSTEM_INODE_USAGE_PERCENT : FILESYSTEM_USAGE_PERCENT;
|
||||
expect(expression).toBe(name.endsWith('Warning') ? `(${base} > 80) and (${base} <= 90)` : `${base} > 90`);
|
||||
expect(body).not.toContain('$labels.mountpoint');
|
||||
expect(body).toContain('$labels.device');
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
it('rejects unknown keys and warning thresholds that are not below critical', () => {
|
||||
|
||||
@@ -6,6 +6,7 @@ import { mkdir, readFile, rename, rm, writeFile } from 'node:fs/promises';
|
||||
import { dirname } from 'node:path';
|
||||
import { promisify } from 'node:util';
|
||||
import { PrismaService } from '../prisma/prisma.service';
|
||||
import { FILESYSTEM_USAGE_PERCENT } from './filesystem-metrics';
|
||||
import type { InfrastructureAlertSettings, InfrastructureAlertThresholds } from './infrastructure-monitoring.contracts';
|
||||
|
||||
const execFileAsync = promisify(execFile);
|
||||
@@ -13,7 +14,7 @@ const execFileAsync = promisify(execFile);
|
||||
export const ALERT_THRESHOLD_DEFINITIONS = [
|
||||
{ key: 'hostCpu', label: '主机 CPU 使用率', unit: '%', min: 1, max: 100, step: 1, warning: 80, critical: 90, expr: '100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100)', names: ['HostCpuUsageWarning', 'HostCpuUsageCritical'], service: 'host', durations: ['10m', '5m'] },
|
||||
{ key: 'hostMemory', label: '主机内存使用率', unit: '%', min: 1, max: 100, step: 1, warning: 85, critical: 95, expr: '(1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100', names: ['HostMemoryUsageWarning', 'HostMemoryUsageCritical'], service: 'host', durations: ['10m', '5m'] },
|
||||
{ key: 'hostDisk', label: '磁盘（所有挂载点）使用率', unit: '%', min: 1, max: 100, step: 1, warning: 80, critical: 90, expr: '(1 - node_filesystem_avail_bytes{device=~"/dev/.+",fstype!~"tmpfs|devtmpfs|overlay|squashfs|ramfs"} / node_filesystem_size_bytes{device=~"/dev/.+",fstype!~"tmpfs|devtmpfs|overlay|squashfs|ramfs"}) * 100', names: ['HostRootDiskUsageWarning', 'HostRootDiskUsageCritical'], service: 'host', durations: ['15m', '5m'] },
|
||||
{ key: 'hostDisk', label: '磁盘（独立文件系统）使用率', unit: '%', min: 1, max: 100, step: 1, warning: 80, critical: 90, expr: FILESYSTEM_USAGE_PERCENT, names: ['HostRootDiskUsageWarning', 'HostRootDiskUsageCritical'], service: 'host', durations: ['15m', '5m'] },
|
||||
{ key: 'apiError', label: 'API 5xx 错误率', unit: '%', min: 0.1, max: 100, step: 0.1, warning: 1, critical: 5, expr: '100 * sum(rate(cmpp_api_http_requests_total{status=~"5.."}[5m])) / clamp_min(sum(rate(cmpp_api_http_requests_total[5m])), 0.001)', guard: 'sum(increase(cmpp_api_http_requests_total{status=~"5.."}[5m])) >= 5', names: ['CmppApiHttpErrorRateWarning', 'CmppApiHttpErrorRateCritical'], service: 'api', durations: ['5m', '5m'] },
|
||||
{ key: 'apiLatency', label: 'API P95 响应时间', unit: '秒', min: 0.1, max: 60, step: 0.1, warning: 1, critical: 3, expr: 'histogram_quantile(0.95, sum by (le) (rate(cmpp_api_http_request_duration_seconds_bucket[10m])))', names: ['CmppApiLatencyWarning', 'CmppApiLatencyCritical'], service: 'api', durations: ['10m', '5m'] },
|
||||
{ key: 'apiEventLoop', label: 'API 事件循环 P99', unit: '秒', min: 0.01, max: 10, step: 0.01, warning: 0.2, critical: 1, expr: 'cmpp_api_nodejs_event_loop_lag_p99_seconds', names: ['CmppApiEventLoopLagWarning', 'CmppApiEventLoopLagCritical'], service: 'api', durations: ['10m', '5m'] },
|
||||
@@ -113,7 +114,7 @@ export class InfrastructureAlertSettingsService {
|
||||
// 低样本量与“未配置容量上限”必须继续作为固定保护条件，避免单次错误或除零结果触发伪告警。
|
||||
const guard = 'guard' in definition ? ` and (${definition.guard})` : '';
|
||||
const expr = isWarning ? `(${definition.expr} > ${values[0]}) and (${definition.expr} <= ${values[1]})${guard}` : `(${definition.expr} > ${values[1]})${guard}`;
|
||||
const diskLocation = definition.key === 'hostDisk' ? ' 挂载点：{{ $labels.mountpoint }}；设备：{{ $labels.device }}。' : '';
|
||||
const diskLocation = definition.key === 'hostDisk' ? ' 设备：{{ $labels.device }}；文件系统：{{ $labels.fstype }}（绑定挂载已合并）。' : '';
|
||||
lines.push(` - alert: ${definition.names[index]}`, ` expr: ${expr}`, ` for: ${definition.durations[index]}`, ' labels:', ` severity: ${isWarning ? 'warning' : 'critical'}`, ` service: ${definition.service}`, ' annotations:', ` summary: "${definition.label}${isWarning ? '达到警告阈值' : '达到严重阈值'}"`, ` description: "${definition.label}持续超过${values[index]}${definition.unit}。${diskLocation}"`, ' currentValue: "{{ $value }}"', ` threshold: "${values[index]}${definition.unit}"`);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -46,6 +46,7 @@ export type InfrastructureMonitoringOverview = {
|
||||
instance: string;
|
||||
device: string;
|
||||
mountpoint: string;
|
||||
mountpoints: string[];
|
||||
filesystem: string;
|
||||
usagePercent: number | null;
|
||||
totalBytes: number | null;
|
||||
|
||||
@@ -2,6 +2,7 @@ import { BadRequestException } from '@nestjs/common';
|
||||
import { ConfigService } from '@nestjs/config';
|
||||
import { createHash } from 'node:crypto';
|
||||
import { InfrastructureMonitoringService } from './infrastructure-monitoring.service';
|
||||
import { FILESYSTEM_USAGE_PERCENT, filesystemIdentity } from './filesystem-metrics';
|
||||
|
||||
function success(data: unknown) {
|
||||
return {
|
||||
@@ -124,7 +125,7 @@ describe('InfrastructureMonitoringService', () => {
|
||||
if (!query.includes('node_filesystem_')) return success({ result: [] });
|
||||
expect(query).not.toContain('mountpoint="/"');
|
||||
if (url.pathname.endsWith('/query_range')) return success({ result: [...metrics].reverse().map((metric) => ({ metric, values: [[1765000060, metric.mountpoint === '/' ? '91' : '12']] })) });
|
||||
return success({ result: metrics.map((metric) => ({ metric, value: [1765000060, query.startsWith('(1') ? (metric.mountpoint === '/' ? '91' : '12') : query.includes('avail') ? '9' : '100'] })) });
|
||||
return success({ result: metrics.map((metric) => ({ metric, value: [1765000060, query === FILESYSTEM_USAGE_PERCENT ? (metric.mountpoint === '/' ? '91' : '12') : query.includes('avail') ? '9' : '100'] })) });
|
||||
});
|
||||
const result = await new InfrastructureMonitoringService(new ConfigService(), prisma as never).overview('1h');
|
||||
expect(result.disks.map((disk) => disk.mountpoint)).toEqual(['/', '/archive', '/data']);
|
||||
@@ -134,6 +135,74 @@ describe('InfrastructureMonitoringService', () => {
|
||||
expect(result.trends.diskUsagePercent[0].value).toBe(91);
|
||||
});
|
||||
|
||||
it('merges bind mounts without summing capacity, keeps a stable identity and uses one aggregated trend', async () => {
|
||||
const data = { instance: 'host:9100', device: '/dev/sdb1', fstype: 'ext4' };
|
||||
const root = { instance: 'host:9100', device: '/dev/sda2', fstype: 'ext4' };
|
||||
let mounts = ['/var/lib/redis', '/var/lib/pgsql', '/data', '/var/lib/minio'];
|
||||
jest.spyOn(global, 'fetch').mockImplementation(async (input) => {
|
||||
const url = new URL(String(input));
|
||||
const query = url.searchParams.get('query') ?? '';
|
||||
if (url.pathname.endsWith('/alerts')) return success({ alerts: [] });
|
||||
if (!query.includes('node_filesystem_')) return success({ result: [] });
|
||||
if (url.pathname.endsWith('/query_range')) {
|
||||
expect(query).toBe(FILESYSTEM_USAGE_PERCENT);
|
||||
return success({ result: [
|
||||
{ metric: data, values: [[1765000000, '82'], [1765000060, 'NaN'], [1765000120, '83.5']] },
|
||||
{ metric: root, values: [[1765000000, '91']] },
|
||||
] });
|
||||
}
|
||||
if (query.startsWith('node_filesystem_size_bytes')) return success({ result: [
|
||||
...mounts.map((mountpoint) => ({ metric: { ...data, mountpoint }, value: [1765000120, '100'] })),
|
||||
...['/var/root-bind', '/'].map((mountpoint) => ({ metric: { ...root, mountpoint }, value: [1765000120, '200'] })),
|
||||
] });
|
||||
if (query === FILESYSTEM_USAGE_PERCENT) return success({ result: [
|
||||
{ metric: data, value: [1765000120, '83.5'] }, { metric: root, value: [1765000120, '91'] },
|
||||
] });
|
||||
expect(query).toContain('min by (instance, device, fstype)');
|
||||
return success({ result: [{ metric: data, value: [1765000120, '16.5'] }, { metric: root, value: [1765000120, '18'] }] });
|
||||
});
|
||||
const service = new InfrastructureMonitoringService(new ConfigService(), prisma as never);
|
||||
const result = await service.overview('1h');
|
||||
expect(result.available).toBe(true);
|
||||
expect(result.disks).toHaveLength(2);
|
||||
expect(result.disks[0]).toMatchObject({ mountpoint: '/', mountpoints: ['/', '/var/root-bind'], totalBytes: 200 });
|
||||
expect(result.metrics.diskUsagePercent).toBe(91);
|
||||
expect(result.trends.diskUsagePercent[0].value).toBe(91);
|
||||
const disk = result.disks[1];
|
||||
expect(disk).toMatchObject({ id: filesystemIdentity(data), mountpoint: '/data', totalBytes: 100, availableBytes: 16.5, usagePercent: 83.5 });
|
||||
expect(disk.mountpoints).toEqual(['/data', '/var/lib/minio', '/var/lib/pgsql', '/var/lib/redis']);
|
||||
expect(disk.trend.map((point) => point.value)).toEqual([82, 83.5]);
|
||||
mounts.reverse();
|
||||
expect((await service.overview('1h')).disks).toEqual(result.disks);
|
||||
mounts = ['/var/lib/redis'];
|
||||
const aliasOnly = (await service.overview('1h')).disks.find((item) => item.id === disk.id)!;
|
||||
expect(aliasOnly.mountpoint).toBe('/var/lib/redis');
|
||||
expect(aliasOnly.trend).toEqual(disk.trend);
|
||||
});
|
||||
|
||||
it('does not merge different hosts, devices or filesystem types with identical capacity', async () => {
|
||||
const metrics = [
|
||||
{ instance: 'a:9100', device: '/dev/sdb1', fstype: 'ext4', mountpoint: '/data' },
|
||||
{ instance: 'b:9100', device: '/dev/sdb1', fstype: 'ext4', mountpoint: '/data' },
|
||||
{ instance: 'a:9100', device: '/dev/sdc1', fstype: 'ext4', mountpoint: '/archive' },
|
||||
{ instance: 'a:9100', device: '/dev/sdb1', fstype: 'xfs', mountpoint: '/other' },
|
||||
];
|
||||
jest.spyOn(global, 'fetch').mockImplementation(async (input) => {
|
||||
const url = new URL(String(input));
|
||||
if (url.pathname.endsWith('/alerts')) return success({ alerts: [] });
|
||||
if (url.searchParams.get('query')?.startsWith('node_filesystem_size_bytes')) {
|
||||
return success({ result: metrics.map((metric) => ({ metric, value: [1765000060, '100'] })) });
|
||||
}
|
||||
return success({ result: [] });
|
||||
});
|
||||
const result = await new InfrastructureMonitoringService(new ConfigService(), prisma as never).overview('1h');
|
||||
expect(result.disks).toHaveLength(4);
|
||||
expect(new Set(result.disks.map((disk) => disk.id)).size).toBe(4);
|
||||
expect(result.disks.every((disk) => disk.usagePercent === null && disk.availableBytes === null && disk.trend.length === 0)).toBe(true);
|
||||
expect(result.metrics.diskUsagePercent).toBeNull();
|
||||
expect(result.trends.diskUsagePercent).toEqual([]);
|
||||
});
|
||||
|
||||
it('excludes only the current alert occurrence after the current administrator marks it read', async () => {
|
||||
const labels = { alertname: 'QaWarning', severity: 'warning', service: 'qa-preview' };
|
||||
const fingerprint = createHash('sha256').update(JSON.stringify(Object.entries(labels).sort(([left], [right]) => left.localeCompare(right)))).digest('hex').slice(0, 24);
|
||||
|
||||
@@ -3,6 +3,7 @@ import { ConfigService } from '@nestjs/config';
|
||||
import { Prisma } from '@prisma/client';
|
||||
import { createHash } from 'node:crypto';
|
||||
import { PrismaService } from '../prisma/prisma.service';
|
||||
import { compareMountpoints, FILESYSTEM_LABELS, FILESYSTEM_SELECTOR, FILESYSTEM_USAGE_PERCENT, filesystemIdentity } from './filesystem-metrics';
|
||||
import type {
|
||||
InfrastructureAlert,
|
||||
InfrastructureMetricPoint,
|
||||
@@ -47,9 +48,9 @@ const QUERIES = {
|
||||
memoryUsagePercent: '(1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) * 100',
|
||||
memoryTotalBytes: 'node_memory_MemTotal_bytes',
|
||||
memoryAvailableBytes: 'node_memory_MemAvailable_bytes',
|
||||
diskUsagePercent: '(1 - (node_filesystem_avail_bytes{device=~"/dev/.+",fstype!~"tmpfs|devtmpfs|overlay|squashfs|ramfs"} / node_filesystem_size_bytes{device=~"/dev/.+",fstype!~"tmpfs|devtmpfs|overlay|squashfs|ramfs"})) * 100',
|
||||
diskTotalBytes: 'node_filesystem_size_bytes{device=~"/dev/.+",fstype!~"tmpfs|devtmpfs|overlay|squashfs|ramfs"}',
|
||||
diskAvailableBytes: 'node_filesystem_avail_bytes{device=~"/dev/.+",fstype!~"tmpfs|devtmpfs|overlay|squashfs|ramfs"}',
|
||||
diskUsagePercent: FILESYSTEM_USAGE_PERCENT,
|
||||
diskTotalBytes: `node_filesystem_size_bytes${FILESYSTEM_SELECTOR}`,
|
||||
diskAvailableBytes: `min by (${FILESYSTEM_LABELS}) (node_filesystem_avail_bytes${FILESYSTEM_SELECTOR})`,
|
||||
networkReceiveBytesPerSecond: 'sum(rate(node_network_receive_bytes_total{device!~"lo"}[5m]))',
|
||||
networkTransmitBytesPerSecond: 'sum(rate(node_network_transmit_bytes_total{device!~"lo"}[5m]))',
|
||||
load1: 'node_load1',
|
||||
@@ -132,15 +133,6 @@ function matrixValues(response: PrometheusQueryResponse): InfrastructureMetricPo
|
||||
});
|
||||
}
|
||||
|
||||
function diskIdentity(metric: Record<string, string>) {
|
||||
return JSON.stringify([metric.instance ?? '', metric.device ?? '', metric.mountpoint ?? '', metric.fstype ?? '']);
|
||||
}
|
||||
|
||||
// Preserve the legacy scalar fields as root-only; never silently pick the first disk.
|
||||
function rootSeries(response: PrometheusQueryResponse): PrometheusQueryResponse {
|
||||
return { ...response, data: { result: (response.data?.result ?? []).filter((item) => item.metric.mountpoint === '/') } };
|
||||
}
|
||||
|
||||
function emptyMetrics(): InfrastructureMonitoringOverview['metrics'] {
|
||||
return {
|
||||
cpuUsagePercent: null,
|
||||
@@ -195,6 +187,7 @@ export class InfrastructureMonitoringService {
|
||||
const warningAlerts = alerts.filter((item) => item.severity === 'warning').length;
|
||||
const criticalAlerts = alerts.filter((item) => item.severity === 'critical').length;
|
||||
const overallStatus = criticalAlerts > 0 ? 'critical' : warningAlerts > 0 ? 'warning' : 'healthy';
|
||||
const rootDisk = instant.disks.find((disk) => disk.mountpoints.includes('/'));
|
||||
return {
|
||||
available: true,
|
||||
range,
|
||||
@@ -209,7 +202,7 @@ export class InfrastructureMonitoringService {
|
||||
activeAlerts: alerts.length,
|
||||
},
|
||||
metrics: instant.metrics,
|
||||
trends: trends.metrics,
|
||||
trends: { ...trends.metrics, diskUsagePercent: rootDisk ? trends.disks.get(rootDisk.id) ?? [] : [] },
|
||||
disks: instant.disks.map((disk) => ({ ...disk, trend: trends.disks.get(disk.id) ?? [] })),
|
||||
services,
|
||||
serviceMetrics,
|
||||
@@ -273,20 +266,34 @@ export class InfrastructureMonitoringService {
|
||||
const keys = Object.keys(emptyMetrics()) as Array<keyof InfrastructureMonitoringOverview['metrics']>;
|
||||
const responses = await Promise.all([...keys.map((key) => this.query(QUERIES[key])), this.query(QUERIES.lastSampleAt)]);
|
||||
const metrics = emptyMetrics();
|
||||
keys.forEach((key, index) => { metrics[key] = vectorValue(key.startsWith('disk') ? rootSeries(responses[index]) : responses[index]); });
|
||||
keys.forEach((key, index) => { if (!key.startsWith('disk')) metrics[key] = vectorValue(responses[index]); });
|
||||
const diskSamples = (key: keyof typeof metrics) => responses[keys.indexOf(key)].data?.result ?? [];
|
||||
const usage = new Map(diskSamples('diskUsagePercent').map((item) => [diskIdentity(item.metric), finiteNumber(item.value?.[1])]));
|
||||
const available = new Map(diskSamples('diskAvailableBytes').map((item) => [diskIdentity(item.metric), finiteNumber(item.value?.[1])]));
|
||||
const disks = diskSamples('diskTotalBytes')
|
||||
.filter((item) => item.metric.mountpoint && (finiteNumber(item.value?.[1]) ?? 0) > 0)
|
||||
.map((item) => ({
|
||||
id: diskIdentity(item.metric), instance: item.metric.instance ?? '', device: item.metric.device ?? '',
|
||||
mountpoint: item.metric.mountpoint, filesystem: item.metric.fstype ?? '',
|
||||
totalBytes: finiteNumber(item.value?.[1]),
|
||||
availableBytes: available.get(diskIdentity(item.metric)) ?? null,
|
||||
usagePercent: usage.get(diskIdentity(item.metric)) ?? null,
|
||||
}))
|
||||
const usage = new Map(diskSamples('diskUsagePercent').map((item) => [filesystemIdentity(item.metric), finiteNumber(item.value?.[1])]));
|
||||
const available = new Map(diskSamples('diskAvailableBytes').map((item) => [filesystemIdentity(item.metric), finiteNumber(item.value?.[1])]));
|
||||
const groups = new Map<string, PrometheusSeries[]>();
|
||||
for (const item of diskSamples('diskTotalBytes')) {
|
||||
if (!item.metric.device || !item.metric.mountpoint || (finiteNumber(item.value?.[1]) ?? 0) <= 0) continue;
|
||||
const id = filesystemIdentity(item.metric);
|
||||
const group = groups.get(id) ?? [];
|
||||
group.push(item);
|
||||
groups.set(id, group);
|
||||
}
|
||||
const disks = [...groups].map(([id, items]) => {
|
||||
const mountpoints = [...new Set(items.map((item) => item.metric.mountpoint))].sort(compareMountpoints);
|
||||
const metric = items[0].metric;
|
||||
return {
|
||||
id, instance: metric.instance ?? '', device: metric.device, filesystem: metric.fstype ?? '',
|
||||
mountpoint: mountpoints[0], mountpoints,
|
||||
// Never sum aliases. Max/min also tolerate slight sampling differences.
|
||||
totalBytes: Math.max(...items.map((item) => finiteNumber(item.value?.[1])!)),
|
||||
availableBytes: available.get(id) ?? null, usagePercent: usage.get(id) ?? null,
|
||||
};
|
||||
})
|
||||
.sort((left, right) => left.instance.localeCompare(right.instance) || (left.mountpoint === '/' ? -1 : right.mountpoint === '/' ? 1 : left.mountpoint.localeCompare(right.mountpoint)));
|
||||
const rootDisk = disks.find((disk) => disk.mountpoints.includes('/'));
|
||||
metrics.diskUsagePercent = rootDisk?.usagePercent ?? null;
|
||||
metrics.diskTotalBytes = rootDisk?.totalBytes ?? null;
|
||||
metrics.diskAvailableBytes = rootDisk?.availableBytes ?? null;
|
||||
return { metrics, disks, lastSampleAt: vectorValue(responses[responses.length - 1]) };
|
||||
}
|
||||
|
||||
@@ -297,9 +304,9 @@ export class InfrastructureMonitoringService {
|
||||
const keys = Object.keys(emptyTrends()) as Array<keyof InfrastructureMonitoringOverview['trends']>;
|
||||
const responses = await Promise.all(keys.map((key) => this.queryRange(QUERIES[key], start, end, config.step)));
|
||||
return {
|
||||
metrics: Object.fromEntries(keys.map((key, index) => [key, matrixValues(key === 'diskUsagePercent' ? rootSeries(responses[index]) : responses[index])])) as InfrastructureMonitoringOverview['trends'],
|
||||
metrics: Object.fromEntries(keys.map((key, index) => [key, key === 'diskUsagePercent' ? [] : matrixValues(responses[index])])) as InfrastructureMonitoringOverview['trends'],
|
||||
disks: new Map((responses[keys.indexOf('diskUsagePercent')].data?.result ?? []).map((item) => [
|
||||
diskIdentity(item.metric), matrixValues({ status: 'success', data: { result: [item] } }),
|
||||
filesystemIdentity(item.metric), matrixValues({ status: 'success', data: { result: [item] } }),
|
||||
])),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -4977,8 +4977,17 @@ npm run verify:phase8
|
||||
| TC-PORTAL-005 | 含引流、不含引流、未检测三类历史记录 | 列表仅含引流记录在发送状态下显示标签，无负向标签；详情明确三种状态，按真实后端位置高亮URL/号码 |
|
||||
| TC-PORTAL-006 | 查看最终已回执及未回执短信 | 列表无回执时间列；详情“最终回执时间”来自消息最终结果，无值显示“-”；各通道回执时间仍保留 |
|
||||
| TC-PORTAL-007 | 三个客户端查询页修改输入、日期、下拉后等待并翻页；查询/重置 | 输入不发搜索请求；翻页保持已应用条件；查询/重置回到第一页，只发一次新查询；上行返回第一页重新加载 |
|
||||
| TC-PORTAL-008 | Prometheus同时返回系统盘、数据盘、第三块磁盘，顺序打乱或有缺失点 | 全部挂载点各有容量卡片及独立趋势；按设备/挂载点匹配，不串盘，不以0填缺失点；采集失败清空指标 |
|
||||
| TC-PORTAL-009 | 系统盘或任一数据盘分别超过容量阈值 | 使用原有效阈值逐盘告警，信息含挂载点与设备；基础和托管规则无同名重复；tmpfs/overlay等虚拟盘不参与 |
|
||||
| TC-PORTAL-008 | Prometheus同时返回系统盘、数据盘、第三块磁盘，顺序打乱或有缺失点 | 全部独立文件系统各有容量卡片及独立趋势；按instance/device/fstype匹配，绑定挂载合并，不串盘，不以0填缺失点；采集失败清空指标 |
|
||||
| TC-PORTAL-009 | 系统盘或任一数据盘分别超过容量阈值 | 使用原有效阈值按独立文件系统告警，信息含设备及文件系统类型，同一文件系统绑定路径不重复告警；实际加载的基础和托管规则无同名重复；tmpfs/overlay等虚拟盘不参与 |
|
||||
| TC-PORTAL-010 | 测试发布与安全边界 | 新独立恢复资产的custom dump、运行tar、配置tar、原标记和SHA全部验证后才能发布；查服务、health、Stream、窗口日志与资源哈希；不发/补发/重投短信，不修改客户/余额/通道配置 |
|
||||
| TC-STORAGE-DEPLOY-001 | 数据盘迁移后安装/发布前置检查；在隔离挂载命名空间隐藏数据盘或绑定挂载 | 两个入口均在初始化/发布写入前拒绝继续；验证UUID、源/目标inode、读写属性及既有存储标记；宿主挂载及服务PID不变 |
|
||||
| TC-STORAGE-DEPLOY-002 | 数据盘迁移后发布业务代码 | 新恢复资产在数据盘上，custom dump/tar/SHA验证通过；存储服务PID、绑定挂载、fstab、环境及systemd保护保持不变；代码回退不恢复旧系统盘业务数据；系统盘和数据盘均有真实监控指标 |
|
||||
|
||||
## TC-DISK-DEDUP-20260831 绑定挂载磁盘去重
|
||||
|
||||
| 用例ID | 场景 | 预期 |
|
||||
| --- | --- | --- |
|
||||
| TC-DISK-DEDUP-001 | 同一设备同时采集/data、/var/lib/pgsql、/var/lib/redis、/var/lib/minio | API仅返回一个数据盘，容量不累加，mountpoints保留四个路径；/data优先展示，展开“其他挂载点（3）”可查看绑定路径；系统盘和EFI独立保留 |
|
||||
| TC-DISK-DEDUP-002 | 打乱采集顺序、根目录有绑定路径、主路径停止采集、趋势缺失或NaN | 文件系统ID保持稳定；根盘兼容指标及趋势正确；主路径按层级/长度/字典序确定；同一文件系统只有一条趋势，不补零，不因路径改变丢失历史 |
|
||||
| TC-DISK-DEDUP-003 | 不同instance/device/fstype的样本容量相同，或指标不可用 | 不同文件系统不误合并；缺失指标为null/空趋势，页面显示暂无数据而非旧卡片；不得仅凭容量相同合并 |
|
||||
| TC-DISK-DEDUP-004 | 核对API查询、基础规则和按当前有效阈值生成的托管规则 | 容量和inode均max by(instance,device,fstype)，可用容量min聚合，不按mountpoint重复告警；阈值不变；promtool校验通过，未来发布后再核对实际加载规则、告警及真实页面；告警标签变化会影响指纹和for计时，不虚报已保留 |
|
||||
|
||||
@@ -4227,3 +4227,14 @@ git diff --check
|
||||
- 外部首页、两端登录入口及API健康均HTTP 200；主JS/CSS和8个改动页面JS实际HTTP下载SHA全部与服务器产物一致。主资源 `index-BaM_U9uq.js` SHA为 `b14c2ee21f80d718e6f78289cc6e676f215c2e16bd24712202df731aeadb3d6f`，`index-D8PbKTXI.css` SHA为 `86b17c3845da62022861df97536db00df9d241ceac9677434d7028286e0ac0ca`。
|
||||
- 本轮API51套581项通过；前端初次fork工作进程启动超时，改threads/单worker重跑8文件45项通过；前后端TypeScript、Vite、依赖安全、部署契约、包体积、bash语法及隔离缺挂载负向检查通过。browser技能路径中Chrome标签发现可用，但旧标签DOM读取及新标签导航均超时，真实页面身份/非空/错误遮罩/控制台/截图/交互六项均未完成；没有切换未授权的浏览器控制替代方式。
|
||||
- 线上证据：`/tmp/cmpp-preprod-sixfix-activate.log`、`/tmp/cmpp-preprod-sixfix-backend-after.json`、`/tmp/cmpp-preprod-sixfix-postcheck.json`（保留首次日志失败）和 `/tmp/cmpp-preprod-sixfix-stability.json`。除明确接续的三份迁移保护脚本外，原有4份修改文档及4项未跟踪文件仍未提交；发布记录采用独立追加暂存，没有夹带另一会话的迁移文档。
|
||||
|
||||
## 2026-08-31 磁盘绑定挂载去重修复（仅本地提交，待授权发布）
|
||||
|
||||
- 用户要求修复并提交，下个版本发布须另等指令。本轮基于`main / 12668cee87400fdacf2d5609e2f18a75caa71cbf`，重新核验Git和最近进度；保留另一会话4份修改文档及4项未跟踪文件，不推送、不部署、不重载规则、不重启服务。
|
||||
- 根因：前次“全部磁盘”按`instance/device/mountpoint/fstype`区分，把同一数据盘的`/data`及三个存储绑定路径当成四块盘。现改为`instance/device/fstype`独立文件系统标识，卡片、瞬时使用率、趋势和容量/inode规则一致去重；总容量取最大值、可用容量取最小值，不累加别名容量，不按容量相同误合并不同设备/主机/文件系统。
|
||||
- API新增`mountpoints`保留所有路径，主路径按层级、长度及字典序稳定选择；前端一张卡片展示主路径，原生可展开区域展示其他挂载点，趋势计数改为文件系统。保留系统盘及EFI等独立分区、根盘兼容字段，采集缺失不补零。未变更业务数据、数据库结构、迁盘绑定和存储保护配置。
|
||||
- 23:35（北京时间）对预生产Prometheus执行只读即时/范围查询：原始容量6条，聚合后3个文件系统（根、EFI、数据盘），每个趋势61点；数据盘容量`105087164416`字节，四个路径只产生一条使用率/可用容量及趋势。此证据验证新查询，不代表线上API或页面已升级。
|
||||
- 候选基础规则63条、托管规则20条通过线上现有`promtool check rules /dev/stdin`只读语法校验；未写入线上规则文件。补充生成规则与基础容量/inode规则表达式一致性测试；发布时仍须按当前有效阈值生成托管规则、移除实际加载的同名基础规则并核验health。聚合会改变告警标签、指纹及for计时，不能承诺保留原告警已读状态或连续计时。
|
||||
- 本地全量API 51套584项、前端9文件48项通过；前后端TypeScript检查、Vite构建、依赖安全、部署契约、包体积门禁及`git diff --check`通过。Vite仍提示既有Chart分块超过500kB，但入口gzip约107.39KiB，符合250KiB预算。新增回归覆盖绑定路径、乱序、主路径缺失、不同主机/设备/类型、缺失样本及展开交互；更新`TC-PORTAL-008/009`，新增`TC-DISK-DEDUP-001..004`。
|
||||
- 本轮仅本地单元/组件回归及真实采集查询核验，没有部署新API/页面；登录浏览器视觉、控制台、真实页面展开及告警加载验收留待下一次明确授权发布，不以测试替身或构建通过冒充真实页面验收。既存日报5秒事务超时不在本轮修复范围。
|
||||
- 提交仅包含本轮监控源码、规则、测试用例和本节进度追加；其他会话的迁盘文档及进度追加不纳入。本轮没有发送、补发、重投短信，也没有修改余额、通道或客户配置；待下一版本指令后再决定推送和目标环境发布。
|
||||
|
||||
@@ -46,6 +46,7 @@ export type InfrastructureMonitoringOverview = {
|
||||
instance: string;
|
||||
device: string;
|
||||
mountpoint: string;
|
||||
mountpoints: string[];
|
||||
filesystem: string;
|
||||
usagePercent: number | null;
|
||||
totalBytes: number | null;
|
||||
|
||||
@@ -176,6 +176,9 @@
|
||||
.system-monitoring-metric small { color: var(--color-text-muted); font-size: 12px; }
|
||||
.system-monitoring-metric strong { color: var(--color-text-strong); font-size: 23px; line-height: 1.25; }
|
||||
.system-monitoring-metric small { overflow: hidden; text-overflow: ellipsis; white-space: nowrap; }
|
||||
.system-monitoring-metric__mounts { color: var(--color-text-muted); font-size: 11px; margin-top: 3px; }
|
||||
.system-monitoring-metric__mounts summary { cursor: pointer; }
|
||||
.system-monitoring-metric__mounts ul { margin: 5px 0 0; padding-left: 16px; overflow-wrap: anywhere; }
|
||||
|
||||
.system-monitoring-main-grid {
|
||||
align-items: start;
|
||||
|
||||
@@ -61,6 +61,26 @@ function formatRate(value: number | null) {
|
||||
return value === null ? '—' : `${formatBytes(value)}/s`;
|
||||
}
|
||||
|
||||
export function DiskMetricCards({ disks }: { disks: InfrastructureMonitoringOverview['disks'] }) {
|
||||
if (!disks.length) return <article className="surface system-monitoring-metric"><HardDrive size={19} /><div><span>磁盘</span><strong>暂无数据</strong></div></article>;
|
||||
return <>{disks.map((disk) => {
|
||||
const aliases = (disk.mountpoints ?? [disk.mountpoint]).filter((path) => path !== disk.mountpoint);
|
||||
return <article className="surface system-monitoring-metric" key={disk.id}>
|
||||
<div className="system-monitoring-metric__icon is-amber"><HardDrive size={19} /></div>
|
||||
<div>
|
||||
<span>{disk.mountpoint === '/' ? '系统盘' : '磁盘'} {disk.mountpoint}</span>
|
||||
<strong>{formatPercent(disk.usagePercent)}</strong>
|
||||
<small title={`${disk.device} · ${disk.filesystem} · ${disk.instance}`}>{disk.device} · {disk.filesystem}</small>
|
||||
<small>{formatBytes(disk.availableBytes)} 可用 / {formatBytes(disk.totalBytes)}</small>
|
||||
{aliases.length > 0 ? <details className="system-monitoring-metric__mounts">
|
||||
<summary>其他挂载点（{aliases.length}）</summary>
|
||||
<ul>{aliases.map((path) => <li key={path}>{path}</li>)}</ul>
|
||||
</details> : null}
|
||||
</div>
|
||||
</article>;
|
||||
})}</>;
|
||||
}
|
||||
|
||||
function totalNetworkRate(receive: number | null | undefined, transmit: number | null | undefined) {
|
||||
if (receive === null || receive === undefined || transmit === null || transmit === undefined) return null;
|
||||
return receive + transmit;
|
||||
@@ -352,11 +372,7 @@ export function AdminSystemMonitoringPage() {
|
||||
<div className="system-monitoring-metrics">
|
||||
<article className="surface system-monitoring-metric"><div className="system-monitoring-metric__icon is-blue"><Cpu size={19} /></div><div><span>CPU 使用率</span><strong>{formatPercent(metrics?.cpuUsagePercent ?? null)}</strong><small>5分钟平均</small></div></article>
|
||||
<article className="surface system-monitoring-metric"><div className="system-monitoring-metric__icon is-violet"><MemoryStick size={19} /></div><div><span>内存使用率</span><strong>{formatPercent(metrics?.memoryUsagePercent ?? null)}</strong><small>{formatBytes(metrics?.memoryAvailableBytes ?? null)} 可用 / {formatBytes(metrics?.memoryTotalBytes ?? null)}</small></div></article>
|
||||
{(overview?.disks ?? []).map((disk) => <article className="surface system-monitoring-metric" key={disk.id}>
|
||||
<div className="system-monitoring-metric__icon is-amber"><HardDrive size={19} /></div>
|
||||
<div><span>{disk.mountpoint === '/' ? '系统盘' : '磁盘'} {disk.mountpoint}</span><strong>{formatPercent(disk.usagePercent)}</strong><small title={`${disk.device} · ${disk.filesystem} · ${disk.instance}`}>{disk.device} · {disk.filesystem}</small><small>{formatBytes(disk.availableBytes)} 可用 / {formatBytes(disk.totalBytes)}</small></div>
|
||||
</article>)}
|
||||
{!overview?.disks?.length ? <article className="surface system-monitoring-metric"><HardDrive size={19} /><div><span>磁盘</span><strong>暂无数据</strong></div></article> : null}
|
||||
<DiskMetricCards disks={overview?.disks ?? []} />
|
||||
<article className="surface system-monitoring-metric"><div className="system-monitoring-metric__icon is-green"><Network size={19} /></div><div><span>网络吞吐</span><strong>{formatRate(totalNetworkRate(metrics?.networkReceiveBytesPerSecond, metrics?.networkTransmitBytesPerSecond))}</strong><small>接收 {formatRate(metrics?.networkReceiveBytesPerSecond ?? null)} · 发送 {formatRate(metrics?.networkTransmitBytesPerSecond ?? null)}</small></div></article>
|
||||
</div>
|
||||
|
||||
@@ -364,7 +380,7 @@ export function AdminSystemMonitoringPage() {
|
||||
<div className="system-monitoring-chart-stack">
|
||||
<article className="surface system-monitoring-chart-card"><header><div><Cpu size={17} /><strong>CPU 趋势</strong></div><span>{formatPercent(metrics?.cpuUsagePercent ?? null)}</span></header>{overview?.trends.cpuUsagePercent.length ? <Chart height={230} option={cpuOption} /> : <EmptyChart />}</article>
|
||||
<article className="surface system-monitoring-chart-card"><header><div><MemoryStick size={17} /><strong>内存趋势</strong></div><span>{formatPercent(metrics?.memoryUsagePercent ?? null)}</span></header>{overview?.trends.memoryUsagePercent.length ? <Chart height={230} option={memoryOption} /> : <EmptyChart />}</article>
|
||||
<article className="surface system-monitoring-chart-card"><header><div><HardDrive size={17} /><strong>全部磁盘趋势</strong></div><span>{overview?.disks?.length ?? 0} 个挂载点</span></header>{overview?.disks?.some((disk) => disk.trend.length) ? <Chart height={230} option={diskOption} /> : <EmptyChart />}</article>
|
||||
<article className="surface system-monitoring-chart-card"><header><div><HardDrive size={17} /><strong>全部磁盘趋势</strong></div><span>{overview?.disks?.length ?? 0} 个文件系统</span></header>{overview?.disks?.some((disk) => disk.trend.length) ? <Chart height={230} option={diskOption} /> : <EmptyChart />}</article>
|
||||
<article className="surface system-monitoring-chart-card"><header><div><Activity size={17} /><strong>网络趋势</strong></div><span>{formatRate(totalNetworkRate(metrics?.networkReceiveBytesPerSecond, metrics?.networkTransmitBytesPerSecond))}</span></header>{overview?.trends.networkReceiveBytesPerSecond.length ? <Chart height={230} option={networkOption} /> : <EmptyChart />}</article>
|
||||
</div>
|
||||
|
||||
|
||||
@@ -0,0 +1,42 @@
|
||||
import { render, screen } from '@testing-library/react';
|
||||
import userEvent from '@testing-library/user-event';
|
||||
import { describe, expect, it, vi } from 'vitest';
|
||||
import type { InfrastructureMonitoringOverview } from '@/api/adminApi';
|
||||
import { DiskMetricCards } from './AdminSystemMonitoringPage';
|
||||
|
||||
vi.mock('@/components/ui/Chart', () => ({ Chart: () => null }));
|
||||
|
||||
const disk: InfrastructureMonitoringOverview['disks'][number] = {
|
||||
id: '["host:9100","/dev/sdb1","ext4"]', instance: 'host:9100', device: '/dev/sdb1',
|
||||
filesystem: 'ext4', mountpoint: '/data', mountpoints: ['/data', '/var/lib/minio', '/var/lib/pgsql', '/var/lib/redis'],
|
||||
totalBytes: 100 * 1024 ** 3, availableBytes: 90 * 1024 ** 3, usagePercent: 10, trend: [],
|
||||
};
|
||||
|
||||
describe('filesystem metric cards', () => {
|
||||
it('shows one capacity card and expandable alias paths for a bind-mounted filesystem', async () => {
|
||||
render(<DiskMetricCards disks={[disk]} />);
|
||||
expect(screen.getAllByRole('article')).toHaveLength(1);
|
||||
expect(screen.getByText('磁盘 /data')).toBeInTheDocument();
|
||||
expect(screen.getByText('90.0 GB 可用 / 100.0 GB')).toBeInTheDocument();
|
||||
const summary = screen.getByText('其他挂载点（3）');
|
||||
expect(summary.closest('details')).not.toHaveAttribute('open');
|
||||
await userEvent.click(summary);
|
||||
expect(summary.closest('details')).toHaveAttribute('open');
|
||||
for (const path of disk.mountpoints.slice(1)) expect(screen.getByText(path)).toBeVisible();
|
||||
});
|
||||
|
||||
it('keeps the root filesystem distinct and does not invent aliases or missing metrics', () => {
|
||||
render(<DiskMetricCards disks={[disk, { ...disk, id: 'root', device: '/dev/sda2', mountpoint: '/', mountpoints: ['/'], usagePercent: null }]} />);
|
||||
expect(screen.getAllByRole('article')).toHaveLength(2);
|
||||
expect(screen.getByText('系统盘 /')).toBeInTheDocument();
|
||||
expect(screen.getAllByText('其他挂载点（3）')).toHaveLength(1);
|
||||
expect(screen.getByText('—')).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it('shows an explicit unavailable state without stale disk cards', () => {
|
||||
const { rerender } = render(<DiskMetricCards disks={[disk]} />);
|
||||
rerender(<DiskMetricCards disks={[]} />);
|
||||
expect(screen.getByText('暂无数据')).toBeInTheDocument();
|
||||
expect(screen.queryByText('磁盘 /data')).not.toBeInTheDocument();
|
||||
});
|
||||
});
|
||||
@@ -108,50 +108,50 @@ groups:
|
||||
threshold: "95%"
|
||||
|
||||
- alert: HostRootDiskUsageWarning
|
||||
expr: ((1 - node_filesystem_avail_bytes{device=~"/dev/.+",fstype!~"tmpfs|devtmpfs|overlay|squashfs|ramfs"} / node_filesystem_size_bytes{device=~"/dev/.+",fstype!~"tmpfs|devtmpfs|overlay|squashfs|ramfs"}) * 100 > 80) and ((1 - node_filesystem_avail_bytes{device=~"/dev/.+",fstype!~"tmpfs|devtmpfs|overlay|squashfs|ramfs"} / node_filesystem_size_bytes{device=~"/dev/.+",fstype!~"tmpfs|devtmpfs|overlay|squashfs|ramfs"}) * 100 <= 90)
|
||||
expr: (max by (instance, device, fstype) ((1 - node_filesystem_avail_bytes{device=~"/dev/.+",fstype!~"tmpfs|devtmpfs|overlay|squashfs|ramfs"} / node_filesystem_size_bytes{device=~"/dev/.+",fstype!~"tmpfs|devtmpfs|overlay|squashfs|ramfs"}) * 100) > 80) and (max by (instance, device, fstype) ((1 - node_filesystem_avail_bytes{device=~"/dev/.+",fstype!~"tmpfs|devtmpfs|overlay|squashfs|ramfs"} / node_filesystem_size_bytes{device=~"/dev/.+",fstype!~"tmpfs|devtmpfs|overlay|squashfs|ramfs"}) * 100) <= 90)
|
||||
for: 15m
|
||||
labels:
|
||||
severity: warning
|
||||
service: host
|
||||
annotations:
|
||||
summary: 磁盘文件系统空间不足
|
||||
description: "磁盘文件系统使用率连续15分钟高于80%。 挂载点：{{ $labels.mountpoint }}；设备：{{ $labels.device }}。"
|
||||
description: "磁盘文件系统使用率连续15分钟高于80%。 设备：{{ $labels.device }}；文件系统：{{ $labels.fstype }}（绑定挂载已合并）。"
|
||||
currentValue: "{{ printf \"%.1f\" $value }}%"
|
||||
threshold: "80%"
|
||||
|
||||
- alert: HostRootDiskUsageCritical
|
||||
expr: (1 - node_filesystem_avail_bytes{device=~"/dev/.+",fstype!~"tmpfs|devtmpfs|overlay|squashfs|ramfs"} / node_filesystem_size_bytes{device=~"/dev/.+",fstype!~"tmpfs|devtmpfs|overlay|squashfs|ramfs"}) * 100 > 90
|
||||
expr: max by (instance, device, fstype) ((1 - node_filesystem_avail_bytes{device=~"/dev/.+",fstype!~"tmpfs|devtmpfs|overlay|squashfs|ramfs"} / node_filesystem_size_bytes{device=~"/dev/.+",fstype!~"tmpfs|devtmpfs|overlay|squashfs|ramfs"}) * 100) > 90
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
service: host
|
||||
annotations:
|
||||
summary: 磁盘文件系统空间严重不足
|
||||
description: "磁盘文件系统使用率连续5分钟高于90%。 挂载点：{{ $labels.mountpoint }}；设备：{{ $labels.device }}。"
|
||||
description: "磁盘文件系统使用率连续5分钟高于90%。 设备：{{ $labels.device }}；文件系统：{{ $labels.fstype }}（绑定挂载已合并）。"
|
||||
currentValue: "{{ printf \"%.1f\" $value }}%"
|
||||
threshold: "90%"
|
||||
|
||||
- alert: HostRootInodeUsageWarning
|
||||
expr: ((1 - node_filesystem_files_free{device=~"/dev/.+",fstype!~"tmpfs|devtmpfs|overlay|squashfs|ramfs"} / node_filesystem_files{device=~"/dev/.+",fstype!~"tmpfs|devtmpfs|overlay|squashfs|ramfs"}) * 100 > 80) and ((1 - node_filesystem_files_free{device=~"/dev/.+",fstype!~"tmpfs|devtmpfs|overlay|squashfs|ramfs"} / node_filesystem_files{device=~"/dev/.+",fstype!~"tmpfs|devtmpfs|overlay|squashfs|ramfs"}) * 100 <= 90)
|
||||
expr: (max by (instance, device, fstype) ((1 - node_filesystem_files_free{device=~"/dev/.+",fstype!~"tmpfs|devtmpfs|overlay|squashfs|ramfs"} / node_filesystem_files{device=~"/dev/.+",fstype!~"tmpfs|devtmpfs|overlay|squashfs|ramfs"}) * 100) > 80) and (max by (instance, device, fstype) ((1 - node_filesystem_files_free{device=~"/dev/.+",fstype!~"tmpfs|devtmpfs|overlay|squashfs|ramfs"} / node_filesystem_files{device=~"/dev/.+",fstype!~"tmpfs|devtmpfs|overlay|squashfs|ramfs"}) * 100) <= 90)
|
||||
for: 15m
|
||||
labels:
|
||||
severity: warning
|
||||
service: host
|
||||
annotations:
|
||||
summary: 磁盘文件系统inode余量偏低
|
||||
description: "磁盘文件系统inode使用率连续15分钟高于80%。 挂载点：{{ $labels.mountpoint }}；设备：{{ $labels.device }}。"
|
||||
description: "磁盘文件系统inode使用率连续15分钟高于80%。 设备：{{ $labels.device }}；文件系统：{{ $labels.fstype }}（绑定挂载已合并）。"
|
||||
currentValue: "{{ printf \"%.1f\" $value }}%"
|
||||
threshold: "80%"
|
||||
|
||||
- alert: HostRootInodeUsageCritical
|
||||
expr: (1 - node_filesystem_files_free{device=~"/dev/.+",fstype!~"tmpfs|devtmpfs|overlay|squashfs|ramfs"} / node_filesystem_files{device=~"/dev/.+",fstype!~"tmpfs|devtmpfs|overlay|squashfs|ramfs"}) * 100 > 90
|
||||
expr: max by (instance, device, fstype) ((1 - node_filesystem_files_free{device=~"/dev/.+",fstype!~"tmpfs|devtmpfs|overlay|squashfs|ramfs"} / node_filesystem_files{device=~"/dev/.+",fstype!~"tmpfs|devtmpfs|overlay|squashfs|ramfs"}) * 100) > 90
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
service: host
|
||||
annotations:
|
||||
summary: 磁盘文件系统inode严重不足
|
||||
description: "磁盘文件系统inode使用率连续5分钟高于90%。 挂载点：{{ $labels.mountpoint }}；设备：{{ $labels.device }}。"
|
||||
description: "磁盘文件系统inode使用率连续5分钟高于90%。 设备：{{ $labels.device }}；文件系统：{{ $labels.fstype }}（绑定挂载已合并）。"
|
||||
currentValue: "{{ printf \"%.1f\" $value }}%"
|
||||
threshold: "90%"
|
||||
|
||||
|
||||
@@ -22,15 +22,15 @@ groups:
|
||||
labels: { severity: critical, service: host }
|
||||
annotations: { summary: "主机内存使用率达到严重阈值", description: "主机内存使用率持续超过95%。", currentValue: "{{ $value }}", threshold: "95%" }
|
||||
- alert: HostRootDiskUsageWarning
|
||||
expr: ((1 - node_filesystem_avail_bytes{device=~"/dev/.+",fstype!~"tmpfs|devtmpfs|overlay|squashfs|ramfs"} / node_filesystem_size_bytes{device=~"/dev/.+",fstype!~"tmpfs|devtmpfs|overlay|squashfs|ramfs"}) * 100 > 80) and ((1 - node_filesystem_avail_bytes{device=~"/dev/.+",fstype!~"tmpfs|devtmpfs|overlay|squashfs|ramfs"} / node_filesystem_size_bytes{device=~"/dev/.+",fstype!~"tmpfs|devtmpfs|overlay|squashfs|ramfs"}) * 100 <= 90)
|
||||
expr: (max by (instance, device, fstype) ((1 - node_filesystem_avail_bytes{device=~"/dev/.+",fstype!~"tmpfs|devtmpfs|overlay|squashfs|ramfs"} / node_filesystem_size_bytes{device=~"/dev/.+",fstype!~"tmpfs|devtmpfs|overlay|squashfs|ramfs"}) * 100) > 80) and (max by (instance, device, fstype) ((1 - node_filesystem_avail_bytes{device=~"/dev/.+",fstype!~"tmpfs|devtmpfs|overlay|squashfs|ramfs"} / node_filesystem_size_bytes{device=~"/dev/.+",fstype!~"tmpfs|devtmpfs|overlay|squashfs|ramfs"}) * 100) <= 90)
|
||||
for: 15m
|
||||
labels: { severity: warning, service: host }
|
||||
annotations: { summary: "磁盘（所有挂载点）使用率达到警告阈值", description: "磁盘（所有挂载点）使用率持续超过80%。", currentValue: "{{ $value }}", threshold: "80%" }
|
||||
annotations: { summary: "磁盘（独立文件系统）使用率达到警告阈值", description: "磁盘（独立文件系统）使用率持续超过80%。 设备：{{ $labels.device }}；文件系统：{{ $labels.fstype }}（绑定挂载已合并）。", currentValue: "{{ $value }}", threshold: "80%" }
|
||||
- alert: HostRootDiskUsageCritical
|
||||
expr: (1 - node_filesystem_avail_bytes{device=~"/dev/.+",fstype!~"tmpfs|devtmpfs|overlay|squashfs|ramfs"} / node_filesystem_size_bytes{device=~"/dev/.+",fstype!~"tmpfs|devtmpfs|overlay|squashfs|ramfs"}) * 100 > 90
|
||||
expr: max by (instance, device, fstype) ((1 - node_filesystem_avail_bytes{device=~"/dev/.+",fstype!~"tmpfs|devtmpfs|overlay|squashfs|ramfs"} / node_filesystem_size_bytes{device=~"/dev/.+",fstype!~"tmpfs|devtmpfs|overlay|squashfs|ramfs"}) * 100) > 90
|
||||
for: 5m
|
||||
labels: { severity: critical, service: host }
|
||||
annotations: { summary: "磁盘（所有挂载点）使用率达到严重阈值", description: "磁盘（所有挂载点）使用率持续超过90%。", currentValue: "{{ $value }}", threshold: "90%" }
|
||||
annotations: { summary: "磁盘（独立文件系统）使用率达到严重阈值", description: "磁盘（独立文件系统）使用率持续超过90%。 设备：{{ $labels.device }}；文件系统：{{ $labels.fstype }}（绑定挂载已合并）。", currentValue: "{{ $value }}", threshold: "90%" }
|
||||
- alert: CmppApiHttpErrorRateWarning
|
||||
expr: (100 * sum(rate(cmpp_api_http_requests_total{status=~"5.."}[5m])) / clamp_min(sum(rate(cmpp_api_http_requests_total[5m])), 0.001) > 1) and (100 * sum(rate(cmpp_api_http_requests_total{status=~"5.."}[5m])) / clamp_min(sum(rate(cmpp_api_http_requests_total[5m])), 0.001) <= 5) and (sum(increase(cmpp_api_http_requests_total{status=~"5.."}[5m])) >= 5)
|
||||
for: 5m
|
||||
|
||||
Reference in New Issue
Block a user