feat: add Prometheus system monitoring
This commit is contained in:
hectorzhao
@@ -25,6 +25,9 @@ OPERATION_LOG_ARCHIVE_INTERVAL_MS=86400000
|
||||
SMS_RECEIPT_TIMEOUT_SCAN_ENABLED=true
|
||||
SMS_RECEIPT_TIMEOUT_HOURS=72
|
||||
SMS_RECEIPT_TIMEOUT_SCAN_INTERVAL_MS=300000
|
||||
# System monitoring reads only fixed queries from a loopback Prometheus instance.
|
||||
PROMETHEUS_URL=http://127.0.0.1:9090
|
||||
PROMETHEUS_QUERY_TIMEOUT_MS=5000
|
||||
# Local HTTP development only. Production must use HTTPS and true.
|
||||
SESSION_COOKIE_SECURE=false
|
||||
MINIO_ENDPOINT=localhost:9000
|
||||
|
||||
@@ -12,6 +12,7 @@ import { DictionariesModule } from './dictionaries/dictionaries.module';
|
||||
import { DeletionGovernanceModule } from './deletion-governance/deletion-governance.module';
|
||||
import { FilesModule } from './files/files.module';
|
||||
import { HealthController } from './health.controller';
|
||||
import { InfrastructureMonitoringModule } from './infrastructure-monitoring/infrastructure-monitoring.module';
|
||||
import { OperationsModule } from './operations/operations.module';
|
||||
import { OpenApiModule } from './open-api/open-api.module';
|
||||
import { PrismaModule } from './prisma/prisma.module';
|
||||
@@ -49,6 +50,7 @@ import { SignatureRetirementModule } from './signature-retirement/signature-reti
|
||||
ReportMaterialsModule,
|
||||
SendChainModule,
|
||||
OperationsModule,
|
||||
InfrastructureMonitoringModule,
|
||||
OpenApiModule,
|
||||
SignatureRetirementModule,
|
||||
],
|
||||
|
||||
@@ -0,0 +1,65 @@
|
||||
export type InfrastructureMonitoringRange = '1h' | '24h' | '7d';
|
||||
|
||||
export type InfrastructureMetricPoint = {
|
||||
timestamp: string;
|
||||
value: number;
|
||||
};
|
||||
|
||||
export type InfrastructureServiceStatus = {
|
||||
key: string;
|
||||
name: string;
|
||||
unit: string;
|
||||
status: 'healthy' | 'unhealthy' | 'unknown';
|
||||
};
|
||||
|
||||
export type InfrastructureAlert = {
|
||||
fingerprint: string;
|
||||
name: string;
|
||||
severity: 'info' | 'warning' | 'critical';
|
||||
status: string;
|
||||
startedAt: string;
|
||||
summary: string;
|
||||
description?: string;
|
||||
currentValue?: string;
|
||||
threshold?: string;
|
||||
service?: string;
|
||||
instance?: string;
|
||||
};
|
||||
|
||||
export type InfrastructureMonitoringOverview = {
|
||||
available: boolean;
|
||||
range: InfrastructureMonitoringRange;
|
||||
collectedAt: string;
|
||||
lastSampleAt: string | null;
|
||||
error?: string;
|
||||
summary: {
|
||||
overallStatus: 'healthy' | 'warning' | 'critical' | 'unknown';
|
||||
serviceTotal: number;
|
||||
serviceHealthy: number;
|
||||
warningAlerts: number;
|
||||
criticalAlerts: number;
|
||||
activeAlerts: number;
|
||||
};
|
||||
metrics: {
|
||||
cpuUsagePercent: number | null;
|
||||
memoryUsagePercent: number | null;
|
||||
memoryTotalBytes: number | null;
|
||||
memoryAvailableBytes: number | null;
|
||||
diskUsagePercent: number | null;
|
||||
diskTotalBytes: number | null;
|
||||
diskAvailableBytes: number | null;
|
||||
networkReceiveBytesPerSecond: number | null;
|
||||
networkTransmitBytesPerSecond: number | null;
|
||||
load1: number | null;
|
||||
uptimeSeconds: number | null;
|
||||
};
|
||||
trends: {
|
||||
cpuUsagePercent: InfrastructureMetricPoint[];
|
||||
memoryUsagePercent: InfrastructureMetricPoint[];
|
||||
diskUsagePercent: InfrastructureMetricPoint[];
|
||||
networkReceiveBytesPerSecond: InfrastructureMetricPoint[];
|
||||
networkTransmitBytesPerSecond: InfrastructureMetricPoint[];
|
||||
};
|
||||
services: InfrastructureServiceStatus[];
|
||||
alerts: InfrastructureAlert[];
|
||||
};
|
||||
@@ -0,0 +1,14 @@
|
||||
import { Controller, Get, Query } from '@nestjs/common';
|
||||
import { ApiTags } from '@nestjs/swagger';
|
||||
import { InfrastructureMonitoringService } from './infrastructure-monitoring.service';
|
||||
|
||||
@ApiTags('infrastructure-monitoring')
|
||||
@Controller('admin/infrastructure-monitoring')
|
||||
export class InfrastructureMonitoringController {
|
||||
constructor(private readonly monitoring: InfrastructureMonitoringService) {}
|
||||
|
||||
@Get('overview')
|
||||
overview(@Query('range') range?: string) {
|
||||
return this.monitoring.overview(range);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
import { Module } from '@nestjs/common';
|
||||
import { InfrastructureMonitoringController } from './infrastructure-monitoring.controller';
|
||||
import { InfrastructureMonitoringService } from './infrastructure-monitoring.service';
|
||||
|
||||
@Module({
|
||||
controllers: [InfrastructureMonitoringController],
|
||||
providers: [InfrastructureMonitoringService],
|
||||
})
|
||||
export class InfrastructureMonitoringModule {}
|
||||
@@ -0,0 +1,89 @@
|
||||
import { BadRequestException } from '@nestjs/common';
|
||||
import { ConfigService } from '@nestjs/config';
|
||||
import { InfrastructureMonitoringService } from './infrastructure-monitoring.service';
|
||||
|
||||
function success(data: unknown) {
|
||||
return {
|
||||
ok: true,
|
||||
status: 200,
|
||||
json: async () => ({ status: 'success', data }),
|
||||
} as Response;
|
||||
}
|
||||
|
||||
describe('InfrastructureMonitoringService', () => {
|
||||
afterEach(() => jest.restoreAllMocks());
|
||||
|
||||
it('rejects ranges outside the fixed whitelist before querying Prometheus', async () => {
|
||||
const fetchSpy = jest.spyOn(global, 'fetch');
|
||||
const service = new InfrastructureMonitoringService(new ConfigService());
|
||||
|
||||
await expect(service.overview('30d')).rejects.toBeInstanceOf(BadRequestException);
|
||||
expect(fetchSpy).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('rejects credential-bearing or remote plaintext Prometheus endpoints at startup', () => {
|
||||
expect(() => new InfrastructureMonitoringService(new ConfigService({ PROMETHEUS_URL: 'http://user:secret@127.0.0.1:9090' }))).toThrow('must not contain credentials');
|
||||
expect(() => new InfrastructureMonitoringService(new ConfigService({ PROMETHEUS_URL: 'http://monitor.example.com:9090' }))).toThrow('must use HTTPS');
|
||||
expect(() => new InfrastructureMonitoringService(new ConfigService({ PROMETHEUS_URL: 'https://monitor.example.com' }))).not.toThrow();
|
||||
});
|
||||
|
||||
it('loads real Prometheus vectors, ranges, services and active alerts', async () => {
|
||||
const requestedUrls: URL[] = [];
|
||||
jest.spyOn(global, 'fetch').mockImplementation(async (input) => {
|
||||
const url = new URL(String(input));
|
||||
requestedUrls.push(url);
|
||||
if (url.pathname.endsWith('/alerts')) {
|
||||
return success({ alerts: [{
|
||||
labels: { alertname: 'HostCpuHigh', severity: 'warning', instance: '127.0.0.1:9100' },
|
||||
annotations: { summary: 'CPU持续偏高', threshold: '85%' },
|
||||
state: 'firing',
|
||||
activeAt: '2026-08-14T03:00:00.000Z',
|
||||
value: '88.2',
|
||||
}] });
|
||||
}
|
||||
const query = url.searchParams.get('query') ?? '';
|
||||
if (url.pathname.endsWith('/query_range')) {
|
||||
return success({ result: [{ metric: {}, values: [[1_765_000_000, '12.5'], [1_765_000_060, '14.5']] }] });
|
||||
}
|
||||
if (query.includes('node_systemd_unit_state')) {
|
||||
return success({ result: [
|
||||
{ metric: { name: 'cmpp-api.service' }, value: [1_765_000_060, '1'] },
|
||||
{ metric: { name: 'cmpp-gateway.service' }, value: [1_765_000_060, '1'] },
|
||||
{ metric: { name: 'postgresql.service' }, value: [1_765_000_060, '1'] },
|
||||
{ metric: { name: 'redis-server.service' }, value: [1_765_000_060, '1'] },
|
||||
{ metric: { name: 'cmpp-minio.service' }, value: [1_765_000_060, '1'] },
|
||||
{ metric: { name: 'nginx.service' }, value: [1_765_000_060, '1'] },
|
||||
] });
|
||||
}
|
||||
if (query.includes('timestamp(node_uname_info)')) {
|
||||
return success({ result: [{ metric: {}, value: [1_765_000_060, '1765000060'] }] });
|
||||
}
|
||||
return success({ result: [{ metric: {}, value: [1_765_000_060, '25'] }] });
|
||||
});
|
||||
|
||||
const service = new InfrastructureMonitoringService(new ConfigService());
|
||||
const result = await service.overview('1h');
|
||||
|
||||
expect(result.available).toBe(true);
|
||||
expect(result.metrics.cpuUsagePercent).toBe(25);
|
||||
expect(result.trends.cpuUsagePercent).toHaveLength(2);
|
||||
expect(result.summary).toMatchObject({ overallStatus: 'warning', serviceHealthy: 6, warningAlerts: 1 });
|
||||
expect(result.services.find((item) => item.key === 'redis')).toMatchObject({ unit: 'redis-server.service', status: 'healthy' });
|
||||
expect(result.alerts[0]).toMatchObject({ name: 'HostCpuHigh', severity: 'warning', currentValue: '88.2' });
|
||||
expect(requestedUrls.filter((url) => url.pathname.endsWith('/query_range'))).toHaveLength(5);
|
||||
expect(requestedUrls.filter((url) => url.pathname.endsWith('/query_range')).every((url) => url.searchParams.get('step') === '60')).toBe(true);
|
||||
});
|
||||
|
||||
it('returns an explicit unavailable payload without stale metrics when Prometheus fails', async () => {
|
||||
jest.spyOn(global, 'fetch').mockRejectedValue(new Error('ECONNREFUSED'));
|
||||
const service = new InfrastructureMonitoringService(new ConfigService());
|
||||
|
||||
const result = await service.overview('24h');
|
||||
|
||||
expect(result.available).toBe(false);
|
||||
expect(result.summary.overallStatus).toBe('unknown');
|
||||
expect(result.metrics.cpuUsagePercent).toBeNull();
|
||||
expect(result.trends.cpuUsagePercent).toEqual([]);
|
||||
expect(result.error).not.toContain('ECONNREFUSED');
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,264 @@
|
||||
import { BadRequestException, Injectable } from '@nestjs/common';
|
||||
import { ConfigService } from '@nestjs/config';
|
||||
import { createHash } from 'node:crypto';
|
||||
import type {
|
||||
InfrastructureAlert,
|
||||
InfrastructureMetricPoint,
|
||||
InfrastructureMonitoringOverview,
|
||||
InfrastructureMonitoringRange,
|
||||
InfrastructureServiceStatus,
|
||||
} from './infrastructure-monitoring.contracts';
|
||||
|
||||
type PrometheusSample = [number, string];
|
||||
type PrometheusSeries = {
|
||||
metric: Record<string, string>;
|
||||
value?: PrometheusSample;
|
||||
values?: PrometheusSample[];
|
||||
};
|
||||
type PrometheusQueryResponse = {
|
||||
status: 'success' | 'error';
|
||||
data?: { result?: PrometheusSeries[] };
|
||||
error?: string;
|
||||
};
|
||||
type PrometheusAlertResponse = {
|
||||
status: 'success' | 'error';
|
||||
data?: {
|
||||
alerts?: Array<{
|
||||
labels?: Record<string, string>;
|
||||
annotations?: Record<string, string>;
|
||||
state?: string;
|
||||
activeAt?: string;
|
||||
value?: string;
|
||||
}>;
|
||||
};
|
||||
};
|
||||
|
||||
const RANGE_CONFIG: Record<InfrastructureMonitoringRange, { seconds: number; step: number }> = {
|
||||
'1h': { seconds: 60 * 60, step: 60 },
|
||||
'24h': { seconds: 24 * 60 * 60, step: 300 },
|
||||
'7d': { seconds: 7 * 24 * 60 * 60, step: 1800 },
|
||||
};
|
||||
|
||||
const QUERIES = {
|
||||
cpuUsagePercent: '100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100)',
|
||||
memoryUsagePercent: '(1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) * 100',
|
||||
memoryTotalBytes: 'node_memory_MemTotal_bytes',
|
||||
memoryAvailableBytes: 'node_memory_MemAvailable_bytes',
|
||||
diskUsagePercent: '(1 - (node_filesystem_avail_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"} / node_filesystem_size_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"})) * 100',
|
||||
diskTotalBytes: 'node_filesystem_size_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"}',
|
||||
diskAvailableBytes: 'node_filesystem_avail_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"}',
|
||||
networkReceiveBytesPerSecond: 'sum(rate(node_network_receive_bytes_total{device!~"lo"}[5m]))',
|
||||
networkTransmitBytesPerSecond: 'sum(rate(node_network_transmit_bytes_total{device!~"lo"}[5m]))',
|
||||
load1: 'node_load1',
|
||||
uptimeSeconds: 'time() - node_boot_time_seconds',
|
||||
lastSampleAt: 'max(timestamp(node_uname_info))',
|
||||
services: 'max by (name) (node_systemd_unit_state{name=~"cmpp-api\\.service|cmpp-gateway\\.service|postgresql\\.service|redis(-server)?\\.service|cmpp-minio\\.service|nginx\\.service",state="active"})',
|
||||
} as const;
|
||||
|
||||
const SERVICE_DEFINITIONS = [
|
||||
{ key: 'api', name: 'API服务', units: ['cmpp-api.service'] },
|
||||
{ key: 'gateway', name: 'Gateway服务', units: ['cmpp-gateway.service'] },
|
||||
{ key: 'postgresql', name: 'PostgreSQL', units: ['postgresql.service'] },
|
||||
{ key: 'redis', name: 'Redis', units: ['redis.service', 'redis-server.service'] },
|
||||
{ key: 'minio', name: 'MinIO', units: ['cmpp-minio.service'] },
|
||||
{ key: 'nginx', name: 'Nginx', units: ['nginx.service'] },
|
||||
] as const;
|
||||
|
||||
function finiteNumber(value: string | number | undefined): number | null {
|
||||
const parsed = Number(value);
|
||||
return Number.isFinite(parsed) ? parsed : null;
|
||||
}
|
||||
|
||||
function normalizePrometheusUrl(rawValue: unknown) {
|
||||
const url = new URL(String(rawValue ?? 'http://127.0.0.1:9090'));
|
||||
if (url.protocol !== 'http:' && url.protocol !== 'https:') throw new Error('PROMETHEUS_URL must use HTTP or HTTPS');
|
||||
if (url.username || url.password) throw new Error('PROMETHEUS_URL must not contain credentials');
|
||||
const privateIpv4 = /^(10\.|192\.168\.|172\.(1[6-9]|2\d|3[01])\.)/.test(url.hostname);
|
||||
const loopback = url.hostname === '127.0.0.1' || url.hostname === 'localhost' || url.hostname === '[::1]';
|
||||
// Plain HTTP is only safe on loopback or an explicit RFC1918 address; named remote endpoints must use HTTPS.
|
||||
if (url.protocol === 'http:' && !loopback && !privateIpv4) throw new Error('Remote PROMETHEUS_URL must use HTTPS');
|
||||
return url.toString().replace(/\/$/, '');
|
||||
}
|
||||
|
||||
function vectorValue(response: PrometheusQueryResponse): number | null {
|
||||
return finiteNumber(response.data?.result?.[0]?.value?.[1]);
|
||||
}
|
||||
|
||||
function matrixValues(response: PrometheusQueryResponse): InfrastructureMetricPoint[] {
|
||||
return (response.data?.result?.[0]?.values ?? []).flatMap(([timestamp, value]) => {
|
||||
const parsed = finiteNumber(value);
|
||||
return parsed === null ? [] : [{ timestamp: new Date(timestamp * 1000).toISOString(), value: parsed }];
|
||||
});
|
||||
}
|
||||
|
||||
function emptyMetrics(): InfrastructureMonitoringOverview['metrics'] {
|
||||
return {
|
||||
cpuUsagePercent: null,
|
||||
memoryUsagePercent: null,
|
||||
memoryTotalBytes: null,
|
||||
memoryAvailableBytes: null,
|
||||
diskUsagePercent: null,
|
||||
diskTotalBytes: null,
|
||||
diskAvailableBytes: null,
|
||||
networkReceiveBytesPerSecond: null,
|
||||
networkTransmitBytesPerSecond: null,
|
||||
load1: null,
|
||||
uptimeSeconds: null,
|
||||
};
|
||||
}
|
||||
|
||||
function emptyTrends(): InfrastructureMonitoringOverview['trends'] {
|
||||
return {
|
||||
cpuUsagePercent: [],
|
||||
memoryUsagePercent: [],
|
||||
diskUsagePercent: [],
|
||||
networkReceiveBytesPerSecond: [],
|
||||
networkTransmitBytesPerSecond: [],
|
||||
};
|
||||
}
|
||||
|
||||
@Injectable()
|
||||
export class InfrastructureMonitoringService {
|
||||
private readonly prometheusUrl: string;
|
||||
private readonly queryTimeoutMs: number;
|
||||
|
||||
constructor(config: ConfigService) {
|
||||
this.prometheusUrl = normalizePrometheusUrl(config.get('PROMETHEUS_URL'));
|
||||
this.queryTimeoutMs = Math.min(15_000, Math.max(1_000, Number(config.get('PROMETHEUS_QUERY_TIMEOUT_MS') ?? 5_000)));
|
||||
}
|
||||
|
||||
async overview(rawRange?: string): Promise<InfrastructureMonitoringOverview> {
|
||||
const range = this.parseRange(rawRange);
|
||||
const collectedAt = new Date().toISOString();
|
||||
try {
|
||||
const [instant, trends, serviceResponse, alertResponse] = await Promise.all([
|
||||
this.loadInstantMetrics(),
|
||||
this.loadTrends(range),
|
||||
this.query(QUERIES.services),
|
||||
this.getJson<PrometheusAlertResponse>('/api/v1/alerts'),
|
||||
]);
|
||||
const services = this.parseServices(serviceResponse);
|
||||
const alerts = this.parseAlerts(alertResponse);
|
||||
const warningAlerts = alerts.filter((item) => item.severity === 'warning').length;
|
||||
const criticalAlerts = alerts.filter((item) => item.severity === 'critical').length;
|
||||
const overallStatus = criticalAlerts > 0 ? 'critical' : warningAlerts > 0 ? 'warning' : 'healthy';
|
||||
return {
|
||||
available: true,
|
||||
range,
|
||||
collectedAt,
|
||||
lastSampleAt: instant.lastSampleAt === null ? null : new Date(instant.lastSampleAt * 1000).toISOString(),
|
||||
summary: {
|
||||
overallStatus,
|
||||
serviceTotal: services.length,
|
||||
serviceHealthy: services.filter((item) => item.status === 'healthy').length,
|
||||
warningAlerts,
|
||||
criticalAlerts,
|
||||
activeAlerts: alerts.length,
|
||||
},
|
||||
metrics: instant.metrics,
|
||||
trends,
|
||||
services,
|
||||
alerts,
|
||||
};
|
||||
} catch {
|
||||
return this.unavailable(range, collectedAt);
|
||||
}
|
||||
}
|
||||
|
||||
private parseRange(value?: string): InfrastructureMonitoringRange {
|
||||
const range = value || '24h';
|
||||
if (!(range in RANGE_CONFIG)) throw new BadRequestException('监控时间范围只支持1h、24h或7d');
|
||||
return range as InfrastructureMonitoringRange;
|
||||
}
|
||||
|
||||
private async loadInstantMetrics() {
|
||||
const keys = Object.keys(emptyMetrics()) as Array<keyof InfrastructureMonitoringOverview['metrics']>;
|
||||
const responses = await Promise.all([...keys.map((key) => this.query(QUERIES[key])), this.query(QUERIES.lastSampleAt)]);
|
||||
const metrics = emptyMetrics();
|
||||
keys.forEach((key, index) => { metrics[key] = vectorValue(responses[index]); });
|
||||
return { metrics, lastSampleAt: vectorValue(responses[responses.length - 1]) };
|
||||
}
|
||||
|
||||
private async loadTrends(range: InfrastructureMonitoringRange) {
|
||||
const config = RANGE_CONFIG[range];
|
||||
const end = Math.floor(Date.now() / 1000);
|
||||
const start = end - config.seconds;
|
||||
const keys = Object.keys(emptyTrends()) as Array<keyof InfrastructureMonitoringOverview['trends']>;
|
||||
const responses = await Promise.all(keys.map((key) => this.queryRange(QUERIES[key], start, end, config.step)));
|
||||
return Object.fromEntries(keys.map((key, index) => [key, matrixValues(responses[index])])) as InfrastructureMonitoringOverview['trends'];
|
||||
}
|
||||
|
||||
private parseServices(response: PrometheusQueryResponse): InfrastructureServiceStatus[] {
|
||||
const values = new Map<string, number>();
|
||||
for (const item of response.data?.result ?? []) {
|
||||
if (item.metric.name) values.set(item.metric.name, vectorValue({ status: 'success', data: { result: [item] } }) ?? 0);
|
||||
}
|
||||
return SERVICE_DEFINITIONS.map((definition) => {
|
||||
const present = definition.units.filter((unit) => values.has(unit));
|
||||
const status = present.length === 0 ? 'unknown' : present.some((unit) => (values.get(unit) ?? 0) >= 1) ? 'healthy' : 'unhealthy';
|
||||
return { key: definition.key, name: definition.name, unit: present[0] ?? definition.units[0], status };
|
||||
});
|
||||
}
|
||||
|
||||
private parseAlerts(response: PrometheusAlertResponse): InfrastructureAlert[] {
|
||||
return (response.data?.alerts ?? [])
|
||||
.filter((item) => item.state === 'firing' || item.state === 'pending')
|
||||
.map<InfrastructureAlert>((item) => {
|
||||
const labels = item.labels ?? {};
|
||||
const annotations = item.annotations ?? {};
|
||||
const severity: InfrastructureAlert['severity'] = labels.severity === 'critical' ? 'critical' : labels.severity === 'warning' ? 'warning' : 'info';
|
||||
const identity = JSON.stringify(Object.entries(labels).sort(([left], [right]) => left.localeCompare(right)));
|
||||
return {
|
||||
fingerprint: createHash('sha256').update(identity).digest('hex').slice(0, 24),
|
||||
name: labels.alertname || '未命名告警',
|
||||
severity,
|
||||
status: item.state || 'unknown',
|
||||
startedAt: item.activeAt || new Date().toISOString(),
|
||||
summary: annotations.summary || annotations.description || labels.alertname || '监控告警',
|
||||
description: annotations.description,
|
||||
currentValue: annotations.currentValue || item.value,
|
||||
threshold: annotations.threshold,
|
||||
service: labels.service,
|
||||
instance: labels.instance,
|
||||
};
|
||||
})
|
||||
.sort((left, right) => {
|
||||
const priority: Record<InfrastructureAlert['severity'], number> = { critical: 0, warning: 1, info: 2 };
|
||||
return priority[left.severity] - priority[right.severity] || Date.parse(left.startedAt) - Date.parse(right.startedAt);
|
||||
});
|
||||
}
|
||||
|
||||
private unavailable(range: InfrastructureMonitoringRange, collectedAt: string): InfrastructureMonitoringOverview {
|
||||
const services = SERVICE_DEFINITIONS.map((item) => ({ key: item.key, name: item.name, unit: item.units[0], status: 'unknown' as const }));
|
||||
return {
|
||||
available: false,
|
||||
range,
|
||||
collectedAt,
|
||||
lastSampleAt: null,
|
||||
error: 'Prometheus监控数据当前不可用，请检查采集与服务状态',
|
||||
summary: { overallStatus: 'unknown', serviceTotal: services.length, serviceHealthy: 0, warningAlerts: 0, criticalAlerts: 0, activeAlerts: 0 },
|
||||
metrics: emptyMetrics(),
|
||||
trends: emptyTrends(),
|
||||
services,
|
||||
alerts: [],
|
||||
};
|
||||
}
|
||||
|
||||
private query(query: string) {
|
||||
return this.getJson<PrometheusQueryResponse>('/api/v1/query', { query });
|
||||
}
|
||||
|
||||
private queryRange(query: string, start: number, end: number, step: number) {
|
||||
return this.getJson<PrometheusQueryResponse>('/api/v1/query_range', { query, start: String(start), end: String(end), step: String(step) });
|
||||
}
|
||||
|
||||
private async getJson<T extends { status: 'success' | 'error'; error?: string }>(path: string, params: Record<string, string> = {}): Promise<T> {
|
||||
const url = new URL(`${this.prometheusUrl}${path}`);
|
||||
Object.entries(params).forEach(([key, value]) => url.searchParams.set(key, value));
|
||||
const response = await fetch(url, { headers: { Accept: 'application/json' }, signal: AbortSignal.timeout(this.queryTimeoutMs) });
|
||||
if (!response.ok) throw new Error(`Prometheus HTTP ${response.status}`);
|
||||
const result = await response.json() as T;
|
||||
if (result.status !== 'success') throw new Error('Prometheus query failed');
|
||||
return result;
|
||||
}
|
||||
}
|
||||
@@ -2025,6 +2025,17 @@
|
||||
- 运营端“短信通道组管理”在通道组名称条件之外增加“通道”筛选。选定一个通道后，只展示成员配置中真实包含该`channelId`的未删除通道组；与通道组名称同时输入时按两个条件取交集。
|
||||
- 通道选项必须来自真实通道API，不使用静态列表、Mock或localStorage。页面首次加载时通道组与通道两个独立请求并行执行；选项同时展示通道名称和编码，已删除通道明确标记“已删除”。未加入任何通道组的真实通道仍可选择，选中后结果为零而不得隐藏该选项。
|
||||
- 通道选择控件必须为通用下拉可搜索控件，支持按通道名称或编码搜索；“全部通道”表示不按通道限制，点击“重置”必须同时清空通道组名称和通道条件并回到第一页。
|
||||
|
||||
## Prometheus 系统监控（2026-08-14）
|
||||
|
||||
- 运营端“系统管理”新增“系统监控”，路由为`/admin/system-monitoring`；原“发送监控”继续负责短信通道和消息业务指标，两个页面、接口和统计口径不得混用。
|
||||
- 系统监控使用Prometheus和Node Exporter作为真实指标基础设施，但全部用户界面由平台React原生实现，不嵌入Grafana、Netdata、Zabbix、Prometheus页面或第三方登录界面。
|
||||
- 浏览器只请求`GET /api/admin/infrastructure-monitoring/overview?range=1h|24h|7d`。NestJS以固定PromQL模板查询Prometheus，禁止前端传任意PromQL、step、时间戳或标签选择器；9090和9100只监听本机或内网，不向公网开放。
|
||||
- 第一版展示CPU、内存、根文件系统、网络收发、1分钟负载和系统运行时长，并展示API、Gateway、PostgreSQL、Redis、MinIO、Nginx六类systemd服务状态。指标缺失必须显示“暂无指标/未知”，不得用0、静态数据、Mock或localStorage冒充真实采集值。
|
||||
- 支持近1小时、近24小时和近7天固定范围，步长分别为60秒、300秒和1800秒。页面可见时每30秒刷新，隐藏或卸载后停止；手动刷新保留当前范围。
|
||||
- 页面展示Prometheus当前firing/pending告警，严重性使用`info/warning/critical`。告警阈值由Prometheus规则计算，前端不重复判断；第一版仅只读展示，不提供缺少审计模型的确认、备注、静默或关闭操作。
|
||||
- Prometheus不可用、查询超时、响应非法时接口返回`available=false`及安全错误摘要，页面清空陈旧指标并展示监控不可用；不能继续显示上一次数据造成误判。
|
||||
- 完整采集、查询、安全、响应契约、视觉规格和验收口径见`docs/prometheus-system-monitoring-design-20260814.md`。
|
||||
# 下游投递后台重投任务（2026-08-12）
|
||||
|
||||
## 完整设计口径与安全整改（2026-08-13）
|
||||
|
||||
@@ -48,6 +48,8 @@ OPERATION_LOG_ARCHIVE_INTERVAL_MS=86400000
|
||||
SMS_RECEIPT_TIMEOUT_SCAN_ENABLED=true
|
||||
SMS_RECEIPT_TIMEOUT_HOURS=72
|
||||
SMS_RECEIPT_TIMEOUT_SCAN_INTERVAL_MS=300000
|
||||
PROMETHEUS_URL=http://127.0.0.1:9090
|
||||
PROMETHEUS_QUERY_TIMEOUT_MS=5000
|
||||
REPORT_DAILY_REFRESH_ENABLED=true
|
||||
REPORT_REFRESH_INTERVAL_MS=3600000
|
||||
CMPP_DOWNSTREAM_ACK_TIMEOUT_SECONDS=30
|
||||
@@ -62,6 +64,8 @@ PROD_ADMIN_USERNAME=prod_admin
|
||||
PROD_ADMIN_PASSWORD='change-me'
|
||||
```
|
||||
|
||||
系统监控的Prometheus和Node Exporter需要在发布前单独安装，配置和规则位于`tools/monitoring/`。在Debian/Ubuntu服务器执行`bash tools/monitoring/install-prometheus-monitoring.sh`；脚本会先备份现有Prometheus配置并运行`promtool`校验，只重启Prometheus和Node Exporter，不重启CMPP业务服务。9090和9100必须只监听`127.0.0.1`，不得加入Nginx公网反向代理或安全组放行。安装完成且确认`curl http://127.0.0.1:9090/-/ready`成功后，才可在正常发布窗口重启API使系统监控接口生效；详细口径见`docs/prometheus-system-monitoring-design-20260814.md`。
|
||||
|
||||
安全会话使用 HttpOnly Cookie，正式生产必须先为页面和管理 API 配置 HTTPS，并保持 `SESSION_COOKIE_SECURE=true`；纯 HTTP 的 `IP:12026` 不作为受支持的登录入口，即使切换期仍保留其监听，也只允许用于非登录的兼容检查并应尽快下线。`sms.lisglo.com` 只允许 Cloudflare 回源，`api.lisglo.com` 通过独立 Nginx SNI 虚拟主机只开放客户接口、客户 Swagger 和健康检查；Let’s Encrypt 使用 DNS-01 自动续期，不依赖开放 80 端口。
|
||||
|
||||
系统操作日志默认在线保留 180 天。API 每日以最多 20 个、每批 1000 条的小事务将过期记录搬入 `OperationLogArchive`，并用 `archiveMonth=YYYY-MM` 标记归档月份；归档记录不会自动删除。调整保留期或批量参数前，应先评估数据库、备份窗口和审计要求。归档表达到千万级或清理窗口不能满足要求时，再实施按 `createdAt` 的月度 PostgreSQL 分区，不在当前数据规模下提前改造主表分区。
|
||||
|
||||
@@ -0,0 +1,194 @@
|
||||
# Prometheus 系统监控设计
|
||||
|
||||
> 版本：V1.0<br>
|
||||
> 设计日期：2026-08-14<br>
|
||||
> 适用范围：运营端 → 系统管理 → 系统监控<br>
|
||||
> 实施边界：使用平台原生 React UI，不引入 Grafana、Netdata、Zabbix 或 Prometheus 自带 UI
|
||||
|
||||
## 1. 建设目标
|
||||
|
||||
运营人员需要在现有权限体系和视觉体系内查看服务器硬件资源、核心服务状态、历史趋势和活动告警。Prometheus 仅承担指标抓取、时序存储、PromQL 计算和告警规则执行；浏览器只访问 CMPP 平台 NestJS API，不直接访问 Prometheus、Node Exporter 或 Alertmanager。
|
||||
|
||||
第一版解决以下问题：
|
||||
|
||||
1. 统一查看 CPU、内存、根文件系统、网络流量、负载和运行时长。
|
||||
2. 查看 `cmpp-api`、`cmpp-gateway`、PostgreSQL、Redis、MinIO、Nginx 六类核心服务状态。
|
||||
3. 在近 1 小时、近 24 小时、近 7 天之间切换真实历史趋势。
|
||||
4. 查看 Prometheus 当前 firing/pending 告警，不在前端伪造阈值判断。
|
||||
5. Prometheus 不可用、查询超时或指标缺失时明确展示“监控不可用/暂无指标”，不得回退静态值、Mock 或 localStorage。
|
||||
|
||||
## 2. 非目标与边界
|
||||
|
||||
- 第一版不提供任意 PromQL 控制台，避免越权查询、高基数查询和资源耗尽。
|
||||
- 第一版不提供告警确认、备注和关闭操作；需要审计留痕的告警处置另立需求并增加 PostgreSQL 模型。
|
||||
- 第一版不采集短信正文、手机号、账号、密钥、数据库查询文本或日志原文。
|
||||
- 云主机通常只能提供虚拟 CPU、内存、云盘和虚拟网卡指标；风扇、物理温度、电源、RAID 和物理硬盘 SMART 需 IPMI/Redfish 或厂商接口，未接入时不得显示伪造数据。
|
||||
- Prometheus、Node Exporter 和 Alertmanager 不暴露在公网；仅 API 服务端可以访问 Prometheus。
|
||||
|
||||
## 3. 总体架构
|
||||
|
||||
```mermaid
|
||||
flowchart LR
|
||||
Node["Node Exporter\n主机与 systemd 指标"] --> Prom["Prometheus\n抓取、存储、PromQL、告警规则"]
|
||||
API["NestJS 监控模块\n固定查询模板与响应归一化"] --> Prom
|
||||
UI["运营端原生 React UI"] --> API
|
||||
Prom --> Alert["Prometheus 活动告警"]
|
||||
Alert --> API
|
||||
```
|
||||
|
||||
数据流必须是 `Exporter → Prometheus → NestJS → 运营端`。前端不得直接拼接 Prometheus URL，不得保存 Prometheus Token，不得接受服务端返回的原始 PromQL。
|
||||
|
||||
## 4. 采集与部署设计
|
||||
|
||||
### 4.1 Node Exporter
|
||||
|
||||
Node Exporter 仅监听 `127.0.0.1:9100`，启用默认 CPU、内存、文件系统、磁盘、网络、负载和启动时间采集器，并显式启用 `systemd` 采集器。systemd 只允许采集：
|
||||
|
||||
- `cmpp-api.service`
|
||||
- `cmpp-gateway.service`
|
||||
- `postgresql.service`
|
||||
- `redis.service` 或 `redis-server.service`
|
||||
- `cmpp-minio.service`
|
||||
- `nginx.service`
|
||||
|
||||
排除 `/dev`、`/proc`、`/sys`、`/run` 等伪文件系统和临时挂载，避免磁盘指标重复和高基数。
|
||||
|
||||
### 4.2 Prometheus
|
||||
|
||||
- 仅监听 `127.0.0.1:9090`。
|
||||
- 默认每 15 秒抓取一次，查询超时 5 秒。
|
||||
- 第一版保留 30 天或不超过 8GB 的时序数据，达到任一边界即按 Prometheus TSDB 策略清理。
|
||||
- 配置文件由仓库 `tools/monitoring/` 管理；安装脚本只安装、校验和启动监控服务，不重启 CMPP Gateway 或发送 Worker。
|
||||
- 生产环境变量使用 `PROMETHEUS_URL=http://127.0.0.1:9090`，API 不向响应透出该地址。
|
||||
|
||||
### 4.3 告警规则
|
||||
|
||||
规则使用持续窗口而非瞬时尖峰：
|
||||
|
||||
| 告警 | Warning | Critical |
|
||||
|---|---:|---:|
|
||||
| 主机指标失联 | — | 连续 2 分钟无数据 |
|
||||
| CPU 使用率 | 连续 10 分钟 > 85% | 连续 5 分钟 > 95% |
|
||||
| 内存使用率 | 连续 10 分钟 > 85% | 连续 5 分钟 > 95% |
|
||||
| 根文件系统使用率 | 连续 15 分钟 > 80% | 连续 5 分钟 > 90% |
|
||||
| inode 使用率 | 连续 15 分钟 > 80% | 连续 5 分钟 > 90% |
|
||||
| CPU iowait | 连续 10 分钟 > 20% | 连续 10 分钟 > 35% |
|
||||
| 核心 systemd 服务 | — | 非 active 2 分钟 |
|
||||
|
||||
告警标签至少包含 `alertname`、`severity`、`instance`、`service`；注解至少包含中文 `summary`、`description`、`currentValue`、`threshold`。敏感环境变量和凭据不得进入标签或注解。
|
||||
|
||||
## 5. 后端设计
|
||||
|
||||
### 5.1 API
|
||||
|
||||
```text
|
||||
GET /api/admin/infrastructure-monitoring/overview?range=1h|24h|7d
|
||||
```
|
||||
|
||||
该接口受现有运营端 Session 中间件保护，仅提供只读数据。`range` 只接受白名单；非法值返回 400。
|
||||
|
||||
### 5.2 固定查询
|
||||
|
||||
后端维护具名查询表，不接受前端 PromQL：
|
||||
|
||||
- CPU：非 idle CPU 秒率。
|
||||
- 内存：`1 - MemAvailable / MemTotal`。
|
||||
- 根文件系统：`1 - available / size`。
|
||||
- 网络：排除 loopback 后的收发字节率。
|
||||
- 负载：`node_load1`。
|
||||
- 运行时长：当前时间减 `node_boot_time_seconds`。
|
||||
- 服务：指定 unit 的 `node_systemd_unit_state{state="active"}`。
|
||||
- 告警：Prometheus `/api/v1/alerts`。
|
||||
|
||||
瞬时查询和区间查询并行执行。区间步长固定为：1小时/60秒、24小时/300秒、7天/1800秒；后端最多返回约 340 个点/序列，禁止浏览器控制步长。
|
||||
|
||||
### 5.3 可用性与错误语义
|
||||
|
||||
- Prometheus 查询成功：`available=true`，返回真实指标、服务和告警。
|
||||
- Prometheus 未配置、拒绝连接、超时、返回非成功状态或响应结构非法：`available=false`，返回采集时间和安全错误摘要，指标字段为 `null`、趋势为空数组。
|
||||
- 单个指标不存在不影响其他指标，缺失项为 `null`；不得把缺失解释为 0。
|
||||
- 服务状态使用 `healthy/unhealthy/unknown`，只有明确采集到 active 才是 healthy，明确采集到 0 才是 unhealthy，指标缺失是 unknown。
|
||||
|
||||
### 5.4 安全控制
|
||||
|
||||
- Prometheus URL 由服务端环境变量读取并限制为 `http://127.0.0.1` 或明确配置的内网 HTTPS 地址。
|
||||
- 每次请求设置 5 秒 AbortSignal 超时。
|
||||
- 不记录完整响应体；错误日志不得包含URL中的认证信息。
|
||||
- 不允许前端传 query、step、start、end 或 Prometheus 标签选择器。
|
||||
- 不将监控指标复制进业务 PostgreSQL，避免高频写入和业务库膨胀。
|
||||
|
||||
## 6. 前端信息架构
|
||||
|
||||
页面路由为 `/admin/system-monitoring`，保留原 `/admin/monitor` “发送监控”，两者业务含义不得混用。
|
||||
|
||||
### 6.1 页面结构
|
||||
|
||||
1. 页面标题、最近采集时间、刷新按钮、1小时/24小时/7天范围切换。
|
||||
2. 横向健康概览：整体状态、服务总数、正常、警告、严重、活动告警。
|
||||
3. CPU、内存、磁盘、网络四项资源区：当前值、辅助值和当前范围趋势。
|
||||
4. 服务状态侧栏：六类服务、状态和最新采集时间。
|
||||
5. 活动告警表：名称、级别、开始时间、持续时间、当前值、阈值。
|
||||
6. 下方资源趋势：CPU、内存、磁盘和网络收发的完整趋势图。
|
||||
|
||||
### 6.2 视觉规格
|
||||
|
||||
- 复用现有 `surface`、`Button`、`Tag`、`Table`、`Chart` 和主题变量。
|
||||
- 真白背景、低饱和蓝色主色、冷灰边框；正常/警告/严重使用既有语义色。
|
||||
- 不使用第三方Logo、iframe、暗色监控皮肤、霓虹渐变或装饰性假指标。
|
||||
- 桌面优先保持表格和趋势信息密度；1100px 以下两列，780px 以下单列。
|
||||
- 图表缺失时显示“暂无真实指标”，不能绘制零线冒充数据。
|
||||
|
||||
### 6.3 交互与刷新
|
||||
|
||||
- 初次进入立即请求真实接口。
|
||||
- 可见页面每 30 秒刷新；页面隐藏或 Session 锁定导致组件卸载后停止刷新。
|
||||
- 切换时间范围立即重新查询并禁用重复点击；手动刷新不改变范围。
|
||||
- 请求失败保留上一次成功数据会造成陈旧误导，因此本版失败时清空数据并展示不可用状态。
|
||||
|
||||
## 7. 响应契约摘要
|
||||
|
||||
```ts
|
||||
type InfrastructureOverview = {
|
||||
available: boolean;
|
||||
range: '1h' | '24h' | '7d';
|
||||
collectedAt: string;
|
||||
error?: string;
|
||||
summary: {
|
||||
overallStatus: 'healthy' | 'warning' | 'critical' | 'unknown';
|
||||
serviceTotal: number;
|
||||
serviceHealthy: number;
|
||||
warningAlerts: number;
|
||||
criticalAlerts: number;
|
||||
activeAlerts: number;
|
||||
};
|
||||
metrics: {
|
||||
cpuUsagePercent: number | null;
|
||||
memoryUsagePercent: number | null;
|
||||
memoryTotalBytes: number | null;
|
||||
memoryAvailableBytes: number | null;
|
||||
diskUsagePercent: number | null;
|
||||
diskTotalBytes: number | null;
|
||||
diskAvailableBytes: number | null;
|
||||
networkReceiveBytesPerSecond: number | null;
|
||||
networkTransmitBytesPerSecond: number | null;
|
||||
load1: number | null;
|
||||
uptimeSeconds: number | null;
|
||||
};
|
||||
trends: Record<string, Array<{ timestamp: string; value: number }>>;
|
||||
services: Array<{ key: string; name: string; unit: string; status: 'healthy' | 'unhealthy' | 'unknown' }>;
|
||||
alerts: Array<{ fingerprint: string; name: string; severity: 'warning' | 'critical' | 'info'; status: string; startedAt: string; summary: string; currentValue?: string; threshold?: string }>;
|
||||
};
|
||||
```
|
||||
|
||||
## 8. 验收标准
|
||||
|
||||
1. 页面所有指标来自真实 Prometheus API，关闭 Prometheus 后页面明确不可用且没有静态回退。
|
||||
2. 浏览器网络请求只访问 CMPP API，不访问 9090/9100。
|
||||
3. 非法 range、任意 PromQL 和自定义 step 均无法进入后端查询。
|
||||
4. CPU、内存、磁盘、网络当前值与 Prometheus 同时刻查询在允许的采样误差内一致。
|
||||
5. 六类服务状态与 systemd 指标一致；指标缺失展示未知而非故障或正常。
|
||||
6. 1小时、24小时、7天切换后时间轴和点数符合固定步长。
|
||||
7. firing/pending 告警真实展示，告警恢复后不再出现在活动列表。
|
||||
8. Prometheus/Exporter 只监听本机，公网不能连接 9090/9100。
|
||||
9. 页面在桌面和移动宽度下无重叠、截断和横向溢出，控制台无相关错误。
|
||||
10. TypeScript、API专项测试、生产构建、配置校验和 `git diff --check`通过。
|
||||
@@ -4628,3 +4628,25 @@ npm run verify:phase8
|
||||
| TC-DASHBOARD-SIGNATURE-REMOVE-001 | 删除看板签名统计模块并统一金额样式 | 打开运营看板，检查指标卡与后续模块 | 两个“今日签名发送统计”模块均不存在；今日消费金额与今日发送总量主数字字号、字重和颜色一致；今日活跃签名仍来自真实接口 |
|
||||
| TC-ENTERPRISE-APP-WIDTH-001 | 企业应用关键列缩窄 | 在桌面大屏打开企业应用管理并读取表头列宽 | 状态、到达率、单价列宽均约为原宽度80%，字段与操作均未隐藏，横向滚动需求减少 |
|
||||
| TC-UI-CARRIER-TAG-002 | 全局运营商数据展示复用通用标签 | 抽查通道/通道组、监控、报备、企业签名、短信审核、批次号码、发送记录和客户端发送详情 | 移动、联通、电信均使用全局低饱和胶囊及统一色值；有运营商集合的三网通道显示三个标签，历史通道级字段显示中性“三网”；筛选选项、图表图例与导出文本保持纯文本 |
|
||||
|
||||
## Prometheus 系统监控专项用例（2026-08-14）
|
||||
|
||||
| 用例编号 | 场景 | 操作 | 预期结果 |
|
||||
|---|---|---|---|
|
||||
| TC-INFRA-MON-001 | 运营端权限与原生页面 | 登录运营端，打开“系统管理 → 系统监控”，检查页面和浏览器网络请求 | 页面使用平台导航、组件和样式；只请求平台`/api/admin/infrastructure-monitoring/overview`，不加载iframe，不从浏览器访问9090/9100，不出现第三方Logo或登录页 |
|
||||
| TC-INFRA-MON-002 | 未登录访问 | 清除运营端Session后直接访问系统监控API和页面 | API按现有Session中间件拒绝，页面进入运营端登录流程；Prometheus数据不得绕过运营端权限公开 |
|
||||
| TC-INFRA-MON-003 | 当前硬件指标真实值 | 在同一采样窗口分别请求平台监控API和Prometheus固定查询 | CPU、内存、根文件系统、网络、负载、运行时长与Prometheus结果在采样误差内一致，响应不包含Prometheus地址或PromQL |
|
||||
| TC-INFRA-MON-004 | 指标缺失 | 临时禁用某个Node Exporter采集器或查询一个不存在的指标后请求页面 | 对应指标为`null`并显示“暂无真实指标”，其他指标继续展示；不得显示0或生成趋势线 |
|
||||
| TC-INFRA-MON-005 | Prometheus不可用 | 停止本地测试Prometheus或将测试环境指向拒绝连接端口，请求监控API | API返回`available=false`和安全错误摘要，页面显示“监控数据不可用”并清空陈旧指标，不显示Mock或上次数据 |
|
||||
| TC-INFRA-MON-006 | 查询超时 | 让测试Prometheus响应超过5秒 | 请求被AbortSignal终止，接口有限时间返回不可用状态；API进程不积累悬挂请求 |
|
||||
| TC-INFRA-MON-007 | 时间范围白名单 | 分别请求`1h`、`24h`、`7d`、`30d`和注入PromQL字符串 | 前三种成功且step分别为60/300/1800秒；非法值返回400，不能进入Prometheus查询 |
|
||||
| TC-INFRA-MON-008 | 趋势切换 | 页面依次选择近1小时、近24小时、近7天 | 每次只发一个新范围请求；图表时间轴、点数和当前范围同步更新，切换期间防止重复触发 |
|
||||
| TC-INFRA-MON-009 | 自动与手动刷新 | 保持页面可见超过30秒，再隐藏页面并点击手动刷新 | 可见时按30秒刷新；隐藏后停止；重新可见后立即刷新；手动刷新保留范围且不会并发重复请求 |
|
||||
| TC-INFRA-MON-010 | 核心服务状态 | 核对`cmpp-api`、`cmpp-gateway`、PostgreSQL、Redis、MinIO、Nginx的systemd状态与页面 | active显示正常，明确0显示异常，指标不存在显示未知；Redis兼容`redis.service`与`redis-server.service`别名 |
|
||||
| TC-INFRA-MON-011 | 活动告警 | 触发一条warning和一条critical测试规则并等待Prometheus进入firing | 页面显示真实名称、严重性、开始时间、持续时间、当前值和阈值；概览计数准确，恢复后活动列表移除 |
|
||||
| TC-INFRA-MON-012 | 综合状态 | 分别构造无告警、warning、critical和Prometheus不可用状态 | 综合状态依次为正常、警告、严重、未知；critical优先于warning，不按前端瞬时指标重复计算 |
|
||||
| TC-INFRA-MON-013 | 并行查询与响应上限 | 检查后端请求时序，并用7天范围请求最大趋势 | 瞬时、趋势、服务、告警查询并行；每序列约不超过340点，响应不包含原始Prometheus响应体 |
|
||||
| TC-INFRA-MON-014 | 监听与公网暴露 | 在服务器执行`ss -lnt`并从外部探测9090、9100 | Prometheus和Node Exporter仅监听127.0.0.1或明确内网地址；公网9090/9100不可连接，运营端仍能经平台API读取指标 |
|
||||
| TC-INFRA-MON-015 | 响应式与无障碍 | 在1536×1024、1280×800和390×844打开页面，操作范围和刷新按钮 | 桌面信息层级符合设计稿；窄屏无内容重叠和页面横向溢出；按钮有可读名称，活动范围和告警严重性不只依赖颜色表达 |
|
||||
| TC-INFRA-MON-016 | 配置和部署幂等 | 在测试服务器重复执行监控安装脚本和配置校验 | 不重复创建系统用户，不开放公网端口；配置通过`promtool check config/rules`，服务保持active，CMPP API/Gateway不因安装被重启 |
|
||||
| TC-INFRA-MON-017 | 业务数据隔离 | 运行监控24小时并检查PostgreSQL业务库和指标标签 | 监控时序只保存在Prometheus TSDB，业务PostgreSQL无高频指标写入；标签、日志和API响应不含手机号、短信正文、账号或密钥 |
|
||||
|
||||
@@ -3573,3 +3573,12 @@ git diff --check
|
||||
- API、Gateway、Nginx、PostgreSQL、Redis与MinIO均active，内部API/Gateway/MinIO健康、Redis PONG；`gateway.submit.commands`消费者1、`pending=0`、`lag=0`。公网运营端、客户端和API health均HTTP 200，公网CMPP 17890 TCP连通；发布时间窗API/Gateway warning级日志为0。
|
||||
- 首次前台SSH执行因本地等待窗口关闭而收到终止信号，包装流程按设计恢复PostgreSQL和原运行目录，运行标识回到`4c70978d`、migration回到86条、服务全部active；确认恢复资产完整后改为服务器后台日志方式重新执行并成功，未并发重复部署。
|
||||
- 本轮没有创建真实重投任务、调用真实Gateway重投、发送短信、修改通道配置、余额或客户连接。既有`api/tsconfig.build.tsbuildinfo`、`tsconfig.tsbuildinfo`、`outputs/`和空文件`=`继续保护，不归入业务提交。
|
||||
|
||||
# 2026-08-14 Prometheus 系统监控（本地未提交、未发布）
|
||||
|
||||
- 运营端“系统管理”新增独立“系统监控”页面，保留原“发送监控”业务职责。页面使用平台React、ECharts和通用组件原生实现，不嵌入Grafana、Prometheus或第三方iframe；支持近1小时、24小时、7天，展示CPU、内存、根磁盘、网络、负载、运行时长、六类核心systemd服务和Prometheus活动告警。
|
||||
- 新增只读`GET /api/admin/infrastructure-monitoring/overview`。后端只接受`1h/24h/7d`白名单，按固定PromQL并行访问Prometheus，5秒超时；浏览器不能提交PromQL或访问9090/9100。缺失序列返回`null`，Prometheus异常返回`available=false`并清空指标和趋势，不用0、Mock、静态数据或localStorage伪装。
|
||||
- 新增Debian/Ubuntu幂等安装脚本、Prometheus抓取配置和告警规则。Prometheus与Node Exporter只监听`127.0.0.1`，默认保留30天且限制8GB；脚本备份既有配置和override、运行`promtool`校验，只重启两个监控服务，不重启API、Gateway或其他业务服务。本轮未在预生产执行脚本。
|
||||
- 专项Jest 1套/4项、API全量38套/467项通过，覆盖范围白名单、Prometheus HTTP响应契约解析、固定60秒step、服务别名、活动告警、不可用无陈旧数据及监控地址安全约束；全量Jest仍因仓库既有异步句柄使用`--forceExit`收尾。API正式TypeScript、前端TypeScript、Vite 8.1.5生产构建、两个Shell脚本语法及`git diff --check`通过；Vite仅保留既有约2.10MB单chunk提示。
|
||||
- 浏览器优先检查现有页面：本地运营端无有效登录Session，被正常引导到带算术验证码的登录页；未绕过登录、未读取或填写验证码，因此登录后桌面与窄屏视觉验收尚未完成。真实Prometheus/Node Exporter集成和告警触发验收必须在明确授权安装的测试或预生产窗口执行，当前不以单测替代真实基础设施验收。
|
||||
- 本轮代码、配置和文档保持未提交、未推送、未部署，没有发送、补发或重投短信，没有修改通道账号、密码、启停状态、企业余额、客户连接或预生产数据。既有构建缓存、`outputs/`和空文件`=`继续保护；并行会话新增的Fail2ban设计与测试文档不属于本需求，不修改、不归因。
|
||||
|
||||
@@ -0,0 +1,7 @@
|
||||
import { request, withQuery } from '../core/httpClient';
|
||||
import type { InfrastructureMonitoringOverview, InfrastructureMonitoringRange } from '../types';
|
||||
|
||||
export const adminInfrastructureMonitoringApi = {
|
||||
getInfrastructureMonitoringOverview: (range: InfrastructureMonitoringRange) =>
|
||||
request<InfrastructureMonitoringOverview>(withQuery('/admin/infrastructure-monitoring/overview', { range })),
|
||||
};
|
||||
@@ -10,6 +10,7 @@ import { adminOperationsApi } from './admin/operations.api';
|
||||
import { adminGovernanceApi } from './admin/governance.api';
|
||||
import { adminFilesApi } from './admin/files.api';
|
||||
import { adminSignatureRetirementApi } from './admin/signature-retirement.api';
|
||||
import { adminInfrastructureMonitoringApi } from './admin/infrastructure-monitoring.api';
|
||||
|
||||
export const adminApi = {
|
||||
...adminIdentityApi,
|
||||
@@ -18,4 +19,5 @@ export const adminApi = {
|
||||
...adminGovernanceApi,
|
||||
...adminFilesApi,
|
||||
...adminSignatureRetirementApi,
|
||||
...adminInfrastructureMonitoringApi,
|
||||
};
|
||||
|
||||
@@ -4,3 +4,4 @@ export * from './channels-reports';
|
||||
export * from './operations';
|
||||
export * from './governance';
|
||||
export * from './signature-retirement';
|
||||
export * from './infrastructure-monitoring';
|
||||
|
||||
@@ -0,0 +1,65 @@
|
||||
export type InfrastructureMonitoringRange = '1h' | '24h' | '7d';
|
||||
|
||||
export type InfrastructureMetricPoint = {
|
||||
timestamp: string;
|
||||
value: number;
|
||||
};
|
||||
|
||||
export type InfrastructureServiceStatus = {
|
||||
key: string;
|
||||
name: string;
|
||||
unit: string;
|
||||
status: 'healthy' | 'unhealthy' | 'unknown';
|
||||
};
|
||||
|
||||
export type InfrastructureAlert = {
|
||||
fingerprint: string;
|
||||
name: string;
|
||||
severity: 'info' | 'warning' | 'critical';
|
||||
status: string;
|
||||
startedAt: string;
|
||||
summary: string;
|
||||
description?: string;
|
||||
currentValue?: string;
|
||||
threshold?: string;
|
||||
service?: string;
|
||||
instance?: string;
|
||||
};
|
||||
|
||||
export type InfrastructureMonitoringOverview = {
|
||||
available: boolean;
|
||||
range: InfrastructureMonitoringRange;
|
||||
collectedAt: string;
|
||||
lastSampleAt: string | null;
|
||||
error?: string;
|
||||
summary: {
|
||||
overallStatus: 'healthy' | 'warning' | 'critical' | 'unknown';
|
||||
serviceTotal: number;
|
||||
serviceHealthy: number;
|
||||
warningAlerts: number;
|
||||
criticalAlerts: number;
|
||||
activeAlerts: number;
|
||||
};
|
||||
metrics: {
|
||||
cpuUsagePercent: number | null;
|
||||
memoryUsagePercent: number | null;
|
||||
memoryTotalBytes: number | null;
|
||||
memoryAvailableBytes: number | null;
|
||||
diskUsagePercent: number | null;
|
||||
diskTotalBytes: number | null;
|
||||
diskAvailableBytes: number | null;
|
||||
networkReceiveBytesPerSecond: number | null;
|
||||
networkTransmitBytesPerSecond: number | null;
|
||||
load1: number | null;
|
||||
uptimeSeconds: number | null;
|
||||
};
|
||||
trends: {
|
||||
cpuUsagePercent: InfrastructureMetricPoint[];
|
||||
memoryUsagePercent: InfrastructureMetricPoint[];
|
||||
diskUsagePercent: InfrastructureMetricPoint[];
|
||||
networkReceiveBytesPerSecond: InfrastructureMetricPoint[];
|
||||
networkTransmitBytesPerSecond: InfrastructureMetricPoint[];
|
||||
};
|
||||
services: InfrastructureServiceStatus[];
|
||||
alerts: InfrastructureAlert[];
|
||||
};
|
||||
@@ -0,0 +1,309 @@
|
||||
.admin-system-monitoring-page {
|
||||
gap: 18px;
|
||||
}
|
||||
|
||||
.system-monitoring-heading {
|
||||
align-items: flex-end;
|
||||
gap: 24px;
|
||||
}
|
||||
|
||||
.system-monitoring-title-row {
|
||||
align-items: center;
|
||||
display: flex;
|
||||
gap: 12px;
|
||||
margin-top: 10px;
|
||||
}
|
||||
|
||||
.system-monitoring-title-row h1 {
|
||||
color: var(--color-text-strong);
|
||||
font-size: 24px;
|
||||
line-height: 1.25;
|
||||
margin: 0;
|
||||
}
|
||||
|
||||
.system-monitoring-title-row p {
|
||||
color: var(--color-text-muted);
|
||||
font-size: 13px;
|
||||
margin: 4px 0 0;
|
||||
}
|
||||
|
||||
.system-monitoring-controls,
|
||||
.system-monitoring-range {
|
||||
align-items: center;
|
||||
display: flex;
|
||||
}
|
||||
|
||||
.system-monitoring-controls {
|
||||
gap: 10px;
|
||||
}
|
||||
|
||||
.system-monitoring-range {
|
||||
background: var(--color-surface-muted);
|
||||
border: 1px solid var(--color-border);
|
||||
border-radius: var(--radius-md);
|
||||
padding: 3px;
|
||||
}
|
||||
|
||||
.system-monitoring-range button {
|
||||
background: transparent;
|
||||
border: 0;
|
||||
border-radius: 4px;
|
||||
color: var(--color-text-muted);
|
||||
cursor: pointer;
|
||||
font: inherit;
|
||||
min-height: 30px;
|
||||
padding: 0 12px;
|
||||
}
|
||||
|
||||
.system-monitoring-range button:hover {
|
||||
color: var(--color-text-strong);
|
||||
}
|
||||
|
||||
.system-monitoring-range button:focus-visible {
|
||||
box-shadow: var(--focus-ring);
|
||||
outline: none;
|
||||
}
|
||||
|
||||
.system-monitoring-range button.is-active {
|
||||
background: var(--color-surface);
|
||||
box-shadow: var(--shadow-xs);
|
||||
color: var(--color-selected);
|
||||
font-weight: var(--font-weight-semibold);
|
||||
}
|
||||
|
||||
.system-monitoring-unavailable {
|
||||
align-items: flex-start;
|
||||
background: var(--color-danger-soft);
|
||||
border: 1px solid color-mix(in srgb, var(--color-danger) 24%, transparent);
|
||||
border-radius: var(--radius-lg);
|
||||
color: var(--color-danger);
|
||||
display: flex;
|
||||
gap: 12px;
|
||||
padding: 14px 16px;
|
||||
}
|
||||
|
||||
.system-monitoring-unavailable div {
|
||||
display: grid;
|
||||
gap: 2px;
|
||||
}
|
||||
|
||||
.system-monitoring-unavailable span {
|
||||
color: var(--color-text-muted);
|
||||
font-size: 13px;
|
||||
}
|
||||
|
||||
.system-monitoring-health {
|
||||
align-items: center;
|
||||
display: grid;
|
||||
gap: 0;
|
||||
grid-template-columns: minmax(220px, 1.35fr) repeat(4, minmax(120px, 1fr));
|
||||
padding: 18px 20px;
|
||||
}
|
||||
|
||||
.system-monitoring-health__mark {
|
||||
align-items: center;
|
||||
border-radius: 50%;
|
||||
display: flex;
|
||||
height: 48px;
|
||||
justify-content: center;
|
||||
margin-right: 13px;
|
||||
width: 48px;
|
||||
}
|
||||
|
||||
.system-monitoring-health__mark.is-healthy { background: var(--color-success-soft); color: var(--color-success); }
|
||||
.system-monitoring-health__mark.is-warning { background: var(--color-warning-soft); color: var(--color-warning); }
|
||||
.system-monitoring-health__mark.is-critical { background: var(--color-danger-soft); color: var(--color-danger); }
|
||||
.system-monitoring-health__mark.is-unknown { background: var(--color-surface-muted); color: var(--color-text-muted); }
|
||||
|
||||
.system-monitoring-health__copy {
|
||||
align-items: center;
|
||||
display: grid;
|
||||
grid-template-columns: 48px 1fr;
|
||||
grid-template-rows: repeat(3, auto);
|
||||
}
|
||||
|
||||
.system-monitoring-health__copy .system-monitoring-health__mark { grid-row: 1 / 4; }
|
||||
.system-monitoring-health__copy > span,
|
||||
.system-monitoring-health__fact > span {
|
||||
color: var(--color-text-muted);
|
||||
font-size: 12px;
|
||||
}
|
||||
|
||||
.system-monitoring-health__copy > strong { color: var(--color-text-strong); font-size: 17px; }
|
||||
.system-monitoring-health__copy > small,
|
||||
.system-monitoring-health__fact > small { color: var(--color-text-subtle); font-size: 11px; }
|
||||
|
||||
.system-monitoring-health__fact {
|
||||
border-left: 1px solid var(--color-border);
|
||||
display: grid;
|
||||
gap: 3px;
|
||||
padding-left: 20px;
|
||||
}
|
||||
|
||||
.system-monitoring-health__fact strong {
|
||||
color: var(--color-text-strong);
|
||||
font-size: 18px;
|
||||
}
|
||||
|
||||
.system-monitoring-metrics {
|
||||
display: grid;
|
||||
gap: 14px;
|
||||
grid-template-columns: repeat(4, minmax(0, 1fr));
|
||||
}
|
||||
|
||||
.system-monitoring-metric {
|
||||
align-items: center;
|
||||
display: flex;
|
||||
gap: 14px;
|
||||
min-width: 0;
|
||||
padding: 18px;
|
||||
}
|
||||
|
||||
.system-monitoring-metric__icon {
|
||||
align-items: center;
|
||||
border-radius: var(--radius-md);
|
||||
display: flex;
|
||||
flex: 0 0 40px;
|
||||
height: 40px;
|
||||
justify-content: center;
|
||||
}
|
||||
|
||||
.system-monitoring-metric__icon.is-blue { background: var(--color-info-soft); color: var(--color-info); }
|
||||
.system-monitoring-metric__icon.is-violet { background: #f5f3ff; color: #7c3aed; }
|
||||
.system-monitoring-metric__icon.is-amber { background: var(--color-warning-soft); color: var(--color-warning); }
|
||||
.system-monitoring-metric__icon.is-green { background: var(--color-success-soft); color: #0f766e; }
|
||||
|
||||
.system-monitoring-metric > div:last-child {
|
||||
display: grid;
|
||||
gap: 2px;
|
||||
min-width: 0;
|
||||
}
|
||||
|
||||
.system-monitoring-metric span,
|
||||
.system-monitoring-metric small { color: var(--color-text-muted); font-size: 12px; }
|
||||
.system-monitoring-metric strong { color: var(--color-text-strong); font-size: 23px; line-height: 1.25; }
|
||||
.system-monitoring-metric small { overflow: hidden; text-overflow: ellipsis; white-space: nowrap; }
|
||||
|
||||
.system-monitoring-main-grid {
|
||||
align-items: start;
|
||||
display: grid;
|
||||
gap: 16px;
|
||||
grid-template-columns: minmax(0, 1fr) 300px;
|
||||
}
|
||||
|
||||
.system-monitoring-chart-stack {
|
||||
display: grid;
|
||||
gap: 16px;
|
||||
grid-template-columns: repeat(2, minmax(0, 1fr));
|
||||
}
|
||||
|
||||
.system-monitoring-chart-card,
|
||||
.system-monitoring-services,
|
||||
.system-monitoring-alerts { padding: 18px; }
|
||||
|
||||
.system-monitoring-chart-card header,
|
||||
.system-monitoring-services > header,
|
||||
.system-monitoring-alerts > header {
|
||||
align-items: center;
|
||||
display: flex;
|
||||
justify-content: space-between;
|
||||
margin-bottom: 12px;
|
||||
}
|
||||
|
||||
.system-monitoring-chart-card header div,
|
||||
.system-monitoring-services > header div,
|
||||
.system-monitoring-alerts > header div {
|
||||
align-items: center;
|
||||
color: var(--color-text-strong);
|
||||
display: flex;
|
||||
gap: 8px;
|
||||
}
|
||||
|
||||
.system-monitoring-chart-card header > span { color: var(--color-text); font-weight: var(--font-weight-semibold); }
|
||||
|
||||
.system-monitoring-chart-empty {
|
||||
align-items: center;
|
||||
color: var(--color-text-subtle);
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 8px;
|
||||
height: 230px;
|
||||
justify-content: center;
|
||||
}
|
||||
|
||||
.system-monitoring-service-list { display: grid; }
|
||||
|
||||
.system-monitoring-service {
|
||||
align-items: center;
|
||||
border-bottom: 1px solid var(--color-border);
|
||||
display: grid;
|
||||
gap: 9px;
|
||||
grid-template-columns: 9px 1fr auto;
|
||||
padding: 13px 0;
|
||||
}
|
||||
|
||||
.system-monitoring-service:last-child { border-bottom: 0; }
|
||||
.system-monitoring-service__dot { background: var(--color-text-subtle); border-radius: 50%; height: 8px; width: 8px; }
|
||||
.system-monitoring-service__dot.is-healthy { background: var(--color-success); box-shadow: 0 0 0 3px var(--color-success-soft); }
|
||||
.system-monitoring-service__dot.is-unhealthy { background: var(--color-danger); box-shadow: 0 0 0 3px var(--color-danger-soft); }
|
||||
.system-monitoring-service__dot.is-unknown { background: var(--color-text-subtle); box-shadow: 0 0 0 3px var(--color-surface-muted); }
|
||||
.system-monitoring-service div { display: grid; gap: 1px; min-width: 0; }
|
||||
.system-monitoring-service strong { color: var(--color-text); font-size: 13px; }
|
||||
.system-monitoring-service small { color: var(--color-text-subtle); font-size: 11px; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; }
|
||||
.system-monitoring-service > span:last-child { color: var(--color-text-muted); font-size: 12px; }
|
||||
|
||||
.system-monitoring-collector-note {
|
||||
align-items: flex-start;
|
||||
background: var(--color-bg-subtle);
|
||||
border: 1px solid var(--color-border);
|
||||
border-radius: var(--radius-md);
|
||||
color: var(--color-text-muted);
|
||||
display: flex;
|
||||
font-size: 12px;
|
||||
gap: 8px;
|
||||
line-height: 1.55;
|
||||
margin-top: 14px;
|
||||
padding: 11px;
|
||||
}
|
||||
|
||||
.system-monitoring-collector-note svg { flex: 0 0 auto; margin-top: 1px; }
|
||||
|
||||
.system-monitoring-alerts { overflow: hidden; }
|
||||
.system-monitoring-alerts > header > span { align-items: center; color: var(--color-text-muted); display: flex; font-size: 12px; gap: 5px; }
|
||||
.system-monitoring-alert-copy { display: grid; gap: 2px; }
|
||||
.system-monitoring-alert-copy strong { color: var(--color-text-strong); }
|
||||
.system-monitoring-alert-copy span { color: var(--color-text-muted); font-size: 12px; }
|
||||
|
||||
.is-spinning { animation: system-monitoring-spin 0.9s linear infinite; }
|
||||
@keyframes system-monitoring-spin { to { transform: rotate(360deg); } }
|
||||
|
||||
@media (max-width: 1180px) {
|
||||
.system-monitoring-health { grid-template-columns: repeat(3, minmax(0, 1fr)); row-gap: 18px; }
|
||||
.system-monitoring-health__copy { grid-column: span 2; }
|
||||
.system-monitoring-health__fact:nth-last-child(-n + 2) { border-left: 0; padding-left: 0; }
|
||||
.system-monitoring-metrics { grid-template-columns: repeat(2, minmax(0, 1fr)); }
|
||||
.system-monitoring-main-grid { grid-template-columns: minmax(0, 1fr); }
|
||||
.system-monitoring-services { order: -1; }
|
||||
.system-monitoring-service-list { grid-template-columns: repeat(3, minmax(0, 1fr)); }
|
||||
.system-monitoring-service { border-bottom: 0; border-right: 1px solid var(--color-border); padding: 11px 12px; }
|
||||
.system-monitoring-service:nth-child(3n) { border-right: 0; }
|
||||
}
|
||||
|
||||
@media (max-width: 760px) {
|
||||
.system-monitoring-heading { align-items: stretch; flex-direction: column; }
|
||||
.system-monitoring-controls { align-items: stretch; flex-direction: column; }
|
||||
.system-monitoring-range { display: grid; grid-template-columns: repeat(3, 1fr); }
|
||||
.system-monitoring-title-row { align-items: flex-start; justify-content: space-between; }
|
||||
.system-monitoring-health { grid-template-columns: repeat(2, minmax(0, 1fr)); }
|
||||
.system-monitoring-health__copy { grid-column: 1 / -1; }
|
||||
.system-monitoring-health__fact { border-left: 0; border-top: 1px solid var(--color-border); padding: 12px 0 0; }
|
||||
.system-monitoring-metrics,
|
||||
.system-monitoring-chart-stack,
|
||||
.system-monitoring-service-list { grid-template-columns: minmax(0, 1fr); }
|
||||
.system-monitoring-service { border-bottom: 1px solid var(--color-border); border-right: 0; padding: 13px 0; }
|
||||
.system-monitoring-chart-card { padding: 14px 10px; }
|
||||
}
|
||||
|
||||
@media (prefers-reduced-motion: reduce) {
|
||||
.is-spinning { animation: none; }
|
||||
}
|
||||
@@ -0,0 +1,316 @@
|
||||
import { useCallback, useEffect, useMemo, useRef, useState } from 'react';
|
||||
import type { EChartsOption } from 'echarts';
|
||||
import {
|
||||
Activity,
|
||||
AlertTriangle,
|
||||
CheckCircle2,
|
||||
Clock3,
|
||||
Cpu,
|
||||
Database,
|
||||
HardDrive,
|
||||
MemoryStick,
|
||||
Network,
|
||||
RefreshCw,
|
||||
Server,
|
||||
ShieldAlert,
|
||||
} from 'lucide-react';
|
||||
import {
|
||||
adminApi,
|
||||
type InfrastructureAlert,
|
||||
type InfrastructureMetricPoint,
|
||||
type InfrastructureMonitoringOverview,
|
||||
type InfrastructureMonitoringRange,
|
||||
} from '@/api/adminApi';
|
||||
import { Breadcrumb, Button, Chart, Table, Tag, type TableColumn } from '@/components/ui';
|
||||
import './AdminSystemMonitoringPage.css';
|
||||
|
||||
const RANGE_OPTIONS: Array<{ value: InfrastructureMonitoringRange; label: string }> = [
|
||||
{ value: '1h', label: '近1小时' },
|
||||
{ value: '24h', label: '近24小时' },
|
||||
{ value: '7d', label: '近7天' },
|
||||
];
|
||||
|
||||
const STATUS_COPY = {
|
||||
healthy: { label: '运行正常', tone: 'success' as const },
|
||||
warning: { label: '需要关注', tone: 'warning' as const },
|
||||
critical: { label: '严重告警', tone: 'danger' as const },
|
||||
unknown: { label: '状态未知', tone: 'neutral' as const },
|
||||
};
|
||||
|
||||
function formatPercent(value: number | null) {
|
||||
return value === null ? '—' : `${value.toFixed(1)}%`;
|
||||
}
|
||||
|
||||
function formatBytes(value: number | null) {
|
||||
if (value === null) return '—';
|
||||
const units = ['B', 'KB', 'MB', 'GB', 'TB'];
|
||||
let amount = value;
|
||||
let index = 0;
|
||||
while (amount >= 1024 && index < units.length - 1) {
|
||||
amount /= 1024;
|
||||
index += 1;
|
||||
}
|
||||
return `${amount.toFixed(index >= 3 ? 1 : 0)} ${units[index]}`;
|
||||
}
|
||||
|
||||
function formatRate(value: number | null) {
|
||||
return value === null ? '—' : `${formatBytes(value)}/s`;
|
||||
}
|
||||
|
||||
function totalNetworkRate(receive: number | null | undefined, transmit: number | null | undefined) {
|
||||
if (receive === null || receive === undefined || transmit === null || transmit === undefined) return null;
|
||||
return receive + transmit;
|
||||
}
|
||||
|
||||
function formatUptime(value: number | null) {
|
||||
if (value === null) return '—';
|
||||
const days = Math.floor(value / 86400);
|
||||
const hours = Math.floor((value % 86400) / 3600);
|
||||
return days > 0 ? `${days}天 ${hours}小时` : `${hours}小时`;
|
||||
}
|
||||
|
||||
function formatTime(value: string | null) {
|
||||
if (!value) return '暂无采样';
|
||||
return new Intl.DateTimeFormat('zh-CN', {
|
||||
month: '2-digit', day: '2-digit', hour: '2-digit', minute: '2-digit', second: '2-digit', hour12: false,
|
||||
}).format(new Date(value));
|
||||
}
|
||||
|
||||
function formatDuration(startedAt: string) {
|
||||
const milliseconds = Date.now() - Date.parse(startedAt);
|
||||
if (!Number.isFinite(milliseconds) || milliseconds < 0) return '—';
|
||||
const minutes = Math.floor(milliseconds / 60_000);
|
||||
if (minutes < 60) return `${Math.max(minutes, 1)}分钟`;
|
||||
const hours = Math.floor(minutes / 60);
|
||||
return hours < 24 ? `${hours}小时 ${minutes % 60}分钟` : `${Math.floor(hours / 24)}天 ${hours % 24}小时`;
|
||||
}
|
||||
|
||||
function timeLabels(points: InfrastructureMetricPoint[], range: InfrastructureMonitoringRange) {
|
||||
return points.map((point) => new Intl.DateTimeFormat('zh-CN', range === '7d'
|
||||
? { month: '2-digit', day: '2-digit', hour: '2-digit', hour12: false }
|
||||
: { hour: '2-digit', minute: '2-digit', hour12: false })
|
||||
.format(new Date(point.timestamp)));
|
||||
}
|
||||
|
||||
function makeTrendOption(params: {
|
||||
range: InfrastructureMonitoringRange;
|
||||
series: Array<{ name: string; points: InfrastructureMetricPoint[]; color: string }>;
|
||||
suffix: string;
|
||||
maximum?: number;
|
||||
}): EChartsOption {
|
||||
const first = params.series[0]?.points ?? [];
|
||||
return {
|
||||
animationDuration: 280,
|
||||
color: params.series.map((item) => item.color),
|
||||
grid: { left: 8, right: 16, top: 34, bottom: 4, containLabel: true },
|
||||
legend: params.series.length > 1 ? { top: 0, right: 0, textStyle: { color: '#6b7280', fontSize: 12 } } : undefined,
|
||||
tooltip: {
|
||||
trigger: 'axis',
|
||||
valueFormatter: (value) => `${Number(value).toFixed(1)}${params.suffix}`,
|
||||
},
|
||||
xAxis: {
|
||||
type: 'category',
|
||||
boundaryGap: false,
|
||||
data: timeLabels(first, params.range),
|
||||
axisLine: { lineStyle: { color: '#e5e7eb' } },
|
||||
axisTick: { show: false },
|
||||
axisLabel: { color: '#9ca3af', hideOverlap: true, margin: 12 },
|
||||
},
|
||||
yAxis: {
|
||||
type: 'value', min: 0, max: params.maximum,
|
||||
axisLabel: { color: '#9ca3af', formatter: `{value}${params.suffix}` },
|
||||
splitLine: { lineStyle: { color: '#eef0f3' } },
|
||||
},
|
||||
series: params.series.map((item) => ({
|
||||
name: item.name,
|
||||
data: item.points.map((point) => point.value),
|
||||
type: 'line',
|
||||
smooth: true,
|
||||
showSymbol: false,
|
||||
lineStyle: { width: 2.5 },
|
||||
areaStyle: { opacity: 0.07 },
|
||||
})),
|
||||
};
|
||||
}
|
||||
|
||||
function severityTag(severity: InfrastructureAlert['severity']) {
|
||||
if (severity === 'critical') return <Tag tone="danger">严重</Tag>;
|
||||
if (severity === 'warning') return <Tag tone="warning">警告</Tag>;
|
||||
return <Tag tone="info">提示</Tag>;
|
||||
}
|
||||
|
||||
const alertColumns: Array<TableColumn<InfrastructureAlert>> = [
|
||||
{ key: 'severity', title: '级别', width: '82px', render: (record) => severityTag(record.severity) },
|
||||
{
|
||||
key: 'alert', title: '告警', width: '280px', render: (record) => (
|
||||
<div className="system-monitoring-alert-copy"><strong>{record.name}</strong><span>{record.summary}</span></div>
|
||||
),
|
||||
},
|
||||
{ key: 'service', title: '服务 / 实例', width: '190px', render: (record) => record.service || record.instance || '主机资源' },
|
||||
{ key: 'value', title: '当前值 / 阈值', width: '150px', render: (record) => `${record.currentValue || '—'} / ${record.threshold || '—'}` },
|
||||
{ key: 'startedAt', title: '开始时间', width: '150px', render: (record) => formatTime(record.startedAt) },
|
||||
{ key: 'duration', title: '持续时间', width: '120px', render: (record) => formatDuration(record.startedAt) },
|
||||
];
|
||||
|
||||
export function AdminSystemMonitoringPage() {
|
||||
const [range, setRange] = useState<InfrastructureMonitoringRange>('24h');
|
||||
const [overview, setOverview] = useState<InfrastructureMonitoringOverview | null>(null);
|
||||
const [loading, setLoading] = useState(true);
|
||||
const [error, setError] = useState('');
|
||||
const requestSequence = useRef(0);
|
||||
const pendingRequests = useRef(0);
|
||||
|
||||
const loadData = useCallback(async (supersede = false) => {
|
||||
if (!supersede && pendingRequests.current > 0) return;
|
||||
pendingRequests.current += 1;
|
||||
const sequence = ++requestSequence.current;
|
||||
setLoading(true);
|
||||
try {
|
||||
const result = await adminApi.getInfrastructureMonitoringOverview(range);
|
||||
if (sequence !== requestSequence.current) return;
|
||||
setOverview(result);
|
||||
setError(result.available ? '' : result.error || '监控数据当前不可用');
|
||||
} catch (reason) {
|
||||
if (sequence !== requestSequence.current) return;
|
||||
setOverview(null);
|
||||
setError(reason instanceof Error ? reason.message : '监控数据加载失败');
|
||||
} finally {
|
||||
if (sequence === requestSequence.current) setLoading(false);
|
||||
pendingRequests.current -= 1;
|
||||
}
|
||||
}, [range]);
|
||||
|
||||
useEffect(() => {
|
||||
void loadData(true);
|
||||
const intervalId = window.setInterval(() => {
|
||||
if (document.visibilityState === 'visible') void loadData();
|
||||
}, 30_000);
|
||||
const handleVisibility = () => {
|
||||
if (document.visibilityState === 'visible') void loadData();
|
||||
};
|
||||
document.addEventListener('visibilitychange', handleVisibility);
|
||||
return () => {
|
||||
requestSequence.current += 1;
|
||||
window.clearInterval(intervalId);
|
||||
document.removeEventListener('visibilitychange', handleVisibility);
|
||||
};
|
||||
}, [loadData]);
|
||||
|
||||
const status = STATUS_COPY[overview?.summary.overallStatus ?? 'unknown'];
|
||||
const cpuOption = useMemo(() => makeTrendOption({
|
||||
range, maximum: 100, suffix: '%', series: [{ name: 'CPU', points: overview?.trends.cpuUsagePercent ?? [], color: '#2563eb' }],
|
||||
}), [overview?.trends.cpuUsagePercent, range]);
|
||||
const memoryOption = useMemo(() => makeTrendOption({
|
||||
range, maximum: 100, suffix: '%', series: [{ name: '内存', points: overview?.trends.memoryUsagePercent ?? [], color: '#7c3aed' }],
|
||||
}), [overview?.trends.memoryUsagePercent, range]);
|
||||
const diskOption = useMemo(() => makeTrendOption({
|
||||
range, maximum: 100, suffix: '%', series: [{ name: '根磁盘', points: overview?.trends.diskUsagePercent ?? [], color: '#d97706' }],
|
||||
}), [overview?.trends.diskUsagePercent, range]);
|
||||
const networkOption = useMemo(() => makeTrendOption({
|
||||
range, suffix: ' B/s', series: [
|
||||
{ name: '接收', points: overview?.trends.networkReceiveBytesPerSecond ?? [], color: '#0f766e' },
|
||||
{ name: '发送', points: overview?.trends.networkTransmitBytesPerSecond ?? [], color: '#2563eb' },
|
||||
],
|
||||
}), [overview?.trends.networkReceiveBytesPerSecond, overview?.trends.networkTransmitBytesPerSecond, range]);
|
||||
|
||||
const metrics = overview?.metrics;
|
||||
const serviceHealthy = overview?.summary.serviceHealthy ?? 0;
|
||||
const serviceTotal = overview?.summary.serviceTotal ?? 6;
|
||||
|
||||
return (
|
||||
<section className="page-stack admin-system-monitoring-page">
|
||||
<div className="page-heading system-monitoring-heading">
|
||||
<div>
|
||||
<Breadcrumb items={['系统管理', '系统监控']} />
|
||||
<div className="system-monitoring-title-row">
|
||||
<div>
|
||||
<h1>系统监控</h1>
|
||||
<p>服务器资源、核心服务与活动告警</p>
|
||||
</div>
|
||||
<Tag tone={status.tone}>{status.label}</Tag>
|
||||
</div>
|
||||
</div>
|
||||
<div className="system-monitoring-controls">
|
||||
<div className="system-monitoring-range" aria-label="监控时间范围" role="group">
|
||||
{RANGE_OPTIONS.map((option) => (
|
||||
<button
|
||||
aria-pressed={range === option.value}
|
||||
className={range === option.value ? 'is-active' : ''}
|
||||
key={option.value}
|
||||
onClick={() => setRange(option.value)}
|
||||
type="button"
|
||||
>{option.label}</button>
|
||||
))}
|
||||
</div>
|
||||
<Button disabled={loading} icon={<RefreshCw className={loading ? 'is-spinning' : ''} size={16} />} onClick={() => void loadData()} variant="ghost">
|
||||
{loading ? '刷新中' : '刷新'}
|
||||
</Button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{error ? (
|
||||
<div className="system-monitoring-unavailable" role="alert">
|
||||
<ShieldAlert size={20} />
|
||||
<div><strong>监控数据不可用</strong><span>{error}。页面不会展示历史缓存值。</span></div>
|
||||
</div>
|
||||
) : null}
|
||||
|
||||
<div className="system-monitoring-health surface">
|
||||
<div className="system-monitoring-health__copy">
|
||||
<div className={`system-monitoring-health__mark is-${overview?.summary.overallStatus ?? 'unknown'}`}>
|
||||
{overview?.summary.overallStatus === 'healthy' ? <CheckCircle2 size={24} /> : <AlertTriangle size={24} />}
|
||||
</div>
|
||||
<span>平台基础设施</span>
|
||||
<strong>{status.label}</strong>
|
||||
<small>最新采样 {formatTime(overview?.lastSampleAt ?? null)}</small>
|
||||
</div>
|
||||
<div className="system-monitoring-health__fact"><span>核心服务</span><strong>{serviceHealthy}/{serviceTotal}</strong><small>正常运行</small></div>
|
||||
<div className="system-monitoring-health__fact"><span>活动告警</span><strong>{overview?.summary.activeAlerts ?? 0}</strong><small>{overview?.summary.criticalAlerts ?? 0} 严重 · {overview?.summary.warningAlerts ?? 0} 警告</small></div>
|
||||
<div className="system-monitoring-health__fact"><span>系统负载</span><strong>{metrics?.load1 === null || metrics?.load1 === undefined ? '—' : metrics.load1.toFixed(2)}</strong><small>最近1分钟</small></div>
|
||||
<div className="system-monitoring-health__fact"><span>持续运行</span><strong>{formatUptime(metrics?.uptimeSeconds ?? null)}</strong><small>主机启动后</small></div>
|
||||
</div>
|
||||
|
||||
<div className="system-monitoring-metrics">
|
||||
<article className="surface system-monitoring-metric"><div className="system-monitoring-metric__icon is-blue"><Cpu size={19} /></div><div><span>CPU 使用率</span><strong>{formatPercent(metrics?.cpuUsagePercent ?? null)}</strong><small>5分钟平均</small></div></article>
|
||||
<article className="surface system-monitoring-metric"><div className="system-monitoring-metric__icon is-violet"><MemoryStick size={19} /></div><div><span>内存使用率</span><strong>{formatPercent(metrics?.memoryUsagePercent ?? null)}</strong><small>{formatBytes(metrics?.memoryAvailableBytes ?? null)} 可用 / {formatBytes(metrics?.memoryTotalBytes ?? null)}</small></div></article>
|
||||
<article className="surface system-monitoring-metric"><div className="system-monitoring-metric__icon is-amber"><HardDrive size={19} /></div><div><span>根磁盘使用率</span><strong>{formatPercent(metrics?.diskUsagePercent ?? null)}</strong><small>{formatBytes(metrics?.diskAvailableBytes ?? null)} 可用 / {formatBytes(metrics?.diskTotalBytes ?? null)}</small></div></article>
|
||||
<article className="surface system-monitoring-metric"><div className="system-monitoring-metric__icon is-green"><Network size={19} /></div><div><span>网络吞吐</span><strong>{formatRate(totalNetworkRate(metrics?.networkReceiveBytesPerSecond, metrics?.networkTransmitBytesPerSecond))}</strong><small>接收 {formatRate(metrics?.networkReceiveBytesPerSecond ?? null)} · 发送 {formatRate(metrics?.networkTransmitBytesPerSecond ?? null)}</small></div></article>
|
||||
</div>
|
||||
|
||||
<div className="system-monitoring-main-grid">
|
||||
<div className="system-monitoring-chart-stack">
|
||||
<article className="surface system-monitoring-chart-card"><header><div><Cpu size={17} /><strong>CPU 趋势</strong></div><span>{formatPercent(metrics?.cpuUsagePercent ?? null)}</span></header>{overview?.trends.cpuUsagePercent.length ? <Chart height={230} option={cpuOption} /> : <EmptyChart />}</article>
|
||||
<article className="surface system-monitoring-chart-card"><header><div><MemoryStick size={17} /><strong>内存趋势</strong></div><span>{formatPercent(metrics?.memoryUsagePercent ?? null)}</span></header>{overview?.trends.memoryUsagePercent.length ? <Chart height={230} option={memoryOption} /> : <EmptyChart />}</article>
|
||||
<article className="surface system-monitoring-chart-card"><header><div><HardDrive size={17} /><strong>磁盘趋势</strong></div><span>{formatPercent(metrics?.diskUsagePercent ?? null)}</span></header>{overview?.trends.diskUsagePercent.length ? <Chart height={230} option={diskOption} /> : <EmptyChart />}</article>
|
||||
<article className="surface system-monitoring-chart-card"><header><div><Activity size={17} /><strong>网络趋势</strong></div><span>{formatRate(totalNetworkRate(metrics?.networkReceiveBytesPerSecond, metrics?.networkTransmitBytesPerSecond))}</span></header>{overview?.trends.networkReceiveBytesPerSecond.length ? <Chart height={230} option={networkOption} /> : <EmptyChart />}</article>
|
||||
</div>
|
||||
|
||||
<aside className="surface system-monitoring-services">
|
||||
<header><div><Server size={18} /><strong>核心服务</strong></div><Tag tone={serviceHealthy === serviceTotal && overview?.available ? 'success' : 'neutral'}>{serviceHealthy}/{serviceTotal} 正常</Tag></header>
|
||||
<div className="system-monitoring-service-list">
|
||||
{(overview?.services ?? []).map((service) => (
|
||||
<div className="system-monitoring-service" key={service.key}>
|
||||
<span className={`system-monitoring-service__dot is-${service.status}`} />
|
||||
<div><strong>{service.name}</strong><small>{service.unit}</small></div>
|
||||
<span>{service.status === 'healthy' ? '正常' : service.status === 'unhealthy' ? '异常' : '未知'}</span>
|
||||
</div>
|
||||
))}
|
||||
{!overview?.services.length ? [
|
||||
['api', 'API服务'], ['gateway', 'Gateway服务'], ['postgresql', 'PostgreSQL'], ['redis', 'Redis'], ['minio', 'MinIO'], ['nginx', 'Nginx'],
|
||||
].map(([key, name]) => <div className="system-monitoring-service" key={key}><span className="system-monitoring-service__dot is-unknown" /><div><strong>{name}</strong><small>等待真实采集</small></div><span>未知</span></div>) : null}
|
||||
</div>
|
||||
<div className="system-monitoring-collector-note"><Database size={16} /><span>指标由 Prometheus 采集，业务数据库不写入高频时序数据。</span></div>
|
||||
</aside>
|
||||
</div>
|
||||
|
||||
<section className="surface system-monitoring-alerts">
|
||||
<header><div><AlertTriangle size={18} /><strong>活动告警</strong><Tag tone={overview?.summary.activeAlerts ? 'warning' : 'success'}>{overview?.summary.activeAlerts ?? 0}</Tag></div><span><Clock3 size={14} /> 刷新于 {formatTime(overview?.collectedAt ?? null)}</span></header>
|
||||
<Table columns={alertColumns} data={overview?.alerts ?? []} emptyText={overview?.available ? '当前没有活动告警' : '监控不可用，无法读取活动告警'} pagination={false} rowKey="fingerprint" />
|
||||
</section>
|
||||
</section>
|
||||
);
|
||||
}
|
||||
|
||||
function EmptyChart() {
|
||||
return <div className="system-monitoring-chart-empty"><Activity size={22} /><span>暂无真实趋势指标</span></div>;
|
||||
}
|
||||
@@ -25,6 +25,7 @@ import {
|
||||
ReceiptText,
|
||||
ClipboardList,
|
||||
Send,
|
||||
ServerCog,
|
||||
ScanSearch,
|
||||
Settings,
|
||||
Shield,
|
||||
@@ -202,6 +203,7 @@ function AdminAuthenticatedLayout({ session }: { session: LoginSession }) {
|
||||
{ label: '报备字段库', to: '/admin/drainage-fields', icon: Hash },
|
||||
{ label: '引流识别规则', to: '/admin/drainage-detection-rules', icon: ScanSearch },
|
||||
{ label: '系统日志', to: '/admin/system-logs', icon: FileText },
|
||||
{ label: '系统监控', to: '/admin/system-monitoring', icon: ServerCog },
|
||||
],
|
||||
},
|
||||
]}
|
||||
|
||||
@@ -38,6 +38,7 @@ import { AdminSignatureAuditPage } from '@/apps/admin/AdminSignatureAuditPage';
|
||||
import { AdminSignatureRetirementPage } from '@/apps/admin/AdminSignatureRetirementPage';
|
||||
import { AdminDrainageAuditPage } from '@/apps/admin/AdminDrainageAuditPage';
|
||||
import { AdminSystemLogsPage } from '@/apps/admin/AdminSystemLogsPage';
|
||||
import { AdminSystemMonitoringPage } from '@/apps/admin/system-monitoring/AdminSystemMonitoringPage';
|
||||
import { AdminTemplateAuditPage } from '@/apps/admin/AdminTemplateAuditPage';
|
||||
import { AdminUsersPage } from '@/apps/admin/AdminUsersPage';
|
||||
import { AdminEnterpriseAuditPage } from '@/apps/admin/AdminEnterpriseAuditPage';
|
||||
@@ -142,6 +143,7 @@ export function AppRoutes() {
|
||||
<Route path="drainage-fields" element={<AdminDrainageFieldsPage />} />
|
||||
<Route path="drainage-detection-rules" element={<AdminDrainageDetectionRulesPage />} />
|
||||
<Route path="system-logs" element={<AdminSystemLogsPage />} />
|
||||
<Route path="system-monitoring" element={<AdminSystemMonitoringPage />} />
|
||||
<Route path="*" element={<PagePlaceholder />} />
|
||||
</Route>
|
||||
</Routes>
|
||||
|
||||
@@ -31,6 +31,8 @@ OPERATION_LOG_ARCHIVE_INTERVAL_MS="${OPERATION_LOG_ARCHIVE_INTERVAL_MS:-86400000
|
||||
SMS_RECEIPT_TIMEOUT_SCAN_ENABLED="${SMS_RECEIPT_TIMEOUT_SCAN_ENABLED:-true}"
|
||||
SMS_RECEIPT_TIMEOUT_HOURS="${SMS_RECEIPT_TIMEOUT_HOURS:-72}"
|
||||
SMS_RECEIPT_TIMEOUT_SCAN_INTERVAL_MS="${SMS_RECEIPT_TIMEOUT_SCAN_INTERVAL_MS:-300000}"
|
||||
PROMETHEUS_URL="${PROMETHEUS_URL:-http://127.0.0.1:9090}"
|
||||
PROMETHEUS_QUERY_TIMEOUT_MS="${PROMETHEUS_QUERY_TIMEOUT_MS:-5000}"
|
||||
|
||||
if [[ "$(id -u)" -ne 0 ]]; then
|
||||
echo "Run as root." >&2
|
||||
@@ -192,6 +194,8 @@ OPERATION_LOG_ARCHIVE_INTERVAL_MS=${OPERATION_LOG_ARCHIVE_INTERVAL_MS}
|
||||
SMS_RECEIPT_TIMEOUT_SCAN_ENABLED=${SMS_RECEIPT_TIMEOUT_SCAN_ENABLED}
|
||||
SMS_RECEIPT_TIMEOUT_HOURS=${SMS_RECEIPT_TIMEOUT_HOURS}
|
||||
SMS_RECEIPT_TIMEOUT_SCAN_INTERVAL_MS=${SMS_RECEIPT_TIMEOUT_SCAN_INTERVAL_MS}
|
||||
PROMETHEUS_URL=${PROMETHEUS_URL}
|
||||
PROMETHEUS_QUERY_TIMEOUT_MS=${PROMETHEUS_QUERY_TIMEOUT_MS}
|
||||
MINIO_ENDPOINT=127.0.0.1:9000
|
||||
MINIO_ACCESS_KEY=${MINIO_ROOT_USER}
|
||||
MINIO_SECRET_KEY=${MINIO_ROOT_PASSWORD}
|
||||
|
||||
@@ -0,0 +1,31 @@
|
||||
# CMPP Prometheus 监控配置
|
||||
|
||||
本目录提供系统监控第一版所需的真实采集配置。运营端不嵌入Prometheus或Grafana页面；NestJS仅从本机Prometheus读取固定指标，再由平台原生UI展示。
|
||||
|
||||
在目标Debian/Ubuntu测试服务器上，以root执行：
|
||||
|
||||
```bash
|
||||
cd /opt/cmpp-platform
|
||||
bash tools/monitoring/install-prometheus-monitoring.sh
|
||||
```
|
||||
|
||||
脚本会安装Prometheus与Node Exporter、备份已有Prometheus配置及本脚本曾写入的systemd override、校验规则、写入新override，并仅重启这两个监控服务。备份目录会在脚本结束时打印。它不会重启API、Gateway、数据库、Redis、MinIO或Nginx。9090和9100固定监听`127.0.0.1`。
|
||||
|
||||
安装后将下列配置写入`/etc/cmpp-platform/cmpp-platform.env`，再按正常发布窗口重启API：
|
||||
|
||||
```dotenv
|
||||
PROMETHEUS_URL=http://127.0.0.1:9090
|
||||
PROMETHEUS_QUERY_TIMEOUT_MS=5000
|
||||
```
|
||||
|
||||
校验命令：
|
||||
|
||||
```bash
|
||||
promtool check config /etc/prometheus/prometheus.yml
|
||||
promtool check rules /etc/prometheus/cmpp-alerts.yml
|
||||
curl -fsS http://127.0.0.1:9090/-/ready
|
||||
curl -fsS 'http://127.0.0.1:9090/api/v1/query?query=up'
|
||||
ss -lnt | grep -E ':(9090|9100)'
|
||||
```
|
||||
|
||||
完整架构、PromQL口径、故障语义和验收标准见`docs/prometheus-system-monitoring-design-20260814.md`。
|
||||
@@ -0,0 +1,148 @@
|
||||
groups:
|
||||
- name: cmpp-host-resources
|
||||
rules:
|
||||
- alert: NodeExporterDown
|
||||
expr: up{job="node"} == 0
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
service: node-exporter
|
||||
annotations:
|
||||
summary: 主机指标采集不可用
|
||||
description: Prometheus连续2分钟无法采集Node Exporter。
|
||||
currentValue: "{{ $value }}"
|
||||
threshold: "up = 1"
|
||||
|
||||
- alert: HostCpuUsageWarning
|
||||
expr: (100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 85) and (100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) <= 95)
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
service: host
|
||||
annotations:
|
||||
summary: CPU使用率持续偏高
|
||||
description: 主机CPU使用率连续10分钟高于85%。
|
||||
currentValue: "{{ printf \"%.1f\" $value }}%"
|
||||
threshold: "85%"
|
||||
|
||||
- alert: HostCpuUsageCritical
|
||||
expr: 100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 95
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
service: host
|
||||
annotations:
|
||||
summary: CPU使用率严重超限
|
||||
description: 主机CPU使用率连续5分钟高于95%。
|
||||
currentValue: "{{ printf \"%.1f\" $value }}%"
|
||||
threshold: "95%"
|
||||
|
||||
- alert: HostMemoryUsageWarning
|
||||
expr: ((1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100 > 85) and ((1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100 <= 95)
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
service: host
|
||||
annotations:
|
||||
summary: 内存使用率持续偏高
|
||||
description: 主机可用内存连续10分钟低于15%。
|
||||
currentValue: "{{ printf \"%.1f\" $value }}%"
|
||||
threshold: "85%"
|
||||
|
||||
- alert: HostMemoryUsageCritical
|
||||
expr: (1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100 > 95
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
service: host
|
||||
annotations:
|
||||
summary: 内存使用率严重超限
|
||||
description: 主机可用内存连续5分钟低于5%。
|
||||
currentValue: "{{ printf \"%.1f\" $value }}%"
|
||||
threshold: "95%"
|
||||
|
||||
- alert: HostRootDiskUsageWarning
|
||||
expr: ((1 - node_filesystem_avail_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"} / node_filesystem_size_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"}) * 100 > 80) and ((1 - node_filesystem_avail_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"} / node_filesystem_size_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"}) * 100 <= 90)
|
||||
for: 15m
|
||||
labels:
|
||||
severity: warning
|
||||
service: host
|
||||
annotations:
|
||||
summary: 根文件系统空间不足
|
||||
description: 根文件系统使用率连续15分钟高于80%。
|
||||
currentValue: "{{ printf \"%.1f\" $value }}%"
|
||||
threshold: "80%"
|
||||
|
||||
- alert: HostRootDiskUsageCritical
|
||||
expr: (1 - node_filesystem_avail_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"} / node_filesystem_size_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"}) * 100 > 90
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
service: host
|
||||
annotations:
|
||||
summary: 根文件系统空间严重不足
|
||||
description: 根文件系统使用率连续5分钟高于90%。
|
||||
currentValue: "{{ printf \"%.1f\" $value }}%"
|
||||
threshold: "90%"
|
||||
|
||||
- alert: HostRootInodeUsageWarning
|
||||
expr: ((1 - node_filesystem_files_free{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"} / node_filesystem_files{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"}) * 100 > 80) and ((1 - node_filesystem_files_free{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"} / node_filesystem_files{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"}) * 100 <= 90)
|
||||
for: 15m
|
||||
labels:
|
||||
severity: warning
|
||||
service: host
|
||||
annotations:
|
||||
summary: 根文件系统inode余量偏低
|
||||
description: 根文件系统inode使用率连续15分钟高于80%。
|
||||
currentValue: "{{ printf \"%.1f\" $value }}%"
|
||||
threshold: "80%"
|
||||
|
||||
- alert: HostRootInodeUsageCritical
|
||||
expr: (1 - node_filesystem_files_free{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"} / node_filesystem_files{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"}) * 100 > 90
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
service: host
|
||||
annotations:
|
||||
summary: 根文件系统inode严重不足
|
||||
description: 根文件系统inode使用率连续5分钟高于90%。
|
||||
currentValue: "{{ printf \"%.1f\" $value }}%"
|
||||
threshold: "90%"
|
||||
|
||||
- alert: HostCpuIowaitWarning
|
||||
expr: (avg by (instance) (rate(node_cpu_seconds_total{mode="iowait"}[5m])) * 100 > 20) and (avg by (instance) (rate(node_cpu_seconds_total{mode="iowait"}[5m])) * 100 <= 35)
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
service: host
|
||||
annotations:
|
||||
summary: CPU iowait持续偏高
|
||||
description: 主机CPU iowait连续10分钟高于20%，请检查磁盘I/O。
|
||||
currentValue: "{{ printf \"%.1f\" $value }}%"
|
||||
threshold: "20%"
|
||||
|
||||
- alert: HostCpuIowaitCritical
|
||||
expr: avg by (instance) (rate(node_cpu_seconds_total{mode="iowait"}[5m])) * 100 > 35
|
||||
for: 10m
|
||||
labels:
|
||||
severity: critical
|
||||
service: host
|
||||
annotations:
|
||||
summary: CPU iowait严重超限
|
||||
description: 主机CPU iowait连续10分钟高于35%，磁盘I/O可能已成为瓶颈。
|
||||
currentValue: "{{ printf \"%.1f\" $value }}%"
|
||||
threshold: "35%"
|
||||
|
||||
- name: cmpp-core-services
|
||||
rules:
|
||||
- alert: CmppCoreServiceInactive
|
||||
expr: node_systemd_unit_state{name=~"cmpp-api\\.service|cmpp-gateway\\.service|postgresql\\.service|redis(-server)?\\.service|cmpp-minio\\.service|nginx\\.service",state="active"} == 0
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
service: "{{ $labels.name }}"
|
||||
annotations:
|
||||
summary: CMPP核心服务未处于active状态
|
||||
description: "systemd服务 {{ $labels.name }} 连续2分钟未处于active状态。"
|
||||
currentValue: "{{ $value }}"
|
||||
threshold: "active = 1"
|
||||
@@ -0,0 +1,82 @@
|
||||
#!/usr/bin/env bash
|
||||
set -Eeuo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
PROMETHEUS_RETENTION_TIME="${PROMETHEUS_RETENTION_TIME:-30d}"
|
||||
PROMETHEUS_RETENTION_SIZE="${PROMETHEUS_RETENTION_SIZE:-8GB}"
|
||||
|
||||
if [[ "$(id -u)" -ne 0 ]]; then
|
||||
echo "Run as root." >&2
|
||||
exit 1
|
||||
fi
|
||||
if ! command -v apt-get >/dev/null 2>&1; then
|
||||
echo "This installer currently supports Debian/Ubuntu apt packages only." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
log() { printf '\n[%s] %s\n' "$(date '+%F %T')" "$*"; }
|
||||
|
||||
log "Installing Prometheus and Node Exporter packages"
|
||||
apt-get update
|
||||
DEBIAN_FRONTEND=noninteractive apt-get install -y prometheus prometheus-node-exporter curl iproute2
|
||||
|
||||
prometheus_bin="$(command -v prometheus)"
|
||||
node_exporter_bin="$(command -v prometheus-node-exporter)"
|
||||
promtool_bin="$(command -v promtool)"
|
||||
backup_dir="/etc/prometheus/cmpp-backups/$(date '+%Y%m%d-%H%M%S')"
|
||||
mkdir -p "$backup_dir" /etc/systemd/system/prometheus.service.d /etc/systemd/system/prometheus-node-exporter.service.d
|
||||
|
||||
for config_file in /etc/prometheus/prometheus.yml /etc/prometheus/cmpp-alerts.yml; do
|
||||
if [[ -f "$config_file" ]]; then
|
||||
cp --preserve=mode,timestamps "$config_file" "$backup_dir/$(basename "$config_file")"
|
||||
fi
|
||||
done
|
||||
if [[ -f /etc/systemd/system/prometheus.service.d/cmpp-monitoring.conf ]]; then
|
||||
cp --preserve=mode,timestamps /etc/systemd/system/prometheus.service.d/cmpp-monitoring.conf "$backup_dir/prometheus-service-override.conf"
|
||||
fi
|
||||
if [[ -f /etc/systemd/system/prometheus-node-exporter.service.d/cmpp-monitoring.conf ]]; then
|
||||
cp --preserve=mode,timestamps /etc/systemd/system/prometheus-node-exporter.service.d/cmpp-monitoring.conf "$backup_dir/node-exporter-service-override.conf"
|
||||
fi
|
||||
|
||||
log "Installing platform-owned scrape and alert configuration"
|
||||
install -o root -g root -m 0644 "$SCRIPT_DIR/prometheus.yml" /etc/prometheus/prometheus.yml
|
||||
install -o root -g root -m 0644 "$SCRIPT_DIR/cmpp-alerts.yml" /etc/prometheus/cmpp-alerts.yml
|
||||
|
||||
cat >/etc/systemd/system/prometheus.service.d/cmpp-monitoring.conf <<EOF
|
||||
[Service]
|
||||
ExecStart=
|
||||
ExecStart=${prometheus_bin} --config.file=/etc/prometheus/prometheus.yml --storage.tsdb.path=/var/lib/prometheus/metrics2 --storage.tsdb.retention.time=${PROMETHEUS_RETENTION_TIME} --storage.tsdb.retention.size=${PROMETHEUS_RETENTION_SIZE} --web.listen-address=127.0.0.1:9090
|
||||
EOF
|
||||
|
||||
cat >/etc/systemd/system/prometheus-node-exporter.service.d/cmpp-monitoring.conf <<EOF
|
||||
[Service]
|
||||
ExecStart=
|
||||
ExecStart=${node_exporter_bin} --web.listen-address=127.0.0.1:9100 --collector.systemd --collector.systemd.unit-include='cmpp-api\\.service|cmpp-gateway\\.service|postgresql\\.service|redis(-server)?\\.service|cmpp-minio\\.service|nginx\\.service' --collector.filesystem.mount-points-exclude='^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/)'
|
||||
EOF
|
||||
|
||||
log "Validating Prometheus configuration before restart"
|
||||
"$promtool_bin" check rules /etc/prometheus/cmpp-alerts.yml
|
||||
"$promtool_bin" check config /etc/prometheus/prometheus.yml
|
||||
|
||||
systemctl daemon-reload
|
||||
systemctl enable prometheus prometheus-node-exporter
|
||||
systemctl restart prometheus-node-exporter
|
||||
systemctl restart prometheus
|
||||
|
||||
for _ in $(seq 1 30); do
|
||||
if curl -fsS http://127.0.0.1:9090/-/ready >/dev/null; then
|
||||
break
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
curl -fsS http://127.0.0.1:9090/-/ready >/dev/null
|
||||
curl -fsS http://127.0.0.1:9100/metrics >/dev/null
|
||||
|
||||
if ss -lnt | grep -Eq '(^|[[:space:]])(0\.0\.0\.0|\[::\]):(9090|9100)([[:space:]]|$)'; then
|
||||
echo "Prometheus monitoring ports unexpectedly listen on a wildcard address." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
log "Prometheus monitoring is ready on loopback only"
|
||||
echo "Configuration backup: $backup_dir"
|
||||
echo "The CMPP API and Gateway were not restarted."
|
||||
@@ -0,0 +1,20 @@
|
||||
global:
|
||||
scrape_interval: 15s
|
||||
evaluation_interval: 15s
|
||||
external_labels:
|
||||
platform: cmpp
|
||||
environment: preproduction
|
||||
|
||||
rule_files:
|
||||
- /etc/prometheus/cmpp-alerts.yml
|
||||
|
||||
scrape_configs:
|
||||
- job_name: prometheus
|
||||
static_configs:
|
||||
- targets: [127.0.0.1:9090]
|
||||
|
||||
- job_name: node
|
||||
static_configs:
|
||||
- targets: [127.0.0.1:9100]
|
||||
labels:
|
||||
host: cmpp-primary
|
||||
Reference in New Issue
Block a user