From b78faa1aa24a89830892b511a6cb17cc5a5fbe68 Mon Sep 17 00:00:00 2001 From: hectorzhao Date: Fri, 14 Aug 2026 10:10:28 +0800 Subject: [PATCH] feat: add Prometheus system monitoring --- .env.example | 3 + api/src/app.module.ts | 2 + .../infrastructure-monitoring.contracts.ts | 65 ++++ .../infrastructure-monitoring.controller.ts | 14 + .../infrastructure-monitoring.module.ts | 9 + .../infrastructure-monitoring.service.spec.ts | 89 +++++ .../infrastructure-monitoring.service.ts | 264 +++++++++++++++ .../first-version-development-requirements.md | 11 + docs/production-deployment.md | 4 + ...theus-system-monitoring-design-20260814.md | 194 +++++++++++ docs/system-functional-test-cases.md | 22 ++ docs/testing-progress.md | 9 + .../admin/infrastructure-monitoring.api.ts | 7 + src/api/adminApi.ts | 2 + src/api/types/index.ts | 1 + src/api/types/infrastructure-monitoring.ts | 65 ++++ .../AdminSystemMonitoringPage.css | 309 +++++++++++++++++ .../AdminSystemMonitoringPage.tsx | 316 ++++++++++++++++++ src/layouts/AdminLayout.tsx | 2 + src/routes/AppRoutes.tsx | 2 + tools/deploy/production-bootstrap.sh | 4 + tools/monitoring/README.md | 31 ++ tools/monitoring/cmpp-alerts.yml | 148 ++++++++ .../install-prometheus-monitoring.sh | 82 +++++ tools/monitoring/prometheus.yml | 20 ++ 25 files changed, 1675 insertions(+) create mode 100644 api/src/infrastructure-monitoring/infrastructure-monitoring.contracts.ts create mode 100644 api/src/infrastructure-monitoring/infrastructure-monitoring.controller.ts create mode 100644 api/src/infrastructure-monitoring/infrastructure-monitoring.module.ts create mode 100644 api/src/infrastructure-monitoring/infrastructure-monitoring.service.spec.ts create mode 100644 api/src/infrastructure-monitoring/infrastructure-monitoring.service.ts create mode 100644 docs/prometheus-system-monitoring-design-20260814.md create mode 100644 src/api/admin/infrastructure-monitoring.api.ts create mode 100644 src/api/types/infrastructure-monitoring.ts create mode 100644 src/apps/admin/system-monitoring/AdminSystemMonitoringPage.css create mode 100644 src/apps/admin/system-monitoring/AdminSystemMonitoringPage.tsx create mode 100644 tools/monitoring/README.md create mode 100644 tools/monitoring/cmpp-alerts.yml create mode 100644 tools/monitoring/install-prometheus-monitoring.sh create mode 100644 tools/monitoring/prometheus.yml diff --git a/.env.example b/.env.example index 9085d5f..31e7610 100644 --- a/.env.example +++ b/.env.example @@ -25,6 +25,9 @@ OPERATION_LOG_ARCHIVE_INTERVAL_MS=86400000 SMS_RECEIPT_TIMEOUT_SCAN_ENABLED=true SMS_RECEIPT_TIMEOUT_HOURS=72 SMS_RECEIPT_TIMEOUT_SCAN_INTERVAL_MS=300000 +# System monitoring reads only fixed queries from a loopback Prometheus instance. +PROMETHEUS_URL=http://127.0.0.1:9090 +PROMETHEUS_QUERY_TIMEOUT_MS=5000 # Local HTTP development only. Production must use HTTPS and true. SESSION_COOKIE_SECURE=false MINIO_ENDPOINT=localhost:9000 diff --git a/api/src/app.module.ts b/api/src/app.module.ts index e63bbcf..dddb960 100644 --- a/api/src/app.module.ts +++ b/api/src/app.module.ts @@ -12,6 +12,7 @@ import { DictionariesModule } from './dictionaries/dictionaries.module'; import { DeletionGovernanceModule } from './deletion-governance/deletion-governance.module'; import { FilesModule } from './files/files.module'; import { HealthController } from './health.controller'; +import { InfrastructureMonitoringModule } from './infrastructure-monitoring/infrastructure-monitoring.module'; import { OperationsModule } from './operations/operations.module'; import { OpenApiModule } from './open-api/open-api.module'; import { PrismaModule } from './prisma/prisma.module'; @@ -49,6 +50,7 @@ import { SignatureRetirementModule } from './signature-retirement/signature-reti ReportMaterialsModule, SendChainModule, OperationsModule, + InfrastructureMonitoringModule, OpenApiModule, SignatureRetirementModule, ], diff --git a/api/src/infrastructure-monitoring/infrastructure-monitoring.contracts.ts b/api/src/infrastructure-monitoring/infrastructure-monitoring.contracts.ts new file mode 100644 index 0000000..22013a0 --- /dev/null +++ b/api/src/infrastructure-monitoring/infrastructure-monitoring.contracts.ts @@ -0,0 +1,65 @@ +export type InfrastructureMonitoringRange = '1h' | '24h' | '7d'; + +export type InfrastructureMetricPoint = { + timestamp: string; + value: number; +}; + +export type InfrastructureServiceStatus = { + key: string; + name: string; + unit: string; + status: 'healthy' | 'unhealthy' | 'unknown'; +}; + +export type InfrastructureAlert = { + fingerprint: string; + name: string; + severity: 'info' | 'warning' | 'critical'; + status: string; + startedAt: string; + summary: string; + description?: string; + currentValue?: string; + threshold?: string; + service?: string; + instance?: string; +}; + +export type InfrastructureMonitoringOverview = { + available: boolean; + range: InfrastructureMonitoringRange; + collectedAt: string; + lastSampleAt: string | null; + error?: string; + summary: { + overallStatus: 'healthy' | 'warning' | 'critical' | 'unknown'; + serviceTotal: number; + serviceHealthy: number; + warningAlerts: number; + criticalAlerts: number; + activeAlerts: number; + }; + metrics: { + cpuUsagePercent: number | null; + memoryUsagePercent: number | null; + memoryTotalBytes: number | null; + memoryAvailableBytes: number | null; + diskUsagePercent: number | null; + diskTotalBytes: number | null; + diskAvailableBytes: number | null; + networkReceiveBytesPerSecond: number | null; + networkTransmitBytesPerSecond: number | null; + load1: number | null; + uptimeSeconds: number | null; + }; + trends: { + cpuUsagePercent: InfrastructureMetricPoint[]; + memoryUsagePercent: InfrastructureMetricPoint[]; + diskUsagePercent: InfrastructureMetricPoint[]; + networkReceiveBytesPerSecond: InfrastructureMetricPoint[]; + networkTransmitBytesPerSecond: InfrastructureMetricPoint[]; + }; + services: InfrastructureServiceStatus[]; + alerts: InfrastructureAlert[]; +}; diff --git a/api/src/infrastructure-monitoring/infrastructure-monitoring.controller.ts b/api/src/infrastructure-monitoring/infrastructure-monitoring.controller.ts new file mode 100644 index 0000000..c4e588a --- /dev/null +++ b/api/src/infrastructure-monitoring/infrastructure-monitoring.controller.ts @@ -0,0 +1,14 @@ +import { Controller, Get, Query } from '@nestjs/common'; +import { ApiTags } from '@nestjs/swagger'; +import { InfrastructureMonitoringService } from './infrastructure-monitoring.service'; + +@ApiTags('infrastructure-monitoring') +@Controller('admin/infrastructure-monitoring') +export class InfrastructureMonitoringController { + constructor(private readonly monitoring: InfrastructureMonitoringService) {} + + @Get('overview') + overview(@Query('range') range?: string) { + return this.monitoring.overview(range); + } +} diff --git a/api/src/infrastructure-monitoring/infrastructure-monitoring.module.ts b/api/src/infrastructure-monitoring/infrastructure-monitoring.module.ts new file mode 100644 index 0000000..243ec14 --- /dev/null +++ b/api/src/infrastructure-monitoring/infrastructure-monitoring.module.ts @@ -0,0 +1,9 @@ +import { Module } from '@nestjs/common'; +import { InfrastructureMonitoringController } from './infrastructure-monitoring.controller'; +import { InfrastructureMonitoringService } from './infrastructure-monitoring.service'; + +@Module({ + controllers: [InfrastructureMonitoringController], + providers: [InfrastructureMonitoringService], +}) +export class InfrastructureMonitoringModule {} diff --git a/api/src/infrastructure-monitoring/infrastructure-monitoring.service.spec.ts b/api/src/infrastructure-monitoring/infrastructure-monitoring.service.spec.ts new file mode 100644 index 0000000..0bfbfff --- /dev/null +++ b/api/src/infrastructure-monitoring/infrastructure-monitoring.service.spec.ts @@ -0,0 +1,89 @@ +import { BadRequestException } from '@nestjs/common'; +import { ConfigService } from '@nestjs/config'; +import { InfrastructureMonitoringService } from './infrastructure-monitoring.service'; + +function success(data: unknown) { + return { + ok: true, + status: 200, + json: async () => ({ status: 'success', data }), + } as Response; +} + +describe('InfrastructureMonitoringService', () => { + afterEach(() => jest.restoreAllMocks()); + + it('rejects ranges outside the fixed whitelist before querying Prometheus', async () => { + const fetchSpy = jest.spyOn(global, 'fetch'); + const service = new InfrastructureMonitoringService(new ConfigService()); + + await expect(service.overview('30d')).rejects.toBeInstanceOf(BadRequestException); + expect(fetchSpy).not.toHaveBeenCalled(); + }); + + it('rejects credential-bearing or remote plaintext Prometheus endpoints at startup', () => { + expect(() => new InfrastructureMonitoringService(new ConfigService({ PROMETHEUS_URL: 'http://user:secret@127.0.0.1:9090' }))).toThrow('must not contain credentials'); + expect(() => new InfrastructureMonitoringService(new ConfigService({ PROMETHEUS_URL: 'http://monitor.example.com:9090' }))).toThrow('must use HTTPS'); + expect(() => new InfrastructureMonitoringService(new ConfigService({ PROMETHEUS_URL: 'https://monitor.example.com' }))).not.toThrow(); + }); + + it('loads real Prometheus vectors, ranges, services and active alerts', async () => { + const requestedUrls: URL[] = []; + jest.spyOn(global, 'fetch').mockImplementation(async (input) => { + const url = new URL(String(input)); + requestedUrls.push(url); + if (url.pathname.endsWith('/alerts')) { + return success({ alerts: [{ + labels: { alertname: 'HostCpuHigh', severity: 'warning', instance: '127.0.0.1:9100' }, + annotations: { summary: 'CPU持续偏高', threshold: '85%' }, + state: 'firing', + activeAt: '2026-08-14T03:00:00.000Z', + value: '88.2', + }] }); + } + const query = url.searchParams.get('query') ?? ''; + if (url.pathname.endsWith('/query_range')) { + return success({ result: [{ metric: {}, values: [[1_765_000_000, '12.5'], [1_765_000_060, '14.5']] }] }); + } + if (query.includes('node_systemd_unit_state')) { + return success({ result: [ + { metric: { name: 'cmpp-api.service' }, value: [1_765_000_060, '1'] }, + { metric: { name: 'cmpp-gateway.service' }, value: [1_765_000_060, '1'] }, + { metric: { name: 'postgresql.service' }, value: [1_765_000_060, '1'] }, + { metric: { name: 'redis-server.service' }, value: [1_765_000_060, '1'] }, + { metric: { name: 'cmpp-minio.service' }, value: [1_765_000_060, '1'] }, + { metric: { name: 'nginx.service' }, value: [1_765_000_060, '1'] }, + ] }); + } + if (query.includes('timestamp(node_uname_info)')) { + return success({ result: [{ metric: {}, value: [1_765_000_060, '1765000060'] }] }); + } + return success({ result: [{ metric: {}, value: [1_765_000_060, '25'] }] }); + }); + + const service = new InfrastructureMonitoringService(new ConfigService()); + const result = await service.overview('1h'); + + expect(result.available).toBe(true); + expect(result.metrics.cpuUsagePercent).toBe(25); + expect(result.trends.cpuUsagePercent).toHaveLength(2); + expect(result.summary).toMatchObject({ overallStatus: 'warning', serviceHealthy: 6, warningAlerts: 1 }); + expect(result.services.find((item) => item.key === 'redis')).toMatchObject({ unit: 'redis-server.service', status: 'healthy' }); + expect(result.alerts[0]).toMatchObject({ name: 'HostCpuHigh', severity: 'warning', currentValue: '88.2' }); + expect(requestedUrls.filter((url) => url.pathname.endsWith('/query_range'))).toHaveLength(5); + expect(requestedUrls.filter((url) => url.pathname.endsWith('/query_range')).every((url) => url.searchParams.get('step') === '60')).toBe(true); + }); + + it('returns an explicit unavailable payload without stale metrics when Prometheus fails', async () => { + jest.spyOn(global, 'fetch').mockRejectedValue(new Error('ECONNREFUSED')); + const service = new InfrastructureMonitoringService(new ConfigService()); + + const result = await service.overview('24h'); + + expect(result.available).toBe(false); + expect(result.summary.overallStatus).toBe('unknown'); + expect(result.metrics.cpuUsagePercent).toBeNull(); + expect(result.trends.cpuUsagePercent).toEqual([]); + expect(result.error).not.toContain('ECONNREFUSED'); + }); +}); diff --git a/api/src/infrastructure-monitoring/infrastructure-monitoring.service.ts b/api/src/infrastructure-monitoring/infrastructure-monitoring.service.ts new file mode 100644 index 0000000..a36ad33 --- /dev/null +++ b/api/src/infrastructure-monitoring/infrastructure-monitoring.service.ts @@ -0,0 +1,264 @@ +import { BadRequestException, Injectable } from '@nestjs/common'; +import { ConfigService } from '@nestjs/config'; +import { createHash } from 'node:crypto'; +import type { + InfrastructureAlert, + InfrastructureMetricPoint, + InfrastructureMonitoringOverview, + InfrastructureMonitoringRange, + InfrastructureServiceStatus, +} from './infrastructure-monitoring.contracts'; + +type PrometheusSample = [number, string]; +type PrometheusSeries = { + metric: Record; + value?: PrometheusSample; + values?: PrometheusSample[]; +}; +type PrometheusQueryResponse = { + status: 'success' | 'error'; + data?: { result?: PrometheusSeries[] }; + error?: string; +}; +type PrometheusAlertResponse = { + status: 'success' | 'error'; + data?: { + alerts?: Array<{ + labels?: Record; + annotations?: Record; + state?: string; + activeAt?: string; + value?: string; + }>; + }; +}; + +const RANGE_CONFIG: Record = { + '1h': { seconds: 60 * 60, step: 60 }, + '24h': { seconds: 24 * 60 * 60, step: 300 }, + '7d': { seconds: 7 * 24 * 60 * 60, step: 1800 }, +}; + +const QUERIES = { + cpuUsagePercent: '100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100)', + memoryUsagePercent: '(1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) * 100', + memoryTotalBytes: 'node_memory_MemTotal_bytes', + memoryAvailableBytes: 'node_memory_MemAvailable_bytes', + diskUsagePercent: '(1 - (node_filesystem_avail_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"} / node_filesystem_size_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"})) * 100', + diskTotalBytes: 'node_filesystem_size_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"}', + diskAvailableBytes: 'node_filesystem_avail_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"}', + networkReceiveBytesPerSecond: 'sum(rate(node_network_receive_bytes_total{device!~"lo"}[5m]))', + networkTransmitBytesPerSecond: 'sum(rate(node_network_transmit_bytes_total{device!~"lo"}[5m]))', + load1: 'node_load1', + uptimeSeconds: 'time() - node_boot_time_seconds', + lastSampleAt: 'max(timestamp(node_uname_info))', + services: 'max by (name) (node_systemd_unit_state{name=~"cmpp-api\\.service|cmpp-gateway\\.service|postgresql\\.service|redis(-server)?\\.service|cmpp-minio\\.service|nginx\\.service",state="active"})', +} as const; + +const SERVICE_DEFINITIONS = [ + { key: 'api', name: 'API服务', units: ['cmpp-api.service'] }, + { key: 'gateway', name: 'Gateway服务', units: ['cmpp-gateway.service'] }, + { key: 'postgresql', name: 'PostgreSQL', units: ['postgresql.service'] }, + { key: 'redis', name: 'Redis', units: ['redis.service', 'redis-server.service'] }, + { key: 'minio', name: 'MinIO', units: ['cmpp-minio.service'] }, + { key: 'nginx', name: 'Nginx', units: ['nginx.service'] }, +] as const; + +function finiteNumber(value: string | number | undefined): number | null { + const parsed = Number(value); + return Number.isFinite(parsed) ? parsed : null; +} + +function normalizePrometheusUrl(rawValue: unknown) { + const url = new URL(String(rawValue ?? 'http://127.0.0.1:9090')); + if (url.protocol !== 'http:' && url.protocol !== 'https:') throw new Error('PROMETHEUS_URL must use HTTP or HTTPS'); + if (url.username || url.password) throw new Error('PROMETHEUS_URL must not contain credentials'); + const privateIpv4 = /^(10\.|192\.168\.|172\.(1[6-9]|2\d|3[01])\.)/.test(url.hostname); + const loopback = url.hostname === '127.0.0.1' || url.hostname === 'localhost' || url.hostname === '[::1]'; + // Plain HTTP is only safe on loopback or an explicit RFC1918 address; named remote endpoints must use HTTPS. + if (url.protocol === 'http:' && !loopback && !privateIpv4) throw new Error('Remote PROMETHEUS_URL must use HTTPS'); + return url.toString().replace(/\/$/, ''); +} + +function vectorValue(response: PrometheusQueryResponse): number | null { + return finiteNumber(response.data?.result?.[0]?.value?.[1]); +} + +function matrixValues(response: PrometheusQueryResponse): InfrastructureMetricPoint[] { + return (response.data?.result?.[0]?.values ?? []).flatMap(([timestamp, value]) => { + const parsed = finiteNumber(value); + return parsed === null ? [] : [{ timestamp: new Date(timestamp * 1000).toISOString(), value: parsed }]; + }); +} + +function emptyMetrics(): InfrastructureMonitoringOverview['metrics'] { + return { + cpuUsagePercent: null, + memoryUsagePercent: null, + memoryTotalBytes: null, + memoryAvailableBytes: null, + diskUsagePercent: null, + diskTotalBytes: null, + diskAvailableBytes: null, + networkReceiveBytesPerSecond: null, + networkTransmitBytesPerSecond: null, + load1: null, + uptimeSeconds: null, + }; +} + +function emptyTrends(): InfrastructureMonitoringOverview['trends'] { + return { + cpuUsagePercent: [], + memoryUsagePercent: [], + diskUsagePercent: [], + networkReceiveBytesPerSecond: [], + networkTransmitBytesPerSecond: [], + }; +} + +@Injectable() +export class InfrastructureMonitoringService { + private readonly prometheusUrl: string; + private readonly queryTimeoutMs: number; + + constructor(config: ConfigService) { + this.prometheusUrl = normalizePrometheusUrl(config.get('PROMETHEUS_URL')); + this.queryTimeoutMs = Math.min(15_000, Math.max(1_000, Number(config.get('PROMETHEUS_QUERY_TIMEOUT_MS') ?? 5_000))); + } + + async overview(rawRange?: string): Promise { + const range = this.parseRange(rawRange); + const collectedAt = new Date().toISOString(); + try { + const [instant, trends, serviceResponse, alertResponse] = await Promise.all([ + this.loadInstantMetrics(), + this.loadTrends(range), + this.query(QUERIES.services), + this.getJson('/api/v1/alerts'), + ]); + const services = this.parseServices(serviceResponse); + const alerts = this.parseAlerts(alertResponse); + const warningAlerts = alerts.filter((item) => item.severity === 'warning').length; + const criticalAlerts = alerts.filter((item) => item.severity === 'critical').length; + const overallStatus = criticalAlerts > 0 ? 'critical' : warningAlerts > 0 ? 'warning' : 'healthy'; + return { + available: true, + range, + collectedAt, + lastSampleAt: instant.lastSampleAt === null ? null : new Date(instant.lastSampleAt * 1000).toISOString(), + summary: { + overallStatus, + serviceTotal: services.length, + serviceHealthy: services.filter((item) => item.status === 'healthy').length, + warningAlerts, + criticalAlerts, + activeAlerts: alerts.length, + }, + metrics: instant.metrics, + trends, + services, + alerts, + }; + } catch { + return this.unavailable(range, collectedAt); + } + } + + private parseRange(value?: string): InfrastructureMonitoringRange { + const range = value || '24h'; + if (!(range in RANGE_CONFIG)) throw new BadRequestException('监控时间范围只支持1h、24h或7d'); + return range as InfrastructureMonitoringRange; + } + + private async loadInstantMetrics() { + const keys = Object.keys(emptyMetrics()) as Array; + const responses = await Promise.all([...keys.map((key) => this.query(QUERIES[key])), this.query(QUERIES.lastSampleAt)]); + const metrics = emptyMetrics(); + keys.forEach((key, index) => { metrics[key] = vectorValue(responses[index]); }); + return { metrics, lastSampleAt: vectorValue(responses[responses.length - 1]) }; + } + + private async loadTrends(range: InfrastructureMonitoringRange) { + const config = RANGE_CONFIG[range]; + const end = Math.floor(Date.now() / 1000); + const start = end - config.seconds; + const keys = Object.keys(emptyTrends()) as Array; + const responses = await Promise.all(keys.map((key) => this.queryRange(QUERIES[key], start, end, config.step))); + return Object.fromEntries(keys.map((key, index) => [key, matrixValues(responses[index])])) as InfrastructureMonitoringOverview['trends']; + } + + private parseServices(response: PrometheusQueryResponse): InfrastructureServiceStatus[] { + const values = new Map(); + for (const item of response.data?.result ?? []) { + if (item.metric.name) values.set(item.metric.name, vectorValue({ status: 'success', data: { result: [item] } }) ?? 0); + } + return SERVICE_DEFINITIONS.map((definition) => { + const present = definition.units.filter((unit) => values.has(unit)); + const status = present.length === 0 ? 'unknown' : present.some((unit) => (values.get(unit) ?? 0) >= 1) ? 'healthy' : 'unhealthy'; + return { key: definition.key, name: definition.name, unit: present[0] ?? definition.units[0], status }; + }); + } + + private parseAlerts(response: PrometheusAlertResponse): InfrastructureAlert[] { + return (response.data?.alerts ?? []) + .filter((item) => item.state === 'firing' || item.state === 'pending') + .map((item) => { + const labels = item.labels ?? {}; + const annotations = item.annotations ?? {}; + const severity: InfrastructureAlert['severity'] = labels.severity === 'critical' ? 'critical' : labels.severity === 'warning' ? 'warning' : 'info'; + const identity = JSON.stringify(Object.entries(labels).sort(([left], [right]) => left.localeCompare(right))); + return { + fingerprint: createHash('sha256').update(identity).digest('hex').slice(0, 24), + name: labels.alertname || '未命名告警', + severity, + status: item.state || 'unknown', + startedAt: item.activeAt || new Date().toISOString(), + summary: annotations.summary || annotations.description || labels.alertname || '监控告警', + description: annotations.description, + currentValue: annotations.currentValue || item.value, + threshold: annotations.threshold, + service: labels.service, + instance: labels.instance, + }; + }) + .sort((left, right) => { + const priority: Record = { critical: 0, warning: 1, info: 2 }; + return priority[left.severity] - priority[right.severity] || Date.parse(left.startedAt) - Date.parse(right.startedAt); + }); + } + + private unavailable(range: InfrastructureMonitoringRange, collectedAt: string): InfrastructureMonitoringOverview { + const services = SERVICE_DEFINITIONS.map((item) => ({ key: item.key, name: item.name, unit: item.units[0], status: 'unknown' as const })); + return { + available: false, + range, + collectedAt, + lastSampleAt: null, + error: 'Prometheus监控数据当前不可用，请检查采集与服务状态', + summary: { overallStatus: 'unknown', serviceTotal: services.length, serviceHealthy: 0, warningAlerts: 0, criticalAlerts: 0, activeAlerts: 0 }, + metrics: emptyMetrics(), + trends: emptyTrends(), + services, + alerts: [], + }; + } + + private query(query: string) { + return this.getJson('/api/v1/query', { query }); + } + + private queryRange(query: string, start: number, end: number, step: number) { + return this.getJson('/api/v1/query_range', { query, start: String(start), end: String(end), step: String(step) }); + } + + private async getJson(path: string, params: Record = {}): Promise { + const url = new URL(`${this.prometheusUrl}${path}`); + Object.entries(params).forEach(([key, value]) => url.searchParams.set(key, value)); + const response = await fetch(url, { headers: { Accept: 'application/json' }, signal: AbortSignal.timeout(this.queryTimeoutMs) }); + if (!response.ok) throw new Error(`Prometheus HTTP ${response.status}`); + const result = await response.json() as T; + if (result.status !== 'success') throw new Error('Prometheus query failed'); + return result; + } +} diff --git a/docs/first-version-development-requirements.md b/docs/first-version-development-requirements.md index 6535d83..e657106 100644 --- a/docs/first-version-development-requirements.md +++ b/docs/first-version-development-requirements.md @@ -2025,6 +2025,17 @@ - 运营端“短信通道组管理”在通道组名称条件之外增加“通道”筛选。选定一个通道后，只展示成员配置中真实包含该`channelId`的未删除通道组；与通道组名称同时输入时按两个条件取交集。 - 通道选项必须来自真实通道API，不使用静态列表、Mock或localStorage。页面首次加载时通道组与通道两个独立请求并行执行；选项同时展示通道名称和编码，已删除通道明确标记“已删除”。未加入任何通道组的真实通道仍可选择，选中后结果为零而不得隐藏该选项。 - 通道选择控件必须为通用下拉可搜索控件，支持按通道名称或编码搜索；“全部通道”表示不按通道限制，点击“重置”必须同时清空通道组名称和通道条件并回到第一页。 + +## Prometheus 系统监控（2026-08-14） + +- 运营端“系统管理”新增“系统监控”，路由为`/admin/system-monitoring`；原“发送监控”继续负责短信通道和消息业务指标，两个页面、接口和统计口径不得混用。 +- 系统监控使用Prometheus和Node Exporter作为真实指标基础设施，但全部用户界面由平台React原生实现，不嵌入Grafana、Netdata、Zabbix、Prometheus页面或第三方登录界面。 +- 浏览器只请求`GET /api/admin/infrastructure-monitoring/overview?range=1h|24h|7d`。NestJS以固定PromQL模板查询Prometheus，禁止前端传任意PromQL、step、时间戳或标签选择器；9090和9100只监听本机或内网，不向公网开放。 +- 第一版展示CPU、内存、根文件系统、网络收发、1分钟负载和系统运行时长，并展示API、Gateway、PostgreSQL、Redis、MinIO、Nginx六类systemd服务状态。指标缺失必须显示“暂无指标/未知”，不得用0、静态数据、Mock或localStorage冒充真实采集值。 +- 支持近1小时、近24小时和近7天固定范围，步长分别为60秒、300秒和1800秒。页面可见时每30秒刷新，隐藏或卸载后停止；手动刷新保留当前范围。 +- 页面展示Prometheus当前firing/pending告警，严重性使用`info/warning/critical`。告警阈值由Prometheus规则计算，前端不重复判断；第一版仅只读展示，不提供缺少审计模型的确认、备注、静默或关闭操作。 +- Prometheus不可用、查询超时、响应非法时接口返回`available=false`及安全错误摘要，页面清空陈旧指标并展示监控不可用；不能继续显示上一次数据造成误判。 +- 完整采集、查询、安全、响应契约、视觉规格和验收口径见`docs/prometheus-system-monitoring-design-20260814.md`。 # 下游投递后台重投任务（2026-08-12） ## 完整设计口径与安全整改（2026-08-13） diff --git a/docs/production-deployment.md b/docs/production-deployment.md index abca289..eb62926 100644 --- a/docs/production-deployment.md +++ b/docs/production-deployment.md @@ -48,6 +48,8 @@ OPERATION_LOG_ARCHIVE_INTERVAL_MS=86400000 SMS_RECEIPT_TIMEOUT_SCAN_ENABLED=true SMS_RECEIPT_TIMEOUT_HOURS=72 SMS_RECEIPT_TIMEOUT_SCAN_INTERVAL_MS=300000 +PROMETHEUS_URL=http://127.0.0.1:9090 +PROMETHEUS_QUERY_TIMEOUT_MS=5000 REPORT_DAILY_REFRESH_ENABLED=true REPORT_REFRESH_INTERVAL_MS=3600000 CMPP_DOWNSTREAM_ACK_TIMEOUT_SECONDS=30 @@ -62,6 +64,8 @@ PROD_ADMIN_USERNAME=prod_admin PROD_ADMIN_PASSWORD='change-me' ``` +系统监控的Prometheus和Node Exporter需要在发布前单独安装，配置和规则位于`tools/monitoring/`。在Debian/Ubuntu服务器执行`bash tools/monitoring/install-prometheus-monitoring.sh`；脚本会先备份现有Prometheus配置并运行`promtool`校验，只重启Prometheus和Node Exporter，不重启CMPP业务服务。9090和9100必须只监听`127.0.0.1`，不得加入Nginx公网反向代理或安全组放行。安装完成且确认`curl http://127.0.0.1:9090/-/ready`成功后，才可在正常发布窗口重启API使系统监控接口生效；详细口径见`docs/prometheus-system-monitoring-design-20260814.md`。 + 安全会话使用 HttpOnly Cookie，正式生产必须先为页面和管理 API 配置 HTTPS，并保持 `SESSION_COOKIE_SECURE=true`；纯 HTTP 的 `IP:12026` 不作为受支持的登录入口，即使切换期仍保留其监听，也只允许用于非登录的兼容检查并应尽快下线。`sms.lisglo.com` 只允许 Cloudflare 回源，`api.lisglo.com` 通过独立 Nginx SNI 虚拟主机只开放客户接口、客户 Swagger 和健康检查；Let’s Encrypt 使用 DNS-01 自动续期，不依赖开放 80 端口。 系统操作日志默认在线保留 180 天。API 每日以最多 20 个、每批 1000 条的小事务将过期记录搬入 `OperationLogArchive`，并用 `archiveMonth=YYYY-MM` 标记归档月份；归档记录不会自动删除。调整保留期或批量参数前，应先评估数据库、备份窗口和审计要求。归档表达到千万级或清理窗口不能满足要求时，再实施按 `createdAt` 的月度 PostgreSQL 分区，不在当前数据规模下提前改造主表分区。 diff --git a/docs/prometheus-system-monitoring-design-20260814.md b/docs/prometheus-system-monitoring-design-20260814.md new file mode 100644 index 0000000..4453c67 --- /dev/null +++ b/docs/prometheus-system-monitoring-design-20260814.md @@ -0,0 +1,194 @@ +# Prometheus 系统监控设计 + +> 版本：V1.0
+> 设计日期：2026-08-14
+> 适用范围：运营端 → 系统管理 → 系统监控
+> 实施边界：使用平台原生 React UI，不引入 Grafana、Netdata、Zabbix 或 Prometheus 自带 UI + +## 1. 建设目标 + +运营人员需要在现有权限体系和视觉体系内查看服务器硬件资源、核心服务状态、历史趋势和活动告警。Prometheus 仅承担指标抓取、时序存储、PromQL 计算和告警规则执行；浏览器只访问 CMPP 平台 NestJS API，不直接访问 Prometheus、Node Exporter 或 Alertmanager。 + +第一版解决以下问题： + +1. 统一查看 CPU、内存、根文件系统、网络流量、负载和运行时长。 +2. 查看 `cmpp-api`、`cmpp-gateway`、PostgreSQL、Redis、MinIO、Nginx 六类核心服务状态。 +3. 在近 1 小时、近 24 小时、近 7 天之间切换真实历史趋势。 +4. 查看 Prometheus 当前 firing/pending 告警，不在前端伪造阈值判断。 +5. Prometheus 不可用、查询超时或指标缺失时明确展示“监控不可用/暂无指标”，不得回退静态值、Mock 或 localStorage。 + +## 2. 非目标与边界 + +- 第一版不提供任意 PromQL 控制台，避免越权查询、高基数查询和资源耗尽。 +- 第一版不提供告警确认、备注和关闭操作；需要审计留痕的告警处置另立需求并增加 PostgreSQL 模型。 +- 第一版不采集短信正文、手机号、账号、密钥、数据库查询文本或日志原文。 +- 云主机通常只能提供虚拟 CPU、内存、云盘和虚拟网卡指标；风扇、物理温度、电源、RAID 和物理硬盘 SMART 需 IPMI/Redfish 或厂商接口，未接入时不得显示伪造数据。 +- Prometheus、Node Exporter 和 Alertmanager 不暴露在公网；仅 API 服务端可以访问 Prometheus。 + +## 3. 总体架构 + +```mermaid +flowchart LR + Node["Node Exporter\n主机与 systemd 指标"] --> Prom["Prometheus\n抓取、存储、PromQL、告警规则"] + API["NestJS 监控模块\n固定查询模板与响应归一化"] --> Prom + UI["运营端原生 React UI"] --> API + Prom --> Alert["Prometheus 活动告警"] + Alert --> API +``` + +数据流必须是 `Exporter → Prometheus → NestJS → 运营端`。前端不得直接拼接 Prometheus URL，不得保存 Prometheus Token，不得接受服务端返回的原始 PromQL。 + +## 4. 采集与部署设计 + +### 4.1 Node Exporter + +Node Exporter 仅监听 `127.0.0.1:9100`，启用默认 CPU、内存、文件系统、磁盘、网络、负载和启动时间采集器，并显式启用 `systemd` 采集器。systemd 只允许采集： + +- `cmpp-api.service` +- `cmpp-gateway.service` +- `postgresql.service` +- `redis.service` 或 `redis-server.service` +- `cmpp-minio.service` +- `nginx.service` + +排除 `/dev`、`/proc`、`/sys`、`/run` 等伪文件系统和临时挂载，避免磁盘指标重复和高基数。 + +### 4.2 Prometheus + +- 仅监听 `127.0.0.1:9090`。 +- 默认每 15 秒抓取一次，查询超时 5 秒。 +- 第一版保留 30 天或不超过 8GB 的时序数据，达到任一边界即按 Prometheus TSDB 策略清理。 +- 配置文件由仓库 `tools/monitoring/` 管理；安装脚本只安装、校验和启动监控服务，不重启 CMPP Gateway 或发送 Worker。 +- 生产环境变量使用 `PROMETHEUS_URL=http://127.0.0.1:9090`，API 不向响应透出该地址。 + +### 4.3 告警规则 + +规则使用持续窗口而非瞬时尖峰： + +| 告警 | Warning | Critical | +|---|---:|---:| +| 主机指标失联 | — | 连续 2 分钟无数据 | +| CPU 使用率 | 连续 10 分钟 > 85% | 连续 5 分钟 > 95% | +| 内存使用率 | 连续 10 分钟 > 85% | 连续 5 分钟 > 95% | +| 根文件系统使用率 | 连续 15 分钟 > 80% | 连续 5 分钟 > 90% | +| inode 使用率 | 连续 15 分钟 > 80% | 连续 5 分钟 > 90% | +| CPU iowait | 连续 10 分钟 > 20% | 连续 10 分钟 > 35% | +| 核心 systemd 服务 | — | 非 active 2 分钟 | + +告警标签至少包含 `alertname`、`severity`、`instance`、`service`；注解至少包含中文 `summary`、`description`、`currentValue`、`threshold`。敏感环境变量和凭据不得进入标签或注解。 + +## 5. 后端设计 + +### 5.1 API + +```text +GET /api/admin/infrastructure-monitoring/overview?range=1h|24h|7d +``` + +该接口受现有运营端 Session 中间件保护，仅提供只读数据。`range` 只接受白名单；非法值返回 400。 + +### 5.2 固定查询 + +后端维护具名查询表，不接受前端 PromQL： + +- CPU：非 idle CPU 秒率。 +- 内存：`1 - MemAvailable / MemTotal`。 +- 根文件系统：`1 - available / size`。 +- 网络：排除 loopback 后的收发字节率。 +- 负载：`node_load1`。 +- 运行时长：当前时间减 `node_boot_time_seconds`。 +- 服务：指定 unit 的 `node_systemd_unit_state{state="active"}`。 +- 告警：Prometheus `/api/v1/alerts`。 + +瞬时查询和区间查询并行执行。区间步长固定为：1小时/60秒、24小时/300秒、7天/1800秒；后端最多返回约 340 个点/序列，禁止浏览器控制步长。 + +### 5.3 可用性与错误语义 + +- Prometheus 查询成功：`available=true`，返回真实指标、服务和告警。 +- Prometheus 未配置、拒绝连接、超时、返回非成功状态或响应结构非法：`available=false`，返回采集时间和安全错误摘要，指标字段为 `null`、趋势为空数组。 +- 单个指标不存在不影响其他指标，缺失项为 `null`；不得把缺失解释为 0。 +- 服务状态使用 `healthy/unhealthy/unknown`，只有明确采集到 active 才是 healthy，明确采集到 0 才是 unhealthy，指标缺失是 unknown。 + +### 5.4 安全控制 + +- Prometheus URL 由服务端环境变量读取并限制为 `http://127.0.0.1` 或明确配置的内网 HTTPS 地址。 +- 每次请求设置 5 秒 AbortSignal 超时。 +- 不记录完整响应体；错误日志不得包含URL中的认证信息。 +- 不允许前端传 query、step、start、end 或 Prometheus 标签选择器。 +- 不将监控指标复制进业务 PostgreSQL，避免高频写入和业务库膨胀。 + +## 6. 前端信息架构 + +页面路由为 `/admin/system-monitoring`，保留原 `/admin/monitor` “发送监控”，两者业务含义不得混用。 + +### 6.1 页面结构 + +1. 页面标题、最近采集时间、刷新按钮、1小时/24小时/7天范围切换。 +2. 横向健康概览：整体状态、服务总数、正常、警告、严重、活动告警。 +3. CPU、内存、磁盘、网络四项资源区：当前值、辅助值和当前范围趋势。 +4. 服务状态侧栏：六类服务、状态和最新采集时间。 +5. 活动告警表：名称、级别、开始时间、持续时间、当前值、阈值。 +6. 下方资源趋势：CPU、内存、磁盘和网络收发的完整趋势图。 + +### 6.2 视觉规格 + +- 复用现有 `surface`、`Button`、`Tag`、`Table`、`Chart` 和主题变量。 +- 真白背景、低饱和蓝色主色、冷灰边框；正常/警告/严重使用既有语义色。 +- 不使用第三方Logo、iframe、暗色监控皮肤、霓虹渐变或装饰性假指标。 +- 桌面优先保持表格和趋势信息密度；1100px 以下两列，780px 以下单列。 +- 图表缺失时显示“暂无真实指标”，不能绘制零线冒充数据。 + +### 6.3 交互与刷新 + +- 初次进入立即请求真实接口。 +- 可见页面每 30 秒刷新；页面隐藏或 Session 锁定导致组件卸载后停止刷新。 +- 切换时间范围立即重新查询并禁用重复点击；手动刷新不改变范围。 +- 请求失败保留上一次成功数据会造成陈旧误导，因此本版失败时清空数据并展示不可用状态。 + +## 7. 响应契约摘要 + +```ts +type InfrastructureOverview = { + available: boolean; + range: '1h' | '24h' | '7d'; + collectedAt: string; + error?: string; + summary: { + overallStatus: 'healthy' | 'warning' | 'critical' | 'unknown'; + serviceTotal: number; + serviceHealthy: number; + warningAlerts: number; + criticalAlerts: number; + activeAlerts: number; + }; + metrics: { + cpuUsagePercent: number | null; + memoryUsagePercent: number | null; + memoryTotalBytes: number | null; + memoryAvailableBytes: number | null; + diskUsagePercent: number | null; + diskTotalBytes: number | null; + diskAvailableBytes: number | null; + networkReceiveBytesPerSecond: number | null; + networkTransmitBytesPerSecond: number | null; + load1: number | null; + uptimeSeconds: number | null; + }; + trends: Record>; + services: Array<{ key: string; name: string; unit: string; status: 'healthy' | 'unhealthy' | 'unknown' }>; + alerts: Array<{ fingerprint: string; name: string; severity: 'warning' | 'critical' | 'info'; status: string; startedAt: string; summary: string; currentValue?: string; threshold?: string }>; +}; +``` + +## 8. 验收标准 + +1. 页面所有指标来自真实 Prometheus API，关闭 Prometheus 后页面明确不可用且没有静态回退。 +2. 浏览器网络请求只访问 CMPP API，不访问 9090/9100。 +3. 非法 range、任意 PromQL 和自定义 step 均无法进入后端查询。 +4. CPU、内存、磁盘、网络当前值与 Prometheus 同时刻查询在允许的采样误差内一致。 +5. 六类服务状态与 systemd 指标一致；指标缺失展示未知而非故障或正常。 +6. 1小时、24小时、7天切换后时间轴和点数符合固定步长。 +7. firing/pending 告警真实展示，告警恢复后不再出现在活动列表。 +8. Prometheus/Exporter 只监听本机，公网不能连接 9090/9100。 +9. 页面在桌面和移动宽度下无重叠、截断和横向溢出，控制台无相关错误。 +10. TypeScript、API专项测试、生产构建、配置校验和 `git diff --check`通过。 diff --git a/docs/system-functional-test-cases.md b/docs/system-functional-test-cases.md index 93d668d..0dfcc37 100644 --- a/docs/system-functional-test-cases.md +++ b/docs/system-functional-test-cases.md @@ -4628,3 +4628,25 @@ npm run verify:phase8 | TC-DASHBOARD-SIGNATURE-REMOVE-001 | 删除看板签名统计模块并统一金额样式 | 打开运营看板，检查指标卡与后续模块 | 两个“今日签名发送统计”模块均不存在；今日消费金额与今日发送总量主数字字号、字重和颜色一致；今日活跃签名仍来自真实接口 | | TC-ENTERPRISE-APP-WIDTH-001 | 企业应用关键列缩窄 | 在桌面大屏打开企业应用管理并读取表头列宽 | 状态、到达率、单价列宽均约为原宽度80%，字段与操作均未隐藏，横向滚动需求减少 | | TC-UI-CARRIER-TAG-002 | 全局运营商数据展示复用通用标签 | 抽查通道/通道组、监控、报备、企业签名、短信审核、批次号码、发送记录和客户端发送详情 | 移动、联通、电信均使用全局低饱和胶囊及统一色值；有运营商集合的三网通道显示三个标签，历史通道级字段显示中性“三网”；筛选选项、图表图例与导出文本保持纯文本 | + +## Prometheus 系统监控专项用例（2026-08-14） + +| 用例编号 | 场景 | 操作 | 预期结果 | +|---|---|---|---| +| TC-INFRA-MON-001 | 运营端权限与原生页面 | 登录运营端，打开“系统管理 → 系统监控”，检查页面和浏览器网络请求 | 页面使用平台导航、组件和样式；只请求平台`/api/admin/infrastructure-monitoring/overview`，不加载iframe，不从浏览器访问9090/9100，不出现第三方Logo或登录页 | +| TC-INFRA-MON-002 | 未登录访问 | 清除运营端Session后直接访问系统监控API和页面 | API按现有Session中间件拒绝，页面进入运营端登录流程；Prometheus数据不得绕过运营端权限公开 | +| TC-INFRA-MON-003 | 当前硬件指标真实值 | 在同一采样窗口分别请求平台监控API和Prometheus固定查询 | CPU、内存、根文件系统、网络、负载、运行时长与Prometheus结果在采样误差内一致，响应不包含Prometheus地址或PromQL | +| TC-INFRA-MON-004 | 指标缺失 | 临时禁用某个Node Exporter采集器或查询一个不存在的指标后请求页面 | 对应指标为`null`并显示“暂无真实指标”，其他指标继续展示；不得显示0或生成趋势线 | +| TC-INFRA-MON-005 | Prometheus不可用 | 停止本地测试Prometheus或将测试环境指向拒绝连接端口，请求监控API | API返回`available=false`和安全错误摘要，页面显示“监控数据不可用”并清空陈旧指标，不显示Mock或上次数据 | +| TC-INFRA-MON-006 | 查询超时 | 让测试Prometheus响应超过5秒 | 请求被AbortSignal终止，接口有限时间返回不可用状态；API进程不积累悬挂请求 | +| TC-INFRA-MON-007 | 时间范围白名单 | 分别请求`1h`、`24h`、`7d`、`30d`和注入PromQL字符串 | 前三种成功且step分别为60/300/1800秒；非法值返回400，不能进入Prometheus查询 | +| TC-INFRA-MON-008 | 趋势切换 | 页面依次选择近1小时、近24小时、近7天 | 每次只发一个新范围请求；图表时间轴、点数和当前范围同步更新，切换期间防止重复触发 | +| TC-INFRA-MON-009 | 自动与手动刷新 | 保持页面可见超过30秒，再隐藏页面并点击手动刷新 | 可见时按30秒刷新；隐藏后停止；重新可见后立即刷新；手动刷新保留范围且不会并发重复请求 | +| TC-INFRA-MON-010 | 核心服务状态 | 核对`cmpp-api`、`cmpp-gateway`、PostgreSQL、Redis、MinIO、Nginx的systemd状态与页面 | active显示正常，明确0显示异常，指标不存在显示未知；Redis兼容`redis.service`与`redis-server.service`别名 | +| TC-INFRA-MON-011 | 活动告警 | 触发一条warning和一条critical测试规则并等待Prometheus进入firing | 页面显示真实名称、严重性、开始时间、持续时间、当前值和阈值；概览计数准确，恢复后活动列表移除 | +| TC-INFRA-MON-012 | 综合状态 | 分别构造无告警、warning、critical和Prometheus不可用状态 | 综合状态依次为正常、警告、严重、未知；critical优先于warning，不按前端瞬时指标重复计算 | +| TC-INFRA-MON-013 | 并行查询与响应上限 | 检查后端请求时序，并用7天范围请求最大趋势 | 瞬时、趋势、服务、告警查询并行；每序列约不超过340点，响应不包含原始Prometheus响应体 | +| TC-INFRA-MON-014 | 监听与公网暴露 | 在服务器执行`ss -lnt`并从外部探测9090、9100 | Prometheus和Node Exporter仅监听127.0.0.1或明确内网地址；公网9090/9100不可连接，运营端仍能经平台API读取指标 | +| TC-INFRA-MON-015 | 响应式与无障碍 | 在1536×1024、1280×800和390×844打开页面，操作范围和刷新按钮 | 桌面信息层级符合设计稿；窄屏无内容重叠和页面横向溢出；按钮有可读名称，活动范围和告警严重性不只依赖颜色表达 | +| TC-INFRA-MON-016 | 配置和部署幂等 | 在测试服务器重复执行监控安装脚本和配置校验 | 不重复创建系统用户，不开放公网端口；配置通过`promtool check config/rules`，服务保持active，CMPP API/Gateway不因安装被重启 | +| TC-INFRA-MON-017 | 业务数据隔离 | 运行监控24小时并检查PostgreSQL业务库和指标标签 | 监控时序只保存在Prometheus TSDB，业务PostgreSQL无高频指标写入；标签、日志和API响应不含手机号、短信正文、账号或密钥 | diff --git a/docs/testing-progress.md b/docs/testing-progress.md index 428e1f1..a45e027 100644 --- a/docs/testing-progress.md +++ b/docs/testing-progress.md @@ -3573,3 +3573,12 @@ git diff --check - API、Gateway、Nginx、PostgreSQL、Redis与MinIO均active，内部API/Gateway/MinIO健康、Redis PONG；`gateway.submit.commands`消费者1、`pending=0`、`lag=0`。公网运营端、客户端和API health均HTTP 200，公网CMPP 17890 TCP连通；发布时间窗API/Gateway warning级日志为0。 - 首次前台SSH执行因本地等待窗口关闭而收到终止信号，包装流程按设计恢复PostgreSQL和原运行目录，运行标识回到`4c70978d`、migration回到86条、服务全部active；确认恢复资产完整后改为服务器后台日志方式重新执行并成功，未并发重复部署。 - 本轮没有创建真实重投任务、调用真实Gateway重投、发送短信、修改通道配置、余额或客户连接。既有`api/tsconfig.build.tsbuildinfo`、`tsconfig.tsbuildinfo`、`outputs/`和空文件`=`继续保护，不归入业务提交。 + +# 2026-08-14 Prometheus 系统监控（本地未提交、未发布） + +- 运营端“系统管理”新增独立“系统监控”页面，保留原“发送监控”业务职责。页面使用平台React、ECharts和通用组件原生实现，不嵌入Grafana、Prometheus或第三方iframe；支持近1小时、24小时、7天，展示CPU、内存、根磁盘、网络、负载、运行时长、六类核心systemd服务和Prometheus活动告警。 +- 新增只读`GET /api/admin/infrastructure-monitoring/overview`。后端只接受`1h/24h/7d`白名单，按固定PromQL并行访问Prometheus，5秒超时；浏览器不能提交PromQL或访问9090/9100。缺失序列返回`null`，Prometheus异常返回`available=false`并清空指标和趋势，不用0、Mock、静态数据或localStorage伪装。 +- 新增Debian/Ubuntu幂等安装脚本、Prometheus抓取配置和告警规则。Prometheus与Node Exporter只监听`127.0.0.1`，默认保留30天且限制8GB；脚本备份既有配置和override、运行`promtool`校验，只重启两个监控服务，不重启API、Gateway或其他业务服务。本轮未在预生产执行脚本。 +- 专项Jest 1套/4项、API全量38套/467项通过，覆盖范围白名单、Prometheus HTTP响应契约解析、固定60秒step、服务别名、活动告警、不可用无陈旧数据及监控地址安全约束；全量Jest仍因仓库既有异步句柄使用`--forceExit`收尾。API正式TypeScript、前端TypeScript、Vite 8.1.5生产构建、两个Shell脚本语法及`git diff --check`通过；Vite仅保留既有约2.10MB单chunk提示。 +- 浏览器优先检查现有页面：本地运营端无有效登录Session，被正常引导到带算术验证码的登录页；未绕过登录、未读取或填写验证码，因此登录后桌面与窄屏视觉验收尚未完成。真实Prometheus/Node Exporter集成和告警触发验收必须在明确授权安装的测试或预生产窗口执行，当前不以单测替代真实基础设施验收。 +- 本轮代码、配置和文档保持未提交、未推送、未部署，没有发送、补发或重投短信，没有修改通道账号、密码、启停状态、企业余额、客户连接或预生产数据。既有构建缓存、`outputs/`和空文件`=`继续保护；并行会话新增的Fail2ban设计与测试文档不属于本需求，不修改、不归因。 diff --git a/src/api/admin/infrastructure-monitoring.api.ts b/src/api/admin/infrastructure-monitoring.api.ts new file mode 100644 index 0000000..92dab94 --- /dev/null +++ b/src/api/admin/infrastructure-monitoring.api.ts @@ -0,0 +1,7 @@ +import { request, withQuery } from '../core/httpClient'; +import type { InfrastructureMonitoringOverview, InfrastructureMonitoringRange } from '../types'; + +export const adminInfrastructureMonitoringApi = { + getInfrastructureMonitoringOverview: (range: InfrastructureMonitoringRange) => + request(withQuery('/admin/infrastructure-monitoring/overview', { range })), +}; diff --git a/src/api/adminApi.ts b/src/api/adminApi.ts index 36389b5..dca5a3b 100644 --- a/src/api/adminApi.ts +++ b/src/api/adminApi.ts @@ -10,6 +10,7 @@ import { adminOperationsApi } from './admin/operations.api'; import { adminGovernanceApi } from './admin/governance.api'; import { adminFilesApi } from './admin/files.api'; import { adminSignatureRetirementApi } from './admin/signature-retirement.api'; +import { adminInfrastructureMonitoringApi } from './admin/infrastructure-monitoring.api'; export const adminApi = { ...adminIdentityApi, @@ -18,4 +19,5 @@ export const adminApi = { ...adminGovernanceApi, ...adminFilesApi, ...adminSignatureRetirementApi, + ...adminInfrastructureMonitoringApi, }; diff --git a/src/api/types/index.ts b/src/api/types/index.ts index 57035a6..5488e4a 100644 --- a/src/api/types/index.ts +++ b/src/api/types/index.ts @@ -4,3 +4,4 @@ export * from './channels-reports'; export * from './operations'; export * from './governance'; export * from './signature-retirement'; +export * from './infrastructure-monitoring'; diff --git a/src/api/types/infrastructure-monitoring.ts b/src/api/types/infrastructure-monitoring.ts new file mode 100644 index 0000000..22013a0 --- /dev/null +++ b/src/api/types/infrastructure-monitoring.ts @@ -0,0 +1,65 @@ +export type InfrastructureMonitoringRange = '1h' | '24h' | '7d'; + +export type InfrastructureMetricPoint = { + timestamp: string; + value: number; +}; + +export type InfrastructureServiceStatus = { + key: string; + name: string; + unit: string; + status: 'healthy' | 'unhealthy' | 'unknown'; +}; + +export type InfrastructureAlert = { + fingerprint: string; + name: string; + severity: 'info' | 'warning' | 'critical'; + status: string; + startedAt: string; + summary: string; + description?: string; + currentValue?: string; + threshold?: string; + service?: string; + instance?: string; +}; + +export type InfrastructureMonitoringOverview = { + available: boolean; + range: InfrastructureMonitoringRange; + collectedAt: string; + lastSampleAt: string | null; + error?: string; + summary: { + overallStatus: 'healthy' | 'warning' | 'critical' | 'unknown'; + serviceTotal: number; + serviceHealthy: number; + warningAlerts: number; + criticalAlerts: number; + activeAlerts: number; + }; + metrics: { + cpuUsagePercent: number | null; + memoryUsagePercent: number | null; + memoryTotalBytes: number | null; + memoryAvailableBytes: number | null; + diskUsagePercent: number | null; + diskTotalBytes: number | null; + diskAvailableBytes: number | null; + networkReceiveBytesPerSecond: number | null; + networkTransmitBytesPerSecond: number | null; + load1: number | null; + uptimeSeconds: number | null; + }; + trends: { + cpuUsagePercent: InfrastructureMetricPoint[]; + memoryUsagePercent: InfrastructureMetricPoint[]; + diskUsagePercent: InfrastructureMetricPoint[]; + networkReceiveBytesPerSecond: InfrastructureMetricPoint[]; + networkTransmitBytesPerSecond: InfrastructureMetricPoint[]; + }; + services: InfrastructureServiceStatus[]; + alerts: InfrastructureAlert[]; +}; diff --git a/src/apps/admin/system-monitoring/AdminSystemMonitoringPage.css b/src/apps/admin/system-monitoring/AdminSystemMonitoringPage.css new file mode 100644 index 0000000..c785a41 --- /dev/null +++ b/src/apps/admin/system-monitoring/AdminSystemMonitoringPage.css @@ -0,0 +1,309 @@ +.admin-system-monitoring-page { + gap: 18px; +} + +.system-monitoring-heading { + align-items: flex-end; + gap: 24px; +} + +.system-monitoring-title-row { + align-items: center; + display: flex; + gap: 12px; + margin-top: 10px; +} + +.system-monitoring-title-row h1 { + color: var(--color-text-strong); + font-size: 24px; + line-height: 1.25; + margin: 0; +} + +.system-monitoring-title-row p { + color: var(--color-text-muted); + font-size: 13px; + margin: 4px 0 0; +} + +.system-monitoring-controls, +.system-monitoring-range { + align-items: center; + display: flex; +} + +.system-monitoring-controls { + gap: 10px; +} + +.system-monitoring-range { + background: var(--color-surface-muted); + border: 1px solid var(--color-border); + border-radius: var(--radius-md); + padding: 3px; +} + +.system-monitoring-range button { + background: transparent; + border: 0; + border-radius: 4px; + color: var(--color-text-muted); + cursor: pointer; + font: inherit; + min-height: 30px; + padding: 0 12px; +} + +.system-monitoring-range button:hover { + color: var(--color-text-strong); +} + +.system-monitoring-range button:focus-visible { + box-shadow: var(--focus-ring); + outline: none; +} + +.system-monitoring-range button.is-active { + background: var(--color-surface); + box-shadow: var(--shadow-xs); + color: var(--color-selected); + font-weight: var(--font-weight-semibold); +} + +.system-monitoring-unavailable { + align-items: flex-start; + background: var(--color-danger-soft); + border: 1px solid color-mix(in srgb, var(--color-danger) 24%, transparent); + border-radius: var(--radius-lg); + color: var(--color-danger); + display: flex; + gap: 12px; + padding: 14px 16px; +} + +.system-monitoring-unavailable div { + display: grid; + gap: 2px; +} + +.system-monitoring-unavailable span { + color: var(--color-text-muted); + font-size: 13px; +} + +.system-monitoring-health { + align-items: center; + display: grid; + gap: 0; + grid-template-columns: minmax(220px, 1.35fr) repeat(4, minmax(120px, 1fr)); + padding: 18px 20px; +} + +.system-monitoring-health__mark { + align-items: center; + border-radius: 50%; + display: flex; + height: 48px; + justify-content: center; + margin-right: 13px; + width: 48px; +} + +.system-monitoring-health__mark.is-healthy { background: var(--color-success-soft); color: var(--color-success); } +.system-monitoring-health__mark.is-warning { background: var(--color-warning-soft); color: var(--color-warning); } +.system-monitoring-health__mark.is-critical { background: var(--color-danger-soft); color: var(--color-danger); } +.system-monitoring-health__mark.is-unknown { background: var(--color-surface-muted); color: var(--color-text-muted); } + +.system-monitoring-health__copy { + align-items: center; + display: grid; + grid-template-columns: 48px 1fr; + grid-template-rows: repeat(3, auto); +} + +.system-monitoring-health__copy .system-monitoring-health__mark { grid-row: 1 / 4; } +.system-monitoring-health__copy > span, +.system-monitoring-health__fact > span { + color: var(--color-text-muted); + font-size: 12px; +} + +.system-monitoring-health__copy > strong { color: var(--color-text-strong); font-size: 17px; } +.system-monitoring-health__copy > small, +.system-monitoring-health__fact > small { color: var(--color-text-subtle); font-size: 11px; } + +.system-monitoring-health__fact { + border-left: 1px solid var(--color-border); + display: grid; + gap: 3px; + padding-left: 20px; +} + +.system-monitoring-health__fact strong { + color: var(--color-text-strong); + font-size: 18px; +} + +.system-monitoring-metrics { + display: grid; + gap: 14px; + grid-template-columns: repeat(4, minmax(0, 1fr)); +} + +.system-monitoring-metric { + align-items: center; + display: flex; + gap: 14px; + min-width: 0; + padding: 18px; +} + +.system-monitoring-metric__icon { + align-items: center; + border-radius: var(--radius-md); + display: flex; + flex: 0 0 40px; + height: 40px; + justify-content: center; +} + +.system-monitoring-metric__icon.is-blue { background: var(--color-info-soft); color: var(--color-info); } +.system-monitoring-metric__icon.is-violet { background: #f5f3ff; color: #7c3aed; } +.system-monitoring-metric__icon.is-amber { background: var(--color-warning-soft); color: var(--color-warning); } +.system-monitoring-metric__icon.is-green { background: var(--color-success-soft); color: #0f766e; } + +.system-monitoring-metric > div:last-child { + display: grid; + gap: 2px; + min-width: 0; +} + +.system-monitoring-metric span, +.system-monitoring-metric small { color: var(--color-text-muted); font-size: 12px; } +.system-monitoring-metric strong { color: var(--color-text-strong); font-size: 23px; line-height: 1.25; } +.system-monitoring-metric small { overflow: hidden; text-overflow: ellipsis; white-space: nowrap; } + +.system-monitoring-main-grid { + align-items: start; + display: grid; + gap: 16px; + grid-template-columns: minmax(0, 1fr) 300px; +} + +.system-monitoring-chart-stack { + display: grid; + gap: 16px; + grid-template-columns: repeat(2, minmax(0, 1fr)); +} + +.system-monitoring-chart-card, +.system-monitoring-services, +.system-monitoring-alerts { padding: 18px; } + +.system-monitoring-chart-card header, +.system-monitoring-services > header, +.system-monitoring-alerts > header { + align-items: center; + display: flex; + justify-content: space-between; + margin-bottom: 12px; +} + +.system-monitoring-chart-card header div, +.system-monitoring-services > header div, +.system-monitoring-alerts > header div { + align-items: center; + color: var(--color-text-strong); + display: flex; + gap: 8px; +} + +.system-monitoring-chart-card header > span { color: var(--color-text); font-weight: var(--font-weight-semibold); } + +.system-monitoring-chart-empty { + align-items: center; + color: var(--color-text-subtle); + display: flex; + flex-direction: column; + gap: 8px; + height: 230px; + justify-content: center; +} + +.system-monitoring-service-list { display: grid; } + +.system-monitoring-service { + align-items: center; + border-bottom: 1px solid var(--color-border); + display: grid; + gap: 9px; + grid-template-columns: 9px 1fr auto; + padding: 13px 0; +} + +.system-monitoring-service:last-child { border-bottom: 0; } +.system-monitoring-service__dot { background: var(--color-text-subtle); border-radius: 50%; height: 8px; width: 8px; } +.system-monitoring-service__dot.is-healthy { background: var(--color-success); box-shadow: 0 0 0 3px var(--color-success-soft); } +.system-monitoring-service__dot.is-unhealthy { background: var(--color-danger); box-shadow: 0 0 0 3px var(--color-danger-soft); } +.system-monitoring-service__dot.is-unknown { background: var(--color-text-subtle); box-shadow: 0 0 0 3px var(--color-surface-muted); } +.system-monitoring-service div { display: grid; gap: 1px; min-width: 0; } +.system-monitoring-service strong { color: var(--color-text); font-size: 13px; } +.system-monitoring-service small { color: var(--color-text-subtle); font-size: 11px; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; } +.system-monitoring-service > span:last-child { color: var(--color-text-muted); font-size: 12px; } + +.system-monitoring-collector-note { + align-items: flex-start; + background: var(--color-bg-subtle); + border: 1px solid var(--color-border); + border-radius: var(--radius-md); + color: var(--color-text-muted); + display: flex; + font-size: 12px; + gap: 8px; + line-height: 1.55; + margin-top: 14px; + padding: 11px; +} + +.system-monitoring-collector-note svg { flex: 0 0 auto; margin-top: 1px; } + +.system-monitoring-alerts { overflow: hidden; } +.system-monitoring-alerts > header > span { align-items: center; color: var(--color-text-muted); display: flex; font-size: 12px; gap: 5px; } +.system-monitoring-alert-copy { display: grid; gap: 2px; } +.system-monitoring-alert-copy strong { color: var(--color-text-strong); } +.system-monitoring-alert-copy span { color: var(--color-text-muted); font-size: 12px; } + +.is-spinning { animation: system-monitoring-spin 0.9s linear infinite; } +@keyframes system-monitoring-spin { to { transform: rotate(360deg); } } + +@media (max-width: 1180px) { + .system-monitoring-health { grid-template-columns: repeat(3, minmax(0, 1fr)); row-gap: 18px; } + .system-monitoring-health__copy { grid-column: span 2; } + .system-monitoring-health__fact:nth-last-child(-n + 2) { border-left: 0; padding-left: 0; } + .system-monitoring-metrics { grid-template-columns: repeat(2, minmax(0, 1fr)); } + .system-monitoring-main-grid { grid-template-columns: minmax(0, 1fr); } + .system-monitoring-services { order: -1; } + .system-monitoring-service-list { grid-template-columns: repeat(3, minmax(0, 1fr)); } + .system-monitoring-service { border-bottom: 0; border-right: 1px solid var(--color-border); padding: 11px 12px; } + .system-monitoring-service:nth-child(3n) { border-right: 0; } +} + +@media (max-width: 760px) { + .system-monitoring-heading { align-items: stretch; flex-direction: column; } + .system-monitoring-controls { align-items: stretch; flex-direction: column; } + .system-monitoring-range { display: grid; grid-template-columns: repeat(3, 1fr); } + .system-monitoring-title-row { align-items: flex-start; justify-content: space-between; } + .system-monitoring-health { grid-template-columns: repeat(2, minmax(0, 1fr)); } + .system-monitoring-health__copy { grid-column: 1 / -1; } + .system-monitoring-health__fact { border-left: 0; border-top: 1px solid var(--color-border); padding: 12px 0 0; } + .system-monitoring-metrics, + .system-monitoring-chart-stack, + .system-monitoring-service-list { grid-template-columns: minmax(0, 1fr); } + .system-monitoring-service { border-bottom: 1px solid var(--color-border); border-right: 0; padding: 13px 0; } + .system-monitoring-chart-card { padding: 14px 10px; } +} + +@media (prefers-reduced-motion: reduce) { + .is-spinning { animation: none; } +} diff --git a/src/apps/admin/system-monitoring/AdminSystemMonitoringPage.tsx b/src/apps/admin/system-monitoring/AdminSystemMonitoringPage.tsx new file mode 100644 index 0000000..98f780d --- /dev/null +++ b/src/apps/admin/system-monitoring/AdminSystemMonitoringPage.tsx @@ -0,0 +1,316 @@ +import { useCallback, useEffect, useMemo, useRef, useState } from 'react'; +import type { EChartsOption } from 'echarts'; +import { + Activity, + AlertTriangle, + CheckCircle2, + Clock3, + Cpu, + Database, + HardDrive, + MemoryStick, + Network, + RefreshCw, + Server, + ShieldAlert, +} from 'lucide-react'; +import { + adminApi, + type InfrastructureAlert, + type InfrastructureMetricPoint, + type InfrastructureMonitoringOverview, + type InfrastructureMonitoringRange, +} from '@/api/adminApi'; +import { Breadcrumb, Button, Chart, Table, Tag, type TableColumn } from '@/components/ui'; +import './AdminSystemMonitoringPage.css'; + +const RANGE_OPTIONS: Array<{ value: InfrastructureMonitoringRange; label: string }> = [ + { value: '1h', label: '近1小时' }, + { value: '24h', label: '近24小时' }, + { value: '7d', label: '近7天' }, +]; + +const STATUS_COPY = { + healthy: { label: '运行正常', tone: 'success' as const }, + warning: { label: '需要关注', tone: 'warning' as const }, + critical: { label: '严重告警', tone: 'danger' as const }, + unknown: { label: '状态未知', tone: 'neutral' as const }, +}; + +function formatPercent(value: number | null) { + return value === null ? '—' : `${value.toFixed(1)}%`; +} + +function formatBytes(value: number | null) { + if (value === null) return '—'; + const units = ['B', 'KB', 'MB', 'GB', 'TB']; + let amount = value; + let index = 0; + while (amount >= 1024 && index < units.length - 1) { + amount /= 1024; + index += 1; + } + return `${amount.toFixed(index >= 3 ? 1 : 0)} ${units[index]}`; +} + +function formatRate(value: number | null) { + return value === null ? '—' : `${formatBytes(value)}/s`; +} + +function totalNetworkRate(receive: number | null | undefined, transmit: number | null | undefined) { + if (receive === null || receive === undefined || transmit === null || transmit === undefined) return null; + return receive + transmit; +} + +function formatUptime(value: number | null) { + if (value === null) return '—'; + const days = Math.floor(value / 86400); + const hours = Math.floor((value % 86400) / 3600); + return days > 0 ? `${days}天 ${hours}小时` : `${hours}小时`; +} + +function formatTime(value: string | null) { + if (!value) return '暂无采样'; + return new Intl.DateTimeFormat('zh-CN', { + month: '2-digit', day: '2-digit', hour: '2-digit', minute: '2-digit', second: '2-digit', hour12: false, + }).format(new Date(value)); +} + +function formatDuration(startedAt: string) { + const milliseconds = Date.now() - Date.parse(startedAt); + if (!Number.isFinite(milliseconds) || milliseconds < 0) return '—'; + const minutes = Math.floor(milliseconds / 60_000); + if (minutes < 60) return `${Math.max(minutes, 1)}分钟`; + const hours = Math.floor(minutes / 60); + return hours < 24 ? `${hours}小时 ${minutes % 60}分钟` : `${Math.floor(hours / 24)}天 ${hours % 24}小时`; +} + +function timeLabels(points: InfrastructureMetricPoint[], range: InfrastructureMonitoringRange) { + return points.map((point) => new Intl.DateTimeFormat('zh-CN', range === '7d' + ? { month: '2-digit', day: '2-digit', hour: '2-digit', hour12: false } + : { hour: '2-digit', minute: '2-digit', hour12: false }) + .format(new Date(point.timestamp))); +} + +function makeTrendOption(params: { + range: InfrastructureMonitoringRange; + series: Array<{ name: string; points: InfrastructureMetricPoint[]; color: string }>; + suffix: string; + maximum?: number; +}): EChartsOption { + const first = params.series[0]?.points ?? []; + return { + animationDuration: 280, + color: params.series.map((item) => item.color), + grid: { left: 8, right: 16, top: 34, bottom: 4, containLabel: true }, + legend: params.series.length > 1 ? { top: 0, right: 0, textStyle: { color: '#6b7280', fontSize: 12 } } : undefined, + tooltip: { + trigger: 'axis', + valueFormatter: (value) => `${Number(value).toFixed(1)}${params.suffix}`, + }, + xAxis: { + type: 'category', + boundaryGap: false, + data: timeLabels(first, params.range), + axisLine: { lineStyle: { color: '#e5e7eb' } }, + axisTick: { show: false }, + axisLabel: { color: '#9ca3af', hideOverlap: true, margin: 12 }, + }, + yAxis: { + type: 'value', min: 0, max: params.maximum, + axisLabel: { color: '#9ca3af', formatter: `{value}${params.suffix}` }, + splitLine: { lineStyle: { color: '#eef0f3' } }, + }, + series: params.series.map((item) => ({ + name: item.name, + data: item.points.map((point) => point.value), + type: 'line', + smooth: true, + showSymbol: false, + lineStyle: { width: 2.5 }, + areaStyle: { opacity: 0.07 }, + })), + }; +} + +function severityTag(severity: InfrastructureAlert['severity']) { + if (severity === 'critical') return 严重; + if (severity === 'warning') return 警告; + return 提示; +} + +const alertColumns: Array> = [ + { key: 'severity', title: '级别', width: '82px', render: (record) => severityTag(record.severity) }, + { + key: 'alert', title: '告警', width: '280px', render: (record) => ( +
{record.name}{record.summary}
+ ), + }, + { key: 'service', title: '服务 / 实例', width: '190px', render: (record) => record.service || record.instance || '主机资源' }, + { key: 'value', title: '当前值 / 阈值', width: '150px', render: (record) => `${record.currentValue || '—'} / ${record.threshold || '—'}` }, + { key: 'startedAt', title: '开始时间', width: '150px', render: (record) => formatTime(record.startedAt) }, + { key: 'duration', title: '持续时间', width: '120px', render: (record) => formatDuration(record.startedAt) }, +]; + +export function AdminSystemMonitoringPage() { + const [range, setRange] = useState('24h'); + const [overview, setOverview] = useState(null); + const [loading, setLoading] = useState(true); + const [error, setError] = useState(''); + const requestSequence = useRef(0); + const pendingRequests = useRef(0); + + const loadData = useCallback(async (supersede = false) => { + if (!supersede && pendingRequests.current > 0) return; + pendingRequests.current += 1; + const sequence = ++requestSequence.current; + setLoading(true); + try { + const result = await adminApi.getInfrastructureMonitoringOverview(range); + if (sequence !== requestSequence.current) return; + setOverview(result); + setError(result.available ? '' : result.error || '监控数据当前不可用'); + } catch (reason) { + if (sequence !== requestSequence.current) return; + setOverview(null); + setError(reason instanceof Error ? reason.message : '监控数据加载失败'); + } finally { + if (sequence === requestSequence.current) setLoading(false); + pendingRequests.current -= 1; + } + }, [range]); + + useEffect(() => { + void loadData(true); + const intervalId = window.setInterval(() => { + if (document.visibilityState === 'visible') void loadData(); + }, 30_000); + const handleVisibility = () => { + if (document.visibilityState === 'visible') void loadData(); + }; + document.addEventListener('visibilitychange', handleVisibility); + return () => { + requestSequence.current += 1; + window.clearInterval(intervalId); + document.removeEventListener('visibilitychange', handleVisibility); + }; + }, [loadData]); + + const status = STATUS_COPY[overview?.summary.overallStatus ?? 'unknown']; + const cpuOption = useMemo(() => makeTrendOption({ + range, maximum: 100, suffix: '%', series: [{ name: 'CPU', points: overview?.trends.cpuUsagePercent ?? [], color: '#2563eb' }], + }), [overview?.trends.cpuUsagePercent, range]); + const memoryOption = useMemo(() => makeTrendOption({ + range, maximum: 100, suffix: '%', series: [{ name: '内存', points: overview?.trends.memoryUsagePercent ?? [], color: '#7c3aed' }], + }), [overview?.trends.memoryUsagePercent, range]); + const diskOption = useMemo(() => makeTrendOption({ + range, maximum: 100, suffix: '%', series: [{ name: '根磁盘', points: overview?.trends.diskUsagePercent ?? [], color: '#d97706' }], + }), [overview?.trends.diskUsagePercent, range]); + const networkOption = useMemo(() => makeTrendOption({ + range, suffix: ' B/s', series: [ + { name: '接收', points: overview?.trends.networkReceiveBytesPerSecond ?? [], color: '#0f766e' }, + { name: '发送', points: overview?.trends.networkTransmitBytesPerSecond ?? [], color: '#2563eb' }, + ], + }), [overview?.trends.networkReceiveBytesPerSecond, overview?.trends.networkTransmitBytesPerSecond, range]); + + const metrics = overview?.metrics; + const serviceHealthy = overview?.summary.serviceHealthy ?? 0; + const serviceTotal = overview?.summary.serviceTotal ?? 6; + + return ( +
+
+
+ +
+
+

系统监控

+

服务器资源、核心服务与活动告警

+
+ {status.label} +
+
+
+
+ {RANGE_OPTIONS.map((option) => ( + + ))} +
+ +
+
+ + {error ? ( +
+ +
监控数据不可用{error}。页面不会展示历史缓存值。
+
+ ) : null} + +
+
+
+ {overview?.summary.overallStatus === 'healthy' ? : } +
+ 平台基础设施 + {status.label} + 最新采样 {formatTime(overview?.lastSampleAt ?? null)} +
+
核心服务{serviceHealthy}/{serviceTotal}正常运行
+
活动告警{overview?.summary.activeAlerts ?? 0}{overview?.summary.criticalAlerts ?? 0} 严重 · {overview?.summary.warningAlerts ?? 0} 警告
+
系统负载{metrics?.load1 === null || metrics?.load1 === undefined ? '—' : metrics.load1.toFixed(2)}最近1分钟
+
持续运行{formatUptime(metrics?.uptimeSeconds ?? null)}主机启动后
+
+ +
+
CPU 使用率{formatPercent(metrics?.cpuUsagePercent ?? null)}5分钟平均
+
内存使用率{formatPercent(metrics?.memoryUsagePercent ?? null)}{formatBytes(metrics?.memoryAvailableBytes ?? null)} 可用 / {formatBytes(metrics?.memoryTotalBytes ?? null)}
+
根磁盘使用率{formatPercent(metrics?.diskUsagePercent ?? null)}{formatBytes(metrics?.diskAvailableBytes ?? null)} 可用 / {formatBytes(metrics?.diskTotalBytes ?? null)}
+
网络吞吐{formatRate(totalNetworkRate(metrics?.networkReceiveBytesPerSecond, metrics?.networkTransmitBytesPerSecond))}接收 {formatRate(metrics?.networkReceiveBytesPerSecond ?? null)} · 发送 {formatRate(metrics?.networkTransmitBytesPerSecond ?? null)}
+
+ +
+
+
CPU 趋势
{formatPercent(metrics?.cpuUsagePercent ?? null)}
{overview?.trends.cpuUsagePercent.length ? : }
+
内存趋势
{formatPercent(metrics?.memoryUsagePercent ?? null)}
{overview?.trends.memoryUsagePercent.length ? : }
+
磁盘趋势
{formatPercent(metrics?.diskUsagePercent ?? null)}
{overview?.trends.diskUsagePercent.length ? : }
+
网络趋势
{formatRate(totalNetworkRate(metrics?.networkReceiveBytesPerSecond, metrics?.networkTransmitBytesPerSecond))}
{overview?.trends.networkReceiveBytesPerSecond.length ? : }
+
+ + +
+ +
+
活动告警{overview?.summary.activeAlerts ?? 0}
刷新于 {formatTime(overview?.collectedAt ?? null)}
+ + + + ); +} + +function EmptyChart() { + return
暂无真实趋势指标
; +} diff --git a/src/layouts/AdminLayout.tsx b/src/layouts/AdminLayout.tsx index 2383991..ca1f954 100644 --- a/src/layouts/AdminLayout.tsx +++ b/src/layouts/AdminLayout.tsx @@ -25,6 +25,7 @@ import { ReceiptText, ClipboardList, Send, + ServerCog, ScanSearch, Settings, Shield, @@ -202,6 +203,7 @@ function AdminAuthenticatedLayout({ session }: { session: LoginSession }) { { label: '报备字段库', to: '/admin/drainage-fields', icon: Hash }, { label: '引流识别规则', to: '/admin/drainage-detection-rules', icon: ScanSearch }, { label: '系统日志', to: '/admin/system-logs', icon: FileText }, + { label: '系统监控', to: '/admin/system-monitoring', icon: ServerCog }, ], }, ]} diff --git a/src/routes/AppRoutes.tsx b/src/routes/AppRoutes.tsx index cdd4a99..0a487d0 100644 --- a/src/routes/AppRoutes.tsx +++ b/src/routes/AppRoutes.tsx @@ -38,6 +38,7 @@ import { AdminSignatureAuditPage } from '@/apps/admin/AdminSignatureAuditPage'; import { AdminSignatureRetirementPage } from '@/apps/admin/AdminSignatureRetirementPage'; import { AdminDrainageAuditPage } from '@/apps/admin/AdminDrainageAuditPage'; import { AdminSystemLogsPage } from '@/apps/admin/AdminSystemLogsPage'; +import { AdminSystemMonitoringPage } from '@/apps/admin/system-monitoring/AdminSystemMonitoringPage'; import { AdminTemplateAuditPage } from '@/apps/admin/AdminTemplateAuditPage'; import { AdminUsersPage } from '@/apps/admin/AdminUsersPage'; import { AdminEnterpriseAuditPage } from '@/apps/admin/AdminEnterpriseAuditPage'; @@ -142,6 +143,7 @@ export function AppRoutes() { } /> } /> } /> + } /> } /> diff --git a/tools/deploy/production-bootstrap.sh b/tools/deploy/production-bootstrap.sh index 6d526d6..09e8296 100644 --- a/tools/deploy/production-bootstrap.sh +++ b/tools/deploy/production-bootstrap.sh @@ -31,6 +31,8 @@ OPERATION_LOG_ARCHIVE_INTERVAL_MS="${OPERATION_LOG_ARCHIVE_INTERVAL_MS:-86400000 SMS_RECEIPT_TIMEOUT_SCAN_ENABLED="${SMS_RECEIPT_TIMEOUT_SCAN_ENABLED:-true}" SMS_RECEIPT_TIMEOUT_HOURS="${SMS_RECEIPT_TIMEOUT_HOURS:-72}" SMS_RECEIPT_TIMEOUT_SCAN_INTERVAL_MS="${SMS_RECEIPT_TIMEOUT_SCAN_INTERVAL_MS:-300000}" +PROMETHEUS_URL="${PROMETHEUS_URL:-http://127.0.0.1:9090}" +PROMETHEUS_QUERY_TIMEOUT_MS="${PROMETHEUS_QUERY_TIMEOUT_MS:-5000}" if [[ "$(id -u)" -ne 0 ]]; then echo "Run as root." >&2 @@ -192,6 +194,8 @@ OPERATION_LOG_ARCHIVE_INTERVAL_MS=${OPERATION_LOG_ARCHIVE_INTERVAL_MS} SMS_RECEIPT_TIMEOUT_SCAN_ENABLED=${SMS_RECEIPT_TIMEOUT_SCAN_ENABLED} SMS_RECEIPT_TIMEOUT_HOURS=${SMS_RECEIPT_TIMEOUT_HOURS} SMS_RECEIPT_TIMEOUT_SCAN_INTERVAL_MS=${SMS_RECEIPT_TIMEOUT_SCAN_INTERVAL_MS} +PROMETHEUS_URL=${PROMETHEUS_URL} +PROMETHEUS_QUERY_TIMEOUT_MS=${PROMETHEUS_QUERY_TIMEOUT_MS} MINIO_ENDPOINT=127.0.0.1:9000 MINIO_ACCESS_KEY=${MINIO_ROOT_USER} MINIO_SECRET_KEY=${MINIO_ROOT_PASSWORD} diff --git a/tools/monitoring/README.md b/tools/monitoring/README.md new file mode 100644 index 0000000..818c3ff --- /dev/null +++ b/tools/monitoring/README.md @@ -0,0 +1,31 @@ +# CMPP Prometheus 监控配置 + +本目录提供系统监控第一版所需的真实采集配置。运营端不嵌入Prometheus或Grafana页面；NestJS仅从本机Prometheus读取固定指标，再由平台原生UI展示。 + +在目标Debian/Ubuntu测试服务器上，以root执行： + +```bash +cd /opt/cmpp-platform +bash tools/monitoring/install-prometheus-monitoring.sh +``` + +脚本会安装Prometheus与Node Exporter、备份已有Prometheus配置及本脚本曾写入的systemd override、校验规则、写入新override，并仅重启这两个监控服务。备份目录会在脚本结束时打印。它不会重启API、Gateway、数据库、Redis、MinIO或Nginx。9090和9100固定监听`127.0.0.1`。 + +安装后将下列配置写入`/etc/cmpp-platform/cmpp-platform.env`，再按正常发布窗口重启API： + +```dotenv +PROMETHEUS_URL=http://127.0.0.1:9090 +PROMETHEUS_QUERY_TIMEOUT_MS=5000 +``` + +校验命令： + +```bash +promtool check config /etc/prometheus/prometheus.yml +promtool check rules /etc/prometheus/cmpp-alerts.yml +curl -fsS http://127.0.0.1:9090/-/ready +curl -fsS 'http://127.0.0.1:9090/api/v1/query?query=up' +ss -lnt | grep -E ':(9090|9100)' +``` + +完整架构、PromQL口径、故障语义和验收标准见`docs/prometheus-system-monitoring-design-20260814.md`。 diff --git a/tools/monitoring/cmpp-alerts.yml b/tools/monitoring/cmpp-alerts.yml new file mode 100644 index 0000000..446db1d --- /dev/null +++ b/tools/monitoring/cmpp-alerts.yml @@ -0,0 +1,148 @@ +groups: + - name: cmpp-host-resources + rules: + - alert: NodeExporterDown + expr: up{job="node"} == 0 + for: 2m + labels: + severity: critical + service: node-exporter + annotations: + summary: 主机指标采集不可用 + description: Prometheus连续2分钟无法采集Node Exporter。 + currentValue: "{{ $value }}" + threshold: "up = 1" + + - alert: HostCpuUsageWarning + expr: (100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 85) and (100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) <= 95) + for: 10m + labels: + severity: warning + service: host + annotations: + summary: CPU使用率持续偏高 + description: 主机CPU使用率连续10分钟高于85%。 + currentValue: "{{ printf \"%.1f\" $value }}%" + threshold: "85%" + + - alert: HostCpuUsageCritical + expr: 100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 95 + for: 5m + labels: + severity: critical + service: host + annotations: + summary: CPU使用率严重超限 + description: 主机CPU使用率连续5分钟高于95%。 + currentValue: "{{ printf \"%.1f\" $value }}%" + threshold: "95%" + + - alert: HostMemoryUsageWarning + expr: ((1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100 > 85) and ((1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100 <= 95) + for: 10m + labels: + severity: warning + service: host + annotations: + summary: 内存使用率持续偏高 + description: 主机可用内存连续10分钟低于15%。 + currentValue: "{{ printf \"%.1f\" $value }}%" + threshold: "85%" + + - alert: HostMemoryUsageCritical + expr: (1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100 > 95 + for: 5m + labels: + severity: critical + service: host + annotations: + summary: 内存使用率严重超限 + description: 主机可用内存连续5分钟低于5%。 + currentValue: "{{ printf \"%.1f\" $value }}%" + threshold: "95%" + + - alert: HostRootDiskUsageWarning + expr: ((1 - node_filesystem_avail_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"} / node_filesystem_size_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"}) * 100 > 80) and ((1 - node_filesystem_avail_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"} / node_filesystem_size_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"}) * 100 <= 90) + for: 15m + labels: + severity: warning + service: host + annotations: + summary: 根文件系统空间不足 + description: 根文件系统使用率连续15分钟高于80%。 + currentValue: "{{ printf \"%.1f\" $value }}%" + threshold: "80%" + + - alert: HostRootDiskUsageCritical + expr: (1 - node_filesystem_avail_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"} / node_filesystem_size_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"}) * 100 > 90 + for: 5m + labels: + severity: critical + service: host + annotations: + summary: 根文件系统空间严重不足 + description: 根文件系统使用率连续5分钟高于90%。 + currentValue: "{{ printf \"%.1f\" $value }}%" + threshold: "90%" + + - alert: HostRootInodeUsageWarning + expr: ((1 - node_filesystem_files_free{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"} / node_filesystem_files{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"}) * 100 > 80) and ((1 - node_filesystem_files_free{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"} / node_filesystem_files{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"}) * 100 <= 90) + for: 15m + labels: + severity: warning + service: host + annotations: + summary: 根文件系统inode余量偏低 + description: 根文件系统inode使用率连续15分钟高于80%。 + currentValue: "{{ printf \"%.1f\" $value }}%" + threshold: "80%" + + - alert: HostRootInodeUsageCritical + expr: (1 - node_filesystem_files_free{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"} / node_filesystem_files{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"}) * 100 > 90 + for: 5m + labels: + severity: critical + service: host + annotations: + summary: 根文件系统inode严重不足 + description: 根文件系统inode使用率连续5分钟高于90%。 + currentValue: "{{ printf \"%.1f\" $value }}%" + threshold: "90%" + + - alert: HostCpuIowaitWarning + expr: (avg by (instance) (rate(node_cpu_seconds_total{mode="iowait"}[5m])) * 100 > 20) and (avg by (instance) (rate(node_cpu_seconds_total{mode="iowait"}[5m])) * 100 <= 35) + for: 10m + labels: + severity: warning + service: host + annotations: + summary: CPU iowait持续偏高 + description: 主机CPU iowait连续10分钟高于20%，请检查磁盘I/O。 + currentValue: "{{ printf \"%.1f\" $value }}%" + threshold: "20%" + + - alert: HostCpuIowaitCritical + expr: avg by (instance) (rate(node_cpu_seconds_total{mode="iowait"}[5m])) * 100 > 35 + for: 10m + labels: + severity: critical + service: host + annotations: + summary: CPU iowait严重超限 + description: 主机CPU iowait连续10分钟高于35%，磁盘I/O可能已成为瓶颈。 + currentValue: "{{ printf \"%.1f\" $value }}%" + threshold: "35%" + + - name: cmpp-core-services + rules: + - alert: CmppCoreServiceInactive + expr: node_systemd_unit_state{name=~"cmpp-api\\.service|cmpp-gateway\\.service|postgresql\\.service|redis(-server)?\\.service|cmpp-minio\\.service|nginx\\.service",state="active"} == 0 + for: 2m + labels: + severity: critical + service: "{{ $labels.name }}" + annotations: + summary: CMPP核心服务未处于active状态 + description: "systemd服务 {{ $labels.name }} 连续2分钟未处于active状态。" + currentValue: "{{ $value }}" + threshold: "active = 1" diff --git a/tools/monitoring/install-prometheus-monitoring.sh b/tools/monitoring/install-prometheus-monitoring.sh new file mode 100644 index 0000000..75264aa --- /dev/null +++ b/tools/monitoring/install-prometheus-monitoring.sh @@ -0,0 +1,82 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROMETHEUS_RETENTION_TIME="${PROMETHEUS_RETENTION_TIME:-30d}" +PROMETHEUS_RETENTION_SIZE="${PROMETHEUS_RETENTION_SIZE:-8GB}" + +if [[ "$(id -u)" -ne 0 ]]; then + echo "Run as root." >&2 + exit 1 +fi +if ! command -v apt-get >/dev/null 2>&1; then + echo "This installer currently supports Debian/Ubuntu apt packages only." >&2 + exit 1 +fi + +log() { printf '\n[%s] %s\n' "$(date '+%F %T')" "$*"; } + +log "Installing Prometheus and Node Exporter packages" +apt-get update +DEBIAN_FRONTEND=noninteractive apt-get install -y prometheus prometheus-node-exporter curl iproute2 + +prometheus_bin="$(command -v prometheus)" +node_exporter_bin="$(command -v prometheus-node-exporter)" +promtool_bin="$(command -v promtool)" +backup_dir="/etc/prometheus/cmpp-backups/$(date '+%Y%m%d-%H%M%S')" +mkdir -p "$backup_dir" /etc/systemd/system/prometheus.service.d /etc/systemd/system/prometheus-node-exporter.service.d + +for config_file in /etc/prometheus/prometheus.yml /etc/prometheus/cmpp-alerts.yml; do + if [[ -f "$config_file" ]]; then + cp --preserve=mode,timestamps "$config_file" "$backup_dir/$(basename "$config_file")" + fi +done +if [[ -f /etc/systemd/system/prometheus.service.d/cmpp-monitoring.conf ]]; then + cp --preserve=mode,timestamps /etc/systemd/system/prometheus.service.d/cmpp-monitoring.conf "$backup_dir/prometheus-service-override.conf" +fi +if [[ -f /etc/systemd/system/prometheus-node-exporter.service.d/cmpp-monitoring.conf ]]; then + cp --preserve=mode,timestamps /etc/systemd/system/prometheus-node-exporter.service.d/cmpp-monitoring.conf "$backup_dir/node-exporter-service-override.conf" +fi + +log "Installing platform-owned scrape and alert configuration" +install -o root -g root -m 0644 "$SCRIPT_DIR/prometheus.yml" /etc/prometheus/prometheus.yml +install -o root -g root -m 0644 "$SCRIPT_DIR/cmpp-alerts.yml" /etc/prometheus/cmpp-alerts.yml + +cat >/etc/systemd/system/prometheus.service.d/cmpp-monitoring.conf </etc/systemd/system/prometheus-node-exporter.service.d/cmpp-monitoring.conf </dev/null; then + break + fi + sleep 1 +done +curl -fsS http://127.0.0.1:9090/-/ready >/dev/null +curl -fsS http://127.0.0.1:9100/metrics >/dev/null + +if ss -lnt | grep -Eq '(^|[[:space:]])(0\.0\.0\.0|\[::\]):(9090|9100)([[:space:]]|$)'; then + echo "Prometheus monitoring ports unexpectedly listen on a wildcard address." >&2 + exit 1 +fi + +log "Prometheus monitoring is ready on loopback only" +echo "Configuration backup: $backup_dir" +echo "The CMPP API and Gateway were not restarted." diff --git a/tools/monitoring/prometheus.yml b/tools/monitoring/prometheus.yml new file mode 100644 index 0000000..211f521 --- /dev/null +++ b/tools/monitoring/prometheus.yml @@ -0,0 +1,20 @@ +global: + scrape_interval: 15s + evaluation_interval: 15s + external_labels: + platform: cmpp + environment: preproduction + +rule_files: + - /etc/prometheus/cmpp-alerts.yml + +scrape_configs: + - job_name: prometheus + static_configs: + - targets: [127.0.0.1:9090] + + - job_name: node + static_configs: + - targets: [127.0.0.1:9100] + labels: + host: cmpp-primary