feat: 完善服务监控与下游重投

api/src/infrastructure-monitoring/infrastructure-monitoring.contracts.ts

@@ -12,6 +12,18 @@ export type InfrastructureServiceStatus = {
   status: 'healthy' | 'unhealthy' | 'unknown';
 };
 
+export type InfrastructureServiceMetricGroup = {
+  key: string;
+  name: string;
+  available: boolean;
+  metrics: Array<{
+    key: string;
+    label: string;
+    value: number | null;
+    unit: 'percent' | 'seconds' | 'count' | 'per_second' | 'bytes';
+  }>;
+};
+
 export type InfrastructureAlert = {
   fingerprint: string;
   name: string;
@@ -61,5 +73,6 @@ export type InfrastructureMonitoringOverview = {
     networkTransmitBytesPerSecond: InfrastructureMetricPoint[];
   };
   services: InfrastructureServiceStatus[];
+  serviceMetrics: InfrastructureServiceMetricGroup[];
   alerts: InfrastructureAlert[];
 };

api/src/infrastructure-monitoring/infrastructure-monitoring.service.spec.ts

@@ -55,6 +55,13 @@ describe('InfrastructureMonitoringService', () => {
           { metric: { name: 'nginx.service' }, value: [1_765_000_060, '1'] },
         ] });
       }
+      if (query.includes('cmpp:service_.*')) {
+        return success({ result: [
+          { metric: { __name__: 'cmpp:service_api:requests_per_second' }, value: [1_765_000_060, '12.5'] },
+          { metric: { __name__: 'cmpp:service_api:error_percent' }, value: [1_765_000_060, '0.2'] },
+          { metric: { __name__: 'cmpp:service_gateway:queue_pending' }, value: [1_765_000_060, '3'] },
+        ] });
+      }
       if (query.includes('timestamp(node_uname_info)')) {
         return success({ result: [{ metric: {}, value: [1_765_000_060, '1765000060'] }] });
       }
@@ -69,9 +76,13 @@ describe('InfrastructureMonitoringService', () => {
     expect(result.trends.cpuUsagePercent).toHaveLength(2);
     expect(result.summary).toMatchObject({ overallStatus: 'warning', serviceHealthy: 6, warningAlerts: 1 });
     expect(result.services.find((item) => item.key === 'redis')).toMatchObject({ unit: 'redis-server.service', status: 'healthy' });
+    expect(result.serviceMetrics.find((item) => item.key === 'api')).toMatchObject({ available: true });
+    expect(result.serviceMetrics.find((item) => item.key === 'gateway')?.metrics.find((item) => item.key === 'queuePending')?.value).toBe(3);
     expect(result.alerts[0]).toMatchObject({ name: 'HostCpuHigh', severity: 'warning', currentValue: '88.2' });
     expect(requestedUrls.filter((url) => url.pathname.endsWith('/query_range'))).toHaveLength(5);
     expect(requestedUrls.filter((url) => url.pathname.endsWith('/query_range')).every((url) => url.searchParams.get('step') === '60')).toBe(true);
+    expect(requestedUrls.find((url) => url.searchParams.get('query')?.includes('node_systemd_unit_state'))?.searchParams.get('query'))
+      .toContain('cmpp-api\\\\.service');
   });
 
   it('returns an explicit unavailable payload without stale metrics when Prometheus fails', async () => {
@@ -84,6 +95,7 @@ describe('InfrastructureMonitoringService', () => {
     expect(result.summary.overallStatus).toBe('unknown');
     expect(result.metrics.cpuUsagePercent).toBeNull();
     expect(result.trends.cpuUsagePercent).toEqual([]);
+    expect(result.serviceMetrics.every((item) => item.available === false)).toBe(true);
     expect(result.error).not.toContain('ECONNREFUSED');
   });
 });

api/src/infrastructure-monitoring/infrastructure-monitoring.service.ts

@@ -1,4 +1,4 @@
-import { BadRequestException, Injectable } from '@nestjs/common';
+import { BadRequestException, Injectable, Logger } from '@nestjs/common';
 import { ConfigService } from '@nestjs/config';
 import { createHash } from 'node:crypto';
 import type {
@@ -7,6 +7,7 @@ import type {
   InfrastructureMonitoringOverview,
   InfrastructureMonitoringRange,
   InfrastructureServiceStatus,
+  InfrastructureServiceMetricGroup,
 } from './infrastructure-monitoring.contracts';
 
 type PrometheusSample = [number, string];
@@ -52,7 +53,8 @@ const QUERIES = {
   load1: 'node_load1',
   uptimeSeconds: 'time() - node_boot_time_seconds',
   lastSampleAt: 'max(timestamp(node_uname_info))',
-  services: 'max by (name) (node_systemd_unit_state{name=~"cmpp-api\\.service|cmpp-gateway\\.service|postgresql\\.service|redis(-server)?\\.service|cmpp-minio\\.service|nginx\\.service",state="active"})',
+  // PromQL字符串本身需要两个反斜杠才能把正则的“\.”传给RE2；TypeScript字面量因此需要写四个。
+  services: 'max by (name) (node_systemd_unit_state{name=~"cmpp-api\\\\.service|cmpp-gateway\\\\.service|postgresql\\\\.service|redis(-server)?\\\\.service|cmpp-minio\\\\.service|nginx\\\\.service",state="active"})',
 } as const;
 
 const SERVICE_DEFINITIONS = [
@@ -64,6 +66,43 @@ const SERVICE_DEFINITIONS = [
   { key: 'nginx', name: 'Nginx', units: ['nginx.service'] },
 ] as const;
 
+const SERVICE_METRIC_DEFINITIONS = [
+  { key: 'api', name: 'API服务', metrics: [
+    ['requestsPerSecond', '请求速率', 'cmpp:service_api:requests_per_second', 'per_second'],
+    ['errorPercent', '5xx错误率', 'cmpp:service_api:error_percent', 'percent'],
+    ['latencyP95', 'P95响应', 'cmpp:service_api:latency_p95_seconds', 'seconds'],
+    ['eventLoopP99', '事件循环P99', 'cmpp:service_api:event_loop_p99_seconds', 'seconds'],
+  ] },
+  { key: 'gateway', name: 'Gateway服务', metrics: [
+    ['submitsPerSecond', '提交速率', 'cmpp:service_gateway:submits_per_second', 'per_second'],
+    ['failurePercent', '提交失败率', 'cmpp:service_gateway:failure_percent', 'percent'],
+    ['queuePending', 'Stream pending', 'cmpp:service_gateway:queue_pending', 'count'],
+    ['queueOldestSeconds', '最旧pending', 'cmpp:service_gateway:queue_oldest_seconds', 'seconds'],
+  ] },
+  { key: 'postgresql', name: 'PostgreSQL', metrics: [
+    ['connectionPercent', '连接使用率', 'cmpp:service_postgresql:connection_percent', 'percent'],
+    ['deadlocks15m', '15分钟死锁', 'cmpp:service_postgresql:deadlocks_15m', 'count'],
+  ] },
+  { key: 'redis', name: 'Redis', metrics: [
+    ['memoryPercent', '内存使用率', 'cmpp:service_redis:memory_percent', 'percent'],
+    ['memoryUsedBytes', '已用内存', 'cmpp:service_redis:memory_used_bytes', 'bytes'],
+    ['connectedClients', '客户端连接', 'cmpp:service_redis:connected_clients', 'count'],
+    ['evictions5m', '5分钟淘汰', 'cmpp:service_redis:evictions_5m', 'count'],
+  ] },
+  { key: 'minio', name: 'MinIO', metrics: [
+    ['capacityPercent', '存储容量使用率', 'cmpp:service_minio:capacity_percent', 'percent'],
+    ['usageBytes', '对象数据量', 'cmpp:service_minio:usage_bytes', 'bytes'],
+    ['objects', '对象数', 'cmpp:service_minio:objects', 'count'],
+    ['drivesOffline', '离线存储盘', 'cmpp:service_minio:drives_offline', 'count'],
+  ] },
+  { key: 'nginx', name: 'Nginx', metrics: [
+    ['connectionsActive', '活跃连接', 'cmpp:service_nginx:connections_active', 'count'],
+    ['requestsPerSecond', '请求速率', 'cmpp:service_nginx:requests_per_second', 'per_second'],
+  ] },
+] as const;
+
+const SERVICE_METRICS_QUERY = '{__name__=~"cmpp:service_.*"}';
+
 function finiteNumber(value: string | number | undefined): number | null {
   const parsed = Number(value);
   return Number.isFinite(parsed) ? parsed : null;
@@ -119,6 +158,7 @@ function emptyTrends(): InfrastructureMonitoringOverview['trends'] {
 
 @Injectable()
 export class InfrastructureMonitoringService {
+  private readonly logger = new Logger(InfrastructureMonitoringService.name);
   private readonly prometheusUrl: string;
   private readonly queryTimeoutMs: number;
 
@@ -131,13 +171,15 @@ export class InfrastructureMonitoringService {
     const range = this.parseRange(rawRange);
     const collectedAt = new Date().toISOString();
     try {
-      const [instant, trends, serviceResponse, alertResponse] = await Promise.all([
+      const [instant, trends, serviceResponse, serviceMetricResponse, alertResponse] = await Promise.all([
         this.loadInstantMetrics(),
         this.loadTrends(range),
         this.query(QUERIES.services),
+        this.query(SERVICE_METRICS_QUERY),
         this.getJson<PrometheusAlertResponse>('/api/v1/alerts'),
       ]);
       const services = this.parseServices(serviceResponse);
+      const serviceMetrics = this.parseServiceMetrics(serviceMetricResponse);
       const alerts = this.parseAlerts(alertResponse);
       const warningAlerts = alerts.filter((item) => item.severity === 'warning').length;
       const criticalAlerts = alerts.filter((item) => item.severity === 'critical').length;
@@ -158,9 +200,12 @@ export class InfrastructureMonitoringService {
         metrics: instant.metrics,
         trends,
         services,
+        serviceMetrics,
         alerts,
       };
-    } catch {
+    } catch (error) {
+      // 页面必须整体清空陈旧指标，但服务端仍需留下不含PromQL/地址/凭据的根因摘要便于运维诊断。
+      this.logger.warn(`Prometheus monitoring overview unavailable: ${error instanceof Error ? error.message : 'unknown error'}`);
       return this.unavailable(range, collectedAt);
     }
   }
@@ -228,6 +273,21 @@ export class InfrastructureMonitoringService {
       });
   }
 
+  private parseServiceMetrics(response: PrometheusQueryResponse): InfrastructureServiceMetricGroup[] {
+    const values = new Map<string, number>();
+    for (const item of response.data?.result ?? []) {
+      const metricName = item.metric.__name__;
+      const value = vectorValue({ status: 'success', data: { result: [item] } });
+      if (metricName && value !== null) values.set(metricName, value);
+    }
+    return SERVICE_METRIC_DEFINITIONS.map((group) => ({
+      key: group.key,
+      name: group.name,
+      available: group.metrics.some((metric) => values.has(metric[2])),
+      metrics: group.metrics.map(([key, label, metricName, unit]) => ({ key, label, value: values.get(metricName) ?? null, unit })),
+    }));
+  }
+
   private unavailable(range: InfrastructureMonitoringRange, collectedAt: string): InfrastructureMonitoringOverview {
     const services = SERVICE_DEFINITIONS.map((item) => ({ key: item.key, name: item.name, unit: item.units[0], status: 'unknown' as const }));
     return {
@@ -240,6 +300,7 @@ export class InfrastructureMonitoringService {
       metrics: emptyMetrics(),
       trends: emptyTrends(),
       services,
+      serviceMetrics: SERVICE_METRIC_DEFINITIONS.map((group) => ({ key: group.key, name: group.name, available: false, metrics: [] })),
       alerts: [],
     };
   }