feat: 完善服务监控与下游重投

api/src/infrastructure-monitoring/infrastructure-monitoring.service.ts

@@ -1,4 +1,4 @@
-import { BadRequestException, Injectable } from '@nestjs/common';
+import { BadRequestException, Injectable, Logger } from '@nestjs/common';
 import { ConfigService } from '@nestjs/config';
 import { createHash } from 'node:crypto';
 import type {
@@ -7,6 +7,7 @@ import type {
   InfrastructureMonitoringOverview,
   InfrastructureMonitoringRange,
   InfrastructureServiceStatus,
+  InfrastructureServiceMetricGroup,
 } from './infrastructure-monitoring.contracts';
 
 type PrometheusSample = [number, string];
@@ -52,7 +53,8 @@ const QUERIES = {
   load1: 'node_load1',
   uptimeSeconds: 'time() - node_boot_time_seconds',
   lastSampleAt: 'max(timestamp(node_uname_info))',
-  services: 'max by (name) (node_systemd_unit_state{name=~"cmpp-api\\.service|cmpp-gateway\\.service|postgresql\\.service|redis(-server)?\\.service|cmpp-minio\\.service|nginx\\.service",state="active"})',
+  // PromQL字符串本身需要两个反斜杠才能把正则的“\.”传给RE2；TypeScript字面量因此需要写四个。
+  services: 'max by (name) (node_systemd_unit_state{name=~"cmpp-api\\\\.service|cmpp-gateway\\\\.service|postgresql\\\\.service|redis(-server)?\\\\.service|cmpp-minio\\\\.service|nginx\\\\.service",state="active"})',
 } as const;
 
 const SERVICE_DEFINITIONS = [
@@ -64,6 +66,43 @@ const SERVICE_DEFINITIONS = [
   { key: 'nginx', name: 'Nginx', units: ['nginx.service'] },
 ] as const;
 
+const SERVICE_METRIC_DEFINITIONS = [
+  { key: 'api', name: 'API服务', metrics: [
+    ['requestsPerSecond', '请求速率', 'cmpp:service_api:requests_per_second', 'per_second'],
+    ['errorPercent', '5xx错误率', 'cmpp:service_api:error_percent', 'percent'],
+    ['latencyP95', 'P95响应', 'cmpp:service_api:latency_p95_seconds', 'seconds'],
+    ['eventLoopP99', '事件循环P99', 'cmpp:service_api:event_loop_p99_seconds', 'seconds'],
+  ] },
+  { key: 'gateway', name: 'Gateway服务', metrics: [
+    ['submitsPerSecond', '提交速率', 'cmpp:service_gateway:submits_per_second', 'per_second'],
+    ['failurePercent', '提交失败率', 'cmpp:service_gateway:failure_percent', 'percent'],
+    ['queuePending', 'Stream pending', 'cmpp:service_gateway:queue_pending', 'count'],
+    ['queueOldestSeconds', '最旧pending', 'cmpp:service_gateway:queue_oldest_seconds', 'seconds'],
+  ] },
+  { key: 'postgresql', name: 'PostgreSQL', metrics: [
+    ['connectionPercent', '连接使用率', 'cmpp:service_postgresql:connection_percent', 'percent'],
+    ['deadlocks15m', '15分钟死锁', 'cmpp:service_postgresql:deadlocks_15m', 'count'],
+  ] },
+  { key: 'redis', name: 'Redis', metrics: [
+    ['memoryPercent', '内存使用率', 'cmpp:service_redis:memory_percent', 'percent'],
+    ['memoryUsedBytes', '已用内存', 'cmpp:service_redis:memory_used_bytes', 'bytes'],
+    ['connectedClients', '客户端连接', 'cmpp:service_redis:connected_clients', 'count'],
+    ['evictions5m', '5分钟淘汰', 'cmpp:service_redis:evictions_5m', 'count'],
+  ] },
+  { key: 'minio', name: 'MinIO', metrics: [
+    ['capacityPercent', '存储容量使用率', 'cmpp:service_minio:capacity_percent', 'percent'],
+    ['usageBytes', '对象数据量', 'cmpp:service_minio:usage_bytes', 'bytes'],
+    ['objects', '对象数', 'cmpp:service_minio:objects', 'count'],
+    ['drivesOffline', '离线存储盘', 'cmpp:service_minio:drives_offline', 'count'],
+  ] },
+  { key: 'nginx', name: 'Nginx', metrics: [
+    ['connectionsActive', '活跃连接', 'cmpp:service_nginx:connections_active', 'count'],
+    ['requestsPerSecond', '请求速率', 'cmpp:service_nginx:requests_per_second', 'per_second'],
+  ] },
+] as const;
+
+const SERVICE_METRICS_QUERY = '{__name__=~"cmpp:service_.*"}';
+
 function finiteNumber(value: string | number | undefined): number | null {
   const parsed = Number(value);
   return Number.isFinite(parsed) ? parsed : null;
@@ -119,6 +158,7 @@ function emptyTrends(): InfrastructureMonitoringOverview['trends'] {
 
 @Injectable()
 export class InfrastructureMonitoringService {
+  private readonly logger = new Logger(InfrastructureMonitoringService.name);
   private readonly prometheusUrl: string;
   private readonly queryTimeoutMs: number;
 
@@ -131,13 +171,15 @@ export class InfrastructureMonitoringService {
     const range = this.parseRange(rawRange);
     const collectedAt = new Date().toISOString();
     try {
-      const [instant, trends, serviceResponse, alertResponse] = await Promise.all([
+      const [instant, trends, serviceResponse, serviceMetricResponse, alertResponse] = await Promise.all([
         this.loadInstantMetrics(),
         this.loadTrends(range),
         this.query(QUERIES.services),
+        this.query(SERVICE_METRICS_QUERY),
         this.getJson<PrometheusAlertResponse>('/api/v1/alerts'),
       ]);
       const services = this.parseServices(serviceResponse);
+      const serviceMetrics = this.parseServiceMetrics(serviceMetricResponse);
       const alerts = this.parseAlerts(alertResponse);
       const warningAlerts = alerts.filter((item) => item.severity === 'warning').length;
       const criticalAlerts = alerts.filter((item) => item.severity === 'critical').length;
@@ -158,9 +200,12 @@ export class InfrastructureMonitoringService {
         metrics: instant.metrics,
         trends,
         services,
+        serviceMetrics,
         alerts,
       };
-    } catch {
+    } catch (error) {
+      // 页面必须整体清空陈旧指标，但服务端仍需留下不含PromQL/地址/凭据的根因摘要便于运维诊断。
+      this.logger.warn(`Prometheus monitoring overview unavailable: ${error instanceof Error ? error.message : 'unknown error'}`);
       return this.unavailable(range, collectedAt);
     }
   }
@@ -228,6 +273,21 @@ export class InfrastructureMonitoringService {
       });
   }
 
+  private parseServiceMetrics(response: PrometheusQueryResponse): InfrastructureServiceMetricGroup[] {
+    const values = new Map<string, number>();
+    for (const item of response.data?.result ?? []) {
+      const metricName = item.metric.__name__;
+      const value = vectorValue({ status: 'success', data: { result: [item] } });
+      if (metricName && value !== null) values.set(metricName, value);
+    }
+    return SERVICE_METRIC_DEFINITIONS.map((group) => ({
+      key: group.key,
+      name: group.name,
+      available: group.metrics.some((metric) => values.has(metric[2])),
+      metrics: group.metrics.map(([key, label, metricName, unit]) => ({ key, label, value: values.get(metricName) ?? null, unit })),
+    }));
+  }
+
   private unavailable(range: InfrastructureMonitoringRange, collectedAt: string): InfrastructureMonitoringOverview {
     const services = SERVICE_DEFINITIONS.map((item) => ({ key: item.key, name: item.name, unit: item.units[0], status: 'unknown' as const }));
     return {
@@ -240,6 +300,7 @@ export class InfrastructureMonitoringService {
       metrics: emptyMetrics(),
       trends: emptyTrends(),
       services,
+      serviceMetrics: SERVICE_METRIC_DEFINITIONS.map((group) => ({ key: group.key, name: group.name, available: false, metrics: [] })),
       alerts: [],
     };
   }