feat: 完善服务监控与下游重投

2026-08-14 17:21:29 +08:00
parent d30d9ea4d0
commit 1ef4380422
50 changed files with 1279 additions and 82 deletions
@@ -0,0 +1,24 @@
+import { CallHandler, ExecutionContext, Injectable, NestInterceptor } from '@nestjs/common';
+import type { Observable } from 'rxjs';
+import { finalize } from 'rxjs/operators';
+import { MetricsService } from './metrics.service';
+
+type RequestLike = { method?: string; baseUrl?: string; route?: { path?: string } };
+type ResponseLike = { statusCode?: number };
+
+@Injectable()
+export class MetricsInterceptor implements NestInterceptor {
+  constructor(private readonly metrics: MetricsService) {}
+
+  intercept(context: ExecutionContext, next: CallHandler): Observable<unknown> {
+    if (context.getType() !== 'http') return next.handle();
+    const http = context.switchToHttp();
+    const request = http.getRequest<RequestLike>();
+    const response = http.getResponse<ResponseLike>();
+    const startedAt = this.metrics.beginRequest();
+    return next.handle().pipe(finalize(() => {
+      const route = `${request.baseUrl ?? ''}${request.route?.path ?? '/unmatched'}`;
+      this.metrics.finishRequest(startedAt, request.method ?? 'UNKNOWN', route, response.statusCode ?? 500);
+    }));
+  }
+}
@@ -0,0 +1,11 @@
+import { Global, Module } from '@nestjs/common';
+import { APP_INTERCEPTOR } from '@nestjs/core';
+import { MetricsInterceptor } from './metrics.interceptor';
+import { MetricsService } from './metrics.service';
+
+@Global()
+@Module({
+  providers: [MetricsService, { provide: APP_INTERCEPTOR, useClass: MetricsInterceptor }],
+  exports: [MetricsService],
+})
+export class MetricsModule {}
@@ -0,0 +1,16 @@
+import { MetricsService } from './metrics.service';
+
+describe('MetricsService', () => {
+  it('exports bounded API process and HTTP metrics without raw identifiers', () => {
+    const service = new MetricsService();
+    const startedAt = service.beginRequest();
+    service.finishRequest(startedAt, 'GET', '/api/admin/tenants/:id', 200);
+    const output = service.render();
+
+    expect(output).toContain('cmpp_api_process_resident_memory_bytes');
+    expect(output).toContain('cmpp_api_http_requests_total{method="GET",route="/api/admin/tenants/:id",status="200"} 1');
+    expect(output).toContain('cmpp_api_http_request_duration_seconds_bucket');
+    expect(output).not.toContain('phone_number');
+    service.onModuleDestroy();
+  });
+});
@@ -0,0 +1,100 @@
+import { Injectable, OnModuleDestroy } from '@nestjs/common';
+import { monitorEventLoopDelay } from 'node:perf_hooks';
+
+const HTTP_DURATION_BUCKETS = [0.05, 0.1, 0.25, 0.5, 1, 3, 10] as const;
+
+type HttpMetric = {
+  count: number;
+  durationSum: number;
+  buckets: number[];
+};
+
+function escapeLabel(value: string) {
+  return value.replace(/\\/g, '\\\\').replace(/\n/g, '\\n').replace(/"/g, '\\"');
+}
+
+function metricLine(name: string, value: number, labels?: Record<string, string>) {
+  const suffix = labels
+    ? `{${Object.entries(labels).map(([key, item]) => `${key}="${escapeLabel(item)}"`).join(',')}}`
+    : '';
+  return `${name}${suffix} ${Number.isFinite(value) ? value : 0}`;
+}
+
+@Injectable()
+export class MetricsService implements OnModuleDestroy {
+  private readonly startedAt = process.hrtime.bigint();
+  private readonly eventLoopDelay = monitorEventLoopDelay({ resolution: 20 });
+  private readonly http = new Map<string, HttpMetric>();
+  private inFlight = 0;
+
+  constructor() {
+    this.eventLoopDelay.enable();
+  }
+
+  beginRequest() {
+    this.inFlight += 1;
+    return process.hrtime.bigint();
+  }
+
+  finishRequest(startedAt: bigint, method: string, route: string, statusCode: number) {
+    this.inFlight = Math.max(0, this.inFlight - 1);
+    // Only route templates enter labels. Raw URLs, IDs, phone numbers and query strings would create unbounded time series.
+    const normalizedRoute = route.startsWith('/') ? route : `/${route}`;
+    const labels = [method.toUpperCase(), normalizedRoute, String(statusCode)];
+    const key = labels.join('\u0000');
+    const metric = this.http.get(key) ?? { count: 0, durationSum: 0, buckets: HTTP_DURATION_BUCKETS.map(() => 0) };
+    const durationSeconds = Number(process.hrtime.bigint() - startedAt) / 1_000_000_000;
+    metric.count += 1;
+    metric.durationSum += durationSeconds;
+    HTTP_DURATION_BUCKETS.forEach((bucket, index) => {
+      if (durationSeconds <= bucket) metric.buckets[index] += 1;
+    });
+    this.http.set(key, metric);
+  }
+
+  render() {
+    const memory = process.memoryUsage();
+    const uptime = Number(process.hrtime.bigint() - this.startedAt) / 1_000_000_000;
+    const lines = [
+      '# HELP cmpp_api_process_uptime_seconds API process uptime.',
+      '# TYPE cmpp_api_process_uptime_seconds gauge',
+      metricLine('cmpp_api_process_uptime_seconds', uptime),
+      '# HELP cmpp_api_process_resident_memory_bytes API resident memory.',
+      '# TYPE cmpp_api_process_resident_memory_bytes gauge',
+      metricLine('cmpp_api_process_resident_memory_bytes', memory.rss),
+      '# HELP cmpp_api_nodejs_heap_used_bytes Node.js heap currently used.',
+      '# TYPE cmpp_api_nodejs_heap_used_bytes gauge',
+      metricLine('cmpp_api_nodejs_heap_used_bytes', memory.heapUsed),
+      '# HELP cmpp_api_nodejs_heap_total_bytes Node.js allocated heap.',
+      '# TYPE cmpp_api_nodejs_heap_total_bytes gauge',
+      metricLine('cmpp_api_nodejs_heap_total_bytes', memory.heapTotal),
+      '# HELP cmpp_api_nodejs_event_loop_lag_p99_seconds Event loop delay p99 since the previous scrape.',
+      '# TYPE cmpp_api_nodejs_event_loop_lag_p99_seconds gauge',
+      metricLine('cmpp_api_nodejs_event_loop_lag_p99_seconds', this.eventLoopDelay.count ? this.eventLoopDelay.percentile(99) / 1_000_000_000 : 0),
+      '# HELP cmpp_api_http_requests_in_flight Current API requests in flight.',
+      '# TYPE cmpp_api_http_requests_in_flight gauge',
+      metricLine('cmpp_api_http_requests_in_flight', this.inFlight),
+      '# HELP cmpp_api_http_requests_total API requests grouped by bounded route templates.',
+      '# TYPE cmpp_api_http_requests_total counter',
+      '# HELP cmpp_api_http_request_duration_seconds API request duration.',
+      '# TYPE cmpp_api_http_request_duration_seconds histogram',
+    ];
+    for (const [key, metric] of this.http) {
+      const [method, route, status] = key.split('\u0000');
+      const labels = { method, route, status };
+      lines.push(metricLine('cmpp_api_http_requests_total', metric.count, labels));
+      HTTP_DURATION_BUCKETS.forEach((bucket, index) => {
+        lines.push(metricLine('cmpp_api_http_request_duration_seconds_bucket', metric.buckets[index], { ...labels, le: String(bucket) }));
+      });
+      lines.push(metricLine('cmpp_api_http_request_duration_seconds_bucket', metric.count, { ...labels, le: '+Inf' }));
+      lines.push(metricLine('cmpp_api_http_request_duration_seconds_sum', metric.durationSum, labels));
+      lines.push(metricLine('cmpp_api_http_request_duration_seconds_count', metric.count, labels));
+    }
+    this.eventLoopDelay.reset();
+    return `${lines.join('\n')}\n`;
+  }
+
+  onModuleDestroy() {
+    this.eventLoopDelay.disable();
+  }
+}