feat: 完善服务监控与下游重投

2026-08-14 17:21:29 +08:00
parent d30d9ea4d0
commit 1ef4380422
50 changed files with 1279 additions and 82 deletions
@@ -0,0 +1,100 @@
+import { Injectable, OnModuleDestroy } from '@nestjs/common';
+import { monitorEventLoopDelay } from 'node:perf_hooks';
+
+const HTTP_DURATION_BUCKETS = [0.05, 0.1, 0.25, 0.5, 1, 3, 10] as const;
+
+type HttpMetric = {
+  count: number;
+  durationSum: number;
+  buckets: number[];
+};
+
+function escapeLabel(value: string) {
+  return value.replace(/\\/g, '\\\\').replace(/\n/g, '\\n').replace(/"/g, '\\"');
+}
+
+function metricLine(name: string, value: number, labels?: Record<string, string>) {
+  const suffix = labels
+    ? `{${Object.entries(labels).map(([key, item]) => `${key}="${escapeLabel(item)}"`).join(',')}}`
+    : '';
+  return `${name}${suffix} ${Number.isFinite(value) ? value : 0}`;
+}
+
+@Injectable()
+export class MetricsService implements OnModuleDestroy {
+  private readonly startedAt = process.hrtime.bigint();
+  private readonly eventLoopDelay = monitorEventLoopDelay({ resolution: 20 });
+  private readonly http = new Map<string, HttpMetric>();
+  private inFlight = 0;
+
+  constructor() {
+    this.eventLoopDelay.enable();
+  }
+
+  beginRequest() {
+    this.inFlight += 1;
+    return process.hrtime.bigint();
+  }
+
+  finishRequest(startedAt: bigint, method: string, route: string, statusCode: number) {
+    this.inFlight = Math.max(0, this.inFlight - 1);
+    // Only route templates enter labels. Raw URLs, IDs, phone numbers and query strings would create unbounded time series.
+    const normalizedRoute = route.startsWith('/') ? route : `/${route}`;
+    const labels = [method.toUpperCase(), normalizedRoute, String(statusCode)];
+    const key = labels.join('\u0000');
+    const metric = this.http.get(key) ?? { count: 0, durationSum: 0, buckets: HTTP_DURATION_BUCKETS.map(() => 0) };
+    const durationSeconds = Number(process.hrtime.bigint() - startedAt) / 1_000_000_000;
+    metric.count += 1;
+    metric.durationSum += durationSeconds;
+    HTTP_DURATION_BUCKETS.forEach((bucket, index) => {
+      if (durationSeconds <= bucket) metric.buckets[index] += 1;
+    });
+    this.http.set(key, metric);
+  }
+
+  render() {
+    const memory = process.memoryUsage();
+    const uptime = Number(process.hrtime.bigint() - this.startedAt) / 1_000_000_000;
+    const lines = [
+      '# HELP cmpp_api_process_uptime_seconds API process uptime.',
+      '# TYPE cmpp_api_process_uptime_seconds gauge',
+      metricLine('cmpp_api_process_uptime_seconds', uptime),
+      '# HELP cmpp_api_process_resident_memory_bytes API resident memory.',
+      '# TYPE cmpp_api_process_resident_memory_bytes gauge',
+      metricLine('cmpp_api_process_resident_memory_bytes', memory.rss),
+      '# HELP cmpp_api_nodejs_heap_used_bytes Node.js heap currently used.',
+      '# TYPE cmpp_api_nodejs_heap_used_bytes gauge',
+      metricLine('cmpp_api_nodejs_heap_used_bytes', memory.heapUsed),
+      '# HELP cmpp_api_nodejs_heap_total_bytes Node.js allocated heap.',
+      '# TYPE cmpp_api_nodejs_heap_total_bytes gauge',
+      metricLine('cmpp_api_nodejs_heap_total_bytes', memory.heapTotal),
+      '# HELP cmpp_api_nodejs_event_loop_lag_p99_seconds Event loop delay p99 since the previous scrape.',
+      '# TYPE cmpp_api_nodejs_event_loop_lag_p99_seconds gauge',
+      metricLine('cmpp_api_nodejs_event_loop_lag_p99_seconds', this.eventLoopDelay.count ? this.eventLoopDelay.percentile(99) / 1_000_000_000 : 0),
+      '# HELP cmpp_api_http_requests_in_flight Current API requests in flight.',
+      '# TYPE cmpp_api_http_requests_in_flight gauge',
+      metricLine('cmpp_api_http_requests_in_flight', this.inFlight),
+      '# HELP cmpp_api_http_requests_total API requests grouped by bounded route templates.',
+      '# TYPE cmpp_api_http_requests_total counter',
+      '# HELP cmpp_api_http_request_duration_seconds API request duration.',
+      '# TYPE cmpp_api_http_request_duration_seconds histogram',
+    ];
+    for (const [key, metric] of this.http) {
+      const [method, route, status] = key.split('\u0000');
+      const labels = { method, route, status };
+      lines.push(metricLine('cmpp_api_http_requests_total', metric.count, labels));
+      HTTP_DURATION_BUCKETS.forEach((bucket, index) => {
+        lines.push(metricLine('cmpp_api_http_request_duration_seconds_bucket', metric.buckets[index], { ...labels, le: String(bucket) }));
+      });
+      lines.push(metricLine('cmpp_api_http_request_duration_seconds_bucket', metric.count, { ...labels, le: '+Inf' }));
+      lines.push(metricLine('cmpp_api_http_request_duration_seconds_sum', metric.durationSum, labels));
+      lines.push(metricLine('cmpp_api_http_request_duration_seconds_count', metric.count, labels));
+    }
+    this.eventLoopDelay.reset();
+    return `${lines.join('\n')}\n`;
+  }
+
+  onModuleDestroy() {
+    this.eventLoopDelay.disable();
+  }
+}