perf(cmpp): instrument inbound flow and unbatch submit worker

api/src/metrics/metrics.service.spec.ts

@@ -5,11 +5,14 @@ describe('MetricsService', () => {
     const service = new MetricsService();
     const startedAt = service.beginRequest();
     service.finishRequest(startedAt, 'GET', '/api/admin/tenants/:id', 200);
+    const inboundStartedAt = service.beginCmppInboundStage();
+    service.finishCmppInboundStage(inboundStartedAt, 'application_lookup', 'success');
     const output = service.render();
 
     expect(output).toContain('cmpp_api_process_resident_memory_bytes');
     expect(output).toContain('cmpp_api_http_requests_total{method="GET",route="/api/admin/tenants/:id",status="200"} 1');
     expect(output).toContain('cmpp_api_http_request_duration_seconds_bucket');
+    expect(output).toContain('cmpp_api_cmpp_inbound_stage_duration_seconds_count{stage="application_lookup",result="success"} 1');
     expect(output).not.toContain('phone_number');
     service.onModuleDestroy();
   });

api/src/metrics/metrics.service.ts

@@ -2,6 +2,24 @@ import { Injectable, OnModuleDestroy } from '@nestjs/common';
 import { monitorEventLoopDelay } from 'node:perf_hooks';
 
 const HTTP_DURATION_BUCKETS = [0.05, 0.1, 0.25, 0.5, 1, 3, 10] as const;
+const CMPP_INBOUND_DURATION_BUCKETS = [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 3, 10] as const;
+
+export type CmppInboundStage =
+  | 'application_lookup'
+  | 'long_message_fragment'
+  | 'submission_precheck'
+  | 'template_match'
+  | 'task_persist'
+  | 'api_request_persist'
+  | 'content_detection'
+  | 'message_persist'
+  | 'risk_frequency'
+  | 'billing'
+  | 'queue_publish'
+  | 'complete_submit'
+  | 'total';
+
+export type CmppInboundStageResult = 'success' | 'error';
 
 type HttpMetric = {
   count: number;
@@ -25,6 +43,7 @@ export class MetricsService implements OnModuleDestroy {
   private readonly startedAt = process.hrtime.bigint();
   private readonly eventLoopDelay = monitorEventLoopDelay({ resolution: 20 });
   private readonly http = new Map<string, HttpMetric>();
+  private readonly cmppInbound = new Map<string, HttpMetric>();
   private inFlight = 0;
 
   constructor() {
@@ -52,6 +71,26 @@ export class MetricsService implements OnModuleDestroy {
     this.http.set(key, metric);
   }
 
+  beginCmppInboundStage() {
+    return process.hrtime.bigint();
+  }
+
+  finishCmppInboundStage(startedAt: bigint, stage: CmppInboundStage, result: CmppInboundStageResult) {
+    const key = `${stage}\u0000${result}`;
+    const metric = this.cmppInbound.get(key) ?? {
+      count: 0,
+      durationSum: 0,
+      buckets: CMPP_INBOUND_DURATION_BUCKETS.map(() => 0),
+    };
+    const durationSeconds = Number(process.hrtime.bigint() - startedAt) / 1_000_000_000;
+    metric.count += 1;
+    metric.durationSum += durationSeconds;
+    CMPP_INBOUND_DURATION_BUCKETS.forEach((bucket, index) => {
+      if (durationSeconds <= bucket) metric.buckets[index] += 1;
+    });
+    this.cmppInbound.set(key, metric);
+  }
+
   render() {
     const memory = process.memoryUsage();
     const uptime = Number(process.hrtime.bigint() - this.startedAt) / 1_000_000_000;
@@ -78,6 +117,8 @@ export class MetricsService implements OnModuleDestroy {
       '# TYPE cmpp_api_http_requests_total counter',
       '# HELP cmpp_api_http_request_duration_seconds API request duration.',
       '# TYPE cmpp_api_http_request_duration_seconds histogram',
+      '# HELP cmpp_api_cmpp_inbound_stage_duration_seconds CMPP inbound processing duration by bounded stage and result.',
+      '# TYPE cmpp_api_cmpp_inbound_stage_duration_seconds histogram',
     ];
     for (const [key, metric] of this.http) {
       const [method, route, status] = key.split('\u0000');
@@ -90,6 +131,16 @@ export class MetricsService implements OnModuleDestroy {
       lines.push(metricLine('cmpp_api_http_request_duration_seconds_sum', metric.durationSum, labels));
       lines.push(metricLine('cmpp_api_http_request_duration_seconds_count', metric.count, labels));
     }
+    for (const [key, metric] of this.cmppInbound) {
+      const [stage, result] = key.split('\u0000');
+      const labels = { stage, result };
+      CMPP_INBOUND_DURATION_BUCKETS.forEach((bucket, index) => {
+        lines.push(metricLine('cmpp_api_cmpp_inbound_stage_duration_seconds_bucket', metric.buckets[index], { ...labels, le: String(bucket) }));
+      });
+      lines.push(metricLine('cmpp_api_cmpp_inbound_stage_duration_seconds_bucket', metric.count, { ...labels, le: '+Inf' }));
+      lines.push(metricLine('cmpp_api_cmpp_inbound_stage_duration_seconds_sum', metric.durationSum, labels));
+      lines.push(metricLine('cmpp_api_cmpp_inbound_stage_duration_seconds_count', metric.count, labels));
+    }
     this.eventLoopDelay.reset();
     return `${lines.join('\n')}\n`;
   }