perf: batch gateway submits and isolate callbacks

api/src/metrics/metrics.service.ts

@@ -3,6 +3,7 @@ import { monitorEventLoopDelay } from 'node:perf_hooks';
 
 const HTTP_DURATION_BUCKETS = [0.05, 0.1, 0.25, 0.5, 1, 3, 10] as const;
 const CMPP_INBOUND_DURATION_BUCKETS = [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 3, 10] as const;
+const SEND_WORKER_DURATION_BUCKETS = [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 3, 10] as const;
 
 export type CmppInboundStage =
   | 'application_lookup'
@@ -25,6 +26,22 @@ export type CmppInboundStage =
 
 export type CmppInboundStageResult = 'success' | 'error';
 
+export type SendWorkerStage =
+  | 'message_load'
+  | 'phone_routing'
+  | 'route_lookup'
+  | 'signature_candidates'
+  | 'signature_final_check'
+  | 'rate_limit'
+  | 'submit_transaction'
+  | 'gateway_bullmq_publish'
+  | 'gateway_stream_publish'
+  | 'task_progress'
+  | 'total';
+
+export type SendWorkerStageResult = 'success' | 'error' | 'skipped';
+export type SendWorkerQueueState = 'waiting' | 'active' | 'completed' | 'failed' | 'delayed' | 'prioritized';
+
 type HttpMetric = {
   count: number;
   durationSum: number;
@@ -48,6 +65,12 @@ export class MetricsService implements OnModuleDestroy {
   private readonly eventLoopDelay = monitorEventLoopDelay({ resolution: 20 });
   private readonly http = new Map<string, HttpMetric>();
   private readonly cmppInbound = new Map<string, HttpMetric>();
+  private readonly sendWorkerStages = new Map<string, HttpMetric>();
+  private readonly sendWorkerQueueJobs = new Map<SendWorkerQueueState, number>();
+  private readonly sendWorkerResults = new Map<string, number>();
+  private sendWorkerConfiguredSlots = 0;
+  private sendWorkerInFlightSlots = 0;
+  private readonly sendWorkerDatabasePool = new Map<'max' | 'total' | 'idle' | 'waiting', number>();
   private inFlight = 0;
   private inboundWorkflowPending = 0;
   private inboundWorkflowProcessing = 0;
@@ -101,6 +124,43 @@ export class MetricsService implements OnModuleDestroy {
     this.cmppInbound.set(key, metric);
   }
 
+  beginSendWorkerStage() {
+    return process.hrtime.bigint();
+  }
+
+  finishSendWorkerStage(startedAt: bigint, stage: SendWorkerStage, result: SendWorkerStageResult) {
+    const key = `${stage}\u0000${result}`;
+    const metric = this.sendWorkerStages.get(key) ?? {
+      count: 0,
+      durationSum: 0,
+      buckets: SEND_WORKER_DURATION_BUCKETS.map(() => 0),
+    };
+    const durationSeconds = Number(process.hrtime.bigint() - startedAt) / 1_000_000_000;
+    metric.count += 1;
+    metric.durationSum += durationSeconds;
+    SEND_WORKER_DURATION_BUCKETS.forEach((bucket, index) => {
+      if (durationSeconds <= bucket) metric.buckets[index] += 1;
+    });
+    this.sendWorkerStages.set(key, metric);
+  }
+
+  setSendWorkerSlots(configured: number, inFlight: number) {
+    this.sendWorkerConfiguredSlots = Math.max(0, configured);
+    this.sendWorkerInFlightSlots = Math.max(0, inFlight);
+  }
+
+  setSendWorkerQueueJobs(state: SendWorkerQueueState, count: number) {
+    this.sendWorkerQueueJobs.set(state, Math.max(0, count));
+  }
+
+  recordSendWorkerResult(result: 'completed' | 'failed' | 'skipped') {
+    this.sendWorkerResults.set(result, (this.sendWorkerResults.get(result) ?? 0) + 1);
+  }
+
+  setSendWorkerDatabasePool(state: 'max' | 'total' | 'idle' | 'waiting', count: number) {
+    this.sendWorkerDatabasePool.set(state, Math.max(0, count));
+  }
+
   setInboundWorkflowState(pending: number, processing: number, oldestPendingAgeSeconds: number) {
     this.inboundWorkflowPending = Math.max(0, pending);
     this.inboundWorkflowProcessing = Math.max(0, processing);
@@ -144,6 +204,18 @@ export class MetricsService implements OnModuleDestroy {
       '# TYPE cmpp_api_http_request_duration_seconds histogram',
       '# HELP cmpp_api_cmpp_inbound_stage_duration_seconds CMPP inbound processing duration by bounded stage and result.',
       '# TYPE cmpp_api_cmpp_inbound_stage_duration_seconds histogram',
+      '# HELP cmpp_worker_send_stage_duration_seconds Send worker processing duration by bounded stage and result.',
+      '# TYPE cmpp_worker_send_stage_duration_seconds histogram',
+      '# HELP cmpp_worker_send_queue_jobs BullMQ send jobs by queue state.',
+      '# TYPE cmpp_worker_send_queue_jobs gauge',
+      '# HELP cmpp_worker_send_slots Send worker concurrency slots by state.',
+      '# TYPE cmpp_worker_send_slots gauge',
+      metricLine('cmpp_worker_send_slots', this.sendWorkerConfiguredSlots, { state: 'configured' }),
+      metricLine('cmpp_worker_send_slots', this.sendWorkerInFlightSlots, { state: 'in_flight' }),
+      '# HELP cmpp_worker_send_jobs_total Send worker processing outcomes.',
+      '# TYPE cmpp_worker_send_jobs_total counter',
+      '# HELP cmpp_worker_database_pool_connections Worker PostgreSQL client pool slots by state.',
+      '# TYPE cmpp_worker_database_pool_connections gauge',
       '# HELP cmpp_worker_inbound_workflow_items Current durable CMPP inbound workflow items by state.',
       '# TYPE cmpp_worker_inbound_workflow_items gauge',
       metricLine('cmpp_worker_inbound_workflow_items', this.inboundWorkflowPending, { state: 'pending' }),
@@ -179,6 +251,25 @@ export class MetricsService implements OnModuleDestroy {
       lines.push(metricLine('cmpp_api_cmpp_inbound_stage_duration_seconds_sum', metric.durationSum, labels));
       lines.push(metricLine('cmpp_api_cmpp_inbound_stage_duration_seconds_count', metric.count, labels));
     }
+    for (const [key, metric] of this.sendWorkerStages) {
+      const [stage, result] = key.split('\u0000');
+      const labels = { stage, result };
+      SEND_WORKER_DURATION_BUCKETS.forEach((bucket, index) => {
+        lines.push(metricLine('cmpp_worker_send_stage_duration_seconds_bucket', metric.buckets[index], { ...labels, le: String(bucket) }));
+      });
+      lines.push(metricLine('cmpp_worker_send_stage_duration_seconds_bucket', metric.count, { ...labels, le: '+Inf' }));
+      lines.push(metricLine('cmpp_worker_send_stage_duration_seconds_sum', metric.durationSum, labels));
+      lines.push(metricLine('cmpp_worker_send_stage_duration_seconds_count', metric.count, labels));
+    }
+    for (const [state, count] of this.sendWorkerQueueJobs) {
+      lines.push(metricLine('cmpp_worker_send_queue_jobs', count, { state }));
+    }
+    for (const [result, count] of this.sendWorkerResults) {
+      lines.push(metricLine('cmpp_worker_send_jobs_total', count, { result }));
+    }
+    for (const [state, count] of this.sendWorkerDatabasePool) {
+      lines.push(metricLine('cmpp_worker_database_pool_connections', count, { state }));
+    }
     for (const [result, count] of this.inboundWorkflowResults) {
       lines.push(metricLine('cmpp_worker_inbound_workflow_results_total', count, { result }));
     }