fix: coordinate SMS completion and improve operations diagnostics

api/src/send-chain/send-chain.service.ts

@@ -1,3 +1,5 @@
+import { AttemptCompletion } from './attempt-completion';
+import { completionContext, completionDatabase, CompletionRouteRequired } from './completion-context';
 import {
   BadRequestException,
   forwardRef,
@@ -84,6 +86,7 @@ export class SendChainService implements OnModuleInit, OnModuleDestroy {
   private readonly submission: SendSubmissionService;
   private readonly completion: SendCompletionService;
   private readonly downstreamRequeueTasks: SendDownstreamRequeueTaskService;
+  private readonly attemptCompletion?: AttemptCompletion;
 
   constructor(
     private readonly prisma: PrismaService,
@@ -94,6 +97,11 @@ export class SendChainService implements OnModuleInit, OnModuleDestroy {
     @Optional() phoneRouting?: PhoneRoutingLookupService,
     @Optional() metrics?: MetricsService,
   ) {
+    const rootPrisma = prisma;
+    prisma = completionDatabase(prisma);
+    this.prisma = prisma;
+    // Structural unit-test doubles may omit the durable delegate; real Prisma always has it.
+    const coordinatedBilling = rootPrisma.smsAttemptCompletionWork ? new BillingService(prisma) : billing;
     const resolvedPhoneRouting = phoneRouting ?? new PhoneRoutingLookupService(prisma);
     this.submission = new SendSubmissionService(
       prisma,
@@ -109,12 +117,46 @@ export class SendChainService implements OnModuleInit, OnModuleDestroy {
       },
       metrics,
     );
-    this.completion = new SendCompletionService(prisma, billing, openApi, this as unknown as SendCompletionFacade);
+    this.completion = new SendCompletionService(
+      prisma,
+      coordinatedBilling,
+      openApi,
+      this as unknown as SendCompletionFacade,
+    );
+    if (rootPrisma.smsAttemptCompletionWork) {
+      this.attemptCompletion = new AttemptCompletion(
+        rootPrisma,
+        async (kind, payload) => {
+          const event = payload as unknown as {
+            data: GatewayReceiptEventDto & GatewaySubmitResultDto & GatewaySubmitSegmentResultDto;
+            incomingIdentity?: Parameters<SendChainService['handleReceipt']>[1];
+            olderThanHours?: number;
+            errorCode?: string;
+            reason?: string;
+          };
+          if (kind === 'receipt') return this.completion.handleReceipt(event.data, event.incomingIdentity);
+          if (kind === 'submit') return this.completion.handleSubmitResult(event.data);
+          if (kind === 'segment') return this.completion.handleSubmitSegmentResult(event.data);
+          if (kind === 'timeout') return this.completion.markUnknownTimeout({ olderThanHours: event.olderThanHours });
+          const message = await prisma.smsMessageRecord.findUniqueOrThrow({
+            where: { id: completionContext.getStore()!.messageRecordId },
+          });
+          if (
+            message.status === 'delivered' ||
+            (message.status === 'timeout' && message.errorCode === 'RECEIPT_TIMEOUT')
+          )
+            return message;
+          return this.completion.recordCmppFailureReceipt(message, event.errorCode!, event.reason!);
+        },
+        (route) => this.submission.waitForChannelRateLimit(route.channel.id, route.channel.rateLimitPerSecond),
+      );
+    }
     this.downstreamRequeueTasks = new SendDownstreamRequeueTaskService(prisma, this);
   }
 
   onModuleInit() {
     const processRole = process.env.CMPP_PROCESS_ROLE?.trim() || 'all';
+    if (['all', 'worker', 'callback'].includes(processRole)) this.attemptCompletion?.start();
     if (processRole === 'api' || processRole === 'callback') return;
     if (processRole === 'outbox') {
       this.submission.startSubmitOutboxPublisher();
@@ -202,6 +244,7 @@ export class SendChainService implements OnModuleInit, OnModuleDestroy {
   }
 
   async onModuleDestroy() {
+    this.attemptCompletion?.stop();
     if (this.receiptTimeoutInitialTimer) clearTimeout(this.receiptTimeoutInitialTimer);
     if (this.receiptTimeoutIntervalTimer) clearInterval(this.receiptTimeoutIntervalTimer);
     if (this.scheduledDispatchInitialTimer) clearTimeout(this.scheduledDispatchInitialTimer);
@@ -484,6 +527,11 @@ export class SendChainService implements OnModuleInit, OnModuleDestroy {
   }
 
   async handleSubmitSegmentResult(data: GatewaySubmitSegmentResultDto) {
+    if (this.attemptCompletion && !completionContext.getStore()) {
+      const message = await this.completion.requireMessageByGatewayEvent(data.messageId, data.gatewayMessageId);
+      const source = await this.completion.resolveSubmitRecordForGatewaySegmentResult(message.id, data);
+      return this.attemptCompletion.enqueue(message.id, source.id, 'segment', { data });
+    }
     return this.completion.handleSubmitSegmentResult(data);
   }
 
@@ -495,6 +543,11 @@ export class SendChainService implements OnModuleInit, OnModuleDestroy {
   }
 
   async handleSubmitResult(data: GatewaySubmitResultDto) {
+    if (this.attemptCompletion && !completionContext.getStore()) {
+      const message = await this.completion.requireMessageByGatewayEvent(data.messageId, data.gatewayMessageId);
+      const source = await this.completion.resolveSubmitRecordForGatewayResult(message.id, data);
+      return this.attemptCompletion.enqueue(message.id, source.id, 'submit', { data }, data.eventId);
+    }
     return this.completion.handleSubmitResult(data);
   }
 
@@ -528,6 +581,14 @@ export class SendChainService implements OnModuleInit, OnModuleDestroy {
       cmppVersion: string;
     },
   ) {
+    if (this.attemptCompletion && !completionContext.getStore()) {
+      const resolved = await this.completion.resolveReceiptMessage(data, incomingIdentity);
+      if (!resolved.submitRecordId) throw new NotFoundException('回执缺少可确认的提交尝试关联');
+      return this.attemptCompletion.enqueue(resolved.message.id, resolved.submitRecordId, 'receipt', {
+        data,
+        incomingIdentity,
+      });
+    }
     return this.completion.handleReceipt(data, incomingIdentity);
   }
 
@@ -726,6 +787,32 @@ export class SendChainService implements OnModuleInit, OnModuleDestroy {
   }
 
   async markUnknownTimeout(data: TimeoutUnknownDto) {
+    if (this.attemptCompletion && !completionContext.getStore()) {
+      const olderThanHours = data.olderThanHours ?? positiveInteger(process.env.SMS_RECEIPT_TIMEOUT_HOURS, 72);
+      const candidates = await this.prisma.smsMessageRecord.findMany({
+        where: {
+          tenantId: { not: null },
+          OR: [
+            {
+              status: { in: ['submitted', 'unknown'] },
+              submittedAt: { lte: new Date(Date.now() - olderThanHours * 3600_000) },
+            },
+            { status: 'timeout', errorCode: 'RECEIPT_TIMEOUT', timeoutReceiptQueuedAt: null },
+          ],
+        },
+        select: { id: true, submitId: true, status: true },
+        take: 100,
+      });
+      let timeout = 0;
+      for (const message of candidates) {
+        const source = message.submitId
+          ? await this.prisma.smsSubmitRecord.findUnique({ where: { submitId: message.submitId } })
+          : null;
+        const result = await this.attemptCompletion.enqueue(message.id, source?.id, 'timeout', { olderThanHours });
+        if (message.status !== 'timeout' && result?.status === 'timeout') timeout++;
+      }
+      return { timeout };
+    }
     return this.completion.markUnknownTimeout(data);
   }
 
@@ -803,6 +890,14 @@ export class SendChainService implements OnModuleInit, OnModuleDestroy {
     },
     options: { forceNational?: boolean; excludeChannelIds?: string[] } = {},
   ): Promise<RoutedChannel> {
+    const context = completionContext.getStore();
+    if (context) {
+      const prepare = () => this.submission.selectChannelForMessage(message, { ...options, previewOnly: true });
+      if (!context.routePlanned) throw new CompletionRouteRequired(prepare);
+      const current = await this.submission.selectChannelForMessage(message, options);
+      if (!context.route || current.channel.id !== context.route.channel.id) throw new CompletionRouteRequired(prepare);
+      return current;
+    }
     return this.submission.selectChannelForMessage(message, options);
   }
 
@@ -923,6 +1018,9 @@ export class SendChainService implements OnModuleInit, OnModuleDestroy {
     errorCode: string,
     reason: string,
   ) {
+    if (this.attemptCompletion && !completionContext.getStore()) {
+      return this.attemptCompletion.enqueue(message.id, undefined, 'rejection', { errorCode, reason });
+    }
     return this.completion.recordCmppFailureReceipt(message, errorCode, reason);
   }
 
@@ -1004,6 +1102,7 @@ export class SendChainService implements OnModuleInit, OnModuleDestroy {
   }
 
   private async waitForChannelRateLimit(channelId: string, tps: number) {
+    if (completionContext.getStore()) return;
     return this.submission.waitForChannelRateLimit(channelId, tps);
   }