feat: 实现发送质量监控与报备状态消息通知

2026-09-06 19:22:49 +08:00
parent 69e3d7368d
commit 457319e627
66 changed files with 6992 additions and 489 deletions
@@ -1,17 +1,26 @@
-import { BadRequestException, HttpException, HttpStatus, Logger, NotFoundException } from '@nestjs/common';
+import { Logger, NotFoundException } from '@nestjs/common';
 import { Prisma } from '@prisma/client';
-import { createHash } from 'node:crypto';
+
 import { BillingService } from '../billing/billing.service';
-import { moneyToNumber } from '../common/money';
+
 import type { OpenApiService } from '../open-api/open-api.service';
 import { PrismaService } from '../prisma/prisma.service';
-import type { GatewaySubmitResultDto, GatewaySubmitSegmentResultDto, GatewayReceiptEventDto, GatewayUplinkEventDto, GatewayPendingDeliveryQueryDto, GatewayDownstreamSentDto, GatewayDownstreamAcknowledgedDto, GatewayDownstreamFailureType, GatewayControlDeliveryResult, GatewaySubmitDeadLetterDto, RequeueGatewaySubmitExceptionDto, GatewayDownstreamRecoveryStatusDto, TimeoutUnknownDto } from './send-chain.contracts';
-import { positiveInteger, normalizeReceiptStatus, normalizeCarrier, DEFAULT_DOWNSTREAM_RETRY_DELAY_MS, DEFAULT_DOWNSTREAM_RETRY_MAX_DELAY_MS, DEFAULT_DOWNSTREAM_MAX_RETRIES, DEFAULT_DOWNSTREAM_PENDING_TIMEOUT_HOURS, DEFAULT_RECEIPT_TIMEOUT_HOURS, DEFAULT_GATEWAY_SUBMIT_REQUEUE_STALE_MS, DEFAULT_DOWNSTREAM_MANUAL_REQUEUE_STALE_MS, DEFAULT_UPSTREAM_RECEIPT_INBOX_PROCESSING_STALE_MS, DEFAULT_UPSTREAM_RECEIPT_INBOX_MAX_ATTEMPTS, DEFAULT_UPSTREAM_RECEIPT_INBOX_MAX_AGE_HOURS, gatewaySubmitRequeueKey, isObjectRecord, asDateOrNull, downstreamRetryDelayMs, downstreamAckTimeoutMs, downstreamRetryBaseDelayMs, downstreamRetryMaxDelayMs, downstreamMaxRetries, downstreamPendingTimeoutHours, downstreamControlFailureMessage, normalizeSubmitStatus, downstreamDeliveryAttemptKey, hasRecoveryAuditStateChanged, normalizeRecoveryFailureCategory, aggregateReceiptSegmentState, isSameUpstreamEndpointIdentity, receiptEventKey, longMessageReceiptMode } from './send-chain.helpers';
-import type { SendSubmissionService } from './send-submission.service';
+import type { GatewayReceiptEventDto } from './send-chain.contracts';
+import {
+  positiveInteger,
+  normalizeReceiptStatus,
+  DEFAULT_UPSTREAM_RECEIPT_INBOX_PROCESSING_STALE_MS,
+  DEFAULT_UPSTREAM_RECEIPT_INBOX_MAX_ATTEMPTS,
+  DEFAULT_UPSTREAM_RECEIPT_INBOX_MAX_AGE_HOURS,
+  aggregateReceiptSegmentState,
+  isSameUpstreamEndpointIdentity,
+  receiptEventKey,
+  longMessageReceiptMode,
+} from './send-chain.helpers';
+
 import type { SendCompletionCallbacks, SendCompletionFacade } from './send-completion.service';
 import { queueFinalReceiptDeliveries } from './downstream-receipt-targets';

-
 /**
 * R10 receipt implementation.
 * Cross-domain calls return through the stable SendChainService seam to preserve locking and test observability.
@@ -68,6 +77,7 @@ export class SendReceiptService {
        errorCode: data.errorCode,
        errorMessage: data.errorMessage,
        deliveredAt,
+        gatewayReceivedAt: data.deliveredAt ? new Date(data.deliveredAt) : null,
        status: 'pending',
        nextRetryAt: new Date(),
      },
@@ -81,8 +91,8 @@ export class SendReceiptService {
  async processPendingUpstreamReceiptInbox(limit = 100) {
    const now = new Date();
    const staleBefore = new Date(
-      now.getTime()
-        - positiveInteger(
+      now.getTime() -
+        positiveInteger(
          process.env.UPSTREAM_RECEIPT_INBOX_PROCESSING_STALE_MS,
          DEFAULT_UPSTREAM_RECEIPT_INBOX_PROCESSING_STALE_MS,
        ),
@@ -110,8 +120,8 @@ export class SendReceiptService {

  async processUpstreamReceiptInboxRecord(id: string) {
    const staleBefore = new Date(
-      Date.now()
-        - positiveInteger(
+      Date.now() -
+        positiveInteger(
          process.env.UPSTREAM_RECEIPT_INBOX_PROCESSING_STALE_MS,
          DEFAULT_UPSTREAM_RECEIPT_INBOX_PROCESSING_STALE_MS,
        ),
@@ -119,10 +129,7 @@ export class SendReceiptService {
    const claimed = await this.prisma.upstreamReceiptInbox.updateMany({
      where: {
        id,
-        OR: [
-          { status: { in: ['pending', 'retrying'] } },
-          { status: 'processing', updatedAt: { lte: staleBefore } },
-        ],
+        OR: [{ status: { in: ['pending', 'retrying'] } }, { status: 'processing', updatedAt: { lte: staleBefore } }],
      },
      data: { status: 'processing', attemptCount: { increment: 1 }, nextRetryAt: null },
    });
@@ -130,25 +137,28 @@ export class SendReceiptService {
    const inbox = await this.prisma.upstreamReceiptInbox.findUnique({ where: { id } });
    if (!inbox) return false;
    try {
-      const message = await this.facade.handleReceipt({
-        messageId: inbox.provisionalMessageId ?? undefined,
-        channelId: inbox.incomingChannelId,
-        connectionId: inbox.incomingConnectionId ?? undefined,
-        sequenceId: inbox.sequenceId ?? undefined,
-        gatewayMessageId: inbox.gatewayMessageId,
-        phoneNumber: inbox.phoneNumber ?? undefined,
-        receiptStatus: normalizeReceiptStatus(inbox.receiptStatus),
-        rawStatus: inbox.rawStatus,
-        errorCode: inbox.errorCode ?? undefined,
-        errorMessage: inbox.errorMessage ?? undefined,
-        deliveredAt: inbox.deliveredAt.toISOString(),
-      }, {
-        account: inbox.upstreamAccount,
-        gatewayHost: inbox.upstreamHost,
-        gatewayPort: inbox.upstreamPort,
-        protocol: inbox.protocol,
-        cmppVersion: inbox.protocolVersion,
-      });
+      const message = await this.facade.handleReceipt(
+        {
+          messageId: inbox.provisionalMessageId ?? undefined,
+          channelId: inbox.incomingChannelId,
+          connectionId: inbox.incomingConnectionId ?? undefined,
+          sequenceId: inbox.sequenceId ?? undefined,
+          gatewayMessageId: inbox.gatewayMessageId,
+          phoneNumber: inbox.phoneNumber ?? undefined,
+          receiptStatus: normalizeReceiptStatus(inbox.receiptStatus),
+          rawStatus: inbox.rawStatus,
+          errorCode: inbox.errorCode ?? undefined,
+          errorMessage: inbox.errorMessage ?? undefined,
+          deliveredAt: inbox.deliveredAt.toISOString(),
+        },
+        {
+          account: inbox.upstreamAccount,
+          gatewayHost: inbox.upstreamHost,
+          gatewayPort: inbox.upstreamPort,
+          protocol: inbox.protocol,
+          cmppVersion: inbox.protocolVersion,
+        },
+      );
      const matchedMessageRecordId = message && 'id' in message ? message.id : message?.messageRecordId;
      const matchedChannelId = message && 'channelId' in message ? message.channelId : undefined;
      await this.prisma.upstreamReceiptInbox.update({
@@ -171,8 +181,8 @@ export class SendReceiptService {
        process.env.UPSTREAM_RECEIPT_INBOX_MAX_AGE_HOURS,
        DEFAULT_UPSTREAM_RECEIPT_INBOX_MAX_AGE_HOURS,
      );
-      const exhausted = inbox.attemptCount >= maxAttempts
-        || inbox.receivedAt.getTime() <= Date.now() - maxAgeHours * 60 * 60_000;
+      const exhausted =
+        inbox.attemptCount >= maxAttempts || inbox.receivedAt.getTime() <= Date.now() - maxAgeHours * 60 * 60_000;
      const retryDelayMs = Math.min(30 * 60_000, 5_000 * 2 ** Math.min(Math.max(inbox.attemptCount - 1, 0), 8));
      await this.prisma.upstreamReceiptInbox.update({
        where: { id },
@@ -204,7 +214,13 @@ export class SendReceiptService {

  async handleReceipt(
    data: GatewayReceiptEventDto,
-    incomingIdentity?: { account: string; gatewayHost: string; gatewayPort: number; protocol: string; cmppVersion: string },
+    incomingIdentity?: {
+      account: string;
+      gatewayHost: string;
+      gatewayPort: number;
+      protocol: string;
+      cmppVersion: string;
+    },
  ) {
    const resolved = await this.facade.resolveReceiptMessage(data, incomingIdentity);
    const logicalChannelId = resolved.channelId ?? data.channelId;
@@ -263,9 +279,8 @@ export class SendReceiptService {
    }
    const logicalReceipt = { ...data, channelId: logicalChannelId };
    await this.facade.recordReceiptSegment(message, logicalReceipt, deliveredAt, resolved.submitRecordId);
-    const receiptMode = Number(message.billingUnits ?? 1) > 1
-      ? await this.getLongMessageReceiptMode(logicalChannelId)
-      : 'per_segment';
+    const receiptMode =
+      Number(message.billingUnits ?? 1) > 1 ? await this.getLongMessageReceiptMode(logicalChannelId) : 'per_segment';
    if (receiptMode === 'message_level' && data.receiptStatus === 'delivered') {
      await this.applyMessageLevelSuccess(
        message,
@@ -287,12 +302,10 @@ export class SendReceiptService {
    }
    const status = aggregate.status;
    const isCurrentAttempt =
-      (!message.channelId || message.channelId === logicalChannelId)
-      && (
-        !message.gatewayMessageId
-        || message.gatewayMessageId === data.gatewayMessageId
-        || (aggregate.segmentTotal > 1 && (!message.submitId || message.submitId === resolved.submitId))
-      );
+      (!message.channelId || message.channelId === logicalChannelId) &&
+      (!message.gatewayMessageId ||
+        message.gatewayMessageId === data.gatewayMessageId ||
+        (aggregate.segmentTotal > 1 && (!message.submitId || message.submitId === resolved.submitId)));
    if (!isCurrentAttempt) {
      return this.prisma.smsMessageRecord.findUnique({ where: { id: message.id } });
    }
@@ -314,11 +327,7 @@ export class SendReceiptService {
    const isStandaloneChannelTest = !message.tenantId && !message.batchTaskId;
    if (status === 'failed' && !isStandaloneChannelTest && message.tenantId && message.batchTaskId) {
      const businessMessage = message as typeof message & { tenantId: string; batchTaskId: string };
-      const retried = await this.facade.retryMessageIfAllowed(
-        businessMessage,
-        '回执失败补发',
-        resolved.submitRecordId,
-      );
+      const retried = await this.facade.retryMessageIfAllowed(businessMessage, '回执失败补发', resolved.submitRecordId);
      if (retried) {
        await this.facade.refreshTaskProgress(businessMessage.batchTaskId);
        return retried;
@@ -339,32 +348,31 @@ export class SendReceiptService {
      },
    });
    if (!isStandaloneChannelTest && message.tenantId && message.applicationId) {
-      await queueFinalReceiptDeliveries(
-        this.prisma,
-        (request) => this.facade.queueAndTryDownstreamDelivery(request),
-        {
-          message,
-          payload: {
-            messageId: message.messageId,
-            gatewayMessageId: data.gatewayMessageId,
-            phoneNumber: message.phoneNumber,
-            receiptStatus: aggregate.receiptStatus,
-            rawStatus: aggregate.rawStatus,
-            errorCode: aggregate.errorCode,
-            deliveredAt: aggregate.deliveredAt.toISOString(),
-          },
-          segmentPayloads: Object.fromEntries(
-            aggregate.segments
-              .filter((segment) => segment.receiptStatus)
-              .map((segment) => [segment.segmentIndex, {
+      await queueFinalReceiptDeliveries(this.prisma, (request) => this.facade.queueAndTryDownstreamDelivery(request), {
+        message,
+        payload: {
+          messageId: message.messageId,
+          gatewayMessageId: data.gatewayMessageId,
+          phoneNumber: message.phoneNumber,
+          receiptStatus: aggregate.receiptStatus,
+          rawStatus: aggregate.rawStatus,
+          errorCode: aggregate.errorCode,
+          deliveredAt: aggregate.deliveredAt.toISOString(),
+        },
+        segmentPayloads: Object.fromEntries(
+          aggregate.segments
+            .filter((segment) => segment.receiptStatus)
+            .map((segment) => [
+              segment.segmentIndex,
+              {
                receiptStatus: segment.receiptStatus,
                rawStatus: segment.rawStatus,
                errorCode: segment.errorCode,
                deliveredAt: segment.deliveredAt?.toISOString() ?? aggregate.deliveredAt.toISOString(),
-              }]),
-          ),
-        },
-      );
+              },
+            ]),
+        ),
+      });
    }
    if (message.batchTaskId) {
      await this.facade.refreshTaskProgress(message.batchTaskId);
@@ -388,8 +396,9 @@ export class SendReceiptService {
    submitRecordId?: string,
    submitId?: string,
  ) {
-    const belongsToCurrentAttempt = (!message.channelId || message.channelId === data.channelId)
-      && (!message.submitId || message.submitId === submitId);
+    const belongsToCurrentAttempt =
+      (!message.channelId || message.channelId === data.channelId) &&
+      (!message.submitId || message.submitId === submitId);
    if (!belongsToCurrentAttempt) return;
    const attemptWhere = submitRecordId
      ? { messageRecordId: message.id, submitRecordId }
@@ -402,7 +411,9 @@ export class SendReceiptService {
      select: { id: true, receiptStatus: true },
    });
    if (segments.length <= 1) return;
-    if (segments.some((segment) => segment.receiptStatus && !['delivered', 'unknown'].includes(segment.receiptStatus))) {
+    if (
+      segments.some((segment) => segment.receiptStatus && !['delivered', 'unknown'].includes(segment.receiptStatus))
+    ) {
      return;
    }
    // This supplier contract reports one message-level success for a multipart SMS.
@@ -506,9 +517,9 @@ export class SendReceiptService {
    const submitRecord = submitRecordId
      ? await this.prisma.smsSubmitRecord.findUnique({ where: { id: submitRecordId } })
      : await this.prisma.smsSubmitRecord.findFirst({
-        where: { messageRecordId: message.id, gatewayMessageId: data.gatewayMessageId },
-        orderBy: { createdAt: 'desc' },
-      });
+          where: { messageRecordId: message.id, gatewayMessageId: data.gatewayMessageId },
+          orderBy: { createdAt: 'desc' },
+        });
    await segmentAudits.upsert({
      where: {
        messageRecordId_submitId_segmentIndex: {
@@ -575,7 +586,13 @@ export class SendReceiptService {

  async resolveReceiptMessage(
    data: GatewayReceiptEventDto,
-    incomingIdentity?: { account: string; gatewayHost: string; gatewayPort: number; protocol: string; cmppVersion: string },
+    incomingIdentity?: {
+      account: string;
+      gatewayHost: string;
+      gatewayPort: number;
+      protocol: string;
+      cmppVersion: string;
+    },
  ) {
    const exactMessage = data.messageId
      ? await this.prisma.smsMessageRecord.findUnique({ where: { messageId: data.messageId } })
@@ -641,8 +658,8 @@ export class SendReceiptService {
      throw new NotFoundException('SMS message record not found');
    }

-    const incomingChannel = incomingIdentity
-      ?? await this.prisma.smsChannel.findUnique({ where: { id: data.channelId } });
+    const incomingChannel =
+      incomingIdentity ?? (await this.prisma.smsChannel.findUnique({ where: { id: data.channelId } }));
    if (!incomingChannel) {
      throw new NotFoundException('SMS message record not found');
    }
@@ -665,8 +682,9 @@ export class SendReceiptService {
        channelId: exactSegmentMatches[0].channelId,
      };
    }
-    const sameSupplierSegments = segmentMatches.filter((candidate) =>
-      candidate.channel && isSameUpstreamEndpointIdentity(incomingChannel, candidate.channel));
+    const sameSupplierSegments = segmentMatches.filter(
+      (candidate) => candidate.channel && isSameUpstreamEndpointIdentity(incomingChannel, candidate.channel),
+    );
    if (sameSupplierSegments.length === 1 && sameSupplierSegments[0]?.messageRecord) {
      return {
        message: sameSupplierSegments[0].messageRecord,
@@ -685,8 +703,9 @@ export class SendReceiptService {
      orderBy: { createdAt: 'desc' },
      take: 10,
    });
-    const sameSupplierSubmits = crossConnectionSubmits.filter((candidate) =>
-      candidate.channel && isSameUpstreamEndpointIdentity(incomingChannel, candidate.channel));
+    const sameSupplierSubmits = crossConnectionSubmits.filter(
+      (candidate) => candidate.channel && isSameUpstreamEndpointIdentity(incomingChannel, candidate.channel),
+    );
    if (sameSupplierSubmits.length === 1 && sameSupplierSubmits[0]?.messageRecord) {
      return {
        message: sameSupplierSubmits[0].messageRecord,