perf(cmpp): instrument inbound flow and unbatch submit worker

2026-08-20 11:27:35 +08:00
parent c4f36fc50d
commit b9a71fe0b9
23 changed files with 760 additions and 138 deletions
@@ -11,6 +11,7 @@ import { PhoneRoutingLookupService } from '../dictionaries/phone-routing-lookup.
 import { PrismaService } from '../prisma/prisma.service';
 import { RiskReviewService } from '../risk-review/risk-review.service';
 import { PhoneFrequencyService } from '../risk-review/phone-frequency.service';
+import { MetricsService, type CmppInboundStage } from '../metrics/metrics.service';
 import type { CreateBatchTaskDto, CreateHttpBatchTaskDto, GatewayInboundAuthDto, GatewayInboundSubmitDto, GatewayInboundSingleSubmitResult, ImportPreviewDto, ConfirmImportDto, SendJob, QueuePriority, RoutedChannel } from './send-chain.contracts';
 import { SEND_QUEUE, GATEWAY_SUBMIT_QUEUE, GATEWAY_SUBMIT_STREAM, DEFAULT_SCHEDULED_DISPATCH_STALE_MS, DEFAULT_INBOUND_LONG_MESSAGE_PROCESSING_STALE_SECONDS, GATEWAY_SUBMIT_REQUEUE_IDEMPOTENCY_TTL_SECONDS, BULLMQ_PRIORITY, statusFromRisk, parseSchedule, parseImportRows, splitImportLine, cellByHeader, normalizeCarrier, normalizeQueuePriority, getPositiveConfigInteger, getNonNegativeConfigInteger, isCarrierCompatible, matchTemplateContent, isNationalChannel, validateInboundApplicationSrcId, composeUpstreamSrcId, positiveInteger, parseOptionalSequenceId, shanghaiDateKey, bullmqConnection, matchesApplicationSecret, octetString, selectChannelCandidate } from './send-chain.helpers';
 import type { SendSubmissionCallbacks, SendSubmissionService } from './send-submission.service';
@@ -30,8 +31,21 @@ export class SendInboundEntryService {
    private readonly phoneRouting: PhoneRoutingLookupService,
    private readonly facade: SendSubmissionService,
    private readonly callbacks: SendSubmissionCallbacks,
+    private readonly metrics?: MetricsService,
  ) {}

+  private async measureInboundStage<T>(stage: CmppInboundStage, action: () => Promise<T>): Promise<T> {
+    const startedAt = this.metrics?.beginCmppInboundStage();
+    try {
+      const result = await action();
+      if (startedAt != null) this.metrics?.finishCmppInboundStage(startedAt, stage, 'success');
+      return result;
+    } catch (error) {
+      if (startedAt != null) this.metrics?.finishCmppInboundStage(startedAt, stage, 'error');
+      throw error;
+    }
+  }
+
  private releaseMessageReservation(
    message: { tenantId: string; batchTaskId: string; messageId: string; amountCents: number | bigint; billingUnits: number },
    remark: string,
@@ -145,7 +159,10 @@ async submitInboundMessage(data: GatewayInboundSubmitDto) {
      throw new BadRequestException('CMPP submit phone number is invalid');
    }

-    const application = await this.facade.findInboundApplication(data.account);
+    const application = await this.measureInboundStage(
+      'application_lookup',
+      () => this.facade.findInboundApplication(data.account),
+    );
    if (!application) {
      throw new BadRequestException('CMPP account is invalid');
    }
@@ -157,7 +174,10 @@ async submitInboundMessage(data: GatewayInboundSubmitDto) {
        throw new BadRequestException('CMPP source IP is not in application allowlist');
      }
      validateInboundApplicationSrcId(data.srcId, application);
-      const collection = await this.facade.collectInboundLongMessageFragment(data, application, phoneNumbers);
+      const collection = await this.measureInboundStage(
+        'long_message_fragment',
+        () => this.facade.collectInboundLongMessageFragment(data, application, phoneNumbers),
+      );
      if (collection.response) {
        return collection.response;
      }
@@ -180,16 +200,18 @@ async submitInboundMessage(data: GatewayInboundSubmitDto) {
        };
      }
      try {
-        const response = await this.facade.recoverCompletedInboundLongMessageResponse(
-          collection.messageId,
-          phoneNumbers,
-        ) ?? await this.facade.submitCompleteInboundMessage({
-          ...data,
-          content: collection.content,
-          sequenceId: collection.sequenceId,
-          registeredDelivery: collection.registeredDelivery ? 1 : 0,
-          longMessage: undefined,
-        }, phoneNumbers, application, collection.messageId);
+        const response = await this.measureInboundStage('complete_submit', async () => (
+          await this.facade.recoverCompletedInboundLongMessageResponse(
+            collection.messageId,
+            phoneNumbers,
+          ) ?? await this.facade.submitCompleteInboundMessage({
+            ...data,
+            content: collection.content,
+            sequenceId: collection.sequenceId,
+            registeredDelivery: collection.registeredDelivery ? 1 : 0,
+            longMessage: undefined,
+          }, phoneNumbers, application, collection.messageId)
+        ));
        await this.prisma.cmppInboundLongMessage.update({
          where: { id: collection.groupId },
          data: {
@@ -210,7 +232,10 @@ async submitInboundMessage(data: GatewayInboundSubmitDto) {
        throw error;
      }
    }
-    return this.facade.submitCompleteInboundMessage(data, phoneNumbers, application);
+    return this.measureInboundStage(
+      'complete_submit',
+      () => this.facade.submitCompleteInboundMessage(data, phoneNumbers, application),
+    );
  }

 async recoverCompletedInboundLongMessageResponse(messageId: string, phoneNumbers: string[]) {
@@ -267,8 +292,9 @@ async submitCompleteInboundMessage(
    if (!application) {
      throw new BadRequestException('CMPP account is invalid');
    }
-    const persisted = requestedGroupMessageId
-      ? await this.prisma.smsMessageRecord.findMany({
+    const precheck = await this.measureInboundStage('submission_precheck', async () => {
+      const persisted = requestedGroupMessageId
+        ? await this.prisma.smsMessageRecord.findMany({
          where: {
            cmppSubmitGroupMessageId: requestedGroupMessageId,
            phoneNumber: { in: phoneNumbers },
@@ -284,15 +310,18 @@ async submitCompleteInboundMessage(
            errorCode: true,
          },
        })
-      : [];
-    const persistedByPhone = new Map(persisted.map((item) => [item.phoneNumber, item]));
-    const phoneRejections = await this.facade.classifyRejectedPhones(application.tenantId, application.id, phoneNumbers);
-    const missingPhoneCount = phoneNumbers.filter((phoneNumber) => (
-      !persistedByPhone.has(phoneNumber) && !phoneRejections.has(phoneNumber)
-    )).length;
-    const dailyQuota = missingPhoneCount > 0
-      ? await this.facade.tryReserveDailySendQuota(application.id, missingPhoneCount)
-      : { reserved: true, dailyLimit: application.dailyLimit ?? 100000 };
+        : [];
+      const persistedByPhone = new Map(persisted.map((item) => [item.phoneNumber, item]));
+      const phoneRejections = await this.facade.classifyRejectedPhones(application.tenantId, application.id, phoneNumbers);
+      const missingPhoneCount = phoneNumbers.filter((phoneNumber) => (
+        !persistedByPhone.has(phoneNumber) && !phoneRejections.has(phoneNumber)
+      )).length;
+      const dailyQuota = missingPhoneCount > 0
+        ? await this.facade.tryReserveDailySendQuota(application.id, missingPhoneCount)
+        : { reserved: true, dailyLimit: application.dailyLimit ?? 100000 };
+      return { persistedByPhone, phoneRejections, dailyQuota, missingPhoneCount };
+    });
+    const { persistedByPhone, phoneRejections, dailyQuota, missingPhoneCount } = precheck;
    const dailyLimitRejection = dailyQuota.reserved
      ? undefined
      : {
@@ -523,7 +552,10 @@ async submitInboundSingleMessage(
    synchronousRejection?: { code: string; reason: string },
    receiptRejection?: { code: string; reason: string },
  ) {
-    const application = await this.facade.findInboundApplication(data.account);
+    const application = await this.measureInboundStage(
+      'application_lookup',
+      () => this.facade.findInboundApplication(data.account),
+    );
    if (!application) {
      throw new BadRequestException('CMPP account is invalid');
    }
@@ -531,7 +563,10 @@ async submitInboundSingleMessage(
      throw new BadRequestException('CMPP source IP is not in application allowlist');
    }
    const clientSrcId = validateInboundApplicationSrcId(data.srcId, application);
-    const template = await this.facade.resolveInboundTemplateCandidate(application.id, data.content);
+    const template = await this.measureInboundStage(
+      'template_match',
+      () => this.facade.resolveInboundTemplateCandidate(application.id, data.content),
+    );
    const templateVariables = template ? matchTemplateContent(template.content, data.content) ?? {} : {};
    const unitPrice = moneyToNumber(application.customerUnitPrice);
    const queuePriority = normalizeQueuePriority(application.queuePriority);
@@ -542,7 +577,7 @@ async submitInboundSingleMessage(
      phoneCount: 1,
      unitPrice,
    });
-    const task = await this.prisma.smsBatchTask.create({
+    const task = await this.measureInboundStage('task_persist', () => this.prisma.smsBatchTask.create({
      data: {
        tenantId: application.tenantId,
        applicationId: application.id,
@@ -556,8 +591,8 @@ async submitInboundSingleMessage(
        rejectReason: synchronousRejection?.reason,
        progressTotal: 1,
      },
-    });
-    await this.prisma.smsApiRequest.create({
+    }));
+    await this.measureInboundStage('api_request_persist', () => this.prisma.smsApiRequest.create({
      data: {
        tenantId: application.tenantId,
        batchTaskId: task.id,
@@ -567,9 +602,12 @@ async submitInboundSingleMessage(
        payloadSummary: { phoneTotal: 1, contentLength: [...data.content].length, account: data.account },
        status: synchronousRejection ? 'rejected' : 'accepted',
      },
-    });
-    const drainageDetection = await detectDrainageContent(this.prisma, data.content);
-    const message = await this.prisma.smsMessageRecord.create({
+    }));
+    const drainageDetection = await this.measureInboundStage(
+      'content_detection',
+      () => detectDrainageContent(this.prisma, data.content),
+    );
+    const message = await this.measureInboundStage('message_persist', () => this.prisma.smsMessageRecord.create({
      data: {
        tenantId: application.tenantId,
        batchTaskId: task.id,
@@ -592,7 +630,7 @@ async submitInboundSingleMessage(
        errorCode: synchronousRejection?.code,
        errorMessage: synchronousRejection?.reason,
      },
-    });
+    }));

    if (synchronousRejection) {
      return {
@@ -614,16 +652,18 @@ async submitInboundSingleMessage(
      await this.recordCmppFailureReceipt(message, code, reason);
    };
    const queueAfterRiskChecks = async (options: { templateId?: string; signatureId?: string }) => {
-      const drainage = await this.facade.resolveDrainageInfoMatch(options.signatureId, data.content);
-      const drainageInfoId = drainage?.id;
-      const risk = await this.facade.evaluateRiskWithPhoneFrequency({
-        tenantId: application.tenantId,
-        applicationId: application.id,
-        templateId: options.templateId,
-        content: data.content,
-        variables: options.templateId ? templateVariables : undefined,
-        phoneNumber: data.phoneNumber,
-        sourceType: 'cmpp',
+      const { drainageInfoId, risk } = await this.measureInboundStage('risk_frequency', async () => {
+        const drainage = await this.facade.resolveDrainageInfoMatch(options.signatureId, data.content);
+        const evaluatedRisk = await this.facade.evaluateRiskWithPhoneFrequency({
+          tenantId: application.tenantId,
+          applicationId: application.id,
+          templateId: options.templateId,
+          content: data.content,
+          variables: options.templateId ? templateVariables : undefined,
+          phoneNumber: data.phoneNumber,
+          sourceType: 'cmpp',
+        });
+        return { drainageInfoId: drainage?.id, risk: evaluatedRisk };
      });
      if (risk.status === 'rejected') {
        await reject('RISK', risk.reason || '短信被风控拒绝');
@@ -645,32 +685,37 @@ async submitInboundSingleMessage(
        });
        return;
      }
-      const accountCheck = await this.billing.checkAccount({
-        tenantId: application.tenantId,
-        amountCents: billing.amountCents,
+      const accountCheck = await this.measureInboundStage('billing', async () => {
+        const check = await this.billing.checkAccount({
+          tenantId: application.tenantId,
+          amountCents: billing.amountCents,
+        });
+        if (check.canSend && billing.amountCents > 0) {
+          await this.billing.freeze({
+            tenantId: application.tenantId,
+            amountCents: billing.amountCents,
+            relatedType: 'sms_batch_task',
+            relatedId: task.id,
+            remark: 'CMPP 入站短信冻结',
+          });
+        }
+        return check;
      });
      if (!accountCheck.canSend) {
        await reject('BALANCE', '企业账户余额不足');
        return;
      }
-      if (billing.amountCents > 0) {
-        await this.billing.freeze({
-          tenantId: application.tenantId,
-          amountCents: billing.amountCents,
-          relatedType: 'sms_batch_task',
-          relatedId: task.id,
-          remark: 'CMPP 入站短信冻结',
+      await this.measureInboundStage('queue_publish', async () => {
+        await this.prisma.smsMessageRecord.update({
+          where: { id: message.id },
+          data: { status: 'queued', signatureId: options.signatureId, drainageInfoId },
        });
-      }
-      await this.prisma.smsMessageRecord.update({
-        where: { id: message.id },
-        data: { status: 'queued', signatureId: options.signatureId, drainageInfoId },
+        await this.prisma.smsBatchTask.update({
+          where: { id: task.id },
+          data: { status: 'ready', riskTaskId: risk.task?.id, auditStatus: 'approved' },
+        });
+        await this.facade.enqueueBatchTask(task.id);
      });
-      await this.prisma.smsBatchTask.update({
-        where: { id: task.id },
-        data: { status: 'ready', riskTaskId: risk.task?.id, auditStatus: 'approved' },
-      });
-      await this.facade.enqueueBatchTask(task.id);
    };
    if (receiptRejection) {
      await reject(receiptRejection.code, receiptRejection.reason);