perf(cmpp): instrument inbound flow and unbatch submit worker

This commit is contained in:
hectorzhao
2026-08-20 11:27:35 +08:00
parent c4f36fc50d
commit b9a71fe0b9
23 changed files with 760 additions and 138 deletions
@@ -11,7 +11,17 @@ describe('GatewayEventsController protocol logging', () => {
const protocolLogs = {
record: jest.fn(),
};
const controller = new GatewayEventsController(sendChain as never, {} as never, protocolLogs as never, { recordEvent: jest.fn() } as never);
const metrics = {
beginCmppInboundStage: jest.fn().mockReturnValue(1n),
finishCmppInboundStage: jest.fn(),
};
const controller = new GatewayEventsController(
sendChain as never,
{} as never,
protocolLogs as never,
{ recordEvent: jest.fn() } as never,
metrics as never,
);
beforeEach(() => {
jest.clearAllMocks();
@@ -77,6 +87,20 @@ describe('GatewayEventsController protocol logging', () => {
}));
});
it('records a failed inbound total without swallowing the service error', async () => {
const failure = new Error('inbound failed');
(sendChain as Record<string, jest.Mock>).submitInboundMessage = jest.fn().mockRejectedValue(failure);
await expect(controller.submitInbound({
account: '607532',
phoneNumber: '13127620092',
content: 'failed',
sequenceId: 142,
})).rejects.toBe(failure);
expect(metrics.finishCmppInboundStage).toHaveBeenCalledWith(1n, 'total', 'error');
});
it('accepts only safe outbound Gateway packet events', () => {
expect(controller.protocolLog({
protocol: 'cmpp',
@@ -155,6 +179,7 @@ describe('GatewayEventsController protocol logging', () => {
messageId: 'MSG-1',
status: 'success',
}));
expect(metrics.finishCmppInboundStage).toHaveBeenCalledWith(1n, 'total', 'success');
});
it('replaces a fallback receipt identifier with the resolved main message identifier', async () => {
@@ -1,4 +1,4 @@
import { BadRequestException, Body, Controller, Post } from '@nestjs/common';
import { BadRequestException, Body, Controller, Optional, Post } from '@nestjs/common';
import { ApiTags } from '@nestjs/swagger';
import {
GatewayInboundAuthDto,
@@ -19,6 +19,7 @@ import { GatewayDownstreamConnectionEventDto } from '../sms-config/sms-config.co
import { SmsConfigService } from '../sms-config/sms-config.service';
import { ProtocolLogsService, type ProtocolLogInput } from '../protocol-logs/protocol-logs.service';
import { SecurityDetectionService } from '../security-detection/security-detection.service';
import { MetricsService } from '../metrics/metrics.service';
@ApiTags('gateway-events')
@Controller('gateway/events')
@@ -28,6 +29,7 @@ export class GatewayEventsController {
private readonly smsConfig: SmsConfigService,
private readonly protocolLogs: ProtocolLogsService,
private readonly security: SecurityDetectionService,
@Optional() private readonly metrics?: MetricsService,
) {}
@Post('submit-result')
@@ -139,6 +141,9 @@ export class GatewayEventsController {
direction: ProtocolLogInput['direction'] = 'channel_to_platform',
) {
const startedAt = Date.now();
const inboundMetricStartedAt = eventType === 'submit' && direction === 'client_to_platform'
? this.metrics?.beginCmppInboundStage()
: undefined;
const value = body as Record<string, unknown>;
const common: Omit<ProtocolLogInput, 'status'> = {
protocol: 'cmpp',
@@ -173,6 +178,9 @@ export class GatewayEventsController {
status: 'success',
durationMs: Date.now() - startedAt,
});
if (inboundMetricStartedAt != null) {
this.metrics?.finishCmppInboundStage(inboundMetricStartedAt, 'total', 'success');
}
return result;
} catch (error) {
this.protocolLogs.record({
@@ -181,6 +189,9 @@ export class GatewayEventsController {
durationMs: Date.now() - startedAt,
detail: { error: error instanceof Error ? error.message : 'unknown error' },
});
if (inboundMetricStartedAt != null) {
this.metrics?.finishCmppInboundStage(inboundMetricStartedAt, 'total', 'error');
}
throw error;
}
}
+3
View File
@@ -12,6 +12,7 @@ import { PhoneRoutingLookupService } from '../dictionaries/phone-routing-lookup.
import { PrismaService } from '../prisma/prisma.service';
import { RiskReviewService } from '../risk-review/risk-review.service';
import { PhoneFrequencyService } from '../risk-review/phone-frequency.service';
import { MetricsService } from '../metrics/metrics.service';
import { OpenApiService } from '../open-api/open-api.service';
import type { CreateBatchTaskDto, CreateHttpBatchTaskDto, GatewayInboundAuthDto, GatewayInboundSubmitDto, GatewayInboundSingleSubmitResult, GatewaySubmitResultDto, GatewaySubmitSegmentResultDto, GatewayReceiptEventDto, GatewayUplinkEventDto, UplinkMatchCandidateInput, GatewayPendingDeliveryQueryDto, GatewayDownstreamSentDto, GatewayDownstreamAcknowledgedDto, GatewayDownstreamFailureType, GatewayControlDeliveryResult, GatewaySubmitDeadLetterDto, RequeueGatewaySubmitExceptionDto, GatewayDownstreamRecoveryStatusDto, TimeoutUnknownDto, ImportPreviewDto, ConfirmImportDto, SendJob, QueuePriority, RoutedChannel } from './send-chain.contracts';
import { SEND_QUEUE, GATEWAY_SUBMIT_QUEUE, GATEWAY_SUBMIT_STREAM, DEFAULT_DOWNSTREAM_RETRY_DELAY_MS, DEFAULT_DOWNSTREAM_RETRY_MAX_DELAY_MS, DEFAULT_DOWNSTREAM_MAX_RETRIES, DEFAULT_DOWNSTREAM_PENDING_TIMEOUT_HOURS, DEFAULT_RECEIPT_TIMEOUT_HOURS, DEFAULT_RECEIPT_TIMEOUT_SCAN_INTERVAL_MS, RECEIPT_TIMEOUT_INITIAL_DELAY_MS, DEFAULT_SCHEDULED_DISPATCH_SCAN_INTERVAL_MS, DEFAULT_SCHEDULED_DISPATCH_STALE_MS, SCHEDULED_DISPATCH_INITIAL_DELAY_MS, DEFAULT_GATEWAY_SUBMIT_REQUEUE_STALE_MS, DEFAULT_DOWNSTREAM_MANUAL_REQUEUE_STALE_MS, DEFAULT_INBOUND_LONG_MESSAGE_SCAN_INTERVAL_MS, INBOUND_LONG_MESSAGE_SCAN_INITIAL_DELAY_MS, DEFAULT_INBOUND_LONG_MESSAGE_PROCESSING_STALE_SECONDS, DEFAULT_UPSTREAM_RECEIPT_INBOX_SCAN_INTERVAL_MS, UPSTREAM_RECEIPT_INBOX_INITIAL_DELAY_MS, DEFAULT_UPSTREAM_RECEIPT_INBOX_PROCESSING_STALE_MS, DEFAULT_UPSTREAM_RECEIPT_INBOX_MAX_ATTEMPTS, DEFAULT_UPSTREAM_RECEIPT_INBOX_MAX_AGE_HOURS, GATEWAY_SUBMIT_REQUEUE_IDEMPOTENCY_TTL_SECONDS, BULLMQ_PRIORITY, gatewaySubmitRequeueKey, drainageRejectionReason, statusFromRisk, parseSchedule, isObjectRecord, asDateOrNull, downstreamRetryDelayMs, downstreamAckTimeoutMs, downstreamRetryBaseDelayMs, downstreamRetryMaxDelayMs, downstreamMaxRetries, downstreamPendingTimeoutHours, downstreamControlFailureMessage, parseImportRows, splitImportLine, cellByHeader, normalizeCarrier, normalizeQueuePriority, getPositiveConfigInteger, getNonNegativeConfigInteger, isCarrierCompatible, normalizeRegion, matchTemplateContent, escapeRegularExpression, isNationalChannel, isProvinceChannel, validateInboundApplicationSrcId, composeUpstreamSrcId, positiveInteger, parseOptionalSequenceId, normalizeSubmitStatus, normalizeReceiptStatus, downstreamDeliveryAttemptKey, shanghaiDateKey, bullmqConnection, matchesApplicationSecret, octetString, hasRecoveryAuditStateChanged, normalizeRecoveryFailureCategory } from './send-chain.helpers';
@@ -50,6 +51,7 @@ export class SendChainService implements OnModuleInit, OnModuleDestroy {
private readonly phoneFrequency: PhoneFrequencyService,
@Optional() @Inject(forwardRef(() => OpenApiService)) private readonly openApi?: OpenApiService,
@Optional() phoneRouting?: PhoneRoutingLookupService,
@Optional() metrics?: MetricsService,
) {
const resolvedPhoneRouting = phoneRouting ?? new PhoneRoutingLookupService(prisma);
this.submission = new SendSubmissionService(
@@ -64,6 +66,7 @@ export class SendChainService implements OnModuleInit, OnModuleDestroy {
recordCmppFailureReceipt: (message, errorCode, reason) =>
this.recordCmppFailureReceipt(message, errorCode, reason),
},
metrics,
);
this.completion = new SendCompletionService(
prisma,
+107 -62
View File
@@ -11,6 +11,7 @@ import { PhoneRoutingLookupService } from '../dictionaries/phone-routing-lookup.
import { PrismaService } from '../prisma/prisma.service';
import { RiskReviewService } from '../risk-review/risk-review.service';
import { PhoneFrequencyService } from '../risk-review/phone-frequency.service';
import { MetricsService, type CmppInboundStage } from '../metrics/metrics.service';
import type { CreateBatchTaskDto, CreateHttpBatchTaskDto, GatewayInboundAuthDto, GatewayInboundSubmitDto, GatewayInboundSingleSubmitResult, ImportPreviewDto, ConfirmImportDto, SendJob, QueuePriority, RoutedChannel } from './send-chain.contracts';
import { SEND_QUEUE, GATEWAY_SUBMIT_QUEUE, GATEWAY_SUBMIT_STREAM, DEFAULT_SCHEDULED_DISPATCH_STALE_MS, DEFAULT_INBOUND_LONG_MESSAGE_PROCESSING_STALE_SECONDS, GATEWAY_SUBMIT_REQUEUE_IDEMPOTENCY_TTL_SECONDS, BULLMQ_PRIORITY, statusFromRisk, parseSchedule, parseImportRows, splitImportLine, cellByHeader, normalizeCarrier, normalizeQueuePriority, getPositiveConfigInteger, getNonNegativeConfigInteger, isCarrierCompatible, matchTemplateContent, isNationalChannel, validateInboundApplicationSrcId, composeUpstreamSrcId, positiveInteger, parseOptionalSequenceId, shanghaiDateKey, bullmqConnection, matchesApplicationSecret, octetString, selectChannelCandidate } from './send-chain.helpers';
import type { SendSubmissionCallbacks, SendSubmissionService } from './send-submission.service';
@@ -30,8 +31,21 @@ export class SendInboundEntryService {
private readonly phoneRouting: PhoneRoutingLookupService,
private readonly facade: SendSubmissionService,
private readonly callbacks: SendSubmissionCallbacks,
private readonly metrics?: MetricsService,
) {}
private async measureInboundStage<T>(stage: CmppInboundStage, action: () => Promise<T>): Promise<T> {
const startedAt = this.metrics?.beginCmppInboundStage();
try {
const result = await action();
if (startedAt != null) this.metrics?.finishCmppInboundStage(startedAt, stage, 'success');
return result;
} catch (error) {
if (startedAt != null) this.metrics?.finishCmppInboundStage(startedAt, stage, 'error');
throw error;
}
}
private releaseMessageReservation(
message: { tenantId: string; batchTaskId: string; messageId: string; amountCents: number | bigint; billingUnits: number },
remark: string,
@@ -145,7 +159,10 @@ async submitInboundMessage(data: GatewayInboundSubmitDto) {
throw new BadRequestException('CMPP submit phone number is invalid');
}
const application = await this.facade.findInboundApplication(data.account);
const application = await this.measureInboundStage(
'application_lookup',
() => this.facade.findInboundApplication(data.account),
);
if (!application) {
throw new BadRequestException('CMPP account is invalid');
}
@@ -157,7 +174,10 @@ async submitInboundMessage(data: GatewayInboundSubmitDto) {
throw new BadRequestException('CMPP source IP is not in application allowlist');
}
validateInboundApplicationSrcId(data.srcId, application);
const collection = await this.facade.collectInboundLongMessageFragment(data, application, phoneNumbers);
const collection = await this.measureInboundStage(
'long_message_fragment',
() => this.facade.collectInboundLongMessageFragment(data, application, phoneNumbers),
);
if (collection.response) {
return collection.response;
}
@@ -180,16 +200,18 @@ async submitInboundMessage(data: GatewayInboundSubmitDto) {
};
}
try {
const response = await this.facade.recoverCompletedInboundLongMessageResponse(
collection.messageId,
phoneNumbers,
) ?? await this.facade.submitCompleteInboundMessage({
...data,
content: collection.content,
sequenceId: collection.sequenceId,
registeredDelivery: collection.registeredDelivery ? 1 : 0,
longMessage: undefined,
}, phoneNumbers, application, collection.messageId);
const response = await this.measureInboundStage('complete_submit', async () => (
await this.facade.recoverCompletedInboundLongMessageResponse(
collection.messageId,
phoneNumbers,
) ?? await this.facade.submitCompleteInboundMessage({
...data,
content: collection.content,
sequenceId: collection.sequenceId,
registeredDelivery: collection.registeredDelivery ? 1 : 0,
longMessage: undefined,
}, phoneNumbers, application, collection.messageId)
));
await this.prisma.cmppInboundLongMessage.update({
where: { id: collection.groupId },
data: {
@@ -210,7 +232,10 @@ async submitInboundMessage(data: GatewayInboundSubmitDto) {
throw error;
}
}
return this.facade.submitCompleteInboundMessage(data, phoneNumbers, application);
return this.measureInboundStage(
'complete_submit',
() => this.facade.submitCompleteInboundMessage(data, phoneNumbers, application),
);
}
async recoverCompletedInboundLongMessageResponse(messageId: string, phoneNumbers: string[]) {
@@ -267,8 +292,9 @@ async submitCompleteInboundMessage(
if (!application) {
throw new BadRequestException('CMPP account is invalid');
}
const persisted = requestedGroupMessageId
? await this.prisma.smsMessageRecord.findMany({
const precheck = await this.measureInboundStage('submission_precheck', async () => {
const persisted = requestedGroupMessageId
? await this.prisma.smsMessageRecord.findMany({
where: {
cmppSubmitGroupMessageId: requestedGroupMessageId,
phoneNumber: { in: phoneNumbers },
@@ -284,15 +310,18 @@ async submitCompleteInboundMessage(
errorCode: true,
},
})
: [];
const persistedByPhone = new Map(persisted.map((item) => [item.phoneNumber, item]));
const phoneRejections = await this.facade.classifyRejectedPhones(application.tenantId, application.id, phoneNumbers);
const missingPhoneCount = phoneNumbers.filter((phoneNumber) => (
!persistedByPhone.has(phoneNumber) && !phoneRejections.has(phoneNumber)
)).length;
const dailyQuota = missingPhoneCount > 0
? await this.facade.tryReserveDailySendQuota(application.id, missingPhoneCount)
: { reserved: true, dailyLimit: application.dailyLimit ?? 100000 };
: [];
const persistedByPhone = new Map(persisted.map((item) => [item.phoneNumber, item]));
const phoneRejections = await this.facade.classifyRejectedPhones(application.tenantId, application.id, phoneNumbers);
const missingPhoneCount = phoneNumbers.filter((phoneNumber) => (
!persistedByPhone.has(phoneNumber) && !phoneRejections.has(phoneNumber)
)).length;
const dailyQuota = missingPhoneCount > 0
? await this.facade.tryReserveDailySendQuota(application.id, missingPhoneCount)
: { reserved: true, dailyLimit: application.dailyLimit ?? 100000 };
return { persistedByPhone, phoneRejections, dailyQuota, missingPhoneCount };
});
const { persistedByPhone, phoneRejections, dailyQuota, missingPhoneCount } = precheck;
const dailyLimitRejection = dailyQuota.reserved
? undefined
: {
@@ -523,7 +552,10 @@ async submitInboundSingleMessage(
synchronousRejection?: { code: string; reason: string },
receiptRejection?: { code: string; reason: string },
) {
const application = await this.facade.findInboundApplication(data.account);
const application = await this.measureInboundStage(
'application_lookup',
() => this.facade.findInboundApplication(data.account),
);
if (!application) {
throw new BadRequestException('CMPP account is invalid');
}
@@ -531,7 +563,10 @@ async submitInboundSingleMessage(
throw new BadRequestException('CMPP source IP is not in application allowlist');
}
const clientSrcId = validateInboundApplicationSrcId(data.srcId, application);
const template = await this.facade.resolveInboundTemplateCandidate(application.id, data.content);
const template = await this.measureInboundStage(
'template_match',
() => this.facade.resolveInboundTemplateCandidate(application.id, data.content),
);
const templateVariables = template ? matchTemplateContent(template.content, data.content) ?? {} : {};
const unitPrice = moneyToNumber(application.customerUnitPrice);
const queuePriority = normalizeQueuePriority(application.queuePriority);
@@ -542,7 +577,7 @@ async submitInboundSingleMessage(
phoneCount: 1,
unitPrice,
});
const task = await this.prisma.smsBatchTask.create({
const task = await this.measureInboundStage('task_persist', () => this.prisma.smsBatchTask.create({
data: {
tenantId: application.tenantId,
applicationId: application.id,
@@ -556,8 +591,8 @@ async submitInboundSingleMessage(
rejectReason: synchronousRejection?.reason,
progressTotal: 1,
},
});
await this.prisma.smsApiRequest.create({
}));
await this.measureInboundStage('api_request_persist', () => this.prisma.smsApiRequest.create({
data: {
tenantId: application.tenantId,
batchTaskId: task.id,
@@ -567,9 +602,12 @@ async submitInboundSingleMessage(
payloadSummary: { phoneTotal: 1, contentLength: [...data.content].length, account: data.account },
status: synchronousRejection ? 'rejected' : 'accepted',
},
});
const drainageDetection = await detectDrainageContent(this.prisma, data.content);
const message = await this.prisma.smsMessageRecord.create({
}));
const drainageDetection = await this.measureInboundStage(
'content_detection',
() => detectDrainageContent(this.prisma, data.content),
);
const message = await this.measureInboundStage('message_persist', () => this.prisma.smsMessageRecord.create({
data: {
tenantId: application.tenantId,
batchTaskId: task.id,
@@ -592,7 +630,7 @@ async submitInboundSingleMessage(
errorCode: synchronousRejection?.code,
errorMessage: synchronousRejection?.reason,
},
});
}));
if (synchronousRejection) {
return {
@@ -614,16 +652,18 @@ async submitInboundSingleMessage(
await this.recordCmppFailureReceipt(message, code, reason);
};
const queueAfterRiskChecks = async (options: { templateId?: string; signatureId?: string }) => {
const drainage = await this.facade.resolveDrainageInfoMatch(options.signatureId, data.content);
const drainageInfoId = drainage?.id;
const risk = await this.facade.evaluateRiskWithPhoneFrequency({
tenantId: application.tenantId,
applicationId: application.id,
templateId: options.templateId,
content: data.content,
variables: options.templateId ? templateVariables : undefined,
phoneNumber: data.phoneNumber,
sourceType: 'cmpp',
const { drainageInfoId, risk } = await this.measureInboundStage('risk_frequency', async () => {
const drainage = await this.facade.resolveDrainageInfoMatch(options.signatureId, data.content);
const evaluatedRisk = await this.facade.evaluateRiskWithPhoneFrequency({
tenantId: application.tenantId,
applicationId: application.id,
templateId: options.templateId,
content: data.content,
variables: options.templateId ? templateVariables : undefined,
phoneNumber: data.phoneNumber,
sourceType: 'cmpp',
});
return { drainageInfoId: drainage?.id, risk: evaluatedRisk };
});
if (risk.status === 'rejected') {
await reject('RISK', risk.reason || '短信被风控拒绝');
@@ -645,32 +685,37 @@ async submitInboundSingleMessage(
});
return;
}
const accountCheck = await this.billing.checkAccount({
tenantId: application.tenantId,
amountCents: billing.amountCents,
const accountCheck = await this.measureInboundStage('billing', async () => {
const check = await this.billing.checkAccount({
tenantId: application.tenantId,
amountCents: billing.amountCents,
});
if (check.canSend && billing.amountCents > 0) {
await this.billing.freeze({
tenantId: application.tenantId,
amountCents: billing.amountCents,
relatedType: 'sms_batch_task',
relatedId: task.id,
remark: 'CMPP 入站短信冻结',
});
}
return check;
});
if (!accountCheck.canSend) {
await reject('BALANCE', '企业账户余额不足');
return;
}
if (billing.amountCents > 0) {
await this.billing.freeze({
tenantId: application.tenantId,
amountCents: billing.amountCents,
relatedType: 'sms_batch_task',
relatedId: task.id,
remark: 'CMPP 入站短信冻结',
await this.measureInboundStage('queue_publish', async () => {
await this.prisma.smsMessageRecord.update({
where: { id: message.id },
data: { status: 'queued', signatureId: options.signatureId, drainageInfoId },
});
}
await this.prisma.smsMessageRecord.update({
where: { id: message.id },
data: { status: 'queued', signatureId: options.signatureId, drainageInfoId },
await this.prisma.smsBatchTask.update({
where: { id: task.id },
data: { status: 'ready', riskTaskId: risk.task?.id, auditStatus: 'approved' },
});
await this.facade.enqueueBatchTask(task.id);
});
await this.prisma.smsBatchTask.update({
where: { id: task.id },
data: { status: 'ready', riskTaskId: risk.task?.id, auditStatus: 'approved' },
});
await this.facade.enqueueBatchTask(task.id);
};
if (receiptRejection) {
await reject(receiptRejection.code, receiptRejection.reason);
@@ -4,6 +4,7 @@ import { PhoneRoutingLookupService } from '../dictionaries/phone-routing-lookup.
import { PrismaService } from '../prisma/prisma.service';
import { RiskReviewService } from '../risk-review/risk-review.service';
import { PhoneFrequencyService } from '../risk-review/phone-frequency.service';
import { MetricsService } from '../metrics/metrics.service';
import type { CreateBatchTaskDto, CreateHttpBatchTaskDto, GatewayInboundAuthDto, GatewayInboundSubmitDto, ImportPreviewDto, ConfirmImportDto, SendJob, QueuePriority, RoutedChannel } from './send-chain.contracts';
import { SendBatchEntryService } from './send-batch-entry.service';
import { SendGatewaySubmitService } from './send-gateway-submit.service';
@@ -56,9 +57,10 @@ export class SendSubmissionService {
phoneRouting: PhoneRoutingLookupService,
facade: SendSubmissionService,
callbacks: SendSubmissionCallbacks,
metrics?: MetricsService,
) {
this.batchEntry = new SendBatchEntryService(prisma, billing, riskReview, phoneFrequency, phoneRouting, facade, callbacks);
this.inboundEntry = new SendInboundEntryService(prisma, billing, riskReview, phoneFrequency, phoneRouting, facade, callbacks);
this.inboundEntry = new SendInboundEntryService(prisma, billing, riskReview, phoneFrequency, phoneRouting, facade, callbacks, metrics);
this.reviewContinuation = new SendReviewContinuationService(prisma, billing, riskReview, phoneFrequency, phoneRouting, facade, callbacks);
this.scheduledDispatch = new SendScheduledDispatchService(prisma, billing, riskReview, phoneFrequency, phoneRouting, facade, callbacks);
this.gatewaySubmit = new SendGatewaySubmitService(prisma, billing, riskReview, phoneFrequency, phoneRouting, facade, callbacks);