import { BadRequestException, HttpException, HttpStatus, Logger, NotFoundException } from '@nestjs/common'; import { Prisma } from '@prisma/client'; import { Queue, Worker } from 'bullmq'; import IORedis from 'ioredis'; import { createHash, randomUUID } from 'node:crypto'; import { setTimeout as sleep } from 'node:timers/promises'; import { BillingService } from '../billing/billing.service'; import { isIpAllowed } from '../common/ip-allowlist'; import { moneyToNumber } from '../common/money'; import { PhoneRoutingLookupService } from '../dictionaries/phone-routing-lookup.service'; import { MetricsService, SendWorkerQueueState, SendWorkerStage } from '../metrics/metrics.service'; import { PrismaService } from '../prisma/prisma.service'; import { RiskReviewService } from '../risk-review/risk-review.service'; import { PhoneFrequencyService } from '../risk-review/phone-frequency.service'; import type { CreateBatchTaskDto, CreateHttpBatchTaskDto, GatewayInboundAuthDto, GatewayInboundSubmitDto, GatewayInboundSingleSubmitResult, ImportPreviewDto, ConfirmImportDto, SendJob, QueuePriority, RoutedChannel } from './send-chain.contracts'; import { SEND_QUEUE, GATEWAY_SUBMIT_QUEUE, GATEWAY_SUBMIT_STREAM, DEFAULT_SCHEDULED_DISPATCH_STALE_MS, DEFAULT_INBOUND_LONG_MESSAGE_PROCESSING_STALE_SECONDS, GATEWAY_SUBMIT_REQUEUE_IDEMPOTENCY_TTL_SECONDS, BULLMQ_PRIORITY, drainageRejectionReason, statusFromRisk, parseSchedule, parseImportRows, splitImportLine, cellByHeader, normalizeCarrier, normalizeQueuePriority, getPositiveConfigInteger, getNonNegativeConfigInteger, isCarrierCompatible, matchTemplateContent, isNationalChannel, validateInboundApplicationSrcId, composeUpstreamSrcId, positiveInteger, parseOptionalSequenceId, shanghaiDateKey, bullmqConnection, matchesApplicationSecret, octetString, selectChannelCandidate } from './send-chain.helpers'; import type { SendSubmissionCallbacks, SendSubmissionService } from './send-submission.service'; type PendingSendBatchItem = { job: SendJob; resolve: (value: unknown) => void; reject: (reason: unknown) => void; }; /** * R9 gatewaySubmit implementation. Cross-method calls return through the stable SendChainService seam. */ export class SendGatewaySubmitService { private readonly logger = new Logger('SendChainService'); private redis?: IORedis; private sendQueue?: Queue; private gatewayQueue?: Queue; private worker?: Worker; private sendQueueMetricsTimer?: ReturnType; private submitOutboxTimer?: ReturnType; private submitOutboxRunning = false; private readonly submitOutboxLeaseOwner = `send-worker-${process.pid}-${randomUUID()}`; private sendWorkerInFlight = 0; private sendWorkerConfiguredSlots = 0; private pendingSendBatch: PendingSendBatchItem[] = []; private sendBatchTimer?: ReturnType; private sendBatchFlushing = false; private readonly taskProgressRefreshes = new Map>(); private readonly dirtyTaskProgressRefreshes = new Set(); private readonly openSubmitSessionIds = new Map>(); constructor( private readonly prisma: PrismaService, private readonly billing: BillingService, private readonly riskReview: RiskReviewService, private readonly phoneFrequency: PhoneFrequencyService, private readonly phoneRouting: PhoneRoutingLookupService, private readonly facade: SendSubmissionService, private readonly callbacks: SendSubmissionCallbacks, private readonly metrics?: MetricsService, ) {} async onModuleDestroy() { if (this.sendQueueMetricsTimer) clearInterval(this.sendQueueMetricsTimer); if (this.submitOutboxTimer) clearInterval(this.submitOutboxTimer); if (this.sendBatchTimer) clearTimeout(this.sendBatchTimer); await this.flushSendBatch(); await this.worker?.close(); await this.sendQueue?.close(); await this.gatewayQueue?.close(); this.redis?.disconnect(); } private releaseMessageReservation( message: { tenantId: string; batchTaskId: string; messageId: string; amountCents: number | bigint; billingUnits: number }, remark: string, ) { return this.callbacks.releaseMessageReservation(message, remark); } private recordCmppFailureReceipt( message: { id: string; tenantId?: string | null; batchTaskId?: string | null; applicationId?: string | null; messageId: string; phoneNumber: string; cmppSubmitSequenceId?: string | null; cmppSubmitGroupMessageId?: string | null; }, errorCode: string, reason: string, ) { return this.callbacks.recordCmppFailureReceipt(message, errorCode, reason); } async enqueueBatchTask(taskId: string, preparedMessage?: { messageRecordId: string; queuePriority?: string | null }) { if (preparedMessage) { // CMPP内部任务在当前请求内刚完成持久化且不暴露取消入口,可安全复用已知ID; // 普通批量任务仍走下方查询路径,以保留取消检查和多消息枚举语义。 const queuePriority = normalizeQueuePriority(preparedMessage.queuePriority); await this.facade.getSendQueue().add('send-message', { messageRecordId: preparedMessage.messageRecordId }, { jobId: preparedMessage.messageRecordId, attempts: 3, priority: BULLMQ_PRIORITY[queuePriority], }); await this.prisma.smsBatchTask.update({ where: { id: taskId }, data: { status: 'queued' } }); return { taskId, enqueued: 1 }; } const task = await this.prisma.smsBatchTask.findUnique({ where: { id: taskId } }); if (!task) { throw new NotFoundException('SMS batch task not found'); } if (task.status === 'canceled') { throw new BadRequestException('SMS batch task is canceled'); } const messages = await this.prisma.smsMessageRecord.findMany({ where: { batchTaskId: taskId, status: 'queued' }, select: { id: true, queuePriority: true }, take: 100000, }); const queue = this.facade.getSendQueue(); for (const message of messages) { const queuePriority = normalizeQueuePriority(message.queuePriority); await queue.add('send-message', { messageRecordId: message.id }, { jobId: message.id, attempts: 3, priority: BULLMQ_PRIORITY[queuePriority], }); } await this.prisma.smsBatchTask.update({ where: { id: taskId }, data: { status: 'queued' } }); return { taskId, enqueued: messages.length }; } startWorker() { if (this.worker) { return { status: 'already_started' }; } const connection = bullmqConnection(); const configuredConcurrency = Number(process.env.API_SEND_WORKER_CONCURRENCY ?? 20); this.sendWorkerConfiguredSlots = Number.isInteger(configuredConcurrency) && configuredConcurrency > 0 ? configuredConcurrency : 20; this.metrics?.setSendWorkerSlots(this.sendWorkerConfiguredSlots, this.sendWorkerInFlight); this.worker = new Worker( SEND_QUEUE, async (job) => { this.sendWorkerInFlight += 1; this.metrics?.setSendWorkerSlots(this.sendWorkerConfiguredSlots, this.sendWorkerInFlight); try { return process.env.API_SEND_WORKER_BATCH_ENABLED === 'false' ? await this.facade.processSendJob(job.data) : await this.enqueueSendBatch(job.data); } finally { this.sendWorkerInFlight = Math.max(0, this.sendWorkerInFlight - 1); this.metrics?.setSendWorkerSlots(this.sendWorkerConfiguredSlots, this.sendWorkerInFlight); } }, { connection, concurrency: this.sendWorkerConfiguredSlots }, ); void this.refreshSendQueueMetrics(); this.sendQueueMetricsTimer = setInterval(() => void this.refreshSendQueueMetrics(), 5_000); this.sendQueueMetricsTimer.unref?.(); if (this.submitOutboxEnabled() && process.env.SEND_SUBMIT_OUTBOX_SEPARATE_PROCESS_ENABLED !== 'true') { void this.publishSubmitOutboxBatch(); this.submitOutboxTimer = setInterval( () => void this.publishSubmitOutboxBatch(), getPositiveConfigInteger(process.env, 'SEND_SUBMIT_OUTBOX_POLL_INTERVAL_MS', 25), ); this.submitOutboxTimer.unref?.(); } return { status: 'started' }; } startSubmitOutboxPublisher() { if (this.submitOutboxTimer) return { status: 'already_started' }; if (!this.submitOutboxEnabled()) return { status: 'disabled' }; void this.publishSubmitOutboxBatch(); this.submitOutboxTimer = setInterval( () => void this.publishSubmitOutboxBatch(), getPositiveConfigInteger(process.env, 'SEND_SUBMIT_OUTBOX_POLL_INTERVAL_MS', 25), ); this.submitOutboxTimer.unref?.(); return { status: 'started' }; } private enqueueSendBatch(job: SendJob) { return new Promise((resolve, reject) => { this.pendingSendBatch.push({ job, resolve, reject }); const batchSize = Math.min(128, getPositiveConfigInteger(process.env, 'API_SEND_WORKER_BATCH_SIZE', 32)); if (this.pendingSendBatch.length >= batchSize) { if (this.sendBatchTimer) clearTimeout(this.sendBatchTimer); this.sendBatchTimer = undefined; queueMicrotask(() => void this.flushSendBatch()); return; } if (!this.sendBatchTimer) { this.sendBatchTimer = setTimeout( () => { this.sendBatchTimer = undefined; void this.flushSendBatch(); }, Math.min(25, getPositiveConfigInteger(process.env, 'API_SEND_WORKER_BATCH_WAIT_MS', 3)), ); this.sendBatchTimer.unref?.(); } }); } private async flushSendBatch() { if (this.sendBatchFlushing) return; this.sendBatchFlushing = true; try { const batchSize = Math.min(128, getPositiveConfigInteger(process.env, 'API_SEND_WORKER_BATCH_SIZE', 32)); while (this.pendingSendBatch.length > 0) { const batch = this.pendingSendBatch.splice(0, batchSize); try { const results = await this.processSendJobBatch(batch.map((item) => item.job)); for (const item of batch) item.resolve(results.get(item.job.messageRecordId) ?? { skipped: true }); } catch (error) { for (const item of batch) item.reject(error); } } } finally { this.sendBatchFlushing = false; if (this.pendingSendBatch.length > 0) queueMicrotask(() => void this.flushSendBatch()); } } private async processSendJobBatch(jobs: SendJob[]) { if (jobs.length === 1) { return new Map([[jobs[0].messageRecordId, await this.processSendJob(jobs[0])]]); } const ids = [...new Set(jobs.map((job) => job.messageRecordId))]; const messages = await this.measureSendStage('message_load', () => this.prisma.smsMessageRecord.findMany({ where: { id: { in: ids } }, include: { batchTask: true, template: { include: { signature: true } }, signature: true }, })); const messageById = new Map(messages.map((message) => [message.id, message])); const results = new Map(); const businessMessages = messages.filter((message) => { if (message.status !== 'queued' || !message.tenantId || !message.batchTaskId) { results.set(message.id, { skipped: true }); return false; } return true; }) as Array; for (const id of ids) if (!messageById.has(id)) results.set(id, { skipped: true }); if (businessMessages.length === 0) return results; const { planned, failed } = await this.planRoutesBatch(businessMessages); if (failed.length > 0) await this.failRouteBatch(failed, results); if (planned.length === 0) return results; await Promise.all(planned.map(({ routed }) => ( this.facade.waitForChannelRateLimit(routed.channel.id, routed.channel.rateLimitPerSecond) ))); const sessionByChannel = new Map(); await Promise.all([...new Set(planned.map(({ routed }) => routed.channel.id))].map(async (channelId) => { sessionByChannel.set(channelId, await this.getOpenSubmitSessionId(channelId)); })); const prepared = planned.map(({ message, routed }) => { const submitId = `SUB-${randomUUID()}`; const upstreamSrcId = composeUpstreamSrcId(routed.channel.srcId, message.applicationExtension); return { message, routed, submitId, command: this.buildGatewaySubmitCommand(message, routed, 0, submitId, upstreamSrcId), sessionId: sessionByChannel.get(routed.channel.id), }; }); const writeOutbox = this.submitOutboxEnabled(); await this.measureSendStage('submit_transaction', () => this.prisma.$transaction(async (tx) => { await tx.smsSubmitRecord.createMany({ data: prepared.map(({ message, routed, submitId, sessionId }) => ({ id: randomUUID(), tenantId: message.tenantId, batchTaskId: message.batchTaskId, messageRecordId: message.id, channelId: routed.channel.id, channelGroupId: routed.groupId, channelGroupName: routed.groupName, sessionId, submitId, submitStatus: 'queued', costUnitPrice: routed.channel.unitPrice ?? 0, costAmountCents: moneyToNumber(routed.channel.unitPrice) * Math.max(1, message.billingUnits ?? 1), })), }); const updates = Prisma.join(prepared.map(({ message, routed, submitId }) => Prisma.sql`( ${message.id}::text, ${routed.channel.id}::text, ${routed.carrier}::text, ${routed.province ?? null}::text, ${submitId}::text )`)); await tx.$executeRaw(Prisma.sql` UPDATE "SmsMessageRecord" AS message SET "channelId" = updates."channelId", carrier = updates.carrier, province = updates.province, "submitId" = updates."submitId", status = 'submit_queued', "submitStatus" = 'queued', "receiptStatus" = NULL, "errorCode" = NULL, "errorMessage" = NULL, "updatedAt" = CURRENT_TIMESTAMP FROM (VALUES ${updates}) AS updates(id, "channelId", carrier, province, "submitId") WHERE message.id = updates.id AND message.status = 'queued' `); if (writeOutbox) { await tx.gatewaySubmitOutbox.createMany({ data: prepared.map(({ message, routed, submitId, command }) => ({ id: randomUUID(), submitId, messageRecordId: message.id, channelId: routed.channel.id, payload: command as Prisma.InputJsonValue, })), }); } })); if (!this.submitOutboxPublishEnabled()) { await Promise.all(prepared.map(({ command }) => this.facade.publishGatewaySubmitCommand(command))); } await this.refreshTaskProgressBatch(prepared.map(({ message }) => message)); for (const { message, routed, submitId } of prepared) { results.set(message.id, { submitted: true, messageRecordId: message.id, channelId: routed.channel.id, attempt: 0, submitId, }); this.metrics?.recordSendWorkerResult('completed'); } return results; } private async planRoutesBatch(messages: T[]) { const unresolvedPhones = messages.filter((message) => !message.carrier).map((message) => message.phoneNumber); const provinces = await this.measureSendStage('phone_routing', () => this.phoneRouting.identifyProvinces(unresolvedPhones)); const routeInputs = await Promise.all(messages.map(async (message) => ({ message, carrier: message.carrier ? normalizeCarrier(message.carrier) : normalizeCarrier(await this.phoneRouting.identifyCarrier(message.phoneNumber)), province: message.carrier ? message.province ?? null : provinces.get(message.phoneNumber) ?? null, signatureId: message.signatureId ?? message.template?.signature?.id ?? message.signature?.id ?? null, }))); const valid = routeInputs.filter((input) => input.message.applicationId && input.signatureId); const signatures = [...new Set(valid.map((input) => input.signatureId as string))]; const routes = valid.length === 0 ? [] : await this.measureSendStage('route_lookup', () => this.prisma.channelRouteRule.findMany({ where: { status: 'active', channelId: null, province: null, OR: valid.map((input) => ({ tenantId: input.message.tenantId, applicationId: input.message.applicationId, carrier: input.carrier, })), }, include: { group: { include: { items: { include: { channel: { include: { connectionStates: { where: { status: 'connected', currentConnections: { gt: 0 } } }, reportTasks: { where: { signatureId: { in: signatures }, reportType: 'signature', status: 'approved' } }, }, }, }, orderBy: { priority: 'asc' }, }, }, }, }, orderBy: { priority: 'asc' }, })); const routeByKey = new Map(); for (const route of routes) { const key = `${route.tenantId}:${route.applicationId}:${normalizeCarrier(route.carrier)}`; if (!routeByKey.has(key)) routeByKey.set(key, route); } const planned: Array<{ message: T; routed: RoutedChannel }> = []; const failed: Array<{ message: T; reason: string }> = []; for (const input of routeInputs) { if (!input.message.applicationId) { failed.push({ message: input.message, reason: '短信应用未配置,无法选择通道组' }); continue; } if (!input.signatureId) { failed.push({ message: input.message, reason: '短信签名未配置,无法选择已报备通道' }); continue; } const route = routeByKey.get(`${input.message.tenantId}:${input.message.applicationId}:${input.carrier}`); if (!route) { failed.push({ message: input.message, reason: '企业应用未配置对应运营商通道组' }); continue; } if (route.group.status !== 'active' || normalizeCarrier(route.group.carrier) !== input.carrier) { failed.push({ message: input.message, reason: '企业应用绑定的通道组已停用或运营商不一致' }); continue; } const approvedItems = route.group.items.filter((item) => item.channel.status === 'active' && item.channel.connectionStates.length > 0 && item.channel.reportTasks.some((task) => task.signatureId === input.signatureId && (task.carrier === input.carrier || (process.env.SIGNATURE_REPORT_STRICT_CARRIER !== 'true' && task.approvalScope === 'legacy_channel')))); const selected = selectChannelCandidate(approvedItems, { carrier: input.carrier, province: input.province, excludedChannelIds: new Set(), approvedChannelIds: new Set(approvedItems.map((item) => item.channelId)), routingKey: input.message.id, }); if (!selected) { failed.push({ message: input.message, reason: '无已报备通过且在线的可用通道' }); continue; } planned.push({ message: input.message, routed: { channel: { ...selected.channel, unitPrice: moneyToNumber(selected.channel.unitPrice) }, carrier: input.carrier, province: input.province, groupId: route.groupId, groupName: route.group.name, routeScope: isNationalChannel(selected) ? 'national' : 'province', }, }); } return { planned, failed }; } private async failRouteBatch(failed: Array<{ message: T; reason: string }>, results: Map) { const values = Prisma.join(failed.map(({ message, reason }) => Prisma.sql`(${message.id}::text, ${reason.slice(0, 1000)}::text)`)); await this.prisma.$executeRaw(Prisma.sql` UPDATE "SmsMessageRecord" AS message SET status = 'failed', "errorMessage" = failures.reason, "updatedAt" = CURRENT_TIMESTAMP FROM (VALUES ${values}) AS failures(id, reason) WHERE message.id = failures.id AND message.status = 'queued' `); await Promise.all(failed.map(async ({ message, reason }) => { await this.releaseMessageReservation(message, reason); if (message.batchTask?.sourceType === 'cmpp') await this.recordCmppFailureReceipt(message, 'ROUTE', reason); else await this.facade.refreshTaskProgress(message.batchTaskId); results.set(message.id, { submitted: false, messageRecordId: message.id, status: 'failed', reason }); this.metrics?.recordSendWorkerResult('failed'); })); } private async refreshTaskProgressBatch(messages: Array<{ batchTaskId: string; batchTask?: { sourceType?: string | null; phoneTotal?: number | null } | null; }>) { const singleCmppIds = [...new Set(messages .filter((message) => message.batchTask?.sourceType === 'cmpp' && message.batchTask.phoneTotal === 1) .map((message) => message.batchTaskId))]; if (singleCmppIds.length > 0) { await this.prisma.smsBatchTask.updateMany({ where: { id: { in: singleCmppIds }, sourceType: 'cmpp', phoneTotal: 1 }, data: singleMessageTaskProgress('submit_queued'), }); } const otherTaskIds = [...new Set(messages .filter((message) => !singleCmppIds.includes(message.batchTaskId)) .map((message) => message.batchTaskId))]; await Promise.all(otherTaskIds.map((taskId) => this.facade.refreshTaskProgress(taskId))); } async processSendJob(job: SendJob) { const totalStartedAt = this.metrics?.beginSendWorkerStage(); let totalFinished = false; const finish = (result: 'completed' | 'failed' | 'skipped') => { if (totalFinished) return; totalFinished = true; if (totalStartedAt != null) { this.metrics?.finishSendWorkerStage(totalStartedAt, 'total', result === 'completed' ? 'success' : result === 'skipped' ? 'skipped' : 'error'); } this.metrics?.recordSendWorkerResult(result); }; try { const message = await this.measureSendStage('message_load', () => this.prisma.smsMessageRecord.findUnique({ where: { id: job.messageRecordId }, include: { batchTask: true, template: { include: { signature: true } }, signature: true }, })); if (!message || message.status !== 'queued') { finish('skipped'); return { skipped: true }; } if (!message.tenantId || !message.batchTaskId) { finish('skipped'); return { skipped: true, reason: 'standalone channel test message' }; } const businessMessage = message as typeof message & { tenantId: string; batchTaskId: string }; try { const routed = await this.facade.selectChannelForMessage(businessMessage); const result = await this.facade.submitMessageToGateway(businessMessage, routed, 0); finish(result.submitted ? 'completed' : 'skipped'); return result; } catch (error) { const reason = error instanceof Error ? error.message : '无可用通道组或通道'; await this.prisma.smsMessageRecord.update({ where: { id: message.id }, data: { status: 'failed', errorMessage: reason }, }); await this.releaseMessageReservation(businessMessage, reason); if (message.batchTask?.sourceType === 'cmpp') { await this.recordCmppFailureReceipt(businessMessage, 'ROUTE', reason); } else { await this.facade.refreshTaskProgress( businessMessage.batchTaskId, message.batchTask?.sourceType === 'cmpp' && message.batchTask.phoneTotal === 1 ? 'failed' : undefined, ); } finish('failed'); return { submitted: false, messageRecordId: message.id, status: 'failed', reason }; } } catch (error) { finish('failed'); throw error; } } async submitMessageToGateway( message: { id: string; tenantId: string; batchTaskId: string; applicationId?: string | null; templateId?: string | null; signatureId?: string | null; submitId?: string | null; messageId: string; phoneNumber: string; content: string; billingUnits: number; queuePriority?: string | null; clientSrcId?: string | null; applicationExtension?: string | null; template?: { signature?: { id?: string | null; name?: string | null } | null } | null; signature?: { id?: string | null; name?: string | null } | null; batchTask?: { sourceType?: string | null; phoneTotal?: number | null } | null; }, routed: RoutedChannel, attempt: number, retryOfSubmitRecordId?: string, ) { const channel = routed.channel; const upstreamSrcId = composeUpstreamSrcId(channel.srcId, message.applicationExtension); await this.measureSendStage('rate_limit', () => this.facade.waitForChannelRateLimit(channel.id, channel.rateLimitPerSecond)); const submitId = `SUB-${randomUUID()}`; const sessionId = await this.getOpenSubmitSessionId(channel.id); const command = this.buildGatewaySubmitCommand(message, routed, attempt, submitId, upstreamSrcId); const writeOutbox = this.submitOutboxEnabled(); try { await this.measureSendStage('submit_transaction', () => this.prisma.$transaction(async (tx) => { await tx.smsSubmitRecord.create({ data: { tenantId: message.tenantId, batchTaskId: message.batchTaskId, messageRecordId: message.id, channelId: channel.id, channelGroupId: routed.groupId, channelGroupName: routed.groupName, sessionId, retryOfSubmitRecordId, submitId, submitStatus: 'queued', costUnitPrice: channel.unitPrice ?? 0, costAmountCents: moneyToNumber(channel.unitPrice) * Math.max(1, message.billingUnits ?? 1), }, }); await tx.smsMessageRecord.update({ where: { id: message.id }, data: { channelId: channel.id, carrier: routed.carrier, province: routed.province, submitId, status: 'submit_queued', submitStatus: 'queued', receiptStatus: null, errorCode: null, errorMessage: attempt > 0 ? `第 ${attempt + 1} 次提交,路由至${routed.routeScope === 'national' ? '全国' : '省网'}通道` : undefined, }, }); if (writeOutbox) { await tx.gatewaySubmitOutbox.create({ data: { submitId, messageRecordId: message.id, channelId: channel.id, payload: command as Prisma.InputJsonValue, }, }); } })); if (retryOfSubmitRecordId) { this.logger.log(`sms_retry_claim_acquired ${JSON.stringify({ messageId: message.messageId, messageRecordId: message.id, retryOfSubmitRecordId, submitId, channelId: channel.id, })}`); } } catch (error) { if ( retryOfSubmitRecordId && error instanceof Prisma.PrismaClientKnownRequestError && error.code === 'P2002' ) { const existingRetry = await this.prisma.smsSubmitRecord.findUnique({ where: { retryOfSubmitRecordId }, }); if (existingRetry) { this.logger.warn(`sms_retry_claim_reused ${JSON.stringify({ messageId: message.messageId, messageRecordId: message.id, retryOfSubmitRecordId, submitId: existingRetry.submitId, channelId: existingRetry.channelId, })}`); return { submitted: false, duplicateRetry: true, messageRecordId: message.id, channelId: existingRetry.channelId, attempt, submitId: existingRetry.submitId, }; } } throw error; } if (!this.submitOutboxPublishEnabled()) { await this.measureSendStage('gateway_stream_publish', () => this.facade.publishGatewaySubmitCommand(command)); } await this.measureSendStage('task_progress', () => this.facade.refreshTaskProgress( message.batchTaskId, message.batchTask?.sourceType === 'cmpp' && message.batchTask.phoneTotal === 1 ? 'submit_queued' : undefined, )); return { submitted: true, messageRecordId: message.id, channelId: channel.id, attempt }; } private buildGatewaySubmitCommand( message: { id: string; tenantId: string; batchTaskId: string; applicationId?: string | null; templateId?: string | null; messageId: string; phoneNumber: string; content: string; billingUnits: number; queuePriority?: string | null; applicationExtension?: string | null; template?: { signature?: { name?: string | null } | null } | null; signature?: { name?: string | null } | null; }, routed: RoutedChannel, attempt: number, submitId: string, upstreamSrcId: string, ) { const channel = routed.channel; return { schemaVersion: 'v1', messageType: 'SubmitCommand', traceId: randomUUID(), messageId: message.messageId, channelId: channel.id, createdAt: new Date().toISOString(), tenantId: message.tenantId, applicationId: message.applicationId ?? 'unknown', taskId: message.batchTaskId, submitId, queuePriority: normalizeQueuePriority(message.queuePriority), phoneNumber: message.phoneNumber, content: message.content, signature: message.template?.signature?.name ?? message.signature?.name ?? 'SMS', templateId: message.templateId ?? 'unknown', billingUnits: message.billingUnits, route: { channelCode: channel.code, cmppAccountCode: channel.account, priority: attempt, rateLimitPerSecond: channel.rateLimitPerSecond, carrier: routed.carrier, province: routed.province ?? undefined, scope: routed.routeScope, groupId: routed.groupId, }, cmpp: { serviceId: channel.config && typeof channel.config === 'object' && 'serviceId' in channel.config ? String(channel.config.serviceId) : 'SMS', srcId: upstreamSrcId, extensionDigits: getNonNegativeConfigInteger(channel.config, 'extensionDigits', 0), registeredDelivery: 1, msgFmt: 8, }, upstream: { gatewayHost: channel.gatewayHost, gatewayPort: channel.gatewayPort, account: channel.account, passwordCipher: channel.passwordCipher, cmppVersion: channel.cmppVersion, desiredConnections: getPositiveConfigInteger(channel.config, 'desiredConnections', 1), windowSize: getPositiveConfigInteger(channel.config, 'windowSize', 16), heartbeatIntervalSeconds: getPositiveConfigInteger(channel.config, 'heartbeatIntervalSeconds', 30), heartbeatMissThreshold: getPositiveConfigInteger(channel.config, 'heartbeatMissThreshold', 3), connectionWarmupSeconds: getNonNegativeConfigInteger(channel.config, 'connectionWarmupSeconds', 30), connectionDrainTimeoutSeconds: getPositiveConfigInteger(channel.config, 'connectionDrainTimeoutSeconds', 60), submitResponseTimeoutSeconds: getPositiveConfigInteger(channel.config, 'submitResponseTimeoutSeconds', 60), connectionFailureCooldownSeconds: getPositiveConfigInteger(channel.config, 'connectionFailureCooldownSeconds', 30), }, retry: { attempt, maxAttempts: 1 }, }; } private submitOutboxEnabled() { return process.env.SEND_SUBMIT_OUTBOX_SHADOW_ENABLED === 'true' || process.env.SEND_SUBMIT_OUTBOX_PUBLISH_ENABLED === 'true'; } private submitOutboxPublishEnabled() { return process.env.SEND_SUBMIT_OUTBOX_PUBLISH_ENABLED === 'true'; } private async publishSubmitOutboxBatch() { if (this.submitOutboxRunning || !this.submitOutboxEnabled()) return; this.submitOutboxRunning = true; try { const batchSize = Math.min(500, getPositiveConfigInteger(process.env, 'SEND_SUBMIT_OUTBOX_BATCH_SIZE', 64)); const leaseSeconds = Math.min(300, getPositiveConfigInteger(process.env, 'SEND_SUBMIT_OUTBOX_LEASE_SECONDS', 30)); const rows = await this.prisma.$queryRaw>(Prisma.sql` WITH candidates AS ( SELECT id FROM "GatewaySubmitOutbox" WHERE ( (status = 'pending' AND "nextAttemptAt" <= CURRENT_TIMESTAMP) OR (status = 'publishing' AND "leaseExpiresAt" < CURRENT_TIMESTAMP) ) ORDER BY "createdAt", id LIMIT ${batchSize} FOR UPDATE SKIP LOCKED ) UPDATE "GatewaySubmitOutbox" AS outbox SET status = 'publishing', "leaseOwner" = ${this.submitOutboxLeaseOwner}, "leaseExpiresAt" = CURRENT_TIMESTAMP + (${leaseSeconds} * INTERVAL '1 second'), "attemptCount" = outbox."attemptCount" + 1, "updatedAt" = CURRENT_TIMESTAMP FROM candidates WHERE outbox.id = candidates.id RETURNING outbox.id, outbox."submitId", outbox.payload `); if (rows.length === 0) return; const results = this.submitOutboxPublishEnabled() ? await this.publishGatewaySubmitCommandBatch(rows) : rows.map((row) => ({ row, streamEntryId: `shadow:${row.submitId}` })); const succeeded = results.filter((result): result is { row: typeof rows[number]; streamEntryId: string } => 'streamEntryId' in result); if (succeeded.length > 0) { const values = Prisma.join(succeeded.map(({ row, streamEntryId }) => Prisma.sql`(${row.id}, ${streamEntryId})`)); await this.prisma.$executeRaw(Prisma.sql` UPDATE "GatewaySubmitOutbox" AS outbox SET status = 'published', "streamEntryId" = published."streamEntryId", "publishedAt" = CURRENT_TIMESTAMP, "leaseOwner" = NULL, "leaseExpiresAt" = NULL, "lastError" = NULL, "updatedAt" = CURRENT_TIMESTAMP FROM (VALUES ${values}) AS published(id, "streamEntryId") WHERE outbox.id = published.id AND outbox.status = 'publishing' AND outbox."leaseOwner" = ${this.submitOutboxLeaseOwner} `); } for (const result of results) { if ('streamEntryId' in result) continue; const { row, error } = result; try { const message = error instanceof Error ? error.message : String(error); await this.prisma.$executeRaw(Prisma.sql` UPDATE "GatewaySubmitOutbox" SET status = CASE WHEN "attemptCount" >= 10 THEN 'dead' ELSE 'pending' END, "nextAttemptAt" = CURRENT_TIMESTAMP + (LEAST(60, POWER(2, LEAST("attemptCount", 6))) * INTERVAL '1 second'), "leaseOwner" = NULL, "leaseExpiresAt" = NULL, "lastError" = ${message.slice(0, 1000)}, "updatedAt" = CURRENT_TIMESTAMP WHERE id = ${row.id} AND "leaseOwner" = ${this.submitOutboxLeaseOwner} `); } catch (recordError) { this.logger.error(`gateway_submit_outbox_failure_record_failed ${recordError instanceof Error ? recordError.message : String(recordError)}`); } } } catch (error) { this.logger.error(`gateway_submit_outbox_publish_failed ${error instanceof Error ? error.message : String(error)}`); } finally { this.submitOutboxRunning = false; } } private async publishGatewaySubmitCommandBatch( rows: Array<{ id: string; submitId: string; payload: Prisma.JsonValue }>, ): Promise> { const redis = this.facade.getRedis(); const stream = process.env.GATEWAY_SUBMIT_STREAM ?? GATEWAY_SUBMIT_STREAM; const script = `local existing = redis.call('GET', KEYS[2]) if existing then return existing end local streamId = redis.call('XADD', KEYS[1], '*', 'messageType', 'SubmitCommand', 'data', ARGV[1]) redis.call('SET', KEYS[2], streamId, 'EX', ARGV[2]) return streamId`; const pipeline = redis.pipeline(); for (const row of rows) { pipeline.eval( script, 2, stream, `gateway:submit:outbox:${row.submitId}`, JSON.stringify(row.payload), String(GATEWAY_SUBMIT_REQUEUE_IDEMPOTENCY_TTL_SECONDS), ); } const replies = await pipeline.exec(); if (!replies || replies.length !== rows.length) { return rows.map((row) => ({ row, error: new Error('Redis Outbox pipeline result count mismatch') })); } return replies.map(([error, value], index) => error ? { row: rows[index], error } : { row: rows[index], streamEntryId: typeof value === 'string' ? value : String(value ?? '') }); } async selectChannelForMessage( message: { id: string; tenantId: string; applicationId?: string | null; templateId?: string | null; signatureId?: string | null; phoneNumber: string; carrier?: string | null; province?: string | null; template?: { signature?: { id?: string | null } | null } | null; signature?: { id?: string | null } | null }, options: { forceNational?: boolean; excludeChannelIds?: string[] } = {}, ): Promise { if (!message.applicationId) { throw new BadRequestException('短信应用未配置,无法选择通道组'); } const hasPersistedRouting = Boolean(message.carrier); const [carrier, province] = await this.measureSendStage('phone_routing', async () => { const resolved = hasPersistedRouting ? [normalizeCarrier(message.carrier), message.province ?? null] as const : await Promise.all([ this.facade.identifyCarrier(message.phoneNumber), this.facade.identifyProvince(message.phoneNumber), ]); if (!hasPersistedRouting) { await this.prisma.smsMessageRecord.update({ where: { id: message.id }, data: { carrier: resolved[0], province: resolved[1] }, }); } return resolved; }); const signatureId = await this.measureSendStage('signature_candidates', () => this.facade.resolveMessageSignatureId(message)); if (!signatureId) throw new BadRequestException('短信签名未配置,无法选择已报备通道'); const route = await this.measureSendStage('route_lookup', () => this.facade.findApplicationRoute( message.tenantId, message.applicationId ?? undefined, carrier, signatureId, )); const excluded = new Set(options.excludeChannelIds ?? []); const approvedChannelIds = new Set(route.group.items.map((item) => item.channelId)); const selected = selectChannelCandidate(route.group.items, { carrier, province, forceNational: options.forceNational, excludedChannelIds: excluded, approvedChannelIds, routingKey: message.id, }); if (!selected) { throw new NotFoundException('无已报备通过且在线的可用通道'); } return { channel: { ...selected.channel, unitPrice: moneyToNumber(selected.channel.unitPrice) }, carrier, province, groupId: route.groupId, groupName: route.group.name, routeScope: isNationalChannel(selected) ? 'national' : 'province', }; } private async measureSendStage(stage: SendWorkerStage, operation: () => Promise): Promise { const startedAt = this.metrics?.beginSendWorkerStage(); try { const result = await operation(); if (startedAt != null) this.metrics?.finishSendWorkerStage(startedAt, stage, 'success'); return result; } catch (error) { if (startedAt != null) this.metrics?.finishSendWorkerStage(startedAt, stage, 'error'); throw error; } } private async refreshSendQueueMetrics() { if (!this.metrics) return; try { const counts = await this.facade.getSendQueue().getJobCounts('wait', 'active', 'completed', 'failed', 'delayed', 'prioritized'); const mappings: Array<[SendWorkerQueueState, number]> = [ ['waiting', counts.wait ?? 0], ['active', counts.active ?? 0], ['completed', counts.completed ?? 0], ['failed', counts.failed ?? 0], ['delayed', counts.delayed ?? 0], ['prioritized', counts.prioritized ?? 0], ]; for (const [state, count] of mappings) this.metrics.setSendWorkerQueueJobs(state, count); const pool = this.prisma.getPoolState(); for (const state of ['max', 'total', 'idle', 'waiting'] as const) { this.metrics.setSendWorkerDatabasePool(state, pool[state]); } } catch (error) { this.logger.warn(`send_queue_metrics_refresh_failed ${error instanceof Error ? error.message : String(error)}`); } } async findApplicationRoute(tenantId: string, applicationId: string | undefined, carrier: string, signatureId?: string) { const approvedChannelWhere = signatureId ? { status: 'active', connectionStates: { some: { status: 'connected', currentConnections: { gt: 0 } } }, reportTasks: { some: { signatureId, reportType: 'signature', status: 'approved', OR: signatureReportApprovalScopes(carrier), }, }, } : undefined; const route = await this.prisma.channelRouteRule.findFirst({ where: { status: 'active', tenantId, applicationId, carrier, channelId: null, province: null, }, include: { group: { include: { items: { where: approvedChannelWhere ? { channel: approvedChannelWhere } : undefined, include: { channel: { include: { connectionStates: approvedChannelWhere ? { where: { status: 'connected', currentConnections: { gt: 0 } } } : true, }, }, }, orderBy: { priority: 'asc' }, }, }, }, }, orderBy: { priority: 'asc' }, }); if (!route) { throw new NotFoundException('企业应用未配置对应运营商通道组'); } if (route.group.status !== 'active') { throw new BadRequestException('企业应用绑定的通道组已停用'); } if (normalizeCarrier(route.group.carrier) !== carrier) { throw new BadRequestException('企业应用绑定的通道组运营商与路由规则不一致'); } return route; } async identifyCarrier(phoneNumber: string) { return normalizeCarrier(await this.phoneRouting.identifyCarrier(phoneNumber)); } async identifyProvince(phoneNumber: string) { return this.phoneRouting.identifyProvince(phoneNumber); } async ensureSignatureReportedForChannel( message: { id: string; templateId?: string | null; template?: { signature?: { id?: string | null; name?: string | null } | null } | null; signature?: { id?: string | null; name?: string | null } | null; }, channelId: string, carrier: string, ) { const signatureId = await this.facade.resolveMessageSignatureId(message); if (!signatureId) { throw new BadRequestException('短信签名未配置,不能提交到通道'); } const reportTask = await this.prisma.channelSignatureReportTask.findFirst({ where: { signatureId, channelId, reportType: 'signature', status: 'approved', OR: signatureReportApprovalScopes(carrier), }, select: { id: true }, }); if (!reportTask) { throw new BadRequestException('短信签名未在最终通道报备通过'); } } async resolveMessageSignatureId(message: { templateId?: string | null; signatureId?: string | null; template?: { signature?: { id?: string | null } | null } | null; signature?: { id?: string | null } | null }) { const direct = message.signatureId ?? message.template?.signature?.id ?? message.signature?.id ?? null; if (direct || !message.templateId) return direct; const template = await this.prisma.smsTemplate.findUnique({ where: { id: message.templateId }, include: { signature: true } }); return template?.signature?.id ?? null; } async waitForChannelRateLimit(channelId: string, tps: number) { const redis = this.facade.getRedis(); for (;;) { const bucket = `rate:channel:${channelId}:${Math.floor(Date.now() / 1000)}`; const count = await redis.incr(bucket); if (count === 1) { await redis.expire(bucket, 2); } if (count <= Math.max(1, tps)) { return; } await sleep(100); } } async refreshTaskProgress(batchTaskId: string, knownSingleMessageStatus?: string) { if (knownSingleMessageStatus) { const direct = await this.prisma.smsBatchTask.updateMany({ where: { id: batchTaskId, sourceType: 'cmpp', phoneTotal: 1 }, data: singleMessageTaskProgress(knownSingleMessageStatus), }); if (direct.count === 1) return; } else { // Gateway结果、回执和超时回调已经先提交了消息状态。对CMPP单号码内部任务, // 在同一条UPDATE中读取该唯一消息的当前状态并写入精确计数,避免每次回调都 // 对一个只有一行的任务执行GROUP BY;多号码任务继续使用下方聚合路径。 const direct = await this.prisma.$executeRaw(Prisma.sql` UPDATE "SmsBatchTask" AS task SET "progressTotal" = 1, "submittedTotal" = CASE WHEN message.status IN ('submit_queued', 'submitted', 'delivered', 'failed', 'unknown', 'timeout') THEN 1 ELSE 0 END, "successTotal" = CASE WHEN message.status = 'delivered' THEN 1 ELSE 0 END, "failedTotal" = CASE WHEN message.status IN ('submit_failed', 'failed') THEN 1 ELSE 0 END, "unknownTotal" = CASE WHEN message.status = 'unknown' THEN 1 ELSE 0 END, "timeoutTotal" = CASE WHEN message.status = 'timeout' THEN 1 ELSE 0 END, status = CASE WHEN message.status IN ('delivered', 'submit_failed', 'failed', 'timeout') THEN 'finished' WHEN message.status IN ('submit_queued', 'submitted', 'unknown') THEN 'sending' ELSE 'queued' END, "updatedAt" = CURRENT_TIMESTAMP FROM "SmsMessageRecord" AS message WHERE task.id = ${batchTaskId} AND task."sourceType" = 'cmpp' AND task."phoneTotal" = 1 AND message."batchTaskId" = task.id `); if (direct === 1) return; } const running = this.taskProgressRefreshes.get(batchTaskId); if (running) { // A state transition committed after the running aggregate may not be visible // to its snapshot. Mark one trailing pass instead of issuing another identical // GROUP BY concurrently for every message/result/receipt callback. this.dirtyTaskProgressRefreshes.add(batchTaskId); return running; } const refresh = this.refreshTaskProgressUntilClean(batchTaskId); this.taskProgressRefreshes.set(batchTaskId, refresh); try { await refresh; } finally { if (this.taskProgressRefreshes.get(batchTaskId) === refresh) { this.taskProgressRefreshes.delete(batchTaskId); } this.dirtyTaskProgressRefreshes.delete(batchTaskId); } } private async refreshTaskProgressUntilClean(batchTaskId: string) { do { this.dirtyTaskProgressRefreshes.delete(batchTaskId); const groups = await this.prisma.smsMessageRecord.groupBy({ by: ['status'], where: { batchTaskId }, _count: { _all: true }, }); const count = (statuses: string[]) => groups.filter((group) => statuses.includes(group.status)).reduce((sum, group) => sum + group._count._all, 0); const progressTotal = groups.reduce((sum, group) => sum + group._count._all, 0); const submittedTotal = count(['submit_queued', 'submitted', 'delivered', 'failed', 'unknown', 'timeout']); const successTotal = count(['delivered']); const failedTotal = count(['submit_failed', 'failed']); const unknownTotal = count(['unknown']); const timeoutTotal = count(['timeout']); const doneTotal = successTotal + failedTotal + timeoutTotal; const status = progressTotal > 0 && doneTotal >= progressTotal ? 'finished' : submittedTotal > 0 ? 'sending' : 'queued'; await this.prisma.smsBatchTask.update({ where: { id: batchTaskId }, data: { progressTotal, submittedTotal, successTotal, failedTotal, unknownTotal, timeoutTotal, status }, }); } while (this.dirtyTaskProgressRefreshes.has(batchTaskId)); } getSendQueue(): Queue { if (!this.sendQueue) { this.sendQueue = new Queue(SEND_QUEUE, { connection: bullmqConnection() }); } return this.sendQueue; } getGatewayQueue(): Queue { if (!this.gatewayQueue) { this.gatewayQueue = new Queue(GATEWAY_SUBMIT_QUEUE, { connection: bullmqConnection() }); } return this.gatewayQueue; } getRedis() { if (!this.redis) { this.redis = new IORedis(process.env.REDIS_URL ?? 'redis://127.0.0.1:6379', { maxRetriesPerRequest: null, }); } return this.redis; } async publishGatewaySubmitCommand(command: unknown, idempotencyKey?: string) { const redis = this.facade.getRedis(); const stream = process.env.GATEWAY_SUBMIT_STREAM ?? GATEWAY_SUBMIT_STREAM; const payload = JSON.stringify(command); if (!idempotencyKey) { return redis.xadd(stream, '*', 'messageType', 'SubmitCommand', 'data', payload); } const result = await redis.eval( `local existing = redis.call('GET', KEYS[2]) if existing then return existing end local streamId = redis.call('XADD', KEYS[1], '*', 'messageType', 'SubmitCommand', 'data', ARGV[1]) redis.call('SET', KEYS[2], streamId, 'EX', ARGV[2]) return streamId`, 2, stream, idempotencyKey, payload, String(GATEWAY_SUBMIT_REQUEUE_IDEMPOTENCY_TTL_SECONDS), ); return typeof result === 'string' ? result : String(result ?? ''); } private getOpenSubmitSessionId(channelId: string) { const cached = this.openSubmitSessionIds.get(channelId); if (cached) return cached; const sessionNo = `OPEN-${channelId}`; const pending = this.prisma.cmppSubmitSession.findUnique({ where: { sessionNo }, select: { id: true }, }).then((existing) => existing ?? this.prisma.cmppSubmitSession.upsert({ where: { sessionNo }, update: {}, create: { channelId, sessionNo, submitTotal: 0 }, select: { id: true }, })).then((session) => session.id).catch((error) => { this.openSubmitSessionIds.delete(channelId); throw error; }); this.openSubmitSessionIds.set(channelId, pending); return pending; } } function singleMessageTaskProgress(status: string) { const submittedTotal = ['submit_queued', 'submitted', 'delivered', 'failed', 'unknown', 'timeout'].includes(status) ? 1 : 0; const successTotal = status === 'delivered' ? 1 : 0; const failedTotal = ['submit_failed', 'failed'].includes(status) ? 1 : 0; const unknownTotal = status === 'unknown' ? 1 : 0; const timeoutTotal = status === 'timeout' ? 1 : 0; const doneTotal = successTotal + failedTotal + timeoutTotal; return { progressTotal: 1, submittedTotal, successTotal, failedTotal, unknownTotal, timeoutTotal, status: doneTotal >= 1 ? 'finished' : submittedTotal > 0 ? 'sending' : 'queued', }; } function signatureReportApprovalScopes(carrier: string) { const exact = { carrier, approvalScope: 'carrier_specific' }; // 迁移期保留双读用于平滑发布;自动转换migration完成且兼容命中清零后再切换严格口径。 return process.env.SIGNATURE_REPORT_STRICT_CARRIER === 'true' ? [exact] : [exact, { carrier: null, approvalScope: 'legacy_channel' }]; }