import { BadRequestException, HttpException, HttpStatus, Logger, NotFoundException } from '@nestjs/common'; import { Prisma } from '@prisma/client'; import { createHash } from 'node:crypto'; import { BillingService } from '../billing/billing.service'; import { moneyToNumber } from '../common/money'; import type { OpenApiService } from '../open-api/open-api.service'; import { PrismaService } from '../prisma/prisma.service'; import type { GatewaySubmitResultDto, GatewaySubmitSegmentResultDto, GatewayReceiptEventDto, GatewayUplinkEventDto, GatewayPendingDeliveryQueryDto, GatewayDownstreamSentDto, GatewayDownstreamAcknowledgedDto, GatewayDownstreamFailureType, GatewayControlDeliveryResult, GatewaySubmitDeadLetterDto, RequeueGatewaySubmitExceptionDto, GatewayDownstreamRecoveryStatusDto, TimeoutUnknownDto } from './send-chain.contracts'; import { positiveInteger, normalizeReceiptStatus, normalizeCarrier, DEFAULT_DOWNSTREAM_RETRY_DELAY_MS, DEFAULT_DOWNSTREAM_RETRY_MAX_DELAY_MS, DEFAULT_DOWNSTREAM_MAX_RETRIES, DEFAULT_DOWNSTREAM_PENDING_TIMEOUT_HOURS, DEFAULT_RECEIPT_TIMEOUT_HOURS, DEFAULT_GATEWAY_SUBMIT_REQUEUE_STALE_MS, DEFAULT_DOWNSTREAM_MANUAL_REQUEUE_STALE_MS, DEFAULT_UPSTREAM_RECEIPT_INBOX_PROCESSING_STALE_MS, DEFAULT_UPSTREAM_RECEIPT_INBOX_MAX_ATTEMPTS, DEFAULT_UPSTREAM_RECEIPT_INBOX_MAX_AGE_HOURS, gatewaySubmitRequeueKey, isObjectRecord, asDateOrNull, downstreamRetryDelayMs, downstreamAckTimeoutMs, downstreamRetryBaseDelayMs, downstreamRetryMaxDelayMs, downstreamMaxRetries, downstreamPendingTimeoutHours, downstreamControlFailureMessage, normalizeSubmitStatus, downstreamDeliveryAttemptKey, hasRecoveryAuditStateChanged, normalizeRecoveryFailureCategory, aggregateReceiptSegmentState, isSameUpstreamEndpointIdentity, receiptEventKey } from './send-chain.helpers'; import type { SendSubmissionService } from './send-submission.service'; import type { SendCompletionCallbacks, SendCompletionFacade } from './send-completion.service'; /** * R10 receipt implementation. * Cross-domain calls return through the stable SendChainService seam to preserve locking and test observability. */ export class SendReceiptService { private readonly logger = new Logger('SendChainService'); private upstreamReceiptInboxScanRunning = false; constructor( private readonly prisma: PrismaService, private readonly billing: BillingService, private readonly openApi: OpenApiService | undefined, private readonly facade: SendCompletionFacade, private readonly callbacks: SendCompletionCallbacks, ) {} async intakeReceipt(data: GatewayReceiptEventDto) { const channel = await this.prisma.smsChannel.findUnique({ where: { id: data.channelId }, select: { id: true, account: true, gatewayHost: true, gatewayPort: true, protocol: true, cmppVersion: true, }, }); if (!channel) { throw new NotFoundException('SMS channel not found'); } const deliveredAt = data.deliveredAt ? new Date(data.deliveredAt) : new Date(); const receiptKey = receiptEventKey(data, data.channelId); const inbox = await this.prisma.upstreamReceiptInbox.upsert({ where: { receiptKey }, update: { incomingConnectionId: data.connectionId, }, create: { receiptKey, incomingChannelId: data.channelId, incomingConnectionId: data.connectionId, upstreamAccount: channel.account, upstreamHost: channel.gatewayHost, upstreamPort: channel.gatewayPort, protocol: channel.protocol, protocolVersion: channel.cmppVersion, provisionalMessageId: data.messageId, sequenceId: data.sequenceId, gatewayMessageId: data.gatewayMessageId, phoneNumber: data.phoneNumber?.trim() || null, receiptStatus: data.receiptStatus, rawStatus: data.rawStatus, errorCode: data.errorCode, errorMessage: data.errorMessage, deliveredAt, status: 'pending', nextRetryAt: new Date(), }, }); if (['pending', 'retrying'].includes(inbox.status)) { setImmediate(() => void this.facade.processUpstreamReceiptInboxRecord(inbox.id)); } return { accepted: true, inboxId: inbox.id, status: inbox.status }; } async processPendingUpstreamReceiptInbox(limit = 100) { const now = new Date(); const staleBefore = new Date( now.getTime() - positiveInteger( process.env.UPSTREAM_RECEIPT_INBOX_PROCESSING_STALE_MS, DEFAULT_UPSTREAM_RECEIPT_INBOX_PROCESSING_STALE_MS, ), ); const candidates = await this.prisma.upstreamReceiptInbox.findMany({ where: { OR: [ { status: { in: ['pending', 'retrying'] }, OR: [{ nextRetryAt: null }, { nextRetryAt: { lte: now } }], }, { status: 'processing', updatedAt: { lte: staleBefore } }, ], }, orderBy: [{ receivedAt: 'asc' }, { id: 'asc' }], take: Math.min(Math.max(limit, 1), 500), select: { id: true }, }); let processed = 0; for (const candidate of candidates) { if (await this.facade.processUpstreamReceiptInboxRecord(candidate.id)) processed += 1; } return { scanned: candidates.length, processed }; } async processUpstreamReceiptInboxRecord(id: string) { const staleBefore = new Date( Date.now() - positiveInteger( process.env.UPSTREAM_RECEIPT_INBOX_PROCESSING_STALE_MS, DEFAULT_UPSTREAM_RECEIPT_INBOX_PROCESSING_STALE_MS, ), ); const claimed = await this.prisma.upstreamReceiptInbox.updateMany({ where: { id, OR: [ { status: { in: ['pending', 'retrying'] } }, { status: 'processing', updatedAt: { lte: staleBefore } }, ], }, data: { status: 'processing', attemptCount: { increment: 1 }, nextRetryAt: null }, }); if (claimed.count !== 1) return false; const inbox = await this.prisma.upstreamReceiptInbox.findUnique({ where: { id } }); if (!inbox) return false; try { const message = await this.facade.handleReceipt({ messageId: inbox.provisionalMessageId ?? undefined, channelId: inbox.incomingChannelId, connectionId: inbox.incomingConnectionId ?? undefined, sequenceId: inbox.sequenceId ?? undefined, gatewayMessageId: inbox.gatewayMessageId, phoneNumber: inbox.phoneNumber ?? undefined, receiptStatus: normalizeReceiptStatus(inbox.receiptStatus), rawStatus: inbox.rawStatus, errorCode: inbox.errorCode ?? undefined, errorMessage: inbox.errorMessage ?? undefined, deliveredAt: inbox.deliveredAt.toISOString(), }, { account: inbox.upstreamAccount, gatewayHost: inbox.upstreamHost, gatewayPort: inbox.upstreamPort, protocol: inbox.protocol, cmppVersion: inbox.protocolVersion, }); const matchedMessageRecordId = message && 'id' in message ? message.id : message?.messageRecordId; const matchedChannelId = message && 'channelId' in message ? message.channelId : undefined; await this.prisma.upstreamReceiptInbox.update({ where: { id }, data: { status: 'matched', matchedMessageRecordId: matchedMessageRecordId ?? null, matchedChannelId: matchedChannelId ?? null, lastError: null, processedAt: new Date(), }, }); return true; } catch (error) { const maxAttempts = positiveInteger( process.env.UPSTREAM_RECEIPT_INBOX_MAX_ATTEMPTS, DEFAULT_UPSTREAM_RECEIPT_INBOX_MAX_ATTEMPTS, ); const maxAgeHours = positiveInteger( process.env.UPSTREAM_RECEIPT_INBOX_MAX_AGE_HOURS, DEFAULT_UPSTREAM_RECEIPT_INBOX_MAX_AGE_HOURS, ); const exhausted = inbox.attemptCount >= maxAttempts || inbox.receivedAt.getTime() <= Date.now() - maxAgeHours * 60 * 60_000; const retryDelayMs = Math.min(30 * 60_000, 5_000 * 2 ** Math.min(Math.max(inbox.attemptCount - 1, 0), 8)); await this.prisma.upstreamReceiptInbox.update({ where: { id }, data: { status: exhausted ? 'unmatched' : 'retrying', nextRetryAt: exhausted ? null : new Date(Date.now() + retryDelayMs), lastError: error instanceof Error ? error.message : String(error), processedAt: exhausted ? new Date() : null, }, }); return false; } } async runUpstreamReceiptInboxScan() { if (this.upstreamReceiptInboxScanRunning) return; this.upstreamReceiptInboxScanRunning = true; try { const result = await this.facade.processPendingUpstreamReceiptInbox(); if (result.processed > 0) { this.logger.log(`Matched ${result.processed}/${result.scanned} pending upstream receipts`); } } catch (error) { this.logger.error('Upstream receipt inbox scan failed', error instanceof Error ? error.stack : String(error)); } finally { this.upstreamReceiptInboxScanRunning = false; } } async handleReceipt( data: GatewayReceiptEventDto, incomingIdentity?: { account: string; gatewayHost: string; gatewayPort: number; protocol: string; cmppVersion: string }, ) { const resolved = await this.facade.resolveReceiptMessage(data, incomingIdentity); const logicalChannelId = resolved.channelId ?? data.channelId; const receiptKey = receiptEventKey(data, logicalChannelId); const existingReceipt = await this.prisma.smsReceiptRecord.findUnique({ where: { receiptKey }, include: { messageRecord: true }, }); if (existingReceipt?.messageRecord) { return existingReceipt.messageRecord; } const message = resolved.message; const deliveredAt = data.deliveredAt ? new Date(data.deliveredAt) : new Date(); if (resolved.submitRecordId) { await this.prisma.smsSubmitRecord.updateMany({ where: { id: resolved.submitRecordId, gatewayMessageId: null, }, data: { gatewayMessageId: data.gatewayMessageId, sequenceId: data.sequenceId, }, }); } try { await this.prisma.smsReceiptRecord.create({ data: { tenantId: message.tenantId, batchTaskId: message.batchTaskId, messageRecordId: message.id, receiptKey, channelId: logicalChannelId, messageId: resolved.messageId, gatewayMessageId: data.gatewayMessageId, phoneNumber: data.phoneNumber?.trim() || message.phoneNumber, sequenceId: data.sequenceId, receiptStatus: data.receiptStatus, rawStatus: data.rawStatus, errorCode: data.errorCode, errorMessage: data.errorMessage, deliveredAt, }, }); } catch (error) { if (error instanceof Prisma.PrismaClientKnownRequestError && error.code === 'P2002') { const duplicate = await this.prisma.smsReceiptRecord.findUnique({ where: { receiptKey }, include: { messageRecord: true }, }); if (duplicate?.messageRecord) return duplicate.messageRecord; } throw error; } const logicalReceipt = { ...data, channelId: logicalChannelId }; await this.facade.recordReceiptSegment(message, logicalReceipt, deliveredAt, resolved.submitRecordId); const aggregate = await this.facade.aggregateReceiptSegments( message, logicalReceipt, deliveredAt, resolved.submitRecordId, resolved.submitId, ); if (!aggregate.terminal) { return this.prisma.smsMessageRecord.findUnique({ where: { id: message.id } }); } const status = aggregate.status; const isCurrentAttempt = (!message.channelId || message.channelId === logicalChannelId) && ( !message.gatewayMessageId || message.gatewayMessageId === data.gatewayMessageId || (aggregate.segmentTotal > 1 && (!message.submitId || message.submitId === resolved.submitId)) ); if (!isCurrentAttempt || (status === 'failed' && message.status === 'delivered')) { return this.prisma.smsMessageRecord.findUnique({ where: { id: message.id } }); } const isStandaloneChannelTest = !message.tenantId && !message.batchTaskId; if (status === 'failed' && !isStandaloneChannelTest && message.tenantId && message.batchTaskId) { const businessMessage = message as typeof message & { tenantId: string; batchTaskId: string }; const retried = await this.facade.retryMessageIfAllowed( businessMessage, '回执失败补发', resolved.submitRecordId, ); if (retried) { await this.facade.refreshTaskProgress(businessMessage.batchTaskId); return retried; } await this.facade.refundMessage(businessMessage, '最终失败退款'); } await this.prisma.smsMessageRecord.update({ where: { id: message.id }, data: { channelId: logicalChannelId, gatewayMessageId: message.gatewayMessageId ?? data.gatewayMessageId, receiptStatus: aggregate.receiptStatus, receiptRawStatus: aggregate.rawStatus, status, errorCode: aggregate.errorCode, errorMessage: aggregate.errorMessage ?? (status === 'delivered' ? null : aggregate.rawStatus), deliveredAt: aggregate.deliveredAt, }, }); if (!isStandaloneChannelTest && message.tenantId && message.applicationId) { await this.facade.queueAndTryDownstreamDelivery({ tenantId: message.tenantId, applicationId: message.applicationId, messageRecordId: message.id, messageId: message.messageId, deliveryType: 'receipt', payload: { messageId: message.messageId, gatewayMessageId: data.gatewayMessageId, phoneNumber: message.phoneNumber, receiptStatus: aggregate.receiptStatus, rawStatus: aggregate.rawStatus, errorCode: aggregate.errorCode, submitSequenceId: message.cmppSubmitSequenceId ? Number(message.cmppSubmitSequenceId) : undefined, submitGroupMessageId: message.cmppSubmitGroupMessageId ?? undefined, deliveredAt: aggregate.deliveredAt.toISOString(), }, }); } if (message.batchTaskId) { await this.facade.refreshTaskProgress(message.batchTaskId); } return this.prisma.smsMessageRecord.findUnique({ where: { id: message.id } }); } async recordReceiptSegment( message: { id: string; tenantId?: string | null; batchTaskId?: string | null; channelId?: string | null; submitId?: string | null; billingUnits?: number | null; }, data: GatewayReceiptEventDto, deliveredAt: Date, submitRecordId?: string, ) { const segmentAudits = this.facade.smsMessageSegmentAuditDelegate(); const updated = await segmentAudits.updateMany({ where: { messageRecordId: message.id, gatewayMessageId: data.gatewayMessageId, }, data: { receiptStatus: data.receiptStatus, rawStatus: data.rawStatus, errorCode: data.errorCode ?? null, deliveredAt, }, }); if (updated.count > 0) { return; } const submitRecord = submitRecordId ? await this.prisma.smsSubmitRecord.findUnique({ where: { id: submitRecordId } }) : await this.prisma.smsSubmitRecord.findFirst({ where: { messageRecordId: message.id, gatewayMessageId: data.gatewayMessageId }, orderBy: { createdAt: 'desc' }, }); await segmentAudits.upsert({ where: { messageRecordId_submitId_segmentIndex: { messageRecordId: message.id, submitId: submitRecord?.submitId ?? message.submitId ?? `RECEIPT-AUDIT-${data.gatewayMessageId}`, segmentIndex: 1, }, }, update: { submitRecordId: submitRecord?.id ?? submitRecordId ?? null, channelId: data.channelId ?? message.channelId ?? null, sequenceId: data.sequenceId ?? null, gatewayMessageId: data.gatewayMessageId, receiptStatus: data.receiptStatus, rawStatus: data.rawStatus, errorCode: data.errorCode ?? null, deliveredAt, }, create: { tenantId: message.tenantId, batchTaskId: message.batchTaskId, messageRecordId: message.id, submitRecordId: submitRecord?.id ?? submitRecordId ?? null, channelId: data.channelId ?? message.channelId ?? null, submitId: submitRecord?.submitId ?? message.submitId ?? `RECEIPT-AUDIT-${data.gatewayMessageId}`, attempt: 0, segmentTotal: Math.max(1, Number(message.billingUnits ?? 1)), segmentIndex: 1, sequenceId: data.sequenceId ?? null, gatewayMessageId: data.gatewayMessageId, submitStatus: submitRecord?.submitStatus ?? 'accepted', receiptStatus: data.receiptStatus, rawStatus: data.rawStatus, compensationType: 'receipt_recovered', errorCode: data.errorCode ?? null, deliveredAt, }, }); } async aggregateReceiptSegments( message: { id: string; billingUnits?: number | null; }, data: GatewayReceiptEventDto, deliveredAt: Date, submitRecordId?: string, submitId?: string, ) { const audits = await this.facade.smsMessageSegmentAuditDelegate().findMany({ where: submitRecordId ? { messageRecordId: message.id, submitRecordId } : submitId ? { messageRecordId: message.id, submitId } : { messageRecordId: message.id, gatewayMessageId: data.gatewayMessageId }, orderBy: { segmentIndex: 'asc' }, }); return aggregateReceiptSegmentState(audits, message.billingUnits, data, deliveredAt); } async resolveReceiptMessage( data: GatewayReceiptEventDto, incomingIdentity?: { account: string; gatewayHost: string; gatewayPort: number; protocol: string; cmppVersion: string }, ) { const exactMessage = data.messageId ? await this.prisma.smsMessageRecord.findUnique({ where: { messageId: data.messageId } }) : null; if (exactMessage) { const segmentAudit = data.gatewayMessageId ? await this.facade.smsMessageSegmentAuditDelegate().findFirst({ where: { messageRecordId: exactMessage.id, gatewayMessageId: data.gatewayMessageId, }, orderBy: { updatedAt: 'desc' }, }) : null; if (segmentAudit) { return { message: exactMessage, messageId: exactMessage.messageId, submitRecordId: segmentAudit.submitRecordId ?? undefined, submitId: segmentAudit.submitId, channelId: segmentAudit.channelId ?? data.channelId, }; } const submitRecord = await this.prisma.smsSubmitRecord.findFirst({ where: { messageRecordId: exactMessage.id, channelId: data.channelId, gatewayMessageId: data.gatewayMessageId, }, orderBy: { createdAt: 'desc' }, }); return { message: exactMessage, messageId: exactMessage.messageId, submitRecordId: submitRecord?.id, submitId: submitRecord?.submitId, channelId: submitRecord?.channelId ?? data.channelId, }; } const phoneNumber = data.phoneNumber?.trim(); const exactSubmits = await this.prisma.smsSubmitRecord.findMany({ where: { channelId: data.channelId, gatewayMessageId: data.gatewayMessageId, ...(phoneNumber ? { messageRecord: { phoneNumber } } : {}), }, include: { messageRecord: true }, orderBy: { createdAt: 'desc' }, take: 2, }); if (exactSubmits.length === 1 && exactSubmits[0]?.messageRecord) { return { message: exactSubmits[0].messageRecord, messageId: exactSubmits[0].messageRecord.messageId, submitRecordId: exactSubmits[0].id, submitId: exactSubmits[0].submitId, channelId: exactSubmits[0].channelId, }; } if (!phoneNumber) { throw new NotFoundException('SMS message record not found'); } const incomingChannel = incomingIdentity ?? await this.prisma.smsChannel.findUnique({ where: { id: data.channelId } }); if (!incomingChannel) { throw new NotFoundException('SMS message record not found'); } const segmentMatches = await this.facade.smsMessageSegmentAuditDelegate().findMany({ where: { gatewayMessageId: data.gatewayMessageId, messageRecord: { phoneNumber }, }, include: { messageRecord: true, submitRecord: true, channel: true }, orderBy: { createdAt: 'desc' }, take: 10, }); const exactSegmentMatches = segmentMatches.filter((candidate) => candidate.channelId === data.channelId); if (exactSegmentMatches.length === 1 && exactSegmentMatches[0]?.messageRecord) { return { message: exactSegmentMatches[0].messageRecord, messageId: exactSegmentMatches[0].messageRecord.messageId, submitRecordId: exactSegmentMatches[0].submitRecordId ?? undefined, submitId: exactSegmentMatches[0].submitRecord?.submitId ?? exactSegmentMatches[0].submitId, channelId: exactSegmentMatches[0].channelId, }; } const sameSupplierSegments = segmentMatches.filter((candidate) => candidate.channel && isSameUpstreamEndpointIdentity(incomingChannel, candidate.channel)); if (sameSupplierSegments.length === 1 && sameSupplierSegments[0]?.messageRecord) { return { message: sameSupplierSegments[0].messageRecord, messageId: sameSupplierSegments[0].messageRecord.messageId, submitRecordId: sameSupplierSegments[0].submitRecordId ?? undefined, submitId: sameSupplierSegments[0].submitRecord?.submitId ?? sameSupplierSegments[0].submitId, channelId: sameSupplierSegments[0].channelId, }; } const crossConnectionSubmits = await this.prisma.smsSubmitRecord.findMany({ where: { gatewayMessageId: data.gatewayMessageId, messageRecord: { phoneNumber }, }, include: { messageRecord: true, channel: true }, orderBy: { createdAt: 'desc' }, take: 10, }); const sameSupplierSubmits = crossConnectionSubmits.filter((candidate) => candidate.channel && isSameUpstreamEndpointIdentity(incomingChannel, candidate.channel)); if (sameSupplierSubmits.length === 1 && sameSupplierSubmits[0]?.messageRecord) { return { message: sameSupplierSubmits[0].messageRecord, messageId: sameSupplierSubmits[0].messageRecord.messageId, submitRecordId: sameSupplierSubmits[0].id, submitId: sameSupplierSubmits[0].submitId, channelId: sameSupplierSubmits[0].channelId, }; } const deliveredAt = data.deliveredAt ? new Date(data.deliveredAt) : new Date(); const submittedAfter = new Date(deliveredAt.getTime() - 72 * 60 * 60 * 1000); const candidates = await this.prisma.smsSubmitRecord.findMany({ where: { channelId: data.channelId, gatewayMessageId: null, submitStatus: 'timeout', submittedAt: { gte: submittedAfter, lte: deliveredAt, }, messageRecord: { phoneNumber, }, }, include: { messageRecord: true, }, orderBy: { submittedAt: 'desc', }, take: 10, }); if (candidates.length !== 1 || !candidates[0]?.messageRecord) { throw new NotFoundException('SMS message record not found'); } return { message: candidates[0].messageRecord, messageId: candidates[0].messageRecord.messageId, submitRecordId: candidates[0].id, submitId: candidates[0].submitId, channelId: candidates[0].channelId, }; } }