Files
lislgosms/api/src/send-chain/send-receipt.service.ts
T

594 lines
23 KiB
TypeScript

import { BadRequestException, HttpException, HttpStatus, Logger, NotFoundException } from '@nestjs/common';
import { Prisma } from '@prisma/client';
import { createHash } from 'node:crypto';
import { BillingService } from '../billing/billing.service';
import { moneyToNumber } from '../common/money';
import type { OpenApiService } from '../open-api/open-api.service';
import { PrismaService } from '../prisma/prisma.service';
import type { GatewaySubmitResultDto, GatewaySubmitSegmentResultDto, GatewayReceiptEventDto, GatewayUplinkEventDto, GatewayPendingDeliveryQueryDto, GatewayDownstreamSentDto, GatewayDownstreamAcknowledgedDto, GatewayDownstreamFailureType, GatewayControlDeliveryResult, GatewaySubmitDeadLetterDto, RequeueGatewaySubmitExceptionDto, GatewayDownstreamRecoveryStatusDto, TimeoutUnknownDto } from './send-chain.contracts';
import { positiveInteger, normalizeReceiptStatus, normalizeCarrier, DEFAULT_DOWNSTREAM_RETRY_DELAY_MS, DEFAULT_DOWNSTREAM_RETRY_MAX_DELAY_MS, DEFAULT_DOWNSTREAM_MAX_RETRIES, DEFAULT_DOWNSTREAM_PENDING_TIMEOUT_HOURS, DEFAULT_RECEIPT_TIMEOUT_HOURS, DEFAULT_GATEWAY_SUBMIT_REQUEUE_STALE_MS, DEFAULT_DOWNSTREAM_MANUAL_REQUEUE_STALE_MS, DEFAULT_UPSTREAM_RECEIPT_INBOX_PROCESSING_STALE_MS, DEFAULT_UPSTREAM_RECEIPT_INBOX_MAX_ATTEMPTS, DEFAULT_UPSTREAM_RECEIPT_INBOX_MAX_AGE_HOURS, gatewaySubmitRequeueKey, isObjectRecord, asDateOrNull, downstreamRetryDelayMs, downstreamAckTimeoutMs, downstreamRetryBaseDelayMs, downstreamRetryMaxDelayMs, downstreamMaxRetries, downstreamPendingTimeoutHours, downstreamControlFailureMessage, normalizeSubmitStatus, downstreamDeliveryAttemptKey, hasRecoveryAuditStateChanged, normalizeRecoveryFailureCategory, aggregateReceiptSegmentState, isSameUpstreamEndpointIdentity, receiptEventKey } from './send-chain.helpers';
import type { SendSubmissionService } from './send-submission.service';
import type { SendCompletionCallbacks, SendCompletionFacade } from './send-completion.service';
/**
* R10 receipt implementation.
* Cross-domain calls return through the stable SendChainService seam to preserve locking and test observability.
*/
export class SendReceiptService {
private readonly logger = new Logger('SendChainService');
private upstreamReceiptInboxScanRunning = false;
constructor(
private readonly prisma: PrismaService,
private readonly billing: BillingService,
private readonly openApi: OpenApiService | undefined,
private readonly facade: SendCompletionFacade,
private readonly callbacks: SendCompletionCallbacks,
) {}
async intakeReceipt(data: GatewayReceiptEventDto) {
const channel = await this.prisma.smsChannel.findUnique({
where: { id: data.channelId },
select: {
id: true,
account: true,
gatewayHost: true,
gatewayPort: true,
protocol: true,
cmppVersion: true,
},
});
if (!channel) {
throw new NotFoundException('SMS channel not found');
}
const deliveredAt = data.deliveredAt ? new Date(data.deliveredAt) : new Date();
const receiptKey = receiptEventKey(data, data.channelId);
const inbox = await this.prisma.upstreamReceiptInbox.upsert({
where: { receiptKey },
update: {
incomingConnectionId: data.connectionId,
},
create: {
receiptKey,
incomingChannelId: data.channelId,
incomingConnectionId: data.connectionId,
upstreamAccount: channel.account,
upstreamHost: channel.gatewayHost,
upstreamPort: channel.gatewayPort,
protocol: channel.protocol,
protocolVersion: channel.cmppVersion,
provisionalMessageId: data.messageId,
sequenceId: data.sequenceId,
gatewayMessageId: data.gatewayMessageId,
phoneNumber: data.phoneNumber?.trim() || null,
receiptStatus: data.receiptStatus,
rawStatus: data.rawStatus,
errorCode: data.errorCode,
errorMessage: data.errorMessage,
deliveredAt,
status: 'pending',
nextRetryAt: new Date(),
},
});
if (['pending', 'retrying'].includes(inbox.status)) {
setImmediate(() => void this.facade.processUpstreamReceiptInboxRecord(inbox.id));
}
return { accepted: true, inboxId: inbox.id, status: inbox.status };
}
async processPendingUpstreamReceiptInbox(limit = 100) {
const now = new Date();
const staleBefore = new Date(
now.getTime()
- positiveInteger(
process.env.UPSTREAM_RECEIPT_INBOX_PROCESSING_STALE_MS,
DEFAULT_UPSTREAM_RECEIPT_INBOX_PROCESSING_STALE_MS,
),
);
const candidates = await this.prisma.upstreamReceiptInbox.findMany({
where: {
OR: [
{
status: { in: ['pending', 'retrying'] },
OR: [{ nextRetryAt: null }, { nextRetryAt: { lte: now } }],
},
{ status: 'processing', updatedAt: { lte: staleBefore } },
],
},
orderBy: [{ receivedAt: 'asc' }, { id: 'asc' }],
take: Math.min(Math.max(limit, 1), 500),
select: { id: true },
});
let processed = 0;
for (const candidate of candidates) {
if (await this.facade.processUpstreamReceiptInboxRecord(candidate.id)) processed += 1;
}
return { scanned: candidates.length, processed };
}
async processUpstreamReceiptInboxRecord(id: string) {
const staleBefore = new Date(
Date.now()
- positiveInteger(
process.env.UPSTREAM_RECEIPT_INBOX_PROCESSING_STALE_MS,
DEFAULT_UPSTREAM_RECEIPT_INBOX_PROCESSING_STALE_MS,
),
);
const claimed = await this.prisma.upstreamReceiptInbox.updateMany({
where: {
id,
OR: [
{ status: { in: ['pending', 'retrying'] } },
{ status: 'processing', updatedAt: { lte: staleBefore } },
],
},
data: { status: 'processing', attemptCount: { increment: 1 }, nextRetryAt: null },
});
if (claimed.count !== 1) return false;
const inbox = await this.prisma.upstreamReceiptInbox.findUnique({ where: { id } });
if (!inbox) return false;
try {
const message = await this.facade.handleReceipt({
messageId: inbox.provisionalMessageId ?? undefined,
channelId: inbox.incomingChannelId,
connectionId: inbox.incomingConnectionId ?? undefined,
sequenceId: inbox.sequenceId ?? undefined,
gatewayMessageId: inbox.gatewayMessageId,
phoneNumber: inbox.phoneNumber ?? undefined,
receiptStatus: normalizeReceiptStatus(inbox.receiptStatus),
rawStatus: inbox.rawStatus,
errorCode: inbox.errorCode ?? undefined,
errorMessage: inbox.errorMessage ?? undefined,
deliveredAt: inbox.deliveredAt.toISOString(),
}, {
account: inbox.upstreamAccount,
gatewayHost: inbox.upstreamHost,
gatewayPort: inbox.upstreamPort,
protocol: inbox.protocol,
cmppVersion: inbox.protocolVersion,
});
const matchedMessageRecordId = message && 'id' in message ? message.id : message?.messageRecordId;
const matchedChannelId = message && 'channelId' in message ? message.channelId : undefined;
await this.prisma.upstreamReceiptInbox.update({
where: { id },
data: {
status: 'matched',
matchedMessageRecordId: matchedMessageRecordId ?? null,
matchedChannelId: matchedChannelId ?? null,
lastError: null,
processedAt: new Date(),
},
});
return true;
} catch (error) {
const maxAttempts = positiveInteger(
process.env.UPSTREAM_RECEIPT_INBOX_MAX_ATTEMPTS,
DEFAULT_UPSTREAM_RECEIPT_INBOX_MAX_ATTEMPTS,
);
const maxAgeHours = positiveInteger(
process.env.UPSTREAM_RECEIPT_INBOX_MAX_AGE_HOURS,
DEFAULT_UPSTREAM_RECEIPT_INBOX_MAX_AGE_HOURS,
);
const exhausted = inbox.attemptCount >= maxAttempts
|| inbox.receivedAt.getTime() <= Date.now() - maxAgeHours * 60 * 60_000;
const retryDelayMs = Math.min(30 * 60_000, 5_000 * 2 ** Math.min(Math.max(inbox.attemptCount - 1, 0), 8));
await this.prisma.upstreamReceiptInbox.update({
where: { id },
data: {
status: exhausted ? 'unmatched' : 'retrying',
nextRetryAt: exhausted ? null : new Date(Date.now() + retryDelayMs),
lastError: error instanceof Error ? error.message : String(error),
processedAt: exhausted ? new Date() : null,
},
});
return false;
}
}
async runUpstreamReceiptInboxScan() {
if (this.upstreamReceiptInboxScanRunning) return;
this.upstreamReceiptInboxScanRunning = true;
try {
const result = await this.facade.processPendingUpstreamReceiptInbox();
if (result.processed > 0) {
this.logger.log(`Matched ${result.processed}/${result.scanned} pending upstream receipts`);
}
} catch (error) {
this.logger.error('Upstream receipt inbox scan failed', error instanceof Error ? error.stack : String(error));
} finally {
this.upstreamReceiptInboxScanRunning = false;
}
}
async handleReceipt(
data: GatewayReceiptEventDto,
incomingIdentity?: { account: string; gatewayHost: string; gatewayPort: number; protocol: string; cmppVersion: string },
) {
const resolved = await this.facade.resolveReceiptMessage(data, incomingIdentity);
const logicalChannelId = resolved.channelId ?? data.channelId;
const receiptKey = receiptEventKey(data, logicalChannelId);
const existingReceipt = await this.prisma.smsReceiptRecord.findUnique({
where: { receiptKey },
include: { messageRecord: true },
});
if (existingReceipt?.messageRecord) {
return existingReceipt.messageRecord;
}
const message = resolved.message;
const deliveredAt = data.deliveredAt ? new Date(data.deliveredAt) : new Date();
if (resolved.submitRecordId) {
await this.prisma.smsSubmitRecord.updateMany({
where: {
id: resolved.submitRecordId,
gatewayMessageId: null,
},
data: {
gatewayMessageId: data.gatewayMessageId,
sequenceId: data.sequenceId,
},
});
}
try {
await this.prisma.smsReceiptRecord.create({
data: {
tenantId: message.tenantId,
batchTaskId: message.batchTaskId,
messageRecordId: message.id,
receiptKey,
channelId: logicalChannelId,
messageId: resolved.messageId,
gatewayMessageId: data.gatewayMessageId,
phoneNumber: data.phoneNumber?.trim() || message.phoneNumber,
sequenceId: data.sequenceId,
receiptStatus: data.receiptStatus,
rawStatus: data.rawStatus,
errorCode: data.errorCode,
errorMessage: data.errorMessage,
deliveredAt,
},
});
} catch (error) {
if (error instanceof Prisma.PrismaClientKnownRequestError && error.code === 'P2002') {
const duplicate = await this.prisma.smsReceiptRecord.findUnique({
where: { receiptKey },
include: { messageRecord: true },
});
if (duplicate?.messageRecord) return duplicate.messageRecord;
}
throw error;
}
const logicalReceipt = { ...data, channelId: logicalChannelId };
await this.facade.recordReceiptSegment(message, logicalReceipt, deliveredAt, resolved.submitRecordId);
const aggregate = await this.facade.aggregateReceiptSegments(
message,
logicalReceipt,
deliveredAt,
resolved.submitRecordId,
resolved.submitId,
);
if (!aggregate.terminal) {
return this.prisma.smsMessageRecord.findUnique({ where: { id: message.id } });
}
const status = aggregate.status;
const isCurrentAttempt =
(!message.channelId || message.channelId === logicalChannelId)
&& (
!message.gatewayMessageId
|| message.gatewayMessageId === data.gatewayMessageId
|| (aggregate.segmentTotal > 1 && (!message.submitId || message.submitId === resolved.submitId))
);
if (!isCurrentAttempt || (status === 'failed' && message.status === 'delivered')) {
return this.prisma.smsMessageRecord.findUnique({ where: { id: message.id } });
}
const isStandaloneChannelTest = !message.tenantId && !message.batchTaskId;
if (status === 'failed' && !isStandaloneChannelTest && message.tenantId && message.batchTaskId) {
const businessMessage = message as typeof message & { tenantId: string; batchTaskId: string };
const retried = await this.facade.retryMessageIfAllowed(
businessMessage,
'回执失败补发',
resolved.submitRecordId,
);
if (retried) {
await this.facade.refreshTaskProgress(businessMessage.batchTaskId);
return retried;
}
await this.facade.refundMessage(businessMessage, '最终失败退款');
}
await this.prisma.smsMessageRecord.update({
where: { id: message.id },
data: {
channelId: logicalChannelId,
gatewayMessageId: message.gatewayMessageId ?? data.gatewayMessageId,
receiptStatus: aggregate.receiptStatus,
receiptRawStatus: aggregate.rawStatus,
status,
errorCode: aggregate.errorCode,
errorMessage: aggregate.errorMessage ?? (status === 'delivered' ? null : aggregate.rawStatus),
deliveredAt: aggregate.deliveredAt,
},
});
if (!isStandaloneChannelTest && message.tenantId && message.applicationId) {
await this.facade.queueAndTryDownstreamDelivery({
tenantId: message.tenantId,
applicationId: message.applicationId,
messageRecordId: message.id,
messageId: message.messageId,
deliveryType: 'receipt',
payload: {
messageId: message.messageId,
gatewayMessageId: data.gatewayMessageId,
phoneNumber: message.phoneNumber,
receiptStatus: aggregate.receiptStatus,
rawStatus: aggregate.rawStatus,
errorCode: aggregate.errorCode,
submitSequenceId: message.cmppSubmitSequenceId ? Number(message.cmppSubmitSequenceId) : undefined,
submitGroupMessageId: message.cmppSubmitGroupMessageId ?? undefined,
deliveredAt: aggregate.deliveredAt.toISOString(),
},
});
}
if (message.batchTaskId) {
await this.facade.refreshTaskProgress(message.batchTaskId);
}
return this.prisma.smsMessageRecord.findUnique({ where: { id: message.id } });
}
async recordReceiptSegment(
message: {
id: string;
tenantId?: string | null;
batchTaskId?: string | null;
channelId?: string | null;
submitId?: string | null;
billingUnits?: number | null;
},
data: GatewayReceiptEventDto,
deliveredAt: Date,
submitRecordId?: string,
) {
const segmentAudits = this.facade.smsMessageSegmentAuditDelegate();
const updated = await segmentAudits.updateMany({
where: {
messageRecordId: message.id,
gatewayMessageId: data.gatewayMessageId,
},
data: {
receiptStatus: data.receiptStatus,
rawStatus: data.rawStatus,
errorCode: data.errorCode ?? null,
deliveredAt,
},
});
if (updated.count > 0) {
return;
}
const submitRecord = submitRecordId
? await this.prisma.smsSubmitRecord.findUnique({ where: { id: submitRecordId } })
: await this.prisma.smsSubmitRecord.findFirst({
where: { messageRecordId: message.id, gatewayMessageId: data.gatewayMessageId },
orderBy: { createdAt: 'desc' },
});
await segmentAudits.upsert({
where: {
messageRecordId_submitId_segmentIndex: {
messageRecordId: message.id,
submitId: submitRecord?.submitId ?? message.submitId ?? `RECEIPT-AUDIT-${data.gatewayMessageId}`,
segmentIndex: 1,
},
},
update: {
submitRecordId: submitRecord?.id ?? submitRecordId ?? null,
channelId: data.channelId ?? message.channelId ?? null,
sequenceId: data.sequenceId ?? null,
gatewayMessageId: data.gatewayMessageId,
receiptStatus: data.receiptStatus,
rawStatus: data.rawStatus,
errorCode: data.errorCode ?? null,
deliveredAt,
},
create: {
tenantId: message.tenantId,
batchTaskId: message.batchTaskId,
messageRecordId: message.id,
submitRecordId: submitRecord?.id ?? submitRecordId ?? null,
channelId: data.channelId ?? message.channelId ?? null,
submitId: submitRecord?.submitId ?? message.submitId ?? `RECEIPT-AUDIT-${data.gatewayMessageId}`,
attempt: 0,
segmentTotal: Math.max(1, Number(message.billingUnits ?? 1)),
segmentIndex: 1,
sequenceId: data.sequenceId ?? null,
gatewayMessageId: data.gatewayMessageId,
submitStatus: submitRecord?.submitStatus ?? 'accepted',
receiptStatus: data.receiptStatus,
rawStatus: data.rawStatus,
compensationType: 'receipt_recovered',
errorCode: data.errorCode ?? null,
deliveredAt,
},
});
}
async aggregateReceiptSegments(
message: {
id: string;
billingUnits?: number | null;
},
data: GatewayReceiptEventDto,
deliveredAt: Date,
submitRecordId?: string,
submitId?: string,
) {
const audits = await this.facade.smsMessageSegmentAuditDelegate().findMany({
where: submitRecordId
? { messageRecordId: message.id, submitRecordId }
: submitId
? { messageRecordId: message.id, submitId }
: { messageRecordId: message.id, gatewayMessageId: data.gatewayMessageId },
orderBy: { segmentIndex: 'asc' },
});
return aggregateReceiptSegmentState(audits, message.billingUnits, data, deliveredAt);
}
async resolveReceiptMessage(
data: GatewayReceiptEventDto,
incomingIdentity?: { account: string; gatewayHost: string; gatewayPort: number; protocol: string; cmppVersion: string },
) {
const exactMessage = data.messageId
? await this.prisma.smsMessageRecord.findUnique({ where: { messageId: data.messageId } })
: null;
if (exactMessage) {
const segmentAudit = data.gatewayMessageId
? await this.facade.smsMessageSegmentAuditDelegate().findFirst({
where: {
messageRecordId: exactMessage.id,
gatewayMessageId: data.gatewayMessageId,
},
orderBy: { updatedAt: 'desc' },
})
: null;
if (segmentAudit) {
return {
message: exactMessage,
messageId: exactMessage.messageId,
submitRecordId: segmentAudit.submitRecordId ?? undefined,
submitId: segmentAudit.submitId,
channelId: segmentAudit.channelId ?? data.channelId,
};
}
const submitRecord = await this.prisma.smsSubmitRecord.findFirst({
where: {
messageRecordId: exactMessage.id,
channelId: data.channelId,
gatewayMessageId: data.gatewayMessageId,
},
orderBy: { createdAt: 'desc' },
});
return {
message: exactMessage,
messageId: exactMessage.messageId,
submitRecordId: submitRecord?.id,
submitId: submitRecord?.submitId,
channelId: submitRecord?.channelId ?? data.channelId,
};
}
const phoneNumber = data.phoneNumber?.trim();
const exactSubmits = await this.prisma.smsSubmitRecord.findMany({
where: {
channelId: data.channelId,
gatewayMessageId: data.gatewayMessageId,
...(phoneNumber ? { messageRecord: { phoneNumber } } : {}),
},
include: { messageRecord: true },
orderBy: { createdAt: 'desc' },
take: 2,
});
if (exactSubmits.length === 1 && exactSubmits[0]?.messageRecord) {
return {
message: exactSubmits[0].messageRecord,
messageId: exactSubmits[0].messageRecord.messageId,
submitRecordId: exactSubmits[0].id,
submitId: exactSubmits[0].submitId,
channelId: exactSubmits[0].channelId,
};
}
if (!phoneNumber) {
throw new NotFoundException('SMS message record not found');
}
const incomingChannel = incomingIdentity
?? await this.prisma.smsChannel.findUnique({ where: { id: data.channelId } });
if (!incomingChannel) {
throw new NotFoundException('SMS message record not found');
}
const segmentMatches = await this.facade.smsMessageSegmentAuditDelegate().findMany({
where: {
gatewayMessageId: data.gatewayMessageId,
messageRecord: { phoneNumber },
},
include: { messageRecord: true, submitRecord: true, channel: true },
orderBy: { createdAt: 'desc' },
take: 10,
});
const exactSegmentMatches = segmentMatches.filter((candidate) => candidate.channelId === data.channelId);
if (exactSegmentMatches.length === 1 && exactSegmentMatches[0]?.messageRecord) {
return {
message: exactSegmentMatches[0].messageRecord,
messageId: exactSegmentMatches[0].messageRecord.messageId,
submitRecordId: exactSegmentMatches[0].submitRecordId ?? undefined,
submitId: exactSegmentMatches[0].submitRecord?.submitId ?? exactSegmentMatches[0].submitId,
channelId: exactSegmentMatches[0].channelId,
};
}
const sameSupplierSegments = segmentMatches.filter((candidate) =>
candidate.channel && isSameUpstreamEndpointIdentity(incomingChannel, candidate.channel));
if (sameSupplierSegments.length === 1 && sameSupplierSegments[0]?.messageRecord) {
return {
message: sameSupplierSegments[0].messageRecord,
messageId: sameSupplierSegments[0].messageRecord.messageId,
submitRecordId: sameSupplierSegments[0].submitRecordId ?? undefined,
submitId: sameSupplierSegments[0].submitRecord?.submitId ?? sameSupplierSegments[0].submitId,
channelId: sameSupplierSegments[0].channelId,
};
}
const crossConnectionSubmits = await this.prisma.smsSubmitRecord.findMany({
where: {
gatewayMessageId: data.gatewayMessageId,
messageRecord: { phoneNumber },
},
include: { messageRecord: true, channel: true },
orderBy: { createdAt: 'desc' },
take: 10,
});
const sameSupplierSubmits = crossConnectionSubmits.filter((candidate) =>
candidate.channel && isSameUpstreamEndpointIdentity(incomingChannel, candidate.channel));
if (sameSupplierSubmits.length === 1 && sameSupplierSubmits[0]?.messageRecord) {
return {
message: sameSupplierSubmits[0].messageRecord,
messageId: sameSupplierSubmits[0].messageRecord.messageId,
submitRecordId: sameSupplierSubmits[0].id,
submitId: sameSupplierSubmits[0].submitId,
channelId: sameSupplierSubmits[0].channelId,
};
}
const deliveredAt = data.deliveredAt ? new Date(data.deliveredAt) : new Date();
const submittedAfter = new Date(deliveredAt.getTime() - 72 * 60 * 60 * 1000);
const candidates = await this.prisma.smsSubmitRecord.findMany({
where: {
channelId: data.channelId,
gatewayMessageId: null,
submitStatus: 'timeout',
submittedAt: {
gte: submittedAfter,
lte: deliveredAt,
},
messageRecord: {
phoneNumber,
},
},
include: {
messageRecord: true,
},
orderBy: {
submittedAt: 'desc',
},
take: 10,
});
if (candidates.length !== 1 || !candidates[0]?.messageRecord) {
throw new NotFoundException('SMS message record not found');
}
return {
message: candidates[0].messageRecord,
messageId: candidates[0].messageRecord.messageId,
submitRecordId: candidates[0].id,
submitId: candidates[0].submitId,
channelId: candidates[0].channelId,
};
}
}