hectorzhao/lislgosms
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: harden upstream and downstream receipt delivery

Browse Source
This commit is contained in:
hectorzhao
2026-07-25 23:32:13 +08:00
parent 5018167696
commit 54617c927e
18 changed files with 1797 additions and 779 deletions
Download Patch File Download Diff File Expand all files Collapse all files
api/src/operations/operations.service.spec.ts
+6 -1
View File
@@ -668,7 +668,12 @@ describe('OperationsService', () => {
lte: new Date('2026-07-15T23:59:59.999+08:00'),
},
}),
include: { tenant: true, application: true, messageRecord: true },
include: {
tenant: true,
application: true,
messageRecord: true,
attempts: { orderBy: [{ attemptNo: 'desc' }, { createdAt: 'desc' }] },
},
orderBy: { createdAt: 'desc' },
skip: 0,
take: 10,
api/src/operations/operations.service.ts
+6 -1
View File
@@ -814,7 +814,12 @@ export class OperationsService {
const [items, total] = await Promise.all([
this.prisma.cmppDownstreamDelivery.findMany({
where,
include: { tenant: true, application: true, messageRecord: true },
include: {
tenant: true,
application: true,
messageRecord: true,
attempts: { orderBy: [{ attemptNo: 'desc' }, { createdAt: 'desc' }] },
},
orderBy: { createdAt: 'desc' },
skip: (page - 1) * pageSize,
take: pageSize,
api/src/send-chain/gateway-events.controller.spec.ts
+47
View File
@@ -4,7 +4,9 @@ import { GatewayEventsController } from './gateway-events.controller';
describe('GatewayEventsController protocol logging', () => {
const sendChain = {
handleSubmitResult: jest.fn(),
handleSubmitSegmentResult: jest.fn(),
handleReceipt: jest.fn(),
intakeReceipt: jest.fn(),
};
const protocolLogs = {
record: jest.fn(),
@@ -30,6 +32,51 @@ describe('GatewayEventsController protocol logging', () => {
expect(protocolLogs.record).not.toHaveBeenCalled();
});
it('persists each supplier segment response without adding a duplicate protocol log', async () => {
sendChain.handleSubmitSegmentResult.mockResolvedValue({ accepted: true });
const body = {
messageId: 'MSG-LONG-1',
channelId: 'channel-1',
submitId: 'submit-1',
segmentIndex: 2,
segmentTotal: 3,
gatewayMessageId: '456',
sequenceId: 8,
submitStatus: 'accepted' as const,
};
await expect(controller.submitSegmentResult(body)).resolves.toEqual({ accepted: true });
expect(sendChain.handleSubmitSegmentResult).toHaveBeenCalledWith(body);
expect(protocolLogs.record).not.toHaveBeenCalled();
});
it('logs a supplier receipt only after it is durably accepted by the inbox', async () => {
sendChain.intakeReceipt.mockResolvedValue({ accepted: true, inboxId: 'inbox-1', status: 'pending' });
const body = {
messageId: 'receipt-456',
channelId: 'channel-1',
connectionId: 'channel-1:0',
gatewayMessageId: '456',
phoneNumber: '13127620092',
receiptStatus: 'undelivered' as const,
rawStatus: 'UNDELIV',
};
await expect(controller.receiptIntake(body)).resolves.toEqual({
accepted: true,
inboxId: 'inbox-1',
status: 'pending',
});
expect(protocolLogs.record).toHaveBeenCalledWith(expect.objectContaining({
direction: 'channel_to_platform',
eventType: 'deliver_receipt',
channelId: 'channel-1',
messageId: 'receipt-456',
gatewayMessageId: '456',
status: 'success',
}));
});
it('accepts only safe outbound Gateway packet events', () => {
expect(controller.protocolLog({
protocol: 'cmpp',
api/src/send-chain/gateway-events.controller.ts
+13 -2
View File
@@ -11,6 +11,7 @@ import {
GatewayReceiptEventDto,
GatewaySubmitDeadLetterDto,
GatewaySubmitResultDto,
GatewaySubmitSegmentResultDto,
GatewayUplinkEventDto,
SendChainService,
} from './send-chain.service';
@@ -31,6 +32,16 @@ export class GatewayEventsController {
return this.sendChain.handleSubmitResult(body);
}
@Post('submit-segment-result')
submitSegmentResult(@Body() body: GatewaySubmitSegmentResultDto) {
return this.sendChain.handleSubmitSegmentResult(body);
}
@Post('receipt/intake')
receiptIntake(@Body() body: GatewayReceiptEventDto) {
return this.trackGatewayEvent('deliver_receipt', body, () => this.sendChain.intakeReceipt(body));
}
@Post('receipt')
receipt(@Body() body: GatewayReceiptEventDto) {
return this.trackGatewayEvent('deliver_receipt', body, () => this.sendChain.handleReceipt(body));
@@ -104,8 +115,8 @@ export class GatewayEventsController {
}
@Post('downstream/failed')
downstreamFailed(@Body() body: { id: string; errorMessage?: string; failureType?: GatewayDownstreamFailureType }) {
return this.sendChain.markDownstreamDeliveryFailed(body.id, body.errorMessage, body.failureType);
downstreamFailed(@Body() body: GatewayDownstreamSentDto & { errorMessage?: string; failureType?: GatewayDownstreamFailureType }) {
return this.sendChain.markDownstreamDeliveryFailed(body.id, body.errorMessage, body.failureType, body);
}
@Post('downstream/recovery-status')
api/src/send-chain/send-chain.service.spec.ts
+139 -4
View File
@@ -36,6 +36,7 @@ function createPrismaMock() {
sendRegion: '全国',
gatewayHost: '127.0.0.1',
gatewayPort: 17890,
protocol: 'CMPP',
passwordCipher: 'secret',
cmppVersion: '3.0',
config: { serviceId: 'SMS' },
@@ -240,6 +241,22 @@ function createPrismaMock() {
update: jest.fn().mockImplementation(({ data }) => Promise.resolve({ id: 'delivery-1', tenantId: 'tenant-1', applicationId: 'app-1', messageId: 'MSG-1', deliveryType: 'receipt', ...data })),
updateMany: jest.fn().mockResolvedValue({ count: 1 }),
},
cmppDownstreamDeliveryAttempt: {
upsert: jest.fn().mockResolvedValue({ id: 'delivery-attempt-1' }),
findMany: jest.fn().mockResolvedValue([]),
},
upstreamReceiptInbox: {
upsert: jest.fn().mockImplementation(({ create }) => Promise.resolve({
id: 'receipt-inbox-1',
attemptCount: 0,
receivedAt: new Date(),
...create,
})),
findMany: jest.fn().mockResolvedValue([]),
findUnique: jest.fn().mockResolvedValue(null),
update: jest.fn().mockImplementation(({ data }) => Promise.resolve({ id: 'receipt-inbox-1', ...data })),
updateMany: jest.fn().mockResolvedValue({ count: 1 }),
},
gatewaySubmitDeadLetter: {
upsert: jest.fn().mockResolvedValue({ id: 'dead-1' }),
findUnique: jest.fn().mockResolvedValue({
@@ -1765,8 +1782,8 @@ describe('SendChainService', () => {
where: { submitId: 'SUB-1' },
data: expect.objectContaining({ sequenceId: 7, gatewayMessageId: 'GW-1', submitStatus: 'accepted' }),
});
expect(prisma.smsMessageRecord.update).toHaveBeenCalledWith({
where: { id: 'record-1' },
expect(prisma.smsMessageRecord.updateMany).toHaveBeenCalledWith({
where: { id: 'record-1', status: { notIn: ['delivered', 'failed', 'unknown'] } },
data: expect.objectContaining({ gatewayMessageId: 'GW-1', status: 'submitted', submitStatus: 'accepted' }),
});
expect(billing.release).toHaveBeenCalledWith(expect.objectContaining({ amountCents: 3, relatedId: 'task-1' }));
@@ -1824,8 +1841,8 @@ describe('SendChainService', () => {
expect(prisma.channelRouteRule.findFirst).not.toHaveBeenCalled();
expect(billing.release).not.toHaveBeenCalled();
expect(prisma.smsBatchTask.update).not.toHaveBeenCalled();
expect(prisma.smsMessageRecord.update).toHaveBeenCalledWith({
where: { id: 'record-1' },
expect(prisma.smsMessageRecord.updateMany).toHaveBeenCalledWith({
where: { id: 'record-1', status: { not: 'delivered' } },
data: expect.objectContaining({
gatewayMessageId: 'GW-1',
submitStatus: 'timeout',
@@ -2680,6 +2697,108 @@ describe('SendChainService', () => {
}));
});
it('persists each upstream SubmitResp segment before the aggregate result arrives', async () => {
const { service, prisma } = createService();
await service.handleSubmitSegmentResult({
messageId: 'MSG-1',
channelId: 'channel-1',
submitId: 'SUB-1',
segmentTotal: 3,
segmentIndex: 1,
sequenceId: 71,
gatewayMessageId: 'GW-SEG-1',
submitStatus: 'accepted',
submittedAt: '2026-07-25T15:00:00.000Z',
});
expect(prisma.smsMessageSegmentAudit.upsert).toHaveBeenCalledWith(expect.objectContaining({
where: {
messageRecordId_submitId_segmentIndex: {
messageRecordId: 'record-1',
submitId: 'SUB-1',
segmentIndex: 1,
},
},
create: expect.objectContaining({
segmentTotal: 3,
sequenceId: 71,
gatewayMessageId: 'GW-SEG-1',
submitStatus: 'accepted',
}),
}));
expect(prisma.smsSubmitRecord.updateMany).toHaveBeenCalledWith(expect.objectContaining({
where: { submitId: 'SUB-1', gatewayMessageId: null },
data: expect.objectContaining({ sequenceId: 71, gatewayMessageId: 'GW-SEG-1' }),
}));
});
it('durably intakes an upstream receipt before asynchronous business matching', async () => {
const { service, prisma } = createService();
jest.spyOn(service as any, 'processUpstreamReceiptInboxRecord').mockResolvedValue(false);
await expect(service.intakeReceipt({
messageId: 'receipt-9001',
channelId: 'channel-1',
connectionId: 'gateway-connection-2',
sequenceId: 81,
gatewayMessageId: '9001',
phoneNumber: '13800000001',
receiptStatus: 'delivered',
rawStatus: 'DELIVRD',
deliveredAt: '2026-07-25T15:01:00.000Z',
})).resolves.toEqual(expect.objectContaining({
accepted: true,
inboxId: 'receipt-inbox-1',
}));
expect(prisma.upstreamReceiptInbox.upsert).toHaveBeenCalledWith(expect.objectContaining({
create: expect.objectContaining({
incomingChannelId: 'channel-1',
incomingConnectionId: 'gateway-connection-2',
upstreamAccount: 'cmpp-account',
upstreamHost: '127.0.0.1',
upstreamPort: 17890,
protocol: 'CMPP',
protocolVersion: '3.0',
gatewayMessageId: '9001',
status: 'pending',
}),
}));
});
it('does not downgrade an early terminal receipt when the aggregate submit result arrives later', async () => {
const { service, prisma } = createService();
const terminalMessage = {
id: 'record-1', messageId: 'MSG-1', tenantId: null, batchTaskId: null, applicationId: null,
channelId: 'channel-1', submitId: 'SUB-1', gatewayMessageId: 'GW-SEG-1',
phoneNumber: '13800000001', billingUnits: 1, amountCents: 0, status: 'failed',
};
prisma.smsMessageRecord.findFirst.mockResolvedValue(terminalMessage);
prisma.smsMessageRecord.findUnique.mockResolvedValue(terminalMessage);
prisma.smsMessageRecord.updateMany
.mockResolvedValueOnce({ count: 0 })
.mockResolvedValueOnce({ count: 1 });
await service.handleSubmitResult({
messageId: 'MSG-1',
channelId: 'channel-1',
submitId: 'SUB-1',
sequenceId: 7,
gatewayMessageId: 'GW-1',
submitStatus: 'accepted',
});
expect(prisma.smsMessageRecord.updateMany).toHaveBeenNthCalledWith(1, expect.objectContaining({
where: { id: 'record-1', status: { notIn: ['delivered', 'failed', 'unknown'] } },
data: expect.objectContaining({ status: 'submitted' }),
}));
expect(prisma.smsMessageRecord.updateMany).toHaveBeenNthCalledWith(2, {
where: { id: 'record-1', gatewayMessageId: null },
data: expect.objectContaining({ gatewayMessageId: 'GW-1' }),
});
});
it('recovers a stale submit requeue with the same Redis idempotency key', async () => {
const { service, prisma } = createService();
const stale = {
@@ -2929,6 +3048,15 @@ describe('SendChainService', () => {
where: { id: 'delivery-1', status: { not: 'delivered' } },
data: expect.objectContaining({ status: 'awaiting_ack', ackSequenceId: '37', connectionId: 'conn-1' }),
}));
expect(prisma.cmppDownstreamDeliveryAttempt.upsert).toHaveBeenLastCalledWith(expect.objectContaining({
create: expect.objectContaining({
deliveryId: 'delivery-1',
attemptNo: 1,
connectionId: 'conn-1',
sequenceId: '37',
status: 'awaiting_ack',
}),
}));
await service.acknowledgeDownstreamDelivery({
id: 'delivery-1',
@@ -2941,6 +3069,13 @@ describe('SendChainService', () => {
expect(prisma.cmppDownstreamDelivery.updateMany).toHaveBeenLastCalledWith(expect.objectContaining({
data: expect.objectContaining({ status: 'delivered', ackResult: 0, deliveredAt: new Date('2026-07-14T03:40:18.060Z') }),
}));
expect(prisma.cmppDownstreamDeliveryAttempt.upsert).toHaveBeenLastCalledWith(expect.objectContaining({
update: expect.objectContaining({
status: 'acknowledged',
acknowledgedAt: new Date('2026-07-14T03:40:18.060Z'),
ackResult: 0,
}),
}));
});
it('does not treat Result=0 with Msg_Id=0 as a business acknowledgement', async () => {
api/src/send-chain/send-chain.service.ts
+409 -11
View File
@@ -89,6 +89,21 @@ export interface GatewaySubmitResultDto {
}>;
}
export interface GatewaySubmitSegmentResultDto {
traceId?: string;
messageId: string;
channelId: string;
submitId?: string;
segmentTotal: number;
segmentIndex: number;
sequenceId?: number;
gatewayMessageId?: string;
submitStatus: 'accepted' | 'rejected' | 'timeout' | string;
errorCode?: string;
errorMessage?: string;
submittedAt?: string;
}
export interface GatewayReceiptEventDto {
traceId?: string;
messageId?: string;
@@ -101,6 +116,7 @@ export interface GatewayReceiptEventDto {
errorCode?: string;
errorMessage?: string;
deliveredAt?: string;
connectionId?: string;
}
export interface GatewayUplinkEventDto {
@@ -269,6 +285,11 @@ const DEFAULT_DOWNSTREAM_MANUAL_REQUEUE_STALE_MS = 2 * 60_000;
const DEFAULT_INBOUND_LONG_MESSAGE_SCAN_INTERVAL_MS = 60_000;
const INBOUND_LONG_MESSAGE_SCAN_INITIAL_DELAY_MS = 10_000;
const DEFAULT_INBOUND_LONG_MESSAGE_PROCESSING_STALE_SECONDS = 30;
const DEFAULT_UPSTREAM_RECEIPT_INBOX_SCAN_INTERVAL_MS = 5_000;
const UPSTREAM_RECEIPT_INBOX_INITIAL_DELAY_MS = 1_000;
const DEFAULT_UPSTREAM_RECEIPT_INBOX_PROCESSING_STALE_MS = 2 * 60_000;
const DEFAULT_UPSTREAM_RECEIPT_INBOX_MAX_ATTEMPTS = 30;
const DEFAULT_UPSTREAM_RECEIPT_INBOX_MAX_AGE_HOURS = 72;
const GATEWAY_SUBMIT_REQUEUE_IDEMPOTENCY_TTL_SECONDS = 30 * 24 * 60 * 60;
const BULLMQ_PRIORITY: Record<QueuePriority, number> = {
priority: 1,
@@ -290,6 +311,9 @@ export class SendChainService implements OnModuleInit, OnModuleDestroy {
private scheduledDispatchScanRunning = false;
private inboundLongMessageInitialTimer?: ReturnType<typeof setTimeout>;
private inboundLongMessageIntervalTimer?: ReturnType<typeof setInterval>;
private upstreamReceiptInboxInitialTimer?: ReturnType<typeof setTimeout>;
private upstreamReceiptInboxIntervalTimer?: ReturnType<typeof setInterval>;
private upstreamReceiptInboxScanRunning = false;
constructor(
private readonly prisma: PrismaService,
@@ -342,6 +366,21 @@ export class SendChainService implements OnModuleInit, OnModuleDestroy {
);
this.inboundLongMessageIntervalTimer.unref?.();
}
if (process.env.UPSTREAM_RECEIPT_INBOX_SCAN_ENABLED !== 'false') {
this.upstreamReceiptInboxInitialTimer = setTimeout(
() => void this.runUpstreamReceiptInboxScan(),
UPSTREAM_RECEIPT_INBOX_INITIAL_DELAY_MS,
);
this.upstreamReceiptInboxInitialTimer.unref?.();
this.upstreamReceiptInboxIntervalTimer = setInterval(
() => void this.runUpstreamReceiptInboxScan(),
positiveInteger(
process.env.UPSTREAM_RECEIPT_INBOX_SCAN_INTERVAL_MS,
DEFAULT_UPSTREAM_RECEIPT_INBOX_SCAN_INTERVAL_MS,
),
);
this.upstreamReceiptInboxIntervalTimer.unref?.();
}
}
async onModuleDestroy() {
@@ -351,6 +390,8 @@ export class SendChainService implements OnModuleInit, OnModuleDestroy {
if (this.scheduledDispatchIntervalTimer) clearInterval(this.scheduledDispatchIntervalTimer);
if (this.inboundLongMessageInitialTimer) clearTimeout(this.inboundLongMessageInitialTimer);
if (this.inboundLongMessageIntervalTimer) clearInterval(this.inboundLongMessageIntervalTimer);
if (this.upstreamReceiptInboxInitialTimer) clearTimeout(this.upstreamReceiptInboxInitialTimer);
if (this.upstreamReceiptInboxIntervalTimer) clearInterval(this.upstreamReceiptInboxIntervalTimer);
await this.worker?.close();
await this.sendQueue?.close();
await this.gatewayQueue?.close();
@@ -907,6 +948,46 @@ export class SendChainService implements OnModuleInit, OnModuleDestroy {
}
}
async handleSubmitSegmentResult(data: GatewaySubmitSegmentResultDto) {
const message = await this.requireMessageByGatewayEvent(data.messageId, data.gatewayMessageId);
const submittedAt = data.submittedAt ? new Date(data.submittedAt) : new Date();
await this.recordSubmitSegments(message, {
messageId: data.messageId,
channelId: data.channelId,
submitId: data.submitId,
sequenceId: data.sequenceId,
gatewayMessageId: data.gatewayMessageId ?? '',
submitStatus: normalizeSubmitStatus(data.submitStatus),
errorCode: data.errorCode,
errorMessage: data.errorMessage,
submittedAt: submittedAt.toISOString(),
segments: [{
segmentTotal: data.segmentTotal,
segmentIndex: data.segmentIndex,
sequenceId: data.sequenceId,
gatewayMessageId: data.gatewayMessageId,
submitStatus: data.submitStatus,
errorCode: data.errorCode,
errorMessage: data.errorMessage,
submittedAt: submittedAt.toISOString(),
}],
}, submittedAt);
if (data.gatewayMessageId) {
await this.prisma.smsSubmitRecord.updateMany({
where: {
...(data.submitId ? { submitId: data.submitId } : { messageRecordId: message.id }),
gatewayMessageId: null,
},
data: {
sequenceId: data.sequenceId,
gatewayMessageId: data.gatewayMessageId,
submittedAt,
},
});
}
return { accepted: true };
}
async handleSubmitResult(data: GatewaySubmitResultDto) {
const message = await this.requireMessageByGatewayEvent(data.messageId, data.gatewayMessageId);
const batchTask = message.batchTaskId
@@ -933,6 +1014,10 @@ export class SendChainService implements OnModuleInit, OnModuleDestroy {
if (data.submitStatus === 'accepted' && !isStandaloneChannelTest && message.tenantId && message.batchTaskId) {
const businessMessage = message as typeof message & { tenantId: string; batchTaskId: string };
await this.chargeAcceptedMessage(businessMessage);
const latest = await this.prisma.smsMessageRecord.findUnique({ where: { id: message.id } });
if (latest?.status === 'failed') {
await this.refundMessage(businessMessage, '先到失败回执补偿退款');
}
} else if (data.submitStatus !== 'accepted' && !isStandaloneChannelTest && message.tenantId && message.batchTaskId) {
const businessMessage = message as typeof message & { tenantId: string; batchTaskId: string };
const retried = await this.retryMessageIfAllowed(businessMessage, data.submitStatus === 'timeout' ? '提交超时补发' : '提交失败补发');
@@ -942,8 +1027,11 @@ export class SendChainService implements OnModuleInit, OnModuleDestroy {
}
await this.releaseMessageReservation(businessMessage, data.submitStatus === 'timeout' ? '提交超时释放冻结' : '提交失败释放冻结');
}
await this.prisma.smsMessageRecord.update({
where: { id: message.id },
const protectedTerminalStatuses = ['delivered', 'failed', 'unknown'];
const updated = await this.prisma.smsMessageRecord.updateMany({
where: data.submitStatus === 'accepted'
? { id: message.id, status: { notIn: protectedTerminalStatuses } }
: { id: message.id, status: { not: 'delivered' } },
data: {
gatewayMessageId: data.gatewayMessageId,
submitStatus: data.submitStatus,
@@ -954,6 +1042,15 @@ export class SendChainService implements OnModuleInit, OnModuleDestroy {
timeoutAt: data.submitStatus === 'timeout' ? submittedAt : undefined,
},
});
if (updated.count === 0 && data.submitStatus === 'accepted') {
await this.prisma.smsMessageRecord.updateMany({
where: { id: message.id, gatewayMessageId: null },
data: {
gatewayMessageId: data.gatewayMessageId,
submittedAt,
},
});
}
if (data.submitStatus !== 'accepted' && batchTask?.sourceType === 'cmpp' && message.tenantId && message.applicationId) {
await this.recordCmppFailureReceipt(
message as typeof message & { tenantId: string; applicationId: string; batchTaskId: string },
@@ -981,8 +1078,185 @@ export class SendChainService implements OnModuleInit, OnModuleDestroy {
return this.prisma.smsMessageRecord.findUnique({ where: { id: message.id } });
}
async handleReceipt(data: GatewayReceiptEventDto) {
const resolved = await this.resolveReceiptMessage(data);
async intakeReceipt(data: GatewayReceiptEventDto) {
const channel = await this.prisma.smsChannel.findUnique({
where: { id: data.channelId },
select: {
id: true,
account: true,
gatewayHost: true,
gatewayPort: true,
protocol: true,
cmppVersion: true,
},
});
if (!channel) {
throw new NotFoundException('SMS channel not found');
}
const deliveredAt = data.deliveredAt ? new Date(data.deliveredAt) : new Date();
const receiptKey = this.receiptEventKey(data, data.channelId);
const inbox = await this.prisma.upstreamReceiptInbox.upsert({
where: { receiptKey },
update: {
incomingConnectionId: data.connectionId,
},
create: {
receiptKey,
incomingChannelId: data.channelId,
incomingConnectionId: data.connectionId,
upstreamAccount: channel.account,
upstreamHost: channel.gatewayHost,
upstreamPort: channel.gatewayPort,
protocol: channel.protocol,
protocolVersion: channel.cmppVersion,
provisionalMessageId: data.messageId,
sequenceId: data.sequenceId,
gatewayMessageId: data.gatewayMessageId,
phoneNumber: data.phoneNumber?.trim() || null,
receiptStatus: data.receiptStatus,
rawStatus: data.rawStatus,
errorCode: data.errorCode,
errorMessage: data.errorMessage,
deliveredAt,
status: 'pending',
nextRetryAt: new Date(),
},
});
if (['pending', 'retrying'].includes(inbox.status)) {
setImmediate(() => void this.processUpstreamReceiptInboxRecord(inbox.id));
}
return { accepted: true, inboxId: inbox.id, status: inbox.status };
}
async processPendingUpstreamReceiptInbox(limit = 100) {
const now = new Date();
const staleBefore = new Date(
now.getTime()
- positiveInteger(
process.env.UPSTREAM_RECEIPT_INBOX_PROCESSING_STALE_MS,
DEFAULT_UPSTREAM_RECEIPT_INBOX_PROCESSING_STALE_MS,
),
);
const candidates = await this.prisma.upstreamReceiptInbox.findMany({
where: {
OR: [
{
status: { in: ['pending', 'retrying'] },
OR: [{ nextRetryAt: null }, { nextRetryAt: { lte: now } }],
},
{ status: 'processing', updatedAt: { lte: staleBefore } },
],
},
orderBy: [{ receivedAt: 'asc' }, { id: 'asc' }],
take: Math.min(Math.max(limit, 1), 500),
select: { id: true },
});
let processed = 0;
for (const candidate of candidates) {
if (await this.processUpstreamReceiptInboxRecord(candidate.id)) processed += 1;
}
return { scanned: candidates.length, processed };
}
private async processUpstreamReceiptInboxRecord(id: string) {
const staleBefore = new Date(
Date.now()
- positiveInteger(
process.env.UPSTREAM_RECEIPT_INBOX_PROCESSING_STALE_MS,
DEFAULT_UPSTREAM_RECEIPT_INBOX_PROCESSING_STALE_MS,
),
);
const claimed = await this.prisma.upstreamReceiptInbox.updateMany({
where: {
id,
OR: [
{ status: { in: ['pending', 'retrying'] } },
{ status: 'processing', updatedAt: { lte: staleBefore } },
],
},
data: { status: 'processing', attemptCount: { increment: 1 }, nextRetryAt: null },
});
if (claimed.count !== 1) return false;
const inbox = await this.prisma.upstreamReceiptInbox.findUnique({ where: { id } });
if (!inbox) return false;
try {
const message = await this.handleReceipt({
messageId: inbox.provisionalMessageId ?? undefined,
channelId: inbox.incomingChannelId,
connectionId: inbox.incomingConnectionId ?? undefined,
sequenceId: inbox.sequenceId ?? undefined,
gatewayMessageId: inbox.gatewayMessageId,
phoneNumber: inbox.phoneNumber ?? undefined,
receiptStatus: normalizeReceiptStatus(inbox.receiptStatus),
rawStatus: inbox.rawStatus,
errorCode: inbox.errorCode ?? undefined,
errorMessage: inbox.errorMessage ?? undefined,
deliveredAt: inbox.deliveredAt.toISOString(),
}, {
account: inbox.upstreamAccount,
gatewayHost: inbox.upstreamHost,
gatewayPort: inbox.upstreamPort,
protocol: inbox.protocol,
cmppVersion: inbox.protocolVersion,
});
const matchedMessageRecordId = message && 'id' in message ? message.id : message?.messageRecordId;
const matchedChannelId = message && 'channelId' in message ? message.channelId : undefined;
await this.prisma.upstreamReceiptInbox.update({
where: { id },
data: {
status: 'matched',
matchedMessageRecordId: matchedMessageRecordId ?? null,
matchedChannelId: matchedChannelId ?? null,
lastError: null,
processedAt: new Date(),
},
});
return true;
} catch (error) {
const maxAttempts = positiveInteger(
process.env.UPSTREAM_RECEIPT_INBOX_MAX_ATTEMPTS,
DEFAULT_UPSTREAM_RECEIPT_INBOX_MAX_ATTEMPTS,
);
const maxAgeHours = positiveInteger(
process.env.UPSTREAM_RECEIPT_INBOX_MAX_AGE_HOURS,
DEFAULT_UPSTREAM_RECEIPT_INBOX_MAX_AGE_HOURS,
);
const exhausted = inbox.attemptCount >= maxAttempts
|| inbox.receivedAt.getTime() <= Date.now() - maxAgeHours * 60 * 60_000;
const retryDelayMs = Math.min(30 * 60_000, 5_000 * 2 ** Math.min(Math.max(inbox.attemptCount - 1, 0), 8));
await this.prisma.upstreamReceiptInbox.update({
where: { id },
data: {
status: exhausted ? 'unmatched' : 'retrying',
nextRetryAt: exhausted ? null : new Date(Date.now() + retryDelayMs),
lastError: error instanceof Error ? error.message : String(error),
processedAt: exhausted ? new Date() : null,
},
});
return false;
}
}
private async runUpstreamReceiptInboxScan() {
if (this.upstreamReceiptInboxScanRunning) return;
this.upstreamReceiptInboxScanRunning = true;
try {
const result = await this.processPendingUpstreamReceiptInbox();
if (result.processed > 0) {
this.logger.log(`Matched ${result.processed}/${result.scanned} pending upstream receipts`);
}
} catch (error) {
this.logger.error('Upstream receipt inbox scan failed', error instanceof Error ? error.stack : String(error));
} finally {
this.upstreamReceiptInboxScanRunning = false;
}
}
async handleReceipt(
data: GatewayReceiptEventDto,
incomingIdentity?: { account: string; gatewayHost: string; gatewayPort: number; protocol: string; cmppVersion: string },
) {
const resolved = await this.resolveReceiptMessage(data, incomingIdentity);
const logicalChannelId = resolved.channelId ?? data.channelId;
const receiptKey = this.receiptEventKey(data, logicalChannelId);
const existingReceipt = await this.prisma.smsReceiptRecord.findUnique({
@@ -1202,6 +1476,32 @@ export class SendChainService implements OnModuleInit, OnModuleDestroy {
async markDownstreamDeliverySent(data: GatewayDownstreamSentDto) {
const sentAt = asDateOrNull(data.sentAt) ?? new Date();
const ackDeadlineAt = asDateOrNull(data.ackDeadlineAt) ?? new Date(sentAt.getTime() + downstreamAckTimeoutMs());
const delivery = await this.prisma.cmppDownstreamDelivery.findUnique({ where: { id: data.id } });
if (!delivery) {
throw new NotFoundException('Downstream delivery not found');
}
const attemptKey = downstreamDeliveryAttemptKey(data);
await this.prisma.cmppDownstreamDeliveryAttempt.upsert({
where: { attemptKey },
update: {
connectionId: data.connectionId,
sequenceId: data.sequenceId,
messageId: data.messageId,
sentAt,
ackDeadlineAt,
},
create: {
deliveryId: data.id,
attemptKey,
attemptNo: delivery.retryCount + 1,
connectionId: data.connectionId,
sequenceId: data.sequenceId,
messageId: data.messageId,
status: 'awaiting_ack',
sentAt,
ackDeadlineAt,
},
});
await this.prisma.cmppDownstreamDelivery.updateMany({
where: { id: data.id, status: { not: 'delivered' } },
data: {
@@ -1221,7 +1521,48 @@ export class SendChainService implements OnModuleInit, OnModuleDestroy {
async acknowledgeDownstreamDelivery(data: GatewayDownstreamAcknowledgedDto) {
const acknowledgedAt = asDateOrNull(data.acknowledgedAt) ?? new Date();
const acknowledgedMessageId = String(data.messageId ?? '').trim();
if (data.result === 0 && acknowledgedMessageId !== '' && acknowledgedMessageId !== '0') {
const delivery = await this.prisma.cmppDownstreamDelivery.findUnique({ where: { id: data.id } });
if (!delivery) {
throw new NotFoundException('Downstream delivery not found');
}
const attemptKey = downstreamDeliveryAttemptKey(data);
const acknowledgementAccepted = data.result === 0 && acknowledgedMessageId !== '' && acknowledgedMessageId !== '0';
await this.prisma.cmppDownstreamDeliveryAttempt.upsert({
where: { attemptKey },
update: {
status: acknowledgementAccepted ? 'acknowledged' : 'rejected',
connectionId: data.connectionId,
sequenceId: data.sequenceId,
messageId: data.messageId,
acknowledgedAt,
ackResult: data.result,
ackDeadlineAt: null,
failureType: acknowledgementAccepted ? null : data.result === 0 ? 'ack_invalid' : 'ack_rejected',
errorMessage: acknowledgementAccepted
? null
: data.result === 0
? 'CMPP_DELIVER_RESP Msg_Id=0'
: `CMPP_DELIVER_RESP result=${data.result}`,
},
create: {
deliveryId: data.id,
attemptKey,
attemptNo: delivery.retryCount + 1,
connectionId: data.connectionId,
sequenceId: data.sequenceId,
messageId: data.messageId,
status: acknowledgementAccepted ? 'acknowledged' : 'rejected',
acknowledgedAt,
ackResult: data.result,
failureType: acknowledgementAccepted ? null : data.result === 0 ? 'ack_invalid' : 'ack_rejected',
errorMessage: acknowledgementAccepted
? null
: data.result === 0
? 'CMPP_DELIVER_RESP Msg_Id=0'
: `CMPP_DELIVER_RESP result=${data.result}`,
},
});
if (acknowledgementAccepted) {
await this.prisma.cmppDownstreamDelivery.updateMany({
where: { id: data.id, status: { not: 'delivered' } },
data: {
@@ -1255,7 +1596,12 @@ export class SendChainService implements OnModuleInit, OnModuleDestroy {
return this.markDownstreamDeliveryFailed(data.id, `downstream CMPP_DELIVER_RESP result=${data.result}`, 'ack_rejected');
}
async markDownstreamDeliveryFailed(id: string, errorMessage?: string, failureType: GatewayDownstreamFailureType = 'send_failed') {
async markDownstreamDeliveryFailed(
id: string,
errorMessage?: string,
failureType: GatewayDownstreamFailureType = 'send_failed',
attempt?: GatewayDownstreamSentDto,
) {
const delivery = await this.prisma.cmppDownstreamDelivery.findUnique({ where: { id } });
if (!delivery) {
throw new NotFoundException('Downstream delivery not found');
@@ -1272,6 +1618,33 @@ export class SendChainService implements OnModuleInit, OnModuleDestroy {
const nonRetryableFailure = failureType === 'unrecoverable' || failureType === 'queue_timeout';
const finalFailure = nonRetryableFailure || !retryAllowed || retryCount >= downstreamMaxRetries();
const finalStatus = failureType === 'ack_rejected' ? 'rejected' : acknowledgementFailure ? 'unconfirmed' : 'failed';
if (attempt && (attempt.connectionId || attempt.sequenceId || attempt.messageId)) {
const attemptKey = downstreamDeliveryAttemptKey({ ...attempt, id });
await this.prisma.cmppDownstreamDeliveryAttempt.upsert({
where: { attemptKey },
update: {
status: 'failed',
connectionId: attempt.connectionId,
sequenceId: attempt.sequenceId,
messageId: attempt.messageId,
failureType,
errorMessage: errorMessage ?? 'downstream delivery failed',
ackDeadlineAt: null,
},
create: {
deliveryId: id,
attemptKey,
attemptNo: delivery.retryCount + 1,
connectionId: attempt.connectionId,
sequenceId: attempt.sequenceId,
messageId: attempt.messageId,
status: 'failed',
sentAt: asDateOrNull(attempt.sentAt),
failureType,
errorMessage: errorMessage ?? 'downstream delivery failed',
},
});
}
const updated = await this.prisma.cmppDownstreamDelivery.update({
where: { id },
data: {
@@ -1914,11 +2287,14 @@ export class SendChainService implements OnModuleInit, OnModuleDestroy {
) as GatewayControlDeliveryResult;
if (result.sent || result.delivered) {
await this.markDownstreamDeliverySent({ id: delivery.id, ...result });
} else if (result.reasonCode === 'SUBMIT_RESPONSE_PENDING') {
return delivery;
} else {
await this.markDownstreamDeliveryFailed(
delivery.id,
downstreamControlFailureMessage(result),
result.retryable === false ? 'unrecoverable' : 'send_failed',
{ id: delivery.id, ...result },
);
}
} catch (error) {
@@ -3831,7 +4207,10 @@ export class SendChainService implements OnModuleInit, OnModuleDestroy {
return message;
}
private async resolveReceiptMessage(data: GatewayReceiptEventDto) {
private async resolveReceiptMessage(
data: GatewayReceiptEventDto,
incomingIdentity?: { account: string; gatewayHost: string; gatewayPort: number; protocol: string; cmppVersion: string },
) {
const exactMessage = data.messageId
? await this.prisma.smsMessageRecord.findUnique({ where: { messageId: data.messageId } })
: null;
@@ -3896,7 +4275,8 @@ export class SendChainService implements OnModuleInit, OnModuleDestroy {
throw new NotFoundException('SMS message record not found');
}
const incomingChannel = await this.prisma.smsChannel.findUnique({ where: { id: data.channelId } });
const incomingChannel = incomingIdentity
?? await this.prisma.smsChannel.findUnique({ where: { id: data.channelId } });
if (!incomingChannel) {
throw new NotFoundException('SMS message record not found');
}
@@ -3920,7 +4300,7 @@ export class SendChainService implements OnModuleInit, OnModuleDestroy {
};
}
const sameSupplierSegments = segmentMatches.filter((candidate) =>
candidate.channel && this.isSameSupplierConnection(incomingChannel, candidate.channel));
candidate.channel && this.isSameUpstreamEndpointIdentity(incomingChannel, candidate.channel));
if (sameSupplierSegments.length === 1 && sameSupplierSegments[0]?.messageRecord) {
return {
message: sameSupplierSegments[0].messageRecord,
@@ -3940,7 +4320,7 @@ export class SendChainService implements OnModuleInit, OnModuleDestroy {
take: 10,
});
const sameSupplierSubmits = crossConnectionSubmits.filter((candidate) =>
candidate.channel && this.isSameSupplierConnection(incomingChannel, candidate.channel));
candidate.channel && this.isSameUpstreamEndpointIdentity(incomingChannel, candidate.channel));
if (sameSupplierSubmits.length === 1 && sameSupplierSubmits[0]?.messageRecord) {
return {
message: sameSupplierSubmits[0].messageRecord,
@@ -3988,7 +4368,7 @@ export class SendChainService implements OnModuleInit, OnModuleDestroy {
};
}
private isSameSupplierConnection(
private isSameUpstreamEndpointIdentity(
left: { account: string; gatewayHost: string; gatewayPort: number; protocol: string; cmppVersion: string },
right: { account: string; gatewayHost: string; gatewayPort: number; protocol: string; cmppVersion: string },
) {
@@ -4341,6 +4721,24 @@ function parseOptionalSequenceId(value: string | null | undefined) {
return Number.isInteger(parsed) && parsed >= 0 && parsed <= 0xffffffff ? parsed : undefined;
}
function normalizeSubmitStatus(value: string): GatewaySubmitResultDto['submitStatus'] {
return value === 'accepted' || value === 'rejected' || value === 'timeout' ? value : 'timeout';
}
function normalizeReceiptStatus(value: string): GatewayReceiptEventDto['receiptStatus'] {
return value === 'delivered' || value === 'undelivered' || value === 'unknown' ? value : 'unknown';
}
function downstreamDeliveryAttemptKey(data: GatewayDownstreamSentDto) {
return createHash('sha256').update([
data.id,
data.connectionId ?? '',
data.sequenceId ?? '',
data.messageId ?? '',
data.sequenceId ? '' : data.sentAt ?? '',
].join('\u0000')).digest('hex');
}
function shanghaiDateKey(now = new Date()) {
const parts = new Intl.DateTimeFormat('en-CA', {
timeZone: 'Asia/Shanghai',