hectorzhao/lislgosms
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

fix: harden upstream and downstream receipt delivery

Browse Source
This commit is contained in:
hectorzhao
2026-07-25 23:32:13 +08:00
parent 5018167696
commit 54617c927e
18 changed files with 1797 additions and 779 deletions
Download Patch File Download Diff File Expand all files Collapse all files
api/prisma/migrations/20260725160000_add_reliable_receipt_delivery_tracking/migration.sql
+86
View File
@@ -0,0 +1,86 @@
CREATE TABLE "CmppDownstreamDeliveryAttempt" (
"id" TEXT NOT NULL,
"deliveryId" TEXT NOT NULL,
"attemptKey" TEXT NOT NULL,
"attemptNo" INTEGER NOT NULL,
"connectionId" TEXT,
"sequenceId" TEXT,
"messageId" TEXT,
"status" TEXT NOT NULL,
"sentAt" TIMESTAMP(3),
"ackDeadlineAt" TIMESTAMP(3),
"acknowledgedAt" TIMESTAMP(3),
"ackResult" INTEGER,
"failureType" TEXT,
"errorMessage" TEXT,
"createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
"updatedAt" TIMESTAMP(3) NOT NULL,
CONSTRAINT "CmppDownstreamDeliveryAttempt_pkey" PRIMARY KEY ("id")
);
CREATE TABLE "UpstreamReceiptInbox" (
"id" TEXT NOT NULL,
"receiptKey" TEXT NOT NULL,
"incomingChannelId" TEXT NOT NULL,
"incomingConnectionId" TEXT,
"upstreamAccount" TEXT NOT NULL,
"upstreamHost" TEXT NOT NULL,
"upstreamPort" INTEGER NOT NULL,
"protocol" TEXT NOT NULL,
"protocolVersion" TEXT NOT NULL,
"provisionalMessageId" TEXT,
"sequenceId" INTEGER,
"gatewayMessageId" TEXT NOT NULL,
"phoneNumber" TEXT,
"receiptStatus" TEXT NOT NULL,
"rawStatus" TEXT NOT NULL,
"errorCode" TEXT,
"errorMessage" TEXT,
"deliveredAt" TIMESTAMP(3) NOT NULL,
"receivedAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
"status" TEXT NOT NULL DEFAULT 'pending',
"matchedMessageRecordId" TEXT,
"matchedSubmitRecordId" TEXT,
"matchedChannelId" TEXT,
"attemptCount" INTEGER NOT NULL DEFAULT 0,
"nextRetryAt" TIMESTAMP(3),
"lastError" TEXT,
"processedAt" TIMESTAMP(3),
"createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
"updatedAt" TIMESTAMP(3) NOT NULL,
CONSTRAINT "UpstreamReceiptInbox_pkey" PRIMARY KEY ("id")
);
CREATE UNIQUE INDEX "CmppDownstreamDeliveryAttempt_attemptKey_key"
ON "CmppDownstreamDeliveryAttempt"("attemptKey");
CREATE INDEX "CmppDownstreamDeliveryAttempt_deliveryId_attemptNo_idx"
ON "CmppDownstreamDeliveryAttempt"("deliveryId", "attemptNo");
CREATE INDEX "CmppDownstreamDeliveryAttempt_status_ackDeadlineAt_idx"
ON "CmppDownstreamDeliveryAttempt"("status", "ackDeadlineAt");
CREATE INDEX "CmppDownstreamDeliveryAttempt_connectionId_sequenceId_idx"
ON "CmppDownstreamDeliveryAttempt"("connectionId", "sequenceId");
CREATE UNIQUE INDEX "UpstreamReceiptInbox_receiptKey_key"
ON "UpstreamReceiptInbox"("receiptKey");
CREATE INDEX "UpstreamReceiptInbox_status_nextRetryAt_receivedAt_idx"
ON "UpstreamReceiptInbox"("status", "nextRetryAt", "receivedAt");
CREATE INDEX "UpstreamReceiptInbox_gatewayMessageId_phoneNumber_idx"
ON "UpstreamReceiptInbox"("gatewayMessageId", "phoneNumber");
CREATE INDEX "UpstreamReceiptInbox_incomingChannelId_receivedAt_idx"
ON "UpstreamReceiptInbox"("incomingChannelId", "receivedAt");
CREATE INDEX "UpstreamReceiptInbox_matchedMessageRecordId_idx"
ON "UpstreamReceiptInbox"("matchedMessageRecordId");
ALTER TABLE "CmppDownstreamDeliveryAttempt"
ADD CONSTRAINT "CmppDownstreamDeliveryAttempt_deliveryId_fkey"
FOREIGN KEY ("deliveryId") REFERENCES "CmppDownstreamDelivery"("id")
ON DELETE CASCADE ON UPDATE CASCADE;
api/prisma/schema.prisma
+790 -727
View File
File diff suppressed because it is too large Load Diff
api/src/operations/operations.service.spec.ts
+6 -1
View File
@@ -668,7 +668,12 @@ describe('OperationsService', () => {
lte: new Date('2026-07-15T23:59:59.999+08:00'),
},
}),
include: { tenant: true, application: true, messageRecord: true },
include: {
tenant: true,
application: true,
messageRecord: true,
attempts: { orderBy: [{ attemptNo: 'desc' }, { createdAt: 'desc' }] },
},
orderBy: { createdAt: 'desc' },
skip: 0,
take: 10,
api/src/operations/operations.service.ts
+6 -1
View File
@@ -814,7 +814,12 @@ export class OperationsService {
const [items, total] = await Promise.all([
this.prisma.cmppDownstreamDelivery.findMany({
where,
include: { tenant: true, application: true, messageRecord: true },
include: {
tenant: true,
application: true,
messageRecord: true,
attempts: { orderBy: [{ attemptNo: 'desc' }, { createdAt: 'desc' }] },
},
orderBy: { createdAt: 'desc' },
skip: (page - 1) * pageSize,
take: pageSize,
api/src/send-chain/gateway-events.controller.spec.ts
+47
View File
@@ -4,7 +4,9 @@ import { GatewayEventsController } from './gateway-events.controller';
describe('GatewayEventsController protocol logging', () => {
const sendChain = {
handleSubmitResult: jest.fn(),
handleSubmitSegmentResult: jest.fn(),
handleReceipt: jest.fn(),
intakeReceipt: jest.fn(),
};
const protocolLogs = {
record: jest.fn(),
@@ -30,6 +32,51 @@ describe('GatewayEventsController protocol logging', () => {
expect(protocolLogs.record).not.toHaveBeenCalled();
});
it('persists each supplier segment response without adding a duplicate protocol log', async () => {
sendChain.handleSubmitSegmentResult.mockResolvedValue({ accepted: true });
const body = {
messageId: 'MSG-LONG-1',
channelId: 'channel-1',
submitId: 'submit-1',
segmentIndex: 2,
segmentTotal: 3,
gatewayMessageId: '456',
sequenceId: 8,
submitStatus: 'accepted' as const,
};
await expect(controller.submitSegmentResult(body)).resolves.toEqual({ accepted: true });
expect(sendChain.handleSubmitSegmentResult).toHaveBeenCalledWith(body);
expect(protocolLogs.record).not.toHaveBeenCalled();
});
it('logs a supplier receipt only after it is durably accepted by the inbox', async () => {
sendChain.intakeReceipt.mockResolvedValue({ accepted: true, inboxId: 'inbox-1', status: 'pending' });
const body = {
messageId: 'receipt-456',
channelId: 'channel-1',
connectionId: 'channel-1:0',
gatewayMessageId: '456',
phoneNumber: '13127620092',
receiptStatus: 'undelivered' as const,
rawStatus: 'UNDELIV',
};
await expect(controller.receiptIntake(body)).resolves.toEqual({
accepted: true,
inboxId: 'inbox-1',
status: 'pending',
});
expect(protocolLogs.record).toHaveBeenCalledWith(expect.objectContaining({
direction: 'channel_to_platform',
eventType: 'deliver_receipt',
channelId: 'channel-1',
messageId: 'receipt-456',
gatewayMessageId: '456',
status: 'success',
}));
});
it('accepts only safe outbound Gateway packet events', () => {
expect(controller.protocolLog({
protocol: 'cmpp',
api/src/send-chain/gateway-events.controller.ts
+13 -2
View File
@@ -11,6 +11,7 @@ import {
GatewayReceiptEventDto,
GatewaySubmitDeadLetterDto,
GatewaySubmitResultDto,
GatewaySubmitSegmentResultDto,
GatewayUplinkEventDto,
SendChainService,
} from './send-chain.service';
@@ -31,6 +32,16 @@ export class GatewayEventsController {
return this.sendChain.handleSubmitResult(body);
}
@Post('submit-segment-result')
submitSegmentResult(@Body() body: GatewaySubmitSegmentResultDto) {
return this.sendChain.handleSubmitSegmentResult(body);
}
@Post('receipt/intake')
receiptIntake(@Body() body: GatewayReceiptEventDto) {
return this.trackGatewayEvent('deliver_receipt', body, () => this.sendChain.intakeReceipt(body));
}
@Post('receipt')
receipt(@Body() body: GatewayReceiptEventDto) {
return this.trackGatewayEvent('deliver_receipt', body, () => this.sendChain.handleReceipt(body));
@@ -104,8 +115,8 @@ export class GatewayEventsController {
}
@Post('downstream/failed')
downstreamFailed(@Body() body: { id: string; errorMessage?: string; failureType?: GatewayDownstreamFailureType }) {
return this.sendChain.markDownstreamDeliveryFailed(body.id, body.errorMessage, body.failureType);
downstreamFailed(@Body() body: GatewayDownstreamSentDto & { errorMessage?: string; failureType?: GatewayDownstreamFailureType }) {
return this.sendChain.markDownstreamDeliveryFailed(body.id, body.errorMessage, body.failureType, body);
}
@Post('downstream/recovery-status')
api/src/send-chain/send-chain.service.spec.ts
+139 -4
View File
@@ -36,6 +36,7 @@ function createPrismaMock() {
sendRegion: '全国',
gatewayHost: '127.0.0.1',
gatewayPort: 17890,
protocol: 'CMPP',
passwordCipher: 'secret',
cmppVersion: '3.0',
config: { serviceId: 'SMS' },
@@ -240,6 +241,22 @@ function createPrismaMock() {
update: jest.fn().mockImplementation(({ data }) => Promise.resolve({ id: 'delivery-1', tenantId: 'tenant-1', applicationId: 'app-1', messageId: 'MSG-1', deliveryType: 'receipt', ...data })),
updateMany: jest.fn().mockResolvedValue({ count: 1 }),
},
cmppDownstreamDeliveryAttempt: {
upsert: jest.fn().mockResolvedValue({ id: 'delivery-attempt-1' }),
findMany: jest.fn().mockResolvedValue([]),
},
upstreamReceiptInbox: {
upsert: jest.fn().mockImplementation(({ create }) => Promise.resolve({
id: 'receipt-inbox-1',
attemptCount: 0,
receivedAt: new Date(),
...create,
})),
findMany: jest.fn().mockResolvedValue([]),
findUnique: jest.fn().mockResolvedValue(null),
update: jest.fn().mockImplementation(({ data }) => Promise.resolve({ id: 'receipt-inbox-1', ...data })),
updateMany: jest.fn().mockResolvedValue({ count: 1 }),
},
gatewaySubmitDeadLetter: {
upsert: jest.fn().mockResolvedValue({ id: 'dead-1' }),
findUnique: jest.fn().mockResolvedValue({
@@ -1765,8 +1782,8 @@ describe('SendChainService', () => {
where: { submitId: 'SUB-1' },
data: expect.objectContaining({ sequenceId: 7, gatewayMessageId: 'GW-1', submitStatus: 'accepted' }),
});
expect(prisma.smsMessageRecord.update).toHaveBeenCalledWith({
where: { id: 'record-1' },
expect(prisma.smsMessageRecord.updateMany).toHaveBeenCalledWith({
where: { id: 'record-1', status: { notIn: ['delivered', 'failed', 'unknown'] } },
data: expect.objectContaining({ gatewayMessageId: 'GW-1', status: 'submitted', submitStatus: 'accepted' }),
});
expect(billing.release).toHaveBeenCalledWith(expect.objectContaining({ amountCents: 3, relatedId: 'task-1' }));
@@ -1824,8 +1841,8 @@ describe('SendChainService', () => {
expect(prisma.channelRouteRule.findFirst).not.toHaveBeenCalled();
expect(billing.release).not.toHaveBeenCalled();
expect(prisma.smsBatchTask.update).not.toHaveBeenCalled();
expect(prisma.smsMessageRecord.update).toHaveBeenCalledWith({
where: { id: 'record-1' },
expect(prisma.smsMessageRecord.updateMany).toHaveBeenCalledWith({
where: { id: 'record-1', status: { not: 'delivered' } },
data: expect.objectContaining({
gatewayMessageId: 'GW-1',
submitStatus: 'timeout',
@@ -2680,6 +2697,108 @@ describe('SendChainService', () => {
}));
});
it('persists each upstream SubmitResp segment before the aggregate result arrives', async () => {
const { service, prisma } = createService();
await service.handleSubmitSegmentResult({
messageId: 'MSG-1',
channelId: 'channel-1',
submitId: 'SUB-1',
segmentTotal: 3,
segmentIndex: 1,
sequenceId: 71,
gatewayMessageId: 'GW-SEG-1',
submitStatus: 'accepted',
submittedAt: '2026-07-25T15:00:00.000Z',
});
expect(prisma.smsMessageSegmentAudit.upsert).toHaveBeenCalledWith(expect.objectContaining({
where: {
messageRecordId_submitId_segmentIndex: {
messageRecordId: 'record-1',
submitId: 'SUB-1',
segmentIndex: 1,
},
},
create: expect.objectContaining({
segmentTotal: 3,
sequenceId: 71,
gatewayMessageId: 'GW-SEG-1',
submitStatus: 'accepted',
}),
}));
expect(prisma.smsSubmitRecord.updateMany).toHaveBeenCalledWith(expect.objectContaining({
where: { submitId: 'SUB-1', gatewayMessageId: null },
data: expect.objectContaining({ sequenceId: 71, gatewayMessageId: 'GW-SEG-1' }),
}));
});
it('durably intakes an upstream receipt before asynchronous business matching', async () => {
const { service, prisma } = createService();
jest.spyOn(service as any, 'processUpstreamReceiptInboxRecord').mockResolvedValue(false);
await expect(service.intakeReceipt({
messageId: 'receipt-9001',
channelId: 'channel-1',
connectionId: 'gateway-connection-2',
sequenceId: 81,
gatewayMessageId: '9001',
phoneNumber: '13800000001',
receiptStatus: 'delivered',
rawStatus: 'DELIVRD',
deliveredAt: '2026-07-25T15:01:00.000Z',
})).resolves.toEqual(expect.objectContaining({
accepted: true,
inboxId: 'receipt-inbox-1',
}));
expect(prisma.upstreamReceiptInbox.upsert).toHaveBeenCalledWith(expect.objectContaining({
create: expect.objectContaining({
incomingChannelId: 'channel-1',
incomingConnectionId: 'gateway-connection-2',
upstreamAccount: 'cmpp-account',
upstreamHost: '127.0.0.1',
upstreamPort: 17890,
protocol: 'CMPP',
protocolVersion: '3.0',
gatewayMessageId: '9001',
status: 'pending',
}),
}));
});
it('does not downgrade an early terminal receipt when the aggregate submit result arrives later', async () => {
const { service, prisma } = createService();
const terminalMessage = {
id: 'record-1', messageId: 'MSG-1', tenantId: null, batchTaskId: null, applicationId: null,
channelId: 'channel-1', submitId: 'SUB-1', gatewayMessageId: 'GW-SEG-1',
phoneNumber: '13800000001', billingUnits: 1, amountCents: 0, status: 'failed',
};
prisma.smsMessageRecord.findFirst.mockResolvedValue(terminalMessage);
prisma.smsMessageRecord.findUnique.mockResolvedValue(terminalMessage);
prisma.smsMessageRecord.updateMany
.mockResolvedValueOnce({ count: 0 })
.mockResolvedValueOnce({ count: 1 });
await service.handleSubmitResult({
messageId: 'MSG-1',
channelId: 'channel-1',
submitId: 'SUB-1',
sequenceId: 7,
gatewayMessageId: 'GW-1',
submitStatus: 'accepted',
});
expect(prisma.smsMessageRecord.updateMany).toHaveBeenNthCalledWith(1, expect.objectContaining({
where: { id: 'record-1', status: { notIn: ['delivered', 'failed', 'unknown'] } },
data: expect.objectContaining({ status: 'submitted' }),
}));
expect(prisma.smsMessageRecord.updateMany).toHaveBeenNthCalledWith(2, {
where: { id: 'record-1', gatewayMessageId: null },
data: expect.objectContaining({ gatewayMessageId: 'GW-1' }),
});
});
it('recovers a stale submit requeue with the same Redis idempotency key', async () => {
const { service, prisma } = createService();
const stale = {
@@ -2929,6 +3048,15 @@ describe('SendChainService', () => {
where: { id: 'delivery-1', status: { not: 'delivered' } },
data: expect.objectContaining({ status: 'awaiting_ack', ackSequenceId: '37', connectionId: 'conn-1' }),
}));
expect(prisma.cmppDownstreamDeliveryAttempt.upsert).toHaveBeenLastCalledWith(expect.objectContaining({
create: expect.objectContaining({
deliveryId: 'delivery-1',
attemptNo: 1,
connectionId: 'conn-1',
sequenceId: '37',
status: 'awaiting_ack',
}),
}));
await service.acknowledgeDownstreamDelivery({
id: 'delivery-1',
@@ -2941,6 +3069,13 @@ describe('SendChainService', () => {
expect(prisma.cmppDownstreamDelivery.updateMany).toHaveBeenLastCalledWith(expect.objectContaining({
data: expect.objectContaining({ status: 'delivered', ackResult: 0, deliveredAt: new Date('2026-07-14T03:40:18.060Z') }),
}));
expect(prisma.cmppDownstreamDeliveryAttempt.upsert).toHaveBeenLastCalledWith(expect.objectContaining({
update: expect.objectContaining({
status: 'acknowledged',
acknowledgedAt: new Date('2026-07-14T03:40:18.060Z'),
ackResult: 0,
}),
}));
});
it('does not treat Result=0 with Msg_Id=0 as a business acknowledgement', async () => {
api/src/send-chain/send-chain.service.ts
+409 -11
View File
@@ -89,6 +89,21 @@ export interface GatewaySubmitResultDto {
}>;
}
export interface GatewaySubmitSegmentResultDto {
traceId?: string;
messageId: string;
channelId: string;
submitId?: string;
segmentTotal: number;
segmentIndex: number;
sequenceId?: number;
gatewayMessageId?: string;
submitStatus: 'accepted' | 'rejected' | 'timeout' | string;
errorCode?: string;
errorMessage?: string;
submittedAt?: string;
}
export interface GatewayReceiptEventDto {
traceId?: string;
messageId?: string;
@@ -101,6 +116,7 @@ export interface GatewayReceiptEventDto {
errorCode?: string;
errorMessage?: string;
deliveredAt?: string;
connectionId?: string;
}
export interface GatewayUplinkEventDto {
@@ -269,6 +285,11 @@ const DEFAULT_DOWNSTREAM_MANUAL_REQUEUE_STALE_MS = 2 * 60_000;
const DEFAULT_INBOUND_LONG_MESSAGE_SCAN_INTERVAL_MS = 60_000;
const INBOUND_LONG_MESSAGE_SCAN_INITIAL_DELAY_MS = 10_000;
const DEFAULT_INBOUND_LONG_MESSAGE_PROCESSING_STALE_SECONDS = 30;
const DEFAULT_UPSTREAM_RECEIPT_INBOX_SCAN_INTERVAL_MS = 5_000;
const UPSTREAM_RECEIPT_INBOX_INITIAL_DELAY_MS = 1_000;
const DEFAULT_UPSTREAM_RECEIPT_INBOX_PROCESSING_STALE_MS = 2 * 60_000;
const DEFAULT_UPSTREAM_RECEIPT_INBOX_MAX_ATTEMPTS = 30;
const DEFAULT_UPSTREAM_RECEIPT_INBOX_MAX_AGE_HOURS = 72;
const GATEWAY_SUBMIT_REQUEUE_IDEMPOTENCY_TTL_SECONDS = 30 * 24 * 60 * 60;
const BULLMQ_PRIORITY: Record<QueuePriority, number> = {
priority: 1,
@@ -290,6 +311,9 @@ export class SendChainService implements OnModuleInit, OnModuleDestroy {
private scheduledDispatchScanRunning = false;
private inboundLongMessageInitialTimer?: ReturnType<typeof setTimeout>;
private inboundLongMessageIntervalTimer?: ReturnType<typeof setInterval>;
private upstreamReceiptInboxInitialTimer?: ReturnType<typeof setTimeout>;
private upstreamReceiptInboxIntervalTimer?: ReturnType<typeof setInterval>;
private upstreamReceiptInboxScanRunning = false;
constructor(
private readonly prisma: PrismaService,
@@ -342,6 +366,21 @@ export class SendChainService implements OnModuleInit, OnModuleDestroy {
);
this.inboundLongMessageIntervalTimer.unref?.();
}
if (process.env.UPSTREAM_RECEIPT_INBOX_SCAN_ENABLED !== 'false') {
this.upstreamReceiptInboxInitialTimer = setTimeout(
() => void this.runUpstreamReceiptInboxScan(),
UPSTREAM_RECEIPT_INBOX_INITIAL_DELAY_MS,
);
this.upstreamReceiptInboxInitialTimer.unref?.();
this.upstreamReceiptInboxIntervalTimer = setInterval(
() => void this.runUpstreamReceiptInboxScan(),
positiveInteger(
process.env.UPSTREAM_RECEIPT_INBOX_SCAN_INTERVAL_MS,
DEFAULT_UPSTREAM_RECEIPT_INBOX_SCAN_INTERVAL_MS,
),
);
this.upstreamReceiptInboxIntervalTimer.unref?.();
}
}
async onModuleDestroy() {
@@ -351,6 +390,8 @@ export class SendChainService implements OnModuleInit, OnModuleDestroy {
if (this.scheduledDispatchIntervalTimer) clearInterval(this.scheduledDispatchIntervalTimer);
if (this.inboundLongMessageInitialTimer) clearTimeout(this.inboundLongMessageInitialTimer);
if (this.inboundLongMessageIntervalTimer) clearInterval(this.inboundLongMessageIntervalTimer);
if (this.upstreamReceiptInboxInitialTimer) clearTimeout(this.upstreamReceiptInboxInitialTimer);
if (this.upstreamReceiptInboxIntervalTimer) clearInterval(this.upstreamReceiptInboxIntervalTimer);
await this.worker?.close();
await this.sendQueue?.close();
await this.gatewayQueue?.close();
@@ -907,6 +948,46 @@ export class SendChainService implements OnModuleInit, OnModuleDestroy {
}
}
async handleSubmitSegmentResult(data: GatewaySubmitSegmentResultDto) {
const message = await this.requireMessageByGatewayEvent(data.messageId, data.gatewayMessageId);
const submittedAt = data.submittedAt ? new Date(data.submittedAt) : new Date();
await this.recordSubmitSegments(message, {
messageId: data.messageId,
channelId: data.channelId,
submitId: data.submitId,
sequenceId: data.sequenceId,
gatewayMessageId: data.gatewayMessageId ?? '',
submitStatus: normalizeSubmitStatus(data.submitStatus),
errorCode: data.errorCode,
errorMessage: data.errorMessage,
submittedAt: submittedAt.toISOString(),
segments: [{
segmentTotal: data.segmentTotal,
segmentIndex: data.segmentIndex,
sequenceId: data.sequenceId,
gatewayMessageId: data.gatewayMessageId,
submitStatus: data.submitStatus,
errorCode: data.errorCode,
errorMessage: data.errorMessage,
submittedAt: submittedAt.toISOString(),
}],
}, submittedAt);
if (data.gatewayMessageId) {
await this.prisma.smsSubmitRecord.updateMany({
where: {
...(data.submitId ? { submitId: data.submitId } : { messageRecordId: message.id }),
gatewayMessageId: null,
},
data: {
sequenceId: data.sequenceId,
gatewayMessageId: data.gatewayMessageId,
submittedAt,
},
});
}
return { accepted: true };
}
async handleSubmitResult(data: GatewaySubmitResultDto) {
const message = await this.requireMessageByGatewayEvent(data.messageId, data.gatewayMessageId);
const batchTask = message.batchTaskId
@@ -933,6 +1014,10 @@ export class SendChainService implements OnModuleInit, OnModuleDestroy {
if (data.submitStatus === 'accepted' && !isStandaloneChannelTest && message.tenantId && message.batchTaskId) {
const businessMessage = message as typeof message & { tenantId: string; batchTaskId: string };
await this.chargeAcceptedMessage(businessMessage);
const latest = await this.prisma.smsMessageRecord.findUnique({ where: { id: message.id } });
if (latest?.status === 'failed') {
await this.refundMessage(businessMessage, '先到失败回执补偿退款');
}
} else if (data.submitStatus !== 'accepted' && !isStandaloneChannelTest && message.tenantId && message.batchTaskId) {
const businessMessage = message as typeof message & { tenantId: string; batchTaskId: string };
const retried = await this.retryMessageIfAllowed(businessMessage, data.submitStatus === 'timeout' ? '提交超时补发' : '提交失败补发');
@@ -942,8 +1027,11 @@ export class SendChainService implements OnModuleInit, OnModuleDestroy {
}
await this.releaseMessageReservation(businessMessage, data.submitStatus === 'timeout' ? '提交超时释放冻结' : '提交失败释放冻结');
}
await this.prisma.smsMessageRecord.update({
where: { id: message.id },
const protectedTerminalStatuses = ['delivered', 'failed', 'unknown'];
const updated = await this.prisma.smsMessageRecord.updateMany({
where: data.submitStatus === 'accepted'
? { id: message.id, status: { notIn: protectedTerminalStatuses } }
: { id: message.id, status: { not: 'delivered' } },
data: {
gatewayMessageId: data.gatewayMessageId,
submitStatus: data.submitStatus,
@@ -954,6 +1042,15 @@ export class SendChainService implements OnModuleInit, OnModuleDestroy {
timeoutAt: data.submitStatus === 'timeout' ? submittedAt : undefined,
},
});
if (updated.count === 0 && data.submitStatus === 'accepted') {
await this.prisma.smsMessageRecord.updateMany({
where: { id: message.id, gatewayMessageId: null },
data: {
gatewayMessageId: data.gatewayMessageId,
submittedAt,
},
});
}
if (data.submitStatus !== 'accepted' && batchTask?.sourceType === 'cmpp' && message.tenantId && message.applicationId) {
await this.recordCmppFailureReceipt(
message as typeof message & { tenantId: string; applicationId: string; batchTaskId: string },
@@ -981,8 +1078,185 @@ export class SendChainService implements OnModuleInit, OnModuleDestroy {
return this.prisma.smsMessageRecord.findUnique({ where: { id: message.id } });
}
async handleReceipt(data: GatewayReceiptEventDto) {
const resolved = await this.resolveReceiptMessage(data);
async intakeReceipt(data: GatewayReceiptEventDto) {
const channel = await this.prisma.smsChannel.findUnique({
where: { id: data.channelId },
select: {
id: true,
account: true,
gatewayHost: true,
gatewayPort: true,
protocol: true,
cmppVersion: true,
},
});
if (!channel) {
throw new NotFoundException('SMS channel not found');
}
const deliveredAt = data.deliveredAt ? new Date(data.deliveredAt) : new Date();
const receiptKey = this.receiptEventKey(data, data.channelId);
const inbox = await this.prisma.upstreamReceiptInbox.upsert({
where: { receiptKey },
update: {
incomingConnectionId: data.connectionId,
},
create: {
receiptKey,
incomingChannelId: data.channelId,
incomingConnectionId: data.connectionId,
upstreamAccount: channel.account,
upstreamHost: channel.gatewayHost,
upstreamPort: channel.gatewayPort,
protocol: channel.protocol,
protocolVersion: channel.cmppVersion,
provisionalMessageId: data.messageId,
sequenceId: data.sequenceId,
gatewayMessageId: data.gatewayMessageId,
phoneNumber: data.phoneNumber?.trim() || null,
receiptStatus: data.receiptStatus,
rawStatus: data.rawStatus,
errorCode: data.errorCode,
errorMessage: data.errorMessage,
deliveredAt,
status: 'pending',
nextRetryAt: new Date(),
},
});
if (['pending', 'retrying'].includes(inbox.status)) {
setImmediate(() => void this.processUpstreamReceiptInboxRecord(inbox.id));
}
return { accepted: true, inboxId: inbox.id, status: inbox.status };
}
async processPendingUpstreamReceiptInbox(limit = 100) {
const now = new Date();
const staleBefore = new Date(
now.getTime()
- positiveInteger(
process.env.UPSTREAM_RECEIPT_INBOX_PROCESSING_STALE_MS,
DEFAULT_UPSTREAM_RECEIPT_INBOX_PROCESSING_STALE_MS,
),
);
const candidates = await this.prisma.upstreamReceiptInbox.findMany({
where: {
OR: [
{
status: { in: ['pending', 'retrying'] },
OR: [{ nextRetryAt: null }, { nextRetryAt: { lte: now } }],
},
{ status: 'processing', updatedAt: { lte: staleBefore } },
],
},
orderBy: [{ receivedAt: 'asc' }, { id: 'asc' }],
take: Math.min(Math.max(limit, 1), 500),
select: { id: true },
});
let processed = 0;
for (const candidate of candidates) {
if (await this.processUpstreamReceiptInboxRecord(candidate.id)) processed += 1;
}
return { scanned: candidates.length, processed };
}
private async processUpstreamReceiptInboxRecord(id: string) {
const staleBefore = new Date(
Date.now()
- positiveInteger(
process.env.UPSTREAM_RECEIPT_INBOX_PROCESSING_STALE_MS,
DEFAULT_UPSTREAM_RECEIPT_INBOX_PROCESSING_STALE_MS,
),
);
const claimed = await this.prisma.upstreamReceiptInbox.updateMany({
where: {
id,
OR: [
{ status: { in: ['pending', 'retrying'] } },
{ status: 'processing', updatedAt: { lte: staleBefore } },
],
},
data: { status: 'processing', attemptCount: { increment: 1 }, nextRetryAt: null },
});
if (claimed.count !== 1) return false;
const inbox = await this.prisma.upstreamReceiptInbox.findUnique({ where: { id } });
if (!inbox) return false;
try {
const message = await this.handleReceipt({
messageId: inbox.provisionalMessageId ?? undefined,
channelId: inbox.incomingChannelId,
connectionId: inbox.incomingConnectionId ?? undefined,
sequenceId: inbox.sequenceId ?? undefined,
gatewayMessageId: inbox.gatewayMessageId,
phoneNumber: inbox.phoneNumber ?? undefined,
receiptStatus: normalizeReceiptStatus(inbox.receiptStatus),
rawStatus: inbox.rawStatus,
errorCode: inbox.errorCode ?? undefined,
errorMessage: inbox.errorMessage ?? undefined,
deliveredAt: inbox.deliveredAt.toISOString(),
}, {
account: inbox.upstreamAccount,
gatewayHost: inbox.upstreamHost,
gatewayPort: inbox.upstreamPort,
protocol: inbox.protocol,
cmppVersion: inbox.protocolVersion,
});
const matchedMessageRecordId = message && 'id' in message ? message.id : message?.messageRecordId;
const matchedChannelId = message && 'channelId' in message ? message.channelId : undefined;
await this.prisma.upstreamReceiptInbox.update({
where: { id },
data: {
status: 'matched',
matchedMessageRecordId: matchedMessageRecordId ?? null,
matchedChannelId: matchedChannelId ?? null,
lastError: null,
processedAt: new Date(),
},
});
return true;
} catch (error) {
const maxAttempts = positiveInteger(
process.env.UPSTREAM_RECEIPT_INBOX_MAX_ATTEMPTS,
DEFAULT_UPSTREAM_RECEIPT_INBOX_MAX_ATTEMPTS,
);
const maxAgeHours = positiveInteger(
process.env.UPSTREAM_RECEIPT_INBOX_MAX_AGE_HOURS,
DEFAULT_UPSTREAM_RECEIPT_INBOX_MAX_AGE_HOURS,
);
const exhausted = inbox.attemptCount >= maxAttempts
|| inbox.receivedAt.getTime() <= Date.now() - maxAgeHours * 60 * 60_000;
const retryDelayMs = Math.min(30 * 60_000, 5_000 * 2 ** Math.min(Math.max(inbox.attemptCount - 1, 0), 8));
await this.prisma.upstreamReceiptInbox.update({
where: { id },
data: {
status: exhausted ? 'unmatched' : 'retrying',
nextRetryAt: exhausted ? null : new Date(Date.now() + retryDelayMs),
lastError: error instanceof Error ? error.message : String(error),
processedAt: exhausted ? new Date() : null,
},
});
return false;
}
}
private async runUpstreamReceiptInboxScan() {
if (this.upstreamReceiptInboxScanRunning) return;
this.upstreamReceiptInboxScanRunning = true;
try {
const result = await this.processPendingUpstreamReceiptInbox();
if (result.processed > 0) {
this.logger.log(`Matched ${result.processed}/${result.scanned} pending upstream receipts`);
}
} catch (error) {
this.logger.error('Upstream receipt inbox scan failed', error instanceof Error ? error.stack : String(error));
} finally {
this.upstreamReceiptInboxScanRunning = false;
}
}
async handleReceipt(
data: GatewayReceiptEventDto,
incomingIdentity?: { account: string; gatewayHost: string; gatewayPort: number; protocol: string; cmppVersion: string },
) {
const resolved = await this.resolveReceiptMessage(data, incomingIdentity);
const logicalChannelId = resolved.channelId ?? data.channelId;
const receiptKey = this.receiptEventKey(data, logicalChannelId);
const existingReceipt = await this.prisma.smsReceiptRecord.findUnique({
@@ -1202,6 +1476,32 @@ export class SendChainService implements OnModuleInit, OnModuleDestroy {
async markDownstreamDeliverySent(data: GatewayDownstreamSentDto) {
const sentAt = asDateOrNull(data.sentAt) ?? new Date();
const ackDeadlineAt = asDateOrNull(data.ackDeadlineAt) ?? new Date(sentAt.getTime() + downstreamAckTimeoutMs());
const delivery = await this.prisma.cmppDownstreamDelivery.findUnique({ where: { id: data.id } });
if (!delivery) {
throw new NotFoundException('Downstream delivery not found');
}
const attemptKey = downstreamDeliveryAttemptKey(data);
await this.prisma.cmppDownstreamDeliveryAttempt.upsert({
where: { attemptKey },
update: {
connectionId: data.connectionId,
sequenceId: data.sequenceId,
messageId: data.messageId,
sentAt,
ackDeadlineAt,
},
create: {
deliveryId: data.id,
attemptKey,
attemptNo: delivery.retryCount + 1,
connectionId: data.connectionId,
sequenceId: data.sequenceId,
messageId: data.messageId,
status: 'awaiting_ack',
sentAt,
ackDeadlineAt,
},
});
await this.prisma.cmppDownstreamDelivery.updateMany({
where: { id: data.id, status: { not: 'delivered' } },
data: {
@@ -1221,7 +1521,48 @@ export class SendChainService implements OnModuleInit, OnModuleDestroy {
async acknowledgeDownstreamDelivery(data: GatewayDownstreamAcknowledgedDto) {
const acknowledgedAt = asDateOrNull(data.acknowledgedAt) ?? new Date();
const acknowledgedMessageId = String(data.messageId ?? '').trim();
if (data.result === 0 && acknowledgedMessageId !== '' && acknowledgedMessageId !== '0') {
const delivery = await this.prisma.cmppDownstreamDelivery.findUnique({ where: { id: data.id } });
if (!delivery) {
throw new NotFoundException('Downstream delivery not found');
}
const attemptKey = downstreamDeliveryAttemptKey(data);
const acknowledgementAccepted = data.result === 0 && acknowledgedMessageId !== '' && acknowledgedMessageId !== '0';
await this.prisma.cmppDownstreamDeliveryAttempt.upsert({
where: { attemptKey },
update: {
status: acknowledgementAccepted ? 'acknowledged' : 'rejected',
connectionId: data.connectionId,
sequenceId: data.sequenceId,
messageId: data.messageId,
acknowledgedAt,
ackResult: data.result,
ackDeadlineAt: null,
failureType: acknowledgementAccepted ? null : data.result === 0 ? 'ack_invalid' : 'ack_rejected',
errorMessage: acknowledgementAccepted
? null
: data.result === 0
? 'CMPP_DELIVER_RESP Msg_Id=0'
: `CMPP_DELIVER_RESP result=${data.result}`,
},
create: {
deliveryId: data.id,
attemptKey,
attemptNo: delivery.retryCount + 1,
connectionId: data.connectionId,
sequenceId: data.sequenceId,
messageId: data.messageId,
status: acknowledgementAccepted ? 'acknowledged' : 'rejected',
acknowledgedAt,
ackResult: data.result,
failureType: acknowledgementAccepted ? null : data.result === 0 ? 'ack_invalid' : 'ack_rejected',
errorMessage: acknowledgementAccepted
? null
: data.result === 0
? 'CMPP_DELIVER_RESP Msg_Id=0'
: `CMPP_DELIVER_RESP result=${data.result}`,
},
});
if (acknowledgementAccepted) {
await this.prisma.cmppDownstreamDelivery.updateMany({
where: { id: data.id, status: { not: 'delivered' } },
data: {
@@ -1255,7 +1596,12 @@ export class SendChainService implements OnModuleInit, OnModuleDestroy {
return this.markDownstreamDeliveryFailed(data.id, `downstream CMPP_DELIVER_RESP result=${data.result}`, 'ack_rejected');
}
async markDownstreamDeliveryFailed(id: string, errorMessage?: string, failureType: GatewayDownstreamFailureType = 'send_failed') {
async markDownstreamDeliveryFailed(
id: string,
errorMessage?: string,
failureType: GatewayDownstreamFailureType = 'send_failed',
attempt?: GatewayDownstreamSentDto,
) {
const delivery = await this.prisma.cmppDownstreamDelivery.findUnique({ where: { id } });
if (!delivery) {
throw new NotFoundException('Downstream delivery not found');
@@ -1272,6 +1618,33 @@ export class SendChainService implements OnModuleInit, OnModuleDestroy {
const nonRetryableFailure = failureType === 'unrecoverable' || failureType === 'queue_timeout';
const finalFailure = nonRetryableFailure || !retryAllowed || retryCount >= downstreamMaxRetries();
const finalStatus = failureType === 'ack_rejected' ? 'rejected' : acknowledgementFailure ? 'unconfirmed' : 'failed';
if (attempt && (attempt.connectionId || attempt.sequenceId || attempt.messageId)) {
const attemptKey = downstreamDeliveryAttemptKey({ ...attempt, id });
await this.prisma.cmppDownstreamDeliveryAttempt.upsert({
where: { attemptKey },
update: {
status: 'failed',
connectionId: attempt.connectionId,
sequenceId: attempt.sequenceId,
messageId: attempt.messageId,
failureType,
errorMessage: errorMessage ?? 'downstream delivery failed',
ackDeadlineAt: null,
},
create: {
deliveryId: id,
attemptKey,
attemptNo: delivery.retryCount + 1,
connectionId: attempt.connectionId,
sequenceId: attempt.sequenceId,
messageId: attempt.messageId,
status: 'failed',
sentAt: asDateOrNull(attempt.sentAt),
failureType,
errorMessage: errorMessage ?? 'downstream delivery failed',
},
});
}
const updated = await this.prisma.cmppDownstreamDelivery.update({
where: { id },
data: {
@@ -1914,11 +2287,14 @@ export class SendChainService implements OnModuleInit, OnModuleDestroy {
) as GatewayControlDeliveryResult;
if (result.sent || result.delivered) {
await this.markDownstreamDeliverySent({ id: delivery.id, ...result });
} else if (result.reasonCode === 'SUBMIT_RESPONSE_PENDING') {
return delivery;
} else {
await this.markDownstreamDeliveryFailed(
delivery.id,
downstreamControlFailureMessage(result),
result.retryable === false ? 'unrecoverable' : 'send_failed',
{ id: delivery.id, ...result },
);
}
} catch (error) {
@@ -3831,7 +4207,10 @@ export class SendChainService implements OnModuleInit, OnModuleDestroy {
return message;
}
private async resolveReceiptMessage(data: GatewayReceiptEventDto) {
private async resolveReceiptMessage(
data: GatewayReceiptEventDto,
incomingIdentity?: { account: string; gatewayHost: string; gatewayPort: number; protocol: string; cmppVersion: string },
) {
const exactMessage = data.messageId
? await this.prisma.smsMessageRecord.findUnique({ where: { messageId: data.messageId } })
: null;
@@ -3896,7 +4275,8 @@ export class SendChainService implements OnModuleInit, OnModuleDestroy {
throw new NotFoundException('SMS message record not found');
}
const incomingChannel = await this.prisma.smsChannel.findUnique({ where: { id: data.channelId } });
const incomingChannel = incomingIdentity
?? await this.prisma.smsChannel.findUnique({ where: { id: data.channelId } });
if (!incomingChannel) {
throw new NotFoundException('SMS message record not found');
}
@@ -3920,7 +4300,7 @@ export class SendChainService implements OnModuleInit, OnModuleDestroy {
};
}
const sameSupplierSegments = segmentMatches.filter((candidate) =>
candidate.channel && this.isSameSupplierConnection(incomingChannel, candidate.channel));
candidate.channel && this.isSameUpstreamEndpointIdentity(incomingChannel, candidate.channel));
if (sameSupplierSegments.length === 1 && sameSupplierSegments[0]?.messageRecord) {
return {
message: sameSupplierSegments[0].messageRecord,
@@ -3940,7 +4320,7 @@ export class SendChainService implements OnModuleInit, OnModuleDestroy {
take: 10,
});
const sameSupplierSubmits = crossConnectionSubmits.filter((candidate) =>
candidate.channel && this.isSameSupplierConnection(incomingChannel, candidate.channel));
candidate.channel && this.isSameUpstreamEndpointIdentity(incomingChannel, candidate.channel));
if (sameSupplierSubmits.length === 1 && sameSupplierSubmits[0]?.messageRecord) {
return {
message: sameSupplierSubmits[0].messageRecord,
@@ -3988,7 +4368,7 @@ export class SendChainService implements OnModuleInit, OnModuleDestroy {
};
}
private isSameSupplierConnection(
private isSameUpstreamEndpointIdentity(
left: { account: string; gatewayHost: string; gatewayPort: number; protocol: string; cmppVersion: string },
right: { account: string; gatewayHost: string; gatewayPort: number; protocol: string; cmppVersion: string },
) {
@@ -4341,6 +4721,24 @@ function parseOptionalSequenceId(value: string | null | undefined) {
return Number.isInteger(parsed) && parsed >= 0 && parsed <= 0xffffffff ? parsed : undefined;
}
function normalizeSubmitStatus(value: string): GatewaySubmitResultDto['submitStatus'] {
return value === 'accepted' || value === 'rejected' || value === 'timeout' ? value : 'timeout';
}
function normalizeReceiptStatus(value: string): GatewayReceiptEventDto['receiptStatus'] {
return value === 'delivered' || value === 'undelivered' || value === 'unknown' ? value : 'unknown';
}
function downstreamDeliveryAttemptKey(data: GatewayDownstreamSentDto) {
return createHash('sha256').update([
data.id,
data.connectionId ?? '',
data.sequenceId ?? '',
data.messageId ?? '',
data.sequenceId ? '' : data.sentAt ?? '',
].join('\u0000')).digest('hex');
}
function shanghaiDateKey(now = new Date()) {
const parts = new Intl.DateTimeFormat('en-CA', {
timeZone: 'Asia/Shanghai',