hectorzhao/lislgosms
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: harden sessions and track downstream acknowledgements

Browse Source
This commit is contained in:
hectorzhao
2026-07-14 14:18:43 +08:00
parent 3d37adcc9f
commit 8c03663f24
43 changed files with 1733 additions and 150 deletions
Download Patch File Download Diff File Expand all files Collapse all files
api/src/send-chain/gateway-events.controller.ts
+15 -2
View File
@@ -4,6 +4,9 @@ import {
GatewayInboundAuthDto,
GatewayInboundSubmitDto,
GatewayPendingDeliveryQueryDto,
GatewayDownstreamAcknowledgedDto,
GatewayDownstreamFailureType,
GatewayDownstreamSentDto,
GatewayDownstreamRecoveryStatusDto,
GatewayReceiptEventDto,
GatewaySubmitDeadLetterDto,
@@ -66,9 +69,19 @@ export class GatewayEventsController {
return this.sendChain.markDownstreamDeliveryDelivered(body.id);
}
@Post('downstream/sent')
downstreamSent(@Body() body: GatewayDownstreamSentDto) {
return this.sendChain.markDownstreamDeliverySent(body);
}
@Post('downstream/acknowledged')
downstreamAcknowledged(@Body() body: GatewayDownstreamAcknowledgedDto) {
return this.sendChain.acknowledgeDownstreamDelivery(body);
}
@Post('downstream/failed')
downstreamFailed(@Body() body: { id: string; errorMessage?: string }) {
return this.sendChain.markDownstreamDeliveryFailed(body.id, body.errorMessage);
downstreamFailed(@Body() body: { id: string; errorMessage?: string; failureType?: GatewayDownstreamFailureType }) {
return this.sendChain.markDownstreamDeliveryFailed(body.id, body.errorMessage, body.failureType);
}
@Post('downstream/recovery-status')
api/src/send-chain/send-chain.service.spec.ts
+44
View File
@@ -213,6 +213,7 @@ function createPrismaMock() {
}),
findMany: jest.fn().mockResolvedValue([]),
update: jest.fn().mockImplementation(({ data }) => Promise.resolve({ id: 'delivery-1', tenantId: 'tenant-1', applicationId: 'app-1', messageId: 'MSG-1', deliveryType: 'receipt', ...data })),
updateMany: jest.fn().mockResolvedValue({ count: 1 }),
},
gatewaySubmitDeadLetter: {
upsert: jest.fn().mockResolvedValue({ id: 'dead-1' }),
@@ -1454,6 +1455,49 @@ describe('SendChainService', () => {
});
});
it('only marks downstream delivery delivered after a successful CMPP_DELIVER_RESP', async () => {
const { service, prisma } = createService();
await service.markDownstreamDeliverySent({
id: 'delivery-1',
connectionId: 'conn-1',
sequenceId: '37',
messageId: '9016479179509871733',
sentAt: '2026-07-14T03:40:18.030Z',
ackDeadlineAt: '2026-07-14T03:40:48.030Z',
});
expect(prisma.cmppDownstreamDelivery.updateMany).toHaveBeenLastCalledWith(expect.objectContaining({
where: { id: 'delivery-1', status: { not: 'delivered' } },
data: expect.objectContaining({ status: 'awaiting_ack', ackSequenceId: '37', connectionId: 'conn-1' }),
}));
await service.acknowledgeDownstreamDelivery({
id: 'delivery-1',
connectionId: 'conn-1',
sequenceId: '37',
messageId: '9016479179509871733',
result: 0,
acknowledgedAt: '2026-07-14T03:40:18.060Z',
});
expect(prisma.cmppDownstreamDelivery.updateMany).toHaveBeenLastCalledWith(expect.objectContaining({
data: expect.objectContaining({ status: 'delivered', ackResult: 0, deliveredAt: new Date('2026-07-14T03:40:18.060Z') }),
}));
});
it('does not automatically retry an unacknowledged delivery when its policy snapshot is disabled', async () => {
const { service, prisma } = createService();
prisma.cmppDownstreamDelivery.findUnique.mockResolvedValue({
id: 'delivery-1', tenantId: 'tenant-1', applicationId: 'app-1', messageId: 'MSG-1',
deliveryType: 'receipt', status: 'awaiting_ack', retryEnabled: false, retryCount: 0,
});
await service.markDownstreamDeliveryFailed('delivery-1', 'CMPP_DELIVER_RESP timeout', 'ack_timeout');
expect(prisma.cmppDownstreamDelivery.update).toHaveBeenCalledWith(expect.objectContaining({
data: expect.objectContaining({ status: 'unconfirmed', retryCount: 1, nextRetryAt: null }),
}));
});
it('uses exponential backoff for downstream delivery retries before final failure', async () => {
const { service, prisma } = createService();
const previousBase = process.env.CMPP_DOWNSTREAM_RETRY_DELAY_MS;
api/src/send-chain/send-chain.service.ts
+118 -13
View File
@@ -105,6 +105,32 @@ export interface GatewayPendingDeliveryQueryDto {
limit?: number;
}
export interface GatewayDownstreamSentDto {
id: string;
connectionId?: string;
sequenceId?: string;
messageId?: string;
sentAt?: string;
ackDeadlineAt?: string;
}
export interface GatewayDownstreamAcknowledgedDto extends GatewayDownstreamSentDto {
result: number;
acknowledgedAt?: string;
}
export type GatewayDownstreamFailureType = 'send_failed' | 'ack_timeout' | 'ack_rejected' | 'connection_lost';
type GatewayControlDeliveryResult = {
sent?: boolean;
delivered?: boolean;
connectionId?: string;
sequenceId?: string;
messageId?: string;
sentAt?: string;
ackDeadlineAt?: string;
};
export interface GatewaySubmitDeadLetterDto {
streamMessageId: string;
traceId?: string;
@@ -910,6 +936,14 @@ export class SendChainService implements OnModuleInit, OnModuleDestroy {
if (!application || application.status !== 'active' || application.tenant.status !== 'active') {
throw new BadRequestException('CMPP account is invalid or disabled');
}
const expiredAcknowledgements = await this.prisma.cmppDownstreamDelivery.findMany({
where: { applicationId: application.id, status: 'awaiting_ack', ackDeadlineAt: { lte: new Date() } },
select: { id: true },
take: 500,
});
for (const expired of expiredAcknowledgements) {
await this.markDownstreamDeliveryFailed(expired.id, 'CMPP_DELIVER_RESP timeout recovered after Gateway restart', 'ack_timeout');
}
return this.prisma.cmppDownstreamDelivery.findMany({
where: {
applicationId: application.id,
@@ -932,19 +966,78 @@ export class SendChainService implements OnModuleInit, OnModuleDestroy {
});
}
async markDownstreamDeliveryFailed(id: string, errorMessage?: string) {
async markDownstreamDeliverySent(data: GatewayDownstreamSentDto) {
const sentAt = asDateOrNull(data.sentAt) ?? new Date();
const ackDeadlineAt = asDateOrNull(data.ackDeadlineAt) ?? new Date(sentAt.getTime() + downstreamAckTimeoutMs());
await this.prisma.cmppDownstreamDelivery.updateMany({
where: { id: data.id, status: { not: 'delivered' } },
data: {
status: 'awaiting_ack',
sentAt,
ackDeadlineAt,
ackSequenceId: data.sequenceId,
ackMessageId: data.messageId,
connectionId: data.connectionId,
nextRetryAt: null,
lastError: null,
},
});
return this.prisma.cmppDownstreamDelivery.findUnique({ where: { id: data.id } });
}
async acknowledgeDownstreamDelivery(data: GatewayDownstreamAcknowledgedDto) {
const acknowledgedAt = asDateOrNull(data.acknowledgedAt) ?? new Date();
if (data.result === 0) {
await this.prisma.cmppDownstreamDelivery.updateMany({
where: { id: data.id, status: { not: 'delivered' } },
data: {
status: 'delivered',
acknowledgedAt,
deliveredAt: acknowledgedAt,
ackDeadlineAt: null,
ackResult: data.result,
ackSequenceId: data.sequenceId,
ackMessageId: data.messageId,
connectionId: data.connectionId,
nextRetryAt: null,
lastError: null,
},
});
return this.prisma.cmppDownstreamDelivery.findUnique({ where: { id: data.id } });
}
await this.prisma.cmppDownstreamDelivery.updateMany({
where: { id: data.id, status: { not: 'delivered' } },
data: {
acknowledgedAt,
ackResult: data.result,
ackSequenceId: data.sequenceId,
ackMessageId: data.messageId,
connectionId: data.connectionId,
},
});
return this.markDownstreamDeliveryFailed(data.id, `downstream CMPP_DELIVER_RESP result=${data.result}`, 'ack_rejected');
}
async markDownstreamDeliveryFailed(id: string, errorMessage?: string, failureType: GatewayDownstreamFailureType = 'send_failed') {
const delivery = await this.prisma.cmppDownstreamDelivery.findUnique({ where: { id } });
if (!delivery) {
throw new NotFoundException('Downstream delivery not found');
}
if (delivery.status === 'delivered') {
return delivery;
}
const retryCount = (delivery.retryCount ?? 0) + 1;
const finalFailure = retryCount >= downstreamMaxRetries();
const acknowledgementFailure = failureType === 'ack_timeout' || failureType === 'ack_rejected' || failureType === 'connection_lost';
const retryAllowed = !acknowledgementFailure || delivery.retryEnabled !== false;
const finalFailure = !retryAllowed || retryCount >= downstreamMaxRetries();
const finalStatus = failureType === 'ack_rejected' ? 'rejected' : acknowledgementFailure ? 'unconfirmed' : 'failed';
const updated = await this.prisma.cmppDownstreamDelivery.update({
where: { id },
data: {
status: finalFailure ? 'failed' : 'pending',
status: finalFailure ? finalStatus : 'pending',
retryCount,
nextRetryAt: finalFailure ? null : new Date(Date.now() + downstreamRetryDelayMs(retryCount)),
ackDeadlineAt: null,
lastError: errorMessage ?? 'downstream delivery failed',
},
});
@@ -960,6 +1053,8 @@ export class SendChainService implements OnModuleInit, OnModuleDestroy {
applicationId: updated.applicationId,
messageId: updated.messageId,
retryCount,
failureType,
retryEnabled: updated.retryEnabled,
errorMessage: updated.lastError,
},
},
@@ -1168,12 +1263,16 @@ export class SendChainService implements OnModuleInit, OnModuleDestroy {
account: String(payload.account ?? delivery.application?.cmppAccount ?? ''),
...payload,
};
await this.prisma.cmppDownstreamDelivery.update({
where: { id: delivery.id },
data: { status: 'pending', retryCount: 0, nextRetryAt: null, ackDeadlineAt: null, lastError: null },
});
try {
const result = await this.postGatewayControl(path, requestPayload) as { delivered?: boolean };
if (result.delivered) {
return this.markDownstreamDeliveryDelivered(delivery.id);
const result = await this.postGatewayControl(path, requestPayload) as GatewayControlDeliveryResult;
if (result.sent || result.delivered) {
return this.markDownstreamDeliverySent({ id: delivery.id, ...result });
}
return this.markDownstreamDeliveryFailed(delivery.id, 'downstream client is not connected');
return this.prisma.cmppDownstreamDelivery.findUnique({ where: { id: delivery.id } });
} catch (error) {
return this.markDownstreamDeliveryFailed(
delivery.id,
@@ -1326,7 +1425,7 @@ export class SendChainService implements OnModuleInit, OnModuleDestroy {
}
const application = await this.prisma.smsApplication.findUnique({
where: { id: data.applicationId },
select: { cmppAccount: true },
select: { cmppAccount: true, downstreamReceiptRetryEnabled: true, downstreamUplinkRetryEnabled: true },
});
const payload = { account: application?.cmppAccount, applicationId: data.applicationId, ...data.payload };
const delivery = await this.prisma.cmppDownstreamDelivery.create({
@@ -1337,6 +1436,9 @@ export class SendChainService implements OnModuleInit, OnModuleDestroy {
messageId: data.messageId,
deliveryType: data.deliveryType,
payload,
retryEnabled: data.deliveryType === 'uplink'
? application?.downstreamUplinkRetryEnabled ?? true
: application?.downstreamReceiptRetryEnabled ?? true,
status: 'pending',
},
});
@@ -1344,11 +1446,9 @@ export class SendChainService implements OnModuleInit, OnModuleDestroy {
const result = await this.postGatewayControl(
data.deliveryType === 'receipt' ? '/downstream/receipt' : '/downstream/uplink',
{ deliveryId: delivery.id, ...payload },
) as { delivered?: boolean };
if (result.delivered) {
await this.markDownstreamDeliveryDelivered(delivery.id);
} else {
await this.markDownstreamDeliveryFailed(delivery.id, 'downstream client is not connected');
) as GatewayControlDeliveryResult;
if (result.sent || result.delivered) {
await this.markDownstreamDeliverySent({ id: delivery.id, ...result });
}
} catch (error) {
await this.markDownstreamDeliveryFailed(delivery.id, error instanceof Error ? error.message : 'Gateway control delivery failed');
@@ -2646,6 +2746,11 @@ function downstreamRetryDelayMs(retryCount = 1) {
return Math.min(delay, max);
}
function downstreamAckTimeoutMs() {
const configured = Number(process.env.CMPP_DOWNSTREAM_ACK_TIMEOUT_SECONDS ?? 30);
return Math.max(5, Number.isFinite(configured) ? configured : 30) * 1000;
}
function downstreamRetryBaseDelayMs() {
const value = Number(process.env.CMPP_DOWNSTREAM_RETRY_DELAY_MS ?? DEFAULT_DOWNSTREAM_RETRY_DELAY_MS);
return Number.isFinite(value) && value > 0 ? value : DEFAULT_DOWNSTREAM_RETRY_DELAY_MS;