hectorzhao/lislgosms
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: harden sessions and track downstream acknowledgements

Browse Source
This commit is contained in:
hectorzhao
2026-07-14 14:18:43 +08:00
parent 3d37adcc9f
commit 8c03663f24
43 changed files with 1733 additions and 150 deletions
Download Patch File Download Diff File Expand all files Collapse all files
api/src/send-chain/send-chain.service.ts
+118 -13
View File
@@ -105,6 +105,32 @@ export interface GatewayPendingDeliveryQueryDto {
limit?: number;
}
export interface GatewayDownstreamSentDto {
id: string;
connectionId?: string;
sequenceId?: string;
messageId?: string;
sentAt?: string;
ackDeadlineAt?: string;
}
export interface GatewayDownstreamAcknowledgedDto extends GatewayDownstreamSentDto {
result: number;
acknowledgedAt?: string;
}
export type GatewayDownstreamFailureType = 'send_failed' | 'ack_timeout' | 'ack_rejected' | 'connection_lost';
type GatewayControlDeliveryResult = {
sent?: boolean;
delivered?: boolean;
connectionId?: string;
sequenceId?: string;
messageId?: string;
sentAt?: string;
ackDeadlineAt?: string;
};
export interface GatewaySubmitDeadLetterDto {
streamMessageId: string;
traceId?: string;
@@ -910,6 +936,14 @@ export class SendChainService implements OnModuleInit, OnModuleDestroy {
if (!application || application.status !== 'active' || application.tenant.status !== 'active') {
throw new BadRequestException('CMPP account is invalid or disabled');
}
const expiredAcknowledgements = await this.prisma.cmppDownstreamDelivery.findMany({
where: { applicationId: application.id, status: 'awaiting_ack', ackDeadlineAt: { lte: new Date() } },
select: { id: true },
take: 500,
});
for (const expired of expiredAcknowledgements) {
await this.markDownstreamDeliveryFailed(expired.id, 'CMPP_DELIVER_RESP timeout recovered after Gateway restart', 'ack_timeout');
}
return this.prisma.cmppDownstreamDelivery.findMany({
where: {
applicationId: application.id,
@@ -932,19 +966,78 @@ export class SendChainService implements OnModuleInit, OnModuleDestroy {
});
}
async markDownstreamDeliveryFailed(id: string, errorMessage?: string) {
async markDownstreamDeliverySent(data: GatewayDownstreamSentDto) {
const sentAt = asDateOrNull(data.sentAt) ?? new Date();
const ackDeadlineAt = asDateOrNull(data.ackDeadlineAt) ?? new Date(sentAt.getTime() + downstreamAckTimeoutMs());
await this.prisma.cmppDownstreamDelivery.updateMany({
where: { id: data.id, status: { not: 'delivered' } },
data: {
status: 'awaiting_ack',
sentAt,
ackDeadlineAt,
ackSequenceId: data.sequenceId,
ackMessageId: data.messageId,
connectionId: data.connectionId,
nextRetryAt: null,
lastError: null,
},
});
return this.prisma.cmppDownstreamDelivery.findUnique({ where: { id: data.id } });
}
async acknowledgeDownstreamDelivery(data: GatewayDownstreamAcknowledgedDto) {
const acknowledgedAt = asDateOrNull(data.acknowledgedAt) ?? new Date();
if (data.result === 0) {
await this.prisma.cmppDownstreamDelivery.updateMany({
where: { id: data.id, status: { not: 'delivered' } },
data: {
status: 'delivered',
acknowledgedAt,
deliveredAt: acknowledgedAt,
ackDeadlineAt: null,
ackResult: data.result,
ackSequenceId: data.sequenceId,
ackMessageId: data.messageId,
connectionId: data.connectionId,
nextRetryAt: null,
lastError: null,
},
});
return this.prisma.cmppDownstreamDelivery.findUnique({ where: { id: data.id } });
}
await this.prisma.cmppDownstreamDelivery.updateMany({
where: { id: data.id, status: { not: 'delivered' } },
data: {
acknowledgedAt,
ackResult: data.result,
ackSequenceId: data.sequenceId,
ackMessageId: data.messageId,
connectionId: data.connectionId,
},
});
return this.markDownstreamDeliveryFailed(data.id, `downstream CMPP_DELIVER_RESP result=${data.result}`, 'ack_rejected');
}
async markDownstreamDeliveryFailed(id: string, errorMessage?: string, failureType: GatewayDownstreamFailureType = 'send_failed') {
const delivery = await this.prisma.cmppDownstreamDelivery.findUnique({ where: { id } });
if (!delivery) {
throw new NotFoundException('Downstream delivery not found');
}
if (delivery.status === 'delivered') {
return delivery;
}
const retryCount = (delivery.retryCount ?? 0) + 1;
const finalFailure = retryCount >= downstreamMaxRetries();
const acknowledgementFailure = failureType === 'ack_timeout' || failureType === 'ack_rejected' || failureType === 'connection_lost';
const retryAllowed = !acknowledgementFailure || delivery.retryEnabled !== false;
const finalFailure = !retryAllowed || retryCount >= downstreamMaxRetries();
const finalStatus = failureType === 'ack_rejected' ? 'rejected' : acknowledgementFailure ? 'unconfirmed' : 'failed';
const updated = await this.prisma.cmppDownstreamDelivery.update({
where: { id },
data: {
status: finalFailure ? 'failed' : 'pending',
status: finalFailure ? finalStatus : 'pending',
retryCount,
nextRetryAt: finalFailure ? null : new Date(Date.now() + downstreamRetryDelayMs(retryCount)),
ackDeadlineAt: null,
lastError: errorMessage ?? 'downstream delivery failed',
},
});
@@ -960,6 +1053,8 @@ export class SendChainService implements OnModuleInit, OnModuleDestroy {
applicationId: updated.applicationId,
messageId: updated.messageId,
retryCount,
failureType,
retryEnabled: updated.retryEnabled,
errorMessage: updated.lastError,
},
},
@@ -1168,12 +1263,16 @@ export class SendChainService implements OnModuleInit, OnModuleDestroy {
account: String(payload.account ?? delivery.application?.cmppAccount ?? ''),
...payload,
};
await this.prisma.cmppDownstreamDelivery.update({
where: { id: delivery.id },
data: { status: 'pending', retryCount: 0, nextRetryAt: null, ackDeadlineAt: null, lastError: null },
});
try {
const result = await this.postGatewayControl(path, requestPayload) as { delivered?: boolean };
if (result.delivered) {
return this.markDownstreamDeliveryDelivered(delivery.id);
const result = await this.postGatewayControl(path, requestPayload) as GatewayControlDeliveryResult;
if (result.sent || result.delivered) {
return this.markDownstreamDeliverySent({ id: delivery.id, ...result });
}
return this.markDownstreamDeliveryFailed(delivery.id, 'downstream client is not connected');
return this.prisma.cmppDownstreamDelivery.findUnique({ where: { id: delivery.id } });
} catch (error) {
return this.markDownstreamDeliveryFailed(
delivery.id,
@@ -1326,7 +1425,7 @@ export class SendChainService implements OnModuleInit, OnModuleDestroy {
}
const application = await this.prisma.smsApplication.findUnique({
where: { id: data.applicationId },
select: { cmppAccount: true },
select: { cmppAccount: true, downstreamReceiptRetryEnabled: true, downstreamUplinkRetryEnabled: true },
});
const payload = { account: application?.cmppAccount, applicationId: data.applicationId, ...data.payload };
const delivery = await this.prisma.cmppDownstreamDelivery.create({
@@ -1337,6 +1436,9 @@ export class SendChainService implements OnModuleInit, OnModuleDestroy {
messageId: data.messageId,
deliveryType: data.deliveryType,
payload,
retryEnabled: data.deliveryType === 'uplink'
? application?.downstreamUplinkRetryEnabled ?? true
: application?.downstreamReceiptRetryEnabled ?? true,
status: 'pending',
},
});
@@ -1344,11 +1446,9 @@ export class SendChainService implements OnModuleInit, OnModuleDestroy {
const result = await this.postGatewayControl(
data.deliveryType === 'receipt' ? '/downstream/receipt' : '/downstream/uplink',
{ deliveryId: delivery.id, ...payload },
) as { delivered?: boolean };
if (result.delivered) {
await this.markDownstreamDeliveryDelivered(delivery.id);
} else {
await this.markDownstreamDeliveryFailed(delivery.id, 'downstream client is not connected');
) as GatewayControlDeliveryResult;
if (result.sent || result.delivered) {
await this.markDownstreamDeliverySent({ id: delivery.id, ...result });
}
} catch (error) {
await this.markDownstreamDeliveryFailed(delivery.id, error instanceof Error ? error.message : 'Gateway control delivery failed');
@@ -2646,6 +2746,11 @@ function downstreamRetryDelayMs(retryCount = 1) {
return Math.min(delay, max);
}
function downstreamAckTimeoutMs() {
const configured = Number(process.env.CMPP_DOWNSTREAM_ACK_TIMEOUT_SECONDS ?? 30);
return Math.max(5, Number.isFinite(configured) ? configured : 30) * 1000;
}
function downstreamRetryBaseDelayMs() {
const value = Number(process.env.CMPP_DOWNSTREAM_RETRY_DELAY_MS ?? DEFAULT_DOWNSTREAM_RETRY_DELAY_MS);
return Number.isFinite(value) && value > 0 ? value : DEFAULT_DOWNSTREAM_RETRY_DELAY_MS;