hectorzhao/lislgosms
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

perf: expand gateway capacity and prevent receipt replay

Browse Source
This commit is contained in:
hectorzhao
2026-08-25 16:08:30 +08:00
parent 761c123b65
commit 9292352be1
48 changed files with 2001 additions and 144 deletions
Download Patch File Download Diff File Expand all files Collapse all files
api/src/send-chain/gateway-callback.controller.spec.ts
+18 -1
View File
@@ -11,7 +11,8 @@ describe('GatewayCallbackController', () => {
const prisma = { $queryRaw: jest.fn(), getPoolState: jest.fn().mockReturnValue({ max: 16, total: 2, idle: 1, waiting: 0 }) };
const controller = new GatewayCallbackController(sendChain as never, protocolLogs as never, prisma as never);
beforeEach(() => jest.clearAllMocks());
beforeEach(() => { jest.clearAllMocks(); process.env.PROTOCOL_LOG_CALLBACK_TRACKING_ENABLED = 'true'; });
afterAll(() => { delete process.env.PROTOCOL_LOG_CALLBACK_TRACKING_ENABLED; });
it('keeps Submit result persistence on the callback process without duplicating protocol logs', async () => {
sendChain.handleSubmitResult.mockResolvedValue({ accepted: true });
@@ -40,4 +41,20 @@ describe('GatewayCallbackController', () => {
protocol: 'cmpp', direction: 'client_to_platform', eventType: 'submit', status: 'success',
})).toThrow(BadRequestException);
});
it('returns an independent result for every event in a callback batch', async () => {
sendChain.handleSubmitResult.mockResolvedValue({ accepted: true });
sendChain.handleUplink.mockResolvedValue({ accepted: true });
await expect(controller.batch({
batchId: 'CB-1', gatewayInstanceId: 'gateway-1', events: [
{ eventId: 'EV-1', type: 'submit_result', payload: { messageId: 'MSG-1', channelId: 'channel-1', submitStatus: 'accepted' } },
{ eventId: 'EV-2', type: 'uplink', payload: { messageId: 'MSG-1', channelId: 'channel-1', content: '1' } },
{ eventId: 'EV-3', type: 'unsupported', payload: {} },
],
})).resolves.toEqual({ batchId: 'CB-1', results: [
{ eventId: 'EV-1', accepted: true },
{ eventId: 'EV-2', accepted: true },
{ eventId: 'EV-3', accepted: false, retryable: false, errorCode: 'INVALID_EVENT' },
] });
});
});
api/src/send-chain/gateway-callback.controller.ts
+47 -2
View File
@@ -68,6 +68,51 @@ export class GatewayCallbackController {
return this.sendChain.recordGatewaySubmitDeadLetter(body);
}
@Post('gateway/events/batch')
async batch(@Body() body: {
batchId?: string;
gatewayInstanceId?: string;
createdAt?: string;
events?: Array<{ eventId?: string; type?: string; payload?: Record<string, unknown> }>;
}) {
if (!body.batchId || !body.gatewayInstanceId || !Array.isArray(body.events) || body.events.length < 1 || body.events.length > 100) {
throw new BadRequestException('batchId, gatewayInstanceId and 1 to 100 events are required');
}
if (Buffer.byteLength(JSON.stringify(body), 'utf8') > 1024 * 1024) {
throw new BadRequestException('Gateway callback batch exceeds 1MB');
}
const seen = new Set<string>();
const results: Array<{ eventId: string; accepted: boolean; retryable?: boolean; errorCode?: string }> = [];
for (let offset = 0; offset < body.events.length; offset += 25) {
results.push(...await Promise.all(body.events.slice(offset, offset + 25).map(async (event) => {
const eventId = String(event.eventId ?? '').trim();
if (!eventId || seen.has(eventId) || !event.payload || typeof event.payload !== 'object') {
return { eventId, accepted: false, retryable: false, errorCode: seen.has(eventId) ? 'DUPLICATE_EVENT_ID' : 'INVALID_EVENT' };
}
seen.add(eventId);
try {
await this.dispatchBatchEvent(String(event.type ?? ''), { ...event.payload, eventId });
return { eventId, accepted: true };
} catch (error) {
const invalid = error instanceof BadRequestException;
return { eventId, accepted: false, retryable: !invalid, errorCode: invalid ? 'INVALID_EVENT' : 'PROCESSING_FAILED' };
}
})));
}
return { batchId: body.batchId, results };
}
private dispatchBatchEvent(type: string, payload: Record<string, unknown>) {
switch (type) {
case 'submit_result': return this.sendChain.handleSubmitResult(payload as unknown as GatewaySubmitResultDto);
case 'submit_segment_result': return this.sendChain.handleSubmitSegmentResult(payload as unknown as GatewaySubmitSegmentResultDto);
case 'receipt_intake': return this.track('deliver_receipt', payload, () => this.sendChain.intakeReceipt(payload as unknown as GatewayReceiptEventDto));
case 'uplink': return this.track('deliver_uplink', payload, () => this.sendChain.handleUplink(payload as unknown as GatewayUplinkEventDto));
case 'dead_letter': return this.sendChain.recordGatewaySubmitDeadLetter(payload as unknown as GatewaySubmitDeadLetterDto);
default: throw new BadRequestException(`Unsupported batch event type ${type}`);
}
}
private async track<T>(eventType: string, body: object, action: () => Promise<T> | T) {
const startedAt = Date.now();
const value = body as Record<string, unknown>;
@@ -83,7 +128,7 @@ export class GatewayCallbackController {
try {
const result = await action();
const resolved = result && typeof result === 'object' ? result as Record<string, unknown> : {};
this.protocolLogs.record({
if (process.env.PROTOCOL_LOG_CALLBACK_TRACKING_ENABLED === 'true') this.protocolLogs.record({
...common,
tenantId: (resolved.tenantId ?? common.tenantId) as string,
applicationId: (resolved.applicationId ?? common.applicationId) as string,
@@ -92,7 +137,7 @@ export class GatewayCallbackController {
});
return result;
} catch (error) {
this.protocolLogs.record({
if (process.env.PROTOCOL_LOG_CALLBACK_TRACKING_ENABLED === 'true') this.protocolLogs.record({
...common, status: 'failed', durationMs: Date.now() - startedAt,
detail: { error: error instanceof Error ? error.message : String(error) },
});
api/src/send-chain/send-chain.contracts.ts
+6
View File
@@ -99,6 +99,7 @@ export interface GatewaySubmitSegmentResultDto {
}
export interface GatewayReceiptEventDto {
eventId?: string;
traceId?: string;
messageId?: string;
channelId: string;
@@ -114,6 +115,7 @@ export interface GatewayReceiptEventDto {
}
export interface GatewayUplinkEventDto {
eventId?: string;
traceId?: string;
messageId?: string;
channelId: string;
@@ -136,10 +138,13 @@ export type UplinkMatchCandidateInput = {
export interface GatewayPendingDeliveryQueryDto {
account: string;
limit?: number;
claimId?: string;
leaseMs?: number;
}
export interface GatewayDownstreamSentDto {
id: string;
claimId?: string;
connectionId?: string;
sequenceId?: string;
messageId?: string;
@@ -153,6 +158,7 @@ export interface GatewayDownstreamAcknowledgedDto extends GatewayDownstreamSentD
}
export type GatewayDownstreamFailureType =
| 'claim_released'
| 'send_failed'
| 'ack_timeout'
| 'ack_rejected'
api/src/send-chain/send-chain.service.spec.ts
+13 -1
View File
@@ -3355,6 +3355,14 @@ describe('SendChainService', () => {
'delivery-once',
]);
expect(service['postGatewayControl']).toHaveBeenCalledTimes(1);
expect(service['postGatewayControl']).toHaveBeenCalledWith(
'/downstream/receipt',
expect.objectContaining({ deliveryId: 'delivery-once', claimId: expect.stringMatching(/^api-direct:/) }),
);
expect(prisma.cmppDownstreamDelivery.updateMany).toHaveBeenCalledWith(expect.objectContaining({
where: { id: 'delivery-once', status: 'pending' },
data: expect.objectContaining({ status: 'dispatching', connectionId: expect.stringMatching(/^api-direct:/) }),
}));
expect(prisma.cmppDownstreamDelivery.create).toHaveBeenCalledTimes(3);
});
@@ -3895,8 +3903,12 @@ describe('SendChainService', () => {
});
prisma.cmppDownstreamDelivery.findMany.mockResolvedValue([]);
await expect(service.listPendingDownstreamDeliveries({ account: '100001', limit: 100 }))
prisma.$queryRaw.mockResolvedValueOnce([]);
await expect(service.listPendingDownstreamDeliveries({ account: '100001', limit: 100, claimId: 'gateway-a:100001:1' }))
.resolves.toEqual([]);
const claimSql = prisma.$queryRaw.mock.calls[0][0].strings.join(' ');
expect(claimSql).toContain('FOR UPDATE SKIP LOCKED');
expect(claimSql).toContain("status = 'dispatching'");
});
it('persists each upstream SubmitResp segment before the aggregate result arrives', async () => {
api/src/send-chain/send-downstream-delivery.service.ts
+29 -5
View File
@@ -1,6 +1,6 @@
import { BadRequestException, HttpException, HttpStatus, Logger, NotFoundException } from '@nestjs/common';
import { Prisma } from '@prisma/client';
import { createHash } from 'node:crypto';
import { createHash, randomUUID } from 'node:crypto';
import { BillingService } from '../billing/billing.service';
import { moneyToNumber } from '../common/money';
import type { OpenApiService } from '../open-api/open-api.service';
@@ -28,6 +28,10 @@ export class SendDownstreamDeliveryService {
) {}
async handleUplink(data: GatewayUplinkEventDto) {
if (data.eventId) {
const existing = await this.prisma.smsUplinkMessage.findUnique({ where: { eventId: data.eventId } });
if (existing) return existing;
}
const channel = await this.prisma.smsChannel.findUnique({ where: { id: data.channelId } });
if (!channel) {
throw new NotFoundException('SMS channel not found');
@@ -35,6 +39,7 @@ export class SendDownstreamDeliveryService {
const match = await this.facade.resolveUplinkMatch(data, channel);
const record = await this.prisma.smsUplinkMessage.create({
data: {
eventId: data.eventId,
tenantId: match.tenantId,
applicationId: match.applicationId,
messageRecordId: match.messageRecordId,
@@ -276,21 +281,40 @@ export class SendDownstreamDeliveryService {
if (!deliveryAllowed) {
return delivery;
}
const claimId = `api-direct:${process.pid}:${randomUUID()}`;
const claim = await this.prisma.cmppDownstreamDelivery.updateMany({
where: { id: delivery.id, status: 'pending' },
data: {
status: 'dispatching',
connectionId: claimId,
ackDeadlineAt: new Date(Date.now() + 30_000),
nextRetryAt: null,
lastError: null,
},
});
if (claim.count !== 1) {
return this.prisma.cmppDownstreamDelivery.findUnique({ where: { id: delivery.id } });
}
try {
const result = await this.facade.postGatewayControl(
data.deliveryType === 'receipt' ? '/downstream/receipt' : '/downstream/uplink',
{ deliveryId: delivery.id, ...payload },
{ deliveryId: delivery.id, claimId, ...payload },
) as GatewayControlDeliveryResult;
if (result.sent || result.delivered) {
await this.facade.markDownstreamDeliverySent({ id: delivery.id, ...result });
await this.facade.markDownstreamDeliverySent({ id: delivery.id, claimId, ...result });
} else if (result.reasonCode === 'SUBMIT_RESPONSE_PENDING') {
return delivery;
await this.facade.markDownstreamDeliveryFailed(
delivery.id,
downstreamControlFailureMessage(result),
'claim_released',
{ id: delivery.id, claimId, ...result },
);
} else {
await this.facade.markDownstreamDeliveryFailed(
delivery.id,
downstreamControlFailureMessage(result),
result.retryable === false ? 'unrecoverable' : 'send_failed',
{ id: delivery.id, ...result },
{ id: delivery.id, claimId, ...result },
);
}
} catch (error) {
api/src/send-chain/send-downstream-state.service.ts
+67 -7
View File
@@ -39,15 +39,54 @@ export class SendDownstreamStateService {
for (const expired of expiredAcknowledgements) {
await this.facade.markDownstreamDeliveryFailed(expired.id, 'CMPP_DELIVER_RESP timeout recovered after Gateway restart', 'ack_timeout');
}
return this.prisma.cmppDownstreamDelivery.findMany({
where: {
applicationId: application.id,
const now = new Date();
await this.prisma.cmppDownstreamDelivery.updateMany({
where: { applicationId: application.id, status: 'dispatching', ackDeadlineAt: { lte: now } },
data: {
status: 'pending',
OR: [{ nextRetryAt: null }, { nextRetryAt: { lte: new Date() } }],
connectionId: null,
ackDeadlineAt: null,
nextRetryAt: now,
lastError: 'Gateway delivery claim lease expired and was recovered',
},
orderBy: { createdAt: 'asc' },
take: Math.min(Math.max(data.limit ?? 100, 1), 500),
});
const claimId = String(data.claimId ?? '').trim();
if (!claimId) {
throw new BadRequestException('claimId is required');
}
const limit = Math.min(Math.max(data.limit ?? 100, 1), 500);
const leaseMs = Math.min(Math.max(data.leaseMs ?? 30_000, 5_000), 120_000);
const claimed = await this.prisma.$queryRaw<Array<{ id: string }>>(Prisma.sql`
WITH candidates AS (
SELECT id
FROM "CmppDownstreamDelivery"
WHERE "applicationId" = ${application.id}
AND status = 'pending'
AND ("nextRetryAt" IS NULL OR "nextRetryAt" <= NOW())
ORDER BY "createdAt" ASC
FOR UPDATE SKIP LOCKED
LIMIT ${limit}
)
UPDATE "CmppDownstreamDelivery" AS delivery
SET status = 'dispatching',
"connectionId" = ${claimId},
"ackDeadlineAt" = NOW() + (${leaseMs} * INTERVAL '1 millisecond'),
"nextRetryAt" = NULL,
"lastError" = NULL,
"updatedAt" = NOW()
FROM candidates
WHERE delivery.id = candidates.id
RETURNING delivery.id
`);
if (claimed.length === 0) {
return [];
}
const claimedIds = claimed.map((item) => item.id);
const deliveries = await this.prisma.cmppDownstreamDelivery.findMany({
where: { id: { in: claimedIds }, status: 'dispatching', connectionId: claimId },
orderBy: { createdAt: 'asc' },
});
return deliveries.map((delivery) => ({ ...delivery, claimId }));
}
async markDownstreamDeliveryDelivered(id: string) {
@@ -91,7 +130,11 @@ export class SendDownstreamStateService {
},
});
await this.prisma.cmppDownstreamDelivery.updateMany({
where: { id: data.id, status: { not: 'delivered' } },
where: {
id: data.id,
status: data.claimId ? 'dispatching' : { not: 'delivered' },
...(data.claimId ? { connectionId: data.claimId } : {}),
},
data: {
status: 'awaiting_ack',
sentAt,
@@ -197,6 +240,23 @@ export class SendDownstreamStateService {
if (delivery.status === 'delivered') {
return delivery;
}
if (failureType === 'claim_released') {
await this.prisma.cmppDownstreamDelivery.updateMany({
where: {
id,
status: 'dispatching',
...(attempt?.claimId ? { connectionId: attempt.claimId } : {}),
},
data: {
status: 'pending',
connectionId: null,
ackDeadlineAt: null,
nextRetryAt: new Date(),
lastError: errorMessage ?? 'Gateway released downstream delivery claim',
},
});
return this.prisma.cmppDownstreamDelivery.findUnique({ where: { id } });
}
if (failureType === 'queue_timeout' && delivery.status !== 'pending') {
return delivery;
}
api/src/send-chain/send-gateway-submit.service.ts
+4
View File
@@ -699,6 +699,10 @@ startSubmitOutboxPublisher() {
windowSize: getPositiveConfigInteger(channel.config, 'windowSize', 16),
heartbeatIntervalSeconds: getPositiveConfigInteger(channel.config, 'heartbeatIntervalSeconds', 30),
heartbeatMissThreshold: getPositiveConfigInteger(channel.config, 'heartbeatMissThreshold', 3),
connectionWarmupSeconds: getNonNegativeConfigInteger(channel.config, 'connectionWarmupSeconds', 30),
connectionDrainTimeoutSeconds: getPositiveConfigInteger(channel.config, 'connectionDrainTimeoutSeconds', 60),
submitResponseTimeoutSeconds: getPositiveConfigInteger(channel.config, 'submitResponseTimeoutSeconds', 60),
connectionFailureCooldownSeconds: getPositiveConfigInteger(channel.config, 'connectionFailureCooldownSeconds', 30),
},
retry: { attempt, maxAttempts: 1 },
};