perf: expand gateway capacity and prevent receipt replay

This commit is contained in:
hectorzhao
2026-08-25 16:08:30 +08:00
parent 761c123b65
commit 9292352be1
48 changed files with 2001 additions and 144 deletions
@@ -367,6 +367,10 @@ export class ChannelConnectionService {
cmppVersion: channel.cmppVersion,
rateLimitPerSecond: channel.rateLimitPerSecond,
windowSize: getPositiveRuntimeInteger(getConfigValue(channel.config, 'windowSize'), 16, 'windowSize'),
connectionWarmupSeconds: Number(getConfigValue(channel.config, 'connectionWarmupSeconds') ?? 30),
connectionDrainTimeoutSeconds: getPositiveRuntimeInteger(getConfigValue(channel.config, 'connectionDrainTimeoutSeconds'), 60, 'connectionDrainTimeoutSeconds'),
submitResponseTimeoutSeconds: getPositiveRuntimeInteger(getConfigValue(channel.config, 'submitResponseTimeoutSeconds'), 60, 'submitResponseTimeoutSeconds'),
connectionFailureCooldownSeconds: getPositiveRuntimeInteger(getConfigValue(channel.config, 'connectionFailureCooldownSeconds'), 30, 'connectionFailureCooldownSeconds'),
heartbeatIntervalSeconds: getPositiveRuntimeInteger(
getConfigValue(channel.config, 'heartbeatIntervalSeconds'),
DEFAULT_HEARTBEAT_INTERVAL_SECONDS,
@@ -0,0 +1,14 @@
import { BadRequestException } from '@nestjs/common';
import { normalizeChannelRuntimeConfig } from './channels.helpers';
describe('Gateway channel capacity validation', () => {
it.each([1, 2, 4, 8])('accepts %i supplier connections', (desiredConnections) => {
expect(normalizeChannelRuntimeConfig(undefined, undefined, desiredConnections, 16)).toEqual(expect.objectContaining({ desiredConnections, windowSize: 16 }));
});
it.each([1, 16, 32, 64])('accepts supplier window %i', (windowSize) => {
expect(normalizeChannelRuntimeConfig(undefined, undefined, 1, windowSize)).toEqual(expect.objectContaining({ desiredConnections: 1, windowSize }));
});
it.each([[0, 16], [9, 16], [1, 0], [1, 65]])('rejects capacity outside 1..8 connections and 1..64 window', (connections, window) => {
expect(() => normalizeChannelRuntimeConfig(undefined, undefined, connections, window)).toThrow(BadRequestException);
});
});
+15 -3
View File
@@ -228,7 +228,7 @@ export function defaultChannelConnectionId(channelId: string) {
export function getDesiredConnections(config?: Prisma.JsonValue | null) {
if (config && typeof config === 'object' && !Array.isArray(config) && 'desiredConnections' in config) {
const value = Number(config.desiredConnections);
if (Number.isInteger(value) && value > 0) {
if (Number.isInteger(value) && value >= 1 && value <= 8) {
return value;
}
}
@@ -327,8 +327,12 @@ export function normalizeChannelRuntimeConfig(
? incomingConfig
: {};
const base = { ...existing, ...incoming };
base.desiredConnections = getPositiveRuntimeInteger(desiredConnections ?? base.desiredConnections, 1, 'desiredConnections');
base.windowSize = getPositiveRuntimeInteger(windowSize ?? base.windowSize, 16, 'windowSize');
base.desiredConnections = boundedRuntimeInteger(desiredConnections ?? base.desiredConnections, 1, 8, 1, 'desiredConnections');
base.windowSize = boundedRuntimeInteger(windowSize ?? base.windowSize, 1, 64, 16, 'windowSize');
base.connectionWarmupSeconds = boundedRuntimeInteger(base.connectionWarmupSeconds, 0, 300, 30, 'connectionWarmupSeconds');
base.connectionDrainTimeoutSeconds = boundedRuntimeInteger(base.connectionDrainTimeoutSeconds, 1, 600, 60, 'connectionDrainTimeoutSeconds');
base.submitResponseTimeoutSeconds = boundedRuntimeInteger(base.submitResponseTimeoutSeconds, 1, 300, 60, 'submitResponseTimeoutSeconds');
base.connectionFailureCooldownSeconds = boundedRuntimeInteger(base.connectionFailureCooldownSeconds, 1, 300, 30, 'connectionFailureCooldownSeconds');
base.heartbeatIntervalSeconds = getPositiveRuntimeInteger(
heartbeatIntervalSeconds ?? base.heartbeatIntervalSeconds,
DEFAULT_HEARTBEAT_INTERVAL_SECONDS,
@@ -345,6 +349,14 @@ export function normalizeChannelRuntimeConfig(
return base;
}
function boundedRuntimeInteger(value: unknown, minimum: number, maximum: number, fallback: number, field: string) {
const normalized = value === undefined || value === null || value === '' ? fallback : Number(value);
if (!Number.isInteger(normalized) || normalized < minimum || normalized > maximum) {
throw new BadRequestException(`${field} must be an integer between ${minimum} and ${maximum}`);
}
return normalized;
}
export function normalizeLongMessageReceiptMode(value: unknown) {
const normalized = String(value ?? 'per_segment').trim() || 'per_segment';
if (!['per_segment', 'message_level'].includes(normalized)) {
+8 -3
View File
@@ -14,14 +14,19 @@ export class PrismaService extends PrismaClient implements OnModuleDestroy {
const workerRole = processRole === 'worker';
const outboxRole = processRole === 'outbox';
const callbackRole = processRole === 'callback';
const databaseUrl = outboxRole
const protocolLogRole = processRole === 'protocol-log-worker';
const databaseUrl = protocolLogRole
? process.env.API_PROTOCOL_LOG_DATABASE_URL || process.env.DATABASE_URL
: outboxRole
? process.env.API_OUTBOX_DATABASE_URL || process.env.DATABASE_URL
: callbackRole
? process.env.API_CALLBACK_DATABASE_URL || process.env.DATABASE_URL
: workerRole
? process.env.API_WORKER_DATABASE_URL || process.env.DATABASE_URL
: process.env.DATABASE_URL;
const configuredPoolMax = Number(outboxRole
const configuredPoolMax = Number(protocolLogRole
? process.env.API_PROTOCOL_LOG_DB_POOL_MAX ?? 4
: outboxRole
? process.env.API_OUTBOX_DB_POOL_MAX ?? 6
: callbackRole
? process.env.API_CALLBACK_DB_POOL_MAX ?? 16
@@ -30,7 +35,7 @@ export class PrismaService extends PrismaClient implements OnModuleDestroy {
: process.env.API_DB_POOL_MAX ?? 32);
const poolMax = Number.isInteger(configuredPoolMax) && configuredPoolMax > 0
? configuredPoolMax
: outboxRole ? 6 : callbackRole ? 16 : workerRole ? 8 : 32;
: protocolLogRole ? 4 : outboxRole ? 6 : callbackRole ? 16 : workerRole ? 8 : 32;
const databasePool = new Pool({
connectionString: databaseUrl
?? 'postgresql://cmpp:cmpp_password@localhost:5432/cmpp_platform?schema=public',
+9
View File
@@ -0,0 +1,9 @@
import { Module } from '@nestjs/common';
import { ConfigModule } from '@nestjs/config';
import { PrismaModule } from './prisma/prisma.module';
import { ProtocolLogsModule } from './protocol-logs/protocol-logs.module';
@Module({
imports: [ConfigModule.forRoot({ isGlobal: true, envFilePath: ['.env.local', '.env'] }), PrismaModule, ProtocolLogsModule],
})
export class ProtocolLogWorkerModule {}
+59
View File
@@ -0,0 +1,59 @@
import { NestFactory } from '@nestjs/core';
import IORedis from 'ioredis';
import { ProtocolLogWorkerModule } from './protocol-log-worker.module';
import { ProtocolLogsService, type ProtocolLogInput } from './protocol-logs/protocol-logs.service';
const STREAM = process.env.GATEWAY_PROTOCOL_LOG_STREAM ?? 'gateway.protocol.logs';
const GROUP = process.env.GATEWAY_PROTOCOL_LOG_GROUP ?? 'cmpp-protocol-log-writer';
const CONSUMER = process.env.GATEWAY_PROTOCOL_LOG_CONSUMER ?? `protocol-log-${process.pid}`;
const BATCH_SIZE = boundedEnv('PROTOCOL_LOG_STREAM_BATCH_SIZE', 250, 100, 500);
async function bootstrap() {
process.env.CMPP_PROCESS_ROLE = 'protocol-log-worker';
const app = await NestFactory.createApplicationContext(ProtocolLogWorkerModule, { logger: ['log', 'warn', 'error'] });
const logs = app.get(ProtocolLogsService);
const redis = new IORedis(process.env.REDIS_URL ?? 'redis://127.0.0.1:6379', { maxRetriesPerRequest: null });
try { await redis.xgroup('CREATE', STREAM, GROUP, '0', 'MKSTREAM'); } catch (error) {
if (!String(error).includes('BUSYGROUP')) throw error;
}
let stopping = false;
const stop = () => { stopping = true; };
process.on('SIGTERM', stop); process.on('SIGINT', stop);
while (!stopping) {
const claimed = await redis.xautoclaim(STREAM, GROUP, CONSUMER, 30_000, '0-0', 'COUNT', BATCH_SIZE) as unknown as [string, Array<[string, string[]]>];
let messages = claimed[1] ?? [];
if (!messages.length) {
const reply = await redis.xreadgroup('GROUP', GROUP, CONSUMER, 'COUNT', BATCH_SIZE, 'BLOCK', 2000, 'STREAMS', STREAM, '>') as unknown as Array<[string, Array<[string, string[]]>]> | null;
if (!reply) continue;
messages = reply[0]?.[1] ?? [];
}
const accepted: string[] = [];
const parsed: ProtocolLogInput[] = [];
for (const [id, fields] of messages) {
const dataIndex = fields.indexOf('data');
try {
if (dataIndex < 0) throw new Error('data field missing');
parsed.push(JSON.parse(fields[dataIndex + 1]) as ProtocolLogInput);
accepted.push(id);
} catch (error) {
console.error(`protocol log stream event ${id} is invalid`, error);
await redis.xadd(`${STREAM}.dead`, '*', 'sourceId', id, 'error', String(error), 'data', dataIndex >= 0 ? fields[dataIndex + 1] : '');
await redis.xack(STREAM, GROUP, id); await redis.xdel(STREAM, id);
}
}
logs.recordMany(parsed);
if (accepted.length && await logs.flushNow()) {
const pipeline = redis.pipeline();
for (const id of accepted) pipeline.xack(STREAM, GROUP, id).xdel(STREAM, id);
await pipeline.exec();
}
}
await logs.flushNow(); await redis.quit(); await app.close();
}
function boundedEnv(name: string, fallback: number, minimum: number, maximum: number) {
const value = Number(process.env[name] ?? fallback);
return Number.isInteger(value) && value >= minimum && value <= maximum ? value : fallback;
}
void bootstrap().catch((error) => { console.error(error); process.exitCode = 1; });
+29 -5
View File
@@ -21,6 +21,10 @@ export type ProtocolLogInput = {
payloadBytes?: number | null;
retryCount?: number | null;
detail?: Record<string, unknown> | null;
eventId?: string | null;
gatewayInstanceId?: string | null;
connectionId?: string | null;
submitId?: string | null;
};
export type ProtocolLogQuery = {
@@ -45,8 +49,10 @@ export class ProtocolLogsService implements OnModuleInit, OnModuleDestroy {
constructor(private readonly prisma: PrismaService) {}
onModuleInit() {
this.flushTimer = setInterval(() => void this.flush(), positiveEnv('PROTOCOL_LOG_FLUSH_INTERVAL_MS', 500));
this.flushTimer.unref?.();
if (process.env.CMPP_PROCESS_ROLE !== 'protocol-log-worker') {
this.flushTimer = setInterval(() => void this.flush(), positiveEnv('PROTOCOL_LOG_FLUSH_INTERVAL_MS', 500));
this.flushTimer.unref?.();
}
this.retentionTimer = setInterval(() => void this.purgeExpired(), positiveEnv('PROTOCOL_LOG_RETENTION_INTERVAL_MS', 86_400_000));
this.retentionTimer.unref?.();
setTimeout(() => void this.purgeExpired(), 30_000).unref?.();
@@ -82,11 +88,25 @@ export class ProtocolLogsService implements OnModuleInit, OnModuleDestroy {
durationMs: safeInteger(input.durationMs),
payloadBytes: safeInteger(input.payloadBytes),
retryCount: safeInteger(input.retryCount),
detail: sanitizeDetail(input.detail),
detail: sanitizeDetail({
...input.detail,
eventId: input.eventId,
gatewayInstanceId: input.gatewayInstanceId,
connectionId: input.connectionId,
submitId: input.submitId,
}),
});
if (this.buffer.length >= positiveEnv('PROTOCOL_LOG_BATCH_SIZE', 100)) void this.flush();
}
recordMany(inputs: ProtocolLogInput[]) {
for (const input of inputs) this.record(input);
}
flushNow() {
return this.flush();
}
async list(query: ProtocolLogQuery) {
const page = Math.max(1, Number(query.page) || 1);
const pageSize = Math.min(100, Math.max(1, Number(query.pageSize) || 20));
@@ -119,18 +139,22 @@ export class ProtocolLogsService implements OnModuleInit, OnModuleDestroy {
return { items, total, page, pageSize, eventTypes: eventTypes.map((item) => item.eventType) };
}
private async flush() {
if (this.flushing || this.buffer.length === 0) return;
private async flush(): Promise<boolean> {
if (this.flushing) return false;
if (this.buffer.length === 0) return true;
this.flushing = true;
const batch = this.buffer.splice(0, positiveEnv('PROTOCOL_LOG_BATCH_SIZE', 100));
try {
await this.prisma.protocolInteractionLog.createMany({ data: batch });
} catch (error) {
this.buffer.unshift(...batch);
this.logger.error(`Protocol log batch write failed (${batch.length} events)`, error instanceof Error ? error.stack : String(error));
return false;
} finally {
this.flushing = false;
if (this.buffer.length > 0) setImmediate(() => void this.flush());
}
return true;
}
private async purgeExpired() {
@@ -11,7 +11,8 @@ describe('GatewayCallbackController', () => {
const prisma = { $queryRaw: jest.fn(), getPoolState: jest.fn().mockReturnValue({ max: 16, total: 2, idle: 1, waiting: 0 }) };
const controller = new GatewayCallbackController(sendChain as never, protocolLogs as never, prisma as never);
beforeEach(() => jest.clearAllMocks());
beforeEach(() => { jest.clearAllMocks(); process.env.PROTOCOL_LOG_CALLBACK_TRACKING_ENABLED = 'true'; });
afterAll(() => { delete process.env.PROTOCOL_LOG_CALLBACK_TRACKING_ENABLED; });
it('keeps Submit result persistence on the callback process without duplicating protocol logs', async () => {
sendChain.handleSubmitResult.mockResolvedValue({ accepted: true });
@@ -40,4 +41,20 @@ describe('GatewayCallbackController', () => {
protocol: 'cmpp', direction: 'client_to_platform', eventType: 'submit', status: 'success',
})).toThrow(BadRequestException);
});
it('returns an independent result for every event in a callback batch', async () => {
sendChain.handleSubmitResult.mockResolvedValue({ accepted: true });
sendChain.handleUplink.mockResolvedValue({ accepted: true });
await expect(controller.batch({
batchId: 'CB-1', gatewayInstanceId: 'gateway-1', events: [
{ eventId: 'EV-1', type: 'submit_result', payload: { messageId: 'MSG-1', channelId: 'channel-1', submitStatus: 'accepted' } },
{ eventId: 'EV-2', type: 'uplink', payload: { messageId: 'MSG-1', channelId: 'channel-1', content: '1' } },
{ eventId: 'EV-3', type: 'unsupported', payload: {} },
],
})).resolves.toEqual({ batchId: 'CB-1', results: [
{ eventId: 'EV-1', accepted: true },
{ eventId: 'EV-2', accepted: true },
{ eventId: 'EV-3', accepted: false, retryable: false, errorCode: 'INVALID_EVENT' },
] });
});
});
@@ -68,6 +68,51 @@ export class GatewayCallbackController {
return this.sendChain.recordGatewaySubmitDeadLetter(body);
}
@Post('gateway/events/batch')
async batch(@Body() body: {
batchId?: string;
gatewayInstanceId?: string;
createdAt?: string;
events?: Array<{ eventId?: string; type?: string; payload?: Record<string, unknown> }>;
}) {
if (!body.batchId || !body.gatewayInstanceId || !Array.isArray(body.events) || body.events.length < 1 || body.events.length > 100) {
throw new BadRequestException('batchId, gatewayInstanceId and 1 to 100 events are required');
}
if (Buffer.byteLength(JSON.stringify(body), 'utf8') > 1024 * 1024) {
throw new BadRequestException('Gateway callback batch exceeds 1MB');
}
const seen = new Set<string>();
const results: Array<{ eventId: string; accepted: boolean; retryable?: boolean; errorCode?: string }> = [];
for (let offset = 0; offset < body.events.length; offset += 25) {
results.push(...await Promise.all(body.events.slice(offset, offset + 25).map(async (event) => {
const eventId = String(event.eventId ?? '').trim();
if (!eventId || seen.has(eventId) || !event.payload || typeof event.payload !== 'object') {
return { eventId, accepted: false, retryable: false, errorCode: seen.has(eventId) ? 'DUPLICATE_EVENT_ID' : 'INVALID_EVENT' };
}
seen.add(eventId);
try {
await this.dispatchBatchEvent(String(event.type ?? ''), { ...event.payload, eventId });
return { eventId, accepted: true };
} catch (error) {
const invalid = error instanceof BadRequestException;
return { eventId, accepted: false, retryable: !invalid, errorCode: invalid ? 'INVALID_EVENT' : 'PROCESSING_FAILED' };
}
})));
}
return { batchId: body.batchId, results };
}
private dispatchBatchEvent(type: string, payload: Record<string, unknown>) {
switch (type) {
case 'submit_result': return this.sendChain.handleSubmitResult(payload as unknown as GatewaySubmitResultDto);
case 'submit_segment_result': return this.sendChain.handleSubmitSegmentResult(payload as unknown as GatewaySubmitSegmentResultDto);
case 'receipt_intake': return this.track('deliver_receipt', payload, () => this.sendChain.intakeReceipt(payload as unknown as GatewayReceiptEventDto));
case 'uplink': return this.track('deliver_uplink', payload, () => this.sendChain.handleUplink(payload as unknown as GatewayUplinkEventDto));
case 'dead_letter': return this.sendChain.recordGatewaySubmitDeadLetter(payload as unknown as GatewaySubmitDeadLetterDto);
default: throw new BadRequestException(`Unsupported batch event type ${type}`);
}
}
private async track<T>(eventType: string, body: object, action: () => Promise<T> | T) {
const startedAt = Date.now();
const value = body as Record<string, unknown>;
@@ -83,7 +128,7 @@ export class GatewayCallbackController {
try {
const result = await action();
const resolved = result && typeof result === 'object' ? result as Record<string, unknown> : {};
this.protocolLogs.record({
if (process.env.PROTOCOL_LOG_CALLBACK_TRACKING_ENABLED === 'true') this.protocolLogs.record({
...common,
tenantId: (resolved.tenantId ?? common.tenantId) as string,
applicationId: (resolved.applicationId ?? common.applicationId) as string,
@@ -92,7 +137,7 @@ export class GatewayCallbackController {
});
return result;
} catch (error) {
this.protocolLogs.record({
if (process.env.PROTOCOL_LOG_CALLBACK_TRACKING_ENABLED === 'true') this.protocolLogs.record({
...common, status: 'failed', durationMs: Date.now() - startedAt,
detail: { error: error instanceof Error ? error.message : String(error) },
});
@@ -99,6 +99,7 @@ export interface GatewaySubmitSegmentResultDto {
}
export interface GatewayReceiptEventDto {
eventId?: string;
traceId?: string;
messageId?: string;
channelId: string;
@@ -114,6 +115,7 @@ export interface GatewayReceiptEventDto {
}
export interface GatewayUplinkEventDto {
eventId?: string;
traceId?: string;
messageId?: string;
channelId: string;
@@ -136,10 +138,13 @@ export type UplinkMatchCandidateInput = {
export interface GatewayPendingDeliveryQueryDto {
account: string;
limit?: number;
claimId?: string;
leaseMs?: number;
}
export interface GatewayDownstreamSentDto {
id: string;
claimId?: string;
connectionId?: string;
sequenceId?: string;
messageId?: string;
@@ -153,6 +158,7 @@ export interface GatewayDownstreamAcknowledgedDto extends GatewayDownstreamSentD
}
export type GatewayDownstreamFailureType =
| 'claim_released'
| 'send_failed'
| 'ack_timeout'
| 'ack_rejected'
+13 -1
View File
@@ -3355,6 +3355,14 @@ describe('SendChainService', () => {
'delivery-once',
]);
expect(service['postGatewayControl']).toHaveBeenCalledTimes(1);
expect(service['postGatewayControl']).toHaveBeenCalledWith(
'/downstream/receipt',
expect.objectContaining({ deliveryId: 'delivery-once', claimId: expect.stringMatching(/^api-direct:/) }),
);
expect(prisma.cmppDownstreamDelivery.updateMany).toHaveBeenCalledWith(expect.objectContaining({
where: { id: 'delivery-once', status: 'pending' },
data: expect.objectContaining({ status: 'dispatching', connectionId: expect.stringMatching(/^api-direct:/) }),
}));
expect(prisma.cmppDownstreamDelivery.create).toHaveBeenCalledTimes(3);
});
@@ -3895,8 +3903,12 @@ describe('SendChainService', () => {
});
prisma.cmppDownstreamDelivery.findMany.mockResolvedValue([]);
await expect(service.listPendingDownstreamDeliveries({ account: '100001', limit: 100 }))
prisma.$queryRaw.mockResolvedValueOnce([]);
await expect(service.listPendingDownstreamDeliveries({ account: '100001', limit: 100, claimId: 'gateway-a:100001:1' }))
.resolves.toEqual([]);
const claimSql = prisma.$queryRaw.mock.calls[0][0].strings.join(' ');
expect(claimSql).toContain('FOR UPDATE SKIP LOCKED');
expect(claimSql).toContain("status = 'dispatching'");
});
it('persists each upstream SubmitResp segment before the aggregate result arrives', async () => {
@@ -1,6 +1,6 @@
import { BadRequestException, HttpException, HttpStatus, Logger, NotFoundException } from '@nestjs/common';
import { Prisma } from '@prisma/client';
import { createHash } from 'node:crypto';
import { createHash, randomUUID } from 'node:crypto';
import { BillingService } from '../billing/billing.service';
import { moneyToNumber } from '../common/money';
import type { OpenApiService } from '../open-api/open-api.service';
@@ -28,6 +28,10 @@ export class SendDownstreamDeliveryService {
) {}
async handleUplink(data: GatewayUplinkEventDto) {
if (data.eventId) {
const existing = await this.prisma.smsUplinkMessage.findUnique({ where: { eventId: data.eventId } });
if (existing) return existing;
}
const channel = await this.prisma.smsChannel.findUnique({ where: { id: data.channelId } });
if (!channel) {
throw new NotFoundException('SMS channel not found');
@@ -35,6 +39,7 @@ export class SendDownstreamDeliveryService {
const match = await this.facade.resolveUplinkMatch(data, channel);
const record = await this.prisma.smsUplinkMessage.create({
data: {
eventId: data.eventId,
tenantId: match.tenantId,
applicationId: match.applicationId,
messageRecordId: match.messageRecordId,
@@ -276,21 +281,40 @@ export class SendDownstreamDeliveryService {
if (!deliveryAllowed) {
return delivery;
}
const claimId = `api-direct:${process.pid}:${randomUUID()}`;
const claim = await this.prisma.cmppDownstreamDelivery.updateMany({
where: { id: delivery.id, status: 'pending' },
data: {
status: 'dispatching',
connectionId: claimId,
ackDeadlineAt: new Date(Date.now() + 30_000),
nextRetryAt: null,
lastError: null,
},
});
if (claim.count !== 1) {
return this.prisma.cmppDownstreamDelivery.findUnique({ where: { id: delivery.id } });
}
try {
const result = await this.facade.postGatewayControl(
data.deliveryType === 'receipt' ? '/downstream/receipt' : '/downstream/uplink',
{ deliveryId: delivery.id, ...payload },
{ deliveryId: delivery.id, claimId, ...payload },
) as GatewayControlDeliveryResult;
if (result.sent || result.delivered) {
await this.facade.markDownstreamDeliverySent({ id: delivery.id, ...result });
await this.facade.markDownstreamDeliverySent({ id: delivery.id, claimId, ...result });
} else if (result.reasonCode === 'SUBMIT_RESPONSE_PENDING') {
return delivery;
await this.facade.markDownstreamDeliveryFailed(
delivery.id,
downstreamControlFailureMessage(result),
'claim_released',
{ id: delivery.id, claimId, ...result },
);
} else {
await this.facade.markDownstreamDeliveryFailed(
delivery.id,
downstreamControlFailureMessage(result),
result.retryable === false ? 'unrecoverable' : 'send_failed',
{ id: delivery.id, ...result },
{ id: delivery.id, claimId, ...result },
);
}
} catch (error) {
@@ -39,15 +39,54 @@ export class SendDownstreamStateService {
for (const expired of expiredAcknowledgements) {
await this.facade.markDownstreamDeliveryFailed(expired.id, 'CMPP_DELIVER_RESP timeout recovered after Gateway restart', 'ack_timeout');
}
return this.prisma.cmppDownstreamDelivery.findMany({
where: {
applicationId: application.id,
const now = new Date();
await this.prisma.cmppDownstreamDelivery.updateMany({
where: { applicationId: application.id, status: 'dispatching', ackDeadlineAt: { lte: now } },
data: {
status: 'pending',
OR: [{ nextRetryAt: null }, { nextRetryAt: { lte: new Date() } }],
connectionId: null,
ackDeadlineAt: null,
nextRetryAt: now,
lastError: 'Gateway delivery claim lease expired and was recovered',
},
orderBy: { createdAt: 'asc' },
take: Math.min(Math.max(data.limit ?? 100, 1), 500),
});
const claimId = String(data.claimId ?? '').trim();
if (!claimId) {
throw new BadRequestException('claimId is required');
}
const limit = Math.min(Math.max(data.limit ?? 100, 1), 500);
const leaseMs = Math.min(Math.max(data.leaseMs ?? 30_000, 5_000), 120_000);
const claimed = await this.prisma.$queryRaw<Array<{ id: string }>>(Prisma.sql`
WITH candidates AS (
SELECT id
FROM "CmppDownstreamDelivery"
WHERE "applicationId" = ${application.id}
AND status = 'pending'
AND ("nextRetryAt" IS NULL OR "nextRetryAt" <= NOW())
ORDER BY "createdAt" ASC
FOR UPDATE SKIP LOCKED
LIMIT ${limit}
)
UPDATE "CmppDownstreamDelivery" AS delivery
SET status = 'dispatching',
"connectionId" = ${claimId},
"ackDeadlineAt" = NOW() + (${leaseMs} * INTERVAL '1 millisecond'),
"nextRetryAt" = NULL,
"lastError" = NULL,
"updatedAt" = NOW()
FROM candidates
WHERE delivery.id = candidates.id
RETURNING delivery.id
`);
if (claimed.length === 0) {
return [];
}
const claimedIds = claimed.map((item) => item.id);
const deliveries = await this.prisma.cmppDownstreamDelivery.findMany({
where: { id: { in: claimedIds }, status: 'dispatching', connectionId: claimId },
orderBy: { createdAt: 'asc' },
});
return deliveries.map((delivery) => ({ ...delivery, claimId }));
}
async markDownstreamDeliveryDelivered(id: string) {
@@ -91,7 +130,11 @@ export class SendDownstreamStateService {
},
});
await this.prisma.cmppDownstreamDelivery.updateMany({
where: { id: data.id, status: { not: 'delivered' } },
where: {
id: data.id,
status: data.claimId ? 'dispatching' : { not: 'delivered' },
...(data.claimId ? { connectionId: data.claimId } : {}),
},
data: {
status: 'awaiting_ack',
sentAt,
@@ -197,6 +240,23 @@ export class SendDownstreamStateService {
if (delivery.status === 'delivered') {
return delivery;
}
if (failureType === 'claim_released') {
await this.prisma.cmppDownstreamDelivery.updateMany({
where: {
id,
status: 'dispatching',
...(attempt?.claimId ? { connectionId: attempt.claimId } : {}),
},
data: {
status: 'pending',
connectionId: null,
ackDeadlineAt: null,
nextRetryAt: new Date(),
lastError: errorMessage ?? 'Gateway released downstream delivery claim',
},
});
return this.prisma.cmppDownstreamDelivery.findUnique({ where: { id } });
}
if (failureType === 'queue_timeout' && delivery.status !== 'pending') {
return delivery;
}
@@ -699,6 +699,10 @@ startSubmitOutboxPublisher() {
windowSize: getPositiveConfigInteger(channel.config, 'windowSize', 16),
heartbeatIntervalSeconds: getPositiveConfigInteger(channel.config, 'heartbeatIntervalSeconds', 30),
heartbeatMissThreshold: getPositiveConfigInteger(channel.config, 'heartbeatMissThreshold', 3),
connectionWarmupSeconds: getNonNegativeConfigInteger(channel.config, 'connectionWarmupSeconds', 30),
connectionDrainTimeoutSeconds: getPositiveConfigInteger(channel.config, 'connectionDrainTimeoutSeconds', 60),
submitResponseTimeoutSeconds: getPositiveConfigInteger(channel.config, 'submitResponseTimeoutSeconds', 60),
connectionFailureCooldownSeconds: getPositiveConfigInteger(channel.config, 'connectionFailureCooldownSeconds', 30),
},
retry: { attempt, maxAttempts: 1 },
};