perf: expand gateway capacity and prevent receipt replay
This commit is contained in:
@@ -367,6 +367,10 @@ export class ChannelConnectionService {
|
||||
cmppVersion: channel.cmppVersion,
|
||||
rateLimitPerSecond: channel.rateLimitPerSecond,
|
||||
windowSize: getPositiveRuntimeInteger(getConfigValue(channel.config, 'windowSize'), 16, 'windowSize'),
|
||||
connectionWarmupSeconds: Number(getConfigValue(channel.config, 'connectionWarmupSeconds') ?? 30),
|
||||
connectionDrainTimeoutSeconds: getPositiveRuntimeInteger(getConfigValue(channel.config, 'connectionDrainTimeoutSeconds'), 60, 'connectionDrainTimeoutSeconds'),
|
||||
submitResponseTimeoutSeconds: getPositiveRuntimeInteger(getConfigValue(channel.config, 'submitResponseTimeoutSeconds'), 60, 'submitResponseTimeoutSeconds'),
|
||||
connectionFailureCooldownSeconds: getPositiveRuntimeInteger(getConfigValue(channel.config, 'connectionFailureCooldownSeconds'), 30, 'connectionFailureCooldownSeconds'),
|
||||
heartbeatIntervalSeconds: getPositiveRuntimeInteger(
|
||||
getConfigValue(channel.config, 'heartbeatIntervalSeconds'),
|
||||
DEFAULT_HEARTBEAT_INTERVAL_SECONDS,
|
||||
|
||||
@@ -0,0 +1,14 @@
|
||||
import { BadRequestException } from '@nestjs/common';
|
||||
import { normalizeChannelRuntimeConfig } from './channels.helpers';
|
||||
|
||||
describe('Gateway channel capacity validation', () => {
|
||||
it.each([1, 2, 4, 8])('accepts %i supplier connections', (desiredConnections) => {
|
||||
expect(normalizeChannelRuntimeConfig(undefined, undefined, desiredConnections, 16)).toEqual(expect.objectContaining({ desiredConnections, windowSize: 16 }));
|
||||
});
|
||||
it.each([1, 16, 32, 64])('accepts supplier window %i', (windowSize) => {
|
||||
expect(normalizeChannelRuntimeConfig(undefined, undefined, 1, windowSize)).toEqual(expect.objectContaining({ desiredConnections: 1, windowSize }));
|
||||
});
|
||||
it.each([[0, 16], [9, 16], [1, 0], [1, 65]])('rejects capacity outside 1..8 connections and 1..64 window', (connections, window) => {
|
||||
expect(() => normalizeChannelRuntimeConfig(undefined, undefined, connections, window)).toThrow(BadRequestException);
|
||||
});
|
||||
});
|
||||
@@ -228,7 +228,7 @@ export function defaultChannelConnectionId(channelId: string) {
|
||||
export function getDesiredConnections(config?: Prisma.JsonValue | null) {
|
||||
if (config && typeof config === 'object' && !Array.isArray(config) && 'desiredConnections' in config) {
|
||||
const value = Number(config.desiredConnections);
|
||||
if (Number.isInteger(value) && value > 0) {
|
||||
if (Number.isInteger(value) && value >= 1 && value <= 8) {
|
||||
return value;
|
||||
}
|
||||
}
|
||||
@@ -327,8 +327,12 @@ export function normalizeChannelRuntimeConfig(
|
||||
? incomingConfig
|
||||
: {};
|
||||
const base = { ...existing, ...incoming };
|
||||
base.desiredConnections = getPositiveRuntimeInteger(desiredConnections ?? base.desiredConnections, 1, 'desiredConnections');
|
||||
base.windowSize = getPositiveRuntimeInteger(windowSize ?? base.windowSize, 16, 'windowSize');
|
||||
base.desiredConnections = boundedRuntimeInteger(desiredConnections ?? base.desiredConnections, 1, 8, 1, 'desiredConnections');
|
||||
base.windowSize = boundedRuntimeInteger(windowSize ?? base.windowSize, 1, 64, 16, 'windowSize');
|
||||
base.connectionWarmupSeconds = boundedRuntimeInteger(base.connectionWarmupSeconds, 0, 300, 30, 'connectionWarmupSeconds');
|
||||
base.connectionDrainTimeoutSeconds = boundedRuntimeInteger(base.connectionDrainTimeoutSeconds, 1, 600, 60, 'connectionDrainTimeoutSeconds');
|
||||
base.submitResponseTimeoutSeconds = boundedRuntimeInteger(base.submitResponseTimeoutSeconds, 1, 300, 60, 'submitResponseTimeoutSeconds');
|
||||
base.connectionFailureCooldownSeconds = boundedRuntimeInteger(base.connectionFailureCooldownSeconds, 1, 300, 30, 'connectionFailureCooldownSeconds');
|
||||
base.heartbeatIntervalSeconds = getPositiveRuntimeInteger(
|
||||
heartbeatIntervalSeconds ?? base.heartbeatIntervalSeconds,
|
||||
DEFAULT_HEARTBEAT_INTERVAL_SECONDS,
|
||||
@@ -345,6 +349,14 @@ export function normalizeChannelRuntimeConfig(
|
||||
return base;
|
||||
}
|
||||
|
||||
function boundedRuntimeInteger(value: unknown, minimum: number, maximum: number, fallback: number, field: string) {
|
||||
const normalized = value === undefined || value === null || value === '' ? fallback : Number(value);
|
||||
if (!Number.isInteger(normalized) || normalized < minimum || normalized > maximum) {
|
||||
throw new BadRequestException(`${field} must be an integer between ${minimum} and ${maximum}`);
|
||||
}
|
||||
return normalized;
|
||||
}
|
||||
|
||||
export function normalizeLongMessageReceiptMode(value: unknown) {
|
||||
const normalized = String(value ?? 'per_segment').trim() || 'per_segment';
|
||||
if (!['per_segment', 'message_level'].includes(normalized)) {
|
||||
|
||||
@@ -14,14 +14,19 @@ export class PrismaService extends PrismaClient implements OnModuleDestroy {
|
||||
const workerRole = processRole === 'worker';
|
||||
const outboxRole = processRole === 'outbox';
|
||||
const callbackRole = processRole === 'callback';
|
||||
const databaseUrl = outboxRole
|
||||
const protocolLogRole = processRole === 'protocol-log-worker';
|
||||
const databaseUrl = protocolLogRole
|
||||
? process.env.API_PROTOCOL_LOG_DATABASE_URL || process.env.DATABASE_URL
|
||||
: outboxRole
|
||||
? process.env.API_OUTBOX_DATABASE_URL || process.env.DATABASE_URL
|
||||
: callbackRole
|
||||
? process.env.API_CALLBACK_DATABASE_URL || process.env.DATABASE_URL
|
||||
: workerRole
|
||||
? process.env.API_WORKER_DATABASE_URL || process.env.DATABASE_URL
|
||||
: process.env.DATABASE_URL;
|
||||
const configuredPoolMax = Number(outboxRole
|
||||
const configuredPoolMax = Number(protocolLogRole
|
||||
? process.env.API_PROTOCOL_LOG_DB_POOL_MAX ?? 4
|
||||
: outboxRole
|
||||
? process.env.API_OUTBOX_DB_POOL_MAX ?? 6
|
||||
: callbackRole
|
||||
? process.env.API_CALLBACK_DB_POOL_MAX ?? 16
|
||||
@@ -30,7 +35,7 @@ export class PrismaService extends PrismaClient implements OnModuleDestroy {
|
||||
: process.env.API_DB_POOL_MAX ?? 32);
|
||||
const poolMax = Number.isInteger(configuredPoolMax) && configuredPoolMax > 0
|
||||
? configuredPoolMax
|
||||
: outboxRole ? 6 : callbackRole ? 16 : workerRole ? 8 : 32;
|
||||
: protocolLogRole ? 4 : outboxRole ? 6 : callbackRole ? 16 : workerRole ? 8 : 32;
|
||||
const databasePool = new Pool({
|
||||
connectionString: databaseUrl
|
||||
?? 'postgresql://cmpp:cmpp_password@localhost:5432/cmpp_platform?schema=public',
|
||||
|
||||
@@ -0,0 +1,9 @@
|
||||
import { Module } from '@nestjs/common';
|
||||
import { ConfigModule } from '@nestjs/config';
|
||||
import { PrismaModule } from './prisma/prisma.module';
|
||||
import { ProtocolLogsModule } from './protocol-logs/protocol-logs.module';
|
||||
|
||||
@Module({
|
||||
imports: [ConfigModule.forRoot({ isGlobal: true, envFilePath: ['.env.local', '.env'] }), PrismaModule, ProtocolLogsModule],
|
||||
})
|
||||
export class ProtocolLogWorkerModule {}
|
||||
@@ -0,0 +1,59 @@
|
||||
import { NestFactory } from '@nestjs/core';
|
||||
import IORedis from 'ioredis';
|
||||
import { ProtocolLogWorkerModule } from './protocol-log-worker.module';
|
||||
import { ProtocolLogsService, type ProtocolLogInput } from './protocol-logs/protocol-logs.service';
|
||||
|
||||
const STREAM = process.env.GATEWAY_PROTOCOL_LOG_STREAM ?? 'gateway.protocol.logs';
|
||||
const GROUP = process.env.GATEWAY_PROTOCOL_LOG_GROUP ?? 'cmpp-protocol-log-writer';
|
||||
const CONSUMER = process.env.GATEWAY_PROTOCOL_LOG_CONSUMER ?? `protocol-log-${process.pid}`;
|
||||
const BATCH_SIZE = boundedEnv('PROTOCOL_LOG_STREAM_BATCH_SIZE', 250, 100, 500);
|
||||
|
||||
async function bootstrap() {
|
||||
process.env.CMPP_PROCESS_ROLE = 'protocol-log-worker';
|
||||
const app = await NestFactory.createApplicationContext(ProtocolLogWorkerModule, { logger: ['log', 'warn', 'error'] });
|
||||
const logs = app.get(ProtocolLogsService);
|
||||
const redis = new IORedis(process.env.REDIS_URL ?? 'redis://127.0.0.1:6379', { maxRetriesPerRequest: null });
|
||||
try { await redis.xgroup('CREATE', STREAM, GROUP, '0', 'MKSTREAM'); } catch (error) {
|
||||
if (!String(error).includes('BUSYGROUP')) throw error;
|
||||
}
|
||||
let stopping = false;
|
||||
const stop = () => { stopping = true; };
|
||||
process.on('SIGTERM', stop); process.on('SIGINT', stop);
|
||||
while (!stopping) {
|
||||
const claimed = await redis.xautoclaim(STREAM, GROUP, CONSUMER, 30_000, '0-0', 'COUNT', BATCH_SIZE) as unknown as [string, Array<[string, string[]]>];
|
||||
let messages = claimed[1] ?? [];
|
||||
if (!messages.length) {
|
||||
const reply = await redis.xreadgroup('GROUP', GROUP, CONSUMER, 'COUNT', BATCH_SIZE, 'BLOCK', 2000, 'STREAMS', STREAM, '>') as unknown as Array<[string, Array<[string, string[]]>]> | null;
|
||||
if (!reply) continue;
|
||||
messages = reply[0]?.[1] ?? [];
|
||||
}
|
||||
const accepted: string[] = [];
|
||||
const parsed: ProtocolLogInput[] = [];
|
||||
for (const [id, fields] of messages) {
|
||||
const dataIndex = fields.indexOf('data');
|
||||
try {
|
||||
if (dataIndex < 0) throw new Error('data field missing');
|
||||
parsed.push(JSON.parse(fields[dataIndex + 1]) as ProtocolLogInput);
|
||||
accepted.push(id);
|
||||
} catch (error) {
|
||||
console.error(`protocol log stream event ${id} is invalid`, error);
|
||||
await redis.xadd(`${STREAM}.dead`, '*', 'sourceId', id, 'error', String(error), 'data', dataIndex >= 0 ? fields[dataIndex + 1] : '');
|
||||
await redis.xack(STREAM, GROUP, id); await redis.xdel(STREAM, id);
|
||||
}
|
||||
}
|
||||
logs.recordMany(parsed);
|
||||
if (accepted.length && await logs.flushNow()) {
|
||||
const pipeline = redis.pipeline();
|
||||
for (const id of accepted) pipeline.xack(STREAM, GROUP, id).xdel(STREAM, id);
|
||||
await pipeline.exec();
|
||||
}
|
||||
}
|
||||
await logs.flushNow(); await redis.quit(); await app.close();
|
||||
}
|
||||
|
||||
function boundedEnv(name: string, fallback: number, minimum: number, maximum: number) {
|
||||
const value = Number(process.env[name] ?? fallback);
|
||||
return Number.isInteger(value) && value >= minimum && value <= maximum ? value : fallback;
|
||||
}
|
||||
|
||||
void bootstrap().catch((error) => { console.error(error); process.exitCode = 1; });
|
||||
@@ -21,6 +21,10 @@ export type ProtocolLogInput = {
|
||||
payloadBytes?: number | null;
|
||||
retryCount?: number | null;
|
||||
detail?: Record<string, unknown> | null;
|
||||
eventId?: string | null;
|
||||
gatewayInstanceId?: string | null;
|
||||
connectionId?: string | null;
|
||||
submitId?: string | null;
|
||||
};
|
||||
|
||||
export type ProtocolLogQuery = {
|
||||
@@ -45,8 +49,10 @@ export class ProtocolLogsService implements OnModuleInit, OnModuleDestroy {
|
||||
constructor(private readonly prisma: PrismaService) {}
|
||||
|
||||
onModuleInit() {
|
||||
this.flushTimer = setInterval(() => void this.flush(), positiveEnv('PROTOCOL_LOG_FLUSH_INTERVAL_MS', 500));
|
||||
this.flushTimer.unref?.();
|
||||
if (process.env.CMPP_PROCESS_ROLE !== 'protocol-log-worker') {
|
||||
this.flushTimer = setInterval(() => void this.flush(), positiveEnv('PROTOCOL_LOG_FLUSH_INTERVAL_MS', 500));
|
||||
this.flushTimer.unref?.();
|
||||
}
|
||||
this.retentionTimer = setInterval(() => void this.purgeExpired(), positiveEnv('PROTOCOL_LOG_RETENTION_INTERVAL_MS', 86_400_000));
|
||||
this.retentionTimer.unref?.();
|
||||
setTimeout(() => void this.purgeExpired(), 30_000).unref?.();
|
||||
@@ -82,11 +88,25 @@ export class ProtocolLogsService implements OnModuleInit, OnModuleDestroy {
|
||||
durationMs: safeInteger(input.durationMs),
|
||||
payloadBytes: safeInteger(input.payloadBytes),
|
||||
retryCount: safeInteger(input.retryCount),
|
||||
detail: sanitizeDetail(input.detail),
|
||||
detail: sanitizeDetail({
|
||||
...input.detail,
|
||||
eventId: input.eventId,
|
||||
gatewayInstanceId: input.gatewayInstanceId,
|
||||
connectionId: input.connectionId,
|
||||
submitId: input.submitId,
|
||||
}),
|
||||
});
|
||||
if (this.buffer.length >= positiveEnv('PROTOCOL_LOG_BATCH_SIZE', 100)) void this.flush();
|
||||
}
|
||||
|
||||
recordMany(inputs: ProtocolLogInput[]) {
|
||||
for (const input of inputs) this.record(input);
|
||||
}
|
||||
|
||||
flushNow() {
|
||||
return this.flush();
|
||||
}
|
||||
|
||||
async list(query: ProtocolLogQuery) {
|
||||
const page = Math.max(1, Number(query.page) || 1);
|
||||
const pageSize = Math.min(100, Math.max(1, Number(query.pageSize) || 20));
|
||||
@@ -119,18 +139,22 @@ export class ProtocolLogsService implements OnModuleInit, OnModuleDestroy {
|
||||
return { items, total, page, pageSize, eventTypes: eventTypes.map((item) => item.eventType) };
|
||||
}
|
||||
|
||||
private async flush() {
|
||||
if (this.flushing || this.buffer.length === 0) return;
|
||||
private async flush(): Promise<boolean> {
|
||||
if (this.flushing) return false;
|
||||
if (this.buffer.length === 0) return true;
|
||||
this.flushing = true;
|
||||
const batch = this.buffer.splice(0, positiveEnv('PROTOCOL_LOG_BATCH_SIZE', 100));
|
||||
try {
|
||||
await this.prisma.protocolInteractionLog.createMany({ data: batch });
|
||||
} catch (error) {
|
||||
this.buffer.unshift(...batch);
|
||||
this.logger.error(`Protocol log batch write failed (${batch.length} events)`, error instanceof Error ? error.stack : String(error));
|
||||
return false;
|
||||
} finally {
|
||||
this.flushing = false;
|
||||
if (this.buffer.length > 0) setImmediate(() => void this.flush());
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
private async purgeExpired() {
|
||||
|
||||
@@ -11,7 +11,8 @@ describe('GatewayCallbackController', () => {
|
||||
const prisma = { $queryRaw: jest.fn(), getPoolState: jest.fn().mockReturnValue({ max: 16, total: 2, idle: 1, waiting: 0 }) };
|
||||
const controller = new GatewayCallbackController(sendChain as never, protocolLogs as never, prisma as never);
|
||||
|
||||
beforeEach(() => jest.clearAllMocks());
|
||||
beforeEach(() => { jest.clearAllMocks(); process.env.PROTOCOL_LOG_CALLBACK_TRACKING_ENABLED = 'true'; });
|
||||
afterAll(() => { delete process.env.PROTOCOL_LOG_CALLBACK_TRACKING_ENABLED; });
|
||||
|
||||
it('keeps Submit result persistence on the callback process without duplicating protocol logs', async () => {
|
||||
sendChain.handleSubmitResult.mockResolvedValue({ accepted: true });
|
||||
@@ -40,4 +41,20 @@ describe('GatewayCallbackController', () => {
|
||||
protocol: 'cmpp', direction: 'client_to_platform', eventType: 'submit', status: 'success',
|
||||
})).toThrow(BadRequestException);
|
||||
});
|
||||
|
||||
it('returns an independent result for every event in a callback batch', async () => {
|
||||
sendChain.handleSubmitResult.mockResolvedValue({ accepted: true });
|
||||
sendChain.handleUplink.mockResolvedValue({ accepted: true });
|
||||
await expect(controller.batch({
|
||||
batchId: 'CB-1', gatewayInstanceId: 'gateway-1', events: [
|
||||
{ eventId: 'EV-1', type: 'submit_result', payload: { messageId: 'MSG-1', channelId: 'channel-1', submitStatus: 'accepted' } },
|
||||
{ eventId: 'EV-2', type: 'uplink', payload: { messageId: 'MSG-1', channelId: 'channel-1', content: '1' } },
|
||||
{ eventId: 'EV-3', type: 'unsupported', payload: {} },
|
||||
],
|
||||
})).resolves.toEqual({ batchId: 'CB-1', results: [
|
||||
{ eventId: 'EV-1', accepted: true },
|
||||
{ eventId: 'EV-2', accepted: true },
|
||||
{ eventId: 'EV-3', accepted: false, retryable: false, errorCode: 'INVALID_EVENT' },
|
||||
] });
|
||||
});
|
||||
});
|
||||
|
||||
@@ -68,6 +68,51 @@ export class GatewayCallbackController {
|
||||
return this.sendChain.recordGatewaySubmitDeadLetter(body);
|
||||
}
|
||||
|
||||
@Post('gateway/events/batch')
|
||||
async batch(@Body() body: {
|
||||
batchId?: string;
|
||||
gatewayInstanceId?: string;
|
||||
createdAt?: string;
|
||||
events?: Array<{ eventId?: string; type?: string; payload?: Record<string, unknown> }>;
|
||||
}) {
|
||||
if (!body.batchId || !body.gatewayInstanceId || !Array.isArray(body.events) || body.events.length < 1 || body.events.length > 100) {
|
||||
throw new BadRequestException('batchId, gatewayInstanceId and 1 to 100 events are required');
|
||||
}
|
||||
if (Buffer.byteLength(JSON.stringify(body), 'utf8') > 1024 * 1024) {
|
||||
throw new BadRequestException('Gateway callback batch exceeds 1MB');
|
||||
}
|
||||
const seen = new Set<string>();
|
||||
const results: Array<{ eventId: string; accepted: boolean; retryable?: boolean; errorCode?: string }> = [];
|
||||
for (let offset = 0; offset < body.events.length; offset += 25) {
|
||||
results.push(...await Promise.all(body.events.slice(offset, offset + 25).map(async (event) => {
|
||||
const eventId = String(event.eventId ?? '').trim();
|
||||
if (!eventId || seen.has(eventId) || !event.payload || typeof event.payload !== 'object') {
|
||||
return { eventId, accepted: false, retryable: false, errorCode: seen.has(eventId) ? 'DUPLICATE_EVENT_ID' : 'INVALID_EVENT' };
|
||||
}
|
||||
seen.add(eventId);
|
||||
try {
|
||||
await this.dispatchBatchEvent(String(event.type ?? ''), { ...event.payload, eventId });
|
||||
return { eventId, accepted: true };
|
||||
} catch (error) {
|
||||
const invalid = error instanceof BadRequestException;
|
||||
return { eventId, accepted: false, retryable: !invalid, errorCode: invalid ? 'INVALID_EVENT' : 'PROCESSING_FAILED' };
|
||||
}
|
||||
})));
|
||||
}
|
||||
return { batchId: body.batchId, results };
|
||||
}
|
||||
|
||||
private dispatchBatchEvent(type: string, payload: Record<string, unknown>) {
|
||||
switch (type) {
|
||||
case 'submit_result': return this.sendChain.handleSubmitResult(payload as unknown as GatewaySubmitResultDto);
|
||||
case 'submit_segment_result': return this.sendChain.handleSubmitSegmentResult(payload as unknown as GatewaySubmitSegmentResultDto);
|
||||
case 'receipt_intake': return this.track('deliver_receipt', payload, () => this.sendChain.intakeReceipt(payload as unknown as GatewayReceiptEventDto));
|
||||
case 'uplink': return this.track('deliver_uplink', payload, () => this.sendChain.handleUplink(payload as unknown as GatewayUplinkEventDto));
|
||||
case 'dead_letter': return this.sendChain.recordGatewaySubmitDeadLetter(payload as unknown as GatewaySubmitDeadLetterDto);
|
||||
default: throw new BadRequestException(`Unsupported batch event type ${type}`);
|
||||
}
|
||||
}
|
||||
|
||||
private async track<T>(eventType: string, body: object, action: () => Promise<T> | T) {
|
||||
const startedAt = Date.now();
|
||||
const value = body as Record<string, unknown>;
|
||||
@@ -83,7 +128,7 @@ export class GatewayCallbackController {
|
||||
try {
|
||||
const result = await action();
|
||||
const resolved = result && typeof result === 'object' ? result as Record<string, unknown> : {};
|
||||
this.protocolLogs.record({
|
||||
if (process.env.PROTOCOL_LOG_CALLBACK_TRACKING_ENABLED === 'true') this.protocolLogs.record({
|
||||
...common,
|
||||
tenantId: (resolved.tenantId ?? common.tenantId) as string,
|
||||
applicationId: (resolved.applicationId ?? common.applicationId) as string,
|
||||
@@ -92,7 +137,7 @@ export class GatewayCallbackController {
|
||||
});
|
||||
return result;
|
||||
} catch (error) {
|
||||
this.protocolLogs.record({
|
||||
if (process.env.PROTOCOL_LOG_CALLBACK_TRACKING_ENABLED === 'true') this.protocolLogs.record({
|
||||
...common, status: 'failed', durationMs: Date.now() - startedAt,
|
||||
detail: { error: error instanceof Error ? error.message : String(error) },
|
||||
});
|
||||
|
||||
@@ -99,6 +99,7 @@ export interface GatewaySubmitSegmentResultDto {
|
||||
}
|
||||
|
||||
export interface GatewayReceiptEventDto {
|
||||
eventId?: string;
|
||||
traceId?: string;
|
||||
messageId?: string;
|
||||
channelId: string;
|
||||
@@ -114,6 +115,7 @@ export interface GatewayReceiptEventDto {
|
||||
}
|
||||
|
||||
export interface GatewayUplinkEventDto {
|
||||
eventId?: string;
|
||||
traceId?: string;
|
||||
messageId?: string;
|
||||
channelId: string;
|
||||
@@ -136,10 +138,13 @@ export type UplinkMatchCandidateInput = {
|
||||
export interface GatewayPendingDeliveryQueryDto {
|
||||
account: string;
|
||||
limit?: number;
|
||||
claimId?: string;
|
||||
leaseMs?: number;
|
||||
}
|
||||
|
||||
export interface GatewayDownstreamSentDto {
|
||||
id: string;
|
||||
claimId?: string;
|
||||
connectionId?: string;
|
||||
sequenceId?: string;
|
||||
messageId?: string;
|
||||
@@ -153,6 +158,7 @@ export interface GatewayDownstreamAcknowledgedDto extends GatewayDownstreamSentD
|
||||
}
|
||||
|
||||
export type GatewayDownstreamFailureType =
|
||||
| 'claim_released'
|
||||
| 'send_failed'
|
||||
| 'ack_timeout'
|
||||
| 'ack_rejected'
|
||||
|
||||
@@ -3355,6 +3355,14 @@ describe('SendChainService', () => {
|
||||
'delivery-once',
|
||||
]);
|
||||
expect(service['postGatewayControl']).toHaveBeenCalledTimes(1);
|
||||
expect(service['postGatewayControl']).toHaveBeenCalledWith(
|
||||
'/downstream/receipt',
|
||||
expect.objectContaining({ deliveryId: 'delivery-once', claimId: expect.stringMatching(/^api-direct:/) }),
|
||||
);
|
||||
expect(prisma.cmppDownstreamDelivery.updateMany).toHaveBeenCalledWith(expect.objectContaining({
|
||||
where: { id: 'delivery-once', status: 'pending' },
|
||||
data: expect.objectContaining({ status: 'dispatching', connectionId: expect.stringMatching(/^api-direct:/) }),
|
||||
}));
|
||||
expect(prisma.cmppDownstreamDelivery.create).toHaveBeenCalledTimes(3);
|
||||
});
|
||||
|
||||
@@ -3895,8 +3903,12 @@ describe('SendChainService', () => {
|
||||
});
|
||||
prisma.cmppDownstreamDelivery.findMany.mockResolvedValue([]);
|
||||
|
||||
await expect(service.listPendingDownstreamDeliveries({ account: '100001', limit: 100 }))
|
||||
prisma.$queryRaw.mockResolvedValueOnce([]);
|
||||
await expect(service.listPendingDownstreamDeliveries({ account: '100001', limit: 100, claimId: 'gateway-a:100001:1' }))
|
||||
.resolves.toEqual([]);
|
||||
const claimSql = prisma.$queryRaw.mock.calls[0][0].strings.join(' ');
|
||||
expect(claimSql).toContain('FOR UPDATE SKIP LOCKED');
|
||||
expect(claimSql).toContain("status = 'dispatching'");
|
||||
});
|
||||
|
||||
it('persists each upstream SubmitResp segment before the aggregate result arrives', async () => {
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { BadRequestException, HttpException, HttpStatus, Logger, NotFoundException } from '@nestjs/common';
|
||||
import { Prisma } from '@prisma/client';
|
||||
import { createHash } from 'node:crypto';
|
||||
import { createHash, randomUUID } from 'node:crypto';
|
||||
import { BillingService } from '../billing/billing.service';
|
||||
import { moneyToNumber } from '../common/money';
|
||||
import type { OpenApiService } from '../open-api/open-api.service';
|
||||
@@ -28,6 +28,10 @@ export class SendDownstreamDeliveryService {
|
||||
) {}
|
||||
|
||||
async handleUplink(data: GatewayUplinkEventDto) {
|
||||
if (data.eventId) {
|
||||
const existing = await this.prisma.smsUplinkMessage.findUnique({ where: { eventId: data.eventId } });
|
||||
if (existing) return existing;
|
||||
}
|
||||
const channel = await this.prisma.smsChannel.findUnique({ where: { id: data.channelId } });
|
||||
if (!channel) {
|
||||
throw new NotFoundException('SMS channel not found');
|
||||
@@ -35,6 +39,7 @@ export class SendDownstreamDeliveryService {
|
||||
const match = await this.facade.resolveUplinkMatch(data, channel);
|
||||
const record = await this.prisma.smsUplinkMessage.create({
|
||||
data: {
|
||||
eventId: data.eventId,
|
||||
tenantId: match.tenantId,
|
||||
applicationId: match.applicationId,
|
||||
messageRecordId: match.messageRecordId,
|
||||
@@ -276,21 +281,40 @@ export class SendDownstreamDeliveryService {
|
||||
if (!deliveryAllowed) {
|
||||
return delivery;
|
||||
}
|
||||
const claimId = `api-direct:${process.pid}:${randomUUID()}`;
|
||||
const claim = await this.prisma.cmppDownstreamDelivery.updateMany({
|
||||
where: { id: delivery.id, status: 'pending' },
|
||||
data: {
|
||||
status: 'dispatching',
|
||||
connectionId: claimId,
|
||||
ackDeadlineAt: new Date(Date.now() + 30_000),
|
||||
nextRetryAt: null,
|
||||
lastError: null,
|
||||
},
|
||||
});
|
||||
if (claim.count !== 1) {
|
||||
return this.prisma.cmppDownstreamDelivery.findUnique({ where: { id: delivery.id } });
|
||||
}
|
||||
try {
|
||||
const result = await this.facade.postGatewayControl(
|
||||
data.deliveryType === 'receipt' ? '/downstream/receipt' : '/downstream/uplink',
|
||||
{ deliveryId: delivery.id, ...payload },
|
||||
{ deliveryId: delivery.id, claimId, ...payload },
|
||||
) as GatewayControlDeliveryResult;
|
||||
if (result.sent || result.delivered) {
|
||||
await this.facade.markDownstreamDeliverySent({ id: delivery.id, ...result });
|
||||
await this.facade.markDownstreamDeliverySent({ id: delivery.id, claimId, ...result });
|
||||
} else if (result.reasonCode === 'SUBMIT_RESPONSE_PENDING') {
|
||||
return delivery;
|
||||
await this.facade.markDownstreamDeliveryFailed(
|
||||
delivery.id,
|
||||
downstreamControlFailureMessage(result),
|
||||
'claim_released',
|
||||
{ id: delivery.id, claimId, ...result },
|
||||
);
|
||||
} else {
|
||||
await this.facade.markDownstreamDeliveryFailed(
|
||||
delivery.id,
|
||||
downstreamControlFailureMessage(result),
|
||||
result.retryable === false ? 'unrecoverable' : 'send_failed',
|
||||
{ id: delivery.id, ...result },
|
||||
{ id: delivery.id, claimId, ...result },
|
||||
);
|
||||
}
|
||||
} catch (error) {
|
||||
|
||||
@@ -39,15 +39,54 @@ export class SendDownstreamStateService {
|
||||
for (const expired of expiredAcknowledgements) {
|
||||
await this.facade.markDownstreamDeliveryFailed(expired.id, 'CMPP_DELIVER_RESP timeout recovered after Gateway restart', 'ack_timeout');
|
||||
}
|
||||
return this.prisma.cmppDownstreamDelivery.findMany({
|
||||
where: {
|
||||
applicationId: application.id,
|
||||
const now = new Date();
|
||||
await this.prisma.cmppDownstreamDelivery.updateMany({
|
||||
where: { applicationId: application.id, status: 'dispatching', ackDeadlineAt: { lte: now } },
|
||||
data: {
|
||||
status: 'pending',
|
||||
OR: [{ nextRetryAt: null }, { nextRetryAt: { lte: new Date() } }],
|
||||
connectionId: null,
|
||||
ackDeadlineAt: null,
|
||||
nextRetryAt: now,
|
||||
lastError: 'Gateway delivery claim lease expired and was recovered',
|
||||
},
|
||||
orderBy: { createdAt: 'asc' },
|
||||
take: Math.min(Math.max(data.limit ?? 100, 1), 500),
|
||||
});
|
||||
const claimId = String(data.claimId ?? '').trim();
|
||||
if (!claimId) {
|
||||
throw new BadRequestException('claimId is required');
|
||||
}
|
||||
const limit = Math.min(Math.max(data.limit ?? 100, 1), 500);
|
||||
const leaseMs = Math.min(Math.max(data.leaseMs ?? 30_000, 5_000), 120_000);
|
||||
const claimed = await this.prisma.$queryRaw<Array<{ id: string }>>(Prisma.sql`
|
||||
WITH candidates AS (
|
||||
SELECT id
|
||||
FROM "CmppDownstreamDelivery"
|
||||
WHERE "applicationId" = ${application.id}
|
||||
AND status = 'pending'
|
||||
AND ("nextRetryAt" IS NULL OR "nextRetryAt" <= NOW())
|
||||
ORDER BY "createdAt" ASC
|
||||
FOR UPDATE SKIP LOCKED
|
||||
LIMIT ${limit}
|
||||
)
|
||||
UPDATE "CmppDownstreamDelivery" AS delivery
|
||||
SET status = 'dispatching',
|
||||
"connectionId" = ${claimId},
|
||||
"ackDeadlineAt" = NOW() + (${leaseMs} * INTERVAL '1 millisecond'),
|
||||
"nextRetryAt" = NULL,
|
||||
"lastError" = NULL,
|
||||
"updatedAt" = NOW()
|
||||
FROM candidates
|
||||
WHERE delivery.id = candidates.id
|
||||
RETURNING delivery.id
|
||||
`);
|
||||
if (claimed.length === 0) {
|
||||
return [];
|
||||
}
|
||||
const claimedIds = claimed.map((item) => item.id);
|
||||
const deliveries = await this.prisma.cmppDownstreamDelivery.findMany({
|
||||
where: { id: { in: claimedIds }, status: 'dispatching', connectionId: claimId },
|
||||
orderBy: { createdAt: 'asc' },
|
||||
});
|
||||
return deliveries.map((delivery) => ({ ...delivery, claimId }));
|
||||
}
|
||||
|
||||
async markDownstreamDeliveryDelivered(id: string) {
|
||||
@@ -91,7 +130,11 @@ export class SendDownstreamStateService {
|
||||
},
|
||||
});
|
||||
await this.prisma.cmppDownstreamDelivery.updateMany({
|
||||
where: { id: data.id, status: { not: 'delivered' } },
|
||||
where: {
|
||||
id: data.id,
|
||||
status: data.claimId ? 'dispatching' : { not: 'delivered' },
|
||||
...(data.claimId ? { connectionId: data.claimId } : {}),
|
||||
},
|
||||
data: {
|
||||
status: 'awaiting_ack',
|
||||
sentAt,
|
||||
@@ -197,6 +240,23 @@ export class SendDownstreamStateService {
|
||||
if (delivery.status === 'delivered') {
|
||||
return delivery;
|
||||
}
|
||||
if (failureType === 'claim_released') {
|
||||
await this.prisma.cmppDownstreamDelivery.updateMany({
|
||||
where: {
|
||||
id,
|
||||
status: 'dispatching',
|
||||
...(attempt?.claimId ? { connectionId: attempt.claimId } : {}),
|
||||
},
|
||||
data: {
|
||||
status: 'pending',
|
||||
connectionId: null,
|
||||
ackDeadlineAt: null,
|
||||
nextRetryAt: new Date(),
|
||||
lastError: errorMessage ?? 'Gateway released downstream delivery claim',
|
||||
},
|
||||
});
|
||||
return this.prisma.cmppDownstreamDelivery.findUnique({ where: { id } });
|
||||
}
|
||||
if (failureType === 'queue_timeout' && delivery.status !== 'pending') {
|
||||
return delivery;
|
||||
}
|
||||
|
||||
@@ -699,6 +699,10 @@ startSubmitOutboxPublisher() {
|
||||
windowSize: getPositiveConfigInteger(channel.config, 'windowSize', 16),
|
||||
heartbeatIntervalSeconds: getPositiveConfigInteger(channel.config, 'heartbeatIntervalSeconds', 30),
|
||||
heartbeatMissThreshold: getPositiveConfigInteger(channel.config, 'heartbeatMissThreshold', 3),
|
||||
connectionWarmupSeconds: getNonNegativeConfigInteger(channel.config, 'connectionWarmupSeconds', 30),
|
||||
connectionDrainTimeoutSeconds: getPositiveConfigInteger(channel.config, 'connectionDrainTimeoutSeconds', 60),
|
||||
submitResponseTimeoutSeconds: getPositiveConfigInteger(channel.config, 'submitResponseTimeoutSeconds', 60),
|
||||
connectionFailureCooldownSeconds: getPositiveConfigInteger(channel.config, 'connectionFailureCooldownSeconds', 30),
|
||||
},
|
||||
retry: { attempt, maxAttempts: 1 },
|
||||
};
|
||||
|
||||
Reference in New Issue
Block a user