From 9292352be111641e1de77227417e22bbcbe424c9 Mon Sep 17 00:00:00 2001 From: hectorzhao Date: Tue, 25 Aug 2026 16:08:30 +0800 Subject: [PATCH] perf: expand gateway capacity and prevent receipt replay --- .../migration.sql | 3 + api/prisma/schema.prisma | 1 + .../channels/channel-connection.service.ts | 4 + .../channels.helpers.capacity.spec.ts | 14 + api/src/channels/channels.helpers.ts | 18 +- api/src/prisma/prisma.service.ts | 11 +- api/src/protocol-log-worker.module.ts | 9 + api/src/protocol-log-worker.ts | 59 +++ .../protocol-logs/protocol-logs.service.ts | 34 +- .../gateway-callback.controller.spec.ts | 19 +- .../send-chain/gateway-callback.controller.ts | 49 +- api/src/send-chain/send-chain.contracts.ts | 6 + api/src/send-chain/send-chain.service.spec.ts | 14 +- .../send-downstream-delivery.service.ts | 34 +- .../send-downstream-state.service.ts | 74 ++- .../send-chain/send-gateway-submit.service.ts | 4 + .../first-version-development-requirements.md | 11 + ...phase-5-gateway-capacity-expansion-plan.md | 493 ++++++++++++++++++ docs/system-functional-test-cases.md | 17 + docs/testing-progress.md | 19 + gateway/cmd/gateway/main.go | 30 +- gateway/internal/control/server.go | 14 + gateway/internal/control/server_test.go | 16 + gateway/internal/inbound/acknowledgement.go | 11 +- gateway/internal/inbound/delivery.go | 15 +- gateway/internal/inbound/pending_recovery.go | 96 +++- gateway/internal/inbound/protocol_log.go | 25 +- gateway/internal/inbound/server.go | 5 + gateway/internal/inbound/server_test.go | 41 +- gateway/internal/inbound/submit.go | 6 +- gateway/internal/metrics/metrics.go | 14 + gateway/internal/protocollog/publisher.go | 128 +++++ .../internal/protocollog/publisher_test.go | 34 ++ gateway/internal/queue/messages.go | 8 + gateway/internal/resultoutbox/batch.go | 170 ++++++ gateway/internal/resultoutbox/outbox.go | 48 +- gateway/internal/resultoutbox/outbox_test.go | 94 ++++ gateway/internal/resultoutbox/worker.go | 3 + gateway/internal/upstream/connection.go | 59 ++- gateway/internal/upstream/deliver.go | 22 +- gateway/internal/upstream/flow_control.go | 59 ++- gateway/internal/upstream/manager.go | 69 ++- gateway/internal/upstream/pool.go | 137 ++++- gateway/internal/upstream/pool_test.go | 51 +- gateway/internal/upstream/protocol_log.go | 28 +- gateway/internal/upstream/submit.go | 21 +- tools/deploy/production-deploy.sh | 38 +- tools/security/install-security-agent.sh | 10 + 48 files changed, 2001 insertions(+), 144 deletions(-) create mode 100644 api/prisma/migrations/20260825143000_add_uplink_event_id/migration.sql create mode 100644 api/src/channels/channels.helpers.capacity.spec.ts create mode 100644 api/src/protocol-log-worker.module.ts create mode 100644 api/src/protocol-log-worker.ts create mode 100644 docs/phase-5-gateway-capacity-expansion-plan.md create mode 100644 gateway/internal/protocollog/publisher.go create mode 100644 gateway/internal/protocollog/publisher_test.go create mode 100644 gateway/internal/resultoutbox/batch.go diff --git a/api/prisma/migrations/20260825143000_add_uplink_event_id/migration.sql b/api/prisma/migrations/20260825143000_add_uplink_event_id/migration.sql new file mode 100644 index 0000000..551060a --- /dev/null +++ b/api/prisma/migrations/20260825143000_add_uplink_event_id/migration.sql @@ -0,0 +1,3 @@ +ALTER TABLE "SmsUplinkMessage" ADD COLUMN "eventId" TEXT; + +CREATE UNIQUE INDEX "SmsUplinkMessage_eventId_key" ON "SmsUplinkMessage"("eventId"); diff --git a/api/prisma/schema.prisma b/api/prisma/schema.prisma index 95b985d..7188c52 100644 --- a/api/prisma/schema.prisma +++ b/api/prisma/schema.prisma @@ -2117,6 +2117,7 @@ model SmsReceiptAnomaly { model SmsUplinkMessage { id String @id @default(cuid()) + eventId String? @unique tenantId String? applicationId String? channelId String diff --git a/api/src/channels/channel-connection.service.ts b/api/src/channels/channel-connection.service.ts index d804247..b643039 100644 --- a/api/src/channels/channel-connection.service.ts +++ b/api/src/channels/channel-connection.service.ts @@ -367,6 +367,10 @@ export class ChannelConnectionService { cmppVersion: channel.cmppVersion, rateLimitPerSecond: channel.rateLimitPerSecond, windowSize: getPositiveRuntimeInteger(getConfigValue(channel.config, 'windowSize'), 16, 'windowSize'), + connectionWarmupSeconds: Number(getConfigValue(channel.config, 'connectionWarmupSeconds') ?? 30), + connectionDrainTimeoutSeconds: getPositiveRuntimeInteger(getConfigValue(channel.config, 'connectionDrainTimeoutSeconds'), 60, 'connectionDrainTimeoutSeconds'), + submitResponseTimeoutSeconds: getPositiveRuntimeInteger(getConfigValue(channel.config, 'submitResponseTimeoutSeconds'), 60, 'submitResponseTimeoutSeconds'), + connectionFailureCooldownSeconds: getPositiveRuntimeInteger(getConfigValue(channel.config, 'connectionFailureCooldownSeconds'), 30, 'connectionFailureCooldownSeconds'), heartbeatIntervalSeconds: getPositiveRuntimeInteger( getConfigValue(channel.config, 'heartbeatIntervalSeconds'), DEFAULT_HEARTBEAT_INTERVAL_SECONDS, diff --git a/api/src/channels/channels.helpers.capacity.spec.ts b/api/src/channels/channels.helpers.capacity.spec.ts new file mode 100644 index 0000000..8008421 --- /dev/null +++ b/api/src/channels/channels.helpers.capacity.spec.ts @@ -0,0 +1,14 @@ +import { BadRequestException } from '@nestjs/common'; +import { normalizeChannelRuntimeConfig } from './channels.helpers'; + +describe('Gateway channel capacity validation', () => { + it.each([1, 2, 4, 8])('accepts %i supplier connections', (desiredConnections) => { + expect(normalizeChannelRuntimeConfig(undefined, undefined, desiredConnections, 16)).toEqual(expect.objectContaining({ desiredConnections, windowSize: 16 })); + }); + it.each([1, 16, 32, 64])('accepts supplier window %i', (windowSize) => { + expect(normalizeChannelRuntimeConfig(undefined, undefined, 1, windowSize)).toEqual(expect.objectContaining({ desiredConnections: 1, windowSize })); + }); + it.each([[0, 16], [9, 16], [1, 0], [1, 65]])('rejects capacity outside 1..8 connections and 1..64 window', (connections, window) => { + expect(() => normalizeChannelRuntimeConfig(undefined, undefined, connections, window)).toThrow(BadRequestException); + }); +}); diff --git a/api/src/channels/channels.helpers.ts b/api/src/channels/channels.helpers.ts index c57e348..248f114 100644 --- a/api/src/channels/channels.helpers.ts +++ b/api/src/channels/channels.helpers.ts @@ -228,7 +228,7 @@ export function defaultChannelConnectionId(channelId: string) { export function getDesiredConnections(config?: Prisma.JsonValue | null) { if (config && typeof config === 'object' && !Array.isArray(config) && 'desiredConnections' in config) { const value = Number(config.desiredConnections); - if (Number.isInteger(value) && value > 0) { + if (Number.isInteger(value) && value >= 1 && value <= 8) { return value; } } @@ -327,8 +327,12 @@ export function normalizeChannelRuntimeConfig( ? incomingConfig : {}; const base = { ...existing, ...incoming }; - base.desiredConnections = getPositiveRuntimeInteger(desiredConnections ?? base.desiredConnections, 1, 'desiredConnections'); - base.windowSize = getPositiveRuntimeInteger(windowSize ?? base.windowSize, 16, 'windowSize'); + base.desiredConnections = boundedRuntimeInteger(desiredConnections ?? base.desiredConnections, 1, 8, 1, 'desiredConnections'); + base.windowSize = boundedRuntimeInteger(windowSize ?? base.windowSize, 1, 64, 16, 'windowSize'); + base.connectionWarmupSeconds = boundedRuntimeInteger(base.connectionWarmupSeconds, 0, 300, 30, 'connectionWarmupSeconds'); + base.connectionDrainTimeoutSeconds = boundedRuntimeInteger(base.connectionDrainTimeoutSeconds, 1, 600, 60, 'connectionDrainTimeoutSeconds'); + base.submitResponseTimeoutSeconds = boundedRuntimeInteger(base.submitResponseTimeoutSeconds, 1, 300, 60, 'submitResponseTimeoutSeconds'); + base.connectionFailureCooldownSeconds = boundedRuntimeInteger(base.connectionFailureCooldownSeconds, 1, 300, 30, 'connectionFailureCooldownSeconds'); base.heartbeatIntervalSeconds = getPositiveRuntimeInteger( heartbeatIntervalSeconds ?? base.heartbeatIntervalSeconds, DEFAULT_HEARTBEAT_INTERVAL_SECONDS, @@ -345,6 +349,14 @@ export function normalizeChannelRuntimeConfig( return base; } +function boundedRuntimeInteger(value: unknown, minimum: number, maximum: number, fallback: number, field: string) { + const normalized = value === undefined || value === null || value === '' ? fallback : Number(value); + if (!Number.isInteger(normalized) || normalized < minimum || normalized > maximum) { + throw new BadRequestException(`${field} must be an integer between ${minimum} and ${maximum}`); + } + return normalized; +} + export function normalizeLongMessageReceiptMode(value: unknown) { const normalized = String(value ?? 'per_segment').trim() || 'per_segment'; if (!['per_segment', 'message_level'].includes(normalized)) { diff --git a/api/src/prisma/prisma.service.ts b/api/src/prisma/prisma.service.ts index 09b92ca..c7b0bb8 100644 --- a/api/src/prisma/prisma.service.ts +++ b/api/src/prisma/prisma.service.ts @@ -14,14 +14,19 @@ export class PrismaService extends PrismaClient implements OnModuleDestroy { const workerRole = processRole === 'worker'; const outboxRole = processRole === 'outbox'; const callbackRole = processRole === 'callback'; - const databaseUrl = outboxRole + const protocolLogRole = processRole === 'protocol-log-worker'; + const databaseUrl = protocolLogRole + ? process.env.API_PROTOCOL_LOG_DATABASE_URL || process.env.DATABASE_URL + : outboxRole ? process.env.API_OUTBOX_DATABASE_URL || process.env.DATABASE_URL : callbackRole ? process.env.API_CALLBACK_DATABASE_URL || process.env.DATABASE_URL : workerRole ? process.env.API_WORKER_DATABASE_URL || process.env.DATABASE_URL : process.env.DATABASE_URL; - const configuredPoolMax = Number(outboxRole + const configuredPoolMax = Number(protocolLogRole + ? process.env.API_PROTOCOL_LOG_DB_POOL_MAX ?? 4 + : outboxRole ? process.env.API_OUTBOX_DB_POOL_MAX ?? 6 : callbackRole ? process.env.API_CALLBACK_DB_POOL_MAX ?? 16 @@ -30,7 +35,7 @@ export class PrismaService extends PrismaClient implements OnModuleDestroy { : process.env.API_DB_POOL_MAX ?? 32); const poolMax = Number.isInteger(configuredPoolMax) && configuredPoolMax > 0 ? configuredPoolMax - : outboxRole ? 6 : callbackRole ? 16 : workerRole ? 8 : 32; + : protocolLogRole ? 4 : outboxRole ? 6 : callbackRole ? 16 : workerRole ? 8 : 32; const databasePool = new Pool({ connectionString: databaseUrl ?? 'postgresql://cmpp:cmpp_password@localhost:5432/cmpp_platform?schema=public', diff --git a/api/src/protocol-log-worker.module.ts b/api/src/protocol-log-worker.module.ts new file mode 100644 index 0000000..262dcae --- /dev/null +++ b/api/src/protocol-log-worker.module.ts @@ -0,0 +1,9 @@ +import { Module } from '@nestjs/common'; +import { ConfigModule } from '@nestjs/config'; +import { PrismaModule } from './prisma/prisma.module'; +import { ProtocolLogsModule } from './protocol-logs/protocol-logs.module'; + +@Module({ + imports: [ConfigModule.forRoot({ isGlobal: true, envFilePath: ['.env.local', '.env'] }), PrismaModule, ProtocolLogsModule], +}) +export class ProtocolLogWorkerModule {} diff --git a/api/src/protocol-log-worker.ts b/api/src/protocol-log-worker.ts new file mode 100644 index 0000000..fc3da7c --- /dev/null +++ b/api/src/protocol-log-worker.ts @@ -0,0 +1,59 @@ +import { NestFactory } from '@nestjs/core'; +import IORedis from 'ioredis'; +import { ProtocolLogWorkerModule } from './protocol-log-worker.module'; +import { ProtocolLogsService, type ProtocolLogInput } from './protocol-logs/protocol-logs.service'; + +const STREAM = process.env.GATEWAY_PROTOCOL_LOG_STREAM ?? 'gateway.protocol.logs'; +const GROUP = process.env.GATEWAY_PROTOCOL_LOG_GROUP ?? 'cmpp-protocol-log-writer'; +const CONSUMER = process.env.GATEWAY_PROTOCOL_LOG_CONSUMER ?? `protocol-log-${process.pid}`; +const BATCH_SIZE = boundedEnv('PROTOCOL_LOG_STREAM_BATCH_SIZE', 250, 100, 500); + +async function bootstrap() { + process.env.CMPP_PROCESS_ROLE = 'protocol-log-worker'; + const app = await NestFactory.createApplicationContext(ProtocolLogWorkerModule, { logger: ['log', 'warn', 'error'] }); + const logs = app.get(ProtocolLogsService); + const redis = new IORedis(process.env.REDIS_URL ?? 'redis://127.0.0.1:6379', { maxRetriesPerRequest: null }); + try { await redis.xgroup('CREATE', STREAM, GROUP, '0', 'MKSTREAM'); } catch (error) { + if (!String(error).includes('BUSYGROUP')) throw error; + } + let stopping = false; + const stop = () => { stopping = true; }; + process.on('SIGTERM', stop); process.on('SIGINT', stop); + while (!stopping) { + const claimed = await redis.xautoclaim(STREAM, GROUP, CONSUMER, 30_000, '0-0', 'COUNT', BATCH_SIZE) as unknown as [string, Array<[string, string[]]>]; + let messages = claimed[1] ?? []; + if (!messages.length) { + const reply = await redis.xreadgroup('GROUP', GROUP, CONSUMER, 'COUNT', BATCH_SIZE, 'BLOCK', 2000, 'STREAMS', STREAM, '>') as unknown as Array<[string, Array<[string, string[]]>]> | null; + if (!reply) continue; + messages = reply[0]?.[1] ?? []; + } + const accepted: string[] = []; + const parsed: ProtocolLogInput[] = []; + for (const [id, fields] of messages) { + const dataIndex = fields.indexOf('data'); + try { + if (dataIndex < 0) throw new Error('data field missing'); + parsed.push(JSON.parse(fields[dataIndex + 1]) as ProtocolLogInput); + accepted.push(id); + } catch (error) { + console.error(`protocol log stream event ${id} is invalid`, error); + await redis.xadd(`${STREAM}.dead`, '*', 'sourceId', id, 'error', String(error), 'data', dataIndex >= 0 ? fields[dataIndex + 1] : ''); + await redis.xack(STREAM, GROUP, id); await redis.xdel(STREAM, id); + } + } + logs.recordMany(parsed); + if (accepted.length && await logs.flushNow()) { + const pipeline = redis.pipeline(); + for (const id of accepted) pipeline.xack(STREAM, GROUP, id).xdel(STREAM, id); + await pipeline.exec(); + } + } + await logs.flushNow(); await redis.quit(); await app.close(); +} + +function boundedEnv(name: string, fallback: number, minimum: number, maximum: number) { + const value = Number(process.env[name] ?? fallback); + return Number.isInteger(value) && value >= minimum && value <= maximum ? value : fallback; +} + +void bootstrap().catch((error) => { console.error(error); process.exitCode = 1; }); diff --git a/api/src/protocol-logs/protocol-logs.service.ts b/api/src/protocol-logs/protocol-logs.service.ts index bb53895..3882d2c 100644 --- a/api/src/protocol-logs/protocol-logs.service.ts +++ b/api/src/protocol-logs/protocol-logs.service.ts @@ -21,6 +21,10 @@ export type ProtocolLogInput = { payloadBytes?: number | null; retryCount?: number | null; detail?: Record | null; + eventId?: string | null; + gatewayInstanceId?: string | null; + connectionId?: string | null; + submitId?: string | null; }; export type ProtocolLogQuery = { @@ -45,8 +49,10 @@ export class ProtocolLogsService implements OnModuleInit, OnModuleDestroy { constructor(private readonly prisma: PrismaService) {} onModuleInit() { - this.flushTimer = setInterval(() => void this.flush(), positiveEnv('PROTOCOL_LOG_FLUSH_INTERVAL_MS', 500)); - this.flushTimer.unref?.(); + if (process.env.CMPP_PROCESS_ROLE !== 'protocol-log-worker') { + this.flushTimer = setInterval(() => void this.flush(), positiveEnv('PROTOCOL_LOG_FLUSH_INTERVAL_MS', 500)); + this.flushTimer.unref?.(); + } this.retentionTimer = setInterval(() => void this.purgeExpired(), positiveEnv('PROTOCOL_LOG_RETENTION_INTERVAL_MS', 86_400_000)); this.retentionTimer.unref?.(); setTimeout(() => void this.purgeExpired(), 30_000).unref?.(); @@ -82,11 +88,25 @@ export class ProtocolLogsService implements OnModuleInit, OnModuleDestroy { durationMs: safeInteger(input.durationMs), payloadBytes: safeInteger(input.payloadBytes), retryCount: safeInteger(input.retryCount), - detail: sanitizeDetail(input.detail), + detail: sanitizeDetail({ + ...input.detail, + eventId: input.eventId, + gatewayInstanceId: input.gatewayInstanceId, + connectionId: input.connectionId, + submitId: input.submitId, + }), }); if (this.buffer.length >= positiveEnv('PROTOCOL_LOG_BATCH_SIZE', 100)) void this.flush(); } + recordMany(inputs: ProtocolLogInput[]) { + for (const input of inputs) this.record(input); + } + + flushNow() { + return this.flush(); + } + async list(query: ProtocolLogQuery) { const page = Math.max(1, Number(query.page) || 1); const pageSize = Math.min(100, Math.max(1, Number(query.pageSize) || 20)); @@ -119,18 +139,22 @@ export class ProtocolLogsService implements OnModuleInit, OnModuleDestroy { return { items, total, page, pageSize, eventTypes: eventTypes.map((item) => item.eventType) }; } - private async flush() { - if (this.flushing || this.buffer.length === 0) return; + private async flush(): Promise { + if (this.flushing) return false; + if (this.buffer.length === 0) return true; this.flushing = true; const batch = this.buffer.splice(0, positiveEnv('PROTOCOL_LOG_BATCH_SIZE', 100)); try { await this.prisma.protocolInteractionLog.createMany({ data: batch }); } catch (error) { + this.buffer.unshift(...batch); this.logger.error(`Protocol log batch write failed (${batch.length} events)`, error instanceof Error ? error.stack : String(error)); + return false; } finally { this.flushing = false; if (this.buffer.length > 0) setImmediate(() => void this.flush()); } + return true; } private async purgeExpired() { diff --git a/api/src/send-chain/gateway-callback.controller.spec.ts b/api/src/send-chain/gateway-callback.controller.spec.ts index 6811f74..80bdaf6 100644 --- a/api/src/send-chain/gateway-callback.controller.spec.ts +++ b/api/src/send-chain/gateway-callback.controller.spec.ts @@ -11,7 +11,8 @@ describe('GatewayCallbackController', () => { const prisma = { $queryRaw: jest.fn(), getPoolState: jest.fn().mockReturnValue({ max: 16, total: 2, idle: 1, waiting: 0 }) }; const controller = new GatewayCallbackController(sendChain as never, protocolLogs as never, prisma as never); - beforeEach(() => jest.clearAllMocks()); + beforeEach(() => { jest.clearAllMocks(); process.env.PROTOCOL_LOG_CALLBACK_TRACKING_ENABLED = 'true'; }); + afterAll(() => { delete process.env.PROTOCOL_LOG_CALLBACK_TRACKING_ENABLED; }); it('keeps Submit result persistence on the callback process without duplicating protocol logs', async () => { sendChain.handleSubmitResult.mockResolvedValue({ accepted: true }); @@ -40,4 +41,20 @@ describe('GatewayCallbackController', () => { protocol: 'cmpp', direction: 'client_to_platform', eventType: 'submit', status: 'success', })).toThrow(BadRequestException); }); + + it('returns an independent result for every event in a callback batch', async () => { + sendChain.handleSubmitResult.mockResolvedValue({ accepted: true }); + sendChain.handleUplink.mockResolvedValue({ accepted: true }); + await expect(controller.batch({ + batchId: 'CB-1', gatewayInstanceId: 'gateway-1', events: [ + { eventId: 'EV-1', type: 'submit_result', payload: { messageId: 'MSG-1', channelId: 'channel-1', submitStatus: 'accepted' } }, + { eventId: 'EV-2', type: 'uplink', payload: { messageId: 'MSG-1', channelId: 'channel-1', content: '1' } }, + { eventId: 'EV-3', type: 'unsupported', payload: {} }, + ], + })).resolves.toEqual({ batchId: 'CB-1', results: [ + { eventId: 'EV-1', accepted: true }, + { eventId: 'EV-2', accepted: true }, + { eventId: 'EV-3', accepted: false, retryable: false, errorCode: 'INVALID_EVENT' }, + ] }); + }); }); diff --git a/api/src/send-chain/gateway-callback.controller.ts b/api/src/send-chain/gateway-callback.controller.ts index e1f5421..7cf493d 100644 --- a/api/src/send-chain/gateway-callback.controller.ts +++ b/api/src/send-chain/gateway-callback.controller.ts @@ -68,6 +68,51 @@ export class GatewayCallbackController { return this.sendChain.recordGatewaySubmitDeadLetter(body); } + @Post('gateway/events/batch') + async batch(@Body() body: { + batchId?: string; + gatewayInstanceId?: string; + createdAt?: string; + events?: Array<{ eventId?: string; type?: string; payload?: Record }>; + }) { + if (!body.batchId || !body.gatewayInstanceId || !Array.isArray(body.events) || body.events.length < 1 || body.events.length > 100) { + throw new BadRequestException('batchId, gatewayInstanceId and 1 to 100 events are required'); + } + if (Buffer.byteLength(JSON.stringify(body), 'utf8') > 1024 * 1024) { + throw new BadRequestException('Gateway callback batch exceeds 1MB'); + } + const seen = new Set(); + const results: Array<{ eventId: string; accepted: boolean; retryable?: boolean; errorCode?: string }> = []; + for (let offset = 0; offset < body.events.length; offset += 25) { + results.push(...await Promise.all(body.events.slice(offset, offset + 25).map(async (event) => { + const eventId = String(event.eventId ?? '').trim(); + if (!eventId || seen.has(eventId) || !event.payload || typeof event.payload !== 'object') { + return { eventId, accepted: false, retryable: false, errorCode: seen.has(eventId) ? 'DUPLICATE_EVENT_ID' : 'INVALID_EVENT' }; + } + seen.add(eventId); + try { + await this.dispatchBatchEvent(String(event.type ?? ''), { ...event.payload, eventId }); + return { eventId, accepted: true }; + } catch (error) { + const invalid = error instanceof BadRequestException; + return { eventId, accepted: false, retryable: !invalid, errorCode: invalid ? 'INVALID_EVENT' : 'PROCESSING_FAILED' }; + } + }))); + } + return { batchId: body.batchId, results }; + } + + private dispatchBatchEvent(type: string, payload: Record) { + switch (type) { + case 'submit_result': return this.sendChain.handleSubmitResult(payload as unknown as GatewaySubmitResultDto); + case 'submit_segment_result': return this.sendChain.handleSubmitSegmentResult(payload as unknown as GatewaySubmitSegmentResultDto); + case 'receipt_intake': return this.track('deliver_receipt', payload, () => this.sendChain.intakeReceipt(payload as unknown as GatewayReceiptEventDto)); + case 'uplink': return this.track('deliver_uplink', payload, () => this.sendChain.handleUplink(payload as unknown as GatewayUplinkEventDto)); + case 'dead_letter': return this.sendChain.recordGatewaySubmitDeadLetter(payload as unknown as GatewaySubmitDeadLetterDto); + default: throw new BadRequestException(`Unsupported batch event type ${type}`); + } + } + private async track(eventType: string, body: object, action: () => Promise | T) { const startedAt = Date.now(); const value = body as Record; @@ -83,7 +128,7 @@ export class GatewayCallbackController { try { const result = await action(); const resolved = result && typeof result === 'object' ? result as Record : {}; - this.protocolLogs.record({ + if (process.env.PROTOCOL_LOG_CALLBACK_TRACKING_ENABLED === 'true') this.protocolLogs.record({ ...common, tenantId: (resolved.tenantId ?? common.tenantId) as string, applicationId: (resolved.applicationId ?? common.applicationId) as string, @@ -92,7 +137,7 @@ export class GatewayCallbackController { }); return result; } catch (error) { - this.protocolLogs.record({ + if (process.env.PROTOCOL_LOG_CALLBACK_TRACKING_ENABLED === 'true') this.protocolLogs.record({ ...common, status: 'failed', durationMs: Date.now() - startedAt, detail: { error: error instanceof Error ? error.message : String(error) }, }); diff --git a/api/src/send-chain/send-chain.contracts.ts b/api/src/send-chain/send-chain.contracts.ts index d77c85a..a618d37 100644 --- a/api/src/send-chain/send-chain.contracts.ts +++ b/api/src/send-chain/send-chain.contracts.ts @@ -99,6 +99,7 @@ export interface GatewaySubmitSegmentResultDto { } export interface GatewayReceiptEventDto { + eventId?: string; traceId?: string; messageId?: string; channelId: string; @@ -114,6 +115,7 @@ export interface GatewayReceiptEventDto { } export interface GatewayUplinkEventDto { + eventId?: string; traceId?: string; messageId?: string; channelId: string; @@ -136,10 +138,13 @@ export type UplinkMatchCandidateInput = { export interface GatewayPendingDeliveryQueryDto { account: string; limit?: number; + claimId?: string; + leaseMs?: number; } export interface GatewayDownstreamSentDto { id: string; + claimId?: string; connectionId?: string; sequenceId?: string; messageId?: string; @@ -153,6 +158,7 @@ export interface GatewayDownstreamAcknowledgedDto extends GatewayDownstreamSentD } export type GatewayDownstreamFailureType = + | 'claim_released' | 'send_failed' | 'ack_timeout' | 'ack_rejected' diff --git a/api/src/send-chain/send-chain.service.spec.ts b/api/src/send-chain/send-chain.service.spec.ts index 9b19567..4cc6a2f 100644 --- a/api/src/send-chain/send-chain.service.spec.ts +++ b/api/src/send-chain/send-chain.service.spec.ts @@ -3355,6 +3355,14 @@ describe('SendChainService', () => { 'delivery-once', ]); expect(service['postGatewayControl']).toHaveBeenCalledTimes(1); + expect(service['postGatewayControl']).toHaveBeenCalledWith( + '/downstream/receipt', + expect.objectContaining({ deliveryId: 'delivery-once', claimId: expect.stringMatching(/^api-direct:/) }), + ); + expect(prisma.cmppDownstreamDelivery.updateMany).toHaveBeenCalledWith(expect.objectContaining({ + where: { id: 'delivery-once', status: 'pending' }, + data: expect.objectContaining({ status: 'dispatching', connectionId: expect.stringMatching(/^api-direct:/) }), + })); expect(prisma.cmppDownstreamDelivery.create).toHaveBeenCalledTimes(3); }); @@ -3895,8 +3903,12 @@ describe('SendChainService', () => { }); prisma.cmppDownstreamDelivery.findMany.mockResolvedValue([]); - await expect(service.listPendingDownstreamDeliveries({ account: '100001', limit: 100 })) + prisma.$queryRaw.mockResolvedValueOnce([]); + await expect(service.listPendingDownstreamDeliveries({ account: '100001', limit: 100, claimId: 'gateway-a:100001:1' })) .resolves.toEqual([]); + const claimSql = prisma.$queryRaw.mock.calls[0][0].strings.join(' '); + expect(claimSql).toContain('FOR UPDATE SKIP LOCKED'); + expect(claimSql).toContain("status = 'dispatching'"); }); it('persists each upstream SubmitResp segment before the aggregate result arrives', async () => { diff --git a/api/src/send-chain/send-downstream-delivery.service.ts b/api/src/send-chain/send-downstream-delivery.service.ts index e208b1e..aed170e 100644 --- a/api/src/send-chain/send-downstream-delivery.service.ts +++ b/api/src/send-chain/send-downstream-delivery.service.ts @@ -1,6 +1,6 @@ import { BadRequestException, HttpException, HttpStatus, Logger, NotFoundException } from '@nestjs/common'; import { Prisma } from '@prisma/client'; -import { createHash } from 'node:crypto'; +import { createHash, randomUUID } from 'node:crypto'; import { BillingService } from '../billing/billing.service'; import { moneyToNumber } from '../common/money'; import type { OpenApiService } from '../open-api/open-api.service'; @@ -28,6 +28,10 @@ export class SendDownstreamDeliveryService { ) {} async handleUplink(data: GatewayUplinkEventDto) { + if (data.eventId) { + const existing = await this.prisma.smsUplinkMessage.findUnique({ where: { eventId: data.eventId } }); + if (existing) return existing; + } const channel = await this.prisma.smsChannel.findUnique({ where: { id: data.channelId } }); if (!channel) { throw new NotFoundException('SMS channel not found'); @@ -35,6 +39,7 @@ export class SendDownstreamDeliveryService { const match = await this.facade.resolveUplinkMatch(data, channel); const record = await this.prisma.smsUplinkMessage.create({ data: { + eventId: data.eventId, tenantId: match.tenantId, applicationId: match.applicationId, messageRecordId: match.messageRecordId, @@ -276,21 +281,40 @@ export class SendDownstreamDeliveryService { if (!deliveryAllowed) { return delivery; } + const claimId = `api-direct:${process.pid}:${randomUUID()}`; + const claim = await this.prisma.cmppDownstreamDelivery.updateMany({ + where: { id: delivery.id, status: 'pending' }, + data: { + status: 'dispatching', + connectionId: claimId, + ackDeadlineAt: new Date(Date.now() + 30_000), + nextRetryAt: null, + lastError: null, + }, + }); + if (claim.count !== 1) { + return this.prisma.cmppDownstreamDelivery.findUnique({ where: { id: delivery.id } }); + } try { const result = await this.facade.postGatewayControl( data.deliveryType === 'receipt' ? '/downstream/receipt' : '/downstream/uplink', - { deliveryId: delivery.id, ...payload }, + { deliveryId: delivery.id, claimId, ...payload }, ) as GatewayControlDeliveryResult; if (result.sent || result.delivered) { - await this.facade.markDownstreamDeliverySent({ id: delivery.id, ...result }); + await this.facade.markDownstreamDeliverySent({ id: delivery.id, claimId, ...result }); } else if (result.reasonCode === 'SUBMIT_RESPONSE_PENDING') { - return delivery; + await this.facade.markDownstreamDeliveryFailed( + delivery.id, + downstreamControlFailureMessage(result), + 'claim_released', + { id: delivery.id, claimId, ...result }, + ); } else { await this.facade.markDownstreamDeliveryFailed( delivery.id, downstreamControlFailureMessage(result), result.retryable === false ? 'unrecoverable' : 'send_failed', - { id: delivery.id, ...result }, + { id: delivery.id, claimId, ...result }, ); } } catch (error) { diff --git a/api/src/send-chain/send-downstream-state.service.ts b/api/src/send-chain/send-downstream-state.service.ts index a2ab15f..dd80eaa 100644 --- a/api/src/send-chain/send-downstream-state.service.ts +++ b/api/src/send-chain/send-downstream-state.service.ts @@ -39,15 +39,54 @@ export class SendDownstreamStateService { for (const expired of expiredAcknowledgements) { await this.facade.markDownstreamDeliveryFailed(expired.id, 'CMPP_DELIVER_RESP timeout recovered after Gateway restart', 'ack_timeout'); } - return this.prisma.cmppDownstreamDelivery.findMany({ - where: { - applicationId: application.id, + const now = new Date(); + await this.prisma.cmppDownstreamDelivery.updateMany({ + where: { applicationId: application.id, status: 'dispatching', ackDeadlineAt: { lte: now } }, + data: { status: 'pending', - OR: [{ nextRetryAt: null }, { nextRetryAt: { lte: new Date() } }], + connectionId: null, + ackDeadlineAt: null, + nextRetryAt: now, + lastError: 'Gateway delivery claim lease expired and was recovered', }, - orderBy: { createdAt: 'asc' }, - take: Math.min(Math.max(data.limit ?? 100, 1), 500), }); + const claimId = String(data.claimId ?? '').trim(); + if (!claimId) { + throw new BadRequestException('claimId is required'); + } + const limit = Math.min(Math.max(data.limit ?? 100, 1), 500); + const leaseMs = Math.min(Math.max(data.leaseMs ?? 30_000, 5_000), 120_000); + const claimed = await this.prisma.$queryRaw>(Prisma.sql` + WITH candidates AS ( + SELECT id + FROM "CmppDownstreamDelivery" + WHERE "applicationId" = ${application.id} + AND status = 'pending' + AND ("nextRetryAt" IS NULL OR "nextRetryAt" <= NOW()) + ORDER BY "createdAt" ASC + FOR UPDATE SKIP LOCKED + LIMIT ${limit} + ) + UPDATE "CmppDownstreamDelivery" AS delivery + SET status = 'dispatching', + "connectionId" = ${claimId}, + "ackDeadlineAt" = NOW() + (${leaseMs} * INTERVAL '1 millisecond'), + "nextRetryAt" = NULL, + "lastError" = NULL, + "updatedAt" = NOW() + FROM candidates + WHERE delivery.id = candidates.id + RETURNING delivery.id + `); + if (claimed.length === 0) { + return []; + } + const claimedIds = claimed.map((item) => item.id); + const deliveries = await this.prisma.cmppDownstreamDelivery.findMany({ + where: { id: { in: claimedIds }, status: 'dispatching', connectionId: claimId }, + orderBy: { createdAt: 'asc' }, + }); + return deliveries.map((delivery) => ({ ...delivery, claimId })); } async markDownstreamDeliveryDelivered(id: string) { @@ -91,7 +130,11 @@ export class SendDownstreamStateService { }, }); await this.prisma.cmppDownstreamDelivery.updateMany({ - where: { id: data.id, status: { not: 'delivered' } }, + where: { + id: data.id, + status: data.claimId ? 'dispatching' : { not: 'delivered' }, + ...(data.claimId ? { connectionId: data.claimId } : {}), + }, data: { status: 'awaiting_ack', sentAt, @@ -197,6 +240,23 @@ export class SendDownstreamStateService { if (delivery.status === 'delivered') { return delivery; } + if (failureType === 'claim_released') { + await this.prisma.cmppDownstreamDelivery.updateMany({ + where: { + id, + status: 'dispatching', + ...(attempt?.claimId ? { connectionId: attempt.claimId } : {}), + }, + data: { + status: 'pending', + connectionId: null, + ackDeadlineAt: null, + nextRetryAt: new Date(), + lastError: errorMessage ?? 'Gateway released downstream delivery claim', + }, + }); + return this.prisma.cmppDownstreamDelivery.findUnique({ where: { id } }); + } if (failureType === 'queue_timeout' && delivery.status !== 'pending') { return delivery; } diff --git a/api/src/send-chain/send-gateway-submit.service.ts b/api/src/send-chain/send-gateway-submit.service.ts index fcecc33..6c0b201 100644 --- a/api/src/send-chain/send-gateway-submit.service.ts +++ b/api/src/send-chain/send-gateway-submit.service.ts @@ -699,6 +699,10 @@ startSubmitOutboxPublisher() { windowSize: getPositiveConfigInteger(channel.config, 'windowSize', 16), heartbeatIntervalSeconds: getPositiveConfigInteger(channel.config, 'heartbeatIntervalSeconds', 30), heartbeatMissThreshold: getPositiveConfigInteger(channel.config, 'heartbeatMissThreshold', 3), + connectionWarmupSeconds: getNonNegativeConfigInteger(channel.config, 'connectionWarmupSeconds', 30), + connectionDrainTimeoutSeconds: getPositiveConfigInteger(channel.config, 'connectionDrainTimeoutSeconds', 60), + submitResponseTimeoutSeconds: getPositiveConfigInteger(channel.config, 'submitResponseTimeoutSeconds', 60), + connectionFailureCooldownSeconds: getPositiveConfigInteger(channel.config, 'connectionFailureCooldownSeconds', 30), }, retry: { attempt, maxAttempts: 1 }, }; diff --git a/docs/first-version-development-requirements.md b/docs/first-version-development-requirements.md index be891de..ff394a1 100644 --- a/docs/first-version-development-requirements.md +++ b/docs/first-version-development-requirements.md @@ -2148,3 +2148,14 @@ - Gateway供应商事件必须与主API控制面隔离：Submit结果、分片结果、回执、上行、受限协议日志和死信进入仅绑定回环地址的独立进程及独立有界PostgreSQL池；连接状态控制继续进入主API。客户HTTP Webhook使用异步队列Worker，不能在Gateway事实回写请求中同步调用客户地址。 - 独立回调进程不得加载运营/计费管理Controller，不得经Nginx或公网暴露；部署必须验证回调健康、池上限、回环监听、事件URL和控制URL。回调服务先于Gateway启动/重启，回调失败应由既有结果Outbox恢复且不得重复处理、重复扣费。 - 本阶段测试环境以正价325金额单位验证：50 TPS档必须同时满足客户端受理、非补发首次供应商Submit不低于50条/秒、MessageId/submitId/Outbox唯一、账单笔数和金额一致、Inbox/Outbox/Stream排空及数据库无持续锁等待。零计费结果不得作为付费链路容量结论。 + +## 单 Gateway 容量扩展第五阶段（2026-08-25） + +- 单个供应商通道必须支持1～8条连接、每连接1～64窗口，并在API校验、Gateway控制入口和实际运行状态三层执行相同边界；扩容需串行预热，缩容需先停止分配并等待在途完成，不能因改配置直接丢弃在途映射。 +- Gateway协议摘要日志不得在Submit热路径同步写数据库或逐条HTTP回调。成功事件可确定性采样，拒绝、超时、断链和非零结果必须全量进入Redis Stream，由独立有界数据库池批量持久化；ACK只能发生在数据库提交后。 +- Submit结果、分片结果、回执、上行和死信应由结果Outbox聚合为最多100条、默认50条的回环HTTP批次。API必须逐事件返回accepted/retryable/errorCode；整批传输失败重放，单个非法事件不得拖累其他事件，事件必须依赖稳定幂等键。 +- Gateway必须暴露连接数、配置/在途窗口、批量回调请求/事件/重试/死信和协议日志发布/采样/错误/Stream长度指标。回调、日志Worker和主API数据库池必须独立有界，PostgreSQL总连接预算不得因扩容失控。 +- 正价容量仍以非补发首次供应商Submit和全链路对账为准。2026-08-25测试机实测20/30/50档稳定，70档出现4秒级P95且未达到目标，故当前单Gateway发布建议上限为50 TPS；100/150/200按停止线未执行。多Gateway租约、fencing和分片属于P2，仍未实施。 +- Gateway拉取待投递回执必须按账号进程内单飞，API必须用FOR UPDATE SKIP LOCKED将pending原子领取为带租约的dispatching；未实际发送的领取必须无损释放，租约过期可恢复。SubmitResp后只能合并调度账号级刷新，不得每条并发扫描同一批pending记录。 +- API新建回执后的直推与Gateway恢复拉取必须共享同一dispatching + claimId所有权；直推未抢到记录时必须退出，不得与恢复路径各发一次。ACK超时定时器必须在ACK注册锁内完成指针赋值，避免极短截止时间下的竞态。 +- 2026-08-25修复后正价阶梯客户入口20/30/50/70/100/150/200 TPS均零拒绝、零节流、零连接错误；完整供应商首提在150/200冲击档分别约94.86/92.24 TPS，显示端到端容量天花板约95 TPS。二次直推领取修复后100 TPS正价复验为999/999、P95/P99=102/179ms、999次999个唯一首提在12.488秒完成（80.00 TPS），972条已形成终态回执的下游投递生命周期尝试次数恰为972、重复0、最大1。生产建议仍保留容量余量，建议限速70 TPS，不将客户SubmitResp受理200 TPS误作完整供应商TPS。 diff --git a/docs/phase-5-gateway-capacity-expansion-plan.md b/docs/phase-5-gateway-capacity-expansion-plan.md new file mode 100644 index 0000000..7efbacd --- /dev/null +++ b/docs/phase-5-gateway-capacity-expansion-plan.md @@ -0,0 +1,493 @@ +# CMPP Gateway 容量扩展与回调降载方案 + +更新日期：2026-08-25 +状态：已实施并完成测试环境验证 +适用范围：发送 Worker/Submit Outbox 到供应商 CMPP Submit、SubmitResp、状态报告、上行和计费结算链路 + +## 1. 已确认的供应商能力边界 + +本方案按以下已确认条件设计： + +- 同一个供应商账号支持建立多条 CMPP 长连接； +- 单通道最多支持 8 条连接； +- 单连接滑动窗口最大支持 64； +- 连接数和窗口上限表示技术许可，不等于供应商账号承诺 TPS； +- 实际配置仍必须服从账号总 TPS、通道限速、供应商网关稳定性和生产合同限制。 + +CMPP 规范建议窗口值为 16；本项目允许使用 32/64，是基于供应商明确支持的扩展能力。禁止未经阶梯验证直接启用 `8 连接 × 64 窗口`。 + +单通道理论最大在途 Submit 数量为： + +```text +连接数 × 单连接窗口 = 8 × 64 = 512 +``` + +512 是未收到 SubmitResp 时允许并行等待的最大请求数，不代表可以稳定达到 512 TPS。稳定 TPS 还取决于 SubmitResp 延迟和供应商账号限速，例如平均响应时间为 200ms 时，窗口理论容量不是首要限制；如果平均响应时间接近 1 秒，窗口容量才会直接限制吞吐。 + +## 2. 改造目标与边界 + +目标： + +1. 逐步提高每个供应商通道可承载的并发 Submit 数量； +2. 在单 Gateway 内完成多连接和窗口扩容，多个 Gateway 水平扩展降为 P2 后续可选项； +3. 将协议日志从发送和回调热路径移出； +4. 将逐事件 HTTP 回调改为可重放、逐事件幂等的批量回调； +5. 在扩容过程中保持消息、Submit、回执和计费不重不漏； +6. 使用非零单价验证真实计费并发，不以零计费压测结果作为容量依据。 + +本阶段不改变： + +- 对外 CMPP 协议和企业账号； +- 签名、模板、余额、频控和企业/应用状态拦截语义； +- 通道组主备补发规则； +- `messageId`、`submitId`、结果事件和计费幂等口径； +- PostgreSQL Submit Outbox 作为 Gateway 命令事实来源的设计。 + +## 3. 改造一：单通道多连接与窗口扩容 + +### 3.1 连接池模型 + +每个启用通道维护独立连接池，最多创建 8 条连接。每条连接独立维护： + +- `connectionId` 和 Gateway 实例标识； +- CMPP CONNECT/CONNECT_RESP 认证状态； +- 独立递增并循环使用的 SequenceId； +- 当前未确认 Submit 数量； +- `sequenceId -> submitId` 在途映射； +- 当前窗口大小和剩余窗口； +- SubmitResp 延迟、超时和连续失败次数； +- ACTIVE_TEST 心跳、断链和重连状态； +- 平滑启用、排空和关闭状态。 + +连接选择采用“可用窗口优先 + 最少在途 + 健康度”策略： + +1. 排除未认证、正在重连、正在排空的连接； +2. 排除窗口已满的连接； +3. 优先选择未确认请求最少的连接； +4. 数量相同时优先选择近期 SubmitResp P95 更低的连接； +5. 连续超时连接进入冷却期，不再分配新 Submit。 + +### 3.2 动态配置 + +建议新增或明确以下配置： + +```text +desiredConnections: 1..8 +windowSize: 1..64 +connectionWarmupSeconds: 30 +connectionDrainTimeoutSeconds: 60 +submitResponseTimeoutSeconds: 60 +connectionFailureCooldownSeconds: 30 +``` + +配置校验必须同时存在于管理 API 和 Gateway： + +- `desiredConnections < 1` 或 `> 8`：拒绝保存/拒绝启动； +- `windowSize < 1` 或 `> 64`：拒绝保存/拒绝启动； +- 扩容时逐条建立连接，不能同时突发 8 次登录； +- 缩容时先停止向目标连接分配新 Submit，等待在途请求完成后断开； +- 修改窗口时只影响新请求，不能丢弃已有在途映射。 + +### 3.3 阶梯扩容参数 + +| 阶段 | 每通道连接 | 单连接窗口 | 理论在途/通道 | 用途 | +|---|---:|---:|---:|---| +| M0 | 1 | 16 | 16 | 当前兼容基线 | +| M1 | 2 | 16 | 32 | 优先验证多连接正确性 | +| M2 | 2 | 32 | 64 | 验证窗口扩容 | +| M3 | 4 | 32 | 128 | 中等容量目标 | +| M4 | 4 | 64 | 256 | 高容量目标 | +| M5 | 8 | 64 | 512 | 上限验证，默认不直接用于生产 | + +每一级必须单独通过正价 smoke、阶梯压测、故障注入和队列排空后才能进入下一级。 + +### 3.4 异常与幂等处理 + +- 收到 SubmitResp 后必须先用连接和 SequenceId 找到稳定 `submitId`，再释放窗口； +- 未找到在途映射的迟到 SubmitResp 进入异常对账，不能静默丢弃； +- TCP 写入前失败可安全重新分配连接； +- TCP 写入成功但未收到 SubmitResp时标记 `unknown`，禁止直接补发； +- 连接断开后只处理该连接的在途请求，不能把整个通道的消息统一重发； +- `submitId` 必须跨连接、跨实例保持唯一； +- 主备通道补发必须使用新的 `submitId`，并通过 `retryOfSubmitRecordId` 关联原提交。 + +## 4. P2 后续项：Gateway 水平扩展（当前不实施） + +多个 Gateway 会引入通道连接所有权、分布式连接配额、租约、fencing token、脑裂、跨实例 pending 接管和“已写入供应商但未收到响应”的 unknown 判定，改造风险明显高于单 Gateway 内扩容。 + +因此本项调整为**优先级 P2 的后续可选改造**： + +- 不纳入当前开发和压测范围； +- 不作为本轮达到目标 TPS 的前置条件； +- 当前先使用单 Gateway 管理同账号多连接，单通道仍遵守最多 8 条连接、单连接窗口最多 64； +- 只有单 Gateway 的 CPU、网络、文件描述符或事件循环成为经验证的主要瓶颈，且单实例优化无法继续提升时，才重新评审本项； +- 未完成独立设计评审、故障模型验证和隔离环境演练前，不得直接部署双 Gateway。 + +以下内容仅作为 P2 预研设计保留，不代表当前实施计划。 + +### 4.1 部署模型 + +部署至少两个 Gateway 实例，每个实例具有唯一的： + +- `gatewayInstanceId`； +- Redis Stream Consumer Name； +- 指标端口和健康检查地址； +- 通道连接所有权和连接标识。 + +所有实例加入同一个 Submit Stream Consumer Group，共享： + +- Submit 命令队列； +- 通道全局 TPS 限流； +- Submit 发布和处理幂等键； +- 结果/回执 Outbox； +- 通道启停配置版本； +- 待恢复和死信状态。 + +### 4.2 推荐的首期分片方式 + +如果以后启动 P2，首期采用“通道分片 + 故障接管”： + +- 正常状态下，一个通道只由一个 Gateway 实例持有连接； +- 不同运营商、主通道和备通道分散到不同实例； +- 实例失联后，备用实例经过租约超时再接管通道； +- 同一个通道不能被两个实例无租约地同时建立全部连接。 + +后续如果需要同账号跨实例并发连接，必须增加分布式连接配额：同一通道所有 Gateway 实例的连接总数不得超过 8。 + +### 4.3 接管与脑裂保护 + +- Redis 或 PostgreSQL 中保存带过期时间的通道所有权租约； +- 租约包含 `channelId`、`gatewayInstanceId`、配置版本和 fencing token； +- 只有持有最新 fencing token 的实例可以领取该通道的新 Submit； +- 旧实例恢复后必须先重新竞选租约，不能直接恢复发送； +- Stream 未确认消息通过 `XAUTOCLAIM` 接管； +- 对“可能已写入供应商 TCP”的请求不得盲目重发，应进入 unknown 对账。 + +### 4.4 负载均衡边界 + +Gateway 水平扩展解决进程 CPU、网络、连接和单机故障问题，但不能突破： + +- 单通道最多 8 条连接； +- 单连接窗口最多 64； +- 供应商账号总 TPS； +- PostgreSQL、Redis和回调服务的实际容量。 + +## 5. 改造三：协议日志降载 + +### 5.1 日志分级 + +| 日志等级 | 示例 | 处理方式 | +|---|---|---| +| 必留异常 | 拒绝、超时、断链、认证失败、重复/未知响应 | 100%保留，异步优先写入 | +| 业务摘要 | Submit、SubmitResp、回执、上行、下行应答 | 异步批量写入 | +| 原始协议包 | 完整请求/响应字段或二进制包 | 正常流量采样，异常全量短期保存 | + +### 5.2 实现方案 + +- Gateway 不在 Submit/SubmitResp 热路径同步写协议日志数据库； +- 日志事件写入独立 Redis Stream； +- 独立日志 Worker 每批领取 100～500 条； +- 使用批量插入和批量状态回写； +- 正常成功事件按通道配置 1%～10% 采样； +- 错误、超时、拒绝和断链事件保持 100%； +- 原始包写滚动文件或对象存储，主库只保存检索摘要和对象地址； +- 正常原始包建议保留 3～7 天，异常日志按审计策略保留； +- Redis 日志 Stream 设置最大长度和积压告警。 + +日志积压不得阻塞 Submit。积压超过门槛时允许降低正常成功日志采样率，但禁止丢弃错误、安全和账务相关事件。 + +### 5.3 必须保留的追踪字段 + +- `messageId`、`submitId`、`eventId`； +- `channelId`、`gatewayInstanceId`、`connectionId`； +- SequenceId 和供应商 Msg_Id； +- 事件类型、方向、结果码和耗时； +- 重试关联和原始日志对象地址。 + +## 6. 改造四：Gateway 批量回调协议 + +### 6.1 接口设计 + +新增内部接口： + +```text +POST /api/gateway/events/batch +``` + +批次请求包含： + +- `batchId`； +- `gatewayInstanceId`； +- `createdAt`； +- 1～100 个事件； +- 每个事件独立的 `eventId`、`type` 和 `payload`。 + +首期支持事件类型： + +- `submit_segment_result`； +- `submit_result`； +- `receipt_intake`； +- `uplink`； +- `dead_letter`。 + +计费不由 Gateway直接生成。计费由 Submit结果/回执状态机依据稳定业务键执行，批量回调只负责可靠传递触发事实。 + +### 6.2 成批策略 + +采用“数量或时间先到先发”： + +- 默认最大 50 个事件； +- 可配置上限 100； +- 默认最大等待 10ms； +- 最大请求体 1MB； +- Submit结果优先级高于普通协议日志； +- 回执和上行可以使用独立批次，避免不同慢操作互相阻塞。 + +### 6.3 响应与重试 + +API 必须返回逐事件结果，而不是只有整批成功或失败： + +```json +{ + "batchId": "CB-001", + "results": [ + { "eventId": "EV-001", "accepted": true }, + { "eventId": "EV-002", "accepted": false, "retryable": true, "errorCode": "DB_BUSY" } + ] +} +``` + +- Gateway 只 ACK `accepted=true` 的事件； +- 可重试失败使用指数退避； +- 不可重试失败进入死信并告警； +- HTTP 超时后允许重发整个批次，API 依靠 `eventId` 幂等； +- API 可以按事件类型及每 20～50 条拆成短事务； +- 单个坏事件不能造成整批回滚。 + +### 6.4 数据库批量处理 + +- `eventId` 建唯一索引或使用既有稳定幂等键； +- Submit结果批量更新 `SmsSubmitRecord`； +- 回执先批量写入耐久 Receipt Inbox，再异步完成聚合； +- 计费使用稳定计费键批量创建，重复事件不得重复扣费； +- 消息和任务进度使用集合更新，避免逐事件 `GROUP BY`； +- 事务中禁止调用外部 HTTP、Redis或供应商接口。 + +### 6.5 兼容与回退 + +- 保留原单事件回调接口； +- 使用开关选择单事件或批量接口，同一事件不能双发； +- 新旧入口共享 `eventId` 幂等逻辑； +- 回退只切换发送方式，不删除回调 Outbox 或幂等记录； +- 回退前必须等待在途批次完成并核对 Redis PEL。 + +## 7. 当前实施顺序与 P2 后续项 + +### 当前步骤 1：可观测性和协议日志降载 + +先补齐连接、窗口、供应商响应延迟和日志积压指标，再将成功协议日志异步批量化。该阶段不提高连接数。 + +### 当前步骤 2：单实例多连接 + +实现连接池、窗口隔离、平滑扩缩容和断链恢复，依次验证 `2×16`、`2×32`、`4×32`。 + +### 当前步骤 3：批量回调协议 + +上线批量接口和逐事件幂等，先影子对账，再切换 Submit结果，最后切换回执和上行。 + +### 当前步骤 4：单 Gateway 容量上限验证 + +只有前述阶段稳定后，才在单 Gateway 内测试 `4×64`、`8×64`。生产建议值依据压测结果确定，不以供应商允许的最大参数直接作为生产配置。 + +### 优先级 P2：多 Gateway 水平扩展 + +当前不实施、不部署、不纳入本轮压测。将来只有证据表明单 Gateway 自身成为主要瓶颈时，才单独立项，执行通道分片、租约防脑裂、故障接管和 unknown 对账验证。 + +## 8. 正价压测方案 + +### 8.1 前置条件 + +- 仅在隔离测试环境执行； +- 压测前完成数据库、环境文件、部署目录和 systemd 配置备份； +- 测试应用单价设置为非零值，并记录压测前余额； +- 使用可明确识别的移动、联通、电信混合号码； +- 主备六通道均处于预期状态； +- Submit、结果、回执、日志 Stream 和数据库 Outbox 初始无积压； +- 压测前记录 Gateway 实例、连接数、窗口和供应商限速配置。 + +### 8.2 阶梯 + +每个连接/窗口档位分别执行： + +1. 9 条正价全链路 smoke； +2. 20 TPS； +3. 30 TPS； +4. 50 TPS； +5. 70 TPS； +6. 100 TPS； +7. 达标且无停止条件时再测试 150/200 TPS。 + +每档结束后等待完整排空并对账，不能只检查入口 SubmitResp。 + +### 8.3 验收指标 + +- 客户 SubmitResp 接收数和唯一 MessageId 数； +- Inbox 完成数； +- 首次供应商 Submit 数和每秒吞吐； +- 各通道、各连接的提交数和在途峰值； +- SubmitResp P50/P95/P99； +- 供应商接受、拒绝、超时和 unknown 数； +- 主备补发关联完整性； +- 最终 delivered/failed/submitted/unknown 数； +- 回调批次大小、请求数、重试数和积压； +- 数据库连接池等待、锁等待和慢 SQL； +- 协议日志 Stream 积压和丢弃/采样计数； +- 计费记录数、计费单位、扣费金额与余额变化。 + +正价计费恒等式至少满足： + +```text +压测前余额 - 压测后余额 += 有效计费记录金额合计 += 各消息 billingUnits × 应用单价之和 +``` + +### 8.4 停止条件 + +出现任一情况立即停止升档： + +- SubmitResp 缺失或 MessageId 重复； +- `submitId` 重复提交； +- 重复计费、漏计费或余额不平； +- 供应商超时/unknown 持续增加； +- Redis Stream、Inbox、Outbox或回调持续积压且无法在规定时间排空； +- 数据库连接等待、锁等待或 idle transaction 异常； +- Gateway 频繁断链或连接数不能恢复； +- 通道禁用、主备补发或发送拦截语义异常； +- 日志降载导致错误、安全或账务事件丢失。 + +## 9. 故障回归 + +必须覆盖： + +- 单连接断开，其余连接继续发送； +- 同一通道从 4 条连接缩到 2 条连接，在途请求不丢失； +- 单 Gateway 进程重启后恢复连接、在途状态和 Stream pending； +- Redis 短暂不可用； +- 批量回调 HTTP 超时后整批重放； +- 回调批次中单个非法事件不拖累其他事件； +- PostgreSQL 回调连接池耗尽时不阻塞 Submit Outbox； +- 主通道 Submit拒绝触发备通道关联补发； +- 主通道禁用后只使用备通道； +- 主备同时禁用时发送在供应商前失败； +- 签名/模板未报备、余额不足、企业/应用删除或禁用、号码频控仍能正确拦截； +- 重启和事件重放不造成重复提交、重复回执或重复计费。 + +多 Gateway 的实例宕机接管、租约、fencing token 和脑裂测试属于 P2 后续项目，不纳入当前故障回归范围。 + +## 10. 容量预期 + +容量提升不能简单相加。最终吞吐由最慢环节决定： + +```text +实际 TPS = min( + 入口能力, + Worker/Outbox能力, + Gateway连接与窗口能力, + 供应商账号总TPS, + 回调与计费能力, + PostgreSQL/Redis能力 +) +``` + +以最近一次测试记录的完整供应商首提约 33 TPS 为历史参考，执行前必须重新验证当前基线。工程预期区间如下： + +| 完成范围 | 预期稳定首提吞吐 | +|---|---:| +| 独立回调 + 批量路由/持久化 | 50～70 TPS | +| 加协议日志降载 + `2×16`/`2×32` | 60～100 TPS | +| 加 `4×32` 或 `4×64` | 80～150 TPS | +| 单 Gateway `8×64` 上限验证 | 由供应商总TPS及单机实测决定，不预先承诺 | +| 多 Gateway 水平扩展（P2） | 当前不实施，另行立项后评估 | +| 300～500 TPS | 不作为当前承诺；未来可能需要供应商总TPS、通道数量和基础设施共同扩展 | + +上述数字是容量规划目标，不是承诺值。即使单通道理论支持 512 个在途请求，如果供应商账号总限速为 50 TPS，平台稳定吞吐仍不会超过约 50 TPS。 + +## 11. 生产推荐原则 + +- 生产参数使用“最低但足够”的连接数和窗口，不长期顶格运行； +- 初始推荐从 `2 连接 × 16 窗口` 开始； +- 只有监控证明窗口经常耗尽且供应商响应稳定，才提高到 32/64； +- 当前不增加 Gateway 实例；只有单实例 CPU、网络、文件描述符或事件循环成为经验证且无法继续优化的瓶颈，才启动 P2 水平扩展评审； +- 保留连接数、窗口、批量回调、日志采样和 Gateway分片的独立回退开关； +- 每次只改变一个主要容量变量，保留可比较的正价压测基线。 + +## 12. 2026-08-25 实施与测试结论 + +本轮已实施本方案中除“多 Gateway 水平扩展（P2）”外的改造，并仅发布到测试环境 `100.93.204.60`。发布前恢复点为 `/opt/cmpp-platform-backups/phase5-capacity-20260825T041622Z`，数据库、运行源码和 systemd/环境配置归档均已校验；预生产环境未操作。 + +已交付能力： + +- Gateway 协议摘要日志改为 Redis Stream 异步写入，成功事件确定性采样 10%，拒绝、超时、断链和非零结果 100% 保留；独立 `cmpp-protocol-log-worker` 使用独立 4 槽 PostgreSQL 连接池批量落库，成功后 ACK/XDEL，失败重放，格式错误进入死信 Stream。 +- 单通道支持 1～8 条供应商连接、每连接 1～64 窗口；连接选择按在途数量和近期 RTT，连续失败进入冷却。扩容串行预热；缩容先停止分配、等待在途结束或排水超时后关闭。 +- Gateway 结果 Outbox 支持最多 50 条、最长等待 10ms 的批量 HTTP 回调；Submit 结果、分片结果、回执、上行和死信均返回逐事件结果，整批 HTTP 失败留在 PEL 重放，单个非法事件不拖累同批其他事件。 +- API、Gateway 控制入口和实际运行状态完成三层校验。运行矩阵覆盖 `1x1`、`2x16`、`4x32`、`8x64`、`4x64`、`2x32`，越界 `0/9` 连接及 `0/65` 窗口均被拒绝。 +- 上行增加 `eventId` 唯一幂等键；同一批量上行事件重放两次只生成一条 `SmsUplinkMessage`。 + +发布后曾发现协议日志发布器在仅配置 `REDIS_HOST/REDIS_PORT` 的环境中没有对空 `REDIS_URL` 回退。该问题触发停止条件，未进入阶梯；修正为与其他 Gateway Redis 组件一致的本机回退、增加测试并重发后，冒烟显示发布 16、采样丢弃 47、错误 0，日志 Stream 最终 `pending=0/lag=0`。 + +### 12.1 正价阶梯结果 + +测试单价为 325 分/计费条，三运营商混合号码，六个模拟供应商账号；压测连接配置为每通道 `2x32`。容量按非补发的首次供应商 Submit 统计。 + +| 目标档 | 客户 SubmitResp | 首次供应商 Submit | 首提跨度/实际速率 | 客户 P50/P95/P99 | 计费 | 结论 | +|---:|---:|---:|---:|---:|---:|---| +| smoke | 9/9 | 9 | 约 8.1 秒/低速冒烟 | 17/1727/1727 ms | 9 笔/2925 | 冷启动尾延迟后链路闭合 | +| 20 TPS | 199/199 | 199 | 约 9.8 秒/约 20 TPS | 14/21/29 ms | 199 笔/64675 | 通过 | +| 30 TPS | 299/299 | 299 | 9.840 秒/30.39 TPS | 16/33/61 ms | 299 笔/97175 | 通过 | +| 50 TPS | 499/499 | 499 | 10.086 秒/49.47 TPS | 22/62/89 ms | 499 笔/162175 | 通过 | +| 70 TPS | 699/699 | 690 | 10.351 秒/66.66 TPS | 94/4099/5323 ms | 690 笔/224250 | 触发停止线，不再升档 | +| 100/150/200 TPS | 未执行 | - | - | - | - | 按停止条件立即停止 | + +70 TPS 档少 9 条首次 Submit 和计费并非丢失：9 条均为重复使用的测试号码命中“单号码 24 小时最多 10 条”，最终 `failed/RISK`，供应商提交和计费均为 0。该结果证明频控拦截有效，但 70 TPS 的 P95/P99 已从 50 档的 `62/89ms` 跃升到 `4099/5323ms`，目标 70 TPS 也只完成 66.66 TPS，因此本版本稳定上限定为 **50 TPS**；70 TPS 只能视作非稳定冲击档。 + +### 12.2 完整回归与恢复 + +- Submit/回执：各通过档客户 SubmitResp 无缺失，MessageId、submitId 和供应商 MessageId 无重复；模拟器 DELIVRD/失败/无回执比例按配置落库，三条 Stream 最终均 `pending=0/lag=0`。 +- 计费：20/30/50 档及 70 档未被风控拒绝的消息，计费笔数均等于有效消息数，单价全部 325，金额满足 `条数 x 325`；风控、余额不足和路由失败不计费。 +- 主备：随机结果码 8 产生带 `retryOfSubmitRecordId` 的备通道补发；仅主通道停用时 9 条全部走 `LGST-M-B`，主备六通道全停时 9/9 `failed/ROUTE`、供应商提交 0。 +- 上行/批量回调：有效上行与非法事件同批时互不影响，非法项为不可重试；同一 eventId 重放两次仅落一条上行记录。 +- 拦截：签名未审核、模板强制匹配但未审核、余额不足分别为 9/9 `SIGNATURE/TEMPLATE/BALANCE` 且供应商提交 0；应用或企业停用时客户连接不能鉴权；频控在 70 档真实命中；所有临时状态均恢复。 +- 平滑扩缩容：`LGST-M-P` 从 1 扩到 4，再在 20 TPS、20 秒发送期间缩到 1；399/399 SubmitResp，399 条首次 Submit，无重复 ID，P95/P99=`49/142ms`，最终连接恢复 6/6。 +- 故障：Gateway 重启后由 API 恢复连接；Redis 在队列排空时短暂停止，Gateway 健康进程保持，Redis 恢复后重启 Gateway/API 并恢复 6/6；整批 HTTP 超时重放和单事件隔离另有自动化测试覆盖。 +- 环境恢复：10 个测试应用单价恢复 0，企业/应用/签名/模板/通道均恢复原状态，三条临时号段规则删除，六通道连接恢复 6/6。真实消息、回执和账务事实保留审计，不回写余额。 + +## 13. 2026-08-25 回执回放风暴修复与正价复压 + +旧结论中70 TPS的4秒级尾延迟不是前几轮Worker、Outbox或计费优化失效，而是Gateway在每条SubmitResp后并发扫描同账号pending回执；API仅查询pending，直到异步sent回调才改状态，因此多个扫描会读到同一批。旧版100/150冲击档客户回执重复曾达847/5101条，Gateway goroutine在150档峰值超过1.2万。 + +修复内容： + +- Gateway同账号flushPending单飞，SubmitResp触发改为25ms尾随合并、250ms最长等待，登录和周期恢复保留； +- API使用FOR UPDATE SKIP LOCKED原子将pending领取为dispatching，绑定claimId和5～120秒可控租约；当前Gateway默认30秒； +- 客户离线或SubmitResp屏障时用claim_released无损释放，不消耗业务重试预算；过期dispatching自动恢复为pending； +- API新建回执后的直推也必须先原子取得dispatching + claimId，从根源消除直推与Gateway恢复各发一次的竞态； +- ACK超时定时器在ACK registry锁内完成指针赋值，Linux竞态检查不再报告该竞态。 + +发布仅到测试机100.93.204.60，预生产未操作。恢复点为/opt/cmpp-platform-backups/phase5-replay-hotfix-20260825T070815Z，PostgreSQL custom dump、运行源码、systemd/环境配置和SHA-256均已校验。Linux go test -race ./internal/inbound、Gateway全包测试/go vet、API 44套520项和TypeScript构建通过。最终运行标记为761c123b65f09093bc379096188f3ee9ccb2e618+workspace.phase5.replayhotfix2.directclaim。 + +正价阶梯（单价325）： + +| 目标档 | 客户SubmitResp | 客户P50/P95/P99 | 首次供应商Submit跨度/速率 | 结论 | +|---:|---:|---:|---:|---| +| 20 | 199/199 | 5/10/16ms | 9.930s / 19.84 TPS | 通过 | +| 30 | 299/299 | 5/10/18ms | 9.854s / 30.34 TPS | 通过 | +| 50 | 499/499 | 7/35/47ms | 10.067s / 49.47 TPS | 通过 | +| 70 | 699/699 | 14/44/60ms | 10.666s / 65.54 TPS | 入口稳定，首提接近70档 | +| 100 | 999/999 | 24/51/72ms | 13.345s / 74.86 TPS | 入口通过，端到端未达100 | +| 150 | 1498/1498 | 49/87/114ms | 15.781s / 94.86 TPS | 冲击档，显示端到端天花板 | +| 200 | 1998/1998 | 50/84/110ms | 21.661s / 92.24 TPS | 冲击档，端到端不再增长 | + +直推领取修复后另执行100 TPS正价复验：999/999，P50/P95/P99=54/102/179ms；999个唯一MessageId、999次非补发首提在12.488秒完成（80.00 TPS）。972条已产生最终回执的投递全部ACK，生命周期尝试972、重复0、最大尝试1。计费998条：995 charged、3条供应商最终失败后refunded；1条RISK在供应商前失败且未计费。27条模拟器no-receipt保持submitted，不是队列丢失。Inbox、Submit Outbox、下游pending/dispatching/awaiting_ack、未授权锁和idle transaction最终均为0。 + +最终结论分两种口径：客户入口SubmitResp受理已通过200 TPS冲击；完整供应商首提天花板约95 TPS。测试环境建议稳定限速70 TPS，保留约25%以上端到端余量；不将入口200 TPS宣称为供应商200 TPS。按用户要求，7个测试应用单价保持325，^1380028/^1300028/^1890028三条临时号段规则保留，原100.91.249.119/32和新增127.0.0.1/32白名单均保留，未做测试后删除或归零。 diff --git a/docs/system-functional-test-cases.md b/docs/system-functional-test-cases.md index a8f2ca7..6ae7011 100644 --- a/docs/system-functional-test-cases.md +++ b/docs/system-functional-test-cases.md @@ -4864,3 +4864,20 @@ npm run verify:phase8 | TC-CMPP-CALLBACK-001 | Gateway事件进程隔离 | Submit结果、回执、上行、协议日志和死信只进入回环回调进程及独立12槽池；连接状态仍进入主API | | TC-CMPP-CALLBACK-002 | 回调安全边界 | 回调进程仅监听127.0.0.1，不加载计费管理Controller；健康与指标端点只在回环可见 | | TC-CMPP-BATCH-003 | 正价50 TPS完整提交 | 单价325，499条首次Submit在9.930秒完成（50.25/s）；499笔账单162175，无丢重，队列最终排空 | + +## TC-CMPP-PHASE5 单 Gateway 容量扩展（2026-08-25） + +| 用例ID | 场景 | 验收结果 | +| --- | --- | --- | +| TC-CMPP-PHASE5-001 | API、Gateway控制入口、实际运行三层校验1～8连接/1～64窗口 | `1x1/2x16/4x32/8x64/4x64/2x32`通过；越界拒绝 | +| TC-CMPP-PHASE5-002 | 协议日志降载 | 成功日志确定性采样，异常全留；独立Worker批量写库后ACK/XDEL，错误0且Stream排空 | +| TC-CMPP-PHASE5-003 | 批量HTTP回调与重放 | 多事件少请求；逐事件结果；非法事件不影响有效事件；HTTP失败整批重放；重试/死信可观测 | +| TC-CMPP-PHASE5-004 | 上行eventId幂等 | 同一事件投递两次只生成1条SmsUplinkMessage | +| TC-CMPP-PHASE5-005 | 在途平滑缩容 | 单通道1→4→1并发发送399条，SubmitResp/首次Submit均399，重复ID为0，连接恢复6/6 | +| TC-CMPP-PHASE5-006 | 非零单价阶梯停止线 | 20/30/50通过；70档P95/P99=`4099/5323ms`且实得66.66 TPS，立即停止100/150/200，稳定上限50 TPS | +| TC-CMPP-PHASE5-007 | 拦截与主备回归 | SIGNATURE/TEMPLATE/BALANCE/RISK均供应商前拦截；应用/企业停用拒绝鉴权；主停走备、全停ROUTE；拒绝补发retryOf完整 | +| TC-CMPP-PHASE5-008 | Gateway/Redis故障与最终排空 | Gateway重启恢复连接；排空时Redis短停后恢复；Submit/结果/日志Stream pending/lag均0，无锁等待/idle事务 | +| TC-CMPP-PHASE5-009 | 同账号并发pending拉取 | 8个并发刷新只产生1次API领取；记录以FOR UPDATE SKIP LOCKED从pending转dispatching，携带claimId和租约 | +| TC-CMPP-PHASE5-010 | API直推与Gateway恢复同时命中同一回执 | 仅抢到dispatching + claimId的路径发送；100 TPS正价实测972条终态回执对应972次下游ACK，重复0，最大尝试1 | +| TC-CMPP-PHASE5-011 | claim未发送、租约过期和ACK定时器竞态 | SubmitResp屏障或客户离线时释放claim且不增加重试次数；过期dispatching可恢复；Linux go test -race ./internal/inbound通过 | +| TC-CMPP-PHASE5-012 | 修复后正价容量与口径分离 | 20至200 TPS入口均无拒绝、节流或连接错误；999条100 TPS复验入口P95/P99 102/179ms，供应商首提80.00 TPS；150/200冲击档首提约95 TPS天花板，两种口径分开报告 | diff --git a/docs/testing-progress.md b/docs/testing-progress.md index e9f8872..698d449 100644 --- a/docs/testing-progress.md +++ b/docs/testing-progress.md @@ -3946,3 +3946,22 @@ git diff --check - 修复后正价结果：smoke 9/9、账单2925；20 TPS 199/199、P50/P95/P99=`19/51/94ms`、账单64675；30 TPS 299/299、`20/48/69ms`、账单97175；50 TPS 499/499、`35/76/135ms`、账单162175。四个成功窗口合计1006条、1006笔账单、单价全部325、金额326950。 - 50档499条唯一首次供应商Submit跨度9.930秒，完整提交速率`50.25条/秒`，相对上一版同档33.07条/秒提高约52%。全部尝试/Outbox均542条且ID唯一；状态检查时delivered485、failed4、submitted10，关联补发43次。Inbox、Outbox、双Stream最终排空，数据库无等待锁、无idle in transaction，回调池`max=12,total=1,idle=1,waiting=0`。 - 测试环境最终标记`c6f11014d61a8627ae311098ef81d89dd099ffaa+workspace.phase4batchcallback.callback-surface.599393bb6233`；恢复资产为`/opt/cmpp-platform-backups/phase4-batch-callback-20260825T025611Z`。10个隔离应用单价恢复0，三条临时号段规则删除，真实计费事实保留；预生产未操作，未执行100/200/300/500档。 + +## 2026-08-25 第五阶段单 Gateway 容量扩展与正价压测 + +- 完成协议日志 Redis Stream 降载及独立日志 Worker、单 Gateway 每通道 1～8 连接/每连接 1～64 窗口、RTT/在途择优、失败冷却、平滑扩缩容、Gateway 结果 Outbox 批量 HTTP 回调、逐事件确认和上行 eventId 幂等；明确未实施多 Gateway P2。 +- API单测、Gateway控制入口和测试机实际运行三层覆盖 `1x1/2x16/4x32/8x64/4x64/2x32`。`LGST-M-P` 1→4→1 在20 TPS在途发送期间平滑缩容，399/399 SubmitResp、399条首次Submit、重复ID为0，P95/P99=`49/142ms`。 +- 协议日志首次冒烟发现空`REDIS_URL`未回退，按停止线中止。修复、回归并重发后发布16、采样47、错误0，独立Worker排空。批量回调累计事件数大于HTTP请求数，重试/死信为0；同一上行eventId重放两次只落一条。 +- 正价325阶梯：20档199条约20 TPS、P95/P99=`21/29ms`、64675；30档299条30.39 TPS、`33/61ms`、97175；50档499条49.47 TPS、`62/89ms`、162175；70档699个SubmitResp，690条有效首提在10.351秒完成66.66 TPS，`4099/5323ms`，690笔224250。另9条为频控`RISK`且供应商/计费均0。 +- 70档尾延迟突增且未达到目标，立即停止，未执行100/150/200。稳定TPS上限定为50 TPS。 +- 当前部署回归：签名、模板、余额分别9/9在供应商前拦截；应用/企业停用均无法鉴权；主通道停用9条全部走备通道，主备全停9/9`ROUTE`且提交0；随机主拒绝补发均有retryOf；Submit、回执、上行和计费对账通过。 +- 发布只到`100.93.204.60`，恢复点`/opt/cmpp-platform-backups/phase5-capacity-20260825T041622Z`；预生产未操作。结束时应用单价0、临时号段规则0、业务配置恢复、六连接在线，队列排空且数据库无锁等待/idle事务。 + +## 2026-08-25 第五阶段回执回放热修复与正价复压 + +- 重新验证确认旧70 TPS尾延迟是回执恢复风暴：每条SubmitResp都启动pending扫描，多个扫描在sent回调前取到同一批。实施Gateway账号级单飞与25ms/250ms有界合并、API FOR UPDATE SKIP LOCKED pending到dispatching原子领取、claimId/30秒租约、未发送无损释放及过期恢复。 +- 首轮复压还发现6191条中29条有2次下游尝试，进一步定位为API新建回执直推与Gateway恢复拉取之间没有共享所有权。直推改为先取得dispatching + claimId，未抢到则退出。Linux race门禁另发现并修复ACK超时timer指针赋值竞态。 +- 门禁：API 44套520项、TypeScript构建、Gateway全包测试、go vet、Linux go test -race ./internal/inbound和git diff --check通过。只发布测试机，预生产未操作；恢复点/opt/cmpp-platform-backups/phase5-replay-hotfix-20260825T070815Z，最终标记761c123b65f09093bc379096188f3ee9ccb2e618+workspace.phase5.replayhotfix2.directclaim。 +- 正价325阶梯入口SubmitResp：20=199/199、P95/P99 10/16ms；30=299/299、10/18ms；50=499/499、35/47ms；70=699/699、44/60ms；100=999/999、51/72ms；150=1498/1498、87/114ms；200=1998/1998、84/110ms；全部零拒绝、零节流、零连接错误。完整供应商首提速率依次为19.84/30.34/49.47/65.54/74.86/94.86/92.24 TPS，端到端天花板约95 TPS。 +- 最终100 TPS正价复验：999/999，P50/P95/P99 54/102/179ms；999个唯一MessageId，1087个含主备补发的submitId全部唯一；999次999个唯一首提在12.488秒完成（80.00 TPS）。972条终态回执全部客户ACK，下游尝试972、重复0、最大1。计费998条中995 charged、3 refunded；1条RISK供应商前失败不计费；27条no-receipt保持submitted。Inbox/Outbox/下游pending队列、未授权锁和idle transaction最终均0。 +- 最终建议分口径：客户入口可受理200 TPS冲击，不等于端到端供应商200 TPS；供应商首提峰值约95 TPS，建议稳定限速70 TPS保留余量。按用户要求，7个应用单价保持325，三条临时号段规则、Tailscale白名单及VM回环白名单均保留，未做删除或归零。压测原始证据已复制到相邻测试项目lg-cmpp-stress-lab/results下的phase5-replayfix目录。 diff --git a/gateway/cmd/gateway/main.go b/gateway/cmd/gateway/main.go index cf9b00a..e683953 100644 --- a/gateway/cmd/gateway/main.go +++ b/gateway/cmd/gateway/main.go @@ -13,6 +13,7 @@ import ( "cmpp-platform/gateway/internal/health" "cmpp-platform/gateway/internal/inbound" platformmetrics "cmpp-platform/gateway/internal/metrics" + "cmpp-platform/gateway/internal/protocollog" "cmpp-platform/gateway/internal/queue" "cmpp-platform/gateway/internal/ratelimit" "cmpp-platform/gateway/internal/resultoutbox" @@ -33,7 +34,17 @@ func main() { } apiBaseURL := os.Getenv("API_BASE_URL") callbackBaseURL := getenv("GATEWAY_CALLBACK_API_BASE_URL", apiBaseURL) - upstreamManager := &upstream.Manager{APIBaseURL: apiBaseURL, EventAPIBaseURL: callbackBaseURL} + gatewayInstanceID := getenv("GATEWAY_INSTANCE_ID", hostname()) + protocolLogPublisher, protocolLogErr := protocollog.New(os.Getenv("REDIS_URL")) + if protocolLogErr != nil { + log.Printf("gateway protocol log Redis publisher init failed: %v", protocolLogErr) + } else { + protocolLogPublisher.Stream = getenv("GATEWAY_PROTOCOL_LOG_STREAM", "gateway.protocol.logs") + protocolLogPublisher.GatewayInstanceID = gatewayInstanceID + protocolLogPublisher.SuccessSampleRate = positiveEnvInt("GATEWAY_PROTOCOL_LOG_SUCCESS_SAMPLE_PERCENT", 10) + protocolLogPublisher.MaxLen = int64(positiveEnvInt("GATEWAY_PROTOCOL_LOG_STREAM_MAX_LEN", 200000)) + } + upstreamManager := &upstream.Manager{APIBaseURL: apiBaseURL, EventAPIBaseURL: callbackBaseURL, ProtocolLogPublisher: protocolLogPublisher, GatewayInstanceID: gatewayInstanceID} var worker *submitworker.Worker var resultOutbox *resultoutbox.Outbox channelLimiter, err := ratelimit.New(os.Getenv("REDIS_URL")) @@ -59,9 +70,10 @@ func main() { SubmitHTTPClient: inbound.NewAPIHTTPClient(inboundConcurrency), PresenceStore: presenceStore, RecoveryStore: recoveryStore, - GatewayInstanceID: getenv("GATEWAY_INSTANCE_ID", hostname()), + GatewayInstanceID: gatewayInstanceID, MaxSubmitConcurrency: inboundConcurrency, SecurityEventToken: os.Getenv("SECURITY_EVENT_TOKEN"), + ProtocolLogPublisher: protocolLogPublisher, }).ListenAndServe(); err != nil { log.Fatalf("gateway inbound server stopped: %v", err) } @@ -83,8 +95,14 @@ func main() { resultOutbox.Consumer = getenv("GATEWAY_SUBMIT_RESULT_CONSUMER", "gateway-1") resultOutbox.APIBaseURL = callbackBaseURL resultOutbox.Concurrency = positiveEnvInt("GATEWAY_SUBMIT_RESULT_WORKER_CONCURRENCY", 8) + resultOutbox.BatchEnabled = os.Getenv("GATEWAY_CALLBACK_BATCH_ENABLED") != "false" + resultOutbox.BatchSize = positiveEnvInt("GATEWAY_CALLBACK_BATCH_SIZE", 50) + resultOutbox.BatchWait = time.Duration(positiveEnvInt("GATEWAY_CALLBACK_BATCH_WAIT_MS", 10)) * time.Millisecond + resultOutbox.GatewayInstanceID = gatewayInstanceID + resultOutbox.DeadLetterStream = getenv("GATEWAY_CALLBACK_DEAD_LETTER_STREAM", "gateway.submit.results.dead") worker.ResultOutbox = resultOutbox upstreamManager.SubmitSegmentPublisher = resultOutbox + upstreamManager.EventPublisher = resultOutbox go func() { log.Printf("cmpp gateway submit worker consuming stream=%s group=%s consumer=%s", worker.Stream, worker.Group, worker.Consumer) if err := worker.Run(context.Background()); err != nil { @@ -108,6 +126,13 @@ func main() { UpstreamDesired: desired, UpstreamConnected: connected, DownstreamConnected: inbound.ActiveConnectionCount(), SubmitWorkerUp: worker != nil, } + snapshot.UpstreamWindowConfigured, snapshot.UpstreamWindowInFlight = upstreamManager.WindowCounts() + if protocolLogPublisher != nil { + snapshot.ProtocolLogPublished, snapshot.ProtocolLogSampled, snapshot.ProtocolLogErrors = protocolLogPublisher.Counts() + if length, err := protocolLogPublisher.Redis.XLen(ctx, protocolLogPublisher.StreamName()).Result(); err == nil { + snapshot.ProtocolLogQueueLength = length + } + } if worker != nil { snapshot.SubmitWorkerConcurrency = worker.ConfiguredConcurrency() snapshot.SubmitWorkerInFlight = worker.InFlight() @@ -116,6 +141,7 @@ func main() { snapshot.ResultWorkerUp = true snapshot.ResultWorkerConcurrency = resultOutbox.ConfiguredConcurrency() snapshot.ResultWorkerInFlight = resultOutbox.InFlight() + snapshot.CallbackBatchRequests, snapshot.CallbackBatchEvents, snapshot.CallbackBatchRetries, snapshot.CallbackDeadLetters = resultOutbox.BatchCounts() } snapshot.InboundSubmitConcurrency, snapshot.InboundSubmitInFlight = inbound.SubmitSlotSnapshot() if worker == nil || worker.Redis == nil { diff --git a/gateway/internal/control/server.go b/gateway/internal/control/server.go index 0e1120d..9fb06d4 100644 --- a/gateway/internal/control/server.go +++ b/gateway/internal/control/server.go @@ -41,6 +41,10 @@ type ChannelConfig struct { WindowSize int `json:"windowSize,omitempty"` HeartbeatIntervalSeconds int `json:"heartbeatIntervalSeconds,omitempty"` HeartbeatMissThreshold int `json:"heartbeatMissThreshold,omitempty"` + ConnectionWarmupSeconds int `json:"connectionWarmupSeconds,omitempty"` + ConnectionDrainSeconds int `json:"connectionDrainTimeoutSeconds,omitempty"` + SubmitTimeoutSeconds int `json:"submitResponseTimeoutSeconds,omitempty"` + FailureCooldownSeconds int `json:"connectionFailureCooldownSeconds,omitempty"` } type DisconnectChannelCommand struct { @@ -351,6 +355,10 @@ func (s Server) connectChannel(ctx context.Context, command ConnectChannelComman WindowSize: command.Channel.WindowSize, HeartbeatIntervalSeconds: command.Channel.HeartbeatIntervalSeconds, HeartbeatMissThreshold: command.Channel.HeartbeatMissThreshold, + ConnectionWarmupSeconds: command.Channel.ConnectionWarmupSeconds, + ConnectionDrainSeconds: command.Channel.ConnectionDrainSeconds, + SubmitTimeoutSeconds: command.Channel.SubmitTimeoutSeconds, + FailureCooldownSeconds: command.Channel.FailureCooldownSeconds, }, }) if err != nil { @@ -411,6 +419,12 @@ func validateConnectChannelCommand(command ConnectChannelCommand) error { if command.Channel.Account == "" || command.Channel.PasswordCipher == "" { return fmt.Errorf("account and passwordCipher are required") } + if command.DesiredConnections < 1 || command.DesiredConnections > 8 { + return fmt.Errorf("desiredConnections must be between 1 and 8") + } + if command.Channel.WindowSize != 0 && (command.Channel.WindowSize < 1 || command.Channel.WindowSize > 64) { + return fmt.Errorf("windowSize must be between 1 and 64") + } return nil } diff --git a/gateway/internal/control/server_test.go b/gateway/internal/control/server_test.go index 69dc409..1254be8 100644 --- a/gateway/internal/control/server_test.go +++ b/gateway/internal/control/server_test.go @@ -140,6 +140,22 @@ func TestConnectChannelRejectsInvalidCommand(t *testing.T) { } } +func TestConnectChannelRejectsCapacityOutsideSupportedMatrix(t *testing.T) { + handler := handlerWithConnect(func(context.Context, ConnectChannelCommand) (ConnectionStateCallback, error) { + return ConnectionStateCallback{}, nil + }) + for _, body := range []string{ + `{"schemaVersion":"v1","messageType":"ConnectChannel","channelId":"c","connectionId":"x","desiredConnections":9,"channel":{"gatewayHost":"127.0.0.1","gatewayPort":17890,"account":"a","passwordCipher":"p","windowSize":16}}`, + `{"schemaVersion":"v1","messageType":"ConnectChannel","channelId":"c","connectionId":"x","desiredConnections":1,"channel":{"gatewayHost":"127.0.0.1","gatewayPort":17890,"account":"a","passwordCipher":"p","windowSize":65}}`, + } { + response := httptest.NewRecorder() + handler.ServeHTTP(response, httptest.NewRequest(http.MethodPost, "/connections/connect", strings.NewReader(body))) + if response.Code != http.StatusBadRequest { + t.Fatalf("status=%d body=%s", response.Code, response.Body.String()) + } + } +} + func TestDisconnectChannelStopsSupplierPool(t *testing.T) { var received DisconnectChannelCommand handler := handlerWithServer(Server{ diff --git a/gateway/internal/inbound/acknowledgement.go b/gateway/internal/inbound/acknowledgement.go index 3bf4ab3..192aaa2 100644 --- a/gateway/internal/inbound/acknowledgement.go +++ b/gateway/internal/inbound/acknowledgement.go @@ -18,6 +18,7 @@ const defaultDownstreamAckTimeout = 30 * time.Second type downstreamAckTracker struct { deliveryID string + claimID string connectionID string sequenceID uint32 messageID uint64 @@ -69,29 +70,29 @@ func downstreamAckKey(conn *cmpp.Conn, sequenceID uint32) string { return fmt.Sprintf("%p:%d", conn, sequenceID) } -func registerDownstreamAck(session *downstreamSession, deliveryID string, sequenceID uint32, messageID uint64, deadline time.Time) *downstreamAckTracker { +func registerDownstreamAck(session *downstreamSession, deliveryID string, claimID string, sequenceID uint32, messageID uint64, deadline time.Time) *downstreamAckTracker { if session == nil || session.conn == nil || strings.TrimSpace(deliveryID) == "" { return nil } tracker := &downstreamAckTracker{ - deliveryID: deliveryID, connectionID: session.connectionID, + deliveryID: deliveryID, claimID: claimID, connectionID: session.connectionID, sequenceID: sequenceID, messageID: messageID, session: session, } key := downstreamAckKey(session.conn, sequenceID) downstreamAckRegistry.Lock() downstreamAckRegistry.items[key] = tracker - downstreamAckRegistry.Unlock() tracker.timer = time.AfterFunc(time.Until(deadline), func() { timedOut := takeDownstreamAck(session.conn, sequenceID) if timedOut == nil || timedOut.session == nil || timedOut.session.deliveryReport == nil { return } timedOut.session.deliveryReport(downstreamDeliveryLifecycleEvent{ - Kind: "failed", DeliveryID: timedOut.deliveryID, ConnectionID: timedOut.connectionID, + Kind: "failed", DeliveryID: timedOut.deliveryID, ClaimID: timedOut.claimID, ConnectionID: timedOut.connectionID, SequenceID: timedOut.sequenceID, MessageID: timedOut.messageID, ObservedAt: time.Now().UTC(), FailureType: "ack_timeout", ErrorMessage: "CMPP_DELIVER_RESP timeout", }) }) + downstreamAckRegistry.Unlock() return tracker } @@ -149,7 +150,7 @@ func handleDownstreamAcknowledgement(conn *cmpp.Conn, sequenceID uint32, message } if tracker.session != nil && tracker.session.deliveryReport != nil { go tracker.session.deliveryReport(downstreamDeliveryLifecycleEvent{ - Kind: "acknowledged", DeliveryID: tracker.deliveryID, ConnectionID: tracker.connectionID, + Kind: "acknowledged", DeliveryID: tracker.deliveryID, ClaimID: tracker.claimID, ConnectionID: tracker.connectionID, SequenceID: sequenceID, MessageID: messageID, Result: result, ObservedAt: time.Now().UTC(), }) } diff --git a/gateway/internal/inbound/delivery.go b/gateway/internal/inbound/delivery.go index adee28e..a2924cc 100644 --- a/gateway/internal/inbound/delivery.go +++ b/gateway/internal/inbound/delivery.go @@ -16,6 +16,7 @@ import ( type DownstreamReceipt struct { DeliveryID string `json:"deliveryId,omitempty"` + ClaimID string `json:"claimId,omitempty"` Account string `json:"account,omitempty"` ApplicationID string `json:"applicationId,omitempty"` MessageID string `json:"messageId"` @@ -31,6 +32,7 @@ type DownstreamReceipt struct { type DownstreamUplink struct { DeliveryID string `json:"deliveryId,omitempty"` + ClaimID string `json:"claimId,omitempty"` Account string `json:"account,omitempty"` ApplicationID string `json:"applicationId,omitempty"` MessageID string `json:"messageId,omitempty"` @@ -55,6 +57,7 @@ type DownstreamSendResult struct { type downstreamDeliveryLifecycleEvent struct { Kind string DeliveryID string + ClaimID string ConnectionID string SequenceID uint32 MessageID uint64 @@ -70,7 +73,7 @@ func (s Server) reportDownstreamDelivery(event downstreamDeliveryLifecycleEvent) return } payload := map[string]any{ - "id": event.DeliveryID, "connectionId": event.ConnectionID, + "id": event.DeliveryID, "claimId": event.ClaimID, "connectionId": event.ConnectionID, "sequenceId": strconv.FormatUint(uint64(event.SequenceID), 10), "messageId": strconv.FormatUint(event.MessageID, 10), } @@ -160,7 +163,7 @@ func pushReceiptWithResult(event DownstreamReceipt, allowRecovery bool) (Downstr return DownstreamSendResult{}, err } deliver := downstreamDeliverPacket(session, receiptMessageID, session.srcID, defaultString(event.PhoneNumber, session.phoneNumber), 0, 1, string(receiptBytes)) - return sendDownstream(session, deliver, event.DeliveryID) + return sendDownstream(session, deliver, event.DeliveryID, event.ClaimID) } func downstreamReceiptMessageID(event DownstreamReceipt, session *downstreamSession) uint64 { @@ -228,7 +231,7 @@ func PushUplinkWithResult(event DownstreamUplink) (DownstreamSendResult, error) 0, content, ) - return sendDownstream(session, deliver, event.DeliveryID) + return sendDownstream(session, deliver, event.DeliveryID, event.ClaimID) } func errorMessageWithCode(message string, code string) string { @@ -272,7 +275,7 @@ func findSession(messageID string, account string) *downstreamSession { return nil } -func sendDownstream(session *downstreamSession, deliver cmpp.Packer, deliveryID string) (DownstreamSendResult, error) { +func sendDownstream(session *downstreamSession, deliver cmpp.Packer, deliveryID string, claimID string) (DownstreamSendResult, error) { session.mu.Lock() defer session.mu.Unlock() messageID := downstreamDeliverMessageID(deliver) @@ -289,7 +292,7 @@ func sendDownstream(session *downstreamSession, deliver cmpp.Packer, deliveryID SentAt: formatRFC3339Nano(sentAt), AckDeadlineAt: formatRFC3339Nano(ackDeadlineAt), } - tracker := registerDownstreamAck(session, deliveryID, sequenceID, messageID, ackDeadlineAt) + tracker := registerDownstreamAck(session, deliveryID, claimID, sequenceID, messageID, ackDeadlineAt) if err := session.conn.SendPkt(deliver, sequenceID); err != nil { removeDownstreamAck(tracker) session.recordDownstreamProtocol(deliver, deliveryID, sequenceID, messageID, "failed", "SEND_FAILED", err) @@ -306,7 +309,7 @@ func sendDownstream(session *downstreamSession, deliver cmpp.Packer, deliveryID result.Sent = true if deliveryID != "" && session.deliveryReport != nil { go session.deliveryReport(downstreamDeliveryLifecycleEvent{ - Kind: "sent", DeliveryID: deliveryID, ConnectionID: session.connectionID, + Kind: "sent", DeliveryID: deliveryID, ClaimID: claimID, ConnectionID: session.connectionID, SequenceID: sequenceID, MessageID: messageID, ObservedAt: sentAt, AckDeadlineAt: ackDeadlineAt, }) } diff --git a/gateway/internal/inbound/pending_recovery.go b/gateway/internal/inbound/pending_recovery.go index f592ec4..b03f365 100644 --- a/gateway/internal/inbound/pending_recovery.go +++ b/gateway/internal/inbound/pending_recovery.go @@ -6,6 +6,7 @@ import ( "fmt" "log" "strings" + "sync" "time" ) @@ -15,6 +16,8 @@ import ( type pendingDeliveryRequest struct { Account string `json:"account"` Limit int `json:"limit,omitempty"` + ClaimID string `json:"claimId"` + LeaseMS int `json:"leaseMs"` } type pendingDelivery struct { @@ -22,6 +25,7 @@ type pendingDelivery struct { DeliveryType string `json:"deliveryType"` Payload json.RawMessage `json:"payload"` CreatedAt time.Time `json:"createdAt"` + ClaimID string `json:"claimId"` } type pendingFlushResult struct { @@ -33,25 +37,96 @@ type pendingFlushResult struct { LastError string } +type pendingFlushCall struct { + done chan struct{} + result pendingFlushResult + err error +} + +var pendingFlushSingleflight = struct { + sync.Mutex + byAccount map[string]*pendingFlushCall +}{byAccount: make(map[string]*pendingFlushCall)} + +type pendingFlushSchedule struct { + first time.Time + timer *time.Timer +} + +var pendingFlushDebouncer = struct { + sync.Mutex + byAccount map[string]*pendingFlushSchedule +}{byAccount: make(map[string]*pendingFlushSchedule)} + +func (s Server) schedulePendingFlush(account string, logger *log.Logger) { + account = strings.TrimSpace(account) + if account == "" { + return + } + pendingFlushDebouncer.Lock() + if scheduled := pendingFlushDebouncer.byAccount[account]; scheduled != nil { + if time.Since(scheduled.first) < 250*time.Millisecond { + scheduled.timer.Reset(25 * time.Millisecond) + } + pendingFlushDebouncer.Unlock() + return + } + scheduled := &pendingFlushSchedule{first: time.Now()} + scheduled.timer = time.AfterFunc(25*time.Millisecond, func() { + pendingFlushDebouncer.Lock() + if pendingFlushDebouncer.byAccount[account] == scheduled { + delete(pendingFlushDebouncer.byAccount, account) + } + pendingFlushDebouncer.Unlock() + if _, err := s.flushPending(account, logger); err != nil { + logger.Printf("cmpp inbound event=scheduled_pending_flush_failed account=%s error=%q", account, err.Error()) + } + }) + pendingFlushDebouncer.byAccount[account] = scheduled + pendingFlushDebouncer.Unlock() +} + func (s Server) flushPending(account string, logger *log.Logger) (pendingFlushResult, error) { + account = strings.TrimSpace(account) result := pendingFlushResult{Account: account} if account == "" { return result, nil } + pendingFlushSingleflight.Lock() + if active := pendingFlushSingleflight.byAccount[account]; active != nil { + pendingFlushSingleflight.Unlock() + <-active.done + return active.result, active.err + } + call := &pendingFlushCall{done: make(chan struct{})} + pendingFlushSingleflight.byAccount[account] = call + pendingFlushSingleflight.Unlock() + call.result, call.err = s.flushPendingClaimed(account, logger) + pendingFlushSingleflight.Lock() + delete(pendingFlushSingleflight.byAccount, account) + close(call.done) + pendingFlushSingleflight.Unlock() + return call.result, call.err +} + +func (s Server) flushPendingClaimed(account string, logger *log.Logger) (pendingFlushResult, error) { + result := pendingFlushResult{Account: account} + claimID := fmt.Sprintf("%s:%s:%d", s.gatewayInstanceID(), account, time.Now().UnixNano()) var deliveries []pendingDelivery - if err := s.post(context.Background(), "/gateway/events/downstream/pending", pendingDeliveryRequest{Account: account, Limit: 100}, &deliveries); err != nil { + if err := s.post(context.Background(), "/gateway/events/downstream/pending", pendingDeliveryRequest{Account: account, Limit: 100, ClaimID: claimID, LeaseMS: 30000}, &deliveries); err != nil { logger.Printf("cmpp inbound pending delivery fetch failed account=%s err=%v", account, err) result.LastError = err.Error() return result, err } result.Deliveries = len(deliveries) for _, delivery := range deliveries { + delivery.ClaimID = defaultString(delivery.ClaimID, claimID) sendResult, err := s.pushPendingDelivery(account, delivery) if err != nil { result.FailedCount++ result.LastError = err.Error() _ = s.post(context.Background(), "/gateway/events/downstream/failed", map[string]any{ - "id": delivery.ID, "errorMessage": err.Error(), "failureType": "send_failed", + "id": delivery.ID, "claimId": delivery.ClaimID, "errorMessage": err.Error(), "failureType": "send_failed", "connectionId": sendResult.ConnectionID, "sequenceId": sendResult.SequenceID, "messageId": sendResult.MessageID, "sentAt": sendResult.SentAt, }, nil) @@ -63,6 +138,7 @@ func (s Server) flushPending(account string, logger *log.Logger) (pendingFlushRe } if sendResult.ReasonCode == "SUBMIT_RESPONSE_PENDING" { result.WaitingCount++ + _ = s.releasePendingClaim(delivery, sendResult) continue } errorMessage := defaultString(sendResult.ErrorMessage, "gateway did not complete downstream delivery") @@ -74,8 +150,12 @@ func (s Server) flushPending(account string, logger *log.Logger) (pendingFlushRe result.FailedCount++ } result.LastError = errorMessage + if sendResult.Retryable { + _ = s.releasePendingClaim(delivery, sendResult) + continue + } _ = s.post(context.Background(), "/gateway/events/downstream/failed", map[string]any{ - "id": delivery.ID, "errorMessage": errorMessageWithCode(errorMessage, sendResult.ReasonCode), + "id": delivery.ID, "claimId": delivery.ClaimID, "errorMessage": errorMessageWithCode(errorMessage, sendResult.ReasonCode), "failureType": failureType, "connectionId": sendResult.ConnectionID, "sequenceId": sendResult.SequenceID, "messageId": sendResult.MessageID, "sentAt": sendResult.SentAt, }, nil) @@ -83,6 +163,14 @@ func (s Server) flushPending(account string, logger *log.Logger) (pendingFlushRe return result, nil } +func (s Server) releasePendingClaim(delivery pendingDelivery, sendResult DownstreamSendResult) error { + return s.post(context.Background(), "/gateway/events/downstream/failed", map[string]any{ + "id": delivery.ID, "claimId": delivery.ClaimID, + "errorMessage": errorMessageWithCode(defaultString(sendResult.ErrorMessage, "Gateway released downstream delivery claim"), sendResult.ReasonCode), + "failureType": "claim_released", + }, nil) +} + func (s Server) pushPendingDelivery(account string, delivery pendingDelivery) (DownstreamSendResult, error) { switch delivery.DeliveryType { case "receipt": @@ -91,6 +179,7 @@ func (s Server) pushPendingDelivery(account string, delivery pendingDelivery) (D return DownstreamSendResult{}, err } event.DeliveryID = delivery.ID + event.ClaimID = delivery.ClaimID event.Account = defaultString(event.Account, account) allowRecovery := !delivery.CreatedAt.IsZero() && time.Since(delivery.CreatedAt) >= 5*time.Second return pushReceiptWithResult(event, allowRecovery) @@ -100,6 +189,7 @@ func (s Server) pushPendingDelivery(account string, delivery pendingDelivery) (D return DownstreamSendResult{}, err } event.DeliveryID = delivery.ID + event.ClaimID = delivery.ClaimID event.Account = defaultString(event.Account, account) return PushUplinkWithResult(event) default: diff --git a/gateway/internal/inbound/protocol_log.go b/gateway/internal/inbound/protocol_log.go index ce91900..396e172 100644 --- a/gateway/internal/inbound/protocol_log.go +++ b/gateway/internal/inbound/protocol_log.go @@ -1,6 +1,7 @@ package inbound import ( + "cmpp-platform/gateway/internal/protocollog" "context" "fmt" cmpp "github.com/bigwhite/gocmpp" @@ -8,20 +9,7 @@ import ( "strconv" ) -type protocolLogEvent struct { - Protocol string `json:"protocol"` - Direction string `json:"direction"` - EventType string `json:"eventType"` - Status string `json:"status"` - TenantID string `json:"tenantId,omitempty"` - ApplicationID string `json:"applicationId,omitempty"` - Account string `json:"account,omitempty"` - MessageID string `json:"messageId,omitempty"` - GatewayMessageID string `json:"gatewayMessageId,omitempty"` - Phone string `json:"phone,omitempty"` - ResultCode string `json:"resultCode,omitempty"` - Detail map[string]any `json:"detail,omitempty"` -} +type protocolLogEvent = protocollog.Event func (s Server) submitResponseProtocolLogger( account string, @@ -71,6 +59,15 @@ func protocolSubmitResponseDetail(sequenceID uint32, sendErr error) map[string]a } func (s Server) emitProtocolLog(event protocolLogEvent) { + event.GatewayInstanceID = s.GatewayInstanceID + if s.ProtocolLogPublisher != nil { + go func() { + if err := s.ProtocolLogPublisher.Publish(context.Background(), event); err != nil { + log.Printf("cmpp inbound protocol log Redis publish failed account=%s message_id=%s error=%q", event.Account, event.MessageID, err) + } + }() + return + } go func() { ctx, cancel := context.WithTimeout(context.Background(), defaultHTTPTimeout) defer cancel() diff --git a/gateway/internal/inbound/server.go b/gateway/internal/inbound/server.go index 78b872c..489dca7 100644 --- a/gateway/internal/inbound/server.go +++ b/gateway/internal/inbound/server.go @@ -1,6 +1,8 @@ package inbound import ( + "cmpp-platform/gateway/internal/protocollog" + "context" cmpp "github.com/bigwhite/gocmpp" "io" "log" @@ -22,6 +24,9 @@ type Server struct { RecoveryStore RecoveryStore GatewayInstanceID string MaxSubmitConcurrency int + ProtocolLogPublisher interface { + Publish(context.Context, protocollog.Event) error + } } func (s Server) ListenAndServe() error { diff --git a/gateway/internal/inbound/server_test.go b/gateway/internal/inbound/server_test.go index 0175855..2487602 100644 --- a/gateway/internal/inbound/server_test.go +++ b/gateway/internal/inbound/server_test.go @@ -930,6 +930,42 @@ func TestFlushOnlineAccountsFetchesPendingDeliveries(t *testing.T) { } } +func TestFlushPendingCoalescesConcurrentRequestsPerAccount(t *testing.T) { + var calls atomic.Int32 + release := make(chan struct{}) + api := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/api/gateway/events/downstream/pending" { + t.Fatalf("unexpected api path: %s", r.URL.Path) + } + calls.Add(1) + <-release + _ = json.NewEncoder(w).Encode([]pendingDelivery{}) + })) + defer api.Close() + + server := Server{APIBaseURL: api.URL + "/api", GatewayInstanceID: "gateway-a"} + results := make(chan error, 8) + for range 8 { + go func() { + _, err := server.flushPending("100001", log.Default()) + results <- err + }() + } + deadline := time.Now().Add(time.Second) + for calls.Load() == 0 && time.Now().Before(deadline) { + time.Sleep(time.Millisecond) + } + close(release) + for range 8 { + if err := <-results; err != nil { + t.Fatalf("flush failed: %v", err) + } + } + if got := calls.Load(); got != 1 { + t.Fatalf("pending fetch calls = %d, want 1", got) + } +} + func TestRecoverPendingCandidatesFetchesPresenceAccounts(t *testing.T) { resetDownstreamRegistry() defer resetDownstreamRegistry() @@ -1085,7 +1121,7 @@ func TestDownstreamDeliveryRequiresAcknowledgement(t *testing.T) { messageID: "MSG-LONG-1", phoneNumber: "18821203795", protocolLog: func(event protocolLogEvent) { protocolEvents <- event }, } - registerDownstreamAck(session, "delivery-1", 37, 9016479179509871733, time.Now().Add(time.Second)) + registerDownstreamAck(session, "delivery-1", "", 37, 9016479179509871733, time.Now().Add(time.Second)) handleDownstreamAcknowledgement(conn, 37, 9016479179509871733, 0, log.Default()) select { @@ -1165,6 +1201,7 @@ func TestSendDownstreamRejectsZeroMessageID(t *testing.T) { &downstreamSession{mu: &sync.Mutex{}}, &cmpp.Cmpp2DeliverReqPkt{MsgId: 0}, "delivery-zero", + "", ) if err == nil || !strings.Contains(err.Error(), "Msg_Id=0") { t.Fatalf("expected zero Msg_Id rejection, got %v", err) @@ -1180,7 +1217,7 @@ func TestDownstreamDeliveryReportsAckTimeout(t *testing.T) { conn: &cmpp.Conn{}, connectionID: "conn-1", deliveryReport: func(event downstreamDeliveryLifecycleEvent) { events <- event }, } - registerDownstreamAck(session, "delivery-timeout", 38, 9017467844344255865, time.Now().Add(20*time.Millisecond)) + registerDownstreamAck(session, "delivery-timeout", "", 38, 9017467844344255865, time.Now().Add(20*time.Millisecond)) select { case event := <-events: diff --git a/gateway/internal/inbound/submit.go b/gateway/internal/inbound/submit.go index a7c09ff..9fb54f7 100644 --- a/gateway/internal/inbound/submit.go +++ b/gateway/internal/inbound/submit.go @@ -212,11 +212,7 @@ func (s Server) handleSubmit(response *cmpp.Response, packet *cmpp.Packet, logge if sendErr != nil { return } - go func() { - if _, err := s.flushPending(account, logger); err != nil { - logger.Printf("cmpp inbound event=post_submit_pending_flush_failed account=%s message_id=%s error=%q", account, result.MessageID, err.Error()) - } - }() + s.schedulePendingFlush(account, logger) }) logger.Printf( "cmpp inbound event=submit_accepted protocol=%s packet_type=%s account=%s remote=%s seq=%d phone=%s dest_count=%d accepted_count=%d result=0 message_id=%s gateway_message_id=%d duration_ms=%d content_chars=%d content_hash=%s", diff --git a/gateway/internal/metrics/metrics.go b/gateway/internal/metrics/metrics.go index f3f2d2b..7a6d608 100644 --- a/gateway/internal/metrics/metrics.go +++ b/gateway/internal/metrics/metrics.go @@ -39,6 +39,8 @@ var submitStageHistograms [5][2]durationHistogram type Snapshot struct { UpstreamDesired int UpstreamConnected int + UpstreamWindowConfigured int + UpstreamWindowInFlight int DownstreamConnected int SubmitWorkerUp bool SubmitWorkerConcurrency int @@ -55,6 +57,14 @@ type Snapshot struct { ResultQueueAvailable bool ResultQueuePending int64 ResultQueueLag int64 + CallbackBatchRequests int64 + CallbackBatchEvents int64 + CallbackBatchRetries int64 + CallbackDeadLetters int64 + ProtocolLogPublished int64 + ProtocolLogSampled int64 + ProtocolLogErrors int64 + ProtocolLogQueueLength int64 } type SnapshotFunc func(context.Context) Snapshot @@ -121,6 +131,10 @@ func Handler(load SnapshotFunc) http.Handler { writeDurationHistogram(response, "cmpp_gateway_submit_stage_duration_seconds", stage, &submitStageHistograms[index]) } fmt.Fprintf(response, "# HELP cmpp_gateway_upstream_connections Desired and live supplier connections.\n# TYPE cmpp_gateway_upstream_connections gauge\ncmpp_gateway_upstream_connections{state=\"desired\"} %d\ncmpp_gateway_upstream_connections{state=\"connected\"} %d\n", snapshot.UpstreamDesired, snapshot.UpstreamConnected) + fmt.Fprintf(response, "# HELP cmpp_gateway_upstream_window_slots Configured and in-flight supplier window slots.\n# TYPE cmpp_gateway_upstream_window_slots gauge\ncmpp_gateway_upstream_window_slots{state=\"configured\"} %d\ncmpp_gateway_upstream_window_slots{state=\"in_flight\"} %d\n", snapshot.UpstreamWindowConfigured, snapshot.UpstreamWindowInFlight) + fmt.Fprintf(response, "# HELP cmpp_gateway_callback_batch_total Batch callback requests, events, retries and dead letters.\n# TYPE cmpp_gateway_callback_batch_total counter\ncmpp_gateway_callback_batch_total{result=\"requests\"} %d\ncmpp_gateway_callback_batch_total{result=\"events\"} %d\ncmpp_gateway_callback_batch_total{result=\"retries\"} %d\ncmpp_gateway_callback_batch_total{result=\"dead_letter\"} %d\n", snapshot.CallbackBatchRequests, snapshot.CallbackBatchEvents, snapshot.CallbackBatchRetries, snapshot.CallbackDeadLetters) + fmt.Fprintf(response, "# HELP cmpp_gateway_protocol_log_total Protocol log Stream outcomes.\n# TYPE cmpp_gateway_protocol_log_total counter\ncmpp_gateway_protocol_log_total{result=\"published\"} %d\ncmpp_gateway_protocol_log_total{result=\"sampled_out\"} %d\ncmpp_gateway_protocol_log_total{result=\"error\"} %d\n", snapshot.ProtocolLogPublished, snapshot.ProtocolLogSampled, snapshot.ProtocolLogErrors) + fmt.Fprintf(response, "# HELP cmpp_gateway_protocol_log_stream_length Current protocol log Stream length.\n# TYPE cmpp_gateway_protocol_log_stream_length gauge\ncmpp_gateway_protocol_log_stream_length %d\n", snapshot.ProtocolLogQueueLength) fmt.Fprintf(response, "# HELP cmpp_gateway_downstream_connections Authenticated client connections.\n# TYPE cmpp_gateway_downstream_connections gauge\ncmpp_gateway_downstream_connections %d\n", snapshot.DownstreamConnected) fmt.Fprintf(response, "# HELP cmpp_gateway_submit_worker_up Whether the submit worker was initialized.\n# TYPE cmpp_gateway_submit_worker_up gauge\ncmpp_gateway_submit_worker_up %d\n", boolNumber(snapshot.SubmitWorkerUp)) fmt.Fprintf(response, "# HELP cmpp_gateway_submit_worker_slots Configured and active bounded submit worker slots.\n# TYPE cmpp_gateway_submit_worker_slots gauge\ncmpp_gateway_submit_worker_slots{state=\"configured\"} %d\ncmpp_gateway_submit_worker_slots{state=\"in_flight\"} %d\n", snapshot.SubmitWorkerConcurrency, snapshot.SubmitWorkerInFlight) diff --git a/gateway/internal/protocollog/publisher.go b/gateway/internal/protocollog/publisher.go new file mode 100644 index 0000000..f2662b9 --- /dev/null +++ b/gateway/internal/protocollog/publisher.go @@ -0,0 +1,128 @@ +package protocollog + +import ( + "context" + "encoding/json" + "fmt" + "hash/fnv" + "strconv" + "strings" + "sync/atomic" + "time" + + "github.com/redis/go-redis/v9" +) + +const defaultStream = "gateway.protocol.logs" + +type Event struct { + EventID string `json:"eventId"` + GatewayInstanceID string `json:"gatewayInstanceId,omitempty"` + ConnectionID string `json:"connectionId,omitempty"` + Protocol string `json:"protocol"` + Direction string `json:"direction"` + EventType string `json:"eventType"` + Status string `json:"status"` + TenantID string `json:"tenantId,omitempty"` + ApplicationID string `json:"applicationId,omitempty"` + ChannelID string `json:"channelId,omitempty"` + Account string `json:"account,omitempty"` + MessageID string `json:"messageId,omitempty"` + SubmitID string `json:"submitId,omitempty"` + GatewayMessageID string `json:"gatewayMessageId,omitempty"` + Phone string `json:"phone,omitempty"` + ResultCode string `json:"resultCode,omitempty"` + DurationMs int `json:"durationMs,omitempty"` + PayloadBytes int `json:"payloadBytes,omitempty"` + Detail map[string]any `json:"detail,omitempty"` + CreatedAt time.Time `json:"createdAt"` +} + +type Publisher struct { + Redis *redis.Client + Stream string + GatewayInstanceID string + SuccessSampleRate int + MaxLen int64 + published atomic.Int64 + sampled atomic.Int64 + errors atomic.Int64 +} + +func New(redisURL string) (*Publisher, error) { + if strings.TrimSpace(redisURL) == "" { + redisURL = "redis://127.0.0.1:6379" + } + options, err := redis.ParseURL(redisURL) + if err != nil { + return nil, err + } + return &Publisher{Redis: redis.NewClient(options)}, nil +} + +func (p *Publisher) Publish(ctx context.Context, event Event) error { + if p == nil || p.Redis == nil { + return fmt.Errorf("protocol log Redis publisher is unavailable") + } + if event.CreatedAt.IsZero() { + event.CreatedAt = time.Now().UTC() + } + if event.GatewayInstanceID == "" { + event.GatewayInstanceID = p.GatewayInstanceID + } + if event.EventID == "" { + event.EventID = fmt.Sprintf("PL-%d-%d", event.CreatedAt.UnixNano(), p.published.Load()+p.sampled.Load()+1) + } + if !p.mustKeep(event) && !p.sample(event) { + p.sampled.Add(1) + return nil + } + data, err := json.Marshal(event) + if err != nil { + p.errors.Add(1) + return err + } + args := &redis.XAddArgs{Stream: p.stream(), Values: map[string]any{"data": string(data)}} + if p.MaxLen > 0 { + args.MaxLen = p.MaxLen + args.Approx = true + } + if err := p.Redis.XAdd(ctx, args).Err(); err != nil { + p.errors.Add(1) + return err + } + p.published.Add(1) + return nil +} + +func (p *Publisher) mustKeep(event Event) bool { + if event.Status != "success" { + return true + } + code := strings.TrimSpace(event.ResultCode) + return code != "" && code != "0" +} + +func (p *Publisher) sample(event Event) bool { + rate := p.SuccessSampleRate + if rate <= 0 { + rate = 10 + } + if rate >= 100 { + return true + } + h := fnv.New32a() + _, _ = h.Write([]byte(event.ChannelID + "|" + event.MessageID + "|" + event.EventType + "|" + strconv.FormatInt(event.CreatedAt.UnixNano()/int64(time.Millisecond), 10))) + return int(h.Sum32()%100) < rate +} + +func (p *Publisher) stream() string { + if strings.TrimSpace(p.Stream) != "" { + return p.Stream + } + return defaultStream +} +func (p *Publisher) StreamName() string { return p.stream() } +func (p *Publisher) Counts() (int64, int64, int64) { + return p.published.Load(), p.sampled.Load(), p.errors.Load() +} diff --git a/gateway/internal/protocollog/publisher_test.go b/gateway/internal/protocollog/publisher_test.go new file mode 100644 index 0000000..b3fa0ce --- /dev/null +++ b/gateway/internal/protocollog/publisher_test.go @@ -0,0 +1,34 @@ +package protocollog + +import ( + "context" + "testing" + + "github.com/alicebob/miniredis/v2" + "github.com/redis/go-redis/v9" +) + +func TestNewUsesLocalRedisWhenURLIsEmpty(t *testing.T) { + publisher, err := New("") + if err != nil { + t.Fatalf("New returned error: %v", err) + } + if publisher == nil || publisher.Redis == nil { + t.Fatal("expected an initialized Redis client") + } + if got := publisher.Redis.Options().Addr; got != "127.0.0.1:6379" { + t.Fatalf("expected local Redis fallback, got %q", got) + } +} + +func TestFailuresAreNeverSampledOut(t *testing.T) { + mr := miniredis.RunT(t) + client := redis.NewClient(&redis.Options{Addr: mr.Addr()}) + p := &Publisher{Redis: client, SuccessSampleRate: 1, MaxLen: 100} + if err := p.Publish(context.Background(), Event{Protocol: "cmpp", EventType: "submit", Status: "failed", ResultCode: "TIMEOUT"}); err != nil { + t.Fatal(err) + } + if client.XLen(context.Background(), p.StreamName()).Val() != 1 { + t.Fatal("failed protocol event must be retained") + } +} diff --git a/gateway/internal/queue/messages.go b/gateway/internal/queue/messages.go index 62d48dd..05f3180 100644 --- a/gateway/internal/queue/messages.go +++ b/gateway/internal/queue/messages.go @@ -70,6 +70,10 @@ type UpstreamConfig struct { WindowSize int `json:"windowSize,omitempty"` HeartbeatIntervalSeconds int `json:"heartbeatIntervalSeconds,omitempty"` HeartbeatMissThreshold int `json:"heartbeatMissThreshold,omitempty"` + ConnectionWarmupSeconds int `json:"connectionWarmupSeconds,omitempty"` + ConnectionDrainSeconds int `json:"connectionDrainTimeoutSeconds,omitempty"` + SubmitTimeoutSeconds int `json:"submitResponseTimeoutSeconds,omitempty"` + FailureCooldownSeconds int `json:"connectionFailureCooldownSeconds,omitempty"` } type Retry struct { @@ -146,6 +150,10 @@ type ConnectChannelConfig struct { WindowSize int `json:"windowSize,omitempty"` HeartbeatIntervalSeconds int `json:"heartbeatIntervalSeconds,omitempty"` HeartbeatMissThreshold int `json:"heartbeatMissThreshold,omitempty"` + ConnectionWarmupSeconds int `json:"connectionWarmupSeconds,omitempty"` + ConnectionDrainSeconds int `json:"connectionDrainTimeoutSeconds,omitempty"` + SubmitTimeoutSeconds int `json:"submitResponseTimeoutSeconds,omitempty"` + FailureCooldownSeconds int `json:"connectionFailureCooldownSeconds,omitempty"` } type DisconnectChannelCommand struct { diff --git a/gateway/internal/resultoutbox/batch.go b/gateway/internal/resultoutbox/batch.go new file mode 100644 index 0000000..373964c --- /dev/null +++ b/gateway/internal/resultoutbox/batch.go @@ -0,0 +1,170 @@ +package resultoutbox + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "strings" + "time" + + "github.com/redis/go-redis/v9" +) + +type callbackBatch struct { + BatchID string `json:"batchId"` + GatewayInstanceID string `json:"gatewayInstanceId"` + CreatedAt time.Time `json:"createdAt"` + Events []callbackBatchEvent `json:"events"` +} +type callbackBatchEvent struct { + EventID string `json:"eventId"` + Type string `json:"type"` + Payload json.RawMessage `json:"payload"` +} +type callbackBatchResponse struct { + BatchID string `json:"batchId"` + Results []callbackEventResult `json:"results"` +} +type callbackEventResult struct { + EventID string `json:"eventId"` + Accepted bool `json:"accepted"` + Retryable bool `json:"retryable,omitempty"` + ErrorCode string `json:"errorCode,omitempty"` +} + +func (o *Outbox) runBatches(ctx context.Context) error { + for ctx.Err() == nil { + messages, _, err := o.Redis.XAutoClaim(ctx, &redis.XAutoClaimArgs{Stream: o.stream(), Group: o.group(), Consumer: o.consumer(), MinIdle: o.minIdle(), Start: "0-0", Count: int64(o.batchSize())}).Result() + if err != nil && !errors.Is(err, redis.Nil) { + return err + } + if len(messages) == 0 { + streams, readErr := o.Redis.XReadGroup(ctx, &redis.XReadGroupArgs{Group: o.group(), Consumer: o.consumer(), Streams: []string{o.stream(), ">"}, Count: int64(o.batchSize()), Block: o.batchWait()}).Result() + if errors.Is(readErr, redis.Nil) { + continue + } + if readErr != nil { + return readErr + } + for _, stream := range streams { + messages = append(messages, stream.Messages...) + } + } + if len(messages) == 0 { + continue + } + if err := o.processBatch(ctx, messages); err != nil { + o.batchRetries.Add(int64(len(messages))) + sleep(ctx, 100*time.Millisecond) + } + } + return ctx.Err() +} + +func (o *Outbox) processBatch(ctx context.Context, messages []redis.XMessage) error { + batch := callbackBatch{BatchID: fmt.Sprintf("CB-%d", time.Now().UnixNano()), GatewayInstanceID: o.GatewayInstanceID, CreatedAt: time.Now().UTC()} + byEvent := make(map[string]redis.XMessage, len(messages)) + for _, message := range messages { + event, err := EventFromStreamValues(message.Values) + if err != nil { + _ = o.deadLetter(ctx, message, "INVALID_ENVELOPE", err.Error()) + continue + } + batch.Events = append(batch.Events, callbackBatchEvent{EventID: event.EventID, Type: event.EventType, Payload: event.Payload}) + byEvent[event.EventID] = message + } + if len(batch.Events) == 0 { + return nil + } + body, err := json.Marshal(batch) + if err != nil { + return err + } + if len(body) > 1024*1024 { + return fmt.Errorf("callback batch exceeds 1MB") + } + req, err := http.NewRequestWithContext(ctx, http.MethodPost, strings.TrimRight(o.APIBaseURL, "/")+"/gateway/events/batch", bytes.NewReader(body)) + if err != nil { + return err + } + req.Header.Set("Content-Type", "application/json") + client := &http.Client{Timeout: o.httpTimeout()} + resp, err := client.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + data, _ := io.ReadAll(io.LimitReader(resp.Body, 512)) + return fmt.Errorf("batch callback returned %d: %s", resp.StatusCode, strings.TrimSpace(string(data))) + } + var result callbackBatchResponse + if err := json.NewDecoder(io.LimitReader(resp.Body, 1024*1024)).Decode(&result); err != nil { + return err + } + if result.BatchID != batch.BatchID { + return fmt.Errorf("callback batchId mismatch") + } + o.batchRequests.Add(1) + o.batchEvents.Add(int64(len(batch.Events))) + seen := make(map[string]struct{}, len(result.Results)) + for _, item := range result.Results { + message, ok := byEvent[item.EventID] + if !ok { + continue + } + seen[item.EventID] = struct{}{} + if item.Accepted { + if err := o.ackDelete(ctx, message.ID); err != nil { + return err + } + continue + } + if !item.Retryable { + if err := o.deadLetter(ctx, message, item.ErrorCode, "non-retryable callback result"); err != nil { + return err + } + } + } + for eventID := range byEvent { + if _, ok := seen[eventID]; !ok { + return fmt.Errorf("batch response omitted event %s", eventID) + } + } + return nil +} + +func (o *Outbox) ackDelete(ctx context.Context, id string) error { + return acknowledgeAndDeleteScript.Run(ctx, o.Redis, []string{o.stream()}, o.group(), id).Err() +} +func (o *Outbox) deadLetter(ctx context.Context, message redis.XMessage, code, detail string) error { + stream := o.DeadLetterStream + if stream == "" { + stream = o.stream() + ".dead" + } + pipe := o.Redis.TxPipeline() + pipe.XAdd(ctx, &redis.XAddArgs{Stream: stream, Values: map[string]any{"sourceId": message.ID, "errorCode": code, "detail": detail, "data": fmt.Sprint(message.Values["data"])}}) + pipe.XAck(ctx, o.stream(), o.group(), message.ID) + pipe.XDel(ctx, o.stream(), message.ID) + if _, err := pipe.Exec(ctx); err != nil { + return err + } + o.deadLetters.Add(1) + return nil +} +func (o *Outbox) batchSize() int { + if o.BatchSize < 1 { + return 50 + } + return min(o.BatchSize, 100) +} +func (o *Outbox) batchWait() time.Duration { + if o.BatchWait <= 0 { + return 10 * time.Millisecond + } + return o.BatchWait +} diff --git a/gateway/internal/resultoutbox/outbox.go b/gateway/internal/resultoutbox/outbox.go index 5e90dbb..9a5e285 100644 --- a/gateway/internal/resultoutbox/outbox.go +++ b/gateway/internal/resultoutbox/outbox.go @@ -53,16 +53,25 @@ type Event struct { } type Outbox struct { - Redis *redis.Client - Stream string - Group string - Consumer string - DedupeTTL time.Duration - APIBaseURL string - HTTPTimeout time.Duration - Concurrency int - MinIdle time.Duration - inFlight atomic.Int64 + Redis *redis.Client + Stream string + Group string + Consumer string + DedupeTTL time.Duration + APIBaseURL string + HTTPTimeout time.Duration + Concurrency int + MinIdle time.Duration + BatchEnabled bool + BatchSize int + BatchWait time.Duration + GatewayInstanceID string + DeadLetterStream string + inFlight atomic.Int64 + batchRequests atomic.Int64 + batchEvents atomic.Int64 + batchRetries atomic.Int64 + deadLetters atomic.Int64 } func New(client *redis.Client) *Outbox { @@ -145,6 +154,18 @@ func (o *Outbox) PublishSubmitResult(ctx context.Context, command queue.SubmitCo return o.publish(ctx, event) } +func (o *Outbox) PublishReceipt(ctx context.Context, event queue.ReceiptEvent) error { + return o.publishRaw(ctx, Event{SchemaVersion: queue.SchemaVersion, EventID: fmt.Sprintf("receipt:%s:%s:%d", event.GatewayMessageID, event.RawStatus, event.SequenceID), EventType: "receipt_intake", Path: "/gateway/events/receipt/intake", TraceID: event.TraceID, MessageID: event.MessageID, ChannelID: event.ChannelID, Payload: mustMarshal(event), CreatedAt: time.Now().UTC()}) +} + +func (o *Outbox) PublishUplink(ctx context.Context, event queue.UplinkEvent) error { + return o.publishRaw(ctx, Event{SchemaVersion: queue.SchemaVersion, EventID: fmt.Sprintf("uplink:%s:%d:%d", event.ChannelID, event.SequenceID, event.ReceivedAt.UnixNano()), EventType: "uplink", Path: "/gateway/events/uplink", TraceID: event.TraceID, MessageID: event.MessageID, ChannelID: event.ChannelID, Payload: mustMarshal(event), CreatedAt: time.Now().UTC()}) +} + +func mustMarshal(value any) json.RawMessage { data, _ := json.Marshal(value); return data } + +func (o *Outbox) publishRaw(ctx context.Context, event Event) error { return o.publish(ctx, event) } + func (o *Outbox) publish(ctx context.Context, event Event) error { if o.Redis == nil { return fmt.Errorf("result Outbox Redis client is required") @@ -211,10 +232,10 @@ func EventFromStreamValues(values map[string]interface{}) (Event, error) { if err := json.Unmarshal([]byte(data), &event); err != nil { return Event{}, err } - if event.SchemaVersion != queue.SchemaVersion || event.EventID == "" || event.MessageID == "" || event.SubmitID == "" { + if event.SchemaVersion != queue.SchemaVersion || event.EventID == "" || event.MessageID == "" { return Event{}, fmt.Errorf("invalid result Outbox envelope") } - if event.Path != "/gateway/events/submit-result" && event.Path != "/gateway/events/submit-segment-result" { + if event.Path != "/gateway/events/submit-result" && event.Path != "/gateway/events/submit-segment-result" && event.Path != "/gateway/events/receipt/intake" && event.Path != "/gateway/events/uplink" && event.Path != "/gateway/events/dead-letter" { return Event{}, fmt.Errorf("unsupported result Outbox path %q", event.Path) } if len(event.Payload) == 0 { @@ -258,3 +279,6 @@ func (o *Outbox) dedupeKey(eventID string) string { func (o *Outbox) StreamName() string { return o.stream() } func (o *Outbox) GroupName() string { return o.group() } func (o *Outbox) InFlight() int64 { return o.inFlight.Load() } +func (o *Outbox) BatchCounts() (int64, int64, int64, int64) { + return o.batchRequests.Load(), o.batchEvents.Load(), o.batchRetries.Load(), o.deadLetters.Load() +} diff --git a/gateway/internal/resultoutbox/outbox_test.go b/gateway/internal/resultoutbox/outbox_test.go index eb73567..b7c4371 100644 --- a/gateway/internal/resultoutbox/outbox_test.go +++ b/gateway/internal/resultoutbox/outbox_test.go @@ -2,6 +2,7 @@ package resultoutbox import ( "context" + "encoding/json" "net/http" "net/http/httptest" "sync/atomic" @@ -32,6 +33,99 @@ func TestPublishSubmitSegmentIsIdempotent(t *testing.T) { } } +func TestBatchCallbackSendsMultipleEventsInOneRequest(t *testing.T) { + mr := miniredis.RunT(t) + client := redis.NewClient(&redis.Options{Addr: mr.Addr()}) + var requests atomic.Int32 + var eventCount atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) { + requests.Add(1) + var batch callbackBatch + if err := json.NewDecoder(request.Body).Decode(&batch); err != nil { + t.Errorf("decode batch: %v", err) + return + } + eventCount.Store(int32(len(batch.Events))) + result := callbackBatchResponse{BatchID: batch.BatchID} + for _, event := range batch.Events { + result.Results = append(result.Results, callbackEventResult{EventID: event.EventID, Accepted: true}) + } + _ = json.NewEncoder(response).Encode(result) + })) + defer server.Close() + outbox := New(client) + outbox.APIBaseURL = server.URL + outbox.BatchEnabled = true + outbox.BatchSize = 50 + outbox.BatchWait = 10 * time.Millisecond + outbox.GatewayInstanceID = "gateway-test" + command := testCommand() + for index := 1; index <= 2; index++ { + if err := outbox.PublishSubmitSegment(context.Background(), command, queue.SubmitSegmentResult{SegmentTotal: 2, SegmentIndex: index, SequenceID: uint32(index), GatewayMessageID: "88", SubmitStatus: "accepted"}); err != nil { + t.Fatal(err) + } + } + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { done <- outbox.Run(ctx) }() + deadline := time.Now().Add(3 * time.Second) + for client.XLen(context.Background(), outbox.StreamName()).Val() != 0 { + if time.Now().After(deadline) { + t.Fatal("batch did not drain") + } + time.Sleep(10 * time.Millisecond) + } + cancel() + <-done + if requests.Load() != 1 || eventCount.Load() != 2 { + t.Fatalf("requests/events=%d/%d, want 1/2", requests.Load(), eventCount.Load()) + } +} + +func TestBatchCallbackReplaysWholeRequestAfterHTTPFailure(t *testing.T) { + mr := miniredis.RunT(t) + client := redis.NewClient(&redis.Options{Addr: mr.Addr()}) + var calls atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) { + var batch callbackBatch + _ = json.NewDecoder(request.Body).Decode(&batch) + if calls.Add(1) == 1 { + http.Error(response, "busy", http.StatusServiceUnavailable) + return + } + result := callbackBatchResponse{BatchID: batch.BatchID} + for _, event := range batch.Events { + result.Results = append(result.Results, callbackEventResult{EventID: event.EventID, Accepted: true}) + } + _ = json.NewEncoder(response).Encode(result) + })) + defer server.Close() + outbox := New(client) + outbox.APIBaseURL = server.URL + outbox.BatchEnabled = true + outbox.BatchWait = 5 * time.Millisecond + outbox.MinIdle = 5 * time.Millisecond + outbox.GatewayInstanceID = "g" + if err := outbox.PublishSubmitSegment(context.Background(), testCommand(), queue.SubmitSegmentResult{SegmentTotal: 1, SegmentIndex: 1, SequenceID: 1, GatewayMessageID: "1", SubmitStatus: "accepted"}); err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { done <- outbox.Run(ctx) }() + deadline := time.Now().Add(3 * time.Second) + for client.XLen(context.Background(), outbox.StreamName()).Val() != 0 { + if time.Now().After(deadline) { + t.Fatal("replayed batch did not drain") + } + time.Sleep(10 * time.Millisecond) + } + cancel() + <-done + if calls.Load() < 2 { + t.Fatalf("calls=%d want replay", calls.Load()) + } +} + func TestPublishAggregateAndCommandAckAreAtomicAndIdempotent(t *testing.T) { mr := miniredis.RunT(t) client := redis.NewClient(&redis.Options{Addr: mr.Addr()}) diff --git a/gateway/internal/resultoutbox/worker.go b/gateway/internal/resultoutbox/worker.go index d0af11b..6e3b463 100644 --- a/gateway/internal/resultoutbox/worker.go +++ b/gateway/internal/resultoutbox/worker.go @@ -46,6 +46,9 @@ func (o *Outbox) Run(ctx context.Context) error { if err := o.ensureGroup(ctx); err != nil { return err } + if o.BatchEnabled { + return o.runBatches(ctx) + } pool := newCallbackPool(ctx, o, o.concurrency()) defer pool.wait() for { diff --git a/gateway/internal/upstream/connection.go b/gateway/internal/upstream/connection.go index 8e21bf1..b109c90 100644 --- a/gateway/internal/upstream/connection.go +++ b/gateway/internal/upstream/connection.go @@ -1,6 +1,7 @@ package upstream import ( + "cmpp-platform/gateway/internal/protocollog" "cmpp-platform/gateway/internal/queue" "context" "fmt" @@ -16,24 +17,38 @@ import ( // the connection must wake pending submitters before scheduling pool recovery. type connection struct { - channelID string - config queue.UpstreamConfig - index int - pool *connectionPool - apiBaseURL string - httpClient *http.Client + channelID string + config queue.UpstreamConfig + index int + pool *connectionPool + apiBaseURL string + httpClient *http.Client + protocolLogPublisher interface { + Publish(context.Context, protocollog.Event) error + } + gatewayInstanceID string + eventPublisher interface { + PublishReceipt(context.Context, queue.ReceiptEvent) error + PublishUplink(context.Context, queue.UplinkEvent) error + } - mu sync.Mutex - sendMu sync.Mutex - client *cmpp.Client - window chan struct{} - pending map[uint32]chan submitPartResponse - tracker map[uint64]queue.SubmitCommand - longUplink map[string]*longUplinkAssembly - readOnce sync.Once - closed bool - heartbeatCancel context.CancelFunc - heartbeatPending map[uint32]time.Time + mu sync.Mutex + sendMu sync.Mutex + client *cmpp.Client + window chan struct{} + windowLimit int + draining bool + retired bool + lastSubmitRTT time.Duration + consecutiveFailures int + cooldownUntil time.Time + pending map[uint32]chan submitPartResponse + tracker map[uint64]queue.SubmitCommand + longUplink map[string]*longUplinkAssembly + readOnce sync.Once + closed bool + heartbeatCancel context.CancelFunc + heartbeatPending map[uint32]time.Time } type submitPartResponse struct { @@ -161,6 +176,14 @@ func (c *connection) identity() string { return fmt.Sprintf("%s-%d", c.channelID, c.index) } +func (c *connection) retire() { + c.mu.Lock() + c.draining = true + c.retired = true + c.mu.Unlock() + c.handleConnectionLoss(fmt.Errorf("connection retired after drain")) +} + func (c *connection) close() { c.handleConnectionLoss(fmt.Errorf("connection closed")) } @@ -191,7 +214,7 @@ func (c *connection) handleConnectionLoss(err error) { default: } } - if c.pool != nil { + if c.pool != nil && !c.retired { status := "disconnected" if c.pool.countActiveConnections() > 0 { status = "reconnecting" diff --git a/gateway/internal/upstream/deliver.go b/gateway/internal/upstream/deliver.go index 9fbd573..683bb5f 100644 --- a/gateway/internal/upstream/deliver.go +++ b/gateway/internal/upstream/deliver.go @@ -88,7 +88,16 @@ func (c *connection) handleDeliver(pkt deliverPacket) error { DeliveredAt: time.Now().UTC(), ConnectionID: c.identity(), } - if err := postJSON(context.Background(), c.httpClient, c.apiBaseURL, "/gateway/events/receipt/intake", event); err != nil { + if c.protocolLogPublisher != nil { + c.emitProtocolLog(protocolLogEvent{Protocol: "cmpp", Direction: "channel_to_platform", EventType: "deliver_receipt", Status: "success", ChannelID: channelID, Account: c.config.Account, MessageID: messageID, GatewayMessageID: fmt.Sprint(receipt.MsgId), Phone: strings.TrimSpace(receipt.DestTerminalId), ResultCode: strings.TrimSpace(receipt.Stat), Detail: map[string]any{"sequenceId": pkt.seqID}}) + } + var publishErr error + if c.eventPublisher != nil { + publishErr = c.eventPublisher.PublishReceipt(context.Background(), event) + } else { + publishErr = postJSON(context.Background(), c.httpClient, c.apiBaseURL, "/gateway/events/receipt/intake", event) + } + if err := publishErr; err != nil { log.Printf("protocol_event protocol=cmpp direction=gateway_to_api event=deliver_receipt status=forward_failed channel_id=%s sequence_id=%d gateway_message_id=%d error=%q", c.channelID, pkt.seqID, receipt.MsgId, err) return err } else { @@ -121,7 +130,16 @@ func (c *connection) handleDeliver(pkt deliverPacket) error { Content: content, ReceivedAt: time.Now().UTC(), } - if err := postJSON(context.Background(), c.httpClient, c.apiBaseURL, "/gateway/events/uplink", event); err != nil { + if c.protocolLogPublisher != nil { + c.emitProtocolLog(protocolLogEvent{Protocol: "cmpp", Direction: "channel_to_platform", EventType: "deliver_uplink", Status: "success", ChannelID: c.channelID, Account: c.config.Account, MessageID: cmd.MessageID, Phone: strings.TrimSpace(pkt.srcTerminalID), Detail: map[string]any{"sequenceId": pkt.seqID}}) + } + var publishErr error + if c.eventPublisher != nil { + publishErr = c.eventPublisher.PublishUplink(context.Background(), event) + } else { + publishErr = postJSON(context.Background(), c.httpClient, c.apiBaseURL, "/gateway/events/uplink", event) + } + if err := publishErr; err != nil { log.Printf("protocol_event protocol=cmpp direction=gateway_to_api event=deliver_uplink status=forward_failed channel_id=%s sequence_id=%d packet_msg_id=%d error=%q", c.channelID, pkt.seqID, pkt.msgID, err) } else { log.Printf("protocol_event protocol=cmpp direction=gateway_to_api event=deliver_uplink status=forwarded channel_id=%s sequence_id=%d packet_msg_id=%d", c.channelID, pkt.seqID, pkt.msgID) diff --git a/gateway/internal/upstream/flow_control.go b/gateway/internal/upstream/flow_control.go index e4a40bf..5c9e9be 100644 --- a/gateway/internal/upstream/flow_control.go +++ b/gateway/internal/upstream/flow_control.go @@ -45,11 +45,20 @@ func (p *connectionPool) tryAcquireConnection() (*connection, func()) { if len(p.conns) == 0 { return nil, nil } - for i := 0; i < len(p.conns); i++ { - index := (p.next + i) % len(p.conns) - conn := p.conns[index] + bestIndex := -1 + bestInFlight := int(^uint(0) >> 1) + var bestRTT time.Duration + for index, conn := range p.conns { + inFlight, rtt, usable := conn.capacitySnapshot() + if !usable || inFlight > bestInFlight || (inFlight == bestInFlight && bestIndex >= 0 && bestRTT > 0 && rtt >= bestRTT) { + continue + } + bestIndex, bestInFlight, bestRTT = index, inFlight, rtt + } + if bestIndex >= 0 { + conn := p.conns[bestIndex] if conn.tryAcquireWindow() { - p.next = (index + 1) % len(p.conns) + p.next = (bestIndex + 1) % len(p.conns) return conn, conn.releaseWindow } } @@ -57,8 +66,17 @@ func (p *connectionPool) tryAcquireConnection() (*connection, func()) { } func (c *connection) tryAcquireWindow() bool { + c.mu.Lock() + defer c.mu.Unlock() if c.window == nil { - c.window = make(chan struct{}, defaultWindowSize) + c.window = make(chan struct{}, maximumWindowSize) + } + limit := c.windowLimit + if limit < 1 { + limit = defaultWindowSize + } + if c.closed || c.draining || c.client == nil || len(c.window) >= limit { + return false } select { case c.window <- struct{}{}: @@ -68,6 +86,37 @@ func (c *connection) tryAcquireWindow() bool { } } +func (c *connection) capacitySnapshot() (int, time.Duration, bool) { + c.mu.Lock() + defer c.mu.Unlock() + limit := c.windowLimit + if limit < 1 { + limit = defaultWindowSize + } + inFlight := len(c.window) + return inFlight, c.lastSubmitRTT, !c.closed && !c.draining && c.client != nil && inFlight < limit && !time.Now().Before(c.cooldownUntil) +} + +func (c *connection) markSubmitFailure() { + c.mu.Lock() + defer c.mu.Unlock() + c.consecutiveFailures++ + if c.consecutiveFailures >= 3 { + seconds := c.config.FailureCooldownSeconds + if seconds <= 0 { + seconds = 30 + } + c.cooldownUntil = time.Now().Add(time.Duration(seconds) * time.Second) + } +} + +func (c *connection) markSubmitSuccess() { + c.mu.Lock() + defer c.mu.Unlock() + c.consecutiveFailures = 0 + c.cooldownUntil = time.Time{} +} + func (c *connection) releaseWindow() { if c.window == nil { return diff --git a/gateway/internal/upstream/manager.go b/gateway/internal/upstream/manager.go index 78de777..8c60d64 100644 --- a/gateway/internal/upstream/manager.go +++ b/gateway/internal/upstream/manager.go @@ -1,6 +1,7 @@ package upstream import ( + "cmpp-platform/gateway/internal/protocollog" "cmpp-platform/gateway/internal/queue" "context" "fmt" @@ -19,6 +20,8 @@ const ( defaultReconnectInitialDelay = 5 * time.Second defaultReconnectMaximumDelay = 5 * time.Minute defaultAuthReconnectDelay = 5 * time.Minute + maximumConnections = 8 + maximumWindowSize = 64 ) type Manager struct { @@ -26,6 +29,14 @@ type Manager struct { EventAPIBaseURL string HTTPClient *http.Client SubmitSegmentPublisher SubmitSegmentPublisher + EventPublisher interface { + PublishReceipt(context.Context, queue.ReceiptEvent) error + PublishUplink(context.Context, queue.UplinkEvent) error + } + ProtocolLogPublisher interface { + Publish(context.Context, protocollog.Event) error + } + GatewayInstanceID string mu sync.Mutex conns map[string]*connectionPool @@ -72,8 +83,14 @@ func (m *Manager) ConnectChannel(ctx context.Context, command queue.ConnectChann WindowSize: command.Channel.WindowSize, HeartbeatIntervalSeconds: command.Channel.HeartbeatIntervalSeconds, HeartbeatMissThreshold: command.Channel.HeartbeatMissThreshold, + ConnectionWarmupSeconds: command.Channel.ConnectionWarmupSeconds, + ConnectionDrainSeconds: command.Channel.ConnectionDrainSeconds, + SubmitTimeoutSeconds: command.Channel.SubmitTimeoutSeconds, + FailureCooldownSeconds: command.Channel.FailureCooldownSeconds, }) - if pool == nil || !pool.matches(config) { + if pool != nil && !pool.matches(config) && pool.sameEndpoint(config) { + pool.reconfigure(config) + } else if pool == nil || !pool.matches(config) { if pool != nil { pool.close() } @@ -130,7 +147,9 @@ func (m *Manager) connectionFor(cmd queue.SubmitCommand) (*connectionPool, error m.ensureDefaultsLocked() pool := m.conns[cmd.ChannelID] - if pool == nil || !pool.matches(cmd.Upstream) { + if pool != nil && !pool.matches(cmd.Upstream) && pool.sameEndpoint(cmd.Upstream) { + pool.reconfigure(cmd.Upstream) + } else if pool == nil || !pool.matches(cmd.Upstream) { if pool != nil { pool.close() } @@ -170,6 +189,27 @@ func (m *Manager) ConnectionCounts() (desired int, connected int) { return desired, connected } +func (m *Manager) WindowCounts() (configured int, inFlight int) { + m.mu.Lock() + pools := make([]*connectionPool, 0, len(m.conns)) + for _, pool := range m.conns { + pools = append(pools, pool) + } + m.mu.Unlock() + for _, pool := range pools { + pool.mu.Lock() + conns := append([]*connection(nil), pool.conns...) + pool.mu.Unlock() + for _, conn := range conns { + conn.mu.Lock() + configured += max(1, conn.windowLimit) + inFlight += len(conn.window) + conn.mu.Unlock() + } + } + return configured, inFlight +} + func (m *Manager) newConnectionPool(channelID string, connectionID string, config queue.UpstreamConfig) *connectionPool { eventAPIBaseURL := m.EventAPIBaseURL if eventAPIBaseURL == "" { @@ -184,8 +224,11 @@ func (m *Manager) newConnectionPool(channelID string, connectionID string, confi reporter: func(ctx context.Context, state ConnectionState) error { return m.post(ctx, "/admin/gateway/connections", state) }, - reconnectSignal: make(chan struct{}, 1), - stopCh: make(chan struct{}), + protocolLogPublisher: m.ProtocolLogPublisher, + gatewayInstanceID: m.GatewayInstanceID, + eventPublisher: m.EventPublisher, + reconnectSignal: make(chan struct{}, 1), + stopCh: make(chan struct{}), } } @@ -202,6 +245,12 @@ func validateConnectChannelCommand(command queue.ConnectChannelCommand) error { if command.Channel.Account == "" || command.Channel.PasswordCipher == "" { return fmt.Errorf("account and passwordCipher are required") } + if command.DesiredConnections < 1 || command.DesiredConnections > maximumConnections { + return fmt.Errorf("desiredConnections must be between 1 and %d", maximumConnections) + } + if command.Channel.WindowSize != 0 && (command.Channel.WindowSize < 1 || command.Channel.WindowSize > maximumWindowSize) { + return fmt.Errorf("windowSize must be between 1 and %d", maximumWindowSize) + } return nil } @@ -222,5 +271,17 @@ func normalizeUpstreamConfig(config queue.UpstreamConfig) queue.UpstreamConfig { if config.HeartbeatMissThreshold <= 0 { config.HeartbeatMissThreshold = defaultHeartbeatMissThreshold } + if config.ConnectionWarmupSeconds < 0 { + config.ConnectionWarmupSeconds = 30 + } + if config.ConnectionDrainSeconds <= 0 { + config.ConnectionDrainSeconds = 60 + } + if config.SubmitTimeoutSeconds <= 0 { + config.SubmitTimeoutSeconds = 60 + } + if config.FailureCooldownSeconds <= 0 { + config.FailureCooldownSeconds = 30 + } return config } diff --git a/gateway/internal/upstream/pool.go b/gateway/internal/upstream/pool.go index 75f869b..ec3b886 100644 --- a/gateway/internal/upstream/pool.go +++ b/gateway/internal/upstream/pool.go @@ -1,6 +1,7 @@ package upstream import ( + "cmpp-platform/gateway/internal/protocollog" "cmpp-platform/gateway/internal/queue" "context" "net/http" @@ -9,12 +10,20 @@ import ( ) type connectionPool struct { - channelID string - connectionID string - config queue.UpstreamConfig - apiBaseURL string - httpClient *http.Client - reporter func(context.Context, ConnectionState) error + channelID string + connectionID string + config queue.UpstreamConfig + apiBaseURL string + httpClient *http.Client + reporter func(context.Context, ConnectionState) error + protocolLogPublisher interface { + Publish(context.Context, protocollog.Event) error + } + gatewayInstanceID string + eventPublisher interface { + PublishReceipt(context.Context, queue.ReceiptEvent) error + PublishUplink(context.Context, queue.UplinkEvent) error + } mu sync.Mutex connectMu sync.Mutex @@ -34,6 +43,103 @@ func (p *connectionPool) matches(config queue.UpstreamConfig) bool { return p.config == normalizeUpstreamConfig(config) } +func (p *connectionPool) sameEndpoint(config queue.UpstreamConfig) bool { + next := normalizeUpstreamConfig(config) + current := p.config + return current.GatewayHost == next.GatewayHost && current.GatewayPort == next.GatewayPort && + current.Account == next.Account && current.PasswordCipher == next.PasswordCipher && current.CMPPVersion == next.CMPPVersion +} + +// reconfigure changes only runtime capacity on the existing pool. Existing +// sequence mappings remain attached to their physical connection. Scale down +// marks surplus connections draining before closing them; scale up is serialized. +func (p *connectionPool) reconfigure(config queue.UpstreamConfig) { + next := normalizeUpstreamConfig(config) + p.mu.Lock() + p.config = next + for _, conn := range p.conns { + conn.mu.Lock() + conn.config = next + conn.windowLimit = next.WindowSize + conn.mu.Unlock() + } + p.mu.Unlock() + go p.reconcileCapacity() +} + +func (p *connectionPool) reconcileCapacity() { + p.connectMu.Lock() + defer p.connectMu.Unlock() + if p.stopped() { + return + } + + p.mu.Lock() + desired := max(1, min(maximumConnections, p.config.DesiredConnections)) + if len(p.conns) > desired { + retiring := append([]*connection(nil), p.conns[desired:]...) + p.conns = p.conns[:desired] + for _, conn := range retiring { + conn.mu.Lock() + conn.draining = true + conn.mu.Unlock() + } + p.mu.Unlock() + deadline := time.Now().Add(time.Duration(p.config.ConnectionDrainSeconds) * time.Second) + for _, conn := range retiring { + for len(conn.window) > 0 && time.Now().Before(deadline) { + time.Sleep(10 * time.Millisecond) + } + conn.retire() + } + _ = p.reportState(context.Background(), "connected", nil) + return + } + p.mu.Unlock() + + for { + p.mu.Lock() + if len(p.conns) >= desired || p.stopped() { + p.mu.Unlock() + break + } + index := len(p.conns) + config := p.config + p.mu.Unlock() + conn := p.newConnection(index, config) + if _, err := conn.ensureConnected(); err != nil { + p.scheduleReconnect(err) + _ = p.reportState(context.Background(), "failed", err) + return + } + p.mu.Lock() + p.conns = append(p.conns, conn) + p.mu.Unlock() + _ = p.reportState(context.Background(), "connected", nil) + if len(p.conns) < desired && config.ConnectionWarmupSeconds > 0 { + timer := time.NewTimer(time.Duration(config.ConnectionWarmupSeconds) * time.Second) + select { + case <-p.stopCh: + timer.Stop() + return + case <-timer.C: + } + } + } +} + +func (p *connectionPool) newConnection(index int, config queue.UpstreamConfig) *connection { + return &connection{ + channelID: p.channelID, config: config, index: index, pool: p, + apiBaseURL: p.apiBaseURL, httpClient: p.httpClient, + protocolLogPublisher: p.protocolLogPublisher, gatewayInstanceID: p.gatewayInstanceID, + eventPublisher: p.eventPublisher, + window: make(chan struct{}, maximumWindowSize), windowLimit: config.WindowSize, + pending: make(map[uint32]chan submitPartResponse), tracker: make(map[uint64]queue.SubmitCommand), + longUplink: make(map[string]*longUplinkAssembly), heartbeatPending: make(map[uint32]time.Time), + } +} + func (p *connectionPool) ensureConnected() error { p.connectMu.Lock() defer p.connectMu.Unlock() @@ -58,20 +164,12 @@ func (p *connectionPool) ensureConnected() error { p.mu.Unlock() break } - index := len(p.conns) - conn := &connection{ - channelID: p.channelID, - config: p.config, - index: index, - pool: p, - apiBaseURL: p.apiBaseURL, - httpClient: p.httpClient, - window: make(chan struct{}, p.config.WindowSize), - pending: make(map[uint32]chan submitPartResponse), - tracker: make(map[uint64]queue.SubmitCommand), - longUplink: make(map[string]*longUplinkAssembly), - heartbeatPending: make(map[uint32]time.Time), + if connectedAny { + p.mu.Unlock() + break } + index := len(p.conns) + conn := p.newConnection(index, p.config) p.mu.Unlock() connected, err := conn.ensureConnected() @@ -87,6 +185,7 @@ func (p *connectionPool) ensureConnected() error { if connectedAny { _ = p.reportState(context.Background(), "connected", nil) } + go p.reconcileCapacity() return nil } diff --git a/gateway/internal/upstream/pool_test.go b/gateway/internal/upstream/pool_test.go index 6d796eb..f3062c9 100644 --- a/gateway/internal/upstream/pool_test.go +++ b/gateway/internal/upstream/pool_test.go @@ -2,7 +2,9 @@ package upstream import ( "context" + cmpp "github.com/bigwhite/gocmpp" "testing" + "time" "cmpp-platform/gateway/internal/queue" ) @@ -10,8 +12,8 @@ import ( func TestConnectionPoolAcquiresAcrossConnections(t *testing.T) { pool := &connectionPool{ conns: []*connection{ - {window: make(chan struct{}, 1)}, - {window: make(chan struct{}, 1)}, + {client: &cmpp.Client{}, window: make(chan struct{}, 64), windowLimit: 1}, + {client: &cmpp.Client{}, window: make(chan struct{}, 64), windowLimit: 1}, }, } @@ -40,6 +42,51 @@ func TestConnectionPoolAcquiresAcrossConnections(t *testing.T) { releaseSecond() } +func TestPoolContinuesOnHealthyConnectionWhenPeerIsDraining(t *testing.T) { + draining := &connection{client: &cmpp.Client{}, window: make(chan struct{}, 64), windowLimit: 16, draining: true} + healthy := &connection{client: &cmpp.Client{}, window: make(chan struct{}, 64), windowLimit: 16} + pool := &connectionPool{conns: []*connection{draining, healthy}} + selected, release := pool.tryAcquireConnection() + if selected != healthy { + t.Fatal("expected healthy peer connection") + } + release() +} + +func TestRuntimeCapacityMatrixAndSmoothScaleDown(t *testing.T) { + for _, connections := range []int{1, 2, 4, 8} { + for _, window := range []int{1, 16, 32, 64} { + config := normalizeUpstreamConfig(queue.UpstreamConfig{GatewayHost: "127.0.0.1", GatewayPort: 17890, Account: "a", PasswordCipher: "p", CMPPVersion: "3.0", DesiredConnections: connections, WindowSize: window}) + if config.DesiredConnections != connections || config.WindowSize != window { + t.Fatalf("matrix normalized incorrectly: %+v", config) + } + } + } + pool := &connectionPool{channelID: "channel-1", connectionID: "primary", config: normalizeUpstreamConfig(queue.UpstreamConfig{DesiredConnections: 2, WindowSize: 16, ConnectionDrainSeconds: 1}), stopCh: make(chan struct{}), reconnectSignal: make(chan struct{}, 1)} + pool.conns = []*connection{{pool: pool, window: make(chan struct{}, 64), windowLimit: 16}, {pool: pool, window: make(chan struct{}, 64), windowLimit: 16}} + pool.reconfigure(queue.UpstreamConfig{DesiredConnections: 1, WindowSize: 32, ConnectionDrainSeconds: 1}) + deadline := time.Now().Add(time.Second) + for { + pool.mu.Lock() + count := len(pool.conns) + first := pool.conns[0] + pool.mu.Unlock() + if count == 1 { + first.mu.Lock() + limit := first.windowLimit + first.mu.Unlock() + if limit != 32 { + t.Fatalf("window limit=%d want32", limit) + } + break + } + if time.Now().After(deadline) { + t.Fatal("scale down did not complete") + } + time.Sleep(time.Millisecond) + } +} + func TestManagerDisconnectChannelRemovesPoolAndStopsReconnects(t *testing.T) { pool := &connectionPool{ channelID: "channel-1", diff --git a/gateway/internal/upstream/protocol_log.go b/gateway/internal/upstream/protocol_log.go index 6f1b78e..4445d7c 100644 --- a/gateway/internal/upstream/protocol_log.go +++ b/gateway/internal/upstream/protocol_log.go @@ -1,6 +1,7 @@ package upstream import ( + "cmpp-platform/gateway/internal/protocollog" "context" "fmt" cmpp "github.com/bigwhite/gocmpp" @@ -8,22 +9,7 @@ import ( "strings" ) -type protocolLogEvent struct { - Protocol string `json:"protocol"` - Direction string `json:"direction"` - EventType string `json:"eventType"` - Status string `json:"status"` - TenantID string `json:"tenantId,omitempty"` - ApplicationID string `json:"applicationId,omitempty"` - ChannelID string `json:"channelId,omitempty"` - Account string `json:"account,omitempty"` - MessageID string `json:"messageId,omitempty"` - GatewayMessageID string `json:"gatewayMessageId,omitempty"` - Phone string `json:"phone,omitempty"` - ResultCode string `json:"resultCode,omitempty"` - PayloadBytes int `json:"payloadBytes,omitempty"` - Detail map[string]any `json:"detail,omitempty"` -} +type protocolLogEvent = protocollog.Event func (c *connection) emitDeliverResponse(pkt deliverPacket, responseErr error) { status := "success" @@ -71,6 +57,16 @@ func (c *connection) emitDeliverResponse(pkt deliverPacket, responseErr error) { } func (c *connection) emitProtocolLog(event protocolLogEvent) { + event.ConnectionID = c.identity() + event.GatewayInstanceID = c.gatewayInstanceID + if c.protocolLogPublisher != nil { + go func() { + if err := c.protocolLogPublisher.Publish(context.Background(), event); err != nil { + log.Printf("protocol log Redis publish failed channel_id=%s message_id=%s error=%q", event.ChannelID, event.MessageID, err) + } + }() + return + } go func() { ctx, cancel := context.WithTimeout(context.Background(), defaultHTTPTimeout) defer cancel() diff --git a/gateway/internal/upstream/submit.go b/gateway/internal/upstream/submit.go index d2372fc..1383ae0 100644 --- a/gateway/internal/upstream/submit.go +++ b/gateway/internal/upstream/submit.go @@ -97,6 +97,12 @@ func (p *connectionPool) submit( } func (c *connection) submitPart(ctx context.Context, cmd queue.SubmitCommand, part submitPart) (uint32, string, queue.SubmitResult, error) { + startedAt := time.Now() + defer func() { + c.mu.Lock() + c.lastSubmitRTT = time.Since(startedAt) + c.mu.Unlock() + }() rspCh := make(chan submitPartResponse, 1) pkt := c.submitRequestPacket(cmd, part) @@ -163,14 +169,20 @@ func (c *connection) submitPart(ctx context.Context, cmd queue.SubmitCommand, pa c.mu.Unlock() }() - waitCtx, cancel := context.WithTimeout(ctx, defaultSubmitTimeout) + timeout := time.Duration(c.config.SubmitTimeoutSeconds) * time.Second + if timeout <= 0 { + timeout = defaultSubmitTimeout + } + waitCtx, cancel := context.WithTimeout(ctx, timeout) defer cancel() select { case <-waitCtx.Done(): + c.markSubmitFailure() result := submitResult(cmd, seq, "", "timeout", "SUBMIT_TIMEOUT", waitCtx.Err().Error()) return seq, "", result, waitCtx.Err() case rsp := <-rspCh: if rsp.err != nil { + c.markSubmitFailure() result := submitResult(cmd, seq, "", "timeout", "CONNECTION_LOST", rsp.err.Error()) return seq, "", result, rsp.err } @@ -203,6 +215,7 @@ func (c *connection) submitPart(ctx context.Context, cmd queue.SubmitCommand, pa }, }) if rsp.result == 0 { + c.markSubmitSuccess() c.mu.Lock() c.tracker[rsp.msgID] = cmd c.mu.Unlock() @@ -345,6 +358,12 @@ func validateSubmitCommand(cmd queue.SubmitCommand) error { if cmd.Upstream.Account == "" || cmd.Upstream.PasswordCipher == "" { return fmt.Errorf("upstream account and passwordCipher are required") } + if cmd.Upstream.DesiredConnections != 0 && (cmd.Upstream.DesiredConnections < 1 || cmd.Upstream.DesiredConnections > maximumConnections) { + return fmt.Errorf("upstream desiredConnections must be between 1 and %d", maximumConnections) + } + if cmd.Upstream.WindowSize != 0 && (cmd.Upstream.WindowSize < 1 || cmd.Upstream.WindowSize > maximumWindowSize) { + return fmt.Errorf("upstream windowSize must be between 1 and %d", maximumWindowSize) + } if len(cmd.PhoneNumber) == 0 || len(cmd.Content) == 0 { return fmt.Errorf("phoneNumber and content are required") } diff --git a/tools/deploy/production-deploy.sh b/tools/deploy/production-deploy.sh index a7fa762..cb043c9 100644 --- a/tools/deploy/production-deploy.sh +++ b/tools/deploy/production-deploy.sh @@ -61,6 +61,16 @@ if [[ "${GATEWAY_CALLBACK_SEPARATE_PROCESS_ENABLED:-false}" == "true" ]]; then exit 1 fi fi +if [[ "${GATEWAY_PROTOCOL_LOG_STREAM_ENABLED:-false}" == "true" && ! "${API_PROTOCOL_LOG_DB_POOL_MAX:-}" =~ ^[1-9][0-9]*$ ]]; then + echo "API_PROTOCOL_LOG_DB_POOL_MAX must be a positive integer when protocol log Stream writing is enabled." >&2 + exit 1 +fi +if [[ "${GATEWAY_CALLBACK_BATCH_ENABLED:-false}" == "true" ]]; then + if [[ ! "${GATEWAY_CALLBACK_BATCH_SIZE:-}" =~ ^[1-9][0-9]*$ || "${GATEWAY_CALLBACK_BATCH_SIZE}" -gt 100 ]]; then + echo "GATEWAY_CALLBACK_BATCH_SIZE must be between 1 and 100." >&2 + exit 1 + fi +fi if [[ -z "${CMPP_PUBLIC_HOST:-}" || ! "${CMPP_PUBLIC_PORT:-}" =~ ^[1-9][0-9]*$ ]]; then echo "CMPP_PUBLIC_HOST and a positive CMPP_PUBLIC_PORT are required in $ENV_FILE; these are the customer-facing CMPP endpoint." >&2 @@ -97,7 +107,7 @@ PROD_ADMIN_CREDENTIAL_FILE="$ADMIN_CREDENTIAL_FILE" node tools/deploy/ensure-pro chmod 600 "$ADMIN_CREDENTIAL_FILE" || true echo "[deploy] Ensuring runtime log directories" -install -d -m 0755 "$APP_DIR/logs/api" "$APP_DIR/logs/send-worker" "$APP_DIR/logs/submit-outbox" "$APP_DIR/logs/gateway-callback" "$APP_DIR/logs/gateway" +install -d -m 0755 "$APP_DIR/logs/api" "$APP_DIR/logs/send-worker" "$APP_DIR/logs/submit-outbox" "$APP_DIR/logs/gateway-callback" "$APP_DIR/logs/protocol-log-worker" "$APP_DIR/logs/gateway" echo "[deploy] Installing split API and send-worker services" node_bin="$(command -v node)" @@ -163,6 +173,26 @@ RestartSec=5 StandardOutput=append:$APP_DIR/logs/gateway-callback/stdout.log StandardError=append:$APP_DIR/logs/gateway-callback/stderr.log +[Install] +WantedBy=multi-user.target +EOF +cat >/etc/systemd/system/cmpp-protocol-log-worker.service </dev/null || true fi +if [[ "${GATEWAY_PROTOCOL_LOG_STREAM_ENABLED:-false}" == "true" ]]; then + systemctl enable --now cmpp-protocol-log-worker + systemctl restart cmpp-protocol-log-worker +else + systemctl disable --now cmpp-protocol-log-worker 2>/dev/null || true +fi # Gateway is restarted after the callback listener so supplier events never point # at a callback port that has not completed Nest/Prisma initialization. systemctl restart cmpp-gateway diff --git a/tools/security/install-security-agent.sh b/tools/security/install-security-agent.sh index c66794f..19951d8 100644 --- a/tools/security/install-security-agent.sh +++ b/tools/security/install-security-agent.sh @@ -16,6 +16,7 @@ install -d -o cmpp-api -g cmpp-security -m 0750 "$APP_DIR/logs/api" install -d -o cmpp-api -g cmpp-security -m 0750 "$APP_DIR/logs/send-worker" install -d -o cmpp-api -g cmpp-security -m 0750 "$APP_DIR/logs/submit-outbox" install -d -o cmpp-api -g cmpp-security -m 0750 "$APP_DIR/logs/gateway-callback" +install -d -o cmpp-api -g cmpp-security -m 0750 "$APP_DIR/logs/protocol-log-worker" [[ -d /var/lib/cmpp-platform/object-storage ]] && chown -R cmpp-api:cmpp-security /var/lib/cmpp-platform/object-storage sed "s#@CMPP_SECURITY_AGENT_BIN@#$agent_binary#g" "$APP_DIR/deploy/security/cmpp-report-only.conf" >/etc/fail2ban/action.d/cmpp-report-only.conf @@ -67,6 +68,15 @@ ProtectKernelModules=true ProtectControlGroups=true ReadWritePaths=$APP_DIR/logs/gateway-callback EOF +install -d -m 0755 /etc/systemd/system/cmpp-protocol-log-worker.service.d +cat >/etc/systemd/system/cmpp-protocol-log-worker.service.d/security-boundary.conf <>/etc/nftables.conf nft -c -f /etc/nftables.conf nft list table inet cmpp_security >/dev/null 2>&1 || nft -f /etc/nftables.d/cmpp-security.nft