perf: expand gateway capacity and prevent receipt replay

This commit is contained in:
hectorzhao
2026-08-25 16:08:30 +08:00
parent 761c123b65
commit 9292352be1
48 changed files with 2001 additions and 144 deletions
@@ -0,0 +1,3 @@
ALTER TABLE "SmsUplinkMessage" ADD COLUMN "eventId" TEXT;
CREATE UNIQUE INDEX "SmsUplinkMessage_eventId_key" ON "SmsUplinkMessage"("eventId");
+1
View File
@@ -2117,6 +2117,7 @@ model SmsReceiptAnomaly {
model SmsUplinkMessage { model SmsUplinkMessage {
id String @id @default(cuid()) id String @id @default(cuid())
eventId String? @unique
tenantId String? tenantId String?
applicationId String? applicationId String?
channelId String channelId String
@@ -367,6 +367,10 @@ export class ChannelConnectionService {
cmppVersion: channel.cmppVersion, cmppVersion: channel.cmppVersion,
rateLimitPerSecond: channel.rateLimitPerSecond, rateLimitPerSecond: channel.rateLimitPerSecond,
windowSize: getPositiveRuntimeInteger(getConfigValue(channel.config, 'windowSize'), 16, 'windowSize'), windowSize: getPositiveRuntimeInteger(getConfigValue(channel.config, 'windowSize'), 16, 'windowSize'),
connectionWarmupSeconds: Number(getConfigValue(channel.config, 'connectionWarmupSeconds') ?? 30),
connectionDrainTimeoutSeconds: getPositiveRuntimeInteger(getConfigValue(channel.config, 'connectionDrainTimeoutSeconds'), 60, 'connectionDrainTimeoutSeconds'),
submitResponseTimeoutSeconds: getPositiveRuntimeInteger(getConfigValue(channel.config, 'submitResponseTimeoutSeconds'), 60, 'submitResponseTimeoutSeconds'),
connectionFailureCooldownSeconds: getPositiveRuntimeInteger(getConfigValue(channel.config, 'connectionFailureCooldownSeconds'), 30, 'connectionFailureCooldownSeconds'),
heartbeatIntervalSeconds: getPositiveRuntimeInteger( heartbeatIntervalSeconds: getPositiveRuntimeInteger(
getConfigValue(channel.config, 'heartbeatIntervalSeconds'), getConfigValue(channel.config, 'heartbeatIntervalSeconds'),
DEFAULT_HEARTBEAT_INTERVAL_SECONDS, DEFAULT_HEARTBEAT_INTERVAL_SECONDS,
@@ -0,0 +1,14 @@
import { BadRequestException } from '@nestjs/common';
import { normalizeChannelRuntimeConfig } from './channels.helpers';
describe('Gateway channel capacity validation', () => {
it.each([1, 2, 4, 8])('accepts %i supplier connections', (desiredConnections) => {
expect(normalizeChannelRuntimeConfig(undefined, undefined, desiredConnections, 16)).toEqual(expect.objectContaining({ desiredConnections, windowSize: 16 }));
});
it.each([1, 16, 32, 64])('accepts supplier window %i', (windowSize) => {
expect(normalizeChannelRuntimeConfig(undefined, undefined, 1, windowSize)).toEqual(expect.objectContaining({ desiredConnections: 1, windowSize }));
});
it.each([[0, 16], [9, 16], [1, 0], [1, 65]])('rejects capacity outside 1..8 connections and 1..64 window', (connections, window) => {
expect(() => normalizeChannelRuntimeConfig(undefined, undefined, connections, window)).toThrow(BadRequestException);
});
});
+15 -3
View File
@@ -228,7 +228,7 @@ export function defaultChannelConnectionId(channelId: string) {
export function getDesiredConnections(config?: Prisma.JsonValue | null) { export function getDesiredConnections(config?: Prisma.JsonValue | null) {
if (config && typeof config === 'object' && !Array.isArray(config) && 'desiredConnections' in config) { if (config && typeof config === 'object' && !Array.isArray(config) && 'desiredConnections' in config) {
const value = Number(config.desiredConnections); const value = Number(config.desiredConnections);
if (Number.isInteger(value) && value > 0) { if (Number.isInteger(value) && value >= 1 && value <= 8) {
return value; return value;
} }
} }
@@ -327,8 +327,12 @@ export function normalizeChannelRuntimeConfig(
? incomingConfig ? incomingConfig
: {}; : {};
const base = { ...existing, ...incoming }; const base = { ...existing, ...incoming };
base.desiredConnections = getPositiveRuntimeInteger(desiredConnections ?? base.desiredConnections, 1, 'desiredConnections'); base.desiredConnections = boundedRuntimeInteger(desiredConnections ?? base.desiredConnections, 1, 8, 1, 'desiredConnections');
base.windowSize = getPositiveRuntimeInteger(windowSize ?? base.windowSize, 16, 'windowSize'); base.windowSize = boundedRuntimeInteger(windowSize ?? base.windowSize, 1, 64, 16, 'windowSize');
base.connectionWarmupSeconds = boundedRuntimeInteger(base.connectionWarmupSeconds, 0, 300, 30, 'connectionWarmupSeconds');
base.connectionDrainTimeoutSeconds = boundedRuntimeInteger(base.connectionDrainTimeoutSeconds, 1, 600, 60, 'connectionDrainTimeoutSeconds');
base.submitResponseTimeoutSeconds = boundedRuntimeInteger(base.submitResponseTimeoutSeconds, 1, 300, 60, 'submitResponseTimeoutSeconds');
base.connectionFailureCooldownSeconds = boundedRuntimeInteger(base.connectionFailureCooldownSeconds, 1, 300, 30, 'connectionFailureCooldownSeconds');
base.heartbeatIntervalSeconds = getPositiveRuntimeInteger( base.heartbeatIntervalSeconds = getPositiveRuntimeInteger(
heartbeatIntervalSeconds ?? base.heartbeatIntervalSeconds, heartbeatIntervalSeconds ?? base.heartbeatIntervalSeconds,
DEFAULT_HEARTBEAT_INTERVAL_SECONDS, DEFAULT_HEARTBEAT_INTERVAL_SECONDS,
@@ -345,6 +349,14 @@ export function normalizeChannelRuntimeConfig(
return base; return base;
} }
function boundedRuntimeInteger(value: unknown, minimum: number, maximum: number, fallback: number, field: string) {
const normalized = value === undefined || value === null || value === '' ? fallback : Number(value);
if (!Number.isInteger(normalized) || normalized < minimum || normalized > maximum) {
throw new BadRequestException(`${field} must be an integer between ${minimum} and ${maximum}`);
}
return normalized;
}
export function normalizeLongMessageReceiptMode(value: unknown) { export function normalizeLongMessageReceiptMode(value: unknown) {
const normalized = String(value ?? 'per_segment').trim() || 'per_segment'; const normalized = String(value ?? 'per_segment').trim() || 'per_segment';
if (!['per_segment', 'message_level'].includes(normalized)) { if (!['per_segment', 'message_level'].includes(normalized)) {
+8 -3
View File
@@ -14,14 +14,19 @@ export class PrismaService extends PrismaClient implements OnModuleDestroy {
const workerRole = processRole === 'worker'; const workerRole = processRole === 'worker';
const outboxRole = processRole === 'outbox'; const outboxRole = processRole === 'outbox';
const callbackRole = processRole === 'callback'; const callbackRole = processRole === 'callback';
const databaseUrl = outboxRole const protocolLogRole = processRole === 'protocol-log-worker';
const databaseUrl = protocolLogRole
? process.env.API_PROTOCOL_LOG_DATABASE_URL || process.env.DATABASE_URL
: outboxRole
? process.env.API_OUTBOX_DATABASE_URL || process.env.DATABASE_URL ? process.env.API_OUTBOX_DATABASE_URL || process.env.DATABASE_URL
: callbackRole : callbackRole
? process.env.API_CALLBACK_DATABASE_URL || process.env.DATABASE_URL ? process.env.API_CALLBACK_DATABASE_URL || process.env.DATABASE_URL
: workerRole : workerRole
? process.env.API_WORKER_DATABASE_URL || process.env.DATABASE_URL ? process.env.API_WORKER_DATABASE_URL || process.env.DATABASE_URL
: process.env.DATABASE_URL; : process.env.DATABASE_URL;
const configuredPoolMax = Number(outboxRole const configuredPoolMax = Number(protocolLogRole
? process.env.API_PROTOCOL_LOG_DB_POOL_MAX ?? 4
: outboxRole
? process.env.API_OUTBOX_DB_POOL_MAX ?? 6 ? process.env.API_OUTBOX_DB_POOL_MAX ?? 6
: callbackRole : callbackRole
? process.env.API_CALLBACK_DB_POOL_MAX ?? 16 ? process.env.API_CALLBACK_DB_POOL_MAX ?? 16
@@ -30,7 +35,7 @@ export class PrismaService extends PrismaClient implements OnModuleDestroy {
: process.env.API_DB_POOL_MAX ?? 32); : process.env.API_DB_POOL_MAX ?? 32);
const poolMax = Number.isInteger(configuredPoolMax) && configuredPoolMax > 0 const poolMax = Number.isInteger(configuredPoolMax) && configuredPoolMax > 0
? configuredPoolMax ? configuredPoolMax
: outboxRole ? 6 : callbackRole ? 16 : workerRole ? 8 : 32; : protocolLogRole ? 4 : outboxRole ? 6 : callbackRole ? 16 : workerRole ? 8 : 32;
const databasePool = new Pool({ const databasePool = new Pool({
connectionString: databaseUrl connectionString: databaseUrl
?? 'postgresql://cmpp:cmpp_password@localhost:5432/cmpp_platform?schema=public', ?? 'postgresql://cmpp:cmpp_password@localhost:5432/cmpp_platform?schema=public',
+9
View File
@@ -0,0 +1,9 @@
import { Module } from '@nestjs/common';
import { ConfigModule } from '@nestjs/config';
import { PrismaModule } from './prisma/prisma.module';
import { ProtocolLogsModule } from './protocol-logs/protocol-logs.module';
@Module({
imports: [ConfigModule.forRoot({ isGlobal: true, envFilePath: ['.env.local', '.env'] }), PrismaModule, ProtocolLogsModule],
})
export class ProtocolLogWorkerModule {}
+59
View File
@@ -0,0 +1,59 @@
import { NestFactory } from '@nestjs/core';
import IORedis from 'ioredis';
import { ProtocolLogWorkerModule } from './protocol-log-worker.module';
import { ProtocolLogsService, type ProtocolLogInput } from './protocol-logs/protocol-logs.service';
const STREAM = process.env.GATEWAY_PROTOCOL_LOG_STREAM ?? 'gateway.protocol.logs';
const GROUP = process.env.GATEWAY_PROTOCOL_LOG_GROUP ?? 'cmpp-protocol-log-writer';
const CONSUMER = process.env.GATEWAY_PROTOCOL_LOG_CONSUMER ?? `protocol-log-${process.pid}`;
const BATCH_SIZE = boundedEnv('PROTOCOL_LOG_STREAM_BATCH_SIZE', 250, 100, 500);
async function bootstrap() {
process.env.CMPP_PROCESS_ROLE = 'protocol-log-worker';
const app = await NestFactory.createApplicationContext(ProtocolLogWorkerModule, { logger: ['log', 'warn', 'error'] });
const logs = app.get(ProtocolLogsService);
const redis = new IORedis(process.env.REDIS_URL ?? 'redis://127.0.0.1:6379', { maxRetriesPerRequest: null });
try { await redis.xgroup('CREATE', STREAM, GROUP, '0', 'MKSTREAM'); } catch (error) {
if (!String(error).includes('BUSYGROUP')) throw error;
}
let stopping = false;
const stop = () => { stopping = true; };
process.on('SIGTERM', stop); process.on('SIGINT', stop);
while (!stopping) {
const claimed = await redis.xautoclaim(STREAM, GROUP, CONSUMER, 30_000, '0-0', 'COUNT', BATCH_SIZE) as unknown as [string, Array<[string, string[]]>];
let messages = claimed[1] ?? [];
if (!messages.length) {
const reply = await redis.xreadgroup('GROUP', GROUP, CONSUMER, 'COUNT', BATCH_SIZE, 'BLOCK', 2000, 'STREAMS', STREAM, '>') as unknown as Array<[string, Array<[string, string[]]>]> | null;
if (!reply) continue;
messages = reply[0]?.[1] ?? [];
}
const accepted: string[] = [];
const parsed: ProtocolLogInput[] = [];
for (const [id, fields] of messages) {
const dataIndex = fields.indexOf('data');
try {
if (dataIndex < 0) throw new Error('data field missing');
parsed.push(JSON.parse(fields[dataIndex + 1]) as ProtocolLogInput);
accepted.push(id);
} catch (error) {
console.error(`protocol log stream event ${id} is invalid`, error);
await redis.xadd(`${STREAM}.dead`, '*', 'sourceId', id, 'error', String(error), 'data', dataIndex >= 0 ? fields[dataIndex + 1] : '');
await redis.xack(STREAM, GROUP, id); await redis.xdel(STREAM, id);
}
}
logs.recordMany(parsed);
if (accepted.length && await logs.flushNow()) {
const pipeline = redis.pipeline();
for (const id of accepted) pipeline.xack(STREAM, GROUP, id).xdel(STREAM, id);
await pipeline.exec();
}
}
await logs.flushNow(); await redis.quit(); await app.close();
}
function boundedEnv(name: string, fallback: number, minimum: number, maximum: number) {
const value = Number(process.env[name] ?? fallback);
return Number.isInteger(value) && value >= minimum && value <= maximum ? value : fallback;
}
void bootstrap().catch((error) => { console.error(error); process.exitCode = 1; });
+29 -5
View File
@@ -21,6 +21,10 @@ export type ProtocolLogInput = {
payloadBytes?: number | null; payloadBytes?: number | null;
retryCount?: number | null; retryCount?: number | null;
detail?: Record<string, unknown> | null; detail?: Record<string, unknown> | null;
eventId?: string | null;
gatewayInstanceId?: string | null;
connectionId?: string | null;
submitId?: string | null;
}; };
export type ProtocolLogQuery = { export type ProtocolLogQuery = {
@@ -45,8 +49,10 @@ export class ProtocolLogsService implements OnModuleInit, OnModuleDestroy {
constructor(private readonly prisma: PrismaService) {} constructor(private readonly prisma: PrismaService) {}
onModuleInit() { onModuleInit() {
this.flushTimer = setInterval(() => void this.flush(), positiveEnv('PROTOCOL_LOG_FLUSH_INTERVAL_MS', 500)); if (process.env.CMPP_PROCESS_ROLE !== 'protocol-log-worker') {
this.flushTimer.unref?.(); this.flushTimer = setInterval(() => void this.flush(), positiveEnv('PROTOCOL_LOG_FLUSH_INTERVAL_MS', 500));
this.flushTimer.unref?.();
}
this.retentionTimer = setInterval(() => void this.purgeExpired(), positiveEnv('PROTOCOL_LOG_RETENTION_INTERVAL_MS', 86_400_000)); this.retentionTimer = setInterval(() => void this.purgeExpired(), positiveEnv('PROTOCOL_LOG_RETENTION_INTERVAL_MS', 86_400_000));
this.retentionTimer.unref?.(); this.retentionTimer.unref?.();
setTimeout(() => void this.purgeExpired(), 30_000).unref?.(); setTimeout(() => void this.purgeExpired(), 30_000).unref?.();
@@ -82,11 +88,25 @@ export class ProtocolLogsService implements OnModuleInit, OnModuleDestroy {
durationMs: safeInteger(input.durationMs), durationMs: safeInteger(input.durationMs),
payloadBytes: safeInteger(input.payloadBytes), payloadBytes: safeInteger(input.payloadBytes),
retryCount: safeInteger(input.retryCount), retryCount: safeInteger(input.retryCount),
detail: sanitizeDetail(input.detail), detail: sanitizeDetail({
...input.detail,
eventId: input.eventId,
gatewayInstanceId: input.gatewayInstanceId,
connectionId: input.connectionId,
submitId: input.submitId,
}),
}); });
if (this.buffer.length >= positiveEnv('PROTOCOL_LOG_BATCH_SIZE', 100)) void this.flush(); if (this.buffer.length >= positiveEnv('PROTOCOL_LOG_BATCH_SIZE', 100)) void this.flush();
} }
recordMany(inputs: ProtocolLogInput[]) {
for (const input of inputs) this.record(input);
}
flushNow() {
return this.flush();
}
async list(query: ProtocolLogQuery) { async list(query: ProtocolLogQuery) {
const page = Math.max(1, Number(query.page) || 1); const page = Math.max(1, Number(query.page) || 1);
const pageSize = Math.min(100, Math.max(1, Number(query.pageSize) || 20)); const pageSize = Math.min(100, Math.max(1, Number(query.pageSize) || 20));
@@ -119,18 +139,22 @@ export class ProtocolLogsService implements OnModuleInit, OnModuleDestroy {
return { items, total, page, pageSize, eventTypes: eventTypes.map((item) => item.eventType) }; return { items, total, page, pageSize, eventTypes: eventTypes.map((item) => item.eventType) };
} }
private async flush() { private async flush(): Promise<boolean> {
if (this.flushing || this.buffer.length === 0) return; if (this.flushing) return false;
if (this.buffer.length === 0) return true;
this.flushing = true; this.flushing = true;
const batch = this.buffer.splice(0, positiveEnv('PROTOCOL_LOG_BATCH_SIZE', 100)); const batch = this.buffer.splice(0, positiveEnv('PROTOCOL_LOG_BATCH_SIZE', 100));
try { try {
await this.prisma.protocolInteractionLog.createMany({ data: batch }); await this.prisma.protocolInteractionLog.createMany({ data: batch });
} catch (error) { } catch (error) {
this.buffer.unshift(...batch);
this.logger.error(`Protocol log batch write failed (${batch.length} events)`, error instanceof Error ? error.stack : String(error)); this.logger.error(`Protocol log batch write failed (${batch.length} events)`, error instanceof Error ? error.stack : String(error));
return false;
} finally { } finally {
this.flushing = false; this.flushing = false;
if (this.buffer.length > 0) setImmediate(() => void this.flush()); if (this.buffer.length > 0) setImmediate(() => void this.flush());
} }
return true;
} }
private async purgeExpired() { private async purgeExpired() {
@@ -11,7 +11,8 @@ describe('GatewayCallbackController', () => {
const prisma = { $queryRaw: jest.fn(), getPoolState: jest.fn().mockReturnValue({ max: 16, total: 2, idle: 1, waiting: 0 }) }; const prisma = { $queryRaw: jest.fn(), getPoolState: jest.fn().mockReturnValue({ max: 16, total: 2, idle: 1, waiting: 0 }) };
const controller = new GatewayCallbackController(sendChain as never, protocolLogs as never, prisma as never); const controller = new GatewayCallbackController(sendChain as never, protocolLogs as never, prisma as never);
beforeEach(() => jest.clearAllMocks()); beforeEach(() => { jest.clearAllMocks(); process.env.PROTOCOL_LOG_CALLBACK_TRACKING_ENABLED = 'true'; });
afterAll(() => { delete process.env.PROTOCOL_LOG_CALLBACK_TRACKING_ENABLED; });
it('keeps Submit result persistence on the callback process without duplicating protocol logs', async () => { it('keeps Submit result persistence on the callback process without duplicating protocol logs', async () => {
sendChain.handleSubmitResult.mockResolvedValue({ accepted: true }); sendChain.handleSubmitResult.mockResolvedValue({ accepted: true });
@@ -40,4 +41,20 @@ describe('GatewayCallbackController', () => {
protocol: 'cmpp', direction: 'client_to_platform', eventType: 'submit', status: 'success', protocol: 'cmpp', direction: 'client_to_platform', eventType: 'submit', status: 'success',
})).toThrow(BadRequestException); })).toThrow(BadRequestException);
}); });
it('returns an independent result for every event in a callback batch', async () => {
sendChain.handleSubmitResult.mockResolvedValue({ accepted: true });
sendChain.handleUplink.mockResolvedValue({ accepted: true });
await expect(controller.batch({
batchId: 'CB-1', gatewayInstanceId: 'gateway-1', events: [
{ eventId: 'EV-1', type: 'submit_result', payload: { messageId: 'MSG-1', channelId: 'channel-1', submitStatus: 'accepted' } },
{ eventId: 'EV-2', type: 'uplink', payload: { messageId: 'MSG-1', channelId: 'channel-1', content: '1' } },
{ eventId: 'EV-3', type: 'unsupported', payload: {} },
],
})).resolves.toEqual({ batchId: 'CB-1', results: [
{ eventId: 'EV-1', accepted: true },
{ eventId: 'EV-2', accepted: true },
{ eventId: 'EV-3', accepted: false, retryable: false, errorCode: 'INVALID_EVENT' },
] });
});
}); });
@@ -68,6 +68,51 @@ export class GatewayCallbackController {
return this.sendChain.recordGatewaySubmitDeadLetter(body); return this.sendChain.recordGatewaySubmitDeadLetter(body);
} }
@Post('gateway/events/batch')
async batch(@Body() body: {
batchId?: string;
gatewayInstanceId?: string;
createdAt?: string;
events?: Array<{ eventId?: string; type?: string; payload?: Record<string, unknown> }>;
}) {
if (!body.batchId || !body.gatewayInstanceId || !Array.isArray(body.events) || body.events.length < 1 || body.events.length > 100) {
throw new BadRequestException('batchId, gatewayInstanceId and 1 to 100 events are required');
}
if (Buffer.byteLength(JSON.stringify(body), 'utf8') > 1024 * 1024) {
throw new BadRequestException('Gateway callback batch exceeds 1MB');
}
const seen = new Set<string>();
const results: Array<{ eventId: string; accepted: boolean; retryable?: boolean; errorCode?: string }> = [];
for (let offset = 0; offset < body.events.length; offset += 25) {
results.push(...await Promise.all(body.events.slice(offset, offset + 25).map(async (event) => {
const eventId = String(event.eventId ?? '').trim();
if (!eventId || seen.has(eventId) || !event.payload || typeof event.payload !== 'object') {
return { eventId, accepted: false, retryable: false, errorCode: seen.has(eventId) ? 'DUPLICATE_EVENT_ID' : 'INVALID_EVENT' };
}
seen.add(eventId);
try {
await this.dispatchBatchEvent(String(event.type ?? ''), { ...event.payload, eventId });
return { eventId, accepted: true };
} catch (error) {
const invalid = error instanceof BadRequestException;
return { eventId, accepted: false, retryable: !invalid, errorCode: invalid ? 'INVALID_EVENT' : 'PROCESSING_FAILED' };
}
})));
}
return { batchId: body.batchId, results };
}
private dispatchBatchEvent(type: string, payload: Record<string, unknown>) {
switch (type) {
case 'submit_result': return this.sendChain.handleSubmitResult(payload as unknown as GatewaySubmitResultDto);
case 'submit_segment_result': return this.sendChain.handleSubmitSegmentResult(payload as unknown as GatewaySubmitSegmentResultDto);
case 'receipt_intake': return this.track('deliver_receipt', payload, () => this.sendChain.intakeReceipt(payload as unknown as GatewayReceiptEventDto));
case 'uplink': return this.track('deliver_uplink', payload, () => this.sendChain.handleUplink(payload as unknown as GatewayUplinkEventDto));
case 'dead_letter': return this.sendChain.recordGatewaySubmitDeadLetter(payload as unknown as GatewaySubmitDeadLetterDto);
default: throw new BadRequestException(`Unsupported batch event type ${type}`);
}
}
private async track<T>(eventType: string, body: object, action: () => Promise<T> | T) { private async track<T>(eventType: string, body: object, action: () => Promise<T> | T) {
const startedAt = Date.now(); const startedAt = Date.now();
const value = body as Record<string, unknown>; const value = body as Record<string, unknown>;
@@ -83,7 +128,7 @@ export class GatewayCallbackController {
try { try {
const result = await action(); const result = await action();
const resolved = result && typeof result === 'object' ? result as Record<string, unknown> : {}; const resolved = result && typeof result === 'object' ? result as Record<string, unknown> : {};
this.protocolLogs.record({ if (process.env.PROTOCOL_LOG_CALLBACK_TRACKING_ENABLED === 'true') this.protocolLogs.record({
...common, ...common,
tenantId: (resolved.tenantId ?? common.tenantId) as string, tenantId: (resolved.tenantId ?? common.tenantId) as string,
applicationId: (resolved.applicationId ?? common.applicationId) as string, applicationId: (resolved.applicationId ?? common.applicationId) as string,
@@ -92,7 +137,7 @@ export class GatewayCallbackController {
}); });
return result; return result;
} catch (error) { } catch (error) {
this.protocolLogs.record({ if (process.env.PROTOCOL_LOG_CALLBACK_TRACKING_ENABLED === 'true') this.protocolLogs.record({
...common, status: 'failed', durationMs: Date.now() - startedAt, ...common, status: 'failed', durationMs: Date.now() - startedAt,
detail: { error: error instanceof Error ? error.message : String(error) }, detail: { error: error instanceof Error ? error.message : String(error) },
}); });
@@ -99,6 +99,7 @@ export interface GatewaySubmitSegmentResultDto {
} }
export interface GatewayReceiptEventDto { export interface GatewayReceiptEventDto {
eventId?: string;
traceId?: string; traceId?: string;
messageId?: string; messageId?: string;
channelId: string; channelId: string;
@@ -114,6 +115,7 @@ export interface GatewayReceiptEventDto {
} }
export interface GatewayUplinkEventDto { export interface GatewayUplinkEventDto {
eventId?: string;
traceId?: string; traceId?: string;
messageId?: string; messageId?: string;
channelId: string; channelId: string;
@@ -136,10 +138,13 @@ export type UplinkMatchCandidateInput = {
export interface GatewayPendingDeliveryQueryDto { export interface GatewayPendingDeliveryQueryDto {
account: string; account: string;
limit?: number; limit?: number;
claimId?: string;
leaseMs?: number;
} }
export interface GatewayDownstreamSentDto { export interface GatewayDownstreamSentDto {
id: string; id: string;
claimId?: string;
connectionId?: string; connectionId?: string;
sequenceId?: string; sequenceId?: string;
messageId?: string; messageId?: string;
@@ -153,6 +158,7 @@ export interface GatewayDownstreamAcknowledgedDto extends GatewayDownstreamSentD
} }
export type GatewayDownstreamFailureType = export type GatewayDownstreamFailureType =
| 'claim_released'
| 'send_failed' | 'send_failed'
| 'ack_timeout' | 'ack_timeout'
| 'ack_rejected' | 'ack_rejected'
+13 -1
View File
@@ -3355,6 +3355,14 @@ describe('SendChainService', () => {
'delivery-once', 'delivery-once',
]); ]);
expect(service['postGatewayControl']).toHaveBeenCalledTimes(1); expect(service['postGatewayControl']).toHaveBeenCalledTimes(1);
expect(service['postGatewayControl']).toHaveBeenCalledWith(
'/downstream/receipt',
expect.objectContaining({ deliveryId: 'delivery-once', claimId: expect.stringMatching(/^api-direct:/) }),
);
expect(prisma.cmppDownstreamDelivery.updateMany).toHaveBeenCalledWith(expect.objectContaining({
where: { id: 'delivery-once', status: 'pending' },
data: expect.objectContaining({ status: 'dispatching', connectionId: expect.stringMatching(/^api-direct:/) }),
}));
expect(prisma.cmppDownstreamDelivery.create).toHaveBeenCalledTimes(3); expect(prisma.cmppDownstreamDelivery.create).toHaveBeenCalledTimes(3);
}); });
@@ -3895,8 +3903,12 @@ describe('SendChainService', () => {
}); });
prisma.cmppDownstreamDelivery.findMany.mockResolvedValue([]); prisma.cmppDownstreamDelivery.findMany.mockResolvedValue([]);
await expect(service.listPendingDownstreamDeliveries({ account: '100001', limit: 100 })) prisma.$queryRaw.mockResolvedValueOnce([]);
await expect(service.listPendingDownstreamDeliveries({ account: '100001', limit: 100, claimId: 'gateway-a:100001:1' }))
.resolves.toEqual([]); .resolves.toEqual([]);
const claimSql = prisma.$queryRaw.mock.calls[0][0].strings.join(' ');
expect(claimSql).toContain('FOR UPDATE SKIP LOCKED');
expect(claimSql).toContain("status = 'dispatching'");
}); });
it('persists each upstream SubmitResp segment before the aggregate result arrives', async () => { it('persists each upstream SubmitResp segment before the aggregate result arrives', async () => {
@@ -1,6 +1,6 @@
import { BadRequestException, HttpException, HttpStatus, Logger, NotFoundException } from '@nestjs/common'; import { BadRequestException, HttpException, HttpStatus, Logger, NotFoundException } from '@nestjs/common';
import { Prisma } from '@prisma/client'; import { Prisma } from '@prisma/client';
import { createHash } from 'node:crypto'; import { createHash, randomUUID } from 'node:crypto';
import { BillingService } from '../billing/billing.service'; import { BillingService } from '../billing/billing.service';
import { moneyToNumber } from '../common/money'; import { moneyToNumber } from '../common/money';
import type { OpenApiService } from '../open-api/open-api.service'; import type { OpenApiService } from '../open-api/open-api.service';
@@ -28,6 +28,10 @@ export class SendDownstreamDeliveryService {
) {} ) {}
async handleUplink(data: GatewayUplinkEventDto) { async handleUplink(data: GatewayUplinkEventDto) {
if (data.eventId) {
const existing = await this.prisma.smsUplinkMessage.findUnique({ where: { eventId: data.eventId } });
if (existing) return existing;
}
const channel = await this.prisma.smsChannel.findUnique({ where: { id: data.channelId } }); const channel = await this.prisma.smsChannel.findUnique({ where: { id: data.channelId } });
if (!channel) { if (!channel) {
throw new NotFoundException('SMS channel not found'); throw new NotFoundException('SMS channel not found');
@@ -35,6 +39,7 @@ export class SendDownstreamDeliveryService {
const match = await this.facade.resolveUplinkMatch(data, channel); const match = await this.facade.resolveUplinkMatch(data, channel);
const record = await this.prisma.smsUplinkMessage.create({ const record = await this.prisma.smsUplinkMessage.create({
data: { data: {
eventId: data.eventId,
tenantId: match.tenantId, tenantId: match.tenantId,
applicationId: match.applicationId, applicationId: match.applicationId,
messageRecordId: match.messageRecordId, messageRecordId: match.messageRecordId,
@@ -276,21 +281,40 @@ export class SendDownstreamDeliveryService {
if (!deliveryAllowed) { if (!deliveryAllowed) {
return delivery; return delivery;
} }
const claimId = `api-direct:${process.pid}:${randomUUID()}`;
const claim = await this.prisma.cmppDownstreamDelivery.updateMany({
where: { id: delivery.id, status: 'pending' },
data: {
status: 'dispatching',
connectionId: claimId,
ackDeadlineAt: new Date(Date.now() + 30_000),
nextRetryAt: null,
lastError: null,
},
});
if (claim.count !== 1) {
return this.prisma.cmppDownstreamDelivery.findUnique({ where: { id: delivery.id } });
}
try { try {
const result = await this.facade.postGatewayControl( const result = await this.facade.postGatewayControl(
data.deliveryType === 'receipt' ? '/downstream/receipt' : '/downstream/uplink', data.deliveryType === 'receipt' ? '/downstream/receipt' : '/downstream/uplink',
{ deliveryId: delivery.id, ...payload }, { deliveryId: delivery.id, claimId, ...payload },
) as GatewayControlDeliveryResult; ) as GatewayControlDeliveryResult;
if (result.sent || result.delivered) { if (result.sent || result.delivered) {
await this.facade.markDownstreamDeliverySent({ id: delivery.id, ...result }); await this.facade.markDownstreamDeliverySent({ id: delivery.id, claimId, ...result });
} else if (result.reasonCode === 'SUBMIT_RESPONSE_PENDING') { } else if (result.reasonCode === 'SUBMIT_RESPONSE_PENDING') {
return delivery; await this.facade.markDownstreamDeliveryFailed(
delivery.id,
downstreamControlFailureMessage(result),
'claim_released',
{ id: delivery.id, claimId, ...result },
);
} else { } else {
await this.facade.markDownstreamDeliveryFailed( await this.facade.markDownstreamDeliveryFailed(
delivery.id, delivery.id,
downstreamControlFailureMessage(result), downstreamControlFailureMessage(result),
result.retryable === false ? 'unrecoverable' : 'send_failed', result.retryable === false ? 'unrecoverable' : 'send_failed',
{ id: delivery.id, ...result }, { id: delivery.id, claimId, ...result },
); );
} }
} catch (error) { } catch (error) {
@@ -39,15 +39,54 @@ export class SendDownstreamStateService {
for (const expired of expiredAcknowledgements) { for (const expired of expiredAcknowledgements) {
await this.facade.markDownstreamDeliveryFailed(expired.id, 'CMPP_DELIVER_RESP timeout recovered after Gateway restart', 'ack_timeout'); await this.facade.markDownstreamDeliveryFailed(expired.id, 'CMPP_DELIVER_RESP timeout recovered after Gateway restart', 'ack_timeout');
} }
return this.prisma.cmppDownstreamDelivery.findMany({ const now = new Date();
where: { await this.prisma.cmppDownstreamDelivery.updateMany({
applicationId: application.id, where: { applicationId: application.id, status: 'dispatching', ackDeadlineAt: { lte: now } },
data: {
status: 'pending', status: 'pending',
OR: [{ nextRetryAt: null }, { nextRetryAt: { lte: new Date() } }], connectionId: null,
ackDeadlineAt: null,
nextRetryAt: now,
lastError: 'Gateway delivery claim lease expired and was recovered',
}, },
orderBy: { createdAt: 'asc' },
take: Math.min(Math.max(data.limit ?? 100, 1), 500),
}); });
const claimId = String(data.claimId ?? '').trim();
if (!claimId) {
throw new BadRequestException('claimId is required');
}
const limit = Math.min(Math.max(data.limit ?? 100, 1), 500);
const leaseMs = Math.min(Math.max(data.leaseMs ?? 30_000, 5_000), 120_000);
const claimed = await this.prisma.$queryRaw<Array<{ id: string }>>(Prisma.sql`
WITH candidates AS (
SELECT id
FROM "CmppDownstreamDelivery"
WHERE "applicationId" = ${application.id}
AND status = 'pending'
AND ("nextRetryAt" IS NULL OR "nextRetryAt" <= NOW())
ORDER BY "createdAt" ASC
FOR UPDATE SKIP LOCKED
LIMIT ${limit}
)
UPDATE "CmppDownstreamDelivery" AS delivery
SET status = 'dispatching',
"connectionId" = ${claimId},
"ackDeadlineAt" = NOW() + (${leaseMs} * INTERVAL '1 millisecond'),
"nextRetryAt" = NULL,
"lastError" = NULL,
"updatedAt" = NOW()
FROM candidates
WHERE delivery.id = candidates.id
RETURNING delivery.id
`);
if (claimed.length === 0) {
return [];
}
const claimedIds = claimed.map((item) => item.id);
const deliveries = await this.prisma.cmppDownstreamDelivery.findMany({
where: { id: { in: claimedIds }, status: 'dispatching', connectionId: claimId },
orderBy: { createdAt: 'asc' },
});
return deliveries.map((delivery) => ({ ...delivery, claimId }));
} }
async markDownstreamDeliveryDelivered(id: string) { async markDownstreamDeliveryDelivered(id: string) {
@@ -91,7 +130,11 @@ export class SendDownstreamStateService {
}, },
}); });
await this.prisma.cmppDownstreamDelivery.updateMany({ await this.prisma.cmppDownstreamDelivery.updateMany({
where: { id: data.id, status: { not: 'delivered' } }, where: {
id: data.id,
status: data.claimId ? 'dispatching' : { not: 'delivered' },
...(data.claimId ? { connectionId: data.claimId } : {}),
},
data: { data: {
status: 'awaiting_ack', status: 'awaiting_ack',
sentAt, sentAt,
@@ -197,6 +240,23 @@ export class SendDownstreamStateService {
if (delivery.status === 'delivered') { if (delivery.status === 'delivered') {
return delivery; return delivery;
} }
if (failureType === 'claim_released') {
await this.prisma.cmppDownstreamDelivery.updateMany({
where: {
id,
status: 'dispatching',
...(attempt?.claimId ? { connectionId: attempt.claimId } : {}),
},
data: {
status: 'pending',
connectionId: null,
ackDeadlineAt: null,
nextRetryAt: new Date(),
lastError: errorMessage ?? 'Gateway released downstream delivery claim',
},
});
return this.prisma.cmppDownstreamDelivery.findUnique({ where: { id } });
}
if (failureType === 'queue_timeout' && delivery.status !== 'pending') { if (failureType === 'queue_timeout' && delivery.status !== 'pending') {
return delivery; return delivery;
} }
@@ -699,6 +699,10 @@ startSubmitOutboxPublisher() {
windowSize: getPositiveConfigInteger(channel.config, 'windowSize', 16), windowSize: getPositiveConfigInteger(channel.config, 'windowSize', 16),
heartbeatIntervalSeconds: getPositiveConfigInteger(channel.config, 'heartbeatIntervalSeconds', 30), heartbeatIntervalSeconds: getPositiveConfigInteger(channel.config, 'heartbeatIntervalSeconds', 30),
heartbeatMissThreshold: getPositiveConfigInteger(channel.config, 'heartbeatMissThreshold', 3), heartbeatMissThreshold: getPositiveConfigInteger(channel.config, 'heartbeatMissThreshold', 3),
connectionWarmupSeconds: getNonNegativeConfigInteger(channel.config, 'connectionWarmupSeconds', 30),
connectionDrainTimeoutSeconds: getPositiveConfigInteger(channel.config, 'connectionDrainTimeoutSeconds', 60),
submitResponseTimeoutSeconds: getPositiveConfigInteger(channel.config, 'submitResponseTimeoutSeconds', 60),
connectionFailureCooldownSeconds: getPositiveConfigInteger(channel.config, 'connectionFailureCooldownSeconds', 30),
}, },
retry: { attempt, maxAttempts: 1 }, retry: { attempt, maxAttempts: 1 },
}; };
@@ -2148,3 +2148,14 @@
- Gateway供应商事件必须与主API控制面隔离:Submit结果、分片结果、回执、上行、受限协议日志和死信进入仅绑定回环地址的独立进程及独立有界PostgreSQL池;连接状态控制继续进入主API。客户HTTP Webhook使用异步队列Worker,不能在Gateway事实回写请求中同步调用客户地址。 - Gateway供应商事件必须与主API控制面隔离:Submit结果、分片结果、回执、上行、受限协议日志和死信进入仅绑定回环地址的独立进程及独立有界PostgreSQL池;连接状态控制继续进入主API。客户HTTP Webhook使用异步队列Worker,不能在Gateway事实回写请求中同步调用客户地址。
- 独立回调进程不得加载运营/计费管理Controller,不得经Nginx或公网暴露;部署必须验证回调健康、池上限、回环监听、事件URL和控制URL。回调服务先于Gateway启动/重启,回调失败应由既有结果Outbox恢复且不得重复处理、重复扣费。 - 独立回调进程不得加载运营/计费管理Controller,不得经Nginx或公网暴露;部署必须验证回调健康、池上限、回环监听、事件URL和控制URL。回调服务先于Gateway启动/重启,回调失败应由既有结果Outbox恢复且不得重复处理、重复扣费。
- 本阶段测试环境以正价325金额单位验证:50 TPS档必须同时满足客户端受理、非补发首次供应商Submit不低于50条/秒、MessageId/submitId/Outbox唯一、账单笔数和金额一致、Inbox/Outbox/Stream排空及数据库无持续锁等待。零计费结果不得作为付费链路容量结论。 - 本阶段测试环境以正价325金额单位验证:50 TPS档必须同时满足客户端受理、非补发首次供应商Submit不低于50条/秒、MessageId/submitId/Outbox唯一、账单笔数和金额一致、Inbox/Outbox/Stream排空及数据库无持续锁等待。零计费结果不得作为付费链路容量结论。
## 单 Gateway 容量扩展第五阶段(2026-08-25)
- 单个供应商通道必须支持1~8条连接、每连接1~64窗口,并在API校验、Gateway控制入口和实际运行状态三层执行相同边界;扩容需串行预热,缩容需先停止分配并等待在途完成,不能因改配置直接丢弃在途映射。
- Gateway协议摘要日志不得在Submit热路径同步写数据库或逐条HTTP回调。成功事件可确定性采样,拒绝、超时、断链和非零结果必须全量进入Redis Stream,由独立有界数据库池批量持久化;ACK只能发生在数据库提交后。
- Submit结果、分片结果、回执、上行和死信应由结果Outbox聚合为最多100条、默认50条的回环HTTP批次。API必须逐事件返回accepted/retryable/errorCode;整批传输失败重放,单个非法事件不得拖累其他事件,事件必须依赖稳定幂等键。
- Gateway必须暴露连接数、配置/在途窗口、批量回调请求/事件/重试/死信和协议日志发布/采样/错误/Stream长度指标。回调、日志Worker和主API数据库池必须独立有界,PostgreSQL总连接预算不得因扩容失控。
- 正价容量仍以非补发首次供应商Submit和全链路对账为准。2026-08-25测试机实测20/30/50档稳定,70档出现4秒级P95且未达到目标,故当前单Gateway发布建议上限为50 TPS100/150/200按停止线未执行。多Gateway租约、fencing和分片属于P2,仍未实施。
- Gateway拉取待投递回执必须按账号进程内单飞,API必须用FOR UPDATE SKIP LOCKED将pending原子领取为带租约的dispatching;未实际发送的领取必须无损释放,租约过期可恢复。SubmitResp后只能合并调度账号级刷新,不得每条并发扫描同一批pending记录。
- API新建回执后的直推与Gateway恢复拉取必须共享同一dispatching + claimId所有权;直推未抢到记录时必须退出,不得与恢复路径各发一次。ACK超时定时器必须在ACK注册锁内完成指针赋值,避免极短截止时间下的竞态。
- 2026-08-25修复后正价阶梯客户入口20/30/50/70/100/150/200 TPS均零拒绝、零节流、零连接错误;完整供应商首提在150/200冲击档分别约94.86/92.24 TPS,显示端到端容量天花板约95 TPS。二次直推领取修复后100 TPS正价复验为999/999、P95/P99=102/179ms、999次999个唯一首提在12.488秒完成(80.00 TPS),972条已形成终态回执的下游投递生命周期尝试次数恰为972、重复0、最大1。生产建议仍保留容量余量,建议限速70 TPS,不将客户SubmitResp受理200 TPS误作完整供应商TPS。
@@ -0,0 +1,493 @@
# CMPP Gateway 容量扩展与回调降载方案
更新日期:2026-08-25
状态:已实施并完成测试环境验证
适用范围:发送 Worker/Submit Outbox 到供应商 CMPP Submit、SubmitResp、状态报告、上行和计费结算链路
## 1. 已确认的供应商能力边界
本方案按以下已确认条件设计:
- 同一个供应商账号支持建立多条 CMPP 长连接;
- 单通道最多支持 8 条连接;
- 单连接滑动窗口最大支持 64
- 连接数和窗口上限表示技术许可,不等于供应商账号承诺 TPS;
- 实际配置仍必须服从账号总 TPS、通道限速、供应商网关稳定性和生产合同限制。
CMPP 规范建议窗口值为 16;本项目允许使用 32/64,是基于供应商明确支持的扩展能力。禁止未经阶梯验证直接启用 `8 连接 × 64 窗口`
单通道理论最大在途 Submit 数量为:
```text
连接数 × 单连接窗口 = 8 × 64 = 512
```
512 是未收到 SubmitResp 时允许并行等待的最大请求数,不代表可以稳定达到 512 TPS。稳定 TPS 还取决于 SubmitResp 延迟和供应商账号限速,例如平均响应时间为 200ms 时,窗口理论容量不是首要限制;如果平均响应时间接近 1 秒,窗口容量才会直接限制吞吐。
## 2. 改造目标与边界
目标:
1. 逐步提高每个供应商通道可承载的并发 Submit 数量;
2. 在单 Gateway 内完成多连接和窗口扩容,多个 Gateway 水平扩展降为 P2 后续可选项;
3. 将协议日志从发送和回调热路径移出;
4. 将逐事件 HTTP 回调改为可重放、逐事件幂等的批量回调;
5. 在扩容过程中保持消息、Submit、回执和计费不重不漏;
6. 使用非零单价验证真实计费并发,不以零计费压测结果作为容量依据。
本阶段不改变:
- 对外 CMPP 协议和企业账号;
- 签名、模板、余额、频控和企业/应用状态拦截语义;
- 通道组主备补发规则;
- `messageId``submitId`、结果事件和计费幂等口径;
- PostgreSQL Submit Outbox 作为 Gateway 命令事实来源的设计。
## 3. 改造一:单通道多连接与窗口扩容
### 3.1 连接池模型
每个启用通道维护独立连接池,最多创建 8 条连接。每条连接独立维护:
- `connectionId` 和 Gateway 实例标识;
- CMPP CONNECT/CONNECT_RESP 认证状态;
- 独立递增并循环使用的 SequenceId;
- 当前未确认 Submit 数量;
- `sequenceId -> submitId` 在途映射;
- 当前窗口大小和剩余窗口;
- SubmitResp 延迟、超时和连续失败次数;
- ACTIVE_TEST 心跳、断链和重连状态;
- 平滑启用、排空和关闭状态。
连接选择采用“可用窗口优先 + 最少在途 + 健康度”策略:
1. 排除未认证、正在重连、正在排空的连接;
2. 排除窗口已满的连接;
3. 优先选择未确认请求最少的连接;
4. 数量相同时优先选择近期 SubmitResp P95 更低的连接;
5. 连续超时连接进入冷却期,不再分配新 Submit。
### 3.2 动态配置
建议新增或明确以下配置:
```text
desiredConnections: 1..8
windowSize: 1..64
connectionWarmupSeconds: 30
connectionDrainTimeoutSeconds: 60
submitResponseTimeoutSeconds: 60
connectionFailureCooldownSeconds: 30
```
配置校验必须同时存在于管理 API 和 Gateway
- `desiredConnections < 1``> 8`:拒绝保存/拒绝启动;
- `windowSize < 1``> 64`:拒绝保存/拒绝启动;
- 扩容时逐条建立连接,不能同时突发 8 次登录;
- 缩容时先停止向目标连接分配新 Submit,等待在途请求完成后断开;
- 修改窗口时只影响新请求,不能丢弃已有在途映射。
### 3.3 阶梯扩容参数
| 阶段 | 每通道连接 | 单连接窗口 | 理论在途/通道 | 用途 |
|---|---:|---:|---:|---|
| M0 | 1 | 16 | 16 | 当前兼容基线 |
| M1 | 2 | 16 | 32 | 优先验证多连接正确性 |
| M2 | 2 | 32 | 64 | 验证窗口扩容 |
| M3 | 4 | 32 | 128 | 中等容量目标 |
| M4 | 4 | 64 | 256 | 高容量目标 |
| M5 | 8 | 64 | 512 | 上限验证,默认不直接用于生产 |
每一级必须单独通过正价 smoke、阶梯压测、故障注入和队列排空后才能进入下一级。
### 3.4 异常与幂等处理
- 收到 SubmitResp 后必须先用连接和 SequenceId 找到稳定 `submitId`,再释放窗口;
- 未找到在途映射的迟到 SubmitResp 进入异常对账,不能静默丢弃;
- TCP 写入前失败可安全重新分配连接;
- TCP 写入成功但未收到 SubmitResp时标记 `unknown`,禁止直接补发;
- 连接断开后只处理该连接的在途请求,不能把整个通道的消息统一重发;
- `submitId` 必须跨连接、跨实例保持唯一;
- 主备通道补发必须使用新的 `submitId`,并通过 `retryOfSubmitRecordId` 关联原提交。
## 4. P2 后续项:Gateway 水平扩展(当前不实施)
多个 Gateway 会引入通道连接所有权、分布式连接配额、租约、fencing token、脑裂、跨实例 pending 接管和“已写入供应商但未收到响应”的 unknown 判定,改造风险明显高于单 Gateway 内扩容。
因此本项调整为**优先级 P2 的后续可选改造**:
- 不纳入当前开发和压测范围;
- 不作为本轮达到目标 TPS 的前置条件;
- 当前先使用单 Gateway 管理同账号多连接,单通道仍遵守最多 8 条连接、单连接窗口最多 64;
- 只有单 Gateway 的 CPU、网络、文件描述符或事件循环成为经验证的主要瓶颈,且单实例优化无法继续提升时,才重新评审本项;
- 未完成独立设计评审、故障模型验证和隔离环境演练前,不得直接部署双 Gateway。
以下内容仅作为 P2 预研设计保留,不代表当前实施计划。
### 4.1 部署模型
部署至少两个 Gateway 实例,每个实例具有唯一的:
- `gatewayInstanceId`
- Redis Stream Consumer Name
- 指标端口和健康检查地址;
- 通道连接所有权和连接标识。
所有实例加入同一个 Submit Stream Consumer Group,共享:
- Submit 命令队列;
- 通道全局 TPS 限流;
- Submit 发布和处理幂等键;
- 结果/回执 Outbox
- 通道启停配置版本;
- 待恢复和死信状态。
### 4.2 推荐的首期分片方式
如果以后启动 P2,首期采用“通道分片 + 故障接管”:
- 正常状态下,一个通道只由一个 Gateway 实例持有连接;
- 不同运营商、主通道和备通道分散到不同实例;
- 实例失联后,备用实例经过租约超时再接管通道;
- 同一个通道不能被两个实例无租约地同时建立全部连接。
后续如果需要同账号跨实例并发连接,必须增加分布式连接配额:同一通道所有 Gateway 实例的连接总数不得超过 8。
### 4.3 接管与脑裂保护
- Redis 或 PostgreSQL 中保存带过期时间的通道所有权租约;
- 租约包含 `channelId``gatewayInstanceId`、配置版本和 fencing token
- 只有持有最新 fencing token 的实例可以领取该通道的新 Submit;
- 旧实例恢复后必须先重新竞选租约,不能直接恢复发送;
- Stream 未确认消息通过 `XAUTOCLAIM` 接管;
- 对“可能已写入供应商 TCP”的请求不得盲目重发,应进入 unknown 对账。
### 4.4 负载均衡边界
Gateway 水平扩展解决进程 CPU、网络、连接和单机故障问题,但不能突破:
- 单通道最多 8 条连接;
- 单连接窗口最多 64
- 供应商账号总 TPS
- PostgreSQL、Redis和回调服务的实际容量。
## 5. 改造三:协议日志降载
### 5.1 日志分级
| 日志等级 | 示例 | 处理方式 |
|---|---|---|
| 必留异常 | 拒绝、超时、断链、认证失败、重复/未知响应 | 100%保留,异步优先写入 |
| 业务摘要 | Submit、SubmitResp、回执、上行、下行应答 | 异步批量写入 |
| 原始协议包 | 完整请求/响应字段或二进制包 | 正常流量采样,异常全量短期保存 |
### 5.2 实现方案
- Gateway 不在 Submit/SubmitResp 热路径同步写协议日志数据库;
- 日志事件写入独立 Redis Stream
- 独立日志 Worker 每批领取 100500 条;
- 使用批量插入和批量状态回写;
- 正常成功事件按通道配置 1%~10% 采样;
- 错误、超时、拒绝和断链事件保持 100%;
- 原始包写滚动文件或对象存储,主库只保存检索摘要和对象地址;
- 正常原始包建议保留 3~7 天,异常日志按审计策略保留;
- Redis 日志 Stream 设置最大长度和积压告警。
日志积压不得阻塞 Submit。积压超过门槛时允许降低正常成功日志采样率,但禁止丢弃错误、安全和账务相关事件。
### 5.3 必须保留的追踪字段
- `messageId``submitId``eventId`
- `channelId``gatewayInstanceId``connectionId`
- SequenceId 和供应商 Msg_Id
- 事件类型、方向、结果码和耗时;
- 重试关联和原始日志对象地址。
## 6. 改造四:Gateway 批量回调协议
### 6.1 接口设计
新增内部接口:
```text
POST /api/gateway/events/batch
```
批次请求包含:
- `batchId`
- `gatewayInstanceId`
- `createdAt`
- 1100 个事件;
- 每个事件独立的 `eventId``type``payload`
首期支持事件类型:
- `submit_segment_result`
- `submit_result`
- `receipt_intake`
- `uplink`
- `dead_letter`
计费不由 Gateway直接生成。计费由 Submit结果/回执状态机依据稳定业务键执行,批量回调只负责可靠传递触发事实。
### 6.2 成批策略
采用“数量或时间先到先发”:
- 默认最大 50 个事件;
- 可配置上限 100
- 默认最大等待 10ms
- 最大请求体 1MB
- Submit结果优先级高于普通协议日志;
- 回执和上行可以使用独立批次,避免不同慢操作互相阻塞。
### 6.3 响应与重试
API 必须返回逐事件结果,而不是只有整批成功或失败:
```json
{
"batchId": "CB-001",
"results": [
{ "eventId": "EV-001", "accepted": true },
{ "eventId": "EV-002", "accepted": false, "retryable": true, "errorCode": "DB_BUSY" }
]
}
```
- Gateway 只 ACK `accepted=true` 的事件;
- 可重试失败使用指数退避;
- 不可重试失败进入死信并告警;
- HTTP 超时后允许重发整个批次,API 依靠 `eventId` 幂等;
- API 可以按事件类型及每 20~50 条拆成短事务;
- 单个坏事件不能造成整批回滚。
### 6.4 数据库批量处理
- `eventId` 建唯一索引或使用既有稳定幂等键;
- Submit结果批量更新 `SmsSubmitRecord`
- 回执先批量写入耐久 Receipt Inbox,再异步完成聚合;
- 计费使用稳定计费键批量创建,重复事件不得重复扣费;
- 消息和任务进度使用集合更新,避免逐事件 `GROUP BY`
- 事务中禁止调用外部 HTTP、Redis或供应商接口。
### 6.5 兼容与回退
- 保留原单事件回调接口;
- 使用开关选择单事件或批量接口,同一事件不能双发;
- 新旧入口共享 `eventId` 幂等逻辑;
- 回退只切换发送方式,不删除回调 Outbox 或幂等记录;
- 回退前必须等待在途批次完成并核对 Redis PEL。
## 7. 当前实施顺序与 P2 后续项
### 当前步骤 1:可观测性和协议日志降载
先补齐连接、窗口、供应商响应延迟和日志积压指标,再将成功协议日志异步批量化。该阶段不提高连接数。
### 当前步骤 2:单实例多连接
实现连接池、窗口隔离、平滑扩缩容和断链恢复,依次验证 `2×16``2×32``4×32`
### 当前步骤 3:批量回调协议
上线批量接口和逐事件幂等,先影子对账,再切换 Submit结果,最后切换回执和上行。
### 当前步骤 4:单 Gateway 容量上限验证
只有前述阶段稳定后,才在单 Gateway 内测试 `4×64``8×64`。生产建议值依据压测结果确定,不以供应商允许的最大参数直接作为生产配置。
### 优先级 P2:多 Gateway 水平扩展
当前不实施、不部署、不纳入本轮压测。将来只有证据表明单 Gateway 自身成为主要瓶颈时,才单独立项,执行通道分片、租约防脑裂、故障接管和 unknown 对账验证。
## 8. 正价压测方案
### 8.1 前置条件
- 仅在隔离测试环境执行;
- 压测前完成数据库、环境文件、部署目录和 systemd 配置备份;
- 测试应用单价设置为非零值,并记录压测前余额;
- 使用可明确识别的移动、联通、电信混合号码;
- 主备六通道均处于预期状态;
- Submit、结果、回执、日志 Stream 和数据库 Outbox 初始无积压;
- 压测前记录 Gateway 实例、连接数、窗口和供应商限速配置。
### 8.2 阶梯
每个连接/窗口档位分别执行:
1. 9 条正价全链路 smoke
2. 20 TPS
3. 30 TPS
4. 50 TPS
5. 70 TPS
6. 100 TPS
7. 达标且无停止条件时再测试 150/200 TPS。
每档结束后等待完整排空并对账,不能只检查入口 SubmitResp。
### 8.3 验收指标
- 客户 SubmitResp 接收数和唯一 MessageId 数;
- Inbox 完成数;
- 首次供应商 Submit 数和每秒吞吐;
- 各通道、各连接的提交数和在途峰值;
- SubmitResp P50/P95/P99
- 供应商接受、拒绝、超时和 unknown 数;
- 主备补发关联完整性;
- 最终 delivered/failed/submitted/unknown 数;
- 回调批次大小、请求数、重试数和积压;
- 数据库连接池等待、锁等待和慢 SQL;
- 协议日志 Stream 积压和丢弃/采样计数;
- 计费记录数、计费单位、扣费金额与余额变化。
正价计费恒等式至少满足:
```text
压测前余额 - 压测后余额
= 有效计费记录金额合计
= 各消息 billingUnits × 应用单价之和
```
### 8.4 停止条件
出现任一情况立即停止升档:
- SubmitResp 缺失或 MessageId 重复;
- `submitId` 重复提交;
- 重复计费、漏计费或余额不平;
- 供应商超时/unknown 持续增加;
- Redis Stream、Inbox、Outbox或回调持续积压且无法在规定时间排空;
- 数据库连接等待、锁等待或 idle transaction 异常;
- Gateway 频繁断链或连接数不能恢复;
- 通道禁用、主备补发或发送拦截语义异常;
- 日志降载导致错误、安全或账务事件丢失。
## 9. 故障回归
必须覆盖:
- 单连接断开,其余连接继续发送;
- 同一通道从 4 条连接缩到 2 条连接,在途请求不丢失;
- 单 Gateway 进程重启后恢复连接、在途状态和 Stream pending
- Redis 短暂不可用;
- 批量回调 HTTP 超时后整批重放;
- 回调批次中单个非法事件不拖累其他事件;
- PostgreSQL 回调连接池耗尽时不阻塞 Submit Outbox
- 主通道 Submit拒绝触发备通道关联补发;
- 主通道禁用后只使用备通道;
- 主备同时禁用时发送在供应商前失败;
- 签名/模板未报备、余额不足、企业/应用删除或禁用、号码频控仍能正确拦截;
- 重启和事件重放不造成重复提交、重复回执或重复计费。
多 Gateway 的实例宕机接管、租约、fencing token 和脑裂测试属于 P2 后续项目,不纳入当前故障回归范围。
## 10. 容量预期
容量提升不能简单相加。最终吞吐由最慢环节决定:
```text
实际 TPS = min(
入口能力,
Worker/Outbox能力,
Gateway连接与窗口能力,
供应商账号总TPS,
回调与计费能力,
PostgreSQL/Redis能力
)
```
以最近一次测试记录的完整供应商首提约 33 TPS 为历史参考,执行前必须重新验证当前基线。工程预期区间如下:
| 完成范围 | 预期稳定首提吞吐 |
|---|---:|
| 独立回调 + 批量路由/持久化 | 50~70 TPS |
| 加协议日志降载 + `2×16`/`2×32` | 60100 TPS |
| 加 `4×32``4×64` | 80150 TPS |
| 单 Gateway `8×64` 上限验证 | 由供应商总TPS及单机实测决定,不预先承诺 |
| 多 Gateway 水平扩展(P2) | 当前不实施,另行立项后评估 |
| 300500 TPS | 不作为当前承诺;未来可能需要供应商总TPS、通道数量和基础设施共同扩展 |
上述数字是容量规划目标,不是承诺值。即使单通道理论支持 512 个在途请求,如果供应商账号总限速为 50 TPS,平台稳定吞吐仍不会超过约 50 TPS。
## 11. 生产推荐原则
- 生产参数使用“最低但足够”的连接数和窗口,不长期顶格运行;
- 初始推荐从 `2 连接 × 16 窗口` 开始;
- 只有监控证明窗口经常耗尽且供应商响应稳定,才提高到 32/64;
- 当前不增加 Gateway 实例;只有单实例 CPU、网络、文件描述符或事件循环成为经验证且无法继续优化的瓶颈,才启动 P2 水平扩展评审;
- 保留连接数、窗口、批量回调、日志采样和 Gateway分片的独立回退开关;
- 每次只改变一个主要容量变量,保留可比较的正价压测基线。
## 12. 2026-08-25 实施与测试结论
本轮已实施本方案中除“多 Gateway 水平扩展(P2)”外的改造,并仅发布到测试环境 `100.93.204.60`。发布前恢复点为 `/opt/cmpp-platform-backups/phase5-capacity-20260825T041622Z`,数据库、运行源码和 systemd/环境配置归档均已校验;预生产环境未操作。
已交付能力:
- Gateway 协议摘要日志改为 Redis Stream 异步写入,成功事件确定性采样 10%,拒绝、超时、断链和非零结果 100% 保留;独立 `cmpp-protocol-log-worker` 使用独立 4 槽 PostgreSQL 连接池批量落库,成功后 ACK/XDEL,失败重放,格式错误进入死信 Stream。
- 单通道支持 1~8 条供应商连接、每连接 1~64 窗口;连接选择按在途数量和近期 RTT,连续失败进入冷却。扩容串行预热;缩容先停止分配、等待在途结束或排水超时后关闭。
- Gateway 结果 Outbox 支持最多 50 条、最长等待 10ms 的批量 HTTP 回调;Submit 结果、分片结果、回执、上行和死信均返回逐事件结果,整批 HTTP 失败留在 PEL 重放,单个非法事件不拖累同批其他事件。
- API、Gateway 控制入口和实际运行状态完成三层校验。运行矩阵覆盖 `1x1``2x16``4x32``8x64``4x64``2x32`,越界 `0/9` 连接及 `0/65` 窗口均被拒绝。
- 上行增加 `eventId` 唯一幂等键;同一批量上行事件重放两次只生成一条 `SmsUplinkMessage`
发布后曾发现协议日志发布器在仅配置 `REDIS_HOST/REDIS_PORT` 的环境中没有对空 `REDIS_URL` 回退。该问题触发停止条件,未进入阶梯;修正为与其他 Gateway Redis 组件一致的本机回退、增加测试并重发后,冒烟显示发布 16、采样丢弃 47、错误 0,日志 Stream 最终 `pending=0/lag=0`
### 12.1 正价阶梯结果
测试单价为 325 分/计费条,三运营商混合号码,六个模拟供应商账号;压测连接配置为每通道 `2x32`。容量按非补发的首次供应商 Submit 统计。
| 目标档 | 客户 SubmitResp | 首次供应商 Submit | 首提跨度/实际速率 | 客户 P50/P95/P99 | 计费 | 结论 |
|---:|---:|---:|---:|---:|---:|---|
| smoke | 9/9 | 9 | 约 8.1 秒/低速冒烟 | 17/1727/1727 ms | 9 笔/2925 | 冷启动尾延迟后链路闭合 |
| 20 TPS | 199/199 | 199 | 约 9.8 秒/约 20 TPS | 14/21/29 ms | 199 笔/64675 | 通过 |
| 30 TPS | 299/299 | 299 | 9.840 秒/30.39 TPS | 16/33/61 ms | 299 笔/97175 | 通过 |
| 50 TPS | 499/499 | 499 | 10.086 秒/49.47 TPS | 22/62/89 ms | 499 笔/162175 | 通过 |
| 70 TPS | 699/699 | 690 | 10.351 秒/66.66 TPS | 94/4099/5323 ms | 690 笔/224250 | 触发停止线,不再升档 |
| 100/150/200 TPS | 未执行 | - | - | - | - | 按停止条件立即停止 |
70 TPS 档少 9 条首次 Submit 和计费并非丢失:9 条均为重复使用的测试号码命中“单号码 24 小时最多 10 条”,最终 `failed/RISK`,供应商提交和计费均为 0。该结果证明频控拦截有效,但 70 TPS 的 P95/P99 已从 50 档的 `62/89ms` 跃升到 `4099/5323ms`,目标 70 TPS 也只完成 66.66 TPS,因此本版本稳定上限定为 **50 TPS**70 TPS 只能视作非稳定冲击档。
### 12.2 完整回归与恢复
- Submit/回执:各通过档客户 SubmitResp 无缺失,MessageId、submitId 和供应商 MessageId 无重复;模拟器 DELIVRD/失败/无回执比例按配置落库,三条 Stream 最终均 `pending=0/lag=0`
- 计费:20/30/50 档及 70 档未被风控拒绝的消息,计费笔数均等于有效消息数,单价全部 325,金额满足 `条数 x 325`;风控、余额不足和路由失败不计费。
- 主备:随机结果码 8 产生带 `retryOfSubmitRecordId` 的备通道补发;仅主通道停用时 9 条全部走 `LGST-M-B`,主备六通道全停时 9/9 `failed/ROUTE`、供应商提交 0。
- 上行/批量回调:有效上行与非法事件同批时互不影响,非法项为不可重试;同一 eventId 重放两次仅落一条上行记录。
- 拦截:签名未审核、模板强制匹配但未审核、余额不足分别为 9/9 `SIGNATURE/TEMPLATE/BALANCE` 且供应商提交 0;应用或企业停用时客户连接不能鉴权;频控在 70 档真实命中;所有临时状态均恢复。
- 平滑扩缩容:`LGST-M-P` 从 1 扩到 4,再在 20 TPS、20 秒发送期间缩到 1399/399 SubmitResp399 条首次 Submit,无重复 IDP95/P99=`49/142ms`,最终连接恢复 6/6。
- 故障:Gateway 重启后由 API 恢复连接;Redis 在队列排空时短暂停止,Gateway 健康进程保持,Redis 恢复后重启 Gateway/API 并恢复 6/6;整批 HTTP 超时重放和单事件隔离另有自动化测试覆盖。
- 环境恢复:10 个测试应用单价恢复 0,企业/应用/签名/模板/通道均恢复原状态,三条临时号段规则删除,六通道连接恢复 6/6。真实消息、回执和账务事实保留审计,不回写余额。
## 13. 2026-08-25 回执回放风暴修复与正价复压
旧结论中70 TPS的4秒级尾延迟不是前几轮Worker、Outbox或计费优化失效,而是Gateway在每条SubmitResp后并发扫描同账号pending回执;API仅查询pending,直到异步sent回调才改状态,因此多个扫描会读到同一批。旧版100/150冲击档客户回执重复曾达847/5101条,Gateway goroutine在150档峰值超过1.2万。
修复内容:
- Gateway同账号flushPending单飞,SubmitResp触发改为25ms尾随合并、250ms最长等待,登录和周期恢复保留;
- API使用FOR UPDATE SKIP LOCKED原子将pending领取为dispatching,绑定claimId和5120秒可控租约;当前Gateway默认30秒;
- 客户离线或SubmitResp屏障时用claim_released无损释放,不消耗业务重试预算;过期dispatching自动恢复为pending
- API新建回执后的直推也必须先原子取得dispatching + claimId,从根源消除直推与Gateway恢复各发一次的竞态;
- ACK超时定时器在ACK registry锁内完成指针赋值,Linux竞态检查不再报告该竞态。
发布仅到测试机100.93.204.60,预生产未操作。恢复点为/opt/cmpp-platform-backups/phase5-replay-hotfix-20260825T070815ZPostgreSQL custom dump、运行源码、systemd/环境配置和SHA-256均已校验。Linux go test -race ./internal/inbound、Gateway全包测试/go vet、API 44套520项和TypeScript构建通过。最终运行标记为761c123b65f09093bc379096188f3ee9ccb2e618+workspace.phase5.replayhotfix2.directclaim。
正价阶梯(单价325):
| 目标档 | 客户SubmitResp | 客户P50/P95/P99 | 首次供应商Submit跨度/速率 | 结论 |
|---:|---:|---:|---:|---|
| 20 | 199/199 | 5/10/16ms | 9.930s / 19.84 TPS | 通过 |
| 30 | 299/299 | 5/10/18ms | 9.854s / 30.34 TPS | 通过 |
| 50 | 499/499 | 7/35/47ms | 10.067s / 49.47 TPS | 通过 |
| 70 | 699/699 | 14/44/60ms | 10.666s / 65.54 TPS | 入口稳定,首提接近70档 |
| 100 | 999/999 | 24/51/72ms | 13.345s / 74.86 TPS | 入口通过,端到端未达100 |
| 150 | 1498/1498 | 49/87/114ms | 15.781s / 94.86 TPS | 冲击档,显示端到端天花板 |
| 200 | 1998/1998 | 50/84/110ms | 21.661s / 92.24 TPS | 冲击档,端到端不再增长 |
直推领取修复后另执行100 TPS正价复验:999/999P50/P95/P99=54/102/179ms999个唯一MessageId、999次非补发首提在12.488秒完成(80.00 TPS)。972条已产生最终回执的投递全部ACK,生命周期尝试972、重复0、最大尝试1。计费998条:995 charged、3条供应商最终失败后refunded;1条RISK在供应商前失败且未计费。27条模拟器no-receipt保持submitted,不是队列丢失。Inbox、Submit Outbox、下游pending/dispatching/awaiting_ack、未授权锁和idle transaction最终均为0。
最终结论分两种口径:客户入口SubmitResp受理已通过200 TPS冲击;完整供应商首提天花板约95 TPS。测试环境建议稳定限速70 TPS,保留约25%以上端到端余量;不将入口200 TPS宣称为供应商200 TPS。按用户要求,7个测试应用单价保持325^1380028/^1300028/^1890028三条临时号段规则保留,原100.91.249.119/32和新增127.0.0.1/32白名单均保留,未做测试后删除或归零。
+17
View File
@@ -4864,3 +4864,20 @@ npm run verify:phase8
| TC-CMPP-CALLBACK-001 | Gateway事件进程隔离 | Submit结果、回执、上行、协议日志和死信只进入回环回调进程及独立12槽池;连接状态仍进入主API | | TC-CMPP-CALLBACK-001 | Gateway事件进程隔离 | Submit结果、回执、上行、协议日志和死信只进入回环回调进程及独立12槽池;连接状态仍进入主API |
| TC-CMPP-CALLBACK-002 | 回调安全边界 | 回调进程仅监听127.0.0.1,不加载计费管理Controller;健康与指标端点只在回环可见 | | TC-CMPP-CALLBACK-002 | 回调安全边界 | 回调进程仅监听127.0.0.1,不加载计费管理Controller;健康与指标端点只在回环可见 |
| TC-CMPP-BATCH-003 | 正价50 TPS完整提交 | 单价325499条首次Submit在9.930秒完成(50.25/s);499笔账单162175,无丢重,队列最终排空 | | TC-CMPP-BATCH-003 | 正价50 TPS完整提交 | 单价325499条首次Submit在9.930秒完成(50.25/s);499笔账单162175,无丢重,队列最终排空 |
## TC-CMPP-PHASE5 单 Gateway 容量扩展(2026-08-25
| 用例ID | 场景 | 验收结果 |
| --- | --- | --- |
| TC-CMPP-PHASE5-001 | API、Gateway控制入口、实际运行三层校验1~8连接/1~64窗口 | `1x1/2x16/4x32/8x64/4x64/2x32`通过;越界拒绝 |
| TC-CMPP-PHASE5-002 | 协议日志降载 | 成功日志确定性采样,异常全留;独立Worker批量写库后ACK/XDEL,错误0且Stream排空 |
| TC-CMPP-PHASE5-003 | 批量HTTP回调与重放 | 多事件少请求;逐事件结果;非法事件不影响有效事件;HTTP失败整批重放;重试/死信可观测 |
| TC-CMPP-PHASE5-004 | 上行eventId幂等 | 同一事件投递两次只生成1条SmsUplinkMessage |
| TC-CMPP-PHASE5-005 | 在途平滑缩容 | 单通道1→4→1并发发送399条,SubmitResp/首次Submit均399,重复ID为0,连接恢复6/6 |
| TC-CMPP-PHASE5-006 | 非零单价阶梯停止线 | 20/30/50通过;70档P95/P99=`4099/5323ms`且实得66.66 TPS,立即停止100/150/200,稳定上限50 TPS |
| TC-CMPP-PHASE5-007 | 拦截与主备回归 | SIGNATURE/TEMPLATE/BALANCE/RISK均供应商前拦截;应用/企业停用拒绝鉴权;主停走备、全停ROUTE;拒绝补发retryOf完整 |
| TC-CMPP-PHASE5-008 | Gateway/Redis故障与最终排空 | Gateway重启恢复连接;排空时Redis短停后恢复;Submit/结果/日志Stream pending/lag均0,无锁等待/idle事务 |
| TC-CMPP-PHASE5-009 | 同账号并发pending拉取 | 8个并发刷新只产生1次API领取;记录以FOR UPDATE SKIP LOCKED从pending转dispatching,携带claimId和租约 |
| TC-CMPP-PHASE5-010 | API直推与Gateway恢复同时命中同一回执 | 仅抢到dispatching + claimId的路径发送;100 TPS正价实测972条终态回执对应972次下游ACK,重复0,最大尝试1 |
| TC-CMPP-PHASE5-011 | claim未发送、租约过期和ACK定时器竞态 | SubmitResp屏障或客户离线时释放claim且不增加重试次数;过期dispatching可恢复;Linux go test -race ./internal/inbound通过 |
| TC-CMPP-PHASE5-012 | 修复后正价容量与口径分离 | 20至200 TPS入口均无拒绝、节流或连接错误;999条100 TPS复验入口P95/P99 102/179ms,供应商首提80.00 TPS150/200冲击档首提约95 TPS天花板,两种口径分开报告 |
+19
View File
@@ -3946,3 +3946,22 @@ git diff --check
- 修复后正价结果:smoke 9/9、账单292520 TPS 199/199、P50/P95/P99=`19/51/94ms`、账单6467530 TPS 299/299、`20/48/69ms`、账单9717550 TPS 499/499、`35/76/135ms`、账单162175。四个成功窗口合计1006条、1006笔账单、单价全部325、金额326950。 - 修复后正价结果:smoke 9/9、账单292520 TPS 199/199、P50/P95/P99=`19/51/94ms`、账单6467530 TPS 299/299、`20/48/69ms`、账单9717550 TPS 499/499、`35/76/135ms`、账单162175。四个成功窗口合计1006条、1006笔账单、单价全部325、金额326950。
- 50档499条唯一首次供应商Submit跨度9.930秒,完整提交速率`50.25条/秒`,相对上一版同档33.07条/秒提高约52%。全部尝试/Outbox均542条且ID唯一;状态检查时delivered485、failed4、submitted10,关联补发43次。Inbox、Outbox、双Stream最终排空,数据库无等待锁、无idle in transaction,回调池`max=12,total=1,idle=1,waiting=0` - 50档499条唯一首次供应商Submit跨度9.930秒,完整提交速率`50.25条/秒`,相对上一版同档33.07条/秒提高约52%。全部尝试/Outbox均542条且ID唯一;状态检查时delivered485、failed4、submitted10,关联补发43次。Inbox、Outbox、双Stream最终排空,数据库无等待锁、无idle in transaction,回调池`max=12,total=1,idle=1,waiting=0`
- 测试环境最终标记`c6f11014d61a8627ae311098ef81d89dd099ffaa+workspace.phase4batchcallback.callback-surface.599393bb6233`;恢复资产为`/opt/cmpp-platform-backups/phase4-batch-callback-20260825T025611Z`。10个隔离应用单价恢复0,三条临时号段规则删除,真实计费事实保留;预生产未操作,未执行100/200/300/500档。 - 测试环境最终标记`c6f11014d61a8627ae311098ef81d89dd099ffaa+workspace.phase4batchcallback.callback-surface.599393bb6233`;恢复资产为`/opt/cmpp-platform-backups/phase4-batch-callback-20260825T025611Z`。10个隔离应用单价恢复0,三条临时号段规则删除,真实计费事实保留;预生产未操作,未执行100/200/300/500档。
## 2026-08-25 第五阶段单 Gateway 容量扩展与正价压测
- 完成协议日志 Redis Stream 降载及独立日志 Worker、单 Gateway 每通道 18 连接/每连接 1~64 窗口、RTT/在途择优、失败冷却、平滑扩缩容、Gateway 结果 Outbox 批量 HTTP 回调、逐事件确认和上行 eventId 幂等;明确未实施多 Gateway P2。
- API单测、Gateway控制入口和测试机实际运行三层覆盖 `1x1/2x16/4x32/8x64/4x64/2x32``LGST-M-P` 1→4→1 在20 TPS在途发送期间平滑缩容,399/399 SubmitResp、399条首次Submit、重复ID为0P95/P99=`49/142ms`
- 协议日志首次冒烟发现空`REDIS_URL`未回退,按停止线中止。修复、回归并重发后发布16、采样47、错误0,独立Worker排空。批量回调累计事件数大于HTTP请求数,重试/死信为0;同一上行eventId重放两次只落一条。
- 正价325阶梯:20档199条约20 TPS、P95/P99=`21/29ms`、6467530档299条30.39 TPS、`33/61ms`、9717550档499条49.47 TPS、`62/89ms`、16217570档699个SubmitResp690条有效首提在10.351秒完成66.66 TPS`4099/5323ms`690笔224250。另9条为频控`RISK`且供应商/计费均0。
- 70档尾延迟突增且未达到目标,立即停止,未执行100/150/200。稳定TPS上限定为50 TPS。
- 当前部署回归:签名、模板、余额分别9/9在供应商前拦截;应用/企业停用均无法鉴权;主通道停用9条全部走备通道,主备全停9/9`ROUTE`且提交0;随机主拒绝补发均有retryOf;Submit、回执、上行和计费对账通过。
- 发布只到`100.93.204.60`,恢复点`/opt/cmpp-platform-backups/phase5-capacity-20260825T041622Z`;预生产未操作。结束时应用单价0、临时号段规则0、业务配置恢复、六连接在线,队列排空且数据库无锁等待/idle事务。
## 2026-08-25 第五阶段回执回放热修复与正价复压
- 重新验证确认旧70 TPS尾延迟是回执恢复风暴:每条SubmitResp都启动pending扫描,多个扫描在sent回调前取到同一批。实施Gateway账号级单飞与25ms/250ms有界合并、API FOR UPDATE SKIP LOCKED pending到dispatching原子领取、claimId/30秒租约、未发送无损释放及过期恢复。
- 首轮复压还发现6191条中29条有2次下游尝试,进一步定位为API新建回执直推与Gateway恢复拉取之间没有共享所有权。直推改为先取得dispatching + claimId,未抢到则退出。Linux race门禁另发现并修复ACK超时timer指针赋值竞态。
- 门禁:API 44套520项、TypeScript构建、Gateway全包测试、go vet、Linux go test -race ./internal/inbound和git diff --check通过。只发布测试机,预生产未操作;恢复点/opt/cmpp-platform-backups/phase5-replay-hotfix-20260825T070815Z,最终标记761c123b65f09093bc379096188f3ee9ccb2e618+workspace.phase5.replayhotfix2.directclaim。
- 正价325阶梯入口SubmitResp20=199/199、P95/P99 10/16ms30=299/299、10/18ms50=499/499、35/47ms70=699/699、44/60ms100=999/999、51/72ms150=1498/1498、87/114ms200=1998/1998、84/110ms;全部零拒绝、零节流、零连接错误。完整供应商首提速率依次为19.84/30.34/49.47/65.54/74.86/94.86/92.24 TPS,端到端天花板约95 TPS。
- 最终100 TPS正价复验:999/999P50/P95/P99 54/102/179ms999个唯一MessageId1087个含主备补发的submitId全部唯一;999次999个唯一首提在12.488秒完成(80.00 TPS)。972条终态回执全部客户ACK,下游尝试972、重复0、最大1。计费998条中995 charged、3 refunded1条RISK供应商前失败不计费;27条no-receipt保持submitted。Inbox/Outbox/下游pending队列、未授权锁和idle transaction最终均0。
- 最终建议分口径:客户入口可受理200 TPS冲击,不等于端到端供应商200 TPS;供应商首提峰值约95 TPS,建议稳定限速70 TPS保留余量。按用户要求,7个应用单价保持325,三条临时号段规则、Tailscale白名单及VM回环白名单均保留,未做删除或归零。压测原始证据已复制到相邻测试项目lg-cmpp-stress-lab/results下的phase5-replayfix目录。
+28 -2
View File
@@ -13,6 +13,7 @@ import (
"cmpp-platform/gateway/internal/health" "cmpp-platform/gateway/internal/health"
"cmpp-platform/gateway/internal/inbound" "cmpp-platform/gateway/internal/inbound"
platformmetrics "cmpp-platform/gateway/internal/metrics" platformmetrics "cmpp-platform/gateway/internal/metrics"
"cmpp-platform/gateway/internal/protocollog"
"cmpp-platform/gateway/internal/queue" "cmpp-platform/gateway/internal/queue"
"cmpp-platform/gateway/internal/ratelimit" "cmpp-platform/gateway/internal/ratelimit"
"cmpp-platform/gateway/internal/resultoutbox" "cmpp-platform/gateway/internal/resultoutbox"
@@ -33,7 +34,17 @@ func main() {
} }
apiBaseURL := os.Getenv("API_BASE_URL") apiBaseURL := os.Getenv("API_BASE_URL")
callbackBaseURL := getenv("GATEWAY_CALLBACK_API_BASE_URL", apiBaseURL) callbackBaseURL := getenv("GATEWAY_CALLBACK_API_BASE_URL", apiBaseURL)
upstreamManager := &upstream.Manager{APIBaseURL: apiBaseURL, EventAPIBaseURL: callbackBaseURL} gatewayInstanceID := getenv("GATEWAY_INSTANCE_ID", hostname())
protocolLogPublisher, protocolLogErr := protocollog.New(os.Getenv("REDIS_URL"))
if protocolLogErr != nil {
log.Printf("gateway protocol log Redis publisher init failed: %v", protocolLogErr)
} else {
protocolLogPublisher.Stream = getenv("GATEWAY_PROTOCOL_LOG_STREAM", "gateway.protocol.logs")
protocolLogPublisher.GatewayInstanceID = gatewayInstanceID
protocolLogPublisher.SuccessSampleRate = positiveEnvInt("GATEWAY_PROTOCOL_LOG_SUCCESS_SAMPLE_PERCENT", 10)
protocolLogPublisher.MaxLen = int64(positiveEnvInt("GATEWAY_PROTOCOL_LOG_STREAM_MAX_LEN", 200000))
}
upstreamManager := &upstream.Manager{APIBaseURL: apiBaseURL, EventAPIBaseURL: callbackBaseURL, ProtocolLogPublisher: protocolLogPublisher, GatewayInstanceID: gatewayInstanceID}
var worker *submitworker.Worker var worker *submitworker.Worker
var resultOutbox *resultoutbox.Outbox var resultOutbox *resultoutbox.Outbox
channelLimiter, err := ratelimit.New(os.Getenv("REDIS_URL")) channelLimiter, err := ratelimit.New(os.Getenv("REDIS_URL"))
@@ -59,9 +70,10 @@ func main() {
SubmitHTTPClient: inbound.NewAPIHTTPClient(inboundConcurrency), SubmitHTTPClient: inbound.NewAPIHTTPClient(inboundConcurrency),
PresenceStore: presenceStore, PresenceStore: presenceStore,
RecoveryStore: recoveryStore, RecoveryStore: recoveryStore,
GatewayInstanceID: getenv("GATEWAY_INSTANCE_ID", hostname()), GatewayInstanceID: gatewayInstanceID,
MaxSubmitConcurrency: inboundConcurrency, MaxSubmitConcurrency: inboundConcurrency,
SecurityEventToken: os.Getenv("SECURITY_EVENT_TOKEN"), SecurityEventToken: os.Getenv("SECURITY_EVENT_TOKEN"),
ProtocolLogPublisher: protocolLogPublisher,
}).ListenAndServe(); err != nil { }).ListenAndServe(); err != nil {
log.Fatalf("gateway inbound server stopped: %v", err) log.Fatalf("gateway inbound server stopped: %v", err)
} }
@@ -83,8 +95,14 @@ func main() {
resultOutbox.Consumer = getenv("GATEWAY_SUBMIT_RESULT_CONSUMER", "gateway-1") resultOutbox.Consumer = getenv("GATEWAY_SUBMIT_RESULT_CONSUMER", "gateway-1")
resultOutbox.APIBaseURL = callbackBaseURL resultOutbox.APIBaseURL = callbackBaseURL
resultOutbox.Concurrency = positiveEnvInt("GATEWAY_SUBMIT_RESULT_WORKER_CONCURRENCY", 8) resultOutbox.Concurrency = positiveEnvInt("GATEWAY_SUBMIT_RESULT_WORKER_CONCURRENCY", 8)
resultOutbox.BatchEnabled = os.Getenv("GATEWAY_CALLBACK_BATCH_ENABLED") != "false"
resultOutbox.BatchSize = positiveEnvInt("GATEWAY_CALLBACK_BATCH_SIZE", 50)
resultOutbox.BatchWait = time.Duration(positiveEnvInt("GATEWAY_CALLBACK_BATCH_WAIT_MS", 10)) * time.Millisecond
resultOutbox.GatewayInstanceID = gatewayInstanceID
resultOutbox.DeadLetterStream = getenv("GATEWAY_CALLBACK_DEAD_LETTER_STREAM", "gateway.submit.results.dead")
worker.ResultOutbox = resultOutbox worker.ResultOutbox = resultOutbox
upstreamManager.SubmitSegmentPublisher = resultOutbox upstreamManager.SubmitSegmentPublisher = resultOutbox
upstreamManager.EventPublisher = resultOutbox
go func() { go func() {
log.Printf("cmpp gateway submit worker consuming stream=%s group=%s consumer=%s", worker.Stream, worker.Group, worker.Consumer) log.Printf("cmpp gateway submit worker consuming stream=%s group=%s consumer=%s", worker.Stream, worker.Group, worker.Consumer)
if err := worker.Run(context.Background()); err != nil { if err := worker.Run(context.Background()); err != nil {
@@ -108,6 +126,13 @@ func main() {
UpstreamDesired: desired, UpstreamConnected: connected, UpstreamDesired: desired, UpstreamConnected: connected,
DownstreamConnected: inbound.ActiveConnectionCount(), SubmitWorkerUp: worker != nil, DownstreamConnected: inbound.ActiveConnectionCount(), SubmitWorkerUp: worker != nil,
} }
snapshot.UpstreamWindowConfigured, snapshot.UpstreamWindowInFlight = upstreamManager.WindowCounts()
if protocolLogPublisher != nil {
snapshot.ProtocolLogPublished, snapshot.ProtocolLogSampled, snapshot.ProtocolLogErrors = protocolLogPublisher.Counts()
if length, err := protocolLogPublisher.Redis.XLen(ctx, protocolLogPublisher.StreamName()).Result(); err == nil {
snapshot.ProtocolLogQueueLength = length
}
}
if worker != nil { if worker != nil {
snapshot.SubmitWorkerConcurrency = worker.ConfiguredConcurrency() snapshot.SubmitWorkerConcurrency = worker.ConfiguredConcurrency()
snapshot.SubmitWorkerInFlight = worker.InFlight() snapshot.SubmitWorkerInFlight = worker.InFlight()
@@ -116,6 +141,7 @@ func main() {
snapshot.ResultWorkerUp = true snapshot.ResultWorkerUp = true
snapshot.ResultWorkerConcurrency = resultOutbox.ConfiguredConcurrency() snapshot.ResultWorkerConcurrency = resultOutbox.ConfiguredConcurrency()
snapshot.ResultWorkerInFlight = resultOutbox.InFlight() snapshot.ResultWorkerInFlight = resultOutbox.InFlight()
snapshot.CallbackBatchRequests, snapshot.CallbackBatchEvents, snapshot.CallbackBatchRetries, snapshot.CallbackDeadLetters = resultOutbox.BatchCounts()
} }
snapshot.InboundSubmitConcurrency, snapshot.InboundSubmitInFlight = inbound.SubmitSlotSnapshot() snapshot.InboundSubmitConcurrency, snapshot.InboundSubmitInFlight = inbound.SubmitSlotSnapshot()
if worker == nil || worker.Redis == nil { if worker == nil || worker.Redis == nil {
+14
View File
@@ -41,6 +41,10 @@ type ChannelConfig struct {
WindowSize int `json:"windowSize,omitempty"` WindowSize int `json:"windowSize,omitempty"`
HeartbeatIntervalSeconds int `json:"heartbeatIntervalSeconds,omitempty"` HeartbeatIntervalSeconds int `json:"heartbeatIntervalSeconds,omitempty"`
HeartbeatMissThreshold int `json:"heartbeatMissThreshold,omitempty"` HeartbeatMissThreshold int `json:"heartbeatMissThreshold,omitempty"`
ConnectionWarmupSeconds int `json:"connectionWarmupSeconds,omitempty"`
ConnectionDrainSeconds int `json:"connectionDrainTimeoutSeconds,omitempty"`
SubmitTimeoutSeconds int `json:"submitResponseTimeoutSeconds,omitempty"`
FailureCooldownSeconds int `json:"connectionFailureCooldownSeconds,omitempty"`
} }
type DisconnectChannelCommand struct { type DisconnectChannelCommand struct {
@@ -351,6 +355,10 @@ func (s Server) connectChannel(ctx context.Context, command ConnectChannelComman
WindowSize: command.Channel.WindowSize, WindowSize: command.Channel.WindowSize,
HeartbeatIntervalSeconds: command.Channel.HeartbeatIntervalSeconds, HeartbeatIntervalSeconds: command.Channel.HeartbeatIntervalSeconds,
HeartbeatMissThreshold: command.Channel.HeartbeatMissThreshold, HeartbeatMissThreshold: command.Channel.HeartbeatMissThreshold,
ConnectionWarmupSeconds: command.Channel.ConnectionWarmupSeconds,
ConnectionDrainSeconds: command.Channel.ConnectionDrainSeconds,
SubmitTimeoutSeconds: command.Channel.SubmitTimeoutSeconds,
FailureCooldownSeconds: command.Channel.FailureCooldownSeconds,
}, },
}) })
if err != nil { if err != nil {
@@ -411,6 +419,12 @@ func validateConnectChannelCommand(command ConnectChannelCommand) error {
if command.Channel.Account == "" || command.Channel.PasswordCipher == "" { if command.Channel.Account == "" || command.Channel.PasswordCipher == "" {
return fmt.Errorf("account and passwordCipher are required") return fmt.Errorf("account and passwordCipher are required")
} }
if command.DesiredConnections < 1 || command.DesiredConnections > 8 {
return fmt.Errorf("desiredConnections must be between 1 and 8")
}
if command.Channel.WindowSize != 0 && (command.Channel.WindowSize < 1 || command.Channel.WindowSize > 64) {
return fmt.Errorf("windowSize must be between 1 and 64")
}
return nil return nil
} }
+16
View File
@@ -140,6 +140,22 @@ func TestConnectChannelRejectsInvalidCommand(t *testing.T) {
} }
} }
func TestConnectChannelRejectsCapacityOutsideSupportedMatrix(t *testing.T) {
handler := handlerWithConnect(func(context.Context, ConnectChannelCommand) (ConnectionStateCallback, error) {
return ConnectionStateCallback{}, nil
})
for _, body := range []string{
`{"schemaVersion":"v1","messageType":"ConnectChannel","channelId":"c","connectionId":"x","desiredConnections":9,"channel":{"gatewayHost":"127.0.0.1","gatewayPort":17890,"account":"a","passwordCipher":"p","windowSize":16}}`,
`{"schemaVersion":"v1","messageType":"ConnectChannel","channelId":"c","connectionId":"x","desiredConnections":1,"channel":{"gatewayHost":"127.0.0.1","gatewayPort":17890,"account":"a","passwordCipher":"p","windowSize":65}}`,
} {
response := httptest.NewRecorder()
handler.ServeHTTP(response, httptest.NewRequest(http.MethodPost, "/connections/connect", strings.NewReader(body)))
if response.Code != http.StatusBadRequest {
t.Fatalf("status=%d body=%s", response.Code, response.Body.String())
}
}
}
func TestDisconnectChannelStopsSupplierPool(t *testing.T) { func TestDisconnectChannelStopsSupplierPool(t *testing.T) {
var received DisconnectChannelCommand var received DisconnectChannelCommand
handler := handlerWithServer(Server{ handler := handlerWithServer(Server{
+6 -5
View File
@@ -18,6 +18,7 @@ const defaultDownstreamAckTimeout = 30 * time.Second
type downstreamAckTracker struct { type downstreamAckTracker struct {
deliveryID string deliveryID string
claimID string
connectionID string connectionID string
sequenceID uint32 sequenceID uint32
messageID uint64 messageID uint64
@@ -69,29 +70,29 @@ func downstreamAckKey(conn *cmpp.Conn, sequenceID uint32) string {
return fmt.Sprintf("%p:%d", conn, sequenceID) return fmt.Sprintf("%p:%d", conn, sequenceID)
} }
func registerDownstreamAck(session *downstreamSession, deliveryID string, sequenceID uint32, messageID uint64, deadline time.Time) *downstreamAckTracker { func registerDownstreamAck(session *downstreamSession, deliveryID string, claimID string, sequenceID uint32, messageID uint64, deadline time.Time) *downstreamAckTracker {
if session == nil || session.conn == nil || strings.TrimSpace(deliveryID) == "" { if session == nil || session.conn == nil || strings.TrimSpace(deliveryID) == "" {
return nil return nil
} }
tracker := &downstreamAckTracker{ tracker := &downstreamAckTracker{
deliveryID: deliveryID, connectionID: session.connectionID, deliveryID: deliveryID, claimID: claimID, connectionID: session.connectionID,
sequenceID: sequenceID, messageID: messageID, session: session, sequenceID: sequenceID, messageID: messageID, session: session,
} }
key := downstreamAckKey(session.conn, sequenceID) key := downstreamAckKey(session.conn, sequenceID)
downstreamAckRegistry.Lock() downstreamAckRegistry.Lock()
downstreamAckRegistry.items[key] = tracker downstreamAckRegistry.items[key] = tracker
downstreamAckRegistry.Unlock()
tracker.timer = time.AfterFunc(time.Until(deadline), func() { tracker.timer = time.AfterFunc(time.Until(deadline), func() {
timedOut := takeDownstreamAck(session.conn, sequenceID) timedOut := takeDownstreamAck(session.conn, sequenceID)
if timedOut == nil || timedOut.session == nil || timedOut.session.deliveryReport == nil { if timedOut == nil || timedOut.session == nil || timedOut.session.deliveryReport == nil {
return return
} }
timedOut.session.deliveryReport(downstreamDeliveryLifecycleEvent{ timedOut.session.deliveryReport(downstreamDeliveryLifecycleEvent{
Kind: "failed", DeliveryID: timedOut.deliveryID, ConnectionID: timedOut.connectionID, Kind: "failed", DeliveryID: timedOut.deliveryID, ClaimID: timedOut.claimID, ConnectionID: timedOut.connectionID,
SequenceID: timedOut.sequenceID, MessageID: timedOut.messageID, ObservedAt: time.Now().UTC(), SequenceID: timedOut.sequenceID, MessageID: timedOut.messageID, ObservedAt: time.Now().UTC(),
FailureType: "ack_timeout", ErrorMessage: "CMPP_DELIVER_RESP timeout", FailureType: "ack_timeout", ErrorMessage: "CMPP_DELIVER_RESP timeout",
}) })
}) })
downstreamAckRegistry.Unlock()
return tracker return tracker
} }
@@ -149,7 +150,7 @@ func handleDownstreamAcknowledgement(conn *cmpp.Conn, sequenceID uint32, message
} }
if tracker.session != nil && tracker.session.deliveryReport != nil { if tracker.session != nil && tracker.session.deliveryReport != nil {
go tracker.session.deliveryReport(downstreamDeliveryLifecycleEvent{ go tracker.session.deliveryReport(downstreamDeliveryLifecycleEvent{
Kind: "acknowledged", DeliveryID: tracker.deliveryID, ConnectionID: tracker.connectionID, Kind: "acknowledged", DeliveryID: tracker.deliveryID, ClaimID: tracker.claimID, ConnectionID: tracker.connectionID,
SequenceID: sequenceID, MessageID: messageID, Result: result, ObservedAt: time.Now().UTC(), SequenceID: sequenceID, MessageID: messageID, Result: result, ObservedAt: time.Now().UTC(),
}) })
} }
+9 -6
View File
@@ -16,6 +16,7 @@ import (
type DownstreamReceipt struct { type DownstreamReceipt struct {
DeliveryID string `json:"deliveryId,omitempty"` DeliveryID string `json:"deliveryId,omitempty"`
ClaimID string `json:"claimId,omitempty"`
Account string `json:"account,omitempty"` Account string `json:"account,omitempty"`
ApplicationID string `json:"applicationId,omitempty"` ApplicationID string `json:"applicationId,omitempty"`
MessageID string `json:"messageId"` MessageID string `json:"messageId"`
@@ -31,6 +32,7 @@ type DownstreamReceipt struct {
type DownstreamUplink struct { type DownstreamUplink struct {
DeliveryID string `json:"deliveryId,omitempty"` DeliveryID string `json:"deliveryId,omitempty"`
ClaimID string `json:"claimId,omitempty"`
Account string `json:"account,omitempty"` Account string `json:"account,omitempty"`
ApplicationID string `json:"applicationId,omitempty"` ApplicationID string `json:"applicationId,omitempty"`
MessageID string `json:"messageId,omitempty"` MessageID string `json:"messageId,omitempty"`
@@ -55,6 +57,7 @@ type DownstreamSendResult struct {
type downstreamDeliveryLifecycleEvent struct { type downstreamDeliveryLifecycleEvent struct {
Kind string Kind string
DeliveryID string DeliveryID string
ClaimID string
ConnectionID string ConnectionID string
SequenceID uint32 SequenceID uint32
MessageID uint64 MessageID uint64
@@ -70,7 +73,7 @@ func (s Server) reportDownstreamDelivery(event downstreamDeliveryLifecycleEvent)
return return
} }
payload := map[string]any{ payload := map[string]any{
"id": event.DeliveryID, "connectionId": event.ConnectionID, "id": event.DeliveryID, "claimId": event.ClaimID, "connectionId": event.ConnectionID,
"sequenceId": strconv.FormatUint(uint64(event.SequenceID), 10), "sequenceId": strconv.FormatUint(uint64(event.SequenceID), 10),
"messageId": strconv.FormatUint(event.MessageID, 10), "messageId": strconv.FormatUint(event.MessageID, 10),
} }
@@ -160,7 +163,7 @@ func pushReceiptWithResult(event DownstreamReceipt, allowRecovery bool) (Downstr
return DownstreamSendResult{}, err return DownstreamSendResult{}, err
} }
deliver := downstreamDeliverPacket(session, receiptMessageID, session.srcID, defaultString(event.PhoneNumber, session.phoneNumber), 0, 1, string(receiptBytes)) deliver := downstreamDeliverPacket(session, receiptMessageID, session.srcID, defaultString(event.PhoneNumber, session.phoneNumber), 0, 1, string(receiptBytes))
return sendDownstream(session, deliver, event.DeliveryID) return sendDownstream(session, deliver, event.DeliveryID, event.ClaimID)
} }
func downstreamReceiptMessageID(event DownstreamReceipt, session *downstreamSession) uint64 { func downstreamReceiptMessageID(event DownstreamReceipt, session *downstreamSession) uint64 {
@@ -228,7 +231,7 @@ func PushUplinkWithResult(event DownstreamUplink) (DownstreamSendResult, error)
0, 0,
content, content,
) )
return sendDownstream(session, deliver, event.DeliveryID) return sendDownstream(session, deliver, event.DeliveryID, event.ClaimID)
} }
func errorMessageWithCode(message string, code string) string { func errorMessageWithCode(message string, code string) string {
@@ -272,7 +275,7 @@ func findSession(messageID string, account string) *downstreamSession {
return nil return nil
} }
func sendDownstream(session *downstreamSession, deliver cmpp.Packer, deliveryID string) (DownstreamSendResult, error) { func sendDownstream(session *downstreamSession, deliver cmpp.Packer, deliveryID string, claimID string) (DownstreamSendResult, error) {
session.mu.Lock() session.mu.Lock()
defer session.mu.Unlock() defer session.mu.Unlock()
messageID := downstreamDeliverMessageID(deliver) messageID := downstreamDeliverMessageID(deliver)
@@ -289,7 +292,7 @@ func sendDownstream(session *downstreamSession, deliver cmpp.Packer, deliveryID
SentAt: formatRFC3339Nano(sentAt), SentAt: formatRFC3339Nano(sentAt),
AckDeadlineAt: formatRFC3339Nano(ackDeadlineAt), AckDeadlineAt: formatRFC3339Nano(ackDeadlineAt),
} }
tracker := registerDownstreamAck(session, deliveryID, sequenceID, messageID, ackDeadlineAt) tracker := registerDownstreamAck(session, deliveryID, claimID, sequenceID, messageID, ackDeadlineAt)
if err := session.conn.SendPkt(deliver, sequenceID); err != nil { if err := session.conn.SendPkt(deliver, sequenceID); err != nil {
removeDownstreamAck(tracker) removeDownstreamAck(tracker)
session.recordDownstreamProtocol(deliver, deliveryID, sequenceID, messageID, "failed", "SEND_FAILED", err) session.recordDownstreamProtocol(deliver, deliveryID, sequenceID, messageID, "failed", "SEND_FAILED", err)
@@ -306,7 +309,7 @@ func sendDownstream(session *downstreamSession, deliver cmpp.Packer, deliveryID
result.Sent = true result.Sent = true
if deliveryID != "" && session.deliveryReport != nil { if deliveryID != "" && session.deliveryReport != nil {
go session.deliveryReport(downstreamDeliveryLifecycleEvent{ go session.deliveryReport(downstreamDeliveryLifecycleEvent{
Kind: "sent", DeliveryID: deliveryID, ConnectionID: session.connectionID, Kind: "sent", DeliveryID: deliveryID, ClaimID: claimID, ConnectionID: session.connectionID,
SequenceID: sequenceID, MessageID: messageID, ObservedAt: sentAt, AckDeadlineAt: ackDeadlineAt, SequenceID: sequenceID, MessageID: messageID, ObservedAt: sentAt, AckDeadlineAt: ackDeadlineAt,
}) })
} }
+93 -3
View File
@@ -6,6 +6,7 @@ import (
"fmt" "fmt"
"log" "log"
"strings" "strings"
"sync"
"time" "time"
) )
@@ -15,6 +16,8 @@ import (
type pendingDeliveryRequest struct { type pendingDeliveryRequest struct {
Account string `json:"account"` Account string `json:"account"`
Limit int `json:"limit,omitempty"` Limit int `json:"limit,omitempty"`
ClaimID string `json:"claimId"`
LeaseMS int `json:"leaseMs"`
} }
type pendingDelivery struct { type pendingDelivery struct {
@@ -22,6 +25,7 @@ type pendingDelivery struct {
DeliveryType string `json:"deliveryType"` DeliveryType string `json:"deliveryType"`
Payload json.RawMessage `json:"payload"` Payload json.RawMessage `json:"payload"`
CreatedAt time.Time `json:"createdAt"` CreatedAt time.Time `json:"createdAt"`
ClaimID string `json:"claimId"`
} }
type pendingFlushResult struct { type pendingFlushResult struct {
@@ -33,25 +37,96 @@ type pendingFlushResult struct {
LastError string LastError string
} }
type pendingFlushCall struct {
done chan struct{}
result pendingFlushResult
err error
}
var pendingFlushSingleflight = struct {
sync.Mutex
byAccount map[string]*pendingFlushCall
}{byAccount: make(map[string]*pendingFlushCall)}
type pendingFlushSchedule struct {
first time.Time
timer *time.Timer
}
var pendingFlushDebouncer = struct {
sync.Mutex
byAccount map[string]*pendingFlushSchedule
}{byAccount: make(map[string]*pendingFlushSchedule)}
func (s Server) schedulePendingFlush(account string, logger *log.Logger) {
account = strings.TrimSpace(account)
if account == "" {
return
}
pendingFlushDebouncer.Lock()
if scheduled := pendingFlushDebouncer.byAccount[account]; scheduled != nil {
if time.Since(scheduled.first) < 250*time.Millisecond {
scheduled.timer.Reset(25 * time.Millisecond)
}
pendingFlushDebouncer.Unlock()
return
}
scheduled := &pendingFlushSchedule{first: time.Now()}
scheduled.timer = time.AfterFunc(25*time.Millisecond, func() {
pendingFlushDebouncer.Lock()
if pendingFlushDebouncer.byAccount[account] == scheduled {
delete(pendingFlushDebouncer.byAccount, account)
}
pendingFlushDebouncer.Unlock()
if _, err := s.flushPending(account, logger); err != nil {
logger.Printf("cmpp inbound event=scheduled_pending_flush_failed account=%s error=%q", account, err.Error())
}
})
pendingFlushDebouncer.byAccount[account] = scheduled
pendingFlushDebouncer.Unlock()
}
func (s Server) flushPending(account string, logger *log.Logger) (pendingFlushResult, error) { func (s Server) flushPending(account string, logger *log.Logger) (pendingFlushResult, error) {
account = strings.TrimSpace(account)
result := pendingFlushResult{Account: account} result := pendingFlushResult{Account: account}
if account == "" { if account == "" {
return result, nil return result, nil
} }
pendingFlushSingleflight.Lock()
if active := pendingFlushSingleflight.byAccount[account]; active != nil {
pendingFlushSingleflight.Unlock()
<-active.done
return active.result, active.err
}
call := &pendingFlushCall{done: make(chan struct{})}
pendingFlushSingleflight.byAccount[account] = call
pendingFlushSingleflight.Unlock()
call.result, call.err = s.flushPendingClaimed(account, logger)
pendingFlushSingleflight.Lock()
delete(pendingFlushSingleflight.byAccount, account)
close(call.done)
pendingFlushSingleflight.Unlock()
return call.result, call.err
}
func (s Server) flushPendingClaimed(account string, logger *log.Logger) (pendingFlushResult, error) {
result := pendingFlushResult{Account: account}
claimID := fmt.Sprintf("%s:%s:%d", s.gatewayInstanceID(), account, time.Now().UnixNano())
var deliveries []pendingDelivery var deliveries []pendingDelivery
if err := s.post(context.Background(), "/gateway/events/downstream/pending", pendingDeliveryRequest{Account: account, Limit: 100}, &deliveries); err != nil { if err := s.post(context.Background(), "/gateway/events/downstream/pending", pendingDeliveryRequest{Account: account, Limit: 100, ClaimID: claimID, LeaseMS: 30000}, &deliveries); err != nil {
logger.Printf("cmpp inbound pending delivery fetch failed account=%s err=%v", account, err) logger.Printf("cmpp inbound pending delivery fetch failed account=%s err=%v", account, err)
result.LastError = err.Error() result.LastError = err.Error()
return result, err return result, err
} }
result.Deliveries = len(deliveries) result.Deliveries = len(deliveries)
for _, delivery := range deliveries { for _, delivery := range deliveries {
delivery.ClaimID = defaultString(delivery.ClaimID, claimID)
sendResult, err := s.pushPendingDelivery(account, delivery) sendResult, err := s.pushPendingDelivery(account, delivery)
if err != nil { if err != nil {
result.FailedCount++ result.FailedCount++
result.LastError = err.Error() result.LastError = err.Error()
_ = s.post(context.Background(), "/gateway/events/downstream/failed", map[string]any{ _ = s.post(context.Background(), "/gateway/events/downstream/failed", map[string]any{
"id": delivery.ID, "errorMessage": err.Error(), "failureType": "send_failed", "id": delivery.ID, "claimId": delivery.ClaimID, "errorMessage": err.Error(), "failureType": "send_failed",
"connectionId": sendResult.ConnectionID, "sequenceId": sendResult.SequenceID, "connectionId": sendResult.ConnectionID, "sequenceId": sendResult.SequenceID,
"messageId": sendResult.MessageID, "sentAt": sendResult.SentAt, "messageId": sendResult.MessageID, "sentAt": sendResult.SentAt,
}, nil) }, nil)
@@ -63,6 +138,7 @@ func (s Server) flushPending(account string, logger *log.Logger) (pendingFlushRe
} }
if sendResult.ReasonCode == "SUBMIT_RESPONSE_PENDING" { if sendResult.ReasonCode == "SUBMIT_RESPONSE_PENDING" {
result.WaitingCount++ result.WaitingCount++
_ = s.releasePendingClaim(delivery, sendResult)
continue continue
} }
errorMessage := defaultString(sendResult.ErrorMessage, "gateway did not complete downstream delivery") errorMessage := defaultString(sendResult.ErrorMessage, "gateway did not complete downstream delivery")
@@ -74,8 +150,12 @@ func (s Server) flushPending(account string, logger *log.Logger) (pendingFlushRe
result.FailedCount++ result.FailedCount++
} }
result.LastError = errorMessage result.LastError = errorMessage
if sendResult.Retryable {
_ = s.releasePendingClaim(delivery, sendResult)
continue
}
_ = s.post(context.Background(), "/gateway/events/downstream/failed", map[string]any{ _ = s.post(context.Background(), "/gateway/events/downstream/failed", map[string]any{
"id": delivery.ID, "errorMessage": errorMessageWithCode(errorMessage, sendResult.ReasonCode), "id": delivery.ID, "claimId": delivery.ClaimID, "errorMessage": errorMessageWithCode(errorMessage, sendResult.ReasonCode),
"failureType": failureType, "connectionId": sendResult.ConnectionID, "failureType": failureType, "connectionId": sendResult.ConnectionID,
"sequenceId": sendResult.SequenceID, "messageId": sendResult.MessageID, "sentAt": sendResult.SentAt, "sequenceId": sendResult.SequenceID, "messageId": sendResult.MessageID, "sentAt": sendResult.SentAt,
}, nil) }, nil)
@@ -83,6 +163,14 @@ func (s Server) flushPending(account string, logger *log.Logger) (pendingFlushRe
return result, nil return result, nil
} }
func (s Server) releasePendingClaim(delivery pendingDelivery, sendResult DownstreamSendResult) error {
return s.post(context.Background(), "/gateway/events/downstream/failed", map[string]any{
"id": delivery.ID, "claimId": delivery.ClaimID,
"errorMessage": errorMessageWithCode(defaultString(sendResult.ErrorMessage, "Gateway released downstream delivery claim"), sendResult.ReasonCode),
"failureType": "claim_released",
}, nil)
}
func (s Server) pushPendingDelivery(account string, delivery pendingDelivery) (DownstreamSendResult, error) { func (s Server) pushPendingDelivery(account string, delivery pendingDelivery) (DownstreamSendResult, error) {
switch delivery.DeliveryType { switch delivery.DeliveryType {
case "receipt": case "receipt":
@@ -91,6 +179,7 @@ func (s Server) pushPendingDelivery(account string, delivery pendingDelivery) (D
return DownstreamSendResult{}, err return DownstreamSendResult{}, err
} }
event.DeliveryID = delivery.ID event.DeliveryID = delivery.ID
event.ClaimID = delivery.ClaimID
event.Account = defaultString(event.Account, account) event.Account = defaultString(event.Account, account)
allowRecovery := !delivery.CreatedAt.IsZero() && time.Since(delivery.CreatedAt) >= 5*time.Second allowRecovery := !delivery.CreatedAt.IsZero() && time.Since(delivery.CreatedAt) >= 5*time.Second
return pushReceiptWithResult(event, allowRecovery) return pushReceiptWithResult(event, allowRecovery)
@@ -100,6 +189,7 @@ func (s Server) pushPendingDelivery(account string, delivery pendingDelivery) (D
return DownstreamSendResult{}, err return DownstreamSendResult{}, err
} }
event.DeliveryID = delivery.ID event.DeliveryID = delivery.ID
event.ClaimID = delivery.ClaimID
event.Account = defaultString(event.Account, account) event.Account = defaultString(event.Account, account)
return PushUplinkWithResult(event) return PushUplinkWithResult(event)
default: default:
+11 -14
View File
@@ -1,6 +1,7 @@
package inbound package inbound
import ( import (
"cmpp-platform/gateway/internal/protocollog"
"context" "context"
"fmt" "fmt"
cmpp "github.com/bigwhite/gocmpp" cmpp "github.com/bigwhite/gocmpp"
@@ -8,20 +9,7 @@ import (
"strconv" "strconv"
) )
type protocolLogEvent struct { type protocolLogEvent = protocollog.Event
Protocol string `json:"protocol"`
Direction string `json:"direction"`
EventType string `json:"eventType"`
Status string `json:"status"`
TenantID string `json:"tenantId,omitempty"`
ApplicationID string `json:"applicationId,omitempty"`
Account string `json:"account,omitempty"`
MessageID string `json:"messageId,omitempty"`
GatewayMessageID string `json:"gatewayMessageId,omitempty"`
Phone string `json:"phone,omitempty"`
ResultCode string `json:"resultCode,omitempty"`
Detail map[string]any `json:"detail,omitempty"`
}
func (s Server) submitResponseProtocolLogger( func (s Server) submitResponseProtocolLogger(
account string, account string,
@@ -71,6 +59,15 @@ func protocolSubmitResponseDetail(sequenceID uint32, sendErr error) map[string]a
} }
func (s Server) emitProtocolLog(event protocolLogEvent) { func (s Server) emitProtocolLog(event protocolLogEvent) {
event.GatewayInstanceID = s.GatewayInstanceID
if s.ProtocolLogPublisher != nil {
go func() {
if err := s.ProtocolLogPublisher.Publish(context.Background(), event); err != nil {
log.Printf("cmpp inbound protocol log Redis publish failed account=%s message_id=%s error=%q", event.Account, event.MessageID, err)
}
}()
return
}
go func() { go func() {
ctx, cancel := context.WithTimeout(context.Background(), defaultHTTPTimeout) ctx, cancel := context.WithTimeout(context.Background(), defaultHTTPTimeout)
defer cancel() defer cancel()
+5
View File
@@ -1,6 +1,8 @@
package inbound package inbound
import ( import (
"cmpp-platform/gateway/internal/protocollog"
"context"
cmpp "github.com/bigwhite/gocmpp" cmpp "github.com/bigwhite/gocmpp"
"io" "io"
"log" "log"
@@ -22,6 +24,9 @@ type Server struct {
RecoveryStore RecoveryStore RecoveryStore RecoveryStore
GatewayInstanceID string GatewayInstanceID string
MaxSubmitConcurrency int MaxSubmitConcurrency int
ProtocolLogPublisher interface {
Publish(context.Context, protocollog.Event) error
}
} }
func (s Server) ListenAndServe() error { func (s Server) ListenAndServe() error {
+39 -2
View File
@@ -930,6 +930,42 @@ func TestFlushOnlineAccountsFetchesPendingDeliveries(t *testing.T) {
} }
} }
func TestFlushPendingCoalescesConcurrentRequestsPerAccount(t *testing.T) {
var calls atomic.Int32
release := make(chan struct{})
api := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/api/gateway/events/downstream/pending" {
t.Fatalf("unexpected api path: %s", r.URL.Path)
}
calls.Add(1)
<-release
_ = json.NewEncoder(w).Encode([]pendingDelivery{})
}))
defer api.Close()
server := Server{APIBaseURL: api.URL + "/api", GatewayInstanceID: "gateway-a"}
results := make(chan error, 8)
for range 8 {
go func() {
_, err := server.flushPending("100001", log.Default())
results <- err
}()
}
deadline := time.Now().Add(time.Second)
for calls.Load() == 0 && time.Now().Before(deadline) {
time.Sleep(time.Millisecond)
}
close(release)
for range 8 {
if err := <-results; err != nil {
t.Fatalf("flush failed: %v", err)
}
}
if got := calls.Load(); got != 1 {
t.Fatalf("pending fetch calls = %d, want 1", got)
}
}
func TestRecoverPendingCandidatesFetchesPresenceAccounts(t *testing.T) { func TestRecoverPendingCandidatesFetchesPresenceAccounts(t *testing.T) {
resetDownstreamRegistry() resetDownstreamRegistry()
defer resetDownstreamRegistry() defer resetDownstreamRegistry()
@@ -1085,7 +1121,7 @@ func TestDownstreamDeliveryRequiresAcknowledgement(t *testing.T) {
messageID: "MSG-LONG-1", phoneNumber: "18821203795", messageID: "MSG-LONG-1", phoneNumber: "18821203795",
protocolLog: func(event protocolLogEvent) { protocolEvents <- event }, protocolLog: func(event protocolLogEvent) { protocolEvents <- event },
} }
registerDownstreamAck(session, "delivery-1", 37, 9016479179509871733, time.Now().Add(time.Second)) registerDownstreamAck(session, "delivery-1", "", 37, 9016479179509871733, time.Now().Add(time.Second))
handleDownstreamAcknowledgement(conn, 37, 9016479179509871733, 0, log.Default()) handleDownstreamAcknowledgement(conn, 37, 9016479179509871733, 0, log.Default())
select { select {
@@ -1165,6 +1201,7 @@ func TestSendDownstreamRejectsZeroMessageID(t *testing.T) {
&downstreamSession{mu: &sync.Mutex{}}, &downstreamSession{mu: &sync.Mutex{}},
&cmpp.Cmpp2DeliverReqPkt{MsgId: 0}, &cmpp.Cmpp2DeliverReqPkt{MsgId: 0},
"delivery-zero", "delivery-zero",
"",
) )
if err == nil || !strings.Contains(err.Error(), "Msg_Id=0") { if err == nil || !strings.Contains(err.Error(), "Msg_Id=0") {
t.Fatalf("expected zero Msg_Id rejection, got %v", err) t.Fatalf("expected zero Msg_Id rejection, got %v", err)
@@ -1180,7 +1217,7 @@ func TestDownstreamDeliveryReportsAckTimeout(t *testing.T) {
conn: &cmpp.Conn{}, connectionID: "conn-1", conn: &cmpp.Conn{}, connectionID: "conn-1",
deliveryReport: func(event downstreamDeliveryLifecycleEvent) { events <- event }, deliveryReport: func(event downstreamDeliveryLifecycleEvent) { events <- event },
} }
registerDownstreamAck(session, "delivery-timeout", 38, 9017467844344255865, time.Now().Add(20*time.Millisecond)) registerDownstreamAck(session, "delivery-timeout", "", 38, 9017467844344255865, time.Now().Add(20*time.Millisecond))
select { select {
case event := <-events: case event := <-events:
+1 -5
View File
@@ -212,11 +212,7 @@ func (s Server) handleSubmit(response *cmpp.Response, packet *cmpp.Packet, logge
if sendErr != nil { if sendErr != nil {
return return
} }
go func() { s.schedulePendingFlush(account, logger)
if _, err := s.flushPending(account, logger); err != nil {
logger.Printf("cmpp inbound event=post_submit_pending_flush_failed account=%s message_id=%s error=%q", account, result.MessageID, err.Error())
}
}()
}) })
logger.Printf( logger.Printf(
"cmpp inbound event=submit_accepted protocol=%s packet_type=%s account=%s remote=%s seq=%d phone=%s dest_count=%d accepted_count=%d result=0 message_id=%s gateway_message_id=%d duration_ms=%d content_chars=%d content_hash=%s", "cmpp inbound event=submit_accepted protocol=%s packet_type=%s account=%s remote=%s seq=%d phone=%s dest_count=%d accepted_count=%d result=0 message_id=%s gateway_message_id=%d duration_ms=%d content_chars=%d content_hash=%s",
+14
View File
@@ -39,6 +39,8 @@ var submitStageHistograms [5][2]durationHistogram
type Snapshot struct { type Snapshot struct {
UpstreamDesired int UpstreamDesired int
UpstreamConnected int UpstreamConnected int
UpstreamWindowConfigured int
UpstreamWindowInFlight int
DownstreamConnected int DownstreamConnected int
SubmitWorkerUp bool SubmitWorkerUp bool
SubmitWorkerConcurrency int SubmitWorkerConcurrency int
@@ -55,6 +57,14 @@ type Snapshot struct {
ResultQueueAvailable bool ResultQueueAvailable bool
ResultQueuePending int64 ResultQueuePending int64
ResultQueueLag int64 ResultQueueLag int64
CallbackBatchRequests int64
CallbackBatchEvents int64
CallbackBatchRetries int64
CallbackDeadLetters int64
ProtocolLogPublished int64
ProtocolLogSampled int64
ProtocolLogErrors int64
ProtocolLogQueueLength int64
} }
type SnapshotFunc func(context.Context) Snapshot type SnapshotFunc func(context.Context) Snapshot
@@ -121,6 +131,10 @@ func Handler(load SnapshotFunc) http.Handler {
writeDurationHistogram(response, "cmpp_gateway_submit_stage_duration_seconds", stage, &submitStageHistograms[index]) writeDurationHistogram(response, "cmpp_gateway_submit_stage_duration_seconds", stage, &submitStageHistograms[index])
} }
fmt.Fprintf(response, "# HELP cmpp_gateway_upstream_connections Desired and live supplier connections.\n# TYPE cmpp_gateway_upstream_connections gauge\ncmpp_gateway_upstream_connections{state=\"desired\"} %d\ncmpp_gateway_upstream_connections{state=\"connected\"} %d\n", snapshot.UpstreamDesired, snapshot.UpstreamConnected) fmt.Fprintf(response, "# HELP cmpp_gateway_upstream_connections Desired and live supplier connections.\n# TYPE cmpp_gateway_upstream_connections gauge\ncmpp_gateway_upstream_connections{state=\"desired\"} %d\ncmpp_gateway_upstream_connections{state=\"connected\"} %d\n", snapshot.UpstreamDesired, snapshot.UpstreamConnected)
fmt.Fprintf(response, "# HELP cmpp_gateway_upstream_window_slots Configured and in-flight supplier window slots.\n# TYPE cmpp_gateway_upstream_window_slots gauge\ncmpp_gateway_upstream_window_slots{state=\"configured\"} %d\ncmpp_gateway_upstream_window_slots{state=\"in_flight\"} %d\n", snapshot.UpstreamWindowConfigured, snapshot.UpstreamWindowInFlight)
fmt.Fprintf(response, "# HELP cmpp_gateway_callback_batch_total Batch callback requests, events, retries and dead letters.\n# TYPE cmpp_gateway_callback_batch_total counter\ncmpp_gateway_callback_batch_total{result=\"requests\"} %d\ncmpp_gateway_callback_batch_total{result=\"events\"} %d\ncmpp_gateway_callback_batch_total{result=\"retries\"} %d\ncmpp_gateway_callback_batch_total{result=\"dead_letter\"} %d\n", snapshot.CallbackBatchRequests, snapshot.CallbackBatchEvents, snapshot.CallbackBatchRetries, snapshot.CallbackDeadLetters)
fmt.Fprintf(response, "# HELP cmpp_gateway_protocol_log_total Protocol log Stream outcomes.\n# TYPE cmpp_gateway_protocol_log_total counter\ncmpp_gateway_protocol_log_total{result=\"published\"} %d\ncmpp_gateway_protocol_log_total{result=\"sampled_out\"} %d\ncmpp_gateway_protocol_log_total{result=\"error\"} %d\n", snapshot.ProtocolLogPublished, snapshot.ProtocolLogSampled, snapshot.ProtocolLogErrors)
fmt.Fprintf(response, "# HELP cmpp_gateway_protocol_log_stream_length Current protocol log Stream length.\n# TYPE cmpp_gateway_protocol_log_stream_length gauge\ncmpp_gateway_protocol_log_stream_length %d\n", snapshot.ProtocolLogQueueLength)
fmt.Fprintf(response, "# HELP cmpp_gateway_downstream_connections Authenticated client connections.\n# TYPE cmpp_gateway_downstream_connections gauge\ncmpp_gateway_downstream_connections %d\n", snapshot.DownstreamConnected) fmt.Fprintf(response, "# HELP cmpp_gateway_downstream_connections Authenticated client connections.\n# TYPE cmpp_gateway_downstream_connections gauge\ncmpp_gateway_downstream_connections %d\n", snapshot.DownstreamConnected)
fmt.Fprintf(response, "# HELP cmpp_gateway_submit_worker_up Whether the submit worker was initialized.\n# TYPE cmpp_gateway_submit_worker_up gauge\ncmpp_gateway_submit_worker_up %d\n", boolNumber(snapshot.SubmitWorkerUp)) fmt.Fprintf(response, "# HELP cmpp_gateway_submit_worker_up Whether the submit worker was initialized.\n# TYPE cmpp_gateway_submit_worker_up gauge\ncmpp_gateway_submit_worker_up %d\n", boolNumber(snapshot.SubmitWorkerUp))
fmt.Fprintf(response, "# HELP cmpp_gateway_submit_worker_slots Configured and active bounded submit worker slots.\n# TYPE cmpp_gateway_submit_worker_slots gauge\ncmpp_gateway_submit_worker_slots{state=\"configured\"} %d\ncmpp_gateway_submit_worker_slots{state=\"in_flight\"} %d\n", snapshot.SubmitWorkerConcurrency, snapshot.SubmitWorkerInFlight) fmt.Fprintf(response, "# HELP cmpp_gateway_submit_worker_slots Configured and active bounded submit worker slots.\n# TYPE cmpp_gateway_submit_worker_slots gauge\ncmpp_gateway_submit_worker_slots{state=\"configured\"} %d\ncmpp_gateway_submit_worker_slots{state=\"in_flight\"} %d\n", snapshot.SubmitWorkerConcurrency, snapshot.SubmitWorkerInFlight)
+128
View File
@@ -0,0 +1,128 @@
package protocollog
import (
"context"
"encoding/json"
"fmt"
"hash/fnv"
"strconv"
"strings"
"sync/atomic"
"time"
"github.com/redis/go-redis/v9"
)
const defaultStream = "gateway.protocol.logs"
type Event struct {
EventID string `json:"eventId"`
GatewayInstanceID string `json:"gatewayInstanceId,omitempty"`
ConnectionID string `json:"connectionId,omitempty"`
Protocol string `json:"protocol"`
Direction string `json:"direction"`
EventType string `json:"eventType"`
Status string `json:"status"`
TenantID string `json:"tenantId,omitempty"`
ApplicationID string `json:"applicationId,omitempty"`
ChannelID string `json:"channelId,omitempty"`
Account string `json:"account,omitempty"`
MessageID string `json:"messageId,omitempty"`
SubmitID string `json:"submitId,omitempty"`
GatewayMessageID string `json:"gatewayMessageId,omitempty"`
Phone string `json:"phone,omitempty"`
ResultCode string `json:"resultCode,omitempty"`
DurationMs int `json:"durationMs,omitempty"`
PayloadBytes int `json:"payloadBytes,omitempty"`
Detail map[string]any `json:"detail,omitempty"`
CreatedAt time.Time `json:"createdAt"`
}
type Publisher struct {
Redis *redis.Client
Stream string
GatewayInstanceID string
SuccessSampleRate int
MaxLen int64
published atomic.Int64
sampled atomic.Int64
errors atomic.Int64
}
func New(redisURL string) (*Publisher, error) {
if strings.TrimSpace(redisURL) == "" {
redisURL = "redis://127.0.0.1:6379"
}
options, err := redis.ParseURL(redisURL)
if err != nil {
return nil, err
}
return &Publisher{Redis: redis.NewClient(options)}, nil
}
func (p *Publisher) Publish(ctx context.Context, event Event) error {
if p == nil || p.Redis == nil {
return fmt.Errorf("protocol log Redis publisher is unavailable")
}
if event.CreatedAt.IsZero() {
event.CreatedAt = time.Now().UTC()
}
if event.GatewayInstanceID == "" {
event.GatewayInstanceID = p.GatewayInstanceID
}
if event.EventID == "" {
event.EventID = fmt.Sprintf("PL-%d-%d", event.CreatedAt.UnixNano(), p.published.Load()+p.sampled.Load()+1)
}
if !p.mustKeep(event) && !p.sample(event) {
p.sampled.Add(1)
return nil
}
data, err := json.Marshal(event)
if err != nil {
p.errors.Add(1)
return err
}
args := &redis.XAddArgs{Stream: p.stream(), Values: map[string]any{"data": string(data)}}
if p.MaxLen > 0 {
args.MaxLen = p.MaxLen
args.Approx = true
}
if err := p.Redis.XAdd(ctx, args).Err(); err != nil {
p.errors.Add(1)
return err
}
p.published.Add(1)
return nil
}
func (p *Publisher) mustKeep(event Event) bool {
if event.Status != "success" {
return true
}
code := strings.TrimSpace(event.ResultCode)
return code != "" && code != "0"
}
func (p *Publisher) sample(event Event) bool {
rate := p.SuccessSampleRate
if rate <= 0 {
rate = 10
}
if rate >= 100 {
return true
}
h := fnv.New32a()
_, _ = h.Write([]byte(event.ChannelID + "|" + event.MessageID + "|" + event.EventType + "|" + strconv.FormatInt(event.CreatedAt.UnixNano()/int64(time.Millisecond), 10)))
return int(h.Sum32()%100) < rate
}
func (p *Publisher) stream() string {
if strings.TrimSpace(p.Stream) != "" {
return p.Stream
}
return defaultStream
}
func (p *Publisher) StreamName() string { return p.stream() }
func (p *Publisher) Counts() (int64, int64, int64) {
return p.published.Load(), p.sampled.Load(), p.errors.Load()
}
@@ -0,0 +1,34 @@
package protocollog
import (
"context"
"testing"
"github.com/alicebob/miniredis/v2"
"github.com/redis/go-redis/v9"
)
func TestNewUsesLocalRedisWhenURLIsEmpty(t *testing.T) {
publisher, err := New("")
if err != nil {
t.Fatalf("New returned error: %v", err)
}
if publisher == nil || publisher.Redis == nil {
t.Fatal("expected an initialized Redis client")
}
if got := publisher.Redis.Options().Addr; got != "127.0.0.1:6379" {
t.Fatalf("expected local Redis fallback, got %q", got)
}
}
func TestFailuresAreNeverSampledOut(t *testing.T) {
mr := miniredis.RunT(t)
client := redis.NewClient(&redis.Options{Addr: mr.Addr()})
p := &Publisher{Redis: client, SuccessSampleRate: 1, MaxLen: 100}
if err := p.Publish(context.Background(), Event{Protocol: "cmpp", EventType: "submit", Status: "failed", ResultCode: "TIMEOUT"}); err != nil {
t.Fatal(err)
}
if client.XLen(context.Background(), p.StreamName()).Val() != 1 {
t.Fatal("failed protocol event must be retained")
}
}
+8
View File
@@ -70,6 +70,10 @@ type UpstreamConfig struct {
WindowSize int `json:"windowSize,omitempty"` WindowSize int `json:"windowSize,omitempty"`
HeartbeatIntervalSeconds int `json:"heartbeatIntervalSeconds,omitempty"` HeartbeatIntervalSeconds int `json:"heartbeatIntervalSeconds,omitempty"`
HeartbeatMissThreshold int `json:"heartbeatMissThreshold,omitempty"` HeartbeatMissThreshold int `json:"heartbeatMissThreshold,omitempty"`
ConnectionWarmupSeconds int `json:"connectionWarmupSeconds,omitempty"`
ConnectionDrainSeconds int `json:"connectionDrainTimeoutSeconds,omitempty"`
SubmitTimeoutSeconds int `json:"submitResponseTimeoutSeconds,omitempty"`
FailureCooldownSeconds int `json:"connectionFailureCooldownSeconds,omitempty"`
} }
type Retry struct { type Retry struct {
@@ -146,6 +150,10 @@ type ConnectChannelConfig struct {
WindowSize int `json:"windowSize,omitempty"` WindowSize int `json:"windowSize,omitempty"`
HeartbeatIntervalSeconds int `json:"heartbeatIntervalSeconds,omitempty"` HeartbeatIntervalSeconds int `json:"heartbeatIntervalSeconds,omitempty"`
HeartbeatMissThreshold int `json:"heartbeatMissThreshold,omitempty"` HeartbeatMissThreshold int `json:"heartbeatMissThreshold,omitempty"`
ConnectionWarmupSeconds int `json:"connectionWarmupSeconds,omitempty"`
ConnectionDrainSeconds int `json:"connectionDrainTimeoutSeconds,omitempty"`
SubmitTimeoutSeconds int `json:"submitResponseTimeoutSeconds,omitempty"`
FailureCooldownSeconds int `json:"connectionFailureCooldownSeconds,omitempty"`
} }
type DisconnectChannelCommand struct { type DisconnectChannelCommand struct {
+170
View File
@@ -0,0 +1,170 @@
package resultoutbox
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"strings"
"time"
"github.com/redis/go-redis/v9"
)
type callbackBatch struct {
BatchID string `json:"batchId"`
GatewayInstanceID string `json:"gatewayInstanceId"`
CreatedAt time.Time `json:"createdAt"`
Events []callbackBatchEvent `json:"events"`
}
type callbackBatchEvent struct {
EventID string `json:"eventId"`
Type string `json:"type"`
Payload json.RawMessage `json:"payload"`
}
type callbackBatchResponse struct {
BatchID string `json:"batchId"`
Results []callbackEventResult `json:"results"`
}
type callbackEventResult struct {
EventID string `json:"eventId"`
Accepted bool `json:"accepted"`
Retryable bool `json:"retryable,omitempty"`
ErrorCode string `json:"errorCode,omitempty"`
}
func (o *Outbox) runBatches(ctx context.Context) error {
for ctx.Err() == nil {
messages, _, err := o.Redis.XAutoClaim(ctx, &redis.XAutoClaimArgs{Stream: o.stream(), Group: o.group(), Consumer: o.consumer(), MinIdle: o.minIdle(), Start: "0-0", Count: int64(o.batchSize())}).Result()
if err != nil && !errors.Is(err, redis.Nil) {
return err
}
if len(messages) == 0 {
streams, readErr := o.Redis.XReadGroup(ctx, &redis.XReadGroupArgs{Group: o.group(), Consumer: o.consumer(), Streams: []string{o.stream(), ">"}, Count: int64(o.batchSize()), Block: o.batchWait()}).Result()
if errors.Is(readErr, redis.Nil) {
continue
}
if readErr != nil {
return readErr
}
for _, stream := range streams {
messages = append(messages, stream.Messages...)
}
}
if len(messages) == 0 {
continue
}
if err := o.processBatch(ctx, messages); err != nil {
o.batchRetries.Add(int64(len(messages)))
sleep(ctx, 100*time.Millisecond)
}
}
return ctx.Err()
}
func (o *Outbox) processBatch(ctx context.Context, messages []redis.XMessage) error {
batch := callbackBatch{BatchID: fmt.Sprintf("CB-%d", time.Now().UnixNano()), GatewayInstanceID: o.GatewayInstanceID, CreatedAt: time.Now().UTC()}
byEvent := make(map[string]redis.XMessage, len(messages))
for _, message := range messages {
event, err := EventFromStreamValues(message.Values)
if err != nil {
_ = o.deadLetter(ctx, message, "INVALID_ENVELOPE", err.Error())
continue
}
batch.Events = append(batch.Events, callbackBatchEvent{EventID: event.EventID, Type: event.EventType, Payload: event.Payload})
byEvent[event.EventID] = message
}
if len(batch.Events) == 0 {
return nil
}
body, err := json.Marshal(batch)
if err != nil {
return err
}
if len(body) > 1024*1024 {
return fmt.Errorf("callback batch exceeds 1MB")
}
req, err := http.NewRequestWithContext(ctx, http.MethodPost, strings.TrimRight(o.APIBaseURL, "/")+"/gateway/events/batch", bytes.NewReader(body))
if err != nil {
return err
}
req.Header.Set("Content-Type", "application/json")
client := &http.Client{Timeout: o.httpTimeout()}
resp, err := client.Do(req)
if err != nil {
return err
}
defer resp.Body.Close()
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
data, _ := io.ReadAll(io.LimitReader(resp.Body, 512))
return fmt.Errorf("batch callback returned %d: %s", resp.StatusCode, strings.TrimSpace(string(data)))
}
var result callbackBatchResponse
if err := json.NewDecoder(io.LimitReader(resp.Body, 1024*1024)).Decode(&result); err != nil {
return err
}
if result.BatchID != batch.BatchID {
return fmt.Errorf("callback batchId mismatch")
}
o.batchRequests.Add(1)
o.batchEvents.Add(int64(len(batch.Events)))
seen := make(map[string]struct{}, len(result.Results))
for _, item := range result.Results {
message, ok := byEvent[item.EventID]
if !ok {
continue
}
seen[item.EventID] = struct{}{}
if item.Accepted {
if err := o.ackDelete(ctx, message.ID); err != nil {
return err
}
continue
}
if !item.Retryable {
if err := o.deadLetter(ctx, message, item.ErrorCode, "non-retryable callback result"); err != nil {
return err
}
}
}
for eventID := range byEvent {
if _, ok := seen[eventID]; !ok {
return fmt.Errorf("batch response omitted event %s", eventID)
}
}
return nil
}
func (o *Outbox) ackDelete(ctx context.Context, id string) error {
return acknowledgeAndDeleteScript.Run(ctx, o.Redis, []string{o.stream()}, o.group(), id).Err()
}
func (o *Outbox) deadLetter(ctx context.Context, message redis.XMessage, code, detail string) error {
stream := o.DeadLetterStream
if stream == "" {
stream = o.stream() + ".dead"
}
pipe := o.Redis.TxPipeline()
pipe.XAdd(ctx, &redis.XAddArgs{Stream: stream, Values: map[string]any{"sourceId": message.ID, "errorCode": code, "detail": detail, "data": fmt.Sprint(message.Values["data"])}})
pipe.XAck(ctx, o.stream(), o.group(), message.ID)
pipe.XDel(ctx, o.stream(), message.ID)
if _, err := pipe.Exec(ctx); err != nil {
return err
}
o.deadLetters.Add(1)
return nil
}
func (o *Outbox) batchSize() int {
if o.BatchSize < 1 {
return 50
}
return min(o.BatchSize, 100)
}
func (o *Outbox) batchWait() time.Duration {
if o.BatchWait <= 0 {
return 10 * time.Millisecond
}
return o.BatchWait
}
+36 -12
View File
@@ -53,16 +53,25 @@ type Event struct {
} }
type Outbox struct { type Outbox struct {
Redis *redis.Client Redis *redis.Client
Stream string Stream string
Group string Group string
Consumer string Consumer string
DedupeTTL time.Duration DedupeTTL time.Duration
APIBaseURL string APIBaseURL string
HTTPTimeout time.Duration HTTPTimeout time.Duration
Concurrency int Concurrency int
MinIdle time.Duration MinIdle time.Duration
inFlight atomic.Int64 BatchEnabled bool
BatchSize int
BatchWait time.Duration
GatewayInstanceID string
DeadLetterStream string
inFlight atomic.Int64
batchRequests atomic.Int64
batchEvents atomic.Int64
batchRetries atomic.Int64
deadLetters atomic.Int64
} }
func New(client *redis.Client) *Outbox { func New(client *redis.Client) *Outbox {
@@ -145,6 +154,18 @@ func (o *Outbox) PublishSubmitResult(ctx context.Context, command queue.SubmitCo
return o.publish(ctx, event) return o.publish(ctx, event)
} }
func (o *Outbox) PublishReceipt(ctx context.Context, event queue.ReceiptEvent) error {
return o.publishRaw(ctx, Event{SchemaVersion: queue.SchemaVersion, EventID: fmt.Sprintf("receipt:%s:%s:%d", event.GatewayMessageID, event.RawStatus, event.SequenceID), EventType: "receipt_intake", Path: "/gateway/events/receipt/intake", TraceID: event.TraceID, MessageID: event.MessageID, ChannelID: event.ChannelID, Payload: mustMarshal(event), CreatedAt: time.Now().UTC()})
}
func (o *Outbox) PublishUplink(ctx context.Context, event queue.UplinkEvent) error {
return o.publishRaw(ctx, Event{SchemaVersion: queue.SchemaVersion, EventID: fmt.Sprintf("uplink:%s:%d:%d", event.ChannelID, event.SequenceID, event.ReceivedAt.UnixNano()), EventType: "uplink", Path: "/gateway/events/uplink", TraceID: event.TraceID, MessageID: event.MessageID, ChannelID: event.ChannelID, Payload: mustMarshal(event), CreatedAt: time.Now().UTC()})
}
func mustMarshal(value any) json.RawMessage { data, _ := json.Marshal(value); return data }
func (o *Outbox) publishRaw(ctx context.Context, event Event) error { return o.publish(ctx, event) }
func (o *Outbox) publish(ctx context.Context, event Event) error { func (o *Outbox) publish(ctx context.Context, event Event) error {
if o.Redis == nil { if o.Redis == nil {
return fmt.Errorf("result Outbox Redis client is required") return fmt.Errorf("result Outbox Redis client is required")
@@ -211,10 +232,10 @@ func EventFromStreamValues(values map[string]interface{}) (Event, error) {
if err := json.Unmarshal([]byte(data), &event); err != nil { if err := json.Unmarshal([]byte(data), &event); err != nil {
return Event{}, err return Event{}, err
} }
if event.SchemaVersion != queue.SchemaVersion || event.EventID == "" || event.MessageID == "" || event.SubmitID == "" { if event.SchemaVersion != queue.SchemaVersion || event.EventID == "" || event.MessageID == "" {
return Event{}, fmt.Errorf("invalid result Outbox envelope") return Event{}, fmt.Errorf("invalid result Outbox envelope")
} }
if event.Path != "/gateway/events/submit-result" && event.Path != "/gateway/events/submit-segment-result" { if event.Path != "/gateway/events/submit-result" && event.Path != "/gateway/events/submit-segment-result" && event.Path != "/gateway/events/receipt/intake" && event.Path != "/gateway/events/uplink" && event.Path != "/gateway/events/dead-letter" {
return Event{}, fmt.Errorf("unsupported result Outbox path %q", event.Path) return Event{}, fmt.Errorf("unsupported result Outbox path %q", event.Path)
} }
if len(event.Payload) == 0 { if len(event.Payload) == 0 {
@@ -258,3 +279,6 @@ func (o *Outbox) dedupeKey(eventID string) string {
func (o *Outbox) StreamName() string { return o.stream() } func (o *Outbox) StreamName() string { return o.stream() }
func (o *Outbox) GroupName() string { return o.group() } func (o *Outbox) GroupName() string { return o.group() }
func (o *Outbox) InFlight() int64 { return o.inFlight.Load() } func (o *Outbox) InFlight() int64 { return o.inFlight.Load() }
func (o *Outbox) BatchCounts() (int64, int64, int64, int64) {
return o.batchRequests.Load(), o.batchEvents.Load(), o.batchRetries.Load(), o.deadLetters.Load()
}
@@ -2,6 +2,7 @@ package resultoutbox
import ( import (
"context" "context"
"encoding/json"
"net/http" "net/http"
"net/http/httptest" "net/http/httptest"
"sync/atomic" "sync/atomic"
@@ -32,6 +33,99 @@ func TestPublishSubmitSegmentIsIdempotent(t *testing.T) {
} }
} }
func TestBatchCallbackSendsMultipleEventsInOneRequest(t *testing.T) {
mr := miniredis.RunT(t)
client := redis.NewClient(&redis.Options{Addr: mr.Addr()})
var requests atomic.Int32
var eventCount atomic.Int32
server := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) {
requests.Add(1)
var batch callbackBatch
if err := json.NewDecoder(request.Body).Decode(&batch); err != nil {
t.Errorf("decode batch: %v", err)
return
}
eventCount.Store(int32(len(batch.Events)))
result := callbackBatchResponse{BatchID: batch.BatchID}
for _, event := range batch.Events {
result.Results = append(result.Results, callbackEventResult{EventID: event.EventID, Accepted: true})
}
_ = json.NewEncoder(response).Encode(result)
}))
defer server.Close()
outbox := New(client)
outbox.APIBaseURL = server.URL
outbox.BatchEnabled = true
outbox.BatchSize = 50
outbox.BatchWait = 10 * time.Millisecond
outbox.GatewayInstanceID = "gateway-test"
command := testCommand()
for index := 1; index <= 2; index++ {
if err := outbox.PublishSubmitSegment(context.Background(), command, queue.SubmitSegmentResult{SegmentTotal: 2, SegmentIndex: index, SequenceID: uint32(index), GatewayMessageID: "88", SubmitStatus: "accepted"}); err != nil {
t.Fatal(err)
}
}
ctx, cancel := context.WithCancel(context.Background())
done := make(chan error, 1)
go func() { done <- outbox.Run(ctx) }()
deadline := time.Now().Add(3 * time.Second)
for client.XLen(context.Background(), outbox.StreamName()).Val() != 0 {
if time.Now().After(deadline) {
t.Fatal("batch did not drain")
}
time.Sleep(10 * time.Millisecond)
}
cancel()
<-done
if requests.Load() != 1 || eventCount.Load() != 2 {
t.Fatalf("requests/events=%d/%d, want 1/2", requests.Load(), eventCount.Load())
}
}
func TestBatchCallbackReplaysWholeRequestAfterHTTPFailure(t *testing.T) {
mr := miniredis.RunT(t)
client := redis.NewClient(&redis.Options{Addr: mr.Addr()})
var calls atomic.Int32
server := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) {
var batch callbackBatch
_ = json.NewDecoder(request.Body).Decode(&batch)
if calls.Add(1) == 1 {
http.Error(response, "busy", http.StatusServiceUnavailable)
return
}
result := callbackBatchResponse{BatchID: batch.BatchID}
for _, event := range batch.Events {
result.Results = append(result.Results, callbackEventResult{EventID: event.EventID, Accepted: true})
}
_ = json.NewEncoder(response).Encode(result)
}))
defer server.Close()
outbox := New(client)
outbox.APIBaseURL = server.URL
outbox.BatchEnabled = true
outbox.BatchWait = 5 * time.Millisecond
outbox.MinIdle = 5 * time.Millisecond
outbox.GatewayInstanceID = "g"
if err := outbox.PublishSubmitSegment(context.Background(), testCommand(), queue.SubmitSegmentResult{SegmentTotal: 1, SegmentIndex: 1, SequenceID: 1, GatewayMessageID: "1", SubmitStatus: "accepted"}); err != nil {
t.Fatal(err)
}
ctx, cancel := context.WithCancel(context.Background())
done := make(chan error, 1)
go func() { done <- outbox.Run(ctx) }()
deadline := time.Now().Add(3 * time.Second)
for client.XLen(context.Background(), outbox.StreamName()).Val() != 0 {
if time.Now().After(deadline) {
t.Fatal("replayed batch did not drain")
}
time.Sleep(10 * time.Millisecond)
}
cancel()
<-done
if calls.Load() < 2 {
t.Fatalf("calls=%d want replay", calls.Load())
}
}
func TestPublishAggregateAndCommandAckAreAtomicAndIdempotent(t *testing.T) { func TestPublishAggregateAndCommandAckAreAtomicAndIdempotent(t *testing.T) {
mr := miniredis.RunT(t) mr := miniredis.RunT(t)
client := redis.NewClient(&redis.Options{Addr: mr.Addr()}) client := redis.NewClient(&redis.Options{Addr: mr.Addr()})
+3
View File
@@ -46,6 +46,9 @@ func (o *Outbox) Run(ctx context.Context) error {
if err := o.ensureGroup(ctx); err != nil { if err := o.ensureGroup(ctx); err != nil {
return err return err
} }
if o.BatchEnabled {
return o.runBatches(ctx)
}
pool := newCallbackPool(ctx, o, o.concurrency()) pool := newCallbackPool(ctx, o, o.concurrency())
defer pool.wait() defer pool.wait()
for { for {
+41 -18
View File
@@ -1,6 +1,7 @@
package upstream package upstream
import ( import (
"cmpp-platform/gateway/internal/protocollog"
"cmpp-platform/gateway/internal/queue" "cmpp-platform/gateway/internal/queue"
"context" "context"
"fmt" "fmt"
@@ -16,24 +17,38 @@ import (
// the connection must wake pending submitters before scheduling pool recovery. // the connection must wake pending submitters before scheduling pool recovery.
type connection struct { type connection struct {
channelID string channelID string
config queue.UpstreamConfig config queue.UpstreamConfig
index int index int
pool *connectionPool pool *connectionPool
apiBaseURL string apiBaseURL string
httpClient *http.Client httpClient *http.Client
protocolLogPublisher interface {
Publish(context.Context, protocollog.Event) error
}
gatewayInstanceID string
eventPublisher interface {
PublishReceipt(context.Context, queue.ReceiptEvent) error
PublishUplink(context.Context, queue.UplinkEvent) error
}
mu sync.Mutex mu sync.Mutex
sendMu sync.Mutex sendMu sync.Mutex
client *cmpp.Client client *cmpp.Client
window chan struct{} window chan struct{}
pending map[uint32]chan submitPartResponse windowLimit int
tracker map[uint64]queue.SubmitCommand draining bool
longUplink map[string]*longUplinkAssembly retired bool
readOnce sync.Once lastSubmitRTT time.Duration
closed bool consecutiveFailures int
heartbeatCancel context.CancelFunc cooldownUntil time.Time
heartbeatPending map[uint32]time.Time pending map[uint32]chan submitPartResponse
tracker map[uint64]queue.SubmitCommand
longUplink map[string]*longUplinkAssembly
readOnce sync.Once
closed bool
heartbeatCancel context.CancelFunc
heartbeatPending map[uint32]time.Time
} }
type submitPartResponse struct { type submitPartResponse struct {
@@ -161,6 +176,14 @@ func (c *connection) identity() string {
return fmt.Sprintf("%s-%d", c.channelID, c.index) return fmt.Sprintf("%s-%d", c.channelID, c.index)
} }
func (c *connection) retire() {
c.mu.Lock()
c.draining = true
c.retired = true
c.mu.Unlock()
c.handleConnectionLoss(fmt.Errorf("connection retired after drain"))
}
func (c *connection) close() { func (c *connection) close() {
c.handleConnectionLoss(fmt.Errorf("connection closed")) c.handleConnectionLoss(fmt.Errorf("connection closed"))
} }
@@ -191,7 +214,7 @@ func (c *connection) handleConnectionLoss(err error) {
default: default:
} }
} }
if c.pool != nil { if c.pool != nil && !c.retired {
status := "disconnected" status := "disconnected"
if c.pool.countActiveConnections() > 0 { if c.pool.countActiveConnections() > 0 {
status = "reconnecting" status = "reconnecting"
+20 -2
View File
@@ -88,7 +88,16 @@ func (c *connection) handleDeliver(pkt deliverPacket) error {
DeliveredAt: time.Now().UTC(), DeliveredAt: time.Now().UTC(),
ConnectionID: c.identity(), ConnectionID: c.identity(),
} }
if err := postJSON(context.Background(), c.httpClient, c.apiBaseURL, "/gateway/events/receipt/intake", event); err != nil { if c.protocolLogPublisher != nil {
c.emitProtocolLog(protocolLogEvent{Protocol: "cmpp", Direction: "channel_to_platform", EventType: "deliver_receipt", Status: "success", ChannelID: channelID, Account: c.config.Account, MessageID: messageID, GatewayMessageID: fmt.Sprint(receipt.MsgId), Phone: strings.TrimSpace(receipt.DestTerminalId), ResultCode: strings.TrimSpace(receipt.Stat), Detail: map[string]any{"sequenceId": pkt.seqID}})
}
var publishErr error
if c.eventPublisher != nil {
publishErr = c.eventPublisher.PublishReceipt(context.Background(), event)
} else {
publishErr = postJSON(context.Background(), c.httpClient, c.apiBaseURL, "/gateway/events/receipt/intake", event)
}
if err := publishErr; err != nil {
log.Printf("protocol_event protocol=cmpp direction=gateway_to_api event=deliver_receipt status=forward_failed channel_id=%s sequence_id=%d gateway_message_id=%d error=%q", c.channelID, pkt.seqID, receipt.MsgId, err) log.Printf("protocol_event protocol=cmpp direction=gateway_to_api event=deliver_receipt status=forward_failed channel_id=%s sequence_id=%d gateway_message_id=%d error=%q", c.channelID, pkt.seqID, receipt.MsgId, err)
return err return err
} else { } else {
@@ -121,7 +130,16 @@ func (c *connection) handleDeliver(pkt deliverPacket) error {
Content: content, Content: content,
ReceivedAt: time.Now().UTC(), ReceivedAt: time.Now().UTC(),
} }
if err := postJSON(context.Background(), c.httpClient, c.apiBaseURL, "/gateway/events/uplink", event); err != nil { if c.protocolLogPublisher != nil {
c.emitProtocolLog(protocolLogEvent{Protocol: "cmpp", Direction: "channel_to_platform", EventType: "deliver_uplink", Status: "success", ChannelID: c.channelID, Account: c.config.Account, MessageID: cmd.MessageID, Phone: strings.TrimSpace(pkt.srcTerminalID), Detail: map[string]any{"sequenceId": pkt.seqID}})
}
var publishErr error
if c.eventPublisher != nil {
publishErr = c.eventPublisher.PublishUplink(context.Background(), event)
} else {
publishErr = postJSON(context.Background(), c.httpClient, c.apiBaseURL, "/gateway/events/uplink", event)
}
if err := publishErr; err != nil {
log.Printf("protocol_event protocol=cmpp direction=gateway_to_api event=deliver_uplink status=forward_failed channel_id=%s sequence_id=%d packet_msg_id=%d error=%q", c.channelID, pkt.seqID, pkt.msgID, err) log.Printf("protocol_event protocol=cmpp direction=gateway_to_api event=deliver_uplink status=forward_failed channel_id=%s sequence_id=%d packet_msg_id=%d error=%q", c.channelID, pkt.seqID, pkt.msgID, err)
} else { } else {
log.Printf("protocol_event protocol=cmpp direction=gateway_to_api event=deliver_uplink status=forwarded channel_id=%s sequence_id=%d packet_msg_id=%d", c.channelID, pkt.seqID, pkt.msgID) log.Printf("protocol_event protocol=cmpp direction=gateway_to_api event=deliver_uplink status=forwarded channel_id=%s sequence_id=%d packet_msg_id=%d", c.channelID, pkt.seqID, pkt.msgID)
+54 -5
View File
@@ -45,11 +45,20 @@ func (p *connectionPool) tryAcquireConnection() (*connection, func()) {
if len(p.conns) == 0 { if len(p.conns) == 0 {
return nil, nil return nil, nil
} }
for i := 0; i < len(p.conns); i++ { bestIndex := -1
index := (p.next + i) % len(p.conns) bestInFlight := int(^uint(0) >> 1)
conn := p.conns[index] var bestRTT time.Duration
for index, conn := range p.conns {
inFlight, rtt, usable := conn.capacitySnapshot()
if !usable || inFlight > bestInFlight || (inFlight == bestInFlight && bestIndex >= 0 && bestRTT > 0 && rtt >= bestRTT) {
continue
}
bestIndex, bestInFlight, bestRTT = index, inFlight, rtt
}
if bestIndex >= 0 {
conn := p.conns[bestIndex]
if conn.tryAcquireWindow() { if conn.tryAcquireWindow() {
p.next = (index + 1) % len(p.conns) p.next = (bestIndex + 1) % len(p.conns)
return conn, conn.releaseWindow return conn, conn.releaseWindow
} }
} }
@@ -57,8 +66,17 @@ func (p *connectionPool) tryAcquireConnection() (*connection, func()) {
} }
func (c *connection) tryAcquireWindow() bool { func (c *connection) tryAcquireWindow() bool {
c.mu.Lock()
defer c.mu.Unlock()
if c.window == nil { if c.window == nil {
c.window = make(chan struct{}, defaultWindowSize) c.window = make(chan struct{}, maximumWindowSize)
}
limit := c.windowLimit
if limit < 1 {
limit = defaultWindowSize
}
if c.closed || c.draining || c.client == nil || len(c.window) >= limit {
return false
} }
select { select {
case c.window <- struct{}{}: case c.window <- struct{}{}:
@@ -68,6 +86,37 @@ func (c *connection) tryAcquireWindow() bool {
} }
} }
func (c *connection) capacitySnapshot() (int, time.Duration, bool) {
c.mu.Lock()
defer c.mu.Unlock()
limit := c.windowLimit
if limit < 1 {
limit = defaultWindowSize
}
inFlight := len(c.window)
return inFlight, c.lastSubmitRTT, !c.closed && !c.draining && c.client != nil && inFlight < limit && !time.Now().Before(c.cooldownUntil)
}
func (c *connection) markSubmitFailure() {
c.mu.Lock()
defer c.mu.Unlock()
c.consecutiveFailures++
if c.consecutiveFailures >= 3 {
seconds := c.config.FailureCooldownSeconds
if seconds <= 0 {
seconds = 30
}
c.cooldownUntil = time.Now().Add(time.Duration(seconds) * time.Second)
}
}
func (c *connection) markSubmitSuccess() {
c.mu.Lock()
defer c.mu.Unlock()
c.consecutiveFailures = 0
c.cooldownUntil = time.Time{}
}
func (c *connection) releaseWindow() { func (c *connection) releaseWindow() {
if c.window == nil { if c.window == nil {
return return
+65 -4
View File
@@ -1,6 +1,7 @@
package upstream package upstream
import ( import (
"cmpp-platform/gateway/internal/protocollog"
"cmpp-platform/gateway/internal/queue" "cmpp-platform/gateway/internal/queue"
"context" "context"
"fmt" "fmt"
@@ -19,6 +20,8 @@ const (
defaultReconnectInitialDelay = 5 * time.Second defaultReconnectInitialDelay = 5 * time.Second
defaultReconnectMaximumDelay = 5 * time.Minute defaultReconnectMaximumDelay = 5 * time.Minute
defaultAuthReconnectDelay = 5 * time.Minute defaultAuthReconnectDelay = 5 * time.Minute
maximumConnections = 8
maximumWindowSize = 64
) )
type Manager struct { type Manager struct {
@@ -26,6 +29,14 @@ type Manager struct {
EventAPIBaseURL string EventAPIBaseURL string
HTTPClient *http.Client HTTPClient *http.Client
SubmitSegmentPublisher SubmitSegmentPublisher SubmitSegmentPublisher SubmitSegmentPublisher
EventPublisher interface {
PublishReceipt(context.Context, queue.ReceiptEvent) error
PublishUplink(context.Context, queue.UplinkEvent) error
}
ProtocolLogPublisher interface {
Publish(context.Context, protocollog.Event) error
}
GatewayInstanceID string
mu sync.Mutex mu sync.Mutex
conns map[string]*connectionPool conns map[string]*connectionPool
@@ -72,8 +83,14 @@ func (m *Manager) ConnectChannel(ctx context.Context, command queue.ConnectChann
WindowSize: command.Channel.WindowSize, WindowSize: command.Channel.WindowSize,
HeartbeatIntervalSeconds: command.Channel.HeartbeatIntervalSeconds, HeartbeatIntervalSeconds: command.Channel.HeartbeatIntervalSeconds,
HeartbeatMissThreshold: command.Channel.HeartbeatMissThreshold, HeartbeatMissThreshold: command.Channel.HeartbeatMissThreshold,
ConnectionWarmupSeconds: command.Channel.ConnectionWarmupSeconds,
ConnectionDrainSeconds: command.Channel.ConnectionDrainSeconds,
SubmitTimeoutSeconds: command.Channel.SubmitTimeoutSeconds,
FailureCooldownSeconds: command.Channel.FailureCooldownSeconds,
}) })
if pool == nil || !pool.matches(config) { if pool != nil && !pool.matches(config) && pool.sameEndpoint(config) {
pool.reconfigure(config)
} else if pool == nil || !pool.matches(config) {
if pool != nil { if pool != nil {
pool.close() pool.close()
} }
@@ -130,7 +147,9 @@ func (m *Manager) connectionFor(cmd queue.SubmitCommand) (*connectionPool, error
m.ensureDefaultsLocked() m.ensureDefaultsLocked()
pool := m.conns[cmd.ChannelID] pool := m.conns[cmd.ChannelID]
if pool == nil || !pool.matches(cmd.Upstream) { if pool != nil && !pool.matches(cmd.Upstream) && pool.sameEndpoint(cmd.Upstream) {
pool.reconfigure(cmd.Upstream)
} else if pool == nil || !pool.matches(cmd.Upstream) {
if pool != nil { if pool != nil {
pool.close() pool.close()
} }
@@ -170,6 +189,27 @@ func (m *Manager) ConnectionCounts() (desired int, connected int) {
return desired, connected return desired, connected
} }
func (m *Manager) WindowCounts() (configured int, inFlight int) {
m.mu.Lock()
pools := make([]*connectionPool, 0, len(m.conns))
for _, pool := range m.conns {
pools = append(pools, pool)
}
m.mu.Unlock()
for _, pool := range pools {
pool.mu.Lock()
conns := append([]*connection(nil), pool.conns...)
pool.mu.Unlock()
for _, conn := range conns {
conn.mu.Lock()
configured += max(1, conn.windowLimit)
inFlight += len(conn.window)
conn.mu.Unlock()
}
}
return configured, inFlight
}
func (m *Manager) newConnectionPool(channelID string, connectionID string, config queue.UpstreamConfig) *connectionPool { func (m *Manager) newConnectionPool(channelID string, connectionID string, config queue.UpstreamConfig) *connectionPool {
eventAPIBaseURL := m.EventAPIBaseURL eventAPIBaseURL := m.EventAPIBaseURL
if eventAPIBaseURL == "" { if eventAPIBaseURL == "" {
@@ -184,8 +224,11 @@ func (m *Manager) newConnectionPool(channelID string, connectionID string, confi
reporter: func(ctx context.Context, state ConnectionState) error { reporter: func(ctx context.Context, state ConnectionState) error {
return m.post(ctx, "/admin/gateway/connections", state) return m.post(ctx, "/admin/gateway/connections", state)
}, },
reconnectSignal: make(chan struct{}, 1), protocolLogPublisher: m.ProtocolLogPublisher,
stopCh: make(chan struct{}), gatewayInstanceID: m.GatewayInstanceID,
eventPublisher: m.EventPublisher,
reconnectSignal: make(chan struct{}, 1),
stopCh: make(chan struct{}),
} }
} }
@@ -202,6 +245,12 @@ func validateConnectChannelCommand(command queue.ConnectChannelCommand) error {
if command.Channel.Account == "" || command.Channel.PasswordCipher == "" { if command.Channel.Account == "" || command.Channel.PasswordCipher == "" {
return fmt.Errorf("account and passwordCipher are required") return fmt.Errorf("account and passwordCipher are required")
} }
if command.DesiredConnections < 1 || command.DesiredConnections > maximumConnections {
return fmt.Errorf("desiredConnections must be between 1 and %d", maximumConnections)
}
if command.Channel.WindowSize != 0 && (command.Channel.WindowSize < 1 || command.Channel.WindowSize > maximumWindowSize) {
return fmt.Errorf("windowSize must be between 1 and %d", maximumWindowSize)
}
return nil return nil
} }
@@ -222,5 +271,17 @@ func normalizeUpstreamConfig(config queue.UpstreamConfig) queue.UpstreamConfig {
if config.HeartbeatMissThreshold <= 0 { if config.HeartbeatMissThreshold <= 0 {
config.HeartbeatMissThreshold = defaultHeartbeatMissThreshold config.HeartbeatMissThreshold = defaultHeartbeatMissThreshold
} }
if config.ConnectionWarmupSeconds < 0 {
config.ConnectionWarmupSeconds = 30
}
if config.ConnectionDrainSeconds <= 0 {
config.ConnectionDrainSeconds = 60
}
if config.SubmitTimeoutSeconds <= 0 {
config.SubmitTimeoutSeconds = 60
}
if config.FailureCooldownSeconds <= 0 {
config.FailureCooldownSeconds = 30
}
return config return config
} }
+118 -19
View File
@@ -1,6 +1,7 @@
package upstream package upstream
import ( import (
"cmpp-platform/gateway/internal/protocollog"
"cmpp-platform/gateway/internal/queue" "cmpp-platform/gateway/internal/queue"
"context" "context"
"net/http" "net/http"
@@ -9,12 +10,20 @@ import (
) )
type connectionPool struct { type connectionPool struct {
channelID string channelID string
connectionID string connectionID string
config queue.UpstreamConfig config queue.UpstreamConfig
apiBaseURL string apiBaseURL string
httpClient *http.Client httpClient *http.Client
reporter func(context.Context, ConnectionState) error reporter func(context.Context, ConnectionState) error
protocolLogPublisher interface {
Publish(context.Context, protocollog.Event) error
}
gatewayInstanceID string
eventPublisher interface {
PublishReceipt(context.Context, queue.ReceiptEvent) error
PublishUplink(context.Context, queue.UplinkEvent) error
}
mu sync.Mutex mu sync.Mutex
connectMu sync.Mutex connectMu sync.Mutex
@@ -34,6 +43,103 @@ func (p *connectionPool) matches(config queue.UpstreamConfig) bool {
return p.config == normalizeUpstreamConfig(config) return p.config == normalizeUpstreamConfig(config)
} }
func (p *connectionPool) sameEndpoint(config queue.UpstreamConfig) bool {
next := normalizeUpstreamConfig(config)
current := p.config
return current.GatewayHost == next.GatewayHost && current.GatewayPort == next.GatewayPort &&
current.Account == next.Account && current.PasswordCipher == next.PasswordCipher && current.CMPPVersion == next.CMPPVersion
}
// reconfigure changes only runtime capacity on the existing pool. Existing
// sequence mappings remain attached to their physical connection. Scale down
// marks surplus connections draining before closing them; scale up is serialized.
func (p *connectionPool) reconfigure(config queue.UpstreamConfig) {
next := normalizeUpstreamConfig(config)
p.mu.Lock()
p.config = next
for _, conn := range p.conns {
conn.mu.Lock()
conn.config = next
conn.windowLimit = next.WindowSize
conn.mu.Unlock()
}
p.mu.Unlock()
go p.reconcileCapacity()
}
func (p *connectionPool) reconcileCapacity() {
p.connectMu.Lock()
defer p.connectMu.Unlock()
if p.stopped() {
return
}
p.mu.Lock()
desired := max(1, min(maximumConnections, p.config.DesiredConnections))
if len(p.conns) > desired {
retiring := append([]*connection(nil), p.conns[desired:]...)
p.conns = p.conns[:desired]
for _, conn := range retiring {
conn.mu.Lock()
conn.draining = true
conn.mu.Unlock()
}
p.mu.Unlock()
deadline := time.Now().Add(time.Duration(p.config.ConnectionDrainSeconds) * time.Second)
for _, conn := range retiring {
for len(conn.window) > 0 && time.Now().Before(deadline) {
time.Sleep(10 * time.Millisecond)
}
conn.retire()
}
_ = p.reportState(context.Background(), "connected", nil)
return
}
p.mu.Unlock()
for {
p.mu.Lock()
if len(p.conns) >= desired || p.stopped() {
p.mu.Unlock()
break
}
index := len(p.conns)
config := p.config
p.mu.Unlock()
conn := p.newConnection(index, config)
if _, err := conn.ensureConnected(); err != nil {
p.scheduleReconnect(err)
_ = p.reportState(context.Background(), "failed", err)
return
}
p.mu.Lock()
p.conns = append(p.conns, conn)
p.mu.Unlock()
_ = p.reportState(context.Background(), "connected", nil)
if len(p.conns) < desired && config.ConnectionWarmupSeconds > 0 {
timer := time.NewTimer(time.Duration(config.ConnectionWarmupSeconds) * time.Second)
select {
case <-p.stopCh:
timer.Stop()
return
case <-timer.C:
}
}
}
}
func (p *connectionPool) newConnection(index int, config queue.UpstreamConfig) *connection {
return &connection{
channelID: p.channelID, config: config, index: index, pool: p,
apiBaseURL: p.apiBaseURL, httpClient: p.httpClient,
protocolLogPublisher: p.protocolLogPublisher, gatewayInstanceID: p.gatewayInstanceID,
eventPublisher: p.eventPublisher,
window: make(chan struct{}, maximumWindowSize), windowLimit: config.WindowSize,
pending: make(map[uint32]chan submitPartResponse), tracker: make(map[uint64]queue.SubmitCommand),
longUplink: make(map[string]*longUplinkAssembly), heartbeatPending: make(map[uint32]time.Time),
}
}
func (p *connectionPool) ensureConnected() error { func (p *connectionPool) ensureConnected() error {
p.connectMu.Lock() p.connectMu.Lock()
defer p.connectMu.Unlock() defer p.connectMu.Unlock()
@@ -58,20 +164,12 @@ func (p *connectionPool) ensureConnected() error {
p.mu.Unlock() p.mu.Unlock()
break break
} }
index := len(p.conns) if connectedAny {
conn := &connection{ p.mu.Unlock()
channelID: p.channelID, break
config: p.config,
index: index,
pool: p,
apiBaseURL: p.apiBaseURL,
httpClient: p.httpClient,
window: make(chan struct{}, p.config.WindowSize),
pending: make(map[uint32]chan submitPartResponse),
tracker: make(map[uint64]queue.SubmitCommand),
longUplink: make(map[string]*longUplinkAssembly),
heartbeatPending: make(map[uint32]time.Time),
} }
index := len(p.conns)
conn := p.newConnection(index, p.config)
p.mu.Unlock() p.mu.Unlock()
connected, err := conn.ensureConnected() connected, err := conn.ensureConnected()
@@ -87,6 +185,7 @@ func (p *connectionPool) ensureConnected() error {
if connectedAny { if connectedAny {
_ = p.reportState(context.Background(), "connected", nil) _ = p.reportState(context.Background(), "connected", nil)
} }
go p.reconcileCapacity()
return nil return nil
} }
+49 -2
View File
@@ -2,7 +2,9 @@ package upstream
import ( import (
"context" "context"
cmpp "github.com/bigwhite/gocmpp"
"testing" "testing"
"time"
"cmpp-platform/gateway/internal/queue" "cmpp-platform/gateway/internal/queue"
) )
@@ -10,8 +12,8 @@ import (
func TestConnectionPoolAcquiresAcrossConnections(t *testing.T) { func TestConnectionPoolAcquiresAcrossConnections(t *testing.T) {
pool := &connectionPool{ pool := &connectionPool{
conns: []*connection{ conns: []*connection{
{window: make(chan struct{}, 1)}, {client: &cmpp.Client{}, window: make(chan struct{}, 64), windowLimit: 1},
{window: make(chan struct{}, 1)}, {client: &cmpp.Client{}, window: make(chan struct{}, 64), windowLimit: 1},
}, },
} }
@@ -40,6 +42,51 @@ func TestConnectionPoolAcquiresAcrossConnections(t *testing.T) {
releaseSecond() releaseSecond()
} }
func TestPoolContinuesOnHealthyConnectionWhenPeerIsDraining(t *testing.T) {
draining := &connection{client: &cmpp.Client{}, window: make(chan struct{}, 64), windowLimit: 16, draining: true}
healthy := &connection{client: &cmpp.Client{}, window: make(chan struct{}, 64), windowLimit: 16}
pool := &connectionPool{conns: []*connection{draining, healthy}}
selected, release := pool.tryAcquireConnection()
if selected != healthy {
t.Fatal("expected healthy peer connection")
}
release()
}
func TestRuntimeCapacityMatrixAndSmoothScaleDown(t *testing.T) {
for _, connections := range []int{1, 2, 4, 8} {
for _, window := range []int{1, 16, 32, 64} {
config := normalizeUpstreamConfig(queue.UpstreamConfig{GatewayHost: "127.0.0.1", GatewayPort: 17890, Account: "a", PasswordCipher: "p", CMPPVersion: "3.0", DesiredConnections: connections, WindowSize: window})
if config.DesiredConnections != connections || config.WindowSize != window {
t.Fatalf("matrix normalized incorrectly: %+v", config)
}
}
}
pool := &connectionPool{channelID: "channel-1", connectionID: "primary", config: normalizeUpstreamConfig(queue.UpstreamConfig{DesiredConnections: 2, WindowSize: 16, ConnectionDrainSeconds: 1}), stopCh: make(chan struct{}), reconnectSignal: make(chan struct{}, 1)}
pool.conns = []*connection{{pool: pool, window: make(chan struct{}, 64), windowLimit: 16}, {pool: pool, window: make(chan struct{}, 64), windowLimit: 16}}
pool.reconfigure(queue.UpstreamConfig{DesiredConnections: 1, WindowSize: 32, ConnectionDrainSeconds: 1})
deadline := time.Now().Add(time.Second)
for {
pool.mu.Lock()
count := len(pool.conns)
first := pool.conns[0]
pool.mu.Unlock()
if count == 1 {
first.mu.Lock()
limit := first.windowLimit
first.mu.Unlock()
if limit != 32 {
t.Fatalf("window limit=%d want32", limit)
}
break
}
if time.Now().After(deadline) {
t.Fatal("scale down did not complete")
}
time.Sleep(time.Millisecond)
}
}
func TestManagerDisconnectChannelRemovesPoolAndStopsReconnects(t *testing.T) { func TestManagerDisconnectChannelRemovesPoolAndStopsReconnects(t *testing.T) {
pool := &connectionPool{ pool := &connectionPool{
channelID: "channel-1", channelID: "channel-1",
+12 -16
View File
@@ -1,6 +1,7 @@
package upstream package upstream
import ( import (
"cmpp-platform/gateway/internal/protocollog"
"context" "context"
"fmt" "fmt"
cmpp "github.com/bigwhite/gocmpp" cmpp "github.com/bigwhite/gocmpp"
@@ -8,22 +9,7 @@ import (
"strings" "strings"
) )
type protocolLogEvent struct { type protocolLogEvent = protocollog.Event
Protocol string `json:"protocol"`
Direction string `json:"direction"`
EventType string `json:"eventType"`
Status string `json:"status"`
TenantID string `json:"tenantId,omitempty"`
ApplicationID string `json:"applicationId,omitempty"`
ChannelID string `json:"channelId,omitempty"`
Account string `json:"account,omitempty"`
MessageID string `json:"messageId,omitempty"`
GatewayMessageID string `json:"gatewayMessageId,omitempty"`
Phone string `json:"phone,omitempty"`
ResultCode string `json:"resultCode,omitempty"`
PayloadBytes int `json:"payloadBytes,omitempty"`
Detail map[string]any `json:"detail,omitempty"`
}
func (c *connection) emitDeliverResponse(pkt deliverPacket, responseErr error) { func (c *connection) emitDeliverResponse(pkt deliverPacket, responseErr error) {
status := "success" status := "success"
@@ -71,6 +57,16 @@ func (c *connection) emitDeliverResponse(pkt deliverPacket, responseErr error) {
} }
func (c *connection) emitProtocolLog(event protocolLogEvent) { func (c *connection) emitProtocolLog(event protocolLogEvent) {
event.ConnectionID = c.identity()
event.GatewayInstanceID = c.gatewayInstanceID
if c.protocolLogPublisher != nil {
go func() {
if err := c.protocolLogPublisher.Publish(context.Background(), event); err != nil {
log.Printf("protocol log Redis publish failed channel_id=%s message_id=%s error=%q", event.ChannelID, event.MessageID, err)
}
}()
return
}
go func() { go func() {
ctx, cancel := context.WithTimeout(context.Background(), defaultHTTPTimeout) ctx, cancel := context.WithTimeout(context.Background(), defaultHTTPTimeout)
defer cancel() defer cancel()
+20 -1
View File
@@ -97,6 +97,12 @@ func (p *connectionPool) submit(
} }
func (c *connection) submitPart(ctx context.Context, cmd queue.SubmitCommand, part submitPart) (uint32, string, queue.SubmitResult, error) { func (c *connection) submitPart(ctx context.Context, cmd queue.SubmitCommand, part submitPart) (uint32, string, queue.SubmitResult, error) {
startedAt := time.Now()
defer func() {
c.mu.Lock()
c.lastSubmitRTT = time.Since(startedAt)
c.mu.Unlock()
}()
rspCh := make(chan submitPartResponse, 1) rspCh := make(chan submitPartResponse, 1)
pkt := c.submitRequestPacket(cmd, part) pkt := c.submitRequestPacket(cmd, part)
@@ -163,14 +169,20 @@ func (c *connection) submitPart(ctx context.Context, cmd queue.SubmitCommand, pa
c.mu.Unlock() c.mu.Unlock()
}() }()
waitCtx, cancel := context.WithTimeout(ctx, defaultSubmitTimeout) timeout := time.Duration(c.config.SubmitTimeoutSeconds) * time.Second
if timeout <= 0 {
timeout = defaultSubmitTimeout
}
waitCtx, cancel := context.WithTimeout(ctx, timeout)
defer cancel() defer cancel()
select { select {
case <-waitCtx.Done(): case <-waitCtx.Done():
c.markSubmitFailure()
result := submitResult(cmd, seq, "", "timeout", "SUBMIT_TIMEOUT", waitCtx.Err().Error()) result := submitResult(cmd, seq, "", "timeout", "SUBMIT_TIMEOUT", waitCtx.Err().Error())
return seq, "", result, waitCtx.Err() return seq, "", result, waitCtx.Err()
case rsp := <-rspCh: case rsp := <-rspCh:
if rsp.err != nil { if rsp.err != nil {
c.markSubmitFailure()
result := submitResult(cmd, seq, "", "timeout", "CONNECTION_LOST", rsp.err.Error()) result := submitResult(cmd, seq, "", "timeout", "CONNECTION_LOST", rsp.err.Error())
return seq, "", result, rsp.err return seq, "", result, rsp.err
} }
@@ -203,6 +215,7 @@ func (c *connection) submitPart(ctx context.Context, cmd queue.SubmitCommand, pa
}, },
}) })
if rsp.result == 0 { if rsp.result == 0 {
c.markSubmitSuccess()
c.mu.Lock() c.mu.Lock()
c.tracker[rsp.msgID] = cmd c.tracker[rsp.msgID] = cmd
c.mu.Unlock() c.mu.Unlock()
@@ -345,6 +358,12 @@ func validateSubmitCommand(cmd queue.SubmitCommand) error {
if cmd.Upstream.Account == "" || cmd.Upstream.PasswordCipher == "" { if cmd.Upstream.Account == "" || cmd.Upstream.PasswordCipher == "" {
return fmt.Errorf("upstream account and passwordCipher are required") return fmt.Errorf("upstream account and passwordCipher are required")
} }
if cmd.Upstream.DesiredConnections != 0 && (cmd.Upstream.DesiredConnections < 1 || cmd.Upstream.DesiredConnections > maximumConnections) {
return fmt.Errorf("upstream desiredConnections must be between 1 and %d", maximumConnections)
}
if cmd.Upstream.WindowSize != 0 && (cmd.Upstream.WindowSize < 1 || cmd.Upstream.WindowSize > maximumWindowSize) {
return fmt.Errorf("upstream windowSize must be between 1 and %d", maximumWindowSize)
}
if len(cmd.PhoneNumber) == 0 || len(cmd.Content) == 0 { if len(cmd.PhoneNumber) == 0 || len(cmd.Content) == 0 {
return fmt.Errorf("phoneNumber and content are required") return fmt.Errorf("phoneNumber and content are required")
} }
+37 -1
View File
@@ -61,6 +61,16 @@ if [[ "${GATEWAY_CALLBACK_SEPARATE_PROCESS_ENABLED:-false}" == "true" ]]; then
exit 1 exit 1
fi fi
fi fi
if [[ "${GATEWAY_PROTOCOL_LOG_STREAM_ENABLED:-false}" == "true" && ! "${API_PROTOCOL_LOG_DB_POOL_MAX:-}" =~ ^[1-9][0-9]*$ ]]; then
echo "API_PROTOCOL_LOG_DB_POOL_MAX must be a positive integer when protocol log Stream writing is enabled." >&2
exit 1
fi
if [[ "${GATEWAY_CALLBACK_BATCH_ENABLED:-false}" == "true" ]]; then
if [[ ! "${GATEWAY_CALLBACK_BATCH_SIZE:-}" =~ ^[1-9][0-9]*$ || "${GATEWAY_CALLBACK_BATCH_SIZE}" -gt 100 ]]; then
echo "GATEWAY_CALLBACK_BATCH_SIZE must be between 1 and 100." >&2
exit 1
fi
fi
if [[ -z "${CMPP_PUBLIC_HOST:-}" || ! "${CMPP_PUBLIC_PORT:-}" =~ ^[1-9][0-9]*$ ]]; then if [[ -z "${CMPP_PUBLIC_HOST:-}" || ! "${CMPP_PUBLIC_PORT:-}" =~ ^[1-9][0-9]*$ ]]; then
echo "CMPP_PUBLIC_HOST and a positive CMPP_PUBLIC_PORT are required in $ENV_FILE; these are the customer-facing CMPP endpoint." >&2 echo "CMPP_PUBLIC_HOST and a positive CMPP_PUBLIC_PORT are required in $ENV_FILE; these are the customer-facing CMPP endpoint." >&2
@@ -97,7 +107,7 @@ PROD_ADMIN_CREDENTIAL_FILE="$ADMIN_CREDENTIAL_FILE" node tools/deploy/ensure-pro
chmod 600 "$ADMIN_CREDENTIAL_FILE" || true chmod 600 "$ADMIN_CREDENTIAL_FILE" || true
echo "[deploy] Ensuring runtime log directories" echo "[deploy] Ensuring runtime log directories"
install -d -m 0755 "$APP_DIR/logs/api" "$APP_DIR/logs/send-worker" "$APP_DIR/logs/submit-outbox" "$APP_DIR/logs/gateway-callback" "$APP_DIR/logs/gateway" install -d -m 0755 "$APP_DIR/logs/api" "$APP_DIR/logs/send-worker" "$APP_DIR/logs/submit-outbox" "$APP_DIR/logs/gateway-callback" "$APP_DIR/logs/protocol-log-worker" "$APP_DIR/logs/gateway"
echo "[deploy] Installing split API and send-worker services" echo "[deploy] Installing split API and send-worker services"
node_bin="$(command -v node)" node_bin="$(command -v node)"
@@ -163,6 +173,26 @@ RestartSec=5
StandardOutput=append:$APP_DIR/logs/gateway-callback/stdout.log StandardOutput=append:$APP_DIR/logs/gateway-callback/stdout.log
StandardError=append:$APP_DIR/logs/gateway-callback/stderr.log StandardError=append:$APP_DIR/logs/gateway-callback/stderr.log
[Install]
WantedBy=multi-user.target
EOF
cat >/etc/systemd/system/cmpp-protocol-log-worker.service <<EOF
[Unit]
Description=CMPP protocol log Redis Stream batch writer
After=network.target postgresql.service redis.service
[Service]
User=cmpp-api
Group=cmpp-security
WorkingDirectory=$APP_DIR/api
EnvironmentFile=$ENV_FILE
Environment=CMPP_PROCESS_ROLE=protocol-log-worker
ExecStart=$node_bin dist/protocol-log-worker.js
Restart=always
RestartSec=5
StandardOutput=append:$APP_DIR/logs/protocol-log-worker/stdout.log
StandardError=append:$APP_DIR/logs/protocol-log-worker/stderr.log
[Install] [Install]
WantedBy=multi-user.target WantedBy=multi-user.target
EOF EOF
@@ -212,6 +242,12 @@ if [[ "${GATEWAY_CALLBACK_SEPARATE_PROCESS_ENABLED:-false}" == "true" ]]; then
else else
systemctl disable --now cmpp-gateway-callback 2>/dev/null || true systemctl disable --now cmpp-gateway-callback 2>/dev/null || true
fi fi
if [[ "${GATEWAY_PROTOCOL_LOG_STREAM_ENABLED:-false}" == "true" ]]; then
systemctl enable --now cmpp-protocol-log-worker
systemctl restart cmpp-protocol-log-worker
else
systemctl disable --now cmpp-protocol-log-worker 2>/dev/null || true
fi
# Gateway is restarted after the callback listener so supplier events never point # Gateway is restarted after the callback listener so supplier events never point
# at a callback port that has not completed Nest/Prisma initialization. # at a callback port that has not completed Nest/Prisma initialization.
systemctl restart cmpp-gateway systemctl restart cmpp-gateway
+10
View File
@@ -16,6 +16,7 @@ install -d -o cmpp-api -g cmpp-security -m 0750 "$APP_DIR/logs/api"
install -d -o cmpp-api -g cmpp-security -m 0750 "$APP_DIR/logs/send-worker" install -d -o cmpp-api -g cmpp-security -m 0750 "$APP_DIR/logs/send-worker"
install -d -o cmpp-api -g cmpp-security -m 0750 "$APP_DIR/logs/submit-outbox" install -d -o cmpp-api -g cmpp-security -m 0750 "$APP_DIR/logs/submit-outbox"
install -d -o cmpp-api -g cmpp-security -m 0750 "$APP_DIR/logs/gateway-callback" install -d -o cmpp-api -g cmpp-security -m 0750 "$APP_DIR/logs/gateway-callback"
install -d -o cmpp-api -g cmpp-security -m 0750 "$APP_DIR/logs/protocol-log-worker"
[[ -d /var/lib/cmpp-platform/object-storage ]] && chown -R cmpp-api:cmpp-security /var/lib/cmpp-platform/object-storage [[ -d /var/lib/cmpp-platform/object-storage ]] && chown -R cmpp-api:cmpp-security /var/lib/cmpp-platform/object-storage
sed "s#@CMPP_SECURITY_AGENT_BIN@#$agent_binary#g" "$APP_DIR/deploy/security/cmpp-report-only.conf" >/etc/fail2ban/action.d/cmpp-report-only.conf sed "s#@CMPP_SECURITY_AGENT_BIN@#$agent_binary#g" "$APP_DIR/deploy/security/cmpp-report-only.conf" >/etc/fail2ban/action.d/cmpp-report-only.conf
@@ -67,6 +68,15 @@ ProtectKernelModules=true
ProtectControlGroups=true ProtectControlGroups=true
ReadWritePaths=$APP_DIR/logs/gateway-callback ReadWritePaths=$APP_DIR/logs/gateway-callback
EOF EOF
install -d -m 0755 /etc/systemd/system/cmpp-protocol-log-worker.service.d
cat >/etc/systemd/system/cmpp-protocol-log-worker.service.d/security-boundary.conf <<EOF
[Service]
NoNewPrivileges=true
PrivateTmp=true
ProtectSystem=strict
ProtectHome=true
ReadWritePaths=$APP_DIR/logs/protocol-log-worker
EOF
grep -q 'cmpp-security.nft' /etc/nftables.conf || printf '\ninclude "/etc/nftables.d/cmpp-security.nft"\n' >>/etc/nftables.conf grep -q 'cmpp-security.nft' /etc/nftables.conf || printf '\ninclude "/etc/nftables.d/cmpp-security.nft"\n' >>/etc/nftables.conf
nft -c -f /etc/nftables.conf nft -c -f /etc/nftables.conf
nft list table inet cmpp_security >/dev/null 2>&1 || nft -f /etc/nftables.d/cmpp-security.nft nft list table inet cmpp_security >/dev/null 2>&1 || nft -f /etc/nftables.d/cmpp-security.nft