From 761c123b65f09093bc379096188f3ee9ccb2e618 Mon Sep 17 00:00:00 2001 From: hectorzhao Date: Tue, 25 Aug 2026 11:39:59 +0800 Subject: [PATCH] perf: batch gateway submits and isolate callbacks --- .../migration.sql | 29 + api/prisma/schema.prisma | 22 + .../phone-routing-lookup.service.ts | 21 +- api/src/gateway-callback.module.ts | 31 + api/src/gateway-callback.ts | 44 + api/src/metrics/metrics.service.spec.ts | 16 + api/src/metrics/metrics.service.ts | 91 ++ api/src/open-api/open-api.service.ts | 4 + api/src/prisma/prisma.service.spec.ts | 18 + api/src/prisma/prisma.service.ts | 57 +- .../gateway-callback.controller.spec.ts | 43 + .../send-chain/gateway-callback.controller.ts | 102 +++ api/src/send-chain/send-chain.service.spec.ts | 187 +++- api/src/send-chain/send-chain.service.ts | 14 +- .../send-chain/send-gateway-submit.service.ts | 797 ++++++++++++++++-- .../send-chain/send-inbound-entry.service.ts | 97 +-- api/src/send-chain/send-submission.service.ts | 15 +- api/src/submit-outbox-worker.ts | 30 + docs/codebase-modularization-roadmap.md | 1 + .../first-version-development-requirements.md | 10 + docs/phase-4-send-pipeline-redesign.md | 197 +++++ docs/phase-4-send-worker-optimization-plan.md | 21 +- docs/system-functional-test-cases.md | 54 ++ docs/testing-progress.md | 72 ++ gateway/cmd/gateway/main.go | 9 +- gateway/internal/upstream/manager.go | 7 +- gateway/internal/upstream/pool_test.go | 11 + tools/deploy/production-deploy.sh | 80 +- tools/security/install-security-agent.sh | 25 + 29 files changed, 1912 insertions(+), 193 deletions(-) create mode 100644 api/prisma/migrations/20260825103000_add_gateway_submit_outbox/migration.sql create mode 100644 api/src/gateway-callback.module.ts create mode 100644 api/src/gateway-callback.ts create mode 100644 api/src/send-chain/gateway-callback.controller.spec.ts create mode 100644 api/src/send-chain/gateway-callback.controller.ts create mode 100644 api/src/submit-outbox-worker.ts create mode 100644 docs/phase-4-send-pipeline-redesign.md diff --git a/api/prisma/migrations/20260825103000_add_gateway_submit_outbox/migration.sql b/api/prisma/migrations/20260825103000_add_gateway_submit_outbox/migration.sql new file mode 100644 index 0000000..028d36c --- /dev/null +++ b/api/prisma/migrations/20260825103000_add_gateway_submit_outbox/migration.sql @@ -0,0 +1,29 @@ +CREATE TABLE "GatewaySubmitOutbox" ( + "id" TEXT NOT NULL, + "submitId" TEXT NOT NULL, + "messageRecordId" TEXT NOT NULL, + "channelId" TEXT NOT NULL, + "payload" JSONB NOT NULL, + "schemaVersion" TEXT NOT NULL DEFAULT 'v1', + "status" TEXT NOT NULL DEFAULT 'pending', + "attemptCount" INTEGER NOT NULL DEFAULT 0, + "nextAttemptAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "leaseOwner" TEXT, + "leaseExpiresAt" TIMESTAMP(3), + "streamEntryId" TEXT, + "publishedAt" TIMESTAMP(3), + "lastError" TEXT, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + CONSTRAINT "GatewaySubmitOutbox_pkey" PRIMARY KEY ("id") +); + +CREATE UNIQUE INDEX "GatewaySubmitOutbox_submitId_key" +ON "GatewaySubmitOutbox"("submitId"); + +CREATE INDEX "GatewaySubmitOutbox_pending_claim_idx" +ON "GatewaySubmitOutbox"("nextAttemptAt", "createdAt") +WHERE "status" IN ('pending', 'publishing'); + +CREATE INDEX "GatewaySubmitOutbox_messageRecordId_idx" +ON "GatewaySubmitOutbox"("messageRecordId"); diff --git a/api/prisma/schema.prisma b/api/prisma/schema.prisma index 13c2288..95b985d 100644 --- a/api/prisma/schema.prisma +++ b/api/prisma/schema.prisma @@ -1828,6 +1828,28 @@ model SmsSubmitRecord { @@index([channelGroupId]) } +model GatewaySubmitOutbox { + id String @id @default(cuid()) + submitId String @unique + messageRecordId String + channelId String + payload Json + schemaVersion String @default("v1") + status String @default("pending") + attemptCount Int @default(0) + nextAttemptAt DateTime @default(now()) + leaseOwner String? + leaseExpiresAt DateTime? + streamEntryId String? + publishedAt DateTime? + lastError String? + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + @@index([submitId, status]) + @@index([messageRecordId]) +} + model DailyReconciliationReport { id String @id @default(cuid()) reportDate DateTime @db.Date diff --git a/api/src/dictionaries/phone-routing-lookup.service.ts b/api/src/dictionaries/phone-routing-lookup.service.ts index b335466..e7b6135 100644 --- a/api/src/dictionaries/phone-routing-lookup.service.ts +++ b/api/src/dictionaries/phone-routing-lookup.service.ts @@ -22,18 +22,25 @@ export class PhoneRoutingLookupService { } async identifyProvince(phoneNumber: string) { - const prefixes = phonePrefixes(phoneNumber); - if (prefixes.length === 0) return null; + return (await this.identifyProvinces([phoneNumber])).get(phoneNumber) ?? null; + } + + async identifyProvinces(phoneNumbers: string[]) { + const uniquePhones = [...new Set(phoneNumbers)]; + const prefixesByPhone = new Map(uniquePhones.map((phone) => [phone, phonePrefixes(phone)])); + const prefixes = [...new Set([...prefixesByPhone.values()].flat())]; + if (prefixes.length === 0) return new Map(uniquePhones.map((phone) => [phone, null])); const segments = await this.prisma.phoneSegment.findMany({ where: { prefix: { in: prefixes } }, select: { prefix: true, province: true }, }); const provinceByPrefix = new Map(segments.map((segment) => [segment.prefix, segment.province])); - for (const prefix of prefixes) { - const province = provinceByPrefix.get(prefix); - if (province) return province; - } - return null; + return new Map(uniquePhones.map((phone) => { + const province = (prefixesByPhone.get(phone) ?? []) + .map((prefix) => provinceByPrefix.get(prefix)) + .find((value): value is string => Boolean(value)) ?? null; + return [phone, province]; + })); } invalidateCarrierRules() { diff --git a/api/src/gateway-callback.module.ts b/api/src/gateway-callback.module.ts new file mode 100644 index 0000000..778a8a2 --- /dev/null +++ b/api/src/gateway-callback.module.ts @@ -0,0 +1,31 @@ +import { Module } from '@nestjs/common'; +import { ConfigModule } from '@nestjs/config'; +import { BillingService } from './billing/billing.service'; +import { PhoneRoutingLookupService } from './dictionaries/phone-routing-lookup.service'; +import { MetricsModule } from './metrics/metrics.module'; +import { OpenApiService } from './open-api/open-api.service'; +import { PrismaModule } from './prisma/prisma.module'; +import { ProtocolLogsModule } from './protocol-logs/protocol-logs.module'; +import { PhoneFrequencyService } from './risk-review/phone-frequency.service'; +import { RiskReviewService } from './risk-review/risk-review.service'; +import { GatewayCallbackController } from './send-chain/gateway-callback.controller'; +import { SendChainService } from './send-chain/send-chain.service'; + +@Module({ + imports: [ + ConfigModule.forRoot({ isGlobal: true, envFilePath: ['.env.local', '.env'] }), + PrismaModule, + MetricsModule, + ProtocolLogsModule, + ], + controllers: [GatewayCallbackController], + providers: [ + BillingService, + RiskReviewService, + PhoneFrequencyService, + PhoneRoutingLookupService, + SendChainService, + OpenApiService, + ], +}) +export class GatewayCallbackModule {} diff --git a/api/src/gateway-callback.ts b/api/src/gateway-callback.ts new file mode 100644 index 0000000..d2ebfff --- /dev/null +++ b/api/src/gateway-callback.ts @@ -0,0 +1,44 @@ +import 'reflect-metadata'; +import { NestFactory } from '@nestjs/core'; +import { createServer } from 'node:http'; +import type { NestExpressApplication } from '@nestjs/platform-express'; +import { GatewayCallbackModule } from './gateway-callback.module'; +import { configureHttpBodyParsers } from './http-body-limits'; +import { MetricsService } from './metrics/metrics.service'; + +Object.defineProperty(BigInt.prototype, 'toJSON', { + configurable: true, + value(this: bigint) { + const result = Number(this); + if (!Number.isSafeInteger(result)) throw new RangeError('金额超过 JavaScript 安全整数范围'); + return result; + }, +}); + +async function bootstrap() { + if (process.env.CMPP_PROCESS_ROLE !== 'callback') { + throw new Error('gateway-callback requires CMPP_PROCESS_ROLE=callback'); + } + const app = await NestFactory.create(GatewayCallbackModule, { rawBody: true, bodyParser: false }); + app.setGlobalPrefix('api'); + configureHttpBodyParsers(app); + app.enableShutdownHooks(); + const host = process.env.API_CALLBACK_HOST?.trim() || '127.0.0.1'; + const port = Number(process.env.API_CALLBACK_PORT ?? 3001); + await app.listen(port, host); + + const metrics = app.get(MetricsService); + const metricsHost = process.env.API_CALLBACK_METRICS_HOST?.trim() || '127.0.0.1'; + const metricsPort = Number(process.env.API_CALLBACK_METRICS_PORT ?? 9468); + const metricsServer = createServer((request, response) => { + if (request.method !== 'GET' || request.url !== '/metrics') return void response.writeHead(404).end(); + response.writeHead(200, { 'Content-Type': 'text/plain; version=0.0.4; charset=utf-8', 'Cache-Control': 'no-store' }); + response.end(metrics.render()); + }); + await new Promise((resolve, reject) => { + metricsServer.once('error', reject); + metricsServer.listen(metricsPort, metricsHost, resolve); + }); +} + +void bootstrap(); diff --git a/api/src/metrics/metrics.service.spec.ts b/api/src/metrics/metrics.service.spec.ts index 5b42e48..c7ffa93 100644 --- a/api/src/metrics/metrics.service.spec.ts +++ b/api/src/metrics/metrics.service.spec.ts @@ -7,13 +7,29 @@ describe('MetricsService', () => { service.finishRequest(startedAt, 'GET', '/api/admin/tenants/:id', 200); const inboundStartedAt = service.beginCmppInboundStage(); service.finishCmppInboundStage(inboundStartedAt, 'application_lookup', 'success'); + const sendStartedAt = service.beginSendWorkerStage(); + service.finishSendWorkerStage(sendStartedAt, 'route_lookup', 'success'); + service.setSendWorkerSlots(20, 3); + service.setSendWorkerQueueJobs('waiting', 12); + service.recordSendWorkerResult('completed'); + service.setSendWorkerDatabasePool('max', 8); + service.setSendWorkerDatabasePool('waiting', 2); const output = service.render(); expect(output).toContain('cmpp_api_process_resident_memory_bytes'); expect(output).toContain('cmpp_api_http_requests_total{method="GET",route="/api/admin/tenants/:id",status="200"} 1'); expect(output).toContain('cmpp_api_http_request_duration_seconds_bucket'); expect(output).toContain('cmpp_api_cmpp_inbound_stage_duration_seconds_count{stage="application_lookup",result="success"} 1'); + expect(output).toContain('cmpp_worker_send_stage_duration_seconds_count{stage="route_lookup",result="success"} 1'); + expect(output).toContain('cmpp_worker_send_slots{state="configured"} 20'); + expect(output).toContain('cmpp_worker_send_slots{state="in_flight"} 3'); + expect(output).toContain('cmpp_worker_send_queue_jobs{state="waiting"} 12'); + expect(output).toContain('cmpp_worker_send_jobs_total{result="completed"} 1'); + expect(output).toContain('cmpp_worker_database_pool_connections{state="max"} 8'); + expect(output).toContain('cmpp_worker_database_pool_connections{state="waiting"} 2'); expect(output).not.toContain('phone_number'); + expect(output).not.toContain('tenant_id'); + expect(output).not.toContain('channel_id'); service.onModuleDestroy(); }); }); diff --git a/api/src/metrics/metrics.service.ts b/api/src/metrics/metrics.service.ts index 61b320c..9ad406e 100644 --- a/api/src/metrics/metrics.service.ts +++ b/api/src/metrics/metrics.service.ts @@ -3,6 +3,7 @@ import { monitorEventLoopDelay } from 'node:perf_hooks'; const HTTP_DURATION_BUCKETS = [0.05, 0.1, 0.25, 0.5, 1, 3, 10] as const; const CMPP_INBOUND_DURATION_BUCKETS = [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 3, 10] as const; +const SEND_WORKER_DURATION_BUCKETS = [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 3, 10] as const; export type CmppInboundStage = | 'application_lookup' @@ -25,6 +26,22 @@ export type CmppInboundStage = export type CmppInboundStageResult = 'success' | 'error'; +export type SendWorkerStage = + | 'message_load' + | 'phone_routing' + | 'route_lookup' + | 'signature_candidates' + | 'signature_final_check' + | 'rate_limit' + | 'submit_transaction' + | 'gateway_bullmq_publish' + | 'gateway_stream_publish' + | 'task_progress' + | 'total'; + +export type SendWorkerStageResult = 'success' | 'error' | 'skipped'; +export type SendWorkerQueueState = 'waiting' | 'active' | 'completed' | 'failed' | 'delayed' | 'prioritized'; + type HttpMetric = { count: number; durationSum: number; @@ -48,6 +65,12 @@ export class MetricsService implements OnModuleDestroy { private readonly eventLoopDelay = monitorEventLoopDelay({ resolution: 20 }); private readonly http = new Map(); private readonly cmppInbound = new Map(); + private readonly sendWorkerStages = new Map(); + private readonly sendWorkerQueueJobs = new Map(); + private readonly sendWorkerResults = new Map(); + private sendWorkerConfiguredSlots = 0; + private sendWorkerInFlightSlots = 0; + private readonly sendWorkerDatabasePool = new Map<'max' | 'total' | 'idle' | 'waiting', number>(); private inFlight = 0; private inboundWorkflowPending = 0; private inboundWorkflowProcessing = 0; @@ -101,6 +124,43 @@ export class MetricsService implements OnModuleDestroy { this.cmppInbound.set(key, metric); } + beginSendWorkerStage() { + return process.hrtime.bigint(); + } + + finishSendWorkerStage(startedAt: bigint, stage: SendWorkerStage, result: SendWorkerStageResult) { + const key = `${stage}\u0000${result}`; + const metric = this.sendWorkerStages.get(key) ?? { + count: 0, + durationSum: 0, + buckets: SEND_WORKER_DURATION_BUCKETS.map(() => 0), + }; + const durationSeconds = Number(process.hrtime.bigint() - startedAt) / 1_000_000_000; + metric.count += 1; + metric.durationSum += durationSeconds; + SEND_WORKER_DURATION_BUCKETS.forEach((bucket, index) => { + if (durationSeconds <= bucket) metric.buckets[index] += 1; + }); + this.sendWorkerStages.set(key, metric); + } + + setSendWorkerSlots(configured: number, inFlight: number) { + this.sendWorkerConfiguredSlots = Math.max(0, configured); + this.sendWorkerInFlightSlots = Math.max(0, inFlight); + } + + setSendWorkerQueueJobs(state: SendWorkerQueueState, count: number) { + this.sendWorkerQueueJobs.set(state, Math.max(0, count)); + } + + recordSendWorkerResult(result: 'completed' | 'failed' | 'skipped') { + this.sendWorkerResults.set(result, (this.sendWorkerResults.get(result) ?? 0) + 1); + } + + setSendWorkerDatabasePool(state: 'max' | 'total' | 'idle' | 'waiting', count: number) { + this.sendWorkerDatabasePool.set(state, Math.max(0, count)); + } + setInboundWorkflowState(pending: number, processing: number, oldestPendingAgeSeconds: number) { this.inboundWorkflowPending = Math.max(0, pending); this.inboundWorkflowProcessing = Math.max(0, processing); @@ -144,6 +204,18 @@ export class MetricsService implements OnModuleDestroy { '# TYPE cmpp_api_http_request_duration_seconds histogram', '# HELP cmpp_api_cmpp_inbound_stage_duration_seconds CMPP inbound processing duration by bounded stage and result.', '# TYPE cmpp_api_cmpp_inbound_stage_duration_seconds histogram', + '# HELP cmpp_worker_send_stage_duration_seconds Send worker processing duration by bounded stage and result.', + '# TYPE cmpp_worker_send_stage_duration_seconds histogram', + '# HELP cmpp_worker_send_queue_jobs BullMQ send jobs by queue state.', + '# TYPE cmpp_worker_send_queue_jobs gauge', + '# HELP cmpp_worker_send_slots Send worker concurrency slots by state.', + '# TYPE cmpp_worker_send_slots gauge', + metricLine('cmpp_worker_send_slots', this.sendWorkerConfiguredSlots, { state: 'configured' }), + metricLine('cmpp_worker_send_slots', this.sendWorkerInFlightSlots, { state: 'in_flight' }), + '# HELP cmpp_worker_send_jobs_total Send worker processing outcomes.', + '# TYPE cmpp_worker_send_jobs_total counter', + '# HELP cmpp_worker_database_pool_connections Worker PostgreSQL client pool slots by state.', + '# TYPE cmpp_worker_database_pool_connections gauge', '# HELP cmpp_worker_inbound_workflow_items Current durable CMPP inbound workflow items by state.', '# TYPE cmpp_worker_inbound_workflow_items gauge', metricLine('cmpp_worker_inbound_workflow_items', this.inboundWorkflowPending, { state: 'pending' }), @@ -179,6 +251,25 @@ export class MetricsService implements OnModuleDestroy { lines.push(metricLine('cmpp_api_cmpp_inbound_stage_duration_seconds_sum', metric.durationSum, labels)); lines.push(metricLine('cmpp_api_cmpp_inbound_stage_duration_seconds_count', metric.count, labels)); } + for (const [key, metric] of this.sendWorkerStages) { + const [stage, result] = key.split('\u0000'); + const labels = { stage, result }; + SEND_WORKER_DURATION_BUCKETS.forEach((bucket, index) => { + lines.push(metricLine('cmpp_worker_send_stage_duration_seconds_bucket', metric.buckets[index], { ...labels, le: String(bucket) })); + }); + lines.push(metricLine('cmpp_worker_send_stage_duration_seconds_bucket', metric.count, { ...labels, le: '+Inf' })); + lines.push(metricLine('cmpp_worker_send_stage_duration_seconds_sum', metric.durationSum, labels)); + lines.push(metricLine('cmpp_worker_send_stage_duration_seconds_count', metric.count, labels)); + } + for (const [state, count] of this.sendWorkerQueueJobs) { + lines.push(metricLine('cmpp_worker_send_queue_jobs', count, { state })); + } + for (const [result, count] of this.sendWorkerResults) { + lines.push(metricLine('cmpp_worker_send_jobs_total', count, { result })); + } + for (const [state, count] of this.sendWorkerDatabasePool) { + lines.push(metricLine('cmpp_worker_database_pool_connections', count, { state })); + } for (const [result, count] of this.inboundWorkflowResults) { lines.push(metricLine('cmpp_worker_inbound_workflow_results_total', count, { result })); } diff --git a/api/src/open-api/open-api.service.ts b/api/src/open-api/open-api.service.ts index 64b88e1..81a285a 100644 --- a/api/src/open-api/open-api.service.ts +++ b/api/src/open-api/open-api.service.ts @@ -55,6 +55,10 @@ export class OpenApiService implements OnModuleInit, OnModuleDestroy { onModuleInit() { const connection = bullmqConnection(); this.queue = new Queue(WEBHOOK_QUEUE, { connection }); + // The isolated Gateway callback process only enqueues customer callbacks. + // Delivery remains owned by the main API process so callback DB/HTTP capacity + // cannot be consumed by slow customer webhook endpoints. + if (process.env.CMPP_PROCESS_ROLE === 'callback') return; this.worker = new Worker(WEBHOOK_QUEUE, (job) => this.deliverWebhook(job.data.deliveryId), { connection, concurrency: 10 }); } diff --git a/api/src/prisma/prisma.service.spec.ts b/api/src/prisma/prisma.service.spec.ts index faf2438..11c1a99 100644 --- a/api/src/prisma/prisma.service.spec.ts +++ b/api/src/prisma/prisma.service.spec.ts @@ -7,7 +7,25 @@ describe('PrismaService', () => { expect(Object.getOwnPropertyDescriptor(prisma, 'operationLog')).toEqual( expect.objectContaining({ configurable: true }), ); + expect(prisma.getPoolState()).toEqual({ max: 32, total: 0, idle: 0, waiting: 0 }); await prisma.$disconnect(); }); + + it('reserves a bounded database pool for the isolated Gateway callback process', async () => { + const previousRole = process.env.CMPP_PROCESS_ROLE; + const previousMax = process.env.API_CALLBACK_DB_POOL_MAX; + process.env.CMPP_PROCESS_ROLE = 'callback'; + process.env.API_CALLBACK_DB_POOL_MAX = '12'; + try { + const prisma = new PrismaService(); + expect(prisma.getPoolState()).toEqual({ max: 12, total: 0, idle: 0, waiting: 0 }); + await prisma.$disconnect(); + } finally { + if (previousRole === undefined) delete process.env.CMPP_PROCESS_ROLE; + else process.env.CMPP_PROCESS_ROLE = previousRole; + if (previousMax === undefined) delete process.env.API_CALLBACK_DB_POOL_MAX; + else process.env.API_CALLBACK_DB_POOL_MAX = previousMax; + } + }); }); diff --git a/api/src/prisma/prisma.service.ts b/api/src/prisma/prisma.service.ts index e80e519..09b92ca 100644 --- a/api/src/prisma/prisma.service.ts +++ b/api/src/prisma/prisma.service.ts @@ -1,30 +1,46 @@ import { Injectable, OnModuleDestroy } from '@nestjs/common'; import { PrismaPg } from '@prisma/adapter-pg'; import { PrismaClient } from '@prisma/client'; +import { Pool } from 'pg'; import { requestContext } from '../common/request-context'; @Injectable() export class PrismaService extends PrismaClient implements OnModuleDestroy { + private readonly databasePool: Pool; + private readonly databasePoolMax: number; + constructor() { - const workerRole = process.env.CMPP_PROCESS_ROLE === 'worker'; - const databaseUrl = workerRole - ? process.env.API_WORKER_DATABASE_URL || process.env.DATABASE_URL - : process.env.DATABASE_URL; - const configuredPoolMax = Number(workerRole - ? process.env.API_WORKER_DB_POOL_MAX ?? 8 - : process.env.API_DB_POOL_MAX ?? 32); + const processRole = process.env.CMPP_PROCESS_ROLE?.trim() || 'api'; + const workerRole = processRole === 'worker'; + const outboxRole = processRole === 'outbox'; + const callbackRole = processRole === 'callback'; + const databaseUrl = outboxRole + ? process.env.API_OUTBOX_DATABASE_URL || process.env.DATABASE_URL + : callbackRole + ? process.env.API_CALLBACK_DATABASE_URL || process.env.DATABASE_URL + : workerRole + ? process.env.API_WORKER_DATABASE_URL || process.env.DATABASE_URL + : process.env.DATABASE_URL; + const configuredPoolMax = Number(outboxRole + ? process.env.API_OUTBOX_DB_POOL_MAX ?? 6 + : callbackRole + ? process.env.API_CALLBACK_DB_POOL_MAX ?? 16 + : workerRole + ? process.env.API_WORKER_DB_POOL_MAX ?? 8 + : process.env.API_DB_POOL_MAX ?? 32); const poolMax = Number.isInteger(configuredPoolMax) && configuredPoolMax > 0 ? configuredPoolMax - : workerRole ? 8 : 32; - super({ - adapter: new PrismaPg({ - connectionString: databaseUrl - ?? 'postgresql://cmpp:cmpp_password@localhost:5432/cmpp_platform?schema=public', - // API capacity must be reserved independently from the heavier Worker - // transactions; explicit bounds also protect PostgreSQL max_connections. - max: poolMax, - }), + : outboxRole ? 6 : callbackRole ? 16 : workerRole ? 8 : 32; + const databasePool = new Pool({ + connectionString: databaseUrl + ?? 'postgresql://cmpp:cmpp_password@localhost:5432/cmpp_platform?schema=public', + // API capacity must be reserved independently from the heavier Worker + // transactions; explicit bounds also protect PostgreSQL max_connections. + max: poolMax, }); + super({ adapter: new PrismaPg(databasePool, { disposeExternalPool: true }) }); + this.databasePool = databasePool; + this.databasePoolMax = poolMax; const operationLog = this.operationLog; Object.defineProperty(this, 'operationLog', { value: new Proxy(operationLog, { @@ -46,6 +62,15 @@ export class PrismaService extends PrismaClient implements OnModuleDestroy { }); } + getPoolState() { + return { + max: this.databasePoolMax, + total: this.databasePool.totalCount, + idle: this.databasePool.idleCount, + waiting: this.databasePool.waitingCount, + }; + } + async onModuleDestroy() { await this.$disconnect(); } diff --git a/api/src/send-chain/gateway-callback.controller.spec.ts b/api/src/send-chain/gateway-callback.controller.spec.ts new file mode 100644 index 0000000..6811f74 --- /dev/null +++ b/api/src/send-chain/gateway-callback.controller.spec.ts @@ -0,0 +1,43 @@ +import { BadRequestException } from '@nestjs/common'; +import { GatewayCallbackController } from './gateway-callback.controller'; + +describe('GatewayCallbackController', () => { + const sendChain = { + handleSubmitResult: jest.fn(), handleSubmitSegmentResult: jest.fn(), + intakeReceipt: jest.fn(), handleReceipt: jest.fn(), handleUplink: jest.fn(), + recordGatewaySubmitDeadLetter: jest.fn(), + }; + const protocolLogs = { record: jest.fn() }; + const prisma = { $queryRaw: jest.fn(), getPoolState: jest.fn().mockReturnValue({ max: 16, total: 2, idle: 1, waiting: 0 }) }; + const controller = new GatewayCallbackController(sendChain as never, protocolLogs as never, prisma as never); + + beforeEach(() => jest.clearAllMocks()); + + it('keeps Submit result persistence on the callback process without duplicating protocol logs', async () => { + sendChain.handleSubmitResult.mockResolvedValue({ accepted: true }); + await expect(controller.submitResult({ + messageId: 'MSG-1', channelId: 'channel-1', gatewayMessageId: '1', sequenceId: 1, + submitStatus: 'accepted', + })).resolves.toEqual({ accepted: true }); + expect(sendChain.handleSubmitResult).toHaveBeenCalledTimes(1); + expect(protocolLogs.record).not.toHaveBeenCalled(); + }); + + it('durably intakes receipts before recording the callback protocol event', async () => { + sendChain.intakeReceipt.mockResolvedValue({ accepted: true, inboxId: 'inbox-1' }); + await controller.receiptIntake({ + messageId: 'MSG-1', channelId: 'channel-1', gatewayMessageId: '1', + phoneNumber: '13800000001', receiptStatus: 'delivered', rawStatus: 'DELIVRD', + }); + expect(sendChain.intakeReceipt).toHaveBeenCalledTimes(1); + expect(protocolLogs.record).toHaveBeenCalledWith(expect.objectContaining({ + eventType: 'deliver_receipt', messageId: 'MSG-1', status: 'success', + })); + }); + + it('rejects inbound/client packet logs from the isolated supplier callback surface', () => { + expect(() => controller.protocolLog({ + protocol: 'cmpp', direction: 'client_to_platform', eventType: 'submit', status: 'success', + })).toThrow(BadRequestException); + }); +}); diff --git a/api/src/send-chain/gateway-callback.controller.ts b/api/src/send-chain/gateway-callback.controller.ts new file mode 100644 index 0000000..e1f5421 --- /dev/null +++ b/api/src/send-chain/gateway-callback.controller.ts @@ -0,0 +1,102 @@ +import { BadRequestException, Body, Controller, Get, Post } from '@nestjs/common'; +import { PrismaService } from '../prisma/prisma.service'; +import { ProtocolLogsService, type ProtocolLogInput } from '../protocol-logs/protocol-logs.service'; +import type { + GatewayReceiptEventDto, + GatewaySubmitDeadLetterDto, + GatewaySubmitResultDto, + GatewaySubmitSegmentResultDto, + GatewayUplinkEventDto, +} from './send-chain.contracts'; +import { SendChainService } from './send-chain.service'; + +@Controller() +export class GatewayCallbackController { + constructor( + private readonly sendChain: SendChainService, + private readonly protocolLogs: ProtocolLogsService, + private readonly prisma: PrismaService, + ) {} + + @Get('health') + async health() { + await this.prisma.$queryRaw`SELECT 1`; + return { status: 'ok', role: 'gateway-callback', databasePool: this.prisma.getPoolState() }; + } + + @Post('gateway/events/submit-result') + submitResult(@Body() body: GatewaySubmitResultDto) { + return this.sendChain.handleSubmitResult(body); + } + + @Post('gateway/events/submit-segment-result') + submitSegmentResult(@Body() body: GatewaySubmitSegmentResultDto) { + return this.sendChain.handleSubmitSegmentResult(body); + } + + @Post('gateway/events/receipt/intake') + receiptIntake(@Body() body: GatewayReceiptEventDto) { + return this.track('deliver_receipt', body, () => this.sendChain.intakeReceipt(body)); + } + + @Post('gateway/events/receipt') + receipt(@Body() body: GatewayReceiptEventDto) { + return this.track('deliver_receipt', body, () => this.sendChain.handleReceipt(body)); + } + + @Post('gateway/events/uplink') + uplink(@Body() body: GatewayUplinkEventDto) { + return this.track('deliver_uplink', body, () => this.sendChain.handleUplink(body)); + } + + @Post('gateway/events/protocol-log') + protocolLog(@Body() body: ProtocolLogInput) { + const allowedPacket = ( + body.direction === 'platform_to_channel' && ['submit', 'deliver_resp'].includes(body.eventType) + ) || ( + body.direction === 'channel_to_platform' && body.eventType === 'submit_resp' + ); + if (body.protocol !== 'cmpp' || !allowedPacket || !['success', 'failed'].includes(body.status)) { + throw new BadRequestException('Unsupported Gateway callback protocol log event'); + } + this.protocolLogs.record(body); + return { accepted: true }; + } + + @Post('gateway/events/dead-letter') + deadLetter(@Body() body: GatewaySubmitDeadLetterDto) { + return this.sendChain.recordGatewaySubmitDeadLetter(body); + } + + private async track(eventType: string, body: object, action: () => Promise | T) { + const startedAt = Date.now(); + const value = body as Record; + const common: Omit = { + protocol: 'cmpp', direction: 'channel_to_platform', eventType, + tenantId: value.tenantId as string, applicationId: value.applicationId as string, + channelId: value.channelId as string, + messageId: (value.messageId ?? value.platformMessageId) as string, + gatewayMessageId: (value.gatewayMessageId ?? value.msgId ?? value.upstreamMessageId) as string, + phone: (value.phoneNumber ?? value.srcTerminalId) as string, + resultCode: (value.rawStatus ?? value.status ?? value.stat) as string, + }; + try { + const result = await action(); + const resolved = result && typeof result === 'object' ? result as Record : {}; + this.protocolLogs.record({ + ...common, + tenantId: (resolved.tenantId ?? common.tenantId) as string, + applicationId: (resolved.applicationId ?? common.applicationId) as string, + messageId: (resolved.messageId ?? common.messageId) as string, + status: 'success', durationMs: Date.now() - startedAt, + }); + return result; + } catch (error) { + this.protocolLogs.record({ + ...common, status: 'failed', durationMs: Date.now() - startedAt, + detail: { error: error instanceof Error ? error.message : String(error) }, + }); + throw error; + } + } +} diff --git a/api/src/send-chain/send-chain.service.spec.ts b/api/src/send-chain/send-chain.service.spec.ts index 4f2aa74..9b19567 100644 --- a/api/src/send-chain/send-chain.service.spec.ts +++ b/api/src/send-chain/send-chain.service.spec.ts @@ -158,10 +158,12 @@ function createPrismaMock() { findUnique: jest.fn().mockResolvedValue(channel), }, cmppSubmitSession: { + findUnique: jest.fn().mockResolvedValue({ id: 'session-1' }), upsert: jest.fn().mockResolvedValue({ id: 'session-1' }), }, smsSubmitRecord: { create: jest.fn().mockResolvedValue({ id: 'submit-1' }), + createMany: jest.fn().mockResolvedValue({ count: 2 }), updateMany: jest.fn().mockResolvedValue({ count: 1 }), findFirst: jest.fn().mockResolvedValue({ id: 'submit-1', submitId: 'SUB-1', submitStatus: 'accepted', createdAt: new Date('2026-07-01T10:00:00.000Z') }), findUnique: jest.fn().mockImplementation(({ where }) => Promise.resolve( @@ -329,6 +331,11 @@ function createPrismaMock() { lastError: 'downstream client is not connected', }), }, + gatewaySubmitOutbox: { + create: jest.fn().mockResolvedValue({ id: 'outbox-1', submitId: 'SUB-1', status: 'pending' }), + createMany: jest.fn().mockResolvedValue({ count: 2 }), + updateMany: jest.fn().mockResolvedValue({ count: 1 }), + }, cmppInboundSubmissionInbox: { create: jest.fn().mockResolvedValue({ id: 'inbox-1' }), findUnique: jest.fn().mockResolvedValue(null), @@ -926,6 +933,7 @@ describe('SendChainService', () => { it('coalesces concurrent batch progress refreshes and keeps a trailing refresh', async () => { const { service, prisma } = createService(); + prisma.$executeRaw.mockResolvedValue(0); let resolveFirst: ((value: Array<{ status: string; _count: { _all: number } }>) => void) | undefined; prisma.smsMessageRecord.groupBy .mockImplementationOnce(() => new Promise((resolve) => { resolveFirst = resolve; })) @@ -946,6 +954,37 @@ describe('SendChainService', () => { }); }); + it('updates a known single-message CMPP task without grouping all message states', async () => { + const { service, prisma } = createService(); + + await service['submission'].refreshTaskProgress('task-cmpp-1', 'submit_queued'); + + expect(prisma.smsBatchTask.updateMany).toHaveBeenCalledWith({ + where: { id: 'task-cmpp-1', sourceType: 'cmpp', phoneTotal: 1 }, + data: { + progressTotal: 1, + submittedTotal: 1, + successTotal: 0, + failedTotal: 0, + unknownTotal: 0, + timeoutTotal: 0, + status: 'sending', + }, + }); + expect(prisma.smsMessageRecord.groupBy).not.toHaveBeenCalled(); + }); + + it('refreshes callback progress for a single-message CMPP task in one direct SQL update', async () => { + const { service, prisma } = createService(); + prisma.$executeRaw.mockResolvedValue(1); + + await service['submission'].refreshTaskProgress('task-cmpp-callback'); + + expect(prisma.$executeRaw).toHaveBeenCalledTimes(1); + expect(prisma.smsMessageRecord.groupBy).not.toHaveBeenCalled(); + expect(prisma.smsBatchTask.update).not.toHaveBeenCalled(); + }); + it('does not expose CMPP internal tasks through client task detail or messages', async () => { const { service, prisma } = createService(); prisma.smsBatchTask.findFirst.mockResolvedValue(null); @@ -2038,9 +2077,7 @@ describe('SendChainService', () => { it('routes queued messages to gateway submit commands', async () => { const { service, prisma } = createService(); - const gatewayAdd = jest.fn().mockResolvedValue(undefined); service['waitForChannelRateLimit'] = jest.fn().mockResolvedValue(undefined); - service['getGatewayQueue'] = jest.fn().mockReturnValue({ add: gatewayAdd }); await expect(service.processSendJob({ messageRecordId: 'record-1' })).resolves.toEqual( expect.objectContaining({ submitted: true, messageRecordId: 'record-1', channelId: 'channel-1' }), @@ -2059,8 +2096,7 @@ describe('SendChainService', () => { where: { id: 'record-1' }, data: expect.objectContaining({ channelId: 'channel-1', carrier: 'mobile', province: '山东', status: 'submit_queued' }), }); - expect(gatewayAdd).toHaveBeenCalledWith( - 'submit-command', + expect(service['publishGatewaySubmitCommand']).toHaveBeenCalledWith( expect.objectContaining({ schemaVersion: 'v1', messageType: 'SubmitCommand', @@ -2074,9 +2110,117 @@ describe('SendChainService', () => { }), ); expect(service['publishGatewaySubmitCommand']).toHaveBeenCalledWith(expect.objectContaining({ messageId: 'MSG-1' })); + expect(prisma.cmppSubmitSession.findUnique).toHaveBeenCalledWith({ + where: { sessionNo: 'OPEN-channel-1' }, + select: { id: true }, + }); + expect(prisma.cmppSubmitSession.upsert).not.toHaveBeenCalled(); expect(service['postGatewayControl']).not.toHaveBeenCalledWith('/upstream/submit', expect.anything()); }); + it('shadow-writes the durable submit Outbox without replacing the direct stream path', async () => { + const previousShadow = process.env.SEND_SUBMIT_OUTBOX_SHADOW_ENABLED; + const previousPublish = process.env.SEND_SUBMIT_OUTBOX_PUBLISH_ENABLED; + process.env.SEND_SUBMIT_OUTBOX_SHADOW_ENABLED = 'true'; + delete process.env.SEND_SUBMIT_OUTBOX_PUBLISH_ENABLED; + try { + const { service, prisma } = createService(); + service['waitForChannelRateLimit'] = jest.fn().mockResolvedValue(undefined); + + await service.processSendJob({ messageRecordId: 'record-1' }); + + expect(prisma.gatewaySubmitOutbox.create).toHaveBeenCalledWith({ + data: expect.objectContaining({ + messageRecordId: 'record-1', + channelId: 'channel-1', + payload: expect.objectContaining({ messageType: 'SubmitCommand', messageId: 'MSG-1' }), + }), + }); + expect(service['publishGatewaySubmitCommand']).toHaveBeenCalledTimes(1); + } finally { + if (previousShadow === undefined) delete process.env.SEND_SUBMIT_OUTBOX_SHADOW_ENABLED; + else process.env.SEND_SUBMIT_OUTBOX_SHADOW_ENABLED = previousShadow; + if (previousPublish === undefined) delete process.env.SEND_SUBMIT_OUTBOX_PUBLISH_ENABLED; + else process.env.SEND_SUBMIT_OUTBOX_PUBLISH_ENABLED = previousPublish; + } + }); + + it('uses only the durable Outbox when formal publishing is enabled', async () => { + const previousShadow = process.env.SEND_SUBMIT_OUTBOX_SHADOW_ENABLED; + const previousPublish = process.env.SEND_SUBMIT_OUTBOX_PUBLISH_ENABLED; + process.env.SEND_SUBMIT_OUTBOX_SHADOW_ENABLED = 'true'; + process.env.SEND_SUBMIT_OUTBOX_PUBLISH_ENABLED = 'true'; + try { + const { service, prisma } = createService(); + service['waitForChannelRateLimit'] = jest.fn().mockResolvedValue(undefined); + + await service.processSendJob({ messageRecordId: 'record-1' }); + + expect(prisma.gatewaySubmitOutbox.create).toHaveBeenCalledTimes(1); + expect(service['publishGatewaySubmitCommand']).not.toHaveBeenCalled(); + } finally { + if (previousShadow === undefined) delete process.env.SEND_SUBMIT_OUTBOX_SHADOW_ENABLED; + else process.env.SEND_SUBMIT_OUTBOX_SHADOW_ENABLED = previousShadow; + if (previousPublish === undefined) delete process.env.SEND_SUBMIT_OUTBOX_PUBLISH_ENABLED; + else process.env.SEND_SUBMIT_OUTBOX_PUBLISH_ENABLED = previousPublish; + } + }); + + it('plans routes once and bulk-creates Submit records and Outbox rows for a Worker batch', async () => { + const previousPublish = process.env.SEND_SUBMIT_OUTBOX_PUBLISH_ENABLED; + process.env.SEND_SUBMIT_OUTBOX_PUBLISH_ENABLED = 'true'; + try { + const { service, prisma } = createService(); + const base = await prisma.smsMessageRecord.findUnique({ where: { id: 'record-1' } }); + const messages = [ + { ...base, id: 'record-1', messageId: 'MSG-1', phoneNumber: '13800000001', amountCents: 3n, carrier: null, province: null, batchTask: { sourceType: 'cmpp', phoneTotal: 1 } }, + { ...base, id: 'record-2', batchTaskId: 'task-2', messageId: 'MSG-2', phoneNumber: '13800000002', amountCents: 3n, carrier: null, province: null, batchTask: { sourceType: 'cmpp', phoneTotal: 1 } }, + ]; + const channel = { + id: 'channel-1', code: 'CMPP-A', account: 'cmpp-account', srcId: '10690000', + rateLimitPerSecond: 100, unitPrice: 3n, status: 'active', carrier: 'mobile', sendRegion: '全国', + gatewayHost: '127.0.0.1', gatewayPort: 17890, passwordCipher: 'secret', cmppVersion: '3.0', + config: { serviceId: 'SMS' }, + connectionStates: [{ status: 'connected', currentConnections: 1, desiredConnections: 1 }], + reportTasks: [{ signatureId: 'sig-1', carrier: 'mobile', approvalScope: 'carrier_specific' }], + }; + prisma.smsMessageRecord.findMany.mockResolvedValue(messages); + prisma.channelRouteRule.findMany.mockResolvedValue([{ + tenantId: 'tenant-1', applicationId: 'app-1', carrier: 'mobile', groupId: 'group-1', + group: { + name: '默认通道组', carrier: 'mobile', status: 'active', + items: [{ id: 'item-1', groupId: 'group-1', channelId: 'channel-1', carrier: 'mobile', province: null, priority: 1, weight: 1, isBackup: false, channel }], + }, + }]); + service['waitForChannelRateLimit'] = jest.fn().mockResolvedValue(undefined); + + const gatewaySubmit = (service as any).submission.gatewaySubmit; + const result = await gatewaySubmit.processSendJobBatch([ + { messageRecordId: 'record-1' }, { messageRecordId: 'record-2' }, + ]); + + expect(prisma.channelRouteRule.findMany).toHaveBeenCalledTimes(1); + expect(prisma.phoneSegment.findMany).toHaveBeenCalledTimes(1); + expect(prisma.smsSubmitRecord.createMany).toHaveBeenCalledWith({ + data: expect.arrayContaining([ + expect.objectContaining({ messageRecordId: 'record-1', channelId: 'channel-1' }), + expect.objectContaining({ messageRecordId: 'record-2', channelId: 'channel-1' }), + ]), + }); + expect(prisma.gatewaySubmitOutbox.createMany).toHaveBeenCalledWith({ + data: expect.arrayContaining([ + expect.objectContaining({ messageRecordId: 'record-1', payload: expect.objectContaining({ messageId: 'MSG-1' }) }), + expect.objectContaining({ messageRecordId: 'record-2', payload: expect.objectContaining({ messageId: 'MSG-2' }) }), + ]), + }); + expect(result.get('record-1')).toEqual(expect.objectContaining({ submitted: true })); + expect(result.get('record-2')).toEqual(expect.objectContaining({ submitted: true })); + } finally { + if (previousPublish === undefined) delete process.env.SEND_SUBMIT_OUTBOX_PUBLISH_ENABLED; + else process.env.SEND_SUBMIT_OUTBOX_PUBLISH_ENABLED = previousPublish; + } + }); + it('appends the real application extension to the upstream channel base number', async () => { const { service, prisma } = createService(); const queuedMessage = await prisma.smsMessageRecord.findUnique({ where: { id: 'record-1' } }); @@ -2085,14 +2229,11 @@ describe('SendChainService', () => { applicationExtension: '0001', clientSrcId: '000001', }); - const gatewayAdd = jest.fn().mockResolvedValue(undefined); service['waitForChannelRateLimit'] = jest.fn().mockResolvedValue(undefined); - service['getGatewayQueue'] = jest.fn().mockReturnValue({ add: gatewayAdd }); await service.processSendJob({ messageRecordId: 'record-1' }); - expect(gatewayAdd).toHaveBeenCalledWith( - 'submit-command', + expect(service['publishGatewaySubmitCommand']).toHaveBeenCalledWith( expect.objectContaining({ cmpp: expect.objectContaining({ srcId: '106900000001' }) }), ); }); @@ -2105,17 +2246,14 @@ describe('SendChainService', () => { prisma.channelRouteRule.findFirst.mockResolvedValue({ ...baseRoute, group: { ...baseRoute.group, items: [ - { ...baseRoute.group.items[0], channelId: primary.id, priority: 1, channel: primary }, { ...baseRoute.group.items[0], id: 'item-2', channelId: backup.id, priority: 2, channel: backup }, ] }, }); - prisma.channelSignatureReportTask.findMany.mockResolvedValue([{ channelId: backup.id }]); - const gatewayAdd = jest.fn().mockResolvedValue(undefined); service['waitForChannelRateLimit'] = jest.fn().mockResolvedValue(undefined); - service['getGatewayQueue'] = jest.fn().mockReturnValue({ add: gatewayAdd }); await expect(service.processSendJob({ messageRecordId: 'record-1' })).resolves.toEqual(expect.objectContaining({ submitted: true, channelId: backup.id })); - expect(prisma.channelSignatureReportTask.findMany).toHaveBeenCalledWith(expect.objectContaining({ where: expect.objectContaining({ reportType: 'signature' }) })); + expect(prisma.channelSignatureReportTask.findMany).not.toHaveBeenCalled(); + expect(JSON.stringify(prisma.channelRouteRule.findFirst.mock.calls.at(-1)?.[0])).toContain('sig-1'); expect(prisma.smsMessageRecord.update).toHaveBeenCalledWith(expect.objectContaining({ data: expect.objectContaining({ channelId: backup.id }) })); }); @@ -2127,6 +2265,7 @@ describe('SendChainService', () => { id: 'record-1', tenantId: 'tenant-1', applicationId: 'app-1', + signatureId: 'sig-1', phoneNumber: '13800000001', })).rejects.toThrow('企业应用未配置对应运营商通道组'); @@ -2554,9 +2693,7 @@ describe('SendChainService', () => { queuedAt: new Date(), }, '回执失败补发')).resolves.toEqual(expect.objectContaining({ channelId: backup.id })); - expect(prisma.channelSignatureReportTask.findMany).toHaveBeenCalledWith(expect.objectContaining({ - where: expect.objectContaining({ signatureId: 'sig-direct' }), - })); + expect(JSON.stringify(prisma.channelRouteRule.findFirst.mock.calls.at(-1)?.[0])).toContain('sig-direct'); expect(submitMessageToGateway).toHaveBeenCalledWith( expect.objectContaining({ signatureId: 'sig-direct' }), expect.objectContaining({ channel: expect.objectContaining({ id: backup.id }) }), @@ -2617,7 +2754,7 @@ describe('SendChainService', () => { expect(results.filter((result) => result.submitted)).toHaveLength(1); expect(results.filter((result) => result.duplicateRetry)).toHaveLength(2); - expect(queueAdd).toHaveBeenCalledTimes(1); + expect(queueAdd).not.toHaveBeenCalled(); expect(service['publishGatewaySubmitCommand']).toHaveBeenCalledTimes(1); expect(prisma.smsMessageRecord.update).toHaveBeenCalledTimes(1); }); @@ -3378,10 +3515,9 @@ describe('SendChainService', () => { it('blocks submit when signature is not approved on the selected channel', async () => { const { service, prisma } = createService(); - const gatewayAdd = jest.fn().mockResolvedValue(undefined); service['waitForChannelRateLimit'] = jest.fn().mockResolvedValue(undefined); - service['getGatewayQueue'] = jest.fn().mockReturnValue({ add: gatewayAdd }); - prisma.channelSignatureReportTask.findMany.mockResolvedValue([]); + const route = await prisma.channelRouteRule.findFirst(); + prisma.channelRouteRule.findFirst.mockResolvedValue({ ...route, group: { ...route.group, items: [] } }); prisma.smsMessageRecord.findUnique.mockResolvedValue({ id: 'record-1', tenantId: 'tenant-1', @@ -3396,14 +3532,14 @@ describe('SendChainService', () => { amountCents: 3, status: 'queued', queuePriority: 'normal', - batchTask: { sourceType: 'cmpp' }, + batchTask: { sourceType: 'cmpp', phoneTotal: 1 }, template: { signature: { id: 'sig-1', name: '签名' } }, }); await expect(service.processSendJob({ messageRecordId: 'record-1' })).resolves.toEqual( expect.objectContaining({ submitted: false, status: 'failed', reason: '无已报备通过且在线的可用通道' }), ); - expect(gatewayAdd).not.toHaveBeenCalled(); + expect(service['publishGatewaySubmitCommand']).not.toHaveBeenCalled(); expect(prisma.smsReceiptRecord.create).toHaveBeenCalledWith({ data: expect.objectContaining({ receiptStatus: 'undelivered', errorCode: 'ROUTE' }), }); @@ -3411,9 +3547,7 @@ describe('SendChainService', () => { it('only selects channel group items allocated to the matched carrier', async () => { const { service, prisma } = createService(); - const gatewayAdd = jest.fn().mockResolvedValue(undefined); service['waitForChannelRateLimit'] = jest.fn().mockResolvedValue(undefined); - service['getGatewayQueue'] = jest.fn().mockReturnValue({ add: gatewayAdd }); prisma.channelRouteRule.findFirst.mockResolvedValue({ id: 'route-1', tenantId: 'tenant-1', @@ -3475,8 +3609,7 @@ describe('SendChainService', () => { await expect(service.processSendJob({ messageRecordId: 'record-1' })).resolves.toEqual( expect.objectContaining({ channelId: 'channel-all' }), ); - expect(gatewayAdd).toHaveBeenCalledWith( - 'submit-command', + expect(service['publishGatewaySubmitCommand']).toHaveBeenCalledWith( expect.objectContaining({ channelId: 'channel-all', route: expect.objectContaining({ channelCode: 'CMPP-ALL', carrier: 'mobile' }) }), ); }); @@ -4429,7 +4562,7 @@ describe('SendChainService', () => { where: { id: 'record-1', status: 'timeout', timeoutReceiptQueuedAt: null }, data: { timeoutReceiptQueuedAt: expect.any(Date) }, }); - expect(prisma.smsBatchTask.update).toHaveBeenCalled(); + expect(prisma.$executeRaw).toHaveBeenCalled(); }); it('queues an explicit HTTP failure webhook when a receipt times out', async () => { diff --git a/api/src/send-chain/send-chain.service.ts b/api/src/send-chain/send-chain.service.ts index b52a2b5..b44325d 100644 --- a/api/src/send-chain/send-chain.service.ts +++ b/api/src/send-chain/send-chain.service.ts @@ -79,7 +79,11 @@ export class SendChainService implements OnModuleInit, OnModuleDestroy { onModuleInit() { const processRole = process.env.CMPP_PROCESS_ROLE?.trim() || 'all'; - if (processRole === 'api') return; + if (processRole === 'api' || processRole === 'callback') return; + if (processRole === 'outbox') { + this.submission.startSubmitOutboxPublisher(); + return; + } if (process.env.API_ENABLE_SEND_WORKER === 'true') { this.startWorker(); } @@ -701,8 +705,8 @@ export class SendChainService implements OnModuleInit, OnModuleDestroy { return this.submission.selectChannelForMessage(message, options); } - private async findApplicationRoute(tenantId: string, applicationId: string | undefined, carrier: string) { - return this.submission.findApplicationRoute(tenantId, applicationId, carrier); + private async findApplicationRoute(tenantId: string, applicationId: string | undefined, carrier: string, signatureId?: string) { + return this.submission.findApplicationRoute(tenantId, applicationId, carrier, signatureId); } private async identifyCarrier(phoneNumber: string) { @@ -836,8 +840,8 @@ export class SendChainService implements OnModuleInit, OnModuleDestroy { return this.submission.waitForChannelRateLimit(channelId, tps); } - private async refreshTaskProgress(batchTaskId: string) { - return this.submission.refreshTaskProgress(batchTaskId); + private async refreshTaskProgress(batchTaskId: string, knownSingleMessageStatus?: string) { + return this.submission.refreshTaskProgress(batchTaskId, knownSingleMessageStatus); } private smsMessageSegmentAuditDelegate() { diff --git a/api/src/send-chain/send-gateway-submit.service.ts b/api/src/send-chain/send-gateway-submit.service.ts index cb07e53..fcecc33 100644 --- a/api/src/send-chain/send-gateway-submit.service.ts +++ b/api/src/send-chain/send-gateway-submit.service.ts @@ -8,6 +8,7 @@ import { BillingService } from '../billing/billing.service'; import { isIpAllowed } from '../common/ip-allowlist'; import { moneyToNumber } from '../common/money'; import { PhoneRoutingLookupService } from '../dictionaries/phone-routing-lookup.service'; +import { MetricsService, SendWorkerQueueState, SendWorkerStage } from '../metrics/metrics.service'; import { PrismaService } from '../prisma/prisma.service'; import { RiskReviewService } from '../risk-review/risk-review.service'; import { PhoneFrequencyService } from '../risk-review/phone-frequency.service'; @@ -15,6 +16,12 @@ import type { CreateBatchTaskDto, CreateHttpBatchTaskDto, GatewayInboundAuthDto, import { SEND_QUEUE, GATEWAY_SUBMIT_QUEUE, GATEWAY_SUBMIT_STREAM, DEFAULT_SCHEDULED_DISPATCH_STALE_MS, DEFAULT_INBOUND_LONG_MESSAGE_PROCESSING_STALE_SECONDS, GATEWAY_SUBMIT_REQUEUE_IDEMPOTENCY_TTL_SECONDS, BULLMQ_PRIORITY, drainageRejectionReason, statusFromRisk, parseSchedule, parseImportRows, splitImportLine, cellByHeader, normalizeCarrier, normalizeQueuePriority, getPositiveConfigInteger, getNonNegativeConfigInteger, isCarrierCompatible, matchTemplateContent, isNationalChannel, validateInboundApplicationSrcId, composeUpstreamSrcId, positiveInteger, parseOptionalSequenceId, shanghaiDateKey, bullmqConnection, matchesApplicationSecret, octetString, selectChannelCandidate } from './send-chain.helpers'; import type { SendSubmissionCallbacks, SendSubmissionService } from './send-submission.service'; +type PendingSendBatchItem = { + job: SendJob; + resolve: (value: unknown) => void; + reject: (reason: unknown) => void; +}; + /** * R9 gatewaySubmit implementation. Cross-method calls return through the stable SendChainService seam. */ @@ -24,8 +31,18 @@ export class SendGatewaySubmitService { private sendQueue?: Queue; private gatewayQueue?: Queue; private worker?: Worker; + private sendQueueMetricsTimer?: ReturnType; + private submitOutboxTimer?: ReturnType; + private submitOutboxRunning = false; + private readonly submitOutboxLeaseOwner = `send-worker-${process.pid}-${randomUUID()}`; + private sendWorkerInFlight = 0; + private sendWorkerConfiguredSlots = 0; + private pendingSendBatch: PendingSendBatchItem[] = []; + private sendBatchTimer?: ReturnType; + private sendBatchFlushing = false; private readonly taskProgressRefreshes = new Map>(); private readonly dirtyTaskProgressRefreshes = new Set(); + private readonly openSubmitSessionIds = new Map>(); constructor( private readonly prisma: PrismaService, @@ -35,9 +52,14 @@ export class SendGatewaySubmitService { private readonly phoneRouting: PhoneRoutingLookupService, private readonly facade: SendSubmissionService, private readonly callbacks: SendSubmissionCallbacks, + private readonly metrics?: MetricsService, ) {} async onModuleDestroy() { + if (this.sendQueueMetricsTimer) clearInterval(this.sendQueueMetricsTimer); + if (this.submitOutboxTimer) clearInterval(this.submitOutboxTimer); + if (this.sendBatchTimer) clearTimeout(this.sendBatchTimer); + await this.flushSendBatch(); await this.worker?.close(); await this.sendQueue?.close(); await this.gatewayQueue?.close(); @@ -112,46 +134,390 @@ startWorker() { return { status: 'already_started' }; } const connection = bullmqConnection(); + const configuredConcurrency = Number(process.env.API_SEND_WORKER_CONCURRENCY ?? 20); + this.sendWorkerConfiguredSlots = Number.isInteger(configuredConcurrency) && configuredConcurrency > 0 ? configuredConcurrency : 20; + this.metrics?.setSendWorkerSlots(this.sendWorkerConfiguredSlots, this.sendWorkerInFlight); this.worker = new Worker( SEND_QUEUE, - async (job) => this.facade.processSendJob(job.data), - { connection, concurrency: Number(process.env.API_SEND_WORKER_CONCURRENCY ?? 20) }, + async (job) => { + this.sendWorkerInFlight += 1; + this.metrics?.setSendWorkerSlots(this.sendWorkerConfiguredSlots, this.sendWorkerInFlight); + try { + return process.env.API_SEND_WORKER_BATCH_ENABLED === 'false' + ? await this.facade.processSendJob(job.data) + : await this.enqueueSendBatch(job.data); + } finally { + this.sendWorkerInFlight = Math.max(0, this.sendWorkerInFlight - 1); + this.metrics?.setSendWorkerSlots(this.sendWorkerConfiguredSlots, this.sendWorkerInFlight); + } + }, + { connection, concurrency: this.sendWorkerConfiguredSlots }, ); + void this.refreshSendQueueMetrics(); + this.sendQueueMetricsTimer = setInterval(() => void this.refreshSendQueueMetrics(), 5_000); + this.sendQueueMetricsTimer.unref?.(); + if (this.submitOutboxEnabled() && process.env.SEND_SUBMIT_OUTBOX_SEPARATE_PROCESS_ENABLED !== 'true') { + void this.publishSubmitOutboxBatch(); + this.submitOutboxTimer = setInterval( + () => void this.publishSubmitOutboxBatch(), + getPositiveConfigInteger(process.env, 'SEND_SUBMIT_OUTBOX_POLL_INTERVAL_MS', 25), + ); + this.submitOutboxTimer.unref?.(); + } return { status: 'started' }; } -async processSendJob(job: SendJob) { - const message = await this.prisma.smsMessageRecord.findUnique({ - where: { id: job.messageRecordId }, - include: { batchTask: true, template: { include: { signature: true } }, signature: true }, - }); - if (!message || message.status !== 'queued') { - return { skipped: true }; - } - if (!message.tenantId || !message.batchTaskId) { - return { skipped: true, reason: 'standalone channel test message' }; - } - const businessMessage = message as typeof message & { tenantId: string; batchTaskId: string }; - try { - const routed = await this.facade.selectChannelForMessage(businessMessage); - return await this.facade.submitMessageToGateway(businessMessage, routed, 0); - } catch (error) { - const reason = error instanceof Error ? error.message : '无可用通道组或通道'; - await this.prisma.smsMessageRecord.update({ - where: { id: message.id }, - data: { status: 'failed', errorMessage: reason }, - }); - await this.releaseMessageReservation(businessMessage, reason); - if (message.batchTask?.sourceType === 'cmpp') { - await this.recordCmppFailureReceipt(businessMessage, 'ROUTE', reason); - } else { - await this.facade.refreshTaskProgress(businessMessage.batchTaskId); +startSubmitOutboxPublisher() { + if (this.submitOutboxTimer) return { status: 'already_started' }; + if (!this.submitOutboxEnabled()) return { status: 'disabled' }; + void this.publishSubmitOutboxBatch(); + this.submitOutboxTimer = setInterval( + () => void this.publishSubmitOutboxBatch(), + getPositiveConfigInteger(process.env, 'SEND_SUBMIT_OUTBOX_POLL_INTERVAL_MS', 25), + ); + this.submitOutboxTimer.unref?.(); + return { status: 'started' }; + } + + private enqueueSendBatch(job: SendJob) { + return new Promise((resolve, reject) => { + this.pendingSendBatch.push({ job, resolve, reject }); + const batchSize = Math.min(128, getPositiveConfigInteger(process.env, 'API_SEND_WORKER_BATCH_SIZE', 32)); + if (this.pendingSendBatch.length >= batchSize) { + if (this.sendBatchTimer) clearTimeout(this.sendBatchTimer); + this.sendBatchTimer = undefined; + queueMicrotask(() => void this.flushSendBatch()); + return; } - return { submitted: false, messageRecordId: message.id, status: 'failed', reason }; + if (!this.sendBatchTimer) { + this.sendBatchTimer = setTimeout( + () => { + this.sendBatchTimer = undefined; + void this.flushSendBatch(); + }, + Math.min(25, getPositiveConfigInteger(process.env, 'API_SEND_WORKER_BATCH_WAIT_MS', 3)), + ); + this.sendBatchTimer.unref?.(); + } + }); + } + + private async flushSendBatch() { + if (this.sendBatchFlushing) return; + this.sendBatchFlushing = true; + try { + const batchSize = Math.min(128, getPositiveConfigInteger(process.env, 'API_SEND_WORKER_BATCH_SIZE', 32)); + while (this.pendingSendBatch.length > 0) { + const batch = this.pendingSendBatch.splice(0, batchSize); + try { + const results = await this.processSendJobBatch(batch.map((item) => item.job)); + for (const item of batch) item.resolve(results.get(item.job.messageRecordId) ?? { skipped: true }); + } catch (error) { + for (const item of batch) item.reject(error); + } + } + } finally { + this.sendBatchFlushing = false; + if (this.pendingSendBatch.length > 0) queueMicrotask(() => void this.flushSendBatch()); } } -async submitMessageToGateway( + private async processSendJobBatch(jobs: SendJob[]) { + if (jobs.length === 1) { + return new Map([[jobs[0].messageRecordId, await this.processSendJob(jobs[0])]]); + } + const ids = [...new Set(jobs.map((job) => job.messageRecordId))]; + const messages = await this.measureSendStage('message_load', () => this.prisma.smsMessageRecord.findMany({ + where: { id: { in: ids } }, + include: { batchTask: true, template: { include: { signature: true } }, signature: true }, + })); + const messageById = new Map(messages.map((message) => [message.id, message])); + const results = new Map(); + const businessMessages = messages.filter((message) => { + if (message.status !== 'queued' || !message.tenantId || !message.batchTaskId) { + results.set(message.id, { skipped: true }); + return false; + } + return true; + }) as Array; + for (const id of ids) if (!messageById.has(id)) results.set(id, { skipped: true }); + if (businessMessages.length === 0) return results; + + const { planned, failed } = await this.planRoutesBatch(businessMessages); + if (failed.length > 0) await this.failRouteBatch(failed, results); + if (planned.length === 0) return results; + + await Promise.all(planned.map(({ routed }) => ( + this.facade.waitForChannelRateLimit(routed.channel.id, routed.channel.rateLimitPerSecond) + ))); + const sessionByChannel = new Map(); + await Promise.all([...new Set(planned.map(({ routed }) => routed.channel.id))].map(async (channelId) => { + sessionByChannel.set(channelId, await this.getOpenSubmitSessionId(channelId)); + })); + const prepared = planned.map(({ message, routed }) => { + const submitId = `SUB-${randomUUID()}`; + const upstreamSrcId = composeUpstreamSrcId(routed.channel.srcId, message.applicationExtension); + return { + message, + routed, + submitId, + command: this.buildGatewaySubmitCommand(message, routed, 0, submitId, upstreamSrcId), + sessionId: sessionByChannel.get(routed.channel.id), + }; + }); + const writeOutbox = this.submitOutboxEnabled(); + await this.measureSendStage('submit_transaction', () => this.prisma.$transaction(async (tx) => { + await tx.smsSubmitRecord.createMany({ + data: prepared.map(({ message, routed, submitId, sessionId }) => ({ + id: randomUUID(), + tenantId: message.tenantId, + batchTaskId: message.batchTaskId, + messageRecordId: message.id, + channelId: routed.channel.id, + channelGroupId: routed.groupId, + channelGroupName: routed.groupName, + sessionId, + submitId, + submitStatus: 'queued', + costUnitPrice: routed.channel.unitPrice ?? 0, + costAmountCents: moneyToNumber(routed.channel.unitPrice) * Math.max(1, message.billingUnits ?? 1), + })), + }); + const updates = Prisma.join(prepared.map(({ message, routed, submitId }) => Prisma.sql`( + ${message.id}::text, ${routed.channel.id}::text, ${routed.carrier}::text, + ${routed.province ?? null}::text, ${submitId}::text + )`)); + await tx.$executeRaw(Prisma.sql` + UPDATE "SmsMessageRecord" AS message + SET "channelId" = updates."channelId", + carrier = updates.carrier, + province = updates.province, + "submitId" = updates."submitId", + status = 'submit_queued', + "submitStatus" = 'queued', + "receiptStatus" = NULL, + "errorCode" = NULL, + "errorMessage" = NULL, + "updatedAt" = CURRENT_TIMESTAMP + FROM (VALUES ${updates}) AS updates(id, "channelId", carrier, province, "submitId") + WHERE message.id = updates.id AND message.status = 'queued' + `); + if (writeOutbox) { + await tx.gatewaySubmitOutbox.createMany({ + data: prepared.map(({ message, routed, submitId, command }) => ({ + id: randomUUID(), submitId, messageRecordId: message.id, + channelId: routed.channel.id, payload: command as Prisma.InputJsonValue, + })), + }); + } + })); + if (!this.submitOutboxPublishEnabled()) { + await Promise.all(prepared.map(({ command }) => this.facade.publishGatewaySubmitCommand(command))); + } + await this.refreshTaskProgressBatch(prepared.map(({ message }) => message)); + for (const { message, routed, submitId } of prepared) { + results.set(message.id, { + submitted: true, messageRecordId: message.id, channelId: routed.channel.id, attempt: 0, submitId, + }); + this.metrics?.recordSendWorkerResult('completed'); + } + return results; + } + + private async planRoutesBatch(messages: T[]) { + const unresolvedPhones = messages.filter((message) => !message.carrier).map((message) => message.phoneNumber); + const provinces = await this.measureSendStage('phone_routing', () => this.phoneRouting.identifyProvinces(unresolvedPhones)); + const routeInputs = await Promise.all(messages.map(async (message) => ({ + message, + carrier: message.carrier ? normalizeCarrier(message.carrier) : normalizeCarrier(await this.phoneRouting.identifyCarrier(message.phoneNumber)), + province: message.carrier ? message.province ?? null : provinces.get(message.phoneNumber) ?? null, + signatureId: message.signatureId ?? message.template?.signature?.id ?? message.signature?.id ?? null, + }))); + const valid = routeInputs.filter((input) => input.message.applicationId && input.signatureId); + const signatures = [...new Set(valid.map((input) => input.signatureId as string))]; + const routes = valid.length === 0 ? [] : await this.measureSendStage('route_lookup', () => this.prisma.channelRouteRule.findMany({ + where: { + status: 'active', channelId: null, province: null, + OR: valid.map((input) => ({ + tenantId: input.message.tenantId, + applicationId: input.message.applicationId, + carrier: input.carrier, + })), + }, + include: { + group: { + include: { + items: { + include: { + channel: { + include: { + connectionStates: { where: { status: 'connected', currentConnections: { gt: 0 } } }, + reportTasks: { where: { signatureId: { in: signatures }, reportType: 'signature', status: 'approved' } }, + }, + }, + }, + orderBy: { priority: 'asc' }, + }, + }, + }, + }, + orderBy: { priority: 'asc' }, + })); + const routeByKey = new Map(); + for (const route of routes) { + const key = `${route.tenantId}:${route.applicationId}:${normalizeCarrier(route.carrier)}`; + if (!routeByKey.has(key)) routeByKey.set(key, route); + } + const planned: Array<{ message: T; routed: RoutedChannel }> = []; + const failed: Array<{ message: T; reason: string }> = []; + for (const input of routeInputs) { + if (!input.message.applicationId) { + failed.push({ message: input.message, reason: '短信应用未配置，无法选择通道组' }); + continue; + } + if (!input.signatureId) { + failed.push({ message: input.message, reason: '短信签名未配置，无法选择已报备通道' }); + continue; + } + const route = routeByKey.get(`${input.message.tenantId}:${input.message.applicationId}:${input.carrier}`); + if (!route) { + failed.push({ message: input.message, reason: '企业应用未配置对应运营商通道组' }); + continue; + } + if (route.group.status !== 'active' || normalizeCarrier(route.group.carrier) !== input.carrier) { + failed.push({ message: input.message, reason: '企业应用绑定的通道组已停用或运营商不一致' }); + continue; + } + const approvedItems = route.group.items.filter((item) => item.channel.status === 'active' + && item.channel.connectionStates.length > 0 + && item.channel.reportTasks.some((task) => task.signatureId === input.signatureId + && (task.carrier === input.carrier + || (process.env.SIGNATURE_REPORT_STRICT_CARRIER !== 'true' && task.approvalScope === 'legacy_channel')))); + const selected = selectChannelCandidate(approvedItems, { + carrier: input.carrier, + province: input.province, + excludedChannelIds: new Set(), + approvedChannelIds: new Set(approvedItems.map((item) => item.channelId)), + routingKey: input.message.id, + }); + if (!selected) { + failed.push({ message: input.message, reason: '无已报备通过且在线的可用通道' }); + continue; + } + planned.push({ + message: input.message, + routed: { + channel: { ...selected.channel, unitPrice: moneyToNumber(selected.channel.unitPrice) }, + carrier: input.carrier, province: input.province, + groupId: route.groupId, groupName: route.group.name, + routeScope: isNationalChannel(selected) ? 'national' : 'province', + }, + }); + } + return { planned, failed }; + } + + private async failRouteBatch(failed: Array<{ message: T; reason: string }>, results: Map) { + const values = Prisma.join(failed.map(({ message, reason }) => Prisma.sql`(${message.id}::text, ${reason.slice(0, 1000)}::text)`)); + await this.prisma.$executeRaw(Prisma.sql` + UPDATE "SmsMessageRecord" AS message + SET status = 'failed', "errorMessage" = failures.reason, "updatedAt" = CURRENT_TIMESTAMP + FROM (VALUES ${values}) AS failures(id, reason) + WHERE message.id = failures.id AND message.status = 'queued' + `); + await Promise.all(failed.map(async ({ message, reason }) => { + await this.releaseMessageReservation(message, reason); + if (message.batchTask?.sourceType === 'cmpp') await this.recordCmppFailureReceipt(message, 'ROUTE', reason); + else await this.facade.refreshTaskProgress(message.batchTaskId); + results.set(message.id, { submitted: false, messageRecordId: message.id, status: 'failed', reason }); + this.metrics?.recordSendWorkerResult('failed'); + })); + } + + private async refreshTaskProgressBatch(messages: Array<{ + batchTaskId: string; batchTask?: { sourceType?: string | null; phoneTotal?: number | null } | null; + }>) { + const singleCmppIds = [...new Set(messages + .filter((message) => message.batchTask?.sourceType === 'cmpp' && message.batchTask.phoneTotal === 1) + .map((message) => message.batchTaskId))]; + if (singleCmppIds.length > 0) { + await this.prisma.smsBatchTask.updateMany({ + where: { id: { in: singleCmppIds }, sourceType: 'cmpp', phoneTotal: 1 }, + data: singleMessageTaskProgress('submit_queued'), + }); + } + const otherTaskIds = [...new Set(messages + .filter((message) => !singleCmppIds.includes(message.batchTaskId)) + .map((message) => message.batchTaskId))]; + await Promise.all(otherTaskIds.map((taskId) => this.facade.refreshTaskProgress(taskId))); + } + + async processSendJob(job: SendJob) { + const totalStartedAt = this.metrics?.beginSendWorkerStage(); + let totalFinished = false; + const finish = (result: 'completed' | 'failed' | 'skipped') => { + if (totalFinished) return; + totalFinished = true; + if (totalStartedAt != null) { + this.metrics?.finishSendWorkerStage(totalStartedAt, 'total', result === 'completed' ? 'success' : result === 'skipped' ? 'skipped' : 'error'); + } + this.metrics?.recordSendWorkerResult(result); + }; + try { + const message = await this.measureSendStage('message_load', () => this.prisma.smsMessageRecord.findUnique({ + where: { id: job.messageRecordId }, + include: { batchTask: true, template: { include: { signature: true } }, signature: true }, + })); + if (!message || message.status !== 'queued') { + finish('skipped'); + return { skipped: true }; + } + if (!message.tenantId || !message.batchTaskId) { + finish('skipped'); + return { skipped: true, reason: 'standalone channel test message' }; + } + const businessMessage = message as typeof message & { tenantId: string; batchTaskId: string }; + try { + const routed = await this.facade.selectChannelForMessage(businessMessage); + const result = await this.facade.submitMessageToGateway(businessMessage, routed, 0); + finish(result.submitted ? 'completed' : 'skipped'); + return result; + } catch (error) { + const reason = error instanceof Error ? error.message : '无可用通道组或通道'; + await this.prisma.smsMessageRecord.update({ + where: { id: message.id }, + data: { status: 'failed', errorMessage: reason }, + }); + await this.releaseMessageReservation(businessMessage, reason); + if (message.batchTask?.sourceType === 'cmpp') { + await this.recordCmppFailureReceipt(businessMessage, 'ROUTE', reason); + } else { + await this.facade.refreshTaskProgress( + businessMessage.batchTaskId, + message.batchTask?.sourceType === 'cmpp' && message.batchTask.phoneTotal === 1 ? 'failed' : undefined, + ); + } + finish('failed'); + return { submitted: false, messageRecordId: message.id, status: 'failed', reason }; + } + } catch (error) { + finish('failed'); + throw error; + } + } + + async submitMessageToGateway( message: { id: string; tenantId: string; @@ -169,6 +535,7 @@ async submitMessageToGateway( applicationExtension?: string | null; template?: { signature?: { id?: string | null; name?: string | null } | null } | null; signature?: { id?: string | null; name?: string | null } | null; + batchTask?: { sourceType?: string | null; phoneTotal?: number | null } | null; }, routed: RoutedChannel, attempt: number, @@ -176,16 +543,13 @@ async submitMessageToGateway( ) { const channel = routed.channel; const upstreamSrcId = composeUpstreamSrcId(channel.srcId, message.applicationExtension); - await this.facade.ensureSignatureReportedForChannel(message, channel.id, routed.carrier); - await this.facade.waitForChannelRateLimit(channel.id, channel.rateLimitPerSecond); + await this.measureSendStage('rate_limit', () => this.facade.waitForChannelRateLimit(channel.id, channel.rateLimitPerSecond)); const submitId = `SUB-${randomUUID()}`; + const sessionId = await this.getOpenSubmitSessionId(channel.id); + const command = this.buildGatewaySubmitCommand(message, routed, attempt, submitId, upstreamSrcId); + const writeOutbox = this.submitOutboxEnabled(); try { - await this.prisma.$transaction(async (tx) => { - const session = await tx.cmppSubmitSession.upsert({ - where: { sessionNo: `OPEN-${channel.id}` }, - update: { submitTotal: { increment: 1 } }, - create: { channelId: channel.id, sessionNo: `OPEN-${channel.id}`, submitTotal: 1 }, - }); + await this.measureSendStage('submit_transaction', () => this.prisma.$transaction(async (tx) => { await tx.smsSubmitRecord.create({ data: { tenantId: message.tenantId, @@ -194,7 +558,7 @@ async submitMessageToGateway( channelId: channel.id, channelGroupId: routed.groupId, channelGroupName: routed.groupName, - sessionId: session.id, + sessionId, retryOfSubmitRecordId, submitId, submitStatus: 'queued', @@ -216,7 +580,17 @@ async submitMessageToGateway( errorMessage: attempt > 0 ? `第 ${attempt + 1} 次提交，路由至${routed.routeScope === 'national' ? '全国' : '省网'}通道` : undefined, }, }); - }); + if (writeOutbox) { + await tx.gatewaySubmitOutbox.create({ + data: { + submitId, + messageRecordId: message.id, + channelId: channel.id, + payload: command as Prisma.InputJsonValue, + }, + }); + } + })); if (retryOfSubmitRecordId) { this.logger.log(`sms_retry_claim_acquired ${JSON.stringify({ messageId: message.messageId, @@ -255,7 +629,31 @@ async submitMessageToGateway( } throw error; } - const command = { + if (!this.submitOutboxPublishEnabled()) { + await this.measureSendStage('gateway_stream_publish', () => this.facade.publishGatewaySubmitCommand(command)); + } + await this.measureSendStage('task_progress', () => this.facade.refreshTaskProgress( + message.batchTaskId, + message.batchTask?.sourceType === 'cmpp' && message.batchTask.phoneTotal === 1 ? 'submit_queued' : undefined, + )); + return { submitted: true, messageRecordId: message.id, channelId: channel.id, attempt }; + } + + private buildGatewaySubmitCommand( + message: { + id: string; tenantId: string; batchTaskId: string; applicationId?: string | null; + templateId?: string | null; messageId: string; phoneNumber: string; content: string; + billingUnits: number; queuePriority?: string | null; applicationExtension?: string | null; + template?: { signature?: { name?: string | null } | null } | null; + signature?: { name?: string | null } | null; + }, + routed: RoutedChannel, + attempt: number, + submitId: string, + upstreamSrcId: string, + ) { + const channel = routed.channel; + return { schemaVersion: 'v1', messageType: 'SubmitCommand', traceId: randomUUID(), @@ -304,10 +702,124 @@ async submitMessageToGateway( }, retry: { attempt, maxAttempts: 1 }, }; - await this.facade.getGatewayQueue().add('submit-command', command); - await this.facade.publishGatewaySubmitCommand(command); - await this.facade.refreshTaskProgress(message.batchTaskId); - return { submitted: true, messageRecordId: message.id, channelId: channel.id, attempt }; + } + + private submitOutboxEnabled() { + return process.env.SEND_SUBMIT_OUTBOX_SHADOW_ENABLED === 'true' + || process.env.SEND_SUBMIT_OUTBOX_PUBLISH_ENABLED === 'true'; + } + + private submitOutboxPublishEnabled() { + return process.env.SEND_SUBMIT_OUTBOX_PUBLISH_ENABLED === 'true'; + } + + private async publishSubmitOutboxBatch() { + if (this.submitOutboxRunning || !this.submitOutboxEnabled()) return; + this.submitOutboxRunning = true; + try { + const batchSize = Math.min(500, getPositiveConfigInteger(process.env, 'SEND_SUBMIT_OUTBOX_BATCH_SIZE', 64)); + const leaseSeconds = Math.min(300, getPositiveConfigInteger(process.env, 'SEND_SUBMIT_OUTBOX_LEASE_SECONDS', 30)); + const rows = await this.prisma.$queryRaw>(Prisma.sql` + WITH candidates AS ( + SELECT id + FROM "GatewaySubmitOutbox" + WHERE ( + (status = 'pending' AND "nextAttemptAt" <= CURRENT_TIMESTAMP) + OR (status = 'publishing' AND "leaseExpiresAt" < CURRENT_TIMESTAMP) + ) + ORDER BY "createdAt", id + LIMIT ${batchSize} + FOR UPDATE SKIP LOCKED + ) + UPDATE "GatewaySubmitOutbox" AS outbox + SET status = 'publishing', + "leaseOwner" = ${this.submitOutboxLeaseOwner}, + "leaseExpiresAt" = CURRENT_TIMESTAMP + (${leaseSeconds} * INTERVAL '1 second'), + "attemptCount" = outbox."attemptCount" + 1, + "updatedAt" = CURRENT_TIMESTAMP + FROM candidates + WHERE outbox.id = candidates.id + RETURNING outbox.id, outbox."submitId", outbox.payload + `); + if (rows.length === 0) return; + const results = this.submitOutboxPublishEnabled() + ? await this.publishGatewaySubmitCommandBatch(rows) + : rows.map((row) => ({ row, streamEntryId: `shadow:${row.submitId}` })); + const succeeded = results.filter((result): result is { row: typeof rows[number]; streamEntryId: string } => 'streamEntryId' in result); + if (succeeded.length > 0) { + const values = Prisma.join(succeeded.map(({ row, streamEntryId }) => Prisma.sql`(${row.id}, ${streamEntryId})`)); + await this.prisma.$executeRaw(Prisma.sql` + UPDATE "GatewaySubmitOutbox" AS outbox + SET status = 'published', + "streamEntryId" = published."streamEntryId", + "publishedAt" = CURRENT_TIMESTAMP, + "leaseOwner" = NULL, + "leaseExpiresAt" = NULL, + "lastError" = NULL, + "updatedAt" = CURRENT_TIMESTAMP + FROM (VALUES ${values}) AS published(id, "streamEntryId") + WHERE outbox.id = published.id + AND outbox.status = 'publishing' + AND outbox."leaseOwner" = ${this.submitOutboxLeaseOwner} + `); + } + for (const result of results) { + if ('streamEntryId' in result) continue; + const { row, error } = result; + try { + const message = error instanceof Error ? error.message : String(error); + await this.prisma.$executeRaw(Prisma.sql` + UPDATE "GatewaySubmitOutbox" + SET status = CASE WHEN "attemptCount" >= 10 THEN 'dead' ELSE 'pending' END, + "nextAttemptAt" = CURRENT_TIMESTAMP + (LEAST(60, POWER(2, LEAST("attemptCount", 6))) * INTERVAL '1 second'), + "leaseOwner" = NULL, + "leaseExpiresAt" = NULL, + "lastError" = ${message.slice(0, 1000)}, + "updatedAt" = CURRENT_TIMESTAMP + WHERE id = ${row.id} AND "leaseOwner" = ${this.submitOutboxLeaseOwner} + `); + } catch (recordError) { + this.logger.error(`gateway_submit_outbox_failure_record_failed ${recordError instanceof Error ? recordError.message : String(recordError)}`); + } + } + } catch (error) { + this.logger.error(`gateway_submit_outbox_publish_failed ${error instanceof Error ? error.message : String(error)}`); + } finally { + this.submitOutboxRunning = false; + } + } + + private async publishGatewaySubmitCommandBatch( + rows: Array<{ id: string; submitId: string; payload: Prisma.JsonValue }>, + ): Promise> { + const redis = this.facade.getRedis(); + const stream = process.env.GATEWAY_SUBMIT_STREAM ?? GATEWAY_SUBMIT_STREAM; + const script = `local existing = redis.call('GET', KEYS[2]) +if existing then return existing end +local streamId = redis.call('XADD', KEYS[1], '*', 'messageType', 'SubmitCommand', 'data', ARGV[1]) +redis.call('SET', KEYS[2], streamId, 'EX', ARGV[2]) +return streamId`; + const pipeline = redis.pipeline(); + for (const row of rows) { + pipeline.eval( + script, + 2, + stream, + `gateway:submit:outbox:${row.submitId}`, + JSON.stringify(row.payload), + String(GATEWAY_SUBMIT_REQUEUE_IDEMPOTENCY_TTL_SECONDS), + ); + } + const replies = await pipeline.exec(); + if (!replies || replies.length !== rows.length) { + return rows.map((row) => ({ row, error: new Error('Redis Outbox pipeline result count mismatch') })); + } + return replies.map(([error, value], index) => error + ? { row: rows[index], error } + : { row: rows[index], streamEntryId: typeof value === 'string' ? value : String(value ?? '') }); } async selectChannelForMessage( @@ -318,33 +830,31 @@ async selectChannelForMessage( throw new BadRequestException('短信应用未配置，无法选择通道组'); } const hasPersistedRouting = Boolean(message.carrier); - const [carrier, province] = hasPersistedRouting - ? [normalizeCarrier(message.carrier), message.province ?? null] - : await Promise.all([ - this.facade.identifyCarrier(message.phoneNumber), - this.facade.identifyProvince(message.phoneNumber), - ]); - if (!hasPersistedRouting) { - await this.prisma.smsMessageRecord.update({ - where: { id: message.id }, - data: { carrier, province }, - }); - } - const route = await this.facade.findApplicationRoute(message.tenantId, message.applicationId, carrier); - const excluded = new Set(options.excludeChannelIds ?? []); - const signatureId = await this.facade.resolveMessageSignatureId(message); - if (!signatureId) throw new BadRequestException('短信签名未配置，无法选择已报备通道'); - const approvedTasks = await this.prisma.channelSignatureReportTask.findMany({ - where: { - signatureId, - reportType: 'signature', - status: 'approved', - channelId: { in: route.group.items.map((item) => item.channelId) }, - OR: signatureReportApprovalScopes(carrier), - }, - select: { channelId: true }, + const [carrier, province] = await this.measureSendStage('phone_routing', async () => { + const resolved = hasPersistedRouting + ? [normalizeCarrier(message.carrier), message.province ?? null] as const + : await Promise.all([ + this.facade.identifyCarrier(message.phoneNumber), + this.facade.identifyProvince(message.phoneNumber), + ]); + if (!hasPersistedRouting) { + await this.prisma.smsMessageRecord.update({ + where: { id: message.id }, + data: { carrier: resolved[0], province: resolved[1] }, + }); + } + return resolved; }); - const approvedChannelIds = new Set(approvedTasks.map((task) => task.channelId)); + const signatureId = await this.measureSendStage('signature_candidates', () => this.facade.resolveMessageSignatureId(message)); + if (!signatureId) throw new BadRequestException('短信签名未配置，无法选择已报备通道'); + const route = await this.measureSendStage('route_lookup', () => this.facade.findApplicationRoute( + message.tenantId, + message.applicationId ?? undefined, + carrier, + signatureId, + )); + const excluded = new Set(options.excludeChannelIds ?? []); + const approvedChannelIds = new Set(route.group.items.map((item) => item.channelId)); const selected = selectChannelCandidate(route.group.items, { carrier, province, @@ -366,7 +876,55 @@ async selectChannelForMessage( }; } -async findApplicationRoute(tenantId: string, applicationId: string | undefined, carrier: string) { +private async measureSendStage(stage: SendWorkerStage, operation: () => Promise): Promise { + const startedAt = this.metrics?.beginSendWorkerStage(); + try { + const result = await operation(); + if (startedAt != null) this.metrics?.finishSendWorkerStage(startedAt, stage, 'success'); + return result; + } catch (error) { + if (startedAt != null) this.metrics?.finishSendWorkerStage(startedAt, stage, 'error'); + throw error; + } +} + +private async refreshSendQueueMetrics() { + if (!this.metrics) return; + try { + const counts = await this.facade.getSendQueue().getJobCounts('wait', 'active', 'completed', 'failed', 'delayed', 'prioritized'); + const mappings: Array<[SendWorkerQueueState, number]> = [ + ['waiting', counts.wait ?? 0], + ['active', counts.active ?? 0], + ['completed', counts.completed ?? 0], + ['failed', counts.failed ?? 0], + ['delayed', counts.delayed ?? 0], + ['prioritized', counts.prioritized ?? 0], + ]; + for (const [state, count] of mappings) this.metrics.setSendWorkerQueueJobs(state, count); + const pool = this.prisma.getPoolState(); + for (const state of ['max', 'total', 'idle', 'waiting'] as const) { + this.metrics.setSendWorkerDatabasePool(state, pool[state]); + } + } catch (error) { + this.logger.warn(`send_queue_metrics_refresh_failed ${error instanceof Error ? error.message : String(error)}`); + } +} + +async findApplicationRoute(tenantId: string, applicationId: string | undefined, carrier: string, signatureId?: string) { + const approvedChannelWhere = signatureId + ? { + status: 'active', + connectionStates: { some: { status: 'connected', currentConnections: { gt: 0 } } }, + reportTasks: { + some: { + signatureId, + reportType: 'signature', + status: 'approved', + OR: signatureReportApprovalScopes(carrier), + }, + }, + } + : undefined; const route = await this.prisma.channelRouteRule.findFirst({ where: { status: 'active', @@ -376,7 +934,25 @@ async findApplicationRoute(tenantId: string, applicationId: string | undefined, channelId: null, province: null, }, - include: { group: { include: { items: { include: { channel: { include: { connectionStates: true } } }, orderBy: { priority: 'asc' } } } } }, + include: { + group: { + include: { + items: { + where: approvedChannelWhere ? { channel: approvedChannelWhere } : undefined, + include: { + channel: { + include: { + connectionStates: approvedChannelWhere + ? { where: { status: 'connected', currentConnections: { gt: 0 } } } + : true, + }, + }, + }, + orderBy: { priority: 'asc' }, + }, + }, + }, + }, orderBy: { priority: 'asc' }, }); if (!route) { @@ -450,7 +1026,40 @@ async waitForChannelRateLimit(channelId: string, tps: number) { } } -async refreshTaskProgress(batchTaskId: string) { +async refreshTaskProgress(batchTaskId: string, knownSingleMessageStatus?: string) { + if (knownSingleMessageStatus) { + const direct = await this.prisma.smsBatchTask.updateMany({ + where: { id: batchTaskId, sourceType: 'cmpp', phoneTotal: 1 }, + data: singleMessageTaskProgress(knownSingleMessageStatus), + }); + if (direct.count === 1) return; + } else { + // Gateway结果、回执和超时回调已经先提交了消息状态。对CMPP单号码内部任务， + // 在同一条UPDATE中读取该唯一消息的当前状态并写入精确计数，避免每次回调都 + // 对一个只有一行的任务执行GROUP BY；多号码任务继续使用下方聚合路径。 + const direct = await this.prisma.$executeRaw(Prisma.sql` + UPDATE "SmsBatchTask" AS task + SET + "progressTotal" = 1, + "submittedTotal" = CASE WHEN message.status IN ('submit_queued', 'submitted', 'delivered', 'failed', 'unknown', 'timeout') THEN 1 ELSE 0 END, + "successTotal" = CASE WHEN message.status = 'delivered' THEN 1 ELSE 0 END, + "failedTotal" = CASE WHEN message.status IN ('submit_failed', 'failed') THEN 1 ELSE 0 END, + "unknownTotal" = CASE WHEN message.status = 'unknown' THEN 1 ELSE 0 END, + "timeoutTotal" = CASE WHEN message.status = 'timeout' THEN 1 ELSE 0 END, + status = CASE + WHEN message.status IN ('delivered', 'submit_failed', 'failed', 'timeout') THEN 'finished' + WHEN message.status IN ('submit_queued', 'submitted', 'unknown') THEN 'sending' + ELSE 'queued' + END, + "updatedAt" = CURRENT_TIMESTAMP + FROM "SmsMessageRecord" AS message + WHERE task.id = ${batchTaskId} + AND task."sourceType" = 'cmpp' + AND task."phoneTotal" = 1 + AND message."batchTaskId" = task.id + `); + if (direct === 1) return; + } const running = this.taskProgressRefreshes.get(batchTaskId); if (running) { // A state transition committed after the running aggregate may not be visible @@ -519,7 +1128,7 @@ getRedis() { return this.redis; } -async publishGatewaySubmitCommand(command: unknown, idempotencyKey?: string) { + async publishGatewaySubmitCommand(command: unknown, idempotencyKey?: string) { const redis = this.facade.getRedis(); const stream = process.env.GATEWAY_SUBMIT_STREAM ?? GATEWAY_SUBMIT_STREAM; const payload = JSON.stringify(command); @@ -540,6 +1149,44 @@ return streamId`, ); return typeof result === 'string' ? result : String(result ?? ''); } + + private getOpenSubmitSessionId(channelId: string) { + const cached = this.openSubmitSessionIds.get(channelId); + if (cached) return cached; + const sessionNo = `OPEN-${channelId}`; + const pending = this.prisma.cmppSubmitSession.findUnique({ + where: { sessionNo }, + select: { id: true }, + }).then((existing) => existing ?? this.prisma.cmppSubmitSession.upsert({ + where: { sessionNo }, + update: {}, + create: { channelId, sessionNo, submitTotal: 0 }, + select: { id: true }, + })).then((session) => session.id).catch((error) => { + this.openSubmitSessionIds.delete(channelId); + throw error; + }); + this.openSubmitSessionIds.set(channelId, pending); + return pending; + } +} + +function singleMessageTaskProgress(status: string) { + const submittedTotal = ['submit_queued', 'submitted', 'delivered', 'failed', 'unknown', 'timeout'].includes(status) ? 1 : 0; + const successTotal = status === 'delivered' ? 1 : 0; + const failedTotal = ['submit_failed', 'failed'].includes(status) ? 1 : 0; + const unknownTotal = status === 'unknown' ? 1 : 0; + const timeoutTotal = status === 'timeout' ? 1 : 0; + const doneTotal = successTotal + failedTotal + timeoutTotal; + return { + progressTotal: 1, + submittedTotal, + successTotal, + failedTotal, + unknownTotal, + timeoutTotal, + status: doneTotal >= 1 ? 'finished' : submittedTotal > 0 ? 'sending' : 'queued', + }; } function signatureReportApprovalScopes(carrier: string) { diff --git a/api/src/send-chain/send-inbound-entry.service.ts b/api/src/send-chain/send-inbound-entry.service.ts index 3931ac9..9f405dc 100644 --- a/api/src/send-chain/send-inbound-entry.service.ts +++ b/api/src/send-chain/send-inbound-entry.service.ts @@ -1542,53 +1542,6 @@ startInboundWorkflowWorker() { return { candidate, workflowDigest, taskId, messageRecordId, content, drainageDetection, billing }; })); await this.measureInboundStage('message_persist', () => this.prisma.$transaction(async (tx) => { - // Lock accounts in a stable order and reserve the whole tenant subtotal once. - // Per-message idempotency rows remain separate so retries, releases and charges - // keep the original accounting contract without one account update per Inbox row. - const tenantIds = [...new Set(prepared.map(({ candidate }) => candidate.application.tenantId))].sort(); - for (const tenantId of tenantIds) { - await tx.$executeRaw`SELECT pg_advisory_xact_lock(hashtextextended(${'tenant-account:' + tenantId}, 0))`; - await tx.tenantAccount.upsert({ - where: { tenantId }, update: {}, - create: { tenantId, balanceCents: 0, creditCents: 0, status: 'active' }, - }); - const account = await tx.tenantAccount.findUniqueOrThrow({ where: { tenantId } }); - const paid = prepared.filter(({ candidate, billing }) => candidate.application.tenantId === tenantId && billing.amountCents > 0); - const totalAmount = paid.reduce((sum, entry) => sum + entry.billing.amountCents, 0); - if (totalAmount === 0) continue; - const available = moneyToNumber(account.balanceCents) + moneyToNumber(account.creditCents); - if (account.status !== 'active' || available < totalAmount) { - throw new BadRequestException('企业账户余额不足'); - } - const existing = await tx.accountTransaction.findMany({ - where: { idempotencyKey: { in: paid.map(({ candidate }) => `${candidate.item.requestKey}:freeze`) } }, - select: { idempotencyKey: true }, - }); - if (existing.length > 0) { - throw new ConflictException('批量计费幂等流水已存在，转入逐条恢复'); - } - const balanceBefore = moneyToNumber(account.balanceCents); - let reserved = 0; - await tx.accountTransaction.createMany({ - data: paid.map(({ candidate, taskId, billing }) => { - reserved += billing.amountCents; - return { - tenantId, - transactionType: 'frozen', - idempotencyKey: `${candidate.item.requestKey}:freeze`, - amountCents: -billing.amountCents, - balanceAfter: balanceBefore - reserved, - relatedType: 'sms_batch_task', - relatedId: taskId, - remark: 'CMPP 入站短信批量冻结', - }; - }), - }); - await tx.tenantAccount.update({ - where: { tenantId }, - data: { balanceCents: { decrement: totalAmount } }, - }); - } await tx.smsBatchTask.createMany({ data: prepared.map(({ candidate, workflowDigest, taskId, content }) => ({ id: taskId, @@ -1637,6 +1590,56 @@ startInboundWorkflowWorker() { status: 'queued', })), }); + + // Acquire the tenant account lock only after the independent workflow rows + // have been staged. PostgreSQL holds transaction-scoped locks until commit; + // keeping this section last preserves the all-or-nothing boundary while + // avoiding serialization across task/API/message persistence for one tenant. + // Per-message idempotency rows remain separate for retry/release/charge repair. + const tenantIds = [...new Set(prepared.map(({ candidate }) => candidate.application.tenantId))].sort(); + for (const tenantId of tenantIds) { + const paid = prepared.filter(({ candidate, billing }) => candidate.application.tenantId === tenantId && billing.amountCents > 0); + const totalAmount = paid.reduce((sum, entry) => sum + entry.billing.amountCents, 0); + if (totalAmount === 0) continue; + await tx.$executeRaw`SELECT pg_advisory_xact_lock(hashtextextended(${'tenant-account:' + tenantId}, 0))`; + await tx.tenantAccount.upsert({ + where: { tenantId }, update: {}, + create: { tenantId, balanceCents: 0, creditCents: 0, status: 'active' }, + }); + const account = await tx.tenantAccount.findUniqueOrThrow({ where: { tenantId } }); + const available = moneyToNumber(account.balanceCents) + moneyToNumber(account.creditCents); + if (account.status !== 'active' || available < totalAmount) { + throw new BadRequestException('企业账户余额不足'); + } + const existing = await tx.accountTransaction.findMany({ + where: { idempotencyKey: { in: paid.map(({ candidate }) => `${candidate.item.requestKey}:freeze`) } }, + select: { idempotencyKey: true }, + }); + if (existing.length > 0) { + throw new ConflictException('批量计费幂等流水已存在，转入逐条恢复'); + } + const balanceBefore = moneyToNumber(account.balanceCents); + let reserved = 0; + await tx.accountTransaction.createMany({ + data: paid.map(({ candidate, taskId, billing }) => { + reserved += billing.amountCents; + return { + tenantId, + transactionType: 'frozen', + idempotencyKey: `${candidate.item.requestKey}:freeze`, + amountCents: -billing.amountCents, + balanceAfter: balanceBefore - reserved, + relatedType: 'sms_batch_task', + relatedId: taskId, + remark: 'CMPP 入站短信批量冻结', + }; + }), + }); + await tx.tenantAccount.update({ + where: { tenantId }, + data: { balanceCents: { decrement: totalAmount } }, + }); + } })); await this.measureInboundStage('queue_publish', () => this.facade.getSendQueue().addBulk(prepared.map(({ candidate, messageRecordId }) => ({ name: 'send-message' as const, diff --git a/api/src/send-chain/send-submission.service.ts b/api/src/send-chain/send-submission.service.ts index 96a296d..d08ad6a 100644 --- a/api/src/send-chain/send-submission.service.ts +++ b/api/src/send-chain/send-submission.service.ts @@ -63,7 +63,7 @@ export class SendSubmissionService { this.inboundEntry = new SendInboundEntryService(prisma, billing, riskReview, phoneFrequency, phoneRouting, facade, callbacks, metrics); this.reviewContinuation = new SendReviewContinuationService(prisma, billing, riskReview, phoneFrequency, phoneRouting, facade, callbacks); this.scheduledDispatch = new SendScheduledDispatchService(prisma, billing, riskReview, phoneFrequency, phoneRouting, facade, callbacks); - this.gatewaySubmit = new SendGatewaySubmitService(prisma, billing, riskReview, phoneFrequency, phoneRouting, facade, callbacks); + this.gatewaySubmit = new SendGatewaySubmitService(prisma, billing, riskReview, phoneFrequency, phoneRouting, facade, callbacks, metrics); } onModuleDestroy() { @@ -229,6 +229,10 @@ startWorker() { return this.gatewaySubmit.startWorker(); } +startSubmitOutboxPublisher() { + return this.gatewaySubmit.startSubmitOutboxPublisher(); + } + startInboundWorkflowWorker() { return this.inboundEntry.startInboundWorkflowWorker(); } @@ -259,6 +263,7 @@ async submitMessageToGateway( applicationExtension?: string | null; template?: { signature?: { id?: string | null; name?: string | null } | null } | null; signature?: { id?: string | null; name?: string | null } | null; + batchTask?: { sourceType?: string | null; phoneTotal?: number | null } | null; }, routed: RoutedChannel, attempt: number, @@ -274,8 +279,8 @@ async selectChannelForMessage( return this.gatewaySubmit.selectChannelForMessage(message, options); } -async findApplicationRoute(tenantId: string, applicationId: string | undefined, carrier: string) { - return this.gatewaySubmit.findApplicationRoute(tenantId, applicationId, carrier); +async findApplicationRoute(tenantId: string, applicationId: string | undefined, carrier: string, signatureId?: string) { + return this.gatewaySubmit.findApplicationRoute(tenantId, applicationId, carrier, signatureId); } async identifyCarrier(phoneNumber: string) { @@ -307,8 +312,8 @@ async waitForChannelRateLimit(channelId: string, tps: number) { return this.gatewaySubmit.waitForChannelRateLimit(channelId, tps); } -async refreshTaskProgress(batchTaskId: string) { - return this.gatewaySubmit.refreshTaskProgress(batchTaskId); +async refreshTaskProgress(batchTaskId: string, knownSingleMessageStatus?: string) { + return this.gatewaySubmit.refreshTaskProgress(batchTaskId, knownSingleMessageStatus); } getSendQueue(): Queue { diff --git a/api/src/submit-outbox-worker.ts b/api/src/submit-outbox-worker.ts new file mode 100644 index 0000000..bd9e0c8 --- /dev/null +++ b/api/src/submit-outbox-worker.ts @@ -0,0 +1,30 @@ +import 'reflect-metadata'; +import { NestFactory } from '@nestjs/core'; +import { createServer } from 'node:http'; +import { MetricsService } from './metrics/metrics.service'; +import { SendWorkerModule } from './send-worker.module'; + +async function bootstrap() { + if (process.env.CMPP_PROCESS_ROLE !== 'outbox') { + throw new Error('submit-outbox-worker requires CMPP_PROCESS_ROLE=outbox'); + } + const app = await NestFactory.createApplicationContext(SendWorkerModule); + app.enableShutdownHooks(); + const metrics = app.get(MetricsService); + const host = process.env.API_OUTBOX_METRICS_HOST?.trim() || '127.0.0.1'; + const port = Number(process.env.API_OUTBOX_METRICS_PORT ?? 9467); + const server = createServer((request, response) => { + if (request.method !== 'GET' || request.url !== '/metrics') { + response.writeHead(404).end(); + return; + } + response.writeHead(200, { 'Content-Type': 'text/plain; version=0.0.4; charset=utf-8', 'Cache-Control': 'no-store' }); + response.end(metrics.render()); + }); + await new Promise((resolve, reject) => { + server.once('error', reject); + server.listen(port, host, resolve); + }); +} + +void bootstrap(); diff --git a/docs/codebase-modularization-roadmap.md b/docs/codebase-modularization-roadmap.md index 1525e88..558b484 100644 --- a/docs/codebase-modularization-roadmap.md +++ b/docs/codebase-modularization-roadmap.md @@ -1173,3 +1173,4 @@ global，不能错误归入client。`client-signature-*`、发送页、企业认 - Gateway既有Submit池与结果Outbox继续独立有界；先以正价六通道实测定位容量，只有证据证明单条回调仍触发停止线时才扩展批量回调协议。 - Gateway上游模块对单分片使用聚合结果作为唯一Outbox边界，对多分片保留逐片持久化；结果Outbox不承担业务去重猜测，事件数量由上游已知分片数确定。 - 批次任务进度聚合仍归`send-gateway-submit`统一实现；同进程、同批次并发调用使用单飞与尾随刷新收敛重复`GROUP BY`，不改变消息状态写入、跨进程幂等或PostgreSQL最终事实。 +- 2026-08-25新增`GatewaySubmitOutbox`和独立`submit-outbox-worker`进程。三运营商50 TPS实测首提仅33.07/s，后续优先批量路由/批量Submit持久化，其次拆分结果、回执和计费HTTP回调进程；两项完成前不得通过增加BullMQ并发宣称架构完成。 diff --git a/docs/first-version-development-requirements.md b/docs/first-version-development-requirements.md index 7483dc3..be891de 100644 --- a/docs/first-version-development-requirements.md +++ b/docs/first-version-development-requirements.md @@ -2138,3 +2138,13 @@ - 单分片短短信的聚合SubmitResult已经携带完整分片信息，只发布聚合Outbox事件；不得再发布内容相同的分片事件造成双倍API回调和数据库写入。多分片长短信仍须在发送下一片前持久化当前分片事件，保持崩溃恢复边界。 - 同一批次并发提交结果、最终回执和失败处理触发任务进度刷新时，进程内只允许一个PostgreSQL聚合查询执行；并发触发合并为一次尾随刷新，既避免逐消息并发扫描整个批次，也必须覆盖运行中查询快照之后已经提交的状态变化。消息状态、计费和幂等事实仍以PostgreSQL为准，不得缓存聚合结果替代最终刷新。 - 压测使用隔离测试企业、真实PostgreSQL账务、单价`0.0325元/计费条`、三运营商混合号码和六个模拟供应商账号；逐档核对消息、冻结/释放/扣费/退款、SmsBillingRecord、通道分布、队列和数据库。临时单价与主动双活配置须先快照、测试后恢复。 +- 发送Worker必须暴露固定低基数的`message_load/phone_routing/route_lookup/signature_candidates/signature_final_check/rate_limit/submit_transaction/gateway_bullmq_publish/gateway_stream_publish/task_progress/total`阶段直方图，结果只允许`success/error/skipped`；同时暴露BullMQ固定状态、configured/in_flight槽位、completed/failed/skipped结果和Worker PostgreSQL客户端池`max/total/idle/waiting`。指标不得包含手机号、企业、应用、通道、消息、任务或提交实体ID。 +- 测试环境允许启用`pg_stat_statements`采集归一化SQL的calls、total/mean执行时间与rows；不得把原始参数或业务标识导出为Prometheus标签。连接池等待、PostgreSQL锁等待和`pg_stat_statements`累计等待须结合相同压测窗口判断，不能只看压测结束后的空闲快照。 +- CMPP单号码内部任务的提交、结果、回执和超时进度必须使用唯一消息当前状态直接写入精确计数，不得对每个单行任务执行`GROUP BY`；多号码任务继续保留聚合语义。路由候选查询必须同时受当前路由启用、组启用、通道启用、在线连接和签名运营商报备通过事实约束，提交前不得再执行等价报备查询，也不得用长期缓存绕过实时停用或撤销。 +- 发送提交事务只关联已存在的开放会话ID，不得为统计目的逐短信更新`CmppSubmitSession.submitTotal`热点行；提交总量按`SmsSubmitRecord`事实聚合或异步统计。Gateway命令以Go Gateway实际消费的Redis Stream为唯一同步入口，遗留无消费者BullMQ队列不得继续增长；Stream PEL恢复、逐条ACK、结果Outbox和死信必须保留。PostgreSQL提交事实到Stream发布之间的跨介质崩溃窗口在具备自动Outbox扫描器前必须明确记录为残余风险，不能宣称零丢失。 +- Gateway Submit命令必须与提交记录和消息`submit_queued`在同一事务写入`GatewaySubmitOutbox`；独立发布器使用短租约、`SKIP LOCKED`、Redis pipeline和稳定submitId幂等键。正式Outbox启用时旧直投必须关闭。 +- 容量按非补发的首次供应商Submit完成跨度统计；失败回执后的关联补发单列，不能用入口SubmitResp或混入补发尾时延的错误口径代替。 +- 发送Worker允许使用毫秒级有界聚合窗批量规划首次提交：批内消息、号段和候选路由/连接/签名报备必须集合式读取，Submit、消息状态和Gateway Outbox必须在一个短事务中批量持久化。批处理不得缓存或绕过通道启停、连接在线、签名报备、限速、补发关联和逐消息幂等约束；异常与补发可退回逐消息状态机。 +- Gateway供应商事件必须与主API控制面隔离：Submit结果、分片结果、回执、上行、受限协议日志和死信进入仅绑定回环地址的独立进程及独立有界PostgreSQL池；连接状态控制继续进入主API。客户HTTP Webhook使用异步队列Worker，不能在Gateway事实回写请求中同步调用客户地址。 +- 独立回调进程不得加载运营/计费管理Controller，不得经Nginx或公网暴露；部署必须验证回调健康、池上限、回环监听、事件URL和控制URL。回调服务先于Gateway启动/重启，回调失败应由既有结果Outbox恢复且不得重复处理、重复扣费。 +- 本阶段测试环境以正价325金额单位验证：50 TPS档必须同时满足客户端受理、非补发首次供应商Submit不低于50条/秒、MessageId/submitId/Outbox唯一、账单笔数和金额一致、Inbox/Outbox/Stream排空及数据库无持续锁等待。零计费结果不得作为付费链路容量结论。 diff --git a/docs/phase-4-send-pipeline-redesign.md b/docs/phase-4-send-pipeline-redesign.md new file mode 100644 index 0000000..6a01449 --- /dev/null +++ b/docs/phase-4-send-pipeline-redesign.md @@ -0,0 +1,197 @@ +# CMPP 发送链路重新设计方案 + +更新日期：2026-08-25 +状态：已在测试环境完成实施与正价50 TPS验收 +适用范围：耐久 Inbox 完成业务校验后，到 Gateway 供应商 Submit、结果回调、回执和计费结算的完整链路 + +## 1. 结论 + +现阶段不应继续在 BullMQ 发送 Worker 内做小幅并发、缓存或微批补丁。已完成的耐久 Inbox、入口幂等、路由/报备查询收敛、开放会话热点移除、无消费者 BullMQ 副本移除和计费锁缩短应保留。 + +若以后继续追求完整供应商提交 100～500 条/秒，建议作为独立项目，重新设计为四个隔离阶段： + +1. Inbox 业务处理与计费预留； +2. 批量路由规划与提交事实持久化； +3. PostgreSQL Submit Outbox 批量发布； +4. 独立的提交结果、回执、计费结算与任务进度回调。 + +核心不是把若干 BullMQ Job 临时拼成数组，而是把“应提交什么”先作为 PostgreSQL 事实一次性落库，再由无业务判断的发布器向 Gateway 供给命令。这样才能同时减少数据库往返、消除数据库到 Redis 的崩溃窗口，并隔离供应商提交与回调写入的资源竞争。 + +## 2. 原方案中应保留的部分 + +- 正价测试使用 325 分而不是 0 单价，暴露了真实计费锁竞争； +- `pg_stat_statements` 和固定低基数阶段指标能区分入口、Worker、Gateway 与回调瓶颈； +- 路由、在线连接和签名报备合并查询，取消最终通道重复报备查询； +- CMPP 单号码内部任务不再逐消息执行整批状态 `GROUP BY`； +- 提交事务不再更新 `CmppSubmitSession.submitTotal` 热点行； +- 删除没有消费者的 `gateway.submit.queue` BullMQ 同步副本，保留 Gateway 实际消费的 Redis Stream； +- 账户 advisory lock 移到事务末尾账务段后，平均等待由约 443.963ms 降至约 5.420ms； +- 验收以完整供应商提交、队列排空、消息唯一性和账务恒等式为准，没有用入口 SubmitResp 冒充完整吞吐。 + +这些改动降低了单消息成本和锁等待，但没有改变发送链的逐消息架构，因此未把完整供应商吞吐提升到目标档位。 + +## 3. 原方案存在的问题 + +### 3.1 微批位置选错 + +原 P2 在 BullMQ Worker 已领取单个 Job 后，用 5ms 窗口聚合最多 20 条消息。这个位置太晚：任务领取、调度和并发槽已经发生，批次规模还受 Worker 并发上限约束，无法稳定形成 32/64 条批次。 + +批内慢消息还会形成最慢项栅栏，使本来可以独立完成的消息互相等待。实测微批完整供应商提交约 22.20 条/秒，未优于对照，证明这不是正确的批处理边界。 + +### 3.2 只批量加载，没有批量处理主成本 + +原微批只合并了消息读取，运营商识别、路由、在线/报备判断、`SmsSubmitRecord` 创建、消息更新、Stream 发布、结果/回执/计费和任务进度仍逐消息执行。省下一个 `WHERE id IN (...)` 查询不足以抵消批次协调开销。 + +### 3.3 PostgreSQL 与 Redis 仍不是原子边界 + +当前链路先提交 PostgreSQL，再发布 Redis Stream。进程若在两步之间退出，会留下数据库已进入待提交状态、但 Gateway 未收到命令的窗口。若 Redis 已成功而数据库未记录发布结果，重试又可能重复发布。 + +Redis pipeline 只能减少网络往返，不能解决跨介质一致性。必须增加 PostgreSQL Outbox，并以稳定 `submitId`、命令幂等键和 Gateway 去重实现至少一次发布。 + +### 3.4 提交供给与回调写入争用资源 + +供应商 Submit 结果、分片结果、最终回执、下游投递和计费结算会同时写 PostgreSQL。回调越快，数据库写入越密集，越容易抢占发送 Worker 的连接和 CPU。仅提高 Worker 并发只会把等待转移到数据库池。 + +### 3.5 实时业务事实不能用长期缓存替代 + +通道启停、真实连接、签名报备、应用/企业状态、余额和号码频控都可能变化。正确做法是批次内共享只读快照，并在提交事实落库前用数据库条件最终校验；不能建立跨批次长期缓存决定是否发送。 + +### 3.6 压测口径和夹具容易误导 + +入口接收、Inbox 完成、供应商首次 Submit 和最终回执是不同阶段。入口 100 条/秒成功，不代表完整供应商提交达到 100 条/秒。历史待投递回执、重复号码频控、缺失运营商规则，以及模拟器重启后数据库连接状态尚未恢复，都会污染结论。 + +## 4. 目标架构 + +### 4.1 阶段 A：Inbox 与业务预留 + +保留现有耐久 Inbox。业务 Worker 批量领取时继续执行应用、企业、签名、模板、黑名单、日限额、号码频控和正价余额预留。 + +- 使用 `FOR UPDATE SKIP LOCKED` 在短事务内领取 32 条，验证稳定后最多 64 条； +- 领取事务只更新租约，业务查询和外部操作不放在持锁事务内； +- 日限、频控和冻结继续保留逐消息稳定业务键及数据库唯一约束； +- 同企业账户锁只覆盖最终余额与流水写入。 + +### 4.2 阶段 B：批量路由规划与提交事实 + +按应用、运营商、签名组合批量读取候选，在一个短事务内为每条可发送消息持久化： + +- 独立 `SmsSubmitRecord`； +- 目标通道、`submitId` 和消息 `submit_queued` 状态； +- 一条 `GatewaySubmitOutbox` 命令事实。 + +事务提交前，SQL 再次约束应用/企业 active、通道 active、连接 connected、签名报备 approved。任一消息失效只拒绝该消息，不回滚整批其他消息。供应商网络调用、Redis 写入和任务进度聚合都不进入该事务。 + +### 4.3 阶段 C：PostgreSQL Submit Outbox + +建议新增表： + +```text +GatewaySubmitOutbox +- id / submitId（唯一） +- messageRecordId / channelId +- payload / schemaVersion +- status: pending | publishing | published | dead +- attemptCount / nextAttemptAt +- leaseOwner / leaseExpiresAt +- streamEntryId / publishedAt / lastError +- createdAt / updatedAt +``` + +发布器只做： + +1. 用 `FOR UPDATE SKIP LOCKED` 原子领取一批 pending 行； +2. 使用 Redis pipeline `XADD`，每条命令携带唯一 `submitId`； +3. 批量回写 published 或可重试失败。 + +“Redis 已成功、PostgreSQL 回写前崩溃”会产生重复发布，因此 Gateway 必须按 `submitId` 去重，结果事件继续按 `resultEventId` 幂等。Outbox 扫描器负责恢复数据库已提交但未发布的命令。 + +### 4.4 阶段 D：回调与计费隔离 + +提交结果、回执和下游投递使用独立回调进程/连接池： + +- 单分片只处理聚合结果，多分片保留逐片事实与最终聚合； +- accepted 结算复用冻结事实，保持 released/charged 幂等且净余额正确； +- rejected/timeout 以唯一 `retryOfSubmitRecordId` 认领组内补发； +- 最终失败退款使用稳定幂等键； +- 任务进度从消息事实异步刷新，CMPP 单消息任务继续直接写计数； +- 回调积压不得阻塞 Outbox 向 Gateway 供给命令。 + +### 4.5 连接池预算 + +| 角色 | 建议初始数据库槽 | 说明 | +| --- | ---: | --- | +| API/入口 | 16～24 | 保证客户 Submit 和查询接口可用 | +| Inbox/路由规划 | 16～24 | 批量查询和短事务持久化 | +| 结果/回执/计费 | 12～16 | 吸收供应商回调峰值 | +| Outbox 发布器 | 4～8 | 只领取、发布和回写 | +| 运维/迁移保留 | 不少于 8 | 健康检查、诊断和恢复 | + +实际值必须结合 PostgreSQL CPU、`max_connections`、PgBouncer 模式和 `pg_stat_statements` 复测，不能把表中数字直接当生产配置。 + +## 5. 实施步骤 + +### 第 0 步：固定基线和测试夹具 + +固定专用号段、三运营商比例、隔离应用和六供应商账号；每次故障注入后同时确认模拟器 6/6 和数据库六通道 connected；建立 PostgreSQL、运行源码和环境/systemd 恢复资产。 + +### 第 1 步：Outbox 迁移与影子写入 + +只新增表、索引和指标；原 Stream 发布仍生效，同时影子写 Outbox 但不发布；对比每个 `submitId` 的原命令与影子 payload。回退只需关闭影子写，不删除历史表。 + +### 第 2 步：Outbox 发布器影子验证 + +发布到不连接真实 Gateway 的影子 Stream，验证领取、租约恢复、重复发布、死信和批量大小；做发布前/后崩溃注入，证明可恢复且幂等。 + +### 第 3 步：单应用灰度切换 + +一个隔离应用切换到正式 Outbox；保留旧直接发布回退开关，但同一消息任何时刻只能有一个发布路径；先做正价 smoke 和全部发送拦截/补发回归。 + +### 第 4 步：批量路由与持久化 + +初始批量 32，按应用、运营商、签名共享查询；批量插入 Submit/Outbox，逐消息保留唯一键和失败结果；使用 `EXPLAIN (ANALYZE, BUFFERS)` 与 `pg_stat_statements` 验证真实 SQL。 + +### 第 5 步：回调与连接池隔离 + +拆分 Submit/Receipt 回调连接池；集合式处理可安全合并的流水、账单和任务进度，同时保留逐消息幂等键和确定账务顺序。 + +### 第 6 步：容量验收 + +按 `smoke → 20 → 30 → 50 → 100 → 200 → 300 → 500` 逐档执行。每档完全排空后再升档；只有完整供应商提交达到目标、无丢重、账务一致且数据库稳定才继续。 + +## 6. 功能开关与回退 + +建议提供： + +- `SEND_SUBMIT_OUTBOX_SHADOW_ENABLED`； +- `SEND_SUBMIT_OUTBOX_PUBLISH_ENABLED`； +- `SEND_ROUTE_BATCH_ENABLED`； +- `SEND_ROUTE_BATCH_SIZE`； +- `SEND_CALLBACK_POOL_ENABLED`。 + +回退时先停止新领取，等待在途事务完成，关闭正式 Outbox 发布并恢复旧直接发布；按 `submitId` 对账 Outbox 与 Stream PEL，不能删除 Outbox 行或清空 Redis 来回退。 + +## 7. 验收标准 + +- 客户 SubmitResp 零丢失，MessageId 与业务号码唯一； +- Inbox、Submit Outbox、Gateway/结果 Stream 和 BullMQ 最终排空； +- 每条消息最多一个有效首次 Submit，补发必须关联原提交； +- 供应商完整提交速率达到该档目标，不只看入口速率； +- 签名、模板、余额、应用/企业状态、号码频次、报备和通道停用继续实时拦截； +- 冻结、释放、扣费、退款和 `SmsBillingRecord` 净额一致； +- 无持续锁等待、无 idle in transaction、连接数不超过预算； +- 服务重启和 Outbox 重放不重复提交、不重复计费。 + +## 8. 风险与工作量判断 + +这是跨 Prisma migration、发送状态机、Gateway 命令协议、回调处理、部署配置和压测工具的架构改造，不适合作为当前阶段的继续小修。主要风险是双发布、Outbox 重放重复提交、补发认领冲突、实时通道状态过期和账务顺序错误。 + +建议以后单独立项，先完成 Outbox 影子对账和崩溃恢复，再进入批量路由。没有完成影子验证前，不应直接替换现有正式发布路径。 + +## 9. 2026-08-25 完整实施结果 + +- `GatewaySubmitOutbox`、独立发布器和正式发布路径已完成；发送Worker进一步实现最大32条、3ms聚合窗的有界批次。批内一次加载消息、号段、路由/连接及签名报备事实，按通道执行限速，再以一个短事务`createMany`写Submit、集合式更新消息并`createMany`写Outbox。重试及异常分支保留逐消息状态机和幂等键。 +- Submit结果、分片结果、回执、上行、受限协议日志和死信改由仅绑定`127.0.0.1:3001`的`cmpp-gateway-callback`进程接收，使用独立12槽PostgreSQL连接池；指标绑定`127.0.0.1:9468`。客户HTTP Webhook仍由主API的独立BullMQ Worker处理，避免慢客户回调占用Gateway事实回写池。连接状态控制仍写主API，供应商事件才写回调进程。 +- 初次发布smoke暴露并修复了回调URL职责过宽问题：连接状态误发到回调端点导致数据库显示`connecting`、路由失败。修复为控制面/事件面双URL后，六连接均为`connected`；该失败样本未发生供应商提交或计费，不纳入性能结果。 +- 修复后正价smoke为9/9受理，账单`9×325=2925`；20 TPS为199/199、账单64675；30 TPS为299/299、账单97175；50 TPS为499/499、账单162175。四个成功窗口共1006条，账单1006笔、单价均325、合计326950。 +- 50 TPS档499条非补发首次供应商Submit覆盖9.930秒，即`50.25条/秒`；相对同日改造前Outbox阶段的`33.07条/秒`提高约52%。该档客户端P50/P95/P99为35/76/135ms；首提499条唯一，全部尝试及Outbox各542条唯一，补发均关联原提交。 +- 压测结束Inbox、Submit Outbox和两条Redis Stream全部排空，数据库无重复Submit ID、无等待锁、无idle in transaction；回调池`max=12,total=1,idle=1,waiting=0`。隔离应用单价已恢复0，三条临时运营商规则已删除，真实账务事实保留审计。100/200/300/500未继续执行，当前验收结论限定为50 TPS档通过。 diff --git a/docs/phase-4-send-worker-optimization-plan.md b/docs/phase-4-send-worker-optimization-plan.md index 3edce01..df17642 100644 --- a/docs/phase-4-send-worker-optimization-plan.md +++ b/docs/phase-4-send-worker-optimization-plan.md @@ -1,7 +1,9 @@ -# CMPP 第四阶段后续性能优化方案（暂停实施） +# CMPP 第四阶段后续性能优化方案（执行中） -更新日期：2026-08-21 -当前决定：本方案仅作为后续工作依据，本轮不再修改性能代码、不再部署性能补丁、不再继续阶梯压测。 +更新日期：2026-08-24 +当前状态：已完成第1步“现状复核与恢复准备”、第2步“P0可观测证据”、P1低风险数据库往返收敛和P2计费锁持有时间治理；Worker消息加载微批实测退化已回退，其余P2及P3不再继续。 + +第1步恢复资产：`/opt/cmpp-platform-backups/phase4-step1-20260824T020501Z`。资产包含PostgreSQL自定义格式备份、测试环境运行目录、systemd/Nginx/PostgreSQL/Redis配置、服务状态、迁移清单与SHA-256清单；`pg_restore --list`、tar目录和哈希复核通过。 ## 1. 当前结论 @@ -28,6 +30,8 @@ - 分别记录消息加载、号码识别、路由、签名报备、限速、提交事务、Gateway 发布和任务进度耗时。 - 同时记录 BullMQ waiting/active/completed、数据库连接池等待、事务耗时和六通道命令供给速率。 +执行结果（2026-08-24）：测试环境已启用`pg_stat_statements`；发送Worker已增加11个固定阶段直方图、BullMQ状态、Worker槽位、处理结果和PostgreSQL客户端池状态，标签不含业务实体。正价六通道100条/秒诊断档2999/2999受理，P50/P95/P99=`31/68/233ms`；2999个唯一号码首次到达供应商覆盖`92.994秒`，约`32.25条/秒`，仍未达到完整100条/秒。阶段均值中`submit_transaction≈47.4ms`最高，其次`route_lookup≈33.5ms`、`message_load≈22.7ms`、`phone_routing≈18.5ms`、`task_progress≈18.2ms`；Worker池waiting峰值3。`pg_stat_statements`显示企业账户advisory lock累计`390.932秒/587次`，开放会话累计UPSERT为`32.293秒/3616次`，因此P1仍应优先消除账户批次锁竞争和会话热点写入，再收敛逐消息查询与进度更新。 + ### P1：低风险数据库往返收敛 1. CMPP 单消息内部任务使用已知状态直接更新计数，不再执行整批 `GROUP BY`。 @@ -35,6 +39,10 @@ 3. 提交事务只复用已存在的开放会话 ID；`submitTotal`改为异步批量累计或从提交记录聚合，不让统计字段锁住真实发送。 4. 确认 Gateway BullMQ 与 Redis Stream 的消费、重放和死信职责；若存在重复同步发布，改为单一持久入口加可恢复 Outbox。 +执行结果（2026-08-24）：CMPP单号码内部任务在首次提交时按已知状态直接更新，结果、回执和超时回调在一条`UPDATE ... FROM`中读取唯一消息当前状态并写精确计数；最终压力窗口中应用产生的任务状态`GROUP BY`为0。路由规则、在线连接和签名运营商报备条件已合并到一次Prisma候选查询，取消最终通道二次报备查询且不缓存实时启停/报备事实。Worker按通道首次读取并复用`OPEN-{channelId}`会话ID，提交事务不再更新`submitTotal`热点行。Go Gateway仅消费`gateway.submit.commands` Redis Stream，其PEL自动认领、逐条ACK、结果Outbox和死信上报均有代码与测试；`gateway.submit.queue`没有消费者，因此发送Worker停止写入该BullMQ副本，压力前后遗留wait均为84119且不再增长。Redis Stream发布前的数据库/队列跨介质崩溃窗口仍没有独立PostgreSQL Outbox自动扫描器，本轮不把该残余风险伪称为已消除，留待P2微批Outbox一起治理。 + +最终正价六通道100档2999/2999受理，零拒绝、零节流、零连接错误，入口P50/P95/P99=`33/77/179ms`；2999个唯一号码首次供应商提交覆盖`99.940秒`，约`30.00条/秒`，低于P0对照`32.25条/秒`，因此完整100条/秒仍失败并停止升档。Worker平均总耗时由约175.1ms降至约123.1ms，`submit_transaction`由47.4ms降至26.8ms，路由加两次签名查询的原约47.3ms降为32.4ms，BullMQ发布阶段被消除；但同一企业正价回调的`pg_advisory_xact_lock`仍累计373.373秒/841次、均值443.963ms，继续压住共享数据库并发。这证明P1降低了逐消息往返，却没有解除剩余计费账户锁瓶颈；不应仅凭Worker阶段变快宣称吞吐提升。 + ### P2：发送微批处理 - Worker 在 5～10ms 窗口内领取最多 32 或 64 条任务。 @@ -43,6 +51,12 @@ - 一次短事务批量创建独立 `SmsSubmitRecord`、更新独立 `SmsMessageRecord`，保留每条消息唯一 `submitId`、幂等键、补发和计费关联。 - Gateway 命令使用 Redis pipeline/批量发布；任一部分失败必须能根据 PostgreSQL 事实安全补发，不能重复提交或重复计费。 +执行结果（2026-08-24）：正价CMPP入站冻结仍保持任务、API请求、消息、冻结流水和账户余额同一PostgreSQL事务，但`tenant-account` advisory lock改为在独立业务行写入后、事务提交前才获取。最终正价六通道50档该锁598次累计3.241秒、均值5.420ms；对比P1的373.373秒/841次、均值443.963ms，累计等待下降约99.1%，均值下降约98.8%。 + +同轮试做5ms、最多20条的Worker消息批量加载，六通道50档完整供应商提交仅约22.20条/秒，低于P1约30.00条/秒；去除批次最慢项栅栏后仍无收益，因此已回退。回退后最终正价六通道50档1499/1499受理，入口P50/P95/P99=`31/77/227ms`；唯一号码首次到达供应商覆盖69.054秒，约21.69条/秒，未达50，按停止线未执100及更高档。最终代码只保留已证明有效的计费锁缩短；PostgreSQL Submit Outbox与批量路由/持久化未继续实施。 + +最终正价样本1499条，MessageId、号码均1499个且唯一，`unitPrice` min/max均325。冻结/释放各1499笔487175；提交扣费1496笔486200，最终失败退费21笔6825；`SmsBillingRecord` charged1475笔479375、refunded21笔6825，净扣与账户流水一致。全部压力档均使用325正价，没有执0计费压测。结束后仅按测试前快照恢复10个应用单价为0并删除3条临时号段规则，恢复后未再压测。 + ### P3：容量参数复核 - 完成 P1/P2 后再让 Worker 并发与数据库连接预算匹配，初始建议按 24～32 个有效数据库槽验证，不直接扩大到更高并发。 @@ -62,4 +76,3 @@ 每轮变更均执行：自动化回归 → 正价 smoke → 50 → 100 → 200 → 300 → 500 条/秒。每档必须核对入口受理、Inbox、BullMQ、Gateway Stream、供应商提交、回执、上行、消息终态、冻结/释放/扣费/退款、数据库锁和连接池。出现丢失、重复、账务不一致、队列持续增长或数据库持续不稳定时立即停止升档。 500 条/秒只有在入口和完整供应商提交均持续达到目标、队列可在限定时间稳定排空且零丢重、账务恒等式成立时才判定通过。 - diff --git a/docs/system-functional-test-cases.md b/docs/system-functional-test-cases.md index df67d18..a8f2ca7 100644 --- a/docs/system-functional-test-cases.md +++ b/docs/system-functional-test-cases.md @@ -4669,6 +4669,11 @@ npm run verify:phase8 | TC-CMPP-PERF-OBS-003 | Gateway供应商下发分段耗时 | 在隔离测试环境构造Stream等待、限速等待、连接窗口等待、供应商慢响应和API慢回调 | `stream_wait/rate_limit_wait/connection_wait/supplier_rtt/api_callback`可独立区分；`supplier_rtt`在API回调变慢时不等量增长 | | TC-CMPP-PERF-OBS-004 | 观测标签边界 | 检查API/Gateway新增指标文本和Prometheus时序标签 | 只出现固定`stage/result/le`；不得出现手机号、企业/应用/通道/连接/消息/Submit/任务ID、短信正文或凭据，未知阶段不生成时序 | | TC-CMPP-PERF-OBS-005 | 纯观测语义回归 | 对比启用埋点前后的同一组CMPP Submit结果、数据库记录、扣费冻结、队列命令及回执 | SubmitResp状态、Msg_Id、多号码独立记录、同步拒绝、异步回执、幂等键和业务调用顺序均不改变；埋点不写PostgreSQL/Redis | +| TC-CMPP-PERF-OBS-006 | 发送Worker分段耗时 | 在隔离测试环境完成正价短信发送并抓取Worker回环metrics | 11个固定阶段按实际路径增长，结果仅为`success/error/skipped`；各阶段count与Worker结果可对账，直方图不含业务实体标签 | +| TC-CMPP-PERF-OBS-007 | BullMQ与Worker槽位 | 在空闲、入压、排空三个时点抓取Worker metrics并交叉核对BullMQ | waiting/active/completed/failed/delayed/prioritized、configured/in_flight和completed/failed/skipped均为真实值，排空后waiting/active归零 | +| TC-CMPP-PERF-OBS-008 | Worker数据库连接池等待 | 让发送并发超过Worker数据库池，在压测窗口逐秒抓取metrics | `cmpp_worker_database_pool_connections{state=max|total|idle|waiting}`反映客户端池；waiting峰值可被采到，结束后归零，不通过提高连接上限掩盖等待 | +| TC-CMPP-PERF-OBS-009 | PostgreSQL归一化热SQL | 测试环境启用并重置`pg_stat_statements`后执行正价六通道压力，再按total执行时间排序 | 可得到归一化SQL的calls/total/mean/rows且不含实参；开放会话累计、账户锁、提交记录和状态写入可分别归因 | +| TC-CMPP-PERF-OBS-010 | 正价六通道诊断档 | 快照后设置0.0325元单价、三运营商规则和六通道主动双活，执行smoke及100条/秒30秒并监控至排空 | 入口、三运营商、六账号、供应商首次提交、计费、Stream、数据库和恢复配置均可对账；未达到完整100条/秒时停止升档，不以SubmitResp冒充全链吞吐 | | TC-CMPP-PERF-V2-001 | 持续补位无批次屏障 | 工作池并发设为2，先投递一个阻塞任务和一个快速任务，再投递第三个任务 | 快速任务结束后第三个任务立即开始，不等待第一个慢任务结束；读取批次不形成整批`Wait`屏障 | | TC-CMPP-PERF-V2-002 | 单消息独立ACK | 同一批次投递一快一慢两条消息，慢任务保持在供应商等待 | 快任务完成后Redis PEL立即只剩慢任务；不得等慢任务结束后整批ACK，也不得在供应商结果回传前提前ACK | | TC-CMPP-PERF-V2-003 | 全局并发边界 | 分别配置并发1、64、1024和大于1024的值，持续投递超过槽位数的消息 | 同时处理数不超过有效配置；缺省为64，大于1024按1024执行，空闲槽位持续补充 | @@ -4810,3 +4815,52 @@ npm run verify:phase8 | TC-CMPP-GUARD-012 | 配置恢复审计 | 每项测试后回读企业、应用、余额、签名、报备、通道、连接和临时规则 | 所有临时配置恢复原值，服务健康、队列无异常状态，操作和测试证据可追溯 | 执行记录（2026-08-21，测试环境）：`TC-CMPP-GUARD-001`至`012`全部通过。入口采用耐久异步受理，因此业务拦截用例的SubmitResp仍可为0；最终结论以本次MessageId对应的消息错误码、供应商提交数及客户失败回执为准。通道组补发实测主通道结果码8、备通道accepted、消息最终delivered；全部临时配置已恢复。 + +## TC-CMPP-500-P4-P1 发送Worker低风险数据库往返收敛（2026-08-24） + +| 用例ID | 场景 | 预期 | +| --- | --- | --- | +| TC-CMPP-500-P4-P1-001 | CMPP单号码任务在提交、结果、回执和超时阶段刷新进度 | 使用直接状态更新；压力窗口不产生按batchTaskId的消息状态`GROUP BY`；多号码任务仍走聚合 | +| TC-CMPP-500-P4-P1-002 | 路由、在线通道与签名报备候选 | 一次数据库候选查询只返回active/connected/approved通道；无最终通道二次报备查询；报备或通道撤销能实时拦截 | +| TC-CMPP-500-P4-P1-003 | 开放会话与提交统计 | 每通道首次读取开放会话ID并复用；提交事务不更新`CmppSubmitSession.submitTotal`热点行；提交记录保持独立外键 | +| TC-CMPP-500-P4-P1-004 | Gateway双发布职责 | Go Gateway只消费Redis Stream；发送Worker不再写无消费者BullMQ副本；Stream PEL/ACK/结果Outbox/死信行为不变 | +| TC-CMPP-500-P4-P1-005 | 正价六通道100档 | 入口、消息、提交、回执、计费、唯一性和六账号分布一致；双Stream最终排空、Bull遗留值不增长、数据库无持续锁或idle事务 | + +执行记录：本地API全量42套500项和TypeScript构建通过；正价smoke 9/9通过。最终100档2999/2999受理，P50/P95/P99=`33/77/179ms`，三运营商=`998/996/1005`，六供应商账号均有提交；2999个唯一号码首次供应商提交覆盖99.940秒、约30.00条/秒，未达到完整100条/秒，按停止线未升200/300/500。应用任务进度`GROUP BY`为0、开放会话热点累计更新为0、Bull遗留wait始终84119；两条Stream最终0/0。计费冻结/释放各2999笔974675，正式扣费2990笔971750、退款25笔8125，SmsBillingRecord当前charged2965笔963625/refunded25笔8125，账务恒等。剩余主瓶颈是同企业计费`pg_advisory_xact_lock`累计373.373秒/841次；P1降低单条Worker总耗时但未提高完整供应商吞吐。临时单价和三条号段规则已恢复。 + +## TC-CMPP-500-P4-P2 正价计费并发锁治理（2026-08-24） + +| 用例ID | 场景 | 预期 | +| --- | --- | --- | +| TC-CMPP-500-P4-P2-001 | 同企业多应用正价并发冻结 | 任务、消息、冻结流水和账户余额仍同事务；账户锁仅覆盖事务末尾账务段，等待时间显著低于P1对照 | +| TC-CMPP-500-P4-P2-002 | 单价325的冻结、释放、扣费、退费并发幂等 | MessageId/号码唯一，账户流水与`SmsBillingRecord`净扣一致，无重复扣费或负余额 | +| TC-CMPP-500-P4-P2-003 | Worker微批候选的性能比较 | 只有完整供应商吞吐不退化才保留；退化时回退候选代码并重新部署验证 | +| TC-CMPP-500-P4-P2-004 | 六通道正价50档停止线 | 核对入口、唯一供应商首提交、通道分布、账务和队列；完整吞吐不到50时不升100 | +| TC-CMPP-500-P4-P2-005 | 禁止0计费压测代替正价证据 | smoke及所有压力档`unitPrice` min/max均325；单价0只在测试结束后做环境恢复，恢复后不再压测 | + +执行记录：最经六通道正价50档1499/1499受理，入口P50/P95/P99=`31/77/227ms`，完整供应商首提交约21.69条/秒，因低于50未升100。计费锁598次累计3.241秒/均值5.420ms，较P1累计下降约99.1%、均值下降约98.8%。冻结/释放各1499笔487175，charged1496笔486200，退费21笔6825，账单净扣一致。Worker微批候选因吞吐退化已回退；所有压力运行均为325正价，无0计费压测。 + +## TC-CMPP-RELEASE-CLOSEOUT 发布前主流程与拦截复核（2026-08-24） + +| 用例ID | 场景 | 当前版本结果 | +| --- | --- | --- | +| TC-CMPP-RELEASE-001 | 正价CMPP接收、发送、SubmitResp、回执、计费 | 单价325的消息`MSG-6b537761-9a1c-47eb-88d3-fe5e1d7d33f9`为accepted/DELIVRD/delivered，计费记录325、账户charged -325；通过 | +| TC-CMPP-RELEASE-002 | 上行匹配与客户Deliver/ACK | 上行`cmt75bszh0qjy6vletv06f4pc`按MessageId精确匹配，下游投递最终delivered；通过 | +| TC-CMPP-RELEASE-003 | 签名、模板、余额、号码频次 | 分别为SIGNATURE/TEMPLATE/BALANCE/RISK，供应商提交均0；通过 | +| TC-CMPP-RELEASE-004 | 应用接口、应用状态、企业状态 | 接口关闭、应用inactive/deleted、企业inactive/deleted均bind状态3；通过 | +| TC-CMPP-RELEASE-005 | 报备与通道启停 | 报备缺失和主备全停为ROUTE且提交0；仅主停自动选择备通道并delivered；通过 | +| TC-CMPP-RELEASE-006 | 主通道拒绝后组内补发 | 主通道结果码8/rejected，备通道关联retryOf后accepted，消息delivered；通过 | +| TC-CMPP-RELEASE-007 | 配置、队列和服务恢复 | 应用/企业/签名/报备/通道全部恢复；Inbox、BullMQ、双Stream排空；无锁等待/idle事务；六服务和六连接健康；通过 | + +说明：异步耐久入口的业务拦截仍可先返回成功SubmitResp，必须按精确MessageId核对最终错误码、供应商提交数和客户失败回执。供应商模拟器重启后必须同时等待模拟器连接数和数据库通道连接状态恢复，不能只看TCP连接数。 + +| TC-CMPP-OUTBOX-001 | Submit事实与Outbox原子持久化 | 提交记录、消息状态和Outbox同事务，payload submitId一致且唯一 | +| TC-CMPP-OUTBOX-002 | 影子到正式单路径切换 | 影子只对账；正式启用后旧直投关闭，无双发布 | +| TC-CMPP-OUTBOX-003 | 租约恢复与批量发布 | SKIP LOCKED领取、Redis pipeline、批量回写和有界失败重试 | +| TC-CMPP-OUTBOX-004 | 正价阶梯与计费恒等式 | 账单数等于消息数、金额等于消息数乘325，补发有retryOf | +| TC-CMPP-OUTBOX-005 | 首提容量停止线 | 仅首次Submit计算速率；50 TPS不达标即停止更高档 | +| TC-CMPP-BATCH-001 | 批量路由规划 | 同一聚合批次只执行一次消息/号段/候选路由预载，仍逐消息执行实时状态、报备及限速判断 | +| TC-CMPP-BATCH-002 | Submit与Outbox批量原子持久化 | Submit批量插入、消息集合式更新、Outbox批量插入同属一个短事务；MessageId/submitId唯一 | +| TC-CMPP-CALLBACK-001 | Gateway事件进程隔离 | Submit结果、回执、上行、协议日志和死信只进入回环回调进程及独立12槽池；连接状态仍进入主API | +| TC-CMPP-CALLBACK-002 | 回调安全边界 | 回调进程仅监听127.0.0.1，不加载计费管理Controller；健康与指标端点只在回环可见 | +| TC-CMPP-BATCH-003 | 正价50 TPS完整提交 | 单价325，499条首次Submit在9.930秒完成（50.25/s）；499笔账单162175，无丢重，队列最终排空 | diff --git a/docs/testing-progress.md b/docs/testing-progress.md index 24a39b6..e9f8872 100644 --- a/docs/testing-progress.md +++ b/docs/testing-progress.md @@ -3874,3 +3874,75 @@ git diff --check - 通道禁用行为通过：仅禁用`LGST-M-P`时，消息`MSG-fc4a6384-3dfd-4252-bc5b-855e4e881a1a`自动选择`LGST-M-B`并`delivered`；主备同时禁用时，消息`MSG-5bf5960f-cfc1-4409-a6bc-80e6969ad827`最终为`failed/ROUTE`且供应商提交0条。两通道均已恢复`active`。 - 通道组补发通过：模拟`LGST-M-P/SUP001`对消息`MSG-3c2ad6ea-9b45-435c-8833-7dbf7bf28ebd`返回Submit结果码8，首条提交记录为`rejected`；平台随后在`LGST-M-B`创建带`retryOfSubmitRecordId`的第二条提交，状态`accepted`，消息最终`delivered`。模拟器已恢复普通配置，6条通道连接均为`connected`。 - 最终恢复审计：API及API/Worker/Gateway/PostgreSQL/Redis/MinIO服务健康；10个隔离应用全部`active`、接口全部开启、单价均0、模板模式均`direct_send`；10个签名审核/汇总报备均通过，60条签名通道报备任务全部`approved`；6个LGST通道全部`active/connected`；临时风险规则0条；Inbox仅有`completed`状态；专项窗口Worker严重错误筛查0条。 + +## 2026-08-24 第四阶段后续优化第1步：现状复核与恢复准备 + +- 本地`HEAD`与`origin/main`均为`c6f11014d61a8627ae311098ef81d89dd099ffaa`。既有`api/tsconfig.build.tsbuildinfo`、`api/tsconfig.tsbuildinfo`、`tsconfig.tsbuildinfo`、`outputs/`、`pnpm-lock.yaml`、空文件`=`以及本轮接管时已存在的`tmp_generate_ui_drafts.py`、`tools/local/seed-screenshot-materials.mjs`均未修改、删除或纳入本轮范围。 +- 测试环境运行标记仍为`90345fba22e3ae183e1edb4fdae718fb7cb2d963+workspace.p4progress.d82933c344ec`；API、Worker、Gateway、安全代理、PostgreSQL、Redis、MinIO和Nginx均active，API健康、Redis PONG。数据库约1348MB、92条已完成migration、11/100连接、0等待锁、0 idle in transaction；Inbox只有36968条completed。10个隔离应用均active/接口开启/单价0，测试企业余额/授信为`16142576/0`。 +- 两条Gateway Stream的消费者均为1、`pending=0/lag=0`。BullMQ `sms.send.queue`为wait/active/failed/delayed均0；`gateway.submit.queue`保留76984条wait，而同口径命令Stream已经完整消费，作为后续双发布职责梳理的当前证据，本步未清理或改写任何队列。 +- 当前`pg_stat_statements`未安装且未加入`shared_preload_libraries`，记录为第2步可观测性准备项，本步未修改PostgreSQL扩展或配置。隔离供应商模拟器`100.91.249.119:17900`当前不可达，六个LGST连接状态均为failed且最后心跳停在2026-08-21；平台和数据库服务正常，但后续真实发送/压测前必须先恢复模拟器并确认6/6重连。 +- 已创建root专属恢复目录`/opt/cmpp-platform-backups/phase4-step1-20260824T020501Z`，权限700。包含129MB `database.dump`、183MB `runtime-source.tar.gz`、48KB `system-config.tar.gz`、运行标记、服务状态、PostgreSQL版本、92条迁移清单及内容目录；`pg_restore --list`识别818项，源码归档内运行标记一致，全部SHA-256复核通过。数据库/源码/系统配置SHA-256分别为`5b6aec796314947b44d7d6323665036927f326994e0f598f32559a06802b03c9`、`7be9da3a51a11048bfdc43b4923285bd7e34a07b8a95f7b983bf92735d525d6a`、`1de39e5dbf4a699123c9f2dab9ffae85092c8bd194c0e7f00e630d51257132be`。备份后磁盘仍剩62GB，服务、API、Redis和两条Stream复核正常。 +- 本步没有修改业务代码、数据库业务数据、通道配置、余额、客户连接或预生产环境，没有发送短信、部署或压测。第1步完成；进入第2步前的显式前置条件为恢复隔离供应商模拟器并确认6/6连接。 + +## 2026-08-24 第四阶段后续优化第2步：P0可观测、测试环境发布与正价诊断 + +- 恢复本地隔离供应商模拟器并通过Tailscale Serve将测试机到`100.91.249.119:17900`的访问转发到本机回环仿真器；重启测试Gateway后平台和模拟器均确认6/6连接。测试结束模拟器继续运行且6连接在线；未触达预生产或真实供应商。 +- 测试机已有`postgresql-contrib`和`pg_stat_statements.so`，无需联网安装新包。新增`/etc/postgresql/16/main/conf.d/20-pg-stat-statements.conf`，设置`shared_preload_libraries='pg_stat_statements'`、max10000、track=all、track_utility=off并创建扩展；PostgreSQL重启后扩展、预加载、视图读取均通过，未调整`max_connections`或`work_mem`。 +- 发送Worker增加固定低基数分段直方图、BullMQ六状态、Worker槽位、处理结果和数据库客户端池`max/total/idle/waiting`；Prisma继续使用原API/Worker池上限，仅改为显式持有同一个pg Pool以读取原生计数。指标不含手机号、企业、应用、通道、消息或任务ID。API全量42套498项、SendChain 122项、相关专项124项、TypeScript正式构建和`git diff --check`通过。 +- 发布前恢复资产`/opt/cmpp-platform-backups/phase4-step2-20260824T023615Z`包含128MB PostgreSQL custom dump、181MB运行目录、56KB系统配置、恢复说明、迁移/服务清单及测试配置快照；`pg_restore --list`、两份tar目录和全部SHA-256通过。基础包/工作区补丁SHA-256分别为`d0f3311830932e3ebb4e3aea3a7bb6a9136f5acce73c9e7dd6e327953e35a24a`和`045215c9a9ac3c2a09a8a2402cdb7a1f3d998deb74ff9fdee6e7e266b03af0da`；排除`outputs/`、`pnpm-lock.yaml`、空文件`=`和`*.tsbuildinfo`。 +- 仅发布到`100.93.204.60`测试环境，运行标记为`c6f11014d61a8627ae311098ef81d89dd099ffaa+workspace.phase4step2.045215c9a9ac`；92条migration无待执行项，部署安全门禁、API/Worker/Gateway/PostgreSQL/Redis/MinIO/Nginx及回环metrics均健康。预生产未发布、未回退、未压测。 +- 正价smoke生成9条，9/9受理、P50/P95/P99=`34/54/54ms`，计费charged 9笔共2925金额单位；Worker 11个阶段count均为9，数据库池waiting为0。客户连接同时ACK了2483条历史pending回执，因此该回执数不作为本轮新增回执证据，随后重置`pg_stat_statements`和Worker指标再进入压力档。 +- 首次100档因测试库没有`PhoneCarrierRule`，仿真联通/电信号默认归为移动，只压到移动主备；2999条生成、2997个SubmitResp、2个缺失，257次客户端节流，P99=1773ms，按停止线不升档。该轮仍给出关键对照：两个会话热点下`submit_transaction`累计669.331秒、Worker池waiting峰值38、开放会话UPSERT累计557.234秒。 +- 快照后临时增加只匹配`1380028/1300028/1890028`的三条运营商规则并重启Worker，修正后的六通道100档为2999/2999受理、零拒绝/节流/连接错误，P50/P95/P99=`31/68/233ms`；移动/联通/电信=`998/996/1005`。六供应商账号各收到588至612次提交尝试；2999个唯一号码的首次供应商提交从`02:55:20.074Z`到`02:56:53.068Z`，覆盖`92.994秒`、约`32.25条/秒`，完整100条/秒目标仍失败，未执行200/300/500。 +- 六通道诊断阶段均值：message_load约22.7ms、phone_routing约18.5ms、route_lookup约33.5ms、signature_candidates约7.0ms、signature_final_check约6.8ms、rate_limit约6.8ms、submit_transaction约47.4ms、BullMQ发布约7.1ms、Stream发布约6.9ms、task_progress约18.2ms、total约175.1ms。180秒采样峰值为Worker in_flight24、数据库池total32/waiting3、PostgreSQL未授予锁20；结束均归零。 +- `pg_stat_statements`进一步确认正价入口企业账户`pg_advisory_xact_lock`累计390.932秒/587次、均值665.983ms；开放会话`CmppSubmitSession` UPSERT累计32.293秒/3616次、均值8.931ms。P1应先治理企业账户批次锁竞争和六个开放会话热点行，再处理路由、号码回写、任务进度与重复Gateway入口；本步不执行P1代码优化。 +- 日志复核发现两个非本次指标代码引入但需后续处理的相邻问题：部署后定时报表刷新有1次5秒交互事务过期；压力客户端连接关闭/重建竞争触发`CmppDownstreamConnection`的P2025/P2002。发送Worker无uncaught/fatal，当前窗口6006个Inbox全部`completed/attempts=1`，服务、连接和Stream最终恢复正常；本步只记录证据，不扩展为业务修复。 +- 本轮精确6006条正价消息（smoke 9、单运营商误测2998、六通道2999）冻结/释放各6006笔、金额1951950；正式charged 5991笔/1947075，退款73笔/23725，SmsBillingRecord最终charged5918/refunded73，当前样本净扣1923350，账务链一致。测试窗口还处理了103条历史消息退款，因此账户从16159476到14269601的总变化以窗口全量账务净额核对，不能只用本轮MessageId计算。 +- 最终已恢复10个隔离应用为active/接口开启/单价0，三主通道priority10/非备用、三备priority20/备用、weight100，删除3条临时运营商规则并重启Worker；6通道active/connected，两条Stream pending/lag均0，数据库0 idle-in-transaction、0锁等待、0未授予锁，8项服务active。真实消息、账务、回执和操作日志保留审计；受保护工作区文件未纳入发布或删除。 + +## 2026-08-24 第四阶段后续优化下一步：P1低风险数据库往返收敛 + +- 实施四项低风险收敛：CMPP单号码任务提交按已知状态直写，结果/回执/超时用单条`UPDATE ... FROM`读取唯一消息状态并直写；路由、在线连接与签名运营商报备合并为一次候选查询并删除最终二次报备查询；Worker首次读取后复用开放会话ID，事务不再逐短信更新`submitTotal`；确认Go Gateway只消费Redis Stream且具备PEL自动认领、逐条ACK、结果Outbox和死信，发送Worker停止向无人消费的`gateway.submit.queue`同步写副本。Redis Stream发布前的跨介质崩溃窗口尚无独立PostgreSQL Outbox扫描器，明确留作P2残余风险。 +- 本地API全量42套500项、SendChain 124项、TypeScript正式构建和`git diff --check`通过。两次部署前分别建立`/opt/cmpp-platform-backups/phase4-p1-20260824T033000Z`和最终恢复点`/opt/cmpp-platform-backups/phase4-p1-final-20260824T034900Z`；最终恢复点包含154MB PostgreSQL custom dump、2.4MB精简运行源码和56KB系统配置，`pg_restore --list`、两份tar及SHA-256全部通过。最终发布包SHA-256=`eb07c9fb5dfff3152fed52574895cd4308b69818e78685d6aad5649507ec4521`，仅部署测试机，运行标记=`c6f11014d61a8627ae311098ef81d89dd099ffaa+workspace.phase4p1.eb07c9fb5dff`；92条migration无待执行项。预生产未修改。 +- 最终正价smoke 9/9受理，P50/P95/P99=`29/33/33ms`。最终六通道100档2999/2999受理，零拒绝、零节流、零连接错误，P50/P95/P99=`33/77/179ms`；三运营商=`998/996/1005`，六通道提交尝试分别为主`998/996/1005`、备`182/178/166`。2999个唯一号码首次供应商提交从`03:51:44.049Z`到`03:53:23.989Z`，覆盖99.940秒、约30.00条/秒，低于P0对照32.25条/秒，完整100条/秒仍失败，按停止线未升200/300/500。 +- 最终Worker阶段3008个样本（smoke+100档）平均：message_load约18.5ms、phone_routing约16.4ms、route_lookup约32.4ms、signature_candidates约0.004ms、rate_limit约5.4ms、submit_transaction约26.8ms、Stream发布约5.5ms、task_progress约17.9ms、total约123.1ms；相比P0总耗时175.1ms下降约29.7%，提交事务和重复签名/路由往返明显下降。应用产生的单任务状态`GROUP BY`为0，开放会话`submitTotal`更新为0，Bull遗留wait从测试前到结束始终84119。 +- 剩余决定性瓶颈为正价同企业计费`pg_advisory_xact_lock`，累计373.373秒/841次、均值443.963ms，远高于其余SQL；因此P1只降低Worker单条耗时，未提升完整供应商吞吐。最终样本消息2999/MessageId2999/号码2999均唯一；当前消息delivered2895、failed42、submitted62，提交accepted3142/rejected73/timeout310。冻结/释放各2999笔974675，扣费2990笔971750，退款25笔8125；SmsBillingRecord当前charged2965笔963625/refunded25笔8125，账务一致。 +- 测试后10个隔离应用全部恢复active/接口开启/单价0，3条临时运营商规则删除，6个供应商通道保持connected；命令/结果Stream均pending0/lag0，Bull遗留wait84119未增长，数据库idle-in-transaction0、未授予锁0，API/Worker/Gateway/PostgreSQL/Redis/Nginx均active且发布窗口无error级journal。真实测试消息、账务和回执保留审计；未提交、未push，受保护的`*.tsbuildinfo`、`outputs/`、`pnpm-lock.yaml`、空文件`=`及其他并发产物未删除或纳入发布。 + +## 2026-08-24 第四阶段P2：正价计费并发锁治理与Worker微批取舍 + +- 发布前恢复资产位于`/opt/cmpp-platform-backups/phase4-p2-20260824T101000Z`：PostgreSQL custom dump 161MB、`pg_restore --list` 835行、精简源码1.4MB、系统配置2KB，两个tar可解包且SHA-256完整。最终测试环境标记为`c6f11014d61a8627ae311098ef81d89dd099ffaa+workspace.phase4p2final.879a234fbc86`，仅发布`100.93.204.60`，预生产未操作。 +- 正价入站批量持久化仍保持全部账务与业务行同事务；仅将同企业`tenant-account` advisory lock移到事务末尾账务段。最终正价六通道50档该锁598次累计3.241秒/均值5.420ms，相比P1的373.373秒/841次、均值443.963ms，累计下降约99.1%、均值下降约98.8%。 +- Worker 5ms/最多20条的消息加载微批曾部署实测，去除最慢项栅栏后仍只有约22.20条/秒，低于P1对照，因此已回退且最终发布不包含该负优化。最终代码只保留计费锁缩短，PostgreSQL Submit Outbox、批量路由与批量提交不再执行。 +- 最终六通道正价50条1499/1499受理，零拒绝/节流/连接错误，入口P50/P95/P99=`31/77/227ms`。1499个唯一号码首次到达供应商从`10:37:44.510Z`到`10:38:53.564Z`，覆盖69.054秒、约21.69条/秒，50档失败，按停止线未执100或更高档。 +- 最终样本MessageId/号码均1499个且唯一，客户单价min/max均325；冻结/释放各1499笔487175，charged1496笔486200，最终失败退费21笔6825，`SmsBillingRecord` charged1475笔479375/refunded21笔6825，净扣一致。所有压力运行均为325正价，没有0计费压测。 +- 结束后根据发布前CSV快照恢复10个隔离应用单价为0，删除3条临时号段规则；这是环境还原，还原后未再压测。API/Worker/Gateway/PostgreSQL/Redis/Nginx均active，六供应商连接在线，BullMQ wait/active和两条Stream pending均0。代码未提交、未push，受保护产物未纳入发布。 + +## 2026-08-24 第四阶段发布前收尾回归 + +- 仅在`100.93.204.60`测试环境和本地隔离供应商模拟器执行，运行标记仍为`c6f11014d61a8627ae311098ef81d89dd099ffaa+workspace.phase4p2final.879a234fbc86`；预生产未操作。恢复点继续使用`/opt/cmpp-platform-backups/phase4-p2-20260824T101000Z`。 +- 正价主流程使用`920001`、新号码`13877780000`和单价325：SubmitResp为0、延迟15ms；消息`MSG-6b537761-9a1c-47eb-88d3-fe5e1d7d33f9`最终`delivered`，主通道Submit accepted、真实`DELIVRD`回执闭合，消息/计费记录单价和金额均325，账户生成`charged -325`流水。结束后单价回读为0。首次因远程SQL引号导致单价未生效的0价消息明确作废，不作为计费证据。 +- 上行使用同一真实MessageId从测试机内部Gateway事件入口注入，`SmsUplinkMessage cmt75bszh0qjy6vletv06f4pc`以`messageId 精确匹配`关联应用和原消息；客户连接上线后普通Deliver得到ACK，`CmppDownstreamDelivery`最终`delivered`。Gateway原始CMPP Deliver解析继续由全量Go测试覆盖。 +- 当前版本拦截复测通过：签名未审核`MSG-c52b65a8-2451-465f-ac80-83779d78e341`为`failed/SIGNATURE`；模板未报备`MSG-137ae451-18e2-401f-b908-a4e4db0175ad`为`failed/TEMPLATE`；余额不足`MSG-1287c06b-3ecb-4fde-be1c-401636a2505c`按325计算并为`failed/BALANCE`；号码频次同号第一条delivered、第二条`MSG-6bec92fd-8eab-4450-afbf-555c292a2150`为`failed/RISK`且命中阈值1/实际2。四种失败均供应商提交0，客户失败回执已生成，临时状态和规则均恢复。 +- 应用接口关闭、应用`inactive/deleted`、企业`inactive/deleted`五次CMPP bind均返回状态3且未进入提交；应用、企业和接口均已恢复。 +- 通道报备和禁用复测通过：移动主备报备临时pending时`MSG-44c2b749-62fc-47f7-a200-9ed74d2f1ff7`为`failed/ROUTE`且供应商提交0；仅主通道inactive时`MSG-d256c8fc-e74f-4260-9fa6-595a37b16e17`自动选`LGST-M-B`并delivered；主备都inactive时`MSG-598ea5de-da7f-4c24-ac4d-728e12629489`为`failed/ROUTE`且供应商提交0。 +- 通道组补发在模拟器6/6且数据库六通道均`connected/currentConnections=1`后执行：`MSG-0a91d27a-0d39-4377-a57d-fa98e869d2d9`在`LGST-M-P`结果码8/rejected，随后`LGST-M-B`产生带`retryOfSubmitRecordId`的accepted提交并最终delivered。切换模拟器后未等待数据库连接状态的第一次试验只有主通道拒绝，明确判为夹具无效轮，不作为产品失败。 +- 最终恢复回读：10个隔离应用全部active、接口开启、单价0、模板模式direct_send；隔离企业active；16个签名审核均approved，10个签名汇总报备approved，60条通道报备任务approved；6通道active且connected；临时规则0；Inbox pending0；两条Stream pending/lag均0；BullMQ wait/active均0；数据库锁等待和idle transaction均0；六个核心服务active，普通模拟器6/6连接。 +- 新增`docs/phase-4-send-pipeline-redesign.md`，说明原BullMQ Worker微批边界、跨介质一致性和回调资源争用问题，并提出PostgreSQL Submit Outbox、批量路由持久化及回调池隔离方案。当前不实施该重构。 +- 发布门禁复跑：API全量42套500项通过，TypeScript正式构建通过。pnpm包装器首次在测试启动前因既有`msgpackr-extract`未批准构建脚本退出；未修改依赖或锁文件，随后直接使用已安装的Jest/TypeScript入口完成验证。`git diff --check`与最终Git范围检查另行执行。 + +## 2026-08-25 PostgreSQL Submit Outbox发布与正价压测 + +- 仅部署测试机`100.93.204.60`，标记`c6f11014d61a8627ae311098ef81d89dd099ffaa+workspace.phase4outbox-live.20260825`；恢复资产`/opt/cmpp-platform-backups/phase4-outbox-20260825T094000Z`已验证，预生产未操作。 +- API 42套502项、Gateway全包测试、TypeScript构建和`git diff --check`通过；迁移已应用，API/Worker/Outbox/Gateway/PostgreSQL/Redis/Nginx active。 +- 正价阶梯：20 TPS=20.46/s、计费64675；30 TPS=29.97/s、计费97175；三运营商50 TPS=33.07/s、计费162175。所有档MessageId/submitId唯一，补发均关联retryOf，队列和锁干净。 +- 50 TPS停止线未通过，未执行100/200/300/500。临时号段规则已删除，10个测试应用单价恢复0；最终Outbox published=2172、payload差异0、重复submitId=0、Inbox非completed=0、双Stream pending/lag=0。 + +## 2026-08-25 批量路由持久化、独立回调进程与正价复压 + +- 发送Worker新增最大32条、等待3ms的有界聚合批次：批量读取消息、号段、路由/连接/签名报备，按消息完成实时约束和通道限速；首次提交以一个短事务批量创建`SmsSubmitRecord`、集合式更新消息、批量创建`GatewaySubmitOutbox`。重试、拒绝和异常仍走原逐消息幂等状态机。 +- 新增回环`cmpp-gateway-callback`进程，独立12槽数据库池，专门处理Submit结果、分片结果、回执、上行、受限协议日志和死信；客户HTTP Webhook仍在主API异步Worker。Gateway连接状态控制继续写主API，供应商事件写回调进程。回调管理Controller未加载，端口仅监听`127.0.0.1:3001/9468`。 +- 初次smoke因连接状态控制误发到回调端点而全部`failed/ROUTE`，供应商提交0、计费0；已拆分控制面和事件面URL并复测，六连接数据库状态均为connected。该轮作为发布缺陷审计保留，不纳入容量统计。 +- 修复后正价结果：smoke 9/9、账单2925；20 TPS 199/199、P50/P95/P99=`19/51/94ms`、账单64675；30 TPS 299/299、`20/48/69ms`、账单97175；50 TPS 499/499、`35/76/135ms`、账单162175。四个成功窗口合计1006条、1006笔账单、单价全部325、金额326950。 +- 50档499条唯一首次供应商Submit跨度9.930秒，完整提交速率`50.25条/秒`，相对上一版同档33.07条/秒提高约52%。全部尝试/Outbox均542条且ID唯一；状态检查时delivered485、failed4、submitted10，关联补发43次。Inbox、Outbox、双Stream最终排空，数据库无等待锁、无idle in transaction，回调池`max=12,total=1,idle=1,waiting=0`。 +- 测试环境最终标记`c6f11014d61a8627ae311098ef81d89dd099ffaa+workspace.phase4batchcallback.callback-surface.599393bb6233`；恢复资产为`/opt/cmpp-platform-backups/phase4-batch-callback-20260825T025611Z`。10个隔离应用单价恢复0，三条临时号段规则删除，真实计费事实保留；预生产未操作，未执行100/200/300/500档。 diff --git a/gateway/cmd/gateway/main.go b/gateway/cmd/gateway/main.go index 202754e..cf9b00a 100644 --- a/gateway/cmd/gateway/main.go +++ b/gateway/cmd/gateway/main.go @@ -32,7 +32,8 @@ func main() { cmppAddr = ":17890" } apiBaseURL := os.Getenv("API_BASE_URL") - upstreamManager := &upstream.Manager{APIBaseURL: apiBaseURL} + callbackBaseURL := getenv("GATEWAY_CALLBACK_API_BASE_URL", apiBaseURL) + upstreamManager := &upstream.Manager{APIBaseURL: apiBaseURL, EventAPIBaseURL: callbackBaseURL} var worker *submitworker.Worker var resultOutbox *resultoutbox.Outbox channelLimiter, err := ratelimit.New(os.Getenv("REDIS_URL")) @@ -75,12 +76,12 @@ func main() { worker.Group = getenv("GATEWAY_SUBMIT_GROUP", "cmpp-gateway") worker.Consumer = getenv("GATEWAY_SUBMIT_CONSUMER", "gateway-1") worker.Concurrency = positiveEnvInt("GATEWAY_SUBMIT_WORKER_CONCURRENCY", 64) - worker.APIBaseURL = apiBaseURL + worker.APIBaseURL = callbackBaseURL resultOutbox = resultoutbox.New(worker.Redis) resultOutbox.Stream = getenv("GATEWAY_SUBMIT_RESULT_STREAM", "gateway.submit.results") resultOutbox.Group = getenv("GATEWAY_SUBMIT_RESULT_GROUP", "cmpp-api-callback") resultOutbox.Consumer = getenv("GATEWAY_SUBMIT_RESULT_CONSUMER", "gateway-1") - resultOutbox.APIBaseURL = apiBaseURL + resultOutbox.APIBaseURL = callbackBaseURL resultOutbox.Concurrency = positiveEnvInt("GATEWAY_SUBMIT_RESULT_WORKER_CONCURRENCY", 8) worker.ResultOutbox = resultOutbox upstreamManager.SubmitSegmentPublisher = resultOutbox @@ -161,7 +162,7 @@ func main() { return snapshot })) control.Register(mux, control.Server{ - APIBaseURL: apiBaseURL, + APIBaseURL: callbackBaseURL, Upstream: upstreamManager, Limiter: channelLimiter, Submit: func(ctx context.Context, command queue.SubmitCommand) (queue.SubmitResult, error) { diff --git a/gateway/internal/upstream/manager.go b/gateway/internal/upstream/manager.go index a2ecb91..78de777 100644 --- a/gateway/internal/upstream/manager.go +++ b/gateway/internal/upstream/manager.go @@ -23,6 +23,7 @@ const ( type Manager struct { APIBaseURL string + EventAPIBaseURL string HTTPClient *http.Client SubmitSegmentPublisher SubmitSegmentPublisher @@ -170,11 +171,15 @@ func (m *Manager) ConnectionCounts() (desired int, connected int) { } func (m *Manager) newConnectionPool(channelID string, connectionID string, config queue.UpstreamConfig) *connectionPool { + eventAPIBaseURL := m.EventAPIBaseURL + if eventAPIBaseURL == "" { + eventAPIBaseURL = m.APIBaseURL + } return &connectionPool{ channelID: channelID, connectionID: connectionID, config: config, - apiBaseURL: m.APIBaseURL, + apiBaseURL: eventAPIBaseURL, httpClient: m.HTTPClient, reporter: func(ctx context.Context, state ConnectionState) error { return m.post(ctx, "/admin/gateway/connections", state) diff --git a/gateway/internal/upstream/pool_test.go b/gateway/internal/upstream/pool_test.go index 77dc33c..6d796eb 100644 --- a/gateway/internal/upstream/pool_test.go +++ b/gateway/internal/upstream/pool_test.go @@ -85,6 +85,17 @@ func TestNormalizeUpstreamConfigDefaults(t *testing.T) { } } +func TestManagerSeparatesConnectionStateAndSupplierEventAPIs(t *testing.T) { + manager := &Manager{APIBaseURL: "http://main-api/api", EventAPIBaseURL: "http://callback-api/api"} + pool := manager.newConnectionPool("channel-1", "channel-1:primary", queue.UpstreamConfig{DesiredConnections: 1}) + if pool.apiBaseURL != manager.EventAPIBaseURL { + t.Fatalf("supplier events must use callback API, got %q", pool.apiBaseURL) + } + if manager.APIBaseURL == pool.apiBaseURL { + t.Fatal("connection-state control API must remain separate from supplier events") + } +} + func queueUpstreamConfigForTest() queue.UpstreamConfig { return queue.UpstreamConfig{ GatewayHost: "127.0.0.1", diff --git a/tools/deploy/production-deploy.sh b/tools/deploy/production-deploy.sh index 4ce5455..a7fa762 100644 --- a/tools/deploy/production-deploy.sh +++ b/tools/deploy/production-deploy.sh @@ -46,6 +46,21 @@ if [[ ! "${API_DB_POOL_MAX:-}" =~ ^[1-9][0-9]*$ || ! "${API_WORKER_DB_POOL_MAX:- echo "API_DB_POOL_MAX and API_WORKER_DB_POOL_MAX must be positive integers in $ENV_FILE." >&2 exit 1 fi +if [[ "${SEND_SUBMIT_OUTBOX_SEPARATE_PROCESS_ENABLED:-false}" == "true" && ! "${API_OUTBOX_DB_POOL_MAX:-}" =~ ^[1-9][0-9]*$ ]]; then + echo "API_OUTBOX_DB_POOL_MAX must be a positive integer when the separate Submit Outbox publisher is enabled." >&2 + exit 1 +fi +if [[ "${GATEWAY_CALLBACK_SEPARATE_PROCESS_ENABLED:-false}" == "true" ]]; then + if [[ ! "${API_CALLBACK_DB_POOL_MAX:-}" =~ ^[1-9][0-9]*$ || ! "${API_CALLBACK_PORT:-}" =~ ^[1-9][0-9]*$ ]]; then + echo "API_CALLBACK_DB_POOL_MAX and API_CALLBACK_PORT must be positive integers when the separate Gateway callback is enabled." >&2 + exit 1 + fi + expected_callback_url="http://127.0.0.1:${API_CALLBACK_PORT}/api" + if [[ "${GATEWAY_CALLBACK_API_BASE_URL:-}" != "$expected_callback_url" ]]; then + echo "GATEWAY_CALLBACK_API_BASE_URL must equal $expected_callback_url for the loopback callback process." >&2 + exit 1 + fi +fi if [[ -z "${CMPP_PUBLIC_HOST:-}" || ! "${CMPP_PUBLIC_PORT:-}" =~ ^[1-9][0-9]*$ ]]; then echo "CMPP_PUBLIC_HOST and a positive CMPP_PUBLIC_PORT are required in $ENV_FILE; these are the customer-facing CMPP endpoint." >&2 @@ -82,7 +97,7 @@ PROD_ADMIN_CREDENTIAL_FILE="$ADMIN_CREDENTIAL_FILE" node tools/deploy/ensure-pro chmod 600 "$ADMIN_CREDENTIAL_FILE" || true echo "[deploy] Ensuring runtime log directories" -install -d -m 0755 "$APP_DIR/logs/api" "$APP_DIR/logs/send-worker" "$APP_DIR/logs/gateway" +install -d -m 0755 "$APP_DIR/logs/api" "$APP_DIR/logs/send-worker" "$APP_DIR/logs/submit-outbox" "$APP_DIR/logs/gateway-callback" "$APP_DIR/logs/gateway" echo "[deploy] Installing split API and send-worker services" node_bin="$(command -v node)" @@ -108,6 +123,46 @@ RestartSec=5 StandardOutput=append:$APP_DIR/logs/send-worker/stdout.log StandardError=append:$APP_DIR/logs/send-worker/stderr.log +[Install] +WantedBy=multi-user.target +EOF +cat >/etc/systemd/system/cmpp-submit-outbox.service </etc/systemd/system/cmpp-gateway-callback.service </dev/null || true +fi +if [[ "${GATEWAY_CALLBACK_SEPARATE_PROCESS_ENABLED:-false}" == "true" ]]; then + systemctl enable --now cmpp-gateway-callback + systemctl restart cmpp-gateway-callback +else + systemctl disable --now cmpp-gateway-callback 2>/dev/null || true +fi +# Gateway is restarted after the callback listener so supplier events never point +# at a callback port that has not completed Nest/Prisma initialization. +systemctl restart cmpp-gateway systemctl restart nginx echo "[deploy] Health checks" @@ -165,6 +234,13 @@ wait_for_http() { } wait_for_http "API" "http://127.0.0.1:${API_PORT:-3000}/api/health" wait_for_http "Send worker metrics" "http://127.0.0.1:${API_WORKER_METRICS_PORT:-9465}/metrics" +if [[ "${SEND_SUBMIT_OUTBOX_SEPARATE_PROCESS_ENABLED:-false}" == "true" ]]; then + wait_for_http "Submit Outbox metrics" "http://127.0.0.1:${API_OUTBOX_METRICS_PORT:-9467}/metrics" +fi +if [[ "${GATEWAY_CALLBACK_SEPARATE_PROCESS_ENABLED:-false}" == "true" ]]; then + wait_for_http "Gateway callback" "http://127.0.0.1:${API_CALLBACK_PORT}/api/health" + wait_for_http "Gateway callback metrics" "http://127.0.0.1:${API_CALLBACK_METRICS_PORT:-9468}/metrics" +fi wait_for_http "Gateway" "http://127.0.0.1:8090/health" redis-cli -h "${REDIS_HOST:-127.0.0.1}" -p "${REDIS_PORT:-6379}" ping >/dev/null pg_isready -d "${DATABASE_URL%%\?*}" >/dev/null diff --git a/tools/security/install-security-agent.sh b/tools/security/install-security-agent.sh index e7a7911..c66794f 100644 --- a/tools/security/install-security-agent.sh +++ b/tools/security/install-security-agent.sh @@ -14,6 +14,8 @@ install -d -o root -g cmpp-security -m 0770 /run/cmpp-security-agent install -d -o root -g cmpp-security -m 0750 /var/lib/cmpp-security-agent install -d -o cmpp-api -g cmpp-security -m 0750 "$APP_DIR/logs/api" install -d -o cmpp-api -g cmpp-security -m 0750 "$APP_DIR/logs/send-worker" +install -d -o cmpp-api -g cmpp-security -m 0750 "$APP_DIR/logs/submit-outbox" +install -d -o cmpp-api -g cmpp-security -m 0750 "$APP_DIR/logs/gateway-callback" [[ -d /var/lib/cmpp-platform/object-storage ]] && chown -R cmpp-api:cmpp-security /var/lib/cmpp-platform/object-storage sed "s#@CMPP_SECURITY_AGENT_BIN@#$agent_binary#g" "$APP_DIR/deploy/security/cmpp-report-only.conf" >/etc/fail2ban/action.d/cmpp-report-only.conf @@ -42,6 +44,29 @@ ProtectHome=true ProtectSystem=true ReadWritePaths=$APP_DIR/logs/send-worker /var/lib/cmpp-platform/object-storage EOF +install -d -m 0755 /etc/systemd/system/cmpp-submit-outbox.service.d +cat >/etc/systemd/system/cmpp-submit-outbox.service.d/security-boundary.conf </etc/systemd/system/cmpp-gateway-callback.service.d/security-boundary.conf <>/etc/nftables.conf nft -c -f /etc/nftables.conf nft list table inet cmpp_security >/dev/null 2>&1 || nft -f /etc/nftables.d/cmpp-security.nft