From 8144f0865249d0a13334243c6807902ab968d98f Mon Sep 17 00:00:00 2001 From: hectorzhao Date: Wed, 8 Jul 2026 16:30:06 +0800 Subject: [PATCH] feat: complete cmpp gateway delivery recovery workflows --- .../migration.sql | 56 + .../migration.sql | 3 + .../migration.sql | 38 + .../migration.sql | 33 + .../migration.sql | 5 + .../migration.sql | 6 + .../migration.sql | 64 + .../migration.sql | 45 + api/prisma/schema.prisma | 213 +++- api/src/channels/channels.service.spec.ts | 8 +- api/src/channels/channels.service.ts | 30 +- .../operations/admin-operations.controller.ts | 146 ++- api/src/operations/operations.module.ts | 3 +- api/src/operations/operations.service.spec.ts | 347 +++++- api/src/operations/operations.service.ts | 510 +++++++- .../send-chain/gateway-events.controller.ts | 28 + api/src/send-chain/send-chain.service.spec.ts | 530 +++++++- api/src/send-chain/send-chain.service.ts | 1085 ++++++++++++++++- api/src/sms-config/sms-config.service.spec.ts | 29 +- api/src/sms-config/sms-config.service.ts | 39 +- api/src/spike/bullmq-link-spike.mjs | 7 + docs/contracts/examples/submit-command.json | 9 + docs/contracts/examples/submit-result.json | 12 +- .../gateway-queue-messages.schema.json | 33 +- .../first-version-development-requirements.md | 58 +- docs/system-functional-test-cases.md | 341 +++++- docs/testing-progress.md | 494 ++++++++ gateway/cmd/gateway/main.go | 68 +- gateway/go.mod | 6 + gateway/go.sum | 12 + gateway/internal/control/server.go | 153 ++- gateway/internal/control/server_test.go | 102 +- gateway/internal/inbound/presence.go | 182 +++ gateway/internal/inbound/presence_test.go | 95 ++ gateway/internal/inbound/recovery.go | 323 +++++ gateway/internal/inbound/recovery_test.go | 101 ++ gateway/internal/inbound/server.go | 537 +++++++- gateway/internal/inbound/server_test.go | 259 ++++ gateway/internal/queue/messages.go | 50 +- gateway/internal/queue/messages_test.go | 15 + gateway/internal/spike/simulator.go | 7 + gateway/internal/submitworker/worker.go | 431 +++++++ gateway/internal/submitworker/worker_test.go | 180 +++ .../internal/upstream/connection_loss_test.go | 50 + gateway/internal/upstream/long_message.go | 153 +++ .../internal/upstream/long_message_test.go | 95 ++ gateway/internal/upstream/manager.go | 697 +++++++++++ gateway/internal/upstream/pool_test.go | 60 + src/api/adminApi.ts | 216 +++- src/apps/admin/AdminChannelsPage.tsx | 12 + .../admin/AdminDownstreamDeliveriesPage.tsx | 377 ++++++ .../AdminDownstreamRecoveryStatusesPage.tsx | 345 ++++++ src/apps/admin/AdminHome.tsx | 9 + .../admin/AdminSmsApplicationFormPage.tsx | 12 + src/apps/admin/AdminSmsRecordsPage.tsx | 41 +- src/apps/admin/AdminSmsUplinkRecordsPage.tsx | 106 +- src/layouts/AdminLayout.tsx | 15 +- src/routes/AppRoutes.tsx | 4 + src/styles/global.css | 109 +- tools/start-local.ps1 | 1 + 60 files changed, 8901 insertions(+), 94 deletions(-) create mode 100644 api/prisma/migrations/20260707183000_add_downstream_delivery_and_uplink_match/migration.sql create mode 100644 api/prisma/migrations/20260708102000_add_application_cmpp_limits/migration.sql create mode 100644 api/prisma/migrations/20260708124500_add_gateway_submit_dead_letters/migration.sql create mode 100644 api/prisma/migrations/20260708193000_add_gateway_downstream_recovery_statuses/migration.sql create mode 100644 api/prisma/migrations/20260708203000_add_recovery_failure_category/migration.sql create mode 100644 api/prisma/migrations/20260708213000_add_recovery_lock_observability/migration.sql create mode 100644 api/prisma/migrations/20260708223000_add_sms_message_segment_audits/migration.sql create mode 100644 api/prisma/migrations/20260708233000_add_uplink_match_candidates/migration.sql create mode 100644 gateway/internal/inbound/presence.go create mode 100644 gateway/internal/inbound/presence_test.go create mode 100644 gateway/internal/inbound/recovery.go create mode 100644 gateway/internal/inbound/recovery_test.go create mode 100644 gateway/internal/submitworker/worker.go create mode 100644 gateway/internal/submitworker/worker_test.go create mode 100644 gateway/internal/upstream/connection_loss_test.go create mode 100644 gateway/internal/upstream/long_message.go create mode 100644 gateway/internal/upstream/long_message_test.go create mode 100644 gateway/internal/upstream/manager.go create mode 100644 gateway/internal/upstream/pool_test.go create mode 100644 src/apps/admin/AdminDownstreamDeliveriesPage.tsx create mode 100644 src/apps/admin/AdminDownstreamRecoveryStatusesPage.tsx diff --git a/api/prisma/migrations/20260707183000_add_downstream_delivery_and_uplink_match/migration.sql b/api/prisma/migrations/20260707183000_add_downstream_delivery_and_uplink_match/migration.sql new file mode 100644 index 0000000..f976d65 --- /dev/null +++ b/api/prisma/migrations/20260707183000_add_downstream_delivery_and_uplink_match/migration.sql @@ -0,0 +1,56 @@ +-- Add customer-side downstream delivery persistence and uplink matching fields. + +ALTER TABLE "SmsUplinkMessage" + ADD COLUMN "applicationId" TEXT, + ADD COLUMN "messageRecordId" TEXT, + ADD COLUMN "matchStatus" TEXT NOT NULL DEFAULT 'unmatched', + ADD COLUMN "matchReason" TEXT; + +ALTER TABLE "SmsUplinkMessage" + ADD CONSTRAINT "SmsUplinkMessage_applicationId_fkey" + FOREIGN KEY ("applicationId") REFERENCES "SmsApplication"("id") ON DELETE SET NULL ON UPDATE CASCADE; + +ALTER TABLE "SmsUplinkMessage" + ADD CONSTRAINT "SmsUplinkMessage_messageRecordId_fkey" + FOREIGN KEY ("messageRecordId") REFERENCES "SmsMessageRecord"("id") ON DELETE SET NULL ON UPDATE CASCADE; + +CREATE INDEX "SmsUplinkMessage_applicationId_receivedAt_idx" ON "SmsUplinkMessage"("applicationId", "receivedAt"); +CREATE INDEX "SmsUplinkMessage_messageRecordId_idx" ON "SmsUplinkMessage"("messageRecordId"); +CREATE INDEX "SmsUplinkMessage_matchStatus_idx" ON "SmsUplinkMessage"("matchStatus"); + +CREATE TABLE "CmppDownstreamDelivery" ( + "id" TEXT NOT NULL, + "tenantId" TEXT NOT NULL, + "applicationId" TEXT NOT NULL, + "messageRecordId" TEXT, + "messageId" TEXT, + "deliveryType" TEXT NOT NULL, + "status" TEXT NOT NULL DEFAULT 'pending', + "payload" JSONB NOT NULL, + "retryCount" INTEGER NOT NULL DEFAULT 0, + "nextRetryAt" TIMESTAMP(3), + "deliveredAt" TIMESTAMP(3), + "lastError" TEXT, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL, + + CONSTRAINT "CmppDownstreamDelivery_pkey" PRIMARY KEY ("id") +); + +ALTER TABLE "CmppDownstreamDelivery" + ADD CONSTRAINT "CmppDownstreamDelivery_tenantId_fkey" + FOREIGN KEY ("tenantId") REFERENCES "Tenant"("id") ON DELETE RESTRICT ON UPDATE CASCADE; + +ALTER TABLE "CmppDownstreamDelivery" + ADD CONSTRAINT "CmppDownstreamDelivery_applicationId_fkey" + FOREIGN KEY ("applicationId") REFERENCES "SmsApplication"("id") ON DELETE RESTRICT ON UPDATE CASCADE; + +ALTER TABLE "CmppDownstreamDelivery" + ADD CONSTRAINT "CmppDownstreamDelivery_messageRecordId_fkey" + FOREIGN KEY ("messageRecordId") REFERENCES "SmsMessageRecord"("id") ON DELETE SET NULL ON UPDATE CASCADE; + +CREATE INDEX "CmppDownstreamDelivery_tenantId_status_createdAt_idx" ON "CmppDownstreamDelivery"("tenantId", "status", "createdAt"); +CREATE INDEX "CmppDownstreamDelivery_applicationId_status_createdAt_idx" ON "CmppDownstreamDelivery"("applicationId", "status", "createdAt"); +CREATE INDEX "CmppDownstreamDelivery_messageId_idx" ON "CmppDownstreamDelivery"("messageId"); +CREATE INDEX "CmppDownstreamDelivery_messageRecordId_idx" ON "CmppDownstreamDelivery"("messageRecordId"); +CREATE INDEX "CmppDownstreamDelivery_deliveryType_status_idx" ON "CmppDownstreamDelivery"("deliveryType", "status"); diff --git a/api/prisma/migrations/20260708102000_add_application_cmpp_limits/migration.sql b/api/prisma/migrations/20260708102000_add_application_cmpp_limits/migration.sql new file mode 100644 index 0000000..d1500d1 --- /dev/null +++ b/api/prisma/migrations/20260708102000_add_application_cmpp_limits/migration.sql @@ -0,0 +1,3 @@ +ALTER TABLE "SmsApplication" +ADD COLUMN "cmppMaxConnections" INTEGER NOT NULL DEFAULT 1, +ADD COLUMN "cmppWindowSize" INTEGER NOT NULL DEFAULT 16; diff --git a/api/prisma/migrations/20260708124500_add_gateway_submit_dead_letters/migration.sql b/api/prisma/migrations/20260708124500_add_gateway_submit_dead_letters/migration.sql new file mode 100644 index 0000000..3cfce3c --- /dev/null +++ b/api/prisma/migrations/20260708124500_add_gateway_submit_dead_letters/migration.sql @@ -0,0 +1,38 @@ +CREATE TABLE "GatewaySubmitDeadLetter" ( + "id" TEXT NOT NULL, + "streamMessageId" TEXT NOT NULL, + "tenantId" TEXT, + "applicationId" TEXT, + "channelId" TEXT, + "traceId" TEXT, + "messageId" TEXT, + "submitId" TEXT, + "status" TEXT NOT NULL DEFAULT 'pending', + "failureCode" TEXT NOT NULL, + "failureMessage" TEXT NOT NULL, + "attempts" INTEGER NOT NULL DEFAULT 1, + "maxAttempts" INTEGER NOT NULL DEFAULT 3, + "commandPayload" JSONB, + "rawPayload" TEXT, + "manualRetryCount" INTEGER NOT NULL DEFAULT 0, + "lastRetryStreamId" TEXT, + "lastRetriedAt" TIMESTAMP(3), + "resolvedAt" TIMESTAMP(3), + "resolvedStatus" TEXT, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + + CONSTRAINT "GatewaySubmitDeadLetter_pkey" PRIMARY KEY ("id") +); + +CREATE UNIQUE INDEX "GatewaySubmitDeadLetter_streamMessageId_key" ON "GatewaySubmitDeadLetter"("streamMessageId"); +CREATE INDEX "GatewaySubmitDeadLetter_status_createdAt_idx" ON "GatewaySubmitDeadLetter"("status", "createdAt"); +CREATE INDEX "GatewaySubmitDeadLetter_tenantId_status_createdAt_idx" ON "GatewaySubmitDeadLetter"("tenantId", "status", "createdAt"); +CREATE INDEX "GatewaySubmitDeadLetter_applicationId_status_createdAt_idx" ON "GatewaySubmitDeadLetter"("applicationId", "status", "createdAt"); +CREATE INDEX "GatewaySubmitDeadLetter_channelId_status_createdAt_idx" ON "GatewaySubmitDeadLetter"("channelId", "status", "createdAt"); +CREATE INDEX "GatewaySubmitDeadLetter_messageId_idx" ON "GatewaySubmitDeadLetter"("messageId"); +CREATE INDEX "GatewaySubmitDeadLetter_submitId_idx" ON "GatewaySubmitDeadLetter"("submitId"); + +ALTER TABLE "GatewaySubmitDeadLetter" ADD CONSTRAINT "GatewaySubmitDeadLetter_tenantId_fkey" FOREIGN KEY ("tenantId") REFERENCES "Tenant"("id") ON DELETE SET NULL ON UPDATE CASCADE; +ALTER TABLE "GatewaySubmitDeadLetter" ADD CONSTRAINT "GatewaySubmitDeadLetter_applicationId_fkey" FOREIGN KEY ("applicationId") REFERENCES "SmsApplication"("id") ON DELETE SET NULL ON UPDATE CASCADE; +ALTER TABLE "GatewaySubmitDeadLetter" ADD CONSTRAINT "GatewaySubmitDeadLetter_channelId_fkey" FOREIGN KEY ("channelId") REFERENCES "SmsChannel"("id") ON DELETE SET NULL ON UPDATE CASCADE; diff --git a/api/prisma/migrations/20260708193000_add_gateway_downstream_recovery_statuses/migration.sql b/api/prisma/migrations/20260708193000_add_gateway_downstream_recovery_statuses/migration.sql new file mode 100644 index 0000000..cbb90ff --- /dev/null +++ b/api/prisma/migrations/20260708193000_add_gateway_downstream_recovery_statuses/migration.sql @@ -0,0 +1,33 @@ +CREATE TABLE "GatewayDownstreamRecoveryStatus" ( + "id" TEXT NOT NULL, + "account" TEXT NOT NULL, + "tenantId" TEXT, + "applicationId" TEXT, + "gatewayInstanceId" TEXT, + "state" TEXT NOT NULL, + "lastAttemptAt" TIMESTAMP(3), + "lastSuccessAt" TIMESTAMP(3), + "lastFailureAt" TIMESTAMP(3), + "nextRetryAt" TIMESTAMP(3), + "attemptCount" INTEGER NOT NULL DEFAULT 0, + "lastError" TEXT, + "lastSkipReason" TEXT, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + + CONSTRAINT "GatewayDownstreamRecoveryStatus_pkey" PRIMARY KEY ("id") +); + +CREATE UNIQUE INDEX "GatewayDownstreamRecoveryStatus_account_key" ON "GatewayDownstreamRecoveryStatus"("account"); +CREATE INDEX "GatewayDownstreamRecoveryStatus_tenantId_state_updatedAt_idx" ON "GatewayDownstreamRecoveryStatus"("tenantId", "state", "updatedAt"); +CREATE INDEX "GatewayDownstreamRecoveryStatus_applicationId_state_updatedAt_idx" ON "GatewayDownstreamRecoveryStatus"("applicationId", "state", "updatedAt"); +CREATE INDEX "GatewayDownstreamRecoveryStatus_state_updatedAt_idx" ON "GatewayDownstreamRecoveryStatus"("state", "updatedAt"); +CREATE INDEX "GatewayDownstreamRecoveryStatus_nextRetryAt_idx" ON "GatewayDownstreamRecoveryStatus"("nextRetryAt"); + +ALTER TABLE "GatewayDownstreamRecoveryStatus" +ADD CONSTRAINT "GatewayDownstreamRecoveryStatus_tenantId_fkey" +FOREIGN KEY ("tenantId") REFERENCES "Tenant"("id") ON DELETE SET NULL ON UPDATE CASCADE; + +ALTER TABLE "GatewayDownstreamRecoveryStatus" +ADD CONSTRAINT "GatewayDownstreamRecoveryStatus_applicationId_fkey" +FOREIGN KEY ("applicationId") REFERENCES "SmsApplication"("id") ON DELETE SET NULL ON UPDATE CASCADE; diff --git a/api/prisma/migrations/20260708203000_add_recovery_failure_category/migration.sql b/api/prisma/migrations/20260708203000_add_recovery_failure_category/migration.sql new file mode 100644 index 0000000..7e41adb --- /dev/null +++ b/api/prisma/migrations/20260708203000_add_recovery_failure_category/migration.sql @@ -0,0 +1,5 @@ +ALTER TABLE "GatewayDownstreamRecoveryStatus" +ADD COLUMN "failureCategory" TEXT; + +CREATE INDEX "GatewayDownstreamRecoveryStatus_failureCategory_updatedAt_idx" +ON "GatewayDownstreamRecoveryStatus"("failureCategory", "updatedAt"); diff --git a/api/prisma/migrations/20260708213000_add_recovery_lock_observability/migration.sql b/api/prisma/migrations/20260708213000_add_recovery_lock_observability/migration.sql new file mode 100644 index 0000000..6b60c2b --- /dev/null +++ b/api/prisma/migrations/20260708213000_add_recovery_lock_observability/migration.sql @@ -0,0 +1,6 @@ +ALTER TABLE "GatewayDownstreamRecoveryStatus" +ADD COLUMN "lockOwner" TEXT, +ADD COLUMN "lockExpiresAt" TIMESTAMP(3); + +CREATE INDEX "GatewayDownstreamRecoveryStatus_lockOwner_lockExpiresAt_idx" +ON "GatewayDownstreamRecoveryStatus"("lockOwner", "lockExpiresAt"); diff --git a/api/prisma/migrations/20260708223000_add_sms_message_segment_audits/migration.sql b/api/prisma/migrations/20260708223000_add_sms_message_segment_audits/migration.sql new file mode 100644 index 0000000..bec3230 --- /dev/null +++ b/api/prisma/migrations/20260708223000_add_sms_message_segment_audits/migration.sql @@ -0,0 +1,64 @@ +CREATE TABLE "SmsMessageSegmentAudit" ( + "id" TEXT NOT NULL, + "tenantId" TEXT NOT NULL, + "batchTaskId" TEXT, + "messageRecordId" TEXT NOT NULL, + "submitRecordId" TEXT, + "channelId" TEXT, + "submitId" TEXT NOT NULL, + "attempt" INTEGER NOT NULL DEFAULT 0, + "segmentTotal" INTEGER NOT NULL DEFAULT 1, + "segmentIndex" INTEGER NOT NULL DEFAULT 1, + "sequenceId" INTEGER, + "gatewayMessageId" TEXT, + "submitStatus" TEXT NOT NULL DEFAULT 'queued', + "receiptStatus" TEXT, + "rawStatus" TEXT, + "compensationType" TEXT, + "errorCode" TEXT, + "errorMessage" TEXT, + "submittedAt" TIMESTAMP(3), + "deliveredAt" TIMESTAMP(3), + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL, + + CONSTRAINT "SmsMessageSegmentAudit_pkey" PRIMARY KEY ("id") +); + +CREATE UNIQUE INDEX "SmsMessageSegmentAudit_messageRecordId_submitId_segmentIndex_key" +ON "SmsMessageSegmentAudit"("messageRecordId", "submitId", "segmentIndex"); + +CREATE INDEX "SmsMessageSegmentAudit_tenantId_createdAt_idx" +ON "SmsMessageSegmentAudit"("tenantId", "createdAt"); + +CREATE INDEX "SmsMessageSegmentAudit_messageRecordId_segmentIndex_idx" +ON "SmsMessageSegmentAudit"("messageRecordId", "segmentIndex"); + +CREATE INDEX "SmsMessageSegmentAudit_submitRecordId_idx" +ON "SmsMessageSegmentAudit"("submitRecordId"); + +CREATE INDEX "SmsMessageSegmentAudit_gatewayMessageId_idx" +ON "SmsMessageSegmentAudit"("gatewayMessageId"); + +CREATE INDEX "SmsMessageSegmentAudit_submitStatus_receiptStatus_idx" +ON "SmsMessageSegmentAudit"("submitStatus", "receiptStatus"); + +ALTER TABLE "SmsMessageSegmentAudit" +ADD CONSTRAINT "SmsMessageSegmentAudit_tenantId_fkey" +FOREIGN KEY ("tenantId") REFERENCES "Tenant"("id") ON DELETE RESTRICT ON UPDATE CASCADE; + +ALTER TABLE "SmsMessageSegmentAudit" +ADD CONSTRAINT "SmsMessageSegmentAudit_batchTaskId_fkey" +FOREIGN KEY ("batchTaskId") REFERENCES "SmsBatchTask"("id") ON DELETE SET NULL ON UPDATE CASCADE; + +ALTER TABLE "SmsMessageSegmentAudit" +ADD CONSTRAINT "SmsMessageSegmentAudit_messageRecordId_fkey" +FOREIGN KEY ("messageRecordId") REFERENCES "SmsMessageRecord"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +ALTER TABLE "SmsMessageSegmentAudit" +ADD CONSTRAINT "SmsMessageSegmentAudit_submitRecordId_fkey" +FOREIGN KEY ("submitRecordId") REFERENCES "SmsSubmitRecord"("id") ON DELETE SET NULL ON UPDATE CASCADE; + +ALTER TABLE "SmsMessageSegmentAudit" +ADD CONSTRAINT "SmsMessageSegmentAudit_channelId_fkey" +FOREIGN KEY ("channelId") REFERENCES "SmsChannel"("id") ON DELETE SET NULL ON UPDATE CASCADE; diff --git a/api/prisma/migrations/20260708233000_add_uplink_match_candidates/migration.sql b/api/prisma/migrations/20260708233000_add_uplink_match_candidates/migration.sql new file mode 100644 index 0000000..7699bce --- /dev/null +++ b/api/prisma/migrations/20260708233000_add_uplink_match_candidates/migration.sql @@ -0,0 +1,45 @@ +-- CreateTable +CREATE TABLE "SmsUplinkMatchCandidate" ( + "id" TEXT NOT NULL, + "uplinkMessageId" TEXT NOT NULL, + "tenantId" TEXT NOT NULL, + "applicationId" TEXT NOT NULL, + "messageRecordId" TEXT, + "matchSource" TEXT NOT NULL, + "confidence" INTEGER NOT NULL DEFAULT 50, + "reason" TEXT, + "status" TEXT NOT NULL DEFAULT 'pending', + "claimedAt" TIMESTAMP(3), + "claimedById" TEXT, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL, + + CONSTRAINT "SmsUplinkMatchCandidate_pkey" PRIMARY KEY ("id") +); + +-- CreateIndex +CREATE UNIQUE INDEX "SmsUplinkMatchCandidate_uplinkMessageId_applicationId_messageRecordId_key" ON "SmsUplinkMatchCandidate"("uplinkMessageId", "applicationId", "messageRecordId"); + +-- CreateIndex +CREATE INDEX "SmsUplinkMatchCandidate_uplinkMessageId_status_idx" ON "SmsUplinkMatchCandidate"("uplinkMessageId", "status"); + +-- CreateIndex +CREATE INDEX "SmsUplinkMatchCandidate_tenantId_status_createdAt_idx" ON "SmsUplinkMatchCandidate"("tenantId", "status", "createdAt"); + +-- CreateIndex +CREATE INDEX "SmsUplinkMatchCandidate_applicationId_status_createdAt_idx" ON "SmsUplinkMatchCandidate"("applicationId", "status", "createdAt"); + +-- CreateIndex +CREATE INDEX "SmsUplinkMatchCandidate_messageRecordId_idx" ON "SmsUplinkMatchCandidate"("messageRecordId"); + +-- AddForeignKey +ALTER TABLE "SmsUplinkMatchCandidate" ADD CONSTRAINT "SmsUplinkMatchCandidate_uplinkMessageId_fkey" FOREIGN KEY ("uplinkMessageId") REFERENCES "SmsUplinkMessage"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "SmsUplinkMatchCandidate" ADD CONSTRAINT "SmsUplinkMatchCandidate_tenantId_fkey" FOREIGN KEY ("tenantId") REFERENCES "Tenant"("id") ON DELETE RESTRICT ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "SmsUplinkMatchCandidate" ADD CONSTRAINT "SmsUplinkMatchCandidate_applicationId_fkey" FOREIGN KEY ("applicationId") REFERENCES "SmsApplication"("id") ON DELETE RESTRICT ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "SmsUplinkMatchCandidate" ADD CONSTRAINT "SmsUplinkMatchCandidate_messageRecordId_fkey" FOREIGN KEY ("messageRecordId") REFERENCES "SmsMessageRecord"("id") ON DELETE SET NULL ON UPDATE CASCADE; diff --git a/api/prisma/schema.prisma b/api/prisma/schema.prisma index 13e7541..3a5150a 100644 --- a/api/prisma/schema.prisma +++ b/api/prisma/schema.prisma @@ -36,8 +36,13 @@ model Tenant { smsApiRequests SmsApiRequest[] smsSubmitRecords SmsSubmitRecord[] smsReceiptRecords SmsReceiptRecord[] + smsMessageSegmentAudits SmsMessageSegmentAudit[] smsUplinkMessages SmsUplinkMessage[] + smsUplinkMatchCandidates SmsUplinkMatchCandidate[] + cmppDownstreamDeliveries CmppDownstreamDelivery[] cmppConnectionStates CmppConnectionState[] + gatewayDownstreamRecoveryStatuses GatewayDownstreamRecoveryStatus[] + gatewaySubmitDeadLetters GatewaySubmitDeadLetter[] } model EnterpriseCertification { @@ -338,6 +343,8 @@ model SmsApplication { callbackUrl String? cmppAccount String @unique secretHash String + cmppMaxConnections Int @default(1) + cmppWindowSize Int @default(16) dailyLimit Int? customerUnitPrice Int @default(0) queuePriority String @default("normal") @@ -354,7 +361,12 @@ model SmsApplication { sendTasks SmsSendTask[] batchTasks SmsBatchTask[] messageRecords SmsMessageRecord[] + uplinkMessages SmsUplinkMessage[] + uplinkMatchCandidates SmsUplinkMatchCandidate[] + downstreamDeliveries CmppDownstreamDelivery[] connectionStates CmppConnectionState[] + gatewayDownstreamRecoveryStatuses GatewayDownstreamRecoveryStatus[] + gatewaySubmitDeadLetters GatewaySubmitDeadLetter[] @@index([tenantId, status]) } @@ -496,8 +508,10 @@ model SmsChannel { submitSessions CmppSubmitSession[] submitRecords SmsSubmitRecord[] receiptRecords SmsReceiptRecord[] + segmentAudits SmsMessageSegmentAudit[] uplinkMessages SmsUplinkMessage[] connectionStates CmppConnectionState[] + gatewaySubmitDeadLetters GatewaySubmitDeadLetter[] @@index([status]) } @@ -818,6 +832,7 @@ model SmsBatchTask { messages SmsMessageRecord[] submitRecords SmsSubmitRecord[] receiptRecords SmsReceiptRecord[] + segmentAudits SmsMessageSegmentAudit[] @@index([tenantId, status, createdAt]) @@index([status, scheduledAt]) @@ -876,6 +891,10 @@ model SmsMessageRecord { channel SmsChannel? @relation(fields: [channelId], references: [id]) submitRecords SmsSubmitRecord[] receiptRecords SmsReceiptRecord[] + segmentAudits SmsMessageSegmentAudit[] + matchedUplinks SmsUplinkMessage[] @relation("SmsUplinkMatchedMessage") + uplinkMatchCandidates SmsUplinkMatchCandidate[] + downstreamDeliveries CmppDownstreamDelivery[] @@index([tenantId, status, queuedAt]) @@index([batchTaskId, status]) @@ -922,12 +941,51 @@ model SmsSubmitRecord { messageRecord SmsMessageRecord @relation(fields: [messageRecordId], references: [id], onDelete: Cascade) channel SmsChannel @relation(fields: [channelId], references: [id]) session CmppSubmitSession? @relation(fields: [sessionId], references: [id]) + segmentAudits SmsMessageSegmentAudit[] @@index([tenantId, createdAt]) @@index([messageRecordId]) @@index([gatewayMessageId]) } +model SmsMessageSegmentAudit { + id String @id @default(cuid()) + tenantId String + batchTaskId String? + messageRecordId String + submitRecordId String? + channelId String? + submitId String + attempt Int @default(0) + segmentTotal Int @default(1) + segmentIndex Int @default(1) + sequenceId Int? + gatewayMessageId String? + submitStatus String @default("queued") + receiptStatus String? + rawStatus String? + compensationType String? + errorCode String? + errorMessage String? + submittedAt DateTime? + deliveredAt DateTime? + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + tenant Tenant @relation(fields: [tenantId], references: [id]) + batchTask SmsBatchTask? @relation(fields: [batchTaskId], references: [id]) + messageRecord SmsMessageRecord @relation(fields: [messageRecordId], references: [id], onDelete: Cascade) + submitRecord SmsSubmitRecord? @relation(fields: [submitRecordId], references: [id]) + channel SmsChannel? @relation(fields: [channelId], references: [id]) + + @@unique([messageRecordId, submitId, segmentIndex]) + @@index([tenantId, createdAt]) + @@index([messageRecordId, segmentIndex]) + @@index([submitRecordId]) + @@index([gatewayMessageId]) + @@index([submitStatus, receiptStatus]) +} + model SmsReceiptRecord { id String @id @default(cuid()) tenantId String @@ -954,21 +1012,152 @@ model SmsReceiptRecord { } model SmsUplinkMessage { - id String @id @default(cuid()) - tenantId String? - channelId String - messageId String? - sequenceId Int? - phoneNumber String - destId String - content String - receivedAt DateTime - createdAt DateTime @default(now()) + id String @id @default(cuid()) + tenantId String? + applicationId String? + channelId String + messageRecordId String? + messageId String? + sequenceId Int? + phoneNumber String + destId String + content String + matchStatus String @default("unmatched") + matchReason String? + receivedAt DateTime + createdAt DateTime @default(now()) - tenant Tenant? @relation(fields: [tenantId], references: [id]) - channel SmsChannel @relation(fields: [channelId], references: [id]) + tenant Tenant? @relation(fields: [tenantId], references: [id]) + application SmsApplication? @relation(fields: [applicationId], references: [id]) + messageRecord SmsMessageRecord? @relation("SmsUplinkMatchedMessage", fields: [messageRecordId], references: [id]) + channel SmsChannel @relation(fields: [channelId], references: [id]) + matchCandidates SmsUplinkMatchCandidate[] @@index([tenantId, createdAt]) + @@index([applicationId, receivedAt]) @@index([channelId, receivedAt]) + @@index([messageRecordId]) + @@index([matchStatus]) @@index([phoneNumber]) } + +model SmsUplinkMatchCandidate { + id String @id @default(cuid()) + uplinkMessageId String + tenantId String + applicationId String + messageRecordId String? + matchSource String + confidence Int @default(50) + reason String? + status String @default("pending") + claimedAt DateTime? + claimedById String? + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + uplinkMessage SmsUplinkMessage @relation(fields: [uplinkMessageId], references: [id], onDelete: Cascade) + tenant Tenant @relation(fields: [tenantId], references: [id]) + application SmsApplication @relation(fields: [applicationId], references: [id]) + messageRecord SmsMessageRecord? @relation(fields: [messageRecordId], references: [id]) + + @@unique([uplinkMessageId, applicationId, messageRecordId]) + @@index([uplinkMessageId, status]) + @@index([tenantId, status, createdAt]) + @@index([applicationId, status, createdAt]) + @@index([messageRecordId]) +} + +model CmppDownstreamDelivery { + id String @id @default(cuid()) + tenantId String + applicationId String + messageRecordId String? + messageId String? + deliveryType String + status String @default("pending") + payload Json + retryCount Int @default(0) + nextRetryAt DateTime? + deliveredAt DateTime? + lastError String? + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + tenant Tenant @relation(fields: [tenantId], references: [id]) + application SmsApplication @relation(fields: [applicationId], references: [id]) + messageRecord SmsMessageRecord? @relation(fields: [messageRecordId], references: [id]) + + @@index([tenantId, status, createdAt]) + @@index([applicationId, status, createdAt]) + @@index([messageId]) + @@index([messageRecordId]) + @@index([deliveryType, status]) +} + +model GatewaySubmitDeadLetter { + id String @id @default(cuid()) + streamMessageId String @unique + tenantId String? + applicationId String? + channelId String? + traceId String? + messageId String? + submitId String? + status String @default("pending") + failureCode String + failureMessage String + attempts Int @default(1) + maxAttempts Int @default(3) + commandPayload Json? + rawPayload String? + manualRetryCount Int @default(0) + lastRetryStreamId String? + lastRetriedAt DateTime? + resolvedAt DateTime? + resolvedStatus String? + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + tenant Tenant? @relation(fields: [tenantId], references: [id]) + application SmsApplication? @relation(fields: [applicationId], references: [id]) + channel SmsChannel? @relation(fields: [channelId], references: [id]) + + @@index([status, createdAt]) + @@index([tenantId, status, createdAt]) + @@index([applicationId, status, createdAt]) + @@index([channelId, status, createdAt]) + @@index([messageId]) + @@index([submitId]) +} + +model GatewayDownstreamRecoveryStatus { + id String @id @default(cuid()) + account String @unique + tenantId String? + applicationId String? + gatewayInstanceId String? + state String + lockOwner String? + lockExpiresAt DateTime? + lastAttemptAt DateTime? + lastSuccessAt DateTime? + lastFailureAt DateTime? + nextRetryAt DateTime? + attemptCount Int @default(0) + failureCategory String? + lastError String? + lastSkipReason String? + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + tenant Tenant? @relation(fields: [tenantId], references: [id]) + application SmsApplication? @relation(fields: [applicationId], references: [id]) + + @@index([tenantId, state, updatedAt]) + @@index([applicationId, state, updatedAt]) + @@index([failureCategory, updatedAt]) + @@index([lockOwner, lockExpiresAt]) + @@index([state, updatedAt]) + @@index([nextRetryAt]) +} diff --git a/api/src/channels/channels.service.spec.ts b/api/src/channels/channels.service.spec.ts index 76d059a..e21890b 100644 --- a/api/src/channels/channels.service.spec.ts +++ b/api/src/channels/channels.service.spec.ts @@ -155,6 +155,8 @@ describe('ChannelsService', () => { account: 'sp', passwordCipher: 'secret', srcId: '10690000', + desiredConnections: 2, + windowSize: 32, }); await service.createGroup({ code: 'G-MOBILE', name: '移动组', carrier: 'mobile', retryEnabled: true, retryTimeLimitMinutes: 750 }); await service.createRouteRule({ tenantId: 'tenant-1', applicationId: 'app-1', groupId: 'group-1', carrier: 'mobile' }); @@ -166,6 +168,7 @@ describe('ChannelsService', () => { rateLimitPerSecond: 100, sendRegion: '全国', status: 'active', + config: expect.objectContaining({ desiredConnections: 2, windowSize: 32 }), }), }); expect(prisma.cmppConnectionState.create).toHaveBeenCalledWith({ @@ -173,7 +176,7 @@ describe('ChannelsService', () => { channelId: 'channel-1', connectionId: 'channel-1:primary', status: 'connecting', - desiredConnections: 1, + desiredConnections: 2, currentConnections: 0, }), }); @@ -215,6 +218,8 @@ describe('ChannelsService', () => { sendRegion: '全国', account: 'sp-new', srcId: '10690001', + desiredConnections: 3, + windowSize: 64, unitPrice: 4, })).resolves.toEqual(expect.objectContaining({ id: 'channel-1', @@ -230,6 +235,7 @@ describe('ChannelsService', () => { gatewayPort: 27890, carrier: 'all', passwordCipher: undefined, + config: expect.objectContaining({ desiredConnections: 3, windowSize: 64 }), }), }); expect(prisma.operationLog.create).toHaveBeenCalledWith({ diff --git a/api/src/channels/channels.service.ts b/api/src/channels/channels.service.ts index d9fc351..94274b6 100644 --- a/api/src/channels/channels.service.ts +++ b/api/src/channels/channels.service.ts @@ -20,6 +20,8 @@ export interface CreateChannelDto { rateLimitPerSecond?: number; unitPrice?: number; status?: string; + desiredConnections?: number; + windowSize?: number; config?: Record; } @@ -196,6 +198,7 @@ export class ChannelsService implements OnModuleInit, OnModuleDestroy { if (!Number.isInteger(gatewayPort) || gatewayPort <= 0 || gatewayPort > 65535) { throw new BadRequestException('gatewayPort must be an integer between 1 and 65535'); } + const config = normalizeChannelRuntimeConfig(data.config, data.desiredConnections, data.windowSize); const channel = await this.prisma.smsChannel.create({ data: { code: data.code, @@ -213,7 +216,7 @@ export class ChannelsService implements OnModuleInit, OnModuleDestroy { rateLimitPerSecond: data.rateLimitPerSecond ?? 100, unitPrice: data.unitPrice ?? 0, status: data.status ?? 'active', - config: data.config as Prisma.InputJsonValue | undefined, + config: config as Prisma.InputJsonValue, }, }); if (channel.status === 'active') { @@ -231,6 +234,9 @@ export class ChannelsService implements OnModuleInit, OnModuleDestroy { if (gatewayPort !== undefined && (!Number.isInteger(gatewayPort) || gatewayPort <= 0 || gatewayPort > 65535)) { throw new BadRequestException('gatewayPort must be an integer between 1 and 65535'); } + const config = data.config !== undefined || data.desiredConnections !== undefined || data.windowSize !== undefined + ? normalizeChannelRuntimeConfig(channel.config, data.desiredConnections, data.windowSize) + : undefined; const updated = await this.prisma.smsChannel.update({ where: { id: channelId }, data: { @@ -249,7 +255,7 @@ export class ChannelsService implements OnModuleInit, OnModuleDestroy { rateLimitPerSecond: data.rateLimitPerSecond, unitPrice: data.unitPrice, status: data.status, - config: data.config as Prisma.InputJsonValue | undefined, + config: config as Prisma.InputJsonValue | undefined, }, }); await this.prisma.operationLog.create({ @@ -1105,6 +1111,26 @@ function getDesiredConnections(config?: Prisma.JsonValue | null) { return 1; } +function normalizeChannelRuntimeConfig(config?: Prisma.JsonValue | Record | null, desiredConnections?: number, windowSize?: number) { + const base = config && typeof config === 'object' && !Array.isArray(config) + ? { ...(config as Record) } + : {}; + base.desiredConnections = getPositiveRuntimeInteger(desiredConnections ?? base.desiredConnections, 1, 'desiredConnections'); + base.windowSize = getPositiveRuntimeInteger(windowSize ?? base.windowSize, 16, 'windowSize'); + return base; +} + +function getPositiveRuntimeInteger(value: unknown, fallback: number, fieldName: string) { + if (value === undefined || value === null || value === '') { + return fallback; + } + const normalized = Number(value); + if (!Number.isInteger(normalized) || normalized <= 0) { + throw new BadRequestException(`${fieldName} must be a positive integer`); + } + return normalized; +} + function bullmqConnection() { const redisUrl = new URL(process.env.REDIS_URL ?? 'redis://127.0.0.1:6379'); return { diff --git a/api/src/operations/admin-operations.controller.ts b/api/src/operations/admin-operations.controller.ts index 053840a..8a4a241 100644 --- a/api/src/operations/admin-operations.controller.ts +++ b/api/src/operations/admin-operations.controller.ts @@ -1,11 +1,20 @@ -import { Controller, Get, Query } from '@nestjs/common'; +import { Body, Controller, Get, Param, Post, Query, Res } from '@nestjs/common'; import { ApiTags } from '@nestjs/swagger'; +import { SendChainService } from '../send-chain/send-chain.service'; import { OperationsService } from './operations.service'; +type DownloadResponse = { + setHeader(name: string, value: number | string): void; + send(content: string | Buffer): void; +}; + @ApiTags('operations') @Controller('admin/operations') export class AdminOperationsController { - constructor(private readonly operations: OperationsService) {} + constructor( + private readonly operations: OperationsService, + private readonly sendChain: SendChainService, + ) {} @Get('monitor') monitor(@Query('tenantId') tenantId?: string, @Query('channelId') channelId?: string) { @@ -30,11 +39,27 @@ export class AdminOperationsController { return this.operations.listMessages({ tenantId, applicationId, channelId, taskId, messageId, phoneNumber, status }); } + @Get('message-segment-audits') + messageSegmentAudits( + @Query('messageId') messageId?: string, + @Query('messageRecordId') messageRecordId?: string, + ) { + return this.operations.listMessageSegmentAudits({ messageId, messageRecordId }); + } + @Get('uplink-messages') listUplinkMessages(@Query('tenantId') tenantId?: string, @Query('channelId') channelId?: string) { return this.operations.listUplinkMessages({ tenantId, channelId }); } + @Post('uplink-messages/:id/claim') + claimUplinkMatchCandidate( + @Param('id') id: string, + @Body() body: { candidateId?: string; operatorId?: string }, + ) { + return this.sendChain.claimUplinkMatchCandidate(id, String(body.candidateId ?? ''), body.operatorId); + } + @Get('dashboard') dashboard(@Query('tenantId') tenantId?: string) { return this.operations.dashboard({ tenantId }); @@ -76,6 +101,123 @@ export class AdminOperationsController { reconciliation(@Query('tenantId') tenantId?: string, @Query('taskId') taskId?: string) { return this.operations.reconciliation({ tenantId, taskId }); } + + @Get('gateway-submit-dead-letters') + gatewaySubmitDeadLetters( + @Query('tenantId') tenantId?: string, + @Query('applicationId') applicationId?: string, + @Query('channelId') channelId?: string, + @Query('status') status?: string, + @Query('keyword') keyword?: string, + @Query('page') page?: string, + @Query('pageSize') pageSize?: string, + ) { + return this.operations.listGatewaySubmitDeadLetters({ + tenantId, + applicationId, + channelId, + status, + keyword, + page: Number(page), + pageSize: Number(pageSize), + }); + } + + @Post('gateway-submit-dead-letters/:id/requeue') + requeueGatewaySubmitDeadLetter(@Param('id') id: string) { + return this.sendChain.requeueGatewaySubmitDeadLetter(id); + } + + @Get('downstream-deliveries') + downstreamDeliveries( + @Query('tenantId') tenantId?: string, + @Query('applicationId') applicationId?: string, + @Query('deliveryType') deliveryType?: string, + @Query('status') status?: string, + @Query('keyword') keyword?: string, + @Query('page') page?: string, + @Query('pageSize') pageSize?: string, + ) { + return this.operations.listDownstreamDeliveries({ + tenantId, + applicationId, + deliveryType, + status, + keyword, + page: Number(page), + pageSize: Number(pageSize), + }); + } + + @Get('downstream-deliveries/dashboard') + downstreamDeliveryDashboard( + @Query('tenantId') tenantId?: string, + @Query('applicationId') applicationId?: string, + @Query('deliveryType') deliveryType?: string, + ) { + return this.operations.downstreamDeliveryDashboard({ + tenantId, + applicationId, + deliveryType, + }); + } + + @Get('downstream-recovery-statuses') + downstreamRecoveryStatuses( + @Query('tenantId') tenantId?: string, + @Query('applicationId') applicationId?: string, + @Query('state') state?: string, + @Query('failureCategory') failureCategory?: string, + @Query('keyword') keyword?: string, + @Query('page') page?: string, + @Query('pageSize') pageSize?: string, + ) { + return this.operations.listDownstreamRecoveryStatuses({ + tenantId, + applicationId, + state, + failureCategory, + keyword, + page: Number(page), + pageSize: Number(pageSize), + }); + } + + @Get('downstream-recovery-statuses/export') + async exportDownstreamRecoveryStatuses( + @Query('tenantId') tenantId: string | undefined, + @Query('applicationId') applicationId: string | undefined, + @Query('state') state: string | undefined, + @Query('failureCategory') failureCategory: string | undefined, + @Query('keyword') keyword: string | undefined, + @Res() response: DownloadResponse, + ) { + const exported = await this.operations.exportDownstreamRecoveryStatuses({ + tenantId, + applicationId, + state, + failureCategory, + keyword, + }); + response.setHeader('Content-Type', 'text/csv; charset=utf-8'); + response.setHeader('Content-Disposition', `attachment; filename*=UTF-8''${encodeURIComponent(exported.fileName)}`); + response.send(`\uFEFF${exported.content}`); + } + + @Get('downstream-recovery-statuses/:id') + downstreamRecoveryStatusDetail(@Param('id') id: string) { + return this.operations.getDownstreamRecoveryStatus(id); + } + + @Post('downstream-deliveries/:id/requeue') + requeueDownstreamDelivery(@Param('id') id: string) { + return this.sendChain.requeueDownstreamDelivery(id); + } + + @Post('downstream-deliveries/requeue') + batchRequeueDownstreamDeliveries(@Body() body: { ids?: string[] }) { + return this.sendChain.batchRequeueDownstreamDeliveries(body.ids ?? []); + } } @ApiTags('admin-system-logs') diff --git a/api/src/operations/operations.module.ts b/api/src/operations/operations.module.ts index 19c9217..066eda6 100644 --- a/api/src/operations/operations.module.ts +++ b/api/src/operations/operations.module.ts @@ -1,11 +1,12 @@ import { Module } from '@nestjs/common'; import { PrismaModule } from '../prisma/prisma.module'; +import { SendChainModule } from '../send-chain/send-chain.module'; import { AdminOperationsController, AdminSystemLogsController } from './admin-operations.controller'; import { ClientOperationsController } from './client-operations.controller'; import { OperationsService } from './operations.service'; @Module({ - imports: [PrismaModule], + imports: [PrismaModule, SendChainModule], controllers: [AdminOperationsController, AdminSystemLogsController, ClientOperationsController], providers: [OperationsService], exports: [OperationsService], diff --git a/api/src/operations/operations.service.spec.ts b/api/src/operations/operations.service.spec.ts index bc36073..e374698 100644 --- a/api/src/operations/operations.service.spec.ts +++ b/api/src/operations/operations.service.spec.ts @@ -40,6 +40,12 @@ function createPrismaMock() { enterpriseCertification: { count: jest.fn().mockResolvedValue(1), }, + smsApplication: { + findMany: jest.fn().mockResolvedValue([ + { id: 'app-1', name: '应用A' }, + { id: 'app-2', name: '应用B' }, + ]), + }, cmppConnectionState: { groupBy: jest.fn().mockResolvedValue([{ status: 'connected', _count: { _all: 1 }, _sum: { currentConnections: 2, desiredConnections: 2 } }]), }, @@ -58,6 +64,101 @@ function createPrismaMock() { count: jest.fn().mockResolvedValue(1), groupBy: jest.fn().mockResolvedValue([{ resource: 'recharge_order', _count: { _all: 1 } }]), }, + gatewaySubmitDeadLetter: { + findMany: jest.fn().mockResolvedValue([{ + id: 'dead-1', + streamMessageId: '1710000000000-0', + status: 'pending', + failureCode: 'SUBMIT_PROCESSING_FAILED', + failureMessage: 'network down', + tenant: { name: '租户A' }, + application: { name: '应用A' }, + channel: { code: 'CMPP-A' }, + }]), + count: jest.fn().mockResolvedValue(1), + }, + gatewayDownstreamRecoveryStatus: { + findMany: jest.fn().mockResolvedValue([{ + id: 'recover-1', + account: '100001', + state: 'waiting_connection', + lockOwner: 'gateway-a', + lockExpiresAt: new Date('2026-07-08T12:00:30.000Z'), + attemptCount: 2, + failureCategory: 'client_disconnected', + nextRetryAt: new Date('2026-07-08T12:10:00.000Z'), + lastError: 'downstream client is not connected', + tenant: { name: '租户A' }, + application: { name: '应用A' }, + }]), + findUnique: jest.fn().mockResolvedValue({ + id: 'recover-1', + account: '100001', + state: 'waiting_connection', + lockOwner: 'gateway-a', + lockExpiresAt: new Date('2026-07-08T12:00:30.000Z'), + attemptCount: 2, + failureCategory: 'client_disconnected', + nextRetryAt: new Date('2026-07-08T12:10:00.000Z'), + lastAttemptAt: new Date('2026-07-08T12:00:00.000Z'), + lastSuccessAt: null, + lastFailureAt: new Date('2026-07-08T11:58:00.000Z'), + lastError: 'downstream client is not connected', + lastSkipReason: 'waiting for reconnect', + gatewayInstanceId: 'gw-01', + createdAt: new Date('2026-07-08T11:50:00.000Z'), + updatedAt: new Date('2026-07-08T12:00:00.000Z'), + tenant: { name: '租户A' }, + application: { name: '应用A' }, + }), + count: jest.fn().mockResolvedValue(1), + groupBy: jest.fn().mockResolvedValue([ + { failureCategory: 'client_disconnected', _count: { _all: 1 } }, + ]), + }, + smsMessageSegmentAudit: { + findMany: jest.fn().mockResolvedValue([{ + id: 'segment-1', + messageRecordId: 'record-1', + submitId: 'SUB-1', + segmentTotal: 2, + segmentIndex: 1, + sequenceId: 7, + gatewayMessageId: 'GW-1-A', + submitStatus: 'accepted', + receiptStatus: 'delivered', + channel: { name: '通道A' }, + }]), + }, + cmppDownstreamDelivery: { + findMany: jest.fn().mockResolvedValue([{ + id: 'delivery-1', + tenantId: 'tenant-1', + applicationId: 'app-1', + messageId: 'MSG-1', + deliveryType: 'receipt', + status: 'failed', + payload: { account: '100001', phoneNumber: '13800000001' }, + retryCount: 10, + lastError: 'client offline', + tenant: { name: '租户A' }, + application: { name: '应用A' }, + messageRecord: { messageId: 'MSG-1' }, + }]), + count: jest.fn().mockResolvedValue(1), + groupBy: jest.fn().mockResolvedValue([ + { deliveryType: 'receipt', status: 'pending', _count: { _all: 2 } }, + { deliveryType: 'receipt', status: 'failed', _count: { _all: 1 } }, + { deliveryType: 'receipt', status: 'delivered', _count: { _all: 6 } }, + { deliveryType: 'uplink', status: 'pending', _count: { _all: 1 } }, + { deliveryType: 'uplink', status: 'delivered', _count: { _all: 2 } }, + { applicationId: 'app-1', status: 'pending', _count: { _all: 2 } }, + { applicationId: 'app-1', status: 'failed', _count: { _all: 1 } }, + { applicationId: 'app-1', status: 'delivered', _count: { _all: 5 } }, + { applicationId: 'app-2', status: 'pending', _count: { _all: 1 } }, + { applicationId: 'app-2', status: 'delivered', _count: { _all: 3 } }, + ]), + }, }; } @@ -100,7 +201,20 @@ describe('OperationsService', () => { expect(prisma.smsUplinkMessage.findMany).toHaveBeenCalledWith({ where: { tenantId: 'tenant-1', channelId: 'channel-1' }, - include: { tenant: true, channel: true }, + include: { + tenant: true, + application: true, + channel: true, + messageRecord: { include: { application: true } }, + matchCandidates: { + include: { + tenant: true, + application: true, + messageRecord: { include: { application: true, tenant: true, channel: true } }, + }, + orderBy: [{ status: 'asc' }, { confidence: 'desc' }, { createdAt: 'asc' }], + }, + }, orderBy: { receivedAt: 'desc' }, take: 500, }); @@ -108,6 +222,12 @@ describe('OperationsService', () => { it('builds dashboard and statistics aggregates', async () => { const prisma = createPrismaMock(); + prisma.cmppDownstreamDelivery.count = jest.fn() + .mockResolvedValueOnce(3) + .mockResolvedValueOnce(2) + .mockResolvedValueOnce(8) + .mockResolvedValueOnce(1) + .mockResolvedValueOnce(2); const service = new OperationsService(prisma as never); await expect(service.dashboard({ tenantId: 'tenant-1' })).resolves.toEqual( @@ -116,6 +236,14 @@ describe('OperationsService', () => { uplinkCount: 1, pendingAuditCount: 6, gatewayConnections: [{ status: 'connected', _count: { _all: 1 }, _sum: { currentConnections: 2, desiredConnections: 2 } }], + downstreamDeliverySummary: expect.objectContaining({ + pending: 3, + failed: 2, + delivered: 8, + stalledPending: 1, + recentFailed: 2, + alertCount: 3, + }), }), ); await service.statistics({ tenantId: 'tenant-1', groupBy: 'application' }); @@ -169,4 +297,221 @@ describe('OperationsService', () => { }), ); }); + + it('returns paginated gateway submit dead letters', async () => { + const prisma = createPrismaMock(); + const service = new OperationsService(prisma as never); + + await expect(service.listGatewaySubmitDeadLetters({ + tenantId: 'tenant-1', + status: 'pending', + keyword: 'SUBMIT', + page: 1, + pageSize: 10, + })).resolves.toEqual({ + items: [expect.objectContaining({ id: 'dead-1', status: 'pending' })], + total: 1, + page: 1, + pageSize: 10, + }); + + expect(prisma.gatewaySubmitDeadLetter.findMany).toHaveBeenCalledWith({ + where: expect.objectContaining({ + tenantId: 'tenant-1', + status: 'pending', + }), + include: { tenant: true, application: true, channel: true }, + orderBy: { createdAt: 'desc' }, + skip: 0, + take: 10, + }); + }); + + it('returns paginated downstream deliveries', async () => { + const prisma = createPrismaMock(); + const service = new OperationsService(prisma as never); + + await expect(service.listDownstreamDeliveries({ + tenantId: 'tenant-1', + deliveryType: 'receipt', + status: 'failed', + keyword: '1380', + page: 1, + pageSize: 10, + })).resolves.toEqual({ + items: [expect.objectContaining({ id: 'delivery-1', status: 'failed' })], + total: 1, + page: 1, + pageSize: 10, + }); + + expect(prisma.cmppDownstreamDelivery.findMany).toHaveBeenCalledWith({ + where: expect.objectContaining({ + tenantId: 'tenant-1', + deliveryType: 'receipt', + status: 'failed', + }), + include: { tenant: true, application: true, messageRecord: true }, + orderBy: { createdAt: 'desc' }, + skip: 0, + take: 10, + }); + }); + + it('builds downstream delivery dashboard aggregates', async () => { + const prisma = createPrismaMock(); + prisma.cmppDownstreamDelivery.count = jest.fn() + .mockResolvedValueOnce(12) + .mockResolvedValueOnce(3) + .mockResolvedValueOnce(8) + .mockResolvedValueOnce(1) + .mockResolvedValueOnce(1) + .mockResolvedValueOnce(1) + .mockResolvedValueOnce(2) + .mockResolvedValueOnce(1) + .mockResolvedValueOnce(0); + prisma.cmppDownstreamDelivery.groupBy = jest.fn() + .mockResolvedValueOnce([ + { deliveryType: 'receipt', status: 'pending', _count: { _all: 2 } }, + { deliveryType: 'receipt', status: 'failed', _count: { _all: 1 } }, + { deliveryType: 'receipt', status: 'delivered', _count: { _all: 6 } }, + { deliveryType: 'uplink', status: 'pending', _count: { _all: 1 } }, + { deliveryType: 'uplink', status: 'delivered', _count: { _all: 2 } }, + ]) + .mockResolvedValueOnce([ + { applicationId: 'app-1', status: 'pending', _count: { _all: 2 } }, + { applicationId: 'app-1', status: 'failed', _count: { _all: 1 } }, + { applicationId: 'app-1', status: 'delivered', _count: { _all: 5 } }, + { applicationId: 'app-2', status: 'pending', _count: { _all: 1 } }, + { applicationId: 'app-2', status: 'delivered', _count: { _all: 3 } }, + ]); + const service = new OperationsService(prisma as never); + + await expect(service.downstreamDeliveryDashboard({ + tenantId: 'tenant-1', + applicationId: 'app-1', + deliveryType: 'all', + })).resolves.toEqual({ + summary: { + total: 12, + pending: 3, + delivered: 8, + failed: 1, + stalledPending: 1, + recentFailed: 1, + alertCount: 2, + }, + typeBreakdown: [ + { deliveryType: 'receipt', total: 9, pending: 2, delivered: 6, failed: 1 }, + { deliveryType: 'uplink', total: 3, pending: 1, delivered: 2, failed: 0 }, + ], + retryBuckets: [ + { label: '0次', count: 2 }, + { label: '1-3次', count: 1 }, + { label: '4次及以上', count: 0 }, + ], + topApplications: [ + { applicationId: 'app-1', name: '应用A', pending: 2, failed: 1, delivered: 5, alertCount: 3 }, + { applicationId: 'app-2', name: '应用B', pending: 1, failed: 0, delivered: 3, alertCount: 1 }, + ], + }); + }); + + it('returns paginated downstream recovery statuses', async () => { + const prisma = createPrismaMock(); + prisma.gatewayDownstreamRecoveryStatus.count = jest.fn() + .mockResolvedValueOnce(1) + .mockResolvedValueOnce(0) + .mockResolvedValueOnce(0) + .mockResolvedValueOnce(0) + .mockResolvedValueOnce(1) + .mockResolvedValueOnce(1); + const service = new OperationsService(prisma as never); + + await expect(service.listDownstreamRecoveryStatuses({ + tenantId: 'tenant-1', + applicationId: 'app-1', + state: 'waiting_connection', + failureCategory: 'client_disconnected', + keyword: '100001', + page: 1, + pageSize: 10, + })).resolves.toEqual({ + items: [expect.objectContaining({ id: 'recover-1', account: '100001', state: 'waiting_connection' })], + total: 1, + page: 1, + pageSize: 10, + summary: { + total: 1, + running: 0, + success: 0, + failed: 0, + waitingConnection: 1, + backoff: 1, + failureCategories: [ + { category: 'client_disconnected', count: 1 }, + ], + }, + }); + }); + + it('returns downstream recovery status detail', async () => { + const prisma = createPrismaMock(); + const service = new OperationsService(prisma as never); + + await expect(service.getDownstreamRecoveryStatus('recover-1')).resolves.toEqual( + expect.objectContaining({ + id: 'recover-1', + account: '100001', + gatewayInstanceId: 'gw-01', + }), + ); + + expect(prisma.gatewayDownstreamRecoveryStatus.findUnique).toHaveBeenCalledWith({ + where: { id: 'recover-1' }, + include: { tenant: true, application: true }, + }); + }); + + it('exports downstream recovery statuses as csv rows', async () => { + const prisma = createPrismaMock(); + const service = new OperationsService(prisma as never); + + await expect(service.exportDownstreamRecoveryStatuses({ + tenantId: 'tenant-1', + state: 'waiting_connection', + keyword: '100001', + })).resolves.toEqual(expect.objectContaining({ + total: 1, + fileName: expect.stringMatching(/^gateway-downstream-recovery-statuses-\d{8}-\d{6}\.csv$/), + content: expect.stringContaining('100001'), + })); + expect(prisma.gatewayDownstreamRecoveryStatus.findMany).toHaveBeenCalledWith(expect.objectContaining({ + where: expect.objectContaining({ + failureCategory: undefined, + }), + })); + }); + + it('returns message segment audit rows', async () => { + const prisma = createPrismaMock(); + const service = new OperationsService(prisma as never); + + await expect(service.listMessageSegmentAudits({ messageRecordId: 'record-1' })).resolves.toEqual([ + expect.objectContaining({ + id: 'segment-1', + segmentIndex: 1, + gatewayMessageId: 'GW-1-A', + }), + ]); + + expect(prisma.smsMessageSegmentAudit.findMany).toHaveBeenCalledWith({ + where: { + messageRecordId: 'record-1', + messageRecord: undefined, + }, + include: { channel: true, submitRecord: true }, + orderBy: [{ submitId: 'asc' }, { segmentIndex: 'asc' }], + }); + }); }); diff --git a/api/src/operations/operations.service.ts b/api/src/operations/operations.service.ts index 45760b6..f6a55e9 100644 --- a/api/src/operations/operations.service.ts +++ b/api/src/operations/operations.service.ts @@ -1,4 +1,4 @@ -import { Injectable } from '@nestjs/common'; +import { Injectable, NotFoundException } from '@nestjs/common'; import { Prisma } from '@prisma/client'; import { PrismaService } from '../prisma/prisma.service'; @@ -27,6 +27,47 @@ export interface OperationLogQuery { pageSize?: number; } +export interface GatewaySubmitDeadLetterQuery { + tenantId?: string; + applicationId?: string; + channelId?: string; + status?: string; + keyword?: string; + page?: number; + pageSize?: number; +} + +export interface DownstreamDeliveryQuery { + tenantId?: string; + applicationId?: string; + deliveryType?: string; + status?: string; + keyword?: string; + page?: number; + pageSize?: number; +} + +export interface DownstreamDeliveryDashboardQuery { + tenantId?: string; + applicationId?: string; + deliveryType?: string; +} + +export interface DownstreamRecoveryStatusQuery { + tenantId?: string; + applicationId?: string; + state?: string; + failureCategory?: string; + keyword?: string; + page?: number; + pageSize?: number; +} + +export interface MessageSegmentAuditQuery { + messageId?: string; + messageRecordId?: string; +} + @Injectable() export class OperationsService { constructor(private readonly prisma: PrismaService) {} @@ -52,7 +93,20 @@ export class OperationsService { listUplinkMessages(query: { tenantId?: string; channelId?: string }) { return this.prisma.smsUplinkMessage.findMany({ where: { tenantId: query.tenantId, channelId: query.channelId }, - include: { tenant: true, channel: true }, + include: { + tenant: true, + application: true, + channel: true, + messageRecord: { include: { application: true } }, + matchCandidates: { + include: { + tenant: true, + application: true, + messageRecord: { include: { application: true, tenant: true, channel: true } }, + }, + orderBy: [{ status: 'asc' }, { confidence: 'desc' }, { createdAt: 'asc' }], + }, + }, orderBy: { receivedAt: 'desc' }, take: 500, }); @@ -99,6 +153,11 @@ export class OperationsService { tenantAccounts, recentTasks, recentRecharges, + downstreamPendingCount, + downstreamFailedCount, + downstreamDeliveredCount, + downstreamStalledPendingCount, + downstreamRecentFailedCount, ] = await Promise.all([ this.prisma.smsBatchTask.count({ where: { tenantId: query.tenantId } }), this.prisma.smsMessageRecord.groupBy({ @@ -152,8 +211,32 @@ export class OperationsService { orderBy: { createdAt: 'desc' }, take: 10, }), + this.prisma.cmppDownstreamDelivery.count({ + where: { tenantId: query.tenantId, status: 'pending' }, + }), + this.prisma.cmppDownstreamDelivery.count({ + where: { tenantId: query.tenantId, status: 'failed' }, + }), + this.prisma.cmppDownstreamDelivery.count({ + where: { tenantId: query.tenantId, status: 'delivered' }, + }), + this.prisma.cmppDownstreamDelivery.count({ + where: { + tenantId: query.tenantId, + status: 'pending', + createdAt: { lte: new Date(Date.now() - downstreamAlertPendingMinutes() * 60_000) }, + }, + }), + this.prisma.cmppDownstreamDelivery.count({ + where: { + tenantId: query.tenantId, + status: 'failed', + updatedAt: { gte: new Date(Date.now() - downstreamAlertRecentFailedHours() * 60 * 60_000) }, + }, + }), ]); const todayTotals = summarizeMessageGroups(todayMessageGroups); + const downstreamAlertCount = downstreamStalledPendingCount + downstreamRecentFailedCount; return { taskCount, messageStatus: messageGroups, @@ -171,6 +254,14 @@ export class OperationsService { transactions: transactionAggregate, gatewayConnections: connectionGroups, pendingAuditCount, + downstreamDeliverySummary: { + pending: downstreamPendingCount, + failed: downstreamFailedCount, + delivered: downstreamDeliveredCount, + stalledPending: downstreamStalledPendingCount, + recentFailed: downstreamRecentFailedCount, + alertCount: downstreamAlertCount, + }, accounts: tenantAccounts, recentTasks, recentRecharges, @@ -256,6 +347,303 @@ export class OperationsService { }; } + async listGatewaySubmitDeadLetters(query: GatewaySubmitDeadLetterQuery) { + const page = Math.max(1, Number(query.page ?? 1)); + const pageSize = Math.min(100, Math.max(1, Number(query.pageSize ?? 10))); + const where: Prisma.GatewaySubmitDeadLetterWhereInput = { + tenantId: query.tenantId, + applicationId: query.applicationId, + channelId: query.channelId, + status: query.status && query.status !== 'all' ? query.status : undefined, + OR: query.keyword ? [ + { streamMessageId: { contains: query.keyword } }, + { traceId: { contains: query.keyword } }, + { messageId: { contains: query.keyword } }, + { submitId: { contains: query.keyword } }, + { failureCode: { contains: query.keyword } }, + { failureMessage: { contains: query.keyword } }, + ] : undefined, + }; + const [items, total] = await Promise.all([ + this.prisma.gatewaySubmitDeadLetter.findMany({ + where, + include: { tenant: true, application: true, channel: true }, + orderBy: { createdAt: 'desc' }, + skip: (page - 1) * pageSize, + take: pageSize, + }), + this.prisma.gatewaySubmitDeadLetter.count({ where }), + ]); + return { items, total, page, pageSize }; + } + + async listDownstreamDeliveries(query: DownstreamDeliveryQuery) { + const page = Math.max(1, Number(query.page ?? 1)); + const pageSize = Math.min(100, Math.max(1, Number(query.pageSize ?? 10))); + const where: Prisma.CmppDownstreamDeliveryWhereInput = { + ...downstreamDeliveryScopedWhere(query), + status: query.status && query.status !== 'all' ? query.status : undefined, + OR: query.keyword ? [ + { messageId: { contains: query.keyword } }, + { payload: { path: ['account'], string_contains: query.keyword } }, + { payload: { path: ['phoneNumber'], string_contains: query.keyword } }, + { lastError: { contains: query.keyword } }, + ] : undefined, + }; + const [items, total] = await Promise.all([ + this.prisma.cmppDownstreamDelivery.findMany({ + where, + include: { tenant: true, application: true, messageRecord: true }, + orderBy: { createdAt: 'desc' }, + skip: (page - 1) * pageSize, + take: pageSize, + }), + this.prisma.cmppDownstreamDelivery.count({ where }), + ]); + return { items, total, page, pageSize }; + } + + async downstreamDeliveryDashboard(query: DownstreamDeliveryDashboardQuery) { + const scopedWhere = downstreamDeliveryScopedWhere(query); + const stalledPendingAt = new Date(Date.now() - downstreamAlertPendingMinutes() * 60_000); + const recentFailedAt = new Date(Date.now() - downstreamAlertRecentFailedHours() * 60 * 60_000); + const [total, pending, delivered, failed, stalledPending, recentFailed, typeGroups, applicationGroups, retryZero, retryLow, retryHigh] = await Promise.all([ + this.prisma.cmppDownstreamDelivery.count({ where: scopedWhere }), + this.prisma.cmppDownstreamDelivery.count({ where: { ...scopedWhere, status: 'pending' } }), + this.prisma.cmppDownstreamDelivery.count({ where: { ...scopedWhere, status: 'delivered' } }), + this.prisma.cmppDownstreamDelivery.count({ where: { ...scopedWhere, status: 'failed' } }), + this.prisma.cmppDownstreamDelivery.count({ + where: { + ...scopedWhere, + status: 'pending', + createdAt: { lte: stalledPendingAt }, + }, + }), + this.prisma.cmppDownstreamDelivery.count({ + where: { + ...scopedWhere, + status: 'failed', + updatedAt: { gte: recentFailedAt }, + }, + }), + this.prisma.cmppDownstreamDelivery.groupBy({ + by: ['deliveryType', 'status'], + where: scopedWhere, + _count: { _all: true }, + }), + this.prisma.cmppDownstreamDelivery.groupBy({ + by: ['applicationId', 'status'], + where: scopedWhere, + _count: { _all: true }, + }), + this.prisma.cmppDownstreamDelivery.count({ + where: { + ...scopedWhere, + status: { in: ['pending', 'failed'] }, + retryCount: 0, + }, + }), + this.prisma.cmppDownstreamDelivery.count({ + where: { + ...scopedWhere, + status: { in: ['pending', 'failed'] }, + retryCount: { gte: 1, lte: 3 }, + }, + }), + this.prisma.cmppDownstreamDelivery.count({ + where: { + ...scopedWhere, + status: { in: ['pending', 'failed'] }, + retryCount: { gte: 4 }, + }, + }), + ]); + const applicationIds = [...new Set(applicationGroups.map((item) => item.applicationId).filter((value): value is string => Boolean(value)))]; + const applications: Array<{ id: string; name: string }> = applicationIds.length > 0 + ? await this.prisma.smsApplication.findMany({ + where: { id: { in: applicationIds } }, + select: { id: true, name: true }, + }) + : []; + const applicationMap = new Map(applications.map((item) => [item.id, item.name])); + const groupedByType = groupDownstreamByType(typeGroups); + const groupedByApplication = groupDownstreamByApplication(applicationGroups, applicationMap); + + return { + summary: { + total, + pending, + delivered, + failed, + stalledPending, + recentFailed, + alertCount: stalledPending + recentFailed, + }, + typeBreakdown: ['receipt', 'uplink'].map((deliveryType) => ({ + deliveryType, + total: groupedByType[deliveryType]?.total ?? 0, + pending: groupedByType[deliveryType]?.pending ?? 0, + delivered: groupedByType[deliveryType]?.delivered ?? 0, + failed: groupedByType[deliveryType]?.failed ?? 0, + })), + retryBuckets: [ + { label: '0次', count: retryZero }, + { label: '1-3次', count: retryLow }, + { label: '4次及以上', count: retryHigh }, + ], + topApplications: groupedByApplication + .sort((left, right) => ( + right.alertCount - left.alertCount + || right.failed - left.failed + || right.pending - left.pending + || left.name.localeCompare(right.name, 'zh-CN') + )) + .slice(0, 5), + }; + } + + async listDownstreamRecoveryStatuses(query: DownstreamRecoveryStatusQuery) { + const recoveryStatuses = this.gatewayDownstreamRecoveryStatusDelegate(); + const page = Math.max(1, Number(query.page ?? 1)); + const pageSize = Math.min(100, Math.max(1, Number(query.pageSize ?? 10))); + const where = downstreamRecoveryStatusWhere(query); + const now = new Date(); + const [items, total, runningCount, successCount, failedCount, waitingConnectionCount, backoffCount, categoryGroups] = await Promise.all([ + recoveryStatuses.findMany({ + where, + include: { tenant: true, application: true }, + orderBy: [{ updatedAt: 'desc' }, { account: 'asc' }], + skip: (page - 1) * pageSize, + take: pageSize, + }), + recoveryStatuses.count({ where }), + recoveryStatuses.count({ where: { ...where, state: 'running' } }), + recoveryStatuses.count({ where: { ...where, state: 'success' } }), + recoveryStatuses.count({ where: { ...where, state: 'failed' } }), + recoveryStatuses.count({ where: { ...where, state: 'waiting_connection' } }), + recoveryStatuses.count({ + where: { + ...where, + nextRetryAt: { gt: now }, + }, + }), + recoveryStatuses.groupBy({ + by: ['failureCategory'], + where, + _count: { _all: true }, + }), + ]); + return { + items, + total, + page, + pageSize, + summary: { + total, + running: runningCount, + success: successCount, + failed: failedCount, + waitingConnection: waitingConnectionCount, + backoff: backoffCount, + failureCategories: categoryGroups + .filter((item) => item.failureCategory) + .map((item) => ({ + category: String(item.failureCategory), + count: item._count?._all ?? 0, + })) + .sort((left, right) => right.count - left.count || left.category.localeCompare(right.category)), + }, + }; + } + + async listMessageSegmentAudits(query: MessageSegmentAuditQuery) { + const segmentAudits = (this.prisma as PrismaService & { + smsMessageSegmentAudit: { + findMany: (args: Record) => Promise; + }; + }).smsMessageSegmentAudit; + if (!query.messageId && !query.messageRecordId) { + return []; + } + return segmentAudits.findMany({ + where: { + messageRecordId: query.messageRecordId, + messageRecord: query.messageId ? { messageId: query.messageId } : undefined, + }, + include: { channel: true, submitRecord: true }, + orderBy: [{ submitId: 'asc' }, { segmentIndex: 'asc' }], + }); + } + + async getDownstreamRecoveryStatus(id: string) { + const recoveryStatuses = this.gatewayDownstreamRecoveryStatusDelegate(); + const item = await recoveryStatuses.findUnique({ + where: { id }, + include: { tenant: true, application: true }, + }); + if (!item) { + throw new NotFoundException('Recovery status not found'); + } + return item; + } + + async exportDownstreamRecoveryStatuses(query: DownstreamRecoveryStatusQuery) { + const recoveryStatuses = this.gatewayDownstreamRecoveryStatusDelegate(); + const where = downstreamRecoveryStatusWhere(query); + const items = await recoveryStatuses.findMany({ + where, + include: { tenant: true, application: true }, + orderBy: [{ updatedAt: 'desc' }, { account: 'asc' }], + take: 5000, + }); + const rows = [ + [ + '账号', + '企业', + '应用', + 'Gateway实例', + '恢复状态', + '锁持有实例', + '锁过期时间', + '失败分类', + '尝试次数', + '最后尝试时间', + '恢复成功时间', + '恢复失败时间', + '下次恢复时间', + '最后错误', + '最后跳过原因', + '创建时间', + '更新时间', + ], + ...items.map((item) => [ + item.account ?? '', + item.tenant?.name ?? '', + item.application?.name ?? '', + item.gatewayInstanceId ?? '', + item.state ?? '', + (item as { lockOwner?: string | null }).lockOwner ?? '', + formatCsvDate((item as { lockExpiresAt?: Date | string | null }).lockExpiresAt), + (item as { failureCategory?: string | null }).failureCategory ?? '', + String(item.attemptCount ?? 0), + formatCsvDate(item.lastAttemptAt), + formatCsvDate(item.lastSuccessAt), + formatCsvDate(item.lastFailureAt), + formatCsvDate(item.nextRetryAt), + item.lastError ?? '', + item.lastSkipReason ?? '', + formatCsvDate(item.createdAt), + formatCsvDate(item.updatedAt), + ]), + ]; + + return { + fileName: `gateway-downstream-recovery-statuses-${formatExportTimestamp(new Date())}.csv`, + content: rows.map((row) => row.map(escapeCsvCell).join(',')).join('\n'), + total: items.length, + }; + } + auditSummary(query: { tenantId?: string }) { return this.prisma.operationLog.groupBy({ by: ['action', 'resource'], @@ -346,6 +734,17 @@ export class OperationsService { this.prisma.smsBatchTask.count({ where: { tenantId, auditStatus: 'pending' } }), ]).then((counts) => counts.reduce((sum, value) => sum + value, 0)); } + + private gatewayDownstreamRecoveryStatusDelegate() { + return (this.prisma as PrismaService & { + gatewayDownstreamRecoveryStatus: { + findMany: (args: Record) => Promise; + count: (args: Record) => Promise; + findUnique: (args: Record) => Promise; + groupBy: (args: Record) => Promise; + }; + }).gatewayDownstreamRecoveryStatus; + } } function messageWhere(query: MessageQuery): Prisma.SmsMessageRecordWhereInput { @@ -390,6 +789,68 @@ function createdAtRange(range?: string): Prisma.DateTimeFilter | undefined { return { gte: date }; } +function downstreamAlertPendingMinutes() { + const value = Number(process.env.CMPP_DOWNSTREAM_ALERT_PENDING_MINUTES ?? 10); + return Number.isFinite(value) && value > 0 ? value : 10; +} + +function downstreamAlertRecentFailedHours() { + const value = Number(process.env.CMPP_DOWNSTREAM_ALERT_RECENT_FAILED_HOURS ?? 1); + return Number.isFinite(value) && value > 0 ? value : 1; +} + +function downstreamDeliveryScopedWhere(query: DownstreamDeliveryDashboardQuery): Prisma.CmppDownstreamDeliveryWhereInput { + return { + tenantId: query.tenantId, + applicationId: query.applicationId, + deliveryType: query.deliveryType && query.deliveryType !== 'all' ? query.deliveryType : undefined, + }; +} + +function downstreamRecoveryStatusWhere(query: DownstreamRecoveryStatusQuery) { + return { + tenantId: query.tenantId, + applicationId: query.applicationId, + state: query.state && query.state !== 'all' ? query.state : undefined, + failureCategory: query.failureCategory && query.failureCategory !== 'all' ? query.failureCategory : undefined, + OR: query.keyword ? [ + { account: { contains: query.keyword } }, + { gatewayInstanceId: { contains: query.keyword } }, + { lastError: { contains: query.keyword } }, + { lastSkipReason: { contains: query.keyword } }, + { tenant: { name: { contains: query.keyword } } }, + { application: { name: { contains: query.keyword } } }, + ] : undefined, + }; +} + +function escapeCsvCell(value: string) { + const normalized = value.replace(/\r\n/g, '\n').replace(/\r/g, '\n'); + if (normalized.includes(',') || normalized.includes('"') || normalized.includes('\n')) { + return `"${normalized.replace(/"/g, '""')}"`; + } + return normalized; +} + +function formatCsvDate(value?: Date | string | null) { + if (!value) { + return ''; + } + return value instanceof Date ? value.toISOString() : value; +} + +function formatExportTimestamp(date: Date) { + const parts = [ + date.getFullYear(), + String(date.getMonth() + 1).padStart(2, '0'), + String(date.getDate()).padStart(2, '0'), + String(date.getHours()).padStart(2, '0'), + String(date.getMinutes()).padStart(2, '0'), + String(date.getSeconds()).padStart(2, '0'), + ]; + return `${parts[0]}${parts[1]}${parts[2]}-${parts[3]}${parts[4]}${parts[5]}`; +} + function summarizeMessageGroups(groups: Array<{ status: string; _count: { _all: number }; _sum: { amountCents: number | null; billingUnits: number | null } }>) { return groups.reduce( (summary, group) => { @@ -410,6 +871,51 @@ function summarizeMessageGroups(groups: Array<{ status: string; _count: { _all: ); } +function groupDownstreamByType( + groups: Array<{ deliveryType: string; status: string; _count: { _all: number } }>, +) { + return groups.reduce>((accumulator, item) => { + const current = accumulator[item.deliveryType] ?? { total: 0, pending: 0, delivered: 0, failed: 0 }; + current.total += item._count._all; + if (item.status === 'pending') { + current.pending += item._count._all; + } else if (item.status === 'delivered') { + current.delivered += item._count._all; + } else if (item.status === 'failed') { + current.failed += item._count._all; + } + accumulator[item.deliveryType] = current; + return accumulator; + }, {}); +} + +function groupDownstreamByApplication( + groups: Array<{ applicationId: string; status: string; _count: { _all: number } }>, + applicationMap: Map, +) { + const summaryMap = new Map(); + groups.forEach((item) => { + const current = summaryMap.get(item.applicationId) ?? { + applicationId: item.applicationId, + name: applicationMap.get(item.applicationId) ?? item.applicationId, + pending: 0, + failed: 0, + delivered: 0, + alertCount: 0, + }; + if (item.status === 'pending') { + current.pending += item._count._all; + } else if (item.status === 'failed') { + current.failed += item._count._all; + } else if (item.status === 'delivered') { + current.delivered += item._count._all; + } + current.alertCount = current.pending + current.failed; + summaryMap.set(item.applicationId, current); + }); + return [...summaryMap.values()]; +} + function normalizeOperationLog(log: Prisma.OperationLogGetPayload<{ include: { tenant: true; user: true } }>) { const detail = (log.detail ?? {}) as Record; const result = String(detail.result ?? detail.status ?? ''); diff --git a/api/src/send-chain/gateway-events.controller.ts b/api/src/send-chain/gateway-events.controller.ts index 3eea273..df3aa31 100644 --- a/api/src/send-chain/gateway-events.controller.ts +++ b/api/src/send-chain/gateway-events.controller.ts @@ -3,7 +3,10 @@ import { ApiTags } from '@nestjs/swagger'; import { GatewayInboundAuthDto, GatewayInboundSubmitDto, + GatewayPendingDeliveryQueryDto, + GatewayDownstreamRecoveryStatusDto, GatewayReceiptEventDto, + GatewaySubmitDeadLetterDto, GatewaySubmitResultDto, GatewayUplinkEventDto, SendChainService, @@ -29,6 +32,11 @@ export class GatewayEventsController { return this.sendChain.handleUplink(body); } + @Post('dead-letter') + deadLetter(@Body() body: GatewaySubmitDeadLetterDto) { + return this.sendChain.recordGatewaySubmitDeadLetter(body); + } + @Post('inbound/authenticate') authenticateInbound(@Body() body: GatewayInboundAuthDto) { return this.sendChain.authenticateInboundApplication(body); @@ -38,4 +46,24 @@ export class GatewayEventsController { submitInbound(@Body() body: GatewayInboundSubmitDto) { return this.sendChain.submitInboundMessage(body); } + + @Post('downstream/pending') + pendingDownstream(@Body() body: GatewayPendingDeliveryQueryDto) { + return this.sendChain.listPendingDownstreamDeliveries(body); + } + + @Post('downstream/delivered') + downstreamDelivered(@Body() body: { id: string }) { + return this.sendChain.markDownstreamDeliveryDelivered(body.id); + } + + @Post('downstream/failed') + downstreamFailed(@Body() body: { id: string; errorMessage?: string }) { + return this.sendChain.markDownstreamDeliveryFailed(body.id, body.errorMessage); + } + + @Post('downstream/recovery-status') + downstreamRecoveryStatus(@Body() body: GatewayDownstreamRecoveryStatusDto) { + return this.sendChain.recordGatewayDownstreamRecoveryStatus(body); + } } diff --git a/api/src/send-chain/send-chain.service.spec.ts b/api/src/send-chain/send-chain.service.spec.ts index 3e97559..779dcd7 100644 --- a/api/src/send-chain/send-chain.service.spec.ts +++ b/api/src/send-chain/send-chain.service.spec.ts @@ -33,6 +33,10 @@ function createPrismaMock() { status: 'active', carrier: 'mobile', sendRegion: '全国', + gatewayHost: '127.0.0.1', + gatewayPort: 17890, + passwordCipher: 'secret', + cmppVersion: '3.0', config: { serviceId: 'SMS' }, connectionStates: [{ status: 'connected', currentConnections: 1, desiredConnections: 1 }], }; @@ -60,6 +64,7 @@ function createPrismaMock() { }, smsApplication: { findUnique: jest.fn().mockResolvedValue({ id: 'app-1', tenantId: 'tenant-1', cmppAccount: '100001', status: 'active', customerUnitPrice: 3, queuePriority: 'normal' }), + findMany: jest.fn().mockResolvedValue([{ id: 'app-1', tenantId: 'tenant-1', name: '应用A' }]), findFirst: jest.fn().mockResolvedValue({ id: 'app-1', tenantId: 'tenant-1', @@ -109,6 +114,7 @@ function createPrismaMock() { }, channelRouteRule: { findFirst: jest.fn().mockResolvedValue(route), + findMany: jest.fn().mockResolvedValue([{ tenantId: 'tenant-1', applicationId: 'app-1' }]), }, phoneCarrierRule: { findMany: jest.fn().mockResolvedValue([{ carrier: 'mobile', pattern: '^13[4-9]', priority: 1, status: 'active' }]), @@ -126,8 +132,16 @@ function createPrismaMock() { smsSubmitRecord: { create: jest.fn().mockResolvedValue({ id: 'submit-1' }), updateMany: jest.fn().mockResolvedValue({ count: 1 }), + findFirst: jest.fn().mockResolvedValue({ id: 'submit-1', submitId: 'SUB-1', submitStatus: 'accepted', createdAt: new Date('2026-07-01T10:00:00.000Z') }), + findUnique: jest.fn().mockResolvedValue({ id: 'submit-1', submitId: 'SUB-1', submitStatus: 'accepted' }), + count: jest.fn().mockResolvedValue(1), findMany: jest.fn().mockResolvedValue([]), }, + smsMessageSegmentAudit: { + upsert: jest.fn().mockResolvedValue({ id: 'segment-1' }), + updateMany: jest.fn().mockResolvedValue({ count: 1 }), + findFirst: jest.fn().mockResolvedValue(null), + }, channelSignatureReportTask: { findFirst: jest.fn().mockResolvedValue({ id: 'report-task-1' }), }, @@ -138,6 +152,103 @@ function createPrismaMock() { smsUplinkMessage: { create: jest.fn().mockImplementation(({ data }) => Promise.resolve({ id: 'uplink-1', ...data })), findMany: jest.fn(), + findUnique: jest.fn().mockResolvedValue({ id: 'uplink-1', matchStatus: 'matched', matchCandidates: [] }), + update: jest.fn().mockImplementation(({ data }) => Promise.resolve({ id: 'uplink-1', ...data })), + }, + smsUplinkMatchCandidate: { + createMany: jest.fn().mockResolvedValue({ count: 1 }), + findFirst: jest.fn().mockResolvedValue({ + id: 'candidate-1', + uplinkMessageId: 'uplink-1', + tenantId: 'tenant-1', + applicationId: 'app-1', + messageRecordId: 'record-1', + matchSource: 'phone_window', + confidence: 55, + reason: '手机号 72 小时窗口候选下发 MSG-1', + status: 'pending', + application: { id: 'app-1', name: '应用A', cmppAccount: '100001' }, + messageRecord: { id: 'record-1', messageId: 'MSG-1', content: 'hello' }, + uplinkMessage: { + id: 'uplink-1', + tenantId: null, + applicationId: null, + messageRecordId: null, + messageId: null, + channelId: 'channel-1', + phoneNumber: '13800000001', + destId: '10690000', + content: '回复TD', + matchStatus: 'ambiguous', + receivedAt: new Date('2026-07-08T12:00:00.000Z'), + }, + }), + updateMany: jest.fn().mockResolvedValue({ count: 1 }), + update: jest.fn().mockResolvedValue({ id: 'candidate-1', status: 'claimed' }), + }, + cmppDownstreamDelivery: { + create: jest.fn().mockImplementation(({ data }) => Promise.resolve({ id: 'delivery-1', ...data, createdAt: new Date(), updatedAt: new Date() })), + findUnique: jest.fn().mockResolvedValue({ + id: 'delivery-1', + tenantId: 'tenant-1', + applicationId: 'app-1', + messageId: 'MSG-1', + deliveryType: 'receipt', + retryCount: 0, + lastError: null, + payload: { account: '100001', messageId: 'MSG-1', phoneNumber: '13800000001', receiptStatus: 'delivered' }, + application: { cmppAccount: '100001' }, + }), + findMany: jest.fn().mockResolvedValue([]), + update: jest.fn().mockImplementation(({ data }) => Promise.resolve({ id: 'delivery-1', tenantId: 'tenant-1', applicationId: 'app-1', messageId: 'MSG-1', deliveryType: 'receipt', ...data })), + }, + gatewaySubmitDeadLetter: { + upsert: jest.fn().mockResolvedValue({ id: 'dead-1' }), + findUnique: jest.fn().mockResolvedValue({ + id: 'dead-1', + tenantId: 'tenant-1', + streamMessageId: '1710000000000-0', + submitId: 'SUB-1', + messageId: 'MSG-1', + commandPayload: { + schemaVersion: 'v1', + messageType: 'SubmitCommand', + traceId: 'trace-1', + messageId: 'MSG-1', + channelId: 'channel-1', + createdAt: '2026-07-08T12:00:00.000Z', + tenantId: 'tenant-1', + applicationId: 'app-1', + submitId: 'SUB-1', + phoneNumber: '13800000001', + content: 'hello', + signature: '签名', + templateId: 'tpl-1', + billingUnits: 1, + queuePriority: 'normal', + route: { channelCode: 'CMPP-A', cmppAccountCode: 'account-a', priority: 0 }, + cmpp: { serviceId: 'SMS', srcId: '10690000', registeredDelivery: 1, msgFmt: 8 }, + upstream: { gatewayHost: '127.0.0.1', gatewayPort: 17890, account: 'account-a', passwordCipher: 'secret', cmppVersion: '3.0' }, + retry: { attempt: 0, maxAttempts: 1 }, + }, + }), + update: jest.fn().mockResolvedValue({ id: 'dead-1', tenantId: 'tenant-1', streamMessageId: '1710000000000-0', submitId: 'SUB-1', messageId: 'MSG-1' }), + updateMany: jest.fn().mockResolvedValue({ count: 1 }), + }, + gatewayDownstreamRecoveryStatus: { + upsert: jest.fn().mockResolvedValue({ + id: 'recover-1', + tenantId: 'tenant-1', + applicationId: 'app-1', + account: '100001', + state: 'waiting_connection', + lockOwner: 'gateway-a', + lockExpiresAt: new Date('2026-07-08T12:00:30.000Z'), + attemptCount: 2, + failureCategory: 'client_disconnected', + nextRetryAt: new Date('2026-07-08T12:10:00.000Z'), + lastError: 'downstream client is not connected', + }), }, smsBillingRecord: { findFirst: jest.fn().mockResolvedValue(null), @@ -148,12 +259,16 @@ function createPrismaMock() { accountTransaction: { findFirst: jest.fn().mockResolvedValue(null), }, + operationLog: { + create: jest.fn().mockResolvedValue({ id: 'log-1' }), + }, enterpriseBlacklist: { findMany: jest.fn().mockResolvedValue([]), }, globalBlacklist: { findMany: jest.fn().mockResolvedValue([]), }, + $transaction: jest.fn((operations) => Promise.all(operations)), }; } @@ -178,7 +293,10 @@ function createService(prisma = createPrismaMock()) { task: { id: 'risk-task-1' }, }), } as unknown as RiskReviewService; - return { service: new SendChainService(prisma as never, billing, riskReview), prisma, billing, riskReview }; + const service = new SendChainService(prisma as never, billing, riskReview); + service['postGatewayControl'] = jest.fn().mockResolvedValue({ delivered: true }); + service['publishGatewaySubmitCommand'] = jest.fn().mockResolvedValue(undefined); + return { service, prisma, billing, riskReview }; } describe('SendChainService', () => { @@ -371,8 +489,11 @@ describe('SendChainService', () => { phoneNumber: '13800000001', route: expect.objectContaining({ channelCode: 'CMPP-A', rateLimitPerSecond: 100 }), cmpp: expect.objectContaining({ serviceId: 'SMS', srcId: '10690000' }), + upstream: expect.objectContaining({ gatewayHost: '127.0.0.1', gatewayPort: 17890, account: 'cmpp-account' }), }), ); + expect(service['publishGatewaySubmitCommand']).toHaveBeenCalledWith(expect.objectContaining({ messageId: 'MSG-1' })); + expect(service['postGatewayControl']).not.toHaveBeenCalledWith('/upstream/submit', expect.anything()); }); it('updates submit result status, charges billing, and task progress', async () => { @@ -386,6 +507,24 @@ describe('SendChainService', () => { gatewayMessageId: 'GW-1', submitStatus: 'accepted', submittedAt: '2026-07-01T10:00:00.000Z', + segments: [ + { + segmentTotal: 2, + segmentIndex: 1, + sequenceId: 7, + gatewayMessageId: 'GW-1-A', + submitStatus: 'accepted', + submittedAt: '2026-07-01T10:00:00.000Z', + }, + { + segmentTotal: 2, + segmentIndex: 2, + sequenceId: 8, + gatewayMessageId: 'GW-1-B', + submitStatus: 'accepted', + submittedAt: '2026-07-01T10:00:01.000Z', + }, + ], }); expect(prisma.smsSubmitRecord.updateMany).toHaveBeenCalledWith({ @@ -401,6 +540,18 @@ describe('SendChainService', () => { expect(prisma.smsBillingRecord.create).toHaveBeenCalledWith({ data: expect.objectContaining({ messageId: 'MSG-1', amountCents: 3, billingStatus: 'charged', transactionId: 'tx-charge' }), }); + expect(prisma.smsMessageSegmentAudit.upsert).toHaveBeenCalledTimes(2); + expect(prisma.smsMessageSegmentAudit.upsert).toHaveBeenCalledWith(expect.objectContaining({ + where: { messageRecordId_submitId_segmentIndex: { messageRecordId: 'record-1', submitId: 'SUB-1', segmentIndex: 1 } }, + create: expect.objectContaining({ segmentTotal: 2, segmentIndex: 1, gatewayMessageId: 'GW-1-A', submitStatus: 'accepted' }), + })); + expect(prisma.gatewaySubmitDeadLetter.updateMany).toHaveBeenCalledWith({ + where: { + status: { in: ['pending', 'requeued'] }, + OR: [{ submitId: 'SUB-1' }, { messageId: 'MSG-1' }], + }, + data: expect.objectContaining({ status: 'resolved', resolvedStatus: 'accepted' }), + }); }); it('releases reservation for rejected submit result and refunds failed receipts', async () => { @@ -520,6 +671,88 @@ describe('SendChainService', () => { }); }); + it('matches receipt to a unique timed-out submit attempt when the upstream submit response was lost', async () => { + const { service, prisma } = createService(); + prisma.smsMessageRecord.findFirst.mockResolvedValue(null); + prisma.smsSubmitRecord.findMany.mockResolvedValue([ + { + id: 'submit-timeout-1', + channelId: 'channel-1', + gatewayMessageId: null, + submitStatus: 'timeout', + submittedAt: new Date('2026-07-01T10:00:00.000Z'), + messageRecord: { + id: 'record-1', + tenantId: 'tenant-1', + batchTaskId: 'task-1', + applicationId: 'app-1', + messageId: 'MSG-1', + phoneNumber: '13800000001', + channelId: 'channel-1', + gatewayMessageId: null, + status: 'timeout', + }, + }, + ]); + + await service.handleReceipt({ + messageId: 'receipt-123456789', + channelId: 'channel-1', + sequenceId: 7, + gatewayMessageId: 'GW-RECOVERED-1', + phoneNumber: '13800000001', + receiptStatus: 'delivered', + rawStatus: 'DELIVRD', + deliveredAt: '2026-07-01T10:01:00.000Z', + }); + + expect(prisma.smsSubmitRecord.updateMany).toHaveBeenCalledWith({ + where: { + id: 'submit-timeout-1', + gatewayMessageId: null, + }, + data: { + gatewayMessageId: 'GW-RECOVERED-1', + sequenceId: 7, + }, + }); + expect(prisma.smsReceiptRecord.create).toHaveBeenCalledWith({ + data: expect.objectContaining({ + messageRecordId: 'record-1', + messageId: 'MSG-1', + gatewayMessageId: 'GW-RECOVERED-1', + }), + }); + }); + + it('rejects ambiguous receipt heuristic matches to avoid binding to the wrong message', async () => { + const { service, prisma } = createService(); + prisma.smsMessageRecord.findFirst.mockResolvedValue(null); + prisma.smsSubmitRecord.findMany.mockResolvedValue([ + { + id: 'submit-timeout-1', + messageRecord: { id: 'record-1', messageId: 'MSG-1', phoneNumber: '13800000001' }, + }, + { + id: 'submit-timeout-2', + messageRecord: { id: 'record-2', messageId: 'MSG-2', phoneNumber: '13800000001' }, + }, + ]); + + await expect( + service.handleReceipt({ + messageId: 'receipt-ambiguous', + channelId: 'channel-1', + gatewayMessageId: 'GW-AMBIGUOUS', + phoneNumber: '13800000001', + receiptStatus: 'delivered', + rawStatus: 'DELIVRD', + }), + ).rejects.toThrow('SMS message record not found'); + + expect(prisma.smsReceiptRecord.create).not.toHaveBeenCalled(); + }); + it('blocks submit when signature is not approved on the selected channel', async () => { const { service, prisma } = createService(); const gatewayAdd = jest.fn().mockResolvedValue(undefined); @@ -635,6 +868,301 @@ describe('SendChainService', () => { }); }); + it('records ambiguous uplink match candidates for shared access numbers', async () => { + const { service, prisma } = createService(); + prisma.channelRouteRule.findMany.mockResolvedValue([ + { applicationId: 'app-1' }, + { applicationId: 'app-2' }, + ]); + prisma.smsApplication.findMany.mockResolvedValue([ + { id: 'app-1', tenantId: 'tenant-1', name: '应用A' }, + { id: 'app-2', tenantId: 'tenant-2', name: '应用B' }, + ]); + + await service.handleUplink({ + channelId: 'channel-1', + sequenceId: 8, + phoneNumber: '13800000001', + destId: '10690000', + content: '回复TD', + receivedAt: '2026-07-01T10:02:00.000Z', + }); + + expect(prisma.smsUplinkMessage.create).toHaveBeenCalledWith({ + data: expect.objectContaining({ + tenantId: undefined, + applicationId: undefined, + matchStatus: 'ambiguous', + matchReason: '接入号匹配多个应用', + }), + }); + expect(prisma.smsUplinkMatchCandidate.createMany).toHaveBeenCalledWith({ + data: [ + expect.objectContaining({ tenantId: 'tenant-1', applicationId: 'app-1', matchSource: 'access_number', confidence: 70 }), + expect.objectContaining({ tenantId: 'tenant-2', applicationId: 'app-2', matchSource: 'access_number', confidence: 70 }), + ], + skipDuplicates: true, + }); + }); + + it('claims an ambiguous uplink candidate and queues downstream uplink delivery', async () => { + const { service, prisma } = createService(); + service['postGatewayControl'] = jest.fn().mockResolvedValue({ delivered: true }); + + await service.claimUplinkMatchCandidate('uplink-1', 'candidate-1', 'admin-1'); + + expect(prisma.smsUplinkMessage.update).toHaveBeenCalledWith({ + where: { id: 'uplink-1' }, + data: expect.objectContaining({ + tenantId: 'tenant-1', + applicationId: 'app-1', + messageRecordId: 'record-1', + messageId: 'MSG-1', + matchStatus: 'matched', + }), + }); + expect(prisma.smsUplinkMatchCandidate.updateMany).toHaveBeenCalledWith({ + where: { uplinkMessageId: 'uplink-1', id: { not: 'candidate-1' }, status: 'pending' }, + data: { status: 'rejected' }, + }); + expect(prisma.cmppDownstreamDelivery.create).toHaveBeenCalledWith({ + data: expect.objectContaining({ + tenantId: 'tenant-1', + applicationId: 'app-1', + messageRecordId: 'record-1', + messageId: 'MSG-1', + deliveryType: 'uplink', + status: 'pending', + }), + }); + }); + + it('records gateway submit dead letters and allows manual requeue', async () => { + const { service, prisma } = createService(); + service['publishGatewaySubmitCommand'] = jest.fn().mockResolvedValue('1710000001000-0'); + + await service.recordGatewaySubmitDeadLetter({ + streamMessageId: '1710000000000-0', + tenantId: 'tenant-1', + applicationId: 'app-1', + channelId: 'channel-1', + traceId: 'trace-1', + messageId: 'MSG-1', + submitId: 'SUB-1', + failureCode: 'SUBMIT_PROCESSING_FAILED', + failureMessage: 'network down', + attempts: 3, + maxAttempts: 3, + commandPayload: { messageType: 'SubmitCommand', submitId: 'SUB-1' }, + rawPayload: '{"messageType":"SubmitCommand"}', + deadLetteredAt: '2026-07-08T12:00:00.000Z', + }); + + expect(prisma.gatewaySubmitDeadLetter.upsert).toHaveBeenCalledWith({ + where: { streamMessageId: '1710000000000-0' }, + update: expect.objectContaining({ + submitId: 'SUB-1', + failureCode: 'SUBMIT_PROCESSING_FAILED', + attempts: 3, + }), + create: expect.objectContaining({ + streamMessageId: '1710000000000-0', + failureMessage: 'network down', + }), + }); + + await service.requeueGatewaySubmitDeadLetter('dead-1'); + + expect(service['publishGatewaySubmitCommand']).toHaveBeenCalledWith(expect.objectContaining({ submitId: 'SUB-1' })); + expect(prisma.gatewaySubmitDeadLetter.update).toHaveBeenCalledWith({ + where: { id: 'dead-1' }, + data: expect.objectContaining({ + status: 'requeued', + manualRetryCount: { increment: 1 }, + lastRetryStreamId: '1710000001000-0', + }), + }); + expect(prisma.operationLog.create).toHaveBeenCalledWith({ + data: expect.objectContaining({ + action: 'gateway.submit_dead_letter_requeue', + resource: 'gateway_submit_dead_letter', + resourceId: 'dead-1', + }), + }); + }); + + it('records gateway downstream recovery statuses', async () => { + const { service, prisma } = createService(); + + await expect(service.recordGatewayDownstreamRecoveryStatus({ + account: '100001', + gatewayInstanceId: 'gateway-a', + state: 'waiting_connection', + lastAttemptAt: '2026-07-08T12:00:00.000Z', + nextRetryAt: '2026-07-08T12:10:00.000Z', + attemptCount: 2, + lockOwner: 'gateway-a', + lockExpiresAt: '2026-07-08T12:00:30.000Z', + lastError: 'downstream client is not connected', + })).resolves.toEqual(expect.objectContaining({ + id: 'recover-1', + account: '100001', + state: 'waiting_connection', + })); + + expect(prisma.gatewayDownstreamRecoveryStatus.upsert).toHaveBeenCalledWith(expect.objectContaining({ + where: { account: '100001' }, + update: expect.objectContaining({ + lockOwner: 'gateway-a', + lockExpiresAt: new Date('2026-07-08T12:00:30.000Z'), + failureCategory: 'client_disconnected', + }), + create: expect.objectContaining({ + lockOwner: 'gateway-a', + lockExpiresAt: new Date('2026-07-08T12:00:30.000Z'), + failureCategory: 'client_disconnected', + }), + })); + expect(prisma.operationLog.create).toHaveBeenCalledWith(expect.objectContaining({ + data: expect.objectContaining({ + action: 'gateway.downstream_recovery_status_sync', + resource: 'gateway_downstream_recovery_status', + }), + })); + }); + + it('marks downstream delivery as failed after reaching retry limit', async () => { + const { service, prisma } = createService(); + const previous = process.env.CMPP_DOWNSTREAM_MAX_RETRIES; + process.env.CMPP_DOWNSTREAM_MAX_RETRIES = '2'; + prisma.cmppDownstreamDelivery.findUnique.mockResolvedValue({ + id: 'delivery-1', + tenantId: 'tenant-1', + applicationId: 'app-1', + messageId: 'MSG-1', + deliveryType: 'receipt', + retryCount: 1, + lastError: null, + }); + + try { + await service.markDownstreamDeliveryFailed('delivery-1', 'client offline'); + } finally { + if (previous === undefined) { + delete process.env.CMPP_DOWNSTREAM_MAX_RETRIES; + } else { + process.env.CMPP_DOWNSTREAM_MAX_RETRIES = previous; + } + } + + expect(prisma.cmppDownstreamDelivery.update).toHaveBeenCalledWith({ + where: { id: 'delivery-1' }, + data: expect.objectContaining({ + status: 'failed', + retryCount: 2, + nextRetryAt: null, + lastError: 'client offline', + }), + }); + expect(prisma.operationLog.create).toHaveBeenCalledWith({ + data: expect.objectContaining({ + action: 'gateway.downstream_delivery_failed', + resource: 'cmpp_downstream_delivery', + resourceId: 'delivery-1', + }), + }); + }); + + it('uses exponential backoff for downstream delivery retries before final failure', async () => { + const { service, prisma } = createService(); + const previousBase = process.env.CMPP_DOWNSTREAM_RETRY_DELAY_MS; + const previousMax = process.env.CMPP_DOWNSTREAM_RETRY_MAX_DELAY_MS; + const now = Date.UTC(2026, 6, 8, 12, 0, 0); + const dateNowSpy = jest.spyOn(Date, 'now').mockReturnValue(now); + process.env.CMPP_DOWNSTREAM_RETRY_DELAY_MS = '60000'; + process.env.CMPP_DOWNSTREAM_RETRY_MAX_DELAY_MS = '600000'; + prisma.cmppDownstreamDelivery.findUnique.mockResolvedValue({ + id: 'delivery-1', + tenantId: 'tenant-1', + applicationId: 'app-1', + messageId: 'MSG-1', + deliveryType: 'receipt', + retryCount: 2, + lastError: null, + }); + + try { + await service.markDownstreamDeliveryFailed('delivery-1', 'temporary network jitter'); + } finally { + dateNowSpy.mockRestore(); + if (previousBase === undefined) { + delete process.env.CMPP_DOWNSTREAM_RETRY_DELAY_MS; + } else { + process.env.CMPP_DOWNSTREAM_RETRY_DELAY_MS = previousBase; + } + if (previousMax === undefined) { + delete process.env.CMPP_DOWNSTREAM_RETRY_MAX_DELAY_MS; + } else { + process.env.CMPP_DOWNSTREAM_RETRY_MAX_DELAY_MS = previousMax; + } + } + + expect(prisma.cmppDownstreamDelivery.update).toHaveBeenCalledWith({ + where: { id: 'delivery-1' }, + data: expect.objectContaining({ + status: 'pending', + retryCount: 3, + nextRetryAt: new Date(now + 240_000), + lastError: 'temporary network jitter', + }), + }); + }); + + it('requeues downstream delivery through real gateway control path', async () => { + const { service, prisma } = createService(); + service['postGatewayControl'] = jest.fn().mockResolvedValue({ delivered: true }); + + await service.requeueDownstreamDelivery('delivery-1'); + + expect(service['postGatewayControl']).toHaveBeenCalledWith( + '/downstream/receipt', + expect.objectContaining({ + deliveryId: 'delivery-1', + account: '100001', + messageId: 'MSG-1', + }), + ); + expect(prisma.operationLog.create).toHaveBeenCalledWith({ + data: expect.objectContaining({ + action: 'gateway.downstream_delivery_requeue', + resource: 'cmpp_downstream_delivery', + resourceId: 'delivery-1', + }), + }); + }); + + it('supports batch requeue of downstream deliveries', async () => { + const { service } = createService(); + service.requeueDownstreamDelivery = jest.fn() + .mockResolvedValueOnce({ id: 'delivery-1' }) + .mockRejectedValueOnce(new Error('Gateway control delivery failed')); + + await expect(service.batchRequeueDownstreamDeliveries(['delivery-1', 'delivery-2'])).resolves.toEqual({ + total: 2, + successCount: 1, + failedCount: 1, + results: [ + { id: 'delivery-1', status: 'success' }, + { id: 'delivery-2', status: 'failed', errorMessage: 'Gateway control delivery failed' }, + ], + }); + }); + + it('rejects empty downstream batch requeue selection', async () => { + const { service } = createService(); + await expect(service.batchRequeueDownstreamDeliveries([])).rejects.toThrow('请选择至少一条下游投递记录'); + }); + it('marks 72 hour unknown receipts as timeout', async () => { const { service, prisma } = createService(); prisma.smsMessageRecord.findMany.mockResolvedValue([ diff --git a/api/src/send-chain/send-chain.service.ts b/api/src/send-chain/send-chain.service.ts index 6dba57f..9f1c4de 100644 --- a/api/src/send-chain/send-chain.service.ts +++ b/api/src/send-chain/send-chain.service.ts @@ -1,4 +1,5 @@ import { BadRequestException, Injectable, NotFoundException, OnModuleDestroy, OnModuleInit } from '@nestjs/common'; +import { Prisma } from '@prisma/client'; import { Queue, Worker } from 'bullmq'; import IORedis from 'ioredis'; import { randomUUID } from 'node:crypto'; @@ -54,14 +55,25 @@ export interface GatewaySubmitResultDto { errorCode?: string; errorMessage?: string; submittedAt?: string; + segments?: Array<{ + segmentTotal?: number; + segmentIndex?: number; + sequenceId?: number; + gatewayMessageId?: string; + submitStatus?: 'accepted' | 'rejected' | 'timeout' | string; + errorCode?: string; + errorMessage?: string; + submittedAt?: string; + }>; } export interface GatewayReceiptEventDto { traceId?: string; - messageId: string; + messageId?: string; channelId: string; sequenceId?: number; gatewayMessageId: string; + phoneNumber?: string; receiptStatus: 'delivered' | 'undelivered' | 'unknown'; rawStatus: string; errorCode?: string; @@ -79,6 +91,53 @@ export interface GatewayUplinkEventDto { receivedAt?: string; } +type UplinkMatchCandidateInput = { + tenantId: string; + applicationId: string; + messageRecordId?: string; + matchSource: 'access_number' | 'phone_window'; + confidence: number; + reason: string; +}; + +export interface GatewayPendingDeliveryQueryDto { + account: string; + limit?: number; +} + +export interface GatewaySubmitDeadLetterDto { + streamMessageId: string; + traceId?: string; + messageId?: string; + channelId?: string; + tenantId?: string; + applicationId?: string; + submitId?: string; + failureCode: string; + failureMessage: string; + attempts: number; + maxAttempts: number; + commandPayload?: Record; + rawPayload?: string; + deadLetteredAt?: string; +} + +export interface GatewayDownstreamRecoveryStatusDto { + account: string; + gatewayInstanceId?: string; + state: string; + lockOwner?: string; + lockExpiresAt?: string; + lastAttemptAt?: string; + lastSuccessAt?: string; + lastFailureAt?: string; + nextRetryAt?: string; + attemptCount?: number; + failureCategory?: string; + lastError?: string; + lastSkipReason?: string; +} + export interface TimeoutUnknownDto { olderThanHours?: number; } @@ -114,6 +173,10 @@ type RoutedChannel = { status: string; carrier?: string | null; sendRegion: string; + gatewayHost: string; + gatewayPort: number; + passwordCipher: string; + cmppVersion: string; config?: unknown; }; carrier: string; @@ -124,6 +187,10 @@ type RoutedChannel = { const SEND_QUEUE = 'sms.send.queue'; const GATEWAY_SUBMIT_QUEUE = 'gateway.submit.queue'; +const GATEWAY_SUBMIT_STREAM = 'gateway.submit.commands'; +const DEFAULT_DOWNSTREAM_RETRY_DELAY_MS = 60_000; +const DEFAULT_DOWNSTREAM_RETRY_MAX_DELAY_MS = 30 * 60_000; +const DEFAULT_DOWNSTREAM_MAX_RETRIES = 10; const BULLMQ_PRIORITY: Record = { priority: 1, normal: 100, @@ -312,6 +379,7 @@ export class SendChainService implements OnModuleInit, OnModuleDestroy { listUplinkMessages(tenantId?: string, channelId?: string) { return this.prisma.smsUplinkMessage.findMany({ where: { tenantId, channelId }, + include: { application: true, channel: true, messageRecord: { include: { application: true } } }, orderBy: { receivedAt: 'desc' }, take: 200, }); @@ -537,7 +605,7 @@ export class SendChainService implements OnModuleInit, OnModuleDestroy { } async handleSubmitResult(data: GatewaySubmitResultDto) { - const message = await this.findMessageByGatewayEvent(data.messageId, data.gatewayMessageId); + const message = await this.requireMessageByGatewayEvent(data.messageId, data.gatewayMessageId); const submittedAt = data.submittedAt ? new Date(data.submittedAt) : new Date(); await this.prisma.smsSubmitRecord.updateMany({ where: { OR: [{ submitId: data.submitId }, { messageRecordId: message.id }] }, @@ -550,6 +618,7 @@ export class SendChainService implements OnModuleInit, OnModuleDestroy { submittedAt, }, }); + await this.recordSubmitSegments(message, data, submittedAt); if (data.submitId && message.submitId && data.submitId !== message.submitId) { return this.prisma.smsMessageRecord.findUnique({ where: { id: message.id } }); } @@ -576,22 +645,49 @@ export class SendChainService implements OnModuleInit, OnModuleDestroy { timeoutAt: data.submitStatus === 'timeout' ? submittedAt : undefined, }, }); + await this.prisma.gatewaySubmitDeadLetter.updateMany({ + where: { + status: { in: ['pending', 'requeued'] }, + OR: [ + data.submitId ? { submitId: data.submitId } : undefined, + data.messageId ? { messageId: data.messageId } : undefined, + ].filter(Boolean) as Array<{ submitId?: string; messageId?: string }>, + }, + data: { + status: 'resolved', + resolvedAt: submittedAt, + resolvedStatus: data.submitStatus, + }, + }); await this.refreshTaskProgress(message.batchTaskId); return this.prisma.smsMessageRecord.findUnique({ where: { id: message.id } }); } async handleReceipt(data: GatewayReceiptEventDto) { - const message = await this.findMessageByGatewayEvent(data.messageId, data.gatewayMessageId); + const resolved = await this.resolveReceiptMessage(data); + const message = resolved.message; const deliveredAt = data.deliveredAt ? new Date(data.deliveredAt) : new Date(); const status = data.receiptStatus === 'delivered' ? 'delivered' : data.receiptStatus === 'unknown' ? 'unknown' : 'failed'; + if (resolved.submitRecordId) { + await this.prisma.smsSubmitRecord.updateMany({ + where: { + id: resolved.submitRecordId, + gatewayMessageId: null, + }, + data: { + gatewayMessageId: data.gatewayMessageId, + sequenceId: data.sequenceId, + }, + }); + } await this.prisma.smsReceiptRecord.create({ data: { tenantId: message.tenantId, batchTaskId: message.batchTaskId, messageRecordId: message.id, channelId: data.channelId, - messageId: data.messageId, + messageId: resolved.messageId, gatewayMessageId: data.gatewayMessageId, sequenceId: data.sequenceId, receiptStatus: data.receiptStatus, @@ -600,6 +696,7 @@ export class SendChainService implements OnModuleInit, OnModuleDestroy { deliveredAt, }, }); + await this.recordReceiptSegment(message, data, deliveredAt, resolved.submitRecordId); const isCurrentAttempt = (!message.channelId || message.channelId === data.channelId) && (!message.gatewayMessageId || message.gatewayMessageId === data.gatewayMessageId); @@ -623,6 +720,22 @@ export class SendChainService implements OnModuleInit, OnModuleDestroy { deliveredAt, }, }); + await this.queueAndTryDownstreamDelivery({ + tenantId: message.tenantId, + applicationId: message.applicationId, + messageRecordId: message.id, + messageId: message.messageId, + deliveryType: 'receipt', + payload: { + messageId: message.messageId, + gatewayMessageId: data.gatewayMessageId, + phoneNumber: message.phoneNumber, + receiptStatus: data.receiptStatus, + rawStatus: data.rawStatus, + errorCode: data.errorCode, + deliveredAt: deliveredAt.toISOString(), + }, + }); await this.refreshTaskProgress(message.batchTaskId); return this.prisma.smsMessageRecord.findUnique({ where: { id: message.id } }); } @@ -632,21 +745,613 @@ export class SendChainService implements OnModuleInit, OnModuleDestroy { if (!channel) { throw new NotFoundException('SMS channel not found'); } - const message = data.messageId - ? await this.prisma.smsMessageRecord.findUnique({ where: { messageId: data.messageId } }) - : null; - return this.prisma.smsUplinkMessage.create({ + const match = await this.resolveUplinkMatch(data, channel); + const record = await this.prisma.smsUplinkMessage.create({ data: { - tenantId: message?.tenantId, + tenantId: match.tenantId, + applicationId: match.applicationId, + messageRecordId: match.messageRecordId, channelId: data.channelId, messageId: data.messageId, sequenceId: data.sequenceId, phoneNumber: data.phoneNumber, destId: data.destId, content: data.content, + matchStatus: match.matchStatus, + matchReason: match.matchReason, receivedAt: data.receivedAt ? new Date(data.receivedAt) : new Date(), }, }); + if (match.candidates.length > 0) { + await this.prisma.smsUplinkMatchCandidate.createMany({ + data: match.candidates.map((candidate) => ({ + uplinkMessageId: record.id, + tenantId: candidate.tenantId, + applicationId: candidate.applicationId, + messageRecordId: candidate.messageRecordId, + matchSource: candidate.matchSource, + confidence: candidate.confidence, + reason: candidate.reason, + })), + skipDuplicates: true, + }); + } + if (match.tenantId && match.applicationId) { + await this.queueAndTryDownstreamDelivery({ + tenantId: match.tenantId, + applicationId: match.applicationId, + messageRecordId: match.messageRecordId, + messageId: data.messageId, + deliveryType: 'uplink', + payload: { + messageId: data.messageId, + applicationId: match.applicationId, + phoneNumber: data.phoneNumber, + destId: data.destId, + content: data.content, + receivedAt: record.receivedAt.toISOString(), + }, + }); + } + return record; + } + + async listPendingDownstreamDeliveries(data: GatewayPendingDeliveryQueryDto) { + const application = await this.findInboundApplication(data.account); + if (!application || application.status !== 'active' || application.tenant.status !== 'active') { + throw new BadRequestException('CMPP account is invalid or disabled'); + } + return this.prisma.cmppDownstreamDelivery.findMany({ + where: { + applicationId: application.id, + status: 'pending', + OR: [{ nextRetryAt: null }, { nextRetryAt: { lte: new Date() } }], + }, + orderBy: { createdAt: 'asc' }, + take: Math.min(Math.max(data.limit ?? 100, 1), 500), + }); + } + + async markDownstreamDeliveryDelivered(id: string) { + return this.prisma.cmppDownstreamDelivery.update({ + where: { id }, + data: { + status: 'delivered', + deliveredAt: new Date(), + lastError: null, + }, + }); + } + + async markDownstreamDeliveryFailed(id: string, errorMessage?: string) { + const delivery = await this.prisma.cmppDownstreamDelivery.findUnique({ where: { id } }); + if (!delivery) { + throw new NotFoundException('Downstream delivery not found'); + } + const retryCount = (delivery.retryCount ?? 0) + 1; + const finalFailure = retryCount >= downstreamMaxRetries(); + const updated = await this.prisma.cmppDownstreamDelivery.update({ + where: { id }, + data: { + status: finalFailure ? 'failed' : 'pending', + retryCount, + nextRetryAt: finalFailure ? null : new Date(Date.now() + downstreamRetryDelayMs(retryCount)), + lastError: errorMessage ?? 'downstream delivery failed', + }, + }); + if (finalFailure) { + await this.prisma.operationLog.create({ + data: { + tenantId: updated.tenantId, + action: 'gateway.downstream_delivery_failed', + resource: 'cmpp_downstream_delivery', + resourceId: updated.id, + detail: { + deliveryType: updated.deliveryType, + applicationId: updated.applicationId, + messageId: updated.messageId, + retryCount, + errorMessage: updated.lastError, + }, + }, + }); + } + return updated; + } + + async recordGatewaySubmitDeadLetter(data: GatewaySubmitDeadLetterDto) { + const createdAt = data.deadLetteredAt ? new Date(data.deadLetteredAt) : new Date(); + return this.prisma.gatewaySubmitDeadLetter.upsert({ + where: { streamMessageId: data.streamMessageId }, + update: { + tenantId: data.tenantId, + applicationId: data.applicationId, + channelId: data.channelId, + traceId: data.traceId, + messageId: data.messageId, + submitId: data.submitId, + status: 'pending', + failureCode: data.failureCode, + failureMessage: data.failureMessage, + attempts: data.attempts, + maxAttempts: data.maxAttempts, + commandPayload: data.commandPayload as Prisma.InputJsonValue | undefined, + rawPayload: data.rawPayload, + resolvedAt: null, + resolvedStatus: null, + }, + create: { + streamMessageId: data.streamMessageId, + tenantId: data.tenantId, + applicationId: data.applicationId, + channelId: data.channelId, + traceId: data.traceId, + messageId: data.messageId, + submitId: data.submitId, + failureCode: data.failureCode, + failureMessage: data.failureMessage, + attempts: data.attempts, + maxAttempts: data.maxAttempts, + commandPayload: data.commandPayload as Prisma.InputJsonValue | undefined, + rawPayload: data.rawPayload, + createdAt, + }, + }); + } + + async recordGatewayDownstreamRecoveryStatus(data: GatewayDownstreamRecoveryStatusDto) { + const account = String(data.account ?? '').trim(); + if (!account) { + throw new BadRequestException('account is required'); + } + const recoveryStatuses = (this.prisma as PrismaService & { + gatewayDownstreamRecoveryStatus: { + upsert: (args: Record) => Promise; + }; + }).gatewayDownstreamRecoveryStatus; + const application = await this.prisma.smsApplication.findUnique({ + where: { cmppAccount: account }, + select: { id: true, tenantId: true, name: true }, + }); + const failureCategory = normalizeRecoveryFailureCategory(data); + const updated = await recoveryStatuses.upsert({ + where: { account }, + update: { + tenantId: application?.tenantId ?? null, + applicationId: application?.id ?? null, + gatewayInstanceId: data.gatewayInstanceId ?? null, + state: data.state, + lockOwner: data.lockOwner ?? null, + lockExpiresAt: asDateOrNull(data.lockExpiresAt), + lastAttemptAt: asDateOrNull(data.lastAttemptAt), + lastSuccessAt: asDateOrNull(data.lastSuccessAt), + lastFailureAt: asDateOrNull(data.lastFailureAt), + nextRetryAt: asDateOrNull(data.nextRetryAt), + attemptCount: Number.isFinite(Number(data.attemptCount)) ? Number(data.attemptCount) : 0, + failureCategory, + lastError: data.lastError ?? null, + lastSkipReason: data.lastSkipReason ?? null, + }, + create: { + account, + tenantId: application?.tenantId, + applicationId: application?.id, + gatewayInstanceId: data.gatewayInstanceId, + state: data.state, + lockOwner: data.lockOwner, + lockExpiresAt: asDateOrNull(data.lockExpiresAt), + lastAttemptAt: asDateOrNull(data.lastAttemptAt), + lastSuccessAt: asDateOrNull(data.lastSuccessAt), + lastFailureAt: asDateOrNull(data.lastFailureAt), + nextRetryAt: asDateOrNull(data.nextRetryAt), + attemptCount: Number.isFinite(Number(data.attemptCount)) ? Number(data.attemptCount) : 0, + failureCategory, + lastError: data.lastError, + lastSkipReason: data.lastSkipReason, + }, + include: { + tenant: true, + application: true, + }, + }); + const normalizedUpdated = updated as typeof updated & { + failureCategory?: string | null; + lockOwner?: string | null; + lockExpiresAt?: Date | null; + }; + await this.prisma.operationLog.create({ + data: { + tenantId: updated.tenantId ?? undefined, + action: 'gateway.downstream_recovery_status_sync', + resource: 'gateway_downstream_recovery_status', + resourceId: updated.id, + detail: { + account, + state: updated.state, + lockOwner: normalizedUpdated.lockOwner, + lockExpiresAt: normalizedUpdated.lockExpiresAt, + attemptCount: updated.attemptCount, + nextRetryAt: updated.nextRetryAt, + failureCategory: normalizedUpdated.failureCategory, + applicationId: updated.applicationId, + applicationName: application?.name, + lastError: updated.lastError, + lastSkipReason: updated.lastSkipReason, + }, + }, + }); + return updated; + } + + async requeueGatewaySubmitDeadLetter(id: string) { + const deadLetter = await this.prisma.gatewaySubmitDeadLetter.findUnique({ where: { id } }); + if (!deadLetter) { + throw new NotFoundException('Gateway submit dead letter not found'); + } + if (!deadLetter.commandPayload || typeof deadLetter.commandPayload !== 'object') { + throw new BadRequestException('该死信缺少可重放的 SubmitCommand'); + } + const retryStreamMessageId = await this.publishGatewaySubmitCommand(deadLetter.commandPayload); + const updated = await this.prisma.gatewaySubmitDeadLetter.update({ + where: { id }, + data: { + status: 'requeued', + manualRetryCount: { increment: 1 }, + lastRetryStreamId: retryStreamMessageId, + lastRetriedAt: new Date(), + }, + }); + await this.prisma.operationLog.create({ + data: { + tenantId: updated.tenantId ?? undefined, + action: 'gateway.submit_dead_letter_requeue', + resource: 'gateway_submit_dead_letter', + resourceId: updated.id, + detail: { + streamMessageId: updated.streamMessageId, + retryStreamMessageId, + submitId: updated.submitId, + messageId: updated.messageId, + }, + }, + }); + return updated; + } + + async requeueDownstreamDelivery(id: string) { + const delivery = await this.prisma.cmppDownstreamDelivery.findUnique({ + where: { id }, + include: { application: { select: { cmppAccount: true } } }, + }); + if (!delivery) { + throw new NotFoundException('Downstream delivery not found'); + } + const payload = isObjectRecord(delivery.payload) ? { ...delivery.payload } : null; + if (!payload) { + throw new BadRequestException('下游投递记录缺少可重放 payload'); + } + const path = + delivery.deliveryType === 'receipt' + ? '/downstream/receipt' + : delivery.deliveryType === 'uplink' + ? '/downstream/uplink' + : null; + if (!path) { + throw new BadRequestException(`Unsupported downstream delivery type ${delivery.deliveryType}`); + } + + await this.prisma.operationLog.create({ + data: { + tenantId: delivery.tenantId, + action: 'gateway.downstream_delivery_requeue', + resource: 'cmpp_downstream_delivery', + resourceId: delivery.id, + detail: { + deliveryType: delivery.deliveryType, + applicationId: delivery.applicationId, + messageId: delivery.messageId, + }, + }, + }); + + const requestPayload = { + deliveryId: delivery.id, + account: String(payload.account ?? delivery.application?.cmppAccount ?? ''), + ...payload, + }; + try { + const result = await this.postGatewayControl(path, requestPayload) as { delivered?: boolean }; + if (result.delivered) { + return this.markDownstreamDeliveryDelivered(delivery.id); + } + return this.markDownstreamDeliveryFailed(delivery.id, 'downstream client is not connected'); + } catch (error) { + return this.markDownstreamDeliveryFailed( + delivery.id, + error instanceof Error ? error.message : 'Gateway control delivery failed', + ); + } + } + + async batchRequeueDownstreamDeliveries(ids: string[]) { + const uniqueIds = [...new Set(ids.filter(Boolean))]; + if (uniqueIds.length === 0) { + throw new BadRequestException('请选择至少一条下游投递记录'); + } + const results: Array<{ id: string; status: 'success' | 'failed'; errorMessage?: string }> = []; + for (const id of uniqueIds) { + try { + await this.requeueDownstreamDelivery(id); + results.push({ id, status: 'success' }); + } catch (error) { + results.push({ + id, + status: 'failed', + errorMessage: error instanceof Error ? error.message : '批量重投失败', + }); + } + } + return { + total: uniqueIds.length, + successCount: results.filter((item) => item.status === 'success').length, + failedCount: results.filter((item) => item.status === 'failed').length, + results, + }; + } + + async claimUplinkMatchCandidate(uplinkMessageId: string, candidateId: string, operatorId?: string) { + const candidate = await this.prisma.smsUplinkMatchCandidate.findFirst({ + where: { id: candidateId, uplinkMessageId }, + include: { + application: { select: { id: true, name: true, cmppAccount: true } }, + messageRecord: { select: { id: true, messageId: true, content: true } }, + uplinkMessage: true, + }, + }); + if (!candidate) { + throw new NotFoundException('Uplink match candidate not found'); + } + if (candidate.status === 'rejected') { + throw new BadRequestException('该候选已被排除，不能认领'); + } + if (candidate.uplinkMessage.matchStatus === 'matched' && candidate.status !== 'claimed') { + throw new BadRequestException('该上行记录已完成匹配，不能重复认领'); + } + const claimedAt = new Date(); + const messageId = candidate.uplinkMessage.messageId ?? candidate.messageRecord?.messageId ?? null; + const [updatedUplink] = await this.prisma.$transaction([ + this.prisma.smsUplinkMessage.update({ + where: { id: uplinkMessageId }, + data: { + tenantId: candidate.tenantId, + applicationId: candidate.applicationId, + messageRecordId: candidate.messageRecordId, + messageId, + matchStatus: 'matched', + matchReason: `人工认领：${candidate.reason ?? candidate.matchSource}`, + }, + }), + this.prisma.smsUplinkMatchCandidate.updateMany({ + where: { + uplinkMessageId, + id: { not: candidate.id }, + status: 'pending', + }, + data: { status: 'rejected' }, + }), + this.prisma.smsUplinkMatchCandidate.update({ + where: { id: candidate.id }, + data: { + status: 'claimed', + claimedAt, + claimedById: operatorId, + }, + }), + this.prisma.operationLog.create({ + data: { + tenantId: candidate.tenantId, + userId: operatorId, + action: 'gateway.uplink_manual_claim', + resource: 'sms_uplink_message', + resourceId: uplinkMessageId, + detail: { + candidateId: candidate.id, + applicationId: candidate.applicationId, + applicationName: candidate.application.name, + messageRecordId: candidate.messageRecordId, + messageId, + matchSource: candidate.matchSource, + phoneNumber: candidate.uplinkMessage.phoneNumber, + destId: candidate.uplinkMessage.destId, + }, + }, + }), + ]); + + await this.queueAndTryDownstreamDelivery({ + tenantId: candidate.tenantId, + applicationId: candidate.applicationId, + messageRecordId: candidate.messageRecordId, + messageId, + deliveryType: 'uplink', + payload: { + messageId, + applicationId: candidate.applicationId, + phoneNumber: candidate.uplinkMessage.phoneNumber, + destId: candidate.uplinkMessage.destId, + content: candidate.uplinkMessage.content, + receivedAt: candidate.uplinkMessage.receivedAt.toISOString(), + manualClaim: true, + uplinkMessageId, + }, + }); + return this.prisma.smsUplinkMessage.findUnique({ + where: { id: updatedUplink.id }, + include: { + tenant: true, + application: true, + channel: true, + messageRecord: { include: { application: true } }, + matchCandidates: { + include: { + tenant: true, + application: true, + messageRecord: { include: { application: true, tenant: true, channel: true } }, + }, + orderBy: [{ status: 'asc' }, { confidence: 'desc' }, { createdAt: 'asc' }], + }, + }, + }); + } + + private async queueAndTryDownstreamDelivery(data: { + tenantId: string; + applicationId?: string | null; + messageRecordId?: string | null; + messageId?: string | null; + deliveryType: 'receipt' | 'uplink'; + payload: Record; + }) { + if (!data.applicationId) { + return null; + } + const application = await this.prisma.smsApplication.findUnique({ + where: { id: data.applicationId }, + select: { cmppAccount: true }, + }); + const payload = { account: application?.cmppAccount, applicationId: data.applicationId, ...data.payload }; + const delivery = await this.prisma.cmppDownstreamDelivery.create({ + data: { + tenantId: data.tenantId, + applicationId: data.applicationId, + messageRecordId: data.messageRecordId, + messageId: data.messageId, + deliveryType: data.deliveryType, + payload, + status: 'pending', + }, + }); + try { + const result = await this.postGatewayControl( + data.deliveryType === 'receipt' ? '/downstream/receipt' : '/downstream/uplink', + { deliveryId: delivery.id, ...payload }, + ) as { delivered?: boolean }; + if (result.delivered) { + await this.markDownstreamDeliveryDelivered(delivery.id); + } else { + await this.markDownstreamDeliveryFailed(delivery.id, 'downstream client is not connected'); + } + } catch (error) { + await this.markDownstreamDeliveryFailed(delivery.id, error instanceof Error ? error.message : 'Gateway control delivery failed'); + } + return delivery; + } + + private async resolveUplinkMatch( + data: GatewayUplinkEventDto, + channel: { id: string; srcId?: string | null }, + ): Promise<{ + tenantId?: string; + applicationId?: string; + messageRecordId?: string; + matchStatus: string; + matchReason: string; + candidates: UplinkMatchCandidateInput[]; + }> { + if (data.messageId) { + const message = await this.prisma.smsMessageRecord.findUnique({ where: { messageId: data.messageId } }); + if (message) { + return { + tenantId: message.tenantId, + applicationId: message.applicationId ?? undefined, + messageRecordId: message.id, + matchStatus: message.applicationId ? 'matched' : 'unmatched', + matchReason: message.applicationId ? 'messageId 精确匹配' : 'messageId 匹配到下发记录但无应用', + candidates: [], + }; + } + } + + const accessNumber = data.destId || channel.srcId || ''; + const accessRoutes = accessNumber + ? await this.prisma.channelRouteRule.findMany({ + where: { + applicationId: { not: null }, + status: 'active', + group: { items: { some: { channelId: channel.id, channel: { srcId: accessNumber } } } }, + }, + select: { applicationId: true }, + take: 10, + }) + : []; + const accessApplicationIds = [...new Set(accessRoutes.map((route) => route.applicationId).filter((value): value is string => Boolean(value)))]; + const accessApplications = accessApplicationIds.length > 0 + ? await this.prisma.smsApplication.findMany({ + where: { id: { in: accessApplicationIds }, status: 'active' }, + select: { id: true, tenantId: true, name: true }, + }) + : []; + if (accessApplications.length === 1) { + return { + tenantId: accessApplications[0].tenantId, + applicationId: accessApplications[0].id, + matchStatus: 'matched', + matchReason: '接入号唯一匹配应用', + candidates: [], + }; + } + if (accessApplications.length > 1) { + return { + matchStatus: 'ambiguous', + matchReason: '接入号匹配多个应用', + candidates: accessApplications.map((application) => ({ + tenantId: application.tenantId, + applicationId: application.id, + matchSource: 'access_number', + confidence: 70, + reason: `接入号 ${accessNumber} 可匹配应用 ${application.name}`, + })), + }; + } + + const windowHours = Number(process.env.UPLINK_MATCH_WINDOW_HOURS ?? 72); + const since = new Date(Date.now() - Math.max(1, windowHours) * 60 * 60 * 1000); + const recentMessages = await this.prisma.smsMessageRecord.findMany({ + where: { + phoneNumber: data.phoneNumber, + applicationId: { not: null }, + submittedAt: { gte: since }, + }, + orderBy: { submittedAt: 'desc' }, + take: 2, + }); + if (recentMessages.length === 1) { + return { + tenantId: recentMessages[0].tenantId, + applicationId: recentMessages[0].applicationId ?? undefined, + messageRecordId: recentMessages[0].id, + matchStatus: 'matched', + matchReason: `手机号 ${windowHours} 小时窗口唯一匹配`, + candidates: [], + }; + } + if (recentMessages.length > 1) { + return { + matchStatus: 'ambiguous', + matchReason: `手机号 ${windowHours} 小时窗口匹配多条下发记录`, + candidates: recentMessages + .filter((message) => message.applicationId) + .map((message) => ({ + tenantId: message.tenantId, + applicationId: String(message.applicationId), + messageRecordId: message.id, + matchSource: 'phone_window', + confidence: 55, + reason: `手机号 ${windowHours} 小时窗口候选下发 ${message.messageId}`, + })), + }; + } + return { matchStatus: 'unmatched', matchReason: '未匹配到应用或下发记录', candidates: [] }; } async authenticateInboundApplication(data: GatewayInboundAuthDto) { @@ -782,7 +1487,7 @@ export class SendChainService implements OnModuleInit, OnModuleDestroy { errorMessage: attempt > 0 ? `第 ${attempt + 1} 次提交，路由至${routed.routeScope === 'national' ? '全国' : '省网'}通道` : undefined, }, }); - await this.getGatewayQueue().add('submit-command', { + const command = { schemaVersion: 'v1', messageType: 'SubmitCommand', traceId: randomUUID(), @@ -817,8 +1522,19 @@ export class SendChainService implements OnModuleInit, OnModuleDestroy { registeredDelivery: 1, msgFmt: 8, }, - retry: { attempt, maxAttempts: 0 }, - }); + upstream: { + gatewayHost: channel.gatewayHost, + gatewayPort: channel.gatewayPort, + account: channel.account, + passwordCipher: channel.passwordCipher, + cmppVersion: channel.cmppVersion, + desiredConnections: getPositiveConfigInteger(channel.config, 'desiredConnections', 1), + windowSize: getPositiveConfigInteger(channel.config, 'windowSize', 16), + }, + retry: { attempt, maxAttempts: 1 }, + }; + await this.getGatewayQueue().add('submit-command', command); + await this.publishGatewaySubmitCommand(command); await this.refreshTaskProgress(message.batchTaskId); return { submitted: true, messageRecordId: message.id, channelId: channel.id, attempt }; } @@ -1225,21 +1941,254 @@ export class SendChainService implements OnModuleInit, OnModuleDestroy { }); } - private async findMessageByGatewayEvent(messageId: string, gatewayMessageId?: string) { - const message = await this.prisma.smsMessageRecord.findFirst({ + private smsMessageSegmentAuditDelegate() { + return (this.prisma as PrismaService & { + smsMessageSegmentAudit: { + upsert: (args: Record) => Promise; + updateMany: (args: Record) => Promise<{ count: number }>; + findFirst: (args: Record) => Promise; + }; + }).smsMessageSegmentAudit; + } + + private async recordSubmitSegments( + message: { + id: string; + tenantId: string; + batchTaskId: string; + channelId?: string | null; + submitId?: string | null; + billingUnits?: number | null; + }, + data: GatewaySubmitResultDto, + submittedAt: Date, + ) { + const segmentAudits = this.smsMessageSegmentAuditDelegate(); + const submitRecord = await this.prisma.smsSubmitRecord.findFirst({ where: { - OR: [{ messageId }, gatewayMessageId ? { gatewayMessageId } : undefined].filter(Boolean) as Array<{ - messageId?: string; - gatewayMessageId?: string; - }>, + messageRecordId: message.id, + OR: [ + data.submitId ? { submitId: data.submitId } : undefined, + data.gatewayMessageId ? { gatewayMessageId: data.gatewayMessageId } : undefined, + ].filter(Boolean) as Array<{ submitId?: string; gatewayMessageId?: string }>, + }, + orderBy: { createdAt: 'desc' }, + }); + const submitId = data.submitId ?? submitRecord?.submitId ?? message.submitId ?? `SUB-AUDIT-${message.id}`; + const attempt = submitRecord + ? Math.max(0, await this.prisma.smsSubmitRecord.count({ + where: { + messageRecordId: message.id, + createdAt: { lte: submitRecord.createdAt }, + }, + }) - 1) + : 0; + const fallbackSegments = [{ + segmentTotal: Math.max(1, Number(message.billingUnits ?? 1)), + segmentIndex: 1, + sequenceId: data.sequenceId, + gatewayMessageId: data.gatewayMessageId, + submitStatus: data.submitStatus, + errorCode: data.errorCode, + errorMessage: data.errorMessage, + submittedAt: data.submittedAt, + }]; + const segments = data.segments && data.segments.length > 0 ? data.segments : fallbackSegments; + const segmentTotal = Math.max(1, ...segments.map((item) => Number(item.segmentTotal ?? segments.length ?? 1))); + await Promise.all(segments.map((segment, index) => { + const segmentIndex = Math.max(1, Number(segment.segmentIndex ?? index + 1)); + const status = segment.submitStatus ?? data.submitStatus; + return segmentAudits.upsert({ + where: { + messageRecordId_submitId_segmentIndex: { + messageRecordId: message.id, + submitId, + segmentIndex, + }, + }, + update: { + submitRecordId: submitRecord?.id ?? null, + channelId: data.channelId ?? message.channelId ?? null, + attempt, + segmentTotal, + sequenceId: segment.sequenceId ?? data.sequenceId ?? null, + gatewayMessageId: segment.gatewayMessageId ?? data.gatewayMessageId ?? null, + submitStatus: status, + errorCode: segment.errorCode ?? data.errorCode ?? null, + errorMessage: segment.errorMessage ?? data.errorMessage ?? null, + submittedAt: segment.submittedAt ? new Date(segment.submittedAt) : submittedAt, + }, + create: { + tenantId: message.tenantId, + batchTaskId: message.batchTaskId, + messageRecordId: message.id, + submitRecordId: submitRecord?.id ?? null, + channelId: data.channelId ?? message.channelId ?? null, + submitId, + attempt, + segmentTotal, + segmentIndex, + sequenceId: segment.sequenceId ?? data.sequenceId ?? null, + gatewayMessageId: segment.gatewayMessageId ?? data.gatewayMessageId ?? null, + submitStatus: status, + compensationType: submitRecord && submitRecord.submitId !== message.submitId ? 'retry_submit' : null, + errorCode: segment.errorCode ?? data.errorCode ?? null, + errorMessage: segment.errorMessage ?? data.errorMessage ?? null, + submittedAt: segment.submittedAt ? new Date(segment.submittedAt) : submittedAt, + }, + }); + })); + } + + private async recordReceiptSegment( + message: { + id: string; + tenantId: string; + batchTaskId: string; + channelId?: string | null; + submitId?: string | null; + billingUnits?: number | null; + }, + data: GatewayReceiptEventDto, + deliveredAt: Date, + submitRecordId?: string, + ) { + const segmentAudits = this.smsMessageSegmentAuditDelegate(); + const updated = await segmentAudits.updateMany({ + where: { + messageRecordId: message.id, + gatewayMessageId: data.gatewayMessageId, + }, + data: { + receiptStatus: data.receiptStatus, + rawStatus: data.rawStatus, + errorCode: data.errorCode ?? null, + deliveredAt, }, }); + if (updated.count > 0) { + return; + } + const submitRecord = submitRecordId + ? await this.prisma.smsSubmitRecord.findUnique({ where: { id: submitRecordId } }) + : await this.prisma.smsSubmitRecord.findFirst({ + where: { messageRecordId: message.id, gatewayMessageId: data.gatewayMessageId }, + orderBy: { createdAt: 'desc' }, + }); + await segmentAudits.upsert({ + where: { + messageRecordId_submitId_segmentIndex: { + messageRecordId: message.id, + submitId: submitRecord?.submitId ?? message.submitId ?? `RECEIPT-AUDIT-${data.gatewayMessageId}`, + segmentIndex: 1, + }, + }, + update: { + submitRecordId: submitRecord?.id ?? submitRecordId ?? null, + channelId: data.channelId ?? message.channelId ?? null, + sequenceId: data.sequenceId ?? null, + gatewayMessageId: data.gatewayMessageId, + receiptStatus: data.receiptStatus, + rawStatus: data.rawStatus, + errorCode: data.errorCode ?? null, + deliveredAt, + }, + create: { + tenantId: message.tenantId, + batchTaskId: message.batchTaskId, + messageRecordId: message.id, + submitRecordId: submitRecord?.id ?? submitRecordId ?? null, + channelId: data.channelId ?? message.channelId ?? null, + submitId: submitRecord?.submitId ?? message.submitId ?? `RECEIPT-AUDIT-${data.gatewayMessageId}`, + attempt: 0, + segmentTotal: Math.max(1, Number(message.billingUnits ?? 1)), + segmentIndex: 1, + sequenceId: data.sequenceId ?? null, + gatewayMessageId: data.gatewayMessageId, + submitStatus: submitRecord?.submitStatus ?? 'accepted', + receiptStatus: data.receiptStatus, + rawStatus: data.rawStatus, + compensationType: 'receipt_recovered', + errorCode: data.errorCode ?? null, + deliveredAt, + }, + }); + } + + private async findMessageByGatewayEvent(messageId?: string, gatewayMessageId?: string) { + const conditions = [{ messageId }, gatewayMessageId ? { gatewayMessageId } : undefined].filter( + Boolean, + ) as Array<{ + messageId?: string; + gatewayMessageId?: string; + }>; + if (conditions.length === 0) { + return null; + } + return this.prisma.smsMessageRecord.findFirst({ + where: { + OR: conditions, + }, + }); + } + + private async requireMessageByGatewayEvent(messageId?: string, gatewayMessageId?: string) { + const message = await this.findMessageByGatewayEvent(messageId, gatewayMessageId); if (!message) { throw new NotFoundException('SMS message record not found'); } return message; } + private async resolveReceiptMessage(data: GatewayReceiptEventDto) { + const exactMessage = await this.findMessageByGatewayEvent(data.messageId, data.gatewayMessageId); + if (exactMessage) { + return { + message: exactMessage, + messageId: exactMessage.messageId, + }; + } + + const phoneNumber = data.phoneNumber?.trim(); + if (!phoneNumber) { + throw new NotFoundException('SMS message record not found'); + } + + const deliveredAt = data.deliveredAt ? new Date(data.deliveredAt) : new Date(); + const submittedAfter = new Date(deliveredAt.getTime() - 72 * 60 * 60 * 1000); + const candidates = await this.prisma.smsSubmitRecord.findMany({ + where: { + channelId: data.channelId, + gatewayMessageId: null, + submitStatus: 'timeout', + submittedAt: { + gte: submittedAfter, + lte: deliveredAt, + }, + messageRecord: { + phoneNumber, + }, + }, + include: { + messageRecord: true, + }, + orderBy: { + submittedAt: 'desc', + }, + take: 10, + }); + + if (candidates.length !== 1 || !candidates[0]?.messageRecord) { + throw new NotFoundException('SMS message record not found'); + } + + return { + message: candidates[0].messageRecord, + messageId: candidates[0].messageRecord.messageId, + submitRecordId: candidates[0].id, + }; + } + private getSendQueue(): Queue { if (!this.sendQueue) { this.sendQueue = new Queue(SEND_QUEUE, { connection: bullmqConnection() }); @@ -1262,6 +2211,32 @@ export class SendChainService implements OnModuleInit, OnModuleDestroy { } return this.redis; } + + private async postGatewayControl(path: string, payload: unknown) { + const baseUrl = (process.env.GATEWAY_CONTROL_URL ?? 'http://127.0.0.1:8090').replace(/\/$/, ''); + const response = await fetch(`${baseUrl}${path}`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(payload), + }); + if (!response.ok) { + const body = await response.text().catch(() => ''); + throw new Error(`Gateway control ${path} returned ${response.status}${body ? `: ${body}` : ''}`); + } + return response.json().catch(() => ({})); + } + + private async publishGatewaySubmitCommand(command: unknown) { + return this.getRedis().xadd( + process.env.GATEWAY_SUBMIT_STREAM ?? GATEWAY_SUBMIT_STREAM, + '*', + 'messageType', + 'SubmitCommand', + 'data', + JSON.stringify(command), + ); + } + } function statusFromRisk(status: string, scheduled: boolean) { @@ -1294,6 +2269,41 @@ function parseSchedule(data: CreateBatchTaskDto) { return { scheduledAt }; } +function isObjectRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +function asDateOrNull(value?: string | null) { + if (!value) { + return null; + } + const parsed = new Date(value); + return Number.isNaN(parsed.getTime()) ? null : parsed; +} + +function downstreamRetryDelayMs(retryCount = 1) { + const base = downstreamRetryBaseDelayMs(); + const max = downstreamRetryMaxDelayMs(); + const attempt = Math.max(1, Math.floor(retryCount)); + const delay = base * Math.pow(2, Math.max(0, attempt - 1)); + return Math.min(delay, max); +} + +function downstreamRetryBaseDelayMs() { + const value = Number(process.env.CMPP_DOWNSTREAM_RETRY_DELAY_MS ?? DEFAULT_DOWNSTREAM_RETRY_DELAY_MS); + return Number.isFinite(value) && value > 0 ? value : DEFAULT_DOWNSTREAM_RETRY_DELAY_MS; +} + +function downstreamRetryMaxDelayMs() { + const value = Number(process.env.CMPP_DOWNSTREAM_RETRY_MAX_DELAY_MS ?? DEFAULT_DOWNSTREAM_RETRY_MAX_DELAY_MS); + return Number.isFinite(value) && value > 0 ? value : DEFAULT_DOWNSTREAM_RETRY_MAX_DELAY_MS; +} + +function downstreamMaxRetries() { + const value = Number(process.env.CMPP_DOWNSTREAM_MAX_RETRIES ?? DEFAULT_DOWNSTREAM_MAX_RETRIES); + return Number.isFinite(value) && value > 0 ? Math.floor(value) : DEFAULT_DOWNSTREAM_MAX_RETRIES; +} + function parseImportRows(content: string, delimiter?: ',' | '\t') { const normalized = content.replace(/^\uFEFF/, ''); const lines = normalized.split(/\r?\n/).filter((line) => line.trim().length > 0); @@ -1343,6 +2353,16 @@ function normalizeQueuePriority(queuePriority?: string | null): QueuePriority { return queuePriority === 'priority' ? 'priority' : 'normal'; } +function getPositiveConfigInteger(config: unknown, key: string, fallback: number) { + if (config && typeof config === 'object' && !Array.isArray(config) && key in config) { + const value = Number((config as Record)[key]); + if (Number.isInteger(value) && value > 0) { + return value; + } + } + return fallback; +} + function isCarrierCompatible(channelCarrier: string | null | undefined, targetCarrier: string) { const normalized = normalizeCarrier(channelCarrier); return normalized === 'all' || normalized === targetCarrier; @@ -1407,6 +2427,35 @@ function octetString(value: string, fixedLength: number) { return value + '\0'.repeat(fixedLength - value.length); } +function normalizeRecoveryFailureCategory(data: GatewayDownstreamRecoveryStatusDto) { + const explicit = String(data.failureCategory ?? '').trim(); + if (explicit) { + return explicit; + } + if (data.state === 'success' || data.state === 'running') { + return null; + } + if (data.lastSkipReason === 'backoff') { + return 'backoff'; + } + if (data.lastSkipReason === 'locked') { + return 'lock_contended'; + } + if (data.lastSkipReason === 'lock_lost') { + return 'lock_lost'; + } + if (data.state === 'waiting_connection') { + return 'client_disconnected'; + } + if (data.state === 'partial') { + return 'partial_delivery_failed'; + } + if (data.state === 'failed' && data.lastError) { + return 'flush_failed'; + } + return data.state ? 'unknown' : null; +} + function isApplicationIpAllowed(remoteIp: string, allowlist: string[]) { const normalizedRemoteIp = normalizeIp(remoteIp); if (allowlist.length === 0) { diff --git a/api/src/sms-config/sms-config.service.spec.ts b/api/src/sms-config/sms-config.service.spec.ts index ad3d011..b6ba440 100644 --- a/api/src/sms-config/sms-config.service.spec.ts +++ b/api/src/sms-config/sms-config.service.spec.ts @@ -9,6 +9,8 @@ function createPrismaMock() { name: '应用A', status: 'active', cmppAccount: '100001', + cmppMaxConnections: 2, + cmppWindowSize: 32, queuePriority: 'normal', tenant: { id: 'tenant-1', name: '租户A', code: 'TENANT-A' }, messageRecords: [{ status: 'delivered' }, { status: 'undelivered' }], @@ -19,6 +21,8 @@ function createPrismaMock() { name: '应用A', status: 'active', cmppAccount: '100001', + cmppMaxConnections: 2, + cmppWindowSize: 32, queuePriority: 'normal', secretHash: 'secret-hash', tenant: { id: 'tenant-1', name: '租户A', code: 'TENANT-A' }, @@ -170,6 +174,7 @@ describe('SmsConfigService', () => { gatewayHost: '127.0.0.1', gatewayPort: 17890, maxConnections: 2, + windowSize: 32, })); }); @@ -181,6 +186,9 @@ describe('SmsConfigService', () => { await expect(service.createApplication({ tenantId: 'tenant-1', name: '优先应用', + cmppAccount: '123456', + cmppMaxConnections: 3, + cmppWindowSize: 32, queuePriority: 'priority', ipAllowlist: ['10.0.0.1/32'], })).resolves.toEqual(expect.objectContaining({ id: 'app-new' })); @@ -189,7 +197,9 @@ describe('SmsConfigService', () => { data: expect.objectContaining({ tenantId: 'tenant-1', name: '优先应用', - cmppAccount: expect.stringMatching(/^\d{6}$/), + cmppAccount: '123456', + cmppMaxConnections: 3, + cmppWindowSize: 32, queuePriority: 'priority', ipAllowlist: { create: [{ ipCidr: '10.0.0.1/32' }] }, }), @@ -209,6 +219,23 @@ describe('SmsConfigService', () => { expect(prisma.smsApplication.create).not.toHaveBeenCalled(); }); + it('rejects duplicate or invalid CMPP application accounts', async () => { + const prisma = createPrismaMock(); + const service = new SmsConfigService(prisma as never); + + await expect(service.createApplication({ + tenantId: 'tenant-1', + name: '异常应用', + cmppAccount: 'abc', + })).rejects.toThrow('cmppAccount must be a 6-digit number'); + + await expect(service.createApplication({ + tenantId: 'tenant-1', + name: '重复应用', + cmppAccount: '100001', + })).rejects.toThrow('cmppAccount already exists'); + }); + it('updates enterprise application profile and allowlist through a transaction', async () => { const prisma = createPrismaMock(); const tx = { diff --git a/api/src/sms-config/sms-config.service.ts b/api/src/sms-config/sms-config.service.ts index 566ad78..9506046 100644 --- a/api/src/sms-config/sms-config.service.ts +++ b/api/src/sms-config/sms-config.service.ts @@ -8,6 +8,9 @@ export interface CreateSmsApplicationDto { name: string; scene?: string; callbackUrl?: string; + cmppAccount?: string; + cmppMaxConnections?: number; + cmppWindowSize?: number; dailyLimit?: number; customerUnitPrice?: number; queuePriority?: string; @@ -153,7 +156,7 @@ export class SmsConfigService { async createApplication(data: CreateSmsApplicationDto) { const secret = randomBytes(24).toString('hex'); const queuePriority = normalizeApplicationQueuePriority(data.queuePriority); - const cmppAccount = await this.generateCmppAccount(); + const cmppAccount = data.cmppAccount ? await this.validateAndReserveCmppAccount(data.cmppAccount) : await this.generateCmppAccount(); return this.prisma.smsApplication.create({ data: { tenantId: data.tenantId, @@ -162,6 +165,8 @@ export class SmsConfigService { callbackUrl: data.callbackUrl, cmppAccount, secretHash: hashSecret(secret), + cmppMaxConnections: getPositiveInteger(data.cmppMaxConnections, 1, 'cmppMaxConnections'), + cmppWindowSize: getPositiveInteger(data.cmppWindowSize, 16, 'cmppWindowSize'), dailyLimit: data.dailyLimit, customerUnitPrice: data.customerUnitPrice ?? 0, queuePriority, @@ -183,6 +188,9 @@ export class SmsConfigService { const queuePriority = data.queuePriority === undefined ? undefined : normalizeApplicationQueuePriority(data.queuePriority); + const cmppAccount = data.cmppAccount === undefined + ? undefined + : await this.validateAndReserveCmppAccount(data.cmppAccount, applicationId); return this.prisma.$transaction(async (tx) => { if (data.ipAllowlist) { @@ -194,6 +202,9 @@ export class SmsConfigService { name: data.name, scene: data.scene, callbackUrl: data.callbackUrl, + cmppAccount, + cmppMaxConnections: data.cmppMaxConnections === undefined ? undefined : getPositiveInteger(data.cmppMaxConnections, 1, 'cmppMaxConnections'), + cmppWindowSize: data.cmppWindowSize === undefined ? undefined : getPositiveInteger(data.cmppWindowSize, 16, 'cmppWindowSize'), dailyLimit: data.dailyLimit, customerUnitPrice: data.customerUnitPrice, queuePriority, @@ -350,13 +361,24 @@ export class SmsConfigService { account: application.cmppAccount, passwordCipher: application.secretHash, srcId: channel?.srcId ?? '', - maxConnections: channel?.config && typeof channel.config === 'object' && 'maxConnections' in channel.config ? Number(channel.config.maxConnections) : 1, + maxConnections: application.cmppMaxConnections, heartbeatSeconds: 30, - windowSize: 16, + windowSize: application.cmppWindowSize, protocolVersion: channel?.cmppVersion ?? '3.0', }; } + private async validateAndReserveCmppAccount(cmppAccount: string, currentApplicationId?: string) { + if (!/^\d{6}$/.test(cmppAccount)) { + throw new BadRequestException('cmppAccount must be a 6-digit number'); + } + const exists = await this.prisma.smsApplication.findUnique({ where: { cmppAccount } }); + if (exists && exists.id !== currentApplicationId) { + throw new BadRequestException('cmppAccount already exists'); + } + return cmppAccount; + } + private async generateCmppAccount() { for (let attempt = 0; attempt < 20; attempt += 1) { const cmppAccount = String(randomInt(100000, 1000000)); @@ -769,6 +791,17 @@ function normalizeApplicationQueuePriority(value?: string): ApplicationQueuePrio return queuePriority as ApplicationQueuePriority; } +function getPositiveInteger(value: number | undefined, fallback: number, fieldName: string) { + if (value === undefined || value === null) { + return fallback; + } + const normalized = Number(value); + if (!Number.isInteger(normalized) || normalized <= 0) { + throw new BadRequestException(`${fieldName} must be a positive integer`); + } + return normalized; +} + function normalizeApplicationCmppStatus(connections: Array<{ status: string; currentConnections: number }>, applicationStatus: string) { if (applicationStatus !== 'active') { return 'inactive'; diff --git a/api/src/spike/bullmq-link-spike.mjs b/api/src/spike/bullmq-link-spike.mjs index 14c141a..650a7fd 100644 --- a/api/src/spike/bullmq-link-spike.mjs +++ b/api/src/spike/bullmq-link-spike.mjs @@ -54,6 +54,13 @@ function createSubmitCommand(index) { feeCode: '0', feeType: '01', }, + upstream: { + gatewayHost: '127.0.0.1', + gatewayPort: 17890, + account: 'cmpp-account-spike', + passwordCipher: 'secret-spike', + cmppVersion: '3.0', + }, retry: { attempt: 0, maxAttempts: 3, diff --git a/docs/contracts/examples/submit-command.json b/docs/contracts/examples/submit-command.json index cd87030..b56932d 100644 --- a/docs/contracts/examples/submit-command.json +++ b/docs/contracts/examples/submit-command.json @@ -30,6 +30,15 @@ "feeCode": "0", "feeType": "01" }, + "upstream": { + "gatewayHost": "127.0.0.1", + "gatewayPort": 17890, + "account": "cmpp-account-demo", + "passwordCipher": "secret-demo", + "cmppVersion": "3.0", + "desiredConnections": 2, + "windowSize": 16 + }, "retry": { "attempt": 0, "maxAttempts": 3 diff --git a/docs/contracts/examples/submit-result.json b/docs/contracts/examples/submit-result.json index 0ecc438..3f3f8d3 100644 --- a/docs/contracts/examples/submit-result.json +++ b/docs/contracts/examples/submit-result.json @@ -8,5 +8,15 @@ "sequenceId": 1024, "gatewayMessageId": "gw-msg-20260701-000001", "submitStatus": "accepted", - "submittedAt": "2026-07-01T09:00:00.118Z" + "submittedAt": "2026-07-01T09:00:00.118Z", + "segments": [ + { + "segmentTotal": 1, + "segmentIndex": 1, + "sequenceId": 1024, + "gatewayMessageId": "gw-msg-20260701-000001", + "submitStatus": "accepted", + "submittedAt": "2026-07-01T09:00:00.118Z" + } + ] } diff --git a/docs/contracts/gateway-queue-messages.schema.json b/docs/contracts/gateway-queue-messages.schema.json index 0d7ca97..b782693 100644 --- a/docs/contracts/gateway-queue-messages.schema.json +++ b/docs/contracts/gateway-queue-messages.schema.json @@ -41,6 +41,7 @@ "queuePriority", "route", "cmpp", + "upstream", "retry" ], "properties": { @@ -78,6 +79,19 @@ "feeType": { "type": "string" } } }, + "upstream": { + "type": "object", + "required": ["gatewayHost", "gatewayPort", "account", "passwordCipher", "cmppVersion"], + "properties": { + "gatewayHost": { "type": "string", "minLength": 1 }, + "gatewayPort": { "type": "integer", "minimum": 1, "maximum": 65535 }, + "account": { "type": "string", "minLength": 1 }, + "passwordCipher": { "type": "string", "minLength": 1 }, + "cmppVersion": { "enum": ["2.0", "3.0"] }, + "desiredConnections": { "type": "integer", "minimum": 1, "maximum": 32 }, + "windowSize": { "type": "integer", "minimum": 1, "maximum": 1024 } + } + }, "retry": { "type": "object", "required": ["attempt", "maxAttempts"], @@ -103,7 +117,24 @@ "submitStatus": { "enum": ["accepted", "rejected", "timeout"] }, "errorCode": { "type": "string" }, "errorMessage": { "type": "string" }, - "submittedAt": { "type": "string", "format": "date-time" } + "submittedAt": { "type": "string", "format": "date-time" }, + "segments": { + "type": "array", + "items": { + "type": "object", + "required": ["segmentTotal", "segmentIndex", "sequenceId", "gatewayMessageId", "submitStatus", "submittedAt"], + "properties": { + "segmentTotal": { "type": "integer", "minimum": 1 }, + "segmentIndex": { "type": "integer", "minimum": 1 }, + "sequenceId": { "type": "integer", "minimum": 0 }, + "gatewayMessageId": { "type": "string", "minLength": 1 }, + "submitStatus": { "enum": ["accepted", "rejected", "timeout"] }, + "errorCode": { "type": "string" }, + "errorMessage": { "type": "string" }, + "submittedAt": { "type": "string", "format": "date-time" } + } + } + } } } ] diff --git a/docs/first-version-development-requirements.md b/docs/first-version-development-requirements.md index 60cabf3..f6d8348 100644 --- a/docs/first-version-development-requirements.md +++ b/docs/first-version-development-requirements.md @@ -105,6 +105,8 @@ 5. 短信应用必须配置发送队列等级：普通队列或优先队列。未配置时默认普通队列；优先队列用于验证码、登录确认、交易通知等高时效短信，普通队列用于营销、通知等常规短信。 6. 发送队列等级属于真实业务配置，必须保存到后端数据库，并在客户端、运营端创建/编辑应用时展示和可修改；不得只作为前端展示字段。 7. 运营端代企业新增短信应用时，第一步选择企业必须使用项目通用 Select/下拉控件，选项来自真实企业 API，支持加载中、空数据、错误态，不允许写死企业列表。 +8. 短信应用必须有独立 6 位数字 CMPP 接入账号 `cmppAccount`；运营端添加/编辑应用时可显式配置，留空时后端自动生成且全局唯一。 +9. 短信应用必须可配置客户侧 CMPP 最大连接数 `cmppMaxConnections` 和客户提交窗口 `cmppWindowSize`；这两个字段是平台运行配置，不是 CMPP 协议字段，也不是 gocmpp 库参数。 ### 4.3 签名与引流信息 @@ -187,6 +189,7 @@ 1. Gateway 必须按运营端通道配置连接上游 SMSC，使用通道的 `gatewayHost/gatewayPort/account/passwordCipher/srcId/cmppVersion` 完成 CMPP 2.0/3.0 connect/login。 2. Gateway 必须校验上游 connect/login 返回码，区分 connected、auth_failed、connect_timeout、network_error、protocol_error 等状态，并回写 NestJS 真实连接状态。 3. Gateway 必须支持每个通道配置期望连接数，建立多条长连接，并按连接维度维护 currentConnections、lastConnectedAt、lastHeartbeatAt、lastError、reconnectCount。 + - `desiredConnections`、`windowSize` 是平台对上游通道连接池和提交窗口的运行配置，必须通过运营端通道配置页面保存到真实后端；它们不是 CMPP 标准 PDU 字段，也不是 gocmpp 的原生配置字段。 4. Gateway 必须实现 ActiveTest 心跳与超时检测；连续心跳失败后连接进入 heartbeat_timeout/reconnecting，重连成功前该连接不可参与发送。 5. Gateway 必须支持断线自动重连、指数退避或固定退避、最大重试间隔、重连日志和状态回写。 6. Gateway 必须维护 CMPP sequenceId 与平台 messageId、submitId、channelId 的映射，submit resp 和 deliver 回执必须能追溯到原短信记录和提交尝试。 @@ -221,23 +224,66 @@ #### 4.8.4 当前实现缺口标记 -截至当前版本，Go Gateway 已有 HTTP 控制服务、健康检查、连接上游 SMSC 的 `ConnectChannel` 控制入口、gocmpp 协议 spike、队列消息结构，并已补齐第一阶段下游 CMPP 入站能力： +截至当前版本，Go Gateway 已有 HTTP 控制服务、健康检查、连接上游 SMSC 的 `ConnectChannel` 控制入口、gocmpp 协议 spike、队列消息结构，并已补齐第一阶段下游 CMPP 入站、上游提交和下游回执/上行推送能力： - 已实现 `17890` 入站 CMPP Server 监听，生产部署由 `GATEWAY_CMPP_ADDR=0.0.0.0:17890` 启动。 - 已实现下游客户 connect/login 鉴权：CMPP `Source_Addr` 使用企业应用独立 6 位 `cmppAccount`，密码使用应用 CMPP 参数中的 `passwordCipher`，Gateway 将 CMPP `AuthSource/Timestamp` 交由 NestJS 根据真实数据库校验。 - 已实现客户端应用 IP 白名单、应用状态、企业状态和企业认证状态校验；校验失败返回 CMPP connect 失败。 - 已实现下游 CMPP Submit 到平台发送请求的转换：Gateway 解码 CMPP 3.0 submit 内容，调用 NestJS 真实入站接口，NestJS 复用模板/签名/风控/余额/路由/队列优先级发送链路，接受后返回 CMPP submit_resp。 +- 已实现 NestJS 校验通过后的异步 Gateway 提交：`SubmitCommand` 保留 BullMQ 审计/兼容投递，同时写入 Redis Stream `gateway.submit.commands` 主命令流；Go Gateway 以 consumer group 独立消费该命令流，携带通道 `gatewayHost/gatewayPort/account/passwordCipher/cmppVersion` 作为 SP 客户端连接上游 SMSC 并发送 CMPP Submit。 +- 已实现上游 submit_resp 回传：Gateway 将 accepted/rejected/timeout 转换为 `SubmitResult` 调用 NestJS，NestJS 继续执行 accepted 扣费、失败/超时补发或释放冻结等既有逻辑。 +- 已实现上游 deliver receipt 和普通 deliver 上行解析：Gateway 在上游连接读循环中解析 receipt/uplink，调用 NestJS `/gateway/events/receipt`、`/gateway/events/uplink` 写入真实发送记录、回执和上行表。 +- 已实现上游长短信第一版拆分和上行长短信重组：Gateway 按 CMPP 标准 6 字节 UDH 将超过 140 字节的 Submit 内容拆成多个分片，设置 `PkTotal/PkNumber/TpUdhi` 后逐包提交；上游普通 Deliver 携带 UDH 分片时，Gateway 在同一连接内按通道、主叫、被叫、引用号和总片数缓存并重组后再回传 NestJS。 +- 已实现上游多连接和窗口控制第一版：NestJS 将通道配置中的 `desiredConnections/windowSize` 写入 `SubmitCommand.upstream`，Go Gateway 按通道建立连接池，每条连接独立维护 submit pending 映射、receipt/uplink 处理和窗口令牌；窗口满时等待可用窗口或按提交超时返回。 +- 已实现 SubmitCommand 在途恢复第一版：Go Gateway submit worker 在消费新消息前会对 Redis Stream consumer group 中空闲超过阈值的 pending 命令执行 `XAUTOCLAIM`，重新提交并按正常成功路径 ack，避免 Gateway 重启后命令永久滞留在 PEL。 +- 已实现上游连接断开时的 pending submit 状态补偿第一版：如果某条上游 CMPP 连接在收到 submit resp 前断开，Gateway 会立即唤醒该连接上等待中的 pending submit，请求返回 `timeout/CONNECTION_LOST`，由 NestJS 进入既有补发或释放冻结逻辑，不再只依赖固定超时。 +- 已实现“上游可能已受理但 submit resp 丢失”场景的保守补偿第一版：Gateway 在 receipt 事件中补充手机号；NestJS 对无法按 `messageId/gatewayMessageId` 精确命中的回执，只在“同通道、同手机号、72 小时窗口内、且仅存在 1 条 `timeout + gatewayMessageId=null` 的 submit 记录”时才回填并接收该回执，避免误绑到其他短信。 +- 已实现 SubmitCommand 死信治理第一版：Go Gateway 对多次处理仍失败的 `SubmitCommand` 不再无限滞留在 PEL，而是按阈值写入 NestJS 真实 `GatewaySubmitDeadLetter` 表；运营端后端接口可分页查询死信，并支持人工将原始 `SubmitCommand` 重新写回 Redis Stream。 +- 已实现客户侧下游投递重试第二版：客户系统负责断线后重连；平台在客户离线或投递失败时把 Deliver Receipt/上行 Deliver 保留在 `CmppDownstreamDelivery`，客户 bind 成功后立即拉取 pending，且 Gateway 会对当前在线账号周期补投；超过重试上限后转 `failed` 并写失败审计。 +- 已实现下游投递失败审计与人工重投第一版：运营端后端与页面可分页查看 `CmppDownstreamDelivery` 的 pending/failed/delivered 记录，支持按状态、类型、应用和关键字筛选，并可对单条记录执行人工重投，真实调用 Gateway `/downstream/receipt` 或 `/downstream/uplink`。 +- 已实现下游投递批量重投第一版：运营端可在当前页勾选多条 `pending/failed` 下游投递记录，调用真实批量接口逐条重投并返回成功/失败汇总，不允许用前端循环假装成功。 +- 已实现下游投递告警第一版：运营看板与右上角通知基于真实 `CmppDownstreamDelivery` 聚合显示下游投递告警数，当前告警口径包括“pending 超过阈值仍未投出”和“最近失败记录数”，用于提醒运营及时进入下游投递记录页处理。 +- 已实现下游投递 Dashboard 第一版：运营端“下游投递记录”页面顶部新增真实聚合总览，直接按 `tenantId/applicationId/deliveryType` 统计投递总量、pending/delivered/failed、积压告警、按类型分布、重试压力分布和应用告警排行，数据源必须来自 `CmppDownstreamDelivery`，不能靠前端本地汇总。 +- 已实现下游连接映射持久化第一步：Gateway 在客户 CMPP 账号 bind 成功、下游 submit 建链和回执/上行下发时，会把账号在线状态、实例标识、最近活跃时间写入 Redis presence；该状态不再只保留在 Gateway 进程内存中，为后续“Gateway 重启后的 pending 恢复”提供外部状态基础。 +- 已实现下游连接映射持久化第二步：Gateway 启动时会读取 Redis presence 与当前内存在线账号，形成“恢复候选视图”，并通过控制面 `GET /downstream/recovery-candidates` 暴露候选账号列表，供后续恢复逻辑与运维排查使用；本阶段仍不等同于自动恢复 pending 投递。 +- 已实现下游 pending 恢复执行第一版：Gateway 启动后会立即按恢复候选账号拉取真实 `CmppDownstreamDelivery.pending`，后续每轮补投周期也会继续扫描恢复候选；若账号已有可用下游连接则继续推送回执/上行，若账号尚未重连则保持 `pending` 等待后续恢复，不能因为 Gateway 重启就把未投递记录误标成失败。 +- 已实现下游恢复控制第一版：Gateway 对恢复候选账号增加账号级恢复锁、失败/等待连接退避和恢复状态持久化，避免同一账号被并发重复恢复或每轮高频空转；控制面新增 `GET /downstream/recovery-statuses` 可查看最近一次恢复状态、重试次数、下一次可恢复时间和错误原因。 +- 已实现下游恢复观测第一版：控制面新增 `GET /downstream/recovery-overview`，一次性返回恢复候选账号与恢复状态，便于联调和生产排查。 +- 已实现下游恢复状态回流第一版：Gateway 在每次恢复状态变化后，调用 NestJS `/api/gateway/events/downstream/recovery-status` 真实回传账号恢复状态；NestJS 将状态写入 Prisma/PostgreSQL `GatewayDownstreamRecoveryStatus`。 +- 已实现下游恢复状态运营化第一版：运营端新增独立“恢复状态管理”页面，支持真实列表、分页、详情查看和当前筛选结果 CSV 导出；原“下游投递记录”页面只保留投递记录本身，不再混放恢复状态区块。 +- 已实现下游恢复失败分类第一版：Gateway/NestJS 共同维护 `failureCategory`，覆盖 `client_disconnected`、`backoff`、`lock_contended`、`lock_lost`、`flush_failed`、`partial_delivery_failed`、`unknown`；运营端“恢复状态管理”页面支持失败分类筛选、分类分布统计、详情展示和导出字段。 +- 已实现多 Gateway 恢复抢占协调第一版：恢复锁从单纯实例名升级为 Redis token 租约，状态记录 `lockOwner/lockExpiresAt`；恢复完成时必须通过 Lua 原子校验锁 token，只有持锁实例才能写入最终恢复状态并释放锁，避免旧实例超时后误删新实例锁或覆盖新实例恢复结果；运营端详情/列表可查看锁持有实例。 +- 已实现长短信分片审计第一版：Gateway `SubmitResult` 回传真实 `segments[]`，包含 `segmentTotal/segmentIndex/sequenceId/gatewayMessageId/submitStatus/submittedAt`；NestJS 写入 Prisma/PostgreSQL `SmsMessageSegmentAudit`，回执按 `gatewayMessageId` 回填分片回执状态，补偿归因可记录 `compensationType`；运营端短信记录详情可查看真实分片提交、回执和补偿审计。 +- 下游投递重试已改为指数退避第一版：首次失败后按基础间隔重试，随后按 2 倍递增，并受最大退避上限约束，避免客户长时间离线时平台每分钟机械重试。 +- 已实现客户侧最终 Deliver 推送的第一版能力：Gateway 在下游 Submit 被接受后记录 messageId 到客户连接的内存映射；NestJS 收到最终 receipt/uplink 并入库后调用 Gateway `/downstream/receipt`、`/downstream/uplink`，Gateway 向仍在线的客户 CMPP 连接下发 Deliver Receipt 或普通 Deliver。 +- 已实现客户侧 Deliver 持久化第一版能力：NestJS 收到最终 receipt/uplink 后写入 `CmppDownstreamDelivery` 待投递记录；在线推送成功标记 delivered，客户断线或 Gateway 不可达时保留 pending 并记录 retry 信息；客户重新 bind 后 Gateway 按账号拉取 pending 记录补发。 +- 已实现普通上行匹配与人工认领第一版：优先按 messageId 精确匹配；无 messageId 时按接入号匹配应用路由；仍无唯一应用时按手机号和最近下发时间窗口匹配；多候选标记 ambiguous 并写入 `SmsUplinkMatchCandidate` 候选，运营端可人工认领候选应用/下发记录；认领后更新上行记录、保留候选审计，并创建真实客户侧上行 Deliver 投递记录。 仍缺少生产完整闭环能力： - 应用级下游连接数限制和连接状态回写尚未完整产品化。 - 当前下游 CMPP Submit 通过 `sourceType=cmpp` 的系统批次兼容承载，尚未拆成完全独立于批量任务模型的单条发送模型。 -- 未实现客户侧最终 Deliver Receipt 和上行 Deliver 投递。 -- 未实现 Gateway 消费 `SubmitCommand` 并真实 submit 到上游通道的 worker。 -- 未实现上游 deliver receipt 和普通上行 deliver 的生产解析与事件回传闭环。 -- 未实现多连接窗口管理、在途消息恢复、断线重连后的消息状态处理。 +- Gateway 控制面 `/upstream/submit` 仅保留为本地调试、人工补偿和运维验证入口；生产主链路应由 Gateway submit worker 消费 Redis Stream `gateway.submit.commands` 触发。 +- 客户侧 Deliver 重投当前已经具备账号级周期恢复、退避、Redis token 租约锁和控制面状态观测；恢复状态已同步到 NestJS 持久化审计表并进入运营端独立页面，且具备第一版失败分类分析和多 Gateway 抢占协调。 +- 普通上行匹配已覆盖 messageId、接入号、手机号时间窗口和共享接入号多候选人工认领；后续仍需补更复杂的批量认领、认领规则推荐和认领准确率指标。 +- 长短信分片当前已具备真实分片提交/回执/补偿审计，运营端短信详情可查看 `SmsMessageSegmentAudit`；后续仍需补“按单个分片自动重投”和分片级人工补偿操作。 +- 多连接窗口当前覆盖单进程内连接池和窗口满等待；在途恢复当前覆盖 Redis Stream pending claim、连接断开时的 pending submit 唤醒、receipt 驱动的保守唯一候选补偿、SubmitCommand 死信入库/人工重入队第一版，以及下游客户在线时的周期补投、Gateway 重启后的恢复候选扫描、账号级 Redis token 租约锁/退避/状态观测、恢复状态入库/运营端可视化；尚未实现连接级状态持久化、窗口指标回写、死信后台自动重试策略、长恢复任务锁续租，以及“上游已受理但 submit resp 丢失”场景的强确认或完整幂等补偿。 -这些缺口未补齐前，只能把 `17890` 端口监听、客户账号密码鉴权、客户 IP 白名单和下游 submit 入平台发送链路作为第一阶段验收通过；不能把客户侧最终回执投递、上游真实运营商 submit/receipt/uplink 或完整多连接窗口恢复作为“生产已验收通过”。 +基于当前真实代码，CMPP 端到端链路剩余缺口可以明确收敛为以下几类： + +1. 下游恢复审计产品化： + - 恢复状态已能写回 NestJS/Prisma/PostgreSQL，并已在运营端“恢复状态管理”页面提供独立列表、详情、导出和失败分类分布。 + - 后续仍需补恢复吞吐、趋势、耗时、连续失败账号等更细指标看板。 +2. 高阶幂等与跨实例协调： + - 多 Gateway 实例下的账号级恢复抢占协调已具备 token 租约和完成校验；后续仍需增强分片级/消息级更细粒度去重，以及长恢复任务中的锁续租。 + - “上游已受理但 submit_resp 永久丢失”的强确认补偿还不完整。 +3. 分片和复杂场景补偿： + - 长短信分片级提交明细、回执和补偿归因已进入真实审计表和运营端短信详情；后续仍需补分片级自动重投/人工重投。 + - 共享接入号、多候选普通上行已具备第一版人工认领和认领后下游投递；后续仍需补批量认领、人工认领复核和指标分析。 +4. 运营观测与指标： + - 窗口利用率、连接级心跳、恢复吞吐、恢复耗时趋势等指标尚未回写到运营端真实页面。 + +这些缺口未补齐前，可以把客户 connect/login、IP 白名单、客户 submit 入平台、NestJS 业务校验、Gateway 上游连接池 submit、窗口满等待、长短信基础拆分/重组、submit_resp、receipt/uplink 入库、分片级提交/回执/补偿审计、共享接入号上行人工认领，以及在线或重连客户 Deliver 推送、Gateway 重启后的 pending 恢复、恢复状态入库、运营端恢复状态独立页/详情/导出和失败分类分布作为第一版真实链路验收；不能把连接级指标回写、分片级自动重投、批量认领和认领指标分析作为“生产已验收通过”。 ### 4.9 回执与上行 diff --git a/docs/system-functional-test-cases.md b/docs/system-functional-test-cases.md index ae33c74..af3f63a 100644 --- a/docs/system-functional-test-cases.md +++ b/docs/system-functional-test-cases.md @@ -238,12 +238,13 @@ - 优先级：P0 - 前置条件：运营管理员已登录。 - 步骤： - 1. 创建 CMPP 通道，填写网关地址、端口、账号、密码密文、接入号、限速。 + 1. 创建 CMPP 通道，填写网关地址、端口、账号、密码密文、接入号、限速、期望连接数和提交窗口。 2. 查询通道列表。 3. 停用通道后创建发送任务。 - 预期结果： - 通道协议默认为 CMPP，版本默认为 3.0。 - 通道限速保存正确。 + - 通道真实保存 `desiredConnections/windowSize`，后续 Gateway `ConnectChannel` 与 `SubmitCommand.upstream` 使用该配置。 - 停用通道不会被路由选中。 ### TC-ADMIN-004 通道组与路由规则 @@ -479,13 +480,14 @@ 1. 打开运营端企业应用管理，点击新增短信应用。 2. 在第一步选择企业下拉框中查看企业选项、加载态和空态。 3. 选择企业后进入应用参数表单。 - 4. 配置应用名称、客户单价、IP 白名单、发送队列等级、移动/联通/电信通道组后保存。 + 4. 配置应用名称、客户单价、IP 白名单、发送队列等级、CMPP 6 位账号、客户最大连接数、客户提交窗口、移动/联通/电信通道组后保存。 5. 刷新列表并打开编辑页。 - 预期结果： - 企业选择使用项目通用 Select/下拉控件，样式、禁用态、错误态与系统其他下拉一致。 - 企业选项来自真实企业 API，不使用静态数组、mock 或 localStorage。 - - 请求体包含 tenantId、queuePriority、客户单价、IP 白名单和通道组绑定。 - - 后端真实保存应用队列等级，刷新列表和编辑页后仍显示正确。 + - 请求体包含 tenantId、queuePriority、客户单价、IP 白名单、`cmppAccount`、`cmppMaxConnections`、`cmppWindowSize` 和通道组绑定。 + - `cmppAccount` 可显式填写 6 位数字；留空时由后端自动生成唯一账号；重复或非法格式保存失败并提示可读错误。 + - 后端真实保存应用队列等级和 CMPP 参数，刷新列表、编辑页和 CMPP 参数弹窗后仍显示正确。 - 不选择任何通道组或缺少必填字段时不能保存，并显示可读提示。 ### TC-ADMIN-019 通道连接日志展示 @@ -994,6 +996,337 @@ - submit 被接受后返回 CMPP SubmitResp 成功，并在真实数据库创建 `sourceType=cmpp` 的发送记录，进入真实发送链路。 - submit 内容不匹配审核模板、余额不足、无可用通道时返回明确失败，不得伪造成功。 +### TC-GW-007 CMPP 客户到上游 SMSC 完整闭环 + +- 优先级：P0 +- 前置条件：企业已认证通过；短信应用 active 且配置独立 `cmppAccount/passwordCipher/IP 白名单`；签名、模板、报备、余额、通道组、通道连接状态均满足发送；Go Gateway、NestJS API、Redis、PostgreSQL 均使用真实本地或生产验证服务；上游使用真实 SMSC 测试环境或本地 gocmpp 模拟 SMSC。 +- 步骤： + 1. 客户 CMPP 客户端连接 Gateway `17890` 并完成 bind/login。 + 2. 客户分别提交匹配模板的短短信 SubmitReq，以及超过 140 字节、需要 CMPP UDH 分片的长短信 SubmitReq。 + 3. NestJS 入站接口执行应用状态、企业认证、IP 白名单、手机号、模板、签名报备、余额、黑名单、风控、运营商识别、通道组路由和通道连接可用性校验。 + 4. NestJS 生成真实发送记录、SubmitCommand 和 SmsSubmitRecord，将 SubmitCommand 写入 Redis Stream `gateway.submit.commands`，同时保留 BullMQ 审计/兼容投递。 + 5. Go Gateway submit worker 通过 consumer group 独立消费 SubmitCommand，使用其中的上游通道连接参数连接 SMSC，发送真实 CMPP Submit；长短信应按 6 字节 UDH 分片，设置 `PkTotal/PkNumber/TpUdhi` 并逐包提交。 + 6. 上游 SMSC 返回 SubmitResp，Gateway 回调 NestJS `SubmitResult`。 + 7. 上游 SMSC 下发 deliver receipt，Gateway 解析为 `ReceiptEvent`，NestJS 入库并更新最终状态。 + 8. NestJS 调用 Gateway `/downstream/receipt`，Gateway 向仍在线的客户连接下发 CMPP Deliver Receipt。 + 9. 上游 SMSC 下发普通 deliver 上行；长上行使用 UDH 分片乱序下发时，Gateway 应等待分片齐全后重组成一条 `UplinkEvent`，NestJS 入库。 + 10. NestJS 调用 Gateway `/downstream/uplink`，Gateway 对可关联 messageId 且客户仍在线的上行下发普通 CMPP Deliver。 + 11. 客户断开 CMPP 连接后再次产生 receipt/uplink，确认 NestJS 写入客户侧待投递记录。 + 12. 客户重新 bind/login，Gateway 按账号拉取 pending 投递并补发，成功后回写 delivered。 +- 预期结果： + - 客户 bind/login 使用真实数据库账号、密码、状态和 IP 白名单校验。 + - 业务校验失败时不调用上游 submit，不扣费，不伪造成功。 + - API 入队后不依赖同步调用 Gateway `/upstream/submit`；Gateway 停止时命令留在 Redis Stream，Gateway 恢复后继续消费。 + - 长短信 Submit 每个 CMPP 分片长度不超过 140 字节，分片 UDH 正确；所有 accepted 分片的上游 `MsgId` 均可映射回同一平台消息。 + - 上游 submit accepted 后只按应用客户费率扣费一次；submit rejected/timeout 进入补发或释放冻结。 + - deliver failed 触发补发或最终退款；重复/迟到回执不重复扣费或退款。 + - 客户在线且 Gateway 仍保留 messageId 或账号会话时，可以收到最终 Deliver Receipt 和可关联上行 Deliver。 + - 客户断线或 Gateway 控制面暂不可达时，`CmppDownstreamDelivery` 保留 pending、retryCount、nextRetryAt、lastError；客户重连后可补发并标记 delivered。 + - 长上行分片未齐全前不入库不推送；分片齐全后只入库一条完整上行内容，且可继续执行 messageId、接入号或手机号时间窗口匹配。 + - 无 messageId 上行优先按接入号匹配应用；接入号无法唯一匹配时按手机号和时间窗口匹配；多候选标记 ambiguous，不误推；完全匹配不到标记 unmatched 但仍入库。 + - 后台周期重试、死信队列、过期策略和人工认领流程需按后续用例验收。 + +### TC-GW-008 Gateway 多连接窗口与窗口满等待 + +- 优先级：P0 +- 前置条件：运营端通道配置 `desiredConnections=2`、`windowSize=1`，上游使用可延迟 SubmitResp 的真实测试 SMSC 或本地 gocmpp 模拟 SMSC；NestJS API、Redis、PostgreSQL、Go Gateway 均运行真实服务。 +- 步骤： + 1. 通过真实发送链路连续提交 3 条可通过业务校验的短信，保证前 2 条 SubmitResp 暂不返回。 + 2. 观察 `SubmitCommand.upstream` 是否携带 `desiredConnections=2` 和 `windowSize=1`。 + 3. 观察 Gateway 是否为同一通道建立 2 条上游 CMPP 连接，并将前 2 条短信分别占用两个连接窗口。 + 4. 第 3 条短信在两个窗口均满时等待，不得越过窗口容量继续 submit。 + 5. 释放任一 SubmitResp 后，确认第 3 条短信继续提交。 +- 预期结果： + - Gateway 每条连接独立维护 sequence/pending 映射，SubmitResp 可正确回到原 `messageId/submitId/channelId`。 + - 窗口满时消息等待可用窗口；等待超过提交超时时返回 timeout，并由 NestJS 进入既有补发或释放冻结逻辑。 + - 多连接窗口只改变 Gateway 提交并发，不绕过 API 侧模板、签名、余额、风控、通道组、通道连接可用性和限速校验。 + - 当前阶段不要求断线后的 pending submit 恢复；该能力在后续在途恢复用例验收。 + +### TC-GW-009 Gateway 重启后认领 pending SubmitCommand + +- 优先级：P0 +- 前置条件：Redis Stream `gateway.submit.commands` 已创建 consumer group；Gateway worker A 已消费一条 `SubmitCommand` 但尚未 ack；该消息空闲时间超过 pending claim 阈值。 +- 步骤： + 1. 停止或模拟断开 worker A，使该消息留在 PEL。 + 2. 启动 worker B 或重启 Gateway。 + 3. 观察 worker B 在消费新消息前执行 pending claim。 + 4. 观察该消息被重新提交、回调 `SubmitResult`，成功后 ack。 +- 预期结果： + - 空闲超过阈值的 pending `SubmitCommand` 会被新的 consumer 认领，不会永久卡在 PEL。 + - 被认领消息仍走正常发送链路，`messageId/submitId/channelId` 和上游回执映射保持一致。 + - 提交成功后消息从 PEL 移除；提交失败时保留待后续重试或死信治理，不得静默丢失。 + +### TC-GW-010 上游连接断开时 pending submit 立即补偿 + +- 优先级：P0 +- 前置条件：Gateway 已与上游 SMSC 建立连接；某条连接已成功发送 submit，但上游故意不立即返回 submit resp，随后主动断开 TCP 连接。 +- 步骤： + 1. 提交一条可通过真实业务校验的短信，使 Gateway 进入等待 submit resp 状态。 + 2. 在 `defaultSubmitTimeout` 到达前，模拟上游连接断开。 + 3. 观察 Gateway 对该 pending submit 的处理，以及 NestJS 收到的 `SubmitResult`。 +- 预期结果： + - Gateway 不会一直等到固定超时才处理，而是立即将该 pending submit 补偿为 `timeout`，错误码为 `CONNECTION_LOST` 或等价可读值。 + - NestJS 收到 `SubmitResult` 后走既有补发或释放冻结逻辑，短信状态不会永久卡在 `submit_queued`。 + - 同一连接上的其他 pending submit 也会被明确唤醒，不会静默丢失。 + +### TC-GW-011 submit_resp 丢失后按 receipt 保守归因 + +- 优先级：P0 +- 前置条件：某条短信提交上游后，Gateway 因连接断开或 submit resp 丢失将该次提交记为 `timeout`，对应 `sms_submit_record.gatewayMessageId` 仍为空；后续上游真实回执已到达，receipt 中带有该手机号、通道和运营商侧 `MsgId`。 +- 步骤： + 1. 构造一条无法按平台 `messageId/gatewayMessageId` 精确命中的 receipt 事件，但带上真实手机号、通道和运营商侧 `MsgId`。 + 2. 保证同通道、同手机号、72 小时窗口内只有 1 条 `timeout + gatewayMessageId=null` 的 submit 记录。 + 3. 观察 Gateway 是否将手机号一并上送 NestJS，并检查 NestJS 的归因与入库结果。 + 4. 再构造“多候选”场景，重复执行同类 receipt 归因。 +- 预期结果： + - 只有在唯一候选成立时，NestJS 才接收该 receipt，并回填真实 `sms_submit_record.gatewayMessageId/sequenceId`，同时写入 `sms_receipt_record` 和 `sms_message_record`。 + - 如果存在多条候选或无候选，则拒绝归因，不得误绑到其他短信。 + - 已经由新通道成功送达的短信，旧尝试迟到回执仍只记历史，不覆盖最终送达状态。 + +### TC-GW-012 SubmitCommand 死信入库与人工重入队 + +- 优先级：P0 +- 前置条件：Gateway submit worker 已连接 Redis Stream `gateway.submit.commands`；准备一条会持续触发处理错误的 `SubmitCommand`；NestJS `/gateway/events/dead-letter` 和运营端 `/api/admin/operations/gateway-submit-dead-letters` 真实可用。 +- 步骤： + 1. 让同一条 `SubmitCommand` 连续处理失败，达到 Gateway 配置的死信阈值。 + 2. 检查 Redis PEL 中该消息是否被 ack，不再无限 pending。 + 3. 检查 NestJS 是否在真实数据库写入一条 `GatewaySubmitDeadLetter`，保存失败原因、尝试次数和原始命令载荷。 + 4. 调用运营端真实接口查询死信列表。 + 5. 调用人工重入队接口，将该死信重新写回 `gateway.submit.commands`。 +- 预期结果： + - 达到阈值后，Gateway 会把该消息转为死信，而不是永久卡在 PEL。 + - 死信记录来自真实数据库，包含 `streamMessageId`、`messageId/submitId`、失败原因、尝试次数和原始 `SubmitCommand`。 + - 人工重入队成功后，死信状态更新为 `requeued`，记录新的 Redis Stream 消息 ID，并写系统日志。 + - 重入队后如后续收到真实 `SubmitResult`，对应死信记录应自动转为 `resolved`。 + +### TC-GW-013 下游客户在线时周期补投与失败封顶 + +- 优先级：P0 +- 前置条件：客户应用 CMPP 账号真实可登录；平台已有至少 1 条 `CmppDownstreamDelivery.status=pending` 的回执或上行待投递记录；Gateway 周期补投任务已启动。 +- 步骤： + 1. 让客户先断线，制造一次投递失败，确认 `CmppDownstreamDelivery` 留在 `pending` 且 `retryCount` 递增。 + 2. 让客户重新 bind，检查 Gateway 是否立即拉取一次 pending 进行补发。 + 3. 在客户保持在线的情况下，继续制造一次临时投递失败，等待周期补投触发。 + 4. 把同一条待投递连续失败到重试上限。 +- 预期结果： + - 客户重连后会立即补发 pending 记录；客户在线但上次投递失败时，Gateway 会按周期再次拉取并补投。 + - 重试未超过上限时，`CmppDownstreamDelivery` 维持 `pending`，更新 `retryCount/nextRetryAt/lastError`。 + - 达到上限后，记录转为 `failed`，不再无限 pending，且写真实失败审计日志。 + - 该能力只负责“客户已在线时的平台补投”和“消息不丢”；客户断线后的重新建链仍由客户系统自己负责。 + +### TC-GW-014 运营端下游投递记录查询与人工重投 + +- 优先级：P1 +- 前置条件：数据库中已有 `CmppDownstreamDelivery` 记录，至少覆盖 `pending`、`failed`、`delivered` 三类状态；运营端已登录；Gateway 控制面和 NestJS API 均为真实服务。 +- 步骤： + 1. 进入运营端“下游投递记录”页面。 + 2. 分别按状态、投递类型、应用、关键字进行筛选，检查分页。 + 3. 打开一条记录详情，核对 payload、重试次数、最后错误和时间字段。 + 4. 对一条 `pending` 或 `failed` 记录执行人工重投。 +- 预期结果： + - 页面列表来自真实 `/api/admin/operations/downstream-deliveries`，不是前端静态数组或本地状态拼装。 + - 详情展示真实 payload、`retryCount/nextRetryAt/deliveredAt/lastError`。 + - 人工重投调用真实 `/api/admin/operations/downstream-deliveries/{id}/requeue`，由后端实际触发 Gateway `/downstream/receipt` 或 `/downstream/uplink`。 + - 重投后记录状态、失败原因和系统日志都与真实后端处理结果一致。 + +### TC-GW-015 下游投递指数退避 + +- 优先级：P1 +- 前置条件：存在一条可重复触发失败的 `CmppDownstreamDelivery`；系统已配置真实基础重试间隔和最大退避上限。 +- 步骤： + 1. 连续触发同一条下游投递失败 3 到 4 次。 + 2. 每次失败后记录 `nextRetryAt` 与当前时间的差值。 + 3. 持续失败直到接近退避上限。 +- 预期结果： + - `nextRetryAt` 不是固定 60 秒，而是随失败次数递增。 + - 退避间隔符合基础间隔的 2 倍递增趋势，并在达到最大退避上限后停止继续增大。 + - 达到总重试上限后仍按既有规则转为 `failed`，不会无限重试。 + +### TC-GW-016 下游投递批量重投 + +- 优先级：P1 +- 前置条件：当前页至少有多条 `pending` 或 `failed` 的 `CmppDownstreamDelivery`；运营端“下游投递记录”页面和真实批量重投接口可用。 +- 步骤： + 1. 在页面勾选多条可重投记录。 + 2. 点击“批量重投”。 + 3. 检查后端返回的成功/失败汇总，并刷新列表。 +- 预期结果： + - 页面调用真实 `/api/admin/operations/downstream-deliveries/requeue` 批量接口，不是前端逐条伪造结果。 + - 后端逐条执行真实重投，返回 `total/successCount/failedCount/results`。 + - 成功和失败记录都会保留真实后端状态与错误信息；空选择时接口拒绝执行。 + +### TC-GW-017 下游投递告警聚合 + +- 优先级：P1 +- 前置条件：真实 `CmppDownstreamDelivery` 中准备一批 `pending` 记录，其中部分已超过告警阈值；同时准备一批最近失败的 `failed` 记录。 +- 步骤： + 1. 访问运营端 Dashboard 和右上角通知区域。 + 2. 调用真实 `/api/admin/operations/dashboard/statistics`，核对返回的下游投递告警聚合。 + 3. 点击“下游投递告警”通知，跳转到下游投递记录页进一步筛查。 +- 预期结果： + - Dashboard 返回真实 `downstreamDeliverySummary`，至少包含 `pending/failed/delivered/stalledPending/recentFailed/alertCount`。 + - 右上角通知中的“下游投递告警”数量与真实 Dashboard 聚合一致，不是前端写死值。 + - 点击通知后可以进入真实下游投递记录页继续处理。 + +### TC-GW-018 下游投递 Dashboard 聚合视图 + +- 优先级：P1 +- 前置条件：真实 `CmppDownstreamDelivery` 中存在多应用、多类型和多重试次数的记录，至少覆盖 `receipt/uplink`、`pending/delivered/failed`。 +- 步骤： + 1. 打开运营端“下游投递记录”页面，查看顶部总览卡片、类型分布、重试压力和应用告警排行。 + 2. 调用真实 `/api/admin/operations/downstream-deliveries/dashboard`，核对 `summary/typeBreakdown/retryBuckets/topApplications`。 + 3. 切换应用和类型筛选，确认顶部 Dashboard 与下方记录列表同时切换到同一筛选范围。 +- 预期结果： + - 顶部 Dashboard 必须来自真实聚合接口，不能由当前页列表条目在前端临时汇总。 + - `summary` 中 `total/pending/delivered/failed/stalledPending/recentFailed/alertCount` 与数据库真实结果一致。 + - `typeBreakdown` 能正确区分 `receipt` 和 `uplink` 的状态分布。 + - `retryBuckets` 真实反映 `pending/failed` 记录的重试压力分布。 + - `topApplications` 以告警量优先排序，切换筛选后结果实时刷新。 + +### TC-GW-019 下游在线账号 Presence 持久化 + +- 优先级：P1 +- 前置条件：Gateway 已配置真实 `REDIS_URL`；客户端应用存在可用的 6 位 `cmppAccount`；Gateway inbound 服务可正常接收 bind 和 submit。 +- 步骤： + 1. 使用真实 CMPP 客户端账号 bind Gateway。 + 2. 发送一条 submit，并触发至少一次下游回执或上行下发。 + 3. 检查 Redis 中该账号的下游 presence 记录。 + 4. 断开连接或触发发送失败清理后，再次检查 Redis。 +- 预期结果： + - bind 成功后，Redis 中存在该 `cmppAccount` 的 presence 记录，不再只保存在 Gateway 内存 map。 + - presence 至少包含账号、Gateway 实例标识、最近更新时间等信息。 + - submit 或下游投递后，presence 的最近活跃时间会刷新。 + - 连接清理后，presence 会被删除或过期，不把离线账号长期误判为在线。 + +### TC-GW-020 Gateway 恢复候选视图 + +- 优先级：P1 +- 前置条件：Gateway 已配置真实 `REDIS_URL`；Redis 中已有部分下游账号 presence；另有至少一个账号当前在 Gateway 内存中处于在线状态。 +- 步骤： + 1. 重启 Gateway。 + 2. 观察 Gateway 启动日志中的恢复候选加载结果。 + 3. 调用 `GET /downstream/recovery-candidates`。 + 4. 比对 Redis presence 和当前在线账号，确认返回候选列表。 +- 预期结果： + - Gateway 启动后会读取 Redis presence，不再完全依赖进程内存冷启动。 + - `/downstream/recovery-candidates` 返回恢复候选账号视图，至少包含账号、实例标识、状态、最近更新时间。 + - 当前内存在线账号与 Redis presence 会合并成同一候选视图。 + - 本阶段只提供恢复候选视图，不应误报为“已自动补投所有 pending 下游投递”。 + +### TC-GW-021 Gateway 重启后的 pending 恢复 + +- 优先级：P0 +- 前置条件：真实 `CmppDownstreamDelivery` 中存在某账号的 `pending` 回执或上行；Redis presence 中保留该账号最近在线记录；Gateway 可正常重启。 +- 步骤： + 1. 让客户账号先在线并制造至少一条 `pending` 下游投递。 + 2. 重启 Gateway。 + 3. 检查 Gateway 启动后是否按恢复候选账号重新拉取 pending。 + 4. 若客户已重新 bind，则观察 pending 是否继续投递成功；若客户未重连，则观察记录是否仍保持 `pending`。 +- 预期结果： + - Gateway 重启后会重新尝试按恢复候选账号拉取真实 pending 下游投递。 + - 客户重新 bind 后，pending 回执/上行可继续投递，不依赖重启前的内存连接映射。 + - 若客户尚未重连，记录应继续保留为 `pending`，不能仅因 Gateway 重启或当前无连接就错误转成 `failed`。 + - 恢复执行基于真实后端 `CmppDownstreamDelivery`，不是前端或 Gateway 内存伪造状态。 + +### TC-GW-022 Gateway 恢复退避与状态审计 + +- 优先级：P1 +- 前置条件：Gateway 已配置真实 `REDIS_URL`；存在可恢复账号；能人为制造“客户未重连”或“拉取 pending 失败”等恢复异常。 +- 步骤： + 1. 触发某账号恢复一次，但让客户保持未连接，或让 pending 拉取接口暂时失败。 + 2. 连续观察两轮恢复周期，检查是否出现对同一账号的高频重复恢复。 + 3. 调用 `GET /downstream/recovery-statuses` 查看恢复状态。 + 4. 恢复客户连接后，再次观察状态是否转为成功。 +- 预期结果： + - 同一账号恢复过程中有恢复锁，不能并发重复恢复。 + - 恢复失败或等待连接后会进入退避，下一轮不会无节制重复尝试。 + - `/downstream/recovery-statuses` 能返回真实恢复状态、尝试次数、下一次可恢复时间和错误原因。 + - 客户恢复连接后，后续状态可转为 `success`，而不是长期卡在错误状态。 + +### TC-GW-023 Gateway 恢复总览接口 + +- 优先级：P2 +- 前置条件：Gateway 已配置真实 `REDIS_URL`；恢复候选与恢复状态已有真实数据。 +- 步骤： + 1. 调用 `GET /downstream/recovery-candidates` 和 `GET /downstream/recovery-statuses`。 + 2. 调用 `GET /downstream/recovery-overview`。 + 3. 比较总览接口与两个明细接口返回结果。 +- 预期结果： + - `/downstream/recovery-overview` 同时返回候选账号列表和恢复状态列表。 + - 总览接口中的 `candidates/statuses` 与两个明细接口真实结果一致，不允许返回静态拼装样例。 + - 运维可仅通过总览接口快速判断“哪些账号待恢复、哪些账号处于退避或错误状态”。 + +### TC-GW-024 恢复状态回流与运营端展示 + +- 优先级：P1 +- 前置条件：Gateway 已产生至少一条真实恢复状态；NestJS API、PostgreSQL 和运营端页面可访问。 +- 步骤： + 1. 触发某账号恢复状态变化，例如 `waiting_connection`、`success` 或 `failed`。 + 2. 检查 Gateway 是否调用 `/api/gateway/events/downstream/recovery-status`。 + 3. 查询数据库 `GatewayDownstreamRecoveryStatus`。 + 4. 打开运营端“恢复状态管理”页面，查看恢复摘要、恢复状态列表和详情弹窗。 + 5. 按失败分类筛选，例如“客户未连接”“退避等待”“恢复执行失败”。 + 6. 使用当前筛选条件执行 CSV 导出。 +- 预期结果： + - 恢复状态会从 Gateway 真实回流到 NestJS，并持久化到 PostgreSQL，不只停留在 Redis 或 Gateway 控制面。 + - `GatewayDownstreamRecoveryStatus` 至少能查到账号、状态、失败分类、尝试次数、下一次恢复时间、错误原因、应用和企业关联。 + - 运营端页面展示的数据来自真实 API/数据库，不是前端本地拼装；详情接口返回字段与数据库一致。 + - 失败分类分布来自后端聚合，筛选后列表与统计同步变化。 + - 导出文件来自真实后端接口，包含失败分类字段，内容与当前筛选结果一致。 + - 页面刷新后恢复状态仍然存在，可继续用于生产排查。 + +### TC-GW-025 多 Gateway 恢复抢占协调 + +- 优先级：P0 +- 前置条件：Redis 使用真实实例；至少准备两个不同 `GATEWAY_INSTANCE_ID` 的 Gateway 进程或可重复调用恢复锁逻辑的测试环境。 +- 步骤： + 1. Gateway-A 对同一 `cmppAccount` 发起 pending 恢复，获取 Redis token 租约锁。 + 2. 在 Gateway-A 未完成前，Gateway-B 对同一账号发起恢复，应被识别为 `lock_contended`。 + 3. 等待 Gateway-A 锁过期后，Gateway-B 再次发起恢复，应能接管并获得新的锁 token。 + 4. 模拟 Gateway-A 迟到完成恢复。 + 5. 查看 Redis 锁、Gateway 恢复状态、NestJS `GatewayDownstreamRecoveryStatus` 和运营端“恢复状态管理”详情。 +- 预期结果： + - 同一账号同一时间只能由一个 Gateway 实例持有恢复锁。 + - 迟到的旧实例完成恢复时，因 token 不匹配不能释放新实例锁，也不能覆盖新实例恢复状态。 + - 锁冲突、锁丢失等场景会以 `failureCategory=lock_contended` 或 `lock_lost` 进入真实恢复状态。 + - `lockOwner/lockExpiresAt` 会回流到 PostgreSQL，并在运营端详情/列表中可见。 + +### TC-GW-026 长短信分片补偿审计 + +- 优先级：P0 +- 前置条件：真实 PostgreSQL、Redis、NestJS API、Go Gateway 和上游 SMSC 模拟器均已启动；存在可用企业、应用、签名、模板、通道组和在线上游通道。 +- 步骤： + 1. 通过客户 CMPP 或发送接口提交一条需要 UDH 分片的长短信。 + 2. 确认 Gateway 向上游 SMSC 逐片提交，并在 `SubmitResult.segments[]` 中回传每个分片的 `segmentIndex/sequenceId/gatewayMessageId/submitStatus/submittedAt`。 + 3. 查询数据库 `SmsMessageSegmentAudit`，确认同一平台短信记录下有对应分片审计行。 + 4. 模拟部分分片或全部分片回执，检查 NestJS 是否按 `gatewayMessageId` 回填分片回执状态。 + 5. 对同一短信触发重投或补偿提交，检查新 `submitId` 的分片审计是否保留历史 attempt 与 `compensationType`。 + 6. 打开运营端短信记录详情，查看“分片补偿审计”列表。 +- 预期结果： + - 分片审计来自真实 Gateway SubmitResult、NestJS API 和 PostgreSQL，不允许前端静态拼装。 + - 每个分片至少记录分片序号、总片数、submitId、sequenceId、上游 MsgId、提交状态和提交时间。 + - 回执按分片上游 MsgId 回填到对应审计行，迟到或失败回执不得覆盖短信最终 delivered 状态。 + - 重投/补偿产生的新 submitId 与历史 submitId 可以并存审计，运营端能看到补偿归因。 + - 当前第一版只要求分片级审计可追踪；按单个分片自动重投和分片级人工重投另行验收。 + +### TC-GW-027 共享接入号上行人工认领 + +- 优先级：P0 +- 前置条件：真实 PostgreSQL、NestJS API、Go Gateway 和客户侧下游 CMPP 连接可用；至少两个应用共享同一接入号或同一手机号时间窗口内存在多条候选下发记录。 +- 步骤： + 1. 模拟一条不携带 messageId 的普通上行 Deliver，接入号或手机号时间窗口可匹配多个应用/下发记录。 + 2. 检查 `SmsUplinkMessage.matchStatus` 是否为 `ambiguous`，并查询 `SmsUplinkMatchCandidate` 候选。 + 3. 打开运营端“短信上行记录”详情，查看候选企业、候选应用、候选来源、置信度、候选下发短信和候选原因。 + 4. 选择正确候选执行“认领并推送”。 + 5. 查询 `SmsUplinkMessage`、`SmsUplinkMatchCandidate`、`CmppDownstreamDelivery` 和操作日志。 + 6. 若客户 CMPP 连接在线，检查 Gateway 是否尝试向认领应用下发普通上行 Deliver；若客户离线，检查待投递记录是否保留 pending/failed 重试状态。 +- 预期结果： + - 多候选上行不会误推给任意客户应用，必须先进入 `ambiguous` 并保留候选。 + - 候选来自真实接入号路由或手机号时间窗口下发记录，不允许前端静态生成。 + - 人工认领后上行记录更新为 `matched`，写入 tenant/application/messageRecord 关联。 + - 被选候选状态变为 `claimed`，其他 pending 候选变为 `rejected`，认领动作写入操作日志。 + - 认领后创建真实 `CmppDownstreamDelivery(deliveryType=uplink)`，并按现有下游投递链路在线推送或离线保留重试。 + ### TC-SEND-021 优先队列插队发送 - 优先级：P0 diff --git a/docs/testing-progress.md b/docs/testing-progress.md index cc12047..70af8db 100644 --- a/docs/testing-progress.md +++ b/docs/testing-progress.md @@ -606,6 +606,500 @@ npm run verify:phase8 - 下游 submit 当前通过 `sourceType=cmpp` 的系统批次兼容承载，尚未完全拆成独立单条发送模型。 - 客户侧最终 Deliver Receipt 投递、客户侧上行 Deliver 推送、上游真实 SMSC submit worker、上游 receipt/uplink 生产解析仍未完成。 +## 2026-07-07 Gateway 上游提交与下游 Deliver 闭环补齐 + +### 本轮修复 + +- `SubmitCommand` 契约、示例和 Go 结构增加 `upstream.gatewayHost/gatewayPort/account/passwordCipher/cmppVersion`，API 发送链路在真实业务校验通过后保留 BullMQ 审计投递，同时写入 Redis Stream `gateway.submit.commands` 主命令流。 +- Go Gateway 新增上游提交管理器，按通道建立/复用 gocmpp 客户端连接，发送真实 CMPP Submit，接收 SubmitResp，并回调 NestJS `SubmitResult`。 +- Go Gateway 上游读循环开始处理 deliver receipt 和普通 deliver 上行：receipt 解析后回调 NestJS `/gateway/events/receipt`，普通上行解码后回调 `/gateway/events/uplink`。 +- Gateway 下游入站服务记录客户 Submit 对应的 messageId 到在线客户连接映射；NestJS 收到最终 receipt/uplink 并入库后调用 Gateway `/downstream/receipt`、`/downstream/uplink`，Gateway 向在线客户下发 CMPP Deliver Receipt 或普通 Deliver。 +- `api/src/send-chain/send-chain.service.spec.ts` 覆盖 SubmitCommand 上游配置和 Redis Stream 发布；`gateway/internal/inbound/server_test.go` 覆盖客户 submit 后平台下发 Deliver Receipt；Gateway 契约示例覆盖新 upstream 字段。 + +### 验证状态 + +- `npm --prefix api test -- send-chain.service.spec.ts`：通过。 +- `npm --prefix api test`：通过，12 个 suites、82 个 tests。 +- `npm --prefix api run build`：通过。 +- `go test ./...`（Gateway）：通过。 +- `npm run spike:contracts`：通过，4 个 Gateway 队列契约示例通过。 + +### 剩余缺口 + +- Gateway 控制面 `/upstream/submit` 仅保留为调试/补偿入口；生产主链路由 Go Gateway submit worker 消费 Redis Stream `gateway.submit.commands` 触发。worker 当前覆盖新消息 `>` 消费和 ack，pending 历史消息扫描与精细重试治理放入后续在途恢复阶段。 +- 客户侧 Deliver Receipt/上行 Deliver 当前依赖 Gateway 内存在线连接映射；客户断线、Gateway 重启或映射丢失时尚未实现持久化缓存、重试和投递失败审计。 +- 普通上行只有能关联 messageId 的事件可推送给客户；仅按接入号、手机号、应用和时间窗口匹配客户连接仍待产品化。 +- 长短信拆分/重组、多连接窗口、窗口满、在途消息恢复、断线重连后的状态补偿仍待后续实现和压测。 + +## 2026-07-07 阶段 1：Gateway SubmitCommand 独立消费 + +### 本轮修复 + +- NestJS SendChain 取消主链路同步调用 Gateway `/upstream/submit`；真实业务校验通过后创建 SmsSubmitRecord、保留 BullMQ `gateway.submit.queue` 审计/兼容投递，并向 Redis Stream `gateway.submit.commands` 写入 `SubmitCommand`。 +- Go Gateway 新增 `submitworker`，启动时默认创建/复用 consumer group `cmpp-gateway`，独立消费 Redis Stream 中的 `SubmitCommand`，调用同一个上游提交管理器真实 submit 到上游 SMSC。 +- Gateway `/upstream/submit` 保留为调试/运维补偿接口，不作为 API 主发送路径。 +- Gateway worker 支持环境变量：`REDIS_URL`、`GATEWAY_SUBMIT_STREAM`、`GATEWAY_SUBMIT_GROUP`、`GATEWAY_SUBMIT_CONSUMER`、`GATEWAY_SUBMIT_WORKER_DISABLED=true`。 + +### 验收口径 + +- API 入队后不再因为 Gateway 控制面短暂不可达而自己生成 timeout；SubmitResult 必须由 Gateway worker 真实消费和提交后回调。 +- Gateway 停止时，SubmitCommand 留在 Redis Stream；Gateway 恢复后由 consumer group 继续消费新消息。 +- BullMQ `gateway.submit.queue` 仅作为审计/兼容，不再是唯一主提交通道。 + +### 剩余边界 + +- 当前 worker 先覆盖新消息 `>` 消费和 ack；pending 历史消息扫描、claim、重试退避和死信审计放到在途恢复阶段继续做。 + +## 2026-07-07 阶段 2/3：客户侧 Deliver 持久化重投与普通上行匹配 + +### 本轮修复 + +- Prisma 新增 `CmppDownstreamDelivery`，用于保存客户侧待投递 Deliver Receipt 和普通 Deliver 上行；状态覆盖 pending/delivered，记录 retryCount、nextRetryAt、lastError、payload、message/application 关联。 +- `SmsUplinkMessage` 增加 `applicationId`、`messageRecordId`、`matchStatus`、`matchReason`，并建立应用和匹配下发记录关系。 +- NestJS 收到最终 receipt 后，先写平台回执和消息状态，再创建客户侧待投递记录，尝试调用 Gateway `/downstream/receipt`；成功标记 delivered，客户不在线或 Gateway 不可达时保留 pending 并记录失败原因。 +- NestJS 收到普通上行后执行匹配：messageId 精确匹配优先；无 messageId 时按接入号匹配应用路由；仍无唯一应用时按手机号和最近下发时间窗口匹配；多候选标记 ambiguous，未匹配标记 unmatched，但均真实入库。 +- Gateway 下游客户 bind/login 成功后保存账号级在线连接，并调用 NestJS `/gateway/events/downstream/pending` 拉取 pending 投递；补发成功后回调 `/gateway/events/downstream/delivered`，失败回调 `/gateway/events/downstream/failed`。 +- 运营/客户端上行查询 include 应用和匹配下发记录，便于页面展示 matchStatus/matchReason。 + +### 验证状态 + +- `npm --prefix api run prisma:generate`：通过。 +- `npm --prefix api test -- send-chain.service.spec.ts`：通过。 +- `npm --prefix api test`：通过，12 个 suites、82 个 tests。 +- `npm --prefix api run build`：通过。 +- `go test ./...`（Gateway）：通过。 +- `npm run spike:contracts`：通过。 +- `npm run build`：通过，仅既有 Vite chunk size warning。 + +### 剩余边界 + +- 待投递 pending 目前在客户 bind/login 时拉取补发；后台周期扫描、指数退避、过期策略、死信队列和运营端失败审计页面仍待后续实现。 +- 上行匹配已覆盖 messageId、接入号和手机号时间窗口；共享接入号、多应用多候选时不会误推，但人工认领/改派流程尚未实现。 +- 客户连接断开检测和应用级连接数状态回写仍需继续产品化。 + +## 2026-07-08 阶段 4：Gateway 长短信拆分与长上行重组 + +### 本轮修复 + +- Go Gateway 上游 Submit 支持长短信第一版拆分：超过 140 字节的短信按 CMPP 标准 6 字节 UDH 生成分片，每片总长度不超过 140 字节，并设置 `PkTotal/PkNumber/TpUdhi` 后逐包发送到上游 SMSC。 +- 同一平台 `SubmitCommand` 的多个 accepted 分片 `MsgId` 均登记到 Gateway 映射表，后续任一分片 receipt 可回溯到原 `messageId/submitId/channelId`。 +- Go Gateway 上游普通 Deliver 支持长上行第一版重组：收到 `TpUdhi=1` 且携带标准 UDH 的分片时，按通道、主叫、被叫、引用号和总片数缓存；分片齐全后只回传一条完整 `UplinkEvent` 给 NestJS。 +- 新增 `gateway/internal/upstream/long_message_test.go`，覆盖 UCS2 长短信拆分、短短信不分片、长上行乱序重组。 + +### 验证状态 + +- `go test ./...`（Gateway）：通过。 + +### 剩余边界 + +- 阶段 4 后长短信仍按单条平台消息记录展示，尚未提供运营端分片级提交明细、分片级补发审计和部分分片失败后的精细补偿；该审计缺口已在阶段 23 补齐第一版。 +- 长上行分片缓存当前为 Gateway 进程内内存；Gateway 重启、跨连接分片漂移或超过缓存 TTL 的残片不会恢复，后续在“在途消息恢复/状态补偿”阶段继续做。 + +## 2026-07-08 阶段 5：Gateway 多连接窗口与窗口满控制 + +### 本轮修复 + +- `SubmitCommand.upstream` 契约、示例、Go 结构和 NestJS 生产者增加 `desiredConnections/windowSize`，字段来自通道真实配置；未配置时默认 `desiredConnections=1`、`windowSize=16`。 +- Go Gateway 上游提交管理器从单连接升级为通道级连接池：同一通道按 `desiredConnections` 建立多条 CMPP 客户端连接，每条连接独立维护 submit pending、receipt/uplink 映射和长上行分片缓存。 +- 每条上游连接增加窗口令牌；提交前必须获得窗口，SubmitResp、reject 或 timeout 后释放窗口；所有连接窗口均满时等待可用窗口，超过提交超时时返回 `WINDOW_TIMEOUT`。 +- 长短信分片也复用连接池窗口调度，同一条平台消息的多个 accepted 分片仍映射回原 `messageId/submitId/channelId`。 +- 新增 `gateway/internal/upstream/pool_test.go`，覆盖连接池跨连接获取窗口、窗口满拒绝继续占用、释放后可重新获取。 + +### 验证状态 + +- `go test ./...`（Gateway）：通过。 +- `npm --prefix api test -- send-chain.service.spec.ts`：通过。 + +### 剩余边界 + +- 当前窗口状态为 Gateway 进程内控制，尚未把连接级窗口占用、等待队列长度、submit latency 等指标回写到 NestJS 或运营端页面。 +- 当前阶段只处理窗口容量和多连接发送；Gateway 重启、上游连接断开时的在途 submit 恢复、pending claim、状态补偿和死信审计仍在下一阶段处理。 + +## 2026-07-08 阶段 6：CMPP 配置入口补齐 + +### 本轮修复 + +- 运营端通道创建/编辑表单新增上游 `desiredConnections` 和 `windowSize` 输入，真实提交到 NestJS 通道 API，并规范化写入 `SmsChannel.config`。 +- NestJS `ChannelsService` 对 `desiredConnections/windowSize` 增加正整数校验；通道激活后的 `ConnectChannel` 请求和发送链路 `SubmitCommand.upstream` 均复用该真实配置。 +- Prisma 为 `SmsApplication` 新增 `cmppMaxConnections`、`cmppWindowSize` 字段；运营端短信应用创建/编辑表单新增 `cmppAccount`、客户最大连接数、客户提交窗口输入。 +- 企业应用 `cmppAccount` 现在支持两种真实路径：显式填写 6 位数字账号，或留空由后端自动生成唯一账号；重复账号和非法格式会被后端拒绝。 +- 企业应用 CMPP 参数接口改为从应用真实字段返回 `account/maxConnections/windowSize`，不再借用任意通道默认值拼装客户参数。 + +### 验证状态 + +- `npm --prefix api run prisma:generate`：通过。 +- `npm --prefix api test -- sms-config.service.spec.ts channels.service.spec.ts`：通过，2 个 suites、34 个 tests。 +- `npm --prefix api run build`：通过。 +- `npm run build`：通过，仅既有 Vite chunk size warning。 + +### 说明 + +- `desiredConnections/windowSize` 不是 CMPP 协议标准字段，也不是 gocmpp 的原生配置项；它们是本平台对上游通道连接池和提交窗口的运行参数。 +- `cmppAccount` 是客户侧应用接入账号；当前已支持真实生成、真实保存和显式配置。 + +## 2026-07-08 阶段 7：SubmitCommand 在途恢复第一步 + +### 本轮修复 + +- Go Gateway `submitworker` 在正常消费新消息前新增 pending 恢复流程：对 Redis Stream consumer group 中空闲超过阈值的消息执行 `XAUTOCLAIM`，将滞留在 PEL 的 `SubmitCommand` 认领到当前 consumer。 +- 被认领的 pending 命令复用现有 `handleMessage -> Upstream.Submit -> XAck` 成功路径处理；成功后 ack，失败时保留在 PEL，留给后续重试/死信治理。 +- `submitworker` 增加可注入 `Submit` 函数，便于单测覆盖消息处理路径；新增单测覆盖 injected submit 和默认 `minIdle` 阈值。 + +### 验证状态 + +- `go test ./...`（Gateway）：通过。 + +### 剩余边界 + +- 当前恢复能力只覆盖 Redis Stream PEL 中“已被读走但未 ack”的 pending 命令；尚未实现恢复次数上限、死信队列、失败审计页面和人工补偿入口。 +- Gateway 重启时上游连接内已经发出但尚未收到 submit resp 的 in-flight CMPP 请求，仍未完成状态补偿；这部分继续放在后续“断线重连后的消息状态处理”阶段。 + +## 2026-07-08 阶段 8：上游连接断开时 pending submit 补偿 + +### 本轮修复 + +- Go Gateway 上游连接读循环开始区分“空读超时”和“真实连接断开”；空读超时继续等待，真实断开则进入连接丢失处理。 +- 某条上游连接断开时，Gateway 会把该连接上所有等待 submit resp 的 pending submit 立即唤醒，返回 `timeout` + `CONNECTION_LOST`，不再机械等待固定 `SUBMIT_TIMEOUT`。 +- 连接池在再次分配连接前会重新执行 `ensureConnected()`；旧连接断开后，后续新消息可重新建立物理连接继续提交。 +- 新增 `gateway/internal/upstream/connection_loss_test.go`，覆盖 pending submit 被唤醒和临时读超时识别。 + +### 验证状态 + +- `go test ./...`（Gateway）：通过。 +- `npm --prefix api test -- send-chain.service.spec.ts`：通过。 + +### 剩余边界 + +- 当前补偿只覆盖“连接断开且 submit resp 尚未返回”的场景；尚未覆盖“上游其实已受理，但 submit resp 在断线前后丢失”的二次确认和幂等回查。 +- submit 结果死信队列、失败审计、恢复次数上限和人工补偿入口仍在后续阶段。 + +## 2026-07-08 阶段 9：receipt 驱动的保守二次归因 + +### 本轮修复 + +- Gateway 上游 receipt 事件补充 `phoneNumber`，即使无法从内存 tracker 中精确恢复平台 `messageId`，也会把运营商回执手机号带回 NestJS。 +- NestJS `handleReceipt` 新增保守归因：如果 receipt 无法按平台 `messageId/gatewayMessageId` 精确命中，只在“同通道、同手机号、72 小时窗口内、且仅存在 1 条 `timeout + gatewayMessageId=null` 的 submit 记录”时才接收该回执。 +- 归因成功后会先回填该次 `sms_submit_record.gatewayMessageId/sequenceId`，再写入真实 `sms_receipt_record` 并按既有逻辑更新 `sms_message_record`、下游客户回执推送和幂等保护。 +- 新增 SendChainService 单测，覆盖唯一候选归因成功和多候选拒绝归因两种场景。 + +### 验证状态 + +- `go test ./...`（Gateway）：待本轮统一回归。 +- `npm --prefix api test -- send-chain.service.spec.ts`：待本轮统一回归。 + +### 剩余边界 + +- 当前只做“唯一候选才归因”的保守版本，仍未实现面向运营商或供应商的 submit 结果主动回查。 +- 如果同通道同手机号在窗口内存在多条 timeout 候选，系统会拒绝归因，后续仍需人工补偿或更强的协议级关联键。 + +## 2026-07-08 阶段 10：SubmitCommand 死信治理第一版 + +### 本轮修复 + +- Prisma 新增真实表 `GatewaySubmitDeadLetter`，保存 Gateway SubmitCommand 死信的消息 ID、租户/应用/通道、失败原因、尝试次数、原始命令载荷、人工重入队状态和解决状态。 +- Go Gateway `submitworker` 新增失败次数治理：同一条 Stream 消息处理失败达到阈值后，调用 NestJS `/gateway/events/dead-letter` 入库死信，并对原消息执行 ack，避免它无限滞留在 PEL。 +- Gateway 对非法 `SubmitCommand` 载荷也会直接转死信，防止 poison message 持续阻塞消费。 +- NestJS 新增真实死信接口：Gateway 可上报死信；运营端后端可分页查询 `/api/admin/operations/gateway-submit-dead-letters`；可通过 `/api/admin/operations/gateway-submit-dead-letters/:id/requeue` 将原始 `SubmitCommand` 重新写回 Redis Stream。 +- NestJS 在收到同一 `submitId/messageId` 的后续真实 `SubmitResult` 时，会把对应死信自动标记为 `resolved`。 + +### 验证状态 + +- `npm --prefix api run prisma:generate`：通过。 +- `npm --prefix api test -- send-chain.service.spec.ts operations.service.spec.ts`：通过，2 个 suites、26 个测试通过。 +- `npm --prefix api run build`：通过。 +- `go test ./...`（Gateway）：通过。 + +### 剩余边界 + +- 当前死信治理只提供“达到阈值后入库 + 人工重入队”的第一版，尚未实现后台自动重放、重放节流、过期清理和专门的前端运营页面。 +- 非法载荷死信如果缺少完整 `SubmitCommand`，当前不可人工重放，只能用于审计和人工排查。 + +## 2026-07-08 阶段 11：下游客户在线时周期补投与失败封顶 + +### 本轮修复 + +- 明确责任边界：客户系统负责断线后的重新 bind；平台负责客户不在线或临时投递失败时的消息不丢、待投递保存和补投。 +- Go Gateway 下游入站服务新增在线账号周期补投：除客户 bind 成功后立即拉取 pending 外，Gateway 还会按周期为当前在线账号再次调用 `/gateway/events/downstream/pending`，继续补发未投递成功的 Deliver Receipt/上行 Deliver。 +- Gateway 向下游发送 Deliver 失败时会清理失效的内存会话映射，避免对已失效连接无休止重复尝试。 +- NestJS `markDownstreamDeliveryFailed` 新增失败上限：未超过阈值时继续 `pending` 并推进 `retryCount/nextRetryAt`；达到阈值后转为 `failed`，停止无限重试，并写 `gateway.downstream_delivery_failed` 系统日志。 + +### 验证状态 + +- `npm --prefix api test -- send-chain.service.spec.ts`：通过，1 个 suite、21 个测试通过。 +- `go test ./internal/inbound ./internal/control ./...`（Gateway）：通过。 + +### 剩余边界 + +- 当前周期补投只针对“Gateway 认为客户在线”的账号；尚未实现下游投递失败专门列表、人工重投页面和跨 Gateway 实例共享的客户在线状态。 +- `CmppDownstreamDelivery` 目前仍使用固定重试间隔，尚未实现指数退避、不同消息类型差异化策略和过期归档。 + +## 2026-07-08 阶段 12：下游投递失败审计与人工重投 + +### 本轮修复 + +- 运营端新增真实下游投递查询接口 `/api/admin/operations/downstream-deliveries`，支持按 `tenantId/applicationId/deliveryType/status/keyword` 筛选并分页返回真实 `CmppDownstreamDelivery` 数据。 +- NestJS 新增 `/api/admin/operations/downstream-deliveries/:id/requeue`，可对单条下游投递记录执行人工重投，真实调用 Gateway `/downstream/receipt` 或 `/downstream/uplink`，并写 `gateway.downstream_delivery_requeue` 系统日志。 +- 运营端新增“下游投递记录”页面，列表、详情、筛选和重投均接真实后端，不使用 mock、本地状态或静态数组。 + +### 验证状态 + +- `npm --prefix api test -- send-chain.service.spec.ts operations.service.spec.ts`：通过，2 个 suites、29 个测试通过。 +- `npm --prefix api run build`：通过。 +- `npm run build`：通过，仅有既有 Vite chunk size warning。 + +### 剩余边界 + +- 当前人工重投仍是单条操作，尚未提供批量重投、失败聚合告警和专门的下游投递 Dashboard。 +- 页面侧暂未做自动轮询刷新，需要手动查询或重进页面观察状态变化。 + +## 2026-07-08 阶段 13：下游投递自动退避第一版 + +### 本轮修复 + +- `CmppDownstreamDelivery` 的失败重试从固定 60 秒改为指数退避：基础间隔来自 `CMPP_DOWNSTREAM_RETRY_DELAY_MS`，每次失败按 2 倍递增，并受 `CMPP_DOWNSTREAM_RETRY_MAX_DELAY_MS` 上限约束。 +- 这样在客户长时间离线或网络持续抖动时，平台不会每分钟机械重试同一条下游投递，能更温和地消耗 API、Gateway 和连接资源。 +- 总重试次数上限逻辑保持不变，超过 `CMPP_DOWNSTREAM_MAX_RETRIES` 后仍转 `failed` 并写失败审计。 + +### 验证状态 + +- `npm --prefix api test -- send-chain.service.spec.ts`：通过，新增指数退避单测。 + +### 剩余边界 + +- 当前退避策略还没有加入随机抖动，多个记录在同一时间失败时，后续重试时刻仍可能比较集中。 +- 退避参数当前是全局环境变量，尚未细分到 receipt/uplink 或不同客户应用级别。 + +## 2026-07-08 阶段 14：下游投递批量重投 + +### 本轮修复 + +- 运营端下游投递记录页新增勾选和“批量重投”操作，仅允许对当前页的 `pending/failed` 记录执行批量重投。 +- NestJS 新增真实批量接口 `/api/admin/operations/downstream-deliveries/requeue`，逐条调用既有单条重投逻辑，返回成功/失败汇总，不用前端自行拼结果。 +- SendChainService 新增批量重投结果汇总与空选择拦截单测。 + +### 验证状态 + +- `npm --prefix api test -- send-chain.service.spec.ts`：通过，新增批量重投单测。 +- `npm --prefix api run build`：通过。 +- `npm run build`：待本轮统一回归。 + +### 剩余边界 + +- 当前批量重投只支持“勾选当前页记录”，还不支持“按筛选条件全量重投”或后台异步大批量任务。 +- 批量结果当前以内联提示为主，尚未做专门的批量执行历史与导出。 + +## 2026-07-08 阶段 15：下游投递告警第一版 + +### 本轮修复 + +- `OperationsService.dashboard()` 新增真实下游投递告警聚合 `downstreamDeliverySummary`，统计 `pending/failed/delivered` 总量，以及“积压过久的 pending”和“最近失败”两类告警计数。 +- 运营端右上角通知新增“下游投递告警”，数量直接来自真实 Dashboard 聚合。 +- 运营看板新增下游投递告警摘要卡片，帮助运营从总览页直接感知当前下游投递异常。 + +### 验证状态 + +- `npm --prefix api test -- operations.service.spec.ts`：随定向测试通过。 +- `npm --prefix api run build`：通过。 +- `npm run build`：通过，仅有既有 Vite chunk size warning。 + +### 剩余边界 + +- 当前告警仍是站内聚合提醒，尚未接短信、邮件、企业微信等外部告警通道。 +- 告警口径当前采用全局阈值环境变量，尚未按客户应用、消息类型或时间段细分。 + +## 2026-07-08 阶段 16：下游投递 Dashboard 第一版 + +### 本轮修复 + +- 新增真实接口 `/api/admin/operations/downstream-deliveries/dashboard`，直接按 `CmppDownstreamDelivery` 聚合返回 `summary/typeBreakdown/retryBuckets/topApplications`。 +- 运营端“下游投递记录”页面顶部补上真实 Dashboard 区域，展示投递总量、待投递、已投递、告警、类型分布、重试压力和应用告警排行。 +- Dashboard 筛选范围与页面应用/类型筛选保持一致，不允许由前端只根据当前页列表数据临时拼装。 + +### 验证状态 + +- `npm --prefix api test -- operations.service.spec.ts`：通过。 +- `npm --prefix api run build`：通过。 +- `npm run build`：通过，仅有既有 Vite chunk size warning。 +- `git diff --check`：无空白错误，仅 Windows LF/CRLF 提示。 + +### 剩余边界 + +- 当前 Dashboard 仍偏运营处置视角，尚未补时间趋势、按客户/账号维度的更细颗粒聚合。 +- 应用告警排行当前以 `pending + failed` 为主排序，尚未加入更复杂的权重和 SLA 指标。 + +## 2026-07-08 阶段 17：下游在线账号 Presence 持久化底座 + +### 本轮修复 + +- Gateway inbound 新增 Redis presence store，客户 `cmppAccount` 在 bind 成功、submit 建链和下游回执/上行投递时，会把在线账号状态写入 Redis。 +- presence 数据至少包含 `account/srcId/remoteIp/gatewayInstanceId/state/connectedAt/updatedAt`，并按 TTL 自动过期，避免该状态只存在单进程内存中。 +- Gateway 发送失败触发连接清理时，会同步移除该账号的 Redis presence 记录。 +- 该阶段先完成“在线状态外部化”，尚未宣称“Gateway 重启后 pending 投递自动恢复”已完成；恢复逻辑在后续阶段继续补。 + +### 验证状态 + +- `go test ./internal/inbound/...`：通过。 +- `go test ./cmd/gateway/...`：通过。 + +### 剩余边界 + +- 当前 presence 主要服务于后续恢复能力，Gateway 还未在启动时主动根据 Redis presence 扫描并恢复 pending 投递。 +- 连接断开当前主要依赖发送失败清理和 TTL 过期兜底，尚未建立更完整的显式断线回收机制。 + +## 2026-07-08 阶段 18：Gateway 恢复候选视图 + +### 本轮修复 + +- Gateway 启动时会读取 Redis presence，并输出恢复候选账号加载日志。 +- 新增控制面接口 `GET /downstream/recovery-candidates`，返回 Redis presence 与当前内存在线账号合并后的恢复候选视图。 +- 候选视图当前用于后续恢复逻辑和运维排查，不直接触发 pending 下游投递补发。 + +### 验证状态 + +- `go test ./internal/inbound/...`：通过。 +- `go test ./internal/control/...`：通过。 +- `go test ./cmd/gateway/...`：通过。 + +### 剩余边界 + +- 当前只是“识别谁值得恢复”，还没有执行“把这些账号的 pending 回执/上行自动继续补投”。 +- 候选视图默认按 Redis TTL 和最近活跃时间保留，尚未叠加更复杂的健康判定和跨实例去重策略。 + +## 2026-07-08 阶段 19：Gateway pending 恢复执行第一版 + +### 本轮修复 + +- Gateway 启动时会立即按恢复候选账号执行一次 pending 下游投递恢复扫描。 +- 后续每轮补投周期除扫描当前内存在线账号外，也会继续扫描恢复候选账号，尝试恢复 `CmppDownstreamDelivery.pending`。 +- 当前恢复策略是“能投就投，投不了继续 pending”：若账号尚无可用下游连接，Gateway 不会把记录误标成失败，而是等待客户重连后的后续恢复机会。 + +### 验证状态 + +- `go test ./internal/inbound/...`：通过。 +- `go test ./internal/control/...`：通过。 +- `go test ./cmd/gateway/...`：通过。 + +### 剩余边界 + +- 当前恢复仍按固定扫描周期触发，尚未做更细的按账号退避、恢复批次追踪和恢复告警。 +- 仍未覆盖更复杂的长短信分片恢复、跨实例抢占协调和恢复中的重复投递防抖。 + +## 2026-07-08 阶段 20：Gateway 恢复退避、锁与状态审计 + +### 本轮修复 + +- Gateway 新增账号级恢复锁，避免同一 `cmppAccount` 被并发重复恢复。 +- 恢复失败、等待连接和部分成功场景会写入真实恢复状态，并按指数退避计算下一次可恢复时间，减少无意义高频重试。 +- 控制面新增 `GET /downstream/recovery-statuses`，可查看账号最近恢复状态、尝试次数、下一次重试时间和错误原因。 + +### 验证状态 + +- `go test ./internal/inbound/...`：通过。 +- `go test ./internal/control/...`：通过。 +- `go test ./cmd/gateway/...`：通过。 + +### 剩余边界 + +- 当前恢复状态审计仍停留在 Gateway 控制面和 Redis，尚未同步到运营端页面或 NestJS 持久化审计表。 +- 恢复退避当前按账号统一处理，尚未细分到回执/上行类型、失败类别或跨实例抢占优先级。 + +## 2026-07-08 阶段 21：Gateway 恢复总览与链路缺口收口 + +### 本轮修复 + +- 控制面新增 `GET /downstream/recovery-overview`，一次性返回恢复候选账号和恢复状态，便于生产联调与排查。 +- 需求文档已按当前真实代码重新梳理 CMPP 端到端链路剩余缺口，明确区分“已能验收的真实链路能力”和“尚未产品化完成的恢复审计/指标/复杂补偿能力”。 +- 系统测试用例新增恢复总览接口口径，便于后续生产验证直接对照。 + +### 验证状态 + +- `go test ./internal/control/...`：通过。 +- `go test ./internal/inbound/...`：通过（延续前一阶段验证结果，本轮未改动 inbound 核心分支逻辑）。 +- `go test ./cmd/gateway/...`：通过。 + +### 阶段 21 后剩余真实缺口 + +- 恢复状态仍未写回 NestJS/Prisma/PostgreSQL，运营端暂无真实恢复状态页面。 +- 多 Gateway 实例下更强的恢复抢占协调、分片级补偿审计、共享接入号上行人工认领仍未完成。 +- 连接级窗口利用率、恢复吞吐、恢复失败分布等运营指标仍未进入真实后台页面。 + +## 2026-07-08 阶段 22：恢复状态回流 NestJS 与运营端展示 + +### 本轮修复 + +- NestJS 新增真实恢复状态接收接口 `/api/gateway/events/downstream/recovery-status`。 +- Prisma/PostgreSQL 新增 `GatewayDownstreamRecoveryStatus` 表，按 `cmppAccount` 持久化恢复状态、尝试次数、下一次恢复时间、错误原因及应用/企业关联。 +- 运营端新增独立“恢复状态管理”页面，支持真实恢复状态列表、分页、详情接口和当前筛选结果 CSV 导出。 +- 原“下游投递记录”页面仅保留投递记录与重投能力，不再混放恢复状态列表。 +- 恢复状态新增 `failureCategory` 失败分类字段，Gateway 回传、NestJS 兜底归类并落库，运营端支持分类筛选、分布统计、详情展示和导出。 +- 多 Gateway 恢复抢占协调补强：恢复锁升级为 Redis token 租约，恢复完成时通过 Lua 原子校验 token 后才写状态和释放锁；迟到旧实例不能误删新实例锁。 +- `GatewayDownstreamRecoveryStatus` 新增 `lockOwner/lockExpiresAt`，Gateway 回传并由 NestJS 入库，运营端恢复状态列表和详情可查看锁持有实例。 +- 本地启动脚本补充 `.local-tools\minio.exe` 查找路径，并已验证本机 MinIO 可通过 `npm run start:local:minio` 启动。 + +### 验证状态 + +- `npm --prefix api run prisma:generate`：通过。 +- `npm --prefix api test -- operations.service.spec.ts send-chain.service.spec.ts`：通过。 +- `npm --prefix api run build`：通过。 +- `go test ./internal/inbound/... ./internal/control/... ./cmd/gateway/...`：通过。 +- `npm run build`：通过，仅有既有 Vite chunk size warning。 +- `npm --prefix api run prisma:migrate:deploy`：通过，已应用 `20260708213000_add_recovery_lock_observability`。 +- `npm run start:local:minio`：通过，MinIO API `http://localhost:9000`、Console `http://localhost:9001` 已监听。 + +### 阶段 22 后剩余真实缺口 + +- 恢复状态已回流 NestJS，并已具备独立运营页、详情、导出和第一版失败分类分布；后续仍缺少恢复吞吐、耗时趋势、连续失败账号等更细指标。 +- 多 Gateway 账号级恢复抢占协调已具备 token 租约和完成校验；分片级补偿审计、共享接入号上行人工认领仍未完成。 +- 连接级窗口利用率、连接级心跳、恢复吞吐和恢复耗时等运营指标仍未进入真实后台页面。 + +## 2026-07-08 阶段 23：长短信分片级补偿审计 + +### 本轮修复 + +- Prisma/PostgreSQL 新增 `SmsMessageSegmentAudit`，按短信记录、submitId、分片序号保存真实分片提交、回执和补偿归因。 +- Go Gateway 上游提交结果 `SubmitResult` 增加 `segments[]`，逐片回传 `segmentTotal/segmentIndex/sequenceId/gatewayMessageId/submitStatus/submittedAt`，长短信不再只暴露首个分片结果。 +- NestJS `handleSubmitResult` 写入分片提交审计，`handleReceipt` 按上游 `gatewayMessageId` 回填分片回执状态；重投或补偿产生的新 submitId 与历史 submitId 可并存追踪。 +- 运营端短信记录详情新增“分片补偿审计”列表，从真实 API 查询 `SmsMessageSegmentAudit`，展示分片、submitId、通道、Sequence、MsgId、提交状态、回执状态、补偿类型和错误信息。 +- 契约文档和示例补充 `SubmitResult.segments[]`，系统测试用例新增 `TC-GW-026 长短信分片补偿审计`。 + +### 验证状态 + +- `npm --prefix api run prisma:generate`：通过。 +- `go test ./internal/upstream/... ./internal/queue/... ./internal/submitworker/...`：通过。 +- `npm --prefix api test -- send-chain.service.spec.ts operations.service.spec.ts`：通过。 +- `npm --prefix api run build`：通过。 +- `npm run build`：通过，仅有既有 Vite chunk size warning。 + +### 阶段 23 后剩余真实缺口 + +- 长短信分片级提交、回执和补偿归因已具备真实审计；后续仍需补按单个分片自动重投、分片级人工重投和更细的补偿指标。 +- 共享接入号、多候选普通上行的人工认领流程仍未完成。 +- 连接级窗口利用率、连接级心跳、恢复吞吐、恢复耗时趋势和连续失败账号等运营指标仍未进入真实后台页面。 + +## 2026-07-08 阶段 24：共享接入号上行人工认领 + +### 本轮修复 + +- Prisma/PostgreSQL 新增 `SmsUplinkMatchCandidate`，用于保存普通上行 ambiguous 场景下的候选企业、应用、下发短信、候选来源、置信度、认领状态和认领时间。 +- NestJS 上行匹配逻辑增强：接入号匹配多个应用、或手机号时间窗口匹配多条下发时，不误推客户；上行记录标记 `ambiguous`，并真实写入候选表。 +- 运营端“短信上行记录”详情新增候选认领区，展示候选企业、候选应用、候选来源、置信度、候选下发短信和候选原因，支持“认领并推送”。 +- 新增 `POST /admin/operations/uplink-messages/:id/claim`：认领后更新 `SmsUplinkMessage` 为 `matched`，选中候选置为 `claimed`，其他候选置为 `rejected`，写入操作日志，并创建真实 `CmppDownstreamDelivery(deliveryType=uplink)` 走客户侧下游投递链路。 +- 系统测试用例新增 `TC-GW-027 共享接入号上行人工认领`。 + +### 验证状态 + +- `npm --prefix api run prisma:generate`：通过。 +- `npm --prefix api run prisma:migrate:deploy`：通过，已应用 `20260708233000_add_uplink_match_candidates`。 +- `npm --prefix api test -- send-chain.service.spec.ts operations.service.spec.ts`：通过，40 个测试。 +- `npm --prefix api run build`：通过。 +- `npm run build`：通过，仅有既有 Vite chunk size warning。 + +### 当前剩余真实缺口 + +- 共享接入号上行已具备候选记录、人工认领和认领后下游投递第一版；后续仍需补批量认领、认领复核和认领准确率/积压指标。 +- 长短信分片级提交、回执和补偿归因已具备真实审计；后续仍需补按单个分片自动重投、分片级人工重投和更细的补偿指标。 +- 连接级窗口利用率、连接级心跳、恢复吞吐、恢复耗时趋势和连续失败账号等运营指标仍未进入真实后台页面。 + ## 2026-07-03 阶段 9：运营端报备回执导入真实上传/解析 ### 本轮修复 diff --git a/gateway/cmd/gateway/main.go b/gateway/cmd/gateway/main.go index a6fcb30..319d887 100644 --- a/gateway/cmd/gateway/main.go +++ b/gateway/cmd/gateway/main.go @@ -1,6 +1,7 @@ package main import ( + "context" "log" "net/http" "os" @@ -8,6 +9,8 @@ import ( "cmpp-platform/gateway/internal/control" "cmpp-platform/gateway/internal/health" "cmpp-platform/gateway/internal/inbound" + "cmpp-platform/gateway/internal/submitworker" + "cmpp-platform/gateway/internal/upstream" ) func main() { @@ -20,20 +23,81 @@ func main() { cmppAddr = ":17890" } apiBaseURL := os.Getenv("API_BASE_URL") + upstreamManager := &upstream.Manager{APIBaseURL: apiBaseURL} + presenceStore, err := inbound.NewRedisPresenceStore(os.Getenv("REDIS_URL")) + if err != nil { + log.Printf("gateway downstream presence store init failed: %v", err) + } + recoveryStore, err := inbound.NewRedisRecoveryStore(os.Getenv("REDIS_URL")) + if err != nil { + log.Printf("gateway downstream recovery store init failed: %v", err) + } go func() { log.Printf("cmpp gateway inbound server listening on %s", cmppAddr) - if err := (inbound.Server{Addr: cmppAddr, APIBaseURL: apiBaseURL}).ListenAndServe(); err != nil { + if err := (inbound.Server{ + Addr: cmppAddr, + APIBaseURL: apiBaseURL, + PresenceStore: presenceStore, + RecoveryStore: recoveryStore, + GatewayInstanceID: getenv("GATEWAY_INSTANCE_ID", hostname()), + }).ListenAndServe(); err != nil { log.Fatalf("gateway inbound server stopped: %v", err) } }() + if os.Getenv("GATEWAY_SUBMIT_WORKER_DISABLED") != "true" { + worker, err := submitworker.New(os.Getenv("REDIS_URL"), upstreamManager) + if err != nil { + log.Printf("gateway submit worker init failed: %v", err) + } else { + worker.Stream = getenv("GATEWAY_SUBMIT_STREAM", "gateway.submit.commands") + worker.Group = getenv("GATEWAY_SUBMIT_GROUP", "cmpp-gateway") + worker.Consumer = getenv("GATEWAY_SUBMIT_CONSUMER", "gateway-1") + worker.APIBaseURL = apiBaseURL + go func() { + log.Printf("cmpp gateway submit worker consuming stream=%s group=%s consumer=%s", worker.Stream, worker.Group, worker.Consumer) + if err := worker.Run(context.Background()); err != nil { + log.Printf("gateway submit worker stopped: %v", err) + } + }() + } + } + mux := http.NewServeMux() mux.Handle("/health", health.Handler()) - control.Register(mux, control.Server{APIBaseURL: apiBaseURL}) + control.Register(mux, control.Server{ + APIBaseURL: apiBaseURL, + Upstream: upstreamManager, + RecoveryCandidates: func(ctx context.Context) ([]inbound.DownstreamPresence, error) { + return inbound.ListRecoveryCandidates(ctx, presenceStore) + }, + RecoveryStatuses: func(ctx context.Context) ([]inbound.DownstreamRecoveryStatus, error) { + if recoveryStore == nil { + return nil, nil + } + return recoveryStore.ListRecoveryStatuses(ctx) + }, + }) log.Printf("cmpp gateway control server listening on %s", addr) if err := http.ListenAndServe(addr, mux); err != nil { log.Fatalf("gateway control server stopped: %v", err) } } + +func getenv(key string, fallback string) string { + value := os.Getenv(key) + if value == "" { + return fallback + } + return value +} + +func hostname() string { + name, err := os.Hostname() + if err != nil || name == "" { + return "gateway-1" + } + return name +} diff --git a/gateway/go.mod b/gateway/go.mod index fbe1c38..8c7dc84 100644 --- a/gateway/go.mod +++ b/gateway/go.mod @@ -3,6 +3,12 @@ module cmpp-platform/gateway go 1.26 require ( + github.com/alicebob/gopher-json v0.0.0-20230218143504-906a9b012302 // indirect + github.com/alicebob/miniredis/v2 v2.34.0 // indirect github.com/bigwhite/gocmpp v0.0.0-20240917054108-b238366bff0b // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect + github.com/redis/go-redis/v9 v9.17.0 // indirect + github.com/yuin/gopher-lua v1.1.1 // indirect golang.org/x/text v0.3.8 // indirect ) diff --git a/gateway/go.sum b/gateway/go.sum index 0e4a8e2..9c55d8b 100644 --- a/gateway/go.sum +++ b/gateway/go.sum @@ -1,7 +1,19 @@ +github.com/alicebob/gopher-json v0.0.0-20230218143504-906a9b012302 h1:uvdUDbHQHO85qeSydJtItA4T55Pw6BtAejd0APRJOCE= +github.com/alicebob/gopher-json v0.0.0-20230218143504-906a9b012302/go.mod h1:SGnFV6hVsYE877CKEZ6tDNTjaSXYUk6QqoIK6PrAtcc= +github.com/alicebob/miniredis/v2 v2.34.0 h1:mBFWMaJSNL9RwdGRyEDoAAv8OQc5UlEhLDQggTglU/0= +github.com/alicebob/miniredis/v2 v2.34.0/go.mod h1:kWShP4b58T1CW0Y5dViCd5ztzrDqRWqM3nksiyXk5s8= github.com/bigwhite/gocmpp v0.0.0-20240917054108-b238366bff0b h1:HOIU4bq4fpwWdtkpXASXnWk/fiFjd6o27Q0Kw4aJJAk= github.com/bigwhite/gocmpp v0.0.0-20240917054108-b238366bff0b/go.mod h1:BDWS0X/2jJROFh0iYgdcAdv4jy3cPhVcZXvEkZmoqCM= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/dvyukov/go-fuzz v0.0.0-20190516070045-5cc3605ccbb6/go.mod h1:11Gm+ccJnvAhCNLlf5+cS9KjtbaD5I5zaZpFMsTHWTw= +github.com/redis/go-redis/v9 v9.17.0 h1:K6E+ZlYN95KSMmZeEQPbU/c++wfmEvfFB17yEAq/VhM= +github.com/redis/go-redis/v9 v9.17.0/go.mod h1:u410H11HMLoB+TP67dz8rL9s6QW2j76l0//kSOd3370= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +github.com/yuin/gopher-lua v1.1.1 h1:kYKnWBjvbNP4XLT3+bPEwAXJx262OhaHDWDVOPjL46M= +github.com/yuin/gopher-lua v1.1.1/go.mod h1:GBR0iDaNXjAgGg9zfCvksxSRnQx76gclCIb7kdAd1Pw= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= diff --git a/gateway/internal/control/server.go b/gateway/internal/control/server.go index e0f6d27..6060c66 100644 --- a/gateway/internal/control/server.go +++ b/gateway/internal/control/server.go @@ -9,6 +9,10 @@ import ( "strings" "time" + "cmpp-platform/gateway/internal/inbound" + "cmpp-platform/gateway/internal/queue" + "cmpp-platform/gateway/internal/upstream" + cmpp "github.com/bigwhite/gocmpp" ) @@ -53,9 +57,17 @@ type ConnectionStateCallback struct { } type Server struct { - APIBaseURL string - HTTPClient *http.Client - Dial DialFunc + APIBaseURL string + HTTPClient *http.Client + Dial DialFunc + Upstream *upstream.Manager + RecoveryCandidates func(context.Context) ([]inbound.DownstreamPresence, error) + RecoveryStatuses func(context.Context) ([]inbound.DownstreamRecoveryStatus, error) +} + +type DownstreamRecoveryOverview struct { + Candidates []inbound.DownstreamPresence `json:"candidates"` + Statuses []inbound.DownstreamRecoveryStatus `json:"statuses"` } func Register(mux *http.ServeMux, server Server) { @@ -65,7 +77,16 @@ func Register(mux *http.ServeMux, server Server) { if server.Dial == nil { server.Dial = DialCMPP } + if server.Upstream == nil { + server.Upstream = &upstream.Manager{APIBaseURL: server.APIBaseURL, HTTPClient: server.HTTPClient} + } mux.HandleFunc("/connections/connect", server.handleConnectChannel) + mux.HandleFunc("/upstream/submit", server.handleUpstreamSubmit) + mux.HandleFunc("/downstream/receipt", server.handleDownstreamReceipt) + mux.HandleFunc("/downstream/uplink", server.handleDownstreamUplink) + mux.HandleFunc("/downstream/recovery-candidates", server.handleDownstreamRecoveryCandidates) + mux.HandleFunc("/downstream/recovery-statuses", server.handleDownstreamRecoveryStatuses) + mux.HandleFunc("/downstream/recovery-overview", server.handleDownstreamRecoveryOverview) } func (s Server) handleConnectChannel(w http.ResponseWriter, r *http.Request) { @@ -111,6 +132,132 @@ func (s Server) handleConnectChannel(w http.ResponseWriter, r *http.Request) { _ = json.NewEncoder(w).Encode(status) } +func (s Server) handleUpstreamSubmit(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + http.Error(w, "method not allowed", http.StatusMethodNotAllowed) + return + } + var command queue.SubmitCommand + if err := json.NewDecoder(r.Body).Decode(&command); err != nil { + http.Error(w, fmt.Sprintf("invalid submit command: %v", err), http.StatusBadRequest) + return + } + result, err := s.Upstream.Submit(r.Context(), command) + if err != nil { + w.WriteHeader(http.StatusBadGateway) + _ = json.NewEncoder(w).Encode(result) + return + } + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(result) +} + +func (s Server) handleDownstreamReceipt(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + http.Error(w, "method not allowed", http.StatusMethodNotAllowed) + return + } + var event inbound.DownstreamReceipt + if err := json.NewDecoder(r.Body).Decode(&event); err != nil { + http.Error(w, fmt.Sprintf("invalid downstream receipt: %v", err), http.StatusBadRequest) + return + } + delivered, err := inbound.PushReceipt(event) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{"delivered": delivered}) +} + +func (s Server) handleDownstreamUplink(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + http.Error(w, "method not allowed", http.StatusMethodNotAllowed) + return + } + var event inbound.DownstreamUplink + if err := json.NewDecoder(r.Body).Decode(&event); err != nil { + http.Error(w, fmt.Sprintf("invalid downstream uplink: %v", err), http.StatusBadRequest) + return + } + delivered, err := inbound.PushUplink(event) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{"delivered": delivered}) +} + +func (s Server) handleDownstreamRecoveryCandidates(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + http.Error(w, "method not allowed", http.StatusMethodNotAllowed) + return + } + if s.RecoveryCandidates == nil { + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode([]inbound.DownstreamPresence{}) + return + } + candidates, err := s.RecoveryCandidates(r.Context()) + if err != nil { + http.Error(w, fmt.Sprintf("failed to load recovery candidates: %v", err), http.StatusBadGateway) + return + } + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(candidates) +} + +func (s Server) handleDownstreamRecoveryStatuses(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + http.Error(w, "method not allowed", http.StatusMethodNotAllowed) + return + } + if s.RecoveryStatuses == nil { + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode([]inbound.DownstreamRecoveryStatus{}) + return + } + statuses, err := s.RecoveryStatuses(r.Context()) + if err != nil { + http.Error(w, fmt.Sprintf("failed to load recovery statuses: %v", err), http.StatusBadGateway) + return + } + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(statuses) +} + +func (s Server) handleDownstreamRecoveryOverview(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + http.Error(w, "method not allowed", http.StatusMethodNotAllowed) + return + } + candidates := []inbound.DownstreamPresence{} + if s.RecoveryCandidates != nil { + result, err := s.RecoveryCandidates(r.Context()) + if err != nil { + http.Error(w, fmt.Sprintf("failed to load recovery candidates: %v", err), http.StatusBadGateway) + return + } + candidates = result + } + statuses := []inbound.DownstreamRecoveryStatus{} + if s.RecoveryStatuses != nil { + result, err := s.RecoveryStatuses(r.Context()) + if err != nil { + http.Error(w, fmt.Sprintf("failed to load recovery statuses: %v", err), http.StatusBadGateway) + return + } + statuses = result + } + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(DownstreamRecoveryOverview{ + Candidates: candidates, + Statuses: statuses, + }) +} + func DialCMPP(ctx context.Context, command ConnectChannelCommand) error { ctx, cancel := context.WithTimeout(ctx, defaultConnectTimeout) defer cancel() diff --git a/gateway/internal/control/server_test.go b/gateway/internal/control/server_test.go index 4c89300..d5c1329 100644 --- a/gateway/internal/control/server_test.go +++ b/gateway/internal/control/server_test.go @@ -7,6 +7,9 @@ import ( "net/http/httptest" "strings" "testing" + "time" + + "cmpp-platform/gateway/internal/inbound" ) func TestConnectChannelCallbacksConnectedState(t *testing.T) { @@ -84,6 +87,99 @@ func TestConnectChannelRejectsInvalidCommand(t *testing.T) { } } +func TestRecoveryCandidatesEndpointReturnsView(t *testing.T) { + handler := handlerWithServer(Server{ + RecoveryCandidates: func(context.Context) ([]inbound.DownstreamPresence, error) { + return []inbound.DownstreamPresence{{ + Account: "100001", + GatewayInstanceID: "gateway-a", + State: "connected", + UpdatedAt: time.Now().UTC(), + }}, nil + }, + }) + + resp := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodGet, "/downstream/recovery-candidates", nil) + handler.ServeHTTP(resp, req) + + if resp.Code != http.StatusOK { + t.Fatalf("unexpected response status: %d body=%s", resp.Code, resp.Body.String()) + } + var payload []inbound.DownstreamPresence + if err := json.Unmarshal(resp.Body.Bytes(), &payload); err != nil { + t.Fatalf("decode response: %v", err) + } + if len(payload) != 1 || payload[0].Account != "100001" { + t.Fatalf("unexpected payload: %+v", payload) + } +} + +func TestRecoveryStatusesEndpointReturnsView(t *testing.T) { + handler := handlerWithServer(Server{ + RecoveryStatuses: func(context.Context) ([]inbound.DownstreamRecoveryStatus, error) { + return []inbound.DownstreamRecoveryStatus{{ + Account: "100001", + GatewayInstanceID: "gateway-a", + State: "waiting_connection", + AttemptCount: 2, + }}, nil + }, + }) + + resp := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodGet, "/downstream/recovery-statuses", nil) + handler.ServeHTTP(resp, req) + + if resp.Code != http.StatusOK { + t.Fatalf("unexpected response status: %d body=%s", resp.Code, resp.Body.String()) + } + var payload []inbound.DownstreamRecoveryStatus + if err := json.Unmarshal(resp.Body.Bytes(), &payload); err != nil { + t.Fatalf("decode response: %v", err) + } + if len(payload) != 1 || payload[0].State != "waiting_connection" || payload[0].AttemptCount != 2 { + t.Fatalf("unexpected payload: %+v", payload) + } +} + +func TestRecoveryOverviewEndpointReturnsCombinedView(t *testing.T) { + handler := handlerWithServer(Server{ + RecoveryCandidates: func(context.Context) ([]inbound.DownstreamPresence, error) { + return []inbound.DownstreamPresence{{ + Account: "100001", + GatewayInstanceID: "gateway-a", + State: "connected", + }}, nil + }, + RecoveryStatuses: func(context.Context) ([]inbound.DownstreamRecoveryStatus, error) { + return []inbound.DownstreamRecoveryStatus{{ + Account: "100001", + GatewayInstanceID: "gateway-a", + State: "success", + }}, nil + }, + }) + + resp := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodGet, "/downstream/recovery-overview", nil) + handler.ServeHTTP(resp, req) + + if resp.Code != http.StatusOK { + t.Fatalf("unexpected response status: %d body=%s", resp.Code, resp.Body.String()) + } + var payload DownstreamRecoveryOverview + if err := json.Unmarshal(resp.Body.Bytes(), &payload); err != nil { + t.Fatalf("decode response: %v", err) + } + if len(payload.Candidates) != 1 || payload.Candidates[0].Account != "100001" { + t.Fatalf("unexpected candidate payload: %+v", payload.Candidates) + } + if len(payload.Statuses) != 1 || payload.Statuses[0].State != "success" { + t.Fatalf("unexpected status payload: %+v", payload.Statuses) + } +} + type testDialError struct{} func (testDialError) Error() string { @@ -93,8 +189,12 @@ func (testDialError) Error() string { var errTestDial testDialError func handlerWithDial(apiBaseURL string, dial DialFunc) http.Handler { + return handlerWithServer(Server{APIBaseURL: apiBaseURL, Dial: dial}) +} + +func handlerWithServer(server Server) http.Handler { mux := http.NewServeMux() - Register(mux, Server{APIBaseURL: apiBaseURL, Dial: dial}) + Register(mux, server) return mux } diff --git a/gateway/internal/inbound/presence.go b/gateway/internal/inbound/presence.go new file mode 100644 index 0000000..eeb14b7 --- /dev/null +++ b/gateway/internal/inbound/presence.go @@ -0,0 +1,182 @@ +package inbound + +import ( + "context" + "encoding/json" + "fmt" + "sort" + "strings" + "time" + + "github.com/redis/go-redis/v9" +) + +const defaultPresenceTTL = 3 * time.Minute + +type PresenceStore interface { + TouchAccount(ctx context.Context, snapshot DownstreamPresence) error + RemoveAccount(ctx context.Context, account string) error + ListAccounts(ctx context.Context) ([]DownstreamPresence, error) +} + +type DownstreamPresence struct { + Account string `json:"account"` + SrcID string `json:"srcId,omitempty"` + RemoteIP string `json:"remoteIp,omitempty"` + GatewayInstanceID string `json:"gatewayInstanceId,omitempty"` + State string `json:"state"` + ConnectedAt time.Time `json:"connectedAt"` + UpdatedAt time.Time `json:"updatedAt"` + LastSubmitAt time.Time `json:"lastSubmitAt,omitempty"` + LastDeliverAt time.Time `json:"lastDeliverAt,omitempty"` +} + +type RedisPresenceStore struct { + Client *redis.Client + TTL time.Duration + Prefix string +} + +func NewRedisPresenceStore(redisURL string) (*RedisPresenceStore, error) { + if redisURL == "" { + redisURL = "redis://127.0.0.1:6379" + } + options, err := redis.ParseURL(redisURL) + if err != nil { + return nil, err + } + return &RedisPresenceStore{ + Client: redis.NewClient(options), + TTL: defaultPresenceTTL, + Prefix: "gateway:downstream:presence", + }, nil +} + +func (s *RedisPresenceStore) TouchAccount(ctx context.Context, snapshot DownstreamPresence) error { + if s == nil || s.Client == nil || snapshot.Account == "" { + return nil + } + if snapshot.UpdatedAt.IsZero() { + snapshot.UpdatedAt = time.Now().UTC() + } + if snapshot.ConnectedAt.IsZero() { + snapshot.ConnectedAt = snapshot.UpdatedAt + } + payload, err := json.Marshal(snapshot) + if err != nil { + return err + } + ttl := s.ttl() + pipe := s.Client.TxPipeline() + pipe.Set(ctx, s.accountKey(snapshot.Account), payload, ttl) + pipe.SAdd(ctx, s.accountsKey(), snapshot.Account) + pipe.Expire(ctx, s.accountsKey(), ttl*4) + _, err = pipe.Exec(ctx) + return err +} + +func (s *RedisPresenceStore) RemoveAccount(ctx context.Context, account string) error { + if s == nil || s.Client == nil || account == "" { + return nil + } + pipe := s.Client.TxPipeline() + pipe.Del(ctx, s.accountKey(account)) + pipe.SRem(ctx, s.accountsKey(), account) + _, err := pipe.Exec(ctx) + return err +} + +func (s *RedisPresenceStore) ListAccounts(ctx context.Context) ([]DownstreamPresence, error) { + if s == nil || s.Client == nil { + return nil, nil + } + accounts, err := s.Client.SMembers(ctx, s.accountsKey()).Result() + if err != nil { + return nil, err + } + result := make([]DownstreamPresence, 0, len(accounts)) + for _, account := range accounts { + payload, getErr := s.Client.Get(ctx, s.accountKey(account)).Bytes() + if getErr == redis.Nil { + _ = s.Client.SRem(ctx, s.accountsKey(), account).Err() + continue + } + if getErr != nil { + return nil, getErr + } + var snapshot DownstreamPresence + if err := json.Unmarshal(payload, &snapshot); err != nil { + return nil, fmt.Errorf("decode presence %s: %w", account, err) + } + result = append(result, snapshot) + } + return result, nil +} + +func (s *RedisPresenceStore) ttl() time.Duration { + if s != nil && s.TTL > 0 { + return s.TTL + } + return defaultPresenceTTL +} + +func (s *RedisPresenceStore) accountKey(account string) string { + return fmt.Sprintf("%s:account:%s", s.prefix(), account) +} + +func (s *RedisPresenceStore) accountsKey() string { + return fmt.Sprintf("%s:accounts", s.prefix()) +} + +func (s *RedisPresenceStore) prefix() string { + if s != nil && s.Prefix != "" { + return s.Prefix + } + return "gateway:downstream:presence" +} + +func ListRecoveryCandidates(ctx context.Context, store PresenceStore) ([]DownstreamPresence, error) { + candidateMap := map[string]DownstreamPresence{} + + if store != nil { + snapshots, err := store.ListAccounts(ctx) + if err != nil { + return nil, err + } + for _, snapshot := range snapshots { + account := strings.TrimSpace(snapshot.Account) + if account == "" { + continue + } + candidateMap[account] = snapshot + } + } + + for _, account := range onlineAccounts() { + account = strings.TrimSpace(account) + if account == "" { + continue + } + current := candidateMap[account] + current.Account = account + if strings.TrimSpace(current.State) == "" { + current.State = "connected" + } + if current.UpdatedAt.IsZero() { + current.UpdatedAt = time.Now().UTC() + } + candidateMap[account] = current + } + + result := make([]DownstreamPresence, 0, len(candidateMap)) + for _, snapshot := range candidateMap { + result = append(result, snapshot) + } + sort.Slice(result, func(i, j int) bool { + if result[i].UpdatedAt.Equal(result[j].UpdatedAt) { + return result[i].Account < result[j].Account + } + return result[i].UpdatedAt.After(result[j].UpdatedAt) + }) + return result, nil +} diff --git a/gateway/internal/inbound/presence_test.go b/gateway/internal/inbound/presence_test.go new file mode 100644 index 0000000..b426157 --- /dev/null +++ b/gateway/internal/inbound/presence_test.go @@ -0,0 +1,95 @@ +package inbound + +import ( + "context" + "testing" + "time" + + "github.com/alicebob/miniredis/v2" +) + +func TestRedisPresenceStoreTouchListAndRemove(t *testing.T) { + mr := miniredis.RunT(t) + store, err := NewRedisPresenceStore("redis://" + mr.Addr()) + if err != nil { + t.Fatalf("new presence store: %v", err) + } + store.TTL = time.Minute + store.Prefix = "test:presence" + + now := time.Now().UTC().Truncate(time.Second) + err = store.TouchAccount(context.Background(), DownstreamPresence{ + Account: "100001", + SrcID: "10690000", + RemoteIP: "127.0.0.1", + GatewayInstanceID: "gateway-a", + State: "connected", + ConnectedAt: now, + UpdatedAt: now, + LastSubmitAt: now, + }) + if err != nil { + t.Fatalf("touch account: %v", err) + } + + accounts, err := store.ListAccounts(context.Background()) + if err != nil { + t.Fatalf("list accounts: %v", err) + } + if len(accounts) != 1 { + t.Fatalf("accounts len = %d, want 1", len(accounts)) + } + if accounts[0].Account != "100001" || accounts[0].GatewayInstanceID != "gateway-a" || accounts[0].State != "connected" { + t.Fatalf("unexpected presence snapshot: %+v", accounts[0]) + } + + if err := store.RemoveAccount(context.Background(), "100001"); err != nil { + t.Fatalf("remove account: %v", err) + } + accounts, err = store.ListAccounts(context.Background()) + if err != nil { + t.Fatalf("list accounts after remove: %v", err) + } + if len(accounts) != 0 { + t.Fatalf("accounts len after remove = %d, want 0", len(accounts)) + } +} + +func TestListRecoveryCandidatesMergesPresenceAndInMemory(t *testing.T) { + resetDownstreamRegistry() + defer resetDownstreamRegistry() + + store := &memoryPresenceStore{ + snapshots: map[string]DownstreamPresence{ + "100001": { + Account: "100001", + GatewayInstanceID: "gateway-a", + State: "connected", + UpdatedAt: time.Now().UTC().Add(-time.Minute), + }, + }, + } + + downstreamRegistry.Lock() + downstreamRegistry.byAccount["100002"] = &downstreamSession{account: "100002"} + downstreamRegistry.Unlock() + + candidates, err := ListRecoveryCandidates(context.Background(), store) + if err != nil { + t.Fatalf("list recovery candidates: %v", err) + } + if len(candidates) != 2 { + t.Fatalf("candidates len = %d, want 2", len(candidates)) + } + + accounts := map[string]DownstreamPresence{} + for _, item := range candidates { + accounts[item.Account] = item + } + if _, ok := accounts["100001"]; !ok { + t.Fatal("expected redis presence candidate 100001") + } + if snapshot, ok := accounts["100002"]; !ok || snapshot.State != "connected" { + t.Fatalf("expected in-memory candidate 100002 connected, got %+v", snapshot) + } +} diff --git a/gateway/internal/inbound/recovery.go b/gateway/internal/inbound/recovery.go new file mode 100644 index 0000000..90b7291 --- /dev/null +++ b/gateway/internal/inbound/recovery.go @@ -0,0 +1,323 @@ +package inbound + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "time" + + "github.com/redis/go-redis/v9" +) + +const ( + defaultRecoveryLockTTL = 30 * time.Second + defaultRecoveryBackoffBase = 30 * time.Second + defaultRecoveryBackoffMax = 10 * time.Minute +) + +var ErrRecoveryLockLost = errors.New("recovery lock lost") + +type RecoveryStore interface { + StartAccountRecovery(ctx context.Context, account string, instanceID string) (RecoveryStartDecision, error) + CompleteAccountRecovery(ctx context.Context, status DownstreamRecoveryStatus) error + GetAccountRecoveryStatus(ctx context.Context, account string) (DownstreamRecoveryStatus, error) + ListRecoveryStatuses(ctx context.Context) ([]DownstreamRecoveryStatus, error) +} + +type RecoveryStartDecision struct { + Allowed bool + SkipReason string + Status DownstreamRecoveryStatus +} + +type DownstreamRecoveryStatus struct { + Account string `json:"account"` + GatewayInstanceID string `json:"gatewayInstanceId,omitempty"` + State string `json:"state"` + LockOwner string `json:"lockOwner,omitempty"` + LockToken string `json:"lockToken,omitempty"` + LockAcquiredAt time.Time `json:"lockAcquiredAt,omitempty"` + LockExpiresAt time.Time `json:"lockExpiresAt,omitempty"` + LastAttemptAt time.Time `json:"lastAttemptAt,omitempty"` + LastSuccessAt time.Time `json:"lastSuccessAt,omitempty"` + LastFailureAt time.Time `json:"lastFailureAt,omitempty"` + NextRetryAt time.Time `json:"nextRetryAt,omitempty"` + AttemptCount int `json:"attemptCount"` + FailureCategory string `json:"failureCategory,omitempty"` + LastError string `json:"lastError,omitempty"` + LastSkipReason string `json:"lastSkipReason,omitempty"` +} + +type RedisRecoveryStore struct { + Client *redis.Client + Prefix string +} + +func NewRedisRecoveryStore(redisURL string) (*RedisRecoveryStore, error) { + if redisURL == "" { + redisURL = "redis://127.0.0.1:6379" + } + options, err := redis.ParseURL(redisURL) + if err != nil { + return nil, err + } + return &RedisRecoveryStore{ + Client: redis.NewClient(options), + Prefix: "gateway:downstream:recovery", + }, nil +} + +func (s *RedisRecoveryStore) StartAccountRecovery(ctx context.Context, account string, instanceID string) (RecoveryStartDecision, error) { + if s == nil || s.Client == nil || account == "" { + return RecoveryStartDecision{Allowed: true}, nil + } + status, err := s.getStatus(ctx, account) + if err != nil { + return RecoveryStartDecision{}, err + } + now := time.Now().UTC() + if !status.NextRetryAt.IsZero() && status.NextRetryAt.After(now) { + status.LastSkipReason = "backoff" + status.FailureCategory = "backoff" + return RecoveryStartDecision{Allowed: false, SkipReason: "backoff", Status: status}, nil + } + lockToken := newRecoveryLockToken(instanceID) + acquired, err := s.Client.SetNX(ctx, s.lockKey(account), lockToken, defaultRecoveryLockTTL).Result() + if err != nil { + return RecoveryStartDecision{}, err + } + if !acquired { + status.LockToken, _ = s.Client.Get(ctx, s.lockKey(account)).Result() + status.LockOwner = recoveryLockOwner(status.LockToken) + status.LockExpiresAt = lockExpiresAt(ctx, s.Client, s.lockKey(account), now) + status.LastSkipReason = "locked" + status.FailureCategory = "lock_contended" + return RecoveryStartDecision{Allowed: false, SkipReason: "locked", Status: status}, nil + } + status.Account = account + status.GatewayInstanceID = instanceID + status.State = "running" + status.LockOwner = instanceID + status.LockToken = lockToken + status.LockAcquiredAt = now + status.LockExpiresAt = now.Add(defaultRecoveryLockTTL) + status.LastAttemptAt = now + status.LastSkipReason = "" + status.FailureCategory = "" + if err := s.saveStatus(ctx, status); err != nil { + _ = s.deleteLockIfOwned(ctx, account, lockToken) + return RecoveryStartDecision{}, err + } + return RecoveryStartDecision{Allowed: true, Status: status}, nil +} + +func (s *RedisRecoveryStore) CompleteAccountRecovery(ctx context.Context, status DownstreamRecoveryStatus) error { + if s == nil || s.Client == nil || status.Account == "" { + return nil + } + now := time.Now().UTC() + if status.LastAttemptAt.IsZero() { + status.LastAttemptAt = now + } + if status.LockOwner == "" { + status.LockOwner = status.GatewayInstanceID + } + switch status.State { + case "success": + status.LastSuccessAt = now + status.AttemptCount = 0 + status.NextRetryAt = time.Time{} + status.LastError = "" + status.FailureCategory = "" + case "waiting_connection", "failed", "partial": + status.AttemptCount++ + if status.State == "failed" { + status.LastFailureAt = now + } + if status.FailureCategory == "" { + status.FailureCategory = classifyRecoveryFailure(status) + } + status.NextRetryAt = now.Add(recoveryBackoffDelay(status.AttemptCount)) + default: + status.State = "unknown" + } + payload, err := json.Marshal(status) + if err != nil { + return err + } + result, err := s.Client.Eval(ctx, completeRecoveryScript, []string{s.lockKey(status.Account), s.statusKey(status.Account), s.accountsKey()}, status.LockToken, payload, int64((24*time.Hour)/time.Millisecond), status.Account).Int() + if err != nil { + return err + } + if result == 0 { + return ErrRecoveryLockLost + } + return nil +} + +func classifyRecoveryFailure(status DownstreamRecoveryStatus) string { + switch status.State { + case "waiting_connection": + return "client_disconnected" + case "partial": + return "partial_delivery_failed" + case "failed": + if status.LastSkipReason == "backoff" { + return "backoff" + } + if status.LastSkipReason == "locked" { + return "lock_contended" + } + if status.LastError != "" { + return "flush_failed" + } + } + return "unknown" +} + +func (s *RedisRecoveryStore) ListRecoveryStatuses(ctx context.Context) ([]DownstreamRecoveryStatus, error) { + if s == nil || s.Client == nil { + return nil, nil + } + accounts, err := s.Client.SMembers(ctx, s.accountsKey()).Result() + if err != nil { + return nil, err + } + result := make([]DownstreamRecoveryStatus, 0, len(accounts)) + for _, account := range accounts { + status, getErr := s.getStatus(ctx, account) + if getErr == redis.Nil { + _ = s.Client.SRem(ctx, s.accountsKey(), account).Err() + continue + } + if getErr != nil { + return nil, getErr + } + result = append(result, status) + } + return result, nil +} + +func (s *RedisRecoveryStore) GetAccountRecoveryStatus(ctx context.Context, account string) (DownstreamRecoveryStatus, error) { + if s == nil || s.Client == nil || account == "" { + return DownstreamRecoveryStatus{Account: account}, nil + } + return s.getStatus(ctx, account) +} + +func (s *RedisRecoveryStore) getStatus(ctx context.Context, account string) (DownstreamRecoveryStatus, error) { + payload, err := s.Client.Get(ctx, s.statusKey(account)).Bytes() + if err != nil { + if err == redis.Nil { + return DownstreamRecoveryStatus{Account: account}, nil + } + return DownstreamRecoveryStatus{}, err + } + var status DownstreamRecoveryStatus + if err := json.Unmarshal(payload, &status); err != nil { + return DownstreamRecoveryStatus{}, fmt.Errorf("decode recovery status %s: %w", account, err) + } + if status.Account == "" { + status.Account = account + } + return status, nil +} + +func (s *RedisRecoveryStore) saveStatus(ctx context.Context, status DownstreamRecoveryStatus) error { + payload, err := json.Marshal(status) + if err != nil { + return err + } + pipe := s.Client.TxPipeline() + pipe.Set(ctx, s.statusKey(status.Account), payload, 24*time.Hour) + pipe.SAdd(ctx, s.accountsKey(), status.Account) + pipe.Expire(ctx, s.accountsKey(), 24*time.Hour) + _, err = pipe.Exec(ctx) + return err +} + +func (s *RedisRecoveryStore) deleteLockIfOwned(ctx context.Context, account string, token string) error { + if token == "" { + return nil + } + _, err := s.Client.Eval(ctx, deleteLockIfOwnedScript, []string{s.lockKey(account)}, token).Result() + return err +} + +func (s *RedisRecoveryStore) statusKey(account string) string { + return fmt.Sprintf("%s:status:%s", s.prefix(), account) +} + +func (s *RedisRecoveryStore) lockKey(account string) string { + return fmt.Sprintf("%s:lock:%s", s.prefix(), account) +} + +func (s *RedisRecoveryStore) accountsKey() string { + return fmt.Sprintf("%s:accounts", s.prefix()) +} + +func (s *RedisRecoveryStore) prefix() string { + if s != nil && s.Prefix != "" { + return s.Prefix + } + return "gateway:downstream:recovery" +} + +func recoveryBackoffDelay(attemptCount int) time.Duration { + if attemptCount <= 0 { + return defaultRecoveryBackoffBase + } + delay := defaultRecoveryBackoffBase + for step := 1; step < attemptCount; step++ { + delay *= 2 + if delay >= defaultRecoveryBackoffMax { + return defaultRecoveryBackoffMax + } + } + if delay > defaultRecoveryBackoffMax { + return defaultRecoveryBackoffMax + } + return delay +} + +func newRecoveryLockToken(instanceID string) string { + return fmt.Sprintf("%s:%d", instanceID, time.Now().UTC().UnixNano()) +} + +func recoveryLockOwner(token string) string { + for index, char := range token { + if char == ':' { + return token[:index] + } + } + return token +} + +func lockExpiresAt(ctx context.Context, client *redis.Client, key string, now time.Time) time.Time { + ttl, err := client.TTL(ctx, key).Result() + if err != nil || ttl <= 0 { + return time.Time{} + } + return now.Add(ttl) +} + +const completeRecoveryScript = ` +local current = redis.call("GET", KEYS[1]) +if current ~= ARGV[1] then + return 0 +end +redis.call("SET", KEYS[2], ARGV[2], "PX", ARGV[3]) +redis.call("SADD", KEYS[3], ARGV[4]) +redis.call("PEXPIRE", KEYS[3], ARGV[3]) +redis.call("DEL", KEYS[1]) +return 1 +` + +const deleteLockIfOwnedScript = ` +local current = redis.call("GET", KEYS[1]) +if current == ARGV[1] then + return redis.call("DEL", KEYS[1]) +end +return 0 +` diff --git a/gateway/internal/inbound/recovery_test.go b/gateway/internal/inbound/recovery_test.go new file mode 100644 index 0000000..21ba71f --- /dev/null +++ b/gateway/internal/inbound/recovery_test.go @@ -0,0 +1,101 @@ +package inbound + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/alicebob/miniredis/v2" +) + +func TestRedisRecoveryStoreBackoffAndStatuses(t *testing.T) { + mr := miniredis.RunT(t) + store, err := NewRedisRecoveryStore("redis://" + mr.Addr()) + if err != nil { + t.Fatalf("new recovery store: %v", err) + } + store.Prefix = "test:recovery" + + decision, err := store.StartAccountRecovery(context.Background(), "100001", "gateway-a") + if err != nil { + t.Fatalf("start recovery: %v", err) + } + if !decision.Allowed { + t.Fatalf("expected recovery allowed, got %+v", decision) + } + if decision.Status.LockOwner != "gateway-a" || decision.Status.LockToken == "" || decision.Status.LockExpiresAt.IsZero() { + t.Fatalf("missing recovery lock metadata: %+v", decision.Status) + } + status := decision.Status + status.State = "waiting_connection" + if err := store.CompleteAccountRecovery(context.Background(), status); err != nil { + t.Fatalf("complete recovery: %v", err) + } + + statuses, err := store.ListRecoveryStatuses(context.Background()) + if err != nil { + t.Fatalf("list recovery statuses: %v", err) + } + if len(statuses) != 1 || statuses[0].State != "waiting_connection" || statuses[0].AttemptCount != 1 { + t.Fatalf("unexpected statuses: %+v", statuses) + } + if statuses[0].FailureCategory != "client_disconnected" { + t.Fatalf("failure category = %q, want client_disconnected", statuses[0].FailureCategory) + } + if statuses[0].NextRetryAt.IsZero() { + t.Fatalf("expected next retry at after waiting connection: %+v", statuses[0]) + } + + decision, err = store.StartAccountRecovery(context.Background(), "100001", "gateway-a") + if err != nil { + t.Fatalf("start recovery second time: %v", err) + } + if decision.Allowed || decision.SkipReason != "backoff" { + t.Fatalf("expected backoff skip, got %+v", decision) + } + if decision.Status.FailureCategory != "backoff" { + t.Fatalf("skip failure category = %q, want backoff", decision.Status.FailureCategory) + } +} + +func TestRedisRecoveryStoreDoesNotReleaseLockOwnedByAnotherGateway(t *testing.T) { + mr := miniredis.RunT(t) + store, err := NewRedisRecoveryStore("redis://" + mr.Addr()) + if err != nil { + t.Fatalf("new recovery store: %v", err) + } + store.Prefix = "test:recovery:takeover" + + first, err := store.StartAccountRecovery(context.Background(), "100001", "gateway-a") + if err != nil { + t.Fatalf("start first recovery: %v", err) + } + if !first.Allowed { + t.Fatalf("expected first recovery allowed, got %+v", first) + } + + mr.FastForward(defaultRecoveryLockTTL + time.Second) + second, err := store.StartAccountRecovery(context.Background(), "100001", "gateway-b") + if err != nil { + t.Fatalf("start second recovery: %v", err) + } + if !second.Allowed { + t.Fatalf("expected second recovery to take over expired lock, got %+v", second) + } + + stale := first.Status + stale.State = "success" + err = store.CompleteAccountRecovery(context.Background(), stale) + if !errors.Is(err, ErrRecoveryLockLost) { + t.Fatalf("stale completion error = %v, want ErrRecoveryLockLost", err) + } + + lockValue, err := store.Client.Get(context.Background(), store.lockKey("100001")).Result() + if err != nil { + t.Fatalf("load current lock: %v", err) + } + if lockValue != second.Status.LockToken { + t.Fatalf("current lock was changed by stale completion: got %q want %q", lockValue, second.Status.LockToken) + } +} diff --git a/gateway/internal/inbound/server.go b/gateway/internal/inbound/server.go index 164dbfb..97f3e81 100644 --- a/gateway/internal/inbound/server.go +++ b/gateway/internal/inbound/server.go @@ -11,6 +11,7 @@ import ( "net" "net/http" "strings" + "sync" "time" cmpp "github.com/bigwhite/gocmpp" @@ -20,9 +21,13 @@ import ( const defaultHTTPTimeout = 10 * time.Second type Server struct { - Addr string - APIBaseURL string - HTTPClient *http.Client + Addr string + APIBaseURL string + HTTPClient *http.Client + PendingFlushInterval time.Duration + PresenceStore PresenceStore + RecoveryStore RecoveryStore + GatewayInstanceID string } type authRequest struct { @@ -49,6 +54,56 @@ type submitResponse struct { type authResponse struct { PasswordCipher string `json:"passwordCipher"` + ApplicationID string `json:"applicationId"` + TenantID string `json:"tenantId"` + Account string `json:"account"` +} + +type DownstreamReceipt struct { + DeliveryID string `json:"deliveryId,omitempty"` + Account string `json:"account,omitempty"` + ApplicationID string `json:"applicationId,omitempty"` + MessageID string `json:"messageId"` + GatewayMessageID string `json:"gatewayMessageId,omitempty"` + PhoneNumber string `json:"phoneNumber,omitempty"` + ReceiptStatus string `json:"receiptStatus"` + RawStatus string `json:"rawStatus,omitempty"` + ErrorCode string `json:"errorCode,omitempty"` + DeliveredAt string `json:"deliveredAt,omitempty"` +} + +type DownstreamUplink struct { + DeliveryID string `json:"deliveryId,omitempty"` + Account string `json:"account,omitempty"` + ApplicationID string `json:"applicationId,omitempty"` + MessageID string `json:"messageId,omitempty"` + PhoneNumber string `json:"phoneNumber"` + DestID string `json:"destId"` + Content string `json:"content"` + ReceivedAt string `json:"receivedAt,omitempty"` +} + +type downstreamSession struct { + messageID string + account string + srcID string + phoneNumber string + gatewayMsgID uint64 + remoteIP string + connectedAt time.Time + conn *cmpp.Conn + mu *sync.Mutex + presence PresenceStore + instanceID string +} + +var downstreamRegistry = struct { + sync.RWMutex + byMessageID map[string]*downstreamSession + byAccount map[string]*downstreamSession +}{ + byMessageID: make(map[string]*downstreamSession), + byAccount: make(map[string]*downstreamSession), } func (s Server) ListenAndServe() error { @@ -56,6 +111,9 @@ func (s Server) ListenAndServe() error { if addr == "" { addr = ":17890" } + s.logRecoveryCandidates(log.Default()) + go s.recoverPendingCandidates(log.Default()) + go s.runPendingFlusher(log.Default()) return cmpp.ListenAndServe(addr, cmpp.V30, 30*time.Second, 3, nil, cmpp.HandlerFunc(s.handleLogin), cmpp.HandlerFunc(s.handleSubmit), @@ -83,6 +141,18 @@ func (s Server) handleLogin(response *cmpp.Response, packet *cmpp.Packet, logger authSource := []byte(req.AuthSrc) authISMG := md5.Sum(bytes.Join([][]byte{{byte(resp.Status)}, authSource, []byte(auth.PasswordCipher)}, nil)) resp.AuthIsmg = string(authISMG[:]) + session := downstreamSession{ + account: strings.TrimSpace(defaultString(auth.Account, account)), + srcID: strings.TrimSpace(auth.Account), + remoteIP: remoteIP(packet.Conn.Conn.RemoteAddr()), + connectedAt: time.Now().UTC(), + conn: packet.Conn, + mu: &sync.Mutex{}, + presence: s.PresenceStore, + instanceID: s.gatewayInstanceID(), + } + rememberAccount(session) + go s.flushPending(defaultString(auth.Account, account), logger) logger.Printf("cmpp inbound account=%s login ok remote=%s", account, packet.Conn.Conn.RemoteAddr()) return false, nil } @@ -120,6 +190,19 @@ func (s Server) handleSubmit(response *cmpp.Response, packet *cmpp.Packet, logge } resp.MsgId = messageIDFrom(result.MessageID, req.SeqId) resp.Result = 0 + rememberDownstream(downstreamSession{ + messageID: result.MessageID, + account: account, + srcID: strings.TrimSpace(req.SrcId), + phoneNumber: phone, + gatewayMsgID: resp.MsgId, + remoteIP: remoteIP(packet.Conn.Conn.RemoteAddr()), + connectedAt: time.Now().UTC(), + conn: packet.Conn, + mu: &sync.Mutex{}, + presence: s.PresenceStore, + instanceID: s.gatewayInstanceID(), + }) return false, nil } @@ -142,6 +225,82 @@ func (s Server) submit(remote net.Addr, payload submitRequest) (submitResponse, return result, err } +type pendingDeliveryRequest struct { + Account string `json:"account"` + Limit int `json:"limit,omitempty"` +} + +type pendingDelivery struct { + ID string `json:"id"` + DeliveryType string `json:"deliveryType"` + Payload json.RawMessage `json:"payload"` +} + +type pendingFlushResult struct { + Account string + Deliveries int + DeliveredCount int + FailedCount int + WaitingCount int + LastError string +} + +func (s Server) flushPending(account string, logger *log.Logger) (pendingFlushResult, error) { + result := pendingFlushResult{Account: account} + if account == "" { + return result, nil + } + var deliveries []pendingDelivery + if err := s.post(context.Background(), "/gateway/events/downstream/pending", pendingDeliveryRequest{Account: account, Limit: 100}, &deliveries); err != nil { + logger.Printf("cmpp inbound pending delivery fetch failed account=%s err=%v", account, err) + result.LastError = err.Error() + return result, err + } + result.Deliveries = len(deliveries) + for _, delivery := range deliveries { + delivered, err := s.pushPendingDelivery(account, delivery) + if err != nil { + result.FailedCount++ + result.LastError = err.Error() + _ = s.post(context.Background(), "/gateway/events/downstream/failed", map[string]string{ + "id": delivery.ID, + "errorMessage": err.Error(), + }, nil) + continue + } + if delivered { + result.DeliveredCount++ + _ = s.post(context.Background(), "/gateway/events/downstream/delivered", map[string]string{"id": delivery.ID}, nil) + continue + } + result.WaitingCount++ + } + return result, nil +} + +func (s Server) pushPendingDelivery(account string, delivery pendingDelivery) (bool, error) { + switch delivery.DeliveryType { + case "receipt": + var event DownstreamReceipt + if err := json.Unmarshal(delivery.Payload, &event); err != nil { + return false, err + } + event.DeliveryID = delivery.ID + event.Account = defaultString(event.Account, account) + return PushReceipt(event) + case "uplink": + var event DownstreamUplink + if err := json.Unmarshal(delivery.Payload, &event); err != nil { + return false, err + } + event.DeliveryID = delivery.ID + event.Account = defaultString(event.Account, account) + return PushUplink(event) + default: + return false, fmt.Errorf("unsupported downstream delivery type %q", delivery.DeliveryType) + } +} + func (s Server) post(ctx context.Context, path string, payload any, result any) error { client := s.HTTPClient if client == nil { @@ -210,3 +369,375 @@ func messageIDFrom(value string, seq uint32) uint64 { } return result } + +func rememberDownstream(session downstreamSession) { + if session.messageID == "" || session.conn == nil { + return + } + session.touchPresence("connected", true, false) + downstreamRegistry.Lock() + downstreamRegistry.byMessageID[session.messageID] = &session + if session.account != "" { + downstreamRegistry.byAccount[session.account] = &session + } + downstreamRegistry.Unlock() +} + +func rememberAccount(session downstreamSession) { + if session.account == "" || session.conn == nil { + return + } + session.touchPresence("connected", false, false) + downstreamRegistry.Lock() + downstreamRegistry.byAccount[session.account] = &session + downstreamRegistry.Unlock() +} + +func forgetDownstream(session *downstreamSession) { + if session == nil { + return + } + downstreamRegistry.Lock() + if session.messageID != "" { + if current := downstreamRegistry.byMessageID[session.messageID]; current == session { + delete(downstreamRegistry.byMessageID, session.messageID) + } + } + if session.account != "" { + if current := downstreamRegistry.byAccount[session.account]; current == session { + delete(downstreamRegistry.byAccount, session.account) + } + } + downstreamRegistry.Unlock() + _ = session.removePresence() +} + +func (s Server) runPendingFlusher(logger *log.Logger) { + ticker := time.NewTicker(s.pendingFlushInterval()) + defer ticker.Stop() + for range ticker.C { + s.flushOnlineAccounts(logger) + s.recoverPendingCandidates(logger) + } +} + +func (s Server) flushOnlineAccounts(logger *log.Logger) { + for _, account := range onlineAccounts() { + _, _ = s.flushPending(account, logger) + } +} + +func (s Server) recoverPendingCandidates(logger *log.Logger) { + candidates, err := ListRecoveryCandidates(context.Background(), s.PresenceStore) + if err != nil { + logger.Printf("cmpp inbound recovery candidate refresh failed err=%v", err) + return + } + for _, candidate := range candidates { + account := strings.TrimSpace(candidate.Account) + if account == "" { + continue + } + activeRecovery := DownstreamRecoveryStatus{ + Account: account, + GatewayInstanceID: s.gatewayInstanceID(), + } + if s.RecoveryStore != nil { + decision, recoveryErr := s.RecoveryStore.StartAccountRecovery(context.Background(), account, s.gatewayInstanceID()) + if recoveryErr != nil { + logger.Printf("cmpp inbound recovery start failed account=%s err=%v", account, recoveryErr) + continue + } + if !decision.Allowed { + logger.Printf("cmpp inbound recovery skipped account=%s reason=%s", account, decision.SkipReason) + decision.Status.Account = account + decision.Status.GatewayInstanceID = s.gatewayInstanceID() + decision.Status.State = defaultString(decision.Status.State, "failed") + decision.Status.LastSkipReason = defaultString(decision.Status.LastSkipReason, decision.SkipReason) + if decision.Status.FailureCategory == "" { + decision.Status.FailureCategory = recoveryFailureCategory(decision.Status.State, "", decision.Status.LastSkipReason) + } + s.syncRecoveryStatus(logger, account, decision.Status) + continue + } + activeRecovery = decision.Status + activeRecovery.Account = account + activeRecovery.GatewayInstanceID = s.gatewayInstanceID() + } + result, flushErr := s.flushPending(account, logger) + if s.RecoveryStore != nil { + status := DownstreamRecoveryStatus{ + Account: account, + GatewayInstanceID: s.gatewayInstanceID(), + LockToken: activeRecovery.LockToken, + LockOwner: defaultString(activeRecovery.LockOwner, s.gatewayInstanceID()), + LockAcquiredAt: activeRecovery.LockAcquiredAt, + LockExpiresAt: activeRecovery.LockExpiresAt, + LastAttemptAt: activeRecovery.LastAttemptAt, + } + switch { + case flushErr != nil: + status.State = "failed" + status.LastError = flushErr.Error() + status.FailureCategory = recoveryFailureCategory(status.State, status.LastError, status.LastSkipReason) + case result.WaitingCount > 0 && result.DeliveredCount == 0 && result.FailedCount == 0: + status.State = "waiting_connection" + status.LastError = "downstream client is not connected" + status.FailureCategory = recoveryFailureCategory(status.State, status.LastError, status.LastSkipReason) + case result.FailedCount > 0 && result.DeliveredCount > 0: + status.State = "partial" + status.LastError = result.LastError + status.FailureCategory = recoveryFailureCategory(status.State, status.LastError, status.LastSkipReason) + default: + status.State = "success" + status.LastError = "" + status.FailureCategory = "" + } + if err := s.RecoveryStore.CompleteAccountRecovery(context.Background(), status); err != nil { + logger.Printf("cmpp inbound recovery completion failed account=%s err=%v", account, err) + if err == ErrRecoveryLockLost { + status.State = "failed" + status.LastSkipReason = "lock_lost" + status.FailureCategory = "lock_lost" + s.syncRecoveryStatus(logger, account, status) + } + } else if persisted, err := s.RecoveryStore.GetAccountRecoveryStatus(context.Background(), account); err != nil { + logger.Printf("cmpp inbound recovery status fetch failed account=%s err=%v", account, err) + } else { + s.syncRecoveryStatus(logger, account, persisted) + } + } + } +} + +func (s Server) syncRecoveryStatus(logger *log.Logger, account string, status DownstreamRecoveryStatus) { + if err := s.post(context.Background(), "/gateway/events/downstream/recovery-status", map[string]any{ + "account": status.Account, + "gatewayInstanceId": status.GatewayInstanceID, + "state": status.State, + "lockOwner": status.LockOwner, + "lockExpiresAt": formatRFC3339Nano(status.LockExpiresAt), + "lastAttemptAt": formatRFC3339Nano(status.LastAttemptAt), + "lastSuccessAt": formatRFC3339Nano(status.LastSuccessAt), + "lastFailureAt": formatRFC3339Nano(status.LastFailureAt), + "nextRetryAt": formatRFC3339Nano(status.NextRetryAt), + "attemptCount": status.AttemptCount, + "failureCategory": status.FailureCategory, + "lastError": status.LastError, + "lastSkipReason": status.LastSkipReason, + }, nil); err != nil { + logger.Printf("cmpp inbound recovery status sync failed account=%s err=%v", account, err) + } +} + +func recoveryFailureCategory(state string, lastError string, lastSkipReason string) string { + if state == "success" || state == "running" { + return "" + } + if lastSkipReason == "backoff" { + return "backoff" + } + if lastSkipReason == "locked" { + return "lock_contended" + } + if lastSkipReason == "lock_lost" { + return "lock_lost" + } + if state == "waiting_connection" { + return "client_disconnected" + } + if state == "partial" { + return "partial_delivery_failed" + } + if state == "failed" && strings.TrimSpace(lastError) != "" { + return "flush_failed" + } + return "unknown" +} + +func onlineAccounts() []string { + downstreamRegistry.RLock() + defer downstreamRegistry.RUnlock() + accounts := make([]string, 0, len(downstreamRegistry.byAccount)) + for account := range downstreamRegistry.byAccount { + if strings.TrimSpace(account) != "" { + accounts = append(accounts, account) + } + } + return accounts +} + +func PushReceipt(event DownstreamReceipt) (bool, error) { + session := findSession(event.MessageID, event.Account) + if session == nil { + return false, nil + } + stat := strings.TrimSpace(event.RawStatus) + if stat == "" { + stat = cmppReceiptStatus(event.ReceiptStatus) + } + when := time.Now() + if event.DeliveredAt != "" { + if parsed, err := time.Parse(time.RFC3339Nano, event.DeliveredAt); err == nil { + when = parsed + } + } + receipt := &cmpp.CmppReceiptPkt{ + MsgId: session.gatewayMsgID, + Stat: stat, + SubmitTime: when.Format("0601021504"), + DoneTime: when.Format("0601021504"), + DestTerminalId: defaultString(event.PhoneNumber, session.phoneNumber), + SmscSequence: uint32(time.Now().UnixNano() & 0xffffffff), + } + receiptBytes, err := receipt.Pack() + if err != nil { + return false, err + } + deliver := &cmpp.Cmpp3DeliverReqPkt{ + MsgId: session.gatewayMsgID, + DestId: session.srcID, + ServiceId: "cmpp", + MsgFmt: 0, + SrcTerminalId: defaultString(event.PhoneNumber, session.phoneNumber), + RegisterDelivery: 1, + MsgLength: uint8(cmpp.CmppReceiptPktLen), + MsgContent: string(receiptBytes), + } + return sendDownstream(session, deliver) +} + +func PushUplink(event DownstreamUplink) (bool, error) { + session := findSession(event.MessageID, event.Account) + if session == nil { + return false, nil + } + content, err := cmpputils.Utf8ToUcs2(event.Content) + if err != nil { + return false, err + } + deliver := &cmpp.Cmpp3DeliverReqPkt{ + MsgId: messageIDFrom(defaultString(event.MessageID, event.Account), uint32(time.Now().UnixNano())), + DestId: defaultString(event.DestID, session.srcID), + ServiceId: "cmpp", + MsgFmt: 8, + SrcTerminalId: event.PhoneNumber, + RegisterDelivery: 0, + MsgLength: uint8(len(content)), + MsgContent: content, + } + return sendDownstream(session, deliver) +} + +func findSession(messageID string, account string) *downstreamSession { + downstreamRegistry.RLock() + defer downstreamRegistry.RUnlock() + if messageID != "" { + if session := downstreamRegistry.byMessageID[messageID]; session != nil { + return session + } + } + if account != "" { + return downstreamRegistry.byAccount[account] + } + return nil +} + +func sendDownstream(session *downstreamSession, deliver *cmpp.Cmpp3DeliverReqPkt) (bool, error) { + session.mu.Lock() + defer session.mu.Unlock() + if err := session.conn.SendPkt(deliver, <-session.conn.SeqId); err != nil { + forgetDownstream(session) + return false, err + } + session.touchPresence("connected", false, true) + return true, nil +} + +func (s Server) pendingFlushInterval() time.Duration { + if s.PendingFlushInterval > 0 { + return s.PendingFlushInterval + } + return time.Minute +} + +func (s Server) logRecoveryCandidates(logger *log.Logger) { + candidates, err := ListRecoveryCandidates(context.Background(), s.PresenceStore) + if err != nil { + logger.Printf("cmpp inbound recovery candidates load failed err=%v", err) + return + } + if len(candidates) == 0 { + logger.Printf("cmpp inbound recovery candidates loaded count=0") + return + } + accounts := make([]string, 0, len(candidates)) + for _, item := range candidates { + if strings.TrimSpace(item.Account) != "" { + accounts = append(accounts, item.Account) + } + } + logger.Printf("cmpp inbound recovery candidates loaded count=%d accounts=%s", len(candidates), strings.Join(accounts, ",")) +} + +func cmppReceiptStatus(status string) string { + switch strings.ToLower(strings.TrimSpace(status)) { + case "delivered": + return "DELIVRD" + case "unknown": + return "UNKNOWN" + default: + return "UNDELIV" + } +} + +func defaultString(value string, fallback string) string { + if value == "" { + return fallback + } + return value +} + +func formatRFC3339Nano(value time.Time) string { + if value.IsZero() { + return "" + } + return value.UTC().Format(time.RFC3339Nano) +} + +func (s Server) gatewayInstanceID() string { + if strings.TrimSpace(s.GatewayInstanceID) != "" { + return strings.TrimSpace(s.GatewayInstanceID) + } + return "gateway-1" +} + +func (session downstreamSession) touchPresence(state string, includeSubmit bool, includeDeliver bool) { + if session.presence == nil || strings.TrimSpace(session.account) == "" { + return + } + now := time.Now().UTC() + snapshot := DownstreamPresence{ + Account: strings.TrimSpace(session.account), + SrcID: strings.TrimSpace(session.srcID), + RemoteIP: strings.TrimSpace(session.remoteIP), + GatewayInstanceID: strings.TrimSpace(session.instanceID), + State: defaultString(strings.TrimSpace(state), "connected"), + ConnectedAt: session.connectedAt, + UpdatedAt: now, + } + if includeSubmit { + snapshot.LastSubmitAt = now + } + if includeDeliver { + snapshot.LastDeliverAt = now + } + _ = session.presence.TouchAccount(context.Background(), snapshot) +} + +func (session downstreamSession) removePresence() error { + if session.presence == nil || strings.TrimSpace(session.account) == "" { + return nil + } + return session.presence.RemoveAccount(context.Background(), strings.TrimSpace(session.account)) +} diff --git a/gateway/internal/inbound/server_test.go b/gateway/internal/inbound/server_test.go index 504f862..6670b36 100644 --- a/gateway/internal/inbound/server_test.go +++ b/gateway/internal/inbound/server_test.go @@ -1,10 +1,13 @@ package inbound import ( + "context" "encoding/json" + "log" "net" "net/http" "net/http/httptest" + "sync" "testing" "time" @@ -12,6 +15,63 @@ import ( cmpputils "github.com/bigwhite/gocmpp/utils" ) +type memoryPresenceStore struct { + snapshots map[string]DownstreamPresence + removed []string +} + +type memoryRecoveryStore struct { + decisions map[string]RecoveryStartDecision + completed []DownstreamRecoveryStatus +} + +func (m *memoryPresenceStore) TouchAccount(_ context.Context, snapshot DownstreamPresence) error { + if m.snapshots == nil { + m.snapshots = map[string]DownstreamPresence{} + } + m.snapshots[snapshot.Account] = snapshot + return nil +} + +func (m *memoryPresenceStore) RemoveAccount(_ context.Context, account string) error { + delete(m.snapshots, account) + m.removed = append(m.removed, account) + return nil +} + +func (m *memoryPresenceStore) ListAccounts(_ context.Context) ([]DownstreamPresence, error) { + result := make([]DownstreamPresence, 0, len(m.snapshots)) + for _, snapshot := range m.snapshots { + result = append(result, snapshot) + } + return result, nil +} + +func (m *memoryRecoveryStore) StartAccountRecovery(_ context.Context, account string, _ string) (RecoveryStartDecision, error) { + if decision, ok := m.decisions[account]; ok { + return decision, nil + } + return RecoveryStartDecision{Allowed: true}, nil +} + +func (m *memoryRecoveryStore) CompleteAccountRecovery(_ context.Context, status DownstreamRecoveryStatus) error { + m.completed = append(m.completed, status) + return nil +} + +func (m *memoryRecoveryStore) ListRecoveryStatuses(_ context.Context) ([]DownstreamRecoveryStatus, error) { + return append([]DownstreamRecoveryStatus(nil), m.completed...), nil +} + +func (m *memoryRecoveryStore) GetAccountRecoveryStatus(_ context.Context, account string) (DownstreamRecoveryStatus, error) { + for _, item := range m.completed { + if item.Account == account { + return item, nil + } + } + return DownstreamRecoveryStatus{Account: account}, nil +} + func TestInboundServerAuthenticatesAndSubmits(t *testing.T) { account := "100001" password := "secret-hash" @@ -29,6 +89,8 @@ func TestInboundServerAuthenticatesAndSubmits(t *testing.T) { t.Fatalf("decode submit: %v", err) } _ = json.NewEncoder(w).Encode(submitResponse{Accepted: true, MessageID: "MSG-1"}) + case "/api/gateway/events/downstream/pending": + _ = json.NewEncoder(w).Encode([]pendingDelivery{}) default: t.Fatalf("unexpected api path: %s", r.URL.Path) } @@ -76,6 +138,26 @@ func TestInboundServerAuthenticatesAndSubmits(t *testing.T) { if rsp.Result != 0 || rsp.MsgId == 0 { t.Fatalf("unexpected submit response: %+v", rsp) } + delivered, err := PushReceipt(DownstreamReceipt{ + MessageID: "MSG-1", + PhoneNumber: "13500002696", + ReceiptStatus: "delivered", + DeliveredAt: time.Now().UTC().Format(time.RFC3339Nano), + }) + if err != nil || !delivered { + t.Fatalf("push receipt delivered=%v err=%v", delivered, err) + } + deliver := recvDeliver(t, client) + if deliver.RegisterDelivery != 1 { + t.Fatalf("expected receipt deliver, got %+v", deliver) + } + var receipt cmpp.CmppReceiptPkt + if err := receipt.Unpack([]byte(deliver.MsgContent)); err != nil { + t.Fatalf("unpack pushed receipt: %v", err) + } + if receipt.Stat != "DELIVRD" || receipt.DestTerminalId != "13500002696" { + t.Fatalf("unexpected pushed receipt: %+v", receipt) + } if gotAuth.Account != account || gotAuth.AuthSource == "" || gotAuth.RemoteIP == "" { t.Fatalf("unexpected auth payload: %+v", gotAuth) } @@ -84,6 +166,183 @@ func TestInboundServerAuthenticatesAndSubmits(t *testing.T) { } } +func TestFlushOnlineAccountsFetchesPendingDeliveries(t *testing.T) { + resetDownstreamRegistry() + defer resetDownstreamRegistry() + + account := "100001" + calls := 0 + api := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/api/gateway/events/downstream/pending": + calls++ + var payload pendingDeliveryRequest + if err := json.NewDecoder(r.Body).Decode(&payload); err != nil { + t.Fatalf("decode pending request: %v", err) + } + if payload.Account != account { + t.Fatalf("unexpected account: %+v", payload) + } + _ = json.NewEncoder(w).Encode([]pendingDelivery{}) + default: + t.Fatalf("unexpected api path: %s", r.URL.Path) + } + })) + defer api.Close() + + downstreamRegistry.Lock() + downstreamRegistry.byAccount[account] = &downstreamSession{account: account} + downstreamRegistry.Unlock() + + server := Server{APIBaseURL: api.URL + "/api"} + server.flushOnlineAccounts(log.Default()) + + if calls != 1 { + t.Fatalf("pending fetch calls = %d, want 1", calls) + } +} + +func TestRecoverPendingCandidatesFetchesPresenceAccounts(t *testing.T) { + resetDownstreamRegistry() + defer resetDownstreamRegistry() + + account := "100009" + calls := 0 + api := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/api/gateway/events/downstream/pending": + calls++ + var payload pendingDeliveryRequest + if err := json.NewDecoder(r.Body).Decode(&payload); err != nil { + t.Fatalf("decode pending request: %v", err) + } + if payload.Account != account { + t.Fatalf("unexpected account: %+v", payload) + } + _ = json.NewEncoder(w).Encode([]pendingDelivery{}) + default: + t.Fatalf("unexpected api path: %s", r.URL.Path) + } + })) + defer api.Close() + + store := &memoryPresenceStore{ + snapshots: map[string]DownstreamPresence{ + account: { + Account: account, + GatewayInstanceID: "gateway-a", + State: "connected", + UpdatedAt: time.Now().UTC(), + }, + }, + } + + server := Server{APIBaseURL: api.URL + "/api", PresenceStore: store} + server.recoverPendingCandidates(log.Default()) + + if calls != 1 { + t.Fatalf("recovery pending fetch calls = %d, want 1", calls) + } +} + +func TestRecoverPendingCandidatesWritesWaitingConnectionStatus(t *testing.T) { + resetDownstreamRegistry() + defer resetDownstreamRegistry() + + account := "100010" + api := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/api/gateway/events/downstream/pending": + _ = json.NewEncoder(w).Encode([]pendingDelivery{{ + ID: "delivery-1", + DeliveryType: "receipt", + Payload: json.RawMessage(`{"messageId":"MSG-404","phoneNumber":"13800000001","receiptStatus":"delivered"}`), + }}) + case "/api/gateway/events/downstream/recovery-status": + w.WriteHeader(http.StatusOK) + default: + t.Fatalf("unexpected api path: %s", r.URL.Path) + } + })) + defer api.Close() + + recovery := &memoryRecoveryStore{} + store := &memoryPresenceStore{ + snapshots: map[string]DownstreamPresence{ + account: { + Account: account, + GatewayInstanceID: "gateway-a", + State: "connected", + UpdatedAt: time.Now().UTC(), + }, + }, + } + + server := Server{APIBaseURL: api.URL + "/api", PresenceStore: store, RecoveryStore: recovery} + server.recoverPendingCandidates(log.Default()) + + if len(recovery.completed) != 1 { + t.Fatalf("completed recovery statuses = %d, want 1", len(recovery.completed)) + } + if recovery.completed[0].State != "waiting_connection" { + t.Fatalf("unexpected recovery status: %+v", recovery.completed[0]) + } +} + +func TestRememberAndForgetAccountUpdatesPresenceStore(t *testing.T) { + resetDownstreamRegistry() + defer resetDownstreamRegistry() + + store := &memoryPresenceStore{} + session := downstreamSession{ + account: "100001", + srcID: "10690000", + remoteIP: "127.0.0.1", + connectedAt: time.Now().UTC(), + conn: &cmpp.Conn{}, + mu: &sync.Mutex{}, + presence: store, + instanceID: "gateway-a", + } + + rememberAccount(session) + snapshot, ok := store.snapshots["100001"] + if !ok { + t.Fatal("expected presence snapshot to be stored") + } + if snapshot.Account != "100001" || snapshot.GatewayInstanceID != "gateway-a" || snapshot.State != "connected" { + t.Fatalf("unexpected snapshot: %+v", snapshot) + } + + forgetDownstream(downstreamRegistry.byAccount["100001"]) + if len(store.removed) != 1 || store.removed[0] != "100001" { + t.Fatalf("unexpected removed accounts: %+v", store.removed) + } +} + +func recvDeliver(t *testing.T, client *cmpp.Client) *cmpp.Cmpp3DeliverReqPkt { + t.Helper() + deadline := time.Now().Add(2 * time.Second) + for time.Now().Before(deadline) { + packet, err := client.RecvAndUnpackPkt(200 * time.Millisecond) + if err != nil { + continue + } + if deliver, ok := packet.(*cmpp.Cmpp3DeliverReqPkt); ok { + return deliver + } + } + t.Fatal("timed out waiting deliver request") + return nil +} + +func resetDownstreamRegistry() { + downstreamRegistry.Lock() + defer downstreamRegistry.Unlock() + downstreamRegistry.byAccount = make(map[string]*downstreamSession) + downstreamRegistry.byMessageID = make(map[string]*downstreamSession) +} + func reserveTCPAddr(t *testing.T) string { t.Helper() listener, err := net.Listen("tcp", "127.0.0.1:0") diff --git a/gateway/internal/queue/messages.go b/gateway/internal/queue/messages.go index 50824d0..15f3d9f 100644 --- a/gateway/internal/queue/messages.go +++ b/gateway/internal/queue/messages.go @@ -25,19 +25,20 @@ type Envelope struct { type SubmitCommand struct { Envelope - TenantID string `json:"tenantId"` - ApplicationID string `json:"applicationId"` - TaskID string `json:"taskId,omitempty"` - SubmitID string `json:"submitId"` - PhoneNumber string `json:"phoneNumber"` - Content string `json:"content"` - Signature string `json:"signature"` - TemplateID string `json:"templateId"` - BillingUnits int `json:"billingUnits"` - QueuePriority string `json:"queuePriority"` - Route Route `json:"route"` - CMPP CMPP `json:"cmpp"` - Retry Retry `json:"retry"` + TenantID string `json:"tenantId"` + ApplicationID string `json:"applicationId"` + TaskID string `json:"taskId,omitempty"` + SubmitID string `json:"submitId"` + PhoneNumber string `json:"phoneNumber"` + Content string `json:"content"` + Signature string `json:"signature"` + TemplateID string `json:"templateId"` + BillingUnits int `json:"billingUnits"` + QueuePriority string `json:"queuePriority"` + Route Route `json:"route"` + CMPP CMPP `json:"cmpp"` + Upstream UpstreamConfig `json:"upstream"` + Retry Retry `json:"retry"` } type Route struct { @@ -57,6 +58,16 @@ type CMPP struct { FeeType string `json:"feeType,omitempty"` } +type UpstreamConfig struct { + GatewayHost string `json:"gatewayHost"` + GatewayPort int `json:"gatewayPort"` + Account string `json:"account"` + PasswordCipher string `json:"passwordCipher"` + CMPPVersion string `json:"cmppVersion"` + DesiredConnections int `json:"desiredConnections,omitempty"` + WindowSize int `json:"windowSize,omitempty"` +} + type Retry struct { Attempt int `json:"attempt"` MaxAttempts int `json:"maxAttempts"` @@ -70,12 +81,25 @@ type SubmitResult struct { ErrorCode string `json:"errorCode,omitempty"` ErrorMessage string `json:"errorMessage,omitempty"` SubmittedAt time.Time `json:"submittedAt"` + Segments []SubmitSegmentResult `json:"segments,omitempty"` +} + +type SubmitSegmentResult struct { + SegmentTotal int `json:"segmentTotal"` + SegmentIndex int `json:"segmentIndex"` + SequenceID uint32 `json:"sequenceId"` + GatewayMessageID string `json:"gatewayMessageId"` + SubmitStatus string `json:"submitStatus"` + ErrorCode string `json:"errorCode,omitempty"` + ErrorMessage string `json:"errorMessage,omitempty"` + SubmittedAt time.Time `json:"submittedAt"` } type ReceiptEvent struct { Envelope SequenceID uint32 `json:"sequenceId"` GatewayMessageID string `json:"gatewayMessageId"` + PhoneNumber string `json:"phoneNumber,omitempty"` ReceiptStatus string `json:"receiptStatus"` RawStatus string `json:"rawStatus"` ErrorCode string `json:"errorCode,omitempty"` diff --git a/gateway/internal/queue/messages_test.go b/gateway/internal/queue/messages_test.go index 68b5d5a..6727468 100644 --- a/gateway/internal/queue/messages_test.go +++ b/gateway/internal/queue/messages_test.go @@ -34,6 +34,15 @@ func TestSubmitCommandUnmarshalsQueuePriority(t *testing.T) { "registeredDelivery": 1, "msgFmt": 8 }, + "upstream": { + "gatewayHost": "127.0.0.1", + "gatewayPort": 17890, + "account": "account-a", + "passwordCipher": "secret", + "cmppVersion": "3.0", + "desiredConnections": 2, + "windowSize": 16 + }, "retry": { "attempt": 0, "maxAttempts": 3 @@ -47,4 +56,10 @@ func TestSubmitCommandUnmarshalsQueuePriority(t *testing.T) { if command.QueuePriority != "priority" { t.Fatalf("QueuePriority = %q, want priority", command.QueuePriority) } + if command.Upstream.GatewayHost != "127.0.0.1" || command.Upstream.Account != "account-a" { + t.Fatalf("unexpected upstream config: %+v", command.Upstream) + } + if command.Upstream.DesiredConnections != 2 || command.Upstream.WindowSize != 16 { + t.Fatalf("unexpected upstream window config: %+v", command.Upstream) + } } diff --git a/gateway/internal/spike/simulator.go b/gateway/internal/spike/simulator.go index 73d9ea7..43cb63e 100644 --- a/gateway/internal/spike/simulator.go +++ b/gateway/internal/spike/simulator.go @@ -134,6 +134,13 @@ func NewSubmitCommand(index int) queue.SubmitCommand { FeeCode: "0", FeeType: "01", }, + Upstream: queue.UpstreamConfig{ + GatewayHost: "127.0.0.1", + GatewayPort: 17890, + Account: "cmpp-account-spike", + PasswordCipher: "secret-spike", + CMPPVersion: "3.0", + }, Retry: queue.Retry{Attempt: 0, MaxAttempts: 3}, } } diff --git a/gateway/internal/submitworker/worker.go b/gateway/internal/submitworker/worker.go new file mode 100644 index 0000000..a6f5264 --- /dev/null +++ b/gateway/internal/submitworker/worker.go @@ -0,0 +1,431 @@ +package submitworker + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "log" + "net/http" + "strings" + "time" + + "cmpp-platform/gateway/internal/queue" + "cmpp-platform/gateway/internal/upstream" + + "github.com/redis/go-redis/v9" +) + +const ( + defaultStream = "gateway.submit.commands" + defaultGroup = "cmpp-gateway" + defaultConsumer = "gateway-1" + defaultMinIdle = 30 * time.Second + defaultMaxFails = 3 +) + +type Worker struct { + Redis *redis.Client + Upstream *upstream.Manager + Submit func(context.Context, queue.SubmitCommand) (queue.SubmitResult, error) + ReportDeadLetter func(context.Context, DeadLetterEvent) error + Stream string + Group string + Consumer string + Block time.Duration + Count int64 + MinIdle time.Duration + MaxFailures int + APIBaseURL string + HTTPClient *http.Client + Logger *log.Logger +} + +type DeadLetterEvent struct { + StreamMessageID string `json:"streamMessageId"` + TraceID string `json:"traceId,omitempty"` + MessageID string `json:"messageId,omitempty"` + ChannelID string `json:"channelId,omitempty"` + TenantID string `json:"tenantId,omitempty"` + ApplicationID string `json:"applicationId,omitempty"` + SubmitID string `json:"submitId,omitempty"` + FailureCode string `json:"failureCode"` + FailureMessage string `json:"failureMessage"` + Attempts int `json:"attempts"` + MaxAttempts int `json:"maxAttempts"` + CommandPayload map[string]interface{} `json:"commandPayload,omitempty"` + RawPayload string `json:"rawPayload,omitempty"` + DeadLetteredAt time.Time `json:"deadLetteredAt"` +} + +func New(redisURL string, manager *upstream.Manager) (*Worker, error) { + client, err := redisClient(redisURL) + if err != nil { + return nil, err + } + return &Worker{Redis: client, Upstream: manager}, nil +} + +func (w *Worker) Run(ctx context.Context) error { + if w.Redis == nil { + return fmt.Errorf("redis client is required") + } + if w.Upstream == nil { + return fmt.Errorf("upstream manager is required") + } + for { + if err := w.ensureGroup(ctx); err != nil { + if ctx.Err() != nil { + return ctx.Err() + } + w.logf("gateway submit worker ensure group failed: %v", err) + sleep(ctx, 3*time.Second) + continue + } + if err := w.recoverPending(ctx); err != nil { + if ctx.Err() != nil { + return ctx.Err() + } + w.logf("gateway submit worker pending recovery failed: %v", err) + sleep(ctx, time.Second) + continue + } + if err := w.consumeOnce(ctx); err != nil { + if ctx.Err() != nil { + return ctx.Err() + } + w.logf("gateway submit worker consume failed: %v", err) + sleep(ctx, time.Second) + } + } +} + +func (w *Worker) ensureGroup(ctx context.Context) error { + err := w.Redis.XGroupCreateMkStream(ctx, w.stream(), w.group(), "0").Err() + if err == nil || strings.Contains(err.Error(), "BUSYGROUP") { + return nil + } + return err +} + +func (w *Worker) consumeOnce(ctx context.Context) error { + streams, err := w.Redis.XReadGroup(ctx, &redis.XReadGroupArgs{ + Group: w.group(), + Consumer: w.consumer(), + Streams: []string{w.stream(), ">"}, + Count: w.count(), + Block: w.block(), + }).Result() + if errors.Is(err, redis.Nil) { + return nil + } + if err != nil { + return err + } + for _, stream := range streams { + if err := w.processMessages(ctx, stream.Messages); err != nil { + return err + } + } + return nil +} + +func (w *Worker) recoverPending(ctx context.Context) error { + start := "0-0" + for { + messages, next, err := w.Redis.XAutoClaim(ctx, &redis.XAutoClaimArgs{ + Stream: w.stream(), + Group: w.group(), + Consumer: w.consumer(), + MinIdle: w.minIdle(), + Start: start, + Count: w.count(), + }).Result() + if errors.Is(err, redis.Nil) { + return nil + } + if err != nil { + return err + } + if len(messages) == 0 { + return nil + } + w.logf("gateway submit worker reclaimed %d pending message(s)", len(messages)) + if err := w.processMessages(ctx, messages); err != nil { + return err + } + start = next + if next == "0-0" { + return nil + } + } +} + +func (w *Worker) processMessages(ctx context.Context, messages []redis.XMessage) error { + for _, message := range messages { + if err := w.processMessage(ctx, message); err != nil { + w.logf("gateway submit worker message %s failed: %v", message.ID, err) + continue + } + } + return nil +} + +func (w *Worker) processMessage(ctx context.Context, message redis.XMessage) error { + command, err := CommandFromStreamValues(message.Values) + if err != nil { + return w.deadLetterMalformedMessage(ctx, message, err) + } + if err := w.handleCommand(ctx, command); err != nil { + attempts, attemptsErr := w.incrementFailureAttempt(ctx, message.ID) + if attemptsErr != nil { + w.logf("gateway submit worker increment failure %s failed: %v", message.ID, attemptsErr) + } + if attempts >= w.maxFailures() { + if reportErr := w.deadLetterCommand(ctx, message, command, attempts, err); reportErr != nil { + return reportErr + } + return w.ackAndClearFailure(ctx, message.ID) + } + return err + } + return w.ackAndClearFailure(ctx, message.ID) +} + +func (w *Worker) handleCommand(ctx context.Context, command queue.SubmitCommand) error { + submit := w.Submit + if submit == nil { + if w.Upstream == nil { + return fmt.Errorf("upstream manager is required") + } + submit = w.Upstream.Submit + } + result, err := submit(ctx, command) + if err != nil && (result.SubmitStatus == "" || result.SubmitStatus == "accepted") { + return err + } + return nil +} + +func CommandFromStreamValues(values map[string]interface{}) (queue.SubmitCommand, error) { + raw, ok := values["data"] + if !ok { + return queue.SubmitCommand{}, fmt.Errorf("stream data field is required") + } + var data string + switch value := raw.(type) { + case string: + data = value + case []byte: + data = string(value) + default: + data = fmt.Sprint(value) + } + var command queue.SubmitCommand + if err := json.Unmarshal([]byte(data), &command); err != nil { + return queue.SubmitCommand{}, err + } + if command.MessageType != queue.MessageTypeSubmitCommand { + return queue.SubmitCommand{}, fmt.Errorf("unsupported messageType %q", command.MessageType) + } + return command, nil +} + +func (w *Worker) deadLetterMalformedMessage(ctx context.Context, message redis.XMessage, cause error) error { + event := DeadLetterEvent{ + StreamMessageID: message.ID, + FailureCode: "INVALID_COMMAND_PAYLOAD", + FailureMessage: cause.Error(), + Attempts: 1, + MaxAttempts: 1, + RawPayload: extractRawPayload(message.Values), + DeadLetteredAt: time.Now().UTC(), + } + if err := w.reportDeadLetter(ctx, event); err != nil { + return err + } + return w.ackAndClearFailure(ctx, message.ID) +} + +func (w *Worker) deadLetterCommand(ctx context.Context, message redis.XMessage, command queue.SubmitCommand, attempts int, cause error) error { + payload, payloadErr := commandPayload(command) + if payloadErr != nil { + w.logf("gateway submit worker marshal dead-letter command %s failed: %v", message.ID, payloadErr) + } + return w.reportDeadLetter(ctx, DeadLetterEvent{ + StreamMessageID: message.ID, + TraceID: command.TraceID, + MessageID: command.MessageID, + ChannelID: command.ChannelID, + TenantID: command.TenantID, + ApplicationID: command.ApplicationID, + SubmitID: command.SubmitID, + FailureCode: "SUBMIT_PROCESSING_FAILED", + FailureMessage: cause.Error(), + Attempts: attempts, + MaxAttempts: w.maxFailures(), + CommandPayload: payload, + RawPayload: extractRawPayload(message.Values), + DeadLetteredAt: time.Now().UTC(), + }) +} + +func (w *Worker) reportDeadLetter(ctx context.Context, event DeadLetterEvent) error { + if w.ReportDeadLetter != nil { + return w.ReportDeadLetter(ctx, event) + } + if w.APIBaseURL == "" { + return fmt.Errorf("gateway submit dead-letter reporter is not configured") + } + client := w.HTTPClient + if client == nil { + client = &http.Client{Timeout: 10 * time.Second} + } + body, err := json.Marshal(event) + if err != nil { + return err + } + req, err := http.NewRequestWithContext( + ctx, + http.MethodPost, + strings.TrimRight(w.APIBaseURL, "/")+"/gateway/events/dead-letter", + bytes.NewReader(body), + ) + if err != nil { + return err + } + req.Header.Set("Content-Type", "application/json") + resp, err := client.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + return fmt.Errorf("dead-letter endpoint returned %d", resp.StatusCode) + } + return nil +} + +func redisClient(redisURL string) (*redis.Client, error) { + if redisURL == "" { + redisURL = "redis://127.0.0.1:6379" + } + options, err := redis.ParseURL(redisURL) + if err != nil { + return nil, err + } + return redis.NewClient(options), nil +} + +func (w *Worker) incrementFailureAttempt(ctx context.Context, messageID string) (int, error) { + value, err := w.Redis.HIncrBy(ctx, w.failureAttemptsKey(), messageID, 1).Result() + return int(value), err +} + +func (w *Worker) ackAndClearFailure(ctx context.Context, messageID string) error { + if err := w.Redis.XAck(ctx, w.stream(), w.group(), messageID).Err(); err != nil { + return err + } + if err := w.Redis.HDel(ctx, w.failureAttemptsKey(), messageID).Err(); err != nil { + w.logf("gateway submit worker clear failure %s failed: %v", messageID, err) + } + return nil +} + +func (w *Worker) stream() string { + if w.Stream != "" { + return w.Stream + } + return defaultStream +} + +func (w *Worker) group() string { + if w.Group != "" { + return w.Group + } + return defaultGroup +} + +func (w *Worker) consumer() string { + if w.Consumer != "" { + return w.Consumer + } + return defaultConsumer +} + +func (w *Worker) block() time.Duration { + if w.Block > 0 { + return w.Block + } + return 5 * time.Second +} + +func (w *Worker) count() int64 { + if w.Count > 0 { + return w.Count + } + return 10 +} + +func (w *Worker) minIdle() time.Duration { + if w.MinIdle > 0 { + return w.MinIdle + } + return defaultMinIdle +} + +func (w *Worker) maxFailures() int { + if w.MaxFailures > 0 { + return w.MaxFailures + } + return defaultMaxFails +} + +func (w *Worker) failureAttemptsKey() string { + return w.stream() + ":failure-attempts" +} + +func (w *Worker) logf(format string, args ...interface{}) { + if w.Logger != nil { + w.Logger.Printf(format, args...) + return + } + log.Printf(format, args...) +} + +func sleep(ctx context.Context, duration time.Duration) { + timer := time.NewTimer(duration) + defer timer.Stop() + select { + case <-ctx.Done(): + case <-timer.C: + } +} + +func extractRawPayload(values map[string]interface{}) string { + raw, ok := values["data"] + if !ok { + return "" + } + switch value := raw.(type) { + case string: + return value + case []byte: + return string(value) + default: + return fmt.Sprint(value) + } +} + +func commandPayload(command queue.SubmitCommand) (map[string]interface{}, error) { + data, err := json.Marshal(command) + if err != nil { + return nil, err + } + var payload map[string]interface{} + if err := json.Unmarshal(data, &payload); err != nil { + return nil, err + } + return payload, nil +} diff --git a/gateway/internal/submitworker/worker_test.go b/gateway/internal/submitworker/worker_test.go new file mode 100644 index 0000000..c9f3f28 --- /dev/null +++ b/gateway/internal/submitworker/worker_test.go @@ -0,0 +1,180 @@ +package submitworker + +import ( + "context" + "testing" + + "cmpp-platform/gateway/internal/queue" + "github.com/alicebob/miniredis/v2" + "github.com/redis/go-redis/v9" +) + +func TestCommandFromStreamValuesParsesSubmitCommand(t *testing.T) { + command, err := CommandFromStreamValues(map[string]interface{}{ + "messageType": "SubmitCommand", + "data": `{ + "schemaVersion": "v1", + "messageType": "SubmitCommand", + "traceId": "trace-worker-0001", + "messageId": "msg-worker-0001", + "channelId": "channel-1", + "createdAt": "2026-07-07T10:00:00Z", + "tenantId": "tenant-1", + "applicationId": "app-1", + "submitId": "submit-1", + "phoneNumber": "13800138000", + "content": "hello", + "signature": "测试", + "templateId": "tpl-1", + "billingUnits": 1, + "queuePriority": "normal", + "route": { + "channelCode": "CMPP-A", + "cmppAccountCode": "account-a", + "priority": 0, + "rateLimitPerSecond": 100 + }, + "cmpp": { + "serviceId": "SMS", + "srcId": "10690000", + "registeredDelivery": 1, + "msgFmt": 8 + }, + "upstream": { + "gatewayHost": "127.0.0.1", + "gatewayPort": 17890, + "account": "account-a", + "passwordCipher": "secret", + "cmppVersion": "3.0" + }, + "retry": { + "attempt": 0, + "maxAttempts": 1 + } + }`, + }) + if err != nil { + t.Fatalf("parse stream command: %v", err) + } + if command.MessageID != "msg-worker-0001" || command.Upstream.Account != "account-a" { + t.Fatalf("unexpected command: %+v", command) + } +} + +func TestCommandFromStreamValuesRejectsMissingData(t *testing.T) { + _, err := CommandFromStreamValues(map[string]interface{}{"messageType": "SubmitCommand"}) + if err == nil { + t.Fatal("expected missing data error") + } +} + +func TestHandleMessageUsesInjectedSubmit(t *testing.T) { + var got queue.SubmitCommand + worker := &Worker{ + Submit: func(_ context.Context, command queue.SubmitCommand) (queue.SubmitResult, error) { + got = command + return queue.SubmitResult{SubmitStatus: "accepted"}, nil + }, + } + if err := worker.handleCommand(context.Background(), queue.SubmitCommand{ + Envelope: queue.Envelope{MessageID: "msg-worker-0002"}, + SubmitID: "submit-2", + PhoneNumber: "13800138000", + Content: "hello", + Upstream: queue.UpstreamConfig{GatewayHost: "127.0.0.1", GatewayPort: 17890, Account: "account-a", PasswordCipher: "secret", CMPPVersion: "3.0"}, + CMPP: queue.CMPP{ServiceID: "SMS", SrcID: "10690000", RegisteredDelivery: 1, MsgFmt: 8}, + Route: queue.Route{ChannelCode: "CMPP-A"}, + Retry: queue.Retry{Attempt: 0, MaxAttempts: 1}, + ApplicationID: "app-1", + TenantID: "tenant-1", + }); err != nil { + t.Fatalf("handleCommand returned error: %v", err) + } + if got.MessageID != "msg-worker-0002" || got.SubmitID != "submit-2" { + t.Fatalf("unexpected command: %+v", got) + } +} + +func TestMinIdleDefaultsToThirtySeconds(t *testing.T) { + worker := &Worker{} + if got := worker.minIdle(); got != defaultMinIdle { + t.Fatalf("minIdle = %v, want %v", got, defaultMinIdle) + } +} + +func TestProcessMessageDeadLettersAfterMaxFailures(t *testing.T) { + mr := miniredis.RunT(t) + client := redis.NewClient(&redis.Options{Addr: mr.Addr()}) + reported := []DeadLetterEvent{} + worker := &Worker{ + Redis: client, + Stream: "gateway.submit.commands", + Group: "cmpp-gateway", + MaxFailures: 2, + Submit: func(_ context.Context, command queue.SubmitCommand) (queue.SubmitResult, error) { + return queue.SubmitResult{}, context.DeadlineExceeded + }, + ReportDeadLetter: func(_ context.Context, event DeadLetterEvent) error { + reported = append(reported, event) + return nil + }, + } + ctx := context.Background() + if err := worker.ensureGroup(ctx); err != nil { + t.Fatalf("ensureGroup: %v", err) + } + message := redis.XMessage{ + ID: "1710000000000-0", + Values: map[string]interface{}{ + "messageType": "SubmitCommand", + "data": `{ + "schemaVersion": "v1", + "messageType": "SubmitCommand", + "traceId": "trace-worker-0003", + "messageId": "msg-worker-0003", + "channelId": "channel-1", + "createdAt": "2026-07-08T10:00:00Z", + "tenantId": "tenant-1", + "applicationId": "app-1", + "submitId": "submit-3", + "phoneNumber": "13800138000", + "content": "hello", + "signature": "测试", + "templateId": "tpl-1", + "billingUnits": 1, + "queuePriority": "normal", + "route": { "channelCode": "CMPP-A", "cmppAccountCode": "account-a", "priority": 0 }, + "cmpp": { "serviceId": "SMS", "srcId": "10690000", "registeredDelivery": 1, "msgFmt": 8 }, + "upstream": { "gatewayHost": "127.0.0.1", "gatewayPort": 17890, "account": "account-a", "passwordCipher": "secret", "cmppVersion": "3.0" }, + "retry": { "attempt": 0, "maxAttempts": 1 } + }`, + }, + } + if err := client.XAdd(ctx, &redis.XAddArgs{ + Stream: worker.stream(), + ID: message.ID, + Values: message.Values, + }).Err(); err != nil { + t.Fatalf("xadd: %v", err) + } + + if err := worker.processMessage(ctx, message); err == nil { + t.Fatal("expected first failure") + } + if len(reported) != 0 { + t.Fatalf("unexpected dead letters on first failure: %+v", reported) + } + + if err := worker.processMessage(ctx, message); err != nil { + t.Fatalf("second failure should dead-letter and ack, got %v", err) + } + if len(reported) != 1 { + t.Fatalf("dead letters = %d, want 1", len(reported)) + } + if reported[0].FailureCode != "SUBMIT_PROCESSING_FAILED" || reported[0].Attempts != 2 { + t.Fatalf("unexpected dead letter: %+v", reported[0]) + } + if client.HGet(ctx, worker.failureAttemptsKey(), message.ID).Err() != redis.Nil { + t.Fatalf("failure attempt key was not cleared") + } +} diff --git a/gateway/internal/upstream/connection_loss_test.go b/gateway/internal/upstream/connection_loss_test.go new file mode 100644 index 0000000..db08bf3 --- /dev/null +++ b/gateway/internal/upstream/connection_loss_test.go @@ -0,0 +1,50 @@ +package upstream + +import ( + "errors" + "testing" +) + +func TestHandleConnectionLossNotifiesPendingSubmitters(t *testing.T) { + conn := &connection{ + pending: make(map[uint32]chan submitPartResponse), + } + waiter := make(chan submitPartResponse, 1) + conn.pending[7] = waiter + + loss := errors.New("socket closed") + conn.handleConnectionLoss(loss) + + select { + case result := <-waiter: + if !errors.Is(result.err, loss) { + t.Fatalf("pending waiter err = %v, want %v", result.err, loss) + } + default: + t.Fatal("expected pending waiter to be notified") + } + + if !conn.closed { + t.Fatal("expected connection to be marked closed") + } + if len(conn.pending) != 0 { + t.Fatalf("expected pending map to be reset, got %d entries", len(conn.pending)) + } +} + +func TestTemporaryReadTimeoutDetection(t *testing.T) { + if !isTemporaryReadTimeout(fakeNetError{timeout: true}) { + t.Fatal("expected timeout error to be treated as temporary") + } + if isTemporaryReadTimeout(errors.New("eof")) { + t.Fatal("did not expect non-timeout error to be treated as temporary") + } +} + +type fakeNetError struct { + timeout bool +} + +func (f fakeNetError) Error() string { return "network error" } +func (f fakeNetError) Timeout() bool { return f.timeout } +func (f fakeNetError) Temporary() bool { return f.timeout } diff --git a/gateway/internal/upstream/long_message.go b/gateway/internal/upstream/long_message.go new file mode 100644 index 0000000..7de555a --- /dev/null +++ b/gateway/internal/upstream/long_message.go @@ -0,0 +1,153 @@ +package upstream + +import ( + "fmt" + "strings" + "time" +) + +const ( + maxSingleMessageBytes = 140 + maxMultipartPayloadBytes = 134 + concatUDHLength = 6 + maxMultipartSegments = 255 +) + +type submitPart struct { + PkTotal uint8 + PkNumber uint8 + TpUdhi uint8 + MsgContent string +} + +type longUplinkAssembly struct { + msgFmt uint8 + total uint8 + parts map[uint8]string + updatedAt time.Time +} + +func splitSubmitContent(format int, content string) ([]submitPart, error) { + encoded, err := encodeContent(format, content) + if err != nil { + return nil, err + } + if len(encoded) <= maxSingleMessageBytes { + return []submitPart{{ + PkTotal: 1, + PkNumber: 1, + TpUdhi: 0, + MsgContent: encoded, + }}, nil + } + + chunks, err := splitEncodedContent(format, content, maxMultipartPayloadBytes) + if err != nil { + return nil, err + } + if len(chunks) > maxMultipartSegments { + return nil, fmt.Errorf("message requires %d segments, maximum is %d", len(chunks), maxMultipartSegments) + } + + ref := uint8(time.Now().UnixNano()) + parts := make([]submitPart, 0, len(chunks)) + for i, chunk := range chunks { + total := uint8(len(chunks)) + number := uint8(i + 1) + udh := []byte{0x05, 0x00, 0x03, ref, total, number} + content := append(udh, []byte(chunk)...) + parts = append(parts, submitPart{ + PkTotal: total, + PkNumber: number, + TpUdhi: 1, + MsgContent: string(content), + }) + } + return parts, nil +} + +func splitEncodedContent(format int, content string, limit int) ([]string, error) { + var chunks []string + var current strings.Builder + currentLen := 0 + for _, r := range content { + encoded, err := encodeContent(format, string(r)) + if err != nil { + return nil, err + } + if len(encoded) > limit { + return nil, fmt.Errorf("single character exceeds segment payload limit") + } + if currentLen > 0 && currentLen+len(encoded) > limit { + chunks = append(chunks, current.String()) + current.Reset() + currentLen = 0 + } + current.WriteString(encoded) + currentLen += len(encoded) + } + if currentLen > 0 { + chunks = append(chunks, current.String()) + } + if len(chunks) == 0 { + chunks = append(chunks, "") + } + return chunks, nil +} + +func parseConcatSegment(content string) (ref uint8, total uint8, number uint8, payload string, ok bool) { + raw := []byte(content) + if len(raw) < concatUDHLength { + return 0, 0, 0, "", false + } + if raw[0] != 0x05 || raw[1] != 0x00 || raw[2] != 0x03 { + return 0, 0, 0, "", false + } + ref = raw[3] + total = raw[4] + number = raw[5] + if total == 0 || number == 0 || number > total { + return 0, 0, 0, "", false + } + return ref, total, number, string(raw[concatUDHLength:]), true +} + +func assembleLongUplink(assemblies map[string]*longUplinkAssembly, key string, msgFmt uint8, total uint8, number uint8, payload string) (string, bool, error) { + assembly := assemblies[key] + if assembly == nil || assembly.total != total || assembly.msgFmt != msgFmt { + assembly = &longUplinkAssembly{ + msgFmt: msgFmt, + total: total, + parts: make(map[uint8]string, int(total)), + } + assemblies[key] = assembly + } + assembly.parts[number] = payload + assembly.updatedAt = time.Now() + if len(assembly.parts) < int(total) { + return "", false, nil + } + + var raw strings.Builder + for i := uint8(1); i <= total; i++ { + part, ok := assembly.parts[i] + if !ok { + return "", false, nil + } + raw.WriteString(part) + } + delete(assemblies, key) + content, err := decodeContent(msgFmt, raw.String()) + if err != nil { + return "", false, err + } + return content, true, nil +} + +func pruneLongUplinkAssemblies(assemblies map[string]*longUplinkAssembly, now time.Time, ttl time.Duration) { + for key, assembly := range assemblies { + if now.Sub(assembly.updatedAt) > ttl { + delete(assemblies, key) + } + } +} diff --git a/gateway/internal/upstream/long_message_test.go b/gateway/internal/upstream/long_message_test.go new file mode 100644 index 0000000..6a38301 --- /dev/null +++ b/gateway/internal/upstream/long_message_test.go @@ -0,0 +1,95 @@ +package upstream + +import ( + "strings" + "testing" +) + +func TestSplitSubmitContentUCS2LongMessage(t *testing.T) { + content := strings.Repeat("测试", 40) + + parts, err := splitSubmitContent(8, content) + if err != nil { + t.Fatalf("splitSubmitContent returned error: %v", err) + } + if len(parts) < 2 { + t.Fatalf("expected multipart content, got %d part", len(parts)) + } + total := uint8(len(parts)) + ref := []byte(parts[0].MsgContent)[3] + for i, part := range parts { + if part.PkTotal != total { + t.Fatalf("part %d PkTotal = %d, want %d", i, part.PkTotal, total) + } + if part.PkNumber != uint8(i+1) { + t.Fatalf("part %d PkNumber = %d, want %d", i, part.PkNumber, i+1) + } + if part.TpUdhi != 1 { + t.Fatalf("part %d TpUdhi = %d, want 1", i, part.TpUdhi) + } + raw := []byte(part.MsgContent) + if len(raw) > maxSingleMessageBytes { + t.Fatalf("part %d length = %d, want <= %d", i, len(raw), maxSingleMessageBytes) + } + if raw[0] != 0x05 || raw[1] != 0x00 || raw[2] != 0x03 { + t.Fatalf("part %d missing standard concat UDH: %v", i, raw[:concatUDHLength]) + } + if raw[3] != ref || raw[4] != total || raw[5] != uint8(i+1) { + t.Fatalf("part %d UDH = %v, ref=%d total=%d number=%d", i, raw[:concatUDHLength], ref, total, i+1) + } + if len(raw[concatUDHLength:])%2 != 0 { + t.Fatalf("part %d UCS2 payload length must be even, got %d", i, len(raw[concatUDHLength:])) + } + } +} + +func TestSplitSubmitContentSingleShortMessage(t *testing.T) { + parts, err := splitSubmitContent(15, "hello") + if err != nil { + t.Fatalf("splitSubmitContent returned error: %v", err) + } + if len(parts) != 1 { + t.Fatalf("expected one part, got %d", len(parts)) + } + if parts[0].PkTotal != 1 || parts[0].PkNumber != 1 || parts[0].TpUdhi != 0 { + t.Fatalf("unexpected single part metadata: %+v", parts[0]) + } +} + +func TestAssembleLongUplinkOutOfOrder(t *testing.T) { + content := strings.Repeat("上行", 40) + parts, err := splitSubmitContent(8, content) + if err != nil { + t.Fatalf("splitSubmitContent returned error: %v", err) + } + if len(parts) < 2 { + t.Fatalf("expected multipart content, got %d part", len(parts)) + } + + assemblies := map[string]*longUplinkAssembly{} + key := "channel:phone:dest:ref" + for i := len(parts) - 1; i >= 0; i-- { + _, total, number, payload, ok := parseConcatSegment(parts[i].MsgContent) + if !ok { + t.Fatalf("part %d did not parse as concat segment", i) + } + assembled, complete, err := assembleLongUplink(assemblies, key, 8, total, number, payload) + if err != nil { + t.Fatalf("assembleLongUplink returned error: %v", err) + } + if i > 0 && complete { + t.Fatalf("assembly completed before all parts arrived") + } + if i == 0 { + if !complete { + t.Fatalf("assembly did not complete after all parts arrived") + } + if assembled != content { + t.Fatalf("assembled content mismatch: got %q want %q", assembled, content) + } + } + } + if len(assemblies) != 0 { + t.Fatalf("expected completed assembly to be removed, got %d", len(assemblies)) + } +} diff --git a/gateway/internal/upstream/manager.go b/gateway/internal/upstream/manager.go new file mode 100644 index 0000000..1b0755f --- /dev/null +++ b/gateway/internal/upstream/manager.go @@ -0,0 +1,697 @@ +package upstream + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "net" + "net/http" + "strings" + "sync" + "time" + + "cmpp-platform/gateway/internal/queue" + + cmpp "github.com/bigwhite/gocmpp" + cmpputils "github.com/bigwhite/gocmpp/utils" +) + +const ( + defaultConnectTimeout = 5 * time.Second + defaultSubmitTimeout = 10 * time.Second + defaultHTTPTimeout = 10 * time.Second + defaultWindowSize = 16 +) + +type Manager struct { + APIBaseURL string + HTTPClient *http.Client + + mu sync.Mutex + conns map[string]*connectionPool +} + +func (m *Manager) Submit(ctx context.Context, cmd queue.SubmitCommand) (queue.SubmitResult, error) { + if err := validateSubmitCommand(cmd); err != nil { + result := submitResult(cmd, 0, "", "rejected", "INVALID_COMMAND", err.Error()) + if postErr := m.post(ctx, "/gateway/events/submit-result", result); postErr != nil { + return result, postErr + } + return result, err + } + + pool, err := m.connectionFor(cmd) + if err != nil { + result := submitResult(cmd, 0, "", "rejected", "CONNECT_FAILED", err.Error()) + if postErr := m.post(ctx, "/gateway/events/submit-result", result); postErr != nil { + return result, postErr + } + return result, err + } + + result, err := pool.submit(ctx, cmd) + if err != nil { + if postErr := m.post(ctx, "/gateway/events/submit-result", result); postErr != nil { + return result, postErr + } + return result, err + } + if err := m.post(ctx, "/gateway/events/submit-result", result); err != nil { + return result, err + } + return result, nil +} + +func (m *Manager) connectionFor(cmd queue.SubmitCommand) (*connectionPool, error) { + m.mu.Lock() + defer m.mu.Unlock() + + if m.HTTPClient == nil { + m.HTTPClient = &http.Client{Timeout: defaultHTTPTimeout} + } + if m.conns == nil { + m.conns = make(map[string]*connectionPool) + } + + pool := m.conns[cmd.ChannelID] + if pool == nil || !pool.matches(cmd.Upstream) { + if pool != nil { + pool.close() + } + pool = &connectionPool{ + channelID: cmd.ChannelID, + config: normalizeUpstreamConfig(cmd.Upstream), + apiBaseURL: m.APIBaseURL, + httpClient: m.HTTPClient, + } + m.conns[cmd.ChannelID] = pool + } + if err := pool.ensureConnected(); err != nil { + delete(m.conns, cmd.ChannelID) + return nil, err + } + return pool, nil +} + +func (m *Manager) post(ctx context.Context, path string, payload any) error { + client := m.HTTPClient + if client == nil { + client = &http.Client{Timeout: defaultHTTPTimeout} + } + return postJSON(ctx, client, m.APIBaseURL, path, payload) +} + +type connectionPool struct { + channelID string + config queue.UpstreamConfig + apiBaseURL string + httpClient *http.Client + + mu sync.Mutex + conns []*connection + next int +} + +func (p *connectionPool) matches(config queue.UpstreamConfig) bool { + return p.config == normalizeUpstreamConfig(config) +} + +func (p *connectionPool) ensureConnected() error { + p.mu.Lock() + defer p.mu.Unlock() + + desired := p.config.DesiredConnections + if desired <= 0 { + desired = 1 + } + for len(p.conns) < desired { + index := len(p.conns) + conn := &connection{ + channelID: p.channelID, + config: p.config, + index: index, + apiBaseURL: p.apiBaseURL, + httpClient: p.httpClient, + window: make(chan struct{}, p.config.WindowSize), + pending: make(map[uint32]chan submitPartResponse), + tracker: make(map[uint64]queue.SubmitCommand), + longUplink: make(map[string]*longUplinkAssembly), + } + if err := conn.ensureConnected(); err != nil { + conn.close() + p.closeLocked() + return err + } + p.conns = append(p.conns, conn) + } + return nil +} + +func (p *connectionPool) submit(ctx context.Context, cmd queue.SubmitCommand) (queue.SubmitResult, error) { + parts, err := splitSubmitContent(cmd.CMPP.MsgFmt, cmd.Content) + if err != nil { + result := submitResult(cmd, 0, "", "rejected", "ENCODE_FAILED", err.Error()) + return result, err + } + + var firstSequence uint32 + var firstGatewayMessageID string + segments := make([]queue.SubmitSegmentResult, 0, len(parts)) + for _, part := range parts { + conn, release, err := p.acquireConnection(ctx) + if err != nil { + result := submitResult(cmd, 0, "", "timeout", "WINDOW_TIMEOUT", err.Error()) + result.Segments = segments + return result, err + } + seq, gatewayMessageID, result, err := conn.submitPart(ctx, cmd, part) + release() + segments = append(segments, submitSegmentResult(part, seq, gatewayMessageID, result)) + if firstSequence == 0 { + firstSequence = seq + } + if firstGatewayMessageID == "" { + firstGatewayMessageID = gatewayMessageID + } + if err != nil { + result.Segments = segments + return result, err + } + if result.SubmitStatus != "accepted" { + result.Segments = segments + return result, nil + } + } + + result := submitResult(cmd, firstSequence, firstGatewayMessageID, "accepted", "", "") + result.Segments = segments + return result, nil +} + +func (p *connectionPool) acquireConnection(ctx context.Context) (*connection, func(), error) { + waitCtx, cancel := context.WithTimeout(ctx, defaultSubmitTimeout) + defer cancel() + ticker := time.NewTicker(10 * time.Millisecond) + defer ticker.Stop() + + for { + if conn, release := p.tryAcquireConnection(); conn != nil { + if err := conn.ensureConnected(); err != nil { + release() + select { + case <-waitCtx.Done(): + return nil, nil, waitCtx.Err() + case <-ticker.C: + continue + } + } + return conn, release, nil + } + select { + case <-waitCtx.Done(): + return nil, nil, waitCtx.Err() + case <-ticker.C: + } + } +} + +func (p *connectionPool) tryAcquireConnection() (*connection, func()) { + p.mu.Lock() + defer p.mu.Unlock() + if len(p.conns) == 0 { + return nil, nil + } + for i := 0; i < len(p.conns); i++ { + index := (p.next + i) % len(p.conns) + conn := p.conns[index] + if conn.tryAcquireWindow() { + p.next = (index + 1) % len(p.conns) + return conn, conn.releaseWindow + } + } + return nil, nil +} + +func (p *connectionPool) close() { + p.mu.Lock() + defer p.mu.Unlock() + p.closeLocked() +} + +func (p *connectionPool) closeLocked() { + for _, conn := range p.conns { + conn.close() + } + p.conns = nil +} + +type connection struct { + channelID string + config queue.UpstreamConfig + index int + apiBaseURL string + httpClient *http.Client + + mu sync.Mutex + sendMu sync.Mutex + client *cmpp.Client + window chan struct{} + pending map[uint32]chan submitPartResponse + tracker map[uint64]queue.SubmitCommand + longUplink map[string]*longUplinkAssembly + readOnce sync.Once + closed bool +} + +type submitPartResponse struct { + rsp *cmpp.Cmpp3SubmitRspPkt + err error +} + +func (c *connection) matches(config queue.UpstreamConfig) bool { + return c.config == normalizeUpstreamConfig(config) +} + +func (c *connection) ensureConnected() error { + c.mu.Lock() + defer c.mu.Unlock() + + if c.client != nil && !c.closed { + return nil + } + client := cmpp.NewClient(protocolVersion(c.config.CMPPVersion)) + addr := fmt.Sprintf("%s:%d", c.config.GatewayHost, c.config.GatewayPort) + if err := client.Connect(addr, c.config.Account, c.config.PasswordCipher, defaultConnectTimeout); err != nil { + client.Disconnect() + return err + } + c.client = client + c.closed = false + go c.readLoop() + return nil +} + +func (c *connection) submitPart(ctx context.Context, cmd queue.SubmitCommand, part submitPart) (uint32, string, queue.SubmitResult, error) { + rspCh := make(chan submitPartResponse, 1) + pkt := &cmpp.Cmpp3SubmitReqPkt{ + PkTotal: part.PkTotal, + PkNumber: part.PkNumber, + TpUdhi: part.TpUdhi, + RegisteredDelivery: uint8(cmd.CMPP.RegisteredDelivery), + MsgLevel: 1, + ServiceId: cmd.CMPP.ServiceID, + FeeUserType: uint8(defaultInt(cmd.CMPP.FeeUserType, 2)), + FeeTerminalId: cmd.PhoneNumber, + MsgFmt: uint8(cmd.CMPP.MsgFmt), + MsgSrc: c.config.Account, + FeeType: defaultString(cmd.CMPP.FeeType, "02"), + FeeCode: defaultString(cmd.CMPP.FeeCode, "0"), + SrcId: cmd.CMPP.SrcID, + DestUsrTl: 1, + DestTerminalId: []string{cmd.PhoneNumber}, + MsgLength: uint8(len(part.MsgContent)), + MsgContent: part.MsgContent, + } + + c.sendMu.Lock() + seq, err := c.client.SendReqPkt(pkt) + c.sendMu.Unlock() + if err != nil { + c.close() + result := submitResult(cmd, 0, "", "timeout", "SEND_FAILED", err.Error()) + return 0, "", result, err + } + + c.mu.Lock() + c.pending[seq] = rspCh + c.mu.Unlock() + defer func() { + c.mu.Lock() + delete(c.pending, seq) + c.mu.Unlock() + }() + + waitCtx, cancel := context.WithTimeout(ctx, defaultSubmitTimeout) + defer cancel() + select { + case <-waitCtx.Done(): + result := submitResult(cmd, seq, "", "timeout", "SUBMIT_TIMEOUT", waitCtx.Err().Error()) + return seq, "", result, waitCtx.Err() + case rsp := <-rspCh: + if rsp.err != nil { + result := submitResult(cmd, seq, "", "timeout", "CONNECTION_LOST", rsp.err.Error()) + return seq, "", result, rsp.err + } + if rsp.rsp == nil { + err := fmt.Errorf("submit response is empty") + result := submitResult(cmd, seq, "", "timeout", "EMPTY_SUBMIT_RESPONSE", err.Error()) + return seq, "", result, err + } + gatewayMessageID := fmt.Sprint(rsp.rsp.MsgId) + status := "accepted" + errorCode := "" + errorMessage := "" + if rsp.rsp.Result != 0 { + status = "rejected" + errorCode = fmt.Sprint(rsp.rsp.Result) + errorMessage = fmt.Sprintf("upstream submit rejected with result %d", rsp.rsp.Result) + } + if rsp.rsp.Result == 0 { + c.mu.Lock() + c.tracker[rsp.rsp.MsgId] = cmd + c.mu.Unlock() + } + return seq, gatewayMessageID, submitResult(cmd, seq, gatewayMessageID, status, errorCode, errorMessage), nil + } +} + +func (c *connection) tryAcquireWindow() bool { + if c.window == nil { + c.window = make(chan struct{}, defaultWindowSize) + } + select { + case c.window <- struct{}{}: + return true + default: + return false + } +} + +func (c *connection) releaseWindow() { + if c.window == nil { + return + } + select { + case <-c.window: + default: + } +} + +func (c *connection) readLoop() { + for { + pkt, err := c.client.RecvAndUnpackPkt(time.Second) + if err != nil { + c.mu.Lock() + closed := c.closed + c.mu.Unlock() + if closed { + return + } + if isTemporaryReadTimeout(err) { + continue + } + c.handleConnectionLoss(err) + continue + } + switch p := pkt.(type) { + case *cmpp.Cmpp3SubmitRspPkt: + c.mu.Lock() + ch := c.pending[p.SeqId] + c.mu.Unlock() + if ch != nil { + ch <- submitPartResponse{rsp: p} + } + case *cmpp.Cmpp3DeliverReqPkt: + c.handleDeliver(p) + case *cmpp.CmppActiveTestReqPkt: + _ = c.client.SendRspPkt(&cmpp.CmppActiveTestRspPkt{}, p.SeqId) + } + } +} + +func (c *connection) handleDeliver(pkt *cmpp.Cmpp3DeliverReqPkt) { + _ = c.client.SendRspPkt(&cmpp.Cmpp3DeliverRspPkt{MsgId: pkt.MsgId, Result: 0}, pkt.SeqId) + + if pkt.RegisterDelivery == 1 { + var receipt cmpp.CmppReceiptPkt + if err := receipt.Unpack([]byte(pkt.MsgContent)); err != nil { + return + } + cmd, ok := c.commandFor(receipt.MsgId) + if !ok { + cmd, ok = c.commandFor(pkt.MsgId) + } + traceID := fmt.Sprintf("receipt-%d", receipt.MsgId) + messageID := fmt.Sprintf("receipt-%d", receipt.MsgId) + channelID := c.channelID + if ok { + traceID = cmd.TraceID + messageID = cmd.MessageID + channelID = cmd.ChannelID + } + event := queue.ReceiptEvent{ + Envelope: queue.Envelope{ + SchemaVersion: queue.SchemaVersion, + MessageType: queue.MessageTypeReceiptEvent, + TraceID: traceID, + MessageID: messageID, + ChannelID: channelID, + CreatedAt: time.Now().UTC(), + }, + SequenceID: pkt.SeqId, + GatewayMessageID: fmt.Sprint(receipt.MsgId), + PhoneNumber: strings.TrimSpace(receipt.DestTerminalId), + ReceiptStatus: receiptStatus(receipt.Stat), + RawStatus: strings.TrimSpace(receipt.Stat), + DeliveredAt: time.Now().UTC(), + } + _ = postJSON(context.Background(), c.httpClient, c.apiBaseURL, "/gateway/events/receipt", event) + return + } + + content, complete, err := c.decodeUplinkContent(pkt) + if err != nil { + return + } + if !complete { + return + } + cmd, _ := c.commandFor(pkt.MsgId) + event := queue.UplinkEvent{ + Envelope: queue.Envelope{ + SchemaVersion: queue.SchemaVersion, + MessageType: queue.MessageTypeUplinkEvent, + TraceID: cmd.TraceID, + MessageID: cmd.MessageID, + ChannelID: c.channelID, + CreatedAt: time.Now().UTC(), + }, + SequenceID: pkt.SeqId, + PhoneNumber: strings.TrimSpace(pkt.SrcTerminalId), + DestID: strings.TrimSpace(pkt.DestId), + Content: content, + ReceivedAt: time.Now().UTC(), + } + _ = postJSON(context.Background(), c.httpClient, c.apiBaseURL, "/gateway/events/uplink", event) +} + +func (c *connection) decodeUplinkContent(pkt *cmpp.Cmpp3DeliverReqPkt) (string, bool, error) { + if pkt.TpUdhi != 1 { + content, err := decodeContent(pkt.MsgFmt, pkt.MsgContent) + return content, true, err + } + ref, total, number, payload, ok := parseConcatSegment(pkt.MsgContent) + if !ok { + content, err := decodeContent(pkt.MsgFmt, pkt.MsgContent) + return content, true, err + } + key := fmt.Sprintf("%s:%s:%s:%d:%d", c.channelID, strings.TrimSpace(pkt.SrcTerminalId), strings.TrimSpace(pkt.DestId), ref, total) + c.mu.Lock() + if c.longUplink == nil { + c.longUplink = make(map[string]*longUplinkAssembly) + } + pruneLongUplinkAssemblies(c.longUplink, time.Now(), 10*time.Minute) + content, complete, err := assembleLongUplink(c.longUplink, key, pkt.MsgFmt, total, number, payload) + c.mu.Unlock() + return content, complete, err +} + +func (c *connection) commandFor(gatewayMsgID uint64) (queue.SubmitCommand, bool) { + c.mu.Lock() + defer c.mu.Unlock() + cmd, ok := c.tracker[gatewayMsgID] + return cmd, ok +} + +func (c *connection) close() { + c.handleConnectionLoss(fmt.Errorf("connection closed")) +} + +func (c *connection) handleConnectionLoss(err error) { + c.mu.Lock() + if c.closed && c.client == nil { + c.mu.Unlock() + return + } + c.closed = true + pending := c.pending + c.pending = make(map[uint32]chan submitPartResponse) + if c.client != nil { + c.client.Disconnect() + c.client = nil + } + c.mu.Unlock() + + for _, ch := range pending { + select { + case ch <- submitPartResponse{err: err}: + default: + } + } +} + +func submitResult(cmd queue.SubmitCommand, sequenceID uint32, gatewayMessageID string, status string, code string, message string) queue.SubmitResult { + if gatewayMessageID == "" { + gatewayMessageID = fmt.Sprintf("GW-%s-%d", cmd.SubmitID, time.Now().UnixNano()) + } + return queue.SubmitResult{ + Envelope: queue.Envelope{ + SchemaVersion: queue.SchemaVersion, + MessageType: queue.MessageTypeSubmitResult, + TraceID: cmd.TraceID, + MessageID: cmd.MessageID, + ChannelID: cmd.ChannelID, + CreatedAt: time.Now().UTC(), + }, + SequenceID: sequenceID, + GatewayMessageID: gatewayMessageID, + SubmitStatus: status, + ErrorCode: code, + ErrorMessage: message, + SubmittedAt: time.Now().UTC(), + } +} + +func submitSegmentResult(part submitPart, sequenceID uint32, gatewayMessageID string, result queue.SubmitResult) queue.SubmitSegmentResult { + if gatewayMessageID == "" { + gatewayMessageID = result.GatewayMessageID + } + return queue.SubmitSegmentResult{ + SegmentTotal: int(part.PkTotal), + SegmentIndex: int(part.PkNumber), + SequenceID: sequenceID, + GatewayMessageID: gatewayMessageID, + SubmitStatus: result.SubmitStatus, + ErrorCode: result.ErrorCode, + ErrorMessage: result.ErrorMessage, + SubmittedAt: result.SubmittedAt, + } +} + +func validateSubmitCommand(cmd queue.SubmitCommand) error { + if cmd.MessageType != queue.MessageTypeSubmitCommand { + return fmt.Errorf("unsupported messageType %q", cmd.MessageType) + } + if cmd.MessageID == "" || cmd.ChannelID == "" || cmd.SubmitID == "" { + return fmt.Errorf("messageId, channelId and submitId are required") + } + if cmd.Upstream.GatewayHost == "" || cmd.Upstream.GatewayPort <= 0 { + return fmt.Errorf("upstream gatewayHost and gatewayPort are required") + } + if cmd.Upstream.Account == "" || cmd.Upstream.PasswordCipher == "" { + return fmt.Errorf("upstream account and passwordCipher are required") + } + if len(cmd.PhoneNumber) == 0 || len(cmd.Content) == 0 { + return fmt.Errorf("phoneNumber and content are required") + } + return nil +} + +func normalizeUpstreamConfig(config queue.UpstreamConfig) queue.UpstreamConfig { + if config.DesiredConnections <= 0 { + config.DesiredConnections = 1 + } + if config.WindowSize <= 0 { + config.WindowSize = defaultWindowSize + } + return config +} + +func isTemporaryReadTimeout(err error) bool { + var netErr net.Error + return errors.As(err, &netErr) && netErr.Timeout() +} + +func postJSON(ctx context.Context, client *http.Client, apiBaseURL string, path string, payload any) error { + if client == nil { + client = &http.Client{Timeout: defaultHTTPTimeout} + } + body, err := json.Marshal(payload) + if err != nil { + return err + } + base := strings.TrimRight(apiBaseURL, "/") + if base == "" { + base = "http://127.0.0.1:3000/api" + } + req, err := http.NewRequestWithContext(ctx, http.MethodPost, base+path, bytes.NewReader(body)) + if err != nil { + return err + } + req.Header.Set("Content-Type", "application/json") + resp, err := client.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + return fmt.Errorf("api returned %s", resp.Status) + } + return nil +} + +func encodeContent(format int, content string) (string, error) { + switch format { + case 8: + return cmpputils.Utf8ToUcs2(content) + case 15: + return cmpputils.Utf8ToGB18030(content) + default: + return content, nil + } +} + +func decodeContent(format uint8, content string) (string, error) { + switch format { + case 8: + return cmpputils.Ucs2ToUtf8(content) + case 15: + return cmpputils.GB18030ToUtf8(content) + default: + return content, nil + } +} + +func protocolVersion(version string) cmpp.Type { + if strings.HasPrefix(version, "2") { + return cmpp.V20 + } + return cmpp.V30 +} + +func receiptStatus(stat string) string { + switch strings.ToUpper(strings.TrimSpace(stat)) { + case "DELIVRD": + return "delivered" + case "": + return "unknown" + default: + return "undelivered" + } +} + +func defaultString(value string, fallback string) string { + if value == "" { + return fallback + } + return value +} + +func defaultInt(value int, fallback int) int { + if value == 0 { + return fallback + } + return value +} diff --git a/gateway/internal/upstream/pool_test.go b/gateway/internal/upstream/pool_test.go new file mode 100644 index 0000000..e884d80 --- /dev/null +++ b/gateway/internal/upstream/pool_test.go @@ -0,0 +1,60 @@ +package upstream + +import ( + "testing" + + "cmpp-platform/gateway/internal/queue" +) + +func TestConnectionPoolAcquiresAcrossConnections(t *testing.T) { + pool := &connectionPool{ + conns: []*connection{ + {window: make(chan struct{}, 1)}, + {window: make(chan struct{}, 1)}, + }, + } + + first, releaseFirst := pool.tryAcquireConnection() + if first == nil { + t.Fatalf("expected first connection") + } + second, releaseSecond := pool.tryAcquireConnection() + if second == nil { + t.Fatalf("expected second connection") + } + if first == second { + t.Fatalf("expected pool to use another connection when the first window is full") + } + third, _ := pool.tryAcquireConnection() + if third != nil { + t.Fatalf("expected nil connection while all windows are full") + } + + releaseFirst() + reacquired, releaseReacquired := pool.tryAcquireConnection() + if reacquired == nil { + t.Fatalf("expected a connection after releasing a window") + } + releaseReacquired() + releaseSecond() +} + +func TestNormalizeUpstreamConfigDefaults(t *testing.T) { + config := normalizeUpstreamConfig(queueUpstreamConfigForTest()) + if config.DesiredConnections != 1 { + t.Fatalf("DesiredConnections = %d, want 1", config.DesiredConnections) + } + if config.WindowSize != defaultWindowSize { + t.Fatalf("WindowSize = %d, want %d", config.WindowSize, defaultWindowSize) + } +} + +func queueUpstreamConfigForTest() queue.UpstreamConfig { + return queue.UpstreamConfig{ + GatewayHost: "127.0.0.1", + GatewayPort: 17890, + Account: "account", + PasswordCipher: "secret", + CMPPVersion: "3.0", + } +} diff --git a/src/api/adminApi.ts b/src/api/adminApi.ts index 19a7db0..04099b3 100644 --- a/src/api/adminApi.ts +++ b/src/api/adminApi.ts @@ -24,6 +24,23 @@ async function request(path: string, options: RequestOptions = {}): Promise; } +async function requestBlob(path: string, options: RequestOptions = {}): Promise { + const headers = new Headers(options.headers); + const session = readSession(); + if (session?.accessToken) { + headers.set('Authorization', `${session.tokenType} ${session.accessToken}`); + } + const tenantId = options.tenantId ?? (path.startsWith('/client') ? getSessionTenantId() : undefined); + if (tenantId) { + headers.set('x-tenant-id', tenantId); + } + const response = await fetch(`/api${path}`, { ...options, headers }); + if (!response.ok) { + throw new Error(await response.text()); + } + return response.blob(); +} + export type AdminChannel = { id: string; code: string; @@ -38,7 +55,7 @@ export type AdminChannel = { rateLimitPerSecond: number; unitPrice: number; status: string; - config?: unknown; + config?: { desiredConnections?: number; windowSize?: number; [key: string]: unknown } | null; connectionStates?: CmppConnectionState[]; }; @@ -151,6 +168,14 @@ export type DashboardResponse = { transactions: { _count: { _all: number }; _sum: { amountCents?: number | null; smsUnits?: number | null } }; gatewayConnections: Array<{ status: string; _count: { _all: number }; _sum: { currentConnections?: number | null; desiredConnections?: number | null } }>; pendingAuditCount: number; + downstreamDeliverySummary?: { + pending: number; + failed: number; + delivered: number; + stalledPending: number; + recentFailed: number; + alertCount: number; + }; accounts: Array<{ id: string; tenantId: string; balanceCents: number; smsUnits: number; creditCents: number; status: string; tenant?: TenantOption }>; recentTasks: Array>; recentRecharges: Array; @@ -298,19 +323,71 @@ export type SmsMessageRecord = { receiptRecords?: Array>; }; +export type SmsMessageSegmentAudit = { + id: string; + tenantId: string; + batchTaskId?: string | null; + messageRecordId: string; + submitRecordId?: string | null; + channelId?: string | null; + submitId: string; + attempt: number; + segmentTotal: number; + segmentIndex: number; + sequenceId?: number | null; + gatewayMessageId?: string | null; + submitStatus: string; + receiptStatus?: string | null; + rawStatus?: string | null; + compensationType?: string | null; + errorCode?: string | null; + errorMessage?: string | null; + submittedAt?: string | null; + deliveredAt?: string | null; + createdAt: string; + updatedAt: string; + channel?: AdminChannel | null; +}; + export type SmsUplinkMessage = { id: string; tenantId?: string | null; channelId: string; + applicationId?: string | null; + messageRecordId?: string | null; messageId?: string | null; sequenceId?: number | null; phoneNumber: string; destId: string; content: string; + matchStatus?: string; + matchReason?: string | null; receivedAt: string; createdAt: string; tenant?: TenantOption | null; + application?: { id: string; name: string } | null; + messageRecord?: SmsMessageRecord | null; channel?: AdminChannel | null; + matchCandidates?: SmsUplinkMatchCandidate[]; +}; + +export type SmsUplinkMatchCandidate = { + id: string; + uplinkMessageId: string; + tenantId: string; + applicationId: string; + messageRecordId?: string | null; + matchSource: string; + confidence: number; + reason?: string | null; + status: string; + claimedAt?: string | null; + claimedById?: string | null; + createdAt: string; + updatedAt: string; + tenant?: TenantOption | null; + application?: { id: string; name: string } | null; + messageRecord?: SmsMessageRecord | null; }; export type DictionaryItem = Record & { @@ -448,6 +525,13 @@ export type OperationLogResponse = { modules: string[]; }; +export type PagedResponse = { + items: T[]; + total: number; + page: number; + pageSize: number; +}; + export type EnterpriseApplication = { id: string; tenantId: string; @@ -459,6 +543,9 @@ export type EnterpriseApplication = { queuePriority?: 'normal' | 'priority' | string | null; maxPhonesPerTask?: number | null; templateMismatchMode?: string | null; + cmppAccount?: string | null; + cmppMaxConnections?: number | null; + cmppWindowSize?: number | null; ipAllowlist?: Array<{ id: string; ipCidr: string; remark?: string | null }>; tenant?: TenantOption; sentToday?: number; @@ -508,6 +595,107 @@ export type ApplicationCmppParams = { protocolVersion: string; }; +export type DownstreamDeliveryRecord = { + id: string; + tenantId: string; + applicationId: string; + messageRecordId?: string | null; + messageId?: string | null; + deliveryType: string; + status: string; + payload: Record; + retryCount: number; + nextRetryAt?: string | null; + deliveredAt?: string | null; + lastError?: string | null; + createdAt: string; + updatedAt: string; + tenant?: TenantOption | null; + application?: EnterpriseApplication | null; + messageRecord?: SmsMessageRecord | null; +}; + +export type BatchRequeueResponse = { + total: number; + successCount: number; + failedCount: number; + results: Array<{ id: string; status: 'success' | 'failed'; errorMessage?: string }>; +}; + +export type DownstreamDeliveryDashboard = { + summary: { + total: number; + pending: number; + delivered: number; + failed: number; + stalledPending: number; + recentFailed: number; + alertCount: number; + }; + typeBreakdown: Array<{ + deliveryType: string; + total: number; + pending: number; + delivered: number; + failed: number; + }>; + retryBuckets: Array<{ + label: string; + count: number; + }>; + topApplications: Array<{ + applicationId: string; + name: string; + pending: number; + failed: number; + delivered: number; + alertCount: number; + }>; +}; + +export type GatewayDownstreamRecoveryStatus = { + id: string; + account: string; + tenantId?: string | null; + applicationId?: string | null; + gatewayInstanceId?: string | null; + state: string; + lockOwner?: string | null; + lockExpiresAt?: string | null; + lastAttemptAt?: string | null; + lastSuccessAt?: string | null; + lastFailureAt?: string | null; + nextRetryAt?: string | null; + attemptCount: number; + failureCategory?: string | null; + lastError?: string | null; + lastSkipReason?: string | null; + createdAt: string; + updatedAt: string; + tenant?: TenantOption | null; + application?: EnterpriseApplication | null; +}; + +export type DownstreamRecoveryStatusResponse = PagedResponse & { + summary: { + total: number; + running: number; + success: number; + failed: number; + waitingConnection: number; + backoff: number; + failureCategories: Array<{ category: string; count: number }>; + }; +}; + +export type DownstreamRecoveryStatusExportQuery = { + tenantId?: string; + applicationId?: string; + state?: string; + failureCategory?: string; + keyword?: string; +}; + function withQuery(path: string, query: Record) { const params = new URLSearchParams(); Object.entries(query).forEach(([key, value]) => { @@ -552,9 +740,9 @@ export const adminApi = { request(withQuery('/admin/enterprise-applications', query)), getEnterpriseApplication: (id: string) => request(`/admin/enterprise-applications/${id}`), - createEnterpriseApplication: (body: { tenantId: string; name: string; scene?: string; dailyLimit?: number; customerUnitPrice?: number; queuePriority?: 'normal' | 'priority'; maxPhonesPerTask?: number; templateMismatchMode?: string; ipAllowlist?: string[] }) => + createEnterpriseApplication: (body: { tenantId: string; name: string; scene?: string; dailyLimit?: number; customerUnitPrice?: number; queuePriority?: 'normal' | 'priority'; maxPhonesPerTask?: number; templateMismatchMode?: string; cmppAccount?: string; cmppMaxConnections?: number; cmppWindowSize?: number; ipAllowlist?: string[] }) => request('/admin/enterprise-applications', { method: 'POST', body: JSON.stringify(body) }), - updateEnterpriseApplication: (id: string, body: { name?: string; scene?: string; dailyLimit?: number; customerUnitPrice?: number; queuePriority?: 'normal' | 'priority'; maxPhonesPerTask?: number; templateMismatchMode?: string; ipAllowlist?: string[] }) => + updateEnterpriseApplication: (id: string, body: { name?: string; scene?: string; dailyLimit?: number; customerUnitPrice?: number; queuePriority?: 'normal' | 'priority'; maxPhonesPerTask?: number; templateMismatchMode?: string; cmppAccount?: string; cmppMaxConnections?: number; cmppWindowSize?: number; ipAllowlist?: string[] }) => request(`/admin/enterprise-applications/${id}`, { method: 'PUT', body: JSON.stringify(body) }), changeApplicationStatus: (id: string, status: string, reason?: string) => request(`/admin/enterprise-applications/${id}/status`, { @@ -571,9 +759,9 @@ export const adminApi = { getApplicationCmppParams: (applicationId: string) => request(`/admin/enterprise-applications/${applicationId}/cmpp-params`), listChannels: () => request('/admin/channels'), - createChannel: (body: Partial & { passwordCipher?: string }) => + createChannel: (body: Partial & { passwordCipher?: string; desiredConnections?: number; windowSize?: number }) => request('/admin/channels', { method: 'POST', body: JSON.stringify(body) }), - updateChannel: (id: string, body: Partial & { passwordCipher?: string }) => + updateChannel: (id: string, body: Partial & { passwordCipher?: string; desiredConnections?: number; windowSize?: number }) => request(`/admin/channels/${id}`, { method: 'PUT', body: JSON.stringify(body) }), copyChannel: (id: string, body: { operatorId?: string } = {}) => request(`/admin/channels/${id}/copy`, { method: 'POST', @@ -664,12 +852,30 @@ export const adminApi = { request(`/admin/send/batch-tasks/${id}/terminate`, { method: 'POST', body: JSON.stringify({}) }), listAdminMessages: (query: { tenantId?: string; applicationId?: string; channelId?: string; taskId?: string; phoneNumber?: string; status?: string } = {}) => request(withQuery('/admin/send/messages', query)), + listMessageSegmentAudits: (query: { messageId?: string; messageRecordId?: string }) => + request(withQuery('/admin/operations/message-segment-audits', query)), listOperationMessages: (query: { tenantId?: string; applicationId?: string; channelId?: string; taskId?: string; messageId?: string; phoneNumber?: string; status?: string } = {}) => request(withQuery('/admin/operations/messages', query)), listAdminUplinkMessages: (query: { tenantId?: string; channelId?: string } = {}) => request(withQuery('/admin/operations/uplink-messages', query)), + claimUplinkMatchCandidate: (uplinkMessageId: string, body: { candidateId: string; operatorId?: string }) => + request(`/admin/operations/uplink-messages/${uplinkMessageId}/claim`, { method: 'POST', body: JSON.stringify(body) }), listMonitor: (query: { tenantId?: string; channelId?: string } = {}) => request>(withQuery('/admin/operations/monitor', query)), listStatistics: (query: { tenantId?: string; groupBy?: string } = {}) => request>>(withQuery('/admin/operations/statistics', query)), + getDownstreamDeliveryDashboard: (query: { tenantId?: string; applicationId?: string; deliveryType?: string } = {}) => + request(withQuery('/admin/operations/downstream-deliveries/dashboard', query)), + listDownstreamRecoveryStatuses: (query: { tenantId?: string; applicationId?: string; state?: string; failureCategory?: string; keyword?: string; page?: number; pageSize?: number } = {}) => + request(withQuery('/admin/operations/downstream-recovery-statuses', query)), + getDownstreamRecoveryStatus: (id: string) => + request(`/admin/operations/downstream-recovery-statuses/${id}`), + exportDownstreamRecoveryStatuses: (query: DownstreamRecoveryStatusExportQuery = {}) => + requestBlob(withQuery('/admin/operations/downstream-recovery-statuses/export', query)), + listDownstreamDeliveries: (query: { tenantId?: string; applicationId?: string; deliveryType?: string; status?: string; keyword?: string; page?: number; pageSize?: number } = {}) => + request>(withQuery('/admin/operations/downstream-deliveries', query)), + requeueDownstreamDelivery: (id: string) => + request(`/admin/operations/downstream-deliveries/${id}/requeue`, { method: 'POST', body: JSON.stringify({}) }), + batchRequeueDownstreamDeliveries: (ids: string[]) => + request('/admin/operations/downstream-deliveries/requeue', { method: 'POST', body: JSON.stringify({ ids }) }), listRiskReviewTasks: (query: { tenantId?: string; status?: string } = {}) => request(withQuery('/admin/risk-review/tasks', query)), approveRiskReviewTask: (id: string, reason?: string) => request(`/admin/risk-review/tasks/${id}/approve`, { method: 'POST', body: JSON.stringify({ reason }) }), diff --git a/src/apps/admin/AdminChannelsPage.tsx b/src/apps/admin/AdminChannelsPage.tsx index 2d7b570..cb54bd3 100644 --- a/src/apps/admin/AdminChannelsPage.tsx +++ b/src/apps/admin/AdminChannelsPage.tsx @@ -26,6 +26,8 @@ type SmsChannel = { corpCode: string; account: string; accessNo: string; + desiredConnections: number; + windowSize: number; passwordCipher?: string; }; @@ -143,6 +145,8 @@ function mapApiChannel(channel: AdminChannel, connections: CmppConnectionState[] corpCode: channel.enterpriseCode ?? channel.code, account: channel.account, accessNo: channel.srcId, + desiredConnections: Number(channel.config?.desiredConnections ?? 1), + windowSize: Number(channel.config?.windowSize ?? 16), }; } @@ -163,6 +167,8 @@ function buildChannelPayload(channel: SmsChannel, passwordCipher?: string) { srcId: channel.accessNo, rateLimitPerSecond: 100, unitPrice: Math.round(channel.unitPrice), + desiredConnections: channel.desiredConnections, + windowSize: channel.windowSize, }; } @@ -199,6 +205,8 @@ function ChannelFormModal({ const [accessNo, setAccessNo] = useState(channel?.accessNo ?? ''); const [extensionDigits, setExtensionDigits] = useState('0'); const [flowLimit, setFlowLimit] = useState('1-2000'); + const [desiredConnections, setDesiredConnections] = useState(String(channel?.desiredConnections ?? 1)); + const [windowSize, setWindowSize] = useState(String(channel?.windowSize ?? 16)); function submit() { onSubmit({ @@ -220,6 +228,8 @@ function ChannelFormModal({ corpCode, account, accessNo, + desiredConnections: Number(desiredConnections) || 1, + windowSize: Number(windowSize) || 16, passwordCipher: password || undefined, }); } @@ -272,6 +282,8 @@ function ChannelFormModal({ setFlowLimit(event.target.value)} suffix="条/秒" value={flowLimit} /> + setDesiredConnections(event.target.value)} placeholder="1" value={desiredConnections} /> + setWindowSize(event.target.value)} placeholder="16" value={windowSize} /> diff --git a/src/apps/admin/AdminDownstreamDeliveriesPage.tsx b/src/apps/admin/AdminDownstreamDeliveriesPage.tsx new file mode 100644 index 0000000..3c20016 --- /dev/null +++ b/src/apps/admin/AdminDownstreamDeliveriesPage.tsx @@ -0,0 +1,377 @@ +import { useCallback, useEffect, useMemo, useState } from 'react'; +import { AlertTriangle, BarChart3, CheckCircle2, Eye, RefreshCw, Search, TimerReset } from 'lucide-react'; +import { adminApi, type DownstreamDeliveryDashboard, type DownstreamDeliveryRecord, type EnterpriseApplication } from '@/api/adminApi'; +import { Breadcrumb, Button, Input, Modal, Pagination, Select, Table, Tag, type TableColumn } from '@/components/ui'; + +const statusTone: Record = { + pending: 'warning', + delivered: 'success', + failed: 'danger', +}; + +const deliveryTypeLabel: Record = { + receipt: '状态回执', + uplink: '上行短信', +}; + +function DeliveryDetailModal({ record, onClose }: { record: DownstreamDeliveryRecord; onClose: () => void }) { + const payloadText = useMemo(() => JSON.stringify(record.payload ?? {}, null, 2), [record.payload]); + + return ( +

下游投递详情

{record.id}

} + footer={} + > +
+
+
企业{record.tenant?.name ?? record.tenantId}
+
应用{record.application?.name ?? record.applicationId}
+
投递类型{deliveryTypeLabel[record.deliveryType] ?? record.deliveryType}
+
当前状态{record.status}
+
消息 ID{record.messageId ?? '-'}
+
重试次数{record.retryCount}
+
下次重试{record.nextRetryAt ?? '-'}
+
已投递时间{record.deliveredAt ?? '-'}
+
最后错误{record.lastError ?? '-'}
+
+
+

Payload

+ 
{payloadText}
+
+
+ + ); +} + +export function AdminDownstreamDeliveriesPage() { + const [records, setRecords] = useState([]); + const [dashboard, setDashboard] = useState(null); + const [applications, setApplications] = useState([]); + const [keyword, setKeyword] = useState(''); + const [status, setStatus] = useState('all'); + const [deliveryType, setDeliveryType] = useState('all'); + const [applicationId, setApplicationId] = useState('all'); + const [page, setPage] = useState(1); + const [pageSize] = useState(10); + const [total, setTotal] = useState(0); + const [loading, setLoading] = useState(false); + const [error, setError] = useState(''); + const [detail, setDetail] = useState(null); + const [selectedIds, setSelectedIds] = useState([]); + + const loadData = useCallback(() => { + setLoading(true); + Promise.all([ + adminApi.getDownstreamDeliveryDashboard({ + applicationId, + deliveryType, + }), + adminApi.listDownstreamDeliveries({ + keyword, + status, + deliveryType, + applicationId, + page, + pageSize, + }), + adminApi.listEnterpriseApplications(), + ]) + .then(([dashboardResponse, response, apps]) => { + setDashboard(dashboardResponse); + setRecords(response.items); + setTotal(response.total); + setApplications(apps); + setSelectedIds((current) => current.filter((id) => response.items.some((item) => item.id === id))); + setError(''); + }) + .catch((failure: Error) => setError(failure.message || '下游投递记录加载失败')) + .finally(() => setLoading(false)); + }, [applicationId, deliveryType, keyword, page, pageSize, status]); + + useEffect(() => { + loadData(); + }, [loadData]); + + const selectableIds = useMemo( + () => records.filter((item) => item.status !== 'delivered').map((item) => item.id), + [records], + ); + const allSelected = selectableIds.length > 0 && selectableIds.every((id) => selectedIds.includes(id)); + const summary = dashboard?.summary; + const typeBreakdown = dashboard?.typeBreakdown ?? []; + const retryBuckets = dashboard?.retryBuckets ?? []; + const topApplications = dashboard?.topApplications ?? []; + + const columns: Array> = [ + { + key: 'select', + title: '选择', + width: '52px', + align: 'center', + render: (record) => ( + { + setSelectedIds((current) => + event.target.checked + ? [...current, record.id] + : current.filter((item) => item !== record.id), + ); + }} + aria-label={`选择${record.id}`} + /> + ), + }, + { key: 'createdAt', title: '投递时间', width: '180px', render: (record) => record.createdAt }, + { key: 'tenant', title: '企业', width: '180px', render: (record) => record.tenant?.name ?? '-' }, + { key: 'application', title: '应用', width: '180px', render: (record) => record.application?.name ?? '-' }, + { key: 'type', title: '类型', width: '110px', render: (record) => deliveryTypeLabel[record.deliveryType] ?? record.deliveryType }, + { key: 'messageId', title: '消息 ID', width: '180px', render: (record) => {record.messageId ?? '-'} }, + { key: 'status', title: '状态', width: '110px', render: (record) => {record.status} }, + { key: 'retry', title: '重试', width: '90px', align: 'center', render: (record) => record.retryCount }, + { key: 'error', title: '最后错误', render: (record) => record.lastError ?? '-' }, + { + key: 'actions', + title: '操作', + width: '170px', + align: 'right', + render: (record) => ( +
+ + +
+ ), + }, + ]; + + const totalPages = Math.max(1, Math.ceil(total / pageSize)); + + return ( +
+
+
+ +

下游投递记录

+
+
+ {error ?

{error}

: null} + +
+
+ +
+ 投递总量 + {summary?.total ?? 0} + 当前筛选范围内的真实下游投递记录。 +
+
+
+ +
+ 待投递 + {summary?.pending ?? 0} + 其中积压告警 {summary?.stalledPending ?? 0} 条。 +
+
+
+ +
+ 已投递 + {summary?.delivered ?? 0} + 已成功下发给客户端的记录。 +
+
+
+ +
+ 告警 + {summary?.alertCount ?? 0} + 近期失败 {summary?.recentFailed ?? 0} 条。 +
+
+
+ +
+ setKeyword(event.target.value)} placeholder="请输入关键字" value={keyword} /> + { + setDeliveryType(event.target.value); + setPage(1); + }} + /> + setKeyword(event.target.value)} placeholder="请输入关键字" value={keyword} /> + { + setFailureCategory(event.target.value); + setPage(1); + }} + /> + setScene(event.target.value)} placeholder="行业通知/营销推广/验证码" value={scene} /> setDailyLimit(event.target.value)} placeholder="100000" required value={dailyLimit} /> setCustomerUnitPrice(event.target.value)} placeholder="0.0300" required value={customerUnitPrice} /> + setCmppAccount(event.target.value)} placeholder="留空自动生成" value={cmppAccount} /> + setCmppMaxConnections(event.target.value)} placeholder="1" required value={cmppMaxConnections} /> + setCmppWindowSize(event.target.value)} placeholder="16" required value={cmppWindowSize} />
发送队列
diff --git a/src/apps/admin/AdminSmsRecordsPage.tsx b/src/apps/admin/AdminSmsRecordsPage.tsx index 2330162..c88b824 100644 --- a/src/apps/admin/AdminSmsRecordsPage.tsx +++ b/src/apps/admin/AdminSmsRecordsPage.tsx @@ -1,6 +1,6 @@ import { useEffect, useMemo, useState } from 'react'; import { MessageSquare, Search, Smartphone } from 'lucide-react'; -import { adminApi, type SmsMessageRecord } from '@/api/adminApi'; +import { adminApi, type SmsMessageRecord, type SmsMessageSegmentAudit } from '@/api/adminApi'; import { Breadcrumb, Button, DateRangeInput, Input, Modal, Select, Tag, type DateRangeValue, type TableColumn, Table } from '@/components/ui'; const statusLabelMap: Record = { @@ -28,6 +28,8 @@ export function AdminSmsRecordsPage() { const [contentKeyword, setContentKeyword] = useState(''); const [status, setStatus] = useState('all'); const [selectedRecord, setSelectedRecord] = useState(null); + const [segmentAudits, setSegmentAudits] = useState([]); + const [segmentLoading, setSegmentLoading] = useState(false); const [error, setError] = useState(''); function loadData() { @@ -43,6 +45,21 @@ export function AdminSmsRecordsPage() { loadData(); }, []); + useEffect(() => { + if (!selectedRecord) { + setSegmentAudits([]); + return; + } + setSegmentLoading(true); + adminApi.listMessageSegmentAudits({ messageRecordId: selectedRecord.id }) + .then((items) => { + setSegmentAudits(items); + setError(''); + }) + .catch((failure: Error) => setError(failure.message || '分片审计加载失败')) + .finally(() => setSegmentLoading(false)); + }, [selectedRecord]); + const filteredRows = useMemo( () => records.filter((item) => { const submittedDate = item.queuedAt.slice(0, 10); @@ -64,6 +81,18 @@ export function AdminSmsRecordsPage() { { key: 'actions', title: '操作', width: '120px', align: 'right', render: (record) => }, ]; + const segmentColumns: Array> = [ + { key: 'segment', title: '分片', width: '90px', render: (record) => `${record.segmentIndex}/${record.segmentTotal}` }, + { key: 'submitId', title: '提交ID', width: '190px', render: (record) => {record.submitId} }, + { key: 'channel', title: '通道', width: '150px', render: (record) => record.channel?.name ?? record.channelId ?? '-' }, + { key: 'sequenceId', title: 'Sequence', width: '110px', render: (record) => record.sequenceId ?? '-' }, + { key: 'gatewayMessageId', title: 'MsgId', width: '180px', render: (record) => record.gatewayMessageId ?? '-' }, + { key: 'submitStatus', title: '提交状态', width: '110px', render: (record) => {record.submitStatus} }, + { key: 'receiptStatus', title: '回执状态', width: '110px', render: (record) => record.receiptStatus ? {record.receiptStatus} : '-' }, + { key: 'compensation', title: '补偿', width: '120px', render: (record) => record.compensationType ?? '-' }, + { key: 'error', title: '错误', render: (record) => record.errorMessage ?? record.errorCode ?? '-' }, + ]; + function resetFilters() { setDateRange({}); setPhoneKeyword(''); @@ -125,6 +154,16 @@ export function AdminSmsRecordsPage() {

状态：{statusLabelMap[selectedRecord.status] ?? selectedRecord.status}

失败原因：{selectedRecord.errorMessage ?? '-'}

+
+

分片补偿审计

+ + ) : null} diff --git a/src/apps/admin/AdminSmsUplinkRecordsPage.tsx b/src/apps/admin/AdminSmsUplinkRecordsPage.tsx index 631c9ef..9588dcb 100644 --- a/src/apps/admin/AdminSmsUplinkRecordsPage.tsx +++ b/src/apps/admin/AdminSmsUplinkRecordsPage.tsx @@ -1,6 +1,6 @@ import { useEffect, useMemo, useState } from 'react'; import { Search, Smartphone } from 'lucide-react'; -import { adminApi, type SmsMessageRecord, type SmsUplinkMessage } from '@/api/adminApi'; +import { adminApi, type SmsMessageRecord, type SmsUplinkMatchCandidate, type SmsUplinkMessage } from '@/api/adminApi'; import { Breadcrumb, Button, @@ -20,19 +20,44 @@ function getTime(value?: string | null) { return value ? `${value.slice(0, 10)} ${value.slice(11, 19)}` : '-'; } +function matchStatusText(status?: string | null) { + const map: Record = { + matched: '已匹配', + ambiguous: '待认领', + unmatched: '未匹配', + }; + return status ? (map[status] ?? status) : '-'; +} + +function candidateStatusText(status?: string | null) { + const map: Record = { + pending: '待认领', + claimed: '已认领', + rejected: '已排除', + }; + return status ? (map[status] ?? status) : '-'; +} + function UplinkDetailModal({ + claimError, + claimingId, detailError, matchedRecords, matching, message, + onClaim, onClose, }: { + claimError: string; + claimingId: string; detailError: string; matchedRecords: SmsMessageRecord[]; matching: boolean; message: SmsUplinkMessage; + onClaim: (candidate: SmsUplinkMatchCandidate) => void; onClose: () => void; }) { + const candidates = message.matchCandidates ?? []; return ( 关闭} @@ -69,6 +94,10 @@ function UplinkDetailModal({ 网关消息ID {message.messageId || '-'} +
+ 匹配状态 + {matchStatusText(message.matchStatus)} +
上行内容 {message.content || '-'} @@ -76,6 +105,58 @@ function UplinkDetailModal({
+
+

候选认领

+ {claimError ?

{claimError}

: null} + {candidates.length === 0 ?
暂无人工认领候选
: null} + {candidates.map((candidate) => ( +
+
+
+ 候选企业 + {candidate.tenant?.name ?? candidate.tenantId} +
+
+ 候选应用 + {candidate.application?.name ?? candidate.applicationId} +
+
+ 候选来源 + {candidate.matchSource === 'access_number' ? '接入号' : '手机号时间窗'} +
+
+ 置信度 + {candidate.confidence} +
+
+ 状态 + {candidateStatusText(candidate.status)} +
+
+ 下发短信 + {candidate.messageRecord?.messageId ?? '-'} +
+
+ {candidate.messageRecord ? ( +
+ 下发内容 +

{candidate.messageRecord.content}

+
+ ) : null} +
+ {candidate.reason ?? '-'} + +
+
+ ))} +
+

匹配发送记录

{matching ?

正在查询真实下发记录...

: null} @@ -125,8 +206,10 @@ export function AdminSmsUplinkRecordsPage() { const [selectedMessage, setSelectedMessage] = useState(null); const [loading, setLoading] = useState(true); const [matching, setMatching] = useState(false); + const [claimingId, setClaimingId] = useState(''); const [error, setError] = useState(''); const [detailError, setDetailError] = useState(''); + const [claimError, setClaimError] = useState(''); function loadData() { setLoading(true); @@ -143,6 +226,7 @@ export function AdminSmsUplinkRecordsPage() { setSelectedMessage(message); setMatchedRecords([]); setDetailError(''); + setClaimError(''); if (!message.messageId) { return; @@ -177,6 +261,22 @@ export function AdminSmsUplinkRecordsPage() { setContentKeyword(''); } + function handleClaim(candidate: SmsUplinkMatchCandidate) { + if (!selectedMessage) { + return; + } + setClaimingId(candidate.id); + setClaimError(''); + adminApi.claimUplinkMatchCandidate(selectedMessage.id, { candidateId: candidate.id }) + .then((updated) => { + setMessages((items) => items.map((item) => (item.id === updated.id ? { ...item, ...updated } : item))); + setSelectedMessage((current) => (current && current.id === updated.id ? { ...current, ...updated } : current)); + loadData(); + }) + .catch((reason: Error) => setClaimError(reason.message || '上行认领失败')) + .finally(() => setClaimingId('')); + } + const columns: Array> = [ { key: 'select', @@ -190,6 +290,7 @@ export function AdminSmsUplinkRecordsPage() { { key: 'content', title: '上行内容', render: (record) => {record.content} }, { key: 'channel', title: '上行通道', width: '260px', render: (record) => {record.channel?.name ?? record.channelId} }, { key: 'accessNo', title: '上行接入号', width: '180px', render: (record) => {record.destId} }, + { key: 'matchStatus', title: '匹配状态', width: '140px', render: (record) => {matchStatusText(record.matchStatus)} }, { key: 'actions', title: '操作', @@ -228,10 +329,13 @@ export function AdminSmsUplinkRecordsPage() { {selectedMessage ? ( setSelectedMessage(null)} /> ) : null} diff --git a/src/layouts/AdminLayout.tsx b/src/layouts/AdminLayout.tsx index c967407..8cd5e7e 100644 --- a/src/layouts/AdminLayout.tsx +++ b/src/layouts/AdminLayout.tsx @@ -17,6 +17,7 @@ import { Layers3, MessageSquare, Phone, + RefreshCw, TrendingUp, RadioTower, ReceiptText, @@ -37,10 +38,17 @@ import { AppShell } from '@/layouts/AppShell'; export function AdminLayout() { const session = readSession(); const [pendingAuditCount, setPendingAuditCount] = useState(0); + const [downstreamAlertCount, setDownstreamAlertCount] = useState(0); const loadPendingAuditCount = useCallback(() => { adminApi.getDashboard() - .then((dashboard) => setPendingAuditCount(dashboard.pendingAuditCount ?? 0)) - .catch(() => setPendingAuditCount(0)); + .then((dashboard) => { + setPendingAuditCount(dashboard.pendingAuditCount ?? 0); + setDownstreamAlertCount(dashboard.downstreamDeliverySummary?.alertCount ?? 0); + }) + .catch(() => { + setPendingAuditCount(0); + setDownstreamAlertCount(0); + }); }, []); useEffect(() => { @@ -71,6 +79,7 @@ export function AdminLayout() { userRole="平台管理员" auditNotifications={[ { label: '待处理审核', count: pendingAuditCount, to: '/admin/sms-audit' }, + { label: '下游投递告警', count: downstreamAlertCount, to: '/admin/downstream-deliveries' }, ]} navSections={[ { @@ -134,6 +143,8 @@ export function AdminLayout() { { label: '短信记录', to: '/admin/sms-records', icon: MessageSquare }, { label: '彩信记录', to: '/admin/mms-records', icon: ImageIcon, pending: true }, { label: '短信上行记录', to: '/admin/sms-uplink-records', icon: MessageSquare }, + { label: '下游投递记录', to: '/admin/downstream-deliveries', icon: Send }, + { label: '恢复状态管理', to: '/admin/downstream-recovery-statuses', icon: RefreshCw }, { label: '充值记录', to: '/admin/recharge-records', icon: ReceiptText }, ], }, diff --git a/src/routes/AppRoutes.tsx b/src/routes/AppRoutes.tsx index 09a1516..d9b58d7 100644 --- a/src/routes/AppRoutes.tsx +++ b/src/routes/AppRoutes.tsx @@ -8,6 +8,8 @@ import { AdminCustomerDetailPage } from '@/apps/admin/AdminCustomerDetailPage'; import { AdminCustomerFormPage } from '@/apps/admin/AdminCustomerFormPage'; import { AdminCustomersPage } from '@/apps/admin/AdminCustomersPage'; import { AdminDrainageFieldsPage } from '@/apps/admin/AdminDrainageFieldsPage'; +import { AdminDownstreamDeliveriesPage } from '@/apps/admin/AdminDownstreamDeliveriesPage'; +import { AdminDownstreamRecoveryStatusesPage } from '@/apps/admin/AdminDownstreamRecoveryStatusesPage'; import { AdminEnterpriseApplicationsPage } from '@/apps/admin/AdminEnterpriseApplicationsPage'; import { AdminEnterpriseBlacklistPage } from '@/apps/admin/AdminEnterpriseBlacklistPage'; import { AdminEnterpriseSignaturesPage } from '@/apps/admin/AdminEnterpriseSignaturesPage'; @@ -103,6 +105,8 @@ export function AppRoutes() { } /> } /> } /> + } /> + } /> } /> } /> } /> diff --git a/src/styles/global.css b/src/styles/global.css index 96860f9..3d42725 100644 --- a/src/styles/global.css +++ b/src/styles/global.css @@ -1034,6 +1034,75 @@ h3 { color: var(--color-text-muted); } +.downstream-breakdown-table { + display: grid; +} + +.downstream-breakdown-table__head, +.downstream-breakdown-table__row { + align-items: center; + display: grid; + gap: var(--space-3); + grid-template-columns: minmax(140px, 1.2fr) repeat(4, minmax(72px, 0.7fr)); + min-height: 52px; +} + +.downstream-breakdown-table__head { + border-bottom: 1px solid var(--color-border); + color: var(--color-text-muted); + font-size: var(--font-size-xs); + font-weight: var(--font-weight-semibold); +} + +.downstream-breakdown-table__head--apps, +.downstream-breakdown-table__row--apps { + grid-template-columns: minmax(180px, 1.4fr) repeat(4, minmax(72px, 0.65fr)); +} + +.downstream-breakdown-table__row { + border-bottom: 1px solid var(--color-border); +} + +.downstream-breakdown-table__row strong, +.downstream-breakdown-table__row span { + min-width: 0; +} + +.downstream-breakdown-table__empty { + color: var(--color-text-muted); + padding: var(--space-5) 0 var(--space-1); +} + +.downstream-bucket-list { + display: grid; + gap: var(--space-3); +} + +.downstream-bucket-list--wrap { + grid-template-columns: repeat(auto-fit, minmax(180px, 1fr)); +} + +.downstream-bucket-item { + align-items: center; + background: var(--color-bg-subtle); + border: 1px solid var(--color-border); + border-radius: var(--radius-md); + display: flex; + justify-content: space-between; + min-height: 56px; + padding: 0 var(--space-4); +} + +.downstream-bucket-item span { + color: var(--color-text-muted); + font-size: var(--font-size-sm); +} + +.downstream-bucket-item strong { + color: var(--color-text-strong); + font-size: var(--font-size-lg); +} + .sms-send-page { display: grid; gap: var(--space-6); @@ -4875,6 +4944,18 @@ h3 { margin-top: var(--space-3); } +.admin-detail-metric-grid--compact { + display: grid; + gap: var(--space-4); + margin-bottom: var(--space-5); +} + +.page-inline-hint { + color: var(--color-text-muted); + font-size: var(--font-size-sm); + line-height: var(--line-height-base); +} + .admin-enterprise-profile { display: grid; gap: var(--space-6); @@ -8311,13 +8392,18 @@ h3 { padding-bottom: var(--space-5); } -.admin-uplink-match-card button { - background: transparent; - border: 0; - color: var(--color-selected); - font-weight: var(--font-weight-semibold); - margin-top: var(--space-4); - padding: 0; +.admin-uplink-candidate-footer { + align-items: center; + display: flex; + gap: var(--space-4); + justify-content: space-between; + margin-top: var(--space-5); +} + +.admin-uplink-candidate-footer span { + color: var(--color-text-muted); + line-height: 1.6; + min-width: 0; } .admin-uplink-empty-match { @@ -8797,6 +8883,15 @@ h3 { position: static; } + .downstream-breakdown-table { + overflow-x: auto; + } + + .downstream-breakdown-table__head, + .downstream-breakdown-table__row { + min-width: 560px; + } + .receiver-table__head, .receiver-table__row { grid-template-columns: 64px minmax(0, 1fr) 64px; diff --git a/tools/start-local.ps1 b/tools/start-local.ps1 index e22f3e3..ba2f5d3 100644 --- a/tools/start-local.ps1 +++ b/tools/start-local.ps1 @@ -157,6 +157,7 @@ function Start-LocalRedis { function Find-LocalMinio { $candidates = @( + (Join-Path $root '.local-tools\minio.exe'), 'C:\cmpp-platform-local\minio.exe', 'C:\cmpp-platform-local\minio\minio.exe' )