From a350aca883369b1cfa67ea630a93cf592a9ed6f7 Mon Sep 17 00:00:00 2001 From: hectorzhao Date: Wed, 16 Sep 2026 18:27:29 +0800 Subject: [PATCH] fix: coordinate SMS completion and improve operations diagnostics --- .../migration.sql | 17 + .../migration.sql | 15 + api/prisma/schema.prisma | 56 ++ api/src/channels/channel-test.service.ts | 28 +- .../infrastructure-monitoring.controller.ts | 9 + .../infrastructure-monitoring.service.spec.ts | 275 ++++++-- .../infrastructure-monitoring.service.ts | 45 +- .../persistent-alerts.ts | 143 +++++ api/src/metrics/metrics.service.ts | 2 + api/src/open-api/open-api.service.ts | 32 +- .../official-export.service.ts | 105 +-- api/src/send-chain/attempt-completion.ts | 219 +++++++ api/src/send-chain/completion-context.ts | 35 + api/src/send-chain/completion-metrics.ts | 26 + .../drainage-submit-guard.controller.spec.ts | 54 ++ .../drainage-submit-guard.controller.ts | 16 +- api/src/send-chain/send-accounting.service.ts | 47 +- api/src/send-chain/send-chain.service.spec.ts | 6 +- api/src/send-chain/send-chain.service.ts | 101 ++- .../send-downstream-delivery.service.ts | 53 +- .../send-chain/send-gateway-result.service.ts | 11 +- .../send-chain/send-gateway-submit.service.ts | 36 +- api/src/send-chain/send-receipt.service.ts | 66 +- api/src/send-chain/send-retry.service.ts | 198 +++--- api/src/send-chain/send-submission.service.ts | 2 +- api/src/send-chain/send-timeout.service.ts | 53 +- .../first-version-development-requirements.md | 18 + docs/http-api-assessment-20260910.md | 4 + docs/phase-4-send-pipeline-redesign.md | 151 +++++ ...theus-system-monitoring-design-20260814.md | 8 + docs/system-functional-test-cases.md | 37 ++ docs/testing-progress.md | 20 + .../admin/infrastructure-monitoring.api.ts | 11 +- .../AdminSystemMonitoringPage.test.tsx | 80 +++ .../AdminSystemMonitoringPage.tsx | 127 +++- .../http-signature/HttpSignaturePage.tsx | 6 +- .../SmsRequestDebugger.test.tsx | 69 ++ .../http-signature/SmsRequestDebugger.tsx | 157 +++++ src/components/ui/RechargeReceiptDialog.tsx | 25 +- tools/testing/verify-attempt-completion.mjs | 602 ++++++++++++++++++ 40 files changed, 2599 insertions(+), 366 deletions(-) create mode 100644 api/prisma/migrations/20260916040000_persist_infrastructure_alerts/migration.sql create mode 100644 api/prisma/migrations/20260916041000_attempt_completion_work/migration.sql create mode 100644 api/src/infrastructure-monitoring/persistent-alerts.ts create mode 100644 api/src/send-chain/attempt-completion.ts create mode 100644 api/src/send-chain/completion-context.ts create mode 100644 api/src/send-chain/completion-metrics.ts create mode 100644 api/src/send-chain/drainage-submit-guard.controller.spec.ts create mode 100644 src/apps/admin/system-monitoring/AdminSystemMonitoringPage.test.tsx create mode 100644 src/apps/shared/http-signature/SmsRequestDebugger.test.tsx create mode 100644 src/apps/shared/http-signature/SmsRequestDebugger.tsx create mode 100644 tools/testing/verify-attempt-completion.mjs diff --git a/api/prisma/migrations/20260916040000_persist_infrastructure_alerts/migration.sql b/api/prisma/migrations/20260916040000_persist_infrastructure_alerts/migration.sql new file mode 100644 index 0000000..0f5fdb2 --- /dev/null +++ b/api/prisma/migrations/20260916040000_persist_infrastructure_alerts/migration.sql @@ -0,0 +1,17 @@ +CREATE TABLE "InfrastructureAlertCollection" ( + "id" TEXT NOT NULL PRIMARY KEY, + "observedAt" TIMESTAMP(3) NOT NULL +); +CREATE TABLE "InfrastructureAlertEvent" ( + "id" TEXT NOT NULL PRIMARY KEY, + "fingerprint" TEXT NOT NULL, + "activeAt" TIMESTAMP(3) NOT NULL, + "payload" JSONB NOT NULL, + "lastObservedAt" TIMESTAMP(3) NOT NULL, + "recoveredAt" TIMESTAMP(3), + "clearedAt" TIMESTAMP(3), + "clearedBy" TEXT, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP +); +CREATE UNIQUE INDEX "InfrastructureAlertEvent_fingerprint_activeAt_key" ON "InfrastructureAlertEvent"("fingerprint", "activeAt"); +CREATE INDEX "InfrastructureAlertEvent_clearedAt_activeAt_idx" ON "InfrastructureAlertEvent"("clearedAt", "activeAt"); diff --git a/api/prisma/migrations/20260916041000_attempt_completion_work/migration.sql b/api/prisma/migrations/20260916041000_attempt_completion_work/migration.sql new file mode 100644 index 0000000..078ae31 --- /dev/null +++ b/api/prisma/migrations/20260916041000_attempt_completion_work/migration.sql @@ -0,0 +1,15 @@ +CREATE TABLE "SmsAttemptCompletionWork" ( +"id" TEXT PRIMARY KEY, "workKey" TEXT NOT NULL, "tenantId" TEXT, "messageRecordId" TEXT NOT NULL, "sourceSubmitRecordId" TEXT, +"revision" INTEGER NOT NULL DEFAULT 0, "processedRevision" INTEGER NOT NULL DEFAULT 0, "state" TEXT NOT NULL DEFAULT 'pending', +"leaseOwner" TEXT, "leaseUntil" TIMESTAMP(3), "fenceVersion" INTEGER NOT NULL DEFAULT 0, "attempts" INTEGER NOT NULL DEFAULT 0, +"nextAttemptAt" TIMESTAMP(3) NOT NULL DEFAULT (CURRENT_TIMESTAMP AT TIME ZONE 'UTC'), "decision" TEXT, "retrySubmitRecordId" TEXT, "lastError" TEXT, +"createdAt" TIMESTAMP(3) NOT NULL DEFAULT (CURRENT_TIMESTAMP AT TIME ZONE 'UTC'), "updatedAt" TIMESTAMP(3) NOT NULL DEFAULT (CURRENT_TIMESTAMP AT TIME ZONE 'UTC')); +CREATE UNIQUE INDEX "SmsAttemptCompletionWork_workKey_key" ON "SmsAttemptCompletionWork"("workKey"); +CREATE UNIQUE INDEX "SmsAttemptCompletionWork_sourceSubmitRecordId_key" ON "SmsAttemptCompletionWork"("sourceSubmitRecordId"); +CREATE INDEX "SmsAttemptCompletionWork_state_nextAttemptAt_idx" ON "SmsAttemptCompletionWork"("state", "nextAttemptAt"); +CREATE INDEX "SmsAttemptCompletionWork_state_leaseUntil_idx" ON "SmsAttemptCompletionWork"("state", "leaseUntil"); +CREATE INDEX "SmsAttemptCompletionWork_messageRecordId_idx" ON "SmsAttemptCompletionWork"("messageRecordId"); +CREATE TABLE "SmsCompletionEvent" ("id" TEXT PRIMARY KEY, "eventKey" TEXT NOT NULL, "workId" TEXT NOT NULL REFERENCES "SmsAttemptCompletionWork"("id") ON DELETE RESTRICT, +"kind" TEXT NOT NULL, "payload" JSONB NOT NULL, "processedAt" TIMESTAMP(3), "createdAt" TIMESTAMP(3) NOT NULL DEFAULT (CURRENT_TIMESTAMP AT TIME ZONE 'UTC')); +CREATE UNIQUE INDEX "SmsCompletionEvent_eventKey_key" ON "SmsCompletionEvent"("eventKey"); +CREATE INDEX "SmsCompletionEvent_workId_processedAt_createdAt_idx" ON "SmsCompletionEvent"("workId", "processedAt", "createdAt"); diff --git a/api/prisma/schema.prisma b/api/prisma/schema.prisma index e6489d3..1355f67 100644 --- a/api/prisma/schema.prisma +++ b/api/prisma/schema.prisma @@ -2840,3 +2840,59 @@ model OpenApiDispatchOutbox { updatedAt DateTime @updatedAt @@index([status, leaseUntil]) } + +model InfrastructureAlertCollection { + id String @id + observedAt DateTime +} + +model InfrastructureAlertEvent { + id String @id @default(cuid()) + fingerprint String + activeAt DateTime + payload Json + lastObservedAt DateTime + recoveredAt DateTime? + clearedAt DateTime? + clearedBy String? + createdAt DateTime @default(now()) + @@unique([fingerprint, activeAt]) + @@index([clearedAt, activeAt]) +} + +model SmsAttemptCompletionWork { + id String @id @default(cuid()) + workKey String @unique + tenantId String? + messageRecordId String + sourceSubmitRecordId String? @unique + revision Int @default(0) + processedRevision Int @default(0) + state String @default("pending") + leaseOwner String? + leaseUntil DateTime? + fenceVersion Int @default(0) + attempts Int @default(0) + nextAttemptAt DateTime @default(dbgenerated("(CURRENT_TIMESTAMP AT TIME ZONE 'UTC')")) + decision String? + retrySubmitRecordId String? + lastError String? + createdAt DateTime @default(dbgenerated("(CURRENT_TIMESTAMP AT TIME ZONE 'UTC')")) + updatedAt DateTime @updatedAt + events SmsCompletionEvent[] + @@index([state, nextAttemptAt]) + @@index([state, leaseUntil]) + @@index([messageRecordId]) +} + +model SmsCompletionEvent { + id String @id @default(cuid()) + eventKey String @unique + workId String + kind String + payload Json + processedAt DateTime? + createdAt DateTime @default(dbgenerated("(CURRENT_TIMESTAMP AT TIME ZONE 'UTC')")) + work SmsAttemptCompletionWork @relation(fields: [workId], references: [id], onDelete: Restrict) + @@index([workId, processedAt, createdAt]) +} diff --git a/api/src/channels/channel-test.service.ts b/api/src/channels/channel-test.service.ts index 2a0d899..3792fe3 100644 --- a/api/src/channels/channel-test.service.ts +++ b/api/src/channels/channel-test.service.ts @@ -1,18 +1,24 @@ -import { BadRequestException, Injectable, Logger, NotFoundException, OnModuleDestroy, OnModuleInit } from '@nestjs/common'; -import { Queue } from 'bullmq'; -import IORedis from 'ioredis'; +import { BadRequestException, NotFoundException } from '@nestjs/common'; import { Prisma } from '@prisma/client'; import { randomUUID } from 'crypto'; -import { assertMoneyUnits, moneyToNumber } from '../common/money'; +import { moneyToNumber } from '../common/money'; import { PrismaService } from '../prisma/prisma.service'; -import type { CreateChannelDto, UpdateChannelDto, CreateChannelGroupDto, CreateChannelGroupItemDto, UpdateChannelGroupDto, CreateRouteRuleDto, CreateReportFieldDto, ReplaceReportFieldsDto, CreateReportMaterialDto, CreateReportTaskDto, ChangeReportTaskStatusesDto, CreateReportExportDto, CreateReceiptImportDto, UpsertConnectionStateDto, ChangeChannelStatusDto, CopyChannelDto, TestChannelDto } from './channels.contracts'; -import { GATEWAY_CONNECTION_QUEUE, GATEWAY_SUBMIT_QUEUE, GATEWAY_SUBMIT_STREAM, DEFAULT_GATEWAY_CONTROL_URL, DEFAULT_CHANNEL_CONNECTION_ID, DEFAULT_CONNECTING_TIMEOUT_MS, DEFAULT_CONNECTING_TIMEOUT_SCAN_MS, DEFAULT_GATEWAY_STARTUP_RECONNECT_DELAY_MS, DEFAULT_GATEWAY_RECONCILE_INTERVAL_MS, DEFAULT_GATEWAY_CONTROL_TIMEOUT_MS, DEFAULT_HEARTBEAT_INTERVAL_SECONDS, DEFAULT_HEARTBEAT_MISS_THRESHOLD, HEARTBEAT_AUDIT_INTERVAL_MS, CONNECTING_TIMEOUT_ERROR, DEFAULT_CMPP_VERSION, normalizeTestPhones, normalizeTestContent, calculateBillingUnits, buildChannelTestSubmitCommand, getConfigValue, getStringConfigValue, normalizeConnectionAction, normalizeCmppVersion, normalizeGatewayConnectionStatus, defaultChannelConnectionId, getDesiredConnections, ChannelConnectionSettings, getRuntimeConfigInteger, channelConnectionSettingsChanged, channelGroupAuditSnapshot, normalizeChannelRuntimeConfig, normalizeCmppServiceId, normalizeChannelRateLimit, normalizeExtensionDigits, getPositiveRuntimeInteger, bullmqConnection, getPositiveIntegerEnv, parseReceiptContent, splitReceiptLine, stripReceiptCell, findReceiptStatusIndex, normalizeReceiptStatus, deriveReceiptStatus, ChannelReportDeliveryRow, summarizeChannelReportDelivery, sumReportDelivery, percentage, latestDate, currentShanghaiDayRange, normalizeRetryTimeLimitMinutes, normalizeSpreadsheetSize, normalizeBusinessCarrier, normalizeChannelCarrier, isChannelCarrierCompatible, normalizeRegion, isRegionCompatible, validateGroupItems, normalizeReportType, summarizeReportStatuses, normalizeLinkEvent } from './channels.helpers'; +import type { TestChannelDto } from './channels.contracts'; +import { + normalizeTestPhones, + normalizeTestContent, + normalizeGatewayConnectionStatus, + calculateBillingUnits, + buildChannelTestSubmitCommand, +} from './channels.helpers'; import { ChannelConnectionService } from './channel-connection.service'; -import { detectDrainageContent } from '../send-chain/drainage-content-detection'; /** R5 channel domain service composed behind ChannelsService. */ export class ChannelTestService { - constructor(private readonly prisma: PrismaService, private readonly connection: ChannelConnectionService) {} + constructor( + private readonly prisma: PrismaService, + private readonly connection: ChannelConnectionService, + ) {} async testChannel(channelId: string, data: TestChannelDto = {}) { const phoneNumbers = normalizeTestPhones(data); @@ -27,8 +33,8 @@ export class ChannelTestService { if (channel.status !== 'active') { throw new BadRequestException('通道未启用，不能发送测试短信'); } - const connectedState = channel.connectionStates.find((state) => - normalizeGatewayConnectionStatus(state.status) === 'connected' && (state.currentConnections ?? 0) > 0, + const connectedState = channel.connectionStates.find( + (state) => normalizeGatewayConnectionStatus(state.status) === 'connected' && (state.currentConnections ?? 0) > 0, ); if (!connectedState) { throw new BadRequestException('通道当前没有可用 CMPP 连接，请先连接成功后再测试发送'); @@ -36,7 +42,6 @@ export class ChannelTestService { const createdAt = new Date(); const testNo = `CHTEST-${Date.now()}-${randomUUID().slice(0, 8)}`; - const drainageDetection = await detectDrainageContent(this.prisma, content); const results = []; for (const [index, phoneNumber] of phoneNumbers.entries()) { const messageId = `MSG-TEST-${Date.now()}-${randomUUID().slice(0, 8)}`; @@ -51,7 +56,6 @@ export class ChannelTestService { messageId, phoneNumber, content, - ...drainageDetection, billingUnits: calculateBillingUnits(content), unitPrice: 0, amountCents: 0, diff --git a/api/src/infrastructure-monitoring/infrastructure-monitoring.controller.ts b/api/src/infrastructure-monitoring/infrastructure-monitoring.controller.ts index 21e484b..73e339c 100644 --- a/api/src/infrastructure-monitoring/infrastructure-monitoring.controller.ts +++ b/api/src/infrastructure-monitoring/infrastructure-monitoring.controller.ts @@ -37,6 +37,15 @@ export class InfrastructureMonitoringController { return this.monitoring.markAlertRead(fingerprint, activeAt, userId); } + @Post('alerts/:fingerprint/clear') + clearAlert( + @Param('fingerprint') fingerprint: string, + @Body('activeAt') activeAt: unknown, + @CurrentSessionUserId() userId: string, + ) { + return this.monitoring.clearAlert(fingerprint, activeAt, userId); + } + @Get('alert-thresholds') alertThresholds() { return this.settings.get(); diff --git a/api/src/infrastructure-monitoring/infrastructure-monitoring.service.spec.ts b/api/src/infrastructure-monitoring/infrastructure-monitoring.service.spec.ts index 8c6fe12..8c2bafb 100644 --- a/api/src/infrastructure-monitoring/infrastructure-monitoring.service.spec.ts +++ b/api/src/infrastructure-monitoring/infrastructure-monitoring.service.spec.ts @@ -1,8 +1,15 @@ import { BadRequestException } from '@nestjs/common'; import { ConfigService } from '@nestjs/config'; import { createHash } from 'node:crypto'; -import { InfrastructureMonitoringService } from './infrastructure-monitoring.service'; import { FILESYSTEM_USAGE_PERCENT, filesystemIdentity } from './filesystem-metrics'; +import { InfrastructureMonitoringService } from './infrastructure-monitoring.service'; +import { retainedAlerts } from './persistent-alerts'; + +jest.mock('./persistent-alerts', () => ({ + retainAlerts: jest.fn(async (_prisma, alerts) => alerts), + retainedAlerts: jest.fn(), + clearRetainedAlert: jest.fn(), +})); function success(data: unknown) { return { @@ -14,7 +21,12 @@ function success(data: unknown) { describe('InfrastructureMonitoringService', () => { const prisma = { - infrastructureAlertRead: { findMany: jest.fn().mockResolvedValue([]), create: jest.fn(), update: jest.fn(), findUniqueOrThrow: jest.fn() }, + infrastructureAlertRead: { + findMany: jest.fn().mockResolvedValue([]), + create: jest.fn(), + update: jest.fn(), + findUniqueOrThrow: jest.fn(), + }, operationLog: { create: jest.fn() }, $transaction: jest.fn(), }; @@ -34,9 +46,27 @@ describe('InfrastructureMonitoringService', () => { }); it('rejects credential-bearing or remote plaintext Prometheus endpoints at startup', () => { - expect(() => new InfrastructureMonitoringService(new ConfigService({ PROMETHEUS_URL: 'http://user:secret@127.0.0.1:9090' }), prisma as never)).toThrow('must not contain credentials'); - expect(() => new InfrastructureMonitoringService(new ConfigService({ PROMETHEUS_URL: 'http://monitor.example.com:9090' }), prisma as never)).toThrow('must use HTTPS'); - expect(() => new InfrastructureMonitoringService(new ConfigService({ PROMETHEUS_URL: 'https://monitor.example.com' }), prisma as never)).not.toThrow(); + expect( + () => + new InfrastructureMonitoringService( + new ConfigService({ PROMETHEUS_URL: 'http://user:secret@127.0.0.1:9090' }), + prisma as never, + ), + ).toThrow('must not contain credentials'); + expect( + () => + new InfrastructureMonitoringService( + new ConfigService({ PROMETHEUS_URL: 'http://monitor.example.com:9090' }), + prisma as never, + ), + ).toThrow('must use HTTPS'); + expect( + () => + new InfrastructureMonitoringService( + new ConfigService({ PROMETHEUS_URL: 'https://monitor.example.com' }), + prisma as never, + ), + ).not.toThrow(); }); it('loads real Prometheus vectors, ranges, services and active alerts', async () => { @@ -45,34 +75,52 @@ describe('InfrastructureMonitoringService', () => { const url = new URL(String(input)); requestedUrls.push(url); if (url.pathname.endsWith('/alerts')) { - return success({ alerts: [{ - labels: { alertname: 'HostCpuHigh', severity: 'warning', instance: '127.0.0.1:9100' }, - annotations: { summary: 'CPU持续偏高', threshold: '85%' }, - state: 'firing', - activeAt: '2026-08-14T03:00:00.000Z', - value: '88.2', - }] }); + return success({ + alerts: [ + { + labels: { alertname: 'HostCpuHigh', severity: 'warning', instance: '127.0.0.1:9100' }, + annotations: { summary: 'CPU持续偏高', threshold: '85%' }, + state: 'firing', + activeAt: '2026-08-14T03:00:00.000Z', + value: '88.2', + }, + ], + }); } const query = url.searchParams.get('query') ?? ''; if (url.pathname.endsWith('/query_range')) { - return success({ result: [{ metric: {}, values: [[1_765_000_000, '12.5'], [1_765_000_060, '14.5']] }] }); + return success({ + result: [ + { + metric: {}, + values: [ + [1_765_000_000, '12.5'], + [1_765_000_060, '14.5'], + ], + }, + ], + }); } if (query.includes('node_systemd_unit_state')) { - return success({ result: [ - { metric: { name: 'cmpp-api.service' }, value: [1_765_000_060, '1'] }, - { metric: { name: 'cmpp-gateway.service' }, value: [1_765_000_060, '1'] }, - { metric: { name: 'postgresql.service' }, value: [1_765_000_060, '1'] }, - { metric: { name: 'redis-server.service' }, value: [1_765_000_060, '1'] }, - { metric: { name: 'cmpp-minio.service' }, value: [1_765_000_060, '1'] }, - { metric: { name: 'nginx.service' }, value: [1_765_000_060, '1'] }, - ] }); + return success({ + result: [ + { metric: { name: 'cmpp-api.service' }, value: [1_765_000_060, '1'] }, + { metric: { name: 'cmpp-gateway.service' }, value: [1_765_000_060, '1'] }, + { metric: { name: 'postgresql.service' }, value: [1_765_000_060, '1'] }, + { metric: { name: 'redis-server.service' }, value: [1_765_000_060, '1'] }, + { metric: { name: 'cmpp-minio.service' }, value: [1_765_000_060, '1'] }, + { metric: { name: 'nginx.service' }, value: [1_765_000_060, '1'] }, + ], + }); } if (query.includes('cmpp:service_.*')) { - return success({ result: [ - { metric: { __name__: 'cmpp:service_api:requests_per_second' }, value: [1_765_000_060, '12.5'] }, - { metric: { __name__: 'cmpp:service_api:error_percent' }, value: [1_765_000_060, '0.2'] }, - { metric: { __name__: 'cmpp:service_gateway:queue_pending' }, value: [1_765_000_060, '3'] }, - ] }); + return success({ + result: [ + { metric: { __name__: 'cmpp:service_api:requests_per_second' }, value: [1_765_000_060, '12.5'] }, + { metric: { __name__: 'cmpp:service_api:error_percent' }, value: [1_765_000_060, '0.2'] }, + { metric: { __name__: 'cmpp:service_gateway:queue_pending' }, value: [1_765_000_060, '3'] }, + ], + }); } if (query.includes('timestamp(node_uname_info)')) { return success({ result: [{ metric: {}, value: [1_765_000_060, '1765000060'] }] }); @@ -87,14 +135,27 @@ describe('InfrastructureMonitoringService', () => { expect(result.metrics.cpuUsagePercent).toBe(25); expect(result.trends.cpuUsagePercent).toHaveLength(2); expect(result.summary).toMatchObject({ overallStatus: 'warning', serviceHealthy: 6, warningAlerts: 1 }); - expect(result.services.find((item) => item.key === 'redis')).toMatchObject({ unit: 'redis-server.service', status: 'healthy' }); + expect(result.services.find((item) => item.key === 'redis')).toMatchObject({ + unit: 'redis-server.service', + status: 'healthy', + }); expect(result.serviceMetrics.find((item) => item.key === 'api')).toMatchObject({ available: true }); - expect(result.serviceMetrics.find((item) => item.key === 'gateway')?.metrics.find((item) => item.key === 'queuePending')?.value).toBe(3); + expect( + result.serviceMetrics.find((item) => item.key === 'gateway')?.metrics.find((item) => item.key === 'queuePending') + ?.value, + ).toBe(3); expect(result.alerts[0]).toMatchObject({ name: 'HostCpuHigh', severity: 'warning', currentValue: '88.2' }); expect(requestedUrls.filter((url) => url.pathname.endsWith('/query_range'))).toHaveLength(5); - expect(requestedUrls.filter((url) => url.pathname.endsWith('/query_range')).every((url) => url.searchParams.get('step') === '60')).toBe(true); - expect(requestedUrls.find((url) => url.searchParams.get('query')?.includes('node_systemd_unit_state'))?.searchParams.get('query')) - .toContain('cmpp-api\\\\.service'); + expect( + requestedUrls + .filter((url) => url.pathname.endsWith('/query_range')) + .every((url) => url.searchParams.get('step') === '60'), + ).toBe(true); + expect( + requestedUrls + .find((url) => url.searchParams.get('query')?.includes('node_systemd_unit_state')) + ?.searchParams.get('query'), + ).toContain('cmpp-api\\\\.service'); }); it('returns an explicit unavailable payload without stale metrics when Prometheus fails', async () => { @@ -124,12 +185,36 @@ describe('InfrastructureMonitoringService', () => { if (url.pathname.endsWith('/alerts')) return success({ alerts: [] }); if (!query.includes('node_filesystem_')) return success({ result: [] }); expect(query).not.toContain('mountpoint="/"'); - if (url.pathname.endsWith('/query_range')) return success({ result: [...metrics].reverse().map((metric) => ({ metric, values: [[1765000060, metric.mountpoint === '/' ? '91' : '12']] })) }); - return success({ result: metrics.map((metric) => ({ metric, value: [1765000060, query === FILESYSTEM_USAGE_PERCENT ? (metric.mountpoint === '/' ? '91' : '12') : query.includes('avail') ? '9' : '100'] })) }); + if (url.pathname.endsWith('/query_range')) + return success({ + result: [...metrics] + .reverse() + .map((metric) => ({ metric, values: [[1765000060, metric.mountpoint === '/' ? '91' : '12']] })), + }); + return success({ + result: metrics.map((metric) => ({ + metric, + value: [ + 1765000060, + query === FILESYSTEM_USAGE_PERCENT + ? metric.mountpoint === '/' + ? '91' + : '12' + : query.includes('avail') + ? '9' + : '100', + ], + })), + }); }); const result = await new InfrastructureMonitoringService(new ConfigService(), prisma as never).overview('1h'); expect(result.disks.map((disk) => disk.mountpoint)).toEqual(['/', '/archive', '/data']); - expect(result.disks[0]).toMatchObject({ usagePercent: 91, totalBytes: 100, availableBytes: 9, trend: [{ timestamp: new Date(1765000060000).toISOString(), value: 91 }] }); + expect(result.disks[0]).toMatchObject({ + usagePercent: 91, + totalBytes: 100, + availableBytes: 9, + trend: [{ timestamp: new Date(1765000060000).toISOString(), value: 91 }], + }); expect(result.disks[2].trend[0].value).toBe(12); expect(result.metrics.diskUsagePercent).toBe(91); expect(result.trends.diskUsagePercent[0].value).toBe(91); @@ -146,20 +231,44 @@ describe('InfrastructureMonitoringService', () => { if (!query.includes('node_filesystem_')) return success({ result: [] }); if (url.pathname.endsWith('/query_range')) { expect(query).toBe(FILESYSTEM_USAGE_PERCENT); - return success({ result: [ - { metric: data, values: [[1765000000, '82'], [1765000060, 'NaN'], [1765000120, '83.5']] }, - { metric: root, values: [[1765000000, '91']] }, - ] }); + return success({ + result: [ + { + metric: data, + values: [ + [1765000000, '82'], + [1765000060, 'NaN'], + [1765000120, '83.5'], + ], + }, + { metric: root, values: [[1765000000, '91']] }, + ], + }); } - if (query.startsWith('node_filesystem_size_bytes')) return success({ result: [ - ...mounts.map((mountpoint) => ({ metric: { ...data, mountpoint }, value: [1765000120, '100'] })), - ...['/var/root-bind', '/'].map((mountpoint) => ({ metric: { ...root, mountpoint }, value: [1765000120, '200'] })), - ] }); - if (query === FILESYSTEM_USAGE_PERCENT) return success({ result: [ - { metric: data, value: [1765000120, '83.5'] }, { metric: root, value: [1765000120, '91'] }, - ] }); + if (query.startsWith('node_filesystem_size_bytes')) + return success({ + result: [ + ...mounts.map((mountpoint) => ({ metric: { ...data, mountpoint }, value: [1765000120, '100'] })), + ...['/var/root-bind', '/'].map((mountpoint) => ({ + metric: { ...root, mountpoint }, + value: [1765000120, '200'], + })), + ], + }); + if (query === FILESYSTEM_USAGE_PERCENT) + return success({ + result: [ + { metric: data, value: [1765000120, '83.5'] }, + { metric: root, value: [1765000120, '91'] }, + ], + }); expect(query).toContain('min by (instance, device, fstype)'); - return success({ result: [{ metric: data, value: [1765000120, '16.5'] }, { metric: root, value: [1765000120, '18'] }] }); + return success({ + result: [ + { metric: data, value: [1765000120, '16.5'] }, + { metric: root, value: [1765000120, '18'] }, + ], + }); }); const service = new InfrastructureMonitoringService(new ConfigService(), prisma as never); const result = await service.overview('1h'); @@ -169,7 +278,13 @@ describe('InfrastructureMonitoringService', () => { expect(result.metrics.diskUsagePercent).toBe(91); expect(result.trends.diskUsagePercent[0].value).toBe(91); const disk = result.disks[1]; - expect(disk).toMatchObject({ id: filesystemIdentity(data), mountpoint: '/data', totalBytes: 100, availableBytes: 16.5, usagePercent: 83.5 }); + expect(disk).toMatchObject({ + id: filesystemIdentity(data), + mountpoint: '/data', + totalBytes: 100, + availableBytes: 16.5, + usagePercent: 83.5, + }); expect(disk.mountpoints).toEqual(['/data', '/var/lib/minio', '/var/lib/pgsql', '/var/lib/redis']); expect(disk.trend.map((point) => point.value)).toEqual([82, 83.5]); mounts.reverse(); @@ -198,33 +313,77 @@ describe('InfrastructureMonitoringService', () => { const result = await new InfrastructureMonitoringService(new ConfigService(), prisma as never).overview('1h'); expect(result.disks).toHaveLength(4); expect(new Set(result.disks.map((disk) => disk.id)).size).toBe(4); - expect(result.disks.every((disk) => disk.usagePercent === null && disk.availableBytes === null && disk.trend.length === 0)).toBe(true); + expect( + result.disks.every( + (disk) => disk.usagePercent === null && disk.availableBytes === null && disk.trend.length === 0, + ), + ).toBe(true); expect(result.metrics.diskUsagePercent).toBeNull(); expect(result.trends.diskUsagePercent).toEqual([]); }); it('excludes only the current alert occurrence after the current administrator marks it read', async () => { const labels = { alertname: 'QaWarning', severity: 'warning', service: 'qa-preview' }; - const fingerprint = createHash('sha256').update(JSON.stringify(Object.entries(labels).sort(([left], [right]) => left.localeCompare(right)))).digest('hex').slice(0, 24); - jest.spyOn(global, 'fetch').mockResolvedValue(success({ alerts: [{ labels, annotations: { summary: '演示预警' }, state: 'firing', activeAt: '2026-08-16T01:00:00.000Z' }] })); - prisma.infrastructureAlertRead.findMany.mockResolvedValue([{ fingerprint, activeAt: new Date('2026-08-16T01:00:00.000Z'), readAt: new Date('2026-08-16T01:01:00.000Z') }]); + const fingerprint = createHash('sha256') + .update(JSON.stringify(Object.entries(labels).sort(([left], [right]) => left.localeCompare(right)))) + .digest('hex') + .slice(0, 24); + jest.spyOn(global, 'fetch').mockResolvedValue( + success({ + alerts: [ + { labels, annotations: { summary: '演示预警' }, state: 'firing', activeAt: '2026-08-16T01:00:00.000Z' }, + ], + }), + ); + prisma.infrastructureAlertRead.findMany.mockResolvedValue([ + { fingerprint, activeAt: new Date('2026-08-16T01:00:00.000Z'), readAt: new Date('2026-08-16T01:01:00.000Z') }, + ]); const service = new InfrastructureMonitoringService(new ConfigService(), prisma as never); await expect(service.notificationSummary('admin-1')).resolves.toEqual({ count: 0, criticalCount: 0 }); - prisma.infrastructureAlertRead.findMany.mockResolvedValue([{ fingerprint, activeAt: new Date('2026-08-15T01:00:00.000Z'), readAt: new Date('2026-08-15T01:01:00.000Z') }]); + prisma.infrastructureAlertRead.findMany.mockResolvedValue([ + { fingerprint, activeAt: new Date('2026-08-15T01:00:00.000Z'), readAt: new Date('2026-08-15T01:01:00.000Z') }, + ]); await expect(service.notificationSummary('admin-1')).resolves.toEqual({ count: 1, criticalCount: 0 }); }); it('upserts an idempotent per-user read record only for a currently active occurrence', async () => { const labels = { alertname: 'QaCritical', severity: 'critical', service: 'qa-preview' }; - const fingerprint = createHash('sha256').update(JSON.stringify(Object.entries(labels).sort(([left], [right]) => left.localeCompare(right)))).digest('hex').slice(0, 24); - jest.spyOn(global, 'fetch').mockResolvedValue(success({ alerts: [{ labels, annotations: { summary: '演示严重告警' }, state: 'firing', activeAt: '2026-08-16T02:00:00.000Z' }] })); - prisma.$transaction.mockResolvedValue([{ activeAt: new Date('2026-08-16T02:00:00.000Z'), readAt: new Date('2026-08-16T02:01:00.000Z') }, {}]); + const fingerprint = createHash('sha256') + .update(JSON.stringify(Object.entries(labels).sort(([left], [right]) => left.localeCompare(right)))) + .digest('hex') + .slice(0, 24); + jest.spyOn(global, 'fetch').mockResolvedValue( + success({ + alerts: [ + { labels, annotations: { summary: '演示严重告警' }, state: 'firing', activeAt: '2026-08-16T02:00:00.000Z' }, + ], + }), + ); + prisma.$transaction.mockResolvedValue([ + { activeAt: new Date('2026-08-16T02:00:00.000Z'), readAt: new Date('2026-08-16T02:01:00.000Z') }, + {}, + ]); const service = new InfrastructureMonitoringService(new ConfigService(), prisma as never); - await expect(service.markAlertRead(fingerprint, '2026-08-16T02:00:00.000Z', 'admin-1')).resolves.toMatchObject({ fingerprint, acknowledged: true }); - expect(prisma.infrastructureAlertRead.create).toHaveBeenCalledWith(expect.objectContaining({ data: expect.objectContaining({ fingerprint, userId: 'admin-1' }) })); - await expect(service.markAlertRead(fingerprint, '2026-08-15T02:00:00.000Z', 'admin-1')).rejects.toThrow('已结束或已重新触发'); + jest.mocked(retainedAlerts).mockResolvedValue([ + { + fingerprint, + startedAt: '2026-08-16T02:00:00.000Z', + name: 'QaCritical', + severity: 'critical', + } as never, + ]); + await expect(service.markAlertRead(fingerprint, '2026-08-16T02:00:00.000Z', 'admin-1')).resolves.toMatchObject({ + fingerprint, + acknowledged: true, + }); + expect(prisma.infrastructureAlertRead.create).toHaveBeenCalledWith( + expect.objectContaining({ data: expect.objectContaining({ fingerprint, userId: 'admin-1' }) }), + ); + await expect(service.markAlertRead(fingerprint, '2026-08-15T02:00:00.000Z', 'admin-1')).rejects.toThrow( + '已结束或已重新触发', + ); }); }); diff --git a/api/src/infrastructure-monitoring/infrastructure-monitoring.service.ts b/api/src/infrastructure-monitoring/infrastructure-monitoring.service.ts index e7205d1..922b140 100644 --- a/api/src/infrastructure-monitoring/infrastructure-monitoring.service.ts +++ b/api/src/infrastructure-monitoring/infrastructure-monitoring.service.ts @@ -1,3 +1,4 @@ +import { retainAlerts, retainedAlerts, clearRetainedAlert } from './persistent-alerts'; import { BadRequestException, Injectable, @@ -202,6 +203,8 @@ export class InfrastructureMonitoringService { private readonly logger = new Logger(InfrastructureMonitoringService.name); private readonly prometheusUrl: string; private readonly queryTimeoutMs: number; + private alertTimer?: ReturnType; + private alertPoll?: Promise; constructor( config: ConfigService, @@ -211,6 +214,35 @@ export class InfrastructureMonitoringService { this.queryTimeoutMs = Math.min(15_000, Math.max(1_000, Number(config.get('PROMETHEUS_QUERY_TIMEOUT_MS') ?? 5_000))); } + onModuleInit() { + if (process.env.CMPP_PROCESS_ROLE && !['api', 'all'].includes(process.env.CMPP_PROCESS_ROLE)) return; + const poll = () => + void this.loadRetainedAlerts().catch(() => this.logger.warn('Persistent alert collection unavailable')); + poll(); + this.alertTimer = setInterval(poll, 30_000); + this.alertTimer.unref(); + } + + onModuleDestroy() { + if (this.alertTimer) clearInterval(this.alertTimer); + } + + private loadRetainedAlerts() { + if (!this.alertPoll) { + const observedAt = new Date(); + this.alertPoll = this.getJson('/api/v1/alerts') + .then((response) => retainAlerts(this.prisma, this.parseAlerts(response), observedAt)) + .finally(() => { + this.alertPoll = undefined; + }); + } + return this.alertPoll; + } + + clearAlert(fingerprint: string, activeAt: unknown, userId: string) { + return clearRetainedAlert(this.prisma, fingerprint, activeAt, userId); + } + async overview(rawRange?: string, userId?: string): Promise { const range = this.parseRange(rawRange); const collectedAt = new Date().toISOString(); @@ -220,11 +252,11 @@ export class InfrastructureMonitoringService { this.loadTrends(range), this.query(QUERIES.services), this.query(SERVICE_METRICS_QUERY), - this.getJson('/api/v1/alerts'), + this.loadRetainedAlerts(), ]); const services = this.parseServices(serviceResponse); const serviceMetrics = this.parseServiceMetrics(serviceMetricResponse); - const alerts = await this.attachReadState(this.parseAlerts(alertResponse), userId); + const alerts = await this.attachReadState(alertResponse, userId); const warningAlerts = alerts.filter((item) => item.severity === 'warning').length; const criticalAlerts = alerts.filter((item) => item.severity === 'critical').length; const overallStatus = criticalAlerts > 0 ? 'critical' : warningAlerts > 0 ? 'warning' : 'healthy'; @@ -250,7 +282,7 @@ export class InfrastructureMonitoringService { alerts, }; } catch (error) { - // 页面必须整体清空陈旧指标，但服务端仍需留下不含PromQL/地址/凭据的根因摘要便于运维诊断。 + // 客户端保留最后成功快照；采集失败不推断告警恢复。 this.logger.warn( `Prometheus monitoring overview unavailable: ${error instanceof Error ? error.message : 'unknown error'}`, ); @@ -260,10 +292,7 @@ export class InfrastructureMonitoringService { async notificationSummary(userId?: string) { try { - const alerts = await this.attachReadState( - this.parseAlerts(await this.getJson('/api/v1/alerts')), - userId, - ); + const alerts = await this.attachReadState(await this.loadRetainedAlerts(), userId); const unreadAlerts = alerts.filter((item) => !item.acknowledged); return { count: unreadAlerts.length, @@ -281,7 +310,7 @@ export class InfrastructureMonitoringService { if (!/^[a-f0-9]{24}$/.test(fingerprint)) throw new BadRequestException('告警指纹无效'); const activeAt = new Date(String(rawActiveAt ?? '')); if (!Number.isFinite(activeAt.getTime())) throw new BadRequestException('告警开始时间无效'); - const activeAlerts = this.parseAlerts(await this.getJson('/api/v1/alerts')); + const activeAlerts = await retainedAlerts(this.prisma); const current = activeAlerts.find( (item) => item.fingerprint === fingerprint && Date.parse(item.startedAt) === activeAt.getTime(), ); diff --git a/api/src/infrastructure-monitoring/persistent-alerts.ts b/api/src/infrastructure-monitoring/persistent-alerts.ts new file mode 100644 index 0000000..0de074c --- /dev/null +++ b/api/src/infrastructure-monitoring/persistent-alerts.ts @@ -0,0 +1,143 @@ +import { BadRequestException, NotFoundException } from '@nestjs/common'; +import { Prisma } from '@prisma/client'; +import { createHash } from 'node:crypto'; +import { PrismaService } from '../prisma/prisma.service'; +import type { InfrastructureAlert } from './infrastructure-monitoring.contracts'; + +export async function retainAlerts(prisma: PrismaService, alerts: InfrastructureAlert[], observedAt: Date) { + await prisma.$transaction(async (tx) => { + // Serialize snapshots across API processes; timestamps reject late HTTP results. + await tx.$executeRaw`SELECT pg_advisory_xact_lock(160916, 1)`; + const previous = await tx.infrastructureAlertCollection.findUnique({ where: { id: 'prometheus' } }); + if (previous && previous.observedAt >= observedAt) return; + // Read durable work directly: application releases do not install Prometheus rules. + // Keep one occurrence identity until the condition really recovers, even after manual clear. + const reviewCount = await tx.smsAttemptCompletionWork.count({ where: { state: 'needs_review' } }); + const oldest = await tx.smsCompletionEvent.findFirst({ + where: { processedAt: null, work: { state: { in: ['pending', 'processing', 'retry_wait'] } } }, + orderBy: { createdAt: 'asc' }, + select: { createdAt: true }, + }); + const age = oldest ? Math.max(0, (observedAt.getTime() - oldest.createdAt.getTime()) / 1000) : 0; + alerts = [...alerts]; + for (const condition of [ + { + name: 'SmsCompletionNeedsReview', + active: reviewCount > 0, + severity: 'critical' as const, + summary: '短信收尾工作需要人工排查', + value: String(reviewCount), + threshold: '0', + }, + { + name: 'SmsCompletionBacklog', + active: age > 300, + severity: 'warning' as const, + summary: '短信收尾工作等待超过5分钟', + value: `${Math.floor(age)}秒`, + threshold: '300秒', + }, + ]) { + if (!condition.active) continue; + const fingerprint = createHash('sha256').update(`durable:${condition.name}`).digest('hex').slice(0, 24); + const occurrence = await tx.infrastructureAlertEvent.findFirst({ + where: { fingerprint, recoveredAt: null }, + orderBy: { activeAt: 'desc' }, + }); + alerts.push({ + fingerprint, + name: condition.name, + severity: condition.severity, + status: 'firing', + startedAt: (occurrence?.activeAt ?? observedAt).toISOString(), + summary: condition.summary, + currentValue: condition.value, + threshold: condition.threshold, + service: '短信收尾', + acknowledged: false, + }); + } + await tx.infrastructureAlertCollection.upsert({ + where: { id: 'prometheus' }, + create: { id: 'prometheus', observedAt }, + update: { observedAt }, + }); + for (const alert of alerts) { + const activeAt = new Date(alert.startedAt); + const payload = JSON.parse(JSON.stringify(alert)) as Prisma.InputJsonValue; + await tx.infrastructureAlertEvent.createMany({ + data: [{ fingerprint: alert.fingerprint, activeAt, payload, lastObservedAt: observedAt }], + skipDuplicates: true, + }); + await tx.infrastructureAlertEvent.updateMany({ + where: { fingerprint: alert.fingerprint, activeAt, lastObservedAt: { lte: observedAt } }, + data: { payload, lastObservedAt: observedAt, recoveredAt: null }, + }); + } + await tx.infrastructureAlertEvent.updateMany({ + where: { + recoveredAt: null, + lastObservedAt: { lt: observedAt }, + ...(alerts.length + ? { + NOT: { + OR: alerts.map((alert) => ({ fingerprint: alert.fingerprint, activeAt: new Date(alert.startedAt) })), + }, + } + : {}), + }, + data: { recoveredAt: observedAt }, + }); + }); + return retainedAlerts(prisma); +} + +export async function retainedAlerts(prisma: PrismaService): Promise { + const records = await prisma.infrastructureAlertEvent.findMany({ + where: { clearedAt: null }, + orderBy: [{ activeAt: 'desc' }, { id: 'asc' }], + }); + return records.map((record) => ({ + ...(record.payload as unknown as InfrastructureAlert), + ...(record.recoveredAt ? { status: 'resolved' } : {}), + acknowledged: false, + })); +} + +export async function clearRetainedAlert( + prisma: PrismaService, + fingerprint: string, + rawActiveAt: unknown, + userId: string, +) { + const activeAt = new Date(String(rawActiveAt ?? '')); + if (!/^[a-f0-9]{24}$/.test(fingerprint) || !Number.isFinite(activeAt.getTime())) + throw new BadRequestException('告警标识无效'); + return prisma.$transaction(async (tx) => { + const record = await tx.infrastructureAlertEvent.findUnique({ + where: { fingerprint_activeAt: { fingerprint, activeAt } }, + }); + if (!record) throw new NotFoundException('告警记录不存在'); + const clearedAt = new Date(); + const result = await tx.infrastructureAlertEvent.updateMany({ + where: { id: record.id, clearedAt: null }, + data: { clearedAt, clearedBy: userId }, + }); + if (result.count) + await tx.operationLog.create({ + data: { + userId, + action: 'monitoring.alert_cleared', + resource: 'infrastructure_alert', + resourceId: record.id, + detail: { fingerprint, activeAt: activeAt.toISOString() }, + }, + }); + return { + fingerprint, + activeAt: activeAt.toISOString(), + cleared: true, + clearedAt: (record.clearedAt ?? clearedAt).toISOString(), + }; + }); +} diff --git a/api/src/metrics/metrics.service.ts b/api/src/metrics/metrics.service.ts index fd3a510..32a25f8 100644 --- a/api/src/metrics/metrics.service.ts +++ b/api/src/metrics/metrics.service.ts @@ -1,3 +1,4 @@ +import { renderCompletionMetrics } from '../send-chain/completion-metrics'; import { Injectable, OnModuleDestroy } from '@nestjs/common'; import { monitorEventLoopDelay } from 'node:perf_hooks'; @@ -314,6 +315,7 @@ export class MetricsService implements OnModuleDestroy { lines.push(metricLine('cmpp_api_auth_protection_events_total', count, { event, scope })); } this.eventLoopDelay.reset(); + lines.push(...renderCompletionMetrics()); return `${lines.join('\n')}\n`; } diff --git a/api/src/open-api/open-api.service.ts b/api/src/open-api/open-api.service.ts index 4e895d8..81933bd 100644 --- a/api/src/open-api/open-api.service.ts +++ b/api/src/open-api/open-api.service.ts @@ -526,24 +526,28 @@ export class OpenApiService implements OnModuleInit, OnModuleDestroy { return row; } - async queueWebhookEvent(data: { - tenantId: string; - applicationId?: string | null; - messageRecordId?: string | null; - messageId?: string | null; - uplinkMessageId?: string | null; - eventType: 'receipt' | 'uplink'; - payload: Record; - }) { + async queueWebhookEvent( + data: { + tenantId: string; + applicationId?: string | null; + messageRecordId?: string | null; + messageId?: string | null; + uplinkMessageId?: string | null; + eventType: 'receipt' | 'uplink'; + payload: Record; + }, + transaction?: Prisma.TransactionClient, + ) { + const db = transaction ?? this.prisma; if (!data.applicationId) return null; - const application = await this.prisma.smsApplication.findUnique({ + const application = await db.smsApplication.findUnique({ where: { id: data.applicationId }, include: { httpConfig: true }, }); const config = application?.httpConfig; const enabled = data.eventType === 'receipt' ? config?.receiptWebhookEnabled : config?.uplinkWebhookEnabled; if (!config?.enabled || !enabled) return null; - const endpoint = await this.prisma.httpWebhookEndpoint.findUnique({ + const endpoint = await db.httpWebhookEndpoint.findUnique({ where: { applicationId_eventType: { applicationId: data.applicationId, eventType: data.eventType } }, }); if (!endpoint || endpoint.status !== 'active') return null; @@ -553,7 +557,7 @@ export class OpenApiService implements OnModuleInit, OnModuleDestroy { : data.eventType === 'uplink' && data.uplinkMessageId ? `evt_uplink_${data.uplinkMessageId}` : `evt_${randomUUID()}`; - const delivery = await this.prisma.$transaction(async (tx) => { + const persist = async (tx: Prisma.TransactionClient) => { const event = await tx.httpWebhookEvent.upsert({ where: { eventId }, update: {}, @@ -573,7 +577,9 @@ export class OpenApiService implements OnModuleInit, OnModuleDestroy { update: {}, create: { eventId: event.id, endpointId: endpoint.id, recoveryVersion: 1 }, }); - }); + }; + const delivery = transaction ? await persist(transaction) : await this.prisma.$transaction(persist); + if (transaction) return delivery; if (delivery.status !== 'pending' || delivery.recoveryVersion !== 1) return delivery; await this.queue?.add( 'deliver', diff --git a/api/src/report-materials/official-export.service.ts b/api/src/report-materials/official-export.service.ts index e62e9a4..0a4a228 100644 --- a/api/src/report-materials/official-export.service.ts +++ b/api/src/report-materials/official-export.service.ts @@ -1,58 +1,85 @@ -import { BadRequestException, ConflictException, Injectable, NotFoundException } from '@nestjs/common'; import { Prisma } from '@prisma/client'; import ExcelJS from 'exceljs'; -import { createHash, randomUUID } from 'node:crypto'; -import { extname } from 'node:path'; import { FilesService } from '../files/files.service'; import { PrismaService } from '../prisma/prisma.service'; import { SmsConfigService } from '../sms-config/sms-config.service'; -import type { AnalyzeImportOptions, CreateImportProfileDto, CreateReportBatchDto, EmbeddedImage, ImportCommitDto, ImportMapping, PagedQuery, ReportBatchInspection, ReportBatchTarget, ReviewImportItemsDto } from './report-materials.contracts'; -import { profileData, validateProfile, loadWorkbook, assertSafeWorkbook, safeSpreadsheetText, readEmbeddedImages, suggestMappings, remapProfileColumns, signatureCoreMapping, drainageCoreMapping, normalizeHeader, normalizeFieldCode, clamp, normalizePage, normalizePageSize, dateRange, cellText, transformValue, mappedCoreValue, dynamicValues, jsonRecord, hasValue, isFileRef, resolveExportValue, applyExportTransform, styleHeader, normalizeImageExtension, imageContentType, safeFileName, normalizeBatchIdempotencyKey, jsonStringArray, jsonSafe } from './report-materials.helpers'; import { ReportPendingQueryService } from './pending-query.service'; +import { safeSpreadsheetText, styleHeader } from './report-materials.helpers'; /** R4 report-materials domain service composed behind ReportMaterialsService. */ export class ReportOfficialExportService { - constructor(private readonly prisma: PrismaService, private readonly files: FilesService, private readonly smsConfig: SmsConfigService, private readonly pending: ReportPendingQueryService) {} + constructor( + private readonly prisma: PrismaService, + private readonly files: FilesService, + private readonly smsConfig: SmsConfigService, + private readonly pending: ReportPendingQueryService, + ) {} async buildOfficialTemplate(reportType: 'signature' | 'drainage', operatorId?: string) { - const workbook = new ExcelJS.Workbook(); - workbook.creator = '聆界短信平台'; - const sheet = workbook.addWorksheet(reportType === 'signature' ? '签名资料' : '引流信息', { views: [{ state: 'frozen', ySplit: 1 }] }); - const headers = reportType === 'signature' + const workbook = new ExcelJS.Workbook(); + workbook.creator = '聆界短信平台'; + const sheet = workbook.addWorksheet(reportType === 'signature' ? '签名资料' : '引流信息', { + views: [{ state: 'frozen', ySplit: 1 }], + }); + const headers = + reportType === 'signature' ? ['短信签名', '用途说明', '营业执照图片', '授权书图片', '备注'] : ['短信签名', '引流 URL 或号码', '备注', '主体证明']; - sheet.addRow(headers); - sheet.addRow(reportType === 'signature' + sheet.addRow(headers); + sheet.addRow( + reportType === 'signature' ? ['示例签名', '验证码通知', '请在本单元格插入图片', '请在本单元格插入图片', '示例行，导入前请删除'] - : ['示例签名', 'example.com/path 或 13800138000', '示例行，导入前请删除', '请在本单元格插入图片']); - styleHeader(sheet.getRow(1)); - sheet.columns.forEach((column) => { column.width = 24; }); - sheet.getRow(2).height = 48; - const content = Buffer.from(await workbook.xlsx.writeBuffer()); - const fileName = `${reportType === 'signature' ? '签名' : '引流信息'}报备资料官方模板.xlsx`; - await this.prisma.operationLog.create({ data: { - userId: operatorId, action: 'report_material.template_downloaded', resource: 'report_material', + : ['示例签名', 'example.com/path 或 13800138000', '示例行，导入前请删除', '请在本单元格插入图片'], + ); + styleHeader(sheet.getRow(1)); + sheet.columns.forEach((column) => { + column.width = 24; + }); + sheet.getRow(2).height = 48; + const content = Buffer.from(await workbook.xlsx.writeBuffer()); + const fileName = `${reportType === 'signature' ? '签名' : '引流信息'}报备资料官方模板.xlsx`; + await this.prisma.operationLog.create({ + data: { + userId: operatorId, + action: 'report_material.template_downloaded', + resource: 'report_material', detail: { fileName, filters: { reportType }, successCount: 1, failedCount: 0 } as Prisma.InputJsonValue, - } }); - return { fileName, content }; - } + }, + }); + return { fileName, content }; + } - async exportPending(query: { reportType?: 'signature' | 'drainage'; tenantId?: string; applicationId?: string }, operatorId?: string) { - const items = await this.pending.findPendingItems(query); - const workbook = new ExcelJS.Workbook(); - const sheet = workbook.addWorksheet('待报备资料', { views: [{ state: 'frozen', ySplit: 1 }] }); - sheet.addRow(['资料类型', '企业', '企业应用', '签名/站点', '详情', '变更时间']); - styleHeader(sheet.getRow(1)); - for (const item of items) sheet.addRow([ - item.reportType === 'signature' ? '签名' : '引流信息', safeSpreadsheetText(item.tenant?.name), - safeSpreadsheetText(item.application?.name), safeSpreadsheetText(item.name), safeSpreadsheetText(item.detail), item.changedAt, + async exportPending( + query: { reportType?: 'signature' | 'drainage'; tenantId?: string; applicationId?: string }, + operatorId?: string, + ) { + const items = await this.pending.findPendingItems(query); + const workbook = new ExcelJS.Workbook(); + const sheet = workbook.addWorksheet('待报备资料', { views: [{ state: 'frozen', ySplit: 1 }] }); + sheet.addRow(['资料类型', '企业', '企业应用', '签名/站点', '详情', '变更时间']); + styleHeader(sheet.getRow(1)); + for (const item of items) + sheet.addRow([ + item.reportType === 'signature' ? '签名' : '引流信息', + safeSpreadsheetText(item.tenant?.name), + safeSpreadsheetText(item.application?.name), + safeSpreadsheetText(item.name), + safeSpreadsheetText(item.detail), + item.changedAt, ]); - sheet.columns.forEach((column, index) => { column.width = index === 4 ? 42 : 22; }); - const fileName = `待报备资料-${new Date().toISOString().slice(0, 10)}.xlsx`; - await this.prisma.operationLog.create({ data: { - tenantId: query.tenantId, userId: operatorId, action: 'report_material.pending_export', resource: 'report_material', + sheet.columns.forEach((column, index) => { + column.width = index === 4 ? 42 : 22; + }); + const fileName = `待报备资料-${new Date().toISOString().slice(0, 10)}.xlsx`; + await this.prisma.operationLog.create({ + data: { + tenantId: query.tenantId, + userId: operatorId, + action: 'report_material.pending_export', + resource: 'report_material', detail: { fileName, filters: query, successCount: items.length, failedCount: 0 } as Prisma.InputJsonValue, - } }); - return { fileName, content: Buffer.from(await workbook.xlsx.writeBuffer()) }; - } + }, + }); + return { fileName, content: Buffer.from(await workbook.xlsx.writeBuffer()) }; + } } diff --git a/api/src/send-chain/attempt-completion.ts b/api/src/send-chain/attempt-completion.ts new file mode 100644 index 0000000..20b9930 --- /dev/null +++ b/api/src/send-chain/attempt-completion.ts @@ -0,0 +1,219 @@ +import { HttpException, Logger } from '@nestjs/common'; +import { Prisma, SmsAttemptCompletionWork } from '@prisma/client'; +import { createHash, randomUUID } from 'node:crypto'; +import { PrismaService } from '../prisma/prisma.service'; +import { completionContext, CompletionRouteRequired } from './completion-context'; +import type { RoutedChannel } from './send-chain.contracts'; +import { countCompletion, observeCompletion } from './completion-metrics'; + +export type CompletionEventKind = 'receipt' | 'submit' | 'segment' | 'timeout' | 'rejection'; +export class AttemptCompletion { + private readonly logger = new Logger(AttemptCompletion.name); + private timer?: ReturnType; + private running = false; + constructor( + private readonly prisma: PrismaService, + private readonly execute: (kind: CompletionEventKind, payload: Prisma.JsonValue) => Promise, + private readonly waitForRoute: (route: RoutedChannel) => Promise, + ) {} + + start() { + this.timer = setInterval(() => void this.scan(), 5_000); + this.timer.unref(); + void this.scan(); + } + stop() { + if (this.timer) clearInterval(this.timer); + } + + async enqueue( + messageRecordId: string, + sourceSubmitRecordId: string | undefined, + kind: CompletionEventKind, + payload: unknown, + identity?: string, + ) { + const json = JSON.parse(JSON.stringify(payload)) as Prisma.InputJsonValue; + const workKey = sourceSubmitRecordId ? `attempt:${sourceSubmitRecordId}` : `message:${messageRecordId}`; + const eventKey = createHash('sha256') + .update(`${workKey}:${kind}:${identity ?? JSON.stringify(json)}`) + .digest('hex'); + const work = await this.prisma.$transaction(async (tx) => { + const message = await tx.smsMessageRecord.findUniqueOrThrow({ + where: { id: messageRecordId }, + select: { tenantId: true }, + }); + if (sourceSubmitRecordId) { + const source = await tx.smsSubmitRecord.findUniqueOrThrow({ where: { id: sourceSubmitRecordId } }); + if (source.messageRecordId !== messageRecordId || source.tenantId !== message.tenantId) + throw new Error('completion_source_mismatch'); + } + await tx.smsAttemptCompletionWork.createMany({ + data: [{ workKey, messageRecordId, sourceSubmitRecordId, tenantId: message.tenantId }], + skipDuplicates: true, + }); + const current = await tx.smsAttemptCompletionWork.findUniqueOrThrow({ where: { workKey } }); + if (current.messageRecordId !== messageRecordId) throw new Error('completion_work_mismatch'); + await tx.$queryRaw`SELECT id FROM "SmsAttemptCompletionWork" WHERE id=${current.id} FOR UPDATE`; + const inserted = await tx.smsCompletionEvent.createMany({ + data: [{ workId: current.id, eventKey, kind, payload: json }], + skipDuplicates: true, + }); + if (inserted.count) + await tx.$executeRaw` + UPDATE "SmsAttemptCompletionWork" SET revision=revision+1, + state=CASE WHEN state IN ('processing', 'needs_review') THEN state ELSE 'pending' END, + "nextAttemptAt"=(CURRENT_TIMESTAMP AT TIME ZONE 'UTC'), "updatedAt"=(CURRENT_TIMESTAMP AT TIME ZONE 'UTC') WHERE id=${current.id}`; + return current; + }); + await this.process(work.id); + return this.prisma.smsMessageRecord.findUnique({ where: { id: messageRecordId } }); + } + + async scan() { + if (this.running) return; + this.running = true; + try { + observeCompletion( + await this.prisma.$queryRaw>` + SELECT w.state, COUNT(*)::int AS count, + EXTRACT(EPOCH FROM (CURRENT_TIMESTAMP AT TIME ZONE 'UTC') - MIN(COALESCE(e."createdAt", w."updatedAt")))::float AS age + FROM "SmsAttemptCompletionWork" w + LEFT JOIN LATERAL (SELECT MIN("createdAt") AS "createdAt" FROM "SmsCompletionEvent" WHERE "workId"=w.id AND "processedAt" IS NULL) e ON true + WHERE w.state IN ('pending','processing','retry_wait','needs_review') GROUP BY w.state`, + ); + const rows = await this.prisma.$queryRaw>` + SELECT id FROM "SmsAttemptCompletionWork" + WHERE (state IN ('pending','retry_wait') AND "nextAttemptAt" <= (CURRENT_TIMESTAMP AT TIME ZONE 'UTC')) + OR (state='processing' AND "leaseUntil" < (CURRENT_TIMESTAMP AT TIME ZONE 'UTC')) + ORDER BY "nextAttemptAt", id LIMIT 32`; + for (const row of rows) await this.process(row.id); + } catch { + this.logger.error('completion_scan_failed'); + } finally { + this.running = false; + } + } + + async process(id: string) { + const owner = randomUUID(); + const claims = await this.prisma.$queryRaw` + UPDATE "SmsAttemptCompletionWork" SET state='processing', "leaseOwner"=${owner}, + "leaseUntil"=(CURRENT_TIMESTAMP AT TIME ZONE 'UTC')+INTERVAL '60 seconds', "fenceVersion"="fenceVersion"+1, + attempts=attempts+1, "updatedAt"=(CURRENT_TIMESTAMP AT TIME ZONE 'UTC') + WHERE id=${id} AND ((state IN ('pending','retry_wait') AND "nextAttemptAt" <= (CURRENT_TIMESTAMP AT TIME ZONE 'UTC')) + OR (state='processing' AND "leaseUntil" < (CURRENT_TIMESTAMP AT TIME ZONE 'UTC'))) RETURNING *`; + const claim = claims[0]; + if (!claim) { + countCompletion('not_claimed'); + return; + } + countCompletion('claimed'); + if (claim.attempts > 1) countCompletion('recovered'); + const renew = setInterval(() => { + void this.prisma + .$executeRaw`UPDATE "SmsAttemptCompletionWork" SET "leaseUntil"=(CURRENT_TIMESTAMP AT TIME ZONE 'UTC')+INTERVAL '60 seconds' + WHERE id=${id} AND state='processing' AND "leaseOwner"=${owner} AND "fenceVersion"=${claim.fenceVersion} + AND "leaseUntil">(CURRENT_TIMESTAMP AT TIME ZONE 'UTC')`.catch(() => + this.logger.warn('completion_lease_renew_failed'), + ); + }, 20_000); + renew.unref(); + let route: RoutedChannel | undefined; + let routePlanned = false; + let planRevision: number | undefined; + try { + for (let pass = 0; pass < 64; pass++) { + try { + const done = await this.prisma.$transaction( + async (tx) => { + const rows = await tx.$queryRaw` + SELECT * FROM "SmsAttemptCompletionWork" WHERE id=${id} AND state='processing' + AND "leaseOwner"=${owner} AND "fenceVersion"=${claim.fenceVersion} AND "leaseUntil">(CURRENT_TIMESTAMP AT TIME ZONE 'UTC') FOR UPDATE`; + const work = rows[0]; + if (!work) { + countCompletion('fence_rejected'); + throw new Error('completion_fence_rejected'); + } + await tx.$queryRaw`SELECT id FROM "SmsMessageRecord" WHERE id=${work.messageRecordId} FOR UPDATE`; + if (planRevision !== work.revision) { + route = undefined; + routePlanned = false; + } + planRevision = work.revision; + const event = await tx.smsCompletionEvent.findFirst({ + where: { workId: id, processedAt: null }, + orderBy: [{ createdAt: 'asc' }, { id: 'asc' }], + }); + if (event) { + await completionContext.run({ tx, messageRecordId: work.messageRecordId, route, routePlanned }, () => + this.execute(event.kind as CompletionEventKind, event.payload), + ); + await tx.smsCompletionEvent.update({ where: { id: event.id }, data: { processedAt: new Date() } }); + } + const remaining = await tx.smsCompletionEvent.count({ where: { workId: id, processedAt: null } }); + const message = await tx.smsMessageRecord.findUniqueOrThrow({ where: { id: work.messageRecordId } }); + const retry = work.sourceSubmitRecordId + ? await tx.smsSubmitRecord.findUnique({ + where: { retryOfSubmitRecordId: work.sourceSubmitRecordId }, + select: { id: true }, + }) + : null; + await tx.smsAttemptCompletionWork.update({ + where: { id }, + data: { + processedRevision: work.revision - remaining, + decision: message.status, + retrySubmitRecordId: retry?.id, + state: remaining ? 'processing' : 'idle', + lastError: null, + ...(!remaining ? { leaseOwner: null, leaseUntil: null, attempts: 0 } : {}), + }, + }); + return remaining === 0; + }, + { timeout: 20_000, maxWait: 5_000 }, + ); + route = undefined; + routePlanned = false; + countCompletion('event_committed'); + if (done) return; + } catch (error) { + if (!(error instanceof CompletionRouteRequired)) throw error; + // The probe transaction rolls back. Routing and rate limiting occur without locks. + try { + route = await error.select(); + await this.waitForRoute(route); + } catch (selectionError) { + if (!(selectionError instanceof HttpException) || selectionError.getStatus() >= 500) throw selectionError; + route = undefined; + } + routePlanned = true; + } + } + throw new Error('completion_batch_budget_exhausted'); + } catch (error) { + const code = + error instanceof Prisma.PrismaClientKnownRequestError + ? error.code + : error instanceof Error && error.message.startsWith('completion_') + ? error.message + : 'completion_processing_failed'; + const exhausted = claim.attempts >= 12; + countCompletion(exhausted ? 'needs_review' : 'retry_wait'); + await this.prisma.smsAttemptCompletionWork.updateMany({ + where: { id, leaseOwner: owner, fenceVersion: claim.fenceVersion, state: 'processing' }, + data: { + state: exhausted ? 'needs_review' : 'retry_wait', + leaseOwner: null, + leaseUntil: null, + lastError: code, + nextAttemptAt: new Date(Date.now() + Math.min(300_000, 1000 * 2 ** Math.min(claim.attempts, 8))), + }, + }); + this.logger.error(`${exhausted ? 'completion_needs_review' : 'completion_retry_wait'}:${code}`); + } finally { + clearInterval(renew); + } + } +} diff --git a/api/src/send-chain/completion-context.ts b/api/src/send-chain/completion-context.ts new file mode 100644 index 0000000..1482b4c --- /dev/null +++ b/api/src/send-chain/completion-context.ts @@ -0,0 +1,35 @@ +import { AsyncLocalStorage } from 'node:async_hooks'; +import { Prisma } from '@prisma/client'; +import { PrismaService } from '../prisma/prisma.service'; +import type { RoutedChannel } from './send-chain.contracts'; + +export type CompletionContext = { + tx: Prisma.TransactionClient; + messageRecordId: string; + route?: RoutedChannel; + routePlanned?: boolean; +}; +export const completionContext = new AsyncLocalStorage(); + +// Only send-chain collaborators use this adapter. Nested billing/outbox transactions +// join the explicitly established completion transaction, never start a second one. +export function completionDatabase(prisma: PrismaService): PrismaService { + return new Proxy(prisma, { + get(target, key) { + const tx = completionContext.getStore()?.tx; + if (tx && key === '$transaction') { + return (operation: ((client: Prisma.TransactionClient) => unknown) | Promise[]) => + typeof operation === 'function' ? operation(tx) : Promise.all(operation); + } + const owner = tx && key in tx ? tx : target; + const value = Reflect.get(owner, key); + return typeof value === 'function' ? value.bind(owner) : value; + }, + }); +} + +export class CompletionRouteRequired extends Error { + constructor(readonly select: () => Promise) { + super('completion_route_required'); + } +} diff --git a/api/src/send-chain/completion-metrics.ts b/api/src/send-chain/completion-metrics.ts new file mode 100644 index 0000000..a9ee23b --- /dev/null +++ b/api/src/send-chain/completion-metrics.ts @@ -0,0 +1,26 @@ +type CompletionMetric = + 'claimed' | 'not_claimed' | 'recovered' | 'fence_rejected' | 'event_committed' | 'retry_wait' | 'needs_review'; +const counts = new Map(); +let snapshot: Array<{ state: string; count: number; age: number }> = []; +const states = ['pending', 'processing', 'retry_wait', 'needs_review']; +export function countCompletion(event: CompletionMetric) { + counts.set(event, (counts.get(event) ?? 0) + 1); +} +export function observeCompletion(rows: typeof snapshot) { + snapshot = rows; +} +export function renderCompletionMetrics() { + return [ + '# TYPE cmpp_completion_events_total counter', + ...[...counts].map(([event, value]) => `cmpp_completion_events_total{event="${event}"} ${value}`), + '# TYPE cmpp_completion_work gauge', + '# TYPE cmpp_completion_oldest_seconds gauge', + ...states.flatMap((state) => { + const row = snapshot.find((item) => item.state === state); + return [ + `cmpp_completion_work{state="${state}"} ${Number(row?.count ?? 0)}`, + `cmpp_completion_oldest_seconds{state="${state}"} ${Math.max(0, Number(row?.age ?? 0))}`, + ]; + }), + ]; +} diff --git a/api/src/send-chain/drainage-submit-guard.controller.spec.ts b/api/src/send-chain/drainage-submit-guard.controller.spec.ts new file mode 100644 index 0000000..07c2893 --- /dev/null +++ b/api/src/send-chain/drainage-submit-guard.controller.spec.ts @@ -0,0 +1,54 @@ +import { createHash } from 'node:crypto'; +import { DrainageSubmitGuardController } from './drainage-submit-guard.controller'; + +describe('channel test final guard', () => { + const content = '无签名正文 https://example.test 4001234567'.repeat(5); + function setup(overrides = {}) { + const tx = { + smsSubmitRecord: { + findUnique: jest.fn().mockResolvedValue({ + channelId: 'channel', + submitId: 'submit', + resultProcessedAt: null, + messageRecord: { content, tenantId: null, batchTaskId: null, status: 'submit_queued', ...overrides }, + }), + }, + drainageDetectionRule: { + findMany: jest.fn(() => { + throw new Error('must not detect channel tests'); + }), + }, + }; + const db = { $transaction: (operation: (value: unknown) => unknown) => operation(tx) }; + return { controller: new DrainageSubmitGuardController(db as never), tx }; + } + const local = { socket: { remoteAddress: '127.0.0.1' } }; + const input = { + submitId: 'submit', + channelId: 'channel', + contentHash: createHash('sha256').update(content).digest('hex'), + }; + it('allows unsigned long channel tests containing drainage information without detecting content', async () => { + const { controller, tx } = setup(); + await expect(controller.authorize(local, input)).resolves.toEqual({ allowed: true }); + expect(tx.drainageDetectionRule.findMany).not.toHaveBeenCalled(); + }); + it('still rejects a modified body or a finished submit', async () => { + await expect(setup().controller.authorize(local, { ...input, contentHash: 'a'.repeat(64) })).resolves.toMatchObject( + { allowed: false }, + ); + await expect(setup({ status: 'delivered' }).controller.authorize(local, input)).resolves.toMatchObject({ + allowed: false, + }); + }); + it('does not grant the exemption to a customer message without its required signature', async () => { + await expect( + setup({ tenantId: 'tenant', applicationId: 'app', batchTaskId: 'task' }).controller.authorize(local, input), + ).resolves.toMatchObject({ allowed: false }); + }); + it('rejects external callers before looking up a submit', async () => { + const { controller, tx } = setup(); + await expect(controller.authorize({ socket: { remoteAddress: '10.0.0.2' } }, input)).rejects.toThrow(); + expect(tx.smsSubmitRecord.findUnique).not.toHaveBeenCalled(); + }); +}); diff --git a/api/src/send-chain/drainage-submit-guard.controller.ts b/api/src/send-chain/drainage-submit-guard.controller.ts index 513cb3a..79a1249 100644 --- a/api/src/send-chain/drainage-submit-guard.controller.ts +++ b/api/src/send-chain/drainage-submit-guard.controller.ts @@ -58,19 +58,9 @@ export class DrainageSubmitGuardController { ) return { allowed: false, code: 'DRN', reason: '提交或消息已终结，不得重复发送' }; if (!message.tenantId && !message.batchTaskId) { - try { - await evaluateMessageDrainage( - tx as unknown as PrismaService, - message, - message.carrier ?? undefined, - undefined, - true, - ); - return { allowed: true }; - } catch (error) { - if (!(error instanceof DrainageRejection)) throw error; - return { allowed: false, code: 'DRN', reason: error.message }; - } + // Operations channel tests bypass signature/drainage policy only after + // verifying the durable submit, channel, content and unfinished state. + return { allowed: true }; } if (!message.tenantId || !message.applicationId || !message.signatureId) return { allowed: false, code: 'DRN', reason: '提交消息未关联企业应用和签名' }; diff --git a/api/src/send-chain/send-accounting.service.ts b/api/src/send-chain/send-accounting.service.ts index b71ad9a..9cdc149 100644 --- a/api/src/send-chain/send-accounting.service.ts +++ b/api/src/send-chain/send-accounting.service.ts @@ -1,16 +1,10 @@ -import { BadRequestException, HttpException, HttpStatus, Logger, NotFoundException } from '@nestjs/common'; -import { Prisma } from '@prisma/client'; -import { createHash } from 'node:crypto'; +import { Logger } from '@nestjs/common'; import { BillingService } from '../billing/billing.service'; import { moneyToNumber } from '../common/money'; import type { OpenApiService } from '../open-api/open-api.service'; import { PrismaService } from '../prisma/prisma.service'; -import type { GatewaySubmitResultDto, GatewaySubmitSegmentResultDto, GatewayReceiptEventDto, GatewayUplinkEventDto, GatewayPendingDeliveryQueryDto, GatewayDownstreamSentDto, GatewayDownstreamAcknowledgedDto, GatewayDownstreamFailureType, GatewayControlDeliveryResult, GatewaySubmitDeadLetterDto, RequeueGatewaySubmitExceptionDto, GatewayDownstreamRecoveryStatusDto, TimeoutUnknownDto } from './send-chain.contracts'; -import { positiveInteger, normalizeReceiptStatus, normalizeCarrier, DEFAULT_DOWNSTREAM_RETRY_DELAY_MS, DEFAULT_DOWNSTREAM_RETRY_MAX_DELAY_MS, DEFAULT_DOWNSTREAM_MAX_RETRIES, DEFAULT_DOWNSTREAM_PENDING_TIMEOUT_HOURS, DEFAULT_RECEIPT_TIMEOUT_HOURS, DEFAULT_GATEWAY_SUBMIT_REQUEUE_STALE_MS, DEFAULT_DOWNSTREAM_MANUAL_REQUEUE_STALE_MS, DEFAULT_UPSTREAM_RECEIPT_INBOX_PROCESSING_STALE_MS, DEFAULT_UPSTREAM_RECEIPT_INBOX_MAX_ATTEMPTS, DEFAULT_UPSTREAM_RECEIPT_INBOX_MAX_AGE_HOURS, gatewaySubmitRequeueKey, isObjectRecord, asDateOrNull, downstreamRetryDelayMs, downstreamAckTimeoutMs, downstreamRetryBaseDelayMs, downstreamRetryMaxDelayMs, downstreamMaxRetries, downstreamPendingTimeoutHours, downstreamControlFailureMessage, normalizeSubmitStatus, downstreamDeliveryAttemptKey, hasRecoveryAuditStateChanged, normalizeRecoveryFailureCategory, aggregateReceiptSegmentState, isSameUpstreamEndpointIdentity, receiptEventKey } from './send-chain.helpers'; -import type { SendSubmissionService } from './send-submission.service'; import type { SendCompletionCallbacks, SendCompletionFacade } from './send-completion.service'; - /** * R10 accounting implementation. * Cross-domain calls return through the stable SendChainService seam to preserve locking and test observability. @@ -41,16 +35,19 @@ export class SendAccountingService { const unitPrice = moneyToNumber(message.unitPrice); const billingUnits = message.billingUnits ?? 0; const exists = await this.prisma.smsBillingRecord.findFirst({ where: { messageId: message.messageId } }); - if (exists?.billingStatus === 'charged') { + if (exists && ['charged', 'refunded'].includes(exists.billingStatus)) { return; } - const transaction = amountCents > 0 ? await this.billing.settleFrozenCharge({ - tenantId: message.tenantId, - amountCents, - taskId: message.batchTaskId, - messageId: message.messageId, - remark: `短信 ${message.messageId} 提交成功释放冻结并转扣费`, - }) : null; + const transaction = + amountCents > 0 + ? await this.billing.settleFrozenCharge({ + tenantId: message.tenantId, + amountCents, + taskId: message.batchTaskId, + messageId: message.messageId, + remark: `短信 ${message.messageId} 提交成功释放冻结并转扣费`, + }) + : null; const data = { tenantId: message.tenantId, applicationId: message.applicationId ?? undefined, @@ -72,14 +69,22 @@ export class SendAccountingService { } async releaseMessageReservation( - message: { tenantId: string; batchTaskId: string; messageId: string; amountCents: number | bigint; billingUnits: number }, + message: { + tenantId: string; + batchTaskId: string; + messageId: string; + amountCents: number | bigint; + billingUnits: number; + }, remark: string, ) { const amountCents = moneyToNumber(message.amountCents); if (amountCents <= 0) { return; } - const charged = await this.prisma.smsBillingRecord.findFirst({ where: { messageId: message.messageId, billingStatus: 'charged' } }); + const charged = await this.prisma.smsBillingRecord.findFirst({ + where: { messageId: message.messageId, billingStatus: 'charged' }, + }); if (charged) { return; } @@ -107,11 +112,15 @@ export class SendAccountingService { if (amountCents <= 0) { return; } - const refunded = await this.prisma.smsBillingRecord.findFirst({ where: { messageId: message.messageId, billingStatus: 'refunded' } }); + const refunded = await this.prisma.smsBillingRecord.findFirst({ + where: { messageId: message.messageId, billingStatus: 'refunded' }, + }); if (refunded) { return; } - const charged = await this.prisma.smsBillingRecord.findFirst({ where: { messageId: message.messageId, billingStatus: 'charged' } }); + const charged = await this.prisma.smsBillingRecord.findFirst({ + where: { messageId: message.messageId, billingStatus: 'charged' }, + }); if (!charged) { return; } diff --git a/api/src/send-chain/send-chain.service.spec.ts b/api/src/send-chain/send-chain.service.spec.ts index 98aa700..cf61408 100644 --- a/api/src/send-chain/send-chain.service.spec.ts +++ b/api/src/send-chain/send-chain.service.spec.ts @@ -1694,6 +1694,7 @@ describe('SendChainService', () => { applicationId: 'app-1', eventType: 'receipt', }), + undefined, ); expect(prisma.cmppDownstreamDelivery.create).not.toHaveBeenCalled(); }); @@ -3257,7 +3258,7 @@ describe('SendChainService', () => { data: expect.objectContaining({ sequenceId: 7, gatewayMessageId: 'GW-1', submitStatus: 'accepted' }), }); expect(prisma.smsMessageRecord.updateMany).toHaveBeenCalledWith({ - where: { id: 'record-1', status: { notIn: ['delivered', 'failed', 'unknown'] } }, + where: { id: 'record-1', status: { notIn: ['delivered', 'failed', 'unknown', 'timeout'] } }, data: expect.objectContaining({ gatewayMessageId: 'GW-1', status: 'submitted', submitStatus: 'accepted' }), }); expect(billing.settleFrozenCharge).toHaveBeenCalledWith( @@ -4891,7 +4892,7 @@ describe('SendChainService', () => { expect(prisma.smsMessageRecord.updateMany).toHaveBeenNthCalledWith( 1, expect.objectContaining({ - where: { id: 'record-1', status: { notIn: ['delivered', 'failed', 'unknown'] } }, + where: { id: 'record-1', status: { notIn: ['delivered', 'failed', 'unknown', 'timeout'] } }, data: expect.objectContaining({ status: 'submitted' }), }), ); @@ -5567,6 +5568,7 @@ describe('SendChainService', () => { errorCode: 'RECEIPT_TIMEOUT', }), }), + undefined, ); expect(prisma.cmppDownstreamDelivery.create).not.toHaveBeenCalled(); expect(prisma.smsMessageRecord.updateMany).toHaveBeenCalledWith({ diff --git a/api/src/send-chain/send-chain.service.ts b/api/src/send-chain/send-chain.service.ts index 4f3b3e9..b0e26de 100644 --- a/api/src/send-chain/send-chain.service.ts +++ b/api/src/send-chain/send-chain.service.ts @@ -1,3 +1,5 @@ +import { AttemptCompletion } from './attempt-completion'; +import { completionContext, completionDatabase, CompletionRouteRequired } from './completion-context'; import { BadRequestException, forwardRef, @@ -84,6 +86,7 @@ export class SendChainService implements OnModuleInit, OnModuleDestroy { private readonly submission: SendSubmissionService; private readonly completion: SendCompletionService; private readonly downstreamRequeueTasks: SendDownstreamRequeueTaskService; + private readonly attemptCompletion?: AttemptCompletion; constructor( private readonly prisma: PrismaService, @@ -94,6 +97,11 @@ export class SendChainService implements OnModuleInit, OnModuleDestroy { @Optional() phoneRouting?: PhoneRoutingLookupService, @Optional() metrics?: MetricsService, ) { + const rootPrisma = prisma; + prisma = completionDatabase(prisma); + this.prisma = prisma; + // Structural unit-test doubles may omit the durable delegate; real Prisma always has it. + const coordinatedBilling = rootPrisma.smsAttemptCompletionWork ? new BillingService(prisma) : billing; const resolvedPhoneRouting = phoneRouting ?? new PhoneRoutingLookupService(prisma); this.submission = new SendSubmissionService( prisma, @@ -109,12 +117,46 @@ export class SendChainService implements OnModuleInit, OnModuleDestroy { }, metrics, ); - this.completion = new SendCompletionService(prisma, billing, openApi, this as unknown as SendCompletionFacade); + this.completion = new SendCompletionService( + prisma, + coordinatedBilling, + openApi, + this as unknown as SendCompletionFacade, + ); + if (rootPrisma.smsAttemptCompletionWork) { + this.attemptCompletion = new AttemptCompletion( + rootPrisma, + async (kind, payload) => { + const event = payload as unknown as { + data: GatewayReceiptEventDto & GatewaySubmitResultDto & GatewaySubmitSegmentResultDto; + incomingIdentity?: Parameters[1]; + olderThanHours?: number; + errorCode?: string; + reason?: string; + }; + if (kind === 'receipt') return this.completion.handleReceipt(event.data, event.incomingIdentity); + if (kind === 'submit') return this.completion.handleSubmitResult(event.data); + if (kind === 'segment') return this.completion.handleSubmitSegmentResult(event.data); + if (kind === 'timeout') return this.completion.markUnknownTimeout({ olderThanHours: event.olderThanHours }); + const message = await prisma.smsMessageRecord.findUniqueOrThrow({ + where: { id: completionContext.getStore()!.messageRecordId }, + }); + if ( + message.status === 'delivered' || + (message.status === 'timeout' && message.errorCode === 'RECEIPT_TIMEOUT') + ) + return message; + return this.completion.recordCmppFailureReceipt(message, event.errorCode!, event.reason!); + }, + (route) => this.submission.waitForChannelRateLimit(route.channel.id, route.channel.rateLimitPerSecond), + ); + } this.downstreamRequeueTasks = new SendDownstreamRequeueTaskService(prisma, this); } onModuleInit() { const processRole = process.env.CMPP_PROCESS_ROLE?.trim() || 'all'; + if (['all', 'worker', 'callback'].includes(processRole)) this.attemptCompletion?.start(); if (processRole === 'api' || processRole === 'callback') return; if (processRole === 'outbox') { this.submission.startSubmitOutboxPublisher(); @@ -202,6 +244,7 @@ export class SendChainService implements OnModuleInit, OnModuleDestroy { } async onModuleDestroy() { + this.attemptCompletion?.stop(); if (this.receiptTimeoutInitialTimer) clearTimeout(this.receiptTimeoutInitialTimer); if (this.receiptTimeoutIntervalTimer) clearInterval(this.receiptTimeoutIntervalTimer); if (this.scheduledDispatchInitialTimer) clearTimeout(this.scheduledDispatchInitialTimer); @@ -484,6 +527,11 @@ export class SendChainService implements OnModuleInit, OnModuleDestroy { } async handleSubmitSegmentResult(data: GatewaySubmitSegmentResultDto) { + if (this.attemptCompletion && !completionContext.getStore()) { + const message = await this.completion.requireMessageByGatewayEvent(data.messageId, data.gatewayMessageId); + const source = await this.completion.resolveSubmitRecordForGatewaySegmentResult(message.id, data); + return this.attemptCompletion.enqueue(message.id, source.id, 'segment', { data }); + } return this.completion.handleSubmitSegmentResult(data); } @@ -495,6 +543,11 @@ export class SendChainService implements OnModuleInit, OnModuleDestroy { } async handleSubmitResult(data: GatewaySubmitResultDto) { + if (this.attemptCompletion && !completionContext.getStore()) { + const message = await this.completion.requireMessageByGatewayEvent(data.messageId, data.gatewayMessageId); + const source = await this.completion.resolveSubmitRecordForGatewayResult(message.id, data); + return this.attemptCompletion.enqueue(message.id, source.id, 'submit', { data }, data.eventId); + } return this.completion.handleSubmitResult(data); } @@ -528,6 +581,14 @@ export class SendChainService implements OnModuleInit, OnModuleDestroy { cmppVersion: string; }, ) { + if (this.attemptCompletion && !completionContext.getStore()) { + const resolved = await this.completion.resolveReceiptMessage(data, incomingIdentity); + if (!resolved.submitRecordId) throw new NotFoundException('回执缺少可确认的提交尝试关联'); + return this.attemptCompletion.enqueue(resolved.message.id, resolved.submitRecordId, 'receipt', { + data, + incomingIdentity, + }); + } return this.completion.handleReceipt(data, incomingIdentity); } @@ -726,6 +787,32 @@ export class SendChainService implements OnModuleInit, OnModuleDestroy { } async markUnknownTimeout(data: TimeoutUnknownDto) { + if (this.attemptCompletion && !completionContext.getStore()) { + const olderThanHours = data.olderThanHours ?? positiveInteger(process.env.SMS_RECEIPT_TIMEOUT_HOURS, 72); + const candidates = await this.prisma.smsMessageRecord.findMany({ + where: { + tenantId: { not: null }, + OR: [ + { + status: { in: ['submitted', 'unknown'] }, + submittedAt: { lte: new Date(Date.now() - olderThanHours * 3600_000) }, + }, + { status: 'timeout', errorCode: 'RECEIPT_TIMEOUT', timeoutReceiptQueuedAt: null }, + ], + }, + select: { id: true, submitId: true, status: true }, + take: 100, + }); + let timeout = 0; + for (const message of candidates) { + const source = message.submitId + ? await this.prisma.smsSubmitRecord.findUnique({ where: { submitId: message.submitId } }) + : null; + const result = await this.attemptCompletion.enqueue(message.id, source?.id, 'timeout', { olderThanHours }); + if (message.status !== 'timeout' && result?.status === 'timeout') timeout++; + } + return { timeout }; + } return this.completion.markUnknownTimeout(data); } @@ -803,6 +890,14 @@ export class SendChainService implements OnModuleInit, OnModuleDestroy { }, options: { forceNational?: boolean; excludeChannelIds?: string[] } = {}, ): Promise { + const context = completionContext.getStore(); + if (context) { + const prepare = () => this.submission.selectChannelForMessage(message, { ...options, previewOnly: true }); + if (!context.routePlanned) throw new CompletionRouteRequired(prepare); + const current = await this.submission.selectChannelForMessage(message, options); + if (!context.route || current.channel.id !== context.route.channel.id) throw new CompletionRouteRequired(prepare); + return current; + } return this.submission.selectChannelForMessage(message, options); } @@ -923,6 +1018,9 @@ export class SendChainService implements OnModuleInit, OnModuleDestroy { errorCode: string, reason: string, ) { + if (this.attemptCompletion && !completionContext.getStore()) { + return this.attemptCompletion.enqueue(message.id, undefined, 'rejection', { errorCode, reason }); + } return this.completion.recordCmppFailureReceipt(message, errorCode, reason); } @@ -1004,6 +1102,7 @@ export class SendChainService implements OnModuleInit, OnModuleDestroy { } private async waitForChannelRateLimit(channelId: string, tps: number) { + if (completionContext.getStore()) return; return this.submission.waitForChannelRateLimit(channelId, tps); } diff --git a/api/src/send-chain/send-downstream-delivery.service.ts b/api/src/send-chain/send-downstream-delivery.service.ts index 810df8a..0baf9cf 100644 --- a/api/src/send-chain/send-downstream-delivery.service.ts +++ b/api/src/send-chain/send-downstream-delivery.service.ts @@ -1,3 +1,4 @@ +import { completionContext } from './completion-context'; import { BadRequestException, Logger, NotFoundException } from '@nestjs/common'; import { Prisma } from '@prisma/client'; import { createHash, randomUUID } from 'node:crypto'; @@ -217,20 +218,24 @@ export class SendDownstreamDeliveryService { const cmppDeliveryAllowed = deliveryAllowed || data.allowBusinessRejectionCmppDelivery === true; if (deliveryAllowed && data.queueHttpWebhook !== false) { try { - await this.openApi?.queueWebhookEvent({ - tenantId: data.tenantId, - applicationId: data.applicationId, - messageRecordId: data.messageRecordId, - messageId: data.messageId, - uplinkMessageId: typeof data.payload.uplinkMessageId === 'string' ? data.payload.uplinkMessageId : undefined, - eventType: data.deliveryType, - payload: data.payload, - }); + await this.openApi?.queueWebhookEvent( + { + tenantId: data.tenantId, + applicationId: data.applicationId, + messageRecordId: data.messageRecordId, + messageId: data.messageId, + uplinkMessageId: + typeof data.payload.uplinkMessageId === 'string' ? data.payload.uplinkMessageId : undefined, + eventType: data.deliveryType, + payload: data.payload, + }, + completionContext.getStore()?.tx, + ); } catch (error) { this.logger.error( `HTTP webhook queue failed for ${data.deliveryType}/${data.messageId ?? '-'}: ${error instanceof Error ? error.message : String(error)}`, ); - if (data.propagateHttpQueueError) throw error; + if (data.propagateHttpQueueError || completionContext.getStore()) throw error; } } if (data.queueCmppDelivery === false) { @@ -249,6 +254,34 @@ export class SendDownstreamDeliveryService { : data.deliveryType === 'uplink' && typeof data.payload.uplinkMessageId === 'string' ? `uplink:${data.payload.uplinkMessageId}` : null; + if (completionContext.getStore()) { + if (!dedupeKey) throw new Error('completion_notification_identity_missing'); + await this.prisma.cmppDownstreamDelivery.createMany({ + data: [ + { + tenantId: data.tenantId, + applicationId: data.applicationId, + messageRecordId: data.messageRecordId, + messageId: data.messageId, + dedupeKey, + deliveryType: data.deliveryType, + payload, + retryEnabled: + cmppDeliveryAllowed && + (data.deliveryType === 'uplink' + ? application.downstreamUplinkRetryEnabled + : application.downstreamReceiptRetryEnabled), + status: cmppDeliveryAllowed ? 'pending' : 'abandoned', + lastError: cmppDeliveryAllowed ? null : '企业应用已停用，保留回执但不再向客户应用推送', + }, + ], + skipDuplicates: true, + }); + const retained = await this.prisma.cmppDownstreamDelivery.findUniqueOrThrow({ where: { dedupeKey } }); + if (retained.messageRecordId !== data.messageRecordId || retained.applicationId !== data.applicationId) + throw new Error('completion_notification_identity_mismatch'); + return retained; + } let delivery; try { delivery = await this.prisma.cmppDownstreamDelivery.create({ diff --git a/api/src/send-chain/send-gateway-result.service.ts b/api/src/send-chain/send-gateway-result.service.ts index d48f696..5c27e44 100644 --- a/api/src/send-chain/send-gateway-result.service.ts +++ b/api/src/send-chain/send-gateway-result.service.ts @@ -188,11 +188,18 @@ export class SendGatewayResultService { } const status = data.submitStatus === 'accepted' ? 'submitted' : data.submitStatus === 'timeout' ? 'timeout' : 'submit_failed'; + if ( + data.submitStatus !== 'accepted' && + (message.status === 'delivered' || (message.status === 'timeout' && message.errorCode === 'RECEIPT_TIMEOUT')) + ) { + await this.markSubmitResultProcessed(submitRecord.id, data.eventId, submittedAt); + return message; + } if (data.submitStatus === 'accepted' && !isStandaloneChannelTest && message.tenantId && message.batchTaskId) { const businessMessage = message as typeof message & { tenantId: string; batchTaskId: string }; await this.facade.chargeAcceptedMessage(businessMessage); const latest = await this.prisma.smsMessageRecord.findUnique({ where: { id: message.id } }); - if (latest?.status === 'failed') { + if (latest?.status === 'failed' || (latest?.status === 'timeout' && latest.errorCode === 'RECEIPT_TIMEOUT')) { await this.facade.refundMessage(businessMessage, '先到失败回执补偿退款'); } } else if ( @@ -219,7 +226,7 @@ export class SendGatewayResultService { data.submitStatus === 'timeout' ? '提交超时释放冻结' : '提交失败释放冻结', ); } - const protectedTerminalStatuses = ['delivered', 'failed', 'unknown']; + const protectedTerminalStatuses = ['delivered', 'failed', 'unknown', 'timeout']; const updated = await this.prisma.smsMessageRecord.updateMany({ where: data.submitStatus === 'accepted' diff --git a/api/src/send-chain/send-gateway-submit.service.ts b/api/src/send-chain/send-gateway-submit.service.ts index f465c0b..4d62302 100644 --- a/api/src/send-chain/send-gateway-submit.service.ts +++ b/api/src/send-chain/send-gateway-submit.service.ts @@ -1,3 +1,4 @@ +import { completionContext } from './completion-context'; import { BadRequestException, Logger, NotFoundException } from '@nestjs/common'; import { Prisma } from '@prisma/client'; import { Queue, Worker } from 'bullmq'; @@ -310,7 +311,7 @@ export class SendGatewaySubmitService { sessionId: sessionByChannel.get(routed.channel.id), }; }); - const writeOutbox = this.submitOutboxEnabled(); + const writeOutbox = Boolean(completionContext.getStore()) || this.submitOutboxEnabled(); await this.measureSendStage('submit_transaction', () => this.prisma.$transaction(async (tx) => { await tx.smsSubmitRecord.createMany({ @@ -365,7 +366,7 @@ export class SendGatewaySubmitService { } }), ); - if (!this.submitOutboxPublishEnabled()) { + if (!completionContext.getStore() && !this.submitOutboxPublishEnabled()) { await Promise.all(prepared.map(({ command }) => this.facade.publishGatewaySubmitCommand(command))); } await this.refreshTaskProgressBatch(prepared.map(({ message }) => message)); @@ -762,7 +763,7 @@ export class SendGatewaySubmitService { const submitId = `SUB-${randomUUID()}`; const sessionId = await this.getOpenSubmitSessionId(channel.id); const command = this.buildGatewaySubmitCommand(message, routed, attempt, submitId, upstreamSrcId); - const writeOutbox = this.submitOutboxEnabled(); + const writeOutbox = Boolean(completionContext.getStore()) || this.submitOutboxEnabled(); try { await this.measureSendStage('submit_transaction', () => this.prisma.$transaction(async (tx) => { @@ -849,7 +850,7 @@ export class SendGatewaySubmitService { } throw error; } - if (!this.submitOutboxPublishEnabled()) { + if (!completionContext.getStore() && !this.submitOutboxPublishEnabled()) { await this.measureSendStage('gateway_stream_publish', () => this.facade.publishGatewaySubmitCommand(command)); } await this.measureSendStage('task_progress', () => @@ -1087,7 +1088,7 @@ return streamId`; template?: { signature?: { id?: string | null } | null } | null; signature?: { id?: string | null } | null; }, - options: { forceNational?: boolean; excludeChannelIds?: string[] } = {}, + options: { forceNational?: boolean; excludeChannelIds?: string[]; previewOnly?: boolean } = {}, ): Promise { if (!message.applicationId) { throw new BadRequestException('短信应用未配置，无法选择通道组'); @@ -1100,7 +1101,7 @@ return streamId`; this.facade.identifyCarrier(message.phoneNumber), this.facade.identifyProvince(message.phoneNumber), ]); - if (!hasPersistedRouting) { + if (!hasPersistedRouting && !options.previewOnly) { await this.prisma.smsMessageRecord.update({ where: { id: message.id }, data: { carrier: resolved[0], province: resolved[1] }, @@ -1137,7 +1138,7 @@ return streamId`; approvedChannelIds, routingKey: message.id, }); - await channelWords.persist(this.prisma); + if (!options.previewOnly) await channelWords.persist(this.prisma); if (rejected) throw new ChannelWordRejection(); if (!selected) { throw new NotFoundException('无已报备通过且在线的可用通道'); @@ -1352,6 +1353,10 @@ return streamId`; `); if (direct === 1) return; } + if (completionContext.getStore()) { + await this.prisma.$queryRaw`SELECT id FROM "SmsBatchTask" WHERE id=${batchTaskId} FOR UPDATE`; + return this.refreshTaskProgressUntilClean(batchTaskId, true); + } const running = this.taskProgressRefreshes.get(batchTaskId); if (running) { // A state transition committed after the running aggregate may not be visible @@ -1372,9 +1377,9 @@ return streamId`; } } - private async refreshTaskProgressUntilClean(batchTaskId: string) { + private async refreshTaskProgressUntilClean(batchTaskId: string, transactional = false) { do { - this.dirtyTaskProgressRefreshes.delete(batchTaskId); + if (!transactional) this.dirtyTaskProgressRefreshes.delete(batchTaskId); const groups = await this.prisma.smsMessageRecord.groupBy({ by: ['status'], where: { batchTaskId }, @@ -1401,7 +1406,7 @@ return streamId`; where: { id: batchTaskId }, data: { progressTotal, submittedTotal, successTotal, failedTotal, unknownTotal, timeoutTotal, status }, }); - } while (this.dirtyTaskProgressRefreshes.has(batchTaskId)); + } while (!transactional && this.dirtyTaskProgressRefreshes.has(batchTaskId)); } private async recoverNightReviews() { @@ -1464,6 +1469,17 @@ return streamId`, } private getOpenSubmitSessionId(channelId: string) { + if (completionContext.getStore()) { + const sessionNo = `OPEN-${channelId}`; + return this.prisma.cmppSubmitSession + .upsert({ + where: { sessionNo }, + update: {}, + create: { channelId, sessionNo, submitTotal: 0 }, + select: { id: true }, + }) + .then((session) => session.id); + } const cached = this.openSubmitSessionIds.get(channelId); if (cached) return cached; const sessionNo = `OPEN-${channelId}`; diff --git a/api/src/send-chain/send-receipt.service.ts b/api/src/send-chain/send-receipt.service.ts index 7a4d82e..4ae2b7e 100644 --- a/api/src/send-chain/send-receipt.service.ts +++ b/api/src/send-chain/send-receipt.service.ts @@ -1,3 +1,4 @@ +import { completionContext } from './completion-context'; import { Logger, NotFoundException } from '@nestjs/common'; import { Prisma } from '@prisma/client'; @@ -229,10 +230,12 @@ export class SendReceiptService { where: { receiptKey }, include: { messageRecord: true }, }); - if (existingReceipt?.messageRecord) { + if (existingReceipt?.messageRecord && !completionContext.getStore()) { return existingReceipt.messageRecord; } const message = resolved.message; + if (existingReceipt && existingReceipt.messageRecordId !== message.id) + throw new Error('completion_receipt_owner_mismatch'); const deliveredAt = data.deliveredAt ? new Date(data.deliveredAt) : new Date(); if (resolved.submitRecordId) { await this.prisma.smsSubmitRecord.updateMany({ @@ -246,37 +249,38 @@ export class SendReceiptService { }, }); } - let receiptRecordId: string | undefined; - try { - const createdReceipt = await this.prisma.smsReceiptRecord.create({ - data: { - tenantId: message.tenantId, - batchTaskId: message.batchTaskId, - messageRecordId: message.id, - receiptKey, - channelId: logicalChannelId, - messageId: resolved.messageId, - gatewayMessageId: data.gatewayMessageId, - phoneNumber: data.phoneNumber?.trim() || message.phoneNumber, - sequenceId: data.sequenceId, - receiptStatus: data.receiptStatus, - rawStatus: data.rawStatus, - errorCode: data.errorCode, - errorMessage: data.errorMessage, - deliveredAt, - }, - }); - receiptRecordId = createdReceipt.id; - } catch (error) { - if (error instanceof Prisma.PrismaClientKnownRequestError && error.code === 'P2002') { - const duplicate = await this.prisma.smsReceiptRecord.findUnique({ - where: { receiptKey }, - include: { messageRecord: true }, + let receiptRecordId: string | undefined = existingReceipt?.id; + if (!existingReceipt) + try { + const createdReceipt = await this.prisma.smsReceiptRecord.create({ + data: { + tenantId: message.tenantId, + batchTaskId: message.batchTaskId, + messageRecordId: message.id, + receiptKey, + channelId: logicalChannelId, + messageId: resolved.messageId, + gatewayMessageId: data.gatewayMessageId, + phoneNumber: data.phoneNumber?.trim() || message.phoneNumber, + sequenceId: data.sequenceId, + receiptStatus: data.receiptStatus, + rawStatus: data.rawStatus, + errorCode: data.errorCode, + errorMessage: data.errorMessage, + deliveredAt, + }, }); - if (duplicate?.messageRecord) return duplicate.messageRecord; + receiptRecordId = createdReceipt.id; + } catch (error) { + if (error instanceof Prisma.PrismaClientKnownRequestError && error.code === 'P2002') { + const duplicate = await this.prisma.smsReceiptRecord.findUnique({ + where: { receiptKey }, + include: { messageRecord: true }, + }); + if (duplicate?.messageRecord) return duplicate.messageRecord; + } + throw error; } - throw error; - } const logicalReceipt = { ...data, channelId: logicalChannelId }; await this.facade.recordReceiptSegment(message, logicalReceipt, deliveredAt, resolved.submitRecordId); const receiptMode = @@ -300,8 +304,10 @@ export class SendReceiptService { if (!aggregate.terminal) { return this.prisma.smsMessageRecord.findUnique({ where: { id: message.id } }); } + if (message.status === 'timeout' && message.errorCode === 'RECEIPT_TIMEOUT') return message; const status = aggregate.status; const isCurrentAttempt = + (!message.submitId || message.submitId === resolved.submitId) && (!message.channelId || message.channelId === logicalChannelId) && (!message.gatewayMessageId || message.gatewayMessageId === data.gatewayMessageId || diff --git a/api/src/send-chain/send-retry.service.ts b/api/src/send-chain/send-retry.service.ts index 76bdd62..34f1d1d 100644 --- a/api/src/send-chain/send-retry.service.ts +++ b/api/src/send-chain/send-retry.service.ts @@ -1,16 +1,18 @@ -import { BadRequestException, HttpException, HttpStatus, Logger, NotFoundException } from '@nestjs/common'; +import { BadRequestException, HttpException, Logger, NotFoundException } from '@nestjs/common'; import { Prisma } from '@prisma/client'; -import { createHash } from 'node:crypto'; import { BillingService } from '../billing/billing.service'; -import { moneyToNumber } from '../common/money'; import type { OpenApiService } from '../open-api/open-api.service'; import { PrismaService } from '../prisma/prisma.service'; -import type { GatewaySubmitResultDto, GatewaySubmitSegmentResultDto, GatewayReceiptEventDto, GatewayUplinkEventDto, GatewayPendingDeliveryQueryDto, GatewayDownstreamSentDto, GatewayDownstreamAcknowledgedDto, GatewayDownstreamFailureType, GatewayControlDeliveryResult, GatewaySubmitDeadLetterDto, RequeueGatewaySubmitExceptionDto, GatewayDownstreamRecoveryStatusDto, TimeoutUnknownDto } from './send-chain.contracts'; -import { positiveInteger, normalizeReceiptStatus, normalizeCarrier, DEFAULT_DOWNSTREAM_RETRY_DELAY_MS, DEFAULT_DOWNSTREAM_RETRY_MAX_DELAY_MS, DEFAULT_DOWNSTREAM_MAX_RETRIES, DEFAULT_DOWNSTREAM_PENDING_TIMEOUT_HOURS, DEFAULT_RECEIPT_TIMEOUT_HOURS, DEFAULT_GATEWAY_SUBMIT_REQUEUE_STALE_MS, DEFAULT_DOWNSTREAM_MANUAL_REQUEUE_STALE_MS, DEFAULT_UPSTREAM_RECEIPT_INBOX_PROCESSING_STALE_MS, DEFAULT_UPSTREAM_RECEIPT_INBOX_MAX_ATTEMPTS, DEFAULT_UPSTREAM_RECEIPT_INBOX_MAX_AGE_HOURS, gatewaySubmitRequeueKey, isObjectRecord, asDateOrNull, downstreamRetryDelayMs, downstreamAckTimeoutMs, downstreamRetryBaseDelayMs, downstreamRetryMaxDelayMs, downstreamMaxRetries, downstreamPendingTimeoutHours, downstreamControlFailureMessage, normalizeSubmitStatus, downstreamDeliveryAttemptKey, hasRecoveryAuditStateChanged, normalizeRecoveryFailureCategory, aggregateReceiptSegmentState, isSameUpstreamEndpointIdentity, receiptEventKey } from './send-chain.helpers'; -import type { SendSubmissionService } from './send-submission.service'; +import type { GatewaySubmitDeadLetterDto, RequeueGatewaySubmitExceptionDto } from './send-chain.contracts'; +import { + DEFAULT_GATEWAY_SUBMIT_REQUEUE_STALE_MS, + gatewaySubmitRequeueKey, + isObjectRecord, + normalizeCarrier, + positiveInteger, +} from './send-chain.helpers'; import type { SendCompletionCallbacks, SendCompletionFacade } from './send-completion.service'; - /** * R10 retry implementation. * Cross-domain calls return through the stable SendChainService seam to preserve locking and test observability. @@ -87,11 +89,12 @@ export class SendRetryService { const message = deadLetter.messageId ? await this.prisma.smsMessageRecord.findUnique({ where: { messageId: deadLetter.messageId } }) : null; - if (message && ( - message.submitStatus === 'accepted' - || ['submitted', 'delivered', 'unknown'].includes(message.status) - || ['delivered', 'unknown'].includes(message.receiptStatus ?? '') - )) { + if ( + message && + (message.submitStatus === 'accepted' || + ['submitted', 'delivered', 'unknown'].includes(message.status) || + ['delivered', 'unknown'].includes(message.receiptStatus ?? '')) + ) { throw new BadRequestException('该短信已有成功或不确定的上游结果，为避免重复发送，禁止重新入队'); } const commandChannelId = String(deadLetter.commandPayload.channelId ?? deadLetter.channelId ?? '').trim(); @@ -118,7 +121,10 @@ export class SendRetryService { const requeueKey = gatewaySubmitRequeueKey(deadLetter.id, deadLetter.manualRetryCount + 1); let retryStreamMessageId: string; try { - const publishedStreamMessageId = await this.facade.publishGatewaySubmitCommand(deadLetter.commandPayload, requeueKey); + const publishedStreamMessageId = await this.facade.publishGatewaySubmitCommand( + deadLetter.commandPayload, + requeueKey, + ); if (!publishedStreamMessageId) { throw new Error('Gateway提交异常重新入队未返回Stream消息编号'); } @@ -208,10 +214,10 @@ export class SendRetryService { } async recoverStaleGatewaySubmitRequeues(now = new Date()) { - const staleCutoff = new Date(now.getTime() - positiveInteger( - process.env.GATEWAY_SUBMIT_REQUEUE_STALE_MS, - DEFAULT_GATEWAY_SUBMIT_REQUEUE_STALE_MS, - )); + const staleCutoff = new Date( + now.getTime() - + positiveInteger(process.env.GATEWAY_SUBMIT_REQUEUE_STALE_MS, DEFAULT_GATEWAY_SUBMIT_REQUEUE_STALE_MS), + ); const stale = await this.prisma.gatewaySubmitDeadLetter.findMany({ where: { status: 'requeueing', updatedAt: { lt: staleCutoff } }, orderBy: { updatedAt: 'asc' }, @@ -235,7 +241,10 @@ export class SendRetryService { if (claimed.count !== 1) continue; try { const requeueKey = gatewaySubmitRequeueKey(deadLetter.id, deadLetter.manualRetryCount + 1); - const retryStreamMessageId = await this.facade.publishGatewaySubmitCommand(deadLetter.commandPayload, requeueKey); + const retryStreamMessageId = await this.facade.publishGatewaySubmitCommand( + deadLetter.commandPayload, + requeueKey, + ); if (!retryStreamMessageId) throw new Error('Gateway提交异常恢复未返回Stream消息编号'); const finalized = await this.prisma.gatewaySubmitDeadLetter.updateMany({ where: { id: deadLetter.id, status: 'requeue_recovering' }, @@ -264,7 +273,9 @@ export class SendRetryService { where: { id: deadLetter.id, status: 'requeue_recovering' }, data: { status: 'requeueing' }, }); - this.logger.error(`Gateway submit requeue recovery failed for ${deadLetter.id}: ${error instanceof Error ? error.message : String(error)}`); + this.logger.error( + `Gateway submit requeue recovery failed for ${deadLetter.id}: ${error instanceof Error ? error.message : String(error)}`, + ); } } return { recovered, failed }; @@ -300,100 +311,123 @@ export class SendRetryService { const attemptedChannelIds = attempts.map((attempt) => attempt.channelId); let sourceAttempt = sourceSubmitRecordId ? attempts.find((attempt) => attempt.id === sourceSubmitRecordId) - : attempts.find((attempt) => attempt.submitId === message.submitId) ?? attempts[attempts.length - 1]; + : (attempts.find((attempt) => attempt.submitId === message.submitId) ?? attempts[attempts.length - 1]); if (!sourceAttempt && sourceSubmitRecordId) { - sourceAttempt = await this.prisma.smsSubmitRecord.findUnique({ - where: { id: sourceSubmitRecordId }, - }) ?? undefined; + sourceAttempt = + (await this.prisma.smsSubmitRecord.findUnique({ + where: { id: sourceSubmitRecordId }, + })) ?? undefined; } if (!sourceAttempt || sourceAttempt.messageRecordId !== message.id) { - this.logger.error(`sms_retry_route_failed ${JSON.stringify({ - messageId: message.messageId, - messageRecordId: message.id, - reason, - sourceSubmitRecordId, - sourceMessageRecordId: sourceAttempt?.messageRecordId, - error: sourceAttempt ? 'retry_source_submit_record_mismatch' : 'retry_source_submit_record_missing', - })}`); + this.logger.error( + `sms_retry_route_failed ${JSON.stringify({ + messageId: message.messageId, + messageRecordId: message.id, + reason, + sourceSubmitRecordId, + sourceMessageRecordId: sourceAttempt?.messageRecordId, + error: sourceAttempt ? 'retry_source_submit_record_mismatch' : 'retry_source_submit_record_missing', + })}`, + ); return null; } const existingRetry = await this.prisma.smsSubmitRecord.findUnique({ where: { retryOfSubmitRecordId: sourceAttempt.id }, }); if (existingRetry) { - this.logger.warn(`sms_retry_claim_reused ${JSON.stringify({ - messageId: message.messageId, - messageRecordId: message.id, - retryOfSubmitRecordId: sourceAttempt.id, - submitId: existingRetry.submitId, - channelId: existingRetry.channelId, - })}`); + this.logger.warn( + `sms_retry_claim_reused ${JSON.stringify({ + messageId: message.messageId, + messageRecordId: message.id, + retryOfSubmitRecordId: sourceAttempt.id, + submitId: existingRetry.submitId, + channelId: existingRetry.channelId, + })}`, + ); return this.prisma.smsMessageRecord.findUnique({ where: { id: message.id } }); } const ageMinutes = (Date.now() - new Date(message.queuedAt ?? Date.now()).getTime()) / 60_000; - this.logger.log(`sms_retry_route_started ${JSON.stringify({ - messageId: message.messageId, - messageRecordId: message.id, - reason, - attemptedChannelIds, - ageMinutes: Math.round(ageMinutes * 100) / 100, - })}`); - if (ageMinutes >= 72 * 60) { - this.logger.warn(`sms_retry_route_skipped ${JSON.stringify({ + this.logger.log( + `sms_retry_route_started ${JSON.stringify({ messageId: message.messageId, messageRecordId: message.id, - reason: 'maximum_message_age_exceeded', + reason, + attemptedChannelIds, ageMinutes: Math.round(ageMinutes * 100) / 100, - })}`); + })}`, + ); + if (ageMinutes >= 72 * 60) { + this.logger.warn( + `sms_retry_route_skipped ${JSON.stringify({ + messageId: message.messageId, + messageRecordId: message.id, + reason: 'maximum_message_age_exceeded', + ageMinutes: Math.round(ageMinutes * 100) / 100, + })}`, + ); return null; } const retryCarrier = message.carrier ? normalizeCarrier(message.carrier) : await this.facade.identifyCarrier(message.phoneNumber); - const route = await this.facade.findApplicationRoute(message.tenantId, message.applicationId ?? undefined, retryCarrier); - const retryTimeLimitMinutes = Math.min(route.group.retryTimeLimitMinutes ?? route.group.retryTimeLimitHours * 60, 72 * 60); + const route = await this.facade.findApplicationRoute( + message.tenantId, + message.applicationId ?? undefined, + retryCarrier, + ); + const retryTimeLimitMinutes = Math.min( + route.group.retryTimeLimitMinutes ?? route.group.retryTimeLimitHours * 60, + 72 * 60, + ); if (!route.group.retryEnabled || ageMinutes >= retryTimeLimitMinutes) { - this.logger.warn(`sms_retry_route_skipped ${JSON.stringify({ - messageId: message.messageId, - messageRecordId: message.id, - groupId: route.groupId, - reason: !route.group.retryEnabled ? 'group_retry_disabled' : 'group_retry_time_limit_exceeded', - ageMinutes: Math.round(ageMinutes * 100) / 100, - retryTimeLimitMinutes, - })}`); + this.logger.warn( + `sms_retry_route_skipped ${JSON.stringify({ + messageId: message.messageId, + messageRecordId: message.id, + groupId: route.groupId, + reason: !route.group.retryEnabled ? 'group_retry_disabled' : 'group_retry_time_limit_exceeded', + ageMinutes: Math.round(ageMinutes * 100) / 100, + retryTimeLimitMinutes, + })}`, + ); return null; } try { - const routed = await this.facade.selectChannelForMessage({ ...message, carrier: retryCarrier }, { - forceNational: true, - excludeChannelIds: attemptedChannelIds, - }); + const routed = await this.facade.selectChannelForMessage( + { ...message, carrier: retryCarrier }, + { + forceNational: true, + excludeChannelIds: attemptedChannelIds, + }, + ); await this.prisma.smsMessageRecord.update({ where: { id: message.id }, data: { errorMessage: reason }, }); - const retried = await this.facade.submitMessageToGateway( - message, - routed, - attempts.length, - sourceAttempt.id, + const retried = await this.facade.submitMessageToGateway(message, routed, attempts.length, sourceAttempt.id); + this.logger.log( + `sms_retry_route_selected ${JSON.stringify({ + messageId: message.messageId, + messageRecordId: message.id, + groupId: routed.groupId, + channelId: routed.channel.id, + attempt: attempts.length, + })}`, ); - this.logger.log(`sms_retry_route_selected ${JSON.stringify({ - messageId: message.messageId, - messageRecordId: message.id, - groupId: routed.groupId, - channelId: routed.channel.id, - attempt: attempts.length, - })}`); return retried; } catch (error) { - this.logger.error(`sms_retry_route_failed ${JSON.stringify({ - messageId: message.messageId, - messageRecordId: message.id, - reason, - attemptedChannelIds, - error: error instanceof Error ? error.message : String(error), - })}`); + // Infrastructure failures and the completion routing probe must abort the + // transaction; treating them as "no retry route" would finalize a failure. + if (!(error instanceof HttpException) || error.getStatus() >= 500) throw error; + this.logger.error( + `sms_retry_route_failed ${JSON.stringify({ + messageId: message.messageId, + messageRecordId: message.id, + reason, + attemptedChannelIds, + error: error instanceof Error ? error.message : String(error), + })}`, + ); return null; } } diff --git a/api/src/send-chain/send-submission.service.ts b/api/src/send-chain/send-submission.service.ts index 4c216d1..b539815 100644 --- a/api/src/send-chain/send-submission.service.ts +++ b/api/src/send-chain/send-submission.service.ts @@ -368,7 +368,7 @@ export class SendSubmissionService { template?: { signature?: { id?: string | null } | null } | null; signature?: { id?: string | null } | null; }, - options: { forceNational?: boolean; excludeChannelIds?: string[] } = {}, + options: { forceNational?: boolean; excludeChannelIds?: string[]; previewOnly?: boolean } = {}, ): Promise { return this.gatewaySubmit.selectChannelForMessage(message, options); } diff --git a/api/src/send-chain/send-timeout.service.ts b/api/src/send-chain/send-timeout.service.ts index 9b40f00..58ad965 100644 --- a/api/src/send-chain/send-timeout.service.ts +++ b/api/src/send-chain/send-timeout.service.ts @@ -1,16 +1,12 @@ -import { BadRequestException, HttpException, HttpStatus, Logger, NotFoundException } from '@nestjs/common'; -import { Prisma } from '@prisma/client'; -import { createHash } from 'node:crypto'; +import { Logger } from '@nestjs/common'; import { BillingService } from '../billing/billing.service'; -import { moneyToNumber } from '../common/money'; import type { OpenApiService } from '../open-api/open-api.service'; import { PrismaService } from '../prisma/prisma.service'; -import type { GatewaySubmitResultDto, GatewaySubmitSegmentResultDto, GatewayReceiptEventDto, GatewayUplinkEventDto, GatewayPendingDeliveryQueryDto, GatewayDownstreamSentDto, GatewayDownstreamAcknowledgedDto, GatewayDownstreamFailureType, GatewayControlDeliveryResult, GatewaySubmitDeadLetterDto, RequeueGatewaySubmitExceptionDto, GatewayDownstreamRecoveryStatusDto, TimeoutUnknownDto } from './send-chain.contracts'; -import { positiveInteger, normalizeReceiptStatus, normalizeCarrier, DEFAULT_DOWNSTREAM_RETRY_DELAY_MS, DEFAULT_DOWNSTREAM_RETRY_MAX_DELAY_MS, DEFAULT_DOWNSTREAM_MAX_RETRIES, DEFAULT_DOWNSTREAM_PENDING_TIMEOUT_HOURS, DEFAULT_RECEIPT_TIMEOUT_HOURS, DEFAULT_GATEWAY_SUBMIT_REQUEUE_STALE_MS, DEFAULT_DOWNSTREAM_MANUAL_REQUEUE_STALE_MS, DEFAULT_UPSTREAM_RECEIPT_INBOX_PROCESSING_STALE_MS, DEFAULT_UPSTREAM_RECEIPT_INBOX_MAX_ATTEMPTS, DEFAULT_UPSTREAM_RECEIPT_INBOX_MAX_AGE_HOURS, gatewaySubmitRequeueKey, isObjectRecord, asDateOrNull, downstreamRetryDelayMs, downstreamAckTimeoutMs, downstreamRetryBaseDelayMs, downstreamRetryMaxDelayMs, downstreamMaxRetries, downstreamPendingTimeoutHours, downstreamControlFailureMessage, normalizeSubmitStatus, downstreamDeliveryAttemptKey, hasRecoveryAuditStateChanged, normalizeRecoveryFailureCategory, aggregateReceiptSegmentState, isSameUpstreamEndpointIdentity, receiptEventKey } from './send-chain.helpers'; -import type { SendSubmissionService } from './send-submission.service'; -import type { SendCompletionCallbacks, SendCompletionFacade } from './send-completion.service'; +import { completionContext } from './completion-context'; import { queueFinalReceiptDeliveries } from './downstream-receipt-targets'; - +import type { TimeoutUnknownDto } from './send-chain.contracts'; +import { DEFAULT_RECEIPT_TIMEOUT_HOURS, downstreamPendingTimeoutHours, positiveInteger } from './send-chain.helpers'; +import type { SendCompletionCallbacks, SendCompletionFacade } from './send-completion.service'; /** * R10 timeout implementation. @@ -29,10 +25,12 @@ export class SendTimeoutService { ) {} async markUnknownTimeout(data: TimeoutUnknownDto) { - const olderThanHours = data.olderThanHours ?? positiveInteger(process.env.SMS_RECEIPT_TIMEOUT_HOURS, DEFAULT_RECEIPT_TIMEOUT_HOURS); + const olderThanHours = + data.olderThanHours ?? positiveInteger(process.env.SMS_RECEIPT_TIMEOUT_HOURS, DEFAULT_RECEIPT_TIMEOUT_HOURS); const cutoff = new Date(Date.now() - olderThanHours * 60 * 60 * 1000); const candidates = await this.prisma.smsMessageRecord.findMany({ where: { + id: completionContext.getStore()?.messageRecordId, tenantId: { not: null }, OR: [ { status: { in: ['submitted', 'unknown'] }, submittedAt: { lte: cutoff } }, @@ -79,7 +77,10 @@ export class SendTimeoutService { // Refund uses the platform-message idempotency key. Re-running it for a // timeout whose downstream outbox was not fully queued also recovers a // crash between the state transition and the original refund call. - await this.facade.refundMessage(candidate as typeof candidate & { tenantId: string }, `${olderThanHours}小时未收到明确回执，自动超时退款`); + await this.facade.refundMessage( + candidate as typeof candidate & { tenantId: string }, + `${olderThanHours}小时未收到明确回执，自动超时退款`, + ); const queued = await queueFinalReceiptDeliveries( this.prisma, (request) => this.facade.queueAndTryDownstreamDelivery(request), @@ -117,10 +118,7 @@ export class SendTimeoutService { const expired = await this.prisma.cmppDownstreamDelivery.findMany({ where: { status: 'pending', - OR: [ - { lastRetriedAt: null, createdAt: { lte: cutoff } }, - { lastRetriedAt: { lte: cutoff } }, - ], + OR: [{ lastRetriedAt: null, createdAt: { lte: cutoff } }, { lastRetriedAt: { lte: cutoff } }], }, select: { id: true }, take: 500, @@ -139,16 +137,21 @@ export class SendTimeoutService { if (this.receiptTimeoutScanRunning) return; this.receiptTimeoutScanRunning = true; try { - const [receiptResult, downstreamResult, requeueRecoveryResult, downstreamManualRecoveryResult] = await Promise.all([ - this.facade.markUnknownTimeout({}), - this.facade.markExpiredDownstreamDeliveries(), - this.facade.recoverStaleGatewaySubmitRequeues(), - this.facade.recoverStaleDownstreamManualRequeues(), - ]); - if (receiptResult.timeout > 0) this.logger.log(`Marked ${receiptResult.timeout} messages as receipt timeout and refunded charged messages`); - if (downstreamResult.failed > 0) this.logger.log(`Terminated ${downstreamResult.failed} expired downstream deliveries`); - if (requeueRecoveryResult.recovered > 0) this.logger.log(`Recovered ${requeueRecoveryResult.recovered} stale Gateway submit requeues`); - if (downstreamManualRecoveryResult.recovered > 0) this.logger.log(`Recovered ${downstreamManualRecoveryResult.recovered} stale downstream manual requeues`); + const [receiptResult, downstreamResult, requeueRecoveryResult, downstreamManualRecoveryResult] = + await Promise.all([ + this.facade.markUnknownTimeout({}), + this.facade.markExpiredDownstreamDeliveries(), + this.facade.recoverStaleGatewaySubmitRequeues(), + this.facade.recoverStaleDownstreamManualRequeues(), + ]); + if (receiptResult.timeout > 0) + this.logger.log(`Marked ${receiptResult.timeout} messages as receipt timeout and refunded charged messages`); + if (downstreamResult.failed > 0) + this.logger.log(`Terminated ${downstreamResult.failed} expired downstream deliveries`); + if (requeueRecoveryResult.recovered > 0) + this.logger.log(`Recovered ${requeueRecoveryResult.recovered} stale Gateway submit requeues`); + if (downstreamManualRecoveryResult.recovered > 0) + this.logger.log(`Recovered ${downstreamManualRecoveryResult.recovered} stale downstream manual requeues`); } catch (error) { this.logger.error('Receipt timeout scan failed', error instanceof Error ? error.stack : String(error)); } finally { diff --git a/docs/first-version-development-requirements.md b/docs/first-version-development-requirements.md index c831bb1..b1a255a 100644 --- a/docs/first-version-development-requirements.md +++ b/docs/first-version-development-requirements.md @@ -2321,3 +2321,21 @@ Webhook需在当前受支持Node运行时通过真实HTTPS投递；SSRF校验后 ## 2026-09-16 产品名称统一 产品正式名称为“聆界短信平台”，页面标题、API文档品牌名称、导出表格作者和充值回执替代文本统一。产品3.0、工程3.0.0不变；CMPP协议名称及技术标识保持兼容。名称调整仅涉及展示文案和导出作者元数据，不改变布局、业务流程、协议、接口地址或工程版本；版本元数据属于独立变更。 + +## 2026-09-16 长短信回执并发处理可靠性补充（待实施） + +本节细化既有短信幂等、补发与计费一致性要求，不改变费率、分段汇总、补发条件或客户接口协议。不同分段回执同时到达时，同一次有效发送只能产生一份已提交的收尾决策；同一次失败最多创建一个后继补发。合法分段历史完整保留，HTTP按业务消息、CMPP按客户原始分段生成耐久通知，网络失败沿用同一投递身份重试。 + +处理进程退出后必须接续，不因“已认领”漏处理；旧持有者和旧尝试不得覆盖新的有效结果，不重复扣费、退款或补发。unknown及不完整分段继续等待既有条件，不能被永久完成标记阻止后续处理。历史数据不得因升级自动重发或重退。 + +实施设计和适用边界见 [发送链路第10节](phase-4-send-pipeline-redesign.md)，验收为 [TC-RC-20260916-01～12](system-functional-test-cases.md)。本次仅方案，不代表线上修复或新增发送授权。 + +## 2026-09-16 五项联合整改（实施中） + +1. 实施发送链路方案第10节，长短信回执按尝试耐久协调，保证并发、故障恢复、补发、账务和通知一致。 +2. 运营通道测试不进行签名/引流内容检测与报备校验，正文按输入交给指定通道；仍校验权限、号码/长度、通道启用/连接和协议参数。客户正常发送保持原规则。 +3. 双端签名工具增加短信参数独立输入、真实发送与应用层HTTP请求/响应展示，详见HTTP专项设计2026-09-16补充。 +4. 系统监控告警持久保留，仅手动清除，已读不清除、恢复不清除，新触发周期独立。 +5. 监控刷新失败显示真实上一快照和过期提示，超时/切换请求取消，恢复后刷新；不同查询范围隔离。 + +授权交付：代码、定向/全量测试、精确提交、推送、测试环境标准发布及隔离模拟短信验收；不操作预生产，不向真实运营商发送。 diff --git a/docs/http-api-assessment-20260910.md b/docs/http-api-assessment-20260910.md index acb8cca..9f869e1 100644 --- a/docs/http-api-assessment-20260910.md +++ b/docs/http-api-assessment-20260910.md @@ -336,3 +336,7 @@ HTTP-FULL-B02：去除URL IPv6方括号后区分IP字面量与DNS，DNS失败转 ### HTTP-0915-B02 HTTP发送误依赖CMPP开关 真实测试企业HTTP enabled/sendEnabled=true、CMPP interfaceEnabled=false时，合法模板请求422，未创建短信。根因SendBatchEntryService.validateSendResources无条件读取interfaceEnabled，与需求HTTP客户接口第一版的独立开通矛盾。仅由内部HTTP_REQUEST_CONTEXT符号确认的HTTP请求传递httpRequest校验选项，再读取真实httpConfig.enabled/sendEnabled；普通客户端及CMPP路径保留原检查，企业认证、租户、应用状态及模板校验不变。没有数据库迁移和计费规则变化。须真实复验CMPP关闭HTTP送达、HTTP关闭拒绝及原CMPP拒绝回归。 + +## 2026-09-16 签名工具增加真实发送调试（实施中） + +双端共用页面保留原始正文签名计算，增加短信调试区：AccessKey、AccessSecret、mobile、content、clientMessageId、Idempotency-Key分别输入。发送目标固定当前站点的/api/openapi/v1/sms/messages，按公开协议生成秒时间戳和nonce，使用同一份JSON字节计算签名并发送。密钥只在页面内存，不存储/上传AccessSecret；展示实际应用层请求方法、路径、请求头、正文和HTTP状态、可读取响应头、原始响应正文，不伪称浏览器隐藏头或HTTP版本为抓包结果。一次点击一次请求，发送期间禁用，不自动重试；超时提示结果未知，保留同一业务幂等键用于核对。新业务内容编辑后生成新幂等键，重试同一内容保持业务键。由原OpenAPI鉴权、应用权限、计费、风控处理，无绕过接口。覆盖失败响应、网络失败、超时、长短信、中文和特殊字符、双端三尺寸与真实发送验收。 diff --git a/docs/phase-4-send-pipeline-redesign.md b/docs/phase-4-send-pipeline-redesign.md index 6a01449..7c5598e 100644 --- a/docs/phase-4-send-pipeline-redesign.md +++ b/docs/phase-4-send-pipeline-redesign.md @@ -195,3 +195,154 @@ GatewaySubmitOutbox - 修复后正价smoke为9/9受理，账单`9×325=2925`；20 TPS为199/199、账单64675；30 TPS为299/299、账单97175；50 TPS为499/499、账单162175。四个成功窗口共1006条，账单1006笔、单价均325、合计326950。 - 50 TPS档499条非补发首次供应商Submit覆盖9.930秒，即`50.25条/秒`；相对同日改造前Outbox阶段的`33.07条/秒`提高约52%。该档客户端P50/P95/P99为35/76/135ms；首提499条唯一，全部尝试及Outbox各542条唯一，补发均关联原提交。 - 压测结束Inbox、Submit Outbox和两条Redis Stream全部排空，数据库无重复Submit ID、无等待锁、无idle in transaction；回调池`max=12,total=1,idle=1,waiting=0`。隔离应用单价已恢复0，三条临时运营商规则已删除，真实账务事实保留审计。100/200/300/500未继续执行，当前验收结论限定为50 TPS档通过。 + +## 10. 长短信分段回执并发整改方案（2026-09-16，待实施） + +### 10.1 目的、证据和适用范围 + +业务解释：一条长短信有多段回执，多个处理流程同时发现整条短信已经有结果，重复安排通知或补发。现有数据库唯一键能阻止重复创建，但不能避免此前重复查询、选路和事务开销。整改目标是“先认领处理资格，再做业务动作；中途退出后可以安全接续”。 + +依据：[预生产只读诊断](cpu-diagnosis-20260916.md)。2026-09-16 09:20～09:40，12682次下游回执唯一键冲突关联6335条长短信；1862次补发唯一键冲突关联1859个长短信原始提交。下游冲突关联消息的13373条Inbox均matched且attemptCount=1；补发6组抽样显示不同分段失败回执同时到达并争抢创建补发。核心回执、补发、提交文件在d13ca07至cbc4a03之间无变化。 + +缺陷有两层：单个receiptKey去重不能覆盖不同分段；补发“先查询、后选路、再创建”的流程存在并发窗口。CPU峰值包含I/O等待，缺少业务进程历史CPU与慢SQL，不能承诺消除此缺陷即可消除全部CPU高峰。 + +本节是现有发送链路的专项补充，作为本次并发整改的设计依据；细化第4节结果处理及第7节不重不漏验收要求，不替代路由、计费、长短信协议或历史容量结论。不扩展到批次编号冲突、通道容量改造或历史短信重发。用户本轮仅要求方案文档，以下模型、状态、参数均是拟实施设计。 + +### 10.2 必须保持的业务规则 + +1. 原始回执逐事件持久化，同一事件幂等；不同分段的合法回执不能当成重复而丢弃。 +2. 成功依现有通道回执模式汇总：per_segment须齐段成功，message_level保持既有整条成功语义。不得为降低冲突把第一段成功视为整条成功。 +3. 同一次有效发送的失败只能触发一份补发决策；是否允许补发仍检查当前应用、通道组、报备、风控、时间上限等既有规则。unknown不触发补发，72小时及组内时限规则保持。 +4. 旧尝试的迟到回执保留审计，不覆盖后续有效尝试的最终成功，不再次触发旧尝试补发或退款。矛盾回执按既有规则记录异常，不采用“谁抢到锁谁决定事实”。 +5. 补发不是消息最终失败；仍有有效补发时不得提前生成业务最终失败通知或最终退款。 +6. HTTP回调保持业务消息维度；CMPP回执按客户原始分段及Registered_Delivery要求生成。内部上游分段与客户分段不能混用。 +7. 现有提交、回执、补发、投递和账务唯一约束全部保留；计费单位、价格、扣退费条件及租户权限不变。 +8. 对外投递仍是可重试的至少一次交付，不能承诺网络侧绝对只收到一次；整改保证同一逻辑通知仅创建一份耐久投递事实，重试沿用其身份。 + +### 10.3 影响模块与最小实施边界 + +| 所有者 | 拟修改内容 | +|---|---| +| send-receipt.service.ts、send-chain.helpers.ts | 保存分段后提交汇总工作；按当前持久化事实重新汇总和校验有效尝试，移出重复终态副作用 | +| send-retry.service.ts、send-gateway-submit.service.ts | 统一补发处理资格，认领后选路，事务内验证资格并创建Submit与GatewaySubmitOutbox | +| send-gateway-result.service.ts、send-timeout.service.ts、send-downstream-delivery.service.ts | 清查提交失败、超时、平台失败等旁路，接入同一业务消息终态协调，不留下与回执处理竞争的独立写入口 | +| send-accounting.service.ts、send-completion.service.ts | 核实稳定账务键和事务接口；重复恢复不重复扣退，不能用已完成标记替代实际账务事实 | +| downstream-receipt-targets.ts、HTTP Webhook生产端 | 保持目标分段契约，幂等落库后再独立投递，重复命中不直接再次发送 | +| Prisma、回调进程恢复扫描、定向与集成测试 | 增量状态模型、认领/恢复索引、有限批次消费与故障注入 | + +预期主要为API发送链路和数据库迁移；不新增页面、不改变客户HTTP参数、签名算法、错误码和CMPP协议。Gateway命令契约预期不变，若实现必须变更，先补契约及Go测试，不能隐式扩大范围。需检查全部调用者而非只在回执方法外套进程内锁。 + +### 10.4 拟增数据模型与认领规则 + +采用PostgreSQL耐久工作记录和短事务条件更新，避免仅靠进程内锁或Redis短期锁。拟新增 `SmsAttemptCompletionWork`，每个sourceSubmitRecordId唯一；该名称及字段在实施迁移前与现有模型再次核对。无供应商Submit的业务拒绝使用独立的稳定事件键，不伪造Submit ID。 + +| 字段 | 含义及约束 | +|---|---| +| id、workKey | 主键及唯一工作键；有Submit时固定为attempt: | +| tenantId、messageRecordId、sourceSubmitRecordId | 与真实记录归属一致；不信任外部回调传入的租户或消息归属 | +| revision、processedRevision | 收到新的有效事实递增revision；完成时只确认已处理版本，避免并发新回执被覆盖 | +| state | pending、processing、retry_wait、idle、needs_review；idle表示当前事实已处理，不等于永不再接收新事实 | +| leaseOwner、leaseUntil、fenceVersion | 租约及递增认领版本；旧持有者不能提交后续业务动作 | +| decision、retrySubmitRecordId | 本次已提交决策及已有补发引用；补发一旦创建，不因恢复或后续分段重新选路创建 | +| attempts、nextAttemptAt、lastError、createdAt、updatedAt | 恢复、退避和诊断信息；错误脱敏，不复制短信正文或凭据 | + +索引：唯一workKey、非空sourceSubmitRecordId唯一；按state/nextAttemptAt及state/leaseUntil支持有界扫描；messageRecordId支持对账。不按手机号或高基数业务ID创建监控标签。 + +认领以数据库时间为准：pending、到期retry_wait或租约过期processing可由条件更新取得；认领成功递增fenceVersion。建议初始租约60秒、每20秒续租、每次扫描不超过32条，作为测试起点而非容量结论。连接池预算、扫描间隔及退避上限须通过隔离环境实测定稿。 + +每次持久化副作用都在短事务中锁定工作记录并核对认领版本、租约及当前有效尝试，同时锁定对应业务消息，使不同尝试/超时入口也不能交错覆盖最终状态。锁顺序统一为工作记录→业务消息→既有账务锁；禁止反向调用形成死锁。不得持有事务等待限速、网络或Redis。 + +### 10.5 事实保存、业务收尾和补发流程 + +**A. 接收与汇总** + +1. 维持耐久Inbox接收。处理单个回执时，将回执事实、分段审计和工作记录唤醒放入同一短事务；唯一事件已存在时，也须核实存在对应工作，不直接忽略未完成收尾。 +2. Inbox的matched表示关联事实已可靠保存，业务收尾由工作记录跟踪；两者状态不能混为一个“完成”。在事务提交前退出，由Inbox恢复；提交后退出，由工作扫描恢复。 +3. 工作消费者认领后，重新读取当前Submit、所有有效分段及业务最终状态。未齐段成功则结束本轮为idle，保留processedRevision；后续新事实使其再次pending。unknown也不能被永久锁死，按既有状态机处理后续明确结果或超时。 +4. 确认最终结果时，事务内重新核验revision、当前尝试和认领版本。有新事实或尝试已变更，重新汇总或记历史，不使用认领前的旧message对象直接更新。 + +**B. 允许补发** + +1. Submit失败入口与回执失败入口都唤醒同一sourceSubmitRecordId的工作；取得资格后才进行选路等高成本操作。 +2. 优先查询已有retryOfSubmitRecordId对应的Submit/Outbox。存在则复用，不再次选路、扣费或创建；不存在才评估业务规则并选择候选。 +3. 选路和限速等待不持数据库锁，长等待续租；失去租约即停止。最终短事务核验资格、当前尝试及必要的实时启用/报备条件，原子提交补发Submit、消息当前尝试更新、GatewaySubmitOutbox和工作决策。 +4. 事务失败全部回滚；成功后由原有Outbox发布器发送，不由认领流程直接调用供应商。不可把“处理中”当成“补发已成功创建”，也不可因此提前退款。 +5. 临时数据库/连接错误进入retry_wait，不能误判为“无通道、最终失败”。确定不允许补发后，才走既有最终失败规则。 + +**C. 最终业务结果与客户通知** + +最终消息状态、必要账务变化和全部应生成的耐久通知事实，应在同一短事务中提交。需让相关服务接受事务客户端；HTTP使用既有HttpWebhookEvent/Delivery模型，CMPP使用CmppDownstreamDelivery，不在事务内向客户发网络请求。 + +如实际账务接口无法共用事务，实施必须先补充稳定业务键的耐久分步恢复设计并验证全部崩溃点，不能先标工作完成再异步裸调用退款/通知。整条业务完成与单次发送尝试结束须分开：已安排补发只结束旧尝试，不给客户发最终失败。 + +正常幂等命中使用针对已知唯一键的无异常插入（例如参数化INSERT ON CONFLICT ... DO NOTHING RETURNING）并回读核实归属和内容；不能把所有数据库错误当重复成功。补发/回执唯一约束仍是最后防线，非预期冲突保留告警。 + +### 10.6 故障恢复与历史兼容 + +| 中断点 | 恢复预期 | +|---|---| +| 回执事实事务提交前 | 事务回滚，Inbox按既有机制再处理 | +| 事实已保存、工作尚未被领取 | 耐久pending扫描继续；不得依赖setImmediate一定执行 | +| 认领后、选路中进程退出 | 租约到期由新消费者接续，旧版本写入被fence拒绝 | +| Submit/Outbox事务提交结果不确定 | 按原始提交唯一键回读；已有记录继续原Outbox，不创建新Submit | +| 最终状态/账务/通知事务中退出 | 全部回滚或全部提交；恢复查稳定业务键，账务不重复 | +| 通知已入库、网络投递失败 | 原有投递记录重试；不重新运行短信补发决策 | +| 消费者持续失败 | 有限指数退避，达到配置上限进入needs_review并告警；不得伪造业务成功或静默丢弃 | +| 同时有新分段或迟到事实 | revision差异驱动再次汇总；旧尝试只留审计，已完成副作用不重复 | + +迁移只增表/索引，不删除历史回执、补发、账单或唯一约束。启用前暂停受影响消费者并完成现有任务排空，不能让旧版直接处理与新版协调器长期并行。 + +切换时建立准确切换时间及待处理清单：未完成Inbox、正在处理回执以及尚未完成收尾的业务记录进入受控衔接；有最终通知/账务/补发事实的历史记录以事实作为已完成依据。不能仅因新工作表为空扫描全部历史短信安排补发。历史异常修复另行设计、授权和审计。 + +上线前必须验证回执先于Submit结果、历史缺失Submit关联、无租户通道测试短信和平台业务拒绝等路径。无法可靠关联的记录保留待匹配/人工排查，不推测来源、不误收尾。 + +### 10.7 观测、性能与验收证据 + +增加低基数指标：工作认领成功/竞争未获得、租约接管、旧版本拒绝、恢复次数、needs_review数量、最老未完成时长、最终动作数量。重复命中记录计数和必要采样，不让正常幂等持续生成ERROR堆栈；真实失败仍明确告警。 + +业务验证必须对账：每个源Submit最多一个有效后继补发；每个客户目标最多一份逻辑通知；每个账务业务键最多一笔；最终状态与分段事实/最新有效尝试一致；Inbox、工作表、Submit Outbox、Redis和客户投递都有可解释的完成或等待原因。 + +性能对比使用同环境、同版本以外条件、同长短信比例、同成功失败分布和同负载，分别记录入口、首次Submit、总Submit、最终送达、分段回执速率及窗口。记录CPU用户态/内核态/I/O wait、数据库写入与锁等待、队列积压及回执完成延迟。新增进程CPU/SQL采样需低开销评估，不擅自在线开启全量SQL日志。 + +确定性并发用例要求不再出现上述两类预期竞争导致的23505异常，正常情况下每次有效尝试仅一份已提交决策；故障恢复可重复计算，但不能重复业务副作用。性能无倒退且队列按批准的排空时限清空；执行负载前固定对比基线和延迟阈值，不事后选窗口宣称提升。不得承诺未经测量的CPU降幅或TPS。 + +### 10.8 实施顺序、成本和发布恢复 + +1. 补定向失败用例复现两段同时成功/失败；盘点所有终态和补发调用路径及账务事务能力，形成事务边界清单。 +2. 实施增量模型、原子认领、revision/fence和扫描恢复；先验证多进程争抢及旧持有者失效。 +3. 接入回执、提交失败、超时及平台失败路径；事务化Submit/Outbox与最终通知/账务。保持外部协议和业务规则。 +4. 执行定向测试、API全量回归、类型检查、构建和现有质量门禁；真实PostgreSQL并发及迁移测试必须通过。Gateway有修改再执行Go测试及vet。 +5. 在另行明确的隔离模拟范围内，以真实API、PostgreSQL、Redis、Gateway和Webhook接收端验收，包含非零计费、重启故障、回执乱序及排空对账；页面检查发送详情和批次最终状态。mock只能证明隔离逻辑。 +6. 真实回归通过后，按明确提交/推送/目标环境授权交付；部署使用标准release流程和独立恢复资产。测试与预生产分别验收，预生产不得以历史授权直接补发或压测。 + +这是跨回执、补发、账务和耐久任务恢复的修复，成本主要在事务改造及故障验收，不能仅以修改几行或压低日志级别交付。完成第1步盘点后再给出实现工时；若需改变计费、状态优先级或Gateway契约，先记录范围扩展。 + +恢复策略：迁移保留增量表；发布失败先停止新旧受影响消费者交叉执行，盘点未完成工作、已生成Submit/Outbox/账务/投递事实，再选择经过兼容验证的应用版本恢复。旧版本不理解新工作表，不能直接回退并声称全部恢复；必须证明待处理工作已排空或存在经过验证的接续路径，否则保持暂停并修复。禁止删除工作记录、回滚业务数据或重新入队来掩盖异常。 + +上线停止条件：发现重复供应商Submit、重复扣退费、终态错误覆盖、持续积压、无法接管或恢复缺口立即停止放量，保留现场；不通过手工补发完成验收。 + +### 10.9 完成标准与本轮状态 + +验收用例见 [系统功能测试用例](system-functional-test-cases.md) 的TC-RC-20260916-01～12；原TC-SEND-018/019/020继续执行。记录见 [测试进度](testing-progress.md)。 + +只有代码、真实并发/故障恢复、账务与队列对账均通过，并完成对应环境独立验收，才能标为该环境已修复。2026-09-16本轮只编写整改方案与用例，未实现模型或业务代码，未运行发送、迁移、提交、推送、测试部署或预生产部署。 + +### 10.10 2026-09-16 实施细化（开发中，尚未验收） + +用户已明确授权实施第10节、提交、推送及测试环境部署；第10.1、10.9的“仅方案”状态是此前记录，当前开始实施，不能据此宣称已上线。 + +- 增加 `SmsCompletionEvent` 耐久事件日志，与工作 revision 在同一接收事务提交；Inbox matched 表示原始事件已存入该日志。规范化 Receipt/Segment 事实、汇总和副作用由收尾事务一并提交。此细化替代10.5 A1中先写规范化表的顺序，避免规范化事实落库后收尾未接续的窗口。日志保留原始事件，不能将未处理日志计作已完成业务回执。 +- 回执、整条提交结果、分段提交结果、超时与平台拒绝通过统一入口入日志；先按可靠关联核验消息及租户。无可靠Submit关联的供应商回执继续留在Inbox待匹配；无供应商Submit的平台拒绝使用message工作键。迁移不扫描历史消息重发。 +- 所有收尾协作者通过仅限发送链路的事务适配器加入当前短事务，账务内层事务加入同一事务。锁顺序为工作→消息→账务。HTTP通知生产端显式接收事务客户端，CMPP通知无异常幂等创建；事务内仅写耐久通知，不执行网络投递。 +- 补发首先认领。首次尝试到达选路点时回滚探测事务，在事务外重新选路和限速；最终事务重新读事件、revision、有效尝试与当前路由条件，原子创建Submit/Outbox。回滚产生的会话ID不得进入跨事务缓存。临时故障必须抛出并退避，不能被吞成最终失败。 +- 恢复扫描初始5秒/32条，租约60秒、20秒续期、事务20秒上限。最多12次失败进入needs_review；这些是待真实环境验证的初值。异步通知仍由原恢复消费者投递。 +- 超时与迟到结果保持已退款状态，不重复扣退；每个历史业务键依据实际账务和通知事实恢复。上线前必须以故障与乱序测试核实，不把代码编译当作证明。 + +待补证据：真实PostgreSQL并发、fence过期、事务中断、队列排空、长短信CMPP与HTTP通知以及账务对账。当前仍为开发中。 + +### 10.11 本次实现边界与发布依赖（2026-09-16） + +- 恢复扫描在worker/callback/all角色运行；正式运行必须启用SEND_SUBMIT_OUTBOX_PUBLISH_ENABLED及对应发布进程。测试环境18:20只读核验两项Outbox开关为true、独立服务active。关闭发布器不属于受支持上线组合，不能把pending当成已发送。 +- 收尾needs_review及等待超过300秒由监控采集器直接查询耐久工作/事件表，写入同一告警事件库，恢复仍保留、仅人工清除；不依赖应用发布工具安装Prometheus规则。过程计数仍提供低基数metrics，采集进程口径需区分。 +- 两项迁移仅新增四张表及索引，无历史回填、无删除、无自动重发；全量106项已在新隔离数据库重放。工作时间使用UTC表达式，避免数据库会话时区与Prisma时间不一致。应用回退前必须盘点和接续未完成工作，旧代码不能消费这些新表。 +- 本地验证包括真实PostgreSQL、两个独立OS进程、事务回滚、旧消费者挂起后接管、三段通知目标、非零账务和重试上限可见告警；网络投递与目标环境的重启/排空验收独立记录，不将路由隔离测试冒称整链路通过。 diff --git a/docs/prometheus-system-monitoring-design-20260814.md b/docs/prometheus-system-monitoring-design-20260814.md index 01d4b70..52acf7d 100644 --- a/docs/prometheus-system-monitoring-design-20260814.md +++ b/docs/prometheus-system-monitoring-design-20260814.md @@ -256,3 +256,11 @@ type InfrastructureOverview = { 新增GET /admin/infrastructure-monitoring/alert-history，继承运营端会话鉴权，参数from/to为上海自然日，默认含今天近7日、最多31日，page正整数、每页25条。逐日查询Prometheus原始ALERTS_FOR_STATE范围向量，以标签指纹+activeAt分开触发周期，跨日采样合并；不使用粗粒度步长丢掉短周期，不把等待触发当作已发送告警。保留真实触发时间和范围内最后观测时间，最后观测不代表准确恢复。 历史和活动列表独立；历史不提供伪造恢复状态、旧annotations、批量已读或清理功能。超出Prometheus保留期/采集缺口的历史无法追溯，界面明确说明。失败返回503并展示错误，日期非法返回400；不迁移数据库、不变更阈值或采集配置。完整实现/验收及限制见operations-fixes-20260909.md。 + +## 2026-09-16 人工清除及刷新失败保留真实快照（实施中） + +本节替代活动告警随Prometheus恢复自动消失及失败清空指标的旧规则。新增PostgreSQL告警事件记录，以fingerprint+activeAt唯一标识一次触发；后台定期采集及页面读取时持久化。指标恢复仅标识已恢复，告警仍在待处理列表，只有运营手动清除（全局生效、记录操作人及时间）才退出；已读保持个人维度，不等于清除。同周期清除后不被下次采集重新激活，新周期独立产生。采集失败不自动推断恢复，不删除记录。历史审计保留，不能将Prometheus过期历史伪造为已完整迁入。 + +页面按查询范围在组件内存保留最后成功的真实快照；失败/超时显示错误和最后成功采样时间，不伪装实时值，不通过localStorage生成业务事实。不同时间范围不能串用缓存；请求有真实AbortController超时、切换/卸载取消和过期响应保护，在线恢复触发刷新；保留会话失效/锁定处理。根因验收需记录网络失败与后端采集失败两种路径，并验证长停留、并发切换及三尺寸。 + +2026-09-16实施补充：收尾工作人工处理/积压告警由采集器直接读取PG，不需要额外安装Prometheus规则；持续条件复用同一发生周期，即使已清除也不在同周期重建，恢复后再次触发才是新事件。手动清除同步移除页面各范围缓存中的同一事件及数量，避免后续刷新失败使已清除告警重新显示。 diff --git a/docs/system-functional-test-cases.md b/docs/system-functional-test-cases.md index 737bccd..08b337c 100644 --- a/docs/system-functional-test-cases.md +++ b/docs/system-functional-test-cases.md @@ -5569,3 +5569,40 @@ CLIENT-0914-01～07 的模板样式/顺序、文档归属与检索、中文状 - TC-BRAND-20260916-01：浏览器标题、平台Swagger、客户OpenAPI及HTTP文档展示“聆界短信平台”；充值回执图片替代文本、两类报备导出工作簿作者一致。源码检查与构建结果见测试进度，真实页面/API/导出验收单独记录。 - TC-BRAND-20260916-02：产品3.0/工程3.0.0、客户接口独立版本、CMPP协议、API路径、签名规则、工程包名和基础设施标识保持不变；已有工作区修改不被覆盖。 + +## 长短信回执并发整改验收（2026-09-16，待实施、未执行） + +设计依据：[发送链路整改方案第10节](phase-4-send-pipeline-redesign.md#10-长短信分段回执并发整改方案2026-09-16待实施)。下列用例均为P0，通用前置条件为明确授权的隔离测试企业/应用、模拟号码/通道，真实API、PostgreSQL、Redis、Gateway和Webhook接收端；保留非零计费规则。测试执行本身不因用例存在而获得发送、补发、配置变更或压测授权。 + +每例保留messageId/sourceSubmitRecordId、脱敏回执时间线、工作记录及认领版本、Submit/Outbox数、客户逻辑通知及实际投递次数、账务流水和队列结果。实际投递重试与重复创建分别计数。故障测试使用隔离环境可控断点或进程终止，禁止预生产注入。 + +| 编号 | 场景与步骤 | 通过标准 | +|---|---|---| +| TC-RC-20260916-01 | 两段、三段成功回执通过不同消费者同时进入；再逐段延迟和逆序各执行一组 | per_segment最后一段成功前不判整条成功；齐段后仅一份最终业务决策；按客户目标生成规定数量回执，无多余逻辑通知和去重键异常 | +| TC-RC-20260916-02 | 两段、四段失败回执同时进入，开启允许补发且有可用模拟候选 | 每个源Submit仅一个后继Submit及对应Outbox；竞争失败流程不重复选路；模拟器无因本缺陷多出的Submit；无retryOf唯一键异常 | +| TC-RC-20260916-03 | 同一失败同时由Submit结果和回执入口触发；再与超时处理竞争 | 入口共用协调；按既有状态规则只提交有效决策；不得一边补发一边最终退款/通知失败 | +| TC-RC-20260916-04 | 相同回执重复20次，再混合不同分段并发；用两个回调消费者处理 | 原始合法分段保留，同一事件幂等；副作用数不随事件重复数增长；不存在仅单进程锁有效的问题 | +| TC-RC-20260916-05 | 旧尝试失败后新尝试成功，再注入旧尝试成功/失败回执；另测message_level成功后矛盾失败 | 最新有效成功不覆盖、不重复退款补发；历史完整；矛盾事件按既有异常规则记录 | +| TC-RC-20260916-06 | 注入unknown、缺一段、回执先于Submit结果；后续补齐合法事实；测试72小时超时 | unknown不补发，缺段不误成功，后续事实可唤醒idle；超时后迟到结果遵守既有规则；未匹配事件不被伪造关联 | +| TC-RC-20260916-07 | 在事实保存前、保存后未认领、认领后未提交三个断点退出并重启 | Inbox或耐久工作分别接续；不漏回执、不丢收尾；每例在预先配置的恢复预算内完成或明确告警 | +| TC-RC-20260916-08 | 挂起旧消费者至租约过期，让新消费者接管，再恢复旧消费者；选路期间加入新事实 | 新fence有效，旧fence不能创建Submit/退款/通知或覆盖状态；revision不丢；允许重新计算但无重复副作用 | +| TC-RC-20260916-09 | 在补发Submit/Outbox事务提交前后、最终状态/账务/通知事务提交前后断开连接或退出 | 未提交全部回滚，已提交通过稳定键回读；恢复不新建第二份补发或账务；短信记录、账户流水、Outbox和通知一致 | +| TC-RC-20260916-10 | HTTP通知成功、503、超时；CMPP客户断线重连；含多客户分段和Registered_Delivery=false | HTTP业务事件及CMPP应通知目标身份稳定；网络重试使用已有投递记录；不因客户回调失败补发短信；不要求回执的分段不下发 | +| TC-RC-20260916-11 | 升级前准备历史已完成、在途、缺Submit关联、无租户通道测试、平台拒绝数据；演练迁移、切换与恢复 | 已完成历史不重发、不重退；在途接续清单完整；归属不明保留待处理；测试消息不通知客户；旧版本恢复前必须验证未完成工作接续，跨租户关联拒绝 | +| TC-RC-20260916-12 | 相同负载及长短信/失败比例做修复前后对比，并注入临时数据库错误、候选失效和持续失败 | 认领后实时校验有效；临时故障不当最终失败；持续失败进入可见告警；正常并发无两类预期冲突；无业务重复、账务不符或持续积压，按预设延迟/排空预算通过 | + +附加回归：TC-SEND-018补发停止条件、TC-SEND-019旧回执、TC-SEND-020账务幂等、单段短信、黑名单/频控/签名/引流报备、租户和应用停用、批次进度及发送详情真实回执时间。并发模拟通过不替代API全量、类型、构建与质量门禁；Gateway改动另执行Go测试和vet。任何无法执行项明确登记，不以mock计为真实验收。 + +## 五项联合整改验收补充（2026-09-16） + +此前RC-01～12为本次实施矩阵，不能沿用“待实施”标题作为当前事实；执行结果逐项记录，未完整覆盖的条目保留未完成。 + +| 用例 | 验收内容 | 当前证据 | +|---|---|---| +| TC-OPS-0916-01 | 通道测试无签名、未报备URL及长短信；真实Gateway最终检查仍放行；普通客户保持原拦截 | 最终guard定向4项通过；真实Gateway待执行 | +| TC-OPS-0916-02 | 双端参数独立输入、原始UTF8签名、真实请求/响应、幂等键保留、重复点击/超时不自动重发 | 前端定向通过；真实浏览器待执行 | +| TC-OPS-0916-03 | 告警恢复、刷新、重启不清除；手动清除有审计且并发幂等；同周期不重现，新周期可见 | 真实PG通过；页面待执行 | +| TC-OPS-0916-04 | 监控刷新失败保留当前范围最后成功快照并标明非实时；网络恢复、路由切换与三尺寸 | 组件测试通过；真实浏览器待执行 | +| TC-OPS-0916-05 | 收尾第12次失败进入needs_review，自动出现在耐久告警；恢复后不自动消失 | 真实PG通过 | + +RC-01/02/04/08/09/10/11目前仅部分本地证据：三段齐段、重复与双进程、事务回滚和过期接管、通知目标数与非零账务、迁移重放；尚不能标整条矩阵通过。RC-03/05/06/07/12的完整组合、网络故障/进程重启、修复前后性能对比须独立补验。无真实运营商流量，未声明容量提升。 diff --git a/docs/testing-progress.md b/docs/testing-progress.md index b829ed5..67b240c 100644 --- a/docs/testing-progress.md +++ b/docs/testing-progress.md @@ -5071,3 +5071,23 @@ CUA本轮可用，实际后端文档三尺寸1600×1000/1366×768/390×844无页 ### 2026-09-16 更名本地提交范围核对 用户追加授权本地提交。本次精确暂存更名代码、规范标题及需求/用例/进度中的更名段落；package及lock版本、main.ts的Swagger版本变更和既有未跟踪version-3.0-baseline.md草稿不纳入本次提交。3.0.0版本元数据仍留在工作区，提交不代表版本标识已发布。前述测试为本地工作区结果，不冒称隔离候选精确提交的全量验证。既有metrics、发布工具、部署脚本与其他文档改动保留。推送、测试部署及预生产部署未执行。 + +## 2026-09-16 五项联合整改启动 + +用户确认实施phase-4-send-pipeline-redesign.md第10节，并授权通道测试免签名/引流检测、HTTP参数化发送调试、监控告警人工清除与刷新保留真实快照；提交、推送及测试环境发布/充分长短信验收。main=a0209f93bc8eee6cb08a9d7cace4163b26749ae6，实际远端86cb9ae，暂存区空。61份已有修改/草稿保护摘要及原文位于.local-data/five-fixes-20260916，前轮版本及发布工具不自动纳入。测试SSH首次连接超时，尚未认证；本地工作继续，部署/真实环境待连通后执行。尚未修改业务代码或操作线上状态。 + +## 2026-09-16 五项改造开发进展（18:00，本地，未提交/推送/部署） + +- 承接用户确认：执行发送链路第10节、通道测试跳过签名与引流、HTTP参数化真实发送调试、告警手动清除、监控失败保留最后成功快照。测试机开机后SSH已恢复；在线版本仍为cbc4a033251c7496414b1e7aa6233ed49bd108ec。预生产未操作。 +- 新增耐久工作/事件日志及告警事件/采集水位迁移；按工作→消息→账务/批次顺序收尾，补发选路限速在事务外，通知/账务/Outbox入同一事务。修复探测异常被吞、事务会话缓存、迟到结果重复计费、超时终态覆盖及多消息批次进度事务隔离细节。 +- 通道测试除去入口检测，同时修正Gateway最终引流授权入口；仅按真实Submit归属确认的无租户/无批次测试消息豁免，普通客户消息仍校验。 +- 本地隔离PostgreSQL 16434/cmpp_qa_completion_20260916已执行全量106迁移（后续UTC默认值细化尚需新库重放）。真实数据库验证并发24事件/12唯一事实、事务故障回滚与接续、租约接管、三段齐段成功、CMPP/HTTP通知幂等、非零975计费/超时退款一次/迟到回执、三段失败只创建一份后继Submit/Outbox。路由与限速在补发事务测试中被隔离，不作为真实路由/Redis/Gateway通过依据。 +- 证据脚本tools/testing/verify-attempt-completion.mjs；日志.local-data/five-fixes-20260916/completion-integration.log。故障注入故意产生completion_retry_wait，不是线上异常。 +- 最近定向隔离回归：发送链路+最终通道校验142/142；发送链路+监控服务147/147（范围重叠，不相加）；HTTP页面/签名15/15。后端类型检查通过。后续改动须复验；真实浏览器、完整回归、故障恢复矩阵、实际CMPP/Webhook投递与测试部署仍未完成。 +- 保护既有61项修改快照，未把原metrics/发布工具/文档草稿视作本轮内容。此前产品命名提交a0209f9尚未推送。本轮新增metrics两处接入需精确hunk暂存，不能包含此前指标修改。 + +### 2026-09-16 18:25 本地验证与交付准备 + +前端全量35套163项通过（115.33秒），后端77套835项通过（171.189秒）；类型检查通过。新增隔离PG全量106迁移重放和11组耐久工作/告警验证通过，包含独立OS进程并发及挂起旧消费者恢复，日志在.local-data/five-fixes-20260916。第12次失败转人工处理及数据库告警采集已补验证；日志中的故意故障注入不得当线上异常。最新采集改动需候选精确提交回归。 + +真实环境尚未部署；独立Playwright获用户允许，专用Edge等待用户完成登录。测试Outbox发布开关及独立进程已核验。为通过更名涉及文件的既有门禁，仅移除official-export未使用导入并格式化该文件及RechargeReceiptDialog，业务规则不变。保护项未纳入；开始准备精确暂存，尚未提交/推送。 diff --git a/src/api/admin/infrastructure-monitoring.api.ts b/src/api/admin/infrastructure-monitoring.api.ts index 65df7ef..0b8909b 100644 --- a/src/api/admin/infrastructure-monitoring.api.ts +++ b/src/api/admin/infrastructure-monitoring.api.ts @@ -25,8 +25,10 @@ export const adminInfrastructureMonitoringApi = { startDate: string; endDate: string; }>(withQuery('/admin/infrastructure-monitoring/alert-history', { from, to, page })), - getInfrastructureMonitoringOverview: (range: InfrastructureMonitoringRange) => - request(withQuery('/admin/infrastructure-monitoring/overview', { range })), + getInfrastructureMonitoringOverview: (range: InfrastructureMonitoringRange, signal?: AbortSignal) => + request(withQuery('/admin/infrastructure-monitoring/overview', { range }), { + signal, + }), getInfrastructureMonitoringNotificationSummary: (signal?: AbortSignal) => request<{ count: number; criticalCount: number }>('/admin/infrastructure-monitoring/notification-summary', { signal, @@ -38,6 +40,11 @@ export const adminInfrastructureMonitoringApi = { method: 'PUT', body: JSON.stringify(body), }), + clearInfrastructureAlert: (fingerprint: string, activeAt: string) => + request<{ cleared: boolean }>(`/admin/infrastructure-monitoring/alerts/${fingerprint}/clear`, { + method: 'POST', + body: JSON.stringify({ activeAt }), + }), markInfrastructureAlertRead: (fingerprint: string, activeAt: string) => request<{ fingerprint: string; activeAt: string; acknowledged: true; acknowledgedAt: string }>( `/admin/infrastructure-monitoring/alerts/${fingerprint}/read`, diff --git a/src/apps/admin/system-monitoring/AdminSystemMonitoringPage.test.tsx b/src/apps/admin/system-monitoring/AdminSystemMonitoringPage.test.tsx new file mode 100644 index 0000000..33ff961 --- /dev/null +++ b/src/apps/admin/system-monitoring/AdminSystemMonitoringPage.test.tsx @@ -0,0 +1,80 @@ +import { fireEvent, render, screen, waitFor } from '@testing-library/react'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import { AdminSystemMonitoringPage } from './AdminSystemMonitoringPage'; + +const api = vi.hoisted(() => ({ + getInfrastructureMonitoringOverview: vi.fn(), + getInfrastructureAlertThresholds: vi.fn(), + clearInfrastructureAlert: vi.fn(), +})); +vi.mock('@/api/adminApi', () => ({ adminApi: api })); +vi.mock('@/components/ui/Chart', () => ({ Chart: () =>
趋势图
})); +vi.mock('./AlertHistory', () => ({ AlertHistory: () => null })); +const sample = { + available: true, + range: '24h', + collectedAt: '2026-09-16T00:00:00Z', + lastSampleAt: '2026-09-16T00:00:00Z', + summary: { + overallStatus: 'healthy', + serviceTotal: 0, + serviceHealthy: 0, + warningAlerts: 0, + criticalAlerts: 0, + activeAlerts: 0, + }, + metrics: { cpuUsagePercent: 12.3 }, + trends: { + cpuUsagePercent: [], + memoryUsagePercent: [], + diskUsagePercent: [], + networkReceiveBytesPerSecond: [], + networkTransmitBytesPerSecond: [], + }, + services: [], + serviceMetrics: [], + disks: [], + alerts: [], +}; +beforeEach(() => { + vi.clearAllMocks(); + api.getInfrastructureMonitoringOverview.mockResolvedValue(sample); + api.getInfrastructureAlertThresholds.mockResolvedValue({ thresholds: {}, definitions: [], configVersion: 1 }); +}); +describe('monitoring snapshots', () => { + it('retains the last successful data with a visible stale warning after refresh fails, then recovers', async () => { + render(); + await screen.findAllByText('12.3%'); + api.getInfrastructureMonitoringOverview.mockRejectedValueOnce(new TypeError('Failed to fetch')); + fireEvent.click(screen.getByRole('button', { name: '刷新' })); + await screen.findByText('监控数据更新失败'); + expect(screen.getAllByText('12.3%').length).toBeGreaterThan(0); + expect(screen.getByRole('alert')).toHaveTextContent('上次成功数据'); + expect(screen.getByRole('alert')).toHaveTextContent('并非实时状态'); + api.getInfrastructureMonitoringOverview.mockResolvedValueOnce({ ...sample, metrics: { cpuUsagePercent: 45.6 } }); + fireEvent(window, new Event('online')); + await screen.findAllByText('45.6%'); + expect(screen.queryByText('监控数据更新失败')).not.toBeInTheDocument(); + }); + it('does not label a different time range with the previous range snapshot and aborts superseded requests', async () => { + render(); + await screen.findAllByText('12.3%'); + api.getInfrastructureMonitoringOverview.mockRejectedValueOnce(new TypeError('offline')); + fireEvent.click(screen.getByRole('button', { name: '近1小时' })); + await screen.findByText('监控数据更新失败'); + expect(screen.queryByText('12.3%')).not.toBeInTheDocument(); + expect(screen.getByRole('alert')).toHaveTextContent('尚无成功采样'); + api.getInfrastructureMonitoringOverview.mockImplementationOnce( + (_range, signal) => + new Promise((_resolve, reject) => + signal.addEventListener('abort', () => reject(new DOMException('aborted', 'AbortError'))), + ), + ); + fireEvent.click(screen.getByRole('button', { name: '刷新' })); + const calls = api.getInfrastructureMonitoringOverview.mock.calls; + const signal = calls[calls.length - 1]?.[1] as AbortSignal; + fireEvent.click(screen.getByRole('button', { name: '近24小时' })); + await waitFor(() => expect(signal.aborted).toBe(true)); + await screen.findAllByText('12.3%'); + }); +}); diff --git a/src/apps/admin/system-monitoring/AdminSystemMonitoringPage.tsx b/src/apps/admin/system-monitoring/AdminSystemMonitoringPage.tsx index fe1acce..f561330 100644 --- a/src/apps/admin/system-monitoring/AdminSystemMonitoringPage.tsx +++ b/src/apps/admin/system-monitoring/AdminSystemMonitoringPage.tsx @@ -227,6 +227,7 @@ function severityTag(severity: InfrastructureAlert['severity']) { function makeAlertColumns( onMarkRead: (alert: InfrastructureAlert) => void, readingFingerprint: string, + onClear: (alert: InfrastructureAlert) => void, ): Array> { return [ { key: 'severity', title: '级别', width: '82px', render: (record) => severityTag(record.severity) }, @@ -254,25 +255,42 @@ function makeAlertColumns( render: (record) => `${record.currentValue || '—'} / ${record.threshold || '—'}`, }, { key: 'startedAt', title: '开始时间', width: '150px', render: (record) => formatTime(record.startedAt) }, + { + key: 'status', + title: '状态', + width: '100px', + render: (record) => (record.status === 'resolved' ? '已恢复待清除' : '仍在触发'), + }, { key: 'duration', title: '持续时间', width: '120px', render: (record) => formatDuration(record.startedAt) }, { key: 'actions', title: '操作', - width: '112px', - render: (record) => - record.acknowledged ? ( - 已读 - ) : ( + width: '180px', + render: (record) => ( +
+ {record.acknowledged ? ( + 已读 + ) : ( + + )} - ), +
+ ), }, ]; } @@ -291,25 +309,43 @@ export function AdminSystemMonitoringPage() { const [readError, setReadError] = useState(''); const requestSequence = useRef(0); const pendingRequests = useRef(0); + const activeRequest = useRef(null); + const snapshots = useRef>>({}); const loadData = useCallback( async (supersede = false) => { if (!supersede && pendingRequests.current > 0) return; + if (supersede) activeRequest.current?.abort(); + const controller = new AbortController(); + activeRequest.current = controller; + const timer = window.setTimeout(() => controller.abort(), 20_000); pendingRequests.current += 1; const sequence = ++requestSequence.current; setLoading(true); + setOverview(snapshots.current[range] ?? null); try { - const result = await adminApi.getInfrastructureMonitoringOverview(range); + const result = await adminApi.getInfrastructureMonitoringOverview(range, controller.signal); if (sequence !== requestSequence.current) return; - setOverview(result); + if (result.available) { + snapshots.current[range] = result; + setOverview(result); + } else { + setOverview(snapshots.current[range] ?? result); + } setError(result.available ? '' : result.error || '监控数据当前不可用'); } catch (reason) { if (sequence !== requestSequence.current) return; - setOverview(null); - setError(reason instanceof Error ? reason.message : '监控数据加载失败'); + setOverview(snapshots.current[range] ?? null); + setError( + reason instanceof Error && reason.name === 'AbortError' + ? '刷新超时，请检查网络后重试' + : '监控刷新失败，请检查网络或采集服务', + ); } finally { if (sequence === requestSequence.current) setLoading(false); pendingRequests.current -= 1; + window.clearTimeout(timer); + if (activeRequest.current === controller) activeRequest.current = null; } }, [range], @@ -372,6 +408,43 @@ export function AdminSystemMonitoringPage() { } }, []); + const clearAlert = useCallback( + async (alert: InfrastructureAlert) => { + setReadingFingerprint(alert.fingerprint); + setReadError(''); + try { + await adminApi.clearInfrastructureAlert(alert.fingerprint, alert.startedAt); + const remove = (snapshot: InfrastructureMonitoringOverview) => { + const alerts = snapshot.alerts.filter( + (item) => item.fingerprint !== alert.fingerprint || item.startedAt !== alert.startedAt, + ); + return { + ...snapshot, + alerts, + summary: { + ...snapshot.summary, + activeAlerts: alerts.length, + criticalAlerts: alerts.filter((item) => item.severity === 'critical').length, + warningAlerts: alerts.filter((item) => item.severity === 'warning').length, + }, + }; + }; + for (const range of Object.keys(snapshots.current) as InfrastructureMonitoringRange[]) { + const snapshot = snapshots.current[range]; + if (snapshot) snapshots.current[range] = remove(snapshot); + } + setOverview((snapshot) => (snapshot ? remove(snapshot) : snapshot)); + window.dispatchEvent(new Event('cmpp-infrastructure-alert-count-refresh')); + await loadData(true); + } catch (reason) { + setReadError(reason instanceof Error ? reason.message : '清除告警失败'); + } finally { + setReadingFingerprint(''); + } + }, + [loadData], + ); + useEffect(() => { void loadData(true); void loadSettings(); @@ -382,14 +455,17 @@ export function AdminSystemMonitoringPage() { if (document.visibilityState === 'visible') void loadData(); }; document.addEventListener('visibilitychange', handleVisibility); + window.addEventListener('online', handleVisibility); return () => { requestSequence.current += 1; + activeRequest.current?.abort(); window.clearInterval(intervalId); document.removeEventListener('visibilitychange', handleVisibility); + window.removeEventListener('online', handleVisibility); }; }, [loadData, loadSettings]); - const status = STATUS_COPY[overview?.summary.overallStatus ?? 'unknown']; + const status = STATUS_COPY[error ? 'unknown' : (overview?.summary.overallStatus ?? 'unknown')]; const cpuOption = useMemo( () => makeTrendOption({ @@ -442,10 +518,16 @@ export function AdminSystemMonitoringPage() { const serviceTotal = overview?.summary.serviceTotal ?? 6; const alertColumns = useMemo( () => - makeAlertColumns((alert) => { - void markAlertRead(alert); - }, readingFingerprint), - [markAlertRead, readingFingerprint], + makeAlertColumns( + (alert) => { + void markAlertRead(alert); + }, + readingFingerprint, + (alert) => { + void clearAlert(alert); + }, + ), + [markAlertRead, readingFingerprint, clearAlert], ); return ( @@ -487,8 +569,13 @@ export function AdminSystemMonitoringPage() {
- 监控数据不可用 - {error}。页面不会展示历史缓存值。 + 监控数据更新失败 + + {error}。 + {overview?.available + ? `当前展示上次成功数据，采样时间 ${formatTime(overview.lastSampleAt ?? overview.collectedAt)}，并非实时状态。` + : '尚无成功采样，请稍后重试。'} +
) : null} diff --git a/src/apps/shared/http-signature/HttpSignaturePage.tsx b/src/apps/shared/http-signature/HttpSignaturePage.tsx index ba07664..0836a46 100644 --- a/src/apps/shared/http-signature/HttpSignaturePage.tsx +++ b/src/apps/shared/http-signature/HttpSignaturePage.tsx @@ -2,6 +2,7 @@ import { useRef, useState } from 'react'; import { Breadcrumb, Button, Input, Select, Textarea } from '@/components/ui'; import { calculateSignature, createNonce, type SignatureInput } from './signature'; import './HttpSignaturePage.css'; +import { SmsRequestDebugger } from './SmsRequestDebugger'; function emptyInput(): SignatureInput { return { @@ -52,7 +53,9 @@ export function HttpSignaturePage() { return (
-

仅在当前浏览器计算，不上传或保存密钥。生成结果不会发送请求。

+

+ 上方工具仅在浏览器计算签名，不上传或保存密钥；需要实际发送时，请使用下方短信接口发送调试。 +

) : null}

{notice}

+
); } diff --git a/src/apps/shared/http-signature/SmsRequestDebugger.test.tsx b/src/apps/shared/http-signature/SmsRequestDebugger.test.tsx new file mode 100644 index 0000000..41b4ea0 --- /dev/null +++ b/src/apps/shared/http-signature/SmsRequestDebugger.test.tsx @@ -0,0 +1,69 @@ +import { fireEvent, render, screen, waitFor } from '@testing-library/react'; +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { SmsRequestDebugger } from './SmsRequestDebugger'; +import { calculateSignature } from './signature'; + +afterEach(() => vi.unstubAllGlobals()); +function fill() { + render(); + for (const [label, value] of [ + ['AccessKey', 'qa-key'], + ['发送 AccessSecret', 'qa-secret'], + ['手机号（mobile）', '13800138000'], + ['短信正文（content）', '【测试】长短信中文内容'.repeat(20)], + ]) { + fireEvent.change(screen.getByLabelText(label), { target: { value } }); + } +} +describe('SMS request debugger', () => { + it('signs the exact UTF-8 body and displays real HTTP results without transmitting the secret', async () => { + const fetcher = vi + .fn() + .mockResolvedValue(new Response('{"code":"OK"}', { status: 202, headers: { 'X-Request-ID': 'qa-trace' } })); + vi.stubGlobal('fetch', fetcher); + fill(); + fireEvent.click(screen.getByRole('button', { name: '发送短信' })); + await waitFor(() => + expect((screen.getByLabelText('返回 HTTP 报文') as HTMLTextAreaElement).value).toContain('HTTP 202'), + ); + const [path, options] = fetcher.mock.calls[0]; + const headers = options.headers; + expect(headers['X-Signature']).toBe( + calculateSignature({ + method: 'POST', + path, + body: options.body, + secret: 'qa-secret', + timestamp: headers['X-Timestamp'], + nonce: headers['X-Nonce'], + }).signature, + ); + expect(JSON.stringify(options)).not.toContain('qa-secret'); + expect(options.credentials).toBe('omit'); + expect((screen.getByLabelText('返回 HTTP 报文') as HTMLTextAreaElement).value).toContain('qa-trace'); + }); + + it('preserves the idempotency key after network uncertainty, never auto retries, and changes it when body changes', async () => { + const fetcher = vi.fn().mockRejectedValue(new TypeError('Failed to fetch')); + vi.stubGlobal('fetch', fetcher); + fill(); + const key = (screen.getByLabelText('业务幂等键（Idempotency-Key）') as HTMLInputElement).value; + fireEvent.click(screen.getByRole('button', { name: '发送短信' })); + await screen.findByRole('alert'); + expect(screen.getByRole('alert')).toHaveTextContent('发送结果未知'); + expect(fetcher).toHaveBeenCalledTimes(1); + expect(screen.getByLabelText('业务幂等键（Idempotency-Key）')).toHaveValue(key); + fireEvent.change(screen.getByLabelText('短信正文（content）'), { target: { value: '修改正文' } }); + expect(screen.getByLabelText('业务幂等键（Idempotency-Key）')).not.toHaveValue(key); + }); + + it('blocks duplicate clicks while a response is outstanding', async () => { + const fetcher = vi.fn().mockImplementation(() => new Promise(() => {})); + vi.stubGlobal('fetch', fetcher); + fill(); + const send = screen.getByRole('button', { name: '发送短信' }); + fireEvent.click(send); + fireEvent.click(send); + expect(fetcher).toHaveBeenCalledTimes(1); + }); +}); diff --git a/src/apps/shared/http-signature/SmsRequestDebugger.tsx b/src/apps/shared/http-signature/SmsRequestDebugger.tsx new file mode 100644 index 0000000..437b1f7 --- /dev/null +++ b/src/apps/shared/http-signature/SmsRequestDebugger.tsx @@ -0,0 +1,157 @@ +import { useEffect, useRef, useState } from 'react'; +import { Button, Input, Textarea } from '@/components/ui'; +import { calculateSignature, createNonce } from './signature'; + +const path = '/api/openapi/v1/sms/messages'; + +export function SmsRequestDebugger() { + const [fields, setFields] = useState({ + appKey: '', + secret: '', + mobile: '', + content: '', + clientMessageId: '', + idempotencyKey: createNonce(), + }); + const [sending, setSending] = useState(false); + const [requestText, setRequestText] = useState(''); + const [responseText, setResponseText] = useState(''); + const [error, setError] = useState(''); + const controller = useRef(null); + useEffect(() => () => controller.current?.abort(), []); + + function change(key: keyof typeof fields, value: string) { + setFields((current) => ({ + ...current, + [key]: value, + ...(key === 'idempotencyKey' ? {} : { idempotencyKey: createNonce() }), + })); + setError(''); + } + + async function send() { + if (controller.current) return; + if (!fields.appKey.trim() || !fields.secret || !/^1\d{10}$/.test(fields.mobile) || !fields.content.trim()) { + setError('请填写 AccessKey、AccessSecret、11位手机号和短信正文'); + return; + } + if (!/^[A-Za-z0-9._:-]{8,128}$/.test(fields.idempotencyKey) || fields.clientMessageId.length > 128) { + setError('请检查业务幂等键（8～128位）及客户消息编号（最多128字符）'); + return; + } + const body = JSON.stringify({ + mobile: fields.mobile, + content: fields.content, + ...(fields.clientMessageId ? { clientMessageId: fields.clientMessageId } : {}), + }); + const timestamp = String(Math.floor(Date.now() / 1000)); + const nonce = createNonce(); + const { signature } = calculateSignature({ method: 'POST', path, timestamp, nonce, secret: fields.secret, body }); + const headers = { + 'Content-Type': 'application/json', + 'X-App-Key': fields.appKey.trim(), + 'X-Timestamp': timestamp, + 'X-Nonce': nonce, + 'X-Signature': signature, + 'Idempotency-Key': fields.idempotencyKey, + }; + setRequestText( + `POST ${path}\n${Object.entries(headers) + .map(([key, value]) => `${key}: ${value}`) + .join('\n')}\n\n${body}`, + ); + setResponseText(''); + setError(''); + setSending(true); + const active = new AbortController(); + controller.current = active; + const timer = window.setTimeout(() => active.abort(), 20_000); + let responseHead = ''; + try { + const response = await fetch(path, { + method: 'POST', + credentials: 'omit', + redirect: 'error', + headers, + body, + signal: active.signal, + }); + responseHead = `HTTP ${response.status} ${response.statusText}\n${Array.from(response.headers.entries()) + .map(([key, value]) => `${key}: ${value}`) + .join('\n')}\n\n`; + const responseBody = await response.text(); + setResponseText(responseHead + responseBody); + if (!response.ok) setError(`接口返回 ${response.status}，请查看响应报文`); + } catch (reason) { + if (responseHead) setResponseText(responseHead + '[响应正文未完整读取]'); + setError( + `${reason instanceof Error && reason.name === 'AbortError' ? '请求超时或已取消' : '网络请求失败'}，发送结果未知。请先查询消息结果；如需重试，保持相同业务幂等键。`, + ); + } finally { + window.clearTimeout(timer); + controller.current = null; + setSending(false); + } + } + + return ( +
+

短信接口发送调试

+

+ 发送到当前平台，会真实创建短信并按应用规则计费。密钥只用于本页计算签名，不保存。以下展示应用层报文，浏览器自动添加的头部不在其中。 +

+
+ change('appKey', event.target.value)} + /> + change('secret', event.target.value)} + /> + change('mobile', event.target.value)} + /> + change('clientMessageId', event.target.value)} + /> + change('idempotencyKey', event.target.value)} + /> +
+