diff --git a/api/prisma/migrations/20260917030000_signature_analytics/migration.sql b/api/prisma/migrations/20260917030000_signature_analytics/migration.sql new file mode 100644 index 0000000..3bcad91 --- /dev/null +++ b/api/prisma/migrations/20260917030000_signature_analytics/migration.sql @@ -0,0 +1,130 @@ +-- CreateTable +CREATE TABLE "SignatureAnalyticsGeneration" ( + "id" TEXT NOT NULL, + "businessDate" DATE NOT NULL, + "sourceAsOf" TIMESTAMP(3) NOT NULL, + + CONSTRAINT "SignatureAnalyticsGeneration_pkey" PRIMARY KEY ("id") +); + +-- CreateTable +CREATE TABLE "SignatureAnalyticsDay" ( + "businessDate" DATE NOT NULL, + "publishedGenerationId" TEXT, + "generatedAt" TIMESTAMP(3), + "sourceAsOf" TIMESTAMP(3), + "refreshFor" DATE, + "state" TEXT NOT NULL DEFAULT 'missing', + "error" TEXT, + "provenance" TEXT NOT NULL DEFAULT 'daily', + "schemaVersion" INTEGER NOT NULL DEFAULT 1, + "rowCounts" JSONB, + "updatedAt" TIMESTAMP(3) NOT NULL, + + CONSTRAINT "SignatureAnalyticsDay_pkey" PRIMARY KEY ("businessDate") +); + +-- CreateTable +CREATE TABLE "SignatureAnalyticsRun" ( + "id" TEXT NOT NULL, + "scope" TEXT NOT NULL, + "businessDate" DATE NOT NULL, + "refreshFor" DATE NOT NULL, + "generationId" TEXT NOT NULL, + "state" TEXT NOT NULL DEFAULT 'pending', + "owner" TEXT, + "fence" INTEGER NOT NULL DEFAULT 0, + "leaseUntil" TIMESTAMP(3), + "attempt" INTEGER NOT NULL DEFAULT 0, + "nextAttemptAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "checkpoint" JSONB, + "error" TEXT, + "startedAt" TIMESTAMP(3), + "finishedAt" TIMESTAMP(3), + + CONSTRAINT "SignatureAnalyticsRun_pkey" PRIMARY KEY ("id") +); + +-- CreateTable +CREATE TABLE "SignatureQualityDaily" ( + "generationId" TEXT NOT NULL, + "businessDate" DATE NOT NULL, + "signatureId" TEXT NOT NULL, + "signatureName" TEXT NOT NULL, + "tenantId" TEXT NOT NULL, + "tenantName" TEXT NOT NULL, + "applicationNames" TEXT NOT NULL, + "total" INTEGER NOT NULL, + "payload" JSONB NOT NULL, + + CONSTRAINT "SignatureQualityDaily_pkey" PRIMARY KEY ("generationId","signatureId") +); + +-- CreateTable +CREATE TABLE "SignatureActivityDaily" ( + "generationId" TEXT NOT NULL, + "businessDate" DATE NOT NULL, + "dimensionKey" TEXT NOT NULL, + "dimensionType" TEXT NOT NULL, + "signatureId" TEXT NOT NULL, + "channelKey" TEXT NOT NULL, + "carrier" TEXT NOT NULL, + "tenantId" TEXT NOT NULL, + "applicationId" TEXT, + "signatureName" TEXT NOT NULL, + "tenantName" TEXT NOT NULL, + "applicationName" TEXT NOT NULL, + "channelName" TEXT NOT NULL, + "approvedAt" TIMESTAMP(3), + "submittedAttempts" INTEGER NOT NULL, + "acceptedBusinessCount" INTEGER NOT NULL, + "deliveredBusinessCount" INTEGER NOT NULL, + "applicability" TEXT NOT NULL, + + CONSTRAINT "SignatureActivityDaily_pkey" PRIMARY KEY ("generationId","dimensionKey") +); + +-- CreateTable +CREATE TABLE "UnreportedSignatureDaily" ( + "generationId" TEXT NOT NULL, + "businessDate" DATE NOT NULL, + "dimensionKey" TEXT NOT NULL, + "tenantId" TEXT NOT NULL, + "applicationId" TEXT NOT NULL, + "signatureName" TEXT NOT NULL, + "tenantName" TEXT NOT NULL, + "applicationName" TEXT NOT NULL, + "messageCount" INTEGER NOT NULL, + + CONSTRAINT "UnreportedSignatureDaily_pkey" PRIMARY KEY ("generationId","dimensionKey") +); + +-- CreateIndex +CREATE UNIQUE INDEX "SignatureAnalyticsGeneration_id_businessDate_key" ON "SignatureAnalyticsGeneration"("id", "businessDate"); + +-- CreateIndex +CREATE INDEX "SignatureAnalyticsRun_state_nextAttemptAt_idx" ON "SignatureAnalyticsRun"("state", "nextAttemptAt"); + +-- CreateIndex +CREATE UNIQUE INDEX "SignatureAnalyticsRun_scope_businessDate_key" ON "SignatureAnalyticsRun"("scope", "businessDate"); + +-- CreateIndex +CREATE INDEX "SignatureQualityDaily_businessDate_generationId_total_idx" ON "SignatureQualityDaily"("businessDate", "generationId", "total"); + +-- CreateIndex +CREATE INDEX "SignatureActivityDaily_businessDate_generationId_dimensionT_idx" ON "SignatureActivityDaily"("businessDate", "generationId", "dimensionType", "acceptedBusinessCount"); + +-- CreateIndex +CREATE INDEX "UnreportedSignatureDaily_businessDate_generationId_messageC_idx" ON "UnreportedSignatureDaily"("businessDate", "generationId", "messageCount"); + +-- AddForeignKey +ALTER TABLE "SignatureAnalyticsDay" ADD CONSTRAINT "SignatureAnalyticsDay_publishedGenerationId_businessDate_fkey" FOREIGN KEY ("publishedGenerationId", "businessDate") REFERENCES "SignatureAnalyticsGeneration"("id", "businessDate") ON DELETE RESTRICT ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "SignatureQualityDaily" ADD CONSTRAINT "SignatureQualityDaily_generationId_businessDate_fkey" FOREIGN KEY ("generationId", "businessDate") REFERENCES "SignatureAnalyticsGeneration"("id", "businessDate") ON DELETE RESTRICT ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "SignatureActivityDaily" ADD CONSTRAINT "SignatureActivityDaily_generationId_businessDate_fkey" FOREIGN KEY ("generationId", "businessDate") REFERENCES "SignatureAnalyticsGeneration"("id", "businessDate") ON DELETE RESTRICT ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "UnreportedSignatureDaily" ADD CONSTRAINT "UnreportedSignatureDaily_generationId_businessDate_fkey" FOREIGN KEY ("generationId", "businessDate") REFERENCES "SignatureAnalyticsGeneration"("id", "businessDate") ON DELETE RESTRICT ON UPDATE CASCADE; diff --git a/api/prisma/migrations/20260917031000_signature_submit_effective_at/migration.sql b/api/prisma/migrations/20260917031000_signature_submit_effective_at/migration.sql new file mode 100644 index 0000000..90aa5f4 --- /dev/null +++ b/api/prisma/migrations/20260917031000_signature_submit_effective_at/migration.sql @@ -0,0 +1,4 @@ +-- Expression index matches the actual report/retirement time predicate, including legacy NULL submittedAt. +-- Deliberately outside a transaction: online construction must not block SMS writes. +CREATE INDEX CONCURRENTLY "SmsSubmitRecord_effective_at_idx" +ON "SmsSubmitRecord" ((COALESCE("submittedAt", "createdAt"))); diff --git a/api/prisma/schema.prisma b/api/prisma/schema.prisma index 1355f67..d1cbd93 100644 --- a/api/prisma/schema.prisma +++ b/api/prisma/schema.prisma @@ -2896,3 +2896,103 @@ model SmsCompletionEvent { work SmsAttemptCompletionWork @relation(fields: [workId], references: [id], onDelete: Restrict) @@index([workId, processedAt, createdAt]) } + +model SignatureAnalyticsGeneration { + id String @id + businessDate DateTime @db.Date + sourceAsOf DateTime + days SignatureAnalyticsDay[] + quality SignatureQualityDaily[] + activity SignatureActivityDaily[] + unreported UnreportedSignatureDaily[] + @@unique([id, businessDate]) +} + +model SignatureAnalyticsDay { + businessDate DateTime @id @db.Date + publishedGenerationId String? + publishedGeneration SignatureAnalyticsGeneration? @relation(fields: [publishedGenerationId, businessDate], references: [id, businessDate], onDelete: Restrict) + generatedAt DateTime? + sourceAsOf DateTime? + refreshFor DateTime? @db.Date + state String @default("missing") + error String? + provenance String @default("daily") + schemaVersion Int @default(1) + rowCounts Json? + updatedAt DateTime @updatedAt +} + +model SignatureAnalyticsRun { + id String @id @default(cuid()) + scope String + businessDate DateTime @db.Date + refreshFor DateTime @db.Date + generationId String + state String @default("pending") + owner String? + fence Int @default(0) + leaseUntil DateTime? + attempt Int @default(0) + nextAttemptAt DateTime @default(now()) + checkpoint Json? + error String? + startedAt DateTime? + finishedAt DateTime? + @@unique([scope, businessDate]) + @@index([state, nextAttemptAt]) +} + +model SignatureQualityDaily { + generation SignatureAnalyticsGeneration @relation(fields: [generationId, businessDate], references: [id, businessDate], onDelete: Restrict) + generationId String + businessDate DateTime @db.Date + signatureId String + signatureName String + tenantId String + tenantName String + applicationNames String + total Int + payload Json + @@id([generationId, signatureId]) + @@index([businessDate, generationId, total]) +} + +model SignatureActivityDaily { + generation SignatureAnalyticsGeneration @relation(fields: [generationId, businessDate], references: [id, businessDate], onDelete: Restrict) + generationId String + businessDate DateTime @db.Date + dimensionKey String + dimensionType String + signatureId String + channelKey String + carrier String + tenantId String + applicationId String? + signatureName String + tenantName String + applicationName String + channelName String + approvedAt DateTime? + submittedAttempts Int + acceptedBusinessCount Int + deliveredBusinessCount Int + applicability String + @@id([generationId, dimensionKey]) + @@index([businessDate, generationId, dimensionType, acceptedBusinessCount]) +} + +model UnreportedSignatureDaily { + generation SignatureAnalyticsGeneration @relation(fields: [generationId, businessDate], references: [id, businessDate], onDelete: Restrict) + generationId String + businessDate DateTime @db.Date + dimensionKey String + tenantId String + applicationId String + signatureName String + tenantName String + applicationName String + messageCount Int + @@id([generationId, dimensionKey]) + @@index([businessDate, generationId, messageCount]) +} diff --git a/api/src/app.module.ts b/api/src/app.module.ts index 3c1287f..23507f2 100644 --- a/api/src/app.module.ts +++ b/api/src/app.module.ts @@ -1,3 +1,4 @@ +import { SignatureAnalyticsModule } from './signature-analytics/signature-analytics.module'; import { MiddlewareConsumer, Module, NestModule } from '@nestjs/common'; import { ConfigModule } from '@nestjs/config'; import { AuditModule } from './audit/audit.module'; @@ -57,6 +58,7 @@ import { SendingMonitorModule } from './sending-monitor/sending-monitor.module'; InfrastructureMonitoringModule, OpenApiModule, SignatureRetirementModule, + SignatureAnalyticsModule, SecurityDetectionModule, MetricsModule, ReportNotificationsModule, diff --git a/api/src/operations/admin-operations.controller.ts b/api/src/operations/admin-operations.controller.ts index 9899cba..f63a737 100644 --- a/api/src/operations/admin-operations.controller.ts +++ b/api/src/operations/admin-operations.controller.ts @@ -178,8 +178,8 @@ export class AdminOperationsController { return this.operations.signatureQuality({ date, keyword, - page: Number(page), - pageSize: Number(pageSize), + page: page === undefined ? 1 : Number(page), + pageSize: pageSize === undefined ? 25 : Number(pageSize), }); } diff --git a/api/src/operations/operations.service.spec.ts b/api/src/operations/operations.service.spec.ts index 8613cc8..76b5e92 100644 --- a/api/src/operations/operations.service.spec.ts +++ b/api/src/operations/operations.service.spec.ts @@ -1,3 +1,4 @@ +import { OperationsQualityQueries } from './queries/quality.queries'; import { OperationsService } from './operations.service'; function createPrismaMock() { @@ -936,10 +937,8 @@ describe('OperationsService', () => { averageArrivalMs: 1800, }, ]); - const service = new OperationsService(prisma as never); - await expect( - service.signatureQuality({ + new OperationsQualityQueries(prisma as never).signatureQualityLive({ date: '2026-07-24', keyword: '测试', page: 2, @@ -987,9 +986,9 @@ describe('OperationsService', () => { it('does not query channel details when the selected date has no registered signatures', async () => { const prisma = createPrismaMock(); prisma.$queryRaw.mockResolvedValueOnce([]); - const service = new OperationsService(prisma as never); - - await expect(service.signatureQuality({ date: '2026-07-24' })).resolves.toEqual({ + await expect( + new OperationsQualityQueries(prisma as never).signatureQualityLive({ date: '2026-07-24' }), + ).resolves.toEqual({ date: '2026-07-24', items: [], total: 0, diff --git a/api/src/operations/queries/quality.queries.ts b/api/src/operations/queries/quality.queries.ts index 44c14c0..146bc71 100644 --- a/api/src/operations/queries/quality.queries.ts +++ b/api/src/operations/queries/quality.queries.ts @@ -1,4 +1,6 @@ import { Prisma } from '@prisma/client'; +import { SignatureAnalyticsRead } from '../../signature-analytics/analytics-read'; +import { analyticsDate, analyticsPage, todayKey } from '../../signature-analytics/analytics-date'; import { PrismaService } from '../../prisma/prisma.service'; import type { SignatureQualityQuery } from '../operations.contracts'; @@ -321,9 +323,30 @@ export class OperationsQualityQueries { return { date: day.key, summary, channels, signatures, drainageSignatures, applications }; } async signatureQuality(query: SignatureQualityQuery) { + const date = analyticsDate(query.date); + analyticsPage(query.page, query.pageSize); + if (date !== todayKey()) return new SignatureAnalyticsRead(this.prisma).quality({ ...query, date }); + return this.prisma.$transaction( + async (tx) => { + await tx.$executeRawUnsafe("SET LOCAL statement_timeout='12s'"); + const result = await new OperationsQualityQueries(tx as PrismaService).signatureQualityLive({ ...query, date }); + return { + ...result, + dataSource: 'live', + reportState: 'ready', + frozen: false, + sourceAsOf: new Date(), + serverBusinessDate: date, + }; + }, + { isolationLevel: 'RepeatableRead', timeout: 15000 }, + ); + } + + async signatureQualityLive(query: SignatureQualityQuery, snapshot = false) { const day = qualityBusinessDay(query.date); const page = positiveInteger(query.page, 1); - const pageSize = Math.min(100, positiveInteger(query.pageSize, 25)); + const pageSize = snapshot ? 2147483647 : Math.min(100, positiveInteger(query.pageSize, 25)); const keyword = query.keyword?.trim() || null; const keywordPattern = keyword ? `%${keyword}%` : null; const summaries = await this.prisma.$queryRaw< @@ -341,6 +364,8 @@ export class OperationsQualityQueries { failureCount: number; successRate: number; averageArrivalMs: number | null; + arrivalMsSum?: number; + arrivalSamples?: number; rowCount: number; }> >(Prisma.sql` @@ -361,6 +386,15 @@ export class OperationsQualityQueries { WHERE message."signatureId" IS NOT NULL AND message."queuedAt" >= ${day.startAt} AND message."queuedAt" < ${day.endAt} + ), dimensions AS ( + SELECT signature_id FROM base + UNION + SELECT message."signatureId" FROM "SmsSubmitRecord" submit + JOIN "SmsMessageRecord" message ON message.id=submit."messageRecordId" + WHERE message."signatureId" IS NOT NULL + AND COALESCE(submit."submittedAt",submit."createdAt")>=${day.startAt} + AND COALESCE(submit."submittedAt",submit."createdAt")<${day.endAt} + AND submit."submitStatus" IN ('accepted','rejected','timeout') ) SELECT signature.id AS "signatureId", @@ -368,37 +402,37 @@ export class OperationsQualityQueries { tenant.id AS "tenantId", tenant.name AS "tenantName", STRING_AGG(DISTINCT application.name, '、') FILTER (WHERE application.name IS NOT NULL) AS "applicationNames", - COUNT(*)::integer AS total, - COUNT(*) FILTER ( + COUNT(base.signature_id)::integer AS total, + COUNT(base.signature_id) FILTER ( WHERE COALESCE(base.status, '') <> 'submit_failed' AND COALESCE(base.submit_status, '') NOT IN ('rejected', 'timeout') )::integer AS "acceptedCount", - COUNT(*) FILTER ( + COUNT(base.signature_id) FILTER ( WHERE base.status = 'submit_failed' OR base.submit_status IN ('rejected', 'timeout') )::integer AS "submitFailureCount", - COUNT(*) FILTER (WHERE base.status = 'delivered' OR base.receipt_status = 'delivered')::integer AS "successCount", - COUNT(*) FILTER ( + COUNT(base.signature_id) FILTER (WHERE base.status = 'delivered' OR base.receipt_status = 'delivered')::integer AS "successCount", + COUNT(base.signature_id) FILTER ( WHERE COALESCE(base.status, '') <> 'submit_failed' AND COALESCE(base.submit_status, '') NOT IN ('rejected', 'timeout') AND NOT (COALESCE(base.status = 'delivered', false) OR COALESCE(base.receipt_status = 'delivered', false)) AND NOT (COALESCE(base.status IN ('failed', 'timeout'), false) OR COALESCE(base.receipt_status = 'undelivered', false)) )::integer AS "unknownCount", - COUNT(*) FILTER ( + COUNT(base.signature_id) FILTER ( WHERE COALESCE(base.status, '') <> 'submit_failed' AND COALESCE(base.submit_status, '') NOT IN ('rejected', 'timeout') AND NOT (COALESCE(base.status = 'delivered', false) OR COALESCE(base.receipt_status = 'delivered', false)) AND (COALESCE(base.status IN ('failed', 'timeout'), false) OR COALESCE(base.receipt_status = 'undelivered', false)) )::integer AS "failureCount", CASE - WHEN COUNT(*) FILTER ( + WHEN COUNT(base.signature_id) FILTER ( WHERE COALESCE(base.status, '') <> 'submit_failed' AND COALESCE(base.submit_status, '') NOT IN ('rejected', 'timeout') ) = 0 THEN 0 ELSE ROUND( - COUNT(*) FILTER (WHERE base.status = 'delivered' OR base.receipt_status = 'delivered') + COUNT(base.signature_id) FILTER (WHERE base.status = 'delivered' OR base.receipt_status = 'delivered') * 100.0 - / COUNT(*) FILTER ( + / COUNT(base.signature_id) FILTER ( WHERE COALESCE(base.status, '') <> 'submit_failed' AND COALESCE(base.submit_status, '') NOT IN ('rejected', 'timeout') ), @@ -407,10 +441,11 @@ export class OperationsQualityQueries { END AS "successRate", ROUND(AVG(base.arrival_ms) FILTER (WHERE base.arrival_ms IS NOT NULL))::integer AS "averageArrivalMs", COUNT(*) OVER()::integer AS "rowCount" - FROM base - JOIN "SmsSignature" signature ON signature.id = base.signature_id + FROM dimensions + JOIN "SmsSignature" signature ON signature.id = dimensions.signature_id + LEFT JOIN base ON base.signature_id=signature.id JOIN "Tenant" tenant ON tenant.id = signature."tenantId" - LEFT JOIN "SmsApplication" application ON application.id = base.application_id + LEFT JOIN "SmsApplication" application ON application.id = COALESCE(base.application_id,signature."applicationId") WHERE ( ${keyword}::text IS NULL OR signature.name ILIKE ${keywordPattern} @@ -441,6 +476,8 @@ export class OperationsQualityQueries { failureCount: number; successRate: number; averageArrivalMs: number | null; + arrivalMsSum?: number; + arrivalSamples?: number; }> >(Prisma.sql` WITH base AS ( @@ -457,12 +494,12 @@ export class OperationsQualityQueries { submit."submitStatus" AS submit_status, receipt."deliveredAt" AS delivered_at, failed_receipt."failedAt" AS failed_at, - COALESCE(segment_summary.segment_count, 0) AS segment_count, + COALESCE(segment_summary.expected_count, 0) AS segment_count, COALESCE(segment_summary.delivered_count, 0) AS segment_delivered_count, COALESCE(segment_summary.failure_count, 0) AS segment_failure_count, CASE WHEN segment_summary.segment_count > 0 - AND segment_summary.delivered_count = segment_summary.segment_count + AND segment_summary.delivered_count = segment_summary.expected_count AND segment_summary.completed_at >= COALESCE(submit."submittedAt", submit."createdAt") THEN EXTRACT(EPOCH FROM (segment_summary.completed_at - COALESCE(submit."submittedAt", submit."createdAt"))) * 1000 WHEN segment_summary.segment_count = 0 @@ -474,6 +511,7 @@ export class OperationsQualityQueries { JOIN "SmsChannel" channel ON channel.id = submit."channelId" LEFT JOIN LATERAL ( SELECT + CASE WHEN COUNT(*) > 0 THEN GREATEST(MAX(segment."segmentTotal"), message."billingUnits") ELSE 0 END::integer AS expected_count, COUNT(*)::integer AS segment_count, COUNT(*) FILTER (WHERE segment."receiptStatus" = 'delivered')::integer AS delivered_count, COUNT(*) FILTER (WHERE segment."receiptStatus" = 'undelivered')::integer AS failure_count, @@ -532,6 +570,8 @@ export class OperationsQualityQueries { 1 )::double precision END AS "successRate", + COALESCE(SUM(arrival_ms) FILTER (WHERE delivery_status = 'success' AND arrival_ms IS NOT NULL),0)::double precision AS "arrivalMsSum", + COUNT(arrival_ms) FILTER (WHERE delivery_status = 'success')::integer AS "arrivalSamples", ROUND(AVG(arrival_ms) FILTER (WHERE delivery_status = 'success' AND arrival_ms IS NOT NULL))::integer AS "averageArrivalMs" FROM classified GROUP BY signature_id, channel_id, carrier, drainage_state @@ -548,6 +588,8 @@ export class OperationsQualityQueries { finalSuccessCount: number; finalSuccessRate: number; averageArrivalMs: number | null; + arrivalMsSum?: number; + arrivalSamples?: number; }> >(Prisma.sql` SELECT @@ -620,6 +662,8 @@ type SignatureSplitRow = { failureCount: number; successRate: number; averageArrivalMs: number | null; + arrivalMsSum?: number; + arrivalSamples?: number; }; function aggregateSignatureRows(rows: SignatureSplitRow[]) { @@ -632,7 +676,7 @@ function aggregateSignatureRows(rows: SignatureSplitRow[]) { const acceptedCount = parts.reduce((sum, item) => sum + item.acceptedCount, 0); const successCount = parts.reduce((sum, item) => sum + item.successCount, 0); const arrivalWeight = parts.reduce( - (sum, item) => sum + (item.averageArrivalMs == null ? 0 : item.successCount), + (sum, item) => sum + (item.arrivalSamples ?? (item.averageArrivalMs == null ? 0 : item.successCount)), 0, ); return { @@ -650,7 +694,10 @@ function aggregateSignatureRows(rows: SignatureSplitRow[]) { arrivalWeight === 0 ? null : Math.round( - parts.reduce((sum, item) => sum + (item.averageArrivalMs ?? 0) * item.successCount, 0) / arrivalWeight, + parts.reduce( + (sum, item) => sum + (item.arrivalMsSum ?? (item.averageArrivalMs ?? 0) * item.successCount), + 0, + ) / arrivalWeight, ), }; }) @@ -676,6 +723,8 @@ type DrainageBreakdownRow = { failureCount: number; successRate: number; averageArrivalMs: number | null; + arrivalMsSum?: number; + arrivalSamples?: number; }; function aggregateChannelCarrierRows(rows: DrainageBreakdownRow[]) { @@ -688,7 +737,10 @@ function aggregateChannelCarrierRows(rows: DrainageBreakdownRow[]) { const first = parts[0]; const acceptedCount = parts.reduce((sum, item) => sum + item.acceptedCount, 0); const successCount = parts.reduce((sum, item) => sum + item.successCount, 0); - const arrivalWeight = parts.reduce((sum, item) => sum + (item.averageArrivalMs == null ? 0 : item.successCount), 0); + const arrivalWeight = parts.reduce( + (sum, item) => sum + (item.arrivalSamples ?? (item.averageArrivalMs == null ? 0 : item.successCount)), + 0, + ); return { signatureId: first.signatureId, channelId: first.channelId, @@ -705,7 +757,10 @@ function aggregateChannelCarrierRows(rows: DrainageBreakdownRow[]) { arrivalWeight === 0 ? null : Math.round( - parts.reduce((sum, item) => sum + (item.averageArrivalMs ?? 0) * item.successCount, 0) / arrivalWeight, + parts.reduce( + (sum, item) => sum + (item.arrivalMsSum ?? (item.averageArrivalMs ?? 0) * item.successCount), + 0, + ) / arrivalWeight, ), }; }); diff --git a/api/src/send-chain/send-chain.service.spec.ts b/api/src/send-chain/send-chain.service.spec.ts index cf61408..43514ed 100644 --- a/api/src/send-chain/send-chain.service.spec.ts +++ b/api/src/send-chain/send-chain.service.spec.ts @@ -298,6 +298,10 @@ function createPrismaMock() { update: jest.fn().mockResolvedValue({ id: 'candidate-1', status: 'claimed' }), }, cmppDownstreamDelivery: { + createMany: jest.fn().mockResolvedValue({ count: 1 }), + findUniqueOrThrow: jest + .fn() + .mockResolvedValue({ id: 'delivery-1', messageRecordId: 'record-1', applicationId: 'app-1' }), create: jest .fn() .mockImplementation(({ data }) => @@ -4455,6 +4459,7 @@ describe('SendChainService', () => { it('records ambiguous uplink match candidates for shared access numbers', async () => { const { service, prisma } = createService(); + prisma.smsMessageRecord.findMany.mockResolvedValue([]); prisma.channelRouteRule.findMany.mockResolvedValue([{ applicationId: 'app-1' }, { applicationId: 'app-2' }]); prisma.smsApplication.findMany.mockResolvedValue([ { id: 'app-1', tenantId: 'tenant-1', name: '应用A' }, @@ -4475,7 +4480,7 @@ describe('SendChainService', () => { tenantId: undefined, applicationId: undefined, matchStatus: 'ambiguous', - matchReason: '接入号匹配多个应用', + matchReason: '应用归属仍有多候选或接入号与发送事实冲突', }), }); expect(prisma.smsUplinkMatchCandidate.createMany).toHaveBeenCalledWith({ @@ -4517,15 +4522,18 @@ describe('SendChainService', () => { where: { uplinkMessageId: 'uplink-1', id: { not: 'candidate-1' }, status: 'pending' }, data: { status: 'rejected' }, }); - expect(prisma.cmppDownstreamDelivery.create).toHaveBeenCalledWith({ - data: expect.objectContaining({ - tenantId: 'tenant-1', - applicationId: 'app-1', - messageRecordId: 'record-1', - messageId: 'MSG-1', - deliveryType: 'uplink', - status: 'pending', - }), + expect(prisma.cmppDownstreamDelivery.createMany).toHaveBeenCalledWith({ + data: [ + expect.objectContaining({ + tenantId: 'tenant-1', + applicationId: 'app-1', + messageRecordId: 'record-1', + messageId: 'MSG-1', + deliveryType: 'uplink', + status: 'pending', + }), + ], + skipDuplicates: true, }); }); diff --git a/api/src/send-chain/send-chain.service.ts b/api/src/send-chain/send-chain.service.ts index b0e26de..a91c86b 100644 --- a/api/src/send-chain/send-chain.service.ts +++ b/api/src/send-chain/send-chain.service.ts @@ -698,6 +698,7 @@ export class SendChainService implements OnModuleInit, OnModuleDestroy { tenantId?: string; applicationId?: string; messageRecordId?: string; + messageId?: string; matchStatus: string; matchReason: string; candidates: UplinkMatchCandidateInput[]; diff --git a/api/src/send-chain/send-completion.service.ts b/api/src/send-chain/send-completion.service.ts index 402d3c9..6178d4d 100644 --- a/api/src/send-chain/send-completion.service.ts +++ b/api/src/send-chain/send-completion.service.ts @@ -1,7 +1,21 @@ import { BillingService } from '../billing/billing.service'; import type { OpenApiService } from '../open-api/open-api.service'; import { PrismaService } from '../prisma/prisma.service'; -import type { GatewaySubmitResultDto, GatewaySubmitSegmentResultDto, GatewayReceiptEventDto, GatewayUplinkEventDto, UplinkMatchCandidateInput, GatewayPendingDeliveryQueryDto, GatewayDownstreamSentDto, GatewayDownstreamAcknowledgedDto, GatewayDownstreamFailureType, GatewaySubmitDeadLetterDto, RequeueGatewaySubmitExceptionDto, GatewayDownstreamRecoveryStatusDto, TimeoutUnknownDto } from './send-chain.contracts'; +import type { + GatewaySubmitResultDto, + GatewaySubmitSegmentResultDto, + GatewayReceiptEventDto, + GatewayUplinkEventDto, + UplinkMatchCandidateInput, + GatewayPendingDeliveryQueryDto, + GatewayDownstreamSentDto, + GatewayDownstreamAcknowledgedDto, + GatewayDownstreamFailureType, + GatewaySubmitDeadLetterDto, + RequeueGatewaySubmitExceptionDto, + GatewayDownstreamRecoveryStatusDto, + TimeoutUnknownDto, +} from './send-chain.contracts'; import { downstreamPendingTimeoutHours } from './send-chain.helpers'; import type { SendSubmissionService } from './send-submission.service'; import { SendAccountingService } from './send-accounting.service'; @@ -13,7 +27,6 @@ import { SendRetryService } from './send-retry.service'; import { SendTimeoutService } from './send-timeout.service'; import type { DownstreamDeliveryQueueRequest } from './downstream-receipt-targets'; - export type SendCompletionCallbacks = Record; export type SendCompletionFacade = SendCompletionService & SendSubmissionService; @@ -49,10 +62,7 @@ export class SendCompletionService { return this.gatewayResult.handleSubmitSegmentResult(data); } - async resolveSubmitRecordForGatewaySegmentResult( - messageRecordId: string, - data: GatewaySubmitSegmentResultDto, - ) { + async resolveSubmitRecordForGatewaySegmentResult(messageRecordId: string, data: GatewaySubmitSegmentResultDto) { return this.gatewayResult.resolveSubmitRecordForGatewaySegmentResult(messageRecordId, data); } @@ -109,7 +119,13 @@ export class SendCompletionService { async handleReceipt( data: GatewayReceiptEventDto, - incomingIdentity?: { account: string; gatewayHost: string; gatewayPort: number; protocol: string; cmppVersion: string }, + incomingIdentity?: { + account: string; + gatewayHost: string; + gatewayPort: number; + protocol: string; + cmppVersion: string; + }, ) { return this.receipt.handleReceipt(data, incomingIdentity); } @@ -145,7 +161,13 @@ export class SendCompletionService { async resolveReceiptMessage( data: GatewayReceiptEventDto, - incomingIdentity?: { account: string; gatewayHost: string; gatewayPort: number; protocol: string; cmppVersion: string }, + incomingIdentity?: { + account: string; + gatewayHost: string; + gatewayPort: number; + protocol: string; + cmppVersion: string; + }, ) { return this.receipt.resolveReceiptMessage(data, incomingIdentity); } @@ -206,7 +228,13 @@ export class SendCompletionService { } async releaseMessageReservation( - message: { tenantId: string; batchTaskId: string; messageId: string; amountCents: number | bigint; billingUnits: number }, + message: { + tenantId: string; + batchTaskId: string; + messageId: string; + amountCents: number | bigint; + billingUnits: number; + }, remark: string, ) { return this.accounting.releaseMessageReservation(message, remark); @@ -279,6 +307,7 @@ export class SendCompletionService { tenantId?: string; applicationId?: string; messageRecordId?: string; + messageId?: string; matchStatus: string; matchReason: string; candidates: UplinkMatchCandidateInput[]; diff --git a/api/src/send-chain/send-downstream-delivery.service.ts b/api/src/send-chain/send-downstream-delivery.service.ts index 0baf9cf..9cf5c86 100644 --- a/api/src/send-chain/send-downstream-delivery.service.ts +++ b/api/src/send-chain/send-downstream-delivery.service.ts @@ -1,4 +1,5 @@ import { completionContext } from './completion-context'; +import { resolveUplinkMatch } from './uplink-matching'; import { BadRequestException, Logger, NotFoundException } from '@nestjs/common'; import { Prisma } from '@prisma/client'; import { createHash, randomUUID } from 'node:crypto'; @@ -32,7 +33,19 @@ export class SendDownstreamDeliveryService { ) {} async handleUplink(data: GatewayUplinkEventDto) { + if (!completionContext.getStore()) { + return this.prisma.$transaction( + (tx) => completionContext.run({ tx, messageRecordId: '' }, () => this.persistUplink(data)), + { timeout: 15_000 }, + ); + } + return this.persistUplink(data); + } + + private async persistUplink(data: GatewayUplinkEventDto) { if (data.eventId) { + await this.prisma + .$executeRaw`SELECT pg_advisory_xact_lock(hashtextextended(${`uplink-event:${data.eventId}`},0))`; const existing = await this.prisma.smsUplinkMessage.findUnique({ where: { eventId: data.eventId } }); if (existing) return existing; } @@ -48,7 +61,7 @@ export class SendDownstreamDeliveryService { applicationId: match.applicationId, messageRecordId: match.messageRecordId, channelId: data.channelId, - messageId: data.messageId, + messageId: match.messageId, gatewayMessageId: data.gatewayMessageId, sequenceId: data.sequenceId, phoneNumber: data.phoneNumber, @@ -78,10 +91,10 @@ export class SendDownstreamDeliveryService { tenantId: match.tenantId, applicationId: match.applicationId, messageRecordId: match.messageRecordId, - messageId: data.messageId, + messageId: match.messageId, deliveryType: 'uplink', payload: { - messageId: data.messageId, + messageId: match.messageId, applicationId: match.applicationId, phoneNumber: data.phoneNumber, destId: data.destId, @@ -95,6 +108,21 @@ export class SendDownstreamDeliveryService { } async claimUplinkMatchCandidate(uplinkMessageId: string, candidateId: string, operatorId?: string) { + if (!completionContext.getStore()) { + return this.prisma.$transaction( + (tx) => + completionContext.run({ tx, messageRecordId: '' }, () => + this.persistUplinkClaim(uplinkMessageId, candidateId, operatorId), + ), + { timeout: 15_000 }, + ); + } + return this.persistUplinkClaim(uplinkMessageId, candidateId, operatorId); + } + + private async persistUplinkClaim(uplinkMessageId: string, candidateId: string, operatorId?: string) { + await this.prisma + .$executeRaw`SELECT pg_advisory_xact_lock(hashtextextended(${`uplink-claim:${uplinkMessageId}`},0))`; const candidate = await this.prisma.smsUplinkMatchCandidate.findFirst({ where: { id: candidateId, uplinkMessageId }, include: { @@ -112,56 +140,55 @@ export class SendDownstreamDeliveryService { if (candidate.uplinkMessage.matchStatus === 'matched' && candidate.status !== 'claimed') { throw new BadRequestException('该上行记录已完成匹配，不能重复认领'); } + if (candidate.status === 'claimed') return candidate.uplinkMessage; const claimedAt = new Date(); - const messageId = candidate.uplinkMessage.messageId ?? candidate.messageRecord?.messageId ?? null; - const [updatedUplink] = await this.prisma.$transaction([ - this.prisma.smsUplinkMessage.update({ - where: { id: uplinkMessageId }, - data: { - tenantId: candidate.tenantId, + const messageId = candidate.messageRecord?.messageId ?? null; + const updatedUplink = await this.prisma.smsUplinkMessage.update({ + where: { id: uplinkMessageId }, + data: { + tenantId: candidate.tenantId, + applicationId: candidate.applicationId, + messageRecordId: candidate.messageRecordId, + messageId, + matchStatus: 'matched', + matchReason: `人工认领：${candidate.reason ?? candidate.matchSource}`, + }, + }); + await this.prisma.smsUplinkMatchCandidate.updateMany({ + where: { + uplinkMessageId, + id: { not: candidate.id }, + status: 'pending', + }, + data: { status: 'rejected' }, + }); + await this.prisma.smsUplinkMatchCandidate.update({ + where: { id: candidate.id }, + data: { + status: 'claimed', + claimedAt, + claimedById: operatorId, + }, + }); + await this.prisma.operationLog.create({ + data: { + tenantId: candidate.tenantId, + userId: operatorId, + action: 'gateway.uplink_manual_claim', + resource: 'sms_uplink_message', + resourceId: uplinkMessageId, + detail: { + candidateId: candidate.id, applicationId: candidate.applicationId, + applicationName: candidate.application.name, messageRecordId: candidate.messageRecordId, messageId, - matchStatus: 'matched', - matchReason: `人工认领：${candidate.reason ?? candidate.matchSource}`, + matchSource: candidate.matchSource, + phoneNumber: candidate.uplinkMessage.phoneNumber, + destId: candidate.uplinkMessage.destId, }, - }), - this.prisma.smsUplinkMatchCandidate.updateMany({ - where: { - uplinkMessageId, - id: { not: candidate.id }, - status: 'pending', - }, - data: { status: 'rejected' }, - }), - this.prisma.smsUplinkMatchCandidate.update({ - where: { id: candidate.id }, - data: { - status: 'claimed', - claimedAt, - claimedById: operatorId, - }, - }), - this.prisma.operationLog.create({ - data: { - tenantId: candidate.tenantId, - userId: operatorId, - action: 'gateway.uplink_manual_claim', - resource: 'sms_uplink_message', - resourceId: uplinkMessageId, - detail: { - candidateId: candidate.id, - applicationId: candidate.applicationId, - applicationName: candidate.application.name, - messageRecordId: candidate.messageRecordId, - messageId, - matchSource: candidate.matchSource, - phoneNumber: candidate.uplinkMessage.phoneNumber, - destId: candidate.uplinkMessage.destId, - }, - }, - }), - ]); + }, + }); await this.facade.queueAndTryDownstreamDelivery({ tenantId: candidate.tenantId, @@ -278,7 +305,10 @@ export class SendDownstreamDeliveryService { skipDuplicates: true, }); const retained = await this.prisma.cmppDownstreamDelivery.findUniqueOrThrow({ where: { dedupeKey } }); - if (retained.messageRecordId !== data.messageRecordId || retained.applicationId !== data.applicationId) + if ( + (retained.messageRecordId ?? null) !== (data.messageRecordId ?? null) || + retained.applicationId !== data.applicationId + ) throw new Error('completion_notification_identity_mismatch'); return retained; } @@ -377,107 +407,12 @@ export class SendDownstreamDeliveryService { tenantId?: string; applicationId?: string; messageRecordId?: string; + messageId?: string; matchStatus: string; matchReason: string; candidates: UplinkMatchCandidateInput[]; }> { - if (data.messageId) { - const message = await this.prisma.smsMessageRecord.findUnique({ where: { messageId: data.messageId } }); - if (message?.tenantId) { - return { - tenantId: message.tenantId, - applicationId: message.applicationId ?? undefined, - messageRecordId: message.id, - matchStatus: message.applicationId ? 'matched' : 'unmatched', - matchReason: message.applicationId ? 'messageId 精确匹配' : 'messageId 匹配到下发记录但无应用', - candidates: [], - }; - } - } - - const accessNumber = data.destId || channel.srcId || ''; - const accessRoutes = accessNumber - ? await this.prisma.channelRouteRule.findMany({ - where: { - applicationId: { not: null }, - status: 'active', - group: { items: { some: { channelId: channel.id, channel: { srcId: accessNumber } } } }, - }, - select: { applicationId: true }, - take: 10, - }) - : []; - const accessApplicationIds = [ - ...new Set(accessRoutes.map((route) => route.applicationId).filter((value): value is string => Boolean(value))), - ]; - const accessApplications = - accessApplicationIds.length > 0 - ? await this.prisma.smsApplication.findMany({ - where: { id: { in: accessApplicationIds }, status: 'active' }, - select: { id: true, tenantId: true, name: true }, - }) - : []; - if (accessApplications.length === 1) { - return { - tenantId: accessApplications[0].tenantId, - applicationId: accessApplications[0].id, - matchStatus: 'matched', - matchReason: '接入号唯一匹配应用', - candidates: [], - }; - } - if (accessApplications.length > 1) { - return { - matchStatus: 'ambiguous', - matchReason: '接入号匹配多个应用', - candidates: accessApplications.map((application) => ({ - tenantId: application.tenantId, - applicationId: application.id, - matchSource: 'access_number', - confidence: 70, - reason: `接入号 ${accessNumber} 可匹配应用 ${application.name}`, - })), - }; - } - - const windowHours = Number(process.env.UPLINK_MATCH_WINDOW_HOURS ?? 72); - const since = new Date(Date.now() - Math.max(1, windowHours) * 60 * 60 * 1000); - const recentMessages = await this.prisma.smsMessageRecord.findMany({ - where: { - phoneNumber: data.phoneNumber, - tenantId: { not: null }, - applicationId: { not: null }, - submittedAt: { gte: since }, - }, - orderBy: { submittedAt: 'desc' }, - take: 2, - }); - const matchableRecentMessages = recentMessages.filter((message) => message.tenantId && message.applicationId); - if (matchableRecentMessages.length === 1) { - return { - tenantId: matchableRecentMessages[0].tenantId ?? undefined, - applicationId: matchableRecentMessages[0].applicationId ?? undefined, - messageRecordId: matchableRecentMessages[0].id, - matchStatus: 'matched', - matchReason: `手机号 ${windowHours} 小时窗口唯一匹配`, - candidates: [], - }; - } - if (matchableRecentMessages.length > 1) { - return { - matchStatus: 'ambiguous', - matchReason: `手机号 ${windowHours} 小时窗口匹配多条下发记录`, - candidates: matchableRecentMessages.map((message) => ({ - tenantId: String(message.tenantId), - applicationId: String(message.applicationId), - messageRecordId: message.id, - matchSource: 'phone_window', - confidence: 55, - reason: `手机号 ${windowHours} 小时窗口候选下发 ${message.messageId}`, - })), - }; - } - return { matchStatus: 'unmatched', matchReason: '未匹配到应用或下发记录', candidates: [] }; + return resolveUplinkMatch(this.prisma, data, channel); } async recordCmppFailureReceipt( diff --git a/api/src/send-chain/uplink-matching.ts b/api/src/send-chain/uplink-matching.ts new file mode 100644 index 0000000..dbe9a22 --- /dev/null +++ b/api/src/send-chain/uplink-matching.ts @@ -0,0 +1,146 @@ +import { BadRequestException } from '@nestjs/common'; +import { PrismaService } from '../prisma/prisma.service'; +import type { GatewayUplinkEventDto, UplinkMatchCandidateInput } from './send-chain.contracts'; + +export type UplinkMatch = { + tenantId?: string; + applicationId?: string; + messageRecordId?: string; + messageId?: string; + matchStatus: string; + matchReason: string; + candidates: UplinkMatchCandidateInput[]; +}; + +/** Attribution is application-level; a reply need not identify one original SMS. */ +export async function resolveUplinkMatch( + db: PrismaService, + data: GatewayUplinkEventDto, + channel: { id: string; srcId?: string | null }, +): Promise { + const receivedAt = data.receivedAt ? new Date(data.receivedAt) : new Date(); + if (!Number.isFinite(receivedAt.getTime())) throw new BadRequestException('上行接收时间无效'); + const configuredHours = Number(process.env.UPLINK_MATCH_WINDOW_HOURS ?? 72); + const hours = + Number.isFinite(configuredHours) && configuredHours >= 1 && configuredHours <= 8760 ? configuredHours : 72; + const since = new Date(receivedAt.getTime() - hours * 3_600_000); + const channelEvidence = { + channelId: channel.id, + submitStatus: 'accepted', + submittedAt: { gte: since, lte: receivedAt }, + }; + if (data.messageId) { + const message = await db.smsMessageRecord.findFirst({ + where: { + messageId: data.messageId, + phoneNumber: data.phoneNumber, + tenantId: { not: null }, + applicationId: { not: null }, + submitRecords: { some: channelEvidence }, + }, + select: { id: true, messageId: true, tenantId: true, applicationId: true }, + }); + if (message?.tenantId && message.applicationId) + return { + tenantId: message.tenantId, + applicationId: message.applicationId, + messageRecordId: message.id, + messageId: message.messageId, + matchStatus: 'matched', + matchReason: 'messageId 与手机号、通道发送事实一致', + candidates: [], + }; + } + const accessNumber = data.destId || channel.srcId || ''; + // Do not truncate routes before deduplicating applications: it can manufacture uniqueness. + const routes = accessNumber + ? await db.channelRouteRule.findMany({ + where: { + applicationId: { not: null }, + status: 'active', + group: { items: { some: { channelId: channel.id, channel: { srcId: accessNumber } } } }, + }, + select: { applicationId: true }, + distinct: ['applicationId'], + }) + : []; + const ids = routes.flatMap((r) => (r.applicationId ? [r.applicationId] : [])); + const applications = ids.length + ? await db.smsApplication.findMany({ + where: { id: { in: ids }, status: 'active' }, + select: { id: true, tenantId: true, name: true }, + }) + : []; + // Read only attribution columns, but inspect the complete window, not its last two SMS. + const messages = await db.smsMessageRecord.findMany({ + where: { + phoneNumber: data.phoneNumber, + tenantId: { not: null }, + applicationId: { not: null }, + submitRecords: { some: channelEvidence }, + }, + select: { id: true, messageId: true, tenantId: true, applicationId: true }, + orderBy: { id: 'asc' }, + }); + const groups = new Map(); + for (const message of messages) { + if (!message.tenantId || !message.applicationId) continue; + const key = JSON.stringify([message.tenantId, message.applicationId]); + const group = groups.get(key) ?? []; + group.push(message); + groups.set(key, group); + } + const accessCandidates: UplinkMatchCandidateInput[] = applications.map((a) => ({ + tenantId: a.tenantId, + applicationId: a.id, + matchSource: 'access_number', + confidence: 70, + reason: '共享接入号应用候选，尚无唯一发送证据', + })); + if (groups.size === 1) { + const records = [...groups.values()][0]; + const message = records[0]; + // A conflicting configured access number is evidence against automatic assignment. + if (!applications.length || applications.some((a) => a.id === message.applicationId)) + return { + tenantId: message.tenantId!, + applicationId: message.applicationId!, + messageRecordId: records.length === 1 ? message.id : undefined, + messageId: records.length === 1 ? message.messageId : undefined, + matchStatus: 'matched', + matchReason: + records.length === 1 + ? `手机号、通道及接收前 ${hours} 小时唯一匹配` + : `手机号、通道及接收前 ${hours} 小时应用唯一；原短信不唯一`, + candidates: [], + }; + } + // No sending evidence: retain the existing unique-access application attribution. + if (!groups.size && applications.length === 1) + return { + tenantId: applications[0].tenantId, + applicationId: applications[0].id, + matchStatus: 'matched', + matchReason: '接入号唯一匹配应用，无唯一原短信', + candidates: [], + }; + const candidates = new Map(accessCandidates.map((c) => [c.applicationId, c])); + for (const records of groups.values()) { + const m = records[0]; + candidates.set(m.applicationId!, { + tenantId: m.tenantId!, + applicationId: m.applicationId!, + messageRecordId: records.length === 1 ? m.id : undefined, + matchSource: 'phone_window', + confidence: 70, + reason: `同通道接收前 ${hours} 小时有 ${records.length} 条下发；须确认应用归属`, + }); + } + return candidates.size + ? { + matchStatus: 'ambiguous', + matchReason: '应用归属仍有多候选或接入号与发送事实冲突', + candidates: [...candidates.values()], + } + : { matchStatus: 'unmatched', matchReason: '未匹配到接入号应用或时间窗内同通道发送事实', candidates: [] }; +} diff --git a/api/src/signature-analytics/analytics-aggregate.ts b/api/src/signature-analytics/analytics-aggregate.ts new file mode 100644 index 0000000..7384aa4 --- /dev/null +++ b/api/src/signature-analytics/analytics-aggregate.ts @@ -0,0 +1,124 @@ +import { Prisma } from '@prisma/client'; +import { PrismaService } from '../prisma/prisma.service'; +import { startOfDay, addDays } from './analytics-date'; + +export type ActivityDimension = { + dimensionKey: string; + dimensionType: string; + tenantId: string; + applicationId: string | null; + signatureId: string; + channelKey: string; + carrier: string; + approvedAt: Date; + signatureName: string; + tenantName: string; + applicationName: string; + channelName: string; +}; +export type ActivityCount = { + submittedAttempts: number; + acceptedBusinessCount: number; + deliveredBusinessCount: number; +}; + +/** Reconstruct membership at the end of the activity day, not from today's active routes. */ +export async function activityDimensions(db: PrismaService, date: string): Promise { + const end = startOfDay(addDays(date, 1)); + const tasks = await db.$queryRaw>>(Prisma.sql` + SELECT t."signatureId",t."channelId" AS "channelKey",t.carrier, + s."tenantId",s."applicationId",s.name AS "signatureName",c.name AS "channelName", + tenant.name AS "tenantName",COALESCE(a.name,'') AS "applicationName", + COALESCE(approved."createdAt",t."approvedAt") AS "approvedAt" + FROM "ChannelSignatureReportTask" t + JOIN "SmsSignature" s ON s.id=t."signatureId" JOIN "Tenant" tenant ON tenant.id=s."tenantId" + JOIN "SmsChannel" c ON c.id=t."channelId" LEFT JOIN "SmsApplication" a ON a.id=s."applicationId" + LEFT JOIN LATERAL (SELECT r."statusAfter" FROM "ChannelSignatureReportRecord" r + WHERE r."taskId"=t.id AND r."createdAt"<${end} ORDER BY r."createdAt" DESC,r.id DESC LIMIT 1) history ON TRUE + LEFT JOIN LATERAL (SELECT r."createdAt" FROM "ChannelSignatureReportRecord" r + WHERE r."taskId"=t.id AND r."createdAt"<${end} AND r."statusAfter"='approved' + AND r."statusBefore" IS DISTINCT FROM 'approved' ORDER BY r."createdAt" DESC,r.id DESC LIMIT 1) approved ON TRUE + WHERE t."reportType"='signature' AND t."approvalScope"='carrier_specific' AND t.carrier IS NOT NULL + AND t."createdAt"<${end} + AND COALESCE(history."statusAfter",CASE WHEN t."approvedAt"<${end} THEN t.status END)='approved' + AND COALESCE(approved."createdAt",t."approvedAt")<${end}`); + const dimensions = new Map(); + for (const t of tasks) { + const channelKey = JSON.stringify(['channel', t.tenantId, t.applicationId, t.signatureId, t.channelKey, t.carrier]); + dimensions.set(channelKey, { ...t, dimensionKey: channelKey, dimensionType: 'channel' }); + const enterpriseKey = JSON.stringify(['enterprise', t.tenantId, t.applicationId, t.signatureId, '', t.carrier]); + const existing = dimensions.get(enterpriseKey); + if (!existing || existing.approvedAt > t.approvedAt) + dimensions.set(enterpriseKey, { + ...t, + dimensionKey: enterpriseKey, + dimensionType: 'enterprise', + channelKey: '', + channelName: '', + }); + } + return [...dimensions.values()]; +} + +/** One bounded source scan for all dimensions; no per-signature correlated receipt scan. */ +export async function activityCounts(db: PrismaService, dimensions: ActivityDimension[], date: string) { + if (!dimensions.length) return new Map(); + const start = startOfDay(date), + end = startOfDay(addDays(date, 1)); + const json = JSON.stringify( + dimensions.map((d) => ({ + key: d.dimensionKey, + signature: d.signatureId, + channel: d.channelKey, + carrier: d.carrier, + approved: d.approvedAt.toISOString(), + })), + ); + const rows = await db.$queryRaw>(Prisma.sql` + WITH dims AS (SELECT * FROM jsonb_to_recordset(${json}::jsonb) AS d(key text,signature text,channel text,carrier text,approved timestamptz)), + attempts AS MATERIALIZED ( + SELECT s.id,s."messageRecordId",s."channelId",s."gatewayMessageId",s."submitStatus",m."signatureId",m.carrier,m."billingUnits", + COALESCE(s."submittedAt",s."createdAt") AS at + FROM "SmsSubmitRecord" s JOIN "SmsMessageRecord" m ON m.id=s."messageRecordId" + WHERE COALESCE(s."submittedAt",s."createdAt")>=${start} AND COALESCE(s."submittedAt",s."createdAt")<${end} + AND m."signatureId" IS NOT NULL + ), segments AS ( + SELECT a.id,COUNT(g.id)::int AS present,GREATEST(MAX(g."segmentTotal"),MAX(a."billingUnits")) AS expected, + COUNT(g.id) FILTER(WHERE g."receiptStatus"='delivered') AS delivered + FROM attempts a LEFT JOIN "SmsMessageSegmentAudit" g ON g."submitRecordId"=a.id GROUP BY a.id + ), delivered AS ( + SELECT DISTINCT a.id FROM attempts a JOIN segments g ON g.id=a.id + WHERE (g.present>0 AND g.delivered=g.expected AND g.present=g.expected) + OR (g.present=0 AND EXISTS(SELECT 1 FROM "SmsReceiptRecord" r + WHERE r."channelId"=a."channelId" AND r."gatewayMessageId"=a."gatewayMessageId" AND r."receiptStatus"='delivered')) + ) + SELECT d.key,COUNT(a.id)::int AS "submittedAttempts", + COUNT(DISTINCT a."messageRecordId") FILTER(WHERE a."submitStatus"='accepted')::int AS "acceptedBusinessCount", + COUNT(DISTINCT a."messageRecordId") FILTER(WHERE a."submitStatus"='accepted' AND delivered.id IS NOT NULL)::int AS "deliveredBusinessCount" + FROM dims d LEFT JOIN attempts a ON a."signatureId"=d.signature AND a.carrier=d.carrier + AND (d.channel='' OR a."channelId"=d.channel) AND a.at>=d.approved AT TIME ZONE 'UTC' + LEFT JOIN delivered ON delivered.id=a.id GROUP BY d.key`); + return new Map(rows.map(({ key, ...counts }) => [key, counts])); +} + +export async function unreportedRows(db: PrismaService, date: string) { + return db.$queryRaw< + Array<{ + dimensionKey: string; + tenantId: string; + applicationId: string; + signatureName: string; + tenantName: string; + applicationName: string; + messageCount: number; + }> + >(Prisma.sql` + WITH extracted AS ( + SELECT m."tenantId",m."applicationId",SUBSTRING(m.content FROM '^【[^【】]+】') AS name + FROM "SmsMessageRecord" m WHERE m."queuedAt">=${startOfDay(date)} AND m."queuedAt"<${startOfDay(addDays(date, 1))} AND m."signatureId" IS NULL + ) SELECT jsonb_build_array(e."tenantId",e."applicationId",e.name)::text AS "dimensionKey", + e."tenantId",e."applicationId",e.name AS "signatureName",t.name AS "tenantName",a.name AS "applicationName",COUNT(*)::int AS "messageCount" + FROM extracted e JOIN "Tenant" t ON t.id=e."tenantId" JOIN "SmsApplication" a ON a.id=e."applicationId" + WHERE e.name IS NOT NULL AND NOT EXISTS(SELECT 1 FROM "SmsSignature" s WHERE s."tenantId"=e."tenantId" AND s."applicationId"=e."applicationId" AND s.name=e.name AND s."auditStatus"<>'deleted') + GROUP BY e."tenantId",e."applicationId",e.name,t.name,a.name`); +} diff --git a/api/src/signature-analytics/analytics-date.spec.ts b/api/src/signature-analytics/analytics-date.spec.ts new file mode 100644 index 0000000..1e2ff4b --- /dev/null +++ b/api/src/signature-analytics/analytics-date.spec.ts @@ -0,0 +1,23 @@ +import { addDays, analyticsDate, analyticsPage, mutableDay, todayKey } from './analytics-date'; + +describe('signature analytics business-day contract', () => { + const now = new Date('2026-09-16T16:00:00Z'); + it('uses Shanghai midnight and freezes T-4', () => { + expect(todayKey(now)).toBe('2026-09-17'); + expect(mutableDay('2026-09-14', now)).toBe(true); + expect(mutableDay('2026-09-13', now)).toBe(false); + expect(mutableDay('2026-09-17', now)).toBe(false); + expect(addDays('2024-03-01', -1)).toBe('2024-02-29'); + }); + it.each(['2026-02-29', '2026-09-18', '2026-13-01', '2026-9-1'])('rejects invalid or future date %s', (date) => { + expect(() => analyticsDate(date, now)).toThrow(); + }); + it.each([ + [0, 25], + [1, 1000], + [Number.NaN, 25], + [1.5, 25], + ])('rejects invalid pagination', (page, size) => { + expect(() => analyticsPage(page, size)).toThrow(); + }); +}); diff --git a/api/src/signature-analytics/analytics-date.ts b/api/src/signature-analytics/analytics-date.ts new file mode 100644 index 0000000..716d138 --- /dev/null +++ b/api/src/signature-analytics/analytics-date.ts @@ -0,0 +1,31 @@ +import { BadRequestException } from '@nestjs/common'; + +const dayFormatter = new Intl.DateTimeFormat('en-CA', { + timeZone: 'Asia/Shanghai', + year: 'numeric', + month: '2-digit', + day: '2-digit', +}); +export const todayKey = (now = new Date()) => dayFormatter.format(now); +export const databaseDay = (key: string) => new Date(`${key}T00:00:00.000Z`); +export const startOfDay = (key: string) => new Date(`${key}T00:00:00+08:00`); +export const addDays = (key: string, days: number) => + new Date(databaseDay(key).getTime() + days * 86_400_000).toISOString().slice(0, 10); +export function analyticsDate(value?: string, now = new Date()) { + const key = value || todayKey(now); + if ( + !/^\d{4}-\d{2}-\d{2}$/.test(key) || + !Number.isFinite(databaseDay(key).getTime()) || + databaseDay(key).toISOString().slice(0, 10) !== key || + key > todayKey(now) + ) { + throw new BadRequestException('统计日期必须为有效的北京时间日期，不能晚于今天'); + } + return key; +} +export function analyticsPage(page = 1, pageSize = 25) { + if (!Number.isInteger(page) || page < 1 || ![10, 25, 50, 100].includes(pageSize)) + throw new BadRequestException('分页参数无效'); + return { page, pageSize }; +} +export const mutableDay = (day: string, now = new Date()) => day < todayKey(now) && day >= addDays(todayKey(now), -3); diff --git a/api/src/signature-analytics/analytics-job.ts b/api/src/signature-analytics/analytics-job.ts new file mode 100644 index 0000000..bf46130 --- /dev/null +++ b/api/src/signature-analytics/analytics-job.ts @@ -0,0 +1,92 @@ +import { randomUUID } from 'node:crypto'; +import { Prisma } from '@prisma/client'; +import { PrismaService } from '../prisma/prisma.service'; +import { databaseDay, todayKey } from './analytics-date'; + +/** Lease acquisition is outside the source snapshot. Publishing always rechecks its fencing token. */ +export async function analyticsJob( + db: PrismaService, + scope: string, + date: string, + work: (tx: PrismaService, generation: string, checkpoint: Prisma.JsonValue | null) => Promise, + now = new Date(), + prepare?: () => Promise, +): Promise<{ skipped: boolean; result?: T }> { + const businessDate = databaseDay(date), + refreshFor = databaseDay(todayKey(now)); + await db.signatureAnalyticsRun.createMany({ + data: [{ scope, businessDate, refreshFor, nextAttemptAt: now, generationId: randomUUID() }], + skipDuplicates: true, + }); + const owner = randomUUID(), + generationId = randomUUID(); + const claimed = await db.signatureAnalyticsRun.updateMany({ + where: { + scope, + businessDate, + AND: [ + { OR: [{ leaseUntil: null }, { leaseUntil: { lt: now } }] }, + { + OR: [ + { refreshFor: { lt: refreshFor } }, + { state: { not: 'succeeded' }, attempt: { lt: 5 }, nextAttemptAt: { lte: now } }, + ], + }, + ], + }, + data: { + owner, + generationId, + leaseUntil: new Date(now.getTime() + 300_000), + fence: { increment: 1 }, + state: 'running', + startedAt: now, + error: null, + }, + }); + if (!claimed.count) return { skipped: true }; + const run = await db.signatureAnalyticsRun.findUniqueOrThrow({ + where: { scope_businessDate: { scope, businessDate } }, + }); + const attempt = run.refreshFor < refreshFor ? 1 : run.attempt + 1; + await db.signatureAnalyticsRun.update({ where: { id: run.id }, data: { refreshFor, attempt } }); + try { + // Preserve the first decision's rule/report snapshot across retries. New-day runs take a new snapshot. + const checkpoint = run.refreshFor < refreshFor ? null : run.checkpoint; + const prepared = checkpoint ?? (prepare ? await prepare() : null); + if (prepared !== null && checkpoint === null) { + const saved = await db.signatureAnalyticsRun.updateMany({ + where: { id: run.id, owner, fence: run.fence, state: 'running' }, + data: { checkpoint: prepared as Prisma.InputJsonValue }, + }); + if (saved.count !== 1) throw new Error('签名统计任务认领已失效'); + } + const result = await db.$transaction( + async (tx) => { + await tx.$executeRawUnsafe("SET LOCAL statement_timeout='90s'"); + const result = await work(tx as PrismaService, generationId, prepared as Prisma.JsonValue | null); + const current = new Date(); + const fenced = await tx.signatureAnalyticsRun.updateMany({ + where: { id: run.id, owner, fence: run.fence, state: 'running', leaseUntil: { gt: current } }, + data: { state: 'succeeded', owner: null, leaseUntil: null, finishedAt: current, error: null }, + }); + if (fenced.count !== 1) throw new Error('签名统计任务租约已失效，拒绝发布'); + return result; + }, + { isolationLevel: 'RepeatableRead', timeout: 120_000, maxWait: 5000 }, + ); + return { skipped: false, result }; + } catch (error) { + await db.signatureAnalyticsRun.updateMany({ + where: { id: run.id, owner, fence: run.fence }, + data: { + state: attempt >= 5 ? 'failed' : 'retry_wait', + owner: null, + leaseUntil: null, + nextAttemptAt: new Date(Date.now() + Math.min(900_000, 60_000 * 2 ** (attempt - 1))), + error: '签名统计生成失败，请查看服务日志', + }, + }); + throw error; + } +} diff --git a/api/src/signature-analytics/analytics-read.ts b/api/src/signature-analytics/analytics-read.ts new file mode 100644 index 0000000..0eb97ce --- /dev/null +++ b/api/src/signature-analytics/analytics-read.ts @@ -0,0 +1,217 @@ +import { BadRequestException } from '@nestjs/common'; +import { Prisma } from '@prisma/client'; +import { PrismaService } from '../prisma/prisma.service'; +import { addDays, analyticsDate, analyticsPage, databaseDay, todayKey } from './analytics-date'; + +export interface ActivityQuery { + date?: string; + dimensionType: string; + page?: number; + pageSize?: number; + tenantName?: string; + applicationName?: string; + signatureName?: string; + channelName?: string; +} +export class SignatureAnalyticsRead { + constructor(private readonly db: PrismaService) {} + + async metadata(date: string, now = new Date()) { + const record = await this.db.signatureAnalyticsDay.findUnique({ where: { businessDate: databaseDay(date) } }); + const run = await this.db.signatureAnalyticsRun.findUnique({ + where: { scope_businessDate: { scope: 'daily', businessDate: databaseDay(date) } }, + }); + const reportState = + run?.state === 'running' + ? 'refreshing' + : ['retry_wait', 'failed'].includes(run?.state ?? '') + ? 'failed' + : (record?.state ?? 'missing'); + return { + dataSource: 'report' as const, + businessDate: date, + serverBusinessDate: todayKey(now), + reportState, + frozen: date <= addDays(todayKey(now), -4), + generatedAt: record?.generatedAt ?? null, + sourceAsOf: record?.sourceAsOf ?? null, + generationId: record?.publishedGenerationId ?? null, + schemaVersion: record?.schemaVersion ?? 1, + provenance: record?.provenance ?? null, + }; + } + + async quality(query: { date?: string; keyword?: string; page?: number; pageSize?: number }) { + const date = analyticsDate(query.date); + const { page, pageSize } = analyticsPage(query.page, query.pageSize); + return this.db.$transaction( + async (tx) => { + const meta = await new SignatureAnalyticsRead(tx as PrismaService).metadata(date); + if (!meta.generationId) return { date, items: [], total: 0, page, pageSize, ...meta }; + const where: Prisma.SignatureQualityDailyWhereInput = { + generationId: meta.generationId, + ...(query.keyword?.trim() + ? { + OR: ['signatureName', 'tenantName', 'applicationNames'].map((field) => ({ + [field]: { contains: query.keyword!.trim(), mode: 'insensitive' }, + })), + } + : {}), + }; + const total = await tx.signatureQualityDaily.count({ where }); + const rows = await tx.signatureQualityDaily.findMany({ + where, + orderBy: [{ total: 'desc' }, { signatureName: 'asc' }, { signatureId: 'asc' }], + skip: (page - 1) * pageSize, + take: pageSize, + }); + return { date, items: rows.map((r) => r.payload), total, page, pageSize, ...meta }; + }, + { isolationLevel: 'RepeatableRead', timeout: 15_000 }, + ); + } + + async unreported(query: { date?: string; keyword?: string; page?: number; pageSize?: number }) { + const date = analyticsDate(query.date); + const { page, pageSize } = analyticsPage(query.page, query.pageSize); + return this.db.$transaction( + async (tx) => { + const meta = await new SignatureAnalyticsRead(tx as PrismaService).metadata(date); + if (!meta.generationId) return { date, items: [], total: 0, page, pageSize, ...meta }; + const where: Prisma.UnreportedSignatureDailyWhereInput = { + generationId: meta.generationId, + ...(query.keyword?.trim() + ? { + OR: ['signatureName', 'tenantName', 'applicationName'].map((field) => ({ + [field]: { contains: query.keyword!.trim(), mode: 'insensitive' }, + })), + } + : {}), + }; + const total = await tx.unreportedSignatureDaily.count({ where }); + const rows = await tx.unreportedSignatureDaily.findMany({ + where, + orderBy: [{ messageCount: 'desc' }, { dimensionKey: 'asc' }], + skip: (page - 1) * pageSize, + take: pageSize, + }); + return { + date, + items: rows.map((r) => ({ ...r, signatureId: r.dimensionKey })), + total, + page, + pageSize, + ...meta, + }; + }, + { isolationLevel: 'RepeatableRead', timeout: 15_000 }, + ); + } + + async activity(query: ActivityQuery) { + const date = analyticsDate(query.date); + if (!['enterprise', 'channel'].includes(query.dimensionType)) + throw new BadRequestException('必须指定企业或通道维度'); + const { page, pageSize } = analyticsPage(query.page, query.pageSize); + const dates = Array.from({ length: 30 }, (_, i) => addDays(date, -i - 1)); + return this.db.$transaction( + async (tx) => { + await tx.$executeRawUnsafe("SET LOCAL statement_timeout='12s'"); + const manifests = await tx.signatureAnalyticsDay.findMany({ + where: { businessDate: { in: dates.map(databaseDay) } }, + }); + const runs = await tx.signatureAnalyticsRun.findMany({ + where: { scope: 'daily', businessDate: { in: dates.map(databaseDay) } }, + }); + const runStates = new Map(runs.map((r) => [r.businessDate.toISOString().slice(0, 10), r.state])); + const byDate = new Map(manifests.map((r) => [r.businessDate.toISOString().slice(0, 10), r])); + const coverage = dates.map((d) => { + const r = byDate.get(d); + return { + date: d, + generationId: r?.publishedGenerationId ?? null, + reportState: + runStates.get(d) === 'running' + ? 'refreshing' + : ['failed', 'retry_wait'].includes(runStates.get(d) ?? '') + ? 'failed' + : (r?.state ?? 'missing'), + generatedAt: r?.generatedAt ?? null, + sourceAsOf: r?.sourceAsOf ?? null, + frozen: d <= addDays(todayKey(), -4), + }; + }); + const generations = coverage.flatMap((c) => (c.generationId ? [c.generationId] : [])); + if (!generations.length) + return { date, items: [], dimensions: [], total: 0, page, pageSize, coverage, complete: false }; + const filters = [ + ['tenantName', query.tenantName], + ['applicationName', query.applicationName], + ['signatureName', query.signatureName], + ['channelName', query.channelName], + ] + .filter(([, value]) => value?.trim()) + .map(([field, value]) => Prisma.sql`AND r.${Prisma.raw(`"${field}"`)} ILIKE ${`%${value!.trim()}%`}`); + const dimensions = await tx.$queryRaw< + Array<{ + dimensionKey: string; + dimensionType: string; + signatureId: string; + channelKey: string; + carrier: string; + signatureName: string; + channelName: string; + tenantName: string; + applicationName: string; + approvedAt: Date | null; + total: number; + rowCount: number; + }> + >(Prisma.sql` + WITH selected AS ( + SELECT * FROM "SignatureActivityDaily" WHERE "generationId" IN (${Prisma.join(generations)}) AND "dimensionType"=${query.dimensionType} + ), latest AS ( + SELECT DISTINCT ON ("dimensionKey") * FROM selected ORDER BY "dimensionKey","businessDate" DESC + ), sums AS (SELECT "dimensionKey",SUM("acceptedBusinessCount")::integer AS total FROM selected GROUP BY 1) + SELECT r.*,s.total,COUNT(*) OVER()::integer AS "rowCount" FROM latest r JOIN sums s USING("dimensionKey") + WHERE TRUE ${filters.length ? Prisma.join(filters, ' ') : Prisma.empty} ORDER BY s.total DESC,r."signatureName",r."dimensionKey" + LIMIT ${pageSize} OFFSET ${(page - 1) * pageSize}`); + // An empty out-of-range page still reports the filtered total. + const emptyPageCount = dimensions.length + ? [] + : await tx.$queryRaw>(Prisma.sql` + WITH latest AS ( + SELECT DISTINCT ON ("dimensionKey") * FROM "SignatureActivityDaily" + WHERE "generationId" IN (${Prisma.join(generations)}) AND "dimensionType"=${query.dimensionType} + ORDER BY "dimensionKey","businessDate" DESC + ) SELECT COUNT(*)::integer AS total FROM latest r WHERE TRUE ${filters.length ? Prisma.join(filters, ' ') : Prisma.empty}`); + const items = dimensions.length + ? await tx.signatureActivityDaily.findMany({ + where: { + generationId: { in: generations }, + dimensionType: query.dimensionType, + dimensionKey: { in: dimensions.map((d) => d.dimensionKey) }, + }, + }) + : []; + return { + date, + dimensions: dimensions.map((d) => ({ ...d, channelId: d.channelKey || null })), + items: items.map((r) => ({ + ...r, + id: `${r.generationId}:${r.dimensionKey}`, + channelId: r.channelKey || null, + activityDate: r.businessDate.toISOString().slice(0, 10), + status: r.applicability, + })), + total: dimensions[0]?.rowCount ?? emptyPageCount[0]?.total ?? 0, + page, + pageSize, + coverage, + complete: coverage.every((c) => Boolean(c.generationId)), + }; + }, + { isolationLevel: 'RepeatableRead', timeout: 15_000 }, + ); + } +} diff --git a/api/src/signature-analytics/retirement-batch.ts b/api/src/signature-analytics/retirement-batch.ts new file mode 100644 index 0000000..ea1f10d --- /dev/null +++ b/api/src/signature-analytics/retirement-batch.ts @@ -0,0 +1,210 @@ +import { Prisma, SignatureRetirementRule } from '@prisma/client'; +import { randomUUID } from 'node:crypto'; +import { PrismaService } from '../prisma/prisma.service'; +import { addDays, databaseDay, startOfDay, todayKey } from './analytics-date'; +import { analyticsJob } from './analytics-job'; + +const keyOf = (d: { dimensionType: string; signatureId: string; channelKey: string; carrier: string }) => + JSON.stringify([d.dimensionType, d.signatureId, d.channelKey, d.carrier]); +const carrierNames: Record = { mobile: '移动', unicom: '联通', telecom: '电信' }; + +export async function detectRetirement(db: PrismaService, date: string) { + if (date !== todayKey()) throw new Error('自动退网检测仅处理当天，不补发历史预警'); + const dependency = await db.signatureAnalyticsDay.findUnique({ + where: { businessDate: databaseDay(addDays(date, -1)) }, + }); + if ( + !dependency?.publishedGenerationId || + dependency.state !== 'ready' || + dependency.refreshFor?.getTime() !== databaseDay(date).getTime() + ) + throw new Error('签名退网检测等待昨日活动日报完成'); + const job = await analyticsJob( + db, + 'retirement', + date, + async (tx, _generation, checkpoint) => { + const report = await tx.signatureAnalyticsDay.findUnique({ + where: { businessDate: databaseDay(addDays(date, -1)) }, + }); + if ( + !report?.publishedGenerationId || + report.state !== 'ready' || + report.refreshFor?.getTime() !== databaseDay(date).getTime() + ) + throw new Error('签名退网检测等待昨日活动日报完成'); + const snapshot = checkpoint as unknown as { generationId: string; rules: SignatureRetirementRule[] }; + const daily = await tx.signatureActivityDaily.findMany({ where: { generationId: snapshot.generationId } }); + const rules = snapshot.rules; + const existing = new Set( + (await tx.signatureRetirementDetection.findMany({ where: { detectionDate: databaseDay(date) } })).map(keyOf), + ); + const dimensions = daily.flatMap((d) => { + if (existing.has(keyOf(d)) || !d.approvedAt) return []; + const enterprise = d.dimensionType === 'enterprise'; + const special = enterprise ? 'enterprise_application' : 'channel'; + const global = enterprise ? 'enterprise_global' : 'channel_global'; + const target = enterprise ? d.applicationId : d.channelKey; + const rule = + rules.find((r) => r.ruleType === special && r.targetId === target) ?? + rules.find((r) => r.ruleType === global && r.targetKey === ''); + if (!rule) return []; + const [windowDays, threshold] = + d.carrier === 'mobile' + ? [rule.mobileWindowDays, rule.mobileThreshold] + : d.carrier === 'unicom' + ? [rule.unicomWindowDays, rule.unicomThreshold] + : [rule.telecomWindowDays, rule.telecomThreshold]; + const windowStart = startOfDay(addDays(date, -windowDays)); + return [ + { + ...d, + approvedAt: d.approvedAt, + rule, + windowDays, + threshold, + windowStart, + observing: d.approvedAt > windowStart, + }, + ]; + }); + const eligible = dimensions.filter((d) => !d.observing); + const windowCounts = new Map(); + if (eligible.length) { + const earliest = new Date(Math.min(...eligible.map((d) => d.windowStart.getTime()))); + const defs = JSON.stringify( + eligible.map((d) => ({ + key: d.dimensionKey, + signature: d.signatureId, + carrier: d.carrier, + channel: d.channelKey, + start: d.windowStart.toISOString(), + })), + ); + const rows = await tx.$queryRaw>(Prisma.sql` + WITH dimensions AS (SELECT * FROM jsonb_to_recordset(${defs}::jsonb) AS d(key text,signature text,carrier text,channel text,start timestamptz)), + accepted AS MATERIALIZED ( + SELECT s."messageRecordId",s."channelId",m."signatureId",m.carrier,COALESCE(s."submittedAt",s."createdAt") AS at + FROM "SmsSubmitRecord" s JOIN "SmsMessageRecord" m ON m.id=s."messageRecordId" + WHERE s."submitStatus"='accepted' AND COALESCE(s."submittedAt",s."createdAt")>=${earliest} + AND COALESCE(s."submittedAt",s."createdAt")<${startOfDay(date)} AND m."signatureId" IS NOT NULL + ) SELECT d.key,COUNT(DISTINCT a."messageRecordId")::int AS count FROM dimensions d + LEFT JOIN accepted a ON a."signatureId"=d.signature AND a.carrier=d.carrier + AND (d.channel='' OR a."channelId"=d.channel) AND a.at>=d.start AT TIME ZONE 'UTC' + GROUP BY d.key`); + for (const row of rows) windowCounts.set(row.key, row.count); + } + const suppressions = new Map( + (await tx.signatureRetirementSuppression.findMany({ where: { active: true } })).map((r) => [keyOf(r), r]), + ); + const cycles = new Map( + (await tx.signatureRetirementCycle.findMany({ where: { status: 'open' } })).map((r) => [keyOf(r), r]), + ); + const rows: Prisma.SignatureRetirementDetectionCreateManyInput[] = []; + const continued: string[] = [], + resolved: string[] = []; + const newCycles: Prisma.SignatureRetirementCycleCreateManyInput[] = []; + let alerted = 0, + healthy = 0, + ineligible = 0; + for (const d of dimensions) { + const key = keyOf(d), + count = windowCounts.get(d.dimensionKey) ?? 0; + const alert = !d.observing && count < d.threshold; + let cycleId: string | null = null; + const cycle = cycles.get(key); + if (d.observing) ineligible++; + else if (alert) { + alerted++; + cycleId = cycle?.id ?? randomUUID(); + if (cycle) continued.push(cycle.id); + else + newCycles.push({ + id: cycleId, + dimensionType: d.dimensionType, + signatureId: d.signatureId, + channelId: d.channelKey || null, + channelKey: d.channelKey, + carrier: d.carrier, + startedOn: databaseDay(date), + lastDetectedOn: databaseDay(date), + }); + } else { + healthy++; + if (cycle) resolved.push(cycle.id); + } + const suppression = suppressions.get(key); + const suppressed = Boolean( + suppression && + (suppression.mode === 'permanent' || !suppression.muteUntil || suppression.muteUntil >= databaseDay(date)), + ); + const fallback = + d.dimensionType === 'enterprise' + ? '请通知 {enterprise}：{signature}在{carrier}过去{days}天发送{actual}条，低于{threshold}条，请及时进行保签名发送。' + : '{signature}在通道{channel}的{carrier}过去{days}天发送{actual}条，低于{threshold}条，请及时通知客户或进行保签名发送。'; + let content = d.rule.messageTemplate?.trim() || fallback; + for (const [name, value] of Object.entries({ + enterprise: d.tenantName, + signature: d.signatureName, + carrier: carrierNames[d.carrier] ?? d.carrier, + days: d.windowDays, + actual: count, + threshold: d.threshold, + channel: d.channelName || '-', + })) + content = content.replaceAll(`{${name}}`, String(value)); + rows.push({ + detectionDate: databaseDay(date), + dimensionType: d.dimensionType, + tenantId: d.tenantId, + applicationId: d.applicationId, + signatureId: d.signatureId, + channelId: d.channelKey || null, + channelKey: d.channelKey, + carrier: d.carrier, + windowDays: d.windowDays, + threshold: d.threshold, + submittedAttempts: d.submittedAttempts, + acceptedBusinessCount: d.acceptedBusinessCount, + deliveredBusinessCount: d.deliveredBusinessCount, + approvedAt: d.approvedAt, + ruleId: d.rule.id, + ruleVersion: d.rule.version, + status: d.observing ? 'observing' : alert ? 'alert' : 'healthy', + cycleId, + suppressed, + notificationTitle: alert + ? d.dimensionType === 'enterprise' + ? '企业签名清退预警' + : '通道签名清退预警' + : null, + notificationContent: alert ? content : null, + }); + } + for (let i = 0; i < newCycles.length; i += 250) + await tx.signatureRetirementCycle.createMany({ data: newCycles.slice(i, i + 250) }); + if (continued.length) + await tx.signatureRetirementCycle.updateMany({ + where: { id: { in: continued } }, + data: { lastDetectedOn: databaseDay(date) }, + }); + if (resolved.length) + await tx.signatureRetirementCycle.updateMany({ + where: { id: { in: resolved } }, + data: { status: 'resolved', resolvedOn: databaseDay(date), lastDetectedOn: databaseDay(date) }, + }); + for (let i = 0; i < rows.length; i += 250) + await tx.signatureRetirementDetection.createMany({ data: rows.slice(i, i + 250) }); + return { detectionDate: date, dimensions: dimensions.length, alerted, healthy, ineligible }; + }, + new Date(), + async () => + JSON.parse( + JSON.stringify({ + generationId: dependency.publishedGenerationId, + rules: await db.signatureRetirementRule.findMany({ where: { enabled: true } }), + }), + ) as Prisma.InputJsonValue, + ); + return job.result ?? { detectionDate: date, skipped: true }; +} diff --git a/api/src/signature-analytics/signature-analytics.module.ts b/api/src/signature-analytics/signature-analytics.module.ts new file mode 100644 index 0000000..241034b --- /dev/null +++ b/api/src/signature-analytics/signature-analytics.module.ts @@ -0,0 +1,5 @@ +import { Module } from '@nestjs/common'; +import { PrismaModule } from '../prisma/prisma.module'; +import { SignatureAnalyticsService } from './signature-analytics.service'; +@Module({ imports: [PrismaModule], providers: [SignatureAnalyticsService], exports: [SignatureAnalyticsService] }) +export class SignatureAnalyticsModule {} diff --git a/api/src/signature-analytics/signature-analytics.service.ts b/api/src/signature-analytics/signature-analytics.service.ts new file mode 100644 index 0000000..41e099d --- /dev/null +++ b/api/src/signature-analytics/signature-analytics.service.ts @@ -0,0 +1,122 @@ +import { Injectable, Logger, OnModuleDestroy, OnModuleInit } from '@nestjs/common'; +import { Prisma } from '@prisma/client'; +import { PrismaService } from '../prisma/prisma.service'; +import { OperationsQualityQueries } from '../operations/queries/quality.queries'; +import { activityCounts, activityDimensions, unreportedRows } from './analytics-aggregate'; +import { addDays, analyticsDate, databaseDay, mutableDay, todayKey } from './analytics-date'; +import { analyticsJob } from './analytics-job'; + +@Injectable() +export class SignatureAnalyticsService implements OnModuleInit, OnModuleDestroy { + private readonly logger = new Logger(SignatureAnalyticsService.name); + private timer?: NodeJS.Timeout; + private running = false; + constructor(private readonly db: PrismaService) {} + onModuleInit() { + if (process.env.NODE_ENV === 'test' || process.env.SIGNATURE_ANALYTICS_ENABLED === 'false') return; + this.timer = setInterval(() => void this.tick(), 60_000); + this.timer.unref(); + void this.tick(); + } + onModuleDestroy() { + if (this.timer) clearInterval(this.timer); + } + async tick(now = new Date()) { + const hour = Number( + new Intl.DateTimeFormat('en-GB', { timeZone: 'Asia/Shanghai', hour: '2-digit', hour12: false }).format(now), + ); + if (hour < 3 || this.running) return; + this.running = true; + try { + for (const offset of [-3, -2, -1]) { + try { + await this.generate(addDays(todayKey(now), offset)); + } catch (error) { + this.logger.error( + `signature_analytics_failed date=${addDays(todayKey(now), offset)}`, + error instanceof Error ? error.stack : String(error), + ); + } + } + } finally { + this.running = false; + } + } + + /** Explicit offline backfill only; no controller exposes this write operation. Existing frozen days cannot be overwritten. */ + async generate(value: string, backfill = false) { + const date = analyticsDate(value); + if (!mutableDay(date) && !backfill) return { skipped: true }; + if (date >= todayKey()) throw new Error('日报只生成完整自然日'); + const businessDate = databaseDay(date); + const existing = await this.db.signatureAnalyticsDay.findUnique({ where: { businessDate } }); + if (backfill && existing?.publishedGenerationId) throw new Error('历史补建不得覆盖已发布日报'); + await this.db.signatureAnalyticsDay.upsert({ where: { businessDate }, create: { businessDate }, update: {} }); + return await analyticsJob(this.db, 'daily', date, async (tx, generationId) => { + const sourceAsOf = new Date(); + await tx.signatureAnalyticsGeneration.create({ data: { id: generationId, businessDate, sourceAsOf } }); + const quality = await new OperationsQualityQueries(tx).signatureQualityLive({ date }, true); + const dimensions = await activityDimensions(tx, date); + const counts = await activityCounts(tx, dimensions, date); + const unreported = await unreportedRows(tx, date); + // All candidate rows and manifest publication share this transaction: readers never see half a day. + for (let i = 0; i < quality.items.length; i += 250) + await tx.signatureQualityDaily.createMany({ + data: quality.items.slice(i, i + 250).map((row) => ({ + businessDate, + generationId, + signatureId: row.signatureId, + signatureName: row.signatureName, + tenantId: row.tenantId, + tenantName: row.tenantName, + applicationNames: row.applicationNames ?? '', + total: row.total, + payload: JSON.parse(JSON.stringify(row)) as Prisma.InputJsonValue, + })), + }); + for (let i = 0; i < dimensions.length; i += 250) + await tx.signatureActivityDaily.createMany({ + data: dimensions.slice(i, i + 250).map((d) => ({ + ...d, + businessDate, + generationId, + ...(counts.get(d.dimensionKey) ?? { + submittedAttempts: 0, + acceptedBusinessCount: 0, + deliveredBusinessCount: 0, + }), + applicability: 'applicable', + })), + }); + for (let i = 0; i < unreported.length; i += 250) + await tx.unreportedSignatureDaily.createMany({ + data: unreported.slice(i, i + 250).map((r) => ({ ...r, businessDate, generationId })), + }); + if (!backfill && !mutableDay(date)) throw new Error('日报已进入冻结区，拒绝跨日发布'); + if (backfill) { + const current = await tx.signatureAnalyticsDay.findUniqueOrThrow({ where: { businessDate } }); + if (current.publishedGenerationId) throw new Error('已有发布版本，拒绝覆盖'); + } + await tx.signatureAnalyticsDay.update({ + where: { businessDate }, + data: { + publishedGenerationId: generationId, + state: 'ready', + error: null, + generatedAt: new Date(), + sourceAsOf, + refreshFor: databaseDay(todayKey()), + provenance: backfill ? 'backfill-current-source' : 'daily', + rowCounts: { quality: quality.items.length, activity: dimensions.length, unreported: unreported.length }, + }, + }); + return { + date, + generationId, + quality: quality.items.length, + activity: dimensions.length, + unreported: unreported.length, + }; + }); + } +} diff --git a/api/src/signature-retirement/daily-message.spec.ts b/api/src/signature-retirement/daily-message.spec.ts index 531a8f9..b58e3b7 100644 --- a/api/src/signature-retirement/daily-message.spec.ts +++ b/api/src/signature-retirement/daily-message.spec.ts @@ -15,6 +15,7 @@ describe('daily application messages and date formatting', () => { notificationContent: `冻结正文${item.id}`, })); const prisma = { + signatureAnalyticsRun: { findUnique: jest.fn().mockResolvedValue({ state: 'succeeded' }) }, signatureRetirementDetection: { findMany: jest.fn().mockResolvedValue(detections) }, smsApplication: { findMany: jest.fn().mockResolvedValue([ @@ -60,6 +61,7 @@ describe('daily application messages and date formatting', () => { ['2026-09-01', '2026-08-31'], ])('keeps T-1 semantics for %s without constructing formatters per row', async (date, expected) => { const prisma = { + signatureAnalyticsRun: { findUnique: jest.fn().mockResolvedValue({ state: 'succeeded' }) }, signatureRetirementDetection: { findMany: jest.fn().mockResolvedValue( Array.from({ length: 100 }, (_, i) => ({ diff --git a/api/src/signature-retirement/signature-retirement.controller.ts b/api/src/signature-retirement/signature-retirement.controller.ts index c3c102a..12ee1f8 100644 --- a/api/src/signature-retirement/signature-retirement.controller.ts +++ b/api/src/signature-retirement/signature-retirement.controller.ts @@ -1,14 +1,26 @@ +import { SignatureAnalyticsRead } from '../signature-analytics/analytics-read'; +import { PrismaService } from '../prisma/prisma.service'; import { Body, Controller, Delete, Get, Param, Post, Put, Query } from '@nestjs/common'; import { ApiTags } from '@nestjs/swagger'; import { CurrentSessionUserId } from '../auth/current-session-user.decorator'; import { RequireRecentAuthentication } from '../auth/require-recent-authentication.decorator'; -import type { CancelRetirementSuppressionDto, CreateRetirementWebhookDto, RetirementMessageQuery, SuppressRetirementMessageDto, UnreportedSignatureQuery, UpsertRetirementRuleDto } from './signature-retirement.contracts'; +import type { + CancelRetirementSuppressionDto, + CreateRetirementWebhookDto, + RetirementMessageQuery, + SuppressRetirementMessageDto, + UnreportedSignatureQuery, + UpsertRetirementRuleDto, +} from './signature-retirement.contracts'; import { SignatureRetirementService } from './signature-retirement.service'; @ApiTags('signature-retirement') @Controller('admin/signature-retirement') export class SignatureRetirementController { - constructor(private readonly service: SignatureRetirementService) {} + constructor( + private readonly service: SignatureRetirementService, + private readonly prisma: PrismaService, + ) {} @Get('configuration') getConfiguration() { @@ -45,7 +57,17 @@ export class SignatureRetirementController { @Query('page') page?: string, @Query('pageSize') pageSize?: string, ) { - const query: RetirementMessageQuery = { dateFrom, dateTo, dimensionType, tenantId, applicationId, signatureKeyword, channelId, page: Number(page), pageSize: Number(pageSize) }; + const query: RetirementMessageQuery = { + dateFrom, + dateTo, + dimensionType, + tenantId, + applicationId, + signatureKeyword, + channelId, + page: Number(page), + pageSize: Number(pageSize), + }; return this.service.listMessages(query); } @@ -66,7 +88,11 @@ export class SignatureRetirementController { @Post('messages/:id/suppress') @RequireRecentAuthentication() - suppress(@Param('id') id: string, @Body() body: SuppressRetirementMessageDto, @CurrentSessionUserId() operatorId?: string) { + suppress( + @Param('id') id: string, + @Body() body: SuppressRetirementMessageDto, + @CurrentSessionUserId() operatorId?: string, + ) { return this.service.suppressMessage(id, body, operatorId); } @@ -77,10 +103,35 @@ export class SignatureRetirementController { @Post('suppressions/:id/cancel') @RequireRecentAuthentication() - cancelSuppression(@Param('id') id: string, @Body() body: CancelRetirementSuppressionDto, @CurrentSessionUserId() operatorId?: string) { + cancelSuppression( + @Param('id') id: string, + @Body() body: CancelRetirementSuppressionDto, + @CurrentSessionUserId() operatorId?: string, + ) { return this.service.cancelSuppression(id, body, operatorId); } + @Get('activity') + activity( + @Query() + query: { + date?: string; + dimensionType: string; + page?: string; + pageSize?: string; + tenantName?: string; + applicationName?: string; + signatureName?: string; + channelName?: string; + }, + ) { + return new SignatureAnalyticsRead(this.prisma).activity({ + ...query, + page: query.page === undefined ? 1 : Number(query.page), + pageSize: query.pageSize === undefined ? 25 : Number(query.pageSize), + }); + } + @Get('heatmap') heatmap(@Query('date') date?: string) { return this.service.heatmap(date); @@ -93,8 +144,12 @@ export class SignatureRetirementController { @Query('page') page?: string, @Query('pageSize') pageSize?: string, ) { - const query: UnreportedSignatureQuery = { date, keyword, page: Number(page), pageSize: Number(pageSize) }; + const query: UnreportedSignatureQuery = { + date, + keyword, + page: page === undefined ? 1 : Number(page), + pageSize: pageSize === undefined ? 25 : Number(pageSize), + }; return this.service.unreportedSignatures(query); } - } diff --git a/api/src/signature-retirement/signature-retirement.service.spec.ts b/api/src/signature-retirement/signature-retirement.service.spec.ts index af64437..9629a6c 100644 --- a/api/src/signature-retirement/signature-retirement.service.spec.ts +++ b/api/src/signature-retirement/signature-retirement.service.spec.ts @@ -1,57 +1,6 @@ import { millisecondsUntilShanghaiHour, SignatureRetirementService } from './signature-retirement.service'; describe('SignatureRetirementService dimensions', () => { - const service = new SignatureRetirementService({} as never); - - it('builds enterprise dimensions once and channel dimensions per approved channel and carrier', () => { - const rules = [ - rule('enterprise_global', ''), - rule('enterprise_application', 'app-1'), - rule('channel_global', ''), - rule('channel', 'channel-2'), - ]; - const tasks = [ - task('channel-1', '移动一号', 'mobile', '2026-06-01T00:00:00Z'), - task('channel-2', '移动二号', 'mobile', '2026-06-05T00:00:00Z'), - task('channel-2', '移动二号', 'unicom', '2026-06-05T00:00:00Z'), - ]; - - const dimensions = ( - service as unknown as { - buildDimensions: ( - inputRules: unknown[], - inputTasks: unknown[], - ) => Array<{ dimensionType: string; carrier: string; approvedAt: Date; rule: { ruleType: string } }>; - } - ).buildDimensions(rules, tasks); - - expect(dimensions.filter((item) => item.dimensionType === 'enterprise')).toHaveLength(2); - expect(dimensions.filter((item) => item.dimensionType === 'channel')).toHaveLength(3); - expect( - dimensions - .find((item) => item.dimensionType === 'enterprise' && item.carrier === 'mobile') - ?.approvedAt.toISOString(), - ).toBe('2026-06-01T00:00:00.000Z'); - expect( - dimensions - .filter((item) => item.dimensionType === 'enterprise') - .every((item) => item.rule.ruleType === 'enterprise_application'), - ).toBe(true); - expect( - dimensions.find((item) => item.dimensionType === 'channel' && item.carrier === 'unicom')?.rule.ruleType, - ).toBe('channel'); - }); - - it('does not monitor legacy carrier-null reporting facts', () => { - const dimensions = ( - service as unknown as { buildDimensions: (inputRules: unknown[], inputTasks: unknown[]) => unknown[] } - ).buildDimensions( - [rule('enterprise_global', ''), rule('channel_global', '')], - [task('channel-1', '三网旧通道', null, '2026-06-01T00:00:00Z')], - ); - expect(dimensions).toEqual([]); - }); - it('schedules the next Beijing 04:00 and 08:00 precisely', () => { expect(millisecondsUntilShanghaiHour(new Date('2026-08-09T19:00:00.000Z'), 4)).toBe(60 * 60_000); expect(millisecondsUntilShanghaiHour(new Date('2026-08-09T21:00:00.000Z'), 4)).toBe(23 * 60 * 60_000); @@ -69,6 +18,7 @@ describe('SignatureRetirementService dimensions', () => { notificationContent: '冻结后的预警正文', }; const prisma = { + signatureAnalyticsRun: { findUnique: jest.fn().mockResolvedValue({ state: 'succeeded' }) }, signatureRetirementDetection: { findMany: jest.fn().mockResolvedValueOnce([detection]).mockResolvedValueOnce([detection]), }, @@ -138,47 +88,17 @@ describe('SignatureRetirementService dimensions', () => { expect(result.items[0]).toEqual(expect.objectContaining({ activityDate: '2026-08-09' })); }); - it('persists daily observing snapshots without opening alert cycles', async () => { - const prisma = { - signatureRetirementSuppression: { - updateMany: jest.fn().mockResolvedValue({ count: 0 }), - findUnique: jest.fn().mockResolvedValue(null), - }, - signatureRetirementRule: { - findMany: jest.fn().mockResolvedValue([rule('enterprise_global', ''), rule('channel_global', '')]), - }, - channelSignatureReportTask: { - findMany: jest.fn().mockResolvedValue([task('channel-1', '移动一号', 'mobile', '2026-08-09T00:00:00Z')]), - }, - signatureRetirementDetection: { - findUnique: jest.fn().mockResolvedValue(null), - create: jest.fn().mockResolvedValue({ id: 'detection-1' }), - }, - signatureRetirementCycle: { findFirst: jest.fn(), create: jest.fn(), update: jest.fn() }, - $queryRaw: jest - .fn() - .mockResolvedValue([{ submittedAttempts: 12, acceptedBusinessCount: 10, deliveredBusinessCount: 9 }]), - }; - const observingService = new SignatureRetirementService(prisma as never); - - await expect(observingService.runDetection('2026-08-10')).resolves.toEqual({ - detectionDate: '2026-08-10', - dimensions: 2, - alerted: 0, - healthy: 0, - ineligible: 2, - }); - expect(prisma.signatureRetirementDetection.create).toHaveBeenCalledTimes(2); - expect(prisma.signatureRetirementDetection.create).toHaveBeenCalledWith({ - data: expect.objectContaining({ - status: 'observing', - acceptedBusinessCount: 10, - cycleId: undefined, - notificationTitle: null, - notificationContent: null, - }), - }); - expect(prisma.signatureRetirementCycle.create).not.toHaveBeenCalled(); + it('waits for a complete daily report before retirement detection', async () => { + const prisma = { signatureAnalyticsDay: { findUnique: jest.fn().mockResolvedValue(null) } }; + const date = new Intl.DateTimeFormat('en-CA', { + timeZone: 'Asia/Shanghai', + year: 'numeric', + month: '2-digit', + day: '2-digit', + }).format(new Date()); + await expect(new SignatureRetirementService(prisma as never).runDetection(date)).rejects.toThrow( + '等待昨日活动日报', + ); }); it('maps the real unreported-signature aggregation to an independent page', async () => { @@ -199,7 +119,7 @@ describe('SignatureRetirementService dimensions', () => { const unreportedService = new SignatureRetirementService(prisma as never); await expect( - unreportedService.unreportedSignatures({ date: '2026-08-10', keyword: '测试', page: 2, pageSize: 10 }), + unreportedService.unreportedSignaturesLive({ date: '2026-08-10', keyword: '测试', page: 2, pageSize: 10 }), ).resolves.toEqual({ date: '2026-08-10', items: [ @@ -311,32 +231,3 @@ describe('SignatureRetirementService dimensions', () => { ); }); }); - -function rule(ruleType: string, targetKey: string) { - return { - id: `${ruleType}-${targetKey}`, - ruleType, - targetId: targetKey || null, - targetKey, - enabled: true, - mobileWindowDays: 30, - mobileThreshold: 1, - unicomWindowDays: 30, - unicomThreshold: 1, - telecomWindowDays: 30, - telecomThreshold: 1, - messageTemplate: null, - version: 1, - }; -} - -function task(channelId: string, channelName: string, carrier: string | null, approvedAt: string) { - return { - signatureId: 'signature-1', - channelId, - carrier, - approvedAt: new Date(approvedAt), - signature: { tenantId: 'tenant-1', applicationId: 'app-1', name: '测试签名', tenant: { name: '测试企业' } }, - channel: { name: channelName }, - }; -} diff --git a/api/src/signature-retirement/signature-retirement.service.ts b/api/src/signature-retirement/signature-retirement.service.ts index 144ecf4..7127427 100644 --- a/api/src/signature-retirement/signature-retirement.service.ts +++ b/api/src/signature-retirement/signature-retirement.service.ts @@ -1,3 +1,6 @@ +import { SignatureAnalyticsRead } from '../signature-analytics/analytics-read'; +import { analyticsDate, analyticsPage, todayKey } from '../signature-analytics/analytics-date'; +import { detectRetirement } from '../signature-analytics/retirement-batch'; import { BadRequestException, Injectable, @@ -37,28 +40,6 @@ const shanghaiHourFormatter = new Intl.DateTimeFormat('en-GB', { const DAY_MS = 86_400_000; const CARRIERS = ['mobile', 'unicom', 'telecom'] as const; const DEFAULT_DELIVERY_INTERVAL_MS = 60_000; -const carrierLabels: Record = { mobile: '移动', unicom: '联通', telecom: '电信' }; - -type RuleRecord = Awaited>; -type DetectionDimension = { - dimensionType: 'enterprise' | 'channel'; - tenantId: string; - applicationId: string | null; - signatureId: string; - signatureName: string; - tenantName: string; - channelId: string | null; - channelName: string | null; - carrier: string; - approvedAt: Date; - rule: NonNullable; -}; - -type ActivityCounts = { - submittedAttempts: number; - acceptedBusinessCount: number; - deliveredBusinessCount: number; -}; @Injectable() export class SignatureRetirementService implements OnModuleInit, OnModuleDestroy { @@ -66,6 +47,8 @@ export class SignatureRetirementService implements OnModuleInit, OnModuleDestroy private detectionTimer?: ReturnType; private notificationTimer?: ReturnType; private deliveryTimer?: ReturnType; + private compensationRunning = false; + private publishedDate?: string; private startupTimer?: ReturnType; constructor(private readonly prisma: PrismaService) {} @@ -78,7 +61,7 @@ export class SignatureRetirementService implements OnModuleInit, OnModuleDestroy this.scheduleDetection(); this.scheduleNotification(); this.deliveryTimer = setInterval( - () => void this.deliverPendingWebhooks(), + () => void this.runStartupCompensation(), positiveIntegerEnv('SIGNATURE_RETIREMENT_WEBHOOK_INTERVAL_MS', DEFAULT_DELIVERY_INTERVAL_MS), ); this.deliveryTimer.unref?.(); @@ -540,6 +523,30 @@ export class SignatureRetirementService implements OnModuleInit, OnModuleDestroy } async unreportedSignatures(query: UnreportedSignatureQuery) { + analyticsPage(query.page, query.pageSize); + const date = analyticsDate(query.date); + if (date !== todayKey()) return new SignatureAnalyticsRead(this.prisma).unreported({ ...query, date }); + return this.prisma.$transaction( + async (tx) => { + await tx.$executeRawUnsafe("SET LOCAL statement_timeout='12s'"); + const data = await new SignatureRetirementService(tx as PrismaService).unreportedSignaturesLive({ + ...query, + date, + }); + return { + ...data, + dataSource: 'live', + reportState: 'ready', + frozen: false, + sourceAsOf: new Date(), + serverBusinessDate: date, + }; + }, + { isolationLevel: 'RepeatableRead', timeout: 15000 }, + ); + } + + async unreportedSignaturesLive(query: UnreportedSignatureQuery) { const date = assertDateKey(query.date || shanghaiDateKey()); const page = positiveInteger(query.page, 1); const pageSize = Math.min(100, positiveInteger(query.pageSize, 25)); @@ -630,68 +637,16 @@ export class SignatureRetirementService implements OnModuleInit, OnModuleDestroy } async runDetection(date?: string) { - const detectionKey = assertDateKey(date || shanghaiDateKey()); - await this.prisma.signatureRetirementSuppression.updateMany({ - where: { active: true, mode: 'temporary', muteUntil: { lt: databaseDate(detectionKey) } }, - data: { active: false }, - }); - const [rules, approvedTasks] = await Promise.all([ - this.prisma.signatureRetirementRule.findMany({ where: { enabled: true } }), - this.prisma.channelSignatureReportTask.findMany({ - where: { - reportType: 'signature', - status: 'approved', - carrier: { not: null }, - approvalScope: 'carrier_specific', - approvedAt: { not: null }, - signature: { auditStatus: { not: 'deleted' } }, - channel: { status: { not: 'deleted' } }, - }, - include: { signature: { include: { tenant: true, application: true } }, channel: true }, - }), - ]); - const dimensions = this.buildDimensions(rules, approvedTasks); - let alerted = 0; - let healthy = 0; - let ineligible = 0; - for (const dimension of dimensions) { - const { windowDays, threshold } = carrierRule(dimension.rule, dimension.carrier); - const windowStartKey = addDays(detectionKey, -windowDays); - const windowStart = shanghaiStart(windowStartKey); - const activityStart = shanghaiStart(addDays(detectionKey, -1)); - const activityEnd = shanghaiStart(detectionKey); - const effectiveActivityStart = dimension.approvedAt > activityStart ? dimension.approvedAt : activityStart; - if (effectiveActivityStart >= activityEnd) { - ineligible += 1; - continue; - } - const dailyCounts = await this.activityCounts(dimension, effectiveActivityStart, activityEnd); - if (dimension.approvedAt > windowStart) { - // 观察期只禁止预警，不能吞掉真实发送快照，否则热力图会错误显示无数据。 - await this.persistDetection(detectionKey, dimension, windowDays, threshold, dailyCounts, false, true); - ineligible += 1; - continue; - } - const windowCounts = await this.activityCounts(dimension, windowStart, activityEnd); - const isAlert = windowCounts.acceptedBusinessCount < threshold; - await this.persistDetection( - detectionKey, - dimension, - windowDays, - threshold, - dailyCounts, - isAlert, - false, - windowCounts, - ); - if (isAlert) alerted += 1; - else healthy += 1; - } - return { detectionDate: detectionKey, dimensions: dimensions.length, alerted, healthy, ineligible }; + return detectRetirement(this.prisma, analyticsDate(date)); } async publishNotifications(date?: string) { - const notificationKey = assertDateKey(date || shanghaiDateKey()); + const notificationKey = analyticsDate(date); + if (this.publishedDate === notificationKey) return { notificationDate: notificationKey, created: 0 }; + const completed = await this.prisma.signatureAnalyticsRun.findUnique({ + where: { scope_businessDate: { scope: 'retirement', businessDate: databaseDate(notificationKey) } }, + }); + if (completed?.state !== 'succeeded') throw new Error('签名退网检测尚未完整完成，暂不发布通知'); const detections = await this.prisma.signatureRetirementDetection.findMany({ where: { detectionDate: databaseDate(notificationKey), @@ -749,10 +704,13 @@ export class SignatureRetirementService implements OnModuleInit, OnModuleDestroy } } await this.enqueueWebhookSummaries(notificationKey); + this.publishedDate = notificationKey; return { notificationDate: notificationKey, created }; } private async runStartupCompensation() { + if (this.compensationRunning) return; + this.compensationRunning = true; const now = new Date(); const hour = shanghaiHour(now); try { @@ -765,6 +723,8 @@ export class SignatureRetirementService implements OnModuleInit, OnModuleDestroy this.logger.error( `Signature retirement startup compensation failed: ${error instanceof Error ? error.message : String(error)}`, ); + } finally { + this.compensationRunning = false; } } @@ -801,231 +761,6 @@ export class SignatureRetirementService implements OnModuleInit, OnModuleDestroy this.notificationTimer.unref?.(); } - private buildDimensions( - rules: Array>, - tasks: Array<{ - signatureId: string; - channelId: string; - carrier: string | null; - approvedAt: Date | null; - signature: { tenantId: string; applicationId: string | null; name: string; tenant: { name: string } }; - channel: { name: string }; - }>, - ) { - const dimensions: DetectionDimension[] = []; - const enterprise = new Map(); - for (const task of tasks) { - if (!task.carrier || !task.approvedAt) continue; - const channelRule = selectRule(rules, 'channel', task.channelId); - if (channelRule) - dimensions.push({ - dimensionType: 'channel', - tenantId: task.signature.tenantId, - applicationId: task.signature.applicationId, - signatureId: task.signatureId, - signatureName: task.signature.name, - tenantName: task.signature.tenant.name, - channelId: task.channelId, - channelName: task.channel.name, - carrier: task.carrier, - approvedAt: task.approvedAt, - rule: channelRule, - }); - const enterpriseRule = selectRule(rules, 'enterprise', task.signature.applicationId); - if (!enterpriseRule) continue; - const key = `${task.signatureId}:${task.carrier}`; - const current = enterprise.get(key); - if (!current || task.approvedAt < current.approvedAt) - enterprise.set(key, { - dimensionType: 'enterprise', - tenantId: task.signature.tenantId, - applicationId: task.signature.applicationId, - signatureId: task.signatureId, - signatureName: task.signature.name, - tenantName: task.signature.tenant.name, - channelId: null, - channelName: null, - carrier: task.carrier, - approvedAt: task.approvedAt, - rule: enterpriseRule, - }); - } - return [...enterprise.values(), ...dimensions]; - } - - private async activityCounts(dimension: DetectionDimension, startAt: Date, endAt: Date): Promise { - const channelFilter = dimension.channelId - ? Prisma.sql`AND submit."channelId" = ${dimension.channelId}` - : Prisma.empty; - const rows = await this.prisma.$queryRaw>(Prisma.sql` - WITH attempts AS ( - SELECT - submit.id, - submit."messageRecordId" AS message_id, - submit."submitStatus" AS submit_status, - CASE - WHEN EXISTS (SELECT 1 FROM "SmsMessageSegmentAudit" segment WHERE segment."submitRecordId" = submit.id) - THEN NOT EXISTS ( - SELECT 1 FROM "SmsMessageSegmentAudit" segment - WHERE segment."submitRecordId" = submit.id AND segment."receiptStatus" IS DISTINCT FROM 'delivered' - ) - ELSE EXISTS ( - SELECT 1 FROM "SmsReceiptRecord" receipt - WHERE receipt."gatewayMessageId" = submit."gatewayMessageId" - AND receipt."channelId" = submit."channelId" - AND receipt."receiptStatus" = 'delivered' - ) - END AS delivery_success - FROM "SmsSubmitRecord" submit - JOIN "SmsMessageRecord" message ON message.id = submit."messageRecordId" - WHERE message."signatureId" = ${dimension.signatureId} - AND message.carrier = ${dimension.carrier} - AND COALESCE(submit."submittedAt", submit."createdAt") >= ${startAt} - AND COALESCE(submit."submittedAt", submit."createdAt") < ${endAt} - ${channelFilter} - ) - SELECT - COUNT(id)::integer AS "submittedAttempts", - COUNT(DISTINCT message_id) FILTER (WHERE submit_status = 'accepted')::integer AS "acceptedBusinessCount", - COUNT(DISTINCT message_id) FILTER (WHERE submit_status = 'accepted' AND delivery_success)::integer AS "deliveredBusinessCount" - FROM attempts - `); - return rows[0] ?? { submittedAttempts: 0, acceptedBusinessCount: 0, deliveredBusinessCount: 0 }; - } - - private async persistDetection( - dateKey: string, - dimension: DetectionDimension, - windowDays: number, - threshold: number, - counts: ActivityCounts, - isAlert: boolean, - observing = false, - alertCounts = counts, - ) { - const detectionDate = databaseDate(dateKey); - const channelKey = dimension.channelId ?? ''; - const existingDetection = await this.prisma.signatureRetirementDetection.findUnique({ - where: { - detectionDate_dimensionType_signatureId_channelKey_carrier: { - detectionDate, - dimensionType: dimension.dimensionType, - signatureId: dimension.signatureId, - channelKey, - carrier: dimension.carrier, - }, - }, - select: { id: true }, - }); - // 同一检测日的结果冻结规则版本；规则变更只在下一检测日生效。 - if (existingDetection) return; - const suppression = await this.prisma.signatureRetirementSuppression.findUnique({ - where: { - dimensionType_signatureId_channelKey_carrier: { - dimensionType: dimension.dimensionType, - signatureId: dimension.signatureId, - channelKey, - carrier: dimension.carrier, - }, - }, - }); - const suppressed = Boolean( - suppression?.active && - (suppression.mode === 'permanent' || !suppression.muteUntil || suppression.muteUntil >= detectionDate), - ); - let cycle = await this.prisma.signatureRetirementCycle.findFirst({ - where: { - dimensionType: dimension.dimensionType, - signatureId: dimension.signatureId, - channelKey, - carrier: dimension.carrier, - status: 'open', - }, - }); - if (observing) { - cycle = null; - } else if (isAlert) { - if (!cycle) { - try { - cycle = await this.prisma.signatureRetirementCycle.create({ - data: { - dimensionType: dimension.dimensionType, - signatureId: dimension.signatureId, - channelId: dimension.channelId, - channelKey, - carrier: dimension.carrier, - startedOn: detectionDate, - lastDetectedOn: detectionDate, - }, - }); - } catch (error) { - if (!isPrismaUniqueError(error)) throw error; - cycle = await this.prisma.signatureRetirementCycle.findFirst({ - where: { - dimensionType: dimension.dimensionType, - signatureId: dimension.signatureId, - channelKey, - carrier: dimension.carrier, - status: 'open', - }, - }); - } - } else { - cycle = await this.prisma.signatureRetirementCycle.update({ - where: { id: cycle.id }, - data: { lastDetectedOn: detectionDate }, - }); - } - } else if (cycle) { - await this.prisma.signatureRetirementCycle.update({ - where: { id: cycle.id }, - data: { status: 'resolved', resolvedOn: detectionDate, lastDetectedOn: detectionDate }, - }); - cycle = null; - } - const notificationTitle = - isAlert && cycle ? (dimension.dimensionType === 'enterprise' ? '企业签名清退预警' : '通道签名清退预警') : null; - const notificationContent = - isAlert && cycle - ? renderMessage( - dimension.rule.messageTemplate, - dimension, - windowDays, - threshold, - alertCounts.acceptedBusinessCount, - ) - : null; - try { - await this.prisma.signatureRetirementDetection.create({ - data: { - detectionDate, - dimensionType: dimension.dimensionType, - tenantId: dimension.tenantId, - applicationId: dimension.applicationId, - signatureId: dimension.signatureId, - channelId: dimension.channelId, - channelKey, - carrier: dimension.carrier, - windowDays, - threshold, - ...counts, - approvedAt: dimension.approvedAt, - ruleId: dimension.rule.id, - ruleVersion: dimension.rule.version, - status: observing ? 'observing' : isAlert ? 'alert' : 'healthy', - cycleId: cycle?.id, - suppressed, - notificationTitle, - notificationContent, - }, - }); - } catch (error) { - // 两个检测实例可能同时越过前置查询；唯一快照已由另一实例生成时直接结束本维度。 - if (isPrismaUniqueError(error)) return; - throw error; - } - } - private async enqueueWebhookSummaries(dateKey: string) { const detectionDate = databaseDate(dateKey); const [webhooks, messages] = await Promise.all([ @@ -1119,46 +854,6 @@ export class SignatureRetirementService implements OnModuleInit, OnModuleDestroy } } -function selectRule( - rules: Array>, - dimension: 'enterprise' | 'channel', - targetId: string | null, -) { - const specialType = dimension === 'enterprise' ? 'enterprise_application' : 'channel'; - const globalType = dimension === 'enterprise' ? 'enterprise_global' : 'channel_global'; - return ( - (targetId ? rules.find((rule) => rule.ruleType === specialType && rule.targetId === targetId) : undefined) ?? - rules.find((rule) => rule.ruleType === globalType && rule.targetKey === '') - ); -} - -function carrierRule(rule: NonNullable, carrier: string) { - if (carrier === 'mobile') return { windowDays: rule.mobileWindowDays, threshold: rule.mobileThreshold }; - if (carrier === 'unicom') return { windowDays: rule.unicomWindowDays, threshold: rule.unicomThreshold }; - return { windowDays: rule.telecomWindowDays, threshold: rule.telecomThreshold }; -} - -function renderMessage( - template: string | null, - dimension: DetectionDimension, - windowDays: number, - threshold: number, - actual: number, -) { - const fallback = - dimension.dimensionType === 'enterprise' - ? '请通知 {enterprise}：{signature}在{carrier}过去{days}天发送{actual}条，低于{threshold}条，请及时进行保签名发送。' - : '{signature}在通道{channel}的{carrier}过去{days}天发送{actual}条，低于{threshold}条，请及时通知客户或进行保签名发送。'; - return (template?.trim() || fallback) - .replaceAll('{enterprise}', dimension.tenantName) - .replaceAll('{signature}', dimension.signatureName) - .replaceAll('{channel}', dimension.channelName ?? '-') - .replaceAll('{carrier}', carrierLabels[dimension.carrier] ?? dimension.carrier) - .replaceAll('{days}', String(windowDays)) - .replaceAll('{threshold}', String(threshold)) - .replaceAll('{actual}', String(actual)); -} - function shanghaiDateKey(date = new Date()) { return shanghaiDayFormatter.format(date); } diff --git a/docs/first-version-development-requirements.md b/docs/first-version-development-requirements.md index b1a255a..4049b3b 100644 --- a/docs/first-version-development-requirements.md +++ b/docs/first-version-development-requirements.md @@ -2339,3 +2339,18 @@ Webhook需在当前受支持Node运行时通过真实HTTPS投递；SSRF校验后 5. 监控刷新失败显示真实上一快照和过期提示，超时/切换请求取消，恢复后刷新；不同查询范围隔离。 授权交付：代码、定向/全量测试、精确提交、推送、测试环境标准发布及隔离模拟短信验收；不操作预生产，不向真实运营商发送。 + +## 2026-09-17 签名质量独立查询与日报优化（本地实现，待发布） + +本节对应用户本轮方案要求，权威设计见[签名退网检测与四页查询优化方案](signature-quality-optimization-plan-20260917.md)，本地实施状态见下方补充，线上尚未发布。 + +1. 四TAB的日期、筛选、分页、请求、错误及结果独立；企业/通道活跃度改为带dimensionType的独立服务端查询与分页，不再各自获取两类全量数据。保留默认25条和10/25/50/100档；同TAB顶部和卡片查询应用同一组草稿条件。 +2. T为服务器北京时间今天，实际统计日为d：d=T查询时实时查业务库；T-1～T-3每天凌晨同时刷新、查询只读日报；T-4及以前只读冻结日报，普通查询、调度和重启不得重算。缺报表明确显示未生成，不回退明细或伪装零。 +3. 用户确认热力图保持所选日D之前30天，即D-1～D-30，不包含当天列；每格按实际日期相对真实T判断数据来源，不按页面D重新开放历史刷新。 +4. 日报历史名称、归属和报备适用性快照保留；冻结后迟到回执、补登记和状态变化不覆盖旧报表。预警判断/通知与可刷新的活动日报分开，刷新不得重发历史通知。 +5. 退网检测采用批量聚合、提前跳过已完成、任务原子认领与故障接续，候选索引由真实执行计划决定。保留业务短信/提交尝试/计费片数区别、长短信完整分段判定和跨通道/跨日窗口去重；现有最长365天规则不缩短。 +6. 原方案阶段为只读；本轮本地实现与隔离验收见下方补充，线上历史补建、推送、两环境部署与真实发送均未执行。本节不改变其他财务/质量报表T-4～T-1刷新规则。 + +### 2026-09-17 本轮本地实现 + +签名质量日报按专项优化方案实现：T实时、T-1～T-3每日刷新、T-4冻结；热力图D-1～D-30且企业/通道请求独立，失败保留本TAB上次真实数据并标明日期。退网窗口批量去重、规则快照/租约/fence/失败恢复与日报解耦。上行按应用唯一性而非短信条数归属，同通道接收前accepted作为证据；入库、候选、认领、通知意图保证事务一致。历史待认领不自动处理。本轮仅本地修改和提交，线上发布/补建及实际短信/通知验收另按授权执行。 diff --git a/docs/signature-quality-optimization-plan-20260917.md b/docs/signature-quality-optimization-plan-20260917.md new file mode 100644 index 0000000..2fa3807 --- /dev/null +++ b/docs/signature-quality-optimization-plan-20260917.md @@ -0,0 +1,191 @@ +# 签名退网检测与签名质量四页查询优化方案 + +维护日期：2026-09-17。状态：**本地实现及隔离验收完成，待发布**。用户于本轮授权修改并本地提交；没有授权本轮推送、部署、历史上行认领或真实短信/通知发送。第1节保留实施前取证；本轮实现与验收以第8节为准。 + +本文是本次三项需求的统一实施设计，补充[签名清退设计](signature-retirement-alert-design.md)。实施后替代该设计第6节、第7节第15步中“热力图直接读检测快照、按当前报备维度展示、前端筛选分页”的实现方式，以及[运营修复设计](operations-fixes-20260908.md)第4项中的客户端热力图分页；保留四TAB独立状态、默认25条、10/25/50/100档、D-1至D-30列序和原业务统计单位。现阶段旧代码仍在运行，不能把本方案描述为已上线功能。 + +本方案仅针对“签名质量检测”四TAB与签名退网检测，不改变“报表对账/发送质量报表”的既有T-4～T-1刷新任务，不改变短信门禁、补发、计费、余额或报备状态。 + +## 1. 核查结论与证据 + +### 1.1 基线 + +- 本地main与实际远端main均为`4eb7b16d122da14f921093716d4ca1ed390d9e4c`，暂存区空；已有版本、metrics、发布工具、部署脚本及文档草稿保留。 +- 2026-09-17 09:37北京时间只读核验预生产运行`010ba3216889032a6160cdb14d8536b616ae7102`。上述两个提交间本方案涉及的页面、质量查询和退网检测源码没有差异。 +- 证据：`src/apps/admin/AdminAnalyticsPage.tsx`、`src/api/admin/signature-retirement.api.ts`、`api/src/operations/queries/quality.queries.ts`、`api/src/signature-retirement/signature-retirement.service.ts`、对应controller及`api/prisma/schema.prisma`。 +- 本机原始取证在忽略目录`.local-data/cpu-20260917/`：`prom.json`、`detail.jsonl`、`design-audit.json`及只读脚本。线上DB使用只读事务与8秒语句超时，未启动应用调度器。此次未执行登录后的四TAB真实浏览器/HTTP复测；以下交互结论来自当前源码，不冒称已复现浏览器串数据。 + +### 1.2 凌晨CPU与检测成本 + +今天检测日2026-09-17的结果从04:00:01.620写到04:04:07.792，共5,831条（企业1,431、通道4,400）。前一天4,280条。Prometheus五分钟CPU均值峰值58.79%，04:06:45为32.81%；04:06最近一分钟忙碌率约2.54%，趋势包含此前负载。日报刷新日志完成于00:33:09。 + +同窗口`SmsMessageRecord`与`SmsSubmitRecord`顺序扫描行速率峰值分别约362万、536万行/秒；这是数据库重复访问行的计数，不是短信量或TPS。IO等待较低。检测时段、真实快照与数据库扫描高度吻合，是主要负载线索；缺少历史业务进程CPU与SQL耗时采样，不能量化各进程占比或断言唯一原因。 + +现有`runDetection`逐个维度调用`activityCounts`：至少一次单日查询，完成观察期后再查一次规则窗口；每次关联消息、提交、分段/回执。已有检测是否存在的判断在这些计算之后的`persistDetection`中，重启补偿仍可能先重复计算再退出；周期、抑制、检测记录还存在逐行读写。现有唯一键保证部分去重，不等于任务已具备全程原子认领和完整事务恢复。 + +### 1.3 四TAB是否关联 + +| TAB | 当前请求与数据来源 | 当前关联/独立性 | 与目标的差距 | +|---|---|---|---| +| 签名通道发送质量 | `GET /admin/operations/signature-quality`；所有日期实时聚合消息、提交、分段与回执 | 独立日期、关键字、分页、请求序号；详情使用本TAB结果 | 历史也查明细，无近三天报表/冻结分支 | +| 企业签名活跃度 | `GET /admin/signature-retirement/heatmap?date=D` | 独立组件状态，但接口返回企业和通道全部数据，再在前端选企业、过滤、排序分页 | 使用次日检测快照而非可刷新的活动日报；行集合依赖当前通过任务 | +| 通道签名活跃度 | 同上 | 独立发起请求，但再次拉取相同两类数据，再在前端选通道 | 同上，整月全量传输和处理重复 | +| 未报备签名 | `GET /admin/signature-retirement/unreported-signatures`；所有日期查消息正文及当前有效签名库 | 独立日期、关键字、分页与结果 | 历史实时重算；补登记/删除签名会改变旧日期结果 | + +结论：没有发现四TAB共用筛选状态或一个TAB直接改写另一个结果的实现；存在两个活跃度TAB共用全量接口与数据源，且活跃度展示依赖退网检测任务。底层业务事实本来有关联，不能为“页面独立”复制出四套互相矛盾的短信事实；应解除请求、状态、失败和生成触发之间的耦合。 + +另需同步修复的本TAB日期歧义：未报备TAB编辑顶部日期后，卡片内“查询”仍按`appliedDate`请求，须先点顶部“查询统计”才能应用新日期。这不是跨TAB串数据，但会造成查询条件似乎未生效。 + +### 1.4 三档取数规则当前未实现 + +质量与未报备页面全部日期实时查库；活跃度使用`SignatureRetirementDetection`快照，`persistDetection`已存在即返回，不会连续刷新最近三天。历史展示又拼接当前签名名称和当前报备维度，因此既不是完整的实时统计，也不是稳定冻结日报。 + +已有`DailyQualityReport`服务另一个报表模块，按计费单位统计，维度缺少本页完整的运营商、通道、引流拆分、去重业务数与到达耗时分子分母，不能直接拿来替代本页。既有财务/质量报表T-4～T-1策略也不自动等同本次要求。 + +## 2. 已确认业务规则 + +### 2.1 日期定义和数据来源 + +T始终指服务器按`Asia/Shanghai`确定的真实今天；D指页面所选日期；d指实际统计自然日，均不依赖浏览器所在时区。用户已确认：**每天凌晨同时刷新T-1、T-2、T-3；热力图保留所选日之前30天。** + +| 实际统计日d | 查询行为 | 后台生成/更新 | +|---|---|---| +| d=T | 本次查询访问真实业务库，在一致性读视图中聚合 | 不将昨日缓存冒充当天数据，不写退网检测或通知 | +| T-3≤d≤T-1 | 只读已完整发布的日报 | 每天凌晨刷新最近三个完整自然日，吸收迟到回执 | +| d≤T-4 | 只读冻结日报 | 普通调度、页面查询、进程重启均不得自动重算或覆盖 | + +例：今天9月17日，9月17日实时，9月14～16日报表可刷新，9月13日及以前冻结。9月18日可刷新范围成为9月15～17日；9月14日报表保留9月17日最后成功发布的版本，不在成为T-4后再额外重算。刷新判断相对真实今天，不能相对用户选择的历史日期重新开放冻结区。 + +热力图选D=9月17日仍展示9月16日至8月18日（D-1至D-30），没有当天列；每一列按该格子的d判断可刷新/冻结。单日质量与未报备TAB选今天时采用实时路径。禁止选择未来D来变相让热力图出现今天列。 + +冻结意味着“固定统计截面”，不意味着所有短信此时都已有最终回执。冻结后迟到回执继续正常更新短信业务事实，不改已冻结报表，不把未知自动改成失败；报表与当前详单存在截面差异时明确显示截止时间。跨日补发也不得使已冻结日的历史报表被普通任务改写。 + +### 2.2 独立查询的边界 + +- 四TAB各有日期草稿/已应用日期、过滤草稿/已应用过滤、排序、分页、pageSize、请求取消/序号、loading/error和结果，任何查询只更新本TAB。 +- 首次打开TAB只请求该TAB；切回保留自己的条件及已取得结果，用户查询/刷新只重查当前TAB。隐藏TAB不自动重查，不用另一TAB成功响应填补失败。 +- 企业、通道活跃度请求明确携带不同`dimensionType`，服务端只处理所选维度，按过滤后的完整30日合计排序再分页，只返回当页维度及其30格。 +- 日期与搜索“查询”统一应用本TAB当前全部草稿条件并回到第一页；翻页只使用已应用条件。特别修正未报备卡片查询沿用旧日期的问题。 +- 底层只读聚合器、数据库连接池和已发布日报可以复用；页面查询不触发其他TAB请求、不触发报表重建、不运行退网检测、不创建预警。 + +### 2.3 指标和历史维度 + +- 质量列表/运营商概览以业务短信为单位；通道矩阵以真实发送尝试为单位；保留现有不同成功率分母并在契约逐字段登记，不把计费片数换成业务条数。 +- 业务统计按`SmsMessageRecord.queuedAt`归日；通道尝试及活跃度按现有`COALESCE(submittedAt,createdAt)`归日。跨日补发可能使列表与矩阵日期归属不同，应分别生成，不能先用当天业务列表裁掉当天实际提交、但原消息属于其他日期的事实。 +- 企业活跃度在统计范围内按messageRecordId去重，通道按messageRecordId+channelId去重；不直接累加通道值得到企业值。规则窗口跨日也须重新去重，不能把每日distinct数简单相加。热力图“30日合计”保持每日展示值之和并标注为日活跃量合计，区别于预警窗口去重人数/条数。 +- 成功必须按发送尝试完整分段事实判断，缺片/缺回执仍为未知；长短信不能因已到的几个分段都成功而提前成功。旧记录无分段事实时才按既有明确回执兼容；不把上游接受当送达。 +- 平均到达时长持久化成功耗时总和与样本数，展示时相除；不能平均各组平均值。列表、抽屉、运营商概览及引流三类拆分使用相同generation版本。 +- 日报保存当次统计维度名称、企业应用归属、运营商、报备适用状态/通过时间与统计规则版本。历史筛选使用报表快照字段，不能inner join当前通过任务导致历史行消失。活动日报逐日记录适用性；尚未通过、不适用、已通过且零发送、未生成必须区分。 +- 未报备仍指规范正文签名在该企业应用有效签名库中不存在，且原消息无signatureId，不等于缺少通道报备。实时用查询时签名库；近三日用本轮生成时签名库；冻结日保留最后发布判定及名称。此后补登记不得追溯抹除冻结历史。 + +## 3. 性能优化方案 + +### 3.1 优先消除重复计算 + +1. 将逐个签名×通道×运营商反复扫描，改为按统计日有界批次聚合；先限定日期与候选消息/提交，再对这些submitId批量归集分段/回执，避免每个维度重新读取相同事实。 +2. 一次读取参与检测的规则、通过任务、既有检测键与抑制摘要。已经完成的检测批次直接退出；部分完成只恢复缺失维度，避免昂贵计算后才查存在性。 +3. 单日活动日报与质量日报复用经核验的尝试分类逻辑，按各自归日维度生成结果。退网“是否活跃”的窗口只需要accepted去重，不为判断阈值重复关联所有回执；送达率由日报单独计算。 +4. 预警窗口最长支持现有365天，按实际生效规则的最大窗口限制扫描，批量JOIN规则维度后按message/channel去重。企业跨通道、跨日不能直接SUM日报；在测试库比较批量COUNT DISTINCT与适量窗口分组方案后选定。禁止以改成30天窗口偷换规则。超大窗口必要时分阶段物化紧凑去重键，但须先盘点容量，不能本轮默认复制全量短信正文或全部历史明细。 +5. 批次发布后退网任务引用已完成单日活动版本，并独立保存当时窗口判断及规则版本。刷新日报不更新既有检测决策、不推进旧周期、不重新生成通知。 + +### 3.2 索引和SQL计划 + +预生产真实索引已有message.queuedAt、submit.messageRecordId，但未见消息的signatureId+carrier组合索引，也未见submit的`COALESCE(submittedAt,createdAt)`表达式时间索引。不能仅因没有索引就断言某一SQL必然全表扫描。 + +候选包括消息(signatureId,carrier,id)、提交(COALESCE(submittedAt,createdAt),messageRecordId)、按实际计划需要的通道+有效提交时间，以及日报日期/维度/排序组合。保留表达式原语义，不能直接用createdAt替代submittedAt。候选须通过有代表性数据的`EXPLAIN (ANALYZE, BUFFERS)`验证选择性、loops、临时文件和写入成本后决定，不盲目全部加索引；本轮未执行重查询或建索引。 + +大表索引上线采用平台支持的在线建索引流程，明确非事务DDL、失败无效索引检查及重试；具体迁移方式在实施时结合Prisma发布门禁验证，禁止把不允许在事务中的DDL塞进普通事务而绕过错误。不全局扩大work_mem，不默认增加SQL/worker并发。 + +### 3.3 页面成本 + +- 活跃度改服务端过滤、排序、分页，只传单类维度的当页30格；四TAB仍各自独立，不将两个活跃度结果捆绑返回。 +- 查询快照元数据和维度使用Map/数据库JOIN，不逐条Array.find造成大集合反复查找；保留已有模块级日期格式化器。 +- 首屏与列表查询仅取所需字段；若详情改独立请求，必须携带列表generationId以保证详情一致，不能点击详情又扫原始历史短信。 +- 当天实时查询采用有界日期、分页和超时；失败保留当前TAB最后真实结果并标注其日期/时间，不能把旧值作为新查询成功。跨午夜请求由服务端固定T并回传，前端不自行混合两个自然日结果。 + +## 4. 日报生成、冻结和恢复 + +建议签名日报任务北京时间03:00启动，依次处理T-3、T-2、T-1，与已存在财务日报错峰；04:00退网检测、08:00通知时点保留。固定时间是拟实施配置，尚未上线；也必须检查与其他后台任务是否重叠。 + +生成任务具备耐久状态`pending/running/succeeded/retry_wait/failed`、租约、递增fence和有限重试；按scope+businessDate唯一认领，旧worker过期后不能发布。每个日期在一致性数据库快照下生成候选版本，完成四类校验后短事务切换publishedGenerationId。部分失败保留完整旧版；其他日期可继续，但整轮不得虚报全成功。发布必须再次验证fence、日期是否已冻结和候选水位。 + +页面只读published版本；生成中有旧版则显示旧版及刷新中/上次成功时间，没有旧版显示“报表尚未生成”，失败显示失败，不能返回空数组冒充零数据或回退扫描明细。合法零数据日期也必须生成完成清单与0行标记。 + +日界线进入T-4后自动禁止覆盖；跨午夜仍在运行的旧任务在发布校验时被拒绝。若T-3最后刷新失败，冻结上一成功版本并显示未达到预期截止时间；若从未成功则保留缺口，不伪造冻结完整。错过整个三日窗口的缺口只能进入显式历史补建流程，不能借“启动补偿”无限回算。 + +04:00所需日报未完成时，检测等待该日完成且告警；08:00只发送已完整完成检测的结果，不把半批维度当完整预警。后续恢复沿用当日补偿与幂等，不补发旧日期通知。周期变更、检测行、通知意图在维度事务中一致提交；全批完成标记必须在全部维度校验后置位。通知仍按日期+企业应用聚合及现有抑制规则，重试不能重复创建消息/Webhook。 + +日报刷新三天不等于重新做三天的预警判断。`SignatureRetirementDetection`保留历史判断审计，页面活动日报是另一投影，两者截止时间可能不同，页面标明统计口径而不是互相覆盖。 + +## 5. 拟新增模型与接口 + +以下名称仅为设计，不表示Prisma已有模型： + +| 模型 | 主键/唯一维度与作用 | +|---|---| +| SignatureAnalyticsDay | businessDate唯一；publishedGenerationId、state、generatedAt、sourceAsOf、frozenAt、schemaVersion、coverage/provenance、rowCounts/checksum；成功零行也记录 | +| SignatureAnalyticsRun | scope+businessDate+generationId唯一；owner、leaseUntil、fence、attempt、checkpoint、startedAt/finishedAt、error摘要；每scope/date最多一个有效运行者 | +| SignatureQualityDaily | generationId+metricKind+tenant/application/signature+carrier/channel/drainage规范化维度键唯一；区分business与attempt，计数、耗时分子分母、名称快照 | +| SignatureActivityDaily | generationId+dimensionType+tenant/application/signature/channelKey/carrier唯一；单日提交、accepted业务数、送达数、适用性及报备快照 | +| UnreportedSignatureDaily | generationId+tenantId+applicationId+规范正文签名键唯一；业务数、生成时判定及名称快照 | + +空channel/application使用无碰撞规范键，不能依赖可空字段唯一约束防重复。行均有真实businessDate和generationId外键；普通查询仅取发布版本。预警任务可复用耐久Run机制但scope独立，不与日报共用完成状态。旧版本保留期限和空间预算须在实施/发布阶段核验，不自动清理现有业务记录。 + +API建议： + +- 质量与未报备保留现有GET路径和date/keyword/page/pageSize，增加`dataSource=live|report`、`reportState=ready|refreshing|failed|missing`、`mutable/frozen`、businessDate、serverBusinessDate、generatedAt/sourceAsOf、generationId和schemaVersion。数据源由服务器决定，客户端不能要求重算冻结日。 +- 热力图提供独立版本路径`GET /admin/signature-retirement/activity`，必填dimensionType=enterprise|channel，date为D；企业/应用/签名/通道过滤分别传入，pageSize只接受10/25/50/100。返回单类当页维度、total和D-1至D-30；逐日期带版本/完整性元数据。保留旧heatmap GET只作兼容，最新页面不再调用；记录旧调用后再决定移除，不让旧前端接收截断结果冒充完整数据。 +- 活跃度筛选、30日排序、总数与当页结果在同一个一致性读事务获取；先在过滤全集排序再LIMIT。缺失日期不可当成0参与已完整合计，须展示“部分日期缺失”。冻日前名称和维度只来自报表。 +- 日期严格验证真实日历、拒绝未来日；页码为正整数、pageSize有上限，排序白名单。请求取消和旧响应隔离按TAB独立实现。 +- 沿用运营端鉴权及数据可见范围，在数据库查询前约束tenant/application；不新增匿名查询/生成接口，不以报表缓存绕过权限。真实HTTP401/403、越权筛选和缓存键隔离须专项验收。 + +## 6. 历史切换、发布和恢复 + +1. 在真实数据规模的隔离测试库建立旧口径对照，包括长短信、补发、跨日和未报备；冻结字段契约，修正差异而非迎合旧错误。 +2. 增量新增模型/索引与后台生成器，旧接口继续运行；候选生成器先影子计算，不产生通知，完成准确性与负载对照后切换。 +3. 盘点可支持历史日期的源数据覆盖、报备轨迹、消息/回执保留和磁盘空间。一次性按明确日期清单补建历史日报，先满足热力图至少30日与用户常用查询区间；预警规则窗口另覆盖实际最大365日需求。没有可信历史事实就标记缺口，不把当前报备状态当作过去状态。 +4. 旧检测快照可作为活动量迁移证据，但不能冒充补齐迟到回执后的完整日报。用原始事实重建的历史报表须记录backfilledAt/sourceAsOf/provenance，不能伪造为当年T+3冻结结果。完整性不足的日期不宣称已完成，历史可查询范围明确展示。 +5. 迁移期不复制历史周期、不重发历史消息。生成和读取按schemaVersion切换；切换后禁用旧逐维度检测调度入口，防止双跑。先测试环境验收，再另按授权发布预生产。 +6. 应用回退保留新增表和已发布冻结版本；异常优先暂停新任务、保留只读已发布日报。旧应用会恢复历史实时查询行为，不能称之为满足新冻结规则的等价回退，须在恢复说明中明确影响。不自动恢复数据库或删除候选/旧报表。 + +## 7. 验收、观察指标与实施顺序 + +详见[功能用例](system-functional-test-cases.md)中`TC-SQA-20260917-01～16`，本地执行覆盖及环境未执行项见第8节。 + +- 准确性：真实PostgreSQL逐字段对账；短/长短信、缺片、重复乱序、失败补发、跨通道/跨日去重、未知和迟到回执；旧历史无分段兼容；成功率分母及耗时加权正确。 +- 冻结：T、T-1、T-3、T-4、跨午夜/跨月/闰日；刷新原子性、版本一致、补登记不改变冻结未报备、名称/通过状态变更不让历史行消失；失败与零数据可区分。 +- 独立性：四TAB首次/切换/查询/分页/详情；一个TAB失败、慢响应或修改日期不影响另一个；企业请求不含通道维度；真实HTTP与三尺寸浏览器核验，不仅用mock证明功能。 +- 可靠性：双worker抢占、租约超时接管、旧fence发布被拒、批次部分失败、任务重启、错过冻结窗口、08:00依赖未完成、重复通知去重。报表刷新不写短信/余额/报备/通知状态。 +- 性能记录：整机及PG/API进程CPU、数据库扫描行/块、SQL耗时/p95、任务耗时、锁等待/临时文件、响应字节、API内存、短信队列延迟；分别标注冷/热缓存和相同业务规模，不把一次结果当容量承诺。 +- 拟验收目标：相同数据规模，退网原始明细扫描行数下降至少80%，任务耗时下降至少50%；单类热力图每页最多100×30个格，响应规模不随未选维度总数增长。此为测试目标而非已实测收益；不能单凭CPU目标判定成功。若准确性/业务队列恶化立即停止候选压测，不在预生产重新跑整套重任务验证猜测。 + +实施分三批：①独立接口/页面查询及精确统计契约；②日报三日刷新/冻结/历史补建与版本化读取；③退网批量聚合/原子任务/依赖编排及性能对照。批次①不宣称已完成报表化，③完成前不宣称CPU问题已解决。 + +这是前端、查询后端、数据库模型和后台调度的跨模块改造，主要成本在历史回填、口径对账和故障恢复验收，不能按简单索引补丁处理。实施需API/前端定向与全量回归、类型/构建/质量门禁、真实PG/接口/浏览器；若不修改Gateway无需冒充重做Gateway发布。真实短信、外部通知及压测只在后续明确授权的环境和范围执行。 + +## 8. 2026-09-17 本地实现与验收 + +### 8.1 实现及设计落地 + +- `api/src/signature-analytics/`提供北京时间日期校验、日报聚合、版本读取、后台调度和独立退网批次。03:00后依次刷新T-3/T-2/T-1，60秒补偿检查；04:00检测与08:00通知保留。日报失败不降级实时查询，未生成与合法零结果分开；热力图继续D-1～D-30。 +- 新增六表：Day发布清单、Generation不可变版本、Run耐久任务、QualityDaily、ActivityDaily、UnreportedDaily。日报行和发布指针以generationId+businessDate复合外键保证日期/版本一致。质量嵌套矩阵和概览保存在同一JSON快照，检索字段独立列化；引流分组保存耗时总和/有效样本数并加权合并，不平均组均值。 +- 与原建议“候选后短事务切指针”相比，本期采用**每个自然日一个RepeatableRead事务**，聚合、分批250行写入、发布与fence验证一起提交。理由是保证同日各口径取自同一源快照，避免未持久化源水位的断点续算混入不同截面。单SQL90秒、事务120秒、租约300秒；失败重算该日，成功日期不重算，最多5次指数退避。事务上限短于租约，不需要保持长事务跨租约续期。超过此规模应先扩展分阶段水位设计，不能直接无限增大超时。 +- 退网窗口只扫描accepted提交，批量按实际规则窗口（最长365天）去重，单日送达数复用活动日报。规则与所引用日报版本在首次认领后持久化为checkpoint，重试不改变；检测、周期及冻结通知正文同事务。完成检查前移；日报刷新不修改已有检测/消息。缺昨日当日刷新版本时等待，不消耗检测重试次数。通知沿用原耐久消息/Webhook去重，同进程完成后不每分钟重扫全部消息；重启可补偿，禁止自动补发历史检测。 +- 两个活动TAB改用`/admin/signature-retirement/activity`，明确dimensionType；服务器过滤、排序、分页和完整性读取处于一致性事务。每页只返回所选维度，最多100×30格；空的越界页仍返回正确总数。旧heatmap接口保留兼容旧页面，当前页面不调用。四TAB查询/取消/错误/日期/筛选各自独立，未报备卡片查询应用新日期。 +- 普通历史读取只访问日报快照，无当前报备/名称JOIN。报备轨迹可还原时取历史状态与通过时间，旧数据仅有明确approvedAt时保留兼容证据；缺完整历史轨迹不宣称重建了过去所有状态。无报备资格的格子N/A，缺日报的格子显示缺口。旧记录完全无分段审计时保留明确回执兼容；存在分段时按预期总片数判断，缺片不提前成功。 +- 迁移`20260917030000_signature_analytics`新增表；`20260917031000_signature_submit_effective_at`单独以非事务`CREATE INDEX CONCURRENTLY`建立有效提交时间表达式索引，避免将所有历史源扫描混进三日任务。108项完整迁移已在独立PostgreSQL库执行。线上失败须检查`pg_index.indisvalid`及迁移状态，按发布恢复流程处理；不自动DROP/resolve失败迁移，不绕过门禁。 + +### 8.2 历史补建与发布边界 + +离线入口`tools/testing/backfill-signature-analytics.mjs`使用已构建API，要求环境变量DATABASE_URL和匹配目标主机的`--host`，逐个明确`--dates=YYYY-MM-DD,...`；默认只检查，明确`--execute=yes`才生成。禁止未来/当天、隐式范围和覆盖已有发布版本。最多366个显式日期，逐日顺序生成；不初始化应用调度、不运行检测/通知，结果标记`backfill-current-source`与实际sourceAsOf，页面标明“事后补建”。本轮只在隔离库验证，线上补建未执行。 + +发布时需先核对源数据覆盖、实际报备历史与存储空间，生成影子日报并对账，至少为常用D准备此前30日，再切换页面；缺失日期保持可见缺口。新版本/旧版本表行均保留，不自动清理。回退应用保留新增表与冻结版本，但旧应用将恢复旧历史查询行为，不是业务等价回退。本轮没有推送或部署，也没有声称线上CPU已下降。 + +### 8.3 已执行证据及限度 + +- 真实PostgreSQL16隔离集群127.0.0.1:16435，空库完整迁移；`verify-signature-analytics.mjs`覆盖日报三日、冻结、跨日去重、2/3/4片、缺片、旧无分段回执、耗时加权、超出页码、重复认领、旧fence拒绝、失败回滚/重试及规则快照。验收数据只在本机专用`cmpp_qa_signature_*`库，脚本拒绝其他主机/库名。 +- `verify-uplink-matching.mjs`真实事务验证重复事件、应用归属、供应商MO ID保留、并发人工认领和通知存储故障回滚。故障注入仅模拟存储错误；实际数据/事务/通知意图均为PostgreSQL，未调用外部Gateway发送。 +- 真实完整Nest应用、PostgreSQL和隔离Redis，生产构建页面在1600×1000、1366×768、390×844打开/切TAB/查询/刷新，页面异常0；HTTP正常查询200、匿名401、非法维度/分页400。浏览器认证会话仅内存传递，不保存密码、令牌或storageState。Redis本机为5.0.14.1，有BullMQ建议6.2警告；本轮未将其视为生产队列能力验收。 +- 前端全量35文件/163项、API全量78套/842项；类型检查、构建、代码结构、格式、ESLint、CSS治理、样式和Bundle门禁记录在testing-progress。Gateway代码未改，不重复冒称实发/协议验收。 +- `benchmark-signature-analytics.mjs`对照4eb7b16原活动聚合：30000条消息、600维度，本地热缓存旧逐维度10343.912ms，新批量624.940ms，减少93.96%，accepted计数逐维度相等。EXPLAIN原单维度实际源表访问30113行，按600维度估算约1806.78万；新批量60039行。旧总行数是样本外推，不是全批逐SQL实测；均为热缓存，未产生临时磁盘块。这里只证明活动聚合环节，整个日报+365日检测批次的CPU、p95、内存、线上队列影响、真实高密度回执规模仍须部署前/后专项验收，不能宣称全部性能目标已达标。 +- 本机原始日志、截图和故障证据在忽略目录`.local-data/signature-optimization-20260917/`。初次迁移目录未生成SQL、测试通道缺carriers、断言漏计样本、临时账号缺email等失败已修正并保留原日志，不用最后成功覆盖失败历史。 diff --git a/docs/signature-retirement-alert-design.md b/docs/signature-retirement-alert-design.md index 5006852..959b245 100644 --- a/docs/signature-retirement-alert-design.md +++ b/docs/signature-retirement-alert-design.md @@ -193,3 +193,11 @@ - 第4至10步可回滚到能够读取运营商集合和历史任务的兼容版本；出现双运营商通道后不得回滚到只识别旧单值的版本。 - 第11步严格门禁启用后，如出现真实发送资格异常，优先切回兼容双读版本，不回滚或覆盖已经产生的新业务数据。 - 任一阶段不得为验收发送、补发或重投真实短信，不修改真实通道账号、密码、启停状态、企业余额或客户连接，除非另获明确授权。 + +## 9. 2026-09-17 性能与签名质量日报改造（本地实现，待发布） + +权威增量设计见[签名退网检测与四页查询优化方案](signature-quality-optimization-plan-20260917.md)。实施后替代本文第6节、第7节第15步中检测快照直接作为活动查询、当前通过任务决定历史行及前端全量筛选分页的方式；保留D-1～D-30日期列、原去重单位、规则/抑制和预警历史。四TAB独立查询，日报T实时/T-1～T-3凌晨刷新/T-4起冻结；活动日报与当时预警判断分离。原正文是历史实现描述，本节及专项方案的本地实施状态见下方补充，线上未发布。 + +### 2026-09-17 实施状态补充 + +上述专项方案已完成本地代码与隔离验收，具体表结构、03:00三日刷新、事务边界、恢复、离线补建及性能测量见专项方案第8节。历史正文保留为旧实现说明；尚未推送或部署，线上依然是旧行为。 diff --git a/docs/system-functional-test-cases.md b/docs/system-functional-test-cases.md index 53d9b37..b981992 100644 --- a/docs/system-functional-test-cases.md +++ b/docs/system-functional-test-cases.md @@ -5614,3 +5614,40 @@ RC-01/02/04/08/09/10/11目前仅部分本地证据：三段齐段、重复与双 ### 2026-09-16 五项整改最终测试证据索引 TC-RC-20260916-01～12按[测试交付记录](release-20260916-test-completion.md)逐项区分真实目标链路、隔离PG故障验证及未执行场景，不能整组标全部通过。两段/三段/四段、补发成功/失败退款、20HTTP通知及9CMPP目标、503/超时自动恢复、无客户回执目标和无租户通道测试均有真实证据；全部逐断点退出/72小时/历史组合/旧版恢复/等负载性能对照未完成。TC-RC-13修复前真实TCP复现，修复后5000次即时响应及测试环境新增6条18分段无补发通过。五项运营页面检查包含三尺寸、请求失败保留真实快照与恢复、恢复告警不自动消失及人工清除审计。详细计数与测试资源收尾以交付记录为准。 + +## 2026-09-17 签名质量日报与退网检测优化验收（本地覆盖，环境项待执行） + +设计见[优化方案](signature-quality-optimization-plan-20260917.md)。以下全部为计划用例，本轮只读取证不等于用例通过；不沿用此前短信测试授权。既有TC-ANALYTICS、TC-SIGNATURE-RETIREMENT保留历史记录，与本轮替代口径冲突时以本节和方案为实施目标。 + +| 编号 | 场景 | 验收要求 | +|---|---|---| +| TC-SQA-20260917-01 | 四TAB切换、不同日期/过滤/分页、慢响应及失败 | 仅当前TAB查询；条件与结果互不覆盖；企业请求不返回通道集合，反之亦然；隐藏TAB不重查 | +| TC-SQA-20260917-02 | 未报备TAB编辑日期后分别点顶部/卡片查询，再翻页 | 两个查询入口均应用当前全部草稿条件；翻页只用已应用条件，不串旧日期 | +| TC-SQA-20260917-03 | 今天查询质量和未报备 | 真实业务库一致性聚合，响应标记live及截止时间；不触发生成/检测/通知；过午夜由服务端确定自然日 | +| TC-SQA-20260917-04 | T-1/T-2/T-3迟到回执后再次凌晨生成 | 三天均更新日报，页面只读发布版本；失败保留旧版并标记；同一版本列表、概览、抽屉一致 | +| TC-SQA-20260917-05 | T-4及更早日收到迟到回执/补登记/更名 | 原短信事实可按原业务规则更新，冻结报表内容摘要不变；历史行和名称不依赖当前报备状态 | +| TC-SQA-20260917-06 | 热力图D为今天/历史日，跨月年及闰日 | 严格显示D-1～D-30，不出现D当天；每格按真实T判断冻结；未来D拒绝 | +| TC-SQA-20260917-07 | 有效零数据、缺报表、无报备资格、部分日期失败 | 四种状态明确区分；缺口不能计作已确认零或触发回查明细；部分30日合计明确不完整 | +| TC-SQA-20260917-08 | 短信跨日/跨通道补发，同通道多次accepted | 企业窗口按业务消息去重、通道按消息+通道去重；窗口不简单累加每日distinct数；质量业务日与尝试日分别核对 | +| TC-SQA-20260917-09 | 2/3/4段、缺片、乱序重复、失败与历史无分段记录 | 完整成功才算送达；未知保持未知；去重和旧回执兼容正确，耗时分子分母及成功率口径不漂移 | +| TC-SQA-20260917-10 | 两worker、租约失效、旧worker恢复、重复执行 | 同scope/date原子认领，旧fence不能发布；批次断点恢复，已完成不重新扫描，周期/快照/意图事务一致 | +| TC-SQA-20260917-11 | 候选生成部分失败、发布时跨入T-4、漏跑三日 | 旧版原子保留，跨冻结边界拒绝覆盖；缺口进入明确历史补建流程，不自动无限回算 | +| TC-SQA-20260917-12 | 04:00日报未好、08:00检测未完成、历史日报刷新 | 检测/通知按依赖等待且告警，不用半批发通知；恢复幂等，不重发历史消息/Webhook，不改已发正文 | +| TC-SQA-20260917-13 | 热力图大数据过滤排序分页及一类TAB失败 | 数据库先过滤并按完整30日合计排序再分页；只返回当页单类维度；另一TAB可独立查询；响应最多100×30格 | +| TC-SQA-20260917-14 | 真实接口鉴权/越权、非法日期和分页参数 | 401/403正确，tenant/application可见范围生效；无匿名生成入口；缓存不越权；SQL参数化 | +| TC-SQA-20260917-15 | 同规模新旧方案性能与准确性对照 | 所有指标对账；记录扫描行/块、任务耗时、进程CPU、API内存/响应量与队列影响；目标扫描下降80%、耗时下降50%，未达到不得冒称达标 | +| TC-SQA-20260917-16 | 历史补建、切换/回退和真实浏览器 | 历史覆盖/来源标明，无伪造冻结时间；不重发预警；三尺寸1600×1000/1366×768/390×844覆盖进入、刷新、路由、筛选、分页、详情、失败与权限；发布按后续授权执行 | + +### 2026-09-17 本地执行覆盖与线上未执行项 + +TC-SQA-01～14：真实隔离PG覆盖核心日期/日报/长短信/事务/分页路径，真实HTTP覆盖200/401/400，组件回归与三尺寸浏览器覆盖独立TAB。详细断言见`tools/testing/verify-signature-analytics.mjs`，不能将各用例全部边界都标成通过：真实08:00外部Webhook、旧回退版本、跨午夜运行过程、线上权限配置与365日大批次仍未执行。TC-SQA-15只完成30000消息/600维度活动聚合对照，整机CPU/内存/队列未测；TC-SQA-16只完成隔离历史补建、三尺寸页面，真实环境切换/回退未执行。 + +| 用例 | 操作 | 验证结果 | +|---|---|---| +| TC-MO-20260917-01 | 同手机号同应用多条accepted、共享接入号多应用 | 按应用归并；有唯一发送证据可匹配应用，不强填原短信 | +| TC-MO-20260917-02 | 第三条消息属另一应用、其他通道、接收后发送 | 保留真实歧义；通道/时间边界参与过滤，messageId不能跳过证据 | +| TC-MO-20260917-03 | 重复事件并发入库 | 真实PG仅一个上行和一份通知意图，供应商ID保留 | +| TC-MO-20260917-04 | 两个候选同时认领 | 真实PG仅一个应用成功，另一个拒绝，通知仅一份 | +| TC-MO-20260917-05 | 通知存储失败后重试 | 真实PG整笔回滚，事件重试恢复；未执行外部发送 | + +历史159条认领/重新投递、真实供应商MO扩展号码及最终客户收取未执行，不沿用此前测试发送授权。 diff --git a/docs/testing-progress.md b/docs/testing-progress.md index 8c34d82..660e808 100644 --- a/docs/testing-progress.md +++ b/docs/testing-progress.md @@ -5109,3 +5109,31 @@ CUA本轮可用，实际后端文档三尺寸1600×1000/1366×768/390×844无页 标准工具最终deployed-needs-review仅提示业务数量变化：119597/130835→119603/130841，精确对应补测6条/6次，已人工对账；版本/资源/服务及日志检查通过。工具businessAcceptance未自动更新，不改报告伪造完成。全部证据、原失败、耗时、恢复点、磁盘增量/治理未完成项及矩阵未覆盖项见[交付验收记录](release-20260916-test-completion.md)。预生产未操作，未做性能容量对照、真实72小时等待或全部逐断点/旧版恢复演练。 本轮专用应用/接口/凭据/Webhook和两个模拟通道已停用，模拟器/隧道/监听器/临时hosts/本地隔离PG停止或撤销；短信账务及审计数据保留。61项旧工作保护核对完成，共享文档/metrics只增本轮内容。最终收尾文档及测试日志标签修正单独提交推送，不重启应用。密码及会话不进入Git。 + +## 2026-09-17 签名退网与四TAB查询优化方案（未实施） + +- 授权：只读核查并编写方案。main与实际远端4eb7b16、暂存区空；09:37预生产实际010ba32。原50项已有修改/草稿继续保护，不纳入本轮交付。 +- 已核查代码、线上编译入口、PG检测结果/真实索引及本会话CPU取证：04:00:01～04:04:07产生5831条检测，逐维度重复扫描与CPU窗口高度吻合，缺历史进程CPU/SQL采样不能唯一归因。当前TAB状态独立，但企业/通道各自调用同一全量heatmap；质量及未报备所有日期实时查库；活跃度检测快照不做近三日刷新，历史又受当前报备维度影响。 +- 用户确认每天凌晨刷新T-1～T-3；热力图保留所选日D之前30天，不新增当天列。方案明确实际T和D、T-4冻结、迟到回执、历史维度/未报备判定冻结、生成失败/缺口、批量聚合/索引验证、任务租约/fence、跨日去重及预警通知分离。 +- 新增[优化方案](signature-quality-optimization-plan-20260917.md)，向既有签名清退设计追加替代关系，需求及TC-SQA-20260917-01～16同步，全部实现验收用例待执行。取证脚本与结果在忽略目录.local-data/cpu-20260917，不含认证秘密。 +- 本轮仅执行文档路径/编号/规则一致性及diff检查；未执行业务回归、迁移、重算、登录后四TAB真实HTTP/浏览器验收或新旧SQL性能对照。没有把既有组件mock测试当本轮真实功能通过。 +- 本地修改：五份文档；业务代码无改动。本地提交、推送、测试部署、预生产部署：均未执行；未发送/补发/重投短信、未改余额/通道/客户配置或触发外部通知。 + +## 2026-09-17 上行待认领只读诊断（未修复） + +- 预生产010ba32共253条上行：159待认领、92已匹配、2未匹配。147条共享接入号多应用直接返回ambiguous，其中143条接收前72小时实际仅有一个候选应用的同通道accepted发送；另12条手机号多记录全部同应用。已用当前线上编译匹配函数与真实PG只读事务复现两个分支，未启动调度/写入/投递。 +- 明确应用归属与原短信唯一关联不能混为一谈；155条可进一步收敛不等于可直接自动认领。另发现窗口按处理时刻、无上界/通道限定、路由take10先截断等风险；现有89条自动关联只读边界核对未见手机号不符/未来关联/缺同通道accepted证据。 +- 详见[上行匹配诊断](uplink-matching-diagnosis-20260917.md)，区分已证实根因和未发生证据的风险；取证在.local-data/cpu-20260917/uplink-*。未执行HTTP/浏览器验收、写入故障注入或代码修复，未认领/发送/重推上行。 +- main/实际远端4eb7b16、暂存区空；已有保护项及前轮签名质量方案保留。新增诊断、追加进度，执行文档路径和diff检查；本地提交、推送、测试部署、预生产部署均未执行。 + +## 2026-09-17 上行归属修复与签名质量日报优化（本地实现） + +- 授权：修复上行待认领问题、执行`signature-quality-optimization-plan-20260917.md`并本地提交；本轮不推送、不部署，不操作历史上行认领/重推，不执行真实短信和外部通知。开始及提交前main/实际远端均4eb7b16，暂存区原空；65项已有修改/草稿已做摘要及副本保护。 +- 上行：共享接入号不再提前判歧义；receivedAt前72小时同通道accepted记录按应用归并，原短信不唯一则不填原短信编号。供应商MO编号独立保存；事件入库/候选/通知、人工认领/通知分别事务化，重复事件和并发认领串行核验，通知只创建耐久意图。既有159条待认领没有自动认领/重投。 +- 日报：六个新增表和版本/日期复合外键；03:00三日刷新，T-4起冻结，历史页面不回查明细；缺口、刷新/失败、截止时间和事后补建来源可见。企业/通道新接口只返回各自分页维度；四TAB查询/日期/筛选/错误独立，保留D-1～D-30。退网批量accepted窗口去重，规则/日报版本首次认领冻结，失败重试沿用；检测与通知等待完整上游批次，不重复生成历史预警。 +- 真实验收：本机独立PostgreSQL16端口16435，新库108项迁移通过（含单独非事务并发索引）。`verify-signature-analytics.mjs`12组验收通过：当天/历史数据源、跨日去重、2/3/4段、缺片、旧无分段明确回执、加权耗时、三日刷新/冻结、旧版本保留、页码越界总数、失败回滚/重试、双worker/fence与规则快照。`verify-uplink-matching.mjs`3组通过：重复事件一份CMPP及一份HTTP通知意图、并发认领、存储故障整笔回滚并恢复；没有启动外部通知消费者。 +- 前端：生产构建，完整真实Nest应用+PG+隔离Redis；Edge/Playwright在1600×1000、1366×768、390×844检查首次、四TAB查询、刷新，补充历史日报、抽屉、企业服务端筛选、TAB日期独立和请求失败仍显示上次真实结果；28个相关请求、页面异常0。匿名401、非法分页/维度400、正常查询200。Browser插件/技能未提供，沿用用户已授权的独立Playwright；会话只在内存传递。 +- 自动门禁：API78套842项、前端35文件163项通过；API/前端类型与生产构建、结构检查、ESLint、Prettier、CSS治理/样式、Bundle预算通过。ESLint仍有发送链路既存测试27项any警告，无error；构建既有chunk提示在预算内。覆盖率门禁见下方补充；Gateway代码未变，无本轮Go/实发/性能容量结论。 +- 性能：30000消息/600维度活动聚合，旧10343.912ms，新624.940ms，热缓存下降93.96%，accepted逐维度一致。原EXPLAIN单维度源访问30113行×600外推18067800，新批量60039；样本外推不冒称全批实测。未测整机CPU、365日真实大批次、峰值内存、p95和短信队列；不能据此宣布预生产CPU问题已完全解决。 +- 原始证据保留于忽略目录`.local-data/signature-optimization-20260917/`：integration-6、uplink-real-3、browser-5、performance-1及最终门禁日志。前面失败（迁移SQL未生成、缺carriers、测试断言漏计新增样本、账号缺email、浏览器按钮名称/模糊标签定位）已修正后重跑；不删除原失败日志。Redis5.0.14.1的BullMQ版本建议及PG驱动在嵌套关系读取时的并发query弃用提示保留，未改依赖版本。 +- 文档：优化方案第8节、上行诊断第6节、需求/测试用例/清退设计同步。本地提交仅纳入本轮代码、两项迁移、验收脚本及相关文档精确新增段落；已有metrics、版本、部署和发布工具修改保留。推送：未执行；测试部署：未执行；预生产部署：未执行。历史日报补建、真实外部MO投递、线上权限/供应商扩展码、午夜冻结过程、现场恢复/回退及完整容量指标未验证，按后续授权实施。 diff --git a/docs/uplink-matching-diagnosis-20260917.md b/docs/uplink-matching-diagnosis-20260917.md new file mode 100644 index 0000000..46aff03 --- /dev/null +++ b/docs/uplink-matching-diagnosis-20260917.md @@ -0,0 +1,76 @@ +# 上行短信待认领只读诊断 + +核查日期：2026-09-17，北京时间。状态：诊断完成；2026-09-17本地修复与隔离验收见第6节，线上未认领、未投递。范围为预生产既存上行和当前匹配代码，不沿用测试环境发送/补发授权。 + +## 1. 基线与结论 + +本地main及实际远端均为`4eb7b16d122da14f921093716d4ca1ed390d9e4c`，暂存区为空。预生产实际运行`010ba3216889032a6160cdb14d8536b616ae7102`。已有版本、metrics、发布工具、部署及前轮优化方案等修改全部保留。 + +当前共253条上行：159条ambiguous（页面“待认领”）、92条matched、2条unmatched。“待认领”不是接收失败：上行已入库，但没有确定应用归属，当前不会按正常已匹配路径推送给客户。 + +| 待认领原因 | 数量 | 真实数据复核 | +|---|---:|---| +| 接入号匹配多个应用 | 147 | 143条在接收前72小时仅有一个发送应用，且属于原候选应用、具有同上行通道accepted提交；4条在此限定口径下无下发记录 | +| 手机号窗口有多条下发 | 12 | 保存的两个候选全属同一应用/企业；完整时间窗口复核也只有一个应用，且有同通道accepted提交 | + +因此155/159条具有“应用归属可以进一步收敛”的真实证据，不代表155条均能唯一确定被回复的具体业务短信，也不构成直接历史认领/投递授权。4条无证据记录保持未决，不因为某条路由现在有效就推断历史归属。历史源记录可能变化或被治理，当前回查不能替代完整历史快照。 + +接入号多应用147条分别位于移动物业-富泷78、三网物业-百信互动52、三网物业-铁布衫15、赛邮行业-王斯评中转2；它们的destId均等于当前通道srcId。12条手机号多记录位于三网物业-百信互动10、联电物业-富泷2。9月16日有31条待认领，9月15日12条。 + +## 2. 已确认根因 + +匹配实现：`api/src/send-chain/send-downstream-delivery.service.ts`的`resolveUplinkMatch`。页面`src/apps/admin/AdminSmsUplinkRecordsPage.tsx`将ambiguous直接映射为待认领，不是前端计算错误。 + +### 2.1 共享接入号过早返回 + +当前顺序为messageId精确匹配→接入号路由查应用。接入号得到多个active应用后，立即返回ambiguous与应用候选，完全不执行后面的手机号时间窗查询。 + +这与需求中“仍无唯一应用时按手机号和最近下发时间窗口匹配”的描述存在差距。共享通道接入号只表示有多个可能客户，不足以否定“该手机近期仅被其中一个客户发送”的进一步证据。 + +真实只读复现记录`cmu48o30s00fo34nkyp3phwt9`：运行中的编译类只调用`channelRouteRule.findMany`和`smsApplication.findMany`，返回3个应用候选；没有查询短信记录。同手机号接收前72小时实际只有一个应用的同通道accepted发送。 + +### 2.2 将短信记录歧义等同应用归属歧义 + +手机号分支取最近两条消息，按记录数量判断唯一或歧义，没有先按tenantId/applicationId归并。一个应用给同一手机发送两次，也被阻断客户归属。 + +真实只读复现`cmtmq3vef0ioyeankps3d80yl`：运行类返回两条phone_window候选，但distinct application数为1。12条同类存量均符合这一情况。可以有唯一客户归属而没有唯一原短信，应分别表达，不能为了绑定客户随意填最近一条messageRecordId/messageId。 + +## 3. 代码边界风险（不等于已经发生错误投递） + +1. 时间窗下限由`Date.now()`计算而非事件receivedAt，且没有`submittedAt <= receivedAt`上限。延迟消费/故障恢复可能错过真实历史下发，或把上行之后的发送纳入。 +2. 手机号回查未限定实际发送通道/供应商边界，也未校验真实accepted提交；仅按message.submittedAt查。不能在修复时直接扩大自动匹配，必须先收紧证据范围。确需同供应商跨物理连接兼容时，另核对账号、主机、端口、协议等边界，不能跨任意通道匹配。 +3. 路由查询`take:10`发生在应用去重之前、没有稳定排序：前10条若同属一个应用可能漏掉其他应用，造成假唯一或候选遗漏。手机号`take:2`也只适于证明多条记录，不能据此证明窗口内只有一个应用。 +4. 接入号只与通道静态srcId精确比较，没有完整核对实际发送号码/应用扩展码。其差异会进入宽泛手机号兜底，不能用当前路由代替历史发送证据。 +5. messageId精确分支未同时验证手机号/通道来源；Gateway普通MO通过packet Msg_Id尝试查本地command。供应商上行ID应继续独立保存为gatewayMessageId，不能把它当已证明的原下发ID。本轮未发现碰撞或错误关联实例。 +6. 上行入库、候选写入与投递分别进行；eventId已存在即返回，部分失败恢复可能留缺候选/缺投递。人工认领是先读状态再更新，未见条件更新原子竞争保护。属于附带发现的可靠性风险，本轮未做故障注入或并发写入复现。 + +对现有85条“手机号窗口唯一匹配”和4条“messageId精确匹配”执行只读边界核对：关联消息均存在、手机号一致、关联提交时间不晚于上行，均存在此前同通道accepted提交。未在这89条中检出上述明显异常；此结果不证明所有潜在风险不存在。 + +## 4. 建议修复方向（未实施） + +- 分开“应用归属”与“原短信关联”。限定证据范围后应用唯一即可确定客户；具体消息仍多条时messageRecordId/messageId留空，并保留真实候选及原因。 +- 共享接入号多应用时继续在候选应用中，结合手机号、上行接收时刻之前的72小时以及真实发送通道/接入号收敛；候选间有冲突仍待认领，不按最后一条或最高分随意选。 +- 查询完整的distinct应用集合或以安全的唯一性检测查询证明唯一；展示候选可以分页，唯一性判断不能先截断。时间窗配置须校验正数有限值，上行receivedAt须校验合法性。 +- 原始供应商MO ID、平台业务ID保持不同语义；无可信原短信关联不得伪造业务messageId。 +- 配套原子入库/认领、通知意图与故障接续；客户查询仍严格隔离应用，ambiguous不得泄露给候选客户。 +- 历史159条先生成只读建议清单，核验保留数据、有效应用、完整号码/通道证据及投递历史，再在明确授权后决定是否认领及是否推送。修复代码不自动批量认领、不自动把旧上行重新入队或投递。 + +## 5. 证据、复现限制与后续验证 + +忽略目录`.local-data/cpu-20260917/`中的`uplink-audit.json`、`uplink-correlation.json`、`uplink-reproduce.json`、`uplink-boundaries.json`及同名mjs为本机证据；输出只含统计、内部记录ID和必要通道标签，不导出手机号、正文或凭据。 + +复现直接加载预生产当前编译的`SendDownstreamDeliveryService.resolveUplinkMatch`，使用真实PG只读事务（8～15秒statement_timeout），不启动Nest生命周期，不调用handleUplink/claim/投递方法。仅在该独立诊断进程将Date.now固定到被查上行的receivedAt以复现旧事件分支，配置仍是当前数据库路由，不能宣称重建了当时全部配置。未执行管理HTTP或浏览器交互验收。 + +后续修复验收至少覆盖共享接入号+唯一应用、同应用多消息、多应用真实冲突、无记录、超过10条路由/超过2条消息、延迟消费/跨日、扩展码、跨供应商隔离、MO ID不冒充MT ID、事务中断、重复事件、并发认领、无重复投递及租户隔离。真实发送/推送只能在后续明确授权的隔离范围执行。 + +本轮业务代码、数据库和线上配置均未修改；仅新增诊断及追加进度记录，未提交、推送或部署。 + +## 6. 2026-09-17 本地修复 + +用户授权修复并本地提交。新增`api/src/send-chain/uplink-matching.ts`，共享接入号继续核对手机号在事件receivedAt前72小时的同通道accepted记录；按tenant/application归并，不以消息条数制造歧义；无take10/take2提前截断。messageId分支同时验证手机号和通道发送事实；收到的供应商gatewayMessageId独立保存。只能确定应用时不填写原短信ID，多应用/配置与事实冲突继续待认领。独占接入号无发送证据的原兼容规则保留；供应商扩展号码改写与跨物理通道归并没有扩大自动归属规则，需有真实协议证据后另行处理。 + +上行记录、候选和CMPP/HTTP通知意图纳入同一事务，沿用发送链路事务适配器；按事件ID事务锁和唯一键去重。人工认领按上行ID串行核验，只有一个候选可成功，认领与通知意图同事务；不在事务里执行外部推送，后续沿用已有耐久队列消费。已有事件直接返回，不擅自修补或重推历史半完成事件。 + +真实隔离PG验证：同应用多消息、第三条不同应用、接收前窗口、其他通道拒绝、原短信为空、重复事件只入一次/通知一次、两候选并发仅一次成功、通知存储失败回滚并重试恢复。自动化入口`tools/testing/verify-uplink-matching.mjs`及签名验收脚本；API全量回归见testing-progress。 + +原159条待认领保持原状；155条可收敛是取证结果，不是已经认领/推送。本轮未连接线上执行认领、重投或修配置，未推送/部署。历史建议清单、实际供应商扩展码、客户端最终收到MO和线上故障恢复仍未验证。 diff --git a/src/api/admin/operations.api.ts b/src/api/admin/operations.api.ts index 5a2239c..ff022bc 100644 --- a/src/api/admin/operations.api.ts +++ b/src/api/admin/operations.api.ts @@ -43,8 +43,10 @@ export const adminOperationsApi = { request(withQuery('/admin/operations/pending-audits', { tenantId }), { signal }), getSendQuality: (date?: string) => request(withQuery('/admin/operations/send-quality', { date })), - getSignatureQuality: (query: { date?: string; keyword?: string; page?: number; pageSize?: number } = {}) => - request(withQuery('/admin/operations/signature-quality', query)), + getSignatureQuality: ( + query: { date?: string; keyword?: string; page?: number; pageSize?: number } = {}, + signal?: AbortSignal, + ) => request(withQuery('/admin/operations/signature-quality', query), { signal }), listSystemLogs: (query: { tenantId?: string; keyword?: string; diff --git a/src/api/admin/signature-retirement.api.ts b/src/api/admin/signature-retirement.api.ts index 42a6fab..557e697 100644 --- a/src/api/admin/signature-retirement.api.ts +++ b/src/api/admin/signature-retirement.api.ts @@ -1,6 +1,8 @@ import { request, withQuery } from '../core/httpClient'; import type { PagedResult, + SignatureActivityResponse, + SignatureAnalyticsMetadata, SignatureRetirementHeatmapItem, SignatureRetirementHeatmapDimension, SignatureRetirementMessage, @@ -12,6 +14,19 @@ import type { } from '../types'; export const adminSignatureRetirementApi = { + getSignatureActivity: ( + query: { + date: string; + dimensionType: 'enterprise' | 'channel'; + page: number; + pageSize: number; + tenantName?: string; + applicationName?: string; + signatureName?: string; + channelName?: string; + }, + signal?: AbortSignal, + ) => request(withQuery('/admin/signature-retirement/activity', query), { signal }), getSignatureRetirementConfiguration: () => request<{ rules: SignatureRetirementRule[]; webhooks: SignatureRetirementWebhook[] }>( '/admin/signature-retirement/configuration', @@ -79,8 +94,12 @@ export const adminSignatureRetirementApi = { dimensions: SignatureRetirementHeatmapDimension[]; items: SignatureRetirementHeatmapItem[]; }>(withQuery('/admin/signature-retirement/heatmap', { date })), - getUnreportedSignatures: (query: { date?: string; keyword?: string; page?: number; pageSize?: number } = {}) => - request & { date: string }>( + getUnreportedSignatures: ( + query: { date?: string; keyword?: string; page?: number; pageSize?: number } = {}, + signal?: AbortSignal, + ) => + request & { date: string } & SignatureAnalyticsMetadata>( withQuery('/admin/signature-retirement/unreported-signatures', query), + { signal }, ), }; diff --git a/src/api/types/operations.ts b/src/api/types/operations.ts index 0930a9d..e0992d1 100644 --- a/src/api/types/operations.ts +++ b/src/api/types/operations.ts @@ -1,3 +1,4 @@ +import type { SignatureAnalyticsMetadata } from './signature-retirement'; // R1 compatibility types. Keep public names re-exported from src/api/adminApi.ts. import type { AdminChannel } from './channels-reports'; @@ -85,7 +86,7 @@ export type SignatureChannelQualityItem = { drainageBreakdowns: SignatureChannelCarrierDrainageQualityStat[]; }; -export type SignatureChannelQualityResponse = { +export type SignatureChannelQualityResponse = SignatureAnalyticsMetadata & { date: string; items: SignatureChannelQualityItem[]; total: number; diff --git a/src/api/types/signature-retirement.ts b/src/api/types/signature-retirement.ts index 7c1fbc6..bbcdff4 100644 --- a/src/api/types/signature-retirement.ts +++ b/src/api/types/signature-retirement.ts @@ -105,3 +105,42 @@ export type UnreportedSignatureItem = { applicationName?: string | null; messageCount: number; }; + +export type SignatureAnalyticsMetadata = { + dataSource?: 'live' | 'report'; + reportState?: string; + frozen?: boolean; + provenance?: string; + sourceAsOf?: string | null; + generatedAt?: string | null; + generationId?: string | null; +}; +export type SignatureActivityItem = Pick< + SignatureRetirementHeatmapItem, + | 'id' + | 'dimensionType' + | 'signatureId' + | 'channelId' + | 'carrier' + | 'activityDate' + | 'submittedAttempts' + | 'acceptedBusinessCount' + | 'deliveredBusinessCount' + | 'status' +>; +export type SignatureActivityResponse = { + date: string; + items: SignatureActivityItem[]; + dimensions: (SignatureRetirementHeatmapDimension & { total: number })[]; + page: number; + pageSize: number; + total: number; + complete: boolean; + coverage: { + date: string; + generationId: string | null; + reportState: string; + frozen: boolean; + sourceAsOf: string | null; + }[]; +}; diff --git a/src/apps/admin/AdminAnalyticsPage.test.tsx b/src/apps/admin/AdminAnalyticsPage.test.tsx index 3f85dfc..3b0c490 100644 --- a/src/apps/admin/AdminAnalyticsPage.test.tsx +++ b/src/apps/admin/AdminAnalyticsPage.test.tsx @@ -3,49 +3,33 @@ import { beforeEach, describe, expect, it, vi } from 'vitest'; import { AdminAnalyticsPage } from './AdminAnalyticsPage'; const { api } = vi.hoisted(() => ({ - api: { getSignatureQuality: vi.fn(), getSignatureRetirementHeatmap: vi.fn(), getUnreportedSignatures: vi.fn() }, + api: { getSignatureQuality: vi.fn(), getSignatureActivity: vi.fn(), getUnreportedSignatures: vi.fn() }, })); vi.mock('@/api/adminApi', () => ({ adminApi: api })); describe('independent analytics tabs', () => { - it('combines separate activity search fields with AND and preserves them across tabs', async () => { - api.getSignatureRetirementHeatmap.mockResolvedValue({ - items: [], - dimensions: [ - { - dimensionType: 'channel', - signatureId: 'a', - signatureName: '签名甲', - tenantName: '企业甲', - applicationName: '应用甲', - channelName: '通道甲', - channelId: 'c', - carrier: 'mobile', - approvedAt: '2026-08-01', - }, - { - dimensionType: 'channel', - signatureId: 'b', - signatureName: '签名乙', - tenantName: '企业甲', - applicationName: '应用乙', - channelName: '通道乙', - channelId: 'd', - carrier: 'unicom', - approvedAt: '2026-08-01', - }, - ], - }); + it('submits independent activity fields to server and preserves drafts across tabs', async () => { render(); fireEvent.click(screen.getByRole('tab', { name: '通道签名活跃度' })); const panel = screen.getByRole('region', { name: '通道签名活跃度' }); - await within(panel).findByText('签名甲'); + await waitFor(() => expect(api.getSignatureActivity).toHaveBeenCalledTimes(1)); fireEvent.change(within(panel).getByLabelText('企业'), { target: { value: '企业甲' } }); fireEvent.change(within(panel).getByLabelText('企业应用'), { target: { value: '应用甲' } }); - await waitFor(() => expect(within(panel).queryByText('签名乙')).not.toBeInTheDocument()); - fireEvent.change(within(panel).getByLabelText('通道'), { target: { value: '通道乙' } }); - await waitFor(() => expect(within(panel).queryByText('签名甲')).not.toBeInTheDocument()); + expect(api.getSignatureActivity).toHaveBeenCalledTimes(1); + fireEvent.click(within(panel).getByRole('button', { name: '查询统计' })); + await waitFor(() => + expect(api.getSignatureActivity).toHaveBeenLastCalledWith( + expect.objectContaining({ dimensionType: 'channel', tenantName: '企业甲', applicationName: '应用甲', page: 1 }), + expect.any(AbortSignal), + ), + ); fireEvent.click(screen.getByRole('tab', { name: '企业签名活跃度' })); + await waitFor(() => + expect(api.getSignatureActivity).toHaveBeenLastCalledWith( + expect.objectContaining({ dimensionType: 'enterprise', tenantName: '', applicationName: '' }), + expect.any(AbortSignal), + ), + ); fireEvent.click(screen.getByRole('tab', { name: '通道签名活跃度' })); expect(within(panel).getByLabelText('企业应用')).toHaveValue('应用甲'); }); @@ -53,19 +37,33 @@ describe('independent analytics tabs', () => { vi.resetAllMocks(); api.getSignatureQuality.mockImplementation(async (query) => ({ ...query, total: 0, items: [] })); api.getUnreportedSignatures.mockImplementation(async (query) => ({ ...query, total: 0, items: [] })); - api.getSignatureRetirementHeatmap.mockResolvedValue({ items: [], dimensions: [] }); + api.getSignatureActivity.mockResolvedValue({ + items: [], + dimensions: [], + total: 0, + page: 1, + pageSize: 25, + coverage: [], + complete: false, + }); }); it('loads only the visited tab and preserves independent dates when switching back', async () => { render(); await waitFor(() => - expect(api.getSignatureQuality).toHaveBeenCalledWith(expect.objectContaining({ pageSize: 25 })), + expect(api.getSignatureQuality).toHaveBeenCalledWith( + expect.objectContaining({ pageSize: 25 }), + expect.any(AbortSignal), + ), ); - expect(api.getSignatureRetirementHeatmap).not.toHaveBeenCalled(); + expect(api.getSignatureActivity).not.toHaveBeenCalled(); const quality = screen.getByRole('region', { name: '签名通道发送质量' }); fireEvent.change(within(quality).getByLabelText('统计日期'), { target: { value: '2026-08-20' } }); fireEvent.click(within(quality).getByRole('button', { name: '查询统计' })); await waitFor(() => - expect(api.getSignatureQuality).toHaveBeenLastCalledWith(expect.objectContaining({ date: '2026-08-20' })), + expect(api.getSignatureQuality).toHaveBeenLastCalledWith( + expect.objectContaining({ date: '2026-08-20' }), + expect.any(AbortSignal), + ), ); fireEvent.click(screen.getByRole('tab', { name: '未报备签名' })); await waitFor(() => expect(api.getUnreportedSignatures).toHaveBeenCalledTimes(1)); @@ -75,7 +73,7 @@ describe('independent analytics tabs', () => { expect(within(quality).getByLabelText('统计日期')).toHaveValue('2026-08-20'); fireEvent.click(screen.getByRole('tab', { name: '未报备签名' })); expect(within(unreported).getByLabelText('统计日期')).toHaveValue('2026-08-25'); - expect(api.getSignatureRetirementHeatmap).not.toHaveBeenCalled(); + expect(api.getSignatureActivity).not.toHaveBeenCalled(); }); it.each([ @@ -89,9 +87,16 @@ describe('independent analytics tabs', () => { await waitFor(() => expect(within(panel).getByRole('button', { name: '下一页' })).toBeEnabled()); fireEvent.change(within(panel).getByLabelText('统计日期'), { target: { value: '2026-08-20' } }); fireEvent.click(within(panel).getByRole('button', { name: '查询统计' })); - await waitFor(() => expect(api[method]).toHaveBeenLastCalledWith(expect.objectContaining({ date: '2026-08-20' }))); + await waitFor(() => + expect(api[method]).toHaveBeenLastCalledWith( + expect.objectContaining({ date: '2026-08-20' }), + expect.any(AbortSignal), + ), + ); fireEvent.click(within(panel).getByRole('button', { name: '下一页' })); - await waitFor(() => expect(api[method]).toHaveBeenLastCalledWith(expect.objectContaining({ page: 2 }))); + await waitFor(() => + expect(api[method]).toHaveBeenLastCalledWith(expect.objectContaining({ page: 2 }), expect.any(AbortSignal)), + ); // An unsubmitted date must not alter a pagination request. fireEvent.change(within(panel).getByLabelText('统计日期'), { target: { value: '2026-08-21' } }); const size = within(panel).getByLabelText(/^每页数量/); @@ -108,6 +113,7 @@ describe('independent analytics tabs', () => { await waitFor(() => expect(api[method]).toHaveBeenLastCalledWith( expect.objectContaining({ page: 1, pageSize: 100, date: '2026-08-20' }), + expect.any(AbortSignal), ), ); await waitFor(() => expect(within(panel).getByLabelText('跳转页码')).toHaveValue(1)); @@ -118,7 +124,7 @@ describe('independent analytics tabs', () => { async (tab) => { render(); fireEvent.click(screen.getByRole('tab', { name: tab })); - await waitFor(() => expect(api.getSignatureRetirementHeatmap).toHaveBeenCalledTimes(1)); + await waitFor(() => expect(api.getSignatureActivity).toHaveBeenCalledTimes(1)); const panel = screen.getByRole('region', { name: tab }); const size = within(panel).getByLabelText(/^每页数量/); expect(size.closest('.ui-pagination')).not.toBeNull(); @@ -131,7 +137,7 @@ describe('independent analytics tabs', () => { ).toHaveTextContent('25 条/页'); fireEvent.click(screen.getByRole('tab', { name: tab })); expect(size).toHaveTextContent('50 条/页'); - expect(api.getSignatureRetirementHeatmap).toHaveBeenCalledTimes(1); + await waitFor(() => expect(api.getSignatureActivity).toHaveBeenCalledTimes(2)); }, ); }); diff --git a/src/apps/admin/AdminAnalyticsPage.tsx b/src/apps/admin/AdminAnalyticsPage.tsx index 5921c99..deee5c1 100644 --- a/src/apps/admin/AdminAnalyticsPage.tsx +++ b/src/apps/admin/AdminAnalyticsPage.tsx @@ -1,11 +1,13 @@ -import { useDeferredValue, useEffect, useRef, useState } from 'react'; +import { useEffect, useRef, useState } from 'react'; import { BarChart3, Eye, Search, X } from 'lucide-react'; import { adminApi, type SignatureChannelCarrierQualityStat, type SignatureChannelQualityItem, + type SignatureActivityResponse, + type SignatureAnalyticsMetadata, type SignatureChannelQualityResponse, - type SignatureRetirementHeatmapItem, + type SignatureActivityItem, type SignatureRetirementHeatmapDimension, type UnreportedSignatureItem, type PagedResult, @@ -59,9 +61,19 @@ function AnalyticsPanel({ kind }: { kind: string }) { const [pageSize, setPageSize] = useState(25); const [appliedDate, setAppliedDate] = useState(() => shanghaiDateKey()); const requestId = useRef(0); + const abort = useRef(null); + const [activity, setActivity] = useState(null); + const [activityFilters, setActivityFilters] = useState({ + tenantName: '', + applicationName: '', + signatureName: '', + channelName: '', + }); + const [appliedActivityFilters, setAppliedActivityFilters] = useState(activityFilters); + const [metadata, setMetadata] = useState(null); const [statisticsDate, setStatisticsDate] = useState(() => shanghaiDateKey()); const [signatureQuality, setSignatureQuality] = useState(null); - const [retirementHeatmap, setRetirementHeatmap] = useState([]); + const [retirementHeatmap, setRetirementHeatmap] = useState([]); const [retirementDimensions, setRetirementDimensions] = useState([]); const [unreportedSignatures, setUnreportedSignatures] = useState< (PagedResult & { date: string }) | null @@ -80,30 +92,47 @@ function AnalyticsPanel({ kind }: { kind: string }) { unreported = appliedUnreportedKeyword, date = statisticsDate, size = pageSize, + filters = appliedActivityFilters, ) { + abort.current?.abort(); + const controller = new AbortController(); + abort.current = controller; const id = ++requestId.current; setLoading(true); setError(''); try { if (kind === 'quality') { - const data = await adminApi.getSignatureQuality({ date, keyword: keyword || undefined, page, pageSize: size }); + const data = await adminApi.getSignatureQuality( + { date, keyword: keyword || undefined, page, pageSize: size }, + controller.signal, + ); if (id !== requestId.current) return; setSignatureQuality(data); + setMetadata(data); setAppliedKeyword(keyword); setSelectedSignature(null); } else if (kind === 'unreported') { - const data = await adminApi.getUnreportedSignatures({ - date, - keyword: unreported || undefined, - page, - pageSize: size, - }); + const data = await adminApi.getUnreportedSignatures( + { + date, + keyword: unreported || undefined, + page, + pageSize: size, + }, + controller.signal, + ); if (id !== requestId.current) return; setUnreportedSignatures(data); + setMetadata(data); setAppliedUnreportedKeyword(unreported); } else { - const data = await adminApi.getSignatureRetirementHeatmap(date); + const data = await adminApi.getSignatureActivity( + { date, dimensionType: kind as 'enterprise' | 'channel', page, pageSize: size, ...filters }, + controller.signal, + ); if (id !== requestId.current) return; + setActivity(data); + setAppliedActivityFilters(filters); setRetirementHeatmap(data.items); setRetirementDimensions(data.dimensions); } @@ -115,9 +144,11 @@ function AnalyticsPanel({ kind }: { kind: string }) { } } + const initialLoad = useRef(loadData); useEffect(() => { - void loadData(1, ''); + void initialLoad.current(1, ''); return () => { + abort.current?.abort(); requestId.current += 1; }; }, []); @@ -221,7 +252,7 @@ function AnalyticsPanel({ kind }: { kind: string }) { ]; function queryStatistics() { - void loadData(1, signatureKeyword.trim()); + void loadData(1, signatureKeyword.trim(), unreportedKeyword.trim(), statisticsDate, pageSize, activityFilters); } function changeSignaturePage(page: number) { @@ -233,8 +264,7 @@ function AnalyticsPanel({ kind }: { kind: string }) { function changePageSize(size: number) { setPageSize(size); - if (kind === 'quality' || kind === 'unreported') - void loadData(1, appliedKeyword, appliedUnreportedKeyword, appliedDate, size); + void loadData(1, appliedKeyword, appliedUnreportedKeyword, appliedDate, size); } return ( @@ -256,7 +286,28 @@ function AnalyticsPanel({ kind }: { kind: string }) { - {error ?

{error}

: null} + {error ? ( +

+ {error} + {signatureQuality || activity || unreportedSignatures + ? `；仍显示上次成功查询的数据，日期为 ${appliedDate}。` + : '；尚无成功查询的数据。'} +

+ ) : null} + {metadata ? ( +

+ {metadata.dataSource === 'live' ? '当天实时查询' : metadata.frozen ? '已冻结日报' : '近三日可刷新日报'} ·{' '} + {metadata.reportState === 'missing' + ? '报表尚未生成' + : metadata.reportState === 'failed' + ? '生成失败，保留上次完整日报' + : metadata.reportState === 'refreshing' + ? '正在刷新，显示上次完整日报' + : '查询完成'} + {metadata.provenance === 'backfill-current-source' ? ' · 事后补建' : ''} + {metadata.sourceAsOf ? ` · 数据截止 ${new Date(metadata.sourceAsOf).toLocaleString('zh-CN')}` : ''} +

+ ) : null} {kind === 'quality' ? (
@@ -323,6 +374,11 @@ function AnalyticsPanel({ kind }: { kind: string }) { onPageSizeChange={changePageSize} date={appliedDate} dimensionType="enterprise" + activity={activity} + filters={activityFilters} + onFilterChange={setActivityFilters} + onSearch={queryStatistics} + onPageChange={(page) => void loadData(page, appliedKeyword, appliedUnreportedKeyword, appliedDate)} dimensions={retirementDimensions} items={retirementHeatmap} title="企业签名活跃度热力图" @@ -334,6 +390,11 @@ function AnalyticsPanel({ kind }: { kind: string }) { onPageSizeChange={changePageSize} date={appliedDate} dimensionType="channel" + activity={activity} + filters={activityFilters} + onFilterChange={setActivityFilters} + onSearch={queryStatistics} + onPageChange={(page) => void loadData(page, appliedKeyword, appliedUnreportedKeyword, appliedDate)} dimensions={retirementDimensions} items={retirementHeatmap} title="通道签名活跃度热力图" @@ -349,7 +410,7 @@ function AnalyticsPanel({ kind }: { kind: string }) { loading={loading} onKeywordChange={setUnreportedKeyword} onPageChange={(page) => loadUnreportedSignatures(page)} - onSearch={() => loadUnreportedSignatures(1, unreportedKeyword.trim())} + onSearch={queryStatistics} /> ) : null} @@ -365,6 +426,11 @@ function AnalyticsPanel({ kind }: { kind: string }) { } function RetirementHeatmap({ + activity, + filters, + onFilterChange, + onSearch, + onPageChange, pageSize, onPageSizeChange, date, @@ -378,12 +444,14 @@ function RetirementHeatmap({ date: string; dimensionType: 'enterprise' | 'channel'; dimensions: SignatureRetirementHeatmapDimension[]; - items: SignatureRetirementHeatmapItem[]; + items: SignatureActivityItem[]; title: string; + activity: SignatureActivityResponse | null; + filters: { tenantName: string; applicationName: string; signatureName: string; channelName: string }; + onFilterChange: (value: typeof filters) => void; + onSearch: () => void; + onPageChange: (page: number) => void; }) { - const [pageState, setPageState] = useState({ key: '', page: 1 }); - const [filters, setFilters] = useState({ tenantName: '', applicationName: '', signatureName: '', channelName: '' }); - const deferredFilters = useDeferredValue(filters); const visible = items.filter((item) => item.dimensionType === dimensionType); const dates = previousDateKeys(date, 30); const cellMap = new Map( @@ -392,40 +460,17 @@ function RetirementHeatmap({ item, ]), ); - const rows = dimensions - .filter((item) => item.dimensionType === dimensionType) - .filter((item) => - Object.entries(deferredFilters).every( - ([key, value]) => - !value.trim() || - (item[key as keyof typeof deferredFilters] ?? '') - .toLocaleLowerCase('zh-CN') - .includes(value.trim().toLocaleLowerCase('zh-CN')), - ), - ) - .map((item) => ({ - key: `${item.signatureId}:${item.channelId ?? ''}:${item.carrier}`, - signatureName: item.signatureName, - channelName: item.channelName, - tenantName: item.tenantName, - applicationName: item.applicationName, - carrier: item.carrier, - approvedAt: item.approvedAt.slice(0, 10), - total: dates.reduce( - (sum, dateKey) => - sum + - (cellMap.get(`${item.signatureId}:${item.channelId ?? ''}:${item.carrier}:${dateKey}`) - ?.acceptedBusinessCount ?? 0), - 0, - ), - })) - .sort((left, right) => right.total - left.total || left.signatureName.localeCompare(right.signatureName, 'zh-CN')); - const totalPages = Math.max(1, Math.ceil(rows.length / pageSize)); - const paginationKey = JSON.stringify([date, deferredFilters, dimensionType, dimensions.length, pageSize]); - const page = pageState.key === paginationKey ? pageState.page : 1; - const setPage = (value: number) => setPageState({ key: paginationKey, page: value }); - const currentPage = Math.min(page, totalPages); - const pagedRows = rows.slice((currentPage - 1) * pageSize, currentPage * pageSize); + const rows = dimensions.map((item) => ({ + ...item, + key: `${item.signatureId}:${item.channelId ?? ''}:${item.carrier}`, + approvedAt: item.approvedAt?.slice(0, 10) ?? '', + total: (item as SignatureRetirementHeatmapDimension & { total: number }).total ?? 0, + })); + const totalPages = Math.max(1, Math.ceil((activity?.total ?? 0) / pageSize)); + const currentPage = activity?.page ?? 1; + const pagedRows = rows; + const setPage = onPageChange; + const coverage = new Map(activity?.coverage.map((day) => [day.date, day])); return (
@@ -448,12 +493,18 @@ function RetirementHeatmap({ label={label} placeholder={`搜索${label}`} value={filters[key]} - onChange={(event) => setFilters((current) => ({ ...current, [key]: event.target.value }))} + onChange={(event) => onFilterChange({ ...filters, [key]: event.target.value })} /> ))} - T-1 至 T-30 + + 所选日之前30天
+ {activity && !activity.complete ? ( +

部分日期报表尚未生成，合计仅包含已发布日期，不代表完整30日数据。

+ ) : null} {rows.length ? ( <>
@@ -482,7 +533,7 @@ function RetirementHeatmap({ {row.total.toLocaleString('zh-CN')} {dates.map((dateKey) => { const item = cellMap.get(`${row.key}:${dateKey}`); - const beforeApproval = dateKey < row.approvedAt; + const beforeApproval = Boolean(coverage.get(dateKey)?.generationId && !item); const successRate = item?.acceptedBusinessCount ? (item.deliveredBusinessCount / item.acceptedBusinessCount) * 100 : 0; @@ -494,10 +545,10 @@ function RetirementHeatmap({ ? 'is-zero' : `is-rate-${successRateTone(successRate)}`; const titleText = beforeApproval - ? '报备前，不适用' + ? '当日报备维度不适用' : item - ? `提交条数：${item.submittedAttempts} 条\n上游接受条数：${item.acceptedBusinessCount} 条\n发送成功条数：${item.deliveredBusinessCount} 条\n发送成功率：${successRate.toFixed(1)}%\n检测状态：${item.status === 'observing' ? '观察中（不预警）' : item.status === 'alert' ? '预警' : '正常'}\n预警阈值：${item.threshold} 条` - : '当日无检测快照'; + ? `提交条数：${item.submittedAttempts} 条\n上游接受条数：${item.acceptedBusinessCount} 条\n发送成功条数：${item.deliveredBusinessCount} 条\n发送成功率：${successRate.toFixed(1)}%\n数据来源：${coverage.get(dateKey)?.frozen ? '冻结日报' : '可刷新日报'}` + : '当日报表尚未生成'; return ( {beforeApproval ? 'N/A' : item ? item.acceptedBusinessCount : '—'} @@ -512,7 +563,7 @@ function RetirementHeatmap({ ) : (

- {Object.values(deferredFilters).some((value) => value.trim()) + {Object.values(filters).some((value) => value.trim()) ? '没有匹配企业、企业应用或签名的热力图维度。' : '暂无已确认到运营商的报备事实，尚未形成检测热力图。'}

@@ -526,7 +577,7 @@ function RetirementHeatmap({ onPrevious={() => setPage(currentPage - 1)} page={currentPage} previousDisabled={currentPage <= 1} - total={rows.length} + total={activity?.total ?? 0} totalPages={totalPages} />
diff --git a/tools/testing/backfill-signature-analytics.mjs b/tools/testing/backfill-signature-analytics.mjs new file mode 100644 index 0000000..6f9e677 --- /dev/null +++ b/tools/testing/backfill-signature-analytics.mjs @@ -0,0 +1,41 @@ +// Offline operator entry point. No SMS, detection, notification or history overwrite is performed. +import { createRequire } from 'node:module'; +import assert from 'node:assert/strict'; +const args = Object.fromEntries(process.argv.slice(2).map((a) => a.replace(/^--/, '').split('='))); +const url = new URL(process.env.DATABASE_URL || ''); +assert( + args.host && args.host === url.hostname, + 'Supply --host matching DATABASE_URL and an explicitly authorized target', +); +assert(args.dates, 'Supply --dates=YYYY-MM-DD,YYYY-MM-DD; no implicit historical range'); +const dates = [...new Set(args.dates.split(','))]; +assert(dates.length <= 366, 'At most 366 explicitly selected days per invocation'); +process.env.NODE_ENV = 'test'; // Disable all automatic schedulers; this script only creates the report writer. +const require = createRequire(new URL('../../api/package.json', import.meta.url)); +require('reflect-metadata'); +const { PrismaService } = require('./dist/prisma/prisma.service'); +const { SignatureAnalyticsService } = require('./dist/signature-analytics/signature-analytics.service'); +const { analyticsDate, databaseDay, todayKey } = require('./dist/signature-analytics/analytics-date'); +const db = new PrismaService(); +try { + for (const day of dates) { + analyticsDate(day); + assert(day < todayKey(), 'Only complete natural days may be backfilled'); + const current = await db.signatureAnalyticsDay.findUnique({ where: { businessDate: databaseDay(day) } }); + assert(!current?.publishedGenerationId, `${day}: published report exists; overwriting is forbidden`); + } + console.log( + JSON.stringify({ + host: url.hostname, + dates, + execute: args.execute === 'yes', + provenance: 'backfill-current-source', + }), + ); + if (args.execute === 'yes') { + const writer = new SignatureAnalyticsService(db); + for (const day of dates) console.log(JSON.stringify(await writer.generate(day, true))); + } +} finally { + await db.onModuleDestroy(); +} diff --git a/tools/testing/benchmark-signature-analytics.mjs b/tools/testing/benchmark-signature-analytics.mjs new file mode 100644 index 0000000..8e4859f --- /dev/null +++ b/tools/testing/benchmark-signature-analytics.mjs @@ -0,0 +1,143 @@ +import assert from 'node:assert/strict'; +import { createRequire } from 'node:module'; +import { execFileSync } from 'node:child_process'; +import Module from 'node:module'; +import { randomUUID } from 'node:crypto'; +const require = createRequire(new URL('../../api/package.json', import.meta.url)); +const url = new URL(process.env.SIGNATURE_TEST_DATABASE_URL || ''); +assert(url.hostname === '127.0.0.1' && url.pathname.startsWith('/cmpp_qa_signature_')); +process.env.DATABASE_URL = url.toString(); +process.env.NODE_ENV = 'test'; +require('reflect-metadata'); +const { PrismaService } = require('./dist/prisma/prisma.service'), + { Prisma } = require('@prisma/client'); +const { activityCounts } = require('./dist/signature-analytics/analytics-aggregate'); +const { todayKey, addDays, startOfDay } = require('./dist/signature-analytics/analytics-date'); +const oldSource = execFileSync( + 'git', + ['show', '4eb7b16d122da14f921093716d4ca1ed390d9e4c:api/src/signature-retirement/signature-retirement.service.ts'], + { encoding: 'utf8' }, +); +const compiled = require('typescript').transpileModule(oldSource, { + compilerOptions: { module: 1, target: 9, experimentalDecorators: true, emitDecoratorMetadata: true }, +}).outputText; +const oldModule = new Module(require.resolve('./dist/signature-retirement/signature-retirement.service')); +oldModule.filename = require.resolve('./dist/signature-retirement/signature-retirement.service'); +oldModule.paths = require.resolve.paths('@prisma/client'); +oldModule._compile(compiled, oldModule.filename); +const db = new PrismaService(), + prefix = 'bench' + randomUUID().slice(0, 8), + N = 300, + K = 100, + D = addDays(todayKey(), -1), + start = startOfDay(D), + end = startOfDay(todayKey()); +let last; +const traced = new Proxy(db, { + get(t, k) { + if (k === '$queryRaw') + return async (q) => { + last = q; + return db.$queryRaw(q); + }; + const v = t[k]; + return typeof v === 'function' ? v.bind(t) : v; + }, +}); +try { + const tenant = await db.tenant.create({ data: { name: prefix, code: prefix } }), + app = await db.smsApplication.create({ + data: { + tenantId: tenant.id, + name: prefix, + cmppAccount: prefix, + cmppEnterpriseCode: '000001', + secretHash: 'isolated', + interfaceEnabled: false, + }, + }), + channel = await db.smsChannel.create({ + data: { + name: prefix, + code: prefix, + srcId: '1069', + gatewayHost: '127.0.0.1', + gatewayPort: 1, + account: 'isolated', + passwordCipher: 'not-a-secret', + carriers: ['mobile'], + status: 'disabled', + }, + }); + await db.$executeRaw`INSERT INTO "SmsSignature"(id,"tenantId","applicationId",name,"updatedAt") SELECT ${prefix}||n,${tenant.id},${app.id},${prefix}||n,NOW() FROM generate_series(1,${N}) n`; + await db.$executeRaw`INSERT INTO "SmsMessageRecord"(id,"messageId","tenantId","applicationId","signatureId","phoneNumber",content,carrier,"queuedAt","billingUnits","updatedAt") SELECT ${prefix}||n||'-'||k,${prefix}||n||'-'||k,${tenant.id},${app.id},${prefix}||n,'13800000200','isolated','mobile',${start},1,NOW() FROM generate_series(1,${N}) n CROSS JOIN generate_series(1,${K}) k`; + await db.$executeRaw`INSERT INTO "SmsSubmitRecord"(id,"messageRecordId","submitId","channelId","submitStatus","submittedAt","createdAt","updatedAt") SELECT id,id,id,${channel.id},'accepted',${start},${start},NOW() FROM "SmsMessageRecord" WHERE "tenantId"=${tenant.id}`; + await db.$executeRawUnsafe('ANALYZE "SmsMessageRecord"'); + await db.$executeRawUnsafe('ANALYZE "SmsSubmitRecord"'); + const dims = Array.from({ length: N }, (_, i) => + ['enterprise', 'channel'].map((type) => ({ + dimensionKey: JSON.stringify([type, prefix + (i + 1)]), + dimensionType: type, + tenantId: tenant.id, + applicationId: app.id, + signatureId: prefix + (i + 1), + channelKey: type === 'channel' ? channel.id : '', + channelId: type === 'channel' ? channel.id : null, + carrier: 'mobile', + approvedAt: startOfDay(addDays(D, -60)), + signatureName: prefix + (i + 1), + tenantName: prefix, + applicationName: prefix, + channelName: prefix, + })), + ).flat(); + const old = new oldModule.exports.SignatureRetirementService(traced); + const before = performance.now(); + for (const d of dims) { + const r = await old.activityCounts(d, start, end); + assert.equal(r.acceptedBusinessCount, K); + } + const oldMs = performance.now() - before; + const sampleOld = last; + const now = performance.now(), + counts = await activityCounts(traced, dims, D), + newMs = performance.now() - now, + sampleNew = last; + for (const d of dims) assert.equal(counts.get(d.dimensionKey).acceptedBusinessCount, K); + const explain = async (q) => + (await db.$queryRaw(Prisma.sql`EXPLAIN (ANALYZE,BUFFERS,FORMAT JSON) ${q}`))[0]['QUERY PLAN'][0]; + const oldPlan = await explain(sampleOld), + newPlan = await explain(sampleNew); + const scanRows = (p) => { + let sum = 0; + const walk = (n) => { + if ( + n['Node Type']?.includes('Scan') && + ['SmsSubmitRecord', 'SmsMessageRecord', 'SmsReceiptRecord', 'SmsMessageSegmentAudit'].includes( + n['Relation Name'], + ) + ) + sum += (n['Actual Rows'] + (n['Rows Removed by Filter'] || 0)) * (n['Actual Loops'] || 0); + for (const c of n.Plans || []) walk(c); + }; + walk(p.Plan); + return sum; + }; + console.log( + JSON.stringify({ + messages: N * K, + dimensions: dims.length, + oldMs, + newMs, + reduction: 1 - newMs / oldMs, + oldSampleRows: scanRows(oldPlan), + estimatedOldRows: scanRows(oldPlan) * dims.length, + newRows: scanRows(newPlan), + note: 'warm cache, per-dimension sample extrapolation; not production CPU or whole job capacity', + oldPlan, + newPlan, + }), + ); +} finally { + await db.onModuleDestroy(); +} diff --git a/tools/testing/verify-signature-analytics.mjs b/tools/testing/verify-signature-analytics.mjs new file mode 100644 index 0000000..2997760 --- /dev/null +++ b/tools/testing/verify-signature-analytics.mjs @@ -0,0 +1,400 @@ +import assert from 'node:assert/strict'; +import { randomUUID } from 'node:crypto'; +import { createRequire } from 'node:module'; +const url = new URL(process.env.SIGNATURE_TEST_DATABASE_URL || ''); +assert(['127.0.0.1', 'localhost'].includes(url.hostname) && url.pathname.startsWith('/cmpp_qa_signature_')); +process.env.DATABASE_URL = url.toString(); +process.env.NODE_ENV = 'test'; +const require = createRequire(new URL('../../api/package.json', import.meta.url)); +require('reflect-metadata'); +const { PrismaService } = require('./dist/prisma/prisma.service'); +const { SignatureAnalyticsService } = require('./dist/signature-analytics/signature-analytics.service'); +const { SignatureAnalyticsRead } = require('./dist/signature-analytics/analytics-read'); +const { OperationsQualityQueries } = require('./dist/operations/queries/quality.queries'); +const { analyticsJob } = require('./dist/signature-analytics/analytics-job'); +const { detectRetirement } = require('./dist/signature-analytics/retirement-batch'); +const { resolveUplinkMatch } = require('./dist/send-chain/uplink-matching'); +const { todayKey, addDays, startOfDay, databaseDay } = require('./dist/signature-analytics/analytics-date'); +const db = new PrismaService(), + writer = new SignatureAnalyticsService(db), + reader = new SignatureAnalyticsRead(db); +const prefix = randomUUID().slice(0, 8), + T = todayKey(), + D = addDays(T, -1), + old = addDays(T, -5); +const pass = (name) => console.log('PASS', name); +const at = (date, seconds = 3600) => new Date(startOfDay(date).getTime() + seconds * 1000); +try { + const tenant = await db.tenant.create({ data: { name: `签名日报验收${prefix}`, code: prefix } }); + const app = await db.smsApplication.create({ + data: { + tenantId: tenant.id, + name: '应用甲', + cmppAccount: prefix, + cmppEnterpriseCode: '000001', + secretHash: 'isolated-no-login', + interfaceEnabled: false, + }, + }); + const channel = await db.smsChannel.create({ + data: { + name: '隔离通道', + code: prefix, + gatewayHost: '127.0.0.1', + gatewayPort: 1, + account: 'isolated', + passwordCipher: 'not-a-secret', + srcId: '10690000', + carriers: ['mobile'], + status: 'disabled', + }, + }); + const signature = await db.smsSignature.create({ + data: { tenantId: tenant.id, applicationId: app.id, name: `【验收${prefix}】`, auditStatus: 'approved' }, + }); + await db.channelSignatureReportTask.create({ + data: { + tenantId: tenant.id, + signatureId: signature.id, + channelId: channel.id, + reportType: 'signature', + carrier: 'mobile', + approvalScope: 'carrier_specific', + status: 'approved', + approvedAt: at(addDays(T, -60)), + createdAt: at(addDays(T, -60)), + }, + }); + async function message({ + date = D, + submitDate = date, + units = 1, + segments = units, + delivered = true, + phone = '13800000001', + sig = signature.id, + content = signature.name + '隔离数据', + } = {}) { + const m = await db.smsMessageRecord.create({ + data: { + messageId: randomUUID(), + tenantId: tenant.id, + applicationId: app.id, + signatureId: sig, + phoneNumber: phone, + content, + carrier: 'mobile', + queuedAt: at(date), + submittedAt: at(submitDate), + billingUnits: units, + status: delivered && segments === units ? 'delivered' : 'submitted', + submitStatus: 'accepted', + receiptStatus: delivered && segments === units ? 'delivered' : null, + deliveredAt: delivered && segments === units ? at(submitDate, 3602) : null, + }, + }); + const s = await db.smsSubmitRecord.create({ + data: { + messageRecordId: m.id, + tenantId: tenant.id, + channelId: channel.id, + submitId: randomUUID(), + submitStatus: 'accepted', + submittedAt: at(submitDate), + gatewayMessageId: randomUUID(), + createdAt: at(submitDate), + }, + }); + for (let index = 1; index <= segments; index++) + await db.smsMessageSegmentAudit.create({ + data: { + messageRecordId: m.id, + submitRecordId: s.id, + channelId: channel.id, + submitId: s.submitId, + segmentTotal: units, + segmentIndex: index, + submitStatus: 'accepted', + receiptStatus: delivered ? 'delivered' : null, + deliveredAt: delivered ? at(submitDate, 3600 + index) : null, + }, + }); + return { m, s }; + } + const complete = await message({ units: 3, phone: '13800000011' }); + const partial = await message({ units: 3, segments: 2, phone: '13800000012' }); + await message({ date: addDays(D, -1), submitDate: D, phone: '13800000013' }); + await message({ sig: null, content: `【未登记${prefix}】隔离`, phone: '13800000014' }); + await message({ date: T, phone: '13800000015' }); + const oldMessage = await message({ date: old, phone: '13800000016' }); + // A rerun uses an isolated database; do not reset or touch any real tenant environment. + await writer.generate(D); + const quality = await reader.quality({ date: D, keyword: prefix }); + const item = quality.items.find((x) => x.signatureId === signature.id); + assert(item); + assert.equal(item.total, 2); + assert.equal(item.channelSubmitTotal, 3); + assert.equal(item.breakdowns[0].successCount, 2); + assert.equal(item.breakdowns[0].unknownCount, 1); + pass('日报业务/尝试归日分离、三段成功与缺片未知'); + const activity = await reader.activity({ date: T, dimensionType: 'enterprise', signatureName: prefix }); + assert.equal(activity.dimensions.length, 1); + assert.equal(activity.items.length, 1); + assert.equal(activity.items[0].acceptedBusinessCount, 3); + assert.equal(activity.items[0].deliveredBusinessCount, 2); + assert.equal(activity.coverage[0].date, D); + assert.equal(activity.complete, false); + pass('企业/通道服务端分页与D-1至D-30缺口元数据'); + assert.equal((await reader.unreported({ date: D, keyword: prefix })).total, 1); + const live = await new OperationsQualityQueries(db).signatureQuality({ date: T, keyword: prefix }); + assert.equal(live.dataSource, 'live'); + assert.equal(live.items[0].total, 1); + pass('当天真实查询、历史只读日报、未报备日报'); + await writer.generate(old, true); + const before = await reader.quality({ date: old, keyword: prefix }); + await db.smsMessageRecord.update({ + where: { id: oldMessage.m.id }, + data: { status: 'failed', receiptStatus: 'undelivered' }, + }); + assert.equal((await writer.generate(old)).skipped, true); + assert.deepEqual((await reader.quality({ date: old, keyword: prefix })).items, before.items); + await assert.rejects(writer.generate(old, true), /覆盖/); + pass('T-4及更早冻结、显式补建不覆盖'); + await db.signatureRetirementRule.create({ + data: { + ruleType: 'enterprise_global', + targetKey: '', + mobileWindowDays: 7, + mobileThreshold: 100, + unicomWindowDays: 7, + unicomThreshold: 100, + telecomWindowDays: 7, + telecomThreshold: 100, + }, + }); + await db.smsSubmitRecord.create({ + data: { + messageRecordId: oldMessage.m.id, + channelId: channel.id, + submitId: randomUUID(), + submitStatus: 'accepted', + submittedAt: at(addDays(old, 1)), + }, + }); + const detection = await detectRetirement(db, T); + const decision = await db.signatureRetirementDetection.findFirst({ + where: { signatureId: signature.id, dimensionType: 'enterprise', detectionDate: databaseDay(T) }, + }); + assert(decision.notificationContent.includes('发送4条')); + assert(detection.alerted >= 1); + const count = await db.signatureRetirementDetection.count({ where: { signatureId: signature.id } }); + assert.equal((await detectRetirement(db, T)).skipped, true); + assert.equal(await db.signatureRetirementDetection.count({ where: { signatureId: signature.id } }), count); + pass('批量退网窗口聚合、重复任务提前退出'); + await message({ phone: '13800000021' }); + await message({ phone: '13800000021' }); + const matched = await resolveUplinkMatch( + db, + { channelId: channel.id, phoneNumber: '13800000021', destId: '1069000099', receivedAt: at(D, 7200).toISOString() }, + channel, + ); + assert.equal(matched.matchStatus, 'matched'); + assert.equal(matched.applicationId, app.id); + assert.equal(matched.messageRecordId, undefined); + const beforeSending = await resolveUplinkMatch( + db, + { channelId: channel.id, phoneNumber: '13800000021', destId: '1069000099', receivedAt: at(D, 3000).toISOString() }, + channel, + ); + assert.equal(beforeSending.matchStatus, 'unmatched'); + pass('上行同应用多短信仍归属唯一、未来下发不参与'); + + const channel2 = await db.smsChannel.create({ + data: { + name: '隔离通道乙', + code: prefix + 'b', + gatewayHost: '127.0.0.1', + gatewayPort: 1, + account: 'isolated', + passwordCipher: 'not-a-secret', + srcId: '10690001', + carriers: ['mobile'], + status: 'disabled', + }, + }); + const wrong = await resolveUplinkMatch( + db, + { + channelId: channel2.id, + messageId: complete.m.messageId, + phoneNumber: complete.m.phoneNumber, + destId: '1069000199', + receivedAt: at(D, 7200).toISOString(), + }, + channel2, + ); + assert.equal(wrong.matchStatus, 'unmatched'); + assert.equal(wrong.messageId, undefined); + const app2 = await db.smsApplication.create({ + data: { + tenantId: tenant.id, + name: '应用乙', + cmppAccount: prefix + 'b', + cmppEnterpriseCode: '000002', + secretHash: 'isolated-no-login', + interfaceEnabled: false, + }, + }); + const other = await message({ phone: '13800000021' }); + await db.smsMessageRecord.update({ where: { id: other.m.id }, data: { applicationId: app2.id } }); + const multi = await resolveUplinkMatch( + db, + { channelId: channel.id, phoneNumber: '13800000021', destId: '1069000099', receivedAt: at(D, 7200).toISOString() }, + channel, + ); + assert.equal(multi.matchStatus, 'ambiguous'); + assert.equal(multi.candidates.length, 2); + pass('错误messageId/其他通道拒绝归属、第三条不同应用不会被截断漏掉'); + + // Fault injection is confined to this disposable QA database. + const late = await db.smsMessageSegmentAudit.create({ + data: { + messageRecordId: partial.m.id, + submitRecordId: partial.s.id, + channelId: channel.id, + submitId: partial.s.submitId, + segmentTotal: 3, + segmentIndex: 3, + submitStatus: 'accepted', + receiptStatus: 'delivered', + deliveredAt: at(D, 3610), + }, + }); + await db.signatureAnalyticsRun.update({ + where: { scope_businessDate: { scope: 'daily', businessDate: databaseDay(D) } }, + data: { refreshFor: databaseDay(D) }, + }); + const prior = quality.generationId; + await writer.generate(D); + const refreshed = await reader.quality({ date: D, keyword: prefix }); + assert.notEqual(refreshed.generationId, prior); + assert.equal(refreshed.items[0].breakdowns[0].successCount, 6); + assert((await db.signatureQualityDaily.count({ where: { generationId: prior } })) > 0); + const secondDay = addDays(T, -2); + await message({ date: secondDay, units: 2 }); + await message({ date: secondDay, units: 4 }); + const legacy = await message({ date: secondDay, units: 4, segments: 0 }); + await db.smsReceiptRecord.create({ + data: { + receiptKey: randomUUID(), + messageRecordId: legacy.m.id, + messageId: legacy.m.messageId, + channelId: channel.id, + gatewayMessageId: legacy.s.gatewayMessageId, + receiptStatus: 'delivered', + rawStatus: 'DELIVRD', + deliveredAt: at(secondDay, 3605), + }, + }); + await writer.generate(secondDay); + const multiLength = await reader.quality({ date: secondDay, keyword: prefix }); + assert.equal(multiLength.items[0].breakdowns[0].successCount, 3); + assert.equal(multiLength.items[0].breakdowns[0].averageArrivalMs, 3667); + const outOfRange = await reader.activity({ date: T, dimensionType: 'enterprise', signatureName: prefix, page: 99 }); + assert.equal(outOfRange.total, 1); + assert.equal(outOfRange.dimensions.length, 0); + pass('两段/四段、历史无分段明确回执兼容、耗时加权、超出页码仍有正确总数'); + await writer.generate(addDays(T, -3)); + pass('最近三日生成、迟到长短信分段刷新、旧版本保留'); + + const failedScope = 'qa-failure-' + prefix, + rollbackGeneration = randomUUID(); + await assert.rejects( + analyticsJob(db, failedScope, D, async (tx) => { + await tx.signatureAnalyticsGeneration.create({ + data: { id: rollbackGeneration, businessDate: databaseDay(D), sourceAsOf: new Date() }, + }); + throw new Error('injected'); + }), + /injected/, + ); + assert.equal(await db.signatureAnalyticsGeneration.count({ where: { id: rollbackGeneration } }), 0); + const failed = await db.signatureAnalyticsRun.findUnique({ + where: { scope_businessDate: { scope: failedScope, businessDate: databaseDay(D) } }, + }); + assert.equal(failed.state, 'retry_wait'); + await db.signatureAnalyticsRun.update({ where: { id: failed.id }, data: { nextAttemptAt: new Date(0) } }); + assert.equal((await analyticsJob(db, failedScope, D, async () => 42)).result, 42); + pass('中途失败原子回滚、持久重试与恢复'); + + const scope = 'qa-concurrent-' + prefix; + let release, entered; + const gate = new Promise((r) => (release = r)), + ready = new Promise((r) => (entered = r)); + const owner = analyticsJob(db, scope, D, async () => { + entered(); + await gate; + return 1; + }); + await ready; + assert.equal((await analyticsJob(db, scope, D, async () => 2)).skipped, true); + release(); + assert.equal((await owner).result, 1); + const fencedScope = 'qa-fenced-' + prefix, + fencedGeneration = randomUUID(); + await assert.rejects( + analyticsJob(db, fencedScope, D, async (tx) => { + await tx.signatureAnalyticsGeneration.create({ + data: { id: fencedGeneration, businessDate: databaseDay(D), sourceAsOf: new Date() }, + }); + await db.signatureAnalyticsRun.update({ + where: { scope_businessDate: { scope: fencedScope, businessDate: databaseDay(D) } }, + data: { fence: { increment: 1 }, owner: 'replacement' }, + }); + }), + ); + assert.equal(await db.signatureAnalyticsGeneration.count({ where: { id: fencedGeneration } }), 0); + pass('双worker唯一认领、旧fence禁止发布并回滚候选版本'); + + const checkpointScope = 'qa-checkpoint-' + prefix; + await assert.rejects( + analyticsJob( + db, + checkpointScope, + D, + async () => { + throw new Error('checkpoint fault'); + }, + new Date(), + async () => ({ version: 1 }), + ), + /checkpoint fault/, + ); + await db.signatureAnalyticsRun.update({ + where: { scope_businessDate: { scope: checkpointScope, businessDate: databaseDay(D) } }, + data: { nextAttemptAt: new Date(0) }, + }); + const resumed = await analyticsJob( + db, + checkpointScope, + D, + async (_tx, _generation, checkpoint) => checkpoint, + new Date(), + async () => ({ version: 2 }), + ); + assert.deepEqual(resumed.result, { version: 1 }); + pass('重试沿用首轮冻结规则快照'); + assert(late.id); + console.log( + JSON.stringify({ + fixturePrefix: prefix, + tenantId: tenant.id, + applicationId: app.id, + signatureId: signature.id, + partial: partial.s.id, + complete: complete.s.id, + }), + ); +} finally { + await db.onModuleDestroy(); +} diff --git a/tools/testing/verify-uplink-matching.mjs b/tools/testing/verify-uplink-matching.mjs new file mode 100644 index 0000000..ff7aec9 --- /dev/null +++ b/tools/testing/verify-uplink-matching.mjs @@ -0,0 +1,143 @@ +import assert from 'node:assert/strict'; +import { randomUUID } from 'node:crypto'; +import { createRequire } from 'node:module'; +const url = new URL(process.env.SIGNATURE_TEST_DATABASE_URL || ''); +assert(['localhost', '127.0.0.1'].includes(url.hostname) && url.pathname.startsWith('/cmpp_qa_signature_')); +process.env.DATABASE_URL = url.toString(); +process.env.NODE_ENV = 'test'; +const require = createRequire(new URL('../../api/package.json', import.meta.url)); +require('reflect-metadata'); +const { PrismaService } = require('./dist/prisma/prisma.service'); +const { completionDatabase } = require('./dist/send-chain/completion-context'); +const { SendDownstreamDeliveryService } = require('./dist/send-chain/send-downstream-delivery.service'); +const { OpenApiService } = require('./dist/open-api/open-api.service'); +const { resolveUplinkMatch } = require('./dist/send-chain/uplink-matching'); +const db = new PrismaService(), + proxy = completionDatabase(db), + prefix = randomUUID().slice(0, 8); +function delivery(openApi) { + let service; + const facade = { + resolveUplinkMatch: (event, channel) => resolveUplinkMatch(proxy, event, channel), + queueAndTryDownstreamDelivery: (data) => service.queueAndTryDownstreamDelivery(data), + postGatewayControl: () => { + throw new Error('External delivery is forbidden in QA'); + }, + }; + service = new SendDownstreamDeliveryService(proxy, undefined, openApi, facade, {}); + return service; +} +try { + const tenant = await db.tenant.create({ data: { name: '上行验收', code: prefix } }); + const apps = []; + for (const suffix of ['a', 'b']) + apps.push( + await db.smsApplication.create({ + data: { + tenantId: tenant.id, + name: '应用' + suffix, + cmppAccount: prefix + suffix, + cmppEnterpriseCode: '000001', + secretHash: 'isolated', + interfaceEnabled: true, + }, + }), + ); + const channel = await db.smsChannel.create({ + data: { + name: '仅入库隔离通道', + code: prefix, + srcId: '1069', + gatewayHost: '127.0.0.1', + gatewayPort: 1, + account: 'isolated', + passwordCipher: 'not-a-secret', + carriers: ['mobile'], + status: 'disabled', + }, + }); + const received = new Date(), + sent = new Date(received.getTime() - 60_000); + for (const [phone, index] of [ + ['13800000101', 0], + ['13800000101', 0], + ['13800000102', 0], + ['13800000102', 1], + ]) { + const m = await db.smsMessageRecord.create({ + data: { + messageId: randomUUID(), + tenantId: tenant.id, + applicationId: apps[index].id, + phoneNumber: phone, + content: '隔离上行匹配测试', + }, + }); + await db.smsSubmitRecord.create({ + data: { + messageRecordId: m.id, + channelId: channel.id, + submitId: randomUUID(), + submitStatus: 'accepted', + submittedAt: sent, + }, + }); + } + for (const app of apps) { + await db.smsApplicationHttpConfig.create({ data: { applicationId: app.id, enabled: true, sendEnabled: false } }); + await db.httpWebhookEndpoint.create({ + data: { + applicationId: app.id, + eventType: 'uplink', + url: 'http://127.0.0.1:1/never-called', + secretEncrypted: 'isolated-no-delivery', + secretLast4: 'test', + }, + }); + } + const service = delivery(new OpenApiService(db, undefined)); + const event = { + eventId: randomUUID(), + channelId: channel.id, + phoneNumber: '13800000101', + destId: '10690001', + content: 'TD', + receivedAt: received.toISOString(), + gatewayMessageId: 'supplier-mo-id', + }; + const copies = await Promise.all([service.handleUplink(event), service.handleUplink(event)]); + assert.equal(copies[0].id, copies[1].id); + assert.equal(copies[0].messageId, null); + assert.equal(copies[0].messageRecordId, null); + assert.equal(copies[0].gatewayMessageId, 'supplier-mo-id'); + assert.equal(await db.cmppDownstreamDelivery.count({ where: { dedupeKey: 'uplink:' + copies[0].id } }), 1); + assert.equal(await db.httpWebhookEvent.count({ where: { uplinkMessageId: copies[0].id } }), 1); + console.log('PASS 重复事件原子入库、无唯一原短信不伪造编号、CMPP与HTTP通知意图各一份'); + const ambiguous = await service.handleUplink({ ...event, eventId: randomUUID(), phoneNumber: '13800000102' }); + assert.equal(ambiguous.matchStatus, 'ambiguous'); + const candidates = await db.smsUplinkMatchCandidate.findMany({ where: { uplinkMessageId: ambiguous.id } }); + assert.equal(candidates.length, 2); + const claims = await Promise.allSettled(candidates.map((c) => service.claimUplinkMatchCandidate(ambiguous.id, c.id))); + assert.equal(claims.filter((r) => r.status === 'fulfilled').length, 1); + assert.equal(await db.cmppDownstreamDelivery.count({ where: { dedupeKey: 'uplink:' + ambiguous.id } }), 1); + assert.equal( + await db.smsUplinkMatchCandidate.count({ where: { uplinkMessageId: ambiguous.id, status: 'claimed' } }), + 1, + ); + console.log('PASS 并发认领仅一个应用成功、候选及通知同事务'); + const failedEvent = { ...event, eventId: randomUUID() }; + await assert.rejects( + delivery({ + queueWebhookEvent: async () => { + throw new Error('injected notification storage failure'); + }, + }).handleUplink(failedEvent), + /injected/, + ); + assert.equal(await db.smsUplinkMessage.count({ where: { eventId: failedEvent.eventId } }), 0); + await service.handleUplink(failedEvent); + assert.equal(await db.smsUplinkMessage.count({ where: { eventId: failedEvent.eventId } }), 1); + console.log('PASS 通知存储故障回滚上行、重试恢复，不调用外部发送'); +} finally { + await db.onModuleDestroy(); +}