Compare commits
2
Commits
86cb9aea36
...
a350aca883
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
a350aca883 | ||
|
|
a0209f93bc |
@@ -1,4 +1,4 @@
|
||||
# CMPP 平台仓库开发约束
|
||||
# 聆界短信平台仓库开发约束
|
||||
|
||||
本文件适用于整个仓库。进入子目录工作时,如果存在更具体的 `AGENTS.md`,还应同时遵守子目录规范。
|
||||
|
||||
|
||||
@@ -0,0 +1,17 @@
|
||||
CREATE TABLE "InfrastructureAlertCollection" (
|
||||
"id" TEXT NOT NULL PRIMARY KEY,
|
||||
"observedAt" TIMESTAMP(3) NOT NULL
|
||||
);
|
||||
CREATE TABLE "InfrastructureAlertEvent" (
|
||||
"id" TEXT NOT NULL PRIMARY KEY,
|
||||
"fingerprint" TEXT NOT NULL,
|
||||
"activeAt" TIMESTAMP(3) NOT NULL,
|
||||
"payload" JSONB NOT NULL,
|
||||
"lastObservedAt" TIMESTAMP(3) NOT NULL,
|
||||
"recoveredAt" TIMESTAMP(3),
|
||||
"clearedAt" TIMESTAMP(3),
|
||||
"clearedBy" TEXT,
|
||||
"createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
CREATE UNIQUE INDEX "InfrastructureAlertEvent_fingerprint_activeAt_key" ON "InfrastructureAlertEvent"("fingerprint", "activeAt");
|
||||
CREATE INDEX "InfrastructureAlertEvent_clearedAt_activeAt_idx" ON "InfrastructureAlertEvent"("clearedAt", "activeAt");
|
||||
@@ -0,0 +1,15 @@
|
||||
CREATE TABLE "SmsAttemptCompletionWork" (
|
||||
"id" TEXT PRIMARY KEY, "workKey" TEXT NOT NULL, "tenantId" TEXT, "messageRecordId" TEXT NOT NULL, "sourceSubmitRecordId" TEXT,
|
||||
"revision" INTEGER NOT NULL DEFAULT 0, "processedRevision" INTEGER NOT NULL DEFAULT 0, "state" TEXT NOT NULL DEFAULT 'pending',
|
||||
"leaseOwner" TEXT, "leaseUntil" TIMESTAMP(3), "fenceVersion" INTEGER NOT NULL DEFAULT 0, "attempts" INTEGER NOT NULL DEFAULT 0,
|
||||
"nextAttemptAt" TIMESTAMP(3) NOT NULL DEFAULT (CURRENT_TIMESTAMP AT TIME ZONE 'UTC'), "decision" TEXT, "retrySubmitRecordId" TEXT, "lastError" TEXT,
|
||||
"createdAt" TIMESTAMP(3) NOT NULL DEFAULT (CURRENT_TIMESTAMP AT TIME ZONE 'UTC'), "updatedAt" TIMESTAMP(3) NOT NULL DEFAULT (CURRENT_TIMESTAMP AT TIME ZONE 'UTC'));
|
||||
CREATE UNIQUE INDEX "SmsAttemptCompletionWork_workKey_key" ON "SmsAttemptCompletionWork"("workKey");
|
||||
CREATE UNIQUE INDEX "SmsAttemptCompletionWork_sourceSubmitRecordId_key" ON "SmsAttemptCompletionWork"("sourceSubmitRecordId");
|
||||
CREATE INDEX "SmsAttemptCompletionWork_state_nextAttemptAt_idx" ON "SmsAttemptCompletionWork"("state", "nextAttemptAt");
|
||||
CREATE INDEX "SmsAttemptCompletionWork_state_leaseUntil_idx" ON "SmsAttemptCompletionWork"("state", "leaseUntil");
|
||||
CREATE INDEX "SmsAttemptCompletionWork_messageRecordId_idx" ON "SmsAttemptCompletionWork"("messageRecordId");
|
||||
CREATE TABLE "SmsCompletionEvent" ("id" TEXT PRIMARY KEY, "eventKey" TEXT NOT NULL, "workId" TEXT NOT NULL REFERENCES "SmsAttemptCompletionWork"("id") ON DELETE RESTRICT,
|
||||
"kind" TEXT NOT NULL, "payload" JSONB NOT NULL, "processedAt" TIMESTAMP(3), "createdAt" TIMESTAMP(3) NOT NULL DEFAULT (CURRENT_TIMESTAMP AT TIME ZONE 'UTC'));
|
||||
CREATE UNIQUE INDEX "SmsCompletionEvent_eventKey_key" ON "SmsCompletionEvent"("eventKey");
|
||||
CREATE INDEX "SmsCompletionEvent_workId_processedAt_createdAt_idx" ON "SmsCompletionEvent"("workId", "processedAt", "createdAt");
|
||||
@@ -2840,3 +2840,59 @@ model OpenApiDispatchOutbox {
|
||||
updatedAt DateTime @updatedAt
|
||||
@@index([status, leaseUntil])
|
||||
}
|
||||
|
||||
model InfrastructureAlertCollection {
|
||||
id String @id
|
||||
observedAt DateTime
|
||||
}
|
||||
|
||||
model InfrastructureAlertEvent {
|
||||
id String @id @default(cuid())
|
||||
fingerprint String
|
||||
activeAt DateTime
|
||||
payload Json
|
||||
lastObservedAt DateTime
|
||||
recoveredAt DateTime?
|
||||
clearedAt DateTime?
|
||||
clearedBy String?
|
||||
createdAt DateTime @default(now())
|
||||
@@unique([fingerprint, activeAt])
|
||||
@@index([clearedAt, activeAt])
|
||||
}
|
||||
|
||||
model SmsAttemptCompletionWork {
|
||||
id String @id @default(cuid())
|
||||
workKey String @unique
|
||||
tenantId String?
|
||||
messageRecordId String
|
||||
sourceSubmitRecordId String? @unique
|
||||
revision Int @default(0)
|
||||
processedRevision Int @default(0)
|
||||
state String @default("pending")
|
||||
leaseOwner String?
|
||||
leaseUntil DateTime?
|
||||
fenceVersion Int @default(0)
|
||||
attempts Int @default(0)
|
||||
nextAttemptAt DateTime @default(dbgenerated("(CURRENT_TIMESTAMP AT TIME ZONE 'UTC')"))
|
||||
decision String?
|
||||
retrySubmitRecordId String?
|
||||
lastError String?
|
||||
createdAt DateTime @default(dbgenerated("(CURRENT_TIMESTAMP AT TIME ZONE 'UTC')"))
|
||||
updatedAt DateTime @updatedAt
|
||||
events SmsCompletionEvent[]
|
||||
@@index([state, nextAttemptAt])
|
||||
@@index([state, leaseUntil])
|
||||
@@index([messageRecordId])
|
||||
}
|
||||
|
||||
model SmsCompletionEvent {
|
||||
id String @id @default(cuid())
|
||||
eventKey String @unique
|
||||
workId String
|
||||
kind String
|
||||
payload Json
|
||||
processedAt DateTime?
|
||||
createdAt DateTime @default(dbgenerated("(CURRENT_TIMESTAMP AT TIME ZONE 'UTC')"))
|
||||
work SmsAttemptCompletionWork @relation(fields: [workId], references: [id], onDelete: Restrict)
|
||||
@@index([workId, processedAt, createdAt])
|
||||
}
|
||||
|
||||
@@ -1,18 +1,24 @@
|
||||
import { BadRequestException, Injectable, Logger, NotFoundException, OnModuleDestroy, OnModuleInit } from '@nestjs/common';
|
||||
import { Queue } from 'bullmq';
|
||||
import IORedis from 'ioredis';
|
||||
import { BadRequestException, NotFoundException } from '@nestjs/common';
|
||||
import { Prisma } from '@prisma/client';
|
||||
import { randomUUID } from 'crypto';
|
||||
import { assertMoneyUnits, moneyToNumber } from '../common/money';
|
||||
import { moneyToNumber } from '../common/money';
|
||||
import { PrismaService } from '../prisma/prisma.service';
|
||||
import type { CreateChannelDto, UpdateChannelDto, CreateChannelGroupDto, CreateChannelGroupItemDto, UpdateChannelGroupDto, CreateRouteRuleDto, CreateReportFieldDto, ReplaceReportFieldsDto, CreateReportMaterialDto, CreateReportTaskDto, ChangeReportTaskStatusesDto, CreateReportExportDto, CreateReceiptImportDto, UpsertConnectionStateDto, ChangeChannelStatusDto, CopyChannelDto, TestChannelDto } from './channels.contracts';
|
||||
import { GATEWAY_CONNECTION_QUEUE, GATEWAY_SUBMIT_QUEUE, GATEWAY_SUBMIT_STREAM, DEFAULT_GATEWAY_CONTROL_URL, DEFAULT_CHANNEL_CONNECTION_ID, DEFAULT_CONNECTING_TIMEOUT_MS, DEFAULT_CONNECTING_TIMEOUT_SCAN_MS, DEFAULT_GATEWAY_STARTUP_RECONNECT_DELAY_MS, DEFAULT_GATEWAY_RECONCILE_INTERVAL_MS, DEFAULT_GATEWAY_CONTROL_TIMEOUT_MS, DEFAULT_HEARTBEAT_INTERVAL_SECONDS, DEFAULT_HEARTBEAT_MISS_THRESHOLD, HEARTBEAT_AUDIT_INTERVAL_MS, CONNECTING_TIMEOUT_ERROR, DEFAULT_CMPP_VERSION, normalizeTestPhones, normalizeTestContent, calculateBillingUnits, buildChannelTestSubmitCommand, getConfigValue, getStringConfigValue, normalizeConnectionAction, normalizeCmppVersion, normalizeGatewayConnectionStatus, defaultChannelConnectionId, getDesiredConnections, ChannelConnectionSettings, getRuntimeConfigInteger, channelConnectionSettingsChanged, channelGroupAuditSnapshot, normalizeChannelRuntimeConfig, normalizeCmppServiceId, normalizeChannelRateLimit, normalizeExtensionDigits, getPositiveRuntimeInteger, bullmqConnection, getPositiveIntegerEnv, parseReceiptContent, splitReceiptLine, stripReceiptCell, findReceiptStatusIndex, normalizeReceiptStatus, deriveReceiptStatus, ChannelReportDeliveryRow, summarizeChannelReportDelivery, sumReportDelivery, percentage, latestDate, currentShanghaiDayRange, normalizeRetryTimeLimitMinutes, normalizeSpreadsheetSize, normalizeBusinessCarrier, normalizeChannelCarrier, isChannelCarrierCompatible, normalizeRegion, isRegionCompatible, validateGroupItems, normalizeReportType, summarizeReportStatuses, normalizeLinkEvent } from './channels.helpers';
|
||||
import type { TestChannelDto } from './channels.contracts';
|
||||
import {
|
||||
normalizeTestPhones,
|
||||
normalizeTestContent,
|
||||
normalizeGatewayConnectionStatus,
|
||||
calculateBillingUnits,
|
||||
buildChannelTestSubmitCommand,
|
||||
} from './channels.helpers';
|
||||
import { ChannelConnectionService } from './channel-connection.service';
|
||||
import { detectDrainageContent } from '../send-chain/drainage-content-detection';
|
||||
|
||||
/** R5 channel domain service composed behind ChannelsService. */
|
||||
export class ChannelTestService {
|
||||
constructor(private readonly prisma: PrismaService, private readonly connection: ChannelConnectionService) {}
|
||||
constructor(
|
||||
private readonly prisma: PrismaService,
|
||||
private readonly connection: ChannelConnectionService,
|
||||
) {}
|
||||
|
||||
async testChannel(channelId: string, data: TestChannelDto = {}) {
|
||||
const phoneNumbers = normalizeTestPhones(data);
|
||||
@@ -27,8 +33,8 @@ export class ChannelTestService {
|
||||
if (channel.status !== 'active') {
|
||||
throw new BadRequestException('通道未启用,不能发送测试短信');
|
||||
}
|
||||
const connectedState = channel.connectionStates.find((state) =>
|
||||
normalizeGatewayConnectionStatus(state.status) === 'connected' && (state.currentConnections ?? 0) > 0,
|
||||
const connectedState = channel.connectionStates.find(
|
||||
(state) => normalizeGatewayConnectionStatus(state.status) === 'connected' && (state.currentConnections ?? 0) > 0,
|
||||
);
|
||||
if (!connectedState) {
|
||||
throw new BadRequestException('通道当前没有可用 CMPP 连接,请先连接成功后再测试发送');
|
||||
@@ -36,7 +42,6 @@ export class ChannelTestService {
|
||||
|
||||
const createdAt = new Date();
|
||||
const testNo = `CHTEST-${Date.now()}-${randomUUID().slice(0, 8)}`;
|
||||
const drainageDetection = await detectDrainageContent(this.prisma, content);
|
||||
const results = [];
|
||||
for (const [index, phoneNumber] of phoneNumbers.entries()) {
|
||||
const messageId = `MSG-TEST-${Date.now()}-${randomUUID().slice(0, 8)}`;
|
||||
@@ -51,7 +56,6 @@ export class ChannelTestService {
|
||||
messageId,
|
||||
phoneNumber,
|
||||
content,
|
||||
...drainageDetection,
|
||||
billingUnits: calculateBillingUnits(content),
|
||||
unitPrice: 0,
|
||||
amountCents: 0,
|
||||
|
||||
@@ -37,6 +37,15 @@ export class InfrastructureMonitoringController {
|
||||
return this.monitoring.markAlertRead(fingerprint, activeAt, userId);
|
||||
}
|
||||
|
||||
@Post('alerts/:fingerprint/clear')
|
||||
clearAlert(
|
||||
@Param('fingerprint') fingerprint: string,
|
||||
@Body('activeAt') activeAt: unknown,
|
||||
@CurrentSessionUserId() userId: string,
|
||||
) {
|
||||
return this.monitoring.clearAlert(fingerprint, activeAt, userId);
|
||||
}
|
||||
|
||||
@Get('alert-thresholds')
|
||||
alertThresholds() {
|
||||
return this.settings.get();
|
||||
|
||||
@@ -1,8 +1,15 @@
|
||||
import { BadRequestException } from '@nestjs/common';
|
||||
import { ConfigService } from '@nestjs/config';
|
||||
import { createHash } from 'node:crypto';
|
||||
import { InfrastructureMonitoringService } from './infrastructure-monitoring.service';
|
||||
import { FILESYSTEM_USAGE_PERCENT, filesystemIdentity } from './filesystem-metrics';
|
||||
import { InfrastructureMonitoringService } from './infrastructure-monitoring.service';
|
||||
import { retainedAlerts } from './persistent-alerts';
|
||||
|
||||
jest.mock('./persistent-alerts', () => ({
|
||||
retainAlerts: jest.fn(async (_prisma, alerts) => alerts),
|
||||
retainedAlerts: jest.fn(),
|
||||
clearRetainedAlert: jest.fn(),
|
||||
}));
|
||||
|
||||
function success(data: unknown) {
|
||||
return {
|
||||
@@ -14,7 +21,12 @@ function success(data: unknown) {
|
||||
|
||||
describe('InfrastructureMonitoringService', () => {
|
||||
const prisma = {
|
||||
infrastructureAlertRead: { findMany: jest.fn().mockResolvedValue([]), create: jest.fn(), update: jest.fn(), findUniqueOrThrow: jest.fn() },
|
||||
infrastructureAlertRead: {
|
||||
findMany: jest.fn().mockResolvedValue([]),
|
||||
create: jest.fn(),
|
||||
update: jest.fn(),
|
||||
findUniqueOrThrow: jest.fn(),
|
||||
},
|
||||
operationLog: { create: jest.fn() },
|
||||
$transaction: jest.fn(),
|
||||
};
|
||||
@@ -34,9 +46,27 @@ describe('InfrastructureMonitoringService', () => {
|
||||
});
|
||||
|
||||
it('rejects credential-bearing or remote plaintext Prometheus endpoints at startup', () => {
|
||||
expect(() => new InfrastructureMonitoringService(new ConfigService({ PROMETHEUS_URL: 'http://user:secret@127.0.0.1:9090' }), prisma as never)).toThrow('must not contain credentials');
|
||||
expect(() => new InfrastructureMonitoringService(new ConfigService({ PROMETHEUS_URL: 'http://monitor.example.com:9090' }), prisma as never)).toThrow('must use HTTPS');
|
||||
expect(() => new InfrastructureMonitoringService(new ConfigService({ PROMETHEUS_URL: 'https://monitor.example.com' }), prisma as never)).not.toThrow();
|
||||
expect(
|
||||
() =>
|
||||
new InfrastructureMonitoringService(
|
||||
new ConfigService({ PROMETHEUS_URL: 'http://user:secret@127.0.0.1:9090' }),
|
||||
prisma as never,
|
||||
),
|
||||
).toThrow('must not contain credentials');
|
||||
expect(
|
||||
() =>
|
||||
new InfrastructureMonitoringService(
|
||||
new ConfigService({ PROMETHEUS_URL: 'http://monitor.example.com:9090' }),
|
||||
prisma as never,
|
||||
),
|
||||
).toThrow('must use HTTPS');
|
||||
expect(
|
||||
() =>
|
||||
new InfrastructureMonitoringService(
|
||||
new ConfigService({ PROMETHEUS_URL: 'https://monitor.example.com' }),
|
||||
prisma as never,
|
||||
),
|
||||
).not.toThrow();
|
||||
});
|
||||
|
||||
it('loads real Prometheus vectors, ranges, services and active alerts', async () => {
|
||||
@@ -45,34 +75,52 @@ describe('InfrastructureMonitoringService', () => {
|
||||
const url = new URL(String(input));
|
||||
requestedUrls.push(url);
|
||||
if (url.pathname.endsWith('/alerts')) {
|
||||
return success({ alerts: [{
|
||||
return success({
|
||||
alerts: [
|
||||
{
|
||||
labels: { alertname: 'HostCpuHigh', severity: 'warning', instance: '127.0.0.1:9100' },
|
||||
annotations: { summary: 'CPU持续偏高', threshold: '85%' },
|
||||
state: 'firing',
|
||||
activeAt: '2026-08-14T03:00:00.000Z',
|
||||
value: '88.2',
|
||||
}] });
|
||||
},
|
||||
],
|
||||
});
|
||||
}
|
||||
const query = url.searchParams.get('query') ?? '';
|
||||
if (url.pathname.endsWith('/query_range')) {
|
||||
return success({ result: [{ metric: {}, values: [[1_765_000_000, '12.5'], [1_765_000_060, '14.5']] }] });
|
||||
return success({
|
||||
result: [
|
||||
{
|
||||
metric: {},
|
||||
values: [
|
||||
[1_765_000_000, '12.5'],
|
||||
[1_765_000_060, '14.5'],
|
||||
],
|
||||
},
|
||||
],
|
||||
});
|
||||
}
|
||||
if (query.includes('node_systemd_unit_state')) {
|
||||
return success({ result: [
|
||||
return success({
|
||||
result: [
|
||||
{ metric: { name: 'cmpp-api.service' }, value: [1_765_000_060, '1'] },
|
||||
{ metric: { name: 'cmpp-gateway.service' }, value: [1_765_000_060, '1'] },
|
||||
{ metric: { name: 'postgresql.service' }, value: [1_765_000_060, '1'] },
|
||||
{ metric: { name: 'redis-server.service' }, value: [1_765_000_060, '1'] },
|
||||
{ metric: { name: 'cmpp-minio.service' }, value: [1_765_000_060, '1'] },
|
||||
{ metric: { name: 'nginx.service' }, value: [1_765_000_060, '1'] },
|
||||
] });
|
||||
],
|
||||
});
|
||||
}
|
||||
if (query.includes('cmpp:service_.*')) {
|
||||
return success({ result: [
|
||||
return success({
|
||||
result: [
|
||||
{ metric: { __name__: 'cmpp:service_api:requests_per_second' }, value: [1_765_000_060, '12.5'] },
|
||||
{ metric: { __name__: 'cmpp:service_api:error_percent' }, value: [1_765_000_060, '0.2'] },
|
||||
{ metric: { __name__: 'cmpp:service_gateway:queue_pending' }, value: [1_765_000_060, '3'] },
|
||||
] });
|
||||
],
|
||||
});
|
||||
}
|
||||
if (query.includes('timestamp(node_uname_info)')) {
|
||||
return success({ result: [{ metric: {}, value: [1_765_000_060, '1765000060'] }] });
|
||||
@@ -87,14 +135,27 @@ describe('InfrastructureMonitoringService', () => {
|
||||
expect(result.metrics.cpuUsagePercent).toBe(25);
|
||||
expect(result.trends.cpuUsagePercent).toHaveLength(2);
|
||||
expect(result.summary).toMatchObject({ overallStatus: 'warning', serviceHealthy: 6, warningAlerts: 1 });
|
||||
expect(result.services.find((item) => item.key === 'redis')).toMatchObject({ unit: 'redis-server.service', status: 'healthy' });
|
||||
expect(result.services.find((item) => item.key === 'redis')).toMatchObject({
|
||||
unit: 'redis-server.service',
|
||||
status: 'healthy',
|
||||
});
|
||||
expect(result.serviceMetrics.find((item) => item.key === 'api')).toMatchObject({ available: true });
|
||||
expect(result.serviceMetrics.find((item) => item.key === 'gateway')?.metrics.find((item) => item.key === 'queuePending')?.value).toBe(3);
|
||||
expect(
|
||||
result.serviceMetrics.find((item) => item.key === 'gateway')?.metrics.find((item) => item.key === 'queuePending')
|
||||
?.value,
|
||||
).toBe(3);
|
||||
expect(result.alerts[0]).toMatchObject({ name: 'HostCpuHigh', severity: 'warning', currentValue: '88.2' });
|
||||
expect(requestedUrls.filter((url) => url.pathname.endsWith('/query_range'))).toHaveLength(5);
|
||||
expect(requestedUrls.filter((url) => url.pathname.endsWith('/query_range')).every((url) => url.searchParams.get('step') === '60')).toBe(true);
|
||||
expect(requestedUrls.find((url) => url.searchParams.get('query')?.includes('node_systemd_unit_state'))?.searchParams.get('query'))
|
||||
.toContain('cmpp-api\\\\.service');
|
||||
expect(
|
||||
requestedUrls
|
||||
.filter((url) => url.pathname.endsWith('/query_range'))
|
||||
.every((url) => url.searchParams.get('step') === '60'),
|
||||
).toBe(true);
|
||||
expect(
|
||||
requestedUrls
|
||||
.find((url) => url.searchParams.get('query')?.includes('node_systemd_unit_state'))
|
||||
?.searchParams.get('query'),
|
||||
).toContain('cmpp-api\\\\.service');
|
||||
});
|
||||
|
||||
it('returns an explicit unavailable payload without stale metrics when Prometheus fails', async () => {
|
||||
@@ -124,12 +185,36 @@ describe('InfrastructureMonitoringService', () => {
|
||||
if (url.pathname.endsWith('/alerts')) return success({ alerts: [] });
|
||||
if (!query.includes('node_filesystem_')) return success({ result: [] });
|
||||
expect(query).not.toContain('mountpoint="/"');
|
||||
if (url.pathname.endsWith('/query_range')) return success({ result: [...metrics].reverse().map((metric) => ({ metric, values: [[1765000060, metric.mountpoint === '/' ? '91' : '12']] })) });
|
||||
return success({ result: metrics.map((metric) => ({ metric, value: [1765000060, query === FILESYSTEM_USAGE_PERCENT ? (metric.mountpoint === '/' ? '91' : '12') : query.includes('avail') ? '9' : '100'] })) });
|
||||
if (url.pathname.endsWith('/query_range'))
|
||||
return success({
|
||||
result: [...metrics]
|
||||
.reverse()
|
||||
.map((metric) => ({ metric, values: [[1765000060, metric.mountpoint === '/' ? '91' : '12']] })),
|
||||
});
|
||||
return success({
|
||||
result: metrics.map((metric) => ({
|
||||
metric,
|
||||
value: [
|
||||
1765000060,
|
||||
query === FILESYSTEM_USAGE_PERCENT
|
||||
? metric.mountpoint === '/'
|
||||
? '91'
|
||||
: '12'
|
||||
: query.includes('avail')
|
||||
? '9'
|
||||
: '100',
|
||||
],
|
||||
})),
|
||||
});
|
||||
});
|
||||
const result = await new InfrastructureMonitoringService(new ConfigService(), prisma as never).overview('1h');
|
||||
expect(result.disks.map((disk) => disk.mountpoint)).toEqual(['/', '/archive', '/data']);
|
||||
expect(result.disks[0]).toMatchObject({ usagePercent: 91, totalBytes: 100, availableBytes: 9, trend: [{ timestamp: new Date(1765000060000).toISOString(), value: 91 }] });
|
||||
expect(result.disks[0]).toMatchObject({
|
||||
usagePercent: 91,
|
||||
totalBytes: 100,
|
||||
availableBytes: 9,
|
||||
trend: [{ timestamp: new Date(1765000060000).toISOString(), value: 91 }],
|
||||
});
|
||||
expect(result.disks[2].trend[0].value).toBe(12);
|
||||
expect(result.metrics.diskUsagePercent).toBe(91);
|
||||
expect(result.trends.diskUsagePercent[0].value).toBe(91);
|
||||
@@ -146,20 +231,44 @@ describe('InfrastructureMonitoringService', () => {
|
||||
if (!query.includes('node_filesystem_')) return success({ result: [] });
|
||||
if (url.pathname.endsWith('/query_range')) {
|
||||
expect(query).toBe(FILESYSTEM_USAGE_PERCENT);
|
||||
return success({ result: [
|
||||
{ metric: data, values: [[1765000000, '82'], [1765000060, 'NaN'], [1765000120, '83.5']] },
|
||||
return success({
|
||||
result: [
|
||||
{
|
||||
metric: data,
|
||||
values: [
|
||||
[1765000000, '82'],
|
||||
[1765000060, 'NaN'],
|
||||
[1765000120, '83.5'],
|
||||
],
|
||||
},
|
||||
{ metric: root, values: [[1765000000, '91']] },
|
||||
] });
|
||||
],
|
||||
});
|
||||
}
|
||||
if (query.startsWith('node_filesystem_size_bytes')) return success({ result: [
|
||||
if (query.startsWith('node_filesystem_size_bytes'))
|
||||
return success({
|
||||
result: [
|
||||
...mounts.map((mountpoint) => ({ metric: { ...data, mountpoint }, value: [1765000120, '100'] })),
|
||||
...['/var/root-bind', '/'].map((mountpoint) => ({ metric: { ...root, mountpoint }, value: [1765000120, '200'] })),
|
||||
] });
|
||||
if (query === FILESYSTEM_USAGE_PERCENT) return success({ result: [
|
||||
{ metric: data, value: [1765000120, '83.5'] }, { metric: root, value: [1765000120, '91'] },
|
||||
] });
|
||||
...['/var/root-bind', '/'].map((mountpoint) => ({
|
||||
metric: { ...root, mountpoint },
|
||||
value: [1765000120, '200'],
|
||||
})),
|
||||
],
|
||||
});
|
||||
if (query === FILESYSTEM_USAGE_PERCENT)
|
||||
return success({
|
||||
result: [
|
||||
{ metric: data, value: [1765000120, '83.5'] },
|
||||
{ metric: root, value: [1765000120, '91'] },
|
||||
],
|
||||
});
|
||||
expect(query).toContain('min by (instance, device, fstype)');
|
||||
return success({ result: [{ metric: data, value: [1765000120, '16.5'] }, { metric: root, value: [1765000120, '18'] }] });
|
||||
return success({
|
||||
result: [
|
||||
{ metric: data, value: [1765000120, '16.5'] },
|
||||
{ metric: root, value: [1765000120, '18'] },
|
||||
],
|
||||
});
|
||||
});
|
||||
const service = new InfrastructureMonitoringService(new ConfigService(), prisma as never);
|
||||
const result = await service.overview('1h');
|
||||
@@ -169,7 +278,13 @@ describe('InfrastructureMonitoringService', () => {
|
||||
expect(result.metrics.diskUsagePercent).toBe(91);
|
||||
expect(result.trends.diskUsagePercent[0].value).toBe(91);
|
||||
const disk = result.disks[1];
|
||||
expect(disk).toMatchObject({ id: filesystemIdentity(data), mountpoint: '/data', totalBytes: 100, availableBytes: 16.5, usagePercent: 83.5 });
|
||||
expect(disk).toMatchObject({
|
||||
id: filesystemIdentity(data),
|
||||
mountpoint: '/data',
|
||||
totalBytes: 100,
|
||||
availableBytes: 16.5,
|
||||
usagePercent: 83.5,
|
||||
});
|
||||
expect(disk.mountpoints).toEqual(['/data', '/var/lib/minio', '/var/lib/pgsql', '/var/lib/redis']);
|
||||
expect(disk.trend.map((point) => point.value)).toEqual([82, 83.5]);
|
||||
mounts.reverse();
|
||||
@@ -198,33 +313,77 @@ describe('InfrastructureMonitoringService', () => {
|
||||
const result = await new InfrastructureMonitoringService(new ConfigService(), prisma as never).overview('1h');
|
||||
expect(result.disks).toHaveLength(4);
|
||||
expect(new Set(result.disks.map((disk) => disk.id)).size).toBe(4);
|
||||
expect(result.disks.every((disk) => disk.usagePercent === null && disk.availableBytes === null && disk.trend.length === 0)).toBe(true);
|
||||
expect(
|
||||
result.disks.every(
|
||||
(disk) => disk.usagePercent === null && disk.availableBytes === null && disk.trend.length === 0,
|
||||
),
|
||||
).toBe(true);
|
||||
expect(result.metrics.diskUsagePercent).toBeNull();
|
||||
expect(result.trends.diskUsagePercent).toEqual([]);
|
||||
});
|
||||
|
||||
it('excludes only the current alert occurrence after the current administrator marks it read', async () => {
|
||||
const labels = { alertname: 'QaWarning', severity: 'warning', service: 'qa-preview' };
|
||||
const fingerprint = createHash('sha256').update(JSON.stringify(Object.entries(labels).sort(([left], [right]) => left.localeCompare(right)))).digest('hex').slice(0, 24);
|
||||
jest.spyOn(global, 'fetch').mockResolvedValue(success({ alerts: [{ labels, annotations: { summary: '演示预警' }, state: 'firing', activeAt: '2026-08-16T01:00:00.000Z' }] }));
|
||||
prisma.infrastructureAlertRead.findMany.mockResolvedValue([{ fingerprint, activeAt: new Date('2026-08-16T01:00:00.000Z'), readAt: new Date('2026-08-16T01:01:00.000Z') }]);
|
||||
const fingerprint = createHash('sha256')
|
||||
.update(JSON.stringify(Object.entries(labels).sort(([left], [right]) => left.localeCompare(right))))
|
||||
.digest('hex')
|
||||
.slice(0, 24);
|
||||
jest.spyOn(global, 'fetch').mockResolvedValue(
|
||||
success({
|
||||
alerts: [
|
||||
{ labels, annotations: { summary: '演示预警' }, state: 'firing', activeAt: '2026-08-16T01:00:00.000Z' },
|
||||
],
|
||||
}),
|
||||
);
|
||||
prisma.infrastructureAlertRead.findMany.mockResolvedValue([
|
||||
{ fingerprint, activeAt: new Date('2026-08-16T01:00:00.000Z'), readAt: new Date('2026-08-16T01:01:00.000Z') },
|
||||
]);
|
||||
const service = new InfrastructureMonitoringService(new ConfigService(), prisma as never);
|
||||
|
||||
await expect(service.notificationSummary('admin-1')).resolves.toEqual({ count: 0, criticalCount: 0 });
|
||||
|
||||
prisma.infrastructureAlertRead.findMany.mockResolvedValue([{ fingerprint, activeAt: new Date('2026-08-15T01:00:00.000Z'), readAt: new Date('2026-08-15T01:01:00.000Z') }]);
|
||||
prisma.infrastructureAlertRead.findMany.mockResolvedValue([
|
||||
{ fingerprint, activeAt: new Date('2026-08-15T01:00:00.000Z'), readAt: new Date('2026-08-15T01:01:00.000Z') },
|
||||
]);
|
||||
await expect(service.notificationSummary('admin-1')).resolves.toEqual({ count: 1, criticalCount: 0 });
|
||||
});
|
||||
|
||||
it('upserts an idempotent per-user read record only for a currently active occurrence', async () => {
|
||||
const labels = { alertname: 'QaCritical', severity: 'critical', service: 'qa-preview' };
|
||||
const fingerprint = createHash('sha256').update(JSON.stringify(Object.entries(labels).sort(([left], [right]) => left.localeCompare(right)))).digest('hex').slice(0, 24);
|
||||
jest.spyOn(global, 'fetch').mockResolvedValue(success({ alerts: [{ labels, annotations: { summary: '演示严重告警' }, state: 'firing', activeAt: '2026-08-16T02:00:00.000Z' }] }));
|
||||
prisma.$transaction.mockResolvedValue([{ activeAt: new Date('2026-08-16T02:00:00.000Z'), readAt: new Date('2026-08-16T02:01:00.000Z') }, {}]);
|
||||
const fingerprint = createHash('sha256')
|
||||
.update(JSON.stringify(Object.entries(labels).sort(([left], [right]) => left.localeCompare(right))))
|
||||
.digest('hex')
|
||||
.slice(0, 24);
|
||||
jest.spyOn(global, 'fetch').mockResolvedValue(
|
||||
success({
|
||||
alerts: [
|
||||
{ labels, annotations: { summary: '演示严重告警' }, state: 'firing', activeAt: '2026-08-16T02:00:00.000Z' },
|
||||
],
|
||||
}),
|
||||
);
|
||||
prisma.$transaction.mockResolvedValue([
|
||||
{ activeAt: new Date('2026-08-16T02:00:00.000Z'), readAt: new Date('2026-08-16T02:01:00.000Z') },
|
||||
{},
|
||||
]);
|
||||
const service = new InfrastructureMonitoringService(new ConfigService(), prisma as never);
|
||||
|
||||
await expect(service.markAlertRead(fingerprint, '2026-08-16T02:00:00.000Z', 'admin-1')).resolves.toMatchObject({ fingerprint, acknowledged: true });
|
||||
expect(prisma.infrastructureAlertRead.create).toHaveBeenCalledWith(expect.objectContaining({ data: expect.objectContaining({ fingerprint, userId: 'admin-1' }) }));
|
||||
await expect(service.markAlertRead(fingerprint, '2026-08-15T02:00:00.000Z', 'admin-1')).rejects.toThrow('已结束或已重新触发');
|
||||
jest.mocked(retainedAlerts).mockResolvedValue([
|
||||
{
|
||||
fingerprint,
|
||||
startedAt: '2026-08-16T02:00:00.000Z',
|
||||
name: 'QaCritical',
|
||||
severity: 'critical',
|
||||
} as never,
|
||||
]);
|
||||
await expect(service.markAlertRead(fingerprint, '2026-08-16T02:00:00.000Z', 'admin-1')).resolves.toMatchObject({
|
||||
fingerprint,
|
||||
acknowledged: true,
|
||||
});
|
||||
expect(prisma.infrastructureAlertRead.create).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ data: expect.objectContaining({ fingerprint, userId: 'admin-1' }) }),
|
||||
);
|
||||
await expect(service.markAlertRead(fingerprint, '2026-08-15T02:00:00.000Z', 'admin-1')).rejects.toThrow(
|
||||
'已结束或已重新触发',
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { retainAlerts, retainedAlerts, clearRetainedAlert } from './persistent-alerts';
|
||||
import {
|
||||
BadRequestException,
|
||||
Injectable,
|
||||
@@ -202,6 +203,8 @@ export class InfrastructureMonitoringService {
|
||||
private readonly logger = new Logger(InfrastructureMonitoringService.name);
|
||||
private readonly prometheusUrl: string;
|
||||
private readonly queryTimeoutMs: number;
|
||||
private alertTimer?: ReturnType<typeof setInterval>;
|
||||
private alertPoll?: Promise<InfrastructureAlert[]>;
|
||||
|
||||
constructor(
|
||||
config: ConfigService,
|
||||
@@ -211,6 +214,35 @@ export class InfrastructureMonitoringService {
|
||||
this.queryTimeoutMs = Math.min(15_000, Math.max(1_000, Number(config.get('PROMETHEUS_QUERY_TIMEOUT_MS') ?? 5_000)));
|
||||
}
|
||||
|
||||
onModuleInit() {
|
||||
if (process.env.CMPP_PROCESS_ROLE && !['api', 'all'].includes(process.env.CMPP_PROCESS_ROLE)) return;
|
||||
const poll = () =>
|
||||
void this.loadRetainedAlerts().catch(() => this.logger.warn('Persistent alert collection unavailable'));
|
||||
poll();
|
||||
this.alertTimer = setInterval(poll, 30_000);
|
||||
this.alertTimer.unref();
|
||||
}
|
||||
|
||||
onModuleDestroy() {
|
||||
if (this.alertTimer) clearInterval(this.alertTimer);
|
||||
}
|
||||
|
||||
private loadRetainedAlerts() {
|
||||
if (!this.alertPoll) {
|
||||
const observedAt = new Date();
|
||||
this.alertPoll = this.getJson<PrometheusAlertResponse>('/api/v1/alerts')
|
||||
.then((response) => retainAlerts(this.prisma, this.parseAlerts(response), observedAt))
|
||||
.finally(() => {
|
||||
this.alertPoll = undefined;
|
||||
});
|
||||
}
|
||||
return this.alertPoll;
|
||||
}
|
||||
|
||||
clearAlert(fingerprint: string, activeAt: unknown, userId: string) {
|
||||
return clearRetainedAlert(this.prisma, fingerprint, activeAt, userId);
|
||||
}
|
||||
|
||||
async overview(rawRange?: string, userId?: string): Promise<InfrastructureMonitoringOverview> {
|
||||
const range = this.parseRange(rawRange);
|
||||
const collectedAt = new Date().toISOString();
|
||||
@@ -220,11 +252,11 @@ export class InfrastructureMonitoringService {
|
||||
this.loadTrends(range),
|
||||
this.query(QUERIES.services),
|
||||
this.query(SERVICE_METRICS_QUERY),
|
||||
this.getJson<PrometheusAlertResponse>('/api/v1/alerts'),
|
||||
this.loadRetainedAlerts(),
|
||||
]);
|
||||
const services = this.parseServices(serviceResponse);
|
||||
const serviceMetrics = this.parseServiceMetrics(serviceMetricResponse);
|
||||
const alerts = await this.attachReadState(this.parseAlerts(alertResponse), userId);
|
||||
const alerts = await this.attachReadState(alertResponse, userId);
|
||||
const warningAlerts = alerts.filter((item) => item.severity === 'warning').length;
|
||||
const criticalAlerts = alerts.filter((item) => item.severity === 'critical').length;
|
||||
const overallStatus = criticalAlerts > 0 ? 'critical' : warningAlerts > 0 ? 'warning' : 'healthy';
|
||||
@@ -250,7 +282,7 @@ export class InfrastructureMonitoringService {
|
||||
alerts,
|
||||
};
|
||||
} catch (error) {
|
||||
// 页面必须整体清空陈旧指标,但服务端仍需留下不含PromQL/地址/凭据的根因摘要便于运维诊断。
|
||||
// 客户端保留最后成功快照;采集失败不推断告警恢复。
|
||||
this.logger.warn(
|
||||
`Prometheus monitoring overview unavailable: ${error instanceof Error ? error.message : 'unknown error'}`,
|
||||
);
|
||||
@@ -260,10 +292,7 @@ export class InfrastructureMonitoringService {
|
||||
|
||||
async notificationSummary(userId?: string) {
|
||||
try {
|
||||
const alerts = await this.attachReadState(
|
||||
this.parseAlerts(await this.getJson<PrometheusAlertResponse>('/api/v1/alerts')),
|
||||
userId,
|
||||
);
|
||||
const alerts = await this.attachReadState(await this.loadRetainedAlerts(), userId);
|
||||
const unreadAlerts = alerts.filter((item) => !item.acknowledged);
|
||||
return {
|
||||
count: unreadAlerts.length,
|
||||
@@ -281,7 +310,7 @@ export class InfrastructureMonitoringService {
|
||||
if (!/^[a-f0-9]{24}$/.test(fingerprint)) throw new BadRequestException('告警指纹无效');
|
||||
const activeAt = new Date(String(rawActiveAt ?? ''));
|
||||
if (!Number.isFinite(activeAt.getTime())) throw new BadRequestException('告警开始时间无效');
|
||||
const activeAlerts = this.parseAlerts(await this.getJson<PrometheusAlertResponse>('/api/v1/alerts'));
|
||||
const activeAlerts = await retainedAlerts(this.prisma);
|
||||
const current = activeAlerts.find(
|
||||
(item) => item.fingerprint === fingerprint && Date.parse(item.startedAt) === activeAt.getTime(),
|
||||
);
|
||||
|
||||
@@ -0,0 +1,143 @@
|
||||
import { BadRequestException, NotFoundException } from '@nestjs/common';
|
||||
import { Prisma } from '@prisma/client';
|
||||
import { createHash } from 'node:crypto';
|
||||
import { PrismaService } from '../prisma/prisma.service';
|
||||
import type { InfrastructureAlert } from './infrastructure-monitoring.contracts';
|
||||
|
||||
export async function retainAlerts(prisma: PrismaService, alerts: InfrastructureAlert[], observedAt: Date) {
|
||||
await prisma.$transaction(async (tx) => {
|
||||
// Serialize snapshots across API processes; timestamps reject late HTTP results.
|
||||
await tx.$executeRaw`SELECT pg_advisory_xact_lock(160916, 1)`;
|
||||
const previous = await tx.infrastructureAlertCollection.findUnique({ where: { id: 'prometheus' } });
|
||||
if (previous && previous.observedAt >= observedAt) return;
|
||||
// Read durable work directly: application releases do not install Prometheus rules.
|
||||
// Keep one occurrence identity until the condition really recovers, even after manual clear.
|
||||
const reviewCount = await tx.smsAttemptCompletionWork.count({ where: { state: 'needs_review' } });
|
||||
const oldest = await tx.smsCompletionEvent.findFirst({
|
||||
where: { processedAt: null, work: { state: { in: ['pending', 'processing', 'retry_wait'] } } },
|
||||
orderBy: { createdAt: 'asc' },
|
||||
select: { createdAt: true },
|
||||
});
|
||||
const age = oldest ? Math.max(0, (observedAt.getTime() - oldest.createdAt.getTime()) / 1000) : 0;
|
||||
alerts = [...alerts];
|
||||
for (const condition of [
|
||||
{
|
||||
name: 'SmsCompletionNeedsReview',
|
||||
active: reviewCount > 0,
|
||||
severity: 'critical' as const,
|
||||
summary: '短信收尾工作需要人工排查',
|
||||
value: String(reviewCount),
|
||||
threshold: '0',
|
||||
},
|
||||
{
|
||||
name: 'SmsCompletionBacklog',
|
||||
active: age > 300,
|
||||
severity: 'warning' as const,
|
||||
summary: '短信收尾工作等待超过5分钟',
|
||||
value: `${Math.floor(age)}秒`,
|
||||
threshold: '300秒',
|
||||
},
|
||||
]) {
|
||||
if (!condition.active) continue;
|
||||
const fingerprint = createHash('sha256').update(`durable:${condition.name}`).digest('hex').slice(0, 24);
|
||||
const occurrence = await tx.infrastructureAlertEvent.findFirst({
|
||||
where: { fingerprint, recoveredAt: null },
|
||||
orderBy: { activeAt: 'desc' },
|
||||
});
|
||||
alerts.push({
|
||||
fingerprint,
|
||||
name: condition.name,
|
||||
severity: condition.severity,
|
||||
status: 'firing',
|
||||
startedAt: (occurrence?.activeAt ?? observedAt).toISOString(),
|
||||
summary: condition.summary,
|
||||
currentValue: condition.value,
|
||||
threshold: condition.threshold,
|
||||
service: '短信收尾',
|
||||
acknowledged: false,
|
||||
});
|
||||
}
|
||||
await tx.infrastructureAlertCollection.upsert({
|
||||
where: { id: 'prometheus' },
|
||||
create: { id: 'prometheus', observedAt },
|
||||
update: { observedAt },
|
||||
});
|
||||
for (const alert of alerts) {
|
||||
const activeAt = new Date(alert.startedAt);
|
||||
const payload = JSON.parse(JSON.stringify(alert)) as Prisma.InputJsonValue;
|
||||
await tx.infrastructureAlertEvent.createMany({
|
||||
data: [{ fingerprint: alert.fingerprint, activeAt, payload, lastObservedAt: observedAt }],
|
||||
skipDuplicates: true,
|
||||
});
|
||||
await tx.infrastructureAlertEvent.updateMany({
|
||||
where: { fingerprint: alert.fingerprint, activeAt, lastObservedAt: { lte: observedAt } },
|
||||
data: { payload, lastObservedAt: observedAt, recoveredAt: null },
|
||||
});
|
||||
}
|
||||
await tx.infrastructureAlertEvent.updateMany({
|
||||
where: {
|
||||
recoveredAt: null,
|
||||
lastObservedAt: { lt: observedAt },
|
||||
...(alerts.length
|
||||
? {
|
||||
NOT: {
|
||||
OR: alerts.map((alert) => ({ fingerprint: alert.fingerprint, activeAt: new Date(alert.startedAt) })),
|
||||
},
|
||||
}
|
||||
: {}),
|
||||
},
|
||||
data: { recoveredAt: observedAt },
|
||||
});
|
||||
});
|
||||
return retainedAlerts(prisma);
|
||||
}
|
||||
|
||||
export async function retainedAlerts(prisma: PrismaService): Promise<InfrastructureAlert[]> {
|
||||
const records = await prisma.infrastructureAlertEvent.findMany({
|
||||
where: { clearedAt: null },
|
||||
orderBy: [{ activeAt: 'desc' }, { id: 'asc' }],
|
||||
});
|
||||
return records.map((record) => ({
|
||||
...(record.payload as unknown as InfrastructureAlert),
|
||||
...(record.recoveredAt ? { status: 'resolved' } : {}),
|
||||
acknowledged: false,
|
||||
}));
|
||||
}
|
||||
|
||||
export async function clearRetainedAlert(
|
||||
prisma: PrismaService,
|
||||
fingerprint: string,
|
||||
rawActiveAt: unknown,
|
||||
userId: string,
|
||||
) {
|
||||
const activeAt = new Date(String(rawActiveAt ?? ''));
|
||||
if (!/^[a-f0-9]{24}$/.test(fingerprint) || !Number.isFinite(activeAt.getTime()))
|
||||
throw new BadRequestException('告警标识无效');
|
||||
return prisma.$transaction(async (tx) => {
|
||||
const record = await tx.infrastructureAlertEvent.findUnique({
|
||||
where: { fingerprint_activeAt: { fingerprint, activeAt } },
|
||||
});
|
||||
if (!record) throw new NotFoundException('告警记录不存在');
|
||||
const clearedAt = new Date();
|
||||
const result = await tx.infrastructureAlertEvent.updateMany({
|
||||
where: { id: record.id, clearedAt: null },
|
||||
data: { clearedAt, clearedBy: userId },
|
||||
});
|
||||
if (result.count)
|
||||
await tx.operationLog.create({
|
||||
data: {
|
||||
userId,
|
||||
action: 'monitoring.alert_cleared',
|
||||
resource: 'infrastructure_alert',
|
||||
resourceId: record.id,
|
||||
detail: { fingerprint, activeAt: activeAt.toISOString() },
|
||||
},
|
||||
});
|
||||
return {
|
||||
fingerprint,
|
||||
activeAt: activeAt.toISOString(),
|
||||
cleared: true,
|
||||
clearedAt: (record.clearedAt ?? clearedAt).toISOString(),
|
||||
};
|
||||
});
|
||||
}
|
||||
+3
-3
@@ -26,8 +26,8 @@ async function bootstrap() {
|
||||
configureHttpBodyParsers(app);
|
||||
|
||||
const swaggerConfig = new DocumentBuilder()
|
||||
.setTitle('CMPP Platform API')
|
||||
.setDescription('First-version CMPP SMS platform API')
|
||||
.setTitle('聆界短信平台 API')
|
||||
.setDescription('聆界短信平台 API')
|
||||
.setVersion('0.1.0')
|
||||
.build();
|
||||
const document = SwaggerModule.createDocument(app, swaggerConfig);
|
||||
@@ -36,7 +36,7 @@ async function bootstrap() {
|
||||
const clientDocument = SwaggerModule.createDocument(
|
||||
app,
|
||||
new DocumentBuilder()
|
||||
.setTitle('CMPP短信平台 HTTP 客户接口')
|
||||
.setTitle('聆界短信平台 HTTP 客户接口')
|
||||
.setDescription('单条短信发送、短信状态查询、上行短信查询及回调验签接口')
|
||||
.setVersion('1.0.0')
|
||||
.build(),
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { renderCompletionMetrics } from '../send-chain/completion-metrics';
|
||||
import { Injectable, OnModuleDestroy } from '@nestjs/common';
|
||||
import { monitorEventLoopDelay } from 'node:perf_hooks';
|
||||
|
||||
@@ -314,6 +315,7 @@ export class MetricsService implements OnModuleDestroy {
|
||||
lines.push(metricLine('cmpp_api_auth_protection_events_total', count, { event, scope }));
|
||||
}
|
||||
this.eventLoopDelay.reset();
|
||||
lines.push(...renderCompletionMetrics());
|
||||
return `${lines.join('\n')}\n`;
|
||||
}
|
||||
|
||||
|
||||
@@ -63,5 +63,5 @@ export function renderHttpGuide(markdown: string, origin: string) {
|
||||
if (line.trim() && !/^---+$/.test(line)) current.body.push(`<p>${inline(line.replace(/^>\s?/, '').replace(/^- /, '• '))}</p>`);
|
||||
}
|
||||
closeTable();
|
||||
return `<!doctype html><html lang="zh-CN"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1"><title>聆界短信 · HTTP 接入文档</title><style>${asset('reader.css')}</style></head><body class="http-developer-docs"><header class="http-doc-header"><div><strong>聆界短信 · 开发者文档</strong><h1>HTTP 接口接入文档</h1><p>${escapeHtml(httpDocVersion(markdown))} · 基础地址 ${escapeHtml(origin || '当前环境')}/api/openapi/v1</p></div><div class="http-doc-actions"><a href="/api/client-docs?format=md" download="client-http-api-guide.md">下载 MD</a><a href="/api/client-docs-json" target="_blank" rel="noreferrer">OpenAPI JSON</a></div></header><div class="http-doc-layout"><nav aria-label="文档目录"><details open><summary>目录</summary>${sections.map((section) => `<a href="#${section.id}">${inline(section.title)}</a>`).join('')}</details><label for="doc-search">错误码 / 文档检索</label><input id="doc-search" type="search" placeholder="输入错误码或关键词"><p id="search-status" role="status"></p></nav><main>${sections.map((section) => `<section id="${section.id}" data-doc-section><div class="http-doc-body"><h2>${inline(section.title)}</h2>${section.body.join('')}</div></section>`).join('')}<p id="no-results" hidden>没有匹配的文档内容,请更换关键词。</p></main></div><p class="http-doc-copy-status" role="status" id="copy-status"></p><script>${asset('reader.js')}</script></body></html>`;
|
||||
return `<!doctype html><html lang="zh-CN"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1"><title>聆界短信平台 · HTTP 接入文档</title><style>${asset('reader.css')}</style></head><body class="http-developer-docs"><header class="http-doc-header"><div><strong>聆界短信平台 · 开发者文档</strong><h1>HTTP 接口接入文档</h1><p>${escapeHtml(httpDocVersion(markdown))} · 基础地址 ${escapeHtml(origin || '当前环境')}/api/openapi/v1</p></div><div class="http-doc-actions"><a href="/api/client-docs?format=md" download="client-http-api-guide.md">下载 MD</a><a href="/api/client-docs-json" target="_blank" rel="noreferrer">OpenAPI JSON</a></div></header><div class="http-doc-layout"><nav aria-label="文档目录"><details open><summary>目录</summary>${sections.map((section) => `<a href="#${section.id}">${inline(section.title)}</a>`).join('')}</details><label for="doc-search">错误码 / 文档检索</label><input id="doc-search" type="search" placeholder="输入错误码或关键词"><p id="search-status" role="status"></p></nav><main>${sections.map((section) => `<section id="${section.id}" data-doc-section><div class="http-doc-body"><h2>${inline(section.title)}</h2>${section.body.join('')}</div></section>`).join('')}<p id="no-results" hidden>没有匹配的文档内容,请更换关键词。</p></main></div><p class="http-doc-copy-status" role="status" id="copy-status"></p><script>${asset('reader.js')}</script></body></html>`;
|
||||
}
|
||||
|
||||
@@ -526,7 +526,8 @@ export class OpenApiService implements OnModuleInit, OnModuleDestroy {
|
||||
return row;
|
||||
}
|
||||
|
||||
async queueWebhookEvent(data: {
|
||||
async queueWebhookEvent(
|
||||
data: {
|
||||
tenantId: string;
|
||||
applicationId?: string | null;
|
||||
messageRecordId?: string | null;
|
||||
@@ -534,16 +535,19 @@ export class OpenApiService implements OnModuleInit, OnModuleDestroy {
|
||||
uplinkMessageId?: string | null;
|
||||
eventType: 'receipt' | 'uplink';
|
||||
payload: Record<string, unknown>;
|
||||
}) {
|
||||
},
|
||||
transaction?: Prisma.TransactionClient,
|
||||
) {
|
||||
const db = transaction ?? this.prisma;
|
||||
if (!data.applicationId) return null;
|
||||
const application = await this.prisma.smsApplication.findUnique({
|
||||
const application = await db.smsApplication.findUnique({
|
||||
where: { id: data.applicationId },
|
||||
include: { httpConfig: true },
|
||||
});
|
||||
const config = application?.httpConfig;
|
||||
const enabled = data.eventType === 'receipt' ? config?.receiptWebhookEnabled : config?.uplinkWebhookEnabled;
|
||||
if (!config?.enabled || !enabled) return null;
|
||||
const endpoint = await this.prisma.httpWebhookEndpoint.findUnique({
|
||||
const endpoint = await db.httpWebhookEndpoint.findUnique({
|
||||
where: { applicationId_eventType: { applicationId: data.applicationId, eventType: data.eventType } },
|
||||
});
|
||||
if (!endpoint || endpoint.status !== 'active') return null;
|
||||
@@ -553,7 +557,7 @@ export class OpenApiService implements OnModuleInit, OnModuleDestroy {
|
||||
: data.eventType === 'uplink' && data.uplinkMessageId
|
||||
? `evt_uplink_${data.uplinkMessageId}`
|
||||
: `evt_${randomUUID()}`;
|
||||
const delivery = await this.prisma.$transaction(async (tx) => {
|
||||
const persist = async (tx: Prisma.TransactionClient) => {
|
||||
const event = await tx.httpWebhookEvent.upsert({
|
||||
where: { eventId },
|
||||
update: {},
|
||||
@@ -573,7 +577,9 @@ export class OpenApiService implements OnModuleInit, OnModuleDestroy {
|
||||
update: {},
|
||||
create: { eventId: event.id, endpointId: endpoint.id, recoveryVersion: 1 },
|
||||
});
|
||||
});
|
||||
};
|
||||
const delivery = transaction ? await persist(transaction) : await this.prisma.$transaction(persist);
|
||||
if (transaction) return delivery;
|
||||
if (delivery.status !== 'pending' || delivery.recoveryVersion !== 1) return delivery;
|
||||
await this.queue?.add(
|
||||
'deliver',
|
||||
|
||||
@@ -313,7 +313,7 @@ export class ReportChannelExportService {
|
||||
missingFields: detail.missingFields,
|
||||
});
|
||||
const workbook = new ExcelJS.Workbook();
|
||||
workbook.creator = 'CMPP短信平台';
|
||||
workbook.creator = '聆界短信平台';
|
||||
const sheet = workbook.addWorksheet('签名报备', { views: [{ state: 'frozen', ySplit: 1 }] });
|
||||
sheet.columns = detail.fields.map((field) => ({
|
||||
header: field.exportName || field.name,
|
||||
|
||||
@@ -1,58 +1,85 @@
|
||||
import { BadRequestException, ConflictException, Injectable, NotFoundException } from '@nestjs/common';
|
||||
import { Prisma } from '@prisma/client';
|
||||
import ExcelJS from 'exceljs';
|
||||
import { createHash, randomUUID } from 'node:crypto';
|
||||
import { extname } from 'node:path';
|
||||
import { FilesService } from '../files/files.service';
|
||||
import { PrismaService } from '../prisma/prisma.service';
|
||||
import { SmsConfigService } from '../sms-config/sms-config.service';
|
||||
import type { AnalyzeImportOptions, CreateImportProfileDto, CreateReportBatchDto, EmbeddedImage, ImportCommitDto, ImportMapping, PagedQuery, ReportBatchInspection, ReportBatchTarget, ReviewImportItemsDto } from './report-materials.contracts';
|
||||
import { profileData, validateProfile, loadWorkbook, assertSafeWorkbook, safeSpreadsheetText, readEmbeddedImages, suggestMappings, remapProfileColumns, signatureCoreMapping, drainageCoreMapping, normalizeHeader, normalizeFieldCode, clamp, normalizePage, normalizePageSize, dateRange, cellText, transformValue, mappedCoreValue, dynamicValues, jsonRecord, hasValue, isFileRef, resolveExportValue, applyExportTransform, styleHeader, normalizeImageExtension, imageContentType, safeFileName, normalizeBatchIdempotencyKey, jsonStringArray, jsonSafe } from './report-materials.helpers';
|
||||
import { ReportPendingQueryService } from './pending-query.service';
|
||||
import { safeSpreadsheetText, styleHeader } from './report-materials.helpers';
|
||||
|
||||
/** R4 report-materials domain service composed behind ReportMaterialsService. */
|
||||
export class ReportOfficialExportService {
|
||||
constructor(private readonly prisma: PrismaService, private readonly files: FilesService, private readonly smsConfig: SmsConfigService, private readonly pending: ReportPendingQueryService) {}
|
||||
constructor(
|
||||
private readonly prisma: PrismaService,
|
||||
private readonly files: FilesService,
|
||||
private readonly smsConfig: SmsConfigService,
|
||||
private readonly pending: ReportPendingQueryService,
|
||||
) {}
|
||||
|
||||
async buildOfficialTemplate(reportType: 'signature' | 'drainage', operatorId?: string) {
|
||||
const workbook = new ExcelJS.Workbook();
|
||||
workbook.creator = 'CMPP短信平台';
|
||||
const sheet = workbook.addWorksheet(reportType === 'signature' ? '签名资料' : '引流信息', { views: [{ state: 'frozen', ySplit: 1 }] });
|
||||
const headers = reportType === 'signature'
|
||||
workbook.creator = '聆界短信平台';
|
||||
const sheet = workbook.addWorksheet(reportType === 'signature' ? '签名资料' : '引流信息', {
|
||||
views: [{ state: 'frozen', ySplit: 1 }],
|
||||
});
|
||||
const headers =
|
||||
reportType === 'signature'
|
||||
? ['短信签名', '用途说明', '营业执照图片', '授权书图片', '备注']
|
||||
: ['短信签名', '引流 URL 或号码', '备注', '主体证明'];
|
||||
sheet.addRow(headers);
|
||||
sheet.addRow(reportType === 'signature'
|
||||
sheet.addRow(
|
||||
reportType === 'signature'
|
||||
? ['示例签名', '验证码通知', '请在本单元格插入图片', '请在本单元格插入图片', '示例行,导入前请删除']
|
||||
: ['示例签名', 'example.com/path 或 13800138000', '示例行,导入前请删除', '请在本单元格插入图片']);
|
||||
: ['示例签名', 'example.com/path 或 13800138000', '示例行,导入前请删除', '请在本单元格插入图片'],
|
||||
);
|
||||
styleHeader(sheet.getRow(1));
|
||||
sheet.columns.forEach((column) => { column.width = 24; });
|
||||
sheet.columns.forEach((column) => {
|
||||
column.width = 24;
|
||||
});
|
||||
sheet.getRow(2).height = 48;
|
||||
const content = Buffer.from(await workbook.xlsx.writeBuffer());
|
||||
const fileName = `${reportType === 'signature' ? '签名' : '引流信息'}报备资料官方模板.xlsx`;
|
||||
await this.prisma.operationLog.create({ data: {
|
||||
userId: operatorId, action: 'report_material.template_downloaded', resource: 'report_material',
|
||||
await this.prisma.operationLog.create({
|
||||
data: {
|
||||
userId: operatorId,
|
||||
action: 'report_material.template_downloaded',
|
||||
resource: 'report_material',
|
||||
detail: { fileName, filters: { reportType }, successCount: 1, failedCount: 0 } as Prisma.InputJsonValue,
|
||||
} });
|
||||
},
|
||||
});
|
||||
return { fileName, content };
|
||||
}
|
||||
|
||||
async exportPending(query: { reportType?: 'signature' | 'drainage'; tenantId?: string; applicationId?: string }, operatorId?: string) {
|
||||
async exportPending(
|
||||
query: { reportType?: 'signature' | 'drainage'; tenantId?: string; applicationId?: string },
|
||||
operatorId?: string,
|
||||
) {
|
||||
const items = await this.pending.findPendingItems(query);
|
||||
const workbook = new ExcelJS.Workbook();
|
||||
const sheet = workbook.addWorksheet('待报备资料', { views: [{ state: 'frozen', ySplit: 1 }] });
|
||||
sheet.addRow(['资料类型', '企业', '企业应用', '签名/站点', '详情', '变更时间']);
|
||||
styleHeader(sheet.getRow(1));
|
||||
for (const item of items) sheet.addRow([
|
||||
item.reportType === 'signature' ? '签名' : '引流信息', safeSpreadsheetText(item.tenant?.name),
|
||||
safeSpreadsheetText(item.application?.name), safeSpreadsheetText(item.name), safeSpreadsheetText(item.detail), item.changedAt,
|
||||
for (const item of items)
|
||||
sheet.addRow([
|
||||
item.reportType === 'signature' ? '签名' : '引流信息',
|
||||
safeSpreadsheetText(item.tenant?.name),
|
||||
safeSpreadsheetText(item.application?.name),
|
||||
safeSpreadsheetText(item.name),
|
||||
safeSpreadsheetText(item.detail),
|
||||
item.changedAt,
|
||||
]);
|
||||
sheet.columns.forEach((column, index) => { column.width = index === 4 ? 42 : 22; });
|
||||
sheet.columns.forEach((column, index) => {
|
||||
column.width = index === 4 ? 42 : 22;
|
||||
});
|
||||
const fileName = `待报备资料-${new Date().toISOString().slice(0, 10)}.xlsx`;
|
||||
await this.prisma.operationLog.create({ data: {
|
||||
tenantId: query.tenantId, userId: operatorId, action: 'report_material.pending_export', resource: 'report_material',
|
||||
await this.prisma.operationLog.create({
|
||||
data: {
|
||||
tenantId: query.tenantId,
|
||||
userId: operatorId,
|
||||
action: 'report_material.pending_export',
|
||||
resource: 'report_material',
|
||||
detail: { fileName, filters: query, successCount: items.length, failedCount: 0 } as Prisma.InputJsonValue,
|
||||
} });
|
||||
},
|
||||
});
|
||||
return { fileName, content: Buffer.from(await workbook.xlsx.writeBuffer()) };
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,219 @@
|
||||
import { HttpException, Logger } from '@nestjs/common';
|
||||
import { Prisma, SmsAttemptCompletionWork } from '@prisma/client';
|
||||
import { createHash, randomUUID } from 'node:crypto';
|
||||
import { PrismaService } from '../prisma/prisma.service';
|
||||
import { completionContext, CompletionRouteRequired } from './completion-context';
|
||||
import type { RoutedChannel } from './send-chain.contracts';
|
||||
import { countCompletion, observeCompletion } from './completion-metrics';
|
||||
|
||||
export type CompletionEventKind = 'receipt' | 'submit' | 'segment' | 'timeout' | 'rejection';
|
||||
export class AttemptCompletion {
|
||||
private readonly logger = new Logger(AttemptCompletion.name);
|
||||
private timer?: ReturnType<typeof setInterval>;
|
||||
private running = false;
|
||||
constructor(
|
||||
private readonly prisma: PrismaService,
|
||||
private readonly execute: (kind: CompletionEventKind, payload: Prisma.JsonValue) => Promise<unknown>,
|
||||
private readonly waitForRoute: (route: RoutedChannel) => Promise<void>,
|
||||
) {}
|
||||
|
||||
start() {
|
||||
this.timer = setInterval(() => void this.scan(), 5_000);
|
||||
this.timer.unref();
|
||||
void this.scan();
|
||||
}
|
||||
stop() {
|
||||
if (this.timer) clearInterval(this.timer);
|
||||
}
|
||||
|
||||
async enqueue(
|
||||
messageRecordId: string,
|
||||
sourceSubmitRecordId: string | undefined,
|
||||
kind: CompletionEventKind,
|
||||
payload: unknown,
|
||||
identity?: string,
|
||||
) {
|
||||
const json = JSON.parse(JSON.stringify(payload)) as Prisma.InputJsonValue;
|
||||
const workKey = sourceSubmitRecordId ? `attempt:${sourceSubmitRecordId}` : `message:${messageRecordId}`;
|
||||
const eventKey = createHash('sha256')
|
||||
.update(`${workKey}:${kind}:${identity ?? JSON.stringify(json)}`)
|
||||
.digest('hex');
|
||||
const work = await this.prisma.$transaction(async (tx) => {
|
||||
const message = await tx.smsMessageRecord.findUniqueOrThrow({
|
||||
where: { id: messageRecordId },
|
||||
select: { tenantId: true },
|
||||
});
|
||||
if (sourceSubmitRecordId) {
|
||||
const source = await tx.smsSubmitRecord.findUniqueOrThrow({ where: { id: sourceSubmitRecordId } });
|
||||
if (source.messageRecordId !== messageRecordId || source.tenantId !== message.tenantId)
|
||||
throw new Error('completion_source_mismatch');
|
||||
}
|
||||
await tx.smsAttemptCompletionWork.createMany({
|
||||
data: [{ workKey, messageRecordId, sourceSubmitRecordId, tenantId: message.tenantId }],
|
||||
skipDuplicates: true,
|
||||
});
|
||||
const current = await tx.smsAttemptCompletionWork.findUniqueOrThrow({ where: { workKey } });
|
||||
if (current.messageRecordId !== messageRecordId) throw new Error('completion_work_mismatch');
|
||||
await tx.$queryRaw`SELECT id FROM "SmsAttemptCompletionWork" WHERE id=${current.id} FOR UPDATE`;
|
||||
const inserted = await tx.smsCompletionEvent.createMany({
|
||||
data: [{ workId: current.id, eventKey, kind, payload: json }],
|
||||
skipDuplicates: true,
|
||||
});
|
||||
if (inserted.count)
|
||||
await tx.$executeRaw`
|
||||
UPDATE "SmsAttemptCompletionWork" SET revision=revision+1,
|
||||
state=CASE WHEN state IN ('processing', 'needs_review') THEN state ELSE 'pending' END,
|
||||
"nextAttemptAt"=(CURRENT_TIMESTAMP AT TIME ZONE 'UTC'), "updatedAt"=(CURRENT_TIMESTAMP AT TIME ZONE 'UTC') WHERE id=${current.id}`;
|
||||
return current;
|
||||
});
|
||||
await this.process(work.id);
|
||||
return this.prisma.smsMessageRecord.findUnique({ where: { id: messageRecordId } });
|
||||
}
|
||||
|
||||
async scan() {
|
||||
if (this.running) return;
|
||||
this.running = true;
|
||||
try {
|
||||
observeCompletion(
|
||||
await this.prisma.$queryRaw<Array<{ state: string; count: number; age: number }>>`
|
||||
SELECT w.state, COUNT(*)::int AS count,
|
||||
EXTRACT(EPOCH FROM (CURRENT_TIMESTAMP AT TIME ZONE 'UTC') - MIN(COALESCE(e."createdAt", w."updatedAt")))::float AS age
|
||||
FROM "SmsAttemptCompletionWork" w
|
||||
LEFT JOIN LATERAL (SELECT MIN("createdAt") AS "createdAt" FROM "SmsCompletionEvent" WHERE "workId"=w.id AND "processedAt" IS NULL) e ON true
|
||||
WHERE w.state IN ('pending','processing','retry_wait','needs_review') GROUP BY w.state`,
|
||||
);
|
||||
const rows = await this.prisma.$queryRaw<Array<{ id: string }>>`
|
||||
SELECT id FROM "SmsAttemptCompletionWork"
|
||||
WHERE (state IN ('pending','retry_wait') AND "nextAttemptAt" <= (CURRENT_TIMESTAMP AT TIME ZONE 'UTC'))
|
||||
OR (state='processing' AND "leaseUntil" < (CURRENT_TIMESTAMP AT TIME ZONE 'UTC'))
|
||||
ORDER BY "nextAttemptAt", id LIMIT 32`;
|
||||
for (const row of rows) await this.process(row.id);
|
||||
} catch {
|
||||
this.logger.error('completion_scan_failed');
|
||||
} finally {
|
||||
this.running = false;
|
||||
}
|
||||
}
|
||||
|
||||
async process(id: string) {
|
||||
const owner = randomUUID();
|
||||
const claims = await this.prisma.$queryRaw<SmsAttemptCompletionWork[]>`
|
||||
UPDATE "SmsAttemptCompletionWork" SET state='processing', "leaseOwner"=${owner},
|
||||
"leaseUntil"=(CURRENT_TIMESTAMP AT TIME ZONE 'UTC')+INTERVAL '60 seconds', "fenceVersion"="fenceVersion"+1,
|
||||
attempts=attempts+1, "updatedAt"=(CURRENT_TIMESTAMP AT TIME ZONE 'UTC')
|
||||
WHERE id=${id} AND ((state IN ('pending','retry_wait') AND "nextAttemptAt" <= (CURRENT_TIMESTAMP AT TIME ZONE 'UTC'))
|
||||
OR (state='processing' AND "leaseUntil" < (CURRENT_TIMESTAMP AT TIME ZONE 'UTC'))) RETURNING *`;
|
||||
const claim = claims[0];
|
||||
if (!claim) {
|
||||
countCompletion('not_claimed');
|
||||
return;
|
||||
}
|
||||
countCompletion('claimed');
|
||||
if (claim.attempts > 1) countCompletion('recovered');
|
||||
const renew = setInterval(() => {
|
||||
void this.prisma
|
||||
.$executeRaw`UPDATE "SmsAttemptCompletionWork" SET "leaseUntil"=(CURRENT_TIMESTAMP AT TIME ZONE 'UTC')+INTERVAL '60 seconds'
|
||||
WHERE id=${id} AND state='processing' AND "leaseOwner"=${owner} AND "fenceVersion"=${claim.fenceVersion}
|
||||
AND "leaseUntil">(CURRENT_TIMESTAMP AT TIME ZONE 'UTC')`.catch(() =>
|
||||
this.logger.warn('completion_lease_renew_failed'),
|
||||
);
|
||||
}, 20_000);
|
||||
renew.unref();
|
||||
let route: RoutedChannel | undefined;
|
||||
let routePlanned = false;
|
||||
let planRevision: number | undefined;
|
||||
try {
|
||||
for (let pass = 0; pass < 64; pass++) {
|
||||
try {
|
||||
const done = await this.prisma.$transaction(
|
||||
async (tx) => {
|
||||
const rows = await tx.$queryRaw<SmsAttemptCompletionWork[]>`
|
||||
SELECT * FROM "SmsAttemptCompletionWork" WHERE id=${id} AND state='processing'
|
||||
AND "leaseOwner"=${owner} AND "fenceVersion"=${claim.fenceVersion} AND "leaseUntil">(CURRENT_TIMESTAMP AT TIME ZONE 'UTC') FOR UPDATE`;
|
||||
const work = rows[0];
|
||||
if (!work) {
|
||||
countCompletion('fence_rejected');
|
||||
throw new Error('completion_fence_rejected');
|
||||
}
|
||||
await tx.$queryRaw`SELECT id FROM "SmsMessageRecord" WHERE id=${work.messageRecordId} FOR UPDATE`;
|
||||
if (planRevision !== work.revision) {
|
||||
route = undefined;
|
||||
routePlanned = false;
|
||||
}
|
||||
planRevision = work.revision;
|
||||
const event = await tx.smsCompletionEvent.findFirst({
|
||||
where: { workId: id, processedAt: null },
|
||||
orderBy: [{ createdAt: 'asc' }, { id: 'asc' }],
|
||||
});
|
||||
if (event) {
|
||||
await completionContext.run({ tx, messageRecordId: work.messageRecordId, route, routePlanned }, () =>
|
||||
this.execute(event.kind as CompletionEventKind, event.payload),
|
||||
);
|
||||
await tx.smsCompletionEvent.update({ where: { id: event.id }, data: { processedAt: new Date() } });
|
||||
}
|
||||
const remaining = await tx.smsCompletionEvent.count({ where: { workId: id, processedAt: null } });
|
||||
const message = await tx.smsMessageRecord.findUniqueOrThrow({ where: { id: work.messageRecordId } });
|
||||
const retry = work.sourceSubmitRecordId
|
||||
? await tx.smsSubmitRecord.findUnique({
|
||||
where: { retryOfSubmitRecordId: work.sourceSubmitRecordId },
|
||||
select: { id: true },
|
||||
})
|
||||
: null;
|
||||
await tx.smsAttemptCompletionWork.update({
|
||||
where: { id },
|
||||
data: {
|
||||
processedRevision: work.revision - remaining,
|
||||
decision: message.status,
|
||||
retrySubmitRecordId: retry?.id,
|
||||
state: remaining ? 'processing' : 'idle',
|
||||
lastError: null,
|
||||
...(!remaining ? { leaseOwner: null, leaseUntil: null, attempts: 0 } : {}),
|
||||
},
|
||||
});
|
||||
return remaining === 0;
|
||||
},
|
||||
{ timeout: 20_000, maxWait: 5_000 },
|
||||
);
|
||||
route = undefined;
|
||||
routePlanned = false;
|
||||
countCompletion('event_committed');
|
||||
if (done) return;
|
||||
} catch (error) {
|
||||
if (!(error instanceof CompletionRouteRequired)) throw error;
|
||||
// The probe transaction rolls back. Routing and rate limiting occur without locks.
|
||||
try {
|
||||
route = await error.select();
|
||||
await this.waitForRoute(route);
|
||||
} catch (selectionError) {
|
||||
if (!(selectionError instanceof HttpException) || selectionError.getStatus() >= 500) throw selectionError;
|
||||
route = undefined;
|
||||
}
|
||||
routePlanned = true;
|
||||
}
|
||||
}
|
||||
throw new Error('completion_batch_budget_exhausted');
|
||||
} catch (error) {
|
||||
const code =
|
||||
error instanceof Prisma.PrismaClientKnownRequestError
|
||||
? error.code
|
||||
: error instanceof Error && error.message.startsWith('completion_')
|
||||
? error.message
|
||||
: 'completion_processing_failed';
|
||||
const exhausted = claim.attempts >= 12;
|
||||
countCompletion(exhausted ? 'needs_review' : 'retry_wait');
|
||||
await this.prisma.smsAttemptCompletionWork.updateMany({
|
||||
where: { id, leaseOwner: owner, fenceVersion: claim.fenceVersion, state: 'processing' },
|
||||
data: {
|
||||
state: exhausted ? 'needs_review' : 'retry_wait',
|
||||
leaseOwner: null,
|
||||
leaseUntil: null,
|
||||
lastError: code,
|
||||
nextAttemptAt: new Date(Date.now() + Math.min(300_000, 1000 * 2 ** Math.min(claim.attempts, 8))),
|
||||
},
|
||||
});
|
||||
this.logger.error(`${exhausted ? 'completion_needs_review' : 'completion_retry_wait'}:${code}`);
|
||||
} finally {
|
||||
clearInterval(renew);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
import { AsyncLocalStorage } from 'node:async_hooks';
|
||||
import { Prisma } from '@prisma/client';
|
||||
import { PrismaService } from '../prisma/prisma.service';
|
||||
import type { RoutedChannel } from './send-chain.contracts';
|
||||
|
||||
export type CompletionContext = {
|
||||
tx: Prisma.TransactionClient;
|
||||
messageRecordId: string;
|
||||
route?: RoutedChannel;
|
||||
routePlanned?: boolean;
|
||||
};
|
||||
export const completionContext = new AsyncLocalStorage<CompletionContext>();
|
||||
|
||||
// Only send-chain collaborators use this adapter. Nested billing/outbox transactions
|
||||
// join the explicitly established completion transaction, never start a second one.
|
||||
export function completionDatabase(prisma: PrismaService): PrismaService {
|
||||
return new Proxy(prisma, {
|
||||
get(target, key) {
|
||||
const tx = completionContext.getStore()?.tx;
|
||||
if (tx && key === '$transaction') {
|
||||
return (operation: ((client: Prisma.TransactionClient) => unknown) | Promise<unknown>[]) =>
|
||||
typeof operation === 'function' ? operation(tx) : Promise.all(operation);
|
||||
}
|
||||
const owner = tx && key in tx ? tx : target;
|
||||
const value = Reflect.get(owner, key);
|
||||
return typeof value === 'function' ? value.bind(owner) : value;
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
export class CompletionRouteRequired extends Error {
|
||||
constructor(readonly select: () => Promise<RoutedChannel>) {
|
||||
super('completion_route_required');
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
type CompletionMetric =
|
||||
'claimed' | 'not_claimed' | 'recovered' | 'fence_rejected' | 'event_committed' | 'retry_wait' | 'needs_review';
|
||||
const counts = new Map<CompletionMetric, number>();
|
||||
let snapshot: Array<{ state: string; count: number; age: number }> = [];
|
||||
const states = ['pending', 'processing', 'retry_wait', 'needs_review'];
|
||||
export function countCompletion(event: CompletionMetric) {
|
||||
counts.set(event, (counts.get(event) ?? 0) + 1);
|
||||
}
|
||||
export function observeCompletion(rows: typeof snapshot) {
|
||||
snapshot = rows;
|
||||
}
|
||||
export function renderCompletionMetrics() {
|
||||
return [
|
||||
'# TYPE cmpp_completion_events_total counter',
|
||||
...[...counts].map(([event, value]) => `cmpp_completion_events_total{event="${event}"} ${value}`),
|
||||
'# TYPE cmpp_completion_work gauge',
|
||||
'# TYPE cmpp_completion_oldest_seconds gauge',
|
||||
...states.flatMap((state) => {
|
||||
const row = snapshot.find((item) => item.state === state);
|
||||
return [
|
||||
`cmpp_completion_work{state="${state}"} ${Number(row?.count ?? 0)}`,
|
||||
`cmpp_completion_oldest_seconds{state="${state}"} ${Math.max(0, Number(row?.age ?? 0))}`,
|
||||
];
|
||||
}),
|
||||
];
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
import { createHash } from 'node:crypto';
|
||||
import { DrainageSubmitGuardController } from './drainage-submit-guard.controller';
|
||||
|
||||
describe('channel test final guard', () => {
|
||||
const content = '无签名正文 https://example.test 4001234567'.repeat(5);
|
||||
function setup(overrides = {}) {
|
||||
const tx = {
|
||||
smsSubmitRecord: {
|
||||
findUnique: jest.fn().mockResolvedValue({
|
||||
channelId: 'channel',
|
||||
submitId: 'submit',
|
||||
resultProcessedAt: null,
|
||||
messageRecord: { content, tenantId: null, batchTaskId: null, status: 'submit_queued', ...overrides },
|
||||
}),
|
||||
},
|
||||
drainageDetectionRule: {
|
||||
findMany: jest.fn(() => {
|
||||
throw new Error('must not detect channel tests');
|
||||
}),
|
||||
},
|
||||
};
|
||||
const db = { $transaction: (operation: (value: unknown) => unknown) => operation(tx) };
|
||||
return { controller: new DrainageSubmitGuardController(db as never), tx };
|
||||
}
|
||||
const local = { socket: { remoteAddress: '127.0.0.1' } };
|
||||
const input = {
|
||||
submitId: 'submit',
|
||||
channelId: 'channel',
|
||||
contentHash: createHash('sha256').update(content).digest('hex'),
|
||||
};
|
||||
it('allows unsigned long channel tests containing drainage information without detecting content', async () => {
|
||||
const { controller, tx } = setup();
|
||||
await expect(controller.authorize(local, input)).resolves.toEqual({ allowed: true });
|
||||
expect(tx.drainageDetectionRule.findMany).not.toHaveBeenCalled();
|
||||
});
|
||||
it('still rejects a modified body or a finished submit', async () => {
|
||||
await expect(setup().controller.authorize(local, { ...input, contentHash: 'a'.repeat(64) })).resolves.toMatchObject(
|
||||
{ allowed: false },
|
||||
);
|
||||
await expect(setup({ status: 'delivered' }).controller.authorize(local, input)).resolves.toMatchObject({
|
||||
allowed: false,
|
||||
});
|
||||
});
|
||||
it('does not grant the exemption to a customer message without its required signature', async () => {
|
||||
await expect(
|
||||
setup({ tenantId: 'tenant', applicationId: 'app', batchTaskId: 'task' }).controller.authorize(local, input),
|
||||
).resolves.toMatchObject({ allowed: false });
|
||||
});
|
||||
it('rejects external callers before looking up a submit', async () => {
|
||||
const { controller, tx } = setup();
|
||||
await expect(controller.authorize({ socket: { remoteAddress: '10.0.0.2' } }, input)).rejects.toThrow();
|
||||
expect(tx.smsSubmitRecord.findUnique).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
@@ -58,19 +58,9 @@ export class DrainageSubmitGuardController {
|
||||
)
|
||||
return { allowed: false, code: 'DRN', reason: '提交或消息已终结,不得重复发送' };
|
||||
if (!message.tenantId && !message.batchTaskId) {
|
||||
try {
|
||||
await evaluateMessageDrainage(
|
||||
tx as unknown as PrismaService,
|
||||
message,
|
||||
message.carrier ?? undefined,
|
||||
undefined,
|
||||
true,
|
||||
);
|
||||
// Operations channel tests bypass signature/drainage policy only after
|
||||
// verifying the durable submit, channel, content and unfinished state.
|
||||
return { allowed: true };
|
||||
} catch (error) {
|
||||
if (!(error instanceof DrainageRejection)) throw error;
|
||||
return { allowed: false, code: 'DRN', reason: error.message };
|
||||
}
|
||||
}
|
||||
if (!message.tenantId || !message.applicationId || !message.signatureId)
|
||||
return { allowed: false, code: 'DRN', reason: '提交消息未关联企业应用和签名' };
|
||||
|
||||
@@ -1,16 +1,10 @@
|
||||
import { BadRequestException, HttpException, HttpStatus, Logger, NotFoundException } from '@nestjs/common';
|
||||
import { Prisma } from '@prisma/client';
|
||||
import { createHash } from 'node:crypto';
|
||||
import { Logger } from '@nestjs/common';
|
||||
import { BillingService } from '../billing/billing.service';
|
||||
import { moneyToNumber } from '../common/money';
|
||||
import type { OpenApiService } from '../open-api/open-api.service';
|
||||
import { PrismaService } from '../prisma/prisma.service';
|
||||
import type { GatewaySubmitResultDto, GatewaySubmitSegmentResultDto, GatewayReceiptEventDto, GatewayUplinkEventDto, GatewayPendingDeliveryQueryDto, GatewayDownstreamSentDto, GatewayDownstreamAcknowledgedDto, GatewayDownstreamFailureType, GatewayControlDeliveryResult, GatewaySubmitDeadLetterDto, RequeueGatewaySubmitExceptionDto, GatewayDownstreamRecoveryStatusDto, TimeoutUnknownDto } from './send-chain.contracts';
|
||||
import { positiveInteger, normalizeReceiptStatus, normalizeCarrier, DEFAULT_DOWNSTREAM_RETRY_DELAY_MS, DEFAULT_DOWNSTREAM_RETRY_MAX_DELAY_MS, DEFAULT_DOWNSTREAM_MAX_RETRIES, DEFAULT_DOWNSTREAM_PENDING_TIMEOUT_HOURS, DEFAULT_RECEIPT_TIMEOUT_HOURS, DEFAULT_GATEWAY_SUBMIT_REQUEUE_STALE_MS, DEFAULT_DOWNSTREAM_MANUAL_REQUEUE_STALE_MS, DEFAULT_UPSTREAM_RECEIPT_INBOX_PROCESSING_STALE_MS, DEFAULT_UPSTREAM_RECEIPT_INBOX_MAX_ATTEMPTS, DEFAULT_UPSTREAM_RECEIPT_INBOX_MAX_AGE_HOURS, gatewaySubmitRequeueKey, isObjectRecord, asDateOrNull, downstreamRetryDelayMs, downstreamAckTimeoutMs, downstreamRetryBaseDelayMs, downstreamRetryMaxDelayMs, downstreamMaxRetries, downstreamPendingTimeoutHours, downstreamControlFailureMessage, normalizeSubmitStatus, downstreamDeliveryAttemptKey, hasRecoveryAuditStateChanged, normalizeRecoveryFailureCategory, aggregateReceiptSegmentState, isSameUpstreamEndpointIdentity, receiptEventKey } from './send-chain.helpers';
|
||||
import type { SendSubmissionService } from './send-submission.service';
|
||||
import type { SendCompletionCallbacks, SendCompletionFacade } from './send-completion.service';
|
||||
|
||||
|
||||
/**
|
||||
* R10 accounting implementation.
|
||||
* Cross-domain calls return through the stable SendChainService seam to preserve locking and test observability.
|
||||
@@ -41,16 +35,19 @@ export class SendAccountingService {
|
||||
const unitPrice = moneyToNumber(message.unitPrice);
|
||||
const billingUnits = message.billingUnits ?? 0;
|
||||
const exists = await this.prisma.smsBillingRecord.findFirst({ where: { messageId: message.messageId } });
|
||||
if (exists?.billingStatus === 'charged') {
|
||||
if (exists && ['charged', 'refunded'].includes(exists.billingStatus)) {
|
||||
return;
|
||||
}
|
||||
const transaction = amountCents > 0 ? await this.billing.settleFrozenCharge({
|
||||
const transaction =
|
||||
amountCents > 0
|
||||
? await this.billing.settleFrozenCharge({
|
||||
tenantId: message.tenantId,
|
||||
amountCents,
|
||||
taskId: message.batchTaskId,
|
||||
messageId: message.messageId,
|
||||
remark: `短信 ${message.messageId} 提交成功释放冻结并转扣费`,
|
||||
}) : null;
|
||||
})
|
||||
: null;
|
||||
const data = {
|
||||
tenantId: message.tenantId,
|
||||
applicationId: message.applicationId ?? undefined,
|
||||
@@ -72,14 +69,22 @@ export class SendAccountingService {
|
||||
}
|
||||
|
||||
async releaseMessageReservation(
|
||||
message: { tenantId: string; batchTaskId: string; messageId: string; amountCents: number | bigint; billingUnits: number },
|
||||
message: {
|
||||
tenantId: string;
|
||||
batchTaskId: string;
|
||||
messageId: string;
|
||||
amountCents: number | bigint;
|
||||
billingUnits: number;
|
||||
},
|
||||
remark: string,
|
||||
) {
|
||||
const amountCents = moneyToNumber(message.amountCents);
|
||||
if (amountCents <= 0) {
|
||||
return;
|
||||
}
|
||||
const charged = await this.prisma.smsBillingRecord.findFirst({ where: { messageId: message.messageId, billingStatus: 'charged' } });
|
||||
const charged = await this.prisma.smsBillingRecord.findFirst({
|
||||
where: { messageId: message.messageId, billingStatus: 'charged' },
|
||||
});
|
||||
if (charged) {
|
||||
return;
|
||||
}
|
||||
@@ -107,11 +112,15 @@ export class SendAccountingService {
|
||||
if (amountCents <= 0) {
|
||||
return;
|
||||
}
|
||||
const refunded = await this.prisma.smsBillingRecord.findFirst({ where: { messageId: message.messageId, billingStatus: 'refunded' } });
|
||||
const refunded = await this.prisma.smsBillingRecord.findFirst({
|
||||
where: { messageId: message.messageId, billingStatus: 'refunded' },
|
||||
});
|
||||
if (refunded) {
|
||||
return;
|
||||
}
|
||||
const charged = await this.prisma.smsBillingRecord.findFirst({ where: { messageId: message.messageId, billingStatus: 'charged' } });
|
||||
const charged = await this.prisma.smsBillingRecord.findFirst({
|
||||
where: { messageId: message.messageId, billingStatus: 'charged' },
|
||||
});
|
||||
if (!charged) {
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -1694,6 +1694,7 @@ describe('SendChainService', () => {
|
||||
applicationId: 'app-1',
|
||||
eventType: 'receipt',
|
||||
}),
|
||||
undefined,
|
||||
);
|
||||
expect(prisma.cmppDownstreamDelivery.create).not.toHaveBeenCalled();
|
||||
});
|
||||
@@ -3257,7 +3258,7 @@ describe('SendChainService', () => {
|
||||
data: expect.objectContaining({ sequenceId: 7, gatewayMessageId: 'GW-1', submitStatus: 'accepted' }),
|
||||
});
|
||||
expect(prisma.smsMessageRecord.updateMany).toHaveBeenCalledWith({
|
||||
where: { id: 'record-1', status: { notIn: ['delivered', 'failed', 'unknown'] } },
|
||||
where: { id: 'record-1', status: { notIn: ['delivered', 'failed', 'unknown', 'timeout'] } },
|
||||
data: expect.objectContaining({ gatewayMessageId: 'GW-1', status: 'submitted', submitStatus: 'accepted' }),
|
||||
});
|
||||
expect(billing.settleFrozenCharge).toHaveBeenCalledWith(
|
||||
@@ -4891,7 +4892,7 @@ describe('SendChainService', () => {
|
||||
expect(prisma.smsMessageRecord.updateMany).toHaveBeenNthCalledWith(
|
||||
1,
|
||||
expect.objectContaining({
|
||||
where: { id: 'record-1', status: { notIn: ['delivered', 'failed', 'unknown'] } },
|
||||
where: { id: 'record-1', status: { notIn: ['delivered', 'failed', 'unknown', 'timeout'] } },
|
||||
data: expect.objectContaining({ status: 'submitted' }),
|
||||
}),
|
||||
);
|
||||
@@ -5567,6 +5568,7 @@ describe('SendChainService', () => {
|
||||
errorCode: 'RECEIPT_TIMEOUT',
|
||||
}),
|
||||
}),
|
||||
undefined,
|
||||
);
|
||||
expect(prisma.cmppDownstreamDelivery.create).not.toHaveBeenCalled();
|
||||
expect(prisma.smsMessageRecord.updateMany).toHaveBeenCalledWith({
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
import { AttemptCompletion } from './attempt-completion';
|
||||
import { completionContext, completionDatabase, CompletionRouteRequired } from './completion-context';
|
||||
import {
|
||||
BadRequestException,
|
||||
forwardRef,
|
||||
@@ -84,6 +86,7 @@ export class SendChainService implements OnModuleInit, OnModuleDestroy {
|
||||
private readonly submission: SendSubmissionService;
|
||||
private readonly completion: SendCompletionService;
|
||||
private readonly downstreamRequeueTasks: SendDownstreamRequeueTaskService;
|
||||
private readonly attemptCompletion?: AttemptCompletion;
|
||||
|
||||
constructor(
|
||||
private readonly prisma: PrismaService,
|
||||
@@ -94,6 +97,11 @@ export class SendChainService implements OnModuleInit, OnModuleDestroy {
|
||||
@Optional() phoneRouting?: PhoneRoutingLookupService,
|
||||
@Optional() metrics?: MetricsService,
|
||||
) {
|
||||
const rootPrisma = prisma;
|
||||
prisma = completionDatabase(prisma);
|
||||
this.prisma = prisma;
|
||||
// Structural unit-test doubles may omit the durable delegate; real Prisma always has it.
|
||||
const coordinatedBilling = rootPrisma.smsAttemptCompletionWork ? new BillingService(prisma) : billing;
|
||||
const resolvedPhoneRouting = phoneRouting ?? new PhoneRoutingLookupService(prisma);
|
||||
this.submission = new SendSubmissionService(
|
||||
prisma,
|
||||
@@ -109,12 +117,46 @@ export class SendChainService implements OnModuleInit, OnModuleDestroy {
|
||||
},
|
||||
metrics,
|
||||
);
|
||||
this.completion = new SendCompletionService(prisma, billing, openApi, this as unknown as SendCompletionFacade);
|
||||
this.completion = new SendCompletionService(
|
||||
prisma,
|
||||
coordinatedBilling,
|
||||
openApi,
|
||||
this as unknown as SendCompletionFacade,
|
||||
);
|
||||
if (rootPrisma.smsAttemptCompletionWork) {
|
||||
this.attemptCompletion = new AttemptCompletion(
|
||||
rootPrisma,
|
||||
async (kind, payload) => {
|
||||
const event = payload as unknown as {
|
||||
data: GatewayReceiptEventDto & GatewaySubmitResultDto & GatewaySubmitSegmentResultDto;
|
||||
incomingIdentity?: Parameters<SendChainService['handleReceipt']>[1];
|
||||
olderThanHours?: number;
|
||||
errorCode?: string;
|
||||
reason?: string;
|
||||
};
|
||||
if (kind === 'receipt') return this.completion.handleReceipt(event.data, event.incomingIdentity);
|
||||
if (kind === 'submit') return this.completion.handleSubmitResult(event.data);
|
||||
if (kind === 'segment') return this.completion.handleSubmitSegmentResult(event.data);
|
||||
if (kind === 'timeout') return this.completion.markUnknownTimeout({ olderThanHours: event.olderThanHours });
|
||||
const message = await prisma.smsMessageRecord.findUniqueOrThrow({
|
||||
where: { id: completionContext.getStore()!.messageRecordId },
|
||||
});
|
||||
if (
|
||||
message.status === 'delivered' ||
|
||||
(message.status === 'timeout' && message.errorCode === 'RECEIPT_TIMEOUT')
|
||||
)
|
||||
return message;
|
||||
return this.completion.recordCmppFailureReceipt(message, event.errorCode!, event.reason!);
|
||||
},
|
||||
(route) => this.submission.waitForChannelRateLimit(route.channel.id, route.channel.rateLimitPerSecond),
|
||||
);
|
||||
}
|
||||
this.downstreamRequeueTasks = new SendDownstreamRequeueTaskService(prisma, this);
|
||||
}
|
||||
|
||||
onModuleInit() {
|
||||
const processRole = process.env.CMPP_PROCESS_ROLE?.trim() || 'all';
|
||||
if (['all', 'worker', 'callback'].includes(processRole)) this.attemptCompletion?.start();
|
||||
if (processRole === 'api' || processRole === 'callback') return;
|
||||
if (processRole === 'outbox') {
|
||||
this.submission.startSubmitOutboxPublisher();
|
||||
@@ -202,6 +244,7 @@ export class SendChainService implements OnModuleInit, OnModuleDestroy {
|
||||
}
|
||||
|
||||
async onModuleDestroy() {
|
||||
this.attemptCompletion?.stop();
|
||||
if (this.receiptTimeoutInitialTimer) clearTimeout(this.receiptTimeoutInitialTimer);
|
||||
if (this.receiptTimeoutIntervalTimer) clearInterval(this.receiptTimeoutIntervalTimer);
|
||||
if (this.scheduledDispatchInitialTimer) clearTimeout(this.scheduledDispatchInitialTimer);
|
||||
@@ -484,6 +527,11 @@ export class SendChainService implements OnModuleInit, OnModuleDestroy {
|
||||
}
|
||||
|
||||
async handleSubmitSegmentResult(data: GatewaySubmitSegmentResultDto) {
|
||||
if (this.attemptCompletion && !completionContext.getStore()) {
|
||||
const message = await this.completion.requireMessageByGatewayEvent(data.messageId, data.gatewayMessageId);
|
||||
const source = await this.completion.resolveSubmitRecordForGatewaySegmentResult(message.id, data);
|
||||
return this.attemptCompletion.enqueue(message.id, source.id, 'segment', { data });
|
||||
}
|
||||
return this.completion.handleSubmitSegmentResult(data);
|
||||
}
|
||||
|
||||
@@ -495,6 +543,11 @@ export class SendChainService implements OnModuleInit, OnModuleDestroy {
|
||||
}
|
||||
|
||||
async handleSubmitResult(data: GatewaySubmitResultDto) {
|
||||
if (this.attemptCompletion && !completionContext.getStore()) {
|
||||
const message = await this.completion.requireMessageByGatewayEvent(data.messageId, data.gatewayMessageId);
|
||||
const source = await this.completion.resolveSubmitRecordForGatewayResult(message.id, data);
|
||||
return this.attemptCompletion.enqueue(message.id, source.id, 'submit', { data }, data.eventId);
|
||||
}
|
||||
return this.completion.handleSubmitResult(data);
|
||||
}
|
||||
|
||||
@@ -528,6 +581,14 @@ export class SendChainService implements OnModuleInit, OnModuleDestroy {
|
||||
cmppVersion: string;
|
||||
},
|
||||
) {
|
||||
if (this.attemptCompletion && !completionContext.getStore()) {
|
||||
const resolved = await this.completion.resolveReceiptMessage(data, incomingIdentity);
|
||||
if (!resolved.submitRecordId) throw new NotFoundException('回执缺少可确认的提交尝试关联');
|
||||
return this.attemptCompletion.enqueue(resolved.message.id, resolved.submitRecordId, 'receipt', {
|
||||
data,
|
||||
incomingIdentity,
|
||||
});
|
||||
}
|
||||
return this.completion.handleReceipt(data, incomingIdentity);
|
||||
}
|
||||
|
||||
@@ -726,6 +787,32 @@ export class SendChainService implements OnModuleInit, OnModuleDestroy {
|
||||
}
|
||||
|
||||
async markUnknownTimeout(data: TimeoutUnknownDto) {
|
||||
if (this.attemptCompletion && !completionContext.getStore()) {
|
||||
const olderThanHours = data.olderThanHours ?? positiveInteger(process.env.SMS_RECEIPT_TIMEOUT_HOURS, 72);
|
||||
const candidates = await this.prisma.smsMessageRecord.findMany({
|
||||
where: {
|
||||
tenantId: { not: null },
|
||||
OR: [
|
||||
{
|
||||
status: { in: ['submitted', 'unknown'] },
|
||||
submittedAt: { lte: new Date(Date.now() - olderThanHours * 3600_000) },
|
||||
},
|
||||
{ status: 'timeout', errorCode: 'RECEIPT_TIMEOUT', timeoutReceiptQueuedAt: null },
|
||||
],
|
||||
},
|
||||
select: { id: true, submitId: true, status: true },
|
||||
take: 100,
|
||||
});
|
||||
let timeout = 0;
|
||||
for (const message of candidates) {
|
||||
const source = message.submitId
|
||||
? await this.prisma.smsSubmitRecord.findUnique({ where: { submitId: message.submitId } })
|
||||
: null;
|
||||
const result = await this.attemptCompletion.enqueue(message.id, source?.id, 'timeout', { olderThanHours });
|
||||
if (message.status !== 'timeout' && result?.status === 'timeout') timeout++;
|
||||
}
|
||||
return { timeout };
|
||||
}
|
||||
return this.completion.markUnknownTimeout(data);
|
||||
}
|
||||
|
||||
@@ -803,6 +890,14 @@ export class SendChainService implements OnModuleInit, OnModuleDestroy {
|
||||
},
|
||||
options: { forceNational?: boolean; excludeChannelIds?: string[] } = {},
|
||||
): Promise<RoutedChannel> {
|
||||
const context = completionContext.getStore();
|
||||
if (context) {
|
||||
const prepare = () => this.submission.selectChannelForMessage(message, { ...options, previewOnly: true });
|
||||
if (!context.routePlanned) throw new CompletionRouteRequired(prepare);
|
||||
const current = await this.submission.selectChannelForMessage(message, options);
|
||||
if (!context.route || current.channel.id !== context.route.channel.id) throw new CompletionRouteRequired(prepare);
|
||||
return current;
|
||||
}
|
||||
return this.submission.selectChannelForMessage(message, options);
|
||||
}
|
||||
|
||||
@@ -923,6 +1018,9 @@ export class SendChainService implements OnModuleInit, OnModuleDestroy {
|
||||
errorCode: string,
|
||||
reason: string,
|
||||
) {
|
||||
if (this.attemptCompletion && !completionContext.getStore()) {
|
||||
return this.attemptCompletion.enqueue(message.id, undefined, 'rejection', { errorCode, reason });
|
||||
}
|
||||
return this.completion.recordCmppFailureReceipt(message, errorCode, reason);
|
||||
}
|
||||
|
||||
@@ -1004,6 +1102,7 @@ export class SendChainService implements OnModuleInit, OnModuleDestroy {
|
||||
}
|
||||
|
||||
private async waitForChannelRateLimit(channelId: string, tps: number) {
|
||||
if (completionContext.getStore()) return;
|
||||
return this.submission.waitForChannelRateLimit(channelId, tps);
|
||||
}
|
||||
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { completionContext } from './completion-context';
|
||||
import { BadRequestException, Logger, NotFoundException } from '@nestjs/common';
|
||||
import { Prisma } from '@prisma/client';
|
||||
import { createHash, randomUUID } from 'node:crypto';
|
||||
@@ -217,20 +218,24 @@ export class SendDownstreamDeliveryService {
|
||||
const cmppDeliveryAllowed = deliveryAllowed || data.allowBusinessRejectionCmppDelivery === true;
|
||||
if (deliveryAllowed && data.queueHttpWebhook !== false) {
|
||||
try {
|
||||
await this.openApi?.queueWebhookEvent({
|
||||
await this.openApi?.queueWebhookEvent(
|
||||
{
|
||||
tenantId: data.tenantId,
|
||||
applicationId: data.applicationId,
|
||||
messageRecordId: data.messageRecordId,
|
||||
messageId: data.messageId,
|
||||
uplinkMessageId: typeof data.payload.uplinkMessageId === 'string' ? data.payload.uplinkMessageId : undefined,
|
||||
uplinkMessageId:
|
||||
typeof data.payload.uplinkMessageId === 'string' ? data.payload.uplinkMessageId : undefined,
|
||||
eventType: data.deliveryType,
|
||||
payload: data.payload,
|
||||
});
|
||||
},
|
||||
completionContext.getStore()?.tx,
|
||||
);
|
||||
} catch (error) {
|
||||
this.logger.error(
|
||||
`HTTP webhook queue failed for ${data.deliveryType}/${data.messageId ?? '-'}: ${error instanceof Error ? error.message : String(error)}`,
|
||||
);
|
||||
if (data.propagateHttpQueueError) throw error;
|
||||
if (data.propagateHttpQueueError || completionContext.getStore()) throw error;
|
||||
}
|
||||
}
|
||||
if (data.queueCmppDelivery === false) {
|
||||
@@ -249,6 +254,34 @@ export class SendDownstreamDeliveryService {
|
||||
: data.deliveryType === 'uplink' && typeof data.payload.uplinkMessageId === 'string'
|
||||
? `uplink:${data.payload.uplinkMessageId}`
|
||||
: null;
|
||||
if (completionContext.getStore()) {
|
||||
if (!dedupeKey) throw new Error('completion_notification_identity_missing');
|
||||
await this.prisma.cmppDownstreamDelivery.createMany({
|
||||
data: [
|
||||
{
|
||||
tenantId: data.tenantId,
|
||||
applicationId: data.applicationId,
|
||||
messageRecordId: data.messageRecordId,
|
||||
messageId: data.messageId,
|
||||
dedupeKey,
|
||||
deliveryType: data.deliveryType,
|
||||
payload,
|
||||
retryEnabled:
|
||||
cmppDeliveryAllowed &&
|
||||
(data.deliveryType === 'uplink'
|
||||
? application.downstreamUplinkRetryEnabled
|
||||
: application.downstreamReceiptRetryEnabled),
|
||||
status: cmppDeliveryAllowed ? 'pending' : 'abandoned',
|
||||
lastError: cmppDeliveryAllowed ? null : '企业应用已停用,保留回执但不再向客户应用推送',
|
||||
},
|
||||
],
|
||||
skipDuplicates: true,
|
||||
});
|
||||
const retained = await this.prisma.cmppDownstreamDelivery.findUniqueOrThrow({ where: { dedupeKey } });
|
||||
if (retained.messageRecordId !== data.messageRecordId || retained.applicationId !== data.applicationId)
|
||||
throw new Error('completion_notification_identity_mismatch');
|
||||
return retained;
|
||||
}
|
||||
let delivery;
|
||||
try {
|
||||
delivery = await this.prisma.cmppDownstreamDelivery.create({
|
||||
|
||||
@@ -188,11 +188,18 @@ export class SendGatewayResultService {
|
||||
}
|
||||
const status =
|
||||
data.submitStatus === 'accepted' ? 'submitted' : data.submitStatus === 'timeout' ? 'timeout' : 'submit_failed';
|
||||
if (
|
||||
data.submitStatus !== 'accepted' &&
|
||||
(message.status === 'delivered' || (message.status === 'timeout' && message.errorCode === 'RECEIPT_TIMEOUT'))
|
||||
) {
|
||||
await this.markSubmitResultProcessed(submitRecord.id, data.eventId, submittedAt);
|
||||
return message;
|
||||
}
|
||||
if (data.submitStatus === 'accepted' && !isStandaloneChannelTest && message.tenantId && message.batchTaskId) {
|
||||
const businessMessage = message as typeof message & { tenantId: string; batchTaskId: string };
|
||||
await this.facade.chargeAcceptedMessage(businessMessage);
|
||||
const latest = await this.prisma.smsMessageRecord.findUnique({ where: { id: message.id } });
|
||||
if (latest?.status === 'failed') {
|
||||
if (latest?.status === 'failed' || (latest?.status === 'timeout' && latest.errorCode === 'RECEIPT_TIMEOUT')) {
|
||||
await this.facade.refundMessage(businessMessage, '先到失败回执补偿退款');
|
||||
}
|
||||
} else if (
|
||||
@@ -219,7 +226,7 @@ export class SendGatewayResultService {
|
||||
data.submitStatus === 'timeout' ? '提交超时释放冻结' : '提交失败释放冻结',
|
||||
);
|
||||
}
|
||||
const protectedTerminalStatuses = ['delivered', 'failed', 'unknown'];
|
||||
const protectedTerminalStatuses = ['delivered', 'failed', 'unknown', 'timeout'];
|
||||
const updated = await this.prisma.smsMessageRecord.updateMany({
|
||||
where:
|
||||
data.submitStatus === 'accepted'
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { completionContext } from './completion-context';
|
||||
import { BadRequestException, Logger, NotFoundException } from '@nestjs/common';
|
||||
import { Prisma } from '@prisma/client';
|
||||
import { Queue, Worker } from 'bullmq';
|
||||
@@ -310,7 +311,7 @@ export class SendGatewaySubmitService {
|
||||
sessionId: sessionByChannel.get(routed.channel.id),
|
||||
};
|
||||
});
|
||||
const writeOutbox = this.submitOutboxEnabled();
|
||||
const writeOutbox = Boolean(completionContext.getStore()) || this.submitOutboxEnabled();
|
||||
await this.measureSendStage('submit_transaction', () =>
|
||||
this.prisma.$transaction(async (tx) => {
|
||||
await tx.smsSubmitRecord.createMany({
|
||||
@@ -365,7 +366,7 @@ export class SendGatewaySubmitService {
|
||||
}
|
||||
}),
|
||||
);
|
||||
if (!this.submitOutboxPublishEnabled()) {
|
||||
if (!completionContext.getStore() && !this.submitOutboxPublishEnabled()) {
|
||||
await Promise.all(prepared.map(({ command }) => this.facade.publishGatewaySubmitCommand(command)));
|
||||
}
|
||||
await this.refreshTaskProgressBatch(prepared.map(({ message }) => message));
|
||||
@@ -762,7 +763,7 @@ export class SendGatewaySubmitService {
|
||||
const submitId = `SUB-${randomUUID()}`;
|
||||
const sessionId = await this.getOpenSubmitSessionId(channel.id);
|
||||
const command = this.buildGatewaySubmitCommand(message, routed, attempt, submitId, upstreamSrcId);
|
||||
const writeOutbox = this.submitOutboxEnabled();
|
||||
const writeOutbox = Boolean(completionContext.getStore()) || this.submitOutboxEnabled();
|
||||
try {
|
||||
await this.measureSendStage('submit_transaction', () =>
|
||||
this.prisma.$transaction(async (tx) => {
|
||||
@@ -849,7 +850,7 @@ export class SendGatewaySubmitService {
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
if (!this.submitOutboxPublishEnabled()) {
|
||||
if (!completionContext.getStore() && !this.submitOutboxPublishEnabled()) {
|
||||
await this.measureSendStage('gateway_stream_publish', () => this.facade.publishGatewaySubmitCommand(command));
|
||||
}
|
||||
await this.measureSendStage('task_progress', () =>
|
||||
@@ -1087,7 +1088,7 @@ return streamId`;
|
||||
template?: { signature?: { id?: string | null } | null } | null;
|
||||
signature?: { id?: string | null } | null;
|
||||
},
|
||||
options: { forceNational?: boolean; excludeChannelIds?: string[] } = {},
|
||||
options: { forceNational?: boolean; excludeChannelIds?: string[]; previewOnly?: boolean } = {},
|
||||
): Promise<RoutedChannel> {
|
||||
if (!message.applicationId) {
|
||||
throw new BadRequestException('短信应用未配置,无法选择通道组');
|
||||
@@ -1100,7 +1101,7 @@ return streamId`;
|
||||
this.facade.identifyCarrier(message.phoneNumber),
|
||||
this.facade.identifyProvince(message.phoneNumber),
|
||||
]);
|
||||
if (!hasPersistedRouting) {
|
||||
if (!hasPersistedRouting && !options.previewOnly) {
|
||||
await this.prisma.smsMessageRecord.update({
|
||||
where: { id: message.id },
|
||||
data: { carrier: resolved[0], province: resolved[1] },
|
||||
@@ -1137,7 +1138,7 @@ return streamId`;
|
||||
approvedChannelIds,
|
||||
routingKey: message.id,
|
||||
});
|
||||
await channelWords.persist(this.prisma);
|
||||
if (!options.previewOnly) await channelWords.persist(this.prisma);
|
||||
if (rejected) throw new ChannelWordRejection();
|
||||
if (!selected) {
|
||||
throw new NotFoundException('无已报备通过且在线的可用通道');
|
||||
@@ -1352,6 +1353,10 @@ return streamId`;
|
||||
`);
|
||||
if (direct === 1) return;
|
||||
}
|
||||
if (completionContext.getStore()) {
|
||||
await this.prisma.$queryRaw`SELECT id FROM "SmsBatchTask" WHERE id=${batchTaskId} FOR UPDATE`;
|
||||
return this.refreshTaskProgressUntilClean(batchTaskId, true);
|
||||
}
|
||||
const running = this.taskProgressRefreshes.get(batchTaskId);
|
||||
if (running) {
|
||||
// A state transition committed after the running aggregate may not be visible
|
||||
@@ -1372,9 +1377,9 @@ return streamId`;
|
||||
}
|
||||
}
|
||||
|
||||
private async refreshTaskProgressUntilClean(batchTaskId: string) {
|
||||
private async refreshTaskProgressUntilClean(batchTaskId: string, transactional = false) {
|
||||
do {
|
||||
this.dirtyTaskProgressRefreshes.delete(batchTaskId);
|
||||
if (!transactional) this.dirtyTaskProgressRefreshes.delete(batchTaskId);
|
||||
const groups = await this.prisma.smsMessageRecord.groupBy({
|
||||
by: ['status'],
|
||||
where: { batchTaskId },
|
||||
@@ -1401,7 +1406,7 @@ return streamId`;
|
||||
where: { id: batchTaskId },
|
||||
data: { progressTotal, submittedTotal, successTotal, failedTotal, unknownTotal, timeoutTotal, status },
|
||||
});
|
||||
} while (this.dirtyTaskProgressRefreshes.has(batchTaskId));
|
||||
} while (!transactional && this.dirtyTaskProgressRefreshes.has(batchTaskId));
|
||||
}
|
||||
|
||||
private async recoverNightReviews() {
|
||||
@@ -1464,6 +1469,17 @@ return streamId`,
|
||||
}
|
||||
|
||||
private getOpenSubmitSessionId(channelId: string) {
|
||||
if (completionContext.getStore()) {
|
||||
const sessionNo = `OPEN-${channelId}`;
|
||||
return this.prisma.cmppSubmitSession
|
||||
.upsert({
|
||||
where: { sessionNo },
|
||||
update: {},
|
||||
create: { channelId, sessionNo, submitTotal: 0 },
|
||||
select: { id: true },
|
||||
})
|
||||
.then((session) => session.id);
|
||||
}
|
||||
const cached = this.openSubmitSessionIds.get(channelId);
|
||||
if (cached) return cached;
|
||||
const sessionNo = `OPEN-${channelId}`;
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { completionContext } from './completion-context';
|
||||
import { Logger, NotFoundException } from '@nestjs/common';
|
||||
import { Prisma } from '@prisma/client';
|
||||
|
||||
@@ -229,10 +230,12 @@ export class SendReceiptService {
|
||||
where: { receiptKey },
|
||||
include: { messageRecord: true },
|
||||
});
|
||||
if (existingReceipt?.messageRecord) {
|
||||
if (existingReceipt?.messageRecord && !completionContext.getStore()) {
|
||||
return existingReceipt.messageRecord;
|
||||
}
|
||||
const message = resolved.message;
|
||||
if (existingReceipt && existingReceipt.messageRecordId !== message.id)
|
||||
throw new Error('completion_receipt_owner_mismatch');
|
||||
const deliveredAt = data.deliveredAt ? new Date(data.deliveredAt) : new Date();
|
||||
if (resolved.submitRecordId) {
|
||||
await this.prisma.smsSubmitRecord.updateMany({
|
||||
@@ -246,7 +249,8 @@ export class SendReceiptService {
|
||||
},
|
||||
});
|
||||
}
|
||||
let receiptRecordId: string | undefined;
|
||||
let receiptRecordId: string | undefined = existingReceipt?.id;
|
||||
if (!existingReceipt)
|
||||
try {
|
||||
const createdReceipt = await this.prisma.smsReceiptRecord.create({
|
||||
data: {
|
||||
@@ -300,8 +304,10 @@ export class SendReceiptService {
|
||||
if (!aggregate.terminal) {
|
||||
return this.prisma.smsMessageRecord.findUnique({ where: { id: message.id } });
|
||||
}
|
||||
if (message.status === 'timeout' && message.errorCode === 'RECEIPT_TIMEOUT') return message;
|
||||
const status = aggregate.status;
|
||||
const isCurrentAttempt =
|
||||
(!message.submitId || message.submitId === resolved.submitId) &&
|
||||
(!message.channelId || message.channelId === logicalChannelId) &&
|
||||
(!message.gatewayMessageId ||
|
||||
message.gatewayMessageId === data.gatewayMessageId ||
|
||||
|
||||
@@ -1,16 +1,18 @@
|
||||
import { BadRequestException, HttpException, HttpStatus, Logger, NotFoundException } from '@nestjs/common';
|
||||
import { BadRequestException, HttpException, Logger, NotFoundException } from '@nestjs/common';
|
||||
import { Prisma } from '@prisma/client';
|
||||
import { createHash } from 'node:crypto';
|
||||
import { BillingService } from '../billing/billing.service';
|
||||
import { moneyToNumber } from '../common/money';
|
||||
import type { OpenApiService } from '../open-api/open-api.service';
|
||||
import { PrismaService } from '../prisma/prisma.service';
|
||||
import type { GatewaySubmitResultDto, GatewaySubmitSegmentResultDto, GatewayReceiptEventDto, GatewayUplinkEventDto, GatewayPendingDeliveryQueryDto, GatewayDownstreamSentDto, GatewayDownstreamAcknowledgedDto, GatewayDownstreamFailureType, GatewayControlDeliveryResult, GatewaySubmitDeadLetterDto, RequeueGatewaySubmitExceptionDto, GatewayDownstreamRecoveryStatusDto, TimeoutUnknownDto } from './send-chain.contracts';
|
||||
import { positiveInteger, normalizeReceiptStatus, normalizeCarrier, DEFAULT_DOWNSTREAM_RETRY_DELAY_MS, DEFAULT_DOWNSTREAM_RETRY_MAX_DELAY_MS, DEFAULT_DOWNSTREAM_MAX_RETRIES, DEFAULT_DOWNSTREAM_PENDING_TIMEOUT_HOURS, DEFAULT_RECEIPT_TIMEOUT_HOURS, DEFAULT_GATEWAY_SUBMIT_REQUEUE_STALE_MS, DEFAULT_DOWNSTREAM_MANUAL_REQUEUE_STALE_MS, DEFAULT_UPSTREAM_RECEIPT_INBOX_PROCESSING_STALE_MS, DEFAULT_UPSTREAM_RECEIPT_INBOX_MAX_ATTEMPTS, DEFAULT_UPSTREAM_RECEIPT_INBOX_MAX_AGE_HOURS, gatewaySubmitRequeueKey, isObjectRecord, asDateOrNull, downstreamRetryDelayMs, downstreamAckTimeoutMs, downstreamRetryBaseDelayMs, downstreamRetryMaxDelayMs, downstreamMaxRetries, downstreamPendingTimeoutHours, downstreamControlFailureMessage, normalizeSubmitStatus, downstreamDeliveryAttemptKey, hasRecoveryAuditStateChanged, normalizeRecoveryFailureCategory, aggregateReceiptSegmentState, isSameUpstreamEndpointIdentity, receiptEventKey } from './send-chain.helpers';
|
||||
import type { SendSubmissionService } from './send-submission.service';
|
||||
import type { GatewaySubmitDeadLetterDto, RequeueGatewaySubmitExceptionDto } from './send-chain.contracts';
|
||||
import {
|
||||
DEFAULT_GATEWAY_SUBMIT_REQUEUE_STALE_MS,
|
||||
gatewaySubmitRequeueKey,
|
||||
isObjectRecord,
|
||||
normalizeCarrier,
|
||||
positiveInteger,
|
||||
} from './send-chain.helpers';
|
||||
import type { SendCompletionCallbacks, SendCompletionFacade } from './send-completion.service';
|
||||
|
||||
|
||||
/**
|
||||
* R10 retry implementation.
|
||||
* Cross-domain calls return through the stable SendChainService seam to preserve locking and test observability.
|
||||
@@ -87,11 +89,12 @@ export class SendRetryService {
|
||||
const message = deadLetter.messageId
|
||||
? await this.prisma.smsMessageRecord.findUnique({ where: { messageId: deadLetter.messageId } })
|
||||
: null;
|
||||
if (message && (
|
||||
message.submitStatus === 'accepted'
|
||||
|| ['submitted', 'delivered', 'unknown'].includes(message.status)
|
||||
|| ['delivered', 'unknown'].includes(message.receiptStatus ?? '')
|
||||
)) {
|
||||
if (
|
||||
message &&
|
||||
(message.submitStatus === 'accepted' ||
|
||||
['submitted', 'delivered', 'unknown'].includes(message.status) ||
|
||||
['delivered', 'unknown'].includes(message.receiptStatus ?? ''))
|
||||
) {
|
||||
throw new BadRequestException('该短信已有成功或不确定的上游结果,为避免重复发送,禁止重新入队');
|
||||
}
|
||||
const commandChannelId = String(deadLetter.commandPayload.channelId ?? deadLetter.channelId ?? '').trim();
|
||||
@@ -118,7 +121,10 @@ export class SendRetryService {
|
||||
const requeueKey = gatewaySubmitRequeueKey(deadLetter.id, deadLetter.manualRetryCount + 1);
|
||||
let retryStreamMessageId: string;
|
||||
try {
|
||||
const publishedStreamMessageId = await this.facade.publishGatewaySubmitCommand(deadLetter.commandPayload, requeueKey);
|
||||
const publishedStreamMessageId = await this.facade.publishGatewaySubmitCommand(
|
||||
deadLetter.commandPayload,
|
||||
requeueKey,
|
||||
);
|
||||
if (!publishedStreamMessageId) {
|
||||
throw new Error('Gateway提交异常重新入队未返回Stream消息编号');
|
||||
}
|
||||
@@ -208,10 +214,10 @@ export class SendRetryService {
|
||||
}
|
||||
|
||||
async recoverStaleGatewaySubmitRequeues(now = new Date()) {
|
||||
const staleCutoff = new Date(now.getTime() - positiveInteger(
|
||||
process.env.GATEWAY_SUBMIT_REQUEUE_STALE_MS,
|
||||
DEFAULT_GATEWAY_SUBMIT_REQUEUE_STALE_MS,
|
||||
));
|
||||
const staleCutoff = new Date(
|
||||
now.getTime() -
|
||||
positiveInteger(process.env.GATEWAY_SUBMIT_REQUEUE_STALE_MS, DEFAULT_GATEWAY_SUBMIT_REQUEUE_STALE_MS),
|
||||
);
|
||||
const stale = await this.prisma.gatewaySubmitDeadLetter.findMany({
|
||||
where: { status: 'requeueing', updatedAt: { lt: staleCutoff } },
|
||||
orderBy: { updatedAt: 'asc' },
|
||||
@@ -235,7 +241,10 @@ export class SendRetryService {
|
||||
if (claimed.count !== 1) continue;
|
||||
try {
|
||||
const requeueKey = gatewaySubmitRequeueKey(deadLetter.id, deadLetter.manualRetryCount + 1);
|
||||
const retryStreamMessageId = await this.facade.publishGatewaySubmitCommand(deadLetter.commandPayload, requeueKey);
|
||||
const retryStreamMessageId = await this.facade.publishGatewaySubmitCommand(
|
||||
deadLetter.commandPayload,
|
||||
requeueKey,
|
||||
);
|
||||
if (!retryStreamMessageId) throw new Error('Gateway提交异常恢复未返回Stream消息编号');
|
||||
const finalized = await this.prisma.gatewaySubmitDeadLetter.updateMany({
|
||||
where: { id: deadLetter.id, status: 'requeue_recovering' },
|
||||
@@ -264,7 +273,9 @@ export class SendRetryService {
|
||||
where: { id: deadLetter.id, status: 'requeue_recovering' },
|
||||
data: { status: 'requeueing' },
|
||||
});
|
||||
this.logger.error(`Gateway submit requeue recovery failed for ${deadLetter.id}: ${error instanceof Error ? error.message : String(error)}`);
|
||||
this.logger.error(
|
||||
`Gateway submit requeue recovery failed for ${deadLetter.id}: ${error instanceof Error ? error.message : String(error)}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
return { recovered, failed };
|
||||
@@ -300,100 +311,123 @@ export class SendRetryService {
|
||||
const attemptedChannelIds = attempts.map((attempt) => attempt.channelId);
|
||||
let sourceAttempt = sourceSubmitRecordId
|
||||
? attempts.find((attempt) => attempt.id === sourceSubmitRecordId)
|
||||
: attempts.find((attempt) => attempt.submitId === message.submitId) ?? attempts[attempts.length - 1];
|
||||
: (attempts.find((attempt) => attempt.submitId === message.submitId) ?? attempts[attempts.length - 1]);
|
||||
if (!sourceAttempt && sourceSubmitRecordId) {
|
||||
sourceAttempt = await this.prisma.smsSubmitRecord.findUnique({
|
||||
sourceAttempt =
|
||||
(await this.prisma.smsSubmitRecord.findUnique({
|
||||
where: { id: sourceSubmitRecordId },
|
||||
}) ?? undefined;
|
||||
})) ?? undefined;
|
||||
}
|
||||
if (!sourceAttempt || sourceAttempt.messageRecordId !== message.id) {
|
||||
this.logger.error(`sms_retry_route_failed ${JSON.stringify({
|
||||
this.logger.error(
|
||||
`sms_retry_route_failed ${JSON.stringify({
|
||||
messageId: message.messageId,
|
||||
messageRecordId: message.id,
|
||||
reason,
|
||||
sourceSubmitRecordId,
|
||||
sourceMessageRecordId: sourceAttempt?.messageRecordId,
|
||||
error: sourceAttempt ? 'retry_source_submit_record_mismatch' : 'retry_source_submit_record_missing',
|
||||
})}`);
|
||||
})}`,
|
||||
);
|
||||
return null;
|
||||
}
|
||||
const existingRetry = await this.prisma.smsSubmitRecord.findUnique({
|
||||
where: { retryOfSubmitRecordId: sourceAttempt.id },
|
||||
});
|
||||
if (existingRetry) {
|
||||
this.logger.warn(`sms_retry_claim_reused ${JSON.stringify({
|
||||
this.logger.warn(
|
||||
`sms_retry_claim_reused ${JSON.stringify({
|
||||
messageId: message.messageId,
|
||||
messageRecordId: message.id,
|
||||
retryOfSubmitRecordId: sourceAttempt.id,
|
||||
submitId: existingRetry.submitId,
|
||||
channelId: existingRetry.channelId,
|
||||
})}`);
|
||||
})}`,
|
||||
);
|
||||
return this.prisma.smsMessageRecord.findUnique({ where: { id: message.id } });
|
||||
}
|
||||
const ageMinutes = (Date.now() - new Date(message.queuedAt ?? Date.now()).getTime()) / 60_000;
|
||||
this.logger.log(`sms_retry_route_started ${JSON.stringify({
|
||||
this.logger.log(
|
||||
`sms_retry_route_started ${JSON.stringify({
|
||||
messageId: message.messageId,
|
||||
messageRecordId: message.id,
|
||||
reason,
|
||||
attemptedChannelIds,
|
||||
ageMinutes: Math.round(ageMinutes * 100) / 100,
|
||||
})}`);
|
||||
})}`,
|
||||
);
|
||||
if (ageMinutes >= 72 * 60) {
|
||||
this.logger.warn(`sms_retry_route_skipped ${JSON.stringify({
|
||||
this.logger.warn(
|
||||
`sms_retry_route_skipped ${JSON.stringify({
|
||||
messageId: message.messageId,
|
||||
messageRecordId: message.id,
|
||||
reason: 'maximum_message_age_exceeded',
|
||||
ageMinutes: Math.round(ageMinutes * 100) / 100,
|
||||
})}`);
|
||||
})}`,
|
||||
);
|
||||
return null;
|
||||
}
|
||||
const retryCarrier = message.carrier
|
||||
? normalizeCarrier(message.carrier)
|
||||
: await this.facade.identifyCarrier(message.phoneNumber);
|
||||
const route = await this.facade.findApplicationRoute(message.tenantId, message.applicationId ?? undefined, retryCarrier);
|
||||
const retryTimeLimitMinutes = Math.min(route.group.retryTimeLimitMinutes ?? route.group.retryTimeLimitHours * 60, 72 * 60);
|
||||
const route = await this.facade.findApplicationRoute(
|
||||
message.tenantId,
|
||||
message.applicationId ?? undefined,
|
||||
retryCarrier,
|
||||
);
|
||||
const retryTimeLimitMinutes = Math.min(
|
||||
route.group.retryTimeLimitMinutes ?? route.group.retryTimeLimitHours * 60,
|
||||
72 * 60,
|
||||
);
|
||||
if (!route.group.retryEnabled || ageMinutes >= retryTimeLimitMinutes) {
|
||||
this.logger.warn(`sms_retry_route_skipped ${JSON.stringify({
|
||||
this.logger.warn(
|
||||
`sms_retry_route_skipped ${JSON.stringify({
|
||||
messageId: message.messageId,
|
||||
messageRecordId: message.id,
|
||||
groupId: route.groupId,
|
||||
reason: !route.group.retryEnabled ? 'group_retry_disabled' : 'group_retry_time_limit_exceeded',
|
||||
ageMinutes: Math.round(ageMinutes * 100) / 100,
|
||||
retryTimeLimitMinutes,
|
||||
})}`);
|
||||
})}`,
|
||||
);
|
||||
return null;
|
||||
}
|
||||
try {
|
||||
const routed = await this.facade.selectChannelForMessage({ ...message, carrier: retryCarrier }, {
|
||||
const routed = await this.facade.selectChannelForMessage(
|
||||
{ ...message, carrier: retryCarrier },
|
||||
{
|
||||
forceNational: true,
|
||||
excludeChannelIds: attemptedChannelIds,
|
||||
});
|
||||
},
|
||||
);
|
||||
await this.prisma.smsMessageRecord.update({
|
||||
where: { id: message.id },
|
||||
data: { errorMessage: reason },
|
||||
});
|
||||
const retried = await this.facade.submitMessageToGateway(
|
||||
message,
|
||||
routed,
|
||||
attempts.length,
|
||||
sourceAttempt.id,
|
||||
);
|
||||
this.logger.log(`sms_retry_route_selected ${JSON.stringify({
|
||||
const retried = await this.facade.submitMessageToGateway(message, routed, attempts.length, sourceAttempt.id);
|
||||
this.logger.log(
|
||||
`sms_retry_route_selected ${JSON.stringify({
|
||||
messageId: message.messageId,
|
||||
messageRecordId: message.id,
|
||||
groupId: routed.groupId,
|
||||
channelId: routed.channel.id,
|
||||
attempt: attempts.length,
|
||||
})}`);
|
||||
})}`,
|
||||
);
|
||||
return retried;
|
||||
} catch (error) {
|
||||
this.logger.error(`sms_retry_route_failed ${JSON.stringify({
|
||||
// Infrastructure failures and the completion routing probe must abort the
|
||||
// transaction; treating them as "no retry route" would finalize a failure.
|
||||
if (!(error instanceof HttpException) || error.getStatus() >= 500) throw error;
|
||||
this.logger.error(
|
||||
`sms_retry_route_failed ${JSON.stringify({
|
||||
messageId: message.messageId,
|
||||
messageRecordId: message.id,
|
||||
reason,
|
||||
attemptedChannelIds,
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
})}`);
|
||||
})}`,
|
||||
);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -368,7 +368,7 @@ export class SendSubmissionService {
|
||||
template?: { signature?: { id?: string | null } | null } | null;
|
||||
signature?: { id?: string | null } | null;
|
||||
},
|
||||
options: { forceNational?: boolean; excludeChannelIds?: string[] } = {},
|
||||
options: { forceNational?: boolean; excludeChannelIds?: string[]; previewOnly?: boolean } = {},
|
||||
): Promise<RoutedChannel> {
|
||||
return this.gatewaySubmit.selectChannelForMessage(message, options);
|
||||
}
|
||||
|
||||
@@ -1,16 +1,12 @@
|
||||
import { BadRequestException, HttpException, HttpStatus, Logger, NotFoundException } from '@nestjs/common';
|
||||
import { Prisma } from '@prisma/client';
|
||||
import { createHash } from 'node:crypto';
|
||||
import { Logger } from '@nestjs/common';
|
||||
import { BillingService } from '../billing/billing.service';
|
||||
import { moneyToNumber } from '../common/money';
|
||||
import type { OpenApiService } from '../open-api/open-api.service';
|
||||
import { PrismaService } from '../prisma/prisma.service';
|
||||
import type { GatewaySubmitResultDto, GatewaySubmitSegmentResultDto, GatewayReceiptEventDto, GatewayUplinkEventDto, GatewayPendingDeliveryQueryDto, GatewayDownstreamSentDto, GatewayDownstreamAcknowledgedDto, GatewayDownstreamFailureType, GatewayControlDeliveryResult, GatewaySubmitDeadLetterDto, RequeueGatewaySubmitExceptionDto, GatewayDownstreamRecoveryStatusDto, TimeoutUnknownDto } from './send-chain.contracts';
|
||||
import { positiveInteger, normalizeReceiptStatus, normalizeCarrier, DEFAULT_DOWNSTREAM_RETRY_DELAY_MS, DEFAULT_DOWNSTREAM_RETRY_MAX_DELAY_MS, DEFAULT_DOWNSTREAM_MAX_RETRIES, DEFAULT_DOWNSTREAM_PENDING_TIMEOUT_HOURS, DEFAULT_RECEIPT_TIMEOUT_HOURS, DEFAULT_GATEWAY_SUBMIT_REQUEUE_STALE_MS, DEFAULT_DOWNSTREAM_MANUAL_REQUEUE_STALE_MS, DEFAULT_UPSTREAM_RECEIPT_INBOX_PROCESSING_STALE_MS, DEFAULT_UPSTREAM_RECEIPT_INBOX_MAX_ATTEMPTS, DEFAULT_UPSTREAM_RECEIPT_INBOX_MAX_AGE_HOURS, gatewaySubmitRequeueKey, isObjectRecord, asDateOrNull, downstreamRetryDelayMs, downstreamAckTimeoutMs, downstreamRetryBaseDelayMs, downstreamRetryMaxDelayMs, downstreamMaxRetries, downstreamPendingTimeoutHours, downstreamControlFailureMessage, normalizeSubmitStatus, downstreamDeliveryAttemptKey, hasRecoveryAuditStateChanged, normalizeRecoveryFailureCategory, aggregateReceiptSegmentState, isSameUpstreamEndpointIdentity, receiptEventKey } from './send-chain.helpers';
|
||||
import type { SendSubmissionService } from './send-submission.service';
|
||||
import type { SendCompletionCallbacks, SendCompletionFacade } from './send-completion.service';
|
||||
import { completionContext } from './completion-context';
|
||||
import { queueFinalReceiptDeliveries } from './downstream-receipt-targets';
|
||||
|
||||
import type { TimeoutUnknownDto } from './send-chain.contracts';
|
||||
import { DEFAULT_RECEIPT_TIMEOUT_HOURS, downstreamPendingTimeoutHours, positiveInteger } from './send-chain.helpers';
|
||||
import type { SendCompletionCallbacks, SendCompletionFacade } from './send-completion.service';
|
||||
|
||||
/**
|
||||
* R10 timeout implementation.
|
||||
@@ -29,10 +25,12 @@ export class SendTimeoutService {
|
||||
) {}
|
||||
|
||||
async markUnknownTimeout(data: TimeoutUnknownDto) {
|
||||
const olderThanHours = data.olderThanHours ?? positiveInteger(process.env.SMS_RECEIPT_TIMEOUT_HOURS, DEFAULT_RECEIPT_TIMEOUT_HOURS);
|
||||
const olderThanHours =
|
||||
data.olderThanHours ?? positiveInteger(process.env.SMS_RECEIPT_TIMEOUT_HOURS, DEFAULT_RECEIPT_TIMEOUT_HOURS);
|
||||
const cutoff = new Date(Date.now() - olderThanHours * 60 * 60 * 1000);
|
||||
const candidates = await this.prisma.smsMessageRecord.findMany({
|
||||
where: {
|
||||
id: completionContext.getStore()?.messageRecordId,
|
||||
tenantId: { not: null },
|
||||
OR: [
|
||||
{ status: { in: ['submitted', 'unknown'] }, submittedAt: { lte: cutoff } },
|
||||
@@ -79,7 +77,10 @@ export class SendTimeoutService {
|
||||
// Refund uses the platform-message idempotency key. Re-running it for a
|
||||
// timeout whose downstream outbox was not fully queued also recovers a
|
||||
// crash between the state transition and the original refund call.
|
||||
await this.facade.refundMessage(candidate as typeof candidate & { tenantId: string }, `${olderThanHours}小时未收到明确回执,自动超时退款`);
|
||||
await this.facade.refundMessage(
|
||||
candidate as typeof candidate & { tenantId: string },
|
||||
`${olderThanHours}小时未收到明确回执,自动超时退款`,
|
||||
);
|
||||
const queued = await queueFinalReceiptDeliveries(
|
||||
this.prisma,
|
||||
(request) => this.facade.queueAndTryDownstreamDelivery(request),
|
||||
@@ -117,10 +118,7 @@ export class SendTimeoutService {
|
||||
const expired = await this.prisma.cmppDownstreamDelivery.findMany({
|
||||
where: {
|
||||
status: 'pending',
|
||||
OR: [
|
||||
{ lastRetriedAt: null, createdAt: { lte: cutoff } },
|
||||
{ lastRetriedAt: { lte: cutoff } },
|
||||
],
|
||||
OR: [{ lastRetriedAt: null, createdAt: { lte: cutoff } }, { lastRetriedAt: { lte: cutoff } }],
|
||||
},
|
||||
select: { id: true },
|
||||
take: 500,
|
||||
@@ -139,16 +137,21 @@ export class SendTimeoutService {
|
||||
if (this.receiptTimeoutScanRunning) return;
|
||||
this.receiptTimeoutScanRunning = true;
|
||||
try {
|
||||
const [receiptResult, downstreamResult, requeueRecoveryResult, downstreamManualRecoveryResult] = await Promise.all([
|
||||
const [receiptResult, downstreamResult, requeueRecoveryResult, downstreamManualRecoveryResult] =
|
||||
await Promise.all([
|
||||
this.facade.markUnknownTimeout({}),
|
||||
this.facade.markExpiredDownstreamDeliveries(),
|
||||
this.facade.recoverStaleGatewaySubmitRequeues(),
|
||||
this.facade.recoverStaleDownstreamManualRequeues(),
|
||||
]);
|
||||
if (receiptResult.timeout > 0) this.logger.log(`Marked ${receiptResult.timeout} messages as receipt timeout and refunded charged messages`);
|
||||
if (downstreamResult.failed > 0) this.logger.log(`Terminated ${downstreamResult.failed} expired downstream deliveries`);
|
||||
if (requeueRecoveryResult.recovered > 0) this.logger.log(`Recovered ${requeueRecoveryResult.recovered} stale Gateway submit requeues`);
|
||||
if (downstreamManualRecoveryResult.recovered > 0) this.logger.log(`Recovered ${downstreamManualRecoveryResult.recovered} stale downstream manual requeues`);
|
||||
if (receiptResult.timeout > 0)
|
||||
this.logger.log(`Marked ${receiptResult.timeout} messages as receipt timeout and refunded charged messages`);
|
||||
if (downstreamResult.failed > 0)
|
||||
this.logger.log(`Terminated ${downstreamResult.failed} expired downstream deliveries`);
|
||||
if (requeueRecoveryResult.recovered > 0)
|
||||
this.logger.log(`Recovered ${requeueRecoveryResult.recovered} stale Gateway submit requeues`);
|
||||
if (downstreamManualRecoveryResult.recovered > 0)
|
||||
this.logger.log(`Recovered ${downstreamManualRecoveryResult.recovered} stale downstream manual requeues`);
|
||||
} catch (error) {
|
||||
this.logger.error('Receipt timeout scan failed', error instanceof Error ? error.stack : String(error));
|
||||
} finally {
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
# 聆界短信 HTTP API 参考手册
|
||||
# 聆界短信平台 HTTP API 参考手册
|
||||
|
||||
**接口版本:v1 · 文档日期:2026-09-15**
|
||||
|
||||
## 1. API 概览
|
||||
|
||||
聆界短信 HTTP API 提供单条短信发送、发送结果查询和用户回复查询。平台还可将短信回执和用户回复主动推送到客户系统。
|
||||
聆界短信平台 HTTP API 提供单条短信发送、发送结果查询和用户回复查询。平台还可将短信回执和用户回复主动推送到客户系统。
|
||||
|
||||
### 1.1 服务地址与数据格式
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
# CMPP 短信平台第一版开发需求文档
|
||||
# 聆界短信平台第一版开发需求文档
|
||||
|
||||
## 0. 文档前提
|
||||
|
||||
@@ -2317,3 +2317,25 @@ Webhook需在当前受支持Node运行时通过真实HTTPS投递;SSRF校验后
|
||||
## 2026-09-15 HTTP调试工具与合并整改
|
||||
|
||||
继续前述新签名规则,客户端与运营端新增HTTP签名计算页:输入方法、路径、秒时间戳、nonce、AccessSecret和POST原始正文,本地生成HMAC-SHA256并复制,不上传/存储密钥、不发短信。修改输入使旧结果失效。运营端签名与引流报备统一三运营商列组件;客户端企业认证仅隐藏新申请入口,保留状态/进度。HTTP发送按独立HTTP总开关/发送能力校验,不能依赖CMPP开关。上行手机号过滤须1开头11位,接入号1–21位数字,非法返回400。设计及真实验收边界见http-api-assessment-20260910.md的2026-09-15补充。
|
||||
|
||||
## 2026-09-16 产品名称统一
|
||||
|
||||
产品正式名称为“聆界短信平台”,页面标题、API文档品牌名称、导出表格作者和充值回执替代文本统一。产品3.0、工程3.0.0不变;CMPP协议名称及技术标识保持兼容。名称调整仅涉及展示文案和导出作者元数据,不改变布局、业务流程、协议、接口地址或工程版本;版本元数据属于独立变更。
|
||||
|
||||
## 2026-09-16 长短信回执并发处理可靠性补充(待实施)
|
||||
|
||||
本节细化既有短信幂等、补发与计费一致性要求,不改变费率、分段汇总、补发条件或客户接口协议。不同分段回执同时到达时,同一次有效发送只能产生一份已提交的收尾决策;同一次失败最多创建一个后继补发。合法分段历史完整保留,HTTP按业务消息、CMPP按客户原始分段生成耐久通知,网络失败沿用同一投递身份重试。
|
||||
|
||||
处理进程退出后必须接续,不因“已认领”漏处理;旧持有者和旧尝试不得覆盖新的有效结果,不重复扣费、退款或补发。unknown及不完整分段继续等待既有条件,不能被永久完成标记阻止后续处理。历史数据不得因升级自动重发或重退。
|
||||
|
||||
实施设计和适用边界见 [发送链路第10节](phase-4-send-pipeline-redesign.md),验收为 [TC-RC-20260916-01~12](system-functional-test-cases.md)。本次仅方案,不代表线上修复或新增发送授权。
|
||||
|
||||
## 2026-09-16 五项联合整改(实施中)
|
||||
|
||||
1. 实施发送链路方案第10节,长短信回执按尝试耐久协调,保证并发、故障恢复、补发、账务和通知一致。
|
||||
2. 运营通道测试不进行签名/引流内容检测与报备校验,正文按输入交给指定通道;仍校验权限、号码/长度、通道启用/连接和协议参数。客户正常发送保持原规则。
|
||||
3. 双端签名工具增加短信参数独立输入、真实发送与应用层HTTP请求/响应展示,详见HTTP专项设计2026-09-16补充。
|
||||
4. 系统监控告警持久保留,仅手动清除,已读不清除、恢复不清除,新触发周期独立。
|
||||
5. 监控刷新失败显示真实上一快照和过期提示,超时/切换请求取消,恢复后刷新;不同查询范围隔离。
|
||||
|
||||
授权交付:代码、定向/全量测试、精确提交、推送、测试环境标准发布及隔离模拟短信验收;不操作预生产,不向真实运营商发送。
|
||||
|
||||
@@ -336,3 +336,7 @@ HTTP-FULL-B02:去除URL IPv6方括号后区分IP字面量与DNS,DNS失败转
|
||||
### HTTP-0915-B02 HTTP发送误依赖CMPP开关
|
||||
|
||||
真实测试企业HTTP enabled/sendEnabled=true、CMPP interfaceEnabled=false时,合法模板请求422,未创建短信。根因SendBatchEntryService.validateSendResources无条件读取interfaceEnabled,与需求HTTP客户接口第一版的独立开通矛盾。仅由内部HTTP_REQUEST_CONTEXT符号确认的HTTP请求传递httpRequest校验选项,再读取真实httpConfig.enabled/sendEnabled;普通客户端及CMPP路径保留原检查,企业认证、租户、应用状态及模板校验不变。没有数据库迁移和计费规则变化。须真实复验CMPP关闭HTTP送达、HTTP关闭拒绝及原CMPP拒绝回归。
|
||||
|
||||
## 2026-09-16 签名工具增加真实发送调试(实施中)
|
||||
|
||||
双端共用页面保留原始正文签名计算,增加短信调试区:AccessKey、AccessSecret、mobile、content、clientMessageId、Idempotency-Key分别输入。发送目标固定当前站点的/api/openapi/v1/sms/messages,按公开协议生成秒时间戳和nonce,使用同一份JSON字节计算签名并发送。密钥只在页面内存,不存储/上传AccessSecret;展示实际应用层请求方法、路径、请求头、正文和HTTP状态、可读取响应头、原始响应正文,不伪称浏览器隐藏头或HTTP版本为抓包结果。一次点击一次请求,发送期间禁用,不自动重试;超时提示结果未知,保留同一业务幂等键用于核对。新业务内容编辑后生成新幂等键,重试同一内容保持业务键。由原OpenAPI鉴权、应用权限、计费、风控处理,无绕过接口。覆盖失败响应、网络失败、超时、长短信、中文和特殊字符、双端三尺寸与真实发送验收。
|
||||
|
||||
@@ -195,3 +195,154 @@ GatewaySubmitOutbox
|
||||
- 修复后正价smoke为9/9受理,账单`9×325=2925`;20 TPS为199/199、账单64675;30 TPS为299/299、账单97175;50 TPS为499/499、账单162175。四个成功窗口共1006条,账单1006笔、单价均325、合计326950。
|
||||
- 50 TPS档499条非补发首次供应商Submit覆盖9.930秒,即`50.25条/秒`;相对同日改造前Outbox阶段的`33.07条/秒`提高约52%。该档客户端P50/P95/P99为35/76/135ms;首提499条唯一,全部尝试及Outbox各542条唯一,补发均关联原提交。
|
||||
- 压测结束Inbox、Submit Outbox和两条Redis Stream全部排空,数据库无重复Submit ID、无等待锁、无idle in transaction;回调池`max=12,total=1,idle=1,waiting=0`。隔离应用单价已恢复0,三条临时运营商规则已删除,真实账务事实保留审计。100/200/300/500未继续执行,当前验收结论限定为50 TPS档通过。
|
||||
|
||||
## 10. 长短信分段回执并发整改方案(2026-09-16,待实施)
|
||||
|
||||
### 10.1 目的、证据和适用范围
|
||||
|
||||
业务解释:一条长短信有多段回执,多个处理流程同时发现整条短信已经有结果,重复安排通知或补发。现有数据库唯一键能阻止重复创建,但不能避免此前重复查询、选路和事务开销。整改目标是“先认领处理资格,再做业务动作;中途退出后可以安全接续”。
|
||||
|
||||
依据:[预生产只读诊断](cpu-diagnosis-20260916.md)。2026-09-16 09:20~09:40,12682次下游回执唯一键冲突关联6335条长短信;1862次补发唯一键冲突关联1859个长短信原始提交。下游冲突关联消息的13373条Inbox均matched且attemptCount=1;补发6组抽样显示不同分段失败回执同时到达并争抢创建补发。核心回执、补发、提交文件在d13ca07至cbc4a03之间无变化。
|
||||
|
||||
缺陷有两层:单个receiptKey去重不能覆盖不同分段;补发“先查询、后选路、再创建”的流程存在并发窗口。CPU峰值包含I/O等待,缺少业务进程历史CPU与慢SQL,不能承诺消除此缺陷即可消除全部CPU高峰。
|
||||
|
||||
本节是现有发送链路的专项补充,作为本次并发整改的设计依据;细化第4节结果处理及第7节不重不漏验收要求,不替代路由、计费、长短信协议或历史容量结论。不扩展到批次编号冲突、通道容量改造或历史短信重发。用户本轮仅要求方案文档,以下模型、状态、参数均是拟实施设计。
|
||||
|
||||
### 10.2 必须保持的业务规则
|
||||
|
||||
1. 原始回执逐事件持久化,同一事件幂等;不同分段的合法回执不能当成重复而丢弃。
|
||||
2. 成功依现有通道回执模式汇总:per_segment须齐段成功,message_level保持既有整条成功语义。不得为降低冲突把第一段成功视为整条成功。
|
||||
3. 同一次有效发送的失败只能触发一份补发决策;是否允许补发仍检查当前应用、通道组、报备、风控、时间上限等既有规则。unknown不触发补发,72小时及组内时限规则保持。
|
||||
4. 旧尝试的迟到回执保留审计,不覆盖后续有效尝试的最终成功,不再次触发旧尝试补发或退款。矛盾回执按既有规则记录异常,不采用“谁抢到锁谁决定事实”。
|
||||
5. 补发不是消息最终失败;仍有有效补发时不得提前生成业务最终失败通知或最终退款。
|
||||
6. HTTP回调保持业务消息维度;CMPP回执按客户原始分段及Registered_Delivery要求生成。内部上游分段与客户分段不能混用。
|
||||
7. 现有提交、回执、补发、投递和账务唯一约束全部保留;计费单位、价格、扣退费条件及租户权限不变。
|
||||
8. 对外投递仍是可重试的至少一次交付,不能承诺网络侧绝对只收到一次;整改保证同一逻辑通知仅创建一份耐久投递事实,重试沿用其身份。
|
||||
|
||||
### 10.3 影响模块与最小实施边界
|
||||
|
||||
| 所有者 | 拟修改内容 |
|
||||
|---|---|
|
||||
| send-receipt.service.ts、send-chain.helpers.ts | 保存分段后提交汇总工作;按当前持久化事实重新汇总和校验有效尝试,移出重复终态副作用 |
|
||||
| send-retry.service.ts、send-gateway-submit.service.ts | 统一补发处理资格,认领后选路,事务内验证资格并创建Submit与GatewaySubmitOutbox |
|
||||
| send-gateway-result.service.ts、send-timeout.service.ts、send-downstream-delivery.service.ts | 清查提交失败、超时、平台失败等旁路,接入同一业务消息终态协调,不留下与回执处理竞争的独立写入口 |
|
||||
| send-accounting.service.ts、send-completion.service.ts | 核实稳定账务键和事务接口;重复恢复不重复扣退,不能用已完成标记替代实际账务事实 |
|
||||
| downstream-receipt-targets.ts、HTTP Webhook生产端 | 保持目标分段契约,幂等落库后再独立投递,重复命中不直接再次发送 |
|
||||
| Prisma、回调进程恢复扫描、定向与集成测试 | 增量状态模型、认领/恢复索引、有限批次消费与故障注入 |
|
||||
|
||||
预期主要为API发送链路和数据库迁移;不新增页面、不改变客户HTTP参数、签名算法、错误码和CMPP协议。Gateway命令契约预期不变,若实现必须变更,先补契约及Go测试,不能隐式扩大范围。需检查全部调用者而非只在回执方法外套进程内锁。
|
||||
|
||||
### 10.4 拟增数据模型与认领规则
|
||||
|
||||
采用PostgreSQL耐久工作记录和短事务条件更新,避免仅靠进程内锁或Redis短期锁。拟新增 `SmsAttemptCompletionWork`,每个sourceSubmitRecordId唯一;该名称及字段在实施迁移前与现有模型再次核对。无供应商Submit的业务拒绝使用独立的稳定事件键,不伪造Submit ID。
|
||||
|
||||
| 字段 | 含义及约束 |
|
||||
|---|---|
|
||||
| id、workKey | 主键及唯一工作键;有Submit时固定为attempt:<sourceSubmitRecordId> |
|
||||
| tenantId、messageRecordId、sourceSubmitRecordId | 与真实记录归属一致;不信任外部回调传入的租户或消息归属 |
|
||||
| revision、processedRevision | 收到新的有效事实递增revision;完成时只确认已处理版本,避免并发新回执被覆盖 |
|
||||
| state | pending、processing、retry_wait、idle、needs_review;idle表示当前事实已处理,不等于永不再接收新事实 |
|
||||
| leaseOwner、leaseUntil、fenceVersion | 租约及递增认领版本;旧持有者不能提交后续业务动作 |
|
||||
| decision、retrySubmitRecordId | 本次已提交决策及已有补发引用;补发一旦创建,不因恢复或后续分段重新选路创建 |
|
||||
| attempts、nextAttemptAt、lastError、createdAt、updatedAt | 恢复、退避和诊断信息;错误脱敏,不复制短信正文或凭据 |
|
||||
|
||||
索引:唯一workKey、非空sourceSubmitRecordId唯一;按state/nextAttemptAt及state/leaseUntil支持有界扫描;messageRecordId支持对账。不按手机号或高基数业务ID创建监控标签。
|
||||
|
||||
认领以数据库时间为准:pending、到期retry_wait或租约过期processing可由条件更新取得;认领成功递增fenceVersion。建议初始租约60秒、每20秒续租、每次扫描不超过32条,作为测试起点而非容量结论。连接池预算、扫描间隔及退避上限须通过隔离环境实测定稿。
|
||||
|
||||
每次持久化副作用都在短事务中锁定工作记录并核对认领版本、租约及当前有效尝试,同时锁定对应业务消息,使不同尝试/超时入口也不能交错覆盖最终状态。锁顺序统一为工作记录→业务消息→既有账务锁;禁止反向调用形成死锁。不得持有事务等待限速、网络或Redis。
|
||||
|
||||
### 10.5 事实保存、业务收尾和补发流程
|
||||
|
||||
**A. 接收与汇总**
|
||||
|
||||
1. 维持耐久Inbox接收。处理单个回执时,将回执事实、分段审计和工作记录唤醒放入同一短事务;唯一事件已存在时,也须核实存在对应工作,不直接忽略未完成收尾。
|
||||
2. Inbox的matched表示关联事实已可靠保存,业务收尾由工作记录跟踪;两者状态不能混为一个“完成”。在事务提交前退出,由Inbox恢复;提交后退出,由工作扫描恢复。
|
||||
3. 工作消费者认领后,重新读取当前Submit、所有有效分段及业务最终状态。未齐段成功则结束本轮为idle,保留processedRevision;后续新事实使其再次pending。unknown也不能被永久锁死,按既有状态机处理后续明确结果或超时。
|
||||
4. 确认最终结果时,事务内重新核验revision、当前尝试和认领版本。有新事实或尝试已变更,重新汇总或记历史,不使用认领前的旧message对象直接更新。
|
||||
|
||||
**B. 允许补发**
|
||||
|
||||
1. Submit失败入口与回执失败入口都唤醒同一sourceSubmitRecordId的工作;取得资格后才进行选路等高成本操作。
|
||||
2. 优先查询已有retryOfSubmitRecordId对应的Submit/Outbox。存在则复用,不再次选路、扣费或创建;不存在才评估业务规则并选择候选。
|
||||
3. 选路和限速等待不持数据库锁,长等待续租;失去租约即停止。最终短事务核验资格、当前尝试及必要的实时启用/报备条件,原子提交补发Submit、消息当前尝试更新、GatewaySubmitOutbox和工作决策。
|
||||
4. 事务失败全部回滚;成功后由原有Outbox发布器发送,不由认领流程直接调用供应商。不可把“处理中”当成“补发已成功创建”,也不可因此提前退款。
|
||||
5. 临时数据库/连接错误进入retry_wait,不能误判为“无通道、最终失败”。确定不允许补发后,才走既有最终失败规则。
|
||||
|
||||
**C. 最终业务结果与客户通知**
|
||||
|
||||
最终消息状态、必要账务变化和全部应生成的耐久通知事实,应在同一短事务中提交。需让相关服务接受事务客户端;HTTP使用既有HttpWebhookEvent/Delivery模型,CMPP使用CmppDownstreamDelivery,不在事务内向客户发网络请求。
|
||||
|
||||
如实际账务接口无法共用事务,实施必须先补充稳定业务键的耐久分步恢复设计并验证全部崩溃点,不能先标工作完成再异步裸调用退款/通知。整条业务完成与单次发送尝试结束须分开:已安排补发只结束旧尝试,不给客户发最终失败。
|
||||
|
||||
正常幂等命中使用针对已知唯一键的无异常插入(例如参数化INSERT ON CONFLICT ... DO NOTHING RETURNING)并回读核实归属和内容;不能把所有数据库错误当重复成功。补发/回执唯一约束仍是最后防线,非预期冲突保留告警。
|
||||
|
||||
### 10.6 故障恢复与历史兼容
|
||||
|
||||
| 中断点 | 恢复预期 |
|
||||
|---|---|
|
||||
| 回执事实事务提交前 | 事务回滚,Inbox按既有机制再处理 |
|
||||
| 事实已保存、工作尚未被领取 | 耐久pending扫描继续;不得依赖setImmediate一定执行 |
|
||||
| 认领后、选路中进程退出 | 租约到期由新消费者接续,旧版本写入被fence拒绝 |
|
||||
| Submit/Outbox事务提交结果不确定 | 按原始提交唯一键回读;已有记录继续原Outbox,不创建新Submit |
|
||||
| 最终状态/账务/通知事务中退出 | 全部回滚或全部提交;恢复查稳定业务键,账务不重复 |
|
||||
| 通知已入库、网络投递失败 | 原有投递记录重试;不重新运行短信补发决策 |
|
||||
| 消费者持续失败 | 有限指数退避,达到配置上限进入needs_review并告警;不得伪造业务成功或静默丢弃 |
|
||||
| 同时有新分段或迟到事实 | revision差异驱动再次汇总;旧尝试只留审计,已完成副作用不重复 |
|
||||
|
||||
迁移只增表/索引,不删除历史回执、补发、账单或唯一约束。启用前暂停受影响消费者并完成现有任务排空,不能让旧版直接处理与新版协调器长期并行。
|
||||
|
||||
切换时建立准确切换时间及待处理清单:未完成Inbox、正在处理回执以及尚未完成收尾的业务记录进入受控衔接;有最终通知/账务/补发事实的历史记录以事实作为已完成依据。不能仅因新工作表为空扫描全部历史短信安排补发。历史异常修复另行设计、授权和审计。
|
||||
|
||||
上线前必须验证回执先于Submit结果、历史缺失Submit关联、无租户通道测试短信和平台业务拒绝等路径。无法可靠关联的记录保留待匹配/人工排查,不推测来源、不误收尾。
|
||||
|
||||
### 10.7 观测、性能与验收证据
|
||||
|
||||
增加低基数指标:工作认领成功/竞争未获得、租约接管、旧版本拒绝、恢复次数、needs_review数量、最老未完成时长、最终动作数量。重复命中记录计数和必要采样,不让正常幂等持续生成ERROR堆栈;真实失败仍明确告警。
|
||||
|
||||
业务验证必须对账:每个源Submit最多一个有效后继补发;每个客户目标最多一份逻辑通知;每个账务业务键最多一笔;最终状态与分段事实/最新有效尝试一致;Inbox、工作表、Submit Outbox、Redis和客户投递都有可解释的完成或等待原因。
|
||||
|
||||
性能对比使用同环境、同版本以外条件、同长短信比例、同成功失败分布和同负载,分别记录入口、首次Submit、总Submit、最终送达、分段回执速率及窗口。记录CPU用户态/内核态/I/O wait、数据库写入与锁等待、队列积压及回执完成延迟。新增进程CPU/SQL采样需低开销评估,不擅自在线开启全量SQL日志。
|
||||
|
||||
确定性并发用例要求不再出现上述两类预期竞争导致的23505异常,正常情况下每次有效尝试仅一份已提交决策;故障恢复可重复计算,但不能重复业务副作用。性能无倒退且队列按批准的排空时限清空;执行负载前固定对比基线和延迟阈值,不事后选窗口宣称提升。不得承诺未经测量的CPU降幅或TPS。
|
||||
|
||||
### 10.8 实施顺序、成本和发布恢复
|
||||
|
||||
1. 补定向失败用例复现两段同时成功/失败;盘点所有终态和补发调用路径及账务事务能力,形成事务边界清单。
|
||||
2. 实施增量模型、原子认领、revision/fence和扫描恢复;先验证多进程争抢及旧持有者失效。
|
||||
3. 接入回执、提交失败、超时及平台失败路径;事务化Submit/Outbox与最终通知/账务。保持外部协议和业务规则。
|
||||
4. 执行定向测试、API全量回归、类型检查、构建和现有质量门禁;真实PostgreSQL并发及迁移测试必须通过。Gateway有修改再执行Go测试及vet。
|
||||
5. 在另行明确的隔离模拟范围内,以真实API、PostgreSQL、Redis、Gateway和Webhook接收端验收,包含非零计费、重启故障、回执乱序及排空对账;页面检查发送详情和批次最终状态。mock只能证明隔离逻辑。
|
||||
6. 真实回归通过后,按明确提交/推送/目标环境授权交付;部署使用标准release流程和独立恢复资产。测试与预生产分别验收,预生产不得以历史授权直接补发或压测。
|
||||
|
||||
这是跨回执、补发、账务和耐久任务恢复的修复,成本主要在事务改造及故障验收,不能仅以修改几行或压低日志级别交付。完成第1步盘点后再给出实现工时;若需改变计费、状态优先级或Gateway契约,先记录范围扩展。
|
||||
|
||||
恢复策略:迁移保留增量表;发布失败先停止新旧受影响消费者交叉执行,盘点未完成工作、已生成Submit/Outbox/账务/投递事实,再选择经过兼容验证的应用版本恢复。旧版本不理解新工作表,不能直接回退并声称全部恢复;必须证明待处理工作已排空或存在经过验证的接续路径,否则保持暂停并修复。禁止删除工作记录、回滚业务数据或重新入队来掩盖异常。
|
||||
|
||||
上线停止条件:发现重复供应商Submit、重复扣退费、终态错误覆盖、持续积压、无法接管或恢复缺口立即停止放量,保留现场;不通过手工补发完成验收。
|
||||
|
||||
### 10.9 完成标准与本轮状态
|
||||
|
||||
验收用例见 [系统功能测试用例](system-functional-test-cases.md) 的TC-RC-20260916-01~12;原TC-SEND-018/019/020继续执行。记录见 [测试进度](testing-progress.md)。
|
||||
|
||||
只有代码、真实并发/故障恢复、账务与队列对账均通过,并完成对应环境独立验收,才能标为该环境已修复。2026-09-16本轮只编写整改方案与用例,未实现模型或业务代码,未运行发送、迁移、提交、推送、测试部署或预生产部署。
|
||||
|
||||
### 10.10 2026-09-16 实施细化(开发中,尚未验收)
|
||||
|
||||
用户已明确授权实施第10节、提交、推送及测试环境部署;第10.1、10.9的“仅方案”状态是此前记录,当前开始实施,不能据此宣称已上线。
|
||||
|
||||
- 增加 `SmsCompletionEvent` 耐久事件日志,与工作 revision 在同一接收事务提交;Inbox matched 表示原始事件已存入该日志。规范化 Receipt/Segment 事实、汇总和副作用由收尾事务一并提交。此细化替代10.5 A1中先写规范化表的顺序,避免规范化事实落库后收尾未接续的窗口。日志保留原始事件,不能将未处理日志计作已完成业务回执。
|
||||
- 回执、整条提交结果、分段提交结果、超时与平台拒绝通过统一入口入日志;先按可靠关联核验消息及租户。无可靠Submit关联的供应商回执继续留在Inbox待匹配;无供应商Submit的平台拒绝使用message工作键。迁移不扫描历史消息重发。
|
||||
- 所有收尾协作者通过仅限发送链路的事务适配器加入当前短事务,账务内层事务加入同一事务。锁顺序为工作→消息→账务。HTTP通知生产端显式接收事务客户端,CMPP通知无异常幂等创建;事务内仅写耐久通知,不执行网络投递。
|
||||
- 补发首先认领。首次尝试到达选路点时回滚探测事务,在事务外重新选路和限速;最终事务重新读事件、revision、有效尝试与当前路由条件,原子创建Submit/Outbox。回滚产生的会话ID不得进入跨事务缓存。临时故障必须抛出并退避,不能被吞成最终失败。
|
||||
- 恢复扫描初始5秒/32条,租约60秒、20秒续期、事务20秒上限。最多12次失败进入needs_review;这些是待真实环境验证的初值。异步通知仍由原恢复消费者投递。
|
||||
- 超时与迟到结果保持已退款状态,不重复扣退;每个历史业务键依据实际账务和通知事实恢复。上线前必须以故障与乱序测试核实,不把代码编译当作证明。
|
||||
|
||||
待补证据:真实PostgreSQL并发、fence过期、事务中断、队列排空、长短信CMPP与HTTP通知以及账务对账。当前仍为开发中。
|
||||
|
||||
### 10.11 本次实现边界与发布依赖(2026-09-16)
|
||||
|
||||
- 恢复扫描在worker/callback/all角色运行;正式运行必须启用SEND_SUBMIT_OUTBOX_PUBLISH_ENABLED及对应发布进程。测试环境18:20只读核验两项Outbox开关为true、独立服务active。关闭发布器不属于受支持上线组合,不能把pending当成已发送。
|
||||
- 收尾needs_review及等待超过300秒由监控采集器直接查询耐久工作/事件表,写入同一告警事件库,恢复仍保留、仅人工清除;不依赖应用发布工具安装Prometheus规则。过程计数仍提供低基数metrics,采集进程口径需区分。
|
||||
- 两项迁移仅新增四张表及索引,无历史回填、无删除、无自动重发;全量106项已在新隔离数据库重放。工作时间使用UTC表达式,避免数据库会话时区与Prisma时间不一致。应用回退前必须盘点和接续未完成工作,旧代码不能消费这些新表。
|
||||
- 本地验证包括真实PostgreSQL、两个独立OS进程、事务回滚、旧消费者挂起后接管、三段通知目标、非零账务和重试上限可见告警;网络投递与目标环境的重启/排空验收独立记录,不将路由隔离测试冒称整链路通过。
|
||||
|
||||
@@ -256,3 +256,11 @@ type InfrastructureOverview = {
|
||||
新增GET /admin/infrastructure-monitoring/alert-history,继承运营端会话鉴权,参数from/to为上海自然日,默认含今天近7日、最多31日,page正整数、每页25条。逐日查询Prometheus原始ALERTS_FOR_STATE范围向量,以标签指纹+activeAt分开触发周期,跨日采样合并;不使用粗粒度步长丢掉短周期,不把等待触发当作已发送告警。保留真实触发时间和范围内最后观测时间,最后观测不代表准确恢复。
|
||||
|
||||
历史和活动列表独立;历史不提供伪造恢复状态、旧annotations、批量已读或清理功能。超出Prometheus保留期/采集缺口的历史无法追溯,界面明确说明。失败返回503并展示错误,日期非法返回400;不迁移数据库、不变更阈值或采集配置。完整实现/验收及限制见operations-fixes-20260909.md。
|
||||
|
||||
## 2026-09-16 人工清除及刷新失败保留真实快照(实施中)
|
||||
|
||||
本节替代活动告警随Prometheus恢复自动消失及失败清空指标的旧规则。新增PostgreSQL告警事件记录,以fingerprint+activeAt唯一标识一次触发;后台定期采集及页面读取时持久化。指标恢复仅标识已恢复,告警仍在待处理列表,只有运营手动清除(全局生效、记录操作人及时间)才退出;已读保持个人维度,不等于清除。同周期清除后不被下次采集重新激活,新周期独立产生。采集失败不自动推断恢复,不删除记录。历史审计保留,不能将Prometheus过期历史伪造为已完整迁入。
|
||||
|
||||
页面按查询范围在组件内存保留最后成功的真实快照;失败/超时显示错误和最后成功采样时间,不伪装实时值,不通过localStorage生成业务事实。不同时间范围不能串用缓存;请求有真实AbortController超时、切换/卸载取消和过期响应保护,在线恢复触发刷新;保留会话失效/锁定处理。根因验收需记录网络失败与后端采集失败两种路径,并验证长停留、并发切换及三尺寸。
|
||||
|
||||
2026-09-16实施补充:收尾工作人工处理/积压告警由采集器直接读取PG,不需要额外安装Prometheus规则;持续条件复用同一发生周期,即使已清除也不在同周期重建,恢复后再次触发才是新事件。手动清除同步移除页面各范围缓存中的同一事件及数量,避免后续刷新失败使已清除告警重新显示。
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
# CMPP 短信平台第一版系统功能测试用例
|
||||
# 聆界短信平台第一版系统功能测试用例
|
||||
|
||||
## 1. 用例说明
|
||||
|
||||
@@ -5564,3 +5564,45 @@ CLIENT-0914-01~07 的模板样式/顺序、文档归属与检索、中文状
|
||||
- 回调:成功单次;503 自动重试成功;400 单次终止;超时达到隔离配置两次上限。真实 HTTPS 接收与原字节 HMAC 验证 16/16,保留两项预期失败,不人工重投。
|
||||
- 收尾:25 条消息终态、25 个批次终态、23 个发送 Outbox dispatched;12 条送达按 325 内部单位计费,失败退款 325,最终余额 996100,冻结/释放相抵,三个消费组无积压。专用测试配置停用后数据和余额保留。
|
||||
- 测试环境浏览器 UI 及独立验证码策略不计通过,见 [2026-09-15 记录](http-api-acceptance-20260915.md)。
|
||||
|
||||
## 产品名称统一验收(2026-09-16)
|
||||
|
||||
- TC-BRAND-20260916-01:浏览器标题、平台Swagger、客户OpenAPI及HTTP文档展示“聆界短信平台”;充值回执图片替代文本、两类报备导出工作簿作者一致。源码检查与构建结果见测试进度,真实页面/API/导出验收单独记录。
|
||||
- TC-BRAND-20260916-02:产品3.0/工程3.0.0、客户接口独立版本、CMPP协议、API路径、签名规则、工程包名和基础设施标识保持不变;已有工作区修改不被覆盖。
|
||||
|
||||
## 长短信回执并发整改验收(2026-09-16,待实施、未执行)
|
||||
|
||||
设计依据:[发送链路整改方案第10节](phase-4-send-pipeline-redesign.md#10-长短信分段回执并发整改方案2026-09-16待实施)。下列用例均为P0,通用前置条件为明确授权的隔离测试企业/应用、模拟号码/通道,真实API、PostgreSQL、Redis、Gateway和Webhook接收端;保留非零计费规则。测试执行本身不因用例存在而获得发送、补发、配置变更或压测授权。
|
||||
|
||||
每例保留messageId/sourceSubmitRecordId、脱敏回执时间线、工作记录及认领版本、Submit/Outbox数、客户逻辑通知及实际投递次数、账务流水和队列结果。实际投递重试与重复创建分别计数。故障测试使用隔离环境可控断点或进程终止,禁止预生产注入。
|
||||
|
||||
| 编号 | 场景与步骤 | 通过标准 |
|
||||
|---|---|---|
|
||||
| TC-RC-20260916-01 | 两段、三段成功回执通过不同消费者同时进入;再逐段延迟和逆序各执行一组 | per_segment最后一段成功前不判整条成功;齐段后仅一份最终业务决策;按客户目标生成规定数量回执,无多余逻辑通知和去重键异常 |
|
||||
| TC-RC-20260916-02 | 两段、四段失败回执同时进入,开启允许补发且有可用模拟候选 | 每个源Submit仅一个后继Submit及对应Outbox;竞争失败流程不重复选路;模拟器无因本缺陷多出的Submit;无retryOf唯一键异常 |
|
||||
| TC-RC-20260916-03 | 同一失败同时由Submit结果和回执入口触发;再与超时处理竞争 | 入口共用协调;按既有状态规则只提交有效决策;不得一边补发一边最终退款/通知失败 |
|
||||
| TC-RC-20260916-04 | 相同回执重复20次,再混合不同分段并发;用两个回调消费者处理 | 原始合法分段保留,同一事件幂等;副作用数不随事件重复数增长;不存在仅单进程锁有效的问题 |
|
||||
| TC-RC-20260916-05 | 旧尝试失败后新尝试成功,再注入旧尝试成功/失败回执;另测message_level成功后矛盾失败 | 最新有效成功不覆盖、不重复退款补发;历史完整;矛盾事件按既有异常规则记录 |
|
||||
| TC-RC-20260916-06 | 注入unknown、缺一段、回执先于Submit结果;后续补齐合法事实;测试72小时超时 | unknown不补发,缺段不误成功,后续事实可唤醒idle;超时后迟到结果遵守既有规则;未匹配事件不被伪造关联 |
|
||||
| TC-RC-20260916-07 | 在事实保存前、保存后未认领、认领后未提交三个断点退出并重启 | Inbox或耐久工作分别接续;不漏回执、不丢收尾;每例在预先配置的恢复预算内完成或明确告警 |
|
||||
| TC-RC-20260916-08 | 挂起旧消费者至租约过期,让新消费者接管,再恢复旧消费者;选路期间加入新事实 | 新fence有效,旧fence不能创建Submit/退款/通知或覆盖状态;revision不丢;允许重新计算但无重复副作用 |
|
||||
| TC-RC-20260916-09 | 在补发Submit/Outbox事务提交前后、最终状态/账务/通知事务提交前后断开连接或退出 | 未提交全部回滚,已提交通过稳定键回读;恢复不新建第二份补发或账务;短信记录、账户流水、Outbox和通知一致 |
|
||||
| TC-RC-20260916-10 | HTTP通知成功、503、超时;CMPP客户断线重连;含多客户分段和Registered_Delivery=false | HTTP业务事件及CMPP应通知目标身份稳定;网络重试使用已有投递记录;不因客户回调失败补发短信;不要求回执的分段不下发 |
|
||||
| TC-RC-20260916-11 | 升级前准备历史已完成、在途、缺Submit关联、无租户通道测试、平台拒绝数据;演练迁移、切换与恢复 | 已完成历史不重发、不重退;在途接续清单完整;归属不明保留待处理;测试消息不通知客户;旧版本恢复前必须验证未完成工作接续,跨租户关联拒绝 |
|
||||
| TC-RC-20260916-12 | 相同负载及长短信/失败比例做修复前后对比,并注入临时数据库错误、候选失效和持续失败 | 认领后实时校验有效;临时故障不当最终失败;持续失败进入可见告警;正常并发无两类预期冲突;无业务重复、账务不符或持续积压,按预设延迟/排空预算通过 |
|
||||
|
||||
附加回归:TC-SEND-018补发停止条件、TC-SEND-019旧回执、TC-SEND-020账务幂等、单段短信、黑名单/频控/签名/引流报备、租户和应用停用、批次进度及发送详情真实回执时间。并发模拟通过不替代API全量、类型、构建与质量门禁;Gateway改动另执行Go测试和vet。任何无法执行项明确登记,不以mock计为真实验收。
|
||||
|
||||
## 五项联合整改验收补充(2026-09-16)
|
||||
|
||||
此前RC-01~12为本次实施矩阵,不能沿用“待实施”标题作为当前事实;执行结果逐项记录,未完整覆盖的条目保留未完成。
|
||||
|
||||
| 用例 | 验收内容 | 当前证据 |
|
||||
|---|---|---|
|
||||
| TC-OPS-0916-01 | 通道测试无签名、未报备URL及长短信;真实Gateway最终检查仍放行;普通客户保持原拦截 | 最终guard定向4项通过;真实Gateway待执行 |
|
||||
| TC-OPS-0916-02 | 双端参数独立输入、原始UTF8签名、真实请求/响应、幂等键保留、重复点击/超时不自动重发 | 前端定向通过;真实浏览器待执行 |
|
||||
| TC-OPS-0916-03 | 告警恢复、刷新、重启不清除;手动清除有审计且并发幂等;同周期不重现,新周期可见 | 真实PG通过;页面待执行 |
|
||||
| TC-OPS-0916-04 | 监控刷新失败保留当前范围最后成功快照并标明非实时;网络恢复、路由切换与三尺寸 | 组件测试通过;真实浏览器待执行 |
|
||||
| TC-OPS-0916-05 | 收尾第12次失败进入needs_review,自动出现在耐久告警;恢复后不自动消失 | 真实PG通过 |
|
||||
|
||||
RC-01/02/04/08/09/10/11目前仅部分本地证据:三段齐段、重复与双进程、事务回滚和过期接管、通知目标数与非零账务、迁移重放;尚不能标整条矩阵通过。RC-03/05/06/07/12的完整组合、网络故障/进程重启、修复前后性能对比须独立补验。无真实运营商流量,未声明容量提升。
|
||||
|
||||
@@ -5058,3 +5058,36 @@ CUA本轮可用,实际后端文档三尺寸1600×1000/1366×768/390×844无页
|
||||
代码 c781313 与补充修复 cbc4a03 已提交推送,测试环境精确运行 cbc4a033251c7496414b1e7aa6233ed49bd108ec(17:36 核验)。包含最新签名协议、双端计算页、报备三列及认证入口隐藏,未遗漏前序范围。最终后端 830/830、前端 158/158,类型/构建/质量门禁通过;最终真实参数回归 78/78,前序独立配置回归 15/15,引流六分支复测通过。25 条隔离记录、13 次模拟 Submit,账务余额与流水一致、冻结净额 0、队列 pending/lag 均 0。Webhook 成功/503 重试/400 终止/超时上限及上行已验收,16 次业务回调 HMAC 正确。专用配置、凭据和模拟器已停用,hosts 精确恢复,未发送至真实运营商。
|
||||
|
||||
最终标准 verify 唯一提示为业务计数 +6 条短信/+2 次 Submit,已与本轮六个引流用例逐条核对,工具状态保留 deployed-needs-review,不伪造全绿。浏览器连接超时,测试环境登录后 UI 尚未验收;独立“验证码限制”的具体规则仍需明确。预生产未操作,容量治理未完成。完整证据、失败记录、时间、版本与恢复资产见 [HTTP 接口联合验收记录](http-api-acceptance-20260915.md)。
|
||||
|
||||
## 2026-09-16 产品统一更名为聆界短信平台(本地完成)
|
||||
|
||||
- 用户要求将项目产品名称由CMPP平台更名为“聆界短信平台”。设计与兼容边界维护于version-3.0-baseline.md“产品名称统一”,需求及TC-BRAND-20260916-01/02同步。
|
||||
- 已更新浏览器标题、登录页标题/Logo替代文本、双端AppShell品牌名称、充值回执替代文本、平台Swagger/客户OpenAPI名称、HTTP文档页面与MD、两类报备导出工作簿作者,以及当前根规范/UI规范/需求/测试用例/版本基线名称。历史实施记录不追溯改写;CMPP协议、工程包名、服务/存储标识、目录路径、业务逻辑和接口签名不变,工程版本保持3.0.0。
|
||||
- 11:27 CST验证:前端33文件158项通过,后端76套831项通过;前端tsc --noEmit及最终Vite生产构建、API tsc -p tsconfig.build.json通过;结构质量检查、包体预算检查、git diff --check通过。构建HTML标题与编译后的HTTP文档渲染函数确认新名称。当前终端npm不在PATH,直接调用仓库node_modules内对应CLI完成等价验证,未改全局环境。
|
||||
- 质量门禁遗留:定向ESLint在official-export.service.ts报告47个原有未使用导入错误,另有LoginPage/RechargeReceiptDialog共3个既有警告;Prettier在official-export.service.ts和RechargeReceiptDialog.tsx报告原有格式问题。已将名称替换在内存中还原后复核,两文件修改前格式检查仍失败、official-export修改前仍47个lint错误;未以整文件格式化扩大范围,不能宣称全部门禁通过。初次Vite构建存在既有大chunk提示。
|
||||
- 真实浏览器三尺寸、登录后页面、Swagger真实API及报备导出验收未执行;当前项目无可解析的Playwright依赖,未安装依赖或将隔离测试冒充真实业务验收。未连接/修改线上服务,未发送、补发、重投短信或改业务配置。
|
||||
- Git核验main/实际远端仍为86cb9aea36f7759eef9cc4f0949a9bca7f85617b,暂存区空。已有metrics、发布工具、部署脚本、版本及文档草稿保留,仅对共享文档精确替换名称/追加本轮记录。本地修改完成;本地提交、推送、测试部署、预生产部署均未执行。
|
||||
|
||||
### 2026-09-16 更名本地提交范围核对
|
||||
|
||||
用户追加授权本地提交。本次精确暂存更名代码、规范标题及需求/用例/进度中的更名段落;package及lock版本、main.ts的Swagger版本变更和既有未跟踪version-3.0-baseline.md草稿不纳入本次提交。3.0.0版本元数据仍留在工作区,提交不代表版本标识已发布。前述测试为本地工作区结果,不冒称隔离候选精确提交的全量验证。既有metrics、发布工具、部署脚本与其他文档改动保留。推送、测试部署及预生产部署未执行。
|
||||
|
||||
## 2026-09-16 五项联合整改启动
|
||||
|
||||
用户确认实施phase-4-send-pipeline-redesign.md第10节,并授权通道测试免签名/引流检测、HTTP参数化发送调试、监控告警人工清除与刷新保留真实快照;提交、推送及测试环境发布/充分长短信验收。main=a0209f93bc8eee6cb08a9d7cace4163b26749ae6,实际远端86cb9ae,暂存区空。61份已有修改/草稿保护摘要及原文位于.local-data/five-fixes-20260916,前轮版本及发布工具不自动纳入。测试SSH首次连接超时,尚未认证;本地工作继续,部署/真实环境待连通后执行。尚未修改业务代码或操作线上状态。
|
||||
|
||||
## 2026-09-16 五项改造开发进展(18:00,本地,未提交/推送/部署)
|
||||
|
||||
- 承接用户确认:执行发送链路第10节、通道测试跳过签名与引流、HTTP参数化真实发送调试、告警手动清除、监控失败保留最后成功快照。测试机开机后SSH已恢复;在线版本仍为cbc4a033251c7496414b1e7aa6233ed49bd108ec。预生产未操作。
|
||||
- 新增耐久工作/事件日志及告警事件/采集水位迁移;按工作→消息→账务/批次顺序收尾,补发选路限速在事务外,通知/账务/Outbox入同一事务。修复探测异常被吞、事务会话缓存、迟到结果重复计费、超时终态覆盖及多消息批次进度事务隔离细节。
|
||||
- 通道测试除去入口检测,同时修正Gateway最终引流授权入口;仅按真实Submit归属确认的无租户/无批次测试消息豁免,普通客户消息仍校验。
|
||||
- 本地隔离PostgreSQL 16434/cmpp_qa_completion_20260916已执行全量106迁移(后续UTC默认值细化尚需新库重放)。真实数据库验证并发24事件/12唯一事实、事务故障回滚与接续、租约接管、三段齐段成功、CMPP/HTTP通知幂等、非零975计费/超时退款一次/迟到回执、三段失败只创建一份后继Submit/Outbox。路由与限速在补发事务测试中被隔离,不作为真实路由/Redis/Gateway通过依据。
|
||||
- 证据脚本tools/testing/verify-attempt-completion.mjs;日志.local-data/five-fixes-20260916/completion-integration.log。故障注入故意产生completion_retry_wait,不是线上异常。
|
||||
- 最近定向隔离回归:发送链路+最终通道校验142/142;发送链路+监控服务147/147(范围重叠,不相加);HTTP页面/签名15/15。后端类型检查通过。后续改动须复验;真实浏览器、完整回归、故障恢复矩阵、实际CMPP/Webhook投递与测试部署仍未完成。
|
||||
- 保护既有61项修改快照,未把原metrics/发布工具/文档草稿视作本轮内容。此前产品命名提交a0209f9尚未推送。本轮新增metrics两处接入需精确hunk暂存,不能包含此前指标修改。
|
||||
|
||||
### 2026-09-16 18:25 本地验证与交付准备
|
||||
|
||||
前端全量35套163项通过(115.33秒),后端77套835项通过(171.189秒);类型检查通过。新增隔离PG全量106迁移重放和11组耐久工作/告警验证通过,包含独立OS进程并发及挂起旧消费者恢复,日志在.local-data/five-fixes-20260916。第12次失败转人工处理及数据库告警采集已补验证;日志中的故意故障注入不得当线上异常。最新采集改动需候选精确提交回归。
|
||||
|
||||
真实环境尚未部署;独立Playwright获用户允许,专用Edge等待用户完成登录。测试Outbox发布开关及独立进程已核验。为通过更名涉及文件的既有门禁,仅移除official-export未使用导入并格式化该文件及RechargeReceiptDialog,业务规则不变。保护项未纳入;开始准备精确暂存,尚未提交/推送。
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
# CMPP 平台 UI 设计规范
|
||||
# 聆界短信平台 UI 设计规范
|
||||
|
||||
## 设计原则
|
||||
|
||||
|
||||
+1
-1
@@ -4,7 +4,7 @@
|
||||
<meta charset="UTF-8" />
|
||||
<link rel="icon" href="/logo/fav.ico" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>聆界短信管理平台</title>
|
||||
<title>聆界短信平台</title>
|
||||
</head>
|
||||
<body>
|
||||
<div id="root"></div>
|
||||
|
||||
@@ -25,8 +25,10 @@ export const adminInfrastructureMonitoringApi = {
|
||||
startDate: string;
|
||||
endDate: string;
|
||||
}>(withQuery('/admin/infrastructure-monitoring/alert-history', { from, to, page })),
|
||||
getInfrastructureMonitoringOverview: (range: InfrastructureMonitoringRange) =>
|
||||
request<InfrastructureMonitoringOverview>(withQuery('/admin/infrastructure-monitoring/overview', { range })),
|
||||
getInfrastructureMonitoringOverview: (range: InfrastructureMonitoringRange, signal?: AbortSignal) =>
|
||||
request<InfrastructureMonitoringOverview>(withQuery('/admin/infrastructure-monitoring/overview', { range }), {
|
||||
signal,
|
||||
}),
|
||||
getInfrastructureMonitoringNotificationSummary: (signal?: AbortSignal) =>
|
||||
request<{ count: number; criticalCount: number }>('/admin/infrastructure-monitoring/notification-summary', {
|
||||
signal,
|
||||
@@ -38,6 +40,11 @@ export const adminInfrastructureMonitoringApi = {
|
||||
method: 'PUT',
|
||||
body: JSON.stringify(body),
|
||||
}),
|
||||
clearInfrastructureAlert: (fingerprint: string, activeAt: string) =>
|
||||
request<{ cleared: boolean }>(`/admin/infrastructure-monitoring/alerts/${fingerprint}/clear`, {
|
||||
method: 'POST',
|
||||
body: JSON.stringify({ activeAt }),
|
||||
}),
|
||||
markInfrastructureAlertRead: (fingerprint: string, activeAt: string) =>
|
||||
request<{ fingerprint: string; activeAt: string; acknowledged: true; acknowledgedAt: string }>(
|
||||
`/admin/infrastructure-monitoring/alerts/${fingerprint}/read`,
|
||||
|
||||
@@ -86,9 +86,9 @@ export function LoginPage({ portal }: LoginPageProps) {
|
||||
{!isAdmin ? <ClientLoginCanvas /> : null}
|
||||
<section className="login-panel">
|
||||
<div className="login-brand">
|
||||
<img alt={isAdmin ? 'CMPP 运营端 logo' : 'CMPP 客户端 logo'} src="/logo/logo1.png" />
|
||||
<img alt={isAdmin ? '聆界短信平台运营端 logo' : '聆界短信平台客户端 logo'} src="/logo/logo1.png" />
|
||||
<div>
|
||||
<h1>短信服务平台</h1>
|
||||
<h1>聆界短信平台</h1>
|
||||
<p>{isAdmin ? '运营端登录' : '客户端登录'}</p>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@@ -0,0 +1,80 @@
|
||||
import { fireEvent, render, screen, waitFor } from '@testing-library/react';
|
||||
import { beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import { AdminSystemMonitoringPage } from './AdminSystemMonitoringPage';
|
||||
|
||||
const api = vi.hoisted(() => ({
|
||||
getInfrastructureMonitoringOverview: vi.fn(),
|
||||
getInfrastructureAlertThresholds: vi.fn(),
|
||||
clearInfrastructureAlert: vi.fn(),
|
||||
}));
|
||||
vi.mock('@/api/adminApi', () => ({ adminApi: api }));
|
||||
vi.mock('@/components/ui/Chart', () => ({ Chart: () => <div>趋势图</div> }));
|
||||
vi.mock('./AlertHistory', () => ({ AlertHistory: () => null }));
|
||||
const sample = {
|
||||
available: true,
|
||||
range: '24h',
|
||||
collectedAt: '2026-09-16T00:00:00Z',
|
||||
lastSampleAt: '2026-09-16T00:00:00Z',
|
||||
summary: {
|
||||
overallStatus: 'healthy',
|
||||
serviceTotal: 0,
|
||||
serviceHealthy: 0,
|
||||
warningAlerts: 0,
|
||||
criticalAlerts: 0,
|
||||
activeAlerts: 0,
|
||||
},
|
||||
metrics: { cpuUsagePercent: 12.3 },
|
||||
trends: {
|
||||
cpuUsagePercent: [],
|
||||
memoryUsagePercent: [],
|
||||
diskUsagePercent: [],
|
||||
networkReceiveBytesPerSecond: [],
|
||||
networkTransmitBytesPerSecond: [],
|
||||
},
|
||||
services: [],
|
||||
serviceMetrics: [],
|
||||
disks: [],
|
||||
alerts: [],
|
||||
};
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
api.getInfrastructureMonitoringOverview.mockResolvedValue(sample);
|
||||
api.getInfrastructureAlertThresholds.mockResolvedValue({ thresholds: {}, definitions: [], configVersion: 1 });
|
||||
});
|
||||
describe('monitoring snapshots', () => {
|
||||
it('retains the last successful data with a visible stale warning after refresh fails, then recovers', async () => {
|
||||
render(<AdminSystemMonitoringPage />);
|
||||
await screen.findAllByText('12.3%');
|
||||
api.getInfrastructureMonitoringOverview.mockRejectedValueOnce(new TypeError('Failed to fetch'));
|
||||
fireEvent.click(screen.getByRole('button', { name: '刷新' }));
|
||||
await screen.findByText('监控数据更新失败');
|
||||
expect(screen.getAllByText('12.3%').length).toBeGreaterThan(0);
|
||||
expect(screen.getByRole('alert')).toHaveTextContent('上次成功数据');
|
||||
expect(screen.getByRole('alert')).toHaveTextContent('并非实时状态');
|
||||
api.getInfrastructureMonitoringOverview.mockResolvedValueOnce({ ...sample, metrics: { cpuUsagePercent: 45.6 } });
|
||||
fireEvent(window, new Event('online'));
|
||||
await screen.findAllByText('45.6%');
|
||||
expect(screen.queryByText('监控数据更新失败')).not.toBeInTheDocument();
|
||||
});
|
||||
it('does not label a different time range with the previous range snapshot and aborts superseded requests', async () => {
|
||||
render(<AdminSystemMonitoringPage />);
|
||||
await screen.findAllByText('12.3%');
|
||||
api.getInfrastructureMonitoringOverview.mockRejectedValueOnce(new TypeError('offline'));
|
||||
fireEvent.click(screen.getByRole('button', { name: '近1小时' }));
|
||||
await screen.findByText('监控数据更新失败');
|
||||
expect(screen.queryByText('12.3%')).not.toBeInTheDocument();
|
||||
expect(screen.getByRole('alert')).toHaveTextContent('尚无成功采样');
|
||||
api.getInfrastructureMonitoringOverview.mockImplementationOnce(
|
||||
(_range, signal) =>
|
||||
new Promise((_resolve, reject) =>
|
||||
signal.addEventListener('abort', () => reject(new DOMException('aborted', 'AbortError'))),
|
||||
),
|
||||
);
|
||||
fireEvent.click(screen.getByRole('button', { name: '刷新' }));
|
||||
const calls = api.getInfrastructureMonitoringOverview.mock.calls;
|
||||
const signal = calls[calls.length - 1]?.[1] as AbortSignal;
|
||||
fireEvent.click(screen.getByRole('button', { name: '近24小时' }));
|
||||
await waitFor(() => expect(signal.aborted).toBe(true));
|
||||
await screen.findAllByText('12.3%');
|
||||
});
|
||||
});
|
||||
@@ -227,6 +227,7 @@ function severityTag(severity: InfrastructureAlert['severity']) {
|
||||
function makeAlertColumns(
|
||||
onMarkRead: (alert: InfrastructureAlert) => void,
|
||||
readingFingerprint: string,
|
||||
onClear: (alert: InfrastructureAlert) => void,
|
||||
): Array<TableColumn<InfrastructureAlert>> {
|
||||
return [
|
||||
{ key: 'severity', title: '级别', width: '82px', render: (record) => severityTag(record.severity) },
|
||||
@@ -254,13 +255,20 @@ function makeAlertColumns(
|
||||
render: (record) => `${record.currentValue || '—'} / ${record.threshold || '—'}`,
|
||||
},
|
||||
{ key: 'startedAt', title: '开始时间', width: '150px', render: (record) => formatTime(record.startedAt) },
|
||||
{
|
||||
key: 'status',
|
||||
title: '状态',
|
||||
width: '100px',
|
||||
render: (record) => (record.status === 'resolved' ? '已恢复待清除' : '仍在触发'),
|
||||
},
|
||||
{ key: 'duration', title: '持续时间', width: '120px', render: (record) => formatDuration(record.startedAt) },
|
||||
{
|
||||
key: 'actions',
|
||||
title: '操作',
|
||||
width: '112px',
|
||||
render: (record) =>
|
||||
record.acknowledged ? (
|
||||
width: '180px',
|
||||
render: (record) => (
|
||||
<div className="table-actions">
|
||||
{record.acknowledged ? (
|
||||
<Tag tone="neutral">已读</Tag>
|
||||
) : (
|
||||
<Button
|
||||
@@ -272,6 +280,16 @@ function makeAlertColumns(
|
||||
>
|
||||
{readingFingerprint === record.fingerprint ? '处理中' : '标记已读'}
|
||||
</Button>
|
||||
)}
|
||||
<Button
|
||||
disabled={readingFingerprint === record.fingerprint}
|
||||
size="sm"
|
||||
variant="ghost"
|
||||
onClick={() => onClear(record)}
|
||||
>
|
||||
清除
|
||||
</Button>
|
||||
</div>
|
||||
),
|
||||
},
|
||||
];
|
||||
@@ -291,25 +309,43 @@ export function AdminSystemMonitoringPage() {
|
||||
const [readError, setReadError] = useState('');
|
||||
const requestSequence = useRef(0);
|
||||
const pendingRequests = useRef(0);
|
||||
const activeRequest = useRef<AbortController | null>(null);
|
||||
const snapshots = useRef<Partial<Record<InfrastructureMonitoringRange, InfrastructureMonitoringOverview>>>({});
|
||||
|
||||
const loadData = useCallback(
|
||||
async (supersede = false) => {
|
||||
if (!supersede && pendingRequests.current > 0) return;
|
||||
if (supersede) activeRequest.current?.abort();
|
||||
const controller = new AbortController();
|
||||
activeRequest.current = controller;
|
||||
const timer = window.setTimeout(() => controller.abort(), 20_000);
|
||||
pendingRequests.current += 1;
|
||||
const sequence = ++requestSequence.current;
|
||||
setLoading(true);
|
||||
setOverview(snapshots.current[range] ?? null);
|
||||
try {
|
||||
const result = await adminApi.getInfrastructureMonitoringOverview(range);
|
||||
const result = await adminApi.getInfrastructureMonitoringOverview(range, controller.signal);
|
||||
if (sequence !== requestSequence.current) return;
|
||||
if (result.available) {
|
||||
snapshots.current[range] = result;
|
||||
setOverview(result);
|
||||
} else {
|
||||
setOverview(snapshots.current[range] ?? result);
|
||||
}
|
||||
setError(result.available ? '' : result.error || '监控数据当前不可用');
|
||||
} catch (reason) {
|
||||
if (sequence !== requestSequence.current) return;
|
||||
setOverview(null);
|
||||
setError(reason instanceof Error ? reason.message : '监控数据加载失败');
|
||||
setOverview(snapshots.current[range] ?? null);
|
||||
setError(
|
||||
reason instanceof Error && reason.name === 'AbortError'
|
||||
? '刷新超时,请检查网络后重试'
|
||||
: '监控刷新失败,请检查网络或采集服务',
|
||||
);
|
||||
} finally {
|
||||
if (sequence === requestSequence.current) setLoading(false);
|
||||
pendingRequests.current -= 1;
|
||||
window.clearTimeout(timer);
|
||||
if (activeRequest.current === controller) activeRequest.current = null;
|
||||
}
|
||||
},
|
||||
[range],
|
||||
@@ -372,6 +408,43 @@ export function AdminSystemMonitoringPage() {
|
||||
}
|
||||
}, []);
|
||||
|
||||
const clearAlert = useCallback(
|
||||
async (alert: InfrastructureAlert) => {
|
||||
setReadingFingerprint(alert.fingerprint);
|
||||
setReadError('');
|
||||
try {
|
||||
await adminApi.clearInfrastructureAlert(alert.fingerprint, alert.startedAt);
|
||||
const remove = (snapshot: InfrastructureMonitoringOverview) => {
|
||||
const alerts = snapshot.alerts.filter(
|
||||
(item) => item.fingerprint !== alert.fingerprint || item.startedAt !== alert.startedAt,
|
||||
);
|
||||
return {
|
||||
...snapshot,
|
||||
alerts,
|
||||
summary: {
|
||||
...snapshot.summary,
|
||||
activeAlerts: alerts.length,
|
||||
criticalAlerts: alerts.filter((item) => item.severity === 'critical').length,
|
||||
warningAlerts: alerts.filter((item) => item.severity === 'warning').length,
|
||||
},
|
||||
};
|
||||
};
|
||||
for (const range of Object.keys(snapshots.current) as InfrastructureMonitoringRange[]) {
|
||||
const snapshot = snapshots.current[range];
|
||||
if (snapshot) snapshots.current[range] = remove(snapshot);
|
||||
}
|
||||
setOverview((snapshot) => (snapshot ? remove(snapshot) : snapshot));
|
||||
window.dispatchEvent(new Event('cmpp-infrastructure-alert-count-refresh'));
|
||||
await loadData(true);
|
||||
} catch (reason) {
|
||||
setReadError(reason instanceof Error ? reason.message : '清除告警失败');
|
||||
} finally {
|
||||
setReadingFingerprint('');
|
||||
}
|
||||
},
|
||||
[loadData],
|
||||
);
|
||||
|
||||
useEffect(() => {
|
||||
void loadData(true);
|
||||
void loadSettings();
|
||||
@@ -382,14 +455,17 @@ export function AdminSystemMonitoringPage() {
|
||||
if (document.visibilityState === 'visible') void loadData();
|
||||
};
|
||||
document.addEventListener('visibilitychange', handleVisibility);
|
||||
window.addEventListener('online', handleVisibility);
|
||||
return () => {
|
||||
requestSequence.current += 1;
|
||||
activeRequest.current?.abort();
|
||||
window.clearInterval(intervalId);
|
||||
document.removeEventListener('visibilitychange', handleVisibility);
|
||||
window.removeEventListener('online', handleVisibility);
|
||||
};
|
||||
}, [loadData, loadSettings]);
|
||||
|
||||
const status = STATUS_COPY[overview?.summary.overallStatus ?? 'unknown'];
|
||||
const status = STATUS_COPY[error ? 'unknown' : (overview?.summary.overallStatus ?? 'unknown')];
|
||||
const cpuOption = useMemo(
|
||||
() =>
|
||||
makeTrendOption({
|
||||
@@ -442,10 +518,16 @@ export function AdminSystemMonitoringPage() {
|
||||
const serviceTotal = overview?.summary.serviceTotal ?? 6;
|
||||
const alertColumns = useMemo(
|
||||
() =>
|
||||
makeAlertColumns((alert) => {
|
||||
makeAlertColumns(
|
||||
(alert) => {
|
||||
void markAlertRead(alert);
|
||||
}, readingFingerprint),
|
||||
[markAlertRead, readingFingerprint],
|
||||
},
|
||||
readingFingerprint,
|
||||
(alert) => {
|
||||
void clearAlert(alert);
|
||||
},
|
||||
),
|
||||
[markAlertRead, readingFingerprint, clearAlert],
|
||||
);
|
||||
|
||||
return (
|
||||
@@ -487,8 +569,13 @@ export function AdminSystemMonitoringPage() {
|
||||
<div className="system-monitoring-unavailable" role="alert">
|
||||
<ShieldAlert size={20} />
|
||||
<div>
|
||||
<strong>监控数据不可用</strong>
|
||||
<span>{error}。页面不会展示历史缓存值。</span>
|
||||
<strong>监控数据更新失败</strong>
|
||||
<span>
|
||||
{error}。
|
||||
{overview?.available
|
||||
? `当前展示上次成功数据,采样时间 ${formatTime(overview.lastSampleAt ?? overview.collectedAt)},并非实时状态。`
|
||||
: '尚无成功采样,请稍后重试。'}
|
||||
</span>
|
||||
</div>
|
||||
</div>
|
||||
) : null}
|
||||
|
||||
@@ -2,6 +2,7 @@ import { useRef, useState } from 'react';
|
||||
import { Breadcrumb, Button, Input, Select, Textarea } from '@/components/ui';
|
||||
import { calculateSignature, createNonce, type SignatureInput } from './signature';
|
||||
import './HttpSignaturePage.css';
|
||||
import { SmsRequestDebugger } from './SmsRequestDebugger';
|
||||
|
||||
function emptyInput(): SignatureInput {
|
||||
return {
|
||||
@@ -52,7 +53,9 @@ export function HttpSignaturePage() {
|
||||
return (
|
||||
<section className="page-stack http-signature-page">
|
||||
<Breadcrumb items={['接口工具', 'HTTP签名计算']} />
|
||||
<p className="muted">仅在当前浏览器计算,不上传或保存密钥。生成结果不会发送请求。</p>
|
||||
<p className="muted">
|
||||
上方工具仅在浏览器计算签名,不上传或保存密钥;需要实际发送时,请使用下方短信接口发送调试。
|
||||
</p>
|
||||
<form
|
||||
className="surface http-signature-page__form"
|
||||
autoComplete="off"
|
||||
@@ -154,6 +157,7 @@ export function HttpSignaturePage() {
|
||||
</div>
|
||||
) : null}
|
||||
<p role="status">{notice}</p>
|
||||
<SmsRequestDebugger />
|
||||
</section>
|
||||
);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,69 @@
|
||||
import { fireEvent, render, screen, waitFor } from '@testing-library/react';
|
||||
import { afterEach, describe, expect, it, vi } from 'vitest';
|
||||
import { SmsRequestDebugger } from './SmsRequestDebugger';
|
||||
import { calculateSignature } from './signature';
|
||||
|
||||
afterEach(() => vi.unstubAllGlobals());
|
||||
function fill() {
|
||||
render(<SmsRequestDebugger />);
|
||||
for (const [label, value] of [
|
||||
['AccessKey', 'qa-key'],
|
||||
['发送 AccessSecret', 'qa-secret'],
|
||||
['手机号(mobile)', '13800138000'],
|
||||
['短信正文(content)', '【测试】长短信中文内容'.repeat(20)],
|
||||
]) {
|
||||
fireEvent.change(screen.getByLabelText(label), { target: { value } });
|
||||
}
|
||||
}
|
||||
describe('SMS request debugger', () => {
|
||||
it('signs the exact UTF-8 body and displays real HTTP results without transmitting the secret', async () => {
|
||||
const fetcher = vi
|
||||
.fn()
|
||||
.mockResolvedValue(new Response('{"code":"OK"}', { status: 202, headers: { 'X-Request-ID': 'qa-trace' } }));
|
||||
vi.stubGlobal('fetch', fetcher);
|
||||
fill();
|
||||
fireEvent.click(screen.getByRole('button', { name: '发送短信' }));
|
||||
await waitFor(() =>
|
||||
expect((screen.getByLabelText('返回 HTTP 报文') as HTMLTextAreaElement).value).toContain('HTTP 202'),
|
||||
);
|
||||
const [path, options] = fetcher.mock.calls[0];
|
||||
const headers = options.headers;
|
||||
expect(headers['X-Signature']).toBe(
|
||||
calculateSignature({
|
||||
method: 'POST',
|
||||
path,
|
||||
body: options.body,
|
||||
secret: 'qa-secret',
|
||||
timestamp: headers['X-Timestamp'],
|
||||
nonce: headers['X-Nonce'],
|
||||
}).signature,
|
||||
);
|
||||
expect(JSON.stringify(options)).not.toContain('qa-secret');
|
||||
expect(options.credentials).toBe('omit');
|
||||
expect((screen.getByLabelText('返回 HTTP 报文') as HTMLTextAreaElement).value).toContain('qa-trace');
|
||||
});
|
||||
|
||||
it('preserves the idempotency key after network uncertainty, never auto retries, and changes it when body changes', async () => {
|
||||
const fetcher = vi.fn().mockRejectedValue(new TypeError('Failed to fetch'));
|
||||
vi.stubGlobal('fetch', fetcher);
|
||||
fill();
|
||||
const key = (screen.getByLabelText('业务幂等键(Idempotency-Key)') as HTMLInputElement).value;
|
||||
fireEvent.click(screen.getByRole('button', { name: '发送短信' }));
|
||||
await screen.findByRole('alert');
|
||||
expect(screen.getByRole('alert')).toHaveTextContent('发送结果未知');
|
||||
expect(fetcher).toHaveBeenCalledTimes(1);
|
||||
expect(screen.getByLabelText('业务幂等键(Idempotency-Key)')).toHaveValue(key);
|
||||
fireEvent.change(screen.getByLabelText('短信正文(content)'), { target: { value: '修改正文' } });
|
||||
expect(screen.getByLabelText('业务幂等键(Idempotency-Key)')).not.toHaveValue(key);
|
||||
});
|
||||
|
||||
it('blocks duplicate clicks while a response is outstanding', async () => {
|
||||
const fetcher = vi.fn().mockImplementation(() => new Promise(() => {}));
|
||||
vi.stubGlobal('fetch', fetcher);
|
||||
fill();
|
||||
const send = screen.getByRole('button', { name: '发送短信' });
|
||||
fireEvent.click(send);
|
||||
fireEvent.click(send);
|
||||
expect(fetcher).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,157 @@
|
||||
import { useEffect, useRef, useState } from 'react';
|
||||
import { Button, Input, Textarea } from '@/components/ui';
|
||||
import { calculateSignature, createNonce } from './signature';
|
||||
|
||||
const path = '/api/openapi/v1/sms/messages';
|
||||
|
||||
export function SmsRequestDebugger() {
|
||||
const [fields, setFields] = useState({
|
||||
appKey: '',
|
||||
secret: '',
|
||||
mobile: '',
|
||||
content: '',
|
||||
clientMessageId: '',
|
||||
idempotencyKey: createNonce(),
|
||||
});
|
||||
const [sending, setSending] = useState(false);
|
||||
const [requestText, setRequestText] = useState('');
|
||||
const [responseText, setResponseText] = useState('');
|
||||
const [error, setError] = useState('');
|
||||
const controller = useRef<AbortController | null>(null);
|
||||
useEffect(() => () => controller.current?.abort(), []);
|
||||
|
||||
function change(key: keyof typeof fields, value: string) {
|
||||
setFields((current) => ({
|
||||
...current,
|
||||
[key]: value,
|
||||
...(key === 'idempotencyKey' ? {} : { idempotencyKey: createNonce() }),
|
||||
}));
|
||||
setError('');
|
||||
}
|
||||
|
||||
async function send() {
|
||||
if (controller.current) return;
|
||||
if (!fields.appKey.trim() || !fields.secret || !/^1\d{10}$/.test(fields.mobile) || !fields.content.trim()) {
|
||||
setError('请填写 AccessKey、AccessSecret、11位手机号和短信正文');
|
||||
return;
|
||||
}
|
||||
if (!/^[A-Za-z0-9._:-]{8,128}$/.test(fields.idempotencyKey) || fields.clientMessageId.length > 128) {
|
||||
setError('请检查业务幂等键(8~128位)及客户消息编号(最多128字符)');
|
||||
return;
|
||||
}
|
||||
const body = JSON.stringify({
|
||||
mobile: fields.mobile,
|
||||
content: fields.content,
|
||||
...(fields.clientMessageId ? { clientMessageId: fields.clientMessageId } : {}),
|
||||
});
|
||||
const timestamp = String(Math.floor(Date.now() / 1000));
|
||||
const nonce = createNonce();
|
||||
const { signature } = calculateSignature({ method: 'POST', path, timestamp, nonce, secret: fields.secret, body });
|
||||
const headers = {
|
||||
'Content-Type': 'application/json',
|
||||
'X-App-Key': fields.appKey.trim(),
|
||||
'X-Timestamp': timestamp,
|
||||
'X-Nonce': nonce,
|
||||
'X-Signature': signature,
|
||||
'Idempotency-Key': fields.idempotencyKey,
|
||||
};
|
||||
setRequestText(
|
||||
`POST ${path}\n${Object.entries(headers)
|
||||
.map(([key, value]) => `${key}: ${value}`)
|
||||
.join('\n')}\n\n${body}`,
|
||||
);
|
||||
setResponseText('');
|
||||
setError('');
|
||||
setSending(true);
|
||||
const active = new AbortController();
|
||||
controller.current = active;
|
||||
const timer = window.setTimeout(() => active.abort(), 20_000);
|
||||
let responseHead = '';
|
||||
try {
|
||||
const response = await fetch(path, {
|
||||
method: 'POST',
|
||||
credentials: 'omit',
|
||||
redirect: 'error',
|
||||
headers,
|
||||
body,
|
||||
signal: active.signal,
|
||||
});
|
||||
responseHead = `HTTP ${response.status} ${response.statusText}\n${Array.from(response.headers.entries())
|
||||
.map(([key, value]) => `${key}: ${value}`)
|
||||
.join('\n')}\n\n`;
|
||||
const responseBody = await response.text();
|
||||
setResponseText(responseHead + responseBody);
|
||||
if (!response.ok) setError(`接口返回 ${response.status},请查看响应报文`);
|
||||
} catch (reason) {
|
||||
if (responseHead) setResponseText(responseHead + '[响应正文未完整读取]');
|
||||
setError(
|
||||
`${reason instanceof Error && reason.name === 'AbortError' ? '请求超时或已取消' : '网络请求失败'},发送结果未知。请先查询消息结果;如需重试,保持相同业务幂等键。`,
|
||||
);
|
||||
} finally {
|
||||
window.clearTimeout(timer);
|
||||
controller.current = null;
|
||||
setSending(false);
|
||||
}
|
||||
}
|
||||
|
||||
return (
|
||||
<section className="surface http-signature-page__form" aria-label="短信接口发送调试">
|
||||
<h2>短信接口发送调试</h2>
|
||||
<p className="muted">
|
||||
发送到当前平台,会真实创建短信并按应用规则计费。密钥只用于本页计算签名,不保存。以下展示应用层报文,浏览器自动添加的头部不在其中。
|
||||
</p>
|
||||
<div className="http-signature-page__fields">
|
||||
<Input
|
||||
label="AccessKey"
|
||||
autoComplete="off"
|
||||
disabled={sending}
|
||||
value={fields.appKey}
|
||||
onChange={(event) => change('appKey', event.target.value)}
|
||||
/>
|
||||
<Input
|
||||
label="发送 AccessSecret"
|
||||
type="password"
|
||||
autoComplete="new-password"
|
||||
disabled={sending}
|
||||
value={fields.secret}
|
||||
onChange={(event) => change('secret', event.target.value)}
|
||||
/>
|
||||
<Input
|
||||
label="手机号(mobile)"
|
||||
disabled={sending}
|
||||
value={fields.mobile}
|
||||
onChange={(event) => change('mobile', event.target.value)}
|
||||
/>
|
||||
<Input
|
||||
label="客户消息编号(clientMessageId,可选)"
|
||||
disabled={sending}
|
||||
value={fields.clientMessageId}
|
||||
onChange={(event) => change('clientMessageId', event.target.value)}
|
||||
/>
|
||||
<Input
|
||||
label="业务幂等键(Idempotency-Key)"
|
||||
disabled={sending}
|
||||
value={fields.idempotencyKey}
|
||||
onChange={(event) => change('idempotencyKey', event.target.value)}
|
||||
/>
|
||||
</div>
|
||||
<Textarea
|
||||
label="短信正文(content)"
|
||||
rows={5}
|
||||
disabled={sending}
|
||||
value={fields.content}
|
||||
onChange={(event) => change('content', event.target.value)}
|
||||
/>
|
||||
<Button disabled={sending} onClick={() => void send()}>
|
||||
{sending ? '发送中…' : '发送短信'}
|
||||
</Button>
|
||||
{error ? (
|
||||
<p className="form-error" role="alert">
|
||||
{error}
|
||||
</p>
|
||||
) : null}
|
||||
{requestText ? <Textarea label="发送 HTTP 报文" rows={12} readOnly value={requestText} /> : null}
|
||||
{responseText ? <Textarea label="返回 HTTP 报文" rows={12} readOnly value={responseText} /> : null}
|
||||
</section>
|
||||
);
|
||||
}
|
||||
@@ -16,19 +16,12 @@ export function formatReceiptAmount(moneyUnits: number) {
|
||||
return formatCents(Math.abs(moneyUnits));
|
||||
}
|
||||
|
||||
export function RechargeReceiptDialog({
|
||||
open,
|
||||
record,
|
||||
tenant,
|
||||
onClose,
|
||||
}: RechargeReceiptDialogProps) {
|
||||
export function RechargeReceiptDialog({ open, record, tenant, onClose }: RechargeReceiptDialogProps) {
|
||||
if (!record) return null;
|
||||
|
||||
const isCorrection = record.amountCents < 0;
|
||||
const balanceAfter = record.balanceAfterCents;
|
||||
const balanceBefore = balanceAfter === null || balanceAfter === undefined
|
||||
? null
|
||||
: balanceAfter - record.amountCents;
|
||||
const balanceBefore = balanceAfter === null || balanceAfter === undefined ? null : balanceAfter - record.amountCents;
|
||||
const enterpriseName = record.tenant?.name ?? tenant?.name ?? record.tenantId;
|
||||
const enterpriseCode = record.tenant?.code ?? tenant?.code ?? '-';
|
||||
|
||||
@@ -40,13 +33,11 @@ export function RechargeReceiptDialog({
|
||||
open={open}
|
||||
title="账户充值回执"
|
||||
>
|
||||
<article className={['recharge-receipt', isCorrection ? 'recharge-receipt--correction' : ''].filter(Boolean).join(' ')}>
|
||||
<article
|
||||
className={['recharge-receipt', isCorrection ? 'recharge-receipt--correction' : ''].filter(Boolean).join(' ')}
|
||||
>
|
||||
<header className="recharge-receipt__header">
|
||||
<img
|
||||
alt="聆界短信服务平台"
|
||||
className="recharge-receipt__logo"
|
||||
src="/logo/logo1.png"
|
||||
/>
|
||||
<img alt="聆界短信平台" className="recharge-receipt__logo" src="/logo/logo1.png" />
|
||||
<span className={isCorrection ? 'is-correction' : 'is-posted'}>
|
||||
<CheckCircle2 aria-hidden="true" size={16} />
|
||||
{isCorrection ? '已冲正' : '已入账'}
|
||||
@@ -104,9 +95,7 @@ export function RechargeReceiptDialog({
|
||||
<p>{record.remark || '无'}</p>
|
||||
</section>
|
||||
|
||||
<footer className="recharge-receipt__note">
|
||||
本回执由系统根据真实入账记录自动生成
|
||||
</footer>
|
||||
<footer className="recharge-receipt__note">本回执由系统根据真实入账记录自动生成</footer>
|
||||
</article>
|
||||
</Modal>
|
||||
);
|
||||
|
||||
@@ -137,7 +137,7 @@ function AdminAuthenticatedLayout({ session }: { session: LoginSession }) {
|
||||
|
||||
return (
|
||||
<AppShell
|
||||
title="CMPP 运营端"
|
||||
title="聆界短信平台运营端"
|
||||
subtitle="平台运营管理中心"
|
||||
workspaceName="平台运营工作区"
|
||||
loginPath="/admin/login"
|
||||
|
||||
@@ -23,7 +23,7 @@ export function ClientLayout() {
|
||||
function ClientAuthenticatedLayout({ session }: { session: import('@/api/session').LoginSession }) {
|
||||
return (
|
||||
<AppShell
|
||||
title="短信服务平台"
|
||||
title="聆界短信平台客户端"
|
||||
subtitle="短信服务控制台"
|
||||
workspaceName={session.user.tenantName ?? '企业客户空间'}
|
||||
loginPath="/client/login"
|
||||
|
||||
@@ -0,0 +1,602 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import { createRequire } from 'node:module';
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import { spawn } from 'node:child_process';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
const require = createRequire(new URL('../../api/package.json', import.meta.url));
|
||||
const url = new URL(process.env.COMPLETION_TEST_DATABASE_URL || '');
|
||||
assert(['localhost', '127.0.0.1'].includes(url.hostname) && url.pathname.startsWith('/cmpp_qa_'));
|
||||
process.env.DATABASE_URL = url.toString();
|
||||
process.env.NODE_ENV = 'test';
|
||||
require('reflect-metadata');
|
||||
const { PrismaService } = require('./dist/prisma/prisma.service');
|
||||
const { AttemptCompletion } = require('./dist/send-chain/attempt-completion');
|
||||
const {
|
||||
completionContext,
|
||||
completionDatabase,
|
||||
CompletionRouteRequired,
|
||||
} = require('./dist/send-chain/completion-context');
|
||||
const {
|
||||
retainAlerts,
|
||||
retainedAlerts,
|
||||
clearRetainedAlert,
|
||||
} = require('./dist/infrastructure-monitoring/persistent-alerts');
|
||||
const { SendChainService } = require('./dist/send-chain/send-chain.service');
|
||||
const { BillingService } = require('./dist/billing/billing.service');
|
||||
const { OpenApiService } = require('./dist/open-api/open-api.service');
|
||||
const db = new PrismaService();
|
||||
const scoped = completionDatabase(db);
|
||||
const prefix = randomUUID();
|
||||
const pass = (name) => console.log('PASS', name);
|
||||
const fixture = async () => {
|
||||
const message = await db.smsMessageRecord.create({
|
||||
data: { messageId: randomUUID(), phoneNumber: '13800138000', content: '隔离并发验收'.repeat(30), billingUnits: 3 },
|
||||
});
|
||||
return message;
|
||||
};
|
||||
if (process.argv[2] === '--worker') {
|
||||
try {
|
||||
const worker = new AttemptCompletion(
|
||||
db,
|
||||
async (_kind, payload) => {
|
||||
await scoped.operationLog.create({
|
||||
data: { action: 'qa.process', resource: process.argv[4], resourceId: payload.id },
|
||||
});
|
||||
},
|
||||
async () => {},
|
||||
);
|
||||
await worker.process(process.argv[3]);
|
||||
} finally {
|
||||
await db.onModuleDestroy();
|
||||
}
|
||||
process.exit(0);
|
||||
}
|
||||
try {
|
||||
const message = await fixture();
|
||||
let active = 0;
|
||||
let maximum = 0;
|
||||
const execute = async (_kind, payload) => {
|
||||
active++;
|
||||
maximum = Math.max(maximum, active);
|
||||
try {
|
||||
await scoped.$transaction(async (tx) => {
|
||||
assert.equal(tx, completionContext.getStore().tx);
|
||||
await tx.operationLog.create({ data: { action: 'qa.completion', resource: prefix, resourceId: payload.id } });
|
||||
});
|
||||
} finally {
|
||||
active--;
|
||||
}
|
||||
};
|
||||
const workers = [
|
||||
new AttemptCompletion(db, execute, async () => {}),
|
||||
new AttemptCompletion(db, execute, async () => {}),
|
||||
];
|
||||
await Promise.all(
|
||||
Array.from({ length: 24 }, (_, i) =>
|
||||
workers[i % 2].enqueue(message.id, undefined, 'receipt', { id: `segment-${i % 12}` }),
|
||||
),
|
||||
);
|
||||
await Promise.all(workers.map((worker) => worker.scan()));
|
||||
const work = await db.smsAttemptCompletionWork.findUniqueOrThrow({ where: { workKey: `message:${message.id}` } });
|
||||
assert.equal(maximum, 1);
|
||||
assert.equal(work.revision, 12);
|
||||
assert.equal(work.processedRevision, 12);
|
||||
assert.equal(work.state, 'idle');
|
||||
assert.equal(await db.operationLog.count({ where: { resource: prefix } }), 12);
|
||||
pass('two consumers, 24 concurrent receipts, 12 unique facts, exactly one owner');
|
||||
|
||||
const processMessage = await fixture();
|
||||
const processWork = await db.smsAttemptCompletionWork.create({
|
||||
data: {
|
||||
workKey: `message:${processMessage.id}`,
|
||||
messageRecordId: processMessage.id,
|
||||
revision: 20,
|
||||
nextAttemptAt: new Date(0),
|
||||
},
|
||||
});
|
||||
await db.smsCompletionEvent.createMany({
|
||||
data: Array.from({ length: 20 }, (_, i) => ({
|
||||
workId: processWork.id,
|
||||
eventKey: randomUUID(),
|
||||
kind: 'receipt',
|
||||
payload: { id: `process-${i}` },
|
||||
})),
|
||||
});
|
||||
const child = () =>
|
||||
new Promise((resolve, reject) => {
|
||||
const processWorker = spawn(
|
||||
process.execPath,
|
||||
[fileURLToPath(import.meta.url), '--worker', processWork.id, prefix],
|
||||
{ env: process.env, windowsHide: true, stdio: 'pipe', timeout: 20000 },
|
||||
);
|
||||
processWorker.once('error', reject);
|
||||
processWorker.once('exit', (code) => (code === 0 ? resolve() : reject(new Error(`child worker exit ${code}`))));
|
||||
});
|
||||
await Promise.all([child(), child()]);
|
||||
assert.equal(await db.operationLog.count({ where: { action: 'qa.process', resource: prefix } }), 20);
|
||||
assert.equal((await db.smsAttemptCompletionWork.findUniqueOrThrow({ where: { id: processWork.id } })).state, 'idle');
|
||||
pass('two independent OS processes consume one durable work item without duplicate effects');
|
||||
|
||||
const crashMessage = await fixture();
|
||||
let crash = true;
|
||||
const crashWorker = new AttemptCompletion(
|
||||
db,
|
||||
async () => {
|
||||
await scoped.smsMessageRecord.update({ where: { id: crashMessage.id }, data: { status: 'delivered' } });
|
||||
await scoped.operationLog.create({
|
||||
data: { action: 'qa.atomic', resource: prefix, resourceId: crashMessage.id },
|
||||
});
|
||||
if (crash) throw new Error('injected');
|
||||
},
|
||||
async () => {},
|
||||
);
|
||||
await crashWorker.enqueue(crashMessage.id, undefined, 'receipt', { id: 'crash' });
|
||||
let crashWork = await db.smsAttemptCompletionWork.findUniqueOrThrow({
|
||||
where: { workKey: `message:${crashMessage.id}` },
|
||||
});
|
||||
assert.equal(crashWork.state, 'retry_wait');
|
||||
assert.equal((await db.smsMessageRecord.findUniqueOrThrow({ where: { id: crashMessage.id } })).status, 'queued');
|
||||
assert.equal(await db.operationLog.count({ where: { resourceId: crashMessage.id } }), 0);
|
||||
crash = false;
|
||||
await db.smsAttemptCompletionWork.update({ where: { id: crashWork.id }, data: { nextAttemptAt: new Date(0) } });
|
||||
await crashWorker.process(crashWork.id);
|
||||
crashWork = await db.smsAttemptCompletionWork.findUniqueOrThrow({ where: { id: crashWork.id } });
|
||||
assert.equal(crashWork.state, 'idle');
|
||||
assert.equal(await db.operationLog.count({ where: { resourceId: crashMessage.id } }), 1);
|
||||
pass('failure rolls back all effects; durable retry commits once');
|
||||
|
||||
const routeMessage = await fixture();
|
||||
let selected = 0;
|
||||
let waited = 0;
|
||||
const routed = { channel: { id: 'qa' } };
|
||||
const routeWorker = new AttemptCompletion(
|
||||
db,
|
||||
async () => {
|
||||
await scoped.smsMessageRecord.update({ where: { id: routeMessage.id }, data: { status: 'delivered' } });
|
||||
if (!completionContext.getStore().routePlanned)
|
||||
throw new CompletionRouteRequired(async () => {
|
||||
assert.equal(completionContext.getStore(), undefined);
|
||||
assert.equal(
|
||||
(await db.smsMessageRecord.findUniqueOrThrow({ where: { id: routeMessage.id } })).status,
|
||||
'queued',
|
||||
);
|
||||
selected++;
|
||||
return routed;
|
||||
});
|
||||
assert.equal(completionContext.getStore().route.channel.id, 'qa');
|
||||
},
|
||||
async () => {
|
||||
assert.equal(completionContext.getStore(), undefined);
|
||||
waited++;
|
||||
},
|
||||
);
|
||||
await routeWorker.enqueue(routeMessage.id, undefined, 'receipt', { id: 'route' });
|
||||
assert.equal(selected, 1);
|
||||
assert.equal(waited, 1);
|
||||
assert.equal((await db.smsMessageRecord.findUniqueOrThrow({ where: { id: routeMessage.id } })).status, 'delivered');
|
||||
pass('route planning and rate limiting run outside rolled-back transaction');
|
||||
|
||||
const stalledMessage = await fixture();
|
||||
const stalled = await db.smsAttemptCompletionWork.create({
|
||||
data: {
|
||||
workKey: `message:${stalledMessage.id}`,
|
||||
messageRecordId: stalledMessage.id,
|
||||
state: 'processing',
|
||||
leaseOwner: 'dead-process',
|
||||
leaseUntil: new Date(0),
|
||||
fenceVersion: 3,
|
||||
revision: 1,
|
||||
attempts: 1,
|
||||
},
|
||||
});
|
||||
await db.smsCompletionEvent.create({
|
||||
data: { workId: stalled.id, eventKey: randomUUID(), kind: 'receipt', payload: { id: 'takeover' } },
|
||||
});
|
||||
await workers[0].process(stalled.id);
|
||||
const recovered = await db.smsAttemptCompletionWork.findUniqueOrThrow({ where: { id: stalled.id } });
|
||||
assert.equal(recovered.state, 'idle');
|
||||
assert.equal(recovered.fenceVersion, 4);
|
||||
assert.equal(
|
||||
(
|
||||
await db.smsAttemptCompletionWork.updateMany({
|
||||
where: { id: stalled.id, fenceVersion: 3, leaseOwner: 'dead-process' },
|
||||
data: { state: 'processing' },
|
||||
})
|
||||
).count,
|
||||
0,
|
||||
);
|
||||
pass('expired process lease is taken over; old owner cannot write');
|
||||
|
||||
const fencedMessage = await fixture();
|
||||
let entered;
|
||||
let releaseOld;
|
||||
const enteredRoute = new Promise((resolve) => {
|
||||
entered = resolve;
|
||||
});
|
||||
const holdRoute = new Promise((resolve) => {
|
||||
releaseOld = resolve;
|
||||
});
|
||||
const oldWorker = new AttemptCompletion(
|
||||
db,
|
||||
async () => {
|
||||
if (!completionContext.getStore().routePlanned)
|
||||
throw new CompletionRouteRequired(async () => {
|
||||
entered();
|
||||
await holdRoute;
|
||||
return routed;
|
||||
});
|
||||
await scoped.operationLog.create({
|
||||
data: { action: 'qa.fenced', resource: prefix, resourceId: fencedMessage.id },
|
||||
});
|
||||
},
|
||||
async () => {},
|
||||
);
|
||||
const oldAttempt = oldWorker.enqueue(fencedMessage.id, undefined, 'receipt', { id: 'fenced' });
|
||||
await enteredRoute;
|
||||
const fencedWork = await db.smsAttemptCompletionWork.findUniqueOrThrow({
|
||||
where: { workKey: `message:${fencedMessage.id}` },
|
||||
});
|
||||
await db.smsAttemptCompletionWork.update({ where: { id: fencedWork.id }, data: { leaseUntil: new Date(0) } });
|
||||
const newWorker = new AttemptCompletion(
|
||||
db,
|
||||
async () => {
|
||||
await scoped.operationLog.create({
|
||||
data: { action: 'qa.fenced', resource: prefix, resourceId: fencedMessage.id },
|
||||
});
|
||||
},
|
||||
async () => {},
|
||||
);
|
||||
await newWorker.process(fencedWork.id);
|
||||
releaseOld();
|
||||
await oldAttempt;
|
||||
assert.equal(await db.operationLog.count({ where: { action: 'qa.fenced', resourceId: fencedMessage.id } }), 1);
|
||||
assert.equal((await db.smsAttemptCompletionWork.findUniqueOrThrow({ where: { id: fencedWork.id } })).state, 'idle');
|
||||
pass('suspended old consumer resumes after takeover and is fenced before any side effect');
|
||||
|
||||
const tenant = await db.tenant.create({ data: { name: '并发回执隔离验收', code: randomUUID() } });
|
||||
const app = await db.smsApplication.create({
|
||||
data: {
|
||||
tenantId: tenant.id,
|
||||
name: '隔离验收',
|
||||
cmppAccount: randomUUID(),
|
||||
cmppEnterpriseCode: '000001',
|
||||
secretHash: 'disabled',
|
||||
httpConfig: { create: { enabled: true, receiptWebhookEnabled: true } },
|
||||
},
|
||||
});
|
||||
await db.httpWebhookEndpoint.create({
|
||||
data: {
|
||||
applicationId: app.id,
|
||||
eventType: 'receipt',
|
||||
url: 'https://example.invalid/never-send',
|
||||
secretEncrypted: 'unused',
|
||||
secretLast4: 'none',
|
||||
},
|
||||
});
|
||||
const channel = await db.smsChannel.create({
|
||||
data: {
|
||||
name: '隔离不联网',
|
||||
code: randomUUID(),
|
||||
gatewayHost: '127.0.0.1',
|
||||
gatewayPort: 1,
|
||||
account: randomUUID(),
|
||||
passwordCipher: 'unused',
|
||||
srcId: '1069',
|
||||
status: 'disabled',
|
||||
carriers: ['mobile'],
|
||||
},
|
||||
});
|
||||
const task = await db.smsBatchTask.create({
|
||||
data: {
|
||||
tenantId: tenant.id,
|
||||
applicationId: app.id,
|
||||
taskNo: randomUUID(),
|
||||
sourceType: 'cmpp',
|
||||
content: '测试'.repeat(100),
|
||||
phoneTotal: 1,
|
||||
},
|
||||
});
|
||||
const long = await db.smsMessageRecord.create({
|
||||
data: {
|
||||
tenantId: tenant.id,
|
||||
applicationId: app.id,
|
||||
batchTaskId: task.id,
|
||||
messageId: randomUUID(),
|
||||
phoneNumber: '13800138000',
|
||||
content: task.content,
|
||||
billingUnits: 3,
|
||||
status: 'submitted',
|
||||
channelId: channel.id,
|
||||
submitId: randomUUID(),
|
||||
cmppSubmitSequenceId: '42',
|
||||
cmppRegisteredDelivery: true,
|
||||
},
|
||||
});
|
||||
const submit = await db.smsSubmitRecord.create({
|
||||
data: {
|
||||
tenantId: tenant.id,
|
||||
messageRecordId: long.id,
|
||||
channelId: channel.id,
|
||||
submitId: long.submitId,
|
||||
submitStatus: 'accepted',
|
||||
},
|
||||
});
|
||||
await db.cmppInboundLongMessage.create({
|
||||
data: {
|
||||
tenantId: tenant.id,
|
||||
applicationId: app.id,
|
||||
groupKey: randomUUID(),
|
||||
account: app.cmppAccount,
|
||||
phoneNumbers: [long.phoneNumber],
|
||||
concatReference: 7,
|
||||
segmentTotal: 3,
|
||||
msgFmt: 8,
|
||||
messageId: long.messageId,
|
||||
expiresAt: new Date(Date.now() + 60000),
|
||||
status: 'completed',
|
||||
segments: {
|
||||
create: [1, 2, 3].map((index) => ({
|
||||
segmentIndex: index,
|
||||
sequenceId: String(40 + index),
|
||||
registeredDelivery: index !== 3,
|
||||
content: '测试',
|
||||
contentHash: randomUUID(),
|
||||
})),
|
||||
},
|
||||
},
|
||||
});
|
||||
await db.smsMessageRecord.update({ where: { id: long.id }, data: { cmppSubmitGroupMessageId: long.messageId } });
|
||||
const billing = new BillingService(db);
|
||||
const api = new OpenApiService(db, undefined);
|
||||
// No worker or network publisher is started: this stage verifies real business
|
||||
// persistence. Gateway/HTTP delivery is a separate environment acceptance stage.
|
||||
const chain = new SendChainService(db, billing, {}, {}, api);
|
||||
for (let index = 1; index <= 3; index++)
|
||||
await chain.handleSubmitSegmentResult({
|
||||
messageId: long.messageId,
|
||||
channelId: channel.id,
|
||||
submitId: submit.submitId,
|
||||
gatewayMessageId: `${prefix}-${index}`,
|
||||
sequenceId: index,
|
||||
segmentIndex: index,
|
||||
segmentTotal: 3,
|
||||
submitStatus: 'accepted',
|
||||
});
|
||||
const receipt = (index) => ({
|
||||
messageId: long.messageId,
|
||||
channelId: channel.id,
|
||||
gatewayMessageId: `${prefix}-${index}`,
|
||||
phoneNumber: long.phoneNumber,
|
||||
receiptStatus: 'delivered',
|
||||
rawStatus: 'DELIVRD',
|
||||
deliveredAt: '2026-09-16T00:00:00.000Z',
|
||||
});
|
||||
await chain.handleReceipt(receipt(1));
|
||||
assert.notEqual((await db.smsMessageRecord.findUniqueOrThrow({ where: { id: long.id } })).status, 'delivered');
|
||||
await Promise.all([
|
||||
chain.handleReceipt(receipt(2)),
|
||||
chain.handleReceipt(receipt(3)),
|
||||
chain.handleReceipt(receipt(2)),
|
||||
]);
|
||||
await chain.attemptCompletion.scan();
|
||||
assert.equal((await db.smsMessageRecord.findUniqueOrThrow({ where: { id: long.id } })).status, 'delivered');
|
||||
assert.equal(await db.smsReceiptRecord.count({ where: { messageRecordId: long.id } }), 3);
|
||||
assert.equal(await db.cmppDownstreamDelivery.count({ where: { messageRecordId: long.id } }), 2);
|
||||
assert.equal(await db.httpWebhookEvent.count({ where: { messageRecordId: long.id } }), 1);
|
||||
assert.equal((await db.smsBatchTask.findUniqueOrThrow({ where: { id: task.id } })).successTotal, 1);
|
||||
pass(
|
||||
'real SendChain: all three receipts required; two requested client fragments and one HTTP notification, unrequested fragment omitted',
|
||||
);
|
||||
|
||||
await db.tenantAccount.create({ data: { tenantId: tenant.id, balanceCents: 100000 } });
|
||||
const moneyMessage = await db.smsMessageRecord.create({
|
||||
data: {
|
||||
tenantId: tenant.id,
|
||||
applicationId: app.id,
|
||||
batchTaskId: task.id,
|
||||
messageId: randomUUID(),
|
||||
phoneNumber: '13800138000',
|
||||
content: task.content,
|
||||
billingUnits: 3,
|
||||
unitPrice: 325,
|
||||
amountCents: 975,
|
||||
status: 'submitted',
|
||||
channelId: channel.id,
|
||||
submitId: randomUUID(),
|
||||
submittedAt: new Date(Date.now() - 73 * 3600_000),
|
||||
},
|
||||
});
|
||||
await db.smsSubmitRecord.create({
|
||||
data: {
|
||||
tenantId: tenant.id,
|
||||
messageRecordId: moneyMessage.id,
|
||||
channelId: channel.id,
|
||||
submitId: moneyMessage.submitId,
|
||||
submitStatus: 'accepted',
|
||||
},
|
||||
});
|
||||
await billing.freeze({ tenantId: tenant.id, amountCents: 975, idempotencyKey: `qa-freeze:${moneyMessage.id}` });
|
||||
await chain.handleSubmitResult({
|
||||
messageId: moneyMessage.messageId,
|
||||
channelId: channel.id,
|
||||
submitId: moneyMessage.submitId,
|
||||
gatewayMessageId: `${prefix}-money`,
|
||||
submitStatus: 'accepted',
|
||||
submittedAt: moneyMessage.submittedAt.toISOString(),
|
||||
eventId: randomUUID(),
|
||||
});
|
||||
assert.equal(
|
||||
(await db.smsBillingRecord.findFirstOrThrow({ where: { messageId: moneyMessage.messageId } })).billingStatus,
|
||||
'charged',
|
||||
);
|
||||
await Promise.all([
|
||||
chain.markUnknownTimeout({ olderThanHours: 72 }),
|
||||
chain.markUnknownTimeout({ olderThanHours: 72 }),
|
||||
]);
|
||||
await chain.attemptCompletion.scan();
|
||||
assert.equal((await db.smsMessageRecord.findUniqueOrThrow({ where: { id: moneyMessage.id } })).status, 'timeout');
|
||||
assert.equal((await db.tenantAccount.findUniqueOrThrow({ where: { tenantId: tenant.id } })).balanceCents, 100000n);
|
||||
assert.equal(
|
||||
await db.accountTransaction.count({ where: { idempotencyKey: `sms-refund:${moneyMessage.messageId}` } }),
|
||||
1,
|
||||
);
|
||||
await chain.handleReceipt({
|
||||
messageId: moneyMessage.messageId,
|
||||
channelId: channel.id,
|
||||
gatewayMessageId: `${prefix}-money`,
|
||||
phoneNumber: moneyMessage.phoneNumber,
|
||||
receiptStatus: 'delivered',
|
||||
rawStatus: 'DELIVRD',
|
||||
});
|
||||
assert.equal((await db.smsMessageRecord.findUniqueOrThrow({ where: { id: moneyMessage.id } })).status, 'timeout');
|
||||
assert.equal(
|
||||
(await db.smsBillingRecord.findFirstOrThrow({ where: { messageId: moneyMessage.messageId } })).billingStatus,
|
||||
'refunded',
|
||||
);
|
||||
pass('nonzero charge, concurrent timeout, one refund and late receipt do not change terminal accounting');
|
||||
|
||||
const retryGroup = await db.smsChannelGroup.create({
|
||||
data: { code: randomUUID(), name: '事务补发隔离', retryEnabled: true },
|
||||
});
|
||||
const retryMessage = await db.smsMessageRecord.create({
|
||||
data: {
|
||||
tenantId: tenant.id,
|
||||
applicationId: app.id,
|
||||
batchTaskId: task.id,
|
||||
messageId: randomUUID(),
|
||||
phoneNumber: '13800138000',
|
||||
content: task.content,
|
||||
carrier: 'mobile',
|
||||
billingUnits: 3,
|
||||
status: 'submitted',
|
||||
channelId: channel.id,
|
||||
submitId: randomUUID(),
|
||||
},
|
||||
});
|
||||
const retrySource = await db.smsSubmitRecord.create({
|
||||
data: {
|
||||
tenantId: tenant.id,
|
||||
messageRecordId: retryMessage.id,
|
||||
channelId: channel.id,
|
||||
submitId: retryMessage.submitId,
|
||||
submitStatus: 'accepted',
|
||||
},
|
||||
});
|
||||
// Routing/rate limiting are deliberately isolated here. Production routing and
|
||||
// Redis must additionally pass the authorized simulator acceptance.
|
||||
chain.findApplicationRoute = async () => ({ group: retryGroup, groupId: retryGroup.id });
|
||||
chain.submission.selectChannelForMessage = async () => ({
|
||||
channel,
|
||||
groupId: retryGroup.id,
|
||||
groupName: retryGroup.name,
|
||||
carrier: 'mobile',
|
||||
routeScope: 'national',
|
||||
});
|
||||
chain.submission.waitForChannelRateLimit = async () => {};
|
||||
for (let index = 1; index <= 3; index++)
|
||||
await chain.handleSubmitSegmentResult({
|
||||
messageId: retryMessage.messageId,
|
||||
channelId: channel.id,
|
||||
submitId: retrySource.submitId,
|
||||
gatewayMessageId: `${prefix}-fail-${index}`,
|
||||
sequenceId: index,
|
||||
segmentIndex: index,
|
||||
segmentTotal: 3,
|
||||
submitStatus: 'accepted',
|
||||
});
|
||||
await Promise.all(
|
||||
[1, 2, 3].map((index) =>
|
||||
chain.handleReceipt({
|
||||
messageId: retryMessage.messageId,
|
||||
channelId: channel.id,
|
||||
gatewayMessageId: `${prefix}-fail-${index}`,
|
||||
phoneNumber: retryMessage.phoneNumber,
|
||||
receiptStatus: 'undelivered',
|
||||
rawStatus: 'UNDELIV',
|
||||
}),
|
||||
),
|
||||
);
|
||||
await chain.attemptCompletion.scan();
|
||||
const retries = await db.smsSubmitRecord.findMany({ where: { retryOfSubmitRecordId: retrySource.id } });
|
||||
assert.equal(retries.length, 1);
|
||||
assert.equal(await db.gatewaySubmitOutbox.count({ where: { submitId: retries[0].submitId } }), 1);
|
||||
assert.equal(await db.httpWebhookEvent.count({ where: { messageRecordId: retryMessage.id } }), 0);
|
||||
assert.equal(
|
||||
(await db.smsMessageRecord.findUniqueOrThrow({ where: { id: retryMessage.id } })).submitId,
|
||||
retries[0].submitId,
|
||||
);
|
||||
pass('three failure segments atomically create one successor Submit and Outbox without final failure notification');
|
||||
|
||||
const poisonMessage = await fixture();
|
||||
const poisonWork = await db.smsAttemptCompletionWork.create({
|
||||
data: {
|
||||
workKey: `message:${poisonMessage.id}`,
|
||||
messageRecordId: poisonMessage.id,
|
||||
revision: 1,
|
||||
attempts: 11,
|
||||
},
|
||||
});
|
||||
await db.smsCompletionEvent.create({
|
||||
data: { workId: poisonWork.id, eventKey: randomUUID(), kind: 'receipt', payload: {} },
|
||||
});
|
||||
const poisonWorker = new AttemptCompletion(
|
||||
db,
|
||||
async () => {
|
||||
throw new Error('injected unavailable database');
|
||||
},
|
||||
async () => {},
|
||||
);
|
||||
await poisonWorker.process(poisonWork.id);
|
||||
assert.equal(
|
||||
(await db.smsAttemptCompletionWork.findUniqueOrThrow({ where: { id: poisonWork.id } })).state,
|
||||
'needs_review',
|
||||
);
|
||||
const durableAlerts = await retainAlerts(db, [], new Date());
|
||||
assert(durableAlerts.some((item) => item.name === 'SmsCompletionNeedsReview' && item.status === 'firing'));
|
||||
await db.smsAttemptCompletionWork.update({
|
||||
where: { id: poisonWork.id },
|
||||
data: { state: 'pending', nextAttemptAt: new Date(0) },
|
||||
});
|
||||
await workers[0].process(poisonWork.id);
|
||||
assert.equal((await db.smsAttemptCompletionWork.findUniqueOrThrow({ where: { id: poisonWork.id } })).state, 'idle');
|
||||
const recoveredAlerts = await retainAlerts(db, [], new Date(Date.now() + 1));
|
||||
assert(recoveredAlerts.some((item) => item.name === 'SmsCompletionNeedsReview' && item.status === 'resolved'));
|
||||
pass(
|
||||
'twelfth failure stops in needs_review and is a durable visible alert; audited recovery retains the resolved alert',
|
||||
);
|
||||
|
||||
const at = Date.now() + 10;
|
||||
const alert = {
|
||||
fingerprint: prefix.replaceAll('-', '').slice(0, 24),
|
||||
startedAt: new Date(at).toISOString(),
|
||||
name: 'QA',
|
||||
severity: 'warning',
|
||||
status: 'firing',
|
||||
acknowledged: false,
|
||||
};
|
||||
const first = new Date(at + 1);
|
||||
const second = new Date(at + 2);
|
||||
await retainAlerts(db, [alert], first);
|
||||
const resolved = await retainAlerts(db, [], second);
|
||||
assert.equal(resolved.find((item) => item.fingerprint === alert.fingerprint)?.status, 'resolved');
|
||||
await retainAlerts(db, [alert], first);
|
||||
assert.equal((await retainedAlerts(db)).find((item) => item.fingerprint === alert.fingerprint).status, 'resolved');
|
||||
const operator = await db.user.create({
|
||||
data: { username: randomUUID(), displayName: 'isolated QA', passwordHash: 'disabled', status: 'disabled' },
|
||||
});
|
||||
await Promise.all([
|
||||
clearRetainedAlert(db, alert.fingerprint, alert.startedAt, operator.id),
|
||||
clearRetainedAlert(db, alert.fingerprint, alert.startedAt, operator.id),
|
||||
]);
|
||||
assert.equal((await retainedAlerts(db)).filter((item) => item.fingerprint === alert.fingerprint).length, 0);
|
||||
await retainAlerts(db, [alert], new Date(at + 3));
|
||||
assert.equal((await retainedAlerts(db)).filter((item) => item.fingerprint === alert.fingerprint).length, 0);
|
||||
await retainAlerts(db, [{ ...alert, startedAt: new Date(at + 4).toISOString() }], new Date(at + 5));
|
||||
assert.equal((await retainedAlerts(db)).filter((item) => item.fingerprint === alert.fingerprint).length, 1);
|
||||
pass('recovery retains alert, stale snapshots ignored, manual clear idempotent, new occurrence visible');
|
||||
} catch (error) {
|
||||
console.error(error instanceof Error ? error.message : String(error));
|
||||
process.exitCode = 1;
|
||||
} finally {
|
||||
await db.onModuleDestroy();
|
||||
}
|
||||
Reference in New Issue
Block a user