fix: coordinate SMS completion and improve operations diagnostics
CSS quality / css-quality (push) Has been cancelled

This commit is contained in:
hectorzhao
2026-09-16 18:27:29 +08:00
parent a0209f93bc
commit a350aca883
40 changed files with 2599 additions and 366 deletions
@@ -0,0 +1,17 @@
CREATE TABLE "InfrastructureAlertCollection" (
"id" TEXT NOT NULL PRIMARY KEY,
"observedAt" TIMESTAMP(3) NOT NULL
);
CREATE TABLE "InfrastructureAlertEvent" (
"id" TEXT NOT NULL PRIMARY KEY,
"fingerprint" TEXT NOT NULL,
"activeAt" TIMESTAMP(3) NOT NULL,
"payload" JSONB NOT NULL,
"lastObservedAt" TIMESTAMP(3) NOT NULL,
"recoveredAt" TIMESTAMP(3),
"clearedAt" TIMESTAMP(3),
"clearedBy" TEXT,
"createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP
);
CREATE UNIQUE INDEX "InfrastructureAlertEvent_fingerprint_activeAt_key" ON "InfrastructureAlertEvent"("fingerprint", "activeAt");
CREATE INDEX "InfrastructureAlertEvent_clearedAt_activeAt_idx" ON "InfrastructureAlertEvent"("clearedAt", "activeAt");
@@ -0,0 +1,15 @@
CREATE TABLE "SmsAttemptCompletionWork" (
"id" TEXT PRIMARY KEY, "workKey" TEXT NOT NULL, "tenantId" TEXT, "messageRecordId" TEXT NOT NULL, "sourceSubmitRecordId" TEXT,
"revision" INTEGER NOT NULL DEFAULT 0, "processedRevision" INTEGER NOT NULL DEFAULT 0, "state" TEXT NOT NULL DEFAULT 'pending',
"leaseOwner" TEXT, "leaseUntil" TIMESTAMP(3), "fenceVersion" INTEGER NOT NULL DEFAULT 0, "attempts" INTEGER NOT NULL DEFAULT 0,
"nextAttemptAt" TIMESTAMP(3) NOT NULL DEFAULT (CURRENT_TIMESTAMP AT TIME ZONE 'UTC'), "decision" TEXT, "retrySubmitRecordId" TEXT, "lastError" TEXT,
"createdAt" TIMESTAMP(3) NOT NULL DEFAULT (CURRENT_TIMESTAMP AT TIME ZONE 'UTC'), "updatedAt" TIMESTAMP(3) NOT NULL DEFAULT (CURRENT_TIMESTAMP AT TIME ZONE 'UTC'));
CREATE UNIQUE INDEX "SmsAttemptCompletionWork_workKey_key" ON "SmsAttemptCompletionWork"("workKey");
CREATE UNIQUE INDEX "SmsAttemptCompletionWork_sourceSubmitRecordId_key" ON "SmsAttemptCompletionWork"("sourceSubmitRecordId");
CREATE INDEX "SmsAttemptCompletionWork_state_nextAttemptAt_idx" ON "SmsAttemptCompletionWork"("state", "nextAttemptAt");
CREATE INDEX "SmsAttemptCompletionWork_state_leaseUntil_idx" ON "SmsAttemptCompletionWork"("state", "leaseUntil");
CREATE INDEX "SmsAttemptCompletionWork_messageRecordId_idx" ON "SmsAttemptCompletionWork"("messageRecordId");
CREATE TABLE "SmsCompletionEvent" ("id" TEXT PRIMARY KEY, "eventKey" TEXT NOT NULL, "workId" TEXT NOT NULL REFERENCES "SmsAttemptCompletionWork"("id") ON DELETE RESTRICT,
"kind" TEXT NOT NULL, "payload" JSONB NOT NULL, "processedAt" TIMESTAMP(3), "createdAt" TIMESTAMP(3) NOT NULL DEFAULT (CURRENT_TIMESTAMP AT TIME ZONE 'UTC'));
CREATE UNIQUE INDEX "SmsCompletionEvent_eventKey_key" ON "SmsCompletionEvent"("eventKey");
CREATE INDEX "SmsCompletionEvent_workId_processedAt_createdAt_idx" ON "SmsCompletionEvent"("workId", "processedAt", "createdAt");
+56
View File
@@ -2840,3 +2840,59 @@ model OpenApiDispatchOutbox {
updatedAt DateTime @updatedAt
@@index([status, leaseUntil])
}
model InfrastructureAlertCollection {
id String @id
observedAt DateTime
}
model InfrastructureAlertEvent {
id String @id @default(cuid())
fingerprint String
activeAt DateTime
payload Json
lastObservedAt DateTime
recoveredAt DateTime?
clearedAt DateTime?
clearedBy String?
createdAt DateTime @default(now())
@@unique([fingerprint, activeAt])
@@index([clearedAt, activeAt])
}
model SmsAttemptCompletionWork {
id String @id @default(cuid())
workKey String @unique
tenantId String?
messageRecordId String
sourceSubmitRecordId String? @unique
revision Int @default(0)
processedRevision Int @default(0)
state String @default("pending")
leaseOwner String?
leaseUntil DateTime?
fenceVersion Int @default(0)
attempts Int @default(0)
nextAttemptAt DateTime @default(dbgenerated("(CURRENT_TIMESTAMP AT TIME ZONE 'UTC')"))
decision String?
retrySubmitRecordId String?
lastError String?
createdAt DateTime @default(dbgenerated("(CURRENT_TIMESTAMP AT TIME ZONE 'UTC')"))
updatedAt DateTime @updatedAt
events SmsCompletionEvent[]
@@index([state, nextAttemptAt])
@@index([state, leaseUntil])
@@index([messageRecordId])
}
model SmsCompletionEvent {
id String @id @default(cuid())
eventKey String @unique
workId String
kind String
payload Json
processedAt DateTime?
createdAt DateTime @default(dbgenerated("(CURRENT_TIMESTAMP AT TIME ZONE 'UTC')"))
work SmsAttemptCompletionWork @relation(fields: [workId], references: [id], onDelete: Restrict)
@@index([workId, processedAt, createdAt])
}
+16 -12
View File
@@ -1,18 +1,24 @@
import { BadRequestException, Injectable, Logger, NotFoundException, OnModuleDestroy, OnModuleInit } from '@nestjs/common';
import { Queue } from 'bullmq';
import IORedis from 'ioredis';
import { BadRequestException, NotFoundException } from '@nestjs/common';
import { Prisma } from '@prisma/client';
import { randomUUID } from 'crypto';
import { assertMoneyUnits, moneyToNumber } from '../common/money';
import { moneyToNumber } from '../common/money';
import { PrismaService } from '../prisma/prisma.service';
import type { CreateChannelDto, UpdateChannelDto, CreateChannelGroupDto, CreateChannelGroupItemDto, UpdateChannelGroupDto, CreateRouteRuleDto, CreateReportFieldDto, ReplaceReportFieldsDto, CreateReportMaterialDto, CreateReportTaskDto, ChangeReportTaskStatusesDto, CreateReportExportDto, CreateReceiptImportDto, UpsertConnectionStateDto, ChangeChannelStatusDto, CopyChannelDto, TestChannelDto } from './channels.contracts';
import { GATEWAY_CONNECTION_QUEUE, GATEWAY_SUBMIT_QUEUE, GATEWAY_SUBMIT_STREAM, DEFAULT_GATEWAY_CONTROL_URL, DEFAULT_CHANNEL_CONNECTION_ID, DEFAULT_CONNECTING_TIMEOUT_MS, DEFAULT_CONNECTING_TIMEOUT_SCAN_MS, DEFAULT_GATEWAY_STARTUP_RECONNECT_DELAY_MS, DEFAULT_GATEWAY_RECONCILE_INTERVAL_MS, DEFAULT_GATEWAY_CONTROL_TIMEOUT_MS, DEFAULT_HEARTBEAT_INTERVAL_SECONDS, DEFAULT_HEARTBEAT_MISS_THRESHOLD, HEARTBEAT_AUDIT_INTERVAL_MS, CONNECTING_TIMEOUT_ERROR, DEFAULT_CMPP_VERSION, normalizeTestPhones, normalizeTestContent, calculateBillingUnits, buildChannelTestSubmitCommand, getConfigValue, getStringConfigValue, normalizeConnectionAction, normalizeCmppVersion, normalizeGatewayConnectionStatus, defaultChannelConnectionId, getDesiredConnections, ChannelConnectionSettings, getRuntimeConfigInteger, channelConnectionSettingsChanged, channelGroupAuditSnapshot, normalizeChannelRuntimeConfig, normalizeCmppServiceId, normalizeChannelRateLimit, normalizeExtensionDigits, getPositiveRuntimeInteger, bullmqConnection, getPositiveIntegerEnv, parseReceiptContent, splitReceiptLine, stripReceiptCell, findReceiptStatusIndex, normalizeReceiptStatus, deriveReceiptStatus, ChannelReportDeliveryRow, summarizeChannelReportDelivery, sumReportDelivery, percentage, latestDate, currentShanghaiDayRange, normalizeRetryTimeLimitMinutes, normalizeSpreadsheetSize, normalizeBusinessCarrier, normalizeChannelCarrier, isChannelCarrierCompatible, normalizeRegion, isRegionCompatible, validateGroupItems, normalizeReportType, summarizeReportStatuses, normalizeLinkEvent } from './channels.helpers';
import type { TestChannelDto } from './channels.contracts';
import {
normalizeTestPhones,
normalizeTestContent,
normalizeGatewayConnectionStatus,
calculateBillingUnits,
buildChannelTestSubmitCommand,
} from './channels.helpers';
import { ChannelConnectionService } from './channel-connection.service';
import { detectDrainageContent } from '../send-chain/drainage-content-detection';
/** R5 channel domain service composed behind ChannelsService. */
export class ChannelTestService {
constructor(private readonly prisma: PrismaService, private readonly connection: ChannelConnectionService) {}
constructor(
private readonly prisma: PrismaService,
private readonly connection: ChannelConnectionService,
) {}
async testChannel(channelId: string, data: TestChannelDto = {}) {
const phoneNumbers = normalizeTestPhones(data);
@@ -27,8 +33,8 @@ export class ChannelTestService {
if (channel.status !== 'active') {
throw new BadRequestException('通道未启用,不能发送测试短信');
}
const connectedState = channel.connectionStates.find((state) =>
normalizeGatewayConnectionStatus(state.status) === 'connected' && (state.currentConnections ?? 0) > 0,
const connectedState = channel.connectionStates.find(
(state) => normalizeGatewayConnectionStatus(state.status) === 'connected' && (state.currentConnections ?? 0) > 0,
);
if (!connectedState) {
throw new BadRequestException('通道当前没有可用 CMPP 连接,请先连接成功后再测试发送');
@@ -36,7 +42,6 @@ export class ChannelTestService {
const createdAt = new Date();
const testNo = `CHTEST-${Date.now()}-${randomUUID().slice(0, 8)}`;
const drainageDetection = await detectDrainageContent(this.prisma, content);
const results = [];
for (const [index, phoneNumber] of phoneNumbers.entries()) {
const messageId = `MSG-TEST-${Date.now()}-${randomUUID().slice(0, 8)}`;
@@ -51,7 +56,6 @@ export class ChannelTestService {
messageId,
phoneNumber,
content,
...drainageDetection,
billingUnits: calculateBillingUnits(content),
unitPrice: 0,
amountCents: 0,
@@ -37,6 +37,15 @@ export class InfrastructureMonitoringController {
return this.monitoring.markAlertRead(fingerprint, activeAt, userId);
}
@Post('alerts/:fingerprint/clear')
clearAlert(
@Param('fingerprint') fingerprint: string,
@Body('activeAt') activeAt: unknown,
@CurrentSessionUserId() userId: string,
) {
return this.monitoring.clearAlert(fingerprint, activeAt, userId);
}
@Get('alert-thresholds')
alertThresholds() {
return this.settings.get();
@@ -1,8 +1,15 @@
import { BadRequestException } from '@nestjs/common';
import { ConfigService } from '@nestjs/config';
import { createHash } from 'node:crypto';
import { InfrastructureMonitoringService } from './infrastructure-monitoring.service';
import { FILESYSTEM_USAGE_PERCENT, filesystemIdentity } from './filesystem-metrics';
import { InfrastructureMonitoringService } from './infrastructure-monitoring.service';
import { retainedAlerts } from './persistent-alerts';
jest.mock('./persistent-alerts', () => ({
retainAlerts: jest.fn(async (_prisma, alerts) => alerts),
retainedAlerts: jest.fn(),
clearRetainedAlert: jest.fn(),
}));
function success(data: unknown) {
return {
@@ -14,7 +21,12 @@ function success(data: unknown) {
describe('InfrastructureMonitoringService', () => {
const prisma = {
infrastructureAlertRead: { findMany: jest.fn().mockResolvedValue([]), create: jest.fn(), update: jest.fn(), findUniqueOrThrow: jest.fn() },
infrastructureAlertRead: {
findMany: jest.fn().mockResolvedValue([]),
create: jest.fn(),
update: jest.fn(),
findUniqueOrThrow: jest.fn(),
},
operationLog: { create: jest.fn() },
$transaction: jest.fn(),
};
@@ -34,9 +46,27 @@ describe('InfrastructureMonitoringService', () => {
});
it('rejects credential-bearing or remote plaintext Prometheus endpoints at startup', () => {
expect(() => new InfrastructureMonitoringService(new ConfigService({ PROMETHEUS_URL: 'http://user:secret@127.0.0.1:9090' }), prisma as never)).toThrow('must not contain credentials');
expect(() => new InfrastructureMonitoringService(new ConfigService({ PROMETHEUS_URL: 'http://monitor.example.com:9090' }), prisma as never)).toThrow('must use HTTPS');
expect(() => new InfrastructureMonitoringService(new ConfigService({ PROMETHEUS_URL: 'https://monitor.example.com' }), prisma as never)).not.toThrow();
expect(
() =>
new InfrastructureMonitoringService(
new ConfigService({ PROMETHEUS_URL: 'http://user:secret@127.0.0.1:9090' }),
prisma as never,
),
).toThrow('must not contain credentials');
expect(
() =>
new InfrastructureMonitoringService(
new ConfigService({ PROMETHEUS_URL: 'http://monitor.example.com:9090' }),
prisma as never,
),
).toThrow('must use HTTPS');
expect(
() =>
new InfrastructureMonitoringService(
new ConfigService({ PROMETHEUS_URL: 'https://monitor.example.com' }),
prisma as never,
),
).not.toThrow();
});
it('loads real Prometheus vectors, ranges, services and active alerts', async () => {
@@ -45,34 +75,52 @@ describe('InfrastructureMonitoringService', () => {
const url = new URL(String(input));
requestedUrls.push(url);
if (url.pathname.endsWith('/alerts')) {
return success({ alerts: [{
labels: { alertname: 'HostCpuHigh', severity: 'warning', instance: '127.0.0.1:9100' },
annotations: { summary: 'CPU持续偏高', threshold: '85%' },
state: 'firing',
activeAt: '2026-08-14T03:00:00.000Z',
value: '88.2',
}] });
return success({
alerts: [
{
labels: { alertname: 'HostCpuHigh', severity: 'warning', instance: '127.0.0.1:9100' },
annotations: { summary: 'CPU持续偏高', threshold: '85%' },
state: 'firing',
activeAt: '2026-08-14T03:00:00.000Z',
value: '88.2',
},
],
});
}
const query = url.searchParams.get('query') ?? '';
if (url.pathname.endsWith('/query_range')) {
return success({ result: [{ metric: {}, values: [[1_765_000_000, '12.5'], [1_765_000_060, '14.5']] }] });
return success({
result: [
{
metric: {},
values: [
[1_765_000_000, '12.5'],
[1_765_000_060, '14.5'],
],
},
],
});
}
if (query.includes('node_systemd_unit_state')) {
return success({ result: [
{ metric: { name: 'cmpp-api.service' }, value: [1_765_000_060, '1'] },
{ metric: { name: 'cmpp-gateway.service' }, value: [1_765_000_060, '1'] },
{ metric: { name: 'postgresql.service' }, value: [1_765_000_060, '1'] },
{ metric: { name: 'redis-server.service' }, value: [1_765_000_060, '1'] },
{ metric: { name: 'cmpp-minio.service' }, value: [1_765_000_060, '1'] },
{ metric: { name: 'nginx.service' }, value: [1_765_000_060, '1'] },
] });
return success({
result: [
{ metric: { name: 'cmpp-api.service' }, value: [1_765_000_060, '1'] },
{ metric: { name: 'cmpp-gateway.service' }, value: [1_765_000_060, '1'] },
{ metric: { name: 'postgresql.service' }, value: [1_765_000_060, '1'] },
{ metric: { name: 'redis-server.service' }, value: [1_765_000_060, '1'] },
{ metric: { name: 'cmpp-minio.service' }, value: [1_765_000_060, '1'] },
{ metric: { name: 'nginx.service' }, value: [1_765_000_060, '1'] },
],
});
}
if (query.includes('cmpp:service_.*')) {
return success({ result: [
{ metric: { __name__: 'cmpp:service_api:requests_per_second' }, value: [1_765_000_060, '12.5'] },
{ metric: { __name__: 'cmpp:service_api:error_percent' }, value: [1_765_000_060, '0.2'] },
{ metric: { __name__: 'cmpp:service_gateway:queue_pending' }, value: [1_765_000_060, '3'] },
] });
return success({
result: [
{ metric: { __name__: 'cmpp:service_api:requests_per_second' }, value: [1_765_000_060, '12.5'] },
{ metric: { __name__: 'cmpp:service_api:error_percent' }, value: [1_765_000_060, '0.2'] },
{ metric: { __name__: 'cmpp:service_gateway:queue_pending' }, value: [1_765_000_060, '3'] },
],
});
}
if (query.includes('timestamp(node_uname_info)')) {
return success({ result: [{ metric: {}, value: [1_765_000_060, '1765000060'] }] });
@@ -87,14 +135,27 @@ describe('InfrastructureMonitoringService', () => {
expect(result.metrics.cpuUsagePercent).toBe(25);
expect(result.trends.cpuUsagePercent).toHaveLength(2);
expect(result.summary).toMatchObject({ overallStatus: 'warning', serviceHealthy: 6, warningAlerts: 1 });
expect(result.services.find((item) => item.key === 'redis')).toMatchObject({ unit: 'redis-server.service', status: 'healthy' });
expect(result.services.find((item) => item.key === 'redis')).toMatchObject({
unit: 'redis-server.service',
status: 'healthy',
});
expect(result.serviceMetrics.find((item) => item.key === 'api')).toMatchObject({ available: true });
expect(result.serviceMetrics.find((item) => item.key === 'gateway')?.metrics.find((item) => item.key === 'queuePending')?.value).toBe(3);
expect(
result.serviceMetrics.find((item) => item.key === 'gateway')?.metrics.find((item) => item.key === 'queuePending')
?.value,
).toBe(3);
expect(result.alerts[0]).toMatchObject({ name: 'HostCpuHigh', severity: 'warning', currentValue: '88.2' });
expect(requestedUrls.filter((url) => url.pathname.endsWith('/query_range'))).toHaveLength(5);
expect(requestedUrls.filter((url) => url.pathname.endsWith('/query_range')).every((url) => url.searchParams.get('step') === '60')).toBe(true);
expect(requestedUrls.find((url) => url.searchParams.get('query')?.includes('node_systemd_unit_state'))?.searchParams.get('query'))
.toContain('cmpp-api\\\\.service');
expect(
requestedUrls
.filter((url) => url.pathname.endsWith('/query_range'))
.every((url) => url.searchParams.get('step') === '60'),
).toBe(true);
expect(
requestedUrls
.find((url) => url.searchParams.get('query')?.includes('node_systemd_unit_state'))
?.searchParams.get('query'),
).toContain('cmpp-api\\\\.service');
});
it('returns an explicit unavailable payload without stale metrics when Prometheus fails', async () => {
@@ -124,12 +185,36 @@ describe('InfrastructureMonitoringService', () => {
if (url.pathname.endsWith('/alerts')) return success({ alerts: [] });
if (!query.includes('node_filesystem_')) return success({ result: [] });
expect(query).not.toContain('mountpoint="/"');
if (url.pathname.endsWith('/query_range')) return success({ result: [...metrics].reverse().map((metric) => ({ metric, values: [[1765000060, metric.mountpoint === '/' ? '91' : '12']] })) });
return success({ result: metrics.map((metric) => ({ metric, value: [1765000060, query === FILESYSTEM_USAGE_PERCENT ? (metric.mountpoint === '/' ? '91' : '12') : query.includes('avail') ? '9' : '100'] })) });
if (url.pathname.endsWith('/query_range'))
return success({
result: [...metrics]
.reverse()
.map((metric) => ({ metric, values: [[1765000060, metric.mountpoint === '/' ? '91' : '12']] })),
});
return success({
result: metrics.map((metric) => ({
metric,
value: [
1765000060,
query === FILESYSTEM_USAGE_PERCENT
? metric.mountpoint === '/'
? '91'
: '12'
: query.includes('avail')
? '9'
: '100',
],
})),
});
});
const result = await new InfrastructureMonitoringService(new ConfigService(), prisma as never).overview('1h');
expect(result.disks.map((disk) => disk.mountpoint)).toEqual(['/', '/archive', '/data']);
expect(result.disks[0]).toMatchObject({ usagePercent: 91, totalBytes: 100, availableBytes: 9, trend: [{ timestamp: new Date(1765000060000).toISOString(), value: 91 }] });
expect(result.disks[0]).toMatchObject({
usagePercent: 91,
totalBytes: 100,
availableBytes: 9,
trend: [{ timestamp: new Date(1765000060000).toISOString(), value: 91 }],
});
expect(result.disks[2].trend[0].value).toBe(12);
expect(result.metrics.diskUsagePercent).toBe(91);
expect(result.trends.diskUsagePercent[0].value).toBe(91);
@@ -146,20 +231,44 @@ describe('InfrastructureMonitoringService', () => {
if (!query.includes('node_filesystem_')) return success({ result: [] });
if (url.pathname.endsWith('/query_range')) {
expect(query).toBe(FILESYSTEM_USAGE_PERCENT);
return success({ result: [
{ metric: data, values: [[1765000000, '82'], [1765000060, 'NaN'], [1765000120, '83.5']] },
{ metric: root, values: [[1765000000, '91']] },
] });
return success({
result: [
{
metric: data,
values: [
[1765000000, '82'],
[1765000060, 'NaN'],
[1765000120, '83.5'],
],
},
{ metric: root, values: [[1765000000, '91']] },
],
});
}
if (query.startsWith('node_filesystem_size_bytes')) return success({ result: [
...mounts.map((mountpoint) => ({ metric: { ...data, mountpoint }, value: [1765000120, '100'] })),
...['/var/root-bind', '/'].map((mountpoint) => ({ metric: { ...root, mountpoint }, value: [1765000120, '200'] })),
] });
if (query === FILESYSTEM_USAGE_PERCENT) return success({ result: [
{ metric: data, value: [1765000120, '83.5'] }, { metric: root, value: [1765000120, '91'] },
] });
if (query.startsWith('node_filesystem_size_bytes'))
return success({
result: [
...mounts.map((mountpoint) => ({ metric: { ...data, mountpoint }, value: [1765000120, '100'] })),
...['/var/root-bind', '/'].map((mountpoint) => ({
metric: { ...root, mountpoint },
value: [1765000120, '200'],
})),
],
});
if (query === FILESYSTEM_USAGE_PERCENT)
return success({
result: [
{ metric: data, value: [1765000120, '83.5'] },
{ metric: root, value: [1765000120, '91'] },
],
});
expect(query).toContain('min by (instance, device, fstype)');
return success({ result: [{ metric: data, value: [1765000120, '16.5'] }, { metric: root, value: [1765000120, '18'] }] });
return success({
result: [
{ metric: data, value: [1765000120, '16.5'] },
{ metric: root, value: [1765000120, '18'] },
],
});
});
const service = new InfrastructureMonitoringService(new ConfigService(), prisma as never);
const result = await service.overview('1h');
@@ -169,7 +278,13 @@ describe('InfrastructureMonitoringService', () => {
expect(result.metrics.diskUsagePercent).toBe(91);
expect(result.trends.diskUsagePercent[0].value).toBe(91);
const disk = result.disks[1];
expect(disk).toMatchObject({ id: filesystemIdentity(data), mountpoint: '/data', totalBytes: 100, availableBytes: 16.5, usagePercent: 83.5 });
expect(disk).toMatchObject({
id: filesystemIdentity(data),
mountpoint: '/data',
totalBytes: 100,
availableBytes: 16.5,
usagePercent: 83.5,
});
expect(disk.mountpoints).toEqual(['/data', '/var/lib/minio', '/var/lib/pgsql', '/var/lib/redis']);
expect(disk.trend.map((point) => point.value)).toEqual([82, 83.5]);
mounts.reverse();
@@ -198,33 +313,77 @@ describe('InfrastructureMonitoringService', () => {
const result = await new InfrastructureMonitoringService(new ConfigService(), prisma as never).overview('1h');
expect(result.disks).toHaveLength(4);
expect(new Set(result.disks.map((disk) => disk.id)).size).toBe(4);
expect(result.disks.every((disk) => disk.usagePercent === null && disk.availableBytes === null && disk.trend.length === 0)).toBe(true);
expect(
result.disks.every(
(disk) => disk.usagePercent === null && disk.availableBytes === null && disk.trend.length === 0,
),
).toBe(true);
expect(result.metrics.diskUsagePercent).toBeNull();
expect(result.trends.diskUsagePercent).toEqual([]);
});
it('excludes only the current alert occurrence after the current administrator marks it read', async () => {
const labels = { alertname: 'QaWarning', severity: 'warning', service: 'qa-preview' };
const fingerprint = createHash('sha256').update(JSON.stringify(Object.entries(labels).sort(([left], [right]) => left.localeCompare(right)))).digest('hex').slice(0, 24);
jest.spyOn(global, 'fetch').mockResolvedValue(success({ alerts: [{ labels, annotations: { summary: '演示预警' }, state: 'firing', activeAt: '2026-08-16T01:00:00.000Z' }] }));
prisma.infrastructureAlertRead.findMany.mockResolvedValue([{ fingerprint, activeAt: new Date('2026-08-16T01:00:00.000Z'), readAt: new Date('2026-08-16T01:01:00.000Z') }]);
const fingerprint = createHash('sha256')
.update(JSON.stringify(Object.entries(labels).sort(([left], [right]) => left.localeCompare(right))))
.digest('hex')
.slice(0, 24);
jest.spyOn(global, 'fetch').mockResolvedValue(
success({
alerts: [
{ labels, annotations: { summary: '演示预警' }, state: 'firing', activeAt: '2026-08-16T01:00:00.000Z' },
],
}),
);
prisma.infrastructureAlertRead.findMany.mockResolvedValue([
{ fingerprint, activeAt: new Date('2026-08-16T01:00:00.000Z'), readAt: new Date('2026-08-16T01:01:00.000Z') },
]);
const service = new InfrastructureMonitoringService(new ConfigService(), prisma as never);
await expect(service.notificationSummary('admin-1')).resolves.toEqual({ count: 0, criticalCount: 0 });
prisma.infrastructureAlertRead.findMany.mockResolvedValue([{ fingerprint, activeAt: new Date('2026-08-15T01:00:00.000Z'), readAt: new Date('2026-08-15T01:01:00.000Z') }]);
prisma.infrastructureAlertRead.findMany.mockResolvedValue([
{ fingerprint, activeAt: new Date('2026-08-15T01:00:00.000Z'), readAt: new Date('2026-08-15T01:01:00.000Z') },
]);
await expect(service.notificationSummary('admin-1')).resolves.toEqual({ count: 1, criticalCount: 0 });
});
it('upserts an idempotent per-user read record only for a currently active occurrence', async () => {
const labels = { alertname: 'QaCritical', severity: 'critical', service: 'qa-preview' };
const fingerprint = createHash('sha256').update(JSON.stringify(Object.entries(labels).sort(([left], [right]) => left.localeCompare(right)))).digest('hex').slice(0, 24);
jest.spyOn(global, 'fetch').mockResolvedValue(success({ alerts: [{ labels, annotations: { summary: '演示严重告警' }, state: 'firing', activeAt: '2026-08-16T02:00:00.000Z' }] }));
prisma.$transaction.mockResolvedValue([{ activeAt: new Date('2026-08-16T02:00:00.000Z'), readAt: new Date('2026-08-16T02:01:00.000Z') }, {}]);
const fingerprint = createHash('sha256')
.update(JSON.stringify(Object.entries(labels).sort(([left], [right]) => left.localeCompare(right))))
.digest('hex')
.slice(0, 24);
jest.spyOn(global, 'fetch').mockResolvedValue(
success({
alerts: [
{ labels, annotations: { summary: '演示严重告警' }, state: 'firing', activeAt: '2026-08-16T02:00:00.000Z' },
],
}),
);
prisma.$transaction.mockResolvedValue([
{ activeAt: new Date('2026-08-16T02:00:00.000Z'), readAt: new Date('2026-08-16T02:01:00.000Z') },
{},
]);
const service = new InfrastructureMonitoringService(new ConfigService(), prisma as never);
await expect(service.markAlertRead(fingerprint, '2026-08-16T02:00:00.000Z', 'admin-1')).resolves.toMatchObject({ fingerprint, acknowledged: true });
expect(prisma.infrastructureAlertRead.create).toHaveBeenCalledWith(expect.objectContaining({ data: expect.objectContaining({ fingerprint, userId: 'admin-1' }) }));
await expect(service.markAlertRead(fingerprint, '2026-08-15T02:00:00.000Z', 'admin-1')).rejects.toThrow('已结束或已重新触发');
jest.mocked(retainedAlerts).mockResolvedValue([
{
fingerprint,
startedAt: '2026-08-16T02:00:00.000Z',
name: 'QaCritical',
severity: 'critical',
} as never,
]);
await expect(service.markAlertRead(fingerprint, '2026-08-16T02:00:00.000Z', 'admin-1')).resolves.toMatchObject({
fingerprint,
acknowledged: true,
});
expect(prisma.infrastructureAlertRead.create).toHaveBeenCalledWith(
expect.objectContaining({ data: expect.objectContaining({ fingerprint, userId: 'admin-1' }) }),
);
await expect(service.markAlertRead(fingerprint, '2026-08-15T02:00:00.000Z', 'admin-1')).rejects.toThrow(
'已结束或已重新触发',
);
});
});
@@ -1,3 +1,4 @@
import { retainAlerts, retainedAlerts, clearRetainedAlert } from './persistent-alerts';
import {
BadRequestException,
Injectable,
@@ -202,6 +203,8 @@ export class InfrastructureMonitoringService {
private readonly logger = new Logger(InfrastructureMonitoringService.name);
private readonly prometheusUrl: string;
private readonly queryTimeoutMs: number;
private alertTimer?: ReturnType<typeof setInterval>;
private alertPoll?: Promise<InfrastructureAlert[]>;
constructor(
config: ConfigService,
@@ -211,6 +214,35 @@ export class InfrastructureMonitoringService {
this.queryTimeoutMs = Math.min(15_000, Math.max(1_000, Number(config.get('PROMETHEUS_QUERY_TIMEOUT_MS') ?? 5_000)));
}
onModuleInit() {
if (process.env.CMPP_PROCESS_ROLE && !['api', 'all'].includes(process.env.CMPP_PROCESS_ROLE)) return;
const poll = () =>
void this.loadRetainedAlerts().catch(() => this.logger.warn('Persistent alert collection unavailable'));
poll();
this.alertTimer = setInterval(poll, 30_000);
this.alertTimer.unref();
}
onModuleDestroy() {
if (this.alertTimer) clearInterval(this.alertTimer);
}
private loadRetainedAlerts() {
if (!this.alertPoll) {
const observedAt = new Date();
this.alertPoll = this.getJson<PrometheusAlertResponse>('/api/v1/alerts')
.then((response) => retainAlerts(this.prisma, this.parseAlerts(response), observedAt))
.finally(() => {
this.alertPoll = undefined;
});
}
return this.alertPoll;
}
clearAlert(fingerprint: string, activeAt: unknown, userId: string) {
return clearRetainedAlert(this.prisma, fingerprint, activeAt, userId);
}
async overview(rawRange?: string, userId?: string): Promise<InfrastructureMonitoringOverview> {
const range = this.parseRange(rawRange);
const collectedAt = new Date().toISOString();
@@ -220,11 +252,11 @@ export class InfrastructureMonitoringService {
this.loadTrends(range),
this.query(QUERIES.services),
this.query(SERVICE_METRICS_QUERY),
this.getJson<PrometheusAlertResponse>('/api/v1/alerts'),
this.loadRetainedAlerts(),
]);
const services = this.parseServices(serviceResponse);
const serviceMetrics = this.parseServiceMetrics(serviceMetricResponse);
const alerts = await this.attachReadState(this.parseAlerts(alertResponse), userId);
const alerts = await this.attachReadState(alertResponse, userId);
const warningAlerts = alerts.filter((item) => item.severity === 'warning').length;
const criticalAlerts = alerts.filter((item) => item.severity === 'critical').length;
const overallStatus = criticalAlerts > 0 ? 'critical' : warningAlerts > 0 ? 'warning' : 'healthy';
@@ -250,7 +282,7 @@ export class InfrastructureMonitoringService {
alerts,
};
} catch (error) {
// 页面必须整体清空陈旧指标,但服务端仍需留下不含PromQL/地址/凭据的根因摘要便于运维诊断
// 客户端保留最后成功快照;采集失败不推断告警恢复
this.logger.warn(
`Prometheus monitoring overview unavailable: ${error instanceof Error ? error.message : 'unknown error'}`,
);
@@ -260,10 +292,7 @@ export class InfrastructureMonitoringService {
async notificationSummary(userId?: string) {
try {
const alerts = await this.attachReadState(
this.parseAlerts(await this.getJson<PrometheusAlertResponse>('/api/v1/alerts')),
userId,
);
const alerts = await this.attachReadState(await this.loadRetainedAlerts(), userId);
const unreadAlerts = alerts.filter((item) => !item.acknowledged);
return {
count: unreadAlerts.length,
@@ -281,7 +310,7 @@ export class InfrastructureMonitoringService {
if (!/^[a-f0-9]{24}$/.test(fingerprint)) throw new BadRequestException('告警指纹无效');
const activeAt = new Date(String(rawActiveAt ?? ''));
if (!Number.isFinite(activeAt.getTime())) throw new BadRequestException('告警开始时间无效');
const activeAlerts = this.parseAlerts(await this.getJson<PrometheusAlertResponse>('/api/v1/alerts'));
const activeAlerts = await retainedAlerts(this.prisma);
const current = activeAlerts.find(
(item) => item.fingerprint === fingerprint && Date.parse(item.startedAt) === activeAt.getTime(),
);
@@ -0,0 +1,143 @@
import { BadRequestException, NotFoundException } from '@nestjs/common';
import { Prisma } from '@prisma/client';
import { createHash } from 'node:crypto';
import { PrismaService } from '../prisma/prisma.service';
import type { InfrastructureAlert } from './infrastructure-monitoring.contracts';
export async function retainAlerts(prisma: PrismaService, alerts: InfrastructureAlert[], observedAt: Date) {
await prisma.$transaction(async (tx) => {
// Serialize snapshots across API processes; timestamps reject late HTTP results.
await tx.$executeRaw`SELECT pg_advisory_xact_lock(160916, 1)`;
const previous = await tx.infrastructureAlertCollection.findUnique({ where: { id: 'prometheus' } });
if (previous && previous.observedAt >= observedAt) return;
// Read durable work directly: application releases do not install Prometheus rules.
// Keep one occurrence identity until the condition really recovers, even after manual clear.
const reviewCount = await tx.smsAttemptCompletionWork.count({ where: { state: 'needs_review' } });
const oldest = await tx.smsCompletionEvent.findFirst({
where: { processedAt: null, work: { state: { in: ['pending', 'processing', 'retry_wait'] } } },
orderBy: { createdAt: 'asc' },
select: { createdAt: true },
});
const age = oldest ? Math.max(0, (observedAt.getTime() - oldest.createdAt.getTime()) / 1000) : 0;
alerts = [...alerts];
for (const condition of [
{
name: 'SmsCompletionNeedsReview',
active: reviewCount > 0,
severity: 'critical' as const,
summary: '短信收尾工作需要人工排查',
value: String(reviewCount),
threshold: '0',
},
{
name: 'SmsCompletionBacklog',
active: age > 300,
severity: 'warning' as const,
summary: '短信收尾工作等待超过5分钟',
value: `${Math.floor(age)}`,
threshold: '300秒',
},
]) {
if (!condition.active) continue;
const fingerprint = createHash('sha256').update(`durable:${condition.name}`).digest('hex').slice(0, 24);
const occurrence = await tx.infrastructureAlertEvent.findFirst({
where: { fingerprint, recoveredAt: null },
orderBy: { activeAt: 'desc' },
});
alerts.push({
fingerprint,
name: condition.name,
severity: condition.severity,
status: 'firing',
startedAt: (occurrence?.activeAt ?? observedAt).toISOString(),
summary: condition.summary,
currentValue: condition.value,
threshold: condition.threshold,
service: '短信收尾',
acknowledged: false,
});
}
await tx.infrastructureAlertCollection.upsert({
where: { id: 'prometheus' },
create: { id: 'prometheus', observedAt },
update: { observedAt },
});
for (const alert of alerts) {
const activeAt = new Date(alert.startedAt);
const payload = JSON.parse(JSON.stringify(alert)) as Prisma.InputJsonValue;
await tx.infrastructureAlertEvent.createMany({
data: [{ fingerprint: alert.fingerprint, activeAt, payload, lastObservedAt: observedAt }],
skipDuplicates: true,
});
await tx.infrastructureAlertEvent.updateMany({
where: { fingerprint: alert.fingerprint, activeAt, lastObservedAt: { lte: observedAt } },
data: { payload, lastObservedAt: observedAt, recoveredAt: null },
});
}
await tx.infrastructureAlertEvent.updateMany({
where: {
recoveredAt: null,
lastObservedAt: { lt: observedAt },
...(alerts.length
? {
NOT: {
OR: alerts.map((alert) => ({ fingerprint: alert.fingerprint, activeAt: new Date(alert.startedAt) })),
},
}
: {}),
},
data: { recoveredAt: observedAt },
});
});
return retainedAlerts(prisma);
}
export async function retainedAlerts(prisma: PrismaService): Promise<InfrastructureAlert[]> {
const records = await prisma.infrastructureAlertEvent.findMany({
where: { clearedAt: null },
orderBy: [{ activeAt: 'desc' }, { id: 'asc' }],
});
return records.map((record) => ({
...(record.payload as unknown as InfrastructureAlert),
...(record.recoveredAt ? { status: 'resolved' } : {}),
acknowledged: false,
}));
}
export async function clearRetainedAlert(
prisma: PrismaService,
fingerprint: string,
rawActiveAt: unknown,
userId: string,
) {
const activeAt = new Date(String(rawActiveAt ?? ''));
if (!/^[a-f0-9]{24}$/.test(fingerprint) || !Number.isFinite(activeAt.getTime()))
throw new BadRequestException('告警标识无效');
return prisma.$transaction(async (tx) => {
const record = await tx.infrastructureAlertEvent.findUnique({
where: { fingerprint_activeAt: { fingerprint, activeAt } },
});
if (!record) throw new NotFoundException('告警记录不存在');
const clearedAt = new Date();
const result = await tx.infrastructureAlertEvent.updateMany({
where: { id: record.id, clearedAt: null },
data: { clearedAt, clearedBy: userId },
});
if (result.count)
await tx.operationLog.create({
data: {
userId,
action: 'monitoring.alert_cleared',
resource: 'infrastructure_alert',
resourceId: record.id,
detail: { fingerprint, activeAt: activeAt.toISOString() },
},
});
return {
fingerprint,
activeAt: activeAt.toISOString(),
cleared: true,
clearedAt: (record.clearedAt ?? clearedAt).toISOString(),
};
});
}
+2
View File
@@ -1,3 +1,4 @@
import { renderCompletionMetrics } from '../send-chain/completion-metrics';
import { Injectable, OnModuleDestroy } from '@nestjs/common';
import { monitorEventLoopDelay } from 'node:perf_hooks';
@@ -314,6 +315,7 @@ export class MetricsService implements OnModuleDestroy {
lines.push(metricLine('cmpp_api_auth_protection_events_total', count, { event, scope }));
}
this.eventLoopDelay.reset();
lines.push(...renderCompletionMetrics());
return `${lines.join('\n')}\n`;
}
+19 -13
View File
@@ -526,24 +526,28 @@ export class OpenApiService implements OnModuleInit, OnModuleDestroy {
return row;
}
async queueWebhookEvent(data: {
tenantId: string;
applicationId?: string | null;
messageRecordId?: string | null;
messageId?: string | null;
uplinkMessageId?: string | null;
eventType: 'receipt' | 'uplink';
payload: Record<string, unknown>;
}) {
async queueWebhookEvent(
data: {
tenantId: string;
applicationId?: string | null;
messageRecordId?: string | null;
messageId?: string | null;
uplinkMessageId?: string | null;
eventType: 'receipt' | 'uplink';
payload: Record<string, unknown>;
},
transaction?: Prisma.TransactionClient,
) {
const db = transaction ?? this.prisma;
if (!data.applicationId) return null;
const application = await this.prisma.smsApplication.findUnique({
const application = await db.smsApplication.findUnique({
where: { id: data.applicationId },
include: { httpConfig: true },
});
const config = application?.httpConfig;
const enabled = data.eventType === 'receipt' ? config?.receiptWebhookEnabled : config?.uplinkWebhookEnabled;
if (!config?.enabled || !enabled) return null;
const endpoint = await this.prisma.httpWebhookEndpoint.findUnique({
const endpoint = await db.httpWebhookEndpoint.findUnique({
where: { applicationId_eventType: { applicationId: data.applicationId, eventType: data.eventType } },
});
if (!endpoint || endpoint.status !== 'active') return null;
@@ -553,7 +557,7 @@ export class OpenApiService implements OnModuleInit, OnModuleDestroy {
: data.eventType === 'uplink' && data.uplinkMessageId
? `evt_uplink_${data.uplinkMessageId}`
: `evt_${randomUUID()}`;
const delivery = await this.prisma.$transaction(async (tx) => {
const persist = async (tx: Prisma.TransactionClient) => {
const event = await tx.httpWebhookEvent.upsert({
where: { eventId },
update: {},
@@ -573,7 +577,9 @@ export class OpenApiService implements OnModuleInit, OnModuleDestroy {
update: {},
create: { eventId: event.id, endpointId: endpoint.id, recoveryVersion: 1 },
});
});
};
const delivery = transaction ? await persist(transaction) : await this.prisma.$transaction(persist);
if (transaction) return delivery;
if (delivery.status !== 'pending' || delivery.recoveryVersion !== 1) return delivery;
await this.queue?.add(
'deliver',
@@ -1,58 +1,85 @@
import { BadRequestException, ConflictException, Injectable, NotFoundException } from '@nestjs/common';
import { Prisma } from '@prisma/client';
import ExcelJS from 'exceljs';
import { createHash, randomUUID } from 'node:crypto';
import { extname } from 'node:path';
import { FilesService } from '../files/files.service';
import { PrismaService } from '../prisma/prisma.service';
import { SmsConfigService } from '../sms-config/sms-config.service';
import type { AnalyzeImportOptions, CreateImportProfileDto, CreateReportBatchDto, EmbeddedImage, ImportCommitDto, ImportMapping, PagedQuery, ReportBatchInspection, ReportBatchTarget, ReviewImportItemsDto } from './report-materials.contracts';
import { profileData, validateProfile, loadWorkbook, assertSafeWorkbook, safeSpreadsheetText, readEmbeddedImages, suggestMappings, remapProfileColumns, signatureCoreMapping, drainageCoreMapping, normalizeHeader, normalizeFieldCode, clamp, normalizePage, normalizePageSize, dateRange, cellText, transformValue, mappedCoreValue, dynamicValues, jsonRecord, hasValue, isFileRef, resolveExportValue, applyExportTransform, styleHeader, normalizeImageExtension, imageContentType, safeFileName, normalizeBatchIdempotencyKey, jsonStringArray, jsonSafe } from './report-materials.helpers';
import { ReportPendingQueryService } from './pending-query.service';
import { safeSpreadsheetText, styleHeader } from './report-materials.helpers';
/** R4 report-materials domain service composed behind ReportMaterialsService. */
export class ReportOfficialExportService {
constructor(private readonly prisma: PrismaService, private readonly files: FilesService, private readonly smsConfig: SmsConfigService, private readonly pending: ReportPendingQueryService) {}
constructor(
private readonly prisma: PrismaService,
private readonly files: FilesService,
private readonly smsConfig: SmsConfigService,
private readonly pending: ReportPendingQueryService,
) {}
async buildOfficialTemplate(reportType: 'signature' | 'drainage', operatorId?: string) {
const workbook = new ExcelJS.Workbook();
workbook.creator = '聆界短信平台';
const sheet = workbook.addWorksheet(reportType === 'signature' ? '签名资料' : '引流信息', { views: [{ state: 'frozen', ySplit: 1 }] });
const headers = reportType === 'signature'
const workbook = new ExcelJS.Workbook();
workbook.creator = '聆界短信平台';
const sheet = workbook.addWorksheet(reportType === 'signature' ? '签名资料' : '引流信息', {
views: [{ state: 'frozen', ySplit: 1 }],
});
const headers =
reportType === 'signature'
? ['短信签名', '用途说明', '营业执照图片', '授权书图片', '备注']
: ['短信签名', '引流 URL 或号码', '备注', '主体证明'];
sheet.addRow(headers);
sheet.addRow(reportType === 'signature'
sheet.addRow(headers);
sheet.addRow(
reportType === 'signature'
? ['示例签名', '验证码通知', '请在本单元格插入图片', '请在本单元格插入图片', '示例行,导入前请删除']
: ['示例签名', 'example.com/path 或 13800138000', '示例行,导入前请删除', '请在本单元格插入图片']);
styleHeader(sheet.getRow(1));
sheet.columns.forEach((column) => { column.width = 24; });
sheet.getRow(2).height = 48;
const content = Buffer.from(await workbook.xlsx.writeBuffer());
const fileName = `${reportType === 'signature' ? '签名' : '引流信息'}报备资料官方模板.xlsx`;
await this.prisma.operationLog.create({ data: {
userId: operatorId, action: 'report_material.template_downloaded', resource: 'report_material',
: ['示例签名', 'example.com/path 或 13800138000', '示例行,导入前请删除', '请在本单元格插入图片'],
);
styleHeader(sheet.getRow(1));
sheet.columns.forEach((column) => {
column.width = 24;
});
sheet.getRow(2).height = 48;
const content = Buffer.from(await workbook.xlsx.writeBuffer());
const fileName = `${reportType === 'signature' ? '签名' : '引流信息'}报备资料官方模板.xlsx`;
await this.prisma.operationLog.create({
data: {
userId: operatorId,
action: 'report_material.template_downloaded',
resource: 'report_material',
detail: { fileName, filters: { reportType }, successCount: 1, failedCount: 0 } as Prisma.InputJsonValue,
} });
return { fileName, content };
}
},
});
return { fileName, content };
}
async exportPending(query: { reportType?: 'signature' | 'drainage'; tenantId?: string; applicationId?: string }, operatorId?: string) {
const items = await this.pending.findPendingItems(query);
const workbook = new ExcelJS.Workbook();
const sheet = workbook.addWorksheet('待报备资料', { views: [{ state: 'frozen', ySplit: 1 }] });
sheet.addRow(['资料类型', '企业', '企业应用', '签名/站点', '详情', '变更时间']);
styleHeader(sheet.getRow(1));
for (const item of items) sheet.addRow([
item.reportType === 'signature' ? '签名' : '引流信息', safeSpreadsheetText(item.tenant?.name),
safeSpreadsheetText(item.application?.name), safeSpreadsheetText(item.name), safeSpreadsheetText(item.detail), item.changedAt,
async exportPending(
query: { reportType?: 'signature' | 'drainage'; tenantId?: string; applicationId?: string },
operatorId?: string,
) {
const items = await this.pending.findPendingItems(query);
const workbook = new ExcelJS.Workbook();
const sheet = workbook.addWorksheet('待报备资料', { views: [{ state: 'frozen', ySplit: 1 }] });
sheet.addRow(['资料类型', '企业', '企业应用', '签名/站点', '详情', '变更时间']);
styleHeader(sheet.getRow(1));
for (const item of items)
sheet.addRow([
item.reportType === 'signature' ? '签名' : '引流信息',
safeSpreadsheetText(item.tenant?.name),
safeSpreadsheetText(item.application?.name),
safeSpreadsheetText(item.name),
safeSpreadsheetText(item.detail),
item.changedAt,
]);
sheet.columns.forEach((column, index) => { column.width = index === 4 ? 42 : 22; });
const fileName = `待报备资料-${new Date().toISOString().slice(0, 10)}.xlsx`;
await this.prisma.operationLog.create({ data: {
tenantId: query.tenantId, userId: operatorId, action: 'report_material.pending_export', resource: 'report_material',
sheet.columns.forEach((column, index) => {
column.width = index === 4 ? 42 : 22;
});
const fileName = `待报备资料-${new Date().toISOString().slice(0, 10)}.xlsx`;
await this.prisma.operationLog.create({
data: {
tenantId: query.tenantId,
userId: operatorId,
action: 'report_material.pending_export',
resource: 'report_material',
detail: { fileName, filters: query, successCount: items.length, failedCount: 0 } as Prisma.InputJsonValue,
} });
return { fileName, content: Buffer.from(await workbook.xlsx.writeBuffer()) };
}
},
});
return { fileName, content: Buffer.from(await workbook.xlsx.writeBuffer()) };
}
}
+219
View File
@@ -0,0 +1,219 @@
import { HttpException, Logger } from '@nestjs/common';
import { Prisma, SmsAttemptCompletionWork } from '@prisma/client';
import { createHash, randomUUID } from 'node:crypto';
import { PrismaService } from '../prisma/prisma.service';
import { completionContext, CompletionRouteRequired } from './completion-context';
import type { RoutedChannel } from './send-chain.contracts';
import { countCompletion, observeCompletion } from './completion-metrics';
export type CompletionEventKind = 'receipt' | 'submit' | 'segment' | 'timeout' | 'rejection';
export class AttemptCompletion {
private readonly logger = new Logger(AttemptCompletion.name);
private timer?: ReturnType<typeof setInterval>;
private running = false;
constructor(
private readonly prisma: PrismaService,
private readonly execute: (kind: CompletionEventKind, payload: Prisma.JsonValue) => Promise<unknown>,
private readonly waitForRoute: (route: RoutedChannel) => Promise<void>,
) {}
start() {
this.timer = setInterval(() => void this.scan(), 5_000);
this.timer.unref();
void this.scan();
}
stop() {
if (this.timer) clearInterval(this.timer);
}
async enqueue(
messageRecordId: string,
sourceSubmitRecordId: string | undefined,
kind: CompletionEventKind,
payload: unknown,
identity?: string,
) {
const json = JSON.parse(JSON.stringify(payload)) as Prisma.InputJsonValue;
const workKey = sourceSubmitRecordId ? `attempt:${sourceSubmitRecordId}` : `message:${messageRecordId}`;
const eventKey = createHash('sha256')
.update(`${workKey}:${kind}:${identity ?? JSON.stringify(json)}`)
.digest('hex');
const work = await this.prisma.$transaction(async (tx) => {
const message = await tx.smsMessageRecord.findUniqueOrThrow({
where: { id: messageRecordId },
select: { tenantId: true },
});
if (sourceSubmitRecordId) {
const source = await tx.smsSubmitRecord.findUniqueOrThrow({ where: { id: sourceSubmitRecordId } });
if (source.messageRecordId !== messageRecordId || source.tenantId !== message.tenantId)
throw new Error('completion_source_mismatch');
}
await tx.smsAttemptCompletionWork.createMany({
data: [{ workKey, messageRecordId, sourceSubmitRecordId, tenantId: message.tenantId }],
skipDuplicates: true,
});
const current = await tx.smsAttemptCompletionWork.findUniqueOrThrow({ where: { workKey } });
if (current.messageRecordId !== messageRecordId) throw new Error('completion_work_mismatch');
await tx.$queryRaw`SELECT id FROM "SmsAttemptCompletionWork" WHERE id=${current.id} FOR UPDATE`;
const inserted = await tx.smsCompletionEvent.createMany({
data: [{ workId: current.id, eventKey, kind, payload: json }],
skipDuplicates: true,
});
if (inserted.count)
await tx.$executeRaw`
UPDATE "SmsAttemptCompletionWork" SET revision=revision+1,
state=CASE WHEN state IN ('processing', 'needs_review') THEN state ELSE 'pending' END,
"nextAttemptAt"=(CURRENT_TIMESTAMP AT TIME ZONE 'UTC'), "updatedAt"=(CURRENT_TIMESTAMP AT TIME ZONE 'UTC') WHERE id=${current.id}`;
return current;
});
await this.process(work.id);
return this.prisma.smsMessageRecord.findUnique({ where: { id: messageRecordId } });
}
async scan() {
if (this.running) return;
this.running = true;
try {
observeCompletion(
await this.prisma.$queryRaw<Array<{ state: string; count: number; age: number }>>`
SELECT w.state, COUNT(*)::int AS count,
EXTRACT(EPOCH FROM (CURRENT_TIMESTAMP AT TIME ZONE 'UTC') - MIN(COALESCE(e."createdAt", w."updatedAt")))::float AS age
FROM "SmsAttemptCompletionWork" w
LEFT JOIN LATERAL (SELECT MIN("createdAt") AS "createdAt" FROM "SmsCompletionEvent" WHERE "workId"=w.id AND "processedAt" IS NULL) e ON true
WHERE w.state IN ('pending','processing','retry_wait','needs_review') GROUP BY w.state`,
);
const rows = await this.prisma.$queryRaw<Array<{ id: string }>>`
SELECT id FROM "SmsAttemptCompletionWork"
WHERE (state IN ('pending','retry_wait') AND "nextAttemptAt" <= (CURRENT_TIMESTAMP AT TIME ZONE 'UTC'))
OR (state='processing' AND "leaseUntil" < (CURRENT_TIMESTAMP AT TIME ZONE 'UTC'))
ORDER BY "nextAttemptAt", id LIMIT 32`;
for (const row of rows) await this.process(row.id);
} catch {
this.logger.error('completion_scan_failed');
} finally {
this.running = false;
}
}
async process(id: string) {
const owner = randomUUID();
const claims = await this.prisma.$queryRaw<SmsAttemptCompletionWork[]>`
UPDATE "SmsAttemptCompletionWork" SET state='processing', "leaseOwner"=${owner},
"leaseUntil"=(CURRENT_TIMESTAMP AT TIME ZONE 'UTC')+INTERVAL '60 seconds', "fenceVersion"="fenceVersion"+1,
attempts=attempts+1, "updatedAt"=(CURRENT_TIMESTAMP AT TIME ZONE 'UTC')
WHERE id=${id} AND ((state IN ('pending','retry_wait') AND "nextAttemptAt" <= (CURRENT_TIMESTAMP AT TIME ZONE 'UTC'))
OR (state='processing' AND "leaseUntil" < (CURRENT_TIMESTAMP AT TIME ZONE 'UTC'))) RETURNING *`;
const claim = claims[0];
if (!claim) {
countCompletion('not_claimed');
return;
}
countCompletion('claimed');
if (claim.attempts > 1) countCompletion('recovered');
const renew = setInterval(() => {
void this.prisma
.$executeRaw`UPDATE "SmsAttemptCompletionWork" SET "leaseUntil"=(CURRENT_TIMESTAMP AT TIME ZONE 'UTC')+INTERVAL '60 seconds'
WHERE id=${id} AND state='processing' AND "leaseOwner"=${owner} AND "fenceVersion"=${claim.fenceVersion}
AND "leaseUntil">(CURRENT_TIMESTAMP AT TIME ZONE 'UTC')`.catch(() =>
this.logger.warn('completion_lease_renew_failed'),
);
}, 20_000);
renew.unref();
let route: RoutedChannel | undefined;
let routePlanned = false;
let planRevision: number | undefined;
try {
for (let pass = 0; pass < 64; pass++) {
try {
const done = await this.prisma.$transaction(
async (tx) => {
const rows = await tx.$queryRaw<SmsAttemptCompletionWork[]>`
SELECT * FROM "SmsAttemptCompletionWork" WHERE id=${id} AND state='processing'
AND "leaseOwner"=${owner} AND "fenceVersion"=${claim.fenceVersion} AND "leaseUntil">(CURRENT_TIMESTAMP AT TIME ZONE 'UTC') FOR UPDATE`;
const work = rows[0];
if (!work) {
countCompletion('fence_rejected');
throw new Error('completion_fence_rejected');
}
await tx.$queryRaw`SELECT id FROM "SmsMessageRecord" WHERE id=${work.messageRecordId} FOR UPDATE`;
if (planRevision !== work.revision) {
route = undefined;
routePlanned = false;
}
planRevision = work.revision;
const event = await tx.smsCompletionEvent.findFirst({
where: { workId: id, processedAt: null },
orderBy: [{ createdAt: 'asc' }, { id: 'asc' }],
});
if (event) {
await completionContext.run({ tx, messageRecordId: work.messageRecordId, route, routePlanned }, () =>
this.execute(event.kind as CompletionEventKind, event.payload),
);
await tx.smsCompletionEvent.update({ where: { id: event.id }, data: { processedAt: new Date() } });
}
const remaining = await tx.smsCompletionEvent.count({ where: { workId: id, processedAt: null } });
const message = await tx.smsMessageRecord.findUniqueOrThrow({ where: { id: work.messageRecordId } });
const retry = work.sourceSubmitRecordId
? await tx.smsSubmitRecord.findUnique({
where: { retryOfSubmitRecordId: work.sourceSubmitRecordId },
select: { id: true },
})
: null;
await tx.smsAttemptCompletionWork.update({
where: { id },
data: {
processedRevision: work.revision - remaining,
decision: message.status,
retrySubmitRecordId: retry?.id,
state: remaining ? 'processing' : 'idle',
lastError: null,
...(!remaining ? { leaseOwner: null, leaseUntil: null, attempts: 0 } : {}),
},
});
return remaining === 0;
},
{ timeout: 20_000, maxWait: 5_000 },
);
route = undefined;
routePlanned = false;
countCompletion('event_committed');
if (done) return;
} catch (error) {
if (!(error instanceof CompletionRouteRequired)) throw error;
// The probe transaction rolls back. Routing and rate limiting occur without locks.
try {
route = await error.select();
await this.waitForRoute(route);
} catch (selectionError) {
if (!(selectionError instanceof HttpException) || selectionError.getStatus() >= 500) throw selectionError;
route = undefined;
}
routePlanned = true;
}
}
throw new Error('completion_batch_budget_exhausted');
} catch (error) {
const code =
error instanceof Prisma.PrismaClientKnownRequestError
? error.code
: error instanceof Error && error.message.startsWith('completion_')
? error.message
: 'completion_processing_failed';
const exhausted = claim.attempts >= 12;
countCompletion(exhausted ? 'needs_review' : 'retry_wait');
await this.prisma.smsAttemptCompletionWork.updateMany({
where: { id, leaseOwner: owner, fenceVersion: claim.fenceVersion, state: 'processing' },
data: {
state: exhausted ? 'needs_review' : 'retry_wait',
leaseOwner: null,
leaseUntil: null,
lastError: code,
nextAttemptAt: new Date(Date.now() + Math.min(300_000, 1000 * 2 ** Math.min(claim.attempts, 8))),
},
});
this.logger.error(`${exhausted ? 'completion_needs_review' : 'completion_retry_wait'}:${code}`);
} finally {
clearInterval(renew);
}
}
}
+35
View File
@@ -0,0 +1,35 @@
import { AsyncLocalStorage } from 'node:async_hooks';
import { Prisma } from '@prisma/client';
import { PrismaService } from '../prisma/prisma.service';
import type { RoutedChannel } from './send-chain.contracts';
export type CompletionContext = {
tx: Prisma.TransactionClient;
messageRecordId: string;
route?: RoutedChannel;
routePlanned?: boolean;
};
export const completionContext = new AsyncLocalStorage<CompletionContext>();
// Only send-chain collaborators use this adapter. Nested billing/outbox transactions
// join the explicitly established completion transaction, never start a second one.
export function completionDatabase(prisma: PrismaService): PrismaService {
return new Proxy(prisma, {
get(target, key) {
const tx = completionContext.getStore()?.tx;
if (tx && key === '$transaction') {
return (operation: ((client: Prisma.TransactionClient) => unknown) | Promise<unknown>[]) =>
typeof operation === 'function' ? operation(tx) : Promise.all(operation);
}
const owner = tx && key in tx ? tx : target;
const value = Reflect.get(owner, key);
return typeof value === 'function' ? value.bind(owner) : value;
},
});
}
export class CompletionRouteRequired extends Error {
constructor(readonly select: () => Promise<RoutedChannel>) {
super('completion_route_required');
}
}
+26
View File
@@ -0,0 +1,26 @@
type CompletionMetric =
'claimed' | 'not_claimed' | 'recovered' | 'fence_rejected' | 'event_committed' | 'retry_wait' | 'needs_review';
const counts = new Map<CompletionMetric, number>();
let snapshot: Array<{ state: string; count: number; age: number }> = [];
const states = ['pending', 'processing', 'retry_wait', 'needs_review'];
export function countCompletion(event: CompletionMetric) {
counts.set(event, (counts.get(event) ?? 0) + 1);
}
export function observeCompletion(rows: typeof snapshot) {
snapshot = rows;
}
export function renderCompletionMetrics() {
return [
'# TYPE cmpp_completion_events_total counter',
...[...counts].map(([event, value]) => `cmpp_completion_events_total{event="${event}"} ${value}`),
'# TYPE cmpp_completion_work gauge',
'# TYPE cmpp_completion_oldest_seconds gauge',
...states.flatMap((state) => {
const row = snapshot.find((item) => item.state === state);
return [
`cmpp_completion_work{state="${state}"} ${Number(row?.count ?? 0)}`,
`cmpp_completion_oldest_seconds{state="${state}"} ${Math.max(0, Number(row?.age ?? 0))}`,
];
}),
];
}
@@ -0,0 +1,54 @@
import { createHash } from 'node:crypto';
import { DrainageSubmitGuardController } from './drainage-submit-guard.controller';
describe('channel test final guard', () => {
const content = '无签名正文 https://example.test 4001234567'.repeat(5);
function setup(overrides = {}) {
const tx = {
smsSubmitRecord: {
findUnique: jest.fn().mockResolvedValue({
channelId: 'channel',
submitId: 'submit',
resultProcessedAt: null,
messageRecord: { content, tenantId: null, batchTaskId: null, status: 'submit_queued', ...overrides },
}),
},
drainageDetectionRule: {
findMany: jest.fn(() => {
throw new Error('must not detect channel tests');
}),
},
};
const db = { $transaction: (operation: (value: unknown) => unknown) => operation(tx) };
return { controller: new DrainageSubmitGuardController(db as never), tx };
}
const local = { socket: { remoteAddress: '127.0.0.1' } };
const input = {
submitId: 'submit',
channelId: 'channel',
contentHash: createHash('sha256').update(content).digest('hex'),
};
it('allows unsigned long channel tests containing drainage information without detecting content', async () => {
const { controller, tx } = setup();
await expect(controller.authorize(local, input)).resolves.toEqual({ allowed: true });
expect(tx.drainageDetectionRule.findMany).not.toHaveBeenCalled();
});
it('still rejects a modified body or a finished submit', async () => {
await expect(setup().controller.authorize(local, { ...input, contentHash: 'a'.repeat(64) })).resolves.toMatchObject(
{ allowed: false },
);
await expect(setup({ status: 'delivered' }).controller.authorize(local, input)).resolves.toMatchObject({
allowed: false,
});
});
it('does not grant the exemption to a customer message without its required signature', async () => {
await expect(
setup({ tenantId: 'tenant', applicationId: 'app', batchTaskId: 'task' }).controller.authorize(local, input),
).resolves.toMatchObject({ allowed: false });
});
it('rejects external callers before looking up a submit', async () => {
const { controller, tx } = setup();
await expect(controller.authorize({ socket: { remoteAddress: '10.0.0.2' } }, input)).rejects.toThrow();
expect(tx.smsSubmitRecord.findUnique).not.toHaveBeenCalled();
});
});
@@ -58,19 +58,9 @@ export class DrainageSubmitGuardController {
)
return { allowed: false, code: 'DRN', reason: '提交或消息已终结,不得重复发送' };
if (!message.tenantId && !message.batchTaskId) {
try {
await evaluateMessageDrainage(
tx as unknown as PrismaService,
message,
message.carrier ?? undefined,
undefined,
true,
);
return { allowed: true };
} catch (error) {
if (!(error instanceof DrainageRejection)) throw error;
return { allowed: false, code: 'DRN', reason: error.message };
}
// Operations channel tests bypass signature/drainage policy only after
// verifying the durable submit, channel, content and unfinished state.
return { allowed: true };
}
if (!message.tenantId || !message.applicationId || !message.signatureId)
return { allowed: false, code: 'DRN', reason: '提交消息未关联企业应用和签名' };
+28 -19
View File
@@ -1,16 +1,10 @@
import { BadRequestException, HttpException, HttpStatus, Logger, NotFoundException } from '@nestjs/common';
import { Prisma } from '@prisma/client';
import { createHash } from 'node:crypto';
import { Logger } from '@nestjs/common';
import { BillingService } from '../billing/billing.service';
import { moneyToNumber } from '../common/money';
import type { OpenApiService } from '../open-api/open-api.service';
import { PrismaService } from '../prisma/prisma.service';
import type { GatewaySubmitResultDto, GatewaySubmitSegmentResultDto, GatewayReceiptEventDto, GatewayUplinkEventDto, GatewayPendingDeliveryQueryDto, GatewayDownstreamSentDto, GatewayDownstreamAcknowledgedDto, GatewayDownstreamFailureType, GatewayControlDeliveryResult, GatewaySubmitDeadLetterDto, RequeueGatewaySubmitExceptionDto, GatewayDownstreamRecoveryStatusDto, TimeoutUnknownDto } from './send-chain.contracts';
import { positiveInteger, normalizeReceiptStatus, normalizeCarrier, DEFAULT_DOWNSTREAM_RETRY_DELAY_MS, DEFAULT_DOWNSTREAM_RETRY_MAX_DELAY_MS, DEFAULT_DOWNSTREAM_MAX_RETRIES, DEFAULT_DOWNSTREAM_PENDING_TIMEOUT_HOURS, DEFAULT_RECEIPT_TIMEOUT_HOURS, DEFAULT_GATEWAY_SUBMIT_REQUEUE_STALE_MS, DEFAULT_DOWNSTREAM_MANUAL_REQUEUE_STALE_MS, DEFAULT_UPSTREAM_RECEIPT_INBOX_PROCESSING_STALE_MS, DEFAULT_UPSTREAM_RECEIPT_INBOX_MAX_ATTEMPTS, DEFAULT_UPSTREAM_RECEIPT_INBOX_MAX_AGE_HOURS, gatewaySubmitRequeueKey, isObjectRecord, asDateOrNull, downstreamRetryDelayMs, downstreamAckTimeoutMs, downstreamRetryBaseDelayMs, downstreamRetryMaxDelayMs, downstreamMaxRetries, downstreamPendingTimeoutHours, downstreamControlFailureMessage, normalizeSubmitStatus, downstreamDeliveryAttemptKey, hasRecoveryAuditStateChanged, normalizeRecoveryFailureCategory, aggregateReceiptSegmentState, isSameUpstreamEndpointIdentity, receiptEventKey } from './send-chain.helpers';
import type { SendSubmissionService } from './send-submission.service';
import type { SendCompletionCallbacks, SendCompletionFacade } from './send-completion.service';
/**
* R10 accounting implementation.
* Cross-domain calls return through the stable SendChainService seam to preserve locking and test observability.
@@ -41,16 +35,19 @@ export class SendAccountingService {
const unitPrice = moneyToNumber(message.unitPrice);
const billingUnits = message.billingUnits ?? 0;
const exists = await this.prisma.smsBillingRecord.findFirst({ where: { messageId: message.messageId } });
if (exists?.billingStatus === 'charged') {
if (exists && ['charged', 'refunded'].includes(exists.billingStatus)) {
return;
}
const transaction = amountCents > 0 ? await this.billing.settleFrozenCharge({
tenantId: message.tenantId,
amountCents,
taskId: message.batchTaskId,
messageId: message.messageId,
remark: `短信 ${message.messageId} 提交成功释放冻结并转扣费`,
}) : null;
const transaction =
amountCents > 0
? await this.billing.settleFrozenCharge({
tenantId: message.tenantId,
amountCents,
taskId: message.batchTaskId,
messageId: message.messageId,
remark: `短信 ${message.messageId} 提交成功释放冻结并转扣费`,
})
: null;
const data = {
tenantId: message.tenantId,
applicationId: message.applicationId ?? undefined,
@@ -72,14 +69,22 @@ export class SendAccountingService {
}
async releaseMessageReservation(
message: { tenantId: string; batchTaskId: string; messageId: string; amountCents: number | bigint; billingUnits: number },
message: {
tenantId: string;
batchTaskId: string;
messageId: string;
amountCents: number | bigint;
billingUnits: number;
},
remark: string,
) {
const amountCents = moneyToNumber(message.amountCents);
if (amountCents <= 0) {
return;
}
const charged = await this.prisma.smsBillingRecord.findFirst({ where: { messageId: message.messageId, billingStatus: 'charged' } });
const charged = await this.prisma.smsBillingRecord.findFirst({
where: { messageId: message.messageId, billingStatus: 'charged' },
});
if (charged) {
return;
}
@@ -107,11 +112,15 @@ export class SendAccountingService {
if (amountCents <= 0) {
return;
}
const refunded = await this.prisma.smsBillingRecord.findFirst({ where: { messageId: message.messageId, billingStatus: 'refunded' } });
const refunded = await this.prisma.smsBillingRecord.findFirst({
where: { messageId: message.messageId, billingStatus: 'refunded' },
});
if (refunded) {
return;
}
const charged = await this.prisma.smsBillingRecord.findFirst({ where: { messageId: message.messageId, billingStatus: 'charged' } });
const charged = await this.prisma.smsBillingRecord.findFirst({
where: { messageId: message.messageId, billingStatus: 'charged' },
});
if (!charged) {
return;
}
@@ -1694,6 +1694,7 @@ describe('SendChainService', () => {
applicationId: 'app-1',
eventType: 'receipt',
}),
undefined,
);
expect(prisma.cmppDownstreamDelivery.create).not.toHaveBeenCalled();
});
@@ -3257,7 +3258,7 @@ describe('SendChainService', () => {
data: expect.objectContaining({ sequenceId: 7, gatewayMessageId: 'GW-1', submitStatus: 'accepted' }),
});
expect(prisma.smsMessageRecord.updateMany).toHaveBeenCalledWith({
where: { id: 'record-1', status: { notIn: ['delivered', 'failed', 'unknown'] } },
where: { id: 'record-1', status: { notIn: ['delivered', 'failed', 'unknown', 'timeout'] } },
data: expect.objectContaining({ gatewayMessageId: 'GW-1', status: 'submitted', submitStatus: 'accepted' }),
});
expect(billing.settleFrozenCharge).toHaveBeenCalledWith(
@@ -4891,7 +4892,7 @@ describe('SendChainService', () => {
expect(prisma.smsMessageRecord.updateMany).toHaveBeenNthCalledWith(
1,
expect.objectContaining({
where: { id: 'record-1', status: { notIn: ['delivered', 'failed', 'unknown'] } },
where: { id: 'record-1', status: { notIn: ['delivered', 'failed', 'unknown', 'timeout'] } },
data: expect.objectContaining({ status: 'submitted' }),
}),
);
@@ -5567,6 +5568,7 @@ describe('SendChainService', () => {
errorCode: 'RECEIPT_TIMEOUT',
}),
}),
undefined,
);
expect(prisma.cmppDownstreamDelivery.create).not.toHaveBeenCalled();
expect(prisma.smsMessageRecord.updateMany).toHaveBeenCalledWith({
+100 -1
View File
@@ -1,3 +1,5 @@
import { AttemptCompletion } from './attempt-completion';
import { completionContext, completionDatabase, CompletionRouteRequired } from './completion-context';
import {
BadRequestException,
forwardRef,
@@ -84,6 +86,7 @@ export class SendChainService implements OnModuleInit, OnModuleDestroy {
private readonly submission: SendSubmissionService;
private readonly completion: SendCompletionService;
private readonly downstreamRequeueTasks: SendDownstreamRequeueTaskService;
private readonly attemptCompletion?: AttemptCompletion;
constructor(
private readonly prisma: PrismaService,
@@ -94,6 +97,11 @@ export class SendChainService implements OnModuleInit, OnModuleDestroy {
@Optional() phoneRouting?: PhoneRoutingLookupService,
@Optional() metrics?: MetricsService,
) {
const rootPrisma = prisma;
prisma = completionDatabase(prisma);
this.prisma = prisma;
// Structural unit-test doubles may omit the durable delegate; real Prisma always has it.
const coordinatedBilling = rootPrisma.smsAttemptCompletionWork ? new BillingService(prisma) : billing;
const resolvedPhoneRouting = phoneRouting ?? new PhoneRoutingLookupService(prisma);
this.submission = new SendSubmissionService(
prisma,
@@ -109,12 +117,46 @@ export class SendChainService implements OnModuleInit, OnModuleDestroy {
},
metrics,
);
this.completion = new SendCompletionService(prisma, billing, openApi, this as unknown as SendCompletionFacade);
this.completion = new SendCompletionService(
prisma,
coordinatedBilling,
openApi,
this as unknown as SendCompletionFacade,
);
if (rootPrisma.smsAttemptCompletionWork) {
this.attemptCompletion = new AttemptCompletion(
rootPrisma,
async (kind, payload) => {
const event = payload as unknown as {
data: GatewayReceiptEventDto & GatewaySubmitResultDto & GatewaySubmitSegmentResultDto;
incomingIdentity?: Parameters<SendChainService['handleReceipt']>[1];
olderThanHours?: number;
errorCode?: string;
reason?: string;
};
if (kind === 'receipt') return this.completion.handleReceipt(event.data, event.incomingIdentity);
if (kind === 'submit') return this.completion.handleSubmitResult(event.data);
if (kind === 'segment') return this.completion.handleSubmitSegmentResult(event.data);
if (kind === 'timeout') return this.completion.markUnknownTimeout({ olderThanHours: event.olderThanHours });
const message = await prisma.smsMessageRecord.findUniqueOrThrow({
where: { id: completionContext.getStore()!.messageRecordId },
});
if (
message.status === 'delivered' ||
(message.status === 'timeout' && message.errorCode === 'RECEIPT_TIMEOUT')
)
return message;
return this.completion.recordCmppFailureReceipt(message, event.errorCode!, event.reason!);
},
(route) => this.submission.waitForChannelRateLimit(route.channel.id, route.channel.rateLimitPerSecond),
);
}
this.downstreamRequeueTasks = new SendDownstreamRequeueTaskService(prisma, this);
}
onModuleInit() {
const processRole = process.env.CMPP_PROCESS_ROLE?.trim() || 'all';
if (['all', 'worker', 'callback'].includes(processRole)) this.attemptCompletion?.start();
if (processRole === 'api' || processRole === 'callback') return;
if (processRole === 'outbox') {
this.submission.startSubmitOutboxPublisher();
@@ -202,6 +244,7 @@ export class SendChainService implements OnModuleInit, OnModuleDestroy {
}
async onModuleDestroy() {
this.attemptCompletion?.stop();
if (this.receiptTimeoutInitialTimer) clearTimeout(this.receiptTimeoutInitialTimer);
if (this.receiptTimeoutIntervalTimer) clearInterval(this.receiptTimeoutIntervalTimer);
if (this.scheduledDispatchInitialTimer) clearTimeout(this.scheduledDispatchInitialTimer);
@@ -484,6 +527,11 @@ export class SendChainService implements OnModuleInit, OnModuleDestroy {
}
async handleSubmitSegmentResult(data: GatewaySubmitSegmentResultDto) {
if (this.attemptCompletion && !completionContext.getStore()) {
const message = await this.completion.requireMessageByGatewayEvent(data.messageId, data.gatewayMessageId);
const source = await this.completion.resolveSubmitRecordForGatewaySegmentResult(message.id, data);
return this.attemptCompletion.enqueue(message.id, source.id, 'segment', { data });
}
return this.completion.handleSubmitSegmentResult(data);
}
@@ -495,6 +543,11 @@ export class SendChainService implements OnModuleInit, OnModuleDestroy {
}
async handleSubmitResult(data: GatewaySubmitResultDto) {
if (this.attemptCompletion && !completionContext.getStore()) {
const message = await this.completion.requireMessageByGatewayEvent(data.messageId, data.gatewayMessageId);
const source = await this.completion.resolveSubmitRecordForGatewayResult(message.id, data);
return this.attemptCompletion.enqueue(message.id, source.id, 'submit', { data }, data.eventId);
}
return this.completion.handleSubmitResult(data);
}
@@ -528,6 +581,14 @@ export class SendChainService implements OnModuleInit, OnModuleDestroy {
cmppVersion: string;
},
) {
if (this.attemptCompletion && !completionContext.getStore()) {
const resolved = await this.completion.resolveReceiptMessage(data, incomingIdentity);
if (!resolved.submitRecordId) throw new NotFoundException('回执缺少可确认的提交尝试关联');
return this.attemptCompletion.enqueue(resolved.message.id, resolved.submitRecordId, 'receipt', {
data,
incomingIdentity,
});
}
return this.completion.handleReceipt(data, incomingIdentity);
}
@@ -726,6 +787,32 @@ export class SendChainService implements OnModuleInit, OnModuleDestroy {
}
async markUnknownTimeout(data: TimeoutUnknownDto) {
if (this.attemptCompletion && !completionContext.getStore()) {
const olderThanHours = data.olderThanHours ?? positiveInteger(process.env.SMS_RECEIPT_TIMEOUT_HOURS, 72);
const candidates = await this.prisma.smsMessageRecord.findMany({
where: {
tenantId: { not: null },
OR: [
{
status: { in: ['submitted', 'unknown'] },
submittedAt: { lte: new Date(Date.now() - olderThanHours * 3600_000) },
},
{ status: 'timeout', errorCode: 'RECEIPT_TIMEOUT', timeoutReceiptQueuedAt: null },
],
},
select: { id: true, submitId: true, status: true },
take: 100,
});
let timeout = 0;
for (const message of candidates) {
const source = message.submitId
? await this.prisma.smsSubmitRecord.findUnique({ where: { submitId: message.submitId } })
: null;
const result = await this.attemptCompletion.enqueue(message.id, source?.id, 'timeout', { olderThanHours });
if (message.status !== 'timeout' && result?.status === 'timeout') timeout++;
}
return { timeout };
}
return this.completion.markUnknownTimeout(data);
}
@@ -803,6 +890,14 @@ export class SendChainService implements OnModuleInit, OnModuleDestroy {
},
options: { forceNational?: boolean; excludeChannelIds?: string[] } = {},
): Promise<RoutedChannel> {
const context = completionContext.getStore();
if (context) {
const prepare = () => this.submission.selectChannelForMessage(message, { ...options, previewOnly: true });
if (!context.routePlanned) throw new CompletionRouteRequired(prepare);
const current = await this.submission.selectChannelForMessage(message, options);
if (!context.route || current.channel.id !== context.route.channel.id) throw new CompletionRouteRequired(prepare);
return current;
}
return this.submission.selectChannelForMessage(message, options);
}
@@ -923,6 +1018,9 @@ export class SendChainService implements OnModuleInit, OnModuleDestroy {
errorCode: string,
reason: string,
) {
if (this.attemptCompletion && !completionContext.getStore()) {
return this.attemptCompletion.enqueue(message.id, undefined, 'rejection', { errorCode, reason });
}
return this.completion.recordCmppFailureReceipt(message, errorCode, reason);
}
@@ -1004,6 +1102,7 @@ export class SendChainService implements OnModuleInit, OnModuleDestroy {
}
private async waitForChannelRateLimit(channelId: string, tps: number) {
if (completionContext.getStore()) return;
return this.submission.waitForChannelRateLimit(channelId, tps);
}
@@ -1,3 +1,4 @@
import { completionContext } from './completion-context';
import { BadRequestException, Logger, NotFoundException } from '@nestjs/common';
import { Prisma } from '@prisma/client';
import { createHash, randomUUID } from 'node:crypto';
@@ -217,20 +218,24 @@ export class SendDownstreamDeliveryService {
const cmppDeliveryAllowed = deliveryAllowed || data.allowBusinessRejectionCmppDelivery === true;
if (deliveryAllowed && data.queueHttpWebhook !== false) {
try {
await this.openApi?.queueWebhookEvent({
tenantId: data.tenantId,
applicationId: data.applicationId,
messageRecordId: data.messageRecordId,
messageId: data.messageId,
uplinkMessageId: typeof data.payload.uplinkMessageId === 'string' ? data.payload.uplinkMessageId : undefined,
eventType: data.deliveryType,
payload: data.payload,
});
await this.openApi?.queueWebhookEvent(
{
tenantId: data.tenantId,
applicationId: data.applicationId,
messageRecordId: data.messageRecordId,
messageId: data.messageId,
uplinkMessageId:
typeof data.payload.uplinkMessageId === 'string' ? data.payload.uplinkMessageId : undefined,
eventType: data.deliveryType,
payload: data.payload,
},
completionContext.getStore()?.tx,
);
} catch (error) {
this.logger.error(
`HTTP webhook queue failed for ${data.deliveryType}/${data.messageId ?? '-'}: ${error instanceof Error ? error.message : String(error)}`,
);
if (data.propagateHttpQueueError) throw error;
if (data.propagateHttpQueueError || completionContext.getStore()) throw error;
}
}
if (data.queueCmppDelivery === false) {
@@ -249,6 +254,34 @@ export class SendDownstreamDeliveryService {
: data.deliveryType === 'uplink' && typeof data.payload.uplinkMessageId === 'string'
? `uplink:${data.payload.uplinkMessageId}`
: null;
if (completionContext.getStore()) {
if (!dedupeKey) throw new Error('completion_notification_identity_missing');
await this.prisma.cmppDownstreamDelivery.createMany({
data: [
{
tenantId: data.tenantId,
applicationId: data.applicationId,
messageRecordId: data.messageRecordId,
messageId: data.messageId,
dedupeKey,
deliveryType: data.deliveryType,
payload,
retryEnabled:
cmppDeliveryAllowed &&
(data.deliveryType === 'uplink'
? application.downstreamUplinkRetryEnabled
: application.downstreamReceiptRetryEnabled),
status: cmppDeliveryAllowed ? 'pending' : 'abandoned',
lastError: cmppDeliveryAllowed ? null : '企业应用已停用,保留回执但不再向客户应用推送',
},
],
skipDuplicates: true,
});
const retained = await this.prisma.cmppDownstreamDelivery.findUniqueOrThrow({ where: { dedupeKey } });
if (retained.messageRecordId !== data.messageRecordId || retained.applicationId !== data.applicationId)
throw new Error('completion_notification_identity_mismatch');
return retained;
}
let delivery;
try {
delivery = await this.prisma.cmppDownstreamDelivery.create({
@@ -188,11 +188,18 @@ export class SendGatewayResultService {
}
const status =
data.submitStatus === 'accepted' ? 'submitted' : data.submitStatus === 'timeout' ? 'timeout' : 'submit_failed';
if (
data.submitStatus !== 'accepted' &&
(message.status === 'delivered' || (message.status === 'timeout' && message.errorCode === 'RECEIPT_TIMEOUT'))
) {
await this.markSubmitResultProcessed(submitRecord.id, data.eventId, submittedAt);
return message;
}
if (data.submitStatus === 'accepted' && !isStandaloneChannelTest && message.tenantId && message.batchTaskId) {
const businessMessage = message as typeof message & { tenantId: string; batchTaskId: string };
await this.facade.chargeAcceptedMessage(businessMessage);
const latest = await this.prisma.smsMessageRecord.findUnique({ where: { id: message.id } });
if (latest?.status === 'failed') {
if (latest?.status === 'failed' || (latest?.status === 'timeout' && latest.errorCode === 'RECEIPT_TIMEOUT')) {
await this.facade.refundMessage(businessMessage, '先到失败回执补偿退款');
}
} else if (
@@ -219,7 +226,7 @@ export class SendGatewayResultService {
data.submitStatus === 'timeout' ? '提交超时释放冻结' : '提交失败释放冻结',
);
}
const protectedTerminalStatuses = ['delivered', 'failed', 'unknown'];
const protectedTerminalStatuses = ['delivered', 'failed', 'unknown', 'timeout'];
const updated = await this.prisma.smsMessageRecord.updateMany({
where:
data.submitStatus === 'accepted'
@@ -1,3 +1,4 @@
import { completionContext } from './completion-context';
import { BadRequestException, Logger, NotFoundException } from '@nestjs/common';
import { Prisma } from '@prisma/client';
import { Queue, Worker } from 'bullmq';
@@ -310,7 +311,7 @@ export class SendGatewaySubmitService {
sessionId: sessionByChannel.get(routed.channel.id),
};
});
const writeOutbox = this.submitOutboxEnabled();
const writeOutbox = Boolean(completionContext.getStore()) || this.submitOutboxEnabled();
await this.measureSendStage('submit_transaction', () =>
this.prisma.$transaction(async (tx) => {
await tx.smsSubmitRecord.createMany({
@@ -365,7 +366,7 @@ export class SendGatewaySubmitService {
}
}),
);
if (!this.submitOutboxPublishEnabled()) {
if (!completionContext.getStore() && !this.submitOutboxPublishEnabled()) {
await Promise.all(prepared.map(({ command }) => this.facade.publishGatewaySubmitCommand(command)));
}
await this.refreshTaskProgressBatch(prepared.map(({ message }) => message));
@@ -762,7 +763,7 @@ export class SendGatewaySubmitService {
const submitId = `SUB-${randomUUID()}`;
const sessionId = await this.getOpenSubmitSessionId(channel.id);
const command = this.buildGatewaySubmitCommand(message, routed, attempt, submitId, upstreamSrcId);
const writeOutbox = this.submitOutboxEnabled();
const writeOutbox = Boolean(completionContext.getStore()) || this.submitOutboxEnabled();
try {
await this.measureSendStage('submit_transaction', () =>
this.prisma.$transaction(async (tx) => {
@@ -849,7 +850,7 @@ export class SendGatewaySubmitService {
}
throw error;
}
if (!this.submitOutboxPublishEnabled()) {
if (!completionContext.getStore() && !this.submitOutboxPublishEnabled()) {
await this.measureSendStage('gateway_stream_publish', () => this.facade.publishGatewaySubmitCommand(command));
}
await this.measureSendStage('task_progress', () =>
@@ -1087,7 +1088,7 @@ return streamId`;
template?: { signature?: { id?: string | null } | null } | null;
signature?: { id?: string | null } | null;
},
options: { forceNational?: boolean; excludeChannelIds?: string[] } = {},
options: { forceNational?: boolean; excludeChannelIds?: string[]; previewOnly?: boolean } = {},
): Promise<RoutedChannel> {
if (!message.applicationId) {
throw new BadRequestException('短信应用未配置,无法选择通道组');
@@ -1100,7 +1101,7 @@ return streamId`;
this.facade.identifyCarrier(message.phoneNumber),
this.facade.identifyProvince(message.phoneNumber),
]);
if (!hasPersistedRouting) {
if (!hasPersistedRouting && !options.previewOnly) {
await this.prisma.smsMessageRecord.update({
where: { id: message.id },
data: { carrier: resolved[0], province: resolved[1] },
@@ -1137,7 +1138,7 @@ return streamId`;
approvedChannelIds,
routingKey: message.id,
});
await channelWords.persist(this.prisma);
if (!options.previewOnly) await channelWords.persist(this.prisma);
if (rejected) throw new ChannelWordRejection();
if (!selected) {
throw new NotFoundException('无已报备通过且在线的可用通道');
@@ -1352,6 +1353,10 @@ return streamId`;
`);
if (direct === 1) return;
}
if (completionContext.getStore()) {
await this.prisma.$queryRaw`SELECT id FROM "SmsBatchTask" WHERE id=${batchTaskId} FOR UPDATE`;
return this.refreshTaskProgressUntilClean(batchTaskId, true);
}
const running = this.taskProgressRefreshes.get(batchTaskId);
if (running) {
// A state transition committed after the running aggregate may not be visible
@@ -1372,9 +1377,9 @@ return streamId`;
}
}
private async refreshTaskProgressUntilClean(batchTaskId: string) {
private async refreshTaskProgressUntilClean(batchTaskId: string, transactional = false) {
do {
this.dirtyTaskProgressRefreshes.delete(batchTaskId);
if (!transactional) this.dirtyTaskProgressRefreshes.delete(batchTaskId);
const groups = await this.prisma.smsMessageRecord.groupBy({
by: ['status'],
where: { batchTaskId },
@@ -1401,7 +1406,7 @@ return streamId`;
where: { id: batchTaskId },
data: { progressTotal, submittedTotal, successTotal, failedTotal, unknownTotal, timeoutTotal, status },
});
} while (this.dirtyTaskProgressRefreshes.has(batchTaskId));
} while (!transactional && this.dirtyTaskProgressRefreshes.has(batchTaskId));
}
private async recoverNightReviews() {
@@ -1464,6 +1469,17 @@ return streamId`,
}
private getOpenSubmitSessionId(channelId: string) {
if (completionContext.getStore()) {
const sessionNo = `OPEN-${channelId}`;
return this.prisma.cmppSubmitSession
.upsert({
where: { sessionNo },
update: {},
create: { channelId, sessionNo, submitTotal: 0 },
select: { id: true },
})
.then((session) => session.id);
}
const cached = this.openSubmitSessionIds.get(channelId);
if (cached) return cached;
const sessionNo = `OPEN-${channelId}`;
+36 -30
View File
@@ -1,3 +1,4 @@
import { completionContext } from './completion-context';
import { Logger, NotFoundException } from '@nestjs/common';
import { Prisma } from '@prisma/client';
@@ -229,10 +230,12 @@ export class SendReceiptService {
where: { receiptKey },
include: { messageRecord: true },
});
if (existingReceipt?.messageRecord) {
if (existingReceipt?.messageRecord && !completionContext.getStore()) {
return existingReceipt.messageRecord;
}
const message = resolved.message;
if (existingReceipt && existingReceipt.messageRecordId !== message.id)
throw new Error('completion_receipt_owner_mismatch');
const deliveredAt = data.deliveredAt ? new Date(data.deliveredAt) : new Date();
if (resolved.submitRecordId) {
await this.prisma.smsSubmitRecord.updateMany({
@@ -246,37 +249,38 @@ export class SendReceiptService {
},
});
}
let receiptRecordId: string | undefined;
try {
const createdReceipt = await this.prisma.smsReceiptRecord.create({
data: {
tenantId: message.tenantId,
batchTaskId: message.batchTaskId,
messageRecordId: message.id,
receiptKey,
channelId: logicalChannelId,
messageId: resolved.messageId,
gatewayMessageId: data.gatewayMessageId,
phoneNumber: data.phoneNumber?.trim() || message.phoneNumber,
sequenceId: data.sequenceId,
receiptStatus: data.receiptStatus,
rawStatus: data.rawStatus,
errorCode: data.errorCode,
errorMessage: data.errorMessage,
deliveredAt,
},
});
receiptRecordId = createdReceipt.id;
} catch (error) {
if (error instanceof Prisma.PrismaClientKnownRequestError && error.code === 'P2002') {
const duplicate = await this.prisma.smsReceiptRecord.findUnique({
where: { receiptKey },
include: { messageRecord: true },
let receiptRecordId: string | undefined = existingReceipt?.id;
if (!existingReceipt)
try {
const createdReceipt = await this.prisma.smsReceiptRecord.create({
data: {
tenantId: message.tenantId,
batchTaskId: message.batchTaskId,
messageRecordId: message.id,
receiptKey,
channelId: logicalChannelId,
messageId: resolved.messageId,
gatewayMessageId: data.gatewayMessageId,
phoneNumber: data.phoneNumber?.trim() || message.phoneNumber,
sequenceId: data.sequenceId,
receiptStatus: data.receiptStatus,
rawStatus: data.rawStatus,
errorCode: data.errorCode,
errorMessage: data.errorMessage,
deliveredAt,
},
});
if (duplicate?.messageRecord) return duplicate.messageRecord;
receiptRecordId = createdReceipt.id;
} catch (error) {
if (error instanceof Prisma.PrismaClientKnownRequestError && error.code === 'P2002') {
const duplicate = await this.prisma.smsReceiptRecord.findUnique({
where: { receiptKey },
include: { messageRecord: true },
});
if (duplicate?.messageRecord) return duplicate.messageRecord;
}
throw error;
}
throw error;
}
const logicalReceipt = { ...data, channelId: logicalChannelId };
await this.facade.recordReceiptSegment(message, logicalReceipt, deliveredAt, resolved.submitRecordId);
const receiptMode =
@@ -300,8 +304,10 @@ export class SendReceiptService {
if (!aggregate.terminal) {
return this.prisma.smsMessageRecord.findUnique({ where: { id: message.id } });
}
if (message.status === 'timeout' && message.errorCode === 'RECEIPT_TIMEOUT') return message;
const status = aggregate.status;
const isCurrentAttempt =
(!message.submitId || message.submitId === resolved.submitId) &&
(!message.channelId || message.channelId === logicalChannelId) &&
(!message.gatewayMessageId ||
message.gatewayMessageId === data.gatewayMessageId ||
+116 -82
View File
@@ -1,16 +1,18 @@
import { BadRequestException, HttpException, HttpStatus, Logger, NotFoundException } from '@nestjs/common';
import { BadRequestException, HttpException, Logger, NotFoundException } from '@nestjs/common';
import { Prisma } from '@prisma/client';
import { createHash } from 'node:crypto';
import { BillingService } from '../billing/billing.service';
import { moneyToNumber } from '../common/money';
import type { OpenApiService } from '../open-api/open-api.service';
import { PrismaService } from '../prisma/prisma.service';
import type { GatewaySubmitResultDto, GatewaySubmitSegmentResultDto, GatewayReceiptEventDto, GatewayUplinkEventDto, GatewayPendingDeliveryQueryDto, GatewayDownstreamSentDto, GatewayDownstreamAcknowledgedDto, GatewayDownstreamFailureType, GatewayControlDeliveryResult, GatewaySubmitDeadLetterDto, RequeueGatewaySubmitExceptionDto, GatewayDownstreamRecoveryStatusDto, TimeoutUnknownDto } from './send-chain.contracts';
import { positiveInteger, normalizeReceiptStatus, normalizeCarrier, DEFAULT_DOWNSTREAM_RETRY_DELAY_MS, DEFAULT_DOWNSTREAM_RETRY_MAX_DELAY_MS, DEFAULT_DOWNSTREAM_MAX_RETRIES, DEFAULT_DOWNSTREAM_PENDING_TIMEOUT_HOURS, DEFAULT_RECEIPT_TIMEOUT_HOURS, DEFAULT_GATEWAY_SUBMIT_REQUEUE_STALE_MS, DEFAULT_DOWNSTREAM_MANUAL_REQUEUE_STALE_MS, DEFAULT_UPSTREAM_RECEIPT_INBOX_PROCESSING_STALE_MS, DEFAULT_UPSTREAM_RECEIPT_INBOX_MAX_ATTEMPTS, DEFAULT_UPSTREAM_RECEIPT_INBOX_MAX_AGE_HOURS, gatewaySubmitRequeueKey, isObjectRecord, asDateOrNull, downstreamRetryDelayMs, downstreamAckTimeoutMs, downstreamRetryBaseDelayMs, downstreamRetryMaxDelayMs, downstreamMaxRetries, downstreamPendingTimeoutHours, downstreamControlFailureMessage, normalizeSubmitStatus, downstreamDeliveryAttemptKey, hasRecoveryAuditStateChanged, normalizeRecoveryFailureCategory, aggregateReceiptSegmentState, isSameUpstreamEndpointIdentity, receiptEventKey } from './send-chain.helpers';
import type { SendSubmissionService } from './send-submission.service';
import type { GatewaySubmitDeadLetterDto, RequeueGatewaySubmitExceptionDto } from './send-chain.contracts';
import {
DEFAULT_GATEWAY_SUBMIT_REQUEUE_STALE_MS,
gatewaySubmitRequeueKey,
isObjectRecord,
normalizeCarrier,
positiveInteger,
} from './send-chain.helpers';
import type { SendCompletionCallbacks, SendCompletionFacade } from './send-completion.service';
/**
* R10 retry implementation.
* Cross-domain calls return through the stable SendChainService seam to preserve locking and test observability.
@@ -87,11 +89,12 @@ export class SendRetryService {
const message = deadLetter.messageId
? await this.prisma.smsMessageRecord.findUnique({ where: { messageId: deadLetter.messageId } })
: null;
if (message && (
message.submitStatus === 'accepted'
|| ['submitted', 'delivered', 'unknown'].includes(message.status)
|| ['delivered', 'unknown'].includes(message.receiptStatus ?? '')
)) {
if (
message &&
(message.submitStatus === 'accepted' ||
['submitted', 'delivered', 'unknown'].includes(message.status) ||
['delivered', 'unknown'].includes(message.receiptStatus ?? ''))
) {
throw new BadRequestException('该短信已有成功或不确定的上游结果,为避免重复发送,禁止重新入队');
}
const commandChannelId = String(deadLetter.commandPayload.channelId ?? deadLetter.channelId ?? '').trim();
@@ -118,7 +121,10 @@ export class SendRetryService {
const requeueKey = gatewaySubmitRequeueKey(deadLetter.id, deadLetter.manualRetryCount + 1);
let retryStreamMessageId: string;
try {
const publishedStreamMessageId = await this.facade.publishGatewaySubmitCommand(deadLetter.commandPayload, requeueKey);
const publishedStreamMessageId = await this.facade.publishGatewaySubmitCommand(
deadLetter.commandPayload,
requeueKey,
);
if (!publishedStreamMessageId) {
throw new Error('Gateway提交异常重新入队未返回Stream消息编号');
}
@@ -208,10 +214,10 @@ export class SendRetryService {
}
async recoverStaleGatewaySubmitRequeues(now = new Date()) {
const staleCutoff = new Date(now.getTime() - positiveInteger(
process.env.GATEWAY_SUBMIT_REQUEUE_STALE_MS,
DEFAULT_GATEWAY_SUBMIT_REQUEUE_STALE_MS,
));
const staleCutoff = new Date(
now.getTime() -
positiveInteger(process.env.GATEWAY_SUBMIT_REQUEUE_STALE_MS, DEFAULT_GATEWAY_SUBMIT_REQUEUE_STALE_MS),
);
const stale = await this.prisma.gatewaySubmitDeadLetter.findMany({
where: { status: 'requeueing', updatedAt: { lt: staleCutoff } },
orderBy: { updatedAt: 'asc' },
@@ -235,7 +241,10 @@ export class SendRetryService {
if (claimed.count !== 1) continue;
try {
const requeueKey = gatewaySubmitRequeueKey(deadLetter.id, deadLetter.manualRetryCount + 1);
const retryStreamMessageId = await this.facade.publishGatewaySubmitCommand(deadLetter.commandPayload, requeueKey);
const retryStreamMessageId = await this.facade.publishGatewaySubmitCommand(
deadLetter.commandPayload,
requeueKey,
);
if (!retryStreamMessageId) throw new Error('Gateway提交异常恢复未返回Stream消息编号');
const finalized = await this.prisma.gatewaySubmitDeadLetter.updateMany({
where: { id: deadLetter.id, status: 'requeue_recovering' },
@@ -264,7 +273,9 @@ export class SendRetryService {
where: { id: deadLetter.id, status: 'requeue_recovering' },
data: { status: 'requeueing' },
});
this.logger.error(`Gateway submit requeue recovery failed for ${deadLetter.id}: ${error instanceof Error ? error.message : String(error)}`);
this.logger.error(
`Gateway submit requeue recovery failed for ${deadLetter.id}: ${error instanceof Error ? error.message : String(error)}`,
);
}
}
return { recovered, failed };
@@ -300,100 +311,123 @@ export class SendRetryService {
const attemptedChannelIds = attempts.map((attempt) => attempt.channelId);
let sourceAttempt = sourceSubmitRecordId
? attempts.find((attempt) => attempt.id === sourceSubmitRecordId)
: attempts.find((attempt) => attempt.submitId === message.submitId) ?? attempts[attempts.length - 1];
: (attempts.find((attempt) => attempt.submitId === message.submitId) ?? attempts[attempts.length - 1]);
if (!sourceAttempt && sourceSubmitRecordId) {
sourceAttempt = await this.prisma.smsSubmitRecord.findUnique({
where: { id: sourceSubmitRecordId },
}) ?? undefined;
sourceAttempt =
(await this.prisma.smsSubmitRecord.findUnique({
where: { id: sourceSubmitRecordId },
})) ?? undefined;
}
if (!sourceAttempt || sourceAttempt.messageRecordId !== message.id) {
this.logger.error(`sms_retry_route_failed ${JSON.stringify({
messageId: message.messageId,
messageRecordId: message.id,
reason,
sourceSubmitRecordId,
sourceMessageRecordId: sourceAttempt?.messageRecordId,
error: sourceAttempt ? 'retry_source_submit_record_mismatch' : 'retry_source_submit_record_missing',
})}`);
this.logger.error(
`sms_retry_route_failed ${JSON.stringify({
messageId: message.messageId,
messageRecordId: message.id,
reason,
sourceSubmitRecordId,
sourceMessageRecordId: sourceAttempt?.messageRecordId,
error: sourceAttempt ? 'retry_source_submit_record_mismatch' : 'retry_source_submit_record_missing',
})}`,
);
return null;
}
const existingRetry = await this.prisma.smsSubmitRecord.findUnique({
where: { retryOfSubmitRecordId: sourceAttempt.id },
});
if (existingRetry) {
this.logger.warn(`sms_retry_claim_reused ${JSON.stringify({
messageId: message.messageId,
messageRecordId: message.id,
retryOfSubmitRecordId: sourceAttempt.id,
submitId: existingRetry.submitId,
channelId: existingRetry.channelId,
})}`);
this.logger.warn(
`sms_retry_claim_reused ${JSON.stringify({
messageId: message.messageId,
messageRecordId: message.id,
retryOfSubmitRecordId: sourceAttempt.id,
submitId: existingRetry.submitId,
channelId: existingRetry.channelId,
})}`,
);
return this.prisma.smsMessageRecord.findUnique({ where: { id: message.id } });
}
const ageMinutes = (Date.now() - new Date(message.queuedAt ?? Date.now()).getTime()) / 60_000;
this.logger.log(`sms_retry_route_started ${JSON.stringify({
messageId: message.messageId,
messageRecordId: message.id,
reason,
attemptedChannelIds,
ageMinutes: Math.round(ageMinutes * 100) / 100,
})}`);
if (ageMinutes >= 72 * 60) {
this.logger.warn(`sms_retry_route_skipped ${JSON.stringify({
this.logger.log(
`sms_retry_route_started ${JSON.stringify({
messageId: message.messageId,
messageRecordId: message.id,
reason: 'maximum_message_age_exceeded',
reason,
attemptedChannelIds,
ageMinutes: Math.round(ageMinutes * 100) / 100,
})}`);
})}`,
);
if (ageMinutes >= 72 * 60) {
this.logger.warn(
`sms_retry_route_skipped ${JSON.stringify({
messageId: message.messageId,
messageRecordId: message.id,
reason: 'maximum_message_age_exceeded',
ageMinutes: Math.round(ageMinutes * 100) / 100,
})}`,
);
return null;
}
const retryCarrier = message.carrier
? normalizeCarrier(message.carrier)
: await this.facade.identifyCarrier(message.phoneNumber);
const route = await this.facade.findApplicationRoute(message.tenantId, message.applicationId ?? undefined, retryCarrier);
const retryTimeLimitMinutes = Math.min(route.group.retryTimeLimitMinutes ?? route.group.retryTimeLimitHours * 60, 72 * 60);
const route = await this.facade.findApplicationRoute(
message.tenantId,
message.applicationId ?? undefined,
retryCarrier,
);
const retryTimeLimitMinutes = Math.min(
route.group.retryTimeLimitMinutes ?? route.group.retryTimeLimitHours * 60,
72 * 60,
);
if (!route.group.retryEnabled || ageMinutes >= retryTimeLimitMinutes) {
this.logger.warn(`sms_retry_route_skipped ${JSON.stringify({
messageId: message.messageId,
messageRecordId: message.id,
groupId: route.groupId,
reason: !route.group.retryEnabled ? 'group_retry_disabled' : 'group_retry_time_limit_exceeded',
ageMinutes: Math.round(ageMinutes * 100) / 100,
retryTimeLimitMinutes,
})}`);
this.logger.warn(
`sms_retry_route_skipped ${JSON.stringify({
messageId: message.messageId,
messageRecordId: message.id,
groupId: route.groupId,
reason: !route.group.retryEnabled ? 'group_retry_disabled' : 'group_retry_time_limit_exceeded',
ageMinutes: Math.round(ageMinutes * 100) / 100,
retryTimeLimitMinutes,
})}`,
);
return null;
}
try {
const routed = await this.facade.selectChannelForMessage({ ...message, carrier: retryCarrier }, {
forceNational: true,
excludeChannelIds: attemptedChannelIds,
});
const routed = await this.facade.selectChannelForMessage(
{ ...message, carrier: retryCarrier },
{
forceNational: true,
excludeChannelIds: attemptedChannelIds,
},
);
await this.prisma.smsMessageRecord.update({
where: { id: message.id },
data: { errorMessage: reason },
});
const retried = await this.facade.submitMessageToGateway(
message,
routed,
attempts.length,
sourceAttempt.id,
const retried = await this.facade.submitMessageToGateway(message, routed, attempts.length, sourceAttempt.id);
this.logger.log(
`sms_retry_route_selected ${JSON.stringify({
messageId: message.messageId,
messageRecordId: message.id,
groupId: routed.groupId,
channelId: routed.channel.id,
attempt: attempts.length,
})}`,
);
this.logger.log(`sms_retry_route_selected ${JSON.stringify({
messageId: message.messageId,
messageRecordId: message.id,
groupId: routed.groupId,
channelId: routed.channel.id,
attempt: attempts.length,
})}`);
return retried;
} catch (error) {
this.logger.error(`sms_retry_route_failed ${JSON.stringify({
messageId: message.messageId,
messageRecordId: message.id,
reason,
attemptedChannelIds,
error: error instanceof Error ? error.message : String(error),
})}`);
// Infrastructure failures and the completion routing probe must abort the
// transaction; treating them as "no retry route" would finalize a failure.
if (!(error instanceof HttpException) || error.getStatus() >= 500) throw error;
this.logger.error(
`sms_retry_route_failed ${JSON.stringify({
messageId: message.messageId,
messageRecordId: message.id,
reason,
attemptedChannelIds,
error: error instanceof Error ? error.message : String(error),
})}`,
);
return null;
}
}
@@ -368,7 +368,7 @@ export class SendSubmissionService {
template?: { signature?: { id?: string | null } | null } | null;
signature?: { id?: string | null } | null;
},
options: { forceNational?: boolean; excludeChannelIds?: string[] } = {},
options: { forceNational?: boolean; excludeChannelIds?: string[]; previewOnly?: boolean } = {},
): Promise<RoutedChannel> {
return this.gatewaySubmit.selectChannelForMessage(message, options);
}
+28 -25
View File
@@ -1,16 +1,12 @@
import { BadRequestException, HttpException, HttpStatus, Logger, NotFoundException } from '@nestjs/common';
import { Prisma } from '@prisma/client';
import { createHash } from 'node:crypto';
import { Logger } from '@nestjs/common';
import { BillingService } from '../billing/billing.service';
import { moneyToNumber } from '../common/money';
import type { OpenApiService } from '../open-api/open-api.service';
import { PrismaService } from '../prisma/prisma.service';
import type { GatewaySubmitResultDto, GatewaySubmitSegmentResultDto, GatewayReceiptEventDto, GatewayUplinkEventDto, GatewayPendingDeliveryQueryDto, GatewayDownstreamSentDto, GatewayDownstreamAcknowledgedDto, GatewayDownstreamFailureType, GatewayControlDeliveryResult, GatewaySubmitDeadLetterDto, RequeueGatewaySubmitExceptionDto, GatewayDownstreamRecoveryStatusDto, TimeoutUnknownDto } from './send-chain.contracts';
import { positiveInteger, normalizeReceiptStatus, normalizeCarrier, DEFAULT_DOWNSTREAM_RETRY_DELAY_MS, DEFAULT_DOWNSTREAM_RETRY_MAX_DELAY_MS, DEFAULT_DOWNSTREAM_MAX_RETRIES, DEFAULT_DOWNSTREAM_PENDING_TIMEOUT_HOURS, DEFAULT_RECEIPT_TIMEOUT_HOURS, DEFAULT_GATEWAY_SUBMIT_REQUEUE_STALE_MS, DEFAULT_DOWNSTREAM_MANUAL_REQUEUE_STALE_MS, DEFAULT_UPSTREAM_RECEIPT_INBOX_PROCESSING_STALE_MS, DEFAULT_UPSTREAM_RECEIPT_INBOX_MAX_ATTEMPTS, DEFAULT_UPSTREAM_RECEIPT_INBOX_MAX_AGE_HOURS, gatewaySubmitRequeueKey, isObjectRecord, asDateOrNull, downstreamRetryDelayMs, downstreamAckTimeoutMs, downstreamRetryBaseDelayMs, downstreamRetryMaxDelayMs, downstreamMaxRetries, downstreamPendingTimeoutHours, downstreamControlFailureMessage, normalizeSubmitStatus, downstreamDeliveryAttemptKey, hasRecoveryAuditStateChanged, normalizeRecoveryFailureCategory, aggregateReceiptSegmentState, isSameUpstreamEndpointIdentity, receiptEventKey } from './send-chain.helpers';
import type { SendSubmissionService } from './send-submission.service';
import type { SendCompletionCallbacks, SendCompletionFacade } from './send-completion.service';
import { completionContext } from './completion-context';
import { queueFinalReceiptDeliveries } from './downstream-receipt-targets';
import type { TimeoutUnknownDto } from './send-chain.contracts';
import { DEFAULT_RECEIPT_TIMEOUT_HOURS, downstreamPendingTimeoutHours, positiveInteger } from './send-chain.helpers';
import type { SendCompletionCallbacks, SendCompletionFacade } from './send-completion.service';
/**
* R10 timeout implementation.
@@ -29,10 +25,12 @@ export class SendTimeoutService {
) {}
async markUnknownTimeout(data: TimeoutUnknownDto) {
const olderThanHours = data.olderThanHours ?? positiveInteger(process.env.SMS_RECEIPT_TIMEOUT_HOURS, DEFAULT_RECEIPT_TIMEOUT_HOURS);
const olderThanHours =
data.olderThanHours ?? positiveInteger(process.env.SMS_RECEIPT_TIMEOUT_HOURS, DEFAULT_RECEIPT_TIMEOUT_HOURS);
const cutoff = new Date(Date.now() - olderThanHours * 60 * 60 * 1000);
const candidates = await this.prisma.smsMessageRecord.findMany({
where: {
id: completionContext.getStore()?.messageRecordId,
tenantId: { not: null },
OR: [
{ status: { in: ['submitted', 'unknown'] }, submittedAt: { lte: cutoff } },
@@ -79,7 +77,10 @@ export class SendTimeoutService {
// Refund uses the platform-message idempotency key. Re-running it for a
// timeout whose downstream outbox was not fully queued also recovers a
// crash between the state transition and the original refund call.
await this.facade.refundMessage(candidate as typeof candidate & { tenantId: string }, `${olderThanHours}小时未收到明确回执,自动超时退款`);
await this.facade.refundMessage(
candidate as typeof candidate & { tenantId: string },
`${olderThanHours}小时未收到明确回执,自动超时退款`,
);
const queued = await queueFinalReceiptDeliveries(
this.prisma,
(request) => this.facade.queueAndTryDownstreamDelivery(request),
@@ -117,10 +118,7 @@ export class SendTimeoutService {
const expired = await this.prisma.cmppDownstreamDelivery.findMany({
where: {
status: 'pending',
OR: [
{ lastRetriedAt: null, createdAt: { lte: cutoff } },
{ lastRetriedAt: { lte: cutoff } },
],
OR: [{ lastRetriedAt: null, createdAt: { lte: cutoff } }, { lastRetriedAt: { lte: cutoff } }],
},
select: { id: true },
take: 500,
@@ -139,16 +137,21 @@ export class SendTimeoutService {
if (this.receiptTimeoutScanRunning) return;
this.receiptTimeoutScanRunning = true;
try {
const [receiptResult, downstreamResult, requeueRecoveryResult, downstreamManualRecoveryResult] = await Promise.all([
this.facade.markUnknownTimeout({}),
this.facade.markExpiredDownstreamDeliveries(),
this.facade.recoverStaleGatewaySubmitRequeues(),
this.facade.recoverStaleDownstreamManualRequeues(),
]);
if (receiptResult.timeout > 0) this.logger.log(`Marked ${receiptResult.timeout} messages as receipt timeout and refunded charged messages`);
if (downstreamResult.failed > 0) this.logger.log(`Terminated ${downstreamResult.failed} expired downstream deliveries`);
if (requeueRecoveryResult.recovered > 0) this.logger.log(`Recovered ${requeueRecoveryResult.recovered} stale Gateway submit requeues`);
if (downstreamManualRecoveryResult.recovered > 0) this.logger.log(`Recovered ${downstreamManualRecoveryResult.recovered} stale downstream manual requeues`);
const [receiptResult, downstreamResult, requeueRecoveryResult, downstreamManualRecoveryResult] =
await Promise.all([
this.facade.markUnknownTimeout({}),
this.facade.markExpiredDownstreamDeliveries(),
this.facade.recoverStaleGatewaySubmitRequeues(),
this.facade.recoverStaleDownstreamManualRequeues(),
]);
if (receiptResult.timeout > 0)
this.logger.log(`Marked ${receiptResult.timeout} messages as receipt timeout and refunded charged messages`);
if (downstreamResult.failed > 0)
this.logger.log(`Terminated ${downstreamResult.failed} expired downstream deliveries`);
if (requeueRecoveryResult.recovered > 0)
this.logger.log(`Recovered ${requeueRecoveryResult.recovered} stale Gateway submit requeues`);
if (downstreamManualRecoveryResult.recovered > 0)
this.logger.log(`Recovered ${downstreamManualRecoveryResult.recovered} stale downstream manual requeues`);
} catch (error) {
this.logger.error('Receipt timeout scan failed', error instanceof Error ? error.stack : String(error));
} finally {