fix: coordinate SMS completion and improve operations diagnostics
CSS quality / css-quality (push) Has been cancelled
CSS quality / css-quality (push) Has been cancelled
This commit is contained in:
@@ -0,0 +1,17 @@
|
||||
CREATE TABLE "InfrastructureAlertCollection" (
|
||||
"id" TEXT NOT NULL PRIMARY KEY,
|
||||
"observedAt" TIMESTAMP(3) NOT NULL
|
||||
);
|
||||
CREATE TABLE "InfrastructureAlertEvent" (
|
||||
"id" TEXT NOT NULL PRIMARY KEY,
|
||||
"fingerprint" TEXT NOT NULL,
|
||||
"activeAt" TIMESTAMP(3) NOT NULL,
|
||||
"payload" JSONB NOT NULL,
|
||||
"lastObservedAt" TIMESTAMP(3) NOT NULL,
|
||||
"recoveredAt" TIMESTAMP(3),
|
||||
"clearedAt" TIMESTAMP(3),
|
||||
"clearedBy" TEXT,
|
||||
"createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
CREATE UNIQUE INDEX "InfrastructureAlertEvent_fingerprint_activeAt_key" ON "InfrastructureAlertEvent"("fingerprint", "activeAt");
|
||||
CREATE INDEX "InfrastructureAlertEvent_clearedAt_activeAt_idx" ON "InfrastructureAlertEvent"("clearedAt", "activeAt");
|
||||
@@ -0,0 +1,15 @@
|
||||
CREATE TABLE "SmsAttemptCompletionWork" (
|
||||
"id" TEXT PRIMARY KEY, "workKey" TEXT NOT NULL, "tenantId" TEXT, "messageRecordId" TEXT NOT NULL, "sourceSubmitRecordId" TEXT,
|
||||
"revision" INTEGER NOT NULL DEFAULT 0, "processedRevision" INTEGER NOT NULL DEFAULT 0, "state" TEXT NOT NULL DEFAULT 'pending',
|
||||
"leaseOwner" TEXT, "leaseUntil" TIMESTAMP(3), "fenceVersion" INTEGER NOT NULL DEFAULT 0, "attempts" INTEGER NOT NULL DEFAULT 0,
|
||||
"nextAttemptAt" TIMESTAMP(3) NOT NULL DEFAULT (CURRENT_TIMESTAMP AT TIME ZONE 'UTC'), "decision" TEXT, "retrySubmitRecordId" TEXT, "lastError" TEXT,
|
||||
"createdAt" TIMESTAMP(3) NOT NULL DEFAULT (CURRENT_TIMESTAMP AT TIME ZONE 'UTC'), "updatedAt" TIMESTAMP(3) NOT NULL DEFAULT (CURRENT_TIMESTAMP AT TIME ZONE 'UTC'));
|
||||
CREATE UNIQUE INDEX "SmsAttemptCompletionWork_workKey_key" ON "SmsAttemptCompletionWork"("workKey");
|
||||
CREATE UNIQUE INDEX "SmsAttemptCompletionWork_sourceSubmitRecordId_key" ON "SmsAttemptCompletionWork"("sourceSubmitRecordId");
|
||||
CREATE INDEX "SmsAttemptCompletionWork_state_nextAttemptAt_idx" ON "SmsAttemptCompletionWork"("state", "nextAttemptAt");
|
||||
CREATE INDEX "SmsAttemptCompletionWork_state_leaseUntil_idx" ON "SmsAttemptCompletionWork"("state", "leaseUntil");
|
||||
CREATE INDEX "SmsAttemptCompletionWork_messageRecordId_idx" ON "SmsAttemptCompletionWork"("messageRecordId");
|
||||
CREATE TABLE "SmsCompletionEvent" ("id" TEXT PRIMARY KEY, "eventKey" TEXT NOT NULL, "workId" TEXT NOT NULL REFERENCES "SmsAttemptCompletionWork"("id") ON DELETE RESTRICT,
|
||||
"kind" TEXT NOT NULL, "payload" JSONB NOT NULL, "processedAt" TIMESTAMP(3), "createdAt" TIMESTAMP(3) NOT NULL DEFAULT (CURRENT_TIMESTAMP AT TIME ZONE 'UTC'));
|
||||
CREATE UNIQUE INDEX "SmsCompletionEvent_eventKey_key" ON "SmsCompletionEvent"("eventKey");
|
||||
CREATE INDEX "SmsCompletionEvent_workId_processedAt_createdAt_idx" ON "SmsCompletionEvent"("workId", "processedAt", "createdAt");
|
||||
@@ -2840,3 +2840,59 @@ model OpenApiDispatchOutbox {
|
||||
updatedAt DateTime @updatedAt
|
||||
@@index([status, leaseUntil])
|
||||
}
|
||||
|
||||
model InfrastructureAlertCollection {
|
||||
id String @id
|
||||
observedAt DateTime
|
||||
}
|
||||
|
||||
model InfrastructureAlertEvent {
|
||||
id String @id @default(cuid())
|
||||
fingerprint String
|
||||
activeAt DateTime
|
||||
payload Json
|
||||
lastObservedAt DateTime
|
||||
recoveredAt DateTime?
|
||||
clearedAt DateTime?
|
||||
clearedBy String?
|
||||
createdAt DateTime @default(now())
|
||||
@@unique([fingerprint, activeAt])
|
||||
@@index([clearedAt, activeAt])
|
||||
}
|
||||
|
||||
model SmsAttemptCompletionWork {
|
||||
id String @id @default(cuid())
|
||||
workKey String @unique
|
||||
tenantId String?
|
||||
messageRecordId String
|
||||
sourceSubmitRecordId String? @unique
|
||||
revision Int @default(0)
|
||||
processedRevision Int @default(0)
|
||||
state String @default("pending")
|
||||
leaseOwner String?
|
||||
leaseUntil DateTime?
|
||||
fenceVersion Int @default(0)
|
||||
attempts Int @default(0)
|
||||
nextAttemptAt DateTime @default(dbgenerated("(CURRENT_TIMESTAMP AT TIME ZONE 'UTC')"))
|
||||
decision String?
|
||||
retrySubmitRecordId String?
|
||||
lastError String?
|
||||
createdAt DateTime @default(dbgenerated("(CURRENT_TIMESTAMP AT TIME ZONE 'UTC')"))
|
||||
updatedAt DateTime @updatedAt
|
||||
events SmsCompletionEvent[]
|
||||
@@index([state, nextAttemptAt])
|
||||
@@index([state, leaseUntil])
|
||||
@@index([messageRecordId])
|
||||
}
|
||||
|
||||
model SmsCompletionEvent {
|
||||
id String @id @default(cuid())
|
||||
eventKey String @unique
|
||||
workId String
|
||||
kind String
|
||||
payload Json
|
||||
processedAt DateTime?
|
||||
createdAt DateTime @default(dbgenerated("(CURRENT_TIMESTAMP AT TIME ZONE 'UTC')"))
|
||||
work SmsAttemptCompletionWork @relation(fields: [workId], references: [id], onDelete: Restrict)
|
||||
@@index([workId, processedAt, createdAt])
|
||||
}
|
||||
|
||||
@@ -1,18 +1,24 @@
|
||||
import { BadRequestException, Injectable, Logger, NotFoundException, OnModuleDestroy, OnModuleInit } from '@nestjs/common';
|
||||
import { Queue } from 'bullmq';
|
||||
import IORedis from 'ioredis';
|
||||
import { BadRequestException, NotFoundException } from '@nestjs/common';
|
||||
import { Prisma } from '@prisma/client';
|
||||
import { randomUUID } from 'crypto';
|
||||
import { assertMoneyUnits, moneyToNumber } from '../common/money';
|
||||
import { moneyToNumber } from '../common/money';
|
||||
import { PrismaService } from '../prisma/prisma.service';
|
||||
import type { CreateChannelDto, UpdateChannelDto, CreateChannelGroupDto, CreateChannelGroupItemDto, UpdateChannelGroupDto, CreateRouteRuleDto, CreateReportFieldDto, ReplaceReportFieldsDto, CreateReportMaterialDto, CreateReportTaskDto, ChangeReportTaskStatusesDto, CreateReportExportDto, CreateReceiptImportDto, UpsertConnectionStateDto, ChangeChannelStatusDto, CopyChannelDto, TestChannelDto } from './channels.contracts';
|
||||
import { GATEWAY_CONNECTION_QUEUE, GATEWAY_SUBMIT_QUEUE, GATEWAY_SUBMIT_STREAM, DEFAULT_GATEWAY_CONTROL_URL, DEFAULT_CHANNEL_CONNECTION_ID, DEFAULT_CONNECTING_TIMEOUT_MS, DEFAULT_CONNECTING_TIMEOUT_SCAN_MS, DEFAULT_GATEWAY_STARTUP_RECONNECT_DELAY_MS, DEFAULT_GATEWAY_RECONCILE_INTERVAL_MS, DEFAULT_GATEWAY_CONTROL_TIMEOUT_MS, DEFAULT_HEARTBEAT_INTERVAL_SECONDS, DEFAULT_HEARTBEAT_MISS_THRESHOLD, HEARTBEAT_AUDIT_INTERVAL_MS, CONNECTING_TIMEOUT_ERROR, DEFAULT_CMPP_VERSION, normalizeTestPhones, normalizeTestContent, calculateBillingUnits, buildChannelTestSubmitCommand, getConfigValue, getStringConfigValue, normalizeConnectionAction, normalizeCmppVersion, normalizeGatewayConnectionStatus, defaultChannelConnectionId, getDesiredConnections, ChannelConnectionSettings, getRuntimeConfigInteger, channelConnectionSettingsChanged, channelGroupAuditSnapshot, normalizeChannelRuntimeConfig, normalizeCmppServiceId, normalizeChannelRateLimit, normalizeExtensionDigits, getPositiveRuntimeInteger, bullmqConnection, getPositiveIntegerEnv, parseReceiptContent, splitReceiptLine, stripReceiptCell, findReceiptStatusIndex, normalizeReceiptStatus, deriveReceiptStatus, ChannelReportDeliveryRow, summarizeChannelReportDelivery, sumReportDelivery, percentage, latestDate, currentShanghaiDayRange, normalizeRetryTimeLimitMinutes, normalizeSpreadsheetSize, normalizeBusinessCarrier, normalizeChannelCarrier, isChannelCarrierCompatible, normalizeRegion, isRegionCompatible, validateGroupItems, normalizeReportType, summarizeReportStatuses, normalizeLinkEvent } from './channels.helpers';
|
||||
import type { TestChannelDto } from './channels.contracts';
|
||||
import {
|
||||
normalizeTestPhones,
|
||||
normalizeTestContent,
|
||||
normalizeGatewayConnectionStatus,
|
||||
calculateBillingUnits,
|
||||
buildChannelTestSubmitCommand,
|
||||
} from './channels.helpers';
|
||||
import { ChannelConnectionService } from './channel-connection.service';
|
||||
import { detectDrainageContent } from '../send-chain/drainage-content-detection';
|
||||
|
||||
/** R5 channel domain service composed behind ChannelsService. */
|
||||
export class ChannelTestService {
|
||||
constructor(private readonly prisma: PrismaService, private readonly connection: ChannelConnectionService) {}
|
||||
constructor(
|
||||
private readonly prisma: PrismaService,
|
||||
private readonly connection: ChannelConnectionService,
|
||||
) {}
|
||||
|
||||
async testChannel(channelId: string, data: TestChannelDto = {}) {
|
||||
const phoneNumbers = normalizeTestPhones(data);
|
||||
@@ -27,8 +33,8 @@ export class ChannelTestService {
|
||||
if (channel.status !== 'active') {
|
||||
throw new BadRequestException('通道未启用,不能发送测试短信');
|
||||
}
|
||||
const connectedState = channel.connectionStates.find((state) =>
|
||||
normalizeGatewayConnectionStatus(state.status) === 'connected' && (state.currentConnections ?? 0) > 0,
|
||||
const connectedState = channel.connectionStates.find(
|
||||
(state) => normalizeGatewayConnectionStatus(state.status) === 'connected' && (state.currentConnections ?? 0) > 0,
|
||||
);
|
||||
if (!connectedState) {
|
||||
throw new BadRequestException('通道当前没有可用 CMPP 连接,请先连接成功后再测试发送');
|
||||
@@ -36,7 +42,6 @@ export class ChannelTestService {
|
||||
|
||||
const createdAt = new Date();
|
||||
const testNo = `CHTEST-${Date.now()}-${randomUUID().slice(0, 8)}`;
|
||||
const drainageDetection = await detectDrainageContent(this.prisma, content);
|
||||
const results = [];
|
||||
for (const [index, phoneNumber] of phoneNumbers.entries()) {
|
||||
const messageId = `MSG-TEST-${Date.now()}-${randomUUID().slice(0, 8)}`;
|
||||
@@ -51,7 +56,6 @@ export class ChannelTestService {
|
||||
messageId,
|
||||
phoneNumber,
|
||||
content,
|
||||
...drainageDetection,
|
||||
billingUnits: calculateBillingUnits(content),
|
||||
unitPrice: 0,
|
||||
amountCents: 0,
|
||||
|
||||
@@ -37,6 +37,15 @@ export class InfrastructureMonitoringController {
|
||||
return this.monitoring.markAlertRead(fingerprint, activeAt, userId);
|
||||
}
|
||||
|
||||
@Post('alerts/:fingerprint/clear')
|
||||
clearAlert(
|
||||
@Param('fingerprint') fingerprint: string,
|
||||
@Body('activeAt') activeAt: unknown,
|
||||
@CurrentSessionUserId() userId: string,
|
||||
) {
|
||||
return this.monitoring.clearAlert(fingerprint, activeAt, userId);
|
||||
}
|
||||
|
||||
@Get('alert-thresholds')
|
||||
alertThresholds() {
|
||||
return this.settings.get();
|
||||
|
||||
@@ -1,8 +1,15 @@
|
||||
import { BadRequestException } from '@nestjs/common';
|
||||
import { ConfigService } from '@nestjs/config';
|
||||
import { createHash } from 'node:crypto';
|
||||
import { InfrastructureMonitoringService } from './infrastructure-monitoring.service';
|
||||
import { FILESYSTEM_USAGE_PERCENT, filesystemIdentity } from './filesystem-metrics';
|
||||
import { InfrastructureMonitoringService } from './infrastructure-monitoring.service';
|
||||
import { retainedAlerts } from './persistent-alerts';
|
||||
|
||||
jest.mock('./persistent-alerts', () => ({
|
||||
retainAlerts: jest.fn(async (_prisma, alerts) => alerts),
|
||||
retainedAlerts: jest.fn(),
|
||||
clearRetainedAlert: jest.fn(),
|
||||
}));
|
||||
|
||||
function success(data: unknown) {
|
||||
return {
|
||||
@@ -14,7 +21,12 @@ function success(data: unknown) {
|
||||
|
||||
describe('InfrastructureMonitoringService', () => {
|
||||
const prisma = {
|
||||
infrastructureAlertRead: { findMany: jest.fn().mockResolvedValue([]), create: jest.fn(), update: jest.fn(), findUniqueOrThrow: jest.fn() },
|
||||
infrastructureAlertRead: {
|
||||
findMany: jest.fn().mockResolvedValue([]),
|
||||
create: jest.fn(),
|
||||
update: jest.fn(),
|
||||
findUniqueOrThrow: jest.fn(),
|
||||
},
|
||||
operationLog: { create: jest.fn() },
|
||||
$transaction: jest.fn(),
|
||||
};
|
||||
@@ -34,9 +46,27 @@ describe('InfrastructureMonitoringService', () => {
|
||||
});
|
||||
|
||||
it('rejects credential-bearing or remote plaintext Prometheus endpoints at startup', () => {
|
||||
expect(() => new InfrastructureMonitoringService(new ConfigService({ PROMETHEUS_URL: 'http://user:secret@127.0.0.1:9090' }), prisma as never)).toThrow('must not contain credentials');
|
||||
expect(() => new InfrastructureMonitoringService(new ConfigService({ PROMETHEUS_URL: 'http://monitor.example.com:9090' }), prisma as never)).toThrow('must use HTTPS');
|
||||
expect(() => new InfrastructureMonitoringService(new ConfigService({ PROMETHEUS_URL: 'https://monitor.example.com' }), prisma as never)).not.toThrow();
|
||||
expect(
|
||||
() =>
|
||||
new InfrastructureMonitoringService(
|
||||
new ConfigService({ PROMETHEUS_URL: 'http://user:secret@127.0.0.1:9090' }),
|
||||
prisma as never,
|
||||
),
|
||||
).toThrow('must not contain credentials');
|
||||
expect(
|
||||
() =>
|
||||
new InfrastructureMonitoringService(
|
||||
new ConfigService({ PROMETHEUS_URL: 'http://monitor.example.com:9090' }),
|
||||
prisma as never,
|
||||
),
|
||||
).toThrow('must use HTTPS');
|
||||
expect(
|
||||
() =>
|
||||
new InfrastructureMonitoringService(
|
||||
new ConfigService({ PROMETHEUS_URL: 'https://monitor.example.com' }),
|
||||
prisma as never,
|
||||
),
|
||||
).not.toThrow();
|
||||
});
|
||||
|
||||
it('loads real Prometheus vectors, ranges, services and active alerts', async () => {
|
||||
@@ -45,34 +75,52 @@ describe('InfrastructureMonitoringService', () => {
|
||||
const url = new URL(String(input));
|
||||
requestedUrls.push(url);
|
||||
if (url.pathname.endsWith('/alerts')) {
|
||||
return success({ alerts: [{
|
||||
labels: { alertname: 'HostCpuHigh', severity: 'warning', instance: '127.0.0.1:9100' },
|
||||
annotations: { summary: 'CPU持续偏高', threshold: '85%' },
|
||||
state: 'firing',
|
||||
activeAt: '2026-08-14T03:00:00.000Z',
|
||||
value: '88.2',
|
||||
}] });
|
||||
return success({
|
||||
alerts: [
|
||||
{
|
||||
labels: { alertname: 'HostCpuHigh', severity: 'warning', instance: '127.0.0.1:9100' },
|
||||
annotations: { summary: 'CPU持续偏高', threshold: '85%' },
|
||||
state: 'firing',
|
||||
activeAt: '2026-08-14T03:00:00.000Z',
|
||||
value: '88.2',
|
||||
},
|
||||
],
|
||||
});
|
||||
}
|
||||
const query = url.searchParams.get('query') ?? '';
|
||||
if (url.pathname.endsWith('/query_range')) {
|
||||
return success({ result: [{ metric: {}, values: [[1_765_000_000, '12.5'], [1_765_000_060, '14.5']] }] });
|
||||
return success({
|
||||
result: [
|
||||
{
|
||||
metric: {},
|
||||
values: [
|
||||
[1_765_000_000, '12.5'],
|
||||
[1_765_000_060, '14.5'],
|
||||
],
|
||||
},
|
||||
],
|
||||
});
|
||||
}
|
||||
if (query.includes('node_systemd_unit_state')) {
|
||||
return success({ result: [
|
||||
{ metric: { name: 'cmpp-api.service' }, value: [1_765_000_060, '1'] },
|
||||
{ metric: { name: 'cmpp-gateway.service' }, value: [1_765_000_060, '1'] },
|
||||
{ metric: { name: 'postgresql.service' }, value: [1_765_000_060, '1'] },
|
||||
{ metric: { name: 'redis-server.service' }, value: [1_765_000_060, '1'] },
|
||||
{ metric: { name: 'cmpp-minio.service' }, value: [1_765_000_060, '1'] },
|
||||
{ metric: { name: 'nginx.service' }, value: [1_765_000_060, '1'] },
|
||||
] });
|
||||
return success({
|
||||
result: [
|
||||
{ metric: { name: 'cmpp-api.service' }, value: [1_765_000_060, '1'] },
|
||||
{ metric: { name: 'cmpp-gateway.service' }, value: [1_765_000_060, '1'] },
|
||||
{ metric: { name: 'postgresql.service' }, value: [1_765_000_060, '1'] },
|
||||
{ metric: { name: 'redis-server.service' }, value: [1_765_000_060, '1'] },
|
||||
{ metric: { name: 'cmpp-minio.service' }, value: [1_765_000_060, '1'] },
|
||||
{ metric: { name: 'nginx.service' }, value: [1_765_000_060, '1'] },
|
||||
],
|
||||
});
|
||||
}
|
||||
if (query.includes('cmpp:service_.*')) {
|
||||
return success({ result: [
|
||||
{ metric: { __name__: 'cmpp:service_api:requests_per_second' }, value: [1_765_000_060, '12.5'] },
|
||||
{ metric: { __name__: 'cmpp:service_api:error_percent' }, value: [1_765_000_060, '0.2'] },
|
||||
{ metric: { __name__: 'cmpp:service_gateway:queue_pending' }, value: [1_765_000_060, '3'] },
|
||||
] });
|
||||
return success({
|
||||
result: [
|
||||
{ metric: { __name__: 'cmpp:service_api:requests_per_second' }, value: [1_765_000_060, '12.5'] },
|
||||
{ metric: { __name__: 'cmpp:service_api:error_percent' }, value: [1_765_000_060, '0.2'] },
|
||||
{ metric: { __name__: 'cmpp:service_gateway:queue_pending' }, value: [1_765_000_060, '3'] },
|
||||
],
|
||||
});
|
||||
}
|
||||
if (query.includes('timestamp(node_uname_info)')) {
|
||||
return success({ result: [{ metric: {}, value: [1_765_000_060, '1765000060'] }] });
|
||||
@@ -87,14 +135,27 @@ describe('InfrastructureMonitoringService', () => {
|
||||
expect(result.metrics.cpuUsagePercent).toBe(25);
|
||||
expect(result.trends.cpuUsagePercent).toHaveLength(2);
|
||||
expect(result.summary).toMatchObject({ overallStatus: 'warning', serviceHealthy: 6, warningAlerts: 1 });
|
||||
expect(result.services.find((item) => item.key === 'redis')).toMatchObject({ unit: 'redis-server.service', status: 'healthy' });
|
||||
expect(result.services.find((item) => item.key === 'redis')).toMatchObject({
|
||||
unit: 'redis-server.service',
|
||||
status: 'healthy',
|
||||
});
|
||||
expect(result.serviceMetrics.find((item) => item.key === 'api')).toMatchObject({ available: true });
|
||||
expect(result.serviceMetrics.find((item) => item.key === 'gateway')?.metrics.find((item) => item.key === 'queuePending')?.value).toBe(3);
|
||||
expect(
|
||||
result.serviceMetrics.find((item) => item.key === 'gateway')?.metrics.find((item) => item.key === 'queuePending')
|
||||
?.value,
|
||||
).toBe(3);
|
||||
expect(result.alerts[0]).toMatchObject({ name: 'HostCpuHigh', severity: 'warning', currentValue: '88.2' });
|
||||
expect(requestedUrls.filter((url) => url.pathname.endsWith('/query_range'))).toHaveLength(5);
|
||||
expect(requestedUrls.filter((url) => url.pathname.endsWith('/query_range')).every((url) => url.searchParams.get('step') === '60')).toBe(true);
|
||||
expect(requestedUrls.find((url) => url.searchParams.get('query')?.includes('node_systemd_unit_state'))?.searchParams.get('query'))
|
||||
.toContain('cmpp-api\\\\.service');
|
||||
expect(
|
||||
requestedUrls
|
||||
.filter((url) => url.pathname.endsWith('/query_range'))
|
||||
.every((url) => url.searchParams.get('step') === '60'),
|
||||
).toBe(true);
|
||||
expect(
|
||||
requestedUrls
|
||||
.find((url) => url.searchParams.get('query')?.includes('node_systemd_unit_state'))
|
||||
?.searchParams.get('query'),
|
||||
).toContain('cmpp-api\\\\.service');
|
||||
});
|
||||
|
||||
it('returns an explicit unavailable payload without stale metrics when Prometheus fails', async () => {
|
||||
@@ -124,12 +185,36 @@ describe('InfrastructureMonitoringService', () => {
|
||||
if (url.pathname.endsWith('/alerts')) return success({ alerts: [] });
|
||||
if (!query.includes('node_filesystem_')) return success({ result: [] });
|
||||
expect(query).not.toContain('mountpoint="/"');
|
||||
if (url.pathname.endsWith('/query_range')) return success({ result: [...metrics].reverse().map((metric) => ({ metric, values: [[1765000060, metric.mountpoint === '/' ? '91' : '12']] })) });
|
||||
return success({ result: metrics.map((metric) => ({ metric, value: [1765000060, query === FILESYSTEM_USAGE_PERCENT ? (metric.mountpoint === '/' ? '91' : '12') : query.includes('avail') ? '9' : '100'] })) });
|
||||
if (url.pathname.endsWith('/query_range'))
|
||||
return success({
|
||||
result: [...metrics]
|
||||
.reverse()
|
||||
.map((metric) => ({ metric, values: [[1765000060, metric.mountpoint === '/' ? '91' : '12']] })),
|
||||
});
|
||||
return success({
|
||||
result: metrics.map((metric) => ({
|
||||
metric,
|
||||
value: [
|
||||
1765000060,
|
||||
query === FILESYSTEM_USAGE_PERCENT
|
||||
? metric.mountpoint === '/'
|
||||
? '91'
|
||||
: '12'
|
||||
: query.includes('avail')
|
||||
? '9'
|
||||
: '100',
|
||||
],
|
||||
})),
|
||||
});
|
||||
});
|
||||
const result = await new InfrastructureMonitoringService(new ConfigService(), prisma as never).overview('1h');
|
||||
expect(result.disks.map((disk) => disk.mountpoint)).toEqual(['/', '/archive', '/data']);
|
||||
expect(result.disks[0]).toMatchObject({ usagePercent: 91, totalBytes: 100, availableBytes: 9, trend: [{ timestamp: new Date(1765000060000).toISOString(), value: 91 }] });
|
||||
expect(result.disks[0]).toMatchObject({
|
||||
usagePercent: 91,
|
||||
totalBytes: 100,
|
||||
availableBytes: 9,
|
||||
trend: [{ timestamp: new Date(1765000060000).toISOString(), value: 91 }],
|
||||
});
|
||||
expect(result.disks[2].trend[0].value).toBe(12);
|
||||
expect(result.metrics.diskUsagePercent).toBe(91);
|
||||
expect(result.trends.diskUsagePercent[0].value).toBe(91);
|
||||
@@ -146,20 +231,44 @@ describe('InfrastructureMonitoringService', () => {
|
||||
if (!query.includes('node_filesystem_')) return success({ result: [] });
|
||||
if (url.pathname.endsWith('/query_range')) {
|
||||
expect(query).toBe(FILESYSTEM_USAGE_PERCENT);
|
||||
return success({ result: [
|
||||
{ metric: data, values: [[1765000000, '82'], [1765000060, 'NaN'], [1765000120, '83.5']] },
|
||||
{ metric: root, values: [[1765000000, '91']] },
|
||||
] });
|
||||
return success({
|
||||
result: [
|
||||
{
|
||||
metric: data,
|
||||
values: [
|
||||
[1765000000, '82'],
|
||||
[1765000060, 'NaN'],
|
||||
[1765000120, '83.5'],
|
||||
],
|
||||
},
|
||||
{ metric: root, values: [[1765000000, '91']] },
|
||||
],
|
||||
});
|
||||
}
|
||||
if (query.startsWith('node_filesystem_size_bytes')) return success({ result: [
|
||||
...mounts.map((mountpoint) => ({ metric: { ...data, mountpoint }, value: [1765000120, '100'] })),
|
||||
...['/var/root-bind', '/'].map((mountpoint) => ({ metric: { ...root, mountpoint }, value: [1765000120, '200'] })),
|
||||
] });
|
||||
if (query === FILESYSTEM_USAGE_PERCENT) return success({ result: [
|
||||
{ metric: data, value: [1765000120, '83.5'] }, { metric: root, value: [1765000120, '91'] },
|
||||
] });
|
||||
if (query.startsWith('node_filesystem_size_bytes'))
|
||||
return success({
|
||||
result: [
|
||||
...mounts.map((mountpoint) => ({ metric: { ...data, mountpoint }, value: [1765000120, '100'] })),
|
||||
...['/var/root-bind', '/'].map((mountpoint) => ({
|
||||
metric: { ...root, mountpoint },
|
||||
value: [1765000120, '200'],
|
||||
})),
|
||||
],
|
||||
});
|
||||
if (query === FILESYSTEM_USAGE_PERCENT)
|
||||
return success({
|
||||
result: [
|
||||
{ metric: data, value: [1765000120, '83.5'] },
|
||||
{ metric: root, value: [1765000120, '91'] },
|
||||
],
|
||||
});
|
||||
expect(query).toContain('min by (instance, device, fstype)');
|
||||
return success({ result: [{ metric: data, value: [1765000120, '16.5'] }, { metric: root, value: [1765000120, '18'] }] });
|
||||
return success({
|
||||
result: [
|
||||
{ metric: data, value: [1765000120, '16.5'] },
|
||||
{ metric: root, value: [1765000120, '18'] },
|
||||
],
|
||||
});
|
||||
});
|
||||
const service = new InfrastructureMonitoringService(new ConfigService(), prisma as never);
|
||||
const result = await service.overview('1h');
|
||||
@@ -169,7 +278,13 @@ describe('InfrastructureMonitoringService', () => {
|
||||
expect(result.metrics.diskUsagePercent).toBe(91);
|
||||
expect(result.trends.diskUsagePercent[0].value).toBe(91);
|
||||
const disk = result.disks[1];
|
||||
expect(disk).toMatchObject({ id: filesystemIdentity(data), mountpoint: '/data', totalBytes: 100, availableBytes: 16.5, usagePercent: 83.5 });
|
||||
expect(disk).toMatchObject({
|
||||
id: filesystemIdentity(data),
|
||||
mountpoint: '/data',
|
||||
totalBytes: 100,
|
||||
availableBytes: 16.5,
|
||||
usagePercent: 83.5,
|
||||
});
|
||||
expect(disk.mountpoints).toEqual(['/data', '/var/lib/minio', '/var/lib/pgsql', '/var/lib/redis']);
|
||||
expect(disk.trend.map((point) => point.value)).toEqual([82, 83.5]);
|
||||
mounts.reverse();
|
||||
@@ -198,33 +313,77 @@ describe('InfrastructureMonitoringService', () => {
|
||||
const result = await new InfrastructureMonitoringService(new ConfigService(), prisma as never).overview('1h');
|
||||
expect(result.disks).toHaveLength(4);
|
||||
expect(new Set(result.disks.map((disk) => disk.id)).size).toBe(4);
|
||||
expect(result.disks.every((disk) => disk.usagePercent === null && disk.availableBytes === null && disk.trend.length === 0)).toBe(true);
|
||||
expect(
|
||||
result.disks.every(
|
||||
(disk) => disk.usagePercent === null && disk.availableBytes === null && disk.trend.length === 0,
|
||||
),
|
||||
).toBe(true);
|
||||
expect(result.metrics.diskUsagePercent).toBeNull();
|
||||
expect(result.trends.diskUsagePercent).toEqual([]);
|
||||
});
|
||||
|
||||
it('excludes only the current alert occurrence after the current administrator marks it read', async () => {
|
||||
const labels = { alertname: 'QaWarning', severity: 'warning', service: 'qa-preview' };
|
||||
const fingerprint = createHash('sha256').update(JSON.stringify(Object.entries(labels).sort(([left], [right]) => left.localeCompare(right)))).digest('hex').slice(0, 24);
|
||||
jest.spyOn(global, 'fetch').mockResolvedValue(success({ alerts: [{ labels, annotations: { summary: '演示预警' }, state: 'firing', activeAt: '2026-08-16T01:00:00.000Z' }] }));
|
||||
prisma.infrastructureAlertRead.findMany.mockResolvedValue([{ fingerprint, activeAt: new Date('2026-08-16T01:00:00.000Z'), readAt: new Date('2026-08-16T01:01:00.000Z') }]);
|
||||
const fingerprint = createHash('sha256')
|
||||
.update(JSON.stringify(Object.entries(labels).sort(([left], [right]) => left.localeCompare(right))))
|
||||
.digest('hex')
|
||||
.slice(0, 24);
|
||||
jest.spyOn(global, 'fetch').mockResolvedValue(
|
||||
success({
|
||||
alerts: [
|
||||
{ labels, annotations: { summary: '演示预警' }, state: 'firing', activeAt: '2026-08-16T01:00:00.000Z' },
|
||||
],
|
||||
}),
|
||||
);
|
||||
prisma.infrastructureAlertRead.findMany.mockResolvedValue([
|
||||
{ fingerprint, activeAt: new Date('2026-08-16T01:00:00.000Z'), readAt: new Date('2026-08-16T01:01:00.000Z') },
|
||||
]);
|
||||
const service = new InfrastructureMonitoringService(new ConfigService(), prisma as never);
|
||||
|
||||
await expect(service.notificationSummary('admin-1')).resolves.toEqual({ count: 0, criticalCount: 0 });
|
||||
|
||||
prisma.infrastructureAlertRead.findMany.mockResolvedValue([{ fingerprint, activeAt: new Date('2026-08-15T01:00:00.000Z'), readAt: new Date('2026-08-15T01:01:00.000Z') }]);
|
||||
prisma.infrastructureAlertRead.findMany.mockResolvedValue([
|
||||
{ fingerprint, activeAt: new Date('2026-08-15T01:00:00.000Z'), readAt: new Date('2026-08-15T01:01:00.000Z') },
|
||||
]);
|
||||
await expect(service.notificationSummary('admin-1')).resolves.toEqual({ count: 1, criticalCount: 0 });
|
||||
});
|
||||
|
||||
it('upserts an idempotent per-user read record only for a currently active occurrence', async () => {
|
||||
const labels = { alertname: 'QaCritical', severity: 'critical', service: 'qa-preview' };
|
||||
const fingerprint = createHash('sha256').update(JSON.stringify(Object.entries(labels).sort(([left], [right]) => left.localeCompare(right)))).digest('hex').slice(0, 24);
|
||||
jest.spyOn(global, 'fetch').mockResolvedValue(success({ alerts: [{ labels, annotations: { summary: '演示严重告警' }, state: 'firing', activeAt: '2026-08-16T02:00:00.000Z' }] }));
|
||||
prisma.$transaction.mockResolvedValue([{ activeAt: new Date('2026-08-16T02:00:00.000Z'), readAt: new Date('2026-08-16T02:01:00.000Z') }, {}]);
|
||||
const fingerprint = createHash('sha256')
|
||||
.update(JSON.stringify(Object.entries(labels).sort(([left], [right]) => left.localeCompare(right))))
|
||||
.digest('hex')
|
||||
.slice(0, 24);
|
||||
jest.spyOn(global, 'fetch').mockResolvedValue(
|
||||
success({
|
||||
alerts: [
|
||||
{ labels, annotations: { summary: '演示严重告警' }, state: 'firing', activeAt: '2026-08-16T02:00:00.000Z' },
|
||||
],
|
||||
}),
|
||||
);
|
||||
prisma.$transaction.mockResolvedValue([
|
||||
{ activeAt: new Date('2026-08-16T02:00:00.000Z'), readAt: new Date('2026-08-16T02:01:00.000Z') },
|
||||
{},
|
||||
]);
|
||||
const service = new InfrastructureMonitoringService(new ConfigService(), prisma as never);
|
||||
|
||||
await expect(service.markAlertRead(fingerprint, '2026-08-16T02:00:00.000Z', 'admin-1')).resolves.toMatchObject({ fingerprint, acknowledged: true });
|
||||
expect(prisma.infrastructureAlertRead.create).toHaveBeenCalledWith(expect.objectContaining({ data: expect.objectContaining({ fingerprint, userId: 'admin-1' }) }));
|
||||
await expect(service.markAlertRead(fingerprint, '2026-08-15T02:00:00.000Z', 'admin-1')).rejects.toThrow('已结束或已重新触发');
|
||||
jest.mocked(retainedAlerts).mockResolvedValue([
|
||||
{
|
||||
fingerprint,
|
||||
startedAt: '2026-08-16T02:00:00.000Z',
|
||||
name: 'QaCritical',
|
||||
severity: 'critical',
|
||||
} as never,
|
||||
]);
|
||||
await expect(service.markAlertRead(fingerprint, '2026-08-16T02:00:00.000Z', 'admin-1')).resolves.toMatchObject({
|
||||
fingerprint,
|
||||
acknowledged: true,
|
||||
});
|
||||
expect(prisma.infrastructureAlertRead.create).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ data: expect.objectContaining({ fingerprint, userId: 'admin-1' }) }),
|
||||
);
|
||||
await expect(service.markAlertRead(fingerprint, '2026-08-15T02:00:00.000Z', 'admin-1')).rejects.toThrow(
|
||||
'已结束或已重新触发',
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { retainAlerts, retainedAlerts, clearRetainedAlert } from './persistent-alerts';
|
||||
import {
|
||||
BadRequestException,
|
||||
Injectable,
|
||||
@@ -202,6 +203,8 @@ export class InfrastructureMonitoringService {
|
||||
private readonly logger = new Logger(InfrastructureMonitoringService.name);
|
||||
private readonly prometheusUrl: string;
|
||||
private readonly queryTimeoutMs: number;
|
||||
private alertTimer?: ReturnType<typeof setInterval>;
|
||||
private alertPoll?: Promise<InfrastructureAlert[]>;
|
||||
|
||||
constructor(
|
||||
config: ConfigService,
|
||||
@@ -211,6 +214,35 @@ export class InfrastructureMonitoringService {
|
||||
this.queryTimeoutMs = Math.min(15_000, Math.max(1_000, Number(config.get('PROMETHEUS_QUERY_TIMEOUT_MS') ?? 5_000)));
|
||||
}
|
||||
|
||||
onModuleInit() {
|
||||
if (process.env.CMPP_PROCESS_ROLE && !['api', 'all'].includes(process.env.CMPP_PROCESS_ROLE)) return;
|
||||
const poll = () =>
|
||||
void this.loadRetainedAlerts().catch(() => this.logger.warn('Persistent alert collection unavailable'));
|
||||
poll();
|
||||
this.alertTimer = setInterval(poll, 30_000);
|
||||
this.alertTimer.unref();
|
||||
}
|
||||
|
||||
onModuleDestroy() {
|
||||
if (this.alertTimer) clearInterval(this.alertTimer);
|
||||
}
|
||||
|
||||
private loadRetainedAlerts() {
|
||||
if (!this.alertPoll) {
|
||||
const observedAt = new Date();
|
||||
this.alertPoll = this.getJson<PrometheusAlertResponse>('/api/v1/alerts')
|
||||
.then((response) => retainAlerts(this.prisma, this.parseAlerts(response), observedAt))
|
||||
.finally(() => {
|
||||
this.alertPoll = undefined;
|
||||
});
|
||||
}
|
||||
return this.alertPoll;
|
||||
}
|
||||
|
||||
clearAlert(fingerprint: string, activeAt: unknown, userId: string) {
|
||||
return clearRetainedAlert(this.prisma, fingerprint, activeAt, userId);
|
||||
}
|
||||
|
||||
async overview(rawRange?: string, userId?: string): Promise<InfrastructureMonitoringOverview> {
|
||||
const range = this.parseRange(rawRange);
|
||||
const collectedAt = new Date().toISOString();
|
||||
@@ -220,11 +252,11 @@ export class InfrastructureMonitoringService {
|
||||
this.loadTrends(range),
|
||||
this.query(QUERIES.services),
|
||||
this.query(SERVICE_METRICS_QUERY),
|
||||
this.getJson<PrometheusAlertResponse>('/api/v1/alerts'),
|
||||
this.loadRetainedAlerts(),
|
||||
]);
|
||||
const services = this.parseServices(serviceResponse);
|
||||
const serviceMetrics = this.parseServiceMetrics(serviceMetricResponse);
|
||||
const alerts = await this.attachReadState(this.parseAlerts(alertResponse), userId);
|
||||
const alerts = await this.attachReadState(alertResponse, userId);
|
||||
const warningAlerts = alerts.filter((item) => item.severity === 'warning').length;
|
||||
const criticalAlerts = alerts.filter((item) => item.severity === 'critical').length;
|
||||
const overallStatus = criticalAlerts > 0 ? 'critical' : warningAlerts > 0 ? 'warning' : 'healthy';
|
||||
@@ -250,7 +282,7 @@ export class InfrastructureMonitoringService {
|
||||
alerts,
|
||||
};
|
||||
} catch (error) {
|
||||
// 页面必须整体清空陈旧指标,但服务端仍需留下不含PromQL/地址/凭据的根因摘要便于运维诊断。
|
||||
// 客户端保留最后成功快照;采集失败不推断告警恢复。
|
||||
this.logger.warn(
|
||||
`Prometheus monitoring overview unavailable: ${error instanceof Error ? error.message : 'unknown error'}`,
|
||||
);
|
||||
@@ -260,10 +292,7 @@ export class InfrastructureMonitoringService {
|
||||
|
||||
async notificationSummary(userId?: string) {
|
||||
try {
|
||||
const alerts = await this.attachReadState(
|
||||
this.parseAlerts(await this.getJson<PrometheusAlertResponse>('/api/v1/alerts')),
|
||||
userId,
|
||||
);
|
||||
const alerts = await this.attachReadState(await this.loadRetainedAlerts(), userId);
|
||||
const unreadAlerts = alerts.filter((item) => !item.acknowledged);
|
||||
return {
|
||||
count: unreadAlerts.length,
|
||||
@@ -281,7 +310,7 @@ export class InfrastructureMonitoringService {
|
||||
if (!/^[a-f0-9]{24}$/.test(fingerprint)) throw new BadRequestException('告警指纹无效');
|
||||
const activeAt = new Date(String(rawActiveAt ?? ''));
|
||||
if (!Number.isFinite(activeAt.getTime())) throw new BadRequestException('告警开始时间无效');
|
||||
const activeAlerts = this.parseAlerts(await this.getJson<PrometheusAlertResponse>('/api/v1/alerts'));
|
||||
const activeAlerts = await retainedAlerts(this.prisma);
|
||||
const current = activeAlerts.find(
|
||||
(item) => item.fingerprint === fingerprint && Date.parse(item.startedAt) === activeAt.getTime(),
|
||||
);
|
||||
|
||||
@@ -0,0 +1,143 @@
|
||||
import { BadRequestException, NotFoundException } from '@nestjs/common';
|
||||
import { Prisma } from '@prisma/client';
|
||||
import { createHash } from 'node:crypto';
|
||||
import { PrismaService } from '../prisma/prisma.service';
|
||||
import type { InfrastructureAlert } from './infrastructure-monitoring.contracts';
|
||||
|
||||
export async function retainAlerts(prisma: PrismaService, alerts: InfrastructureAlert[], observedAt: Date) {
|
||||
await prisma.$transaction(async (tx) => {
|
||||
// Serialize snapshots across API processes; timestamps reject late HTTP results.
|
||||
await tx.$executeRaw`SELECT pg_advisory_xact_lock(160916, 1)`;
|
||||
const previous = await tx.infrastructureAlertCollection.findUnique({ where: { id: 'prometheus' } });
|
||||
if (previous && previous.observedAt >= observedAt) return;
|
||||
// Read durable work directly: application releases do not install Prometheus rules.
|
||||
// Keep one occurrence identity until the condition really recovers, even after manual clear.
|
||||
const reviewCount = await tx.smsAttemptCompletionWork.count({ where: { state: 'needs_review' } });
|
||||
const oldest = await tx.smsCompletionEvent.findFirst({
|
||||
where: { processedAt: null, work: { state: { in: ['pending', 'processing', 'retry_wait'] } } },
|
||||
orderBy: { createdAt: 'asc' },
|
||||
select: { createdAt: true },
|
||||
});
|
||||
const age = oldest ? Math.max(0, (observedAt.getTime() - oldest.createdAt.getTime()) / 1000) : 0;
|
||||
alerts = [...alerts];
|
||||
for (const condition of [
|
||||
{
|
||||
name: 'SmsCompletionNeedsReview',
|
||||
active: reviewCount > 0,
|
||||
severity: 'critical' as const,
|
||||
summary: '短信收尾工作需要人工排查',
|
||||
value: String(reviewCount),
|
||||
threshold: '0',
|
||||
},
|
||||
{
|
||||
name: 'SmsCompletionBacklog',
|
||||
active: age > 300,
|
||||
severity: 'warning' as const,
|
||||
summary: '短信收尾工作等待超过5分钟',
|
||||
value: `${Math.floor(age)}秒`,
|
||||
threshold: '300秒',
|
||||
},
|
||||
]) {
|
||||
if (!condition.active) continue;
|
||||
const fingerprint = createHash('sha256').update(`durable:${condition.name}`).digest('hex').slice(0, 24);
|
||||
const occurrence = await tx.infrastructureAlertEvent.findFirst({
|
||||
where: { fingerprint, recoveredAt: null },
|
||||
orderBy: { activeAt: 'desc' },
|
||||
});
|
||||
alerts.push({
|
||||
fingerprint,
|
||||
name: condition.name,
|
||||
severity: condition.severity,
|
||||
status: 'firing',
|
||||
startedAt: (occurrence?.activeAt ?? observedAt).toISOString(),
|
||||
summary: condition.summary,
|
||||
currentValue: condition.value,
|
||||
threshold: condition.threshold,
|
||||
service: '短信收尾',
|
||||
acknowledged: false,
|
||||
});
|
||||
}
|
||||
await tx.infrastructureAlertCollection.upsert({
|
||||
where: { id: 'prometheus' },
|
||||
create: { id: 'prometheus', observedAt },
|
||||
update: { observedAt },
|
||||
});
|
||||
for (const alert of alerts) {
|
||||
const activeAt = new Date(alert.startedAt);
|
||||
const payload = JSON.parse(JSON.stringify(alert)) as Prisma.InputJsonValue;
|
||||
await tx.infrastructureAlertEvent.createMany({
|
||||
data: [{ fingerprint: alert.fingerprint, activeAt, payload, lastObservedAt: observedAt }],
|
||||
skipDuplicates: true,
|
||||
});
|
||||
await tx.infrastructureAlertEvent.updateMany({
|
||||
where: { fingerprint: alert.fingerprint, activeAt, lastObservedAt: { lte: observedAt } },
|
||||
data: { payload, lastObservedAt: observedAt, recoveredAt: null },
|
||||
});
|
||||
}
|
||||
await tx.infrastructureAlertEvent.updateMany({
|
||||
where: {
|
||||
recoveredAt: null,
|
||||
lastObservedAt: { lt: observedAt },
|
||||
...(alerts.length
|
||||
? {
|
||||
NOT: {
|
||||
OR: alerts.map((alert) => ({ fingerprint: alert.fingerprint, activeAt: new Date(alert.startedAt) })),
|
||||
},
|
||||
}
|
||||
: {}),
|
||||
},
|
||||
data: { recoveredAt: observedAt },
|
||||
});
|
||||
});
|
||||
return retainedAlerts(prisma);
|
||||
}
|
||||
|
||||
export async function retainedAlerts(prisma: PrismaService): Promise<InfrastructureAlert[]> {
|
||||
const records = await prisma.infrastructureAlertEvent.findMany({
|
||||
where: { clearedAt: null },
|
||||
orderBy: [{ activeAt: 'desc' }, { id: 'asc' }],
|
||||
});
|
||||
return records.map((record) => ({
|
||||
...(record.payload as unknown as InfrastructureAlert),
|
||||
...(record.recoveredAt ? { status: 'resolved' } : {}),
|
||||
acknowledged: false,
|
||||
}));
|
||||
}
|
||||
|
||||
export async function clearRetainedAlert(
|
||||
prisma: PrismaService,
|
||||
fingerprint: string,
|
||||
rawActiveAt: unknown,
|
||||
userId: string,
|
||||
) {
|
||||
const activeAt = new Date(String(rawActiveAt ?? ''));
|
||||
if (!/^[a-f0-9]{24}$/.test(fingerprint) || !Number.isFinite(activeAt.getTime()))
|
||||
throw new BadRequestException('告警标识无效');
|
||||
return prisma.$transaction(async (tx) => {
|
||||
const record = await tx.infrastructureAlertEvent.findUnique({
|
||||
where: { fingerprint_activeAt: { fingerprint, activeAt } },
|
||||
});
|
||||
if (!record) throw new NotFoundException('告警记录不存在');
|
||||
const clearedAt = new Date();
|
||||
const result = await tx.infrastructureAlertEvent.updateMany({
|
||||
where: { id: record.id, clearedAt: null },
|
||||
data: { clearedAt, clearedBy: userId },
|
||||
});
|
||||
if (result.count)
|
||||
await tx.operationLog.create({
|
||||
data: {
|
||||
userId,
|
||||
action: 'monitoring.alert_cleared',
|
||||
resource: 'infrastructure_alert',
|
||||
resourceId: record.id,
|
||||
detail: { fingerprint, activeAt: activeAt.toISOString() },
|
||||
},
|
||||
});
|
||||
return {
|
||||
fingerprint,
|
||||
activeAt: activeAt.toISOString(),
|
||||
cleared: true,
|
||||
clearedAt: (record.clearedAt ?? clearedAt).toISOString(),
|
||||
};
|
||||
});
|
||||
}
|
||||
@@ -1,3 +1,4 @@
|
||||
import { renderCompletionMetrics } from '../send-chain/completion-metrics';
|
||||
import { Injectable, OnModuleDestroy } from '@nestjs/common';
|
||||
import { monitorEventLoopDelay } from 'node:perf_hooks';
|
||||
|
||||
@@ -314,6 +315,7 @@ export class MetricsService implements OnModuleDestroy {
|
||||
lines.push(metricLine('cmpp_api_auth_protection_events_total', count, { event, scope }));
|
||||
}
|
||||
this.eventLoopDelay.reset();
|
||||
lines.push(...renderCompletionMetrics());
|
||||
return `${lines.join('\n')}\n`;
|
||||
}
|
||||
|
||||
|
||||
@@ -526,24 +526,28 @@ export class OpenApiService implements OnModuleInit, OnModuleDestroy {
|
||||
return row;
|
||||
}
|
||||
|
||||
async queueWebhookEvent(data: {
|
||||
tenantId: string;
|
||||
applicationId?: string | null;
|
||||
messageRecordId?: string | null;
|
||||
messageId?: string | null;
|
||||
uplinkMessageId?: string | null;
|
||||
eventType: 'receipt' | 'uplink';
|
||||
payload: Record<string, unknown>;
|
||||
}) {
|
||||
async queueWebhookEvent(
|
||||
data: {
|
||||
tenantId: string;
|
||||
applicationId?: string | null;
|
||||
messageRecordId?: string | null;
|
||||
messageId?: string | null;
|
||||
uplinkMessageId?: string | null;
|
||||
eventType: 'receipt' | 'uplink';
|
||||
payload: Record<string, unknown>;
|
||||
},
|
||||
transaction?: Prisma.TransactionClient,
|
||||
) {
|
||||
const db = transaction ?? this.prisma;
|
||||
if (!data.applicationId) return null;
|
||||
const application = await this.prisma.smsApplication.findUnique({
|
||||
const application = await db.smsApplication.findUnique({
|
||||
where: { id: data.applicationId },
|
||||
include: { httpConfig: true },
|
||||
});
|
||||
const config = application?.httpConfig;
|
||||
const enabled = data.eventType === 'receipt' ? config?.receiptWebhookEnabled : config?.uplinkWebhookEnabled;
|
||||
if (!config?.enabled || !enabled) return null;
|
||||
const endpoint = await this.prisma.httpWebhookEndpoint.findUnique({
|
||||
const endpoint = await db.httpWebhookEndpoint.findUnique({
|
||||
where: { applicationId_eventType: { applicationId: data.applicationId, eventType: data.eventType } },
|
||||
});
|
||||
if (!endpoint || endpoint.status !== 'active') return null;
|
||||
@@ -553,7 +557,7 @@ export class OpenApiService implements OnModuleInit, OnModuleDestroy {
|
||||
: data.eventType === 'uplink' && data.uplinkMessageId
|
||||
? `evt_uplink_${data.uplinkMessageId}`
|
||||
: `evt_${randomUUID()}`;
|
||||
const delivery = await this.prisma.$transaction(async (tx) => {
|
||||
const persist = async (tx: Prisma.TransactionClient) => {
|
||||
const event = await tx.httpWebhookEvent.upsert({
|
||||
where: { eventId },
|
||||
update: {},
|
||||
@@ -573,7 +577,9 @@ export class OpenApiService implements OnModuleInit, OnModuleDestroy {
|
||||
update: {},
|
||||
create: { eventId: event.id, endpointId: endpoint.id, recoveryVersion: 1 },
|
||||
});
|
||||
});
|
||||
};
|
||||
const delivery = transaction ? await persist(transaction) : await this.prisma.$transaction(persist);
|
||||
if (transaction) return delivery;
|
||||
if (delivery.status !== 'pending' || delivery.recoveryVersion !== 1) return delivery;
|
||||
await this.queue?.add(
|
||||
'deliver',
|
||||
|
||||
@@ -1,58 +1,85 @@
|
||||
import { BadRequestException, ConflictException, Injectable, NotFoundException } from '@nestjs/common';
|
||||
import { Prisma } from '@prisma/client';
|
||||
import ExcelJS from 'exceljs';
|
||||
import { createHash, randomUUID } from 'node:crypto';
|
||||
import { extname } from 'node:path';
|
||||
import { FilesService } from '../files/files.service';
|
||||
import { PrismaService } from '../prisma/prisma.service';
|
||||
import { SmsConfigService } from '../sms-config/sms-config.service';
|
||||
import type { AnalyzeImportOptions, CreateImportProfileDto, CreateReportBatchDto, EmbeddedImage, ImportCommitDto, ImportMapping, PagedQuery, ReportBatchInspection, ReportBatchTarget, ReviewImportItemsDto } from './report-materials.contracts';
|
||||
import { profileData, validateProfile, loadWorkbook, assertSafeWorkbook, safeSpreadsheetText, readEmbeddedImages, suggestMappings, remapProfileColumns, signatureCoreMapping, drainageCoreMapping, normalizeHeader, normalizeFieldCode, clamp, normalizePage, normalizePageSize, dateRange, cellText, transformValue, mappedCoreValue, dynamicValues, jsonRecord, hasValue, isFileRef, resolveExportValue, applyExportTransform, styleHeader, normalizeImageExtension, imageContentType, safeFileName, normalizeBatchIdempotencyKey, jsonStringArray, jsonSafe } from './report-materials.helpers';
|
||||
import { ReportPendingQueryService } from './pending-query.service';
|
||||
import { safeSpreadsheetText, styleHeader } from './report-materials.helpers';
|
||||
|
||||
/** R4 report-materials domain service composed behind ReportMaterialsService. */
|
||||
export class ReportOfficialExportService {
|
||||
constructor(private readonly prisma: PrismaService, private readonly files: FilesService, private readonly smsConfig: SmsConfigService, private readonly pending: ReportPendingQueryService) {}
|
||||
constructor(
|
||||
private readonly prisma: PrismaService,
|
||||
private readonly files: FilesService,
|
||||
private readonly smsConfig: SmsConfigService,
|
||||
private readonly pending: ReportPendingQueryService,
|
||||
) {}
|
||||
|
||||
async buildOfficialTemplate(reportType: 'signature' | 'drainage', operatorId?: string) {
|
||||
const workbook = new ExcelJS.Workbook();
|
||||
workbook.creator = '聆界短信平台';
|
||||
const sheet = workbook.addWorksheet(reportType === 'signature' ? '签名资料' : '引流信息', { views: [{ state: 'frozen', ySplit: 1 }] });
|
||||
const headers = reportType === 'signature'
|
||||
const workbook = new ExcelJS.Workbook();
|
||||
workbook.creator = '聆界短信平台';
|
||||
const sheet = workbook.addWorksheet(reportType === 'signature' ? '签名资料' : '引流信息', {
|
||||
views: [{ state: 'frozen', ySplit: 1 }],
|
||||
});
|
||||
const headers =
|
||||
reportType === 'signature'
|
||||
? ['短信签名', '用途说明', '营业执照图片', '授权书图片', '备注']
|
||||
: ['短信签名', '引流 URL 或号码', '备注', '主体证明'];
|
||||
sheet.addRow(headers);
|
||||
sheet.addRow(reportType === 'signature'
|
||||
sheet.addRow(headers);
|
||||
sheet.addRow(
|
||||
reportType === 'signature'
|
||||
? ['示例签名', '验证码通知', '请在本单元格插入图片', '请在本单元格插入图片', '示例行,导入前请删除']
|
||||
: ['示例签名', 'example.com/path 或 13800138000', '示例行,导入前请删除', '请在本单元格插入图片']);
|
||||
styleHeader(sheet.getRow(1));
|
||||
sheet.columns.forEach((column) => { column.width = 24; });
|
||||
sheet.getRow(2).height = 48;
|
||||
const content = Buffer.from(await workbook.xlsx.writeBuffer());
|
||||
const fileName = `${reportType === 'signature' ? '签名' : '引流信息'}报备资料官方模板.xlsx`;
|
||||
await this.prisma.operationLog.create({ data: {
|
||||
userId: operatorId, action: 'report_material.template_downloaded', resource: 'report_material',
|
||||
: ['示例签名', 'example.com/path 或 13800138000', '示例行,导入前请删除', '请在本单元格插入图片'],
|
||||
);
|
||||
styleHeader(sheet.getRow(1));
|
||||
sheet.columns.forEach((column) => {
|
||||
column.width = 24;
|
||||
});
|
||||
sheet.getRow(2).height = 48;
|
||||
const content = Buffer.from(await workbook.xlsx.writeBuffer());
|
||||
const fileName = `${reportType === 'signature' ? '签名' : '引流信息'}报备资料官方模板.xlsx`;
|
||||
await this.prisma.operationLog.create({
|
||||
data: {
|
||||
userId: operatorId,
|
||||
action: 'report_material.template_downloaded',
|
||||
resource: 'report_material',
|
||||
detail: { fileName, filters: { reportType }, successCount: 1, failedCount: 0 } as Prisma.InputJsonValue,
|
||||
} });
|
||||
return { fileName, content };
|
||||
}
|
||||
},
|
||||
});
|
||||
return { fileName, content };
|
||||
}
|
||||
|
||||
async exportPending(query: { reportType?: 'signature' | 'drainage'; tenantId?: string; applicationId?: string }, operatorId?: string) {
|
||||
const items = await this.pending.findPendingItems(query);
|
||||
const workbook = new ExcelJS.Workbook();
|
||||
const sheet = workbook.addWorksheet('待报备资料', { views: [{ state: 'frozen', ySplit: 1 }] });
|
||||
sheet.addRow(['资料类型', '企业', '企业应用', '签名/站点', '详情', '变更时间']);
|
||||
styleHeader(sheet.getRow(1));
|
||||
for (const item of items) sheet.addRow([
|
||||
item.reportType === 'signature' ? '签名' : '引流信息', safeSpreadsheetText(item.tenant?.name),
|
||||
safeSpreadsheetText(item.application?.name), safeSpreadsheetText(item.name), safeSpreadsheetText(item.detail), item.changedAt,
|
||||
async exportPending(
|
||||
query: { reportType?: 'signature' | 'drainage'; tenantId?: string; applicationId?: string },
|
||||
operatorId?: string,
|
||||
) {
|
||||
const items = await this.pending.findPendingItems(query);
|
||||
const workbook = new ExcelJS.Workbook();
|
||||
const sheet = workbook.addWorksheet('待报备资料', { views: [{ state: 'frozen', ySplit: 1 }] });
|
||||
sheet.addRow(['资料类型', '企业', '企业应用', '签名/站点', '详情', '变更时间']);
|
||||
styleHeader(sheet.getRow(1));
|
||||
for (const item of items)
|
||||
sheet.addRow([
|
||||
item.reportType === 'signature' ? '签名' : '引流信息',
|
||||
safeSpreadsheetText(item.tenant?.name),
|
||||
safeSpreadsheetText(item.application?.name),
|
||||
safeSpreadsheetText(item.name),
|
||||
safeSpreadsheetText(item.detail),
|
||||
item.changedAt,
|
||||
]);
|
||||
sheet.columns.forEach((column, index) => { column.width = index === 4 ? 42 : 22; });
|
||||
const fileName = `待报备资料-${new Date().toISOString().slice(0, 10)}.xlsx`;
|
||||
await this.prisma.operationLog.create({ data: {
|
||||
tenantId: query.tenantId, userId: operatorId, action: 'report_material.pending_export', resource: 'report_material',
|
||||
sheet.columns.forEach((column, index) => {
|
||||
column.width = index === 4 ? 42 : 22;
|
||||
});
|
||||
const fileName = `待报备资料-${new Date().toISOString().slice(0, 10)}.xlsx`;
|
||||
await this.prisma.operationLog.create({
|
||||
data: {
|
||||
tenantId: query.tenantId,
|
||||
userId: operatorId,
|
||||
action: 'report_material.pending_export',
|
||||
resource: 'report_material',
|
||||
detail: { fileName, filters: query, successCount: items.length, failedCount: 0 } as Prisma.InputJsonValue,
|
||||
} });
|
||||
return { fileName, content: Buffer.from(await workbook.xlsx.writeBuffer()) };
|
||||
}
|
||||
},
|
||||
});
|
||||
return { fileName, content: Buffer.from(await workbook.xlsx.writeBuffer()) };
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,219 @@
|
||||
import { HttpException, Logger } from '@nestjs/common';
|
||||
import { Prisma, SmsAttemptCompletionWork } from '@prisma/client';
|
||||
import { createHash, randomUUID } from 'node:crypto';
|
||||
import { PrismaService } from '../prisma/prisma.service';
|
||||
import { completionContext, CompletionRouteRequired } from './completion-context';
|
||||
import type { RoutedChannel } from './send-chain.contracts';
|
||||
import { countCompletion, observeCompletion } from './completion-metrics';
|
||||
|
||||
export type CompletionEventKind = 'receipt' | 'submit' | 'segment' | 'timeout' | 'rejection';
|
||||
export class AttemptCompletion {
|
||||
private readonly logger = new Logger(AttemptCompletion.name);
|
||||
private timer?: ReturnType<typeof setInterval>;
|
||||
private running = false;
|
||||
constructor(
|
||||
private readonly prisma: PrismaService,
|
||||
private readonly execute: (kind: CompletionEventKind, payload: Prisma.JsonValue) => Promise<unknown>,
|
||||
private readonly waitForRoute: (route: RoutedChannel) => Promise<void>,
|
||||
) {}
|
||||
|
||||
start() {
|
||||
this.timer = setInterval(() => void this.scan(), 5_000);
|
||||
this.timer.unref();
|
||||
void this.scan();
|
||||
}
|
||||
stop() {
|
||||
if (this.timer) clearInterval(this.timer);
|
||||
}
|
||||
|
||||
async enqueue(
|
||||
messageRecordId: string,
|
||||
sourceSubmitRecordId: string | undefined,
|
||||
kind: CompletionEventKind,
|
||||
payload: unknown,
|
||||
identity?: string,
|
||||
) {
|
||||
const json = JSON.parse(JSON.stringify(payload)) as Prisma.InputJsonValue;
|
||||
const workKey = sourceSubmitRecordId ? `attempt:${sourceSubmitRecordId}` : `message:${messageRecordId}`;
|
||||
const eventKey = createHash('sha256')
|
||||
.update(`${workKey}:${kind}:${identity ?? JSON.stringify(json)}`)
|
||||
.digest('hex');
|
||||
const work = await this.prisma.$transaction(async (tx) => {
|
||||
const message = await tx.smsMessageRecord.findUniqueOrThrow({
|
||||
where: { id: messageRecordId },
|
||||
select: { tenantId: true },
|
||||
});
|
||||
if (sourceSubmitRecordId) {
|
||||
const source = await tx.smsSubmitRecord.findUniqueOrThrow({ where: { id: sourceSubmitRecordId } });
|
||||
if (source.messageRecordId !== messageRecordId || source.tenantId !== message.tenantId)
|
||||
throw new Error('completion_source_mismatch');
|
||||
}
|
||||
await tx.smsAttemptCompletionWork.createMany({
|
||||
data: [{ workKey, messageRecordId, sourceSubmitRecordId, tenantId: message.tenantId }],
|
||||
skipDuplicates: true,
|
||||
});
|
||||
const current = await tx.smsAttemptCompletionWork.findUniqueOrThrow({ where: { workKey } });
|
||||
if (current.messageRecordId !== messageRecordId) throw new Error('completion_work_mismatch');
|
||||
await tx.$queryRaw`SELECT id FROM "SmsAttemptCompletionWork" WHERE id=${current.id} FOR UPDATE`;
|
||||
const inserted = await tx.smsCompletionEvent.createMany({
|
||||
data: [{ workId: current.id, eventKey, kind, payload: json }],
|
||||
skipDuplicates: true,
|
||||
});
|
||||
if (inserted.count)
|
||||
await tx.$executeRaw`
|
||||
UPDATE "SmsAttemptCompletionWork" SET revision=revision+1,
|
||||
state=CASE WHEN state IN ('processing', 'needs_review') THEN state ELSE 'pending' END,
|
||||
"nextAttemptAt"=(CURRENT_TIMESTAMP AT TIME ZONE 'UTC'), "updatedAt"=(CURRENT_TIMESTAMP AT TIME ZONE 'UTC') WHERE id=${current.id}`;
|
||||
return current;
|
||||
});
|
||||
await this.process(work.id);
|
||||
return this.prisma.smsMessageRecord.findUnique({ where: { id: messageRecordId } });
|
||||
}
|
||||
|
||||
async scan() {
|
||||
if (this.running) return;
|
||||
this.running = true;
|
||||
try {
|
||||
observeCompletion(
|
||||
await this.prisma.$queryRaw<Array<{ state: string; count: number; age: number }>>`
|
||||
SELECT w.state, COUNT(*)::int AS count,
|
||||
EXTRACT(EPOCH FROM (CURRENT_TIMESTAMP AT TIME ZONE 'UTC') - MIN(COALESCE(e."createdAt", w."updatedAt")))::float AS age
|
||||
FROM "SmsAttemptCompletionWork" w
|
||||
LEFT JOIN LATERAL (SELECT MIN("createdAt") AS "createdAt" FROM "SmsCompletionEvent" WHERE "workId"=w.id AND "processedAt" IS NULL) e ON true
|
||||
WHERE w.state IN ('pending','processing','retry_wait','needs_review') GROUP BY w.state`,
|
||||
);
|
||||
const rows = await this.prisma.$queryRaw<Array<{ id: string }>>`
|
||||
SELECT id FROM "SmsAttemptCompletionWork"
|
||||
WHERE (state IN ('pending','retry_wait') AND "nextAttemptAt" <= (CURRENT_TIMESTAMP AT TIME ZONE 'UTC'))
|
||||
OR (state='processing' AND "leaseUntil" < (CURRENT_TIMESTAMP AT TIME ZONE 'UTC'))
|
||||
ORDER BY "nextAttemptAt", id LIMIT 32`;
|
||||
for (const row of rows) await this.process(row.id);
|
||||
} catch {
|
||||
this.logger.error('completion_scan_failed');
|
||||
} finally {
|
||||
this.running = false;
|
||||
}
|
||||
}
|
||||
|
||||
async process(id: string) {
|
||||
const owner = randomUUID();
|
||||
const claims = await this.prisma.$queryRaw<SmsAttemptCompletionWork[]>`
|
||||
UPDATE "SmsAttemptCompletionWork" SET state='processing', "leaseOwner"=${owner},
|
||||
"leaseUntil"=(CURRENT_TIMESTAMP AT TIME ZONE 'UTC')+INTERVAL '60 seconds', "fenceVersion"="fenceVersion"+1,
|
||||
attempts=attempts+1, "updatedAt"=(CURRENT_TIMESTAMP AT TIME ZONE 'UTC')
|
||||
WHERE id=${id} AND ((state IN ('pending','retry_wait') AND "nextAttemptAt" <= (CURRENT_TIMESTAMP AT TIME ZONE 'UTC'))
|
||||
OR (state='processing' AND "leaseUntil" < (CURRENT_TIMESTAMP AT TIME ZONE 'UTC'))) RETURNING *`;
|
||||
const claim = claims[0];
|
||||
if (!claim) {
|
||||
countCompletion('not_claimed');
|
||||
return;
|
||||
}
|
||||
countCompletion('claimed');
|
||||
if (claim.attempts > 1) countCompletion('recovered');
|
||||
const renew = setInterval(() => {
|
||||
void this.prisma
|
||||
.$executeRaw`UPDATE "SmsAttemptCompletionWork" SET "leaseUntil"=(CURRENT_TIMESTAMP AT TIME ZONE 'UTC')+INTERVAL '60 seconds'
|
||||
WHERE id=${id} AND state='processing' AND "leaseOwner"=${owner} AND "fenceVersion"=${claim.fenceVersion}
|
||||
AND "leaseUntil">(CURRENT_TIMESTAMP AT TIME ZONE 'UTC')`.catch(() =>
|
||||
this.logger.warn('completion_lease_renew_failed'),
|
||||
);
|
||||
}, 20_000);
|
||||
renew.unref();
|
||||
let route: RoutedChannel | undefined;
|
||||
let routePlanned = false;
|
||||
let planRevision: number | undefined;
|
||||
try {
|
||||
for (let pass = 0; pass < 64; pass++) {
|
||||
try {
|
||||
const done = await this.prisma.$transaction(
|
||||
async (tx) => {
|
||||
const rows = await tx.$queryRaw<SmsAttemptCompletionWork[]>`
|
||||
SELECT * FROM "SmsAttemptCompletionWork" WHERE id=${id} AND state='processing'
|
||||
AND "leaseOwner"=${owner} AND "fenceVersion"=${claim.fenceVersion} AND "leaseUntil">(CURRENT_TIMESTAMP AT TIME ZONE 'UTC') FOR UPDATE`;
|
||||
const work = rows[0];
|
||||
if (!work) {
|
||||
countCompletion('fence_rejected');
|
||||
throw new Error('completion_fence_rejected');
|
||||
}
|
||||
await tx.$queryRaw`SELECT id FROM "SmsMessageRecord" WHERE id=${work.messageRecordId} FOR UPDATE`;
|
||||
if (planRevision !== work.revision) {
|
||||
route = undefined;
|
||||
routePlanned = false;
|
||||
}
|
||||
planRevision = work.revision;
|
||||
const event = await tx.smsCompletionEvent.findFirst({
|
||||
where: { workId: id, processedAt: null },
|
||||
orderBy: [{ createdAt: 'asc' }, { id: 'asc' }],
|
||||
});
|
||||
if (event) {
|
||||
await completionContext.run({ tx, messageRecordId: work.messageRecordId, route, routePlanned }, () =>
|
||||
this.execute(event.kind as CompletionEventKind, event.payload),
|
||||
);
|
||||
await tx.smsCompletionEvent.update({ where: { id: event.id }, data: { processedAt: new Date() } });
|
||||
}
|
||||
const remaining = await tx.smsCompletionEvent.count({ where: { workId: id, processedAt: null } });
|
||||
const message = await tx.smsMessageRecord.findUniqueOrThrow({ where: { id: work.messageRecordId } });
|
||||
const retry = work.sourceSubmitRecordId
|
||||
? await tx.smsSubmitRecord.findUnique({
|
||||
where: { retryOfSubmitRecordId: work.sourceSubmitRecordId },
|
||||
select: { id: true },
|
||||
})
|
||||
: null;
|
||||
await tx.smsAttemptCompletionWork.update({
|
||||
where: { id },
|
||||
data: {
|
||||
processedRevision: work.revision - remaining,
|
||||
decision: message.status,
|
||||
retrySubmitRecordId: retry?.id,
|
||||
state: remaining ? 'processing' : 'idle',
|
||||
lastError: null,
|
||||
...(!remaining ? { leaseOwner: null, leaseUntil: null, attempts: 0 } : {}),
|
||||
},
|
||||
});
|
||||
return remaining === 0;
|
||||
},
|
||||
{ timeout: 20_000, maxWait: 5_000 },
|
||||
);
|
||||
route = undefined;
|
||||
routePlanned = false;
|
||||
countCompletion('event_committed');
|
||||
if (done) return;
|
||||
} catch (error) {
|
||||
if (!(error instanceof CompletionRouteRequired)) throw error;
|
||||
// The probe transaction rolls back. Routing and rate limiting occur without locks.
|
||||
try {
|
||||
route = await error.select();
|
||||
await this.waitForRoute(route);
|
||||
} catch (selectionError) {
|
||||
if (!(selectionError instanceof HttpException) || selectionError.getStatus() >= 500) throw selectionError;
|
||||
route = undefined;
|
||||
}
|
||||
routePlanned = true;
|
||||
}
|
||||
}
|
||||
throw new Error('completion_batch_budget_exhausted');
|
||||
} catch (error) {
|
||||
const code =
|
||||
error instanceof Prisma.PrismaClientKnownRequestError
|
||||
? error.code
|
||||
: error instanceof Error && error.message.startsWith('completion_')
|
||||
? error.message
|
||||
: 'completion_processing_failed';
|
||||
const exhausted = claim.attempts >= 12;
|
||||
countCompletion(exhausted ? 'needs_review' : 'retry_wait');
|
||||
await this.prisma.smsAttemptCompletionWork.updateMany({
|
||||
where: { id, leaseOwner: owner, fenceVersion: claim.fenceVersion, state: 'processing' },
|
||||
data: {
|
||||
state: exhausted ? 'needs_review' : 'retry_wait',
|
||||
leaseOwner: null,
|
||||
leaseUntil: null,
|
||||
lastError: code,
|
||||
nextAttemptAt: new Date(Date.now() + Math.min(300_000, 1000 * 2 ** Math.min(claim.attempts, 8))),
|
||||
},
|
||||
});
|
||||
this.logger.error(`${exhausted ? 'completion_needs_review' : 'completion_retry_wait'}:${code}`);
|
||||
} finally {
|
||||
clearInterval(renew);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
import { AsyncLocalStorage } from 'node:async_hooks';
|
||||
import { Prisma } from '@prisma/client';
|
||||
import { PrismaService } from '../prisma/prisma.service';
|
||||
import type { RoutedChannel } from './send-chain.contracts';
|
||||
|
||||
export type CompletionContext = {
|
||||
tx: Prisma.TransactionClient;
|
||||
messageRecordId: string;
|
||||
route?: RoutedChannel;
|
||||
routePlanned?: boolean;
|
||||
};
|
||||
export const completionContext = new AsyncLocalStorage<CompletionContext>();
|
||||
|
||||
// Only send-chain collaborators use this adapter. Nested billing/outbox transactions
|
||||
// join the explicitly established completion transaction, never start a second one.
|
||||
export function completionDatabase(prisma: PrismaService): PrismaService {
|
||||
return new Proxy(prisma, {
|
||||
get(target, key) {
|
||||
const tx = completionContext.getStore()?.tx;
|
||||
if (tx && key === '$transaction') {
|
||||
return (operation: ((client: Prisma.TransactionClient) => unknown) | Promise<unknown>[]) =>
|
||||
typeof operation === 'function' ? operation(tx) : Promise.all(operation);
|
||||
}
|
||||
const owner = tx && key in tx ? tx : target;
|
||||
const value = Reflect.get(owner, key);
|
||||
return typeof value === 'function' ? value.bind(owner) : value;
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
export class CompletionRouteRequired extends Error {
|
||||
constructor(readonly select: () => Promise<RoutedChannel>) {
|
||||
super('completion_route_required');
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
type CompletionMetric =
|
||||
'claimed' | 'not_claimed' | 'recovered' | 'fence_rejected' | 'event_committed' | 'retry_wait' | 'needs_review';
|
||||
const counts = new Map<CompletionMetric, number>();
|
||||
let snapshot: Array<{ state: string; count: number; age: number }> = [];
|
||||
const states = ['pending', 'processing', 'retry_wait', 'needs_review'];
|
||||
export function countCompletion(event: CompletionMetric) {
|
||||
counts.set(event, (counts.get(event) ?? 0) + 1);
|
||||
}
|
||||
export function observeCompletion(rows: typeof snapshot) {
|
||||
snapshot = rows;
|
||||
}
|
||||
export function renderCompletionMetrics() {
|
||||
return [
|
||||
'# TYPE cmpp_completion_events_total counter',
|
||||
...[...counts].map(([event, value]) => `cmpp_completion_events_total{event="${event}"} ${value}`),
|
||||
'# TYPE cmpp_completion_work gauge',
|
||||
'# TYPE cmpp_completion_oldest_seconds gauge',
|
||||
...states.flatMap((state) => {
|
||||
const row = snapshot.find((item) => item.state === state);
|
||||
return [
|
||||
`cmpp_completion_work{state="${state}"} ${Number(row?.count ?? 0)}`,
|
||||
`cmpp_completion_oldest_seconds{state="${state}"} ${Math.max(0, Number(row?.age ?? 0))}`,
|
||||
];
|
||||
}),
|
||||
];
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
import { createHash } from 'node:crypto';
|
||||
import { DrainageSubmitGuardController } from './drainage-submit-guard.controller';
|
||||
|
||||
describe('channel test final guard', () => {
|
||||
const content = '无签名正文 https://example.test 4001234567'.repeat(5);
|
||||
function setup(overrides = {}) {
|
||||
const tx = {
|
||||
smsSubmitRecord: {
|
||||
findUnique: jest.fn().mockResolvedValue({
|
||||
channelId: 'channel',
|
||||
submitId: 'submit',
|
||||
resultProcessedAt: null,
|
||||
messageRecord: { content, tenantId: null, batchTaskId: null, status: 'submit_queued', ...overrides },
|
||||
}),
|
||||
},
|
||||
drainageDetectionRule: {
|
||||
findMany: jest.fn(() => {
|
||||
throw new Error('must not detect channel tests');
|
||||
}),
|
||||
},
|
||||
};
|
||||
const db = { $transaction: (operation: (value: unknown) => unknown) => operation(tx) };
|
||||
return { controller: new DrainageSubmitGuardController(db as never), tx };
|
||||
}
|
||||
const local = { socket: { remoteAddress: '127.0.0.1' } };
|
||||
const input = {
|
||||
submitId: 'submit',
|
||||
channelId: 'channel',
|
||||
contentHash: createHash('sha256').update(content).digest('hex'),
|
||||
};
|
||||
it('allows unsigned long channel tests containing drainage information without detecting content', async () => {
|
||||
const { controller, tx } = setup();
|
||||
await expect(controller.authorize(local, input)).resolves.toEqual({ allowed: true });
|
||||
expect(tx.drainageDetectionRule.findMany).not.toHaveBeenCalled();
|
||||
});
|
||||
it('still rejects a modified body or a finished submit', async () => {
|
||||
await expect(setup().controller.authorize(local, { ...input, contentHash: 'a'.repeat(64) })).resolves.toMatchObject(
|
||||
{ allowed: false },
|
||||
);
|
||||
await expect(setup({ status: 'delivered' }).controller.authorize(local, input)).resolves.toMatchObject({
|
||||
allowed: false,
|
||||
});
|
||||
});
|
||||
it('does not grant the exemption to a customer message without its required signature', async () => {
|
||||
await expect(
|
||||
setup({ tenantId: 'tenant', applicationId: 'app', batchTaskId: 'task' }).controller.authorize(local, input),
|
||||
).resolves.toMatchObject({ allowed: false });
|
||||
});
|
||||
it('rejects external callers before looking up a submit', async () => {
|
||||
const { controller, tx } = setup();
|
||||
await expect(controller.authorize({ socket: { remoteAddress: '10.0.0.2' } }, input)).rejects.toThrow();
|
||||
expect(tx.smsSubmitRecord.findUnique).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
@@ -58,19 +58,9 @@ export class DrainageSubmitGuardController {
|
||||
)
|
||||
return { allowed: false, code: 'DRN', reason: '提交或消息已终结,不得重复发送' };
|
||||
if (!message.tenantId && !message.batchTaskId) {
|
||||
try {
|
||||
await evaluateMessageDrainage(
|
||||
tx as unknown as PrismaService,
|
||||
message,
|
||||
message.carrier ?? undefined,
|
||||
undefined,
|
||||
true,
|
||||
);
|
||||
return { allowed: true };
|
||||
} catch (error) {
|
||||
if (!(error instanceof DrainageRejection)) throw error;
|
||||
return { allowed: false, code: 'DRN', reason: error.message };
|
||||
}
|
||||
// Operations channel tests bypass signature/drainage policy only after
|
||||
// verifying the durable submit, channel, content and unfinished state.
|
||||
return { allowed: true };
|
||||
}
|
||||
if (!message.tenantId || !message.applicationId || !message.signatureId)
|
||||
return { allowed: false, code: 'DRN', reason: '提交消息未关联企业应用和签名' };
|
||||
|
||||
@@ -1,16 +1,10 @@
|
||||
import { BadRequestException, HttpException, HttpStatus, Logger, NotFoundException } from '@nestjs/common';
|
||||
import { Prisma } from '@prisma/client';
|
||||
import { createHash } from 'node:crypto';
|
||||
import { Logger } from '@nestjs/common';
|
||||
import { BillingService } from '../billing/billing.service';
|
||||
import { moneyToNumber } from '../common/money';
|
||||
import type { OpenApiService } from '../open-api/open-api.service';
|
||||
import { PrismaService } from '../prisma/prisma.service';
|
||||
import type { GatewaySubmitResultDto, GatewaySubmitSegmentResultDto, GatewayReceiptEventDto, GatewayUplinkEventDto, GatewayPendingDeliveryQueryDto, GatewayDownstreamSentDto, GatewayDownstreamAcknowledgedDto, GatewayDownstreamFailureType, GatewayControlDeliveryResult, GatewaySubmitDeadLetterDto, RequeueGatewaySubmitExceptionDto, GatewayDownstreamRecoveryStatusDto, TimeoutUnknownDto } from './send-chain.contracts';
|
||||
import { positiveInteger, normalizeReceiptStatus, normalizeCarrier, DEFAULT_DOWNSTREAM_RETRY_DELAY_MS, DEFAULT_DOWNSTREAM_RETRY_MAX_DELAY_MS, DEFAULT_DOWNSTREAM_MAX_RETRIES, DEFAULT_DOWNSTREAM_PENDING_TIMEOUT_HOURS, DEFAULT_RECEIPT_TIMEOUT_HOURS, DEFAULT_GATEWAY_SUBMIT_REQUEUE_STALE_MS, DEFAULT_DOWNSTREAM_MANUAL_REQUEUE_STALE_MS, DEFAULT_UPSTREAM_RECEIPT_INBOX_PROCESSING_STALE_MS, DEFAULT_UPSTREAM_RECEIPT_INBOX_MAX_ATTEMPTS, DEFAULT_UPSTREAM_RECEIPT_INBOX_MAX_AGE_HOURS, gatewaySubmitRequeueKey, isObjectRecord, asDateOrNull, downstreamRetryDelayMs, downstreamAckTimeoutMs, downstreamRetryBaseDelayMs, downstreamRetryMaxDelayMs, downstreamMaxRetries, downstreamPendingTimeoutHours, downstreamControlFailureMessage, normalizeSubmitStatus, downstreamDeliveryAttemptKey, hasRecoveryAuditStateChanged, normalizeRecoveryFailureCategory, aggregateReceiptSegmentState, isSameUpstreamEndpointIdentity, receiptEventKey } from './send-chain.helpers';
|
||||
import type { SendSubmissionService } from './send-submission.service';
|
||||
import type { SendCompletionCallbacks, SendCompletionFacade } from './send-completion.service';
|
||||
|
||||
|
||||
/**
|
||||
* R10 accounting implementation.
|
||||
* Cross-domain calls return through the stable SendChainService seam to preserve locking and test observability.
|
||||
@@ -41,16 +35,19 @@ export class SendAccountingService {
|
||||
const unitPrice = moneyToNumber(message.unitPrice);
|
||||
const billingUnits = message.billingUnits ?? 0;
|
||||
const exists = await this.prisma.smsBillingRecord.findFirst({ where: { messageId: message.messageId } });
|
||||
if (exists?.billingStatus === 'charged') {
|
||||
if (exists && ['charged', 'refunded'].includes(exists.billingStatus)) {
|
||||
return;
|
||||
}
|
||||
const transaction = amountCents > 0 ? await this.billing.settleFrozenCharge({
|
||||
tenantId: message.tenantId,
|
||||
amountCents,
|
||||
taskId: message.batchTaskId,
|
||||
messageId: message.messageId,
|
||||
remark: `短信 ${message.messageId} 提交成功释放冻结并转扣费`,
|
||||
}) : null;
|
||||
const transaction =
|
||||
amountCents > 0
|
||||
? await this.billing.settleFrozenCharge({
|
||||
tenantId: message.tenantId,
|
||||
amountCents,
|
||||
taskId: message.batchTaskId,
|
||||
messageId: message.messageId,
|
||||
remark: `短信 ${message.messageId} 提交成功释放冻结并转扣费`,
|
||||
})
|
||||
: null;
|
||||
const data = {
|
||||
tenantId: message.tenantId,
|
||||
applicationId: message.applicationId ?? undefined,
|
||||
@@ -72,14 +69,22 @@ export class SendAccountingService {
|
||||
}
|
||||
|
||||
async releaseMessageReservation(
|
||||
message: { tenantId: string; batchTaskId: string; messageId: string; amountCents: number | bigint; billingUnits: number },
|
||||
message: {
|
||||
tenantId: string;
|
||||
batchTaskId: string;
|
||||
messageId: string;
|
||||
amountCents: number | bigint;
|
||||
billingUnits: number;
|
||||
},
|
||||
remark: string,
|
||||
) {
|
||||
const amountCents = moneyToNumber(message.amountCents);
|
||||
if (amountCents <= 0) {
|
||||
return;
|
||||
}
|
||||
const charged = await this.prisma.smsBillingRecord.findFirst({ where: { messageId: message.messageId, billingStatus: 'charged' } });
|
||||
const charged = await this.prisma.smsBillingRecord.findFirst({
|
||||
where: { messageId: message.messageId, billingStatus: 'charged' },
|
||||
});
|
||||
if (charged) {
|
||||
return;
|
||||
}
|
||||
@@ -107,11 +112,15 @@ export class SendAccountingService {
|
||||
if (amountCents <= 0) {
|
||||
return;
|
||||
}
|
||||
const refunded = await this.prisma.smsBillingRecord.findFirst({ where: { messageId: message.messageId, billingStatus: 'refunded' } });
|
||||
const refunded = await this.prisma.smsBillingRecord.findFirst({
|
||||
where: { messageId: message.messageId, billingStatus: 'refunded' },
|
||||
});
|
||||
if (refunded) {
|
||||
return;
|
||||
}
|
||||
const charged = await this.prisma.smsBillingRecord.findFirst({ where: { messageId: message.messageId, billingStatus: 'charged' } });
|
||||
const charged = await this.prisma.smsBillingRecord.findFirst({
|
||||
where: { messageId: message.messageId, billingStatus: 'charged' },
|
||||
});
|
||||
if (!charged) {
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -1694,6 +1694,7 @@ describe('SendChainService', () => {
|
||||
applicationId: 'app-1',
|
||||
eventType: 'receipt',
|
||||
}),
|
||||
undefined,
|
||||
);
|
||||
expect(prisma.cmppDownstreamDelivery.create).not.toHaveBeenCalled();
|
||||
});
|
||||
@@ -3257,7 +3258,7 @@ describe('SendChainService', () => {
|
||||
data: expect.objectContaining({ sequenceId: 7, gatewayMessageId: 'GW-1', submitStatus: 'accepted' }),
|
||||
});
|
||||
expect(prisma.smsMessageRecord.updateMany).toHaveBeenCalledWith({
|
||||
where: { id: 'record-1', status: { notIn: ['delivered', 'failed', 'unknown'] } },
|
||||
where: { id: 'record-1', status: { notIn: ['delivered', 'failed', 'unknown', 'timeout'] } },
|
||||
data: expect.objectContaining({ gatewayMessageId: 'GW-1', status: 'submitted', submitStatus: 'accepted' }),
|
||||
});
|
||||
expect(billing.settleFrozenCharge).toHaveBeenCalledWith(
|
||||
@@ -4891,7 +4892,7 @@ describe('SendChainService', () => {
|
||||
expect(prisma.smsMessageRecord.updateMany).toHaveBeenNthCalledWith(
|
||||
1,
|
||||
expect.objectContaining({
|
||||
where: { id: 'record-1', status: { notIn: ['delivered', 'failed', 'unknown'] } },
|
||||
where: { id: 'record-1', status: { notIn: ['delivered', 'failed', 'unknown', 'timeout'] } },
|
||||
data: expect.objectContaining({ status: 'submitted' }),
|
||||
}),
|
||||
);
|
||||
@@ -5567,6 +5568,7 @@ describe('SendChainService', () => {
|
||||
errorCode: 'RECEIPT_TIMEOUT',
|
||||
}),
|
||||
}),
|
||||
undefined,
|
||||
);
|
||||
expect(prisma.cmppDownstreamDelivery.create).not.toHaveBeenCalled();
|
||||
expect(prisma.smsMessageRecord.updateMany).toHaveBeenCalledWith({
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
import { AttemptCompletion } from './attempt-completion';
|
||||
import { completionContext, completionDatabase, CompletionRouteRequired } from './completion-context';
|
||||
import {
|
||||
BadRequestException,
|
||||
forwardRef,
|
||||
@@ -84,6 +86,7 @@ export class SendChainService implements OnModuleInit, OnModuleDestroy {
|
||||
private readonly submission: SendSubmissionService;
|
||||
private readonly completion: SendCompletionService;
|
||||
private readonly downstreamRequeueTasks: SendDownstreamRequeueTaskService;
|
||||
private readonly attemptCompletion?: AttemptCompletion;
|
||||
|
||||
constructor(
|
||||
private readonly prisma: PrismaService,
|
||||
@@ -94,6 +97,11 @@ export class SendChainService implements OnModuleInit, OnModuleDestroy {
|
||||
@Optional() phoneRouting?: PhoneRoutingLookupService,
|
||||
@Optional() metrics?: MetricsService,
|
||||
) {
|
||||
const rootPrisma = prisma;
|
||||
prisma = completionDatabase(prisma);
|
||||
this.prisma = prisma;
|
||||
// Structural unit-test doubles may omit the durable delegate; real Prisma always has it.
|
||||
const coordinatedBilling = rootPrisma.smsAttemptCompletionWork ? new BillingService(prisma) : billing;
|
||||
const resolvedPhoneRouting = phoneRouting ?? new PhoneRoutingLookupService(prisma);
|
||||
this.submission = new SendSubmissionService(
|
||||
prisma,
|
||||
@@ -109,12 +117,46 @@ export class SendChainService implements OnModuleInit, OnModuleDestroy {
|
||||
},
|
||||
metrics,
|
||||
);
|
||||
this.completion = new SendCompletionService(prisma, billing, openApi, this as unknown as SendCompletionFacade);
|
||||
this.completion = new SendCompletionService(
|
||||
prisma,
|
||||
coordinatedBilling,
|
||||
openApi,
|
||||
this as unknown as SendCompletionFacade,
|
||||
);
|
||||
if (rootPrisma.smsAttemptCompletionWork) {
|
||||
this.attemptCompletion = new AttemptCompletion(
|
||||
rootPrisma,
|
||||
async (kind, payload) => {
|
||||
const event = payload as unknown as {
|
||||
data: GatewayReceiptEventDto & GatewaySubmitResultDto & GatewaySubmitSegmentResultDto;
|
||||
incomingIdentity?: Parameters<SendChainService['handleReceipt']>[1];
|
||||
olderThanHours?: number;
|
||||
errorCode?: string;
|
||||
reason?: string;
|
||||
};
|
||||
if (kind === 'receipt') return this.completion.handleReceipt(event.data, event.incomingIdentity);
|
||||
if (kind === 'submit') return this.completion.handleSubmitResult(event.data);
|
||||
if (kind === 'segment') return this.completion.handleSubmitSegmentResult(event.data);
|
||||
if (kind === 'timeout') return this.completion.markUnknownTimeout({ olderThanHours: event.olderThanHours });
|
||||
const message = await prisma.smsMessageRecord.findUniqueOrThrow({
|
||||
where: { id: completionContext.getStore()!.messageRecordId },
|
||||
});
|
||||
if (
|
||||
message.status === 'delivered' ||
|
||||
(message.status === 'timeout' && message.errorCode === 'RECEIPT_TIMEOUT')
|
||||
)
|
||||
return message;
|
||||
return this.completion.recordCmppFailureReceipt(message, event.errorCode!, event.reason!);
|
||||
},
|
||||
(route) => this.submission.waitForChannelRateLimit(route.channel.id, route.channel.rateLimitPerSecond),
|
||||
);
|
||||
}
|
||||
this.downstreamRequeueTasks = new SendDownstreamRequeueTaskService(prisma, this);
|
||||
}
|
||||
|
||||
onModuleInit() {
|
||||
const processRole = process.env.CMPP_PROCESS_ROLE?.trim() || 'all';
|
||||
if (['all', 'worker', 'callback'].includes(processRole)) this.attemptCompletion?.start();
|
||||
if (processRole === 'api' || processRole === 'callback') return;
|
||||
if (processRole === 'outbox') {
|
||||
this.submission.startSubmitOutboxPublisher();
|
||||
@@ -202,6 +244,7 @@ export class SendChainService implements OnModuleInit, OnModuleDestroy {
|
||||
}
|
||||
|
||||
async onModuleDestroy() {
|
||||
this.attemptCompletion?.stop();
|
||||
if (this.receiptTimeoutInitialTimer) clearTimeout(this.receiptTimeoutInitialTimer);
|
||||
if (this.receiptTimeoutIntervalTimer) clearInterval(this.receiptTimeoutIntervalTimer);
|
||||
if (this.scheduledDispatchInitialTimer) clearTimeout(this.scheduledDispatchInitialTimer);
|
||||
@@ -484,6 +527,11 @@ export class SendChainService implements OnModuleInit, OnModuleDestroy {
|
||||
}
|
||||
|
||||
async handleSubmitSegmentResult(data: GatewaySubmitSegmentResultDto) {
|
||||
if (this.attemptCompletion && !completionContext.getStore()) {
|
||||
const message = await this.completion.requireMessageByGatewayEvent(data.messageId, data.gatewayMessageId);
|
||||
const source = await this.completion.resolveSubmitRecordForGatewaySegmentResult(message.id, data);
|
||||
return this.attemptCompletion.enqueue(message.id, source.id, 'segment', { data });
|
||||
}
|
||||
return this.completion.handleSubmitSegmentResult(data);
|
||||
}
|
||||
|
||||
@@ -495,6 +543,11 @@ export class SendChainService implements OnModuleInit, OnModuleDestroy {
|
||||
}
|
||||
|
||||
async handleSubmitResult(data: GatewaySubmitResultDto) {
|
||||
if (this.attemptCompletion && !completionContext.getStore()) {
|
||||
const message = await this.completion.requireMessageByGatewayEvent(data.messageId, data.gatewayMessageId);
|
||||
const source = await this.completion.resolveSubmitRecordForGatewayResult(message.id, data);
|
||||
return this.attemptCompletion.enqueue(message.id, source.id, 'submit', { data }, data.eventId);
|
||||
}
|
||||
return this.completion.handleSubmitResult(data);
|
||||
}
|
||||
|
||||
@@ -528,6 +581,14 @@ export class SendChainService implements OnModuleInit, OnModuleDestroy {
|
||||
cmppVersion: string;
|
||||
},
|
||||
) {
|
||||
if (this.attemptCompletion && !completionContext.getStore()) {
|
||||
const resolved = await this.completion.resolveReceiptMessage(data, incomingIdentity);
|
||||
if (!resolved.submitRecordId) throw new NotFoundException('回执缺少可确认的提交尝试关联');
|
||||
return this.attemptCompletion.enqueue(resolved.message.id, resolved.submitRecordId, 'receipt', {
|
||||
data,
|
||||
incomingIdentity,
|
||||
});
|
||||
}
|
||||
return this.completion.handleReceipt(data, incomingIdentity);
|
||||
}
|
||||
|
||||
@@ -726,6 +787,32 @@ export class SendChainService implements OnModuleInit, OnModuleDestroy {
|
||||
}
|
||||
|
||||
async markUnknownTimeout(data: TimeoutUnknownDto) {
|
||||
if (this.attemptCompletion && !completionContext.getStore()) {
|
||||
const olderThanHours = data.olderThanHours ?? positiveInteger(process.env.SMS_RECEIPT_TIMEOUT_HOURS, 72);
|
||||
const candidates = await this.prisma.smsMessageRecord.findMany({
|
||||
where: {
|
||||
tenantId: { not: null },
|
||||
OR: [
|
||||
{
|
||||
status: { in: ['submitted', 'unknown'] },
|
||||
submittedAt: { lte: new Date(Date.now() - olderThanHours * 3600_000) },
|
||||
},
|
||||
{ status: 'timeout', errorCode: 'RECEIPT_TIMEOUT', timeoutReceiptQueuedAt: null },
|
||||
],
|
||||
},
|
||||
select: { id: true, submitId: true, status: true },
|
||||
take: 100,
|
||||
});
|
||||
let timeout = 0;
|
||||
for (const message of candidates) {
|
||||
const source = message.submitId
|
||||
? await this.prisma.smsSubmitRecord.findUnique({ where: { submitId: message.submitId } })
|
||||
: null;
|
||||
const result = await this.attemptCompletion.enqueue(message.id, source?.id, 'timeout', { olderThanHours });
|
||||
if (message.status !== 'timeout' && result?.status === 'timeout') timeout++;
|
||||
}
|
||||
return { timeout };
|
||||
}
|
||||
return this.completion.markUnknownTimeout(data);
|
||||
}
|
||||
|
||||
@@ -803,6 +890,14 @@ export class SendChainService implements OnModuleInit, OnModuleDestroy {
|
||||
},
|
||||
options: { forceNational?: boolean; excludeChannelIds?: string[] } = {},
|
||||
): Promise<RoutedChannel> {
|
||||
const context = completionContext.getStore();
|
||||
if (context) {
|
||||
const prepare = () => this.submission.selectChannelForMessage(message, { ...options, previewOnly: true });
|
||||
if (!context.routePlanned) throw new CompletionRouteRequired(prepare);
|
||||
const current = await this.submission.selectChannelForMessage(message, options);
|
||||
if (!context.route || current.channel.id !== context.route.channel.id) throw new CompletionRouteRequired(prepare);
|
||||
return current;
|
||||
}
|
||||
return this.submission.selectChannelForMessage(message, options);
|
||||
}
|
||||
|
||||
@@ -923,6 +1018,9 @@ export class SendChainService implements OnModuleInit, OnModuleDestroy {
|
||||
errorCode: string,
|
||||
reason: string,
|
||||
) {
|
||||
if (this.attemptCompletion && !completionContext.getStore()) {
|
||||
return this.attemptCompletion.enqueue(message.id, undefined, 'rejection', { errorCode, reason });
|
||||
}
|
||||
return this.completion.recordCmppFailureReceipt(message, errorCode, reason);
|
||||
}
|
||||
|
||||
@@ -1004,6 +1102,7 @@ export class SendChainService implements OnModuleInit, OnModuleDestroy {
|
||||
}
|
||||
|
||||
private async waitForChannelRateLimit(channelId: string, tps: number) {
|
||||
if (completionContext.getStore()) return;
|
||||
return this.submission.waitForChannelRateLimit(channelId, tps);
|
||||
}
|
||||
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { completionContext } from './completion-context';
|
||||
import { BadRequestException, Logger, NotFoundException } from '@nestjs/common';
|
||||
import { Prisma } from '@prisma/client';
|
||||
import { createHash, randomUUID } from 'node:crypto';
|
||||
@@ -217,20 +218,24 @@ export class SendDownstreamDeliveryService {
|
||||
const cmppDeliveryAllowed = deliveryAllowed || data.allowBusinessRejectionCmppDelivery === true;
|
||||
if (deliveryAllowed && data.queueHttpWebhook !== false) {
|
||||
try {
|
||||
await this.openApi?.queueWebhookEvent({
|
||||
tenantId: data.tenantId,
|
||||
applicationId: data.applicationId,
|
||||
messageRecordId: data.messageRecordId,
|
||||
messageId: data.messageId,
|
||||
uplinkMessageId: typeof data.payload.uplinkMessageId === 'string' ? data.payload.uplinkMessageId : undefined,
|
||||
eventType: data.deliveryType,
|
||||
payload: data.payload,
|
||||
});
|
||||
await this.openApi?.queueWebhookEvent(
|
||||
{
|
||||
tenantId: data.tenantId,
|
||||
applicationId: data.applicationId,
|
||||
messageRecordId: data.messageRecordId,
|
||||
messageId: data.messageId,
|
||||
uplinkMessageId:
|
||||
typeof data.payload.uplinkMessageId === 'string' ? data.payload.uplinkMessageId : undefined,
|
||||
eventType: data.deliveryType,
|
||||
payload: data.payload,
|
||||
},
|
||||
completionContext.getStore()?.tx,
|
||||
);
|
||||
} catch (error) {
|
||||
this.logger.error(
|
||||
`HTTP webhook queue failed for ${data.deliveryType}/${data.messageId ?? '-'}: ${error instanceof Error ? error.message : String(error)}`,
|
||||
);
|
||||
if (data.propagateHttpQueueError) throw error;
|
||||
if (data.propagateHttpQueueError || completionContext.getStore()) throw error;
|
||||
}
|
||||
}
|
||||
if (data.queueCmppDelivery === false) {
|
||||
@@ -249,6 +254,34 @@ export class SendDownstreamDeliveryService {
|
||||
: data.deliveryType === 'uplink' && typeof data.payload.uplinkMessageId === 'string'
|
||||
? `uplink:${data.payload.uplinkMessageId}`
|
||||
: null;
|
||||
if (completionContext.getStore()) {
|
||||
if (!dedupeKey) throw new Error('completion_notification_identity_missing');
|
||||
await this.prisma.cmppDownstreamDelivery.createMany({
|
||||
data: [
|
||||
{
|
||||
tenantId: data.tenantId,
|
||||
applicationId: data.applicationId,
|
||||
messageRecordId: data.messageRecordId,
|
||||
messageId: data.messageId,
|
||||
dedupeKey,
|
||||
deliveryType: data.deliveryType,
|
||||
payload,
|
||||
retryEnabled:
|
||||
cmppDeliveryAllowed &&
|
||||
(data.deliveryType === 'uplink'
|
||||
? application.downstreamUplinkRetryEnabled
|
||||
: application.downstreamReceiptRetryEnabled),
|
||||
status: cmppDeliveryAllowed ? 'pending' : 'abandoned',
|
||||
lastError: cmppDeliveryAllowed ? null : '企业应用已停用,保留回执但不再向客户应用推送',
|
||||
},
|
||||
],
|
||||
skipDuplicates: true,
|
||||
});
|
||||
const retained = await this.prisma.cmppDownstreamDelivery.findUniqueOrThrow({ where: { dedupeKey } });
|
||||
if (retained.messageRecordId !== data.messageRecordId || retained.applicationId !== data.applicationId)
|
||||
throw new Error('completion_notification_identity_mismatch');
|
||||
return retained;
|
||||
}
|
||||
let delivery;
|
||||
try {
|
||||
delivery = await this.prisma.cmppDownstreamDelivery.create({
|
||||
|
||||
@@ -188,11 +188,18 @@ export class SendGatewayResultService {
|
||||
}
|
||||
const status =
|
||||
data.submitStatus === 'accepted' ? 'submitted' : data.submitStatus === 'timeout' ? 'timeout' : 'submit_failed';
|
||||
if (
|
||||
data.submitStatus !== 'accepted' &&
|
||||
(message.status === 'delivered' || (message.status === 'timeout' && message.errorCode === 'RECEIPT_TIMEOUT'))
|
||||
) {
|
||||
await this.markSubmitResultProcessed(submitRecord.id, data.eventId, submittedAt);
|
||||
return message;
|
||||
}
|
||||
if (data.submitStatus === 'accepted' && !isStandaloneChannelTest && message.tenantId && message.batchTaskId) {
|
||||
const businessMessage = message as typeof message & { tenantId: string; batchTaskId: string };
|
||||
await this.facade.chargeAcceptedMessage(businessMessage);
|
||||
const latest = await this.prisma.smsMessageRecord.findUnique({ where: { id: message.id } });
|
||||
if (latest?.status === 'failed') {
|
||||
if (latest?.status === 'failed' || (latest?.status === 'timeout' && latest.errorCode === 'RECEIPT_TIMEOUT')) {
|
||||
await this.facade.refundMessage(businessMessage, '先到失败回执补偿退款');
|
||||
}
|
||||
} else if (
|
||||
@@ -219,7 +226,7 @@ export class SendGatewayResultService {
|
||||
data.submitStatus === 'timeout' ? '提交超时释放冻结' : '提交失败释放冻结',
|
||||
);
|
||||
}
|
||||
const protectedTerminalStatuses = ['delivered', 'failed', 'unknown'];
|
||||
const protectedTerminalStatuses = ['delivered', 'failed', 'unknown', 'timeout'];
|
||||
const updated = await this.prisma.smsMessageRecord.updateMany({
|
||||
where:
|
||||
data.submitStatus === 'accepted'
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { completionContext } from './completion-context';
|
||||
import { BadRequestException, Logger, NotFoundException } from '@nestjs/common';
|
||||
import { Prisma } from '@prisma/client';
|
||||
import { Queue, Worker } from 'bullmq';
|
||||
@@ -310,7 +311,7 @@ export class SendGatewaySubmitService {
|
||||
sessionId: sessionByChannel.get(routed.channel.id),
|
||||
};
|
||||
});
|
||||
const writeOutbox = this.submitOutboxEnabled();
|
||||
const writeOutbox = Boolean(completionContext.getStore()) || this.submitOutboxEnabled();
|
||||
await this.measureSendStage('submit_transaction', () =>
|
||||
this.prisma.$transaction(async (tx) => {
|
||||
await tx.smsSubmitRecord.createMany({
|
||||
@@ -365,7 +366,7 @@ export class SendGatewaySubmitService {
|
||||
}
|
||||
}),
|
||||
);
|
||||
if (!this.submitOutboxPublishEnabled()) {
|
||||
if (!completionContext.getStore() && !this.submitOutboxPublishEnabled()) {
|
||||
await Promise.all(prepared.map(({ command }) => this.facade.publishGatewaySubmitCommand(command)));
|
||||
}
|
||||
await this.refreshTaskProgressBatch(prepared.map(({ message }) => message));
|
||||
@@ -762,7 +763,7 @@ export class SendGatewaySubmitService {
|
||||
const submitId = `SUB-${randomUUID()}`;
|
||||
const sessionId = await this.getOpenSubmitSessionId(channel.id);
|
||||
const command = this.buildGatewaySubmitCommand(message, routed, attempt, submitId, upstreamSrcId);
|
||||
const writeOutbox = this.submitOutboxEnabled();
|
||||
const writeOutbox = Boolean(completionContext.getStore()) || this.submitOutboxEnabled();
|
||||
try {
|
||||
await this.measureSendStage('submit_transaction', () =>
|
||||
this.prisma.$transaction(async (tx) => {
|
||||
@@ -849,7 +850,7 @@ export class SendGatewaySubmitService {
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
if (!this.submitOutboxPublishEnabled()) {
|
||||
if (!completionContext.getStore() && !this.submitOutboxPublishEnabled()) {
|
||||
await this.measureSendStage('gateway_stream_publish', () => this.facade.publishGatewaySubmitCommand(command));
|
||||
}
|
||||
await this.measureSendStage('task_progress', () =>
|
||||
@@ -1087,7 +1088,7 @@ return streamId`;
|
||||
template?: { signature?: { id?: string | null } | null } | null;
|
||||
signature?: { id?: string | null } | null;
|
||||
},
|
||||
options: { forceNational?: boolean; excludeChannelIds?: string[] } = {},
|
||||
options: { forceNational?: boolean; excludeChannelIds?: string[]; previewOnly?: boolean } = {},
|
||||
): Promise<RoutedChannel> {
|
||||
if (!message.applicationId) {
|
||||
throw new BadRequestException('短信应用未配置,无法选择通道组');
|
||||
@@ -1100,7 +1101,7 @@ return streamId`;
|
||||
this.facade.identifyCarrier(message.phoneNumber),
|
||||
this.facade.identifyProvince(message.phoneNumber),
|
||||
]);
|
||||
if (!hasPersistedRouting) {
|
||||
if (!hasPersistedRouting && !options.previewOnly) {
|
||||
await this.prisma.smsMessageRecord.update({
|
||||
where: { id: message.id },
|
||||
data: { carrier: resolved[0], province: resolved[1] },
|
||||
@@ -1137,7 +1138,7 @@ return streamId`;
|
||||
approvedChannelIds,
|
||||
routingKey: message.id,
|
||||
});
|
||||
await channelWords.persist(this.prisma);
|
||||
if (!options.previewOnly) await channelWords.persist(this.prisma);
|
||||
if (rejected) throw new ChannelWordRejection();
|
||||
if (!selected) {
|
||||
throw new NotFoundException('无已报备通过且在线的可用通道');
|
||||
@@ -1352,6 +1353,10 @@ return streamId`;
|
||||
`);
|
||||
if (direct === 1) return;
|
||||
}
|
||||
if (completionContext.getStore()) {
|
||||
await this.prisma.$queryRaw`SELECT id FROM "SmsBatchTask" WHERE id=${batchTaskId} FOR UPDATE`;
|
||||
return this.refreshTaskProgressUntilClean(batchTaskId, true);
|
||||
}
|
||||
const running = this.taskProgressRefreshes.get(batchTaskId);
|
||||
if (running) {
|
||||
// A state transition committed after the running aggregate may not be visible
|
||||
@@ -1372,9 +1377,9 @@ return streamId`;
|
||||
}
|
||||
}
|
||||
|
||||
private async refreshTaskProgressUntilClean(batchTaskId: string) {
|
||||
private async refreshTaskProgressUntilClean(batchTaskId: string, transactional = false) {
|
||||
do {
|
||||
this.dirtyTaskProgressRefreshes.delete(batchTaskId);
|
||||
if (!transactional) this.dirtyTaskProgressRefreshes.delete(batchTaskId);
|
||||
const groups = await this.prisma.smsMessageRecord.groupBy({
|
||||
by: ['status'],
|
||||
where: { batchTaskId },
|
||||
@@ -1401,7 +1406,7 @@ return streamId`;
|
||||
where: { id: batchTaskId },
|
||||
data: { progressTotal, submittedTotal, successTotal, failedTotal, unknownTotal, timeoutTotal, status },
|
||||
});
|
||||
} while (this.dirtyTaskProgressRefreshes.has(batchTaskId));
|
||||
} while (!transactional && this.dirtyTaskProgressRefreshes.has(batchTaskId));
|
||||
}
|
||||
|
||||
private async recoverNightReviews() {
|
||||
@@ -1464,6 +1469,17 @@ return streamId`,
|
||||
}
|
||||
|
||||
private getOpenSubmitSessionId(channelId: string) {
|
||||
if (completionContext.getStore()) {
|
||||
const sessionNo = `OPEN-${channelId}`;
|
||||
return this.prisma.cmppSubmitSession
|
||||
.upsert({
|
||||
where: { sessionNo },
|
||||
update: {},
|
||||
create: { channelId, sessionNo, submitTotal: 0 },
|
||||
select: { id: true },
|
||||
})
|
||||
.then((session) => session.id);
|
||||
}
|
||||
const cached = this.openSubmitSessionIds.get(channelId);
|
||||
if (cached) return cached;
|
||||
const sessionNo = `OPEN-${channelId}`;
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { completionContext } from './completion-context';
|
||||
import { Logger, NotFoundException } from '@nestjs/common';
|
||||
import { Prisma } from '@prisma/client';
|
||||
|
||||
@@ -229,10 +230,12 @@ export class SendReceiptService {
|
||||
where: { receiptKey },
|
||||
include: { messageRecord: true },
|
||||
});
|
||||
if (existingReceipt?.messageRecord) {
|
||||
if (existingReceipt?.messageRecord && !completionContext.getStore()) {
|
||||
return existingReceipt.messageRecord;
|
||||
}
|
||||
const message = resolved.message;
|
||||
if (existingReceipt && existingReceipt.messageRecordId !== message.id)
|
||||
throw new Error('completion_receipt_owner_mismatch');
|
||||
const deliveredAt = data.deliveredAt ? new Date(data.deliveredAt) : new Date();
|
||||
if (resolved.submitRecordId) {
|
||||
await this.prisma.smsSubmitRecord.updateMany({
|
||||
@@ -246,37 +249,38 @@ export class SendReceiptService {
|
||||
},
|
||||
});
|
||||
}
|
||||
let receiptRecordId: string | undefined;
|
||||
try {
|
||||
const createdReceipt = await this.prisma.smsReceiptRecord.create({
|
||||
data: {
|
||||
tenantId: message.tenantId,
|
||||
batchTaskId: message.batchTaskId,
|
||||
messageRecordId: message.id,
|
||||
receiptKey,
|
||||
channelId: logicalChannelId,
|
||||
messageId: resolved.messageId,
|
||||
gatewayMessageId: data.gatewayMessageId,
|
||||
phoneNumber: data.phoneNumber?.trim() || message.phoneNumber,
|
||||
sequenceId: data.sequenceId,
|
||||
receiptStatus: data.receiptStatus,
|
||||
rawStatus: data.rawStatus,
|
||||
errorCode: data.errorCode,
|
||||
errorMessage: data.errorMessage,
|
||||
deliveredAt,
|
||||
},
|
||||
});
|
||||
receiptRecordId = createdReceipt.id;
|
||||
} catch (error) {
|
||||
if (error instanceof Prisma.PrismaClientKnownRequestError && error.code === 'P2002') {
|
||||
const duplicate = await this.prisma.smsReceiptRecord.findUnique({
|
||||
where: { receiptKey },
|
||||
include: { messageRecord: true },
|
||||
let receiptRecordId: string | undefined = existingReceipt?.id;
|
||||
if (!existingReceipt)
|
||||
try {
|
||||
const createdReceipt = await this.prisma.smsReceiptRecord.create({
|
||||
data: {
|
||||
tenantId: message.tenantId,
|
||||
batchTaskId: message.batchTaskId,
|
||||
messageRecordId: message.id,
|
||||
receiptKey,
|
||||
channelId: logicalChannelId,
|
||||
messageId: resolved.messageId,
|
||||
gatewayMessageId: data.gatewayMessageId,
|
||||
phoneNumber: data.phoneNumber?.trim() || message.phoneNumber,
|
||||
sequenceId: data.sequenceId,
|
||||
receiptStatus: data.receiptStatus,
|
||||
rawStatus: data.rawStatus,
|
||||
errorCode: data.errorCode,
|
||||
errorMessage: data.errorMessage,
|
||||
deliveredAt,
|
||||
},
|
||||
});
|
||||
if (duplicate?.messageRecord) return duplicate.messageRecord;
|
||||
receiptRecordId = createdReceipt.id;
|
||||
} catch (error) {
|
||||
if (error instanceof Prisma.PrismaClientKnownRequestError && error.code === 'P2002') {
|
||||
const duplicate = await this.prisma.smsReceiptRecord.findUnique({
|
||||
where: { receiptKey },
|
||||
include: { messageRecord: true },
|
||||
});
|
||||
if (duplicate?.messageRecord) return duplicate.messageRecord;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
const logicalReceipt = { ...data, channelId: logicalChannelId };
|
||||
await this.facade.recordReceiptSegment(message, logicalReceipt, deliveredAt, resolved.submitRecordId);
|
||||
const receiptMode =
|
||||
@@ -300,8 +304,10 @@ export class SendReceiptService {
|
||||
if (!aggregate.terminal) {
|
||||
return this.prisma.smsMessageRecord.findUnique({ where: { id: message.id } });
|
||||
}
|
||||
if (message.status === 'timeout' && message.errorCode === 'RECEIPT_TIMEOUT') return message;
|
||||
const status = aggregate.status;
|
||||
const isCurrentAttempt =
|
||||
(!message.submitId || message.submitId === resolved.submitId) &&
|
||||
(!message.channelId || message.channelId === logicalChannelId) &&
|
||||
(!message.gatewayMessageId ||
|
||||
message.gatewayMessageId === data.gatewayMessageId ||
|
||||
|
||||
@@ -1,16 +1,18 @@
|
||||
import { BadRequestException, HttpException, HttpStatus, Logger, NotFoundException } from '@nestjs/common';
|
||||
import { BadRequestException, HttpException, Logger, NotFoundException } from '@nestjs/common';
|
||||
import { Prisma } from '@prisma/client';
|
||||
import { createHash } from 'node:crypto';
|
||||
import { BillingService } from '../billing/billing.service';
|
||||
import { moneyToNumber } from '../common/money';
|
||||
import type { OpenApiService } from '../open-api/open-api.service';
|
||||
import { PrismaService } from '../prisma/prisma.service';
|
||||
import type { GatewaySubmitResultDto, GatewaySubmitSegmentResultDto, GatewayReceiptEventDto, GatewayUplinkEventDto, GatewayPendingDeliveryQueryDto, GatewayDownstreamSentDto, GatewayDownstreamAcknowledgedDto, GatewayDownstreamFailureType, GatewayControlDeliveryResult, GatewaySubmitDeadLetterDto, RequeueGatewaySubmitExceptionDto, GatewayDownstreamRecoveryStatusDto, TimeoutUnknownDto } from './send-chain.contracts';
|
||||
import { positiveInteger, normalizeReceiptStatus, normalizeCarrier, DEFAULT_DOWNSTREAM_RETRY_DELAY_MS, DEFAULT_DOWNSTREAM_RETRY_MAX_DELAY_MS, DEFAULT_DOWNSTREAM_MAX_RETRIES, DEFAULT_DOWNSTREAM_PENDING_TIMEOUT_HOURS, DEFAULT_RECEIPT_TIMEOUT_HOURS, DEFAULT_GATEWAY_SUBMIT_REQUEUE_STALE_MS, DEFAULT_DOWNSTREAM_MANUAL_REQUEUE_STALE_MS, DEFAULT_UPSTREAM_RECEIPT_INBOX_PROCESSING_STALE_MS, DEFAULT_UPSTREAM_RECEIPT_INBOX_MAX_ATTEMPTS, DEFAULT_UPSTREAM_RECEIPT_INBOX_MAX_AGE_HOURS, gatewaySubmitRequeueKey, isObjectRecord, asDateOrNull, downstreamRetryDelayMs, downstreamAckTimeoutMs, downstreamRetryBaseDelayMs, downstreamRetryMaxDelayMs, downstreamMaxRetries, downstreamPendingTimeoutHours, downstreamControlFailureMessage, normalizeSubmitStatus, downstreamDeliveryAttemptKey, hasRecoveryAuditStateChanged, normalizeRecoveryFailureCategory, aggregateReceiptSegmentState, isSameUpstreamEndpointIdentity, receiptEventKey } from './send-chain.helpers';
|
||||
import type { SendSubmissionService } from './send-submission.service';
|
||||
import type { GatewaySubmitDeadLetterDto, RequeueGatewaySubmitExceptionDto } from './send-chain.contracts';
|
||||
import {
|
||||
DEFAULT_GATEWAY_SUBMIT_REQUEUE_STALE_MS,
|
||||
gatewaySubmitRequeueKey,
|
||||
isObjectRecord,
|
||||
normalizeCarrier,
|
||||
positiveInteger,
|
||||
} from './send-chain.helpers';
|
||||
import type { SendCompletionCallbacks, SendCompletionFacade } from './send-completion.service';
|
||||
|
||||
|
||||
/**
|
||||
* R10 retry implementation.
|
||||
* Cross-domain calls return through the stable SendChainService seam to preserve locking and test observability.
|
||||
@@ -87,11 +89,12 @@ export class SendRetryService {
|
||||
const message = deadLetter.messageId
|
||||
? await this.prisma.smsMessageRecord.findUnique({ where: { messageId: deadLetter.messageId } })
|
||||
: null;
|
||||
if (message && (
|
||||
message.submitStatus === 'accepted'
|
||||
|| ['submitted', 'delivered', 'unknown'].includes(message.status)
|
||||
|| ['delivered', 'unknown'].includes(message.receiptStatus ?? '')
|
||||
)) {
|
||||
if (
|
||||
message &&
|
||||
(message.submitStatus === 'accepted' ||
|
||||
['submitted', 'delivered', 'unknown'].includes(message.status) ||
|
||||
['delivered', 'unknown'].includes(message.receiptStatus ?? ''))
|
||||
) {
|
||||
throw new BadRequestException('该短信已有成功或不确定的上游结果,为避免重复发送,禁止重新入队');
|
||||
}
|
||||
const commandChannelId = String(deadLetter.commandPayload.channelId ?? deadLetter.channelId ?? '').trim();
|
||||
@@ -118,7 +121,10 @@ export class SendRetryService {
|
||||
const requeueKey = gatewaySubmitRequeueKey(deadLetter.id, deadLetter.manualRetryCount + 1);
|
||||
let retryStreamMessageId: string;
|
||||
try {
|
||||
const publishedStreamMessageId = await this.facade.publishGatewaySubmitCommand(deadLetter.commandPayload, requeueKey);
|
||||
const publishedStreamMessageId = await this.facade.publishGatewaySubmitCommand(
|
||||
deadLetter.commandPayload,
|
||||
requeueKey,
|
||||
);
|
||||
if (!publishedStreamMessageId) {
|
||||
throw new Error('Gateway提交异常重新入队未返回Stream消息编号');
|
||||
}
|
||||
@@ -208,10 +214,10 @@ export class SendRetryService {
|
||||
}
|
||||
|
||||
async recoverStaleGatewaySubmitRequeues(now = new Date()) {
|
||||
const staleCutoff = new Date(now.getTime() - positiveInteger(
|
||||
process.env.GATEWAY_SUBMIT_REQUEUE_STALE_MS,
|
||||
DEFAULT_GATEWAY_SUBMIT_REQUEUE_STALE_MS,
|
||||
));
|
||||
const staleCutoff = new Date(
|
||||
now.getTime() -
|
||||
positiveInteger(process.env.GATEWAY_SUBMIT_REQUEUE_STALE_MS, DEFAULT_GATEWAY_SUBMIT_REQUEUE_STALE_MS),
|
||||
);
|
||||
const stale = await this.prisma.gatewaySubmitDeadLetter.findMany({
|
||||
where: { status: 'requeueing', updatedAt: { lt: staleCutoff } },
|
||||
orderBy: { updatedAt: 'asc' },
|
||||
@@ -235,7 +241,10 @@ export class SendRetryService {
|
||||
if (claimed.count !== 1) continue;
|
||||
try {
|
||||
const requeueKey = gatewaySubmitRequeueKey(deadLetter.id, deadLetter.manualRetryCount + 1);
|
||||
const retryStreamMessageId = await this.facade.publishGatewaySubmitCommand(deadLetter.commandPayload, requeueKey);
|
||||
const retryStreamMessageId = await this.facade.publishGatewaySubmitCommand(
|
||||
deadLetter.commandPayload,
|
||||
requeueKey,
|
||||
);
|
||||
if (!retryStreamMessageId) throw new Error('Gateway提交异常恢复未返回Stream消息编号');
|
||||
const finalized = await this.prisma.gatewaySubmitDeadLetter.updateMany({
|
||||
where: { id: deadLetter.id, status: 'requeue_recovering' },
|
||||
@@ -264,7 +273,9 @@ export class SendRetryService {
|
||||
where: { id: deadLetter.id, status: 'requeue_recovering' },
|
||||
data: { status: 'requeueing' },
|
||||
});
|
||||
this.logger.error(`Gateway submit requeue recovery failed for ${deadLetter.id}: ${error instanceof Error ? error.message : String(error)}`);
|
||||
this.logger.error(
|
||||
`Gateway submit requeue recovery failed for ${deadLetter.id}: ${error instanceof Error ? error.message : String(error)}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
return { recovered, failed };
|
||||
@@ -300,100 +311,123 @@ export class SendRetryService {
|
||||
const attemptedChannelIds = attempts.map((attempt) => attempt.channelId);
|
||||
let sourceAttempt = sourceSubmitRecordId
|
||||
? attempts.find((attempt) => attempt.id === sourceSubmitRecordId)
|
||||
: attempts.find((attempt) => attempt.submitId === message.submitId) ?? attempts[attempts.length - 1];
|
||||
: (attempts.find((attempt) => attempt.submitId === message.submitId) ?? attempts[attempts.length - 1]);
|
||||
if (!sourceAttempt && sourceSubmitRecordId) {
|
||||
sourceAttempt = await this.prisma.smsSubmitRecord.findUnique({
|
||||
where: { id: sourceSubmitRecordId },
|
||||
}) ?? undefined;
|
||||
sourceAttempt =
|
||||
(await this.prisma.smsSubmitRecord.findUnique({
|
||||
where: { id: sourceSubmitRecordId },
|
||||
})) ?? undefined;
|
||||
}
|
||||
if (!sourceAttempt || sourceAttempt.messageRecordId !== message.id) {
|
||||
this.logger.error(`sms_retry_route_failed ${JSON.stringify({
|
||||
messageId: message.messageId,
|
||||
messageRecordId: message.id,
|
||||
reason,
|
||||
sourceSubmitRecordId,
|
||||
sourceMessageRecordId: sourceAttempt?.messageRecordId,
|
||||
error: sourceAttempt ? 'retry_source_submit_record_mismatch' : 'retry_source_submit_record_missing',
|
||||
})}`);
|
||||
this.logger.error(
|
||||
`sms_retry_route_failed ${JSON.stringify({
|
||||
messageId: message.messageId,
|
||||
messageRecordId: message.id,
|
||||
reason,
|
||||
sourceSubmitRecordId,
|
||||
sourceMessageRecordId: sourceAttempt?.messageRecordId,
|
||||
error: sourceAttempt ? 'retry_source_submit_record_mismatch' : 'retry_source_submit_record_missing',
|
||||
})}`,
|
||||
);
|
||||
return null;
|
||||
}
|
||||
const existingRetry = await this.prisma.smsSubmitRecord.findUnique({
|
||||
where: { retryOfSubmitRecordId: sourceAttempt.id },
|
||||
});
|
||||
if (existingRetry) {
|
||||
this.logger.warn(`sms_retry_claim_reused ${JSON.stringify({
|
||||
messageId: message.messageId,
|
||||
messageRecordId: message.id,
|
||||
retryOfSubmitRecordId: sourceAttempt.id,
|
||||
submitId: existingRetry.submitId,
|
||||
channelId: existingRetry.channelId,
|
||||
})}`);
|
||||
this.logger.warn(
|
||||
`sms_retry_claim_reused ${JSON.stringify({
|
||||
messageId: message.messageId,
|
||||
messageRecordId: message.id,
|
||||
retryOfSubmitRecordId: sourceAttempt.id,
|
||||
submitId: existingRetry.submitId,
|
||||
channelId: existingRetry.channelId,
|
||||
})}`,
|
||||
);
|
||||
return this.prisma.smsMessageRecord.findUnique({ where: { id: message.id } });
|
||||
}
|
||||
const ageMinutes = (Date.now() - new Date(message.queuedAt ?? Date.now()).getTime()) / 60_000;
|
||||
this.logger.log(`sms_retry_route_started ${JSON.stringify({
|
||||
messageId: message.messageId,
|
||||
messageRecordId: message.id,
|
||||
reason,
|
||||
attemptedChannelIds,
|
||||
ageMinutes: Math.round(ageMinutes * 100) / 100,
|
||||
})}`);
|
||||
if (ageMinutes >= 72 * 60) {
|
||||
this.logger.warn(`sms_retry_route_skipped ${JSON.stringify({
|
||||
this.logger.log(
|
||||
`sms_retry_route_started ${JSON.stringify({
|
||||
messageId: message.messageId,
|
||||
messageRecordId: message.id,
|
||||
reason: 'maximum_message_age_exceeded',
|
||||
reason,
|
||||
attemptedChannelIds,
|
||||
ageMinutes: Math.round(ageMinutes * 100) / 100,
|
||||
})}`);
|
||||
})}`,
|
||||
);
|
||||
if (ageMinutes >= 72 * 60) {
|
||||
this.logger.warn(
|
||||
`sms_retry_route_skipped ${JSON.stringify({
|
||||
messageId: message.messageId,
|
||||
messageRecordId: message.id,
|
||||
reason: 'maximum_message_age_exceeded',
|
||||
ageMinutes: Math.round(ageMinutes * 100) / 100,
|
||||
})}`,
|
||||
);
|
||||
return null;
|
||||
}
|
||||
const retryCarrier = message.carrier
|
||||
? normalizeCarrier(message.carrier)
|
||||
: await this.facade.identifyCarrier(message.phoneNumber);
|
||||
const route = await this.facade.findApplicationRoute(message.tenantId, message.applicationId ?? undefined, retryCarrier);
|
||||
const retryTimeLimitMinutes = Math.min(route.group.retryTimeLimitMinutes ?? route.group.retryTimeLimitHours * 60, 72 * 60);
|
||||
const route = await this.facade.findApplicationRoute(
|
||||
message.tenantId,
|
||||
message.applicationId ?? undefined,
|
||||
retryCarrier,
|
||||
);
|
||||
const retryTimeLimitMinutes = Math.min(
|
||||
route.group.retryTimeLimitMinutes ?? route.group.retryTimeLimitHours * 60,
|
||||
72 * 60,
|
||||
);
|
||||
if (!route.group.retryEnabled || ageMinutes >= retryTimeLimitMinutes) {
|
||||
this.logger.warn(`sms_retry_route_skipped ${JSON.stringify({
|
||||
messageId: message.messageId,
|
||||
messageRecordId: message.id,
|
||||
groupId: route.groupId,
|
||||
reason: !route.group.retryEnabled ? 'group_retry_disabled' : 'group_retry_time_limit_exceeded',
|
||||
ageMinutes: Math.round(ageMinutes * 100) / 100,
|
||||
retryTimeLimitMinutes,
|
||||
})}`);
|
||||
this.logger.warn(
|
||||
`sms_retry_route_skipped ${JSON.stringify({
|
||||
messageId: message.messageId,
|
||||
messageRecordId: message.id,
|
||||
groupId: route.groupId,
|
||||
reason: !route.group.retryEnabled ? 'group_retry_disabled' : 'group_retry_time_limit_exceeded',
|
||||
ageMinutes: Math.round(ageMinutes * 100) / 100,
|
||||
retryTimeLimitMinutes,
|
||||
})}`,
|
||||
);
|
||||
return null;
|
||||
}
|
||||
try {
|
||||
const routed = await this.facade.selectChannelForMessage({ ...message, carrier: retryCarrier }, {
|
||||
forceNational: true,
|
||||
excludeChannelIds: attemptedChannelIds,
|
||||
});
|
||||
const routed = await this.facade.selectChannelForMessage(
|
||||
{ ...message, carrier: retryCarrier },
|
||||
{
|
||||
forceNational: true,
|
||||
excludeChannelIds: attemptedChannelIds,
|
||||
},
|
||||
);
|
||||
await this.prisma.smsMessageRecord.update({
|
||||
where: { id: message.id },
|
||||
data: { errorMessage: reason },
|
||||
});
|
||||
const retried = await this.facade.submitMessageToGateway(
|
||||
message,
|
||||
routed,
|
||||
attempts.length,
|
||||
sourceAttempt.id,
|
||||
const retried = await this.facade.submitMessageToGateway(message, routed, attempts.length, sourceAttempt.id);
|
||||
this.logger.log(
|
||||
`sms_retry_route_selected ${JSON.stringify({
|
||||
messageId: message.messageId,
|
||||
messageRecordId: message.id,
|
||||
groupId: routed.groupId,
|
||||
channelId: routed.channel.id,
|
||||
attempt: attempts.length,
|
||||
})}`,
|
||||
);
|
||||
this.logger.log(`sms_retry_route_selected ${JSON.stringify({
|
||||
messageId: message.messageId,
|
||||
messageRecordId: message.id,
|
||||
groupId: routed.groupId,
|
||||
channelId: routed.channel.id,
|
||||
attempt: attempts.length,
|
||||
})}`);
|
||||
return retried;
|
||||
} catch (error) {
|
||||
this.logger.error(`sms_retry_route_failed ${JSON.stringify({
|
||||
messageId: message.messageId,
|
||||
messageRecordId: message.id,
|
||||
reason,
|
||||
attemptedChannelIds,
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
})}`);
|
||||
// Infrastructure failures and the completion routing probe must abort the
|
||||
// transaction; treating them as "no retry route" would finalize a failure.
|
||||
if (!(error instanceof HttpException) || error.getStatus() >= 500) throw error;
|
||||
this.logger.error(
|
||||
`sms_retry_route_failed ${JSON.stringify({
|
||||
messageId: message.messageId,
|
||||
messageRecordId: message.id,
|
||||
reason,
|
||||
attemptedChannelIds,
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
})}`,
|
||||
);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -368,7 +368,7 @@ export class SendSubmissionService {
|
||||
template?: { signature?: { id?: string | null } | null } | null;
|
||||
signature?: { id?: string | null } | null;
|
||||
},
|
||||
options: { forceNational?: boolean; excludeChannelIds?: string[] } = {},
|
||||
options: { forceNational?: boolean; excludeChannelIds?: string[]; previewOnly?: boolean } = {},
|
||||
): Promise<RoutedChannel> {
|
||||
return this.gatewaySubmit.selectChannelForMessage(message, options);
|
||||
}
|
||||
|
||||
@@ -1,16 +1,12 @@
|
||||
import { BadRequestException, HttpException, HttpStatus, Logger, NotFoundException } from '@nestjs/common';
|
||||
import { Prisma } from '@prisma/client';
|
||||
import { createHash } from 'node:crypto';
|
||||
import { Logger } from '@nestjs/common';
|
||||
import { BillingService } from '../billing/billing.service';
|
||||
import { moneyToNumber } from '../common/money';
|
||||
import type { OpenApiService } from '../open-api/open-api.service';
|
||||
import { PrismaService } from '../prisma/prisma.service';
|
||||
import type { GatewaySubmitResultDto, GatewaySubmitSegmentResultDto, GatewayReceiptEventDto, GatewayUplinkEventDto, GatewayPendingDeliveryQueryDto, GatewayDownstreamSentDto, GatewayDownstreamAcknowledgedDto, GatewayDownstreamFailureType, GatewayControlDeliveryResult, GatewaySubmitDeadLetterDto, RequeueGatewaySubmitExceptionDto, GatewayDownstreamRecoveryStatusDto, TimeoutUnknownDto } from './send-chain.contracts';
|
||||
import { positiveInteger, normalizeReceiptStatus, normalizeCarrier, DEFAULT_DOWNSTREAM_RETRY_DELAY_MS, DEFAULT_DOWNSTREAM_RETRY_MAX_DELAY_MS, DEFAULT_DOWNSTREAM_MAX_RETRIES, DEFAULT_DOWNSTREAM_PENDING_TIMEOUT_HOURS, DEFAULT_RECEIPT_TIMEOUT_HOURS, DEFAULT_GATEWAY_SUBMIT_REQUEUE_STALE_MS, DEFAULT_DOWNSTREAM_MANUAL_REQUEUE_STALE_MS, DEFAULT_UPSTREAM_RECEIPT_INBOX_PROCESSING_STALE_MS, DEFAULT_UPSTREAM_RECEIPT_INBOX_MAX_ATTEMPTS, DEFAULT_UPSTREAM_RECEIPT_INBOX_MAX_AGE_HOURS, gatewaySubmitRequeueKey, isObjectRecord, asDateOrNull, downstreamRetryDelayMs, downstreamAckTimeoutMs, downstreamRetryBaseDelayMs, downstreamRetryMaxDelayMs, downstreamMaxRetries, downstreamPendingTimeoutHours, downstreamControlFailureMessage, normalizeSubmitStatus, downstreamDeliveryAttemptKey, hasRecoveryAuditStateChanged, normalizeRecoveryFailureCategory, aggregateReceiptSegmentState, isSameUpstreamEndpointIdentity, receiptEventKey } from './send-chain.helpers';
|
||||
import type { SendSubmissionService } from './send-submission.service';
|
||||
import type { SendCompletionCallbacks, SendCompletionFacade } from './send-completion.service';
|
||||
import { completionContext } from './completion-context';
|
||||
import { queueFinalReceiptDeliveries } from './downstream-receipt-targets';
|
||||
|
||||
import type { TimeoutUnknownDto } from './send-chain.contracts';
|
||||
import { DEFAULT_RECEIPT_TIMEOUT_HOURS, downstreamPendingTimeoutHours, positiveInteger } from './send-chain.helpers';
|
||||
import type { SendCompletionCallbacks, SendCompletionFacade } from './send-completion.service';
|
||||
|
||||
/**
|
||||
* R10 timeout implementation.
|
||||
@@ -29,10 +25,12 @@ export class SendTimeoutService {
|
||||
) {}
|
||||
|
||||
async markUnknownTimeout(data: TimeoutUnknownDto) {
|
||||
const olderThanHours = data.olderThanHours ?? positiveInteger(process.env.SMS_RECEIPT_TIMEOUT_HOURS, DEFAULT_RECEIPT_TIMEOUT_HOURS);
|
||||
const olderThanHours =
|
||||
data.olderThanHours ?? positiveInteger(process.env.SMS_RECEIPT_TIMEOUT_HOURS, DEFAULT_RECEIPT_TIMEOUT_HOURS);
|
||||
const cutoff = new Date(Date.now() - olderThanHours * 60 * 60 * 1000);
|
||||
const candidates = await this.prisma.smsMessageRecord.findMany({
|
||||
where: {
|
||||
id: completionContext.getStore()?.messageRecordId,
|
||||
tenantId: { not: null },
|
||||
OR: [
|
||||
{ status: { in: ['submitted', 'unknown'] }, submittedAt: { lte: cutoff } },
|
||||
@@ -79,7 +77,10 @@ export class SendTimeoutService {
|
||||
// Refund uses the platform-message idempotency key. Re-running it for a
|
||||
// timeout whose downstream outbox was not fully queued also recovers a
|
||||
// crash between the state transition and the original refund call.
|
||||
await this.facade.refundMessage(candidate as typeof candidate & { tenantId: string }, `${olderThanHours}小时未收到明确回执,自动超时退款`);
|
||||
await this.facade.refundMessage(
|
||||
candidate as typeof candidate & { tenantId: string },
|
||||
`${olderThanHours}小时未收到明确回执,自动超时退款`,
|
||||
);
|
||||
const queued = await queueFinalReceiptDeliveries(
|
||||
this.prisma,
|
||||
(request) => this.facade.queueAndTryDownstreamDelivery(request),
|
||||
@@ -117,10 +118,7 @@ export class SendTimeoutService {
|
||||
const expired = await this.prisma.cmppDownstreamDelivery.findMany({
|
||||
where: {
|
||||
status: 'pending',
|
||||
OR: [
|
||||
{ lastRetriedAt: null, createdAt: { lte: cutoff } },
|
||||
{ lastRetriedAt: { lte: cutoff } },
|
||||
],
|
||||
OR: [{ lastRetriedAt: null, createdAt: { lte: cutoff } }, { lastRetriedAt: { lte: cutoff } }],
|
||||
},
|
||||
select: { id: true },
|
||||
take: 500,
|
||||
@@ -139,16 +137,21 @@ export class SendTimeoutService {
|
||||
if (this.receiptTimeoutScanRunning) return;
|
||||
this.receiptTimeoutScanRunning = true;
|
||||
try {
|
||||
const [receiptResult, downstreamResult, requeueRecoveryResult, downstreamManualRecoveryResult] = await Promise.all([
|
||||
this.facade.markUnknownTimeout({}),
|
||||
this.facade.markExpiredDownstreamDeliveries(),
|
||||
this.facade.recoverStaleGatewaySubmitRequeues(),
|
||||
this.facade.recoverStaleDownstreamManualRequeues(),
|
||||
]);
|
||||
if (receiptResult.timeout > 0) this.logger.log(`Marked ${receiptResult.timeout} messages as receipt timeout and refunded charged messages`);
|
||||
if (downstreamResult.failed > 0) this.logger.log(`Terminated ${downstreamResult.failed} expired downstream deliveries`);
|
||||
if (requeueRecoveryResult.recovered > 0) this.logger.log(`Recovered ${requeueRecoveryResult.recovered} stale Gateway submit requeues`);
|
||||
if (downstreamManualRecoveryResult.recovered > 0) this.logger.log(`Recovered ${downstreamManualRecoveryResult.recovered} stale downstream manual requeues`);
|
||||
const [receiptResult, downstreamResult, requeueRecoveryResult, downstreamManualRecoveryResult] =
|
||||
await Promise.all([
|
||||
this.facade.markUnknownTimeout({}),
|
||||
this.facade.markExpiredDownstreamDeliveries(),
|
||||
this.facade.recoverStaleGatewaySubmitRequeues(),
|
||||
this.facade.recoverStaleDownstreamManualRequeues(),
|
||||
]);
|
||||
if (receiptResult.timeout > 0)
|
||||
this.logger.log(`Marked ${receiptResult.timeout} messages as receipt timeout and refunded charged messages`);
|
||||
if (downstreamResult.failed > 0)
|
||||
this.logger.log(`Terminated ${downstreamResult.failed} expired downstream deliveries`);
|
||||
if (requeueRecoveryResult.recovered > 0)
|
||||
this.logger.log(`Recovered ${requeueRecoveryResult.recovered} stale Gateway submit requeues`);
|
||||
if (downstreamManualRecoveryResult.recovered > 0)
|
||||
this.logger.log(`Recovered ${downstreamManualRecoveryResult.recovered} stale downstream manual requeues`);
|
||||
} catch (error) {
|
||||
this.logger.error('Receipt timeout scan failed', error instanceof Error ? error.stack : String(error));
|
||||
} finally {
|
||||
|
||||
Reference in New Issue
Block a user