feat: complete cmpp gateway delivery recovery workflows

This commit is contained in:
hectorzhao
2026-07-08 16:30:06 +08:00
parent cc628d0214
commit 8144f08652
60 changed files with 8901 additions and 94 deletions
@@ -1,11 +1,20 @@
import { Controller, Get, Query } from '@nestjs/common';
import { Body, Controller, Get, Param, Post, Query, Res } from '@nestjs/common';
import { ApiTags } from '@nestjs/swagger';
import { SendChainService } from '../send-chain/send-chain.service';
import { OperationsService } from './operations.service';
type DownloadResponse = {
setHeader(name: string, value: number | string): void;
send(content: string | Buffer): void;
};
@ApiTags('operations')
@Controller('admin/operations')
export class AdminOperationsController {
constructor(private readonly operations: OperationsService) {}
constructor(
private readonly operations: OperationsService,
private readonly sendChain: SendChainService,
) {}
@Get('monitor')
monitor(@Query('tenantId') tenantId?: string, @Query('channelId') channelId?: string) {
@@ -30,11 +39,27 @@ export class AdminOperationsController {
return this.operations.listMessages({ tenantId, applicationId, channelId, taskId, messageId, phoneNumber, status });
}
@Get('message-segment-audits')
messageSegmentAudits(
@Query('messageId') messageId?: string,
@Query('messageRecordId') messageRecordId?: string,
) {
return this.operations.listMessageSegmentAudits({ messageId, messageRecordId });
}
@Get('uplink-messages')
listUplinkMessages(@Query('tenantId') tenantId?: string, @Query('channelId') channelId?: string) {
return this.operations.listUplinkMessages({ tenantId, channelId });
}
@Post('uplink-messages/:id/claim')
claimUplinkMatchCandidate(
@Param('id') id: string,
@Body() body: { candidateId?: string; operatorId?: string },
) {
return this.sendChain.claimUplinkMatchCandidate(id, String(body.candidateId ?? ''), body.operatorId);
}
@Get('dashboard')
dashboard(@Query('tenantId') tenantId?: string) {
return this.operations.dashboard({ tenantId });
@@ -76,6 +101,123 @@ export class AdminOperationsController {
reconciliation(@Query('tenantId') tenantId?: string, @Query('taskId') taskId?: string) {
return this.operations.reconciliation({ tenantId, taskId });
}
@Get('gateway-submit-dead-letters')
gatewaySubmitDeadLetters(
@Query('tenantId') tenantId?: string,
@Query('applicationId') applicationId?: string,
@Query('channelId') channelId?: string,
@Query('status') status?: string,
@Query('keyword') keyword?: string,
@Query('page') page?: string,
@Query('pageSize') pageSize?: string,
) {
return this.operations.listGatewaySubmitDeadLetters({
tenantId,
applicationId,
channelId,
status,
keyword,
page: Number(page),
pageSize: Number(pageSize),
});
}
@Post('gateway-submit-dead-letters/:id/requeue')
requeueGatewaySubmitDeadLetter(@Param('id') id: string) {
return this.sendChain.requeueGatewaySubmitDeadLetter(id);
}
@Get('downstream-deliveries')
downstreamDeliveries(
@Query('tenantId') tenantId?: string,
@Query('applicationId') applicationId?: string,
@Query('deliveryType') deliveryType?: string,
@Query('status') status?: string,
@Query('keyword') keyword?: string,
@Query('page') page?: string,
@Query('pageSize') pageSize?: string,
) {
return this.operations.listDownstreamDeliveries({
tenantId,
applicationId,
deliveryType,
status,
keyword,
page: Number(page),
pageSize: Number(pageSize),
});
}
@Get('downstream-deliveries/dashboard')
downstreamDeliveryDashboard(
@Query('tenantId') tenantId?: string,
@Query('applicationId') applicationId?: string,
@Query('deliveryType') deliveryType?: string,
) {
return this.operations.downstreamDeliveryDashboard({
tenantId,
applicationId,
deliveryType,
});
}
@Get('downstream-recovery-statuses')
downstreamRecoveryStatuses(
@Query('tenantId') tenantId?: string,
@Query('applicationId') applicationId?: string,
@Query('state') state?: string,
@Query('failureCategory') failureCategory?: string,
@Query('keyword') keyword?: string,
@Query('page') page?: string,
@Query('pageSize') pageSize?: string,
) {
return this.operations.listDownstreamRecoveryStatuses({
tenantId,
applicationId,
state,
failureCategory,
keyword,
page: Number(page),
pageSize: Number(pageSize),
});
}
@Get('downstream-recovery-statuses/export')
async exportDownstreamRecoveryStatuses(
@Query('tenantId') tenantId: string | undefined,
@Query('applicationId') applicationId: string | undefined,
@Query('state') state: string | undefined,
@Query('failureCategory') failureCategory: string | undefined,
@Query('keyword') keyword: string | undefined,
@Res() response: DownloadResponse,
) {
const exported = await this.operations.exportDownstreamRecoveryStatuses({
tenantId,
applicationId,
state,
failureCategory,
keyword,
});
response.setHeader('Content-Type', 'text/csv; charset=utf-8');
response.setHeader('Content-Disposition', `attachment; filename*=UTF-8''${encodeURIComponent(exported.fileName)}`);
response.send(`\uFEFF${exported.content}`);
}
@Get('downstream-recovery-statuses/:id')
downstreamRecoveryStatusDetail(@Param('id') id: string) {
return this.operations.getDownstreamRecoveryStatus(id);
}
@Post('downstream-deliveries/:id/requeue')
requeueDownstreamDelivery(@Param('id') id: string) {
return this.sendChain.requeueDownstreamDelivery(id);
}
@Post('downstream-deliveries/requeue')
batchRequeueDownstreamDeliveries(@Body() body: { ids?: string[] }) {
return this.sendChain.batchRequeueDownstreamDeliveries(body.ids ?? []);
}
}
@ApiTags('admin-system-logs')
+2 -1
View File
@@ -1,11 +1,12 @@
import { Module } from '@nestjs/common';
import { PrismaModule } from '../prisma/prisma.module';
import { SendChainModule } from '../send-chain/send-chain.module';
import { AdminOperationsController, AdminSystemLogsController } from './admin-operations.controller';
import { ClientOperationsController } from './client-operations.controller';
import { OperationsService } from './operations.service';
@Module({
imports: [PrismaModule],
imports: [PrismaModule, SendChainModule],
controllers: [AdminOperationsController, AdminSystemLogsController, ClientOperationsController],
providers: [OperationsService],
exports: [OperationsService],
+346 -1
View File
@@ -40,6 +40,12 @@ function createPrismaMock() {
enterpriseCertification: {
count: jest.fn().mockResolvedValue(1),
},
smsApplication: {
findMany: jest.fn().mockResolvedValue([
{ id: 'app-1', name: '应用A' },
{ id: 'app-2', name: '应用B' },
]),
},
cmppConnectionState: {
groupBy: jest.fn().mockResolvedValue([{ status: 'connected', _count: { _all: 1 }, _sum: { currentConnections: 2, desiredConnections: 2 } }]),
},
@@ -58,6 +64,101 @@ function createPrismaMock() {
count: jest.fn().mockResolvedValue(1),
groupBy: jest.fn().mockResolvedValue([{ resource: 'recharge_order', _count: { _all: 1 } }]),
},
gatewaySubmitDeadLetter: {
findMany: jest.fn().mockResolvedValue([{
id: 'dead-1',
streamMessageId: '1710000000000-0',
status: 'pending',
failureCode: 'SUBMIT_PROCESSING_FAILED',
failureMessage: 'network down',
tenant: { name: '租户A' },
application: { name: '应用A' },
channel: { code: 'CMPP-A' },
}]),
count: jest.fn().mockResolvedValue(1),
},
gatewayDownstreamRecoveryStatus: {
findMany: jest.fn().mockResolvedValue([{
id: 'recover-1',
account: '100001',
state: 'waiting_connection',
lockOwner: 'gateway-a',
lockExpiresAt: new Date('2026-07-08T12:00:30.000Z'),
attemptCount: 2,
failureCategory: 'client_disconnected',
nextRetryAt: new Date('2026-07-08T12:10:00.000Z'),
lastError: 'downstream client is not connected',
tenant: { name: '租户A' },
application: { name: '应用A' },
}]),
findUnique: jest.fn().mockResolvedValue({
id: 'recover-1',
account: '100001',
state: 'waiting_connection',
lockOwner: 'gateway-a',
lockExpiresAt: new Date('2026-07-08T12:00:30.000Z'),
attemptCount: 2,
failureCategory: 'client_disconnected',
nextRetryAt: new Date('2026-07-08T12:10:00.000Z'),
lastAttemptAt: new Date('2026-07-08T12:00:00.000Z'),
lastSuccessAt: null,
lastFailureAt: new Date('2026-07-08T11:58:00.000Z'),
lastError: 'downstream client is not connected',
lastSkipReason: 'waiting for reconnect',
gatewayInstanceId: 'gw-01',
createdAt: new Date('2026-07-08T11:50:00.000Z'),
updatedAt: new Date('2026-07-08T12:00:00.000Z'),
tenant: { name: '租户A' },
application: { name: '应用A' },
}),
count: jest.fn().mockResolvedValue(1),
groupBy: jest.fn().mockResolvedValue([
{ failureCategory: 'client_disconnected', _count: { _all: 1 } },
]),
},
smsMessageSegmentAudit: {
findMany: jest.fn().mockResolvedValue([{
id: 'segment-1',
messageRecordId: 'record-1',
submitId: 'SUB-1',
segmentTotal: 2,
segmentIndex: 1,
sequenceId: 7,
gatewayMessageId: 'GW-1-A',
submitStatus: 'accepted',
receiptStatus: 'delivered',
channel: { name: '通道A' },
}]),
},
cmppDownstreamDelivery: {
findMany: jest.fn().mockResolvedValue([{
id: 'delivery-1',
tenantId: 'tenant-1',
applicationId: 'app-1',
messageId: 'MSG-1',
deliveryType: 'receipt',
status: 'failed',
payload: { account: '100001', phoneNumber: '13800000001' },
retryCount: 10,
lastError: 'client offline',
tenant: { name: '租户A' },
application: { name: '应用A' },
messageRecord: { messageId: 'MSG-1' },
}]),
count: jest.fn().mockResolvedValue(1),
groupBy: jest.fn().mockResolvedValue([
{ deliveryType: 'receipt', status: 'pending', _count: { _all: 2 } },
{ deliveryType: 'receipt', status: 'failed', _count: { _all: 1 } },
{ deliveryType: 'receipt', status: 'delivered', _count: { _all: 6 } },
{ deliveryType: 'uplink', status: 'pending', _count: { _all: 1 } },
{ deliveryType: 'uplink', status: 'delivered', _count: { _all: 2 } },
{ applicationId: 'app-1', status: 'pending', _count: { _all: 2 } },
{ applicationId: 'app-1', status: 'failed', _count: { _all: 1 } },
{ applicationId: 'app-1', status: 'delivered', _count: { _all: 5 } },
{ applicationId: 'app-2', status: 'pending', _count: { _all: 1 } },
{ applicationId: 'app-2', status: 'delivered', _count: { _all: 3 } },
]),
},
};
}
@@ -100,7 +201,20 @@ describe('OperationsService', () => {
expect(prisma.smsUplinkMessage.findMany).toHaveBeenCalledWith({
where: { tenantId: 'tenant-1', channelId: 'channel-1' },
include: { tenant: true, channel: true },
include: {
tenant: true,
application: true,
channel: true,
messageRecord: { include: { application: true } },
matchCandidates: {
include: {
tenant: true,
application: true,
messageRecord: { include: { application: true, tenant: true, channel: true } },
},
orderBy: [{ status: 'asc' }, { confidence: 'desc' }, { createdAt: 'asc' }],
},
},
orderBy: { receivedAt: 'desc' },
take: 500,
});
@@ -108,6 +222,12 @@ describe('OperationsService', () => {
it('builds dashboard and statistics aggregates', async () => {
const prisma = createPrismaMock();
prisma.cmppDownstreamDelivery.count = jest.fn()
.mockResolvedValueOnce(3)
.mockResolvedValueOnce(2)
.mockResolvedValueOnce(8)
.mockResolvedValueOnce(1)
.mockResolvedValueOnce(2);
const service = new OperationsService(prisma as never);
await expect(service.dashboard({ tenantId: 'tenant-1' })).resolves.toEqual(
@@ -116,6 +236,14 @@ describe('OperationsService', () => {
uplinkCount: 1,
pendingAuditCount: 6,
gatewayConnections: [{ status: 'connected', _count: { _all: 1 }, _sum: { currentConnections: 2, desiredConnections: 2 } }],
downstreamDeliverySummary: expect.objectContaining({
pending: 3,
failed: 2,
delivered: 8,
stalledPending: 1,
recentFailed: 2,
alertCount: 3,
}),
}),
);
await service.statistics({ tenantId: 'tenant-1', groupBy: 'application' });
@@ -169,4 +297,221 @@ describe('OperationsService', () => {
}),
);
});
it('returns paginated gateway submit dead letters', async () => {
const prisma = createPrismaMock();
const service = new OperationsService(prisma as never);
await expect(service.listGatewaySubmitDeadLetters({
tenantId: 'tenant-1',
status: 'pending',
keyword: 'SUBMIT',
page: 1,
pageSize: 10,
})).resolves.toEqual({
items: [expect.objectContaining({ id: 'dead-1', status: 'pending' })],
total: 1,
page: 1,
pageSize: 10,
});
expect(prisma.gatewaySubmitDeadLetter.findMany).toHaveBeenCalledWith({
where: expect.objectContaining({
tenantId: 'tenant-1',
status: 'pending',
}),
include: { tenant: true, application: true, channel: true },
orderBy: { createdAt: 'desc' },
skip: 0,
take: 10,
});
});
it('returns paginated downstream deliveries', async () => {
const prisma = createPrismaMock();
const service = new OperationsService(prisma as never);
await expect(service.listDownstreamDeliveries({
tenantId: 'tenant-1',
deliveryType: 'receipt',
status: 'failed',
keyword: '1380',
page: 1,
pageSize: 10,
})).resolves.toEqual({
items: [expect.objectContaining({ id: 'delivery-1', status: 'failed' })],
total: 1,
page: 1,
pageSize: 10,
});
expect(prisma.cmppDownstreamDelivery.findMany).toHaveBeenCalledWith({
where: expect.objectContaining({
tenantId: 'tenant-1',
deliveryType: 'receipt',
status: 'failed',
}),
include: { tenant: true, application: true, messageRecord: true },
orderBy: { createdAt: 'desc' },
skip: 0,
take: 10,
});
});
it('builds downstream delivery dashboard aggregates', async () => {
const prisma = createPrismaMock();
prisma.cmppDownstreamDelivery.count = jest.fn()
.mockResolvedValueOnce(12)
.mockResolvedValueOnce(3)
.mockResolvedValueOnce(8)
.mockResolvedValueOnce(1)
.mockResolvedValueOnce(1)
.mockResolvedValueOnce(1)
.mockResolvedValueOnce(2)
.mockResolvedValueOnce(1)
.mockResolvedValueOnce(0);
prisma.cmppDownstreamDelivery.groupBy = jest.fn()
.mockResolvedValueOnce([
{ deliveryType: 'receipt', status: 'pending', _count: { _all: 2 } },
{ deliveryType: 'receipt', status: 'failed', _count: { _all: 1 } },
{ deliveryType: 'receipt', status: 'delivered', _count: { _all: 6 } },
{ deliveryType: 'uplink', status: 'pending', _count: { _all: 1 } },
{ deliveryType: 'uplink', status: 'delivered', _count: { _all: 2 } },
])
.mockResolvedValueOnce([
{ applicationId: 'app-1', status: 'pending', _count: { _all: 2 } },
{ applicationId: 'app-1', status: 'failed', _count: { _all: 1 } },
{ applicationId: 'app-1', status: 'delivered', _count: { _all: 5 } },
{ applicationId: 'app-2', status: 'pending', _count: { _all: 1 } },
{ applicationId: 'app-2', status: 'delivered', _count: { _all: 3 } },
]);
const service = new OperationsService(prisma as never);
await expect(service.downstreamDeliveryDashboard({
tenantId: 'tenant-1',
applicationId: 'app-1',
deliveryType: 'all',
})).resolves.toEqual({
summary: {
total: 12,
pending: 3,
delivered: 8,
failed: 1,
stalledPending: 1,
recentFailed: 1,
alertCount: 2,
},
typeBreakdown: [
{ deliveryType: 'receipt', total: 9, pending: 2, delivered: 6, failed: 1 },
{ deliveryType: 'uplink', total: 3, pending: 1, delivered: 2, failed: 0 },
],
retryBuckets: [
{ label: '0次', count: 2 },
{ label: '1-3次', count: 1 },
{ label: '4次及以上', count: 0 },
],
topApplications: [
{ applicationId: 'app-1', name: '应用A', pending: 2, failed: 1, delivered: 5, alertCount: 3 },
{ applicationId: 'app-2', name: '应用B', pending: 1, failed: 0, delivered: 3, alertCount: 1 },
],
});
});
it('returns paginated downstream recovery statuses', async () => {
const prisma = createPrismaMock();
prisma.gatewayDownstreamRecoveryStatus.count = jest.fn()
.mockResolvedValueOnce(1)
.mockResolvedValueOnce(0)
.mockResolvedValueOnce(0)
.mockResolvedValueOnce(0)
.mockResolvedValueOnce(1)
.mockResolvedValueOnce(1);
const service = new OperationsService(prisma as never);
await expect(service.listDownstreamRecoveryStatuses({
tenantId: 'tenant-1',
applicationId: 'app-1',
state: 'waiting_connection',
failureCategory: 'client_disconnected',
keyword: '100001',
page: 1,
pageSize: 10,
})).resolves.toEqual({
items: [expect.objectContaining({ id: 'recover-1', account: '100001', state: 'waiting_connection' })],
total: 1,
page: 1,
pageSize: 10,
summary: {
total: 1,
running: 0,
success: 0,
failed: 0,
waitingConnection: 1,
backoff: 1,
failureCategories: [
{ category: 'client_disconnected', count: 1 },
],
},
});
});
it('returns downstream recovery status detail', async () => {
const prisma = createPrismaMock();
const service = new OperationsService(prisma as never);
await expect(service.getDownstreamRecoveryStatus('recover-1')).resolves.toEqual(
expect.objectContaining({
id: 'recover-1',
account: '100001',
gatewayInstanceId: 'gw-01',
}),
);
expect(prisma.gatewayDownstreamRecoveryStatus.findUnique).toHaveBeenCalledWith({
where: { id: 'recover-1' },
include: { tenant: true, application: true },
});
});
it('exports downstream recovery statuses as csv rows', async () => {
const prisma = createPrismaMock();
const service = new OperationsService(prisma as never);
await expect(service.exportDownstreamRecoveryStatuses({
tenantId: 'tenant-1',
state: 'waiting_connection',
keyword: '100001',
})).resolves.toEqual(expect.objectContaining({
total: 1,
fileName: expect.stringMatching(/^gateway-downstream-recovery-statuses-\d{8}-\d{6}\.csv$/),
content: expect.stringContaining('100001'),
}));
expect(prisma.gatewayDownstreamRecoveryStatus.findMany).toHaveBeenCalledWith(expect.objectContaining({
where: expect.objectContaining({
failureCategory: undefined,
}),
}));
});
it('returns message segment audit rows', async () => {
const prisma = createPrismaMock();
const service = new OperationsService(prisma as never);
await expect(service.listMessageSegmentAudits({ messageRecordId: 'record-1' })).resolves.toEqual([
expect.objectContaining({
id: 'segment-1',
segmentIndex: 1,
gatewayMessageId: 'GW-1-A',
}),
]);
expect(prisma.smsMessageSegmentAudit.findMany).toHaveBeenCalledWith({
where: {
messageRecordId: 'record-1',
messageRecord: undefined,
},
include: { channel: true, submitRecord: true },
orderBy: [{ submitId: 'asc' }, { segmentIndex: 'asc' }],
});
});
});
+508 -2
View File
@@ -1,4 +1,4 @@
import { Injectable } from '@nestjs/common';
import { Injectable, NotFoundException } from '@nestjs/common';
import { Prisma } from '@prisma/client';
import { PrismaService } from '../prisma/prisma.service';
@@ -27,6 +27,47 @@ export interface OperationLogQuery {
pageSize?: number;
}
export interface GatewaySubmitDeadLetterQuery {
tenantId?: string;
applicationId?: string;
channelId?: string;
status?: string;
keyword?: string;
page?: number;
pageSize?: number;
}
export interface DownstreamDeliveryQuery {
tenantId?: string;
applicationId?: string;
deliveryType?: string;
status?: string;
keyword?: string;
page?: number;
pageSize?: number;
}
export interface DownstreamDeliveryDashboardQuery {
tenantId?: string;
applicationId?: string;
deliveryType?: string;
}
export interface DownstreamRecoveryStatusQuery {
tenantId?: string;
applicationId?: string;
state?: string;
failureCategory?: string;
keyword?: string;
page?: number;
pageSize?: number;
}
export interface MessageSegmentAuditQuery {
messageId?: string;
messageRecordId?: string;
}
@Injectable()
export class OperationsService {
constructor(private readonly prisma: PrismaService) {}
@@ -52,7 +93,20 @@ export class OperationsService {
listUplinkMessages(query: { tenantId?: string; channelId?: string }) {
return this.prisma.smsUplinkMessage.findMany({
where: { tenantId: query.tenantId, channelId: query.channelId },
include: { tenant: true, channel: true },
include: {
tenant: true,
application: true,
channel: true,
messageRecord: { include: { application: true } },
matchCandidates: {
include: {
tenant: true,
application: true,
messageRecord: { include: { application: true, tenant: true, channel: true } },
},
orderBy: [{ status: 'asc' }, { confidence: 'desc' }, { createdAt: 'asc' }],
},
},
orderBy: { receivedAt: 'desc' },
take: 500,
});
@@ -99,6 +153,11 @@ export class OperationsService {
tenantAccounts,
recentTasks,
recentRecharges,
downstreamPendingCount,
downstreamFailedCount,
downstreamDeliveredCount,
downstreamStalledPendingCount,
downstreamRecentFailedCount,
] = await Promise.all([
this.prisma.smsBatchTask.count({ where: { tenantId: query.tenantId } }),
this.prisma.smsMessageRecord.groupBy({
@@ -152,8 +211,32 @@ export class OperationsService {
orderBy: { createdAt: 'desc' },
take: 10,
}),
this.prisma.cmppDownstreamDelivery.count({
where: { tenantId: query.tenantId, status: 'pending' },
}),
this.prisma.cmppDownstreamDelivery.count({
where: { tenantId: query.tenantId, status: 'failed' },
}),
this.prisma.cmppDownstreamDelivery.count({
where: { tenantId: query.tenantId, status: 'delivered' },
}),
this.prisma.cmppDownstreamDelivery.count({
where: {
tenantId: query.tenantId,
status: 'pending',
createdAt: { lte: new Date(Date.now() - downstreamAlertPendingMinutes() * 60_000) },
},
}),
this.prisma.cmppDownstreamDelivery.count({
where: {
tenantId: query.tenantId,
status: 'failed',
updatedAt: { gte: new Date(Date.now() - downstreamAlertRecentFailedHours() * 60 * 60_000) },
},
}),
]);
const todayTotals = summarizeMessageGroups(todayMessageGroups);
const downstreamAlertCount = downstreamStalledPendingCount + downstreamRecentFailedCount;
return {
taskCount,
messageStatus: messageGroups,
@@ -171,6 +254,14 @@ export class OperationsService {
transactions: transactionAggregate,
gatewayConnections: connectionGroups,
pendingAuditCount,
downstreamDeliverySummary: {
pending: downstreamPendingCount,
failed: downstreamFailedCount,
delivered: downstreamDeliveredCount,
stalledPending: downstreamStalledPendingCount,
recentFailed: downstreamRecentFailedCount,
alertCount: downstreamAlertCount,
},
accounts: tenantAccounts,
recentTasks,
recentRecharges,
@@ -256,6 +347,303 @@ export class OperationsService {
};
}
async listGatewaySubmitDeadLetters(query: GatewaySubmitDeadLetterQuery) {
const page = Math.max(1, Number(query.page ?? 1));
const pageSize = Math.min(100, Math.max(1, Number(query.pageSize ?? 10)));
const where: Prisma.GatewaySubmitDeadLetterWhereInput = {
tenantId: query.tenantId,
applicationId: query.applicationId,
channelId: query.channelId,
status: query.status && query.status !== 'all' ? query.status : undefined,
OR: query.keyword ? [
{ streamMessageId: { contains: query.keyword } },
{ traceId: { contains: query.keyword } },
{ messageId: { contains: query.keyword } },
{ submitId: { contains: query.keyword } },
{ failureCode: { contains: query.keyword } },
{ failureMessage: { contains: query.keyword } },
] : undefined,
};
const [items, total] = await Promise.all([
this.prisma.gatewaySubmitDeadLetter.findMany({
where,
include: { tenant: true, application: true, channel: true },
orderBy: { createdAt: 'desc' },
skip: (page - 1) * pageSize,
take: pageSize,
}),
this.prisma.gatewaySubmitDeadLetter.count({ where }),
]);
return { items, total, page, pageSize };
}
async listDownstreamDeliveries(query: DownstreamDeliveryQuery) {
const page = Math.max(1, Number(query.page ?? 1));
const pageSize = Math.min(100, Math.max(1, Number(query.pageSize ?? 10)));
const where: Prisma.CmppDownstreamDeliveryWhereInput = {
...downstreamDeliveryScopedWhere(query),
status: query.status && query.status !== 'all' ? query.status : undefined,
OR: query.keyword ? [
{ messageId: { contains: query.keyword } },
{ payload: { path: ['account'], string_contains: query.keyword } },
{ payload: { path: ['phoneNumber'], string_contains: query.keyword } },
{ lastError: { contains: query.keyword } },
] : undefined,
};
const [items, total] = await Promise.all([
this.prisma.cmppDownstreamDelivery.findMany({
where,
include: { tenant: true, application: true, messageRecord: true },
orderBy: { createdAt: 'desc' },
skip: (page - 1) * pageSize,
take: pageSize,
}),
this.prisma.cmppDownstreamDelivery.count({ where }),
]);
return { items, total, page, pageSize };
}
async downstreamDeliveryDashboard(query: DownstreamDeliveryDashboardQuery) {
const scopedWhere = downstreamDeliveryScopedWhere(query);
const stalledPendingAt = new Date(Date.now() - downstreamAlertPendingMinutes() * 60_000);
const recentFailedAt = new Date(Date.now() - downstreamAlertRecentFailedHours() * 60 * 60_000);
const [total, pending, delivered, failed, stalledPending, recentFailed, typeGroups, applicationGroups, retryZero, retryLow, retryHigh] = await Promise.all([
this.prisma.cmppDownstreamDelivery.count({ where: scopedWhere }),
this.prisma.cmppDownstreamDelivery.count({ where: { ...scopedWhere, status: 'pending' } }),
this.prisma.cmppDownstreamDelivery.count({ where: { ...scopedWhere, status: 'delivered' } }),
this.prisma.cmppDownstreamDelivery.count({ where: { ...scopedWhere, status: 'failed' } }),
this.prisma.cmppDownstreamDelivery.count({
where: {
...scopedWhere,
status: 'pending',
createdAt: { lte: stalledPendingAt },
},
}),
this.prisma.cmppDownstreamDelivery.count({
where: {
...scopedWhere,
status: 'failed',
updatedAt: { gte: recentFailedAt },
},
}),
this.prisma.cmppDownstreamDelivery.groupBy({
by: ['deliveryType', 'status'],
where: scopedWhere,
_count: { _all: true },
}),
this.prisma.cmppDownstreamDelivery.groupBy({
by: ['applicationId', 'status'],
where: scopedWhere,
_count: { _all: true },
}),
this.prisma.cmppDownstreamDelivery.count({
where: {
...scopedWhere,
status: { in: ['pending', 'failed'] },
retryCount: 0,
},
}),
this.prisma.cmppDownstreamDelivery.count({
where: {
...scopedWhere,
status: { in: ['pending', 'failed'] },
retryCount: { gte: 1, lte: 3 },
},
}),
this.prisma.cmppDownstreamDelivery.count({
where: {
...scopedWhere,
status: { in: ['pending', 'failed'] },
retryCount: { gte: 4 },
},
}),
]);
const applicationIds = [...new Set(applicationGroups.map((item) => item.applicationId).filter((value): value is string => Boolean(value)))];
const applications: Array<{ id: string; name: string }> = applicationIds.length > 0
? await this.prisma.smsApplication.findMany({
where: { id: { in: applicationIds } },
select: { id: true, name: true },
})
: [];
const applicationMap = new Map<string, string>(applications.map((item) => [item.id, item.name]));
const groupedByType = groupDownstreamByType(typeGroups);
const groupedByApplication = groupDownstreamByApplication(applicationGroups, applicationMap);
return {
summary: {
total,
pending,
delivered,
failed,
stalledPending,
recentFailed,
alertCount: stalledPending + recentFailed,
},
typeBreakdown: ['receipt', 'uplink'].map((deliveryType) => ({
deliveryType,
total: groupedByType[deliveryType]?.total ?? 0,
pending: groupedByType[deliveryType]?.pending ?? 0,
delivered: groupedByType[deliveryType]?.delivered ?? 0,
failed: groupedByType[deliveryType]?.failed ?? 0,
})),
retryBuckets: [
{ label: '0次', count: retryZero },
{ label: '1-3次', count: retryLow },
{ label: '4次及以上', count: retryHigh },
],
topApplications: groupedByApplication
.sort((left, right) => (
right.alertCount - left.alertCount
|| right.failed - left.failed
|| right.pending - left.pending
|| left.name.localeCompare(right.name, 'zh-CN')
))
.slice(0, 5),
};
}
async listDownstreamRecoveryStatuses(query: DownstreamRecoveryStatusQuery) {
const recoveryStatuses = this.gatewayDownstreamRecoveryStatusDelegate();
const page = Math.max(1, Number(query.page ?? 1));
const pageSize = Math.min(100, Math.max(1, Number(query.pageSize ?? 10)));
const where = downstreamRecoveryStatusWhere(query);
const now = new Date();
const [items, total, runningCount, successCount, failedCount, waitingConnectionCount, backoffCount, categoryGroups] = await Promise.all([
recoveryStatuses.findMany({
where,
include: { tenant: true, application: true },
orderBy: [{ updatedAt: 'desc' }, { account: 'asc' }],
skip: (page - 1) * pageSize,
take: pageSize,
}),
recoveryStatuses.count({ where }),
recoveryStatuses.count({ where: { ...where, state: 'running' } }),
recoveryStatuses.count({ where: { ...where, state: 'success' } }),
recoveryStatuses.count({ where: { ...where, state: 'failed' } }),
recoveryStatuses.count({ where: { ...where, state: 'waiting_connection' } }),
recoveryStatuses.count({
where: {
...where,
nextRetryAt: { gt: now },
},
}),
recoveryStatuses.groupBy({
by: ['failureCategory'],
where,
_count: { _all: true },
}),
]);
return {
items,
total,
page,
pageSize,
summary: {
total,
running: runningCount,
success: successCount,
failed: failedCount,
waitingConnection: waitingConnectionCount,
backoff: backoffCount,
failureCategories: categoryGroups
.filter((item) => item.failureCategory)
.map((item) => ({
category: String(item.failureCategory),
count: item._count?._all ?? 0,
}))
.sort((left, right) => right.count - left.count || left.category.localeCompare(right.category)),
},
};
}
async listMessageSegmentAudits(query: MessageSegmentAuditQuery) {
const segmentAudits = (this.prisma as PrismaService & {
smsMessageSegmentAudit: {
findMany: (args: Record<string, unknown>) => Promise<any[]>;
};
}).smsMessageSegmentAudit;
if (!query.messageId && !query.messageRecordId) {
return [];
}
return segmentAudits.findMany({
where: {
messageRecordId: query.messageRecordId,
messageRecord: query.messageId ? { messageId: query.messageId } : undefined,
},
include: { channel: true, submitRecord: true },
orderBy: [{ submitId: 'asc' }, { segmentIndex: 'asc' }],
});
}
async getDownstreamRecoveryStatus(id: string) {
const recoveryStatuses = this.gatewayDownstreamRecoveryStatusDelegate();
const item = await recoveryStatuses.findUnique({
where: { id },
include: { tenant: true, application: true },
});
if (!item) {
throw new NotFoundException('Recovery status not found');
}
return item;
}
async exportDownstreamRecoveryStatuses(query: DownstreamRecoveryStatusQuery) {
const recoveryStatuses = this.gatewayDownstreamRecoveryStatusDelegate();
const where = downstreamRecoveryStatusWhere(query);
const items = await recoveryStatuses.findMany({
where,
include: { tenant: true, application: true },
orderBy: [{ updatedAt: 'desc' }, { account: 'asc' }],
take: 5000,
});
const rows = [
[
'账号',
'企业',
'应用',
'Gateway实例',
'恢复状态',
'锁持有实例',
'锁过期时间',
'失败分类',
'尝试次数',
'最后尝试时间',
'恢复成功时间',
'恢复失败时间',
'下次恢复时间',
'最后错误',
'最后跳过原因',
'创建时间',
'更新时间',
],
...items.map((item) => [
item.account ?? '',
item.tenant?.name ?? '',
item.application?.name ?? '',
item.gatewayInstanceId ?? '',
item.state ?? '',
(item as { lockOwner?: string | null }).lockOwner ?? '',
formatCsvDate((item as { lockExpiresAt?: Date | string | null }).lockExpiresAt),
(item as { failureCategory?: string | null }).failureCategory ?? '',
String(item.attemptCount ?? 0),
formatCsvDate(item.lastAttemptAt),
formatCsvDate(item.lastSuccessAt),
formatCsvDate(item.lastFailureAt),
formatCsvDate(item.nextRetryAt),
item.lastError ?? '',
item.lastSkipReason ?? '',
formatCsvDate(item.createdAt),
formatCsvDate(item.updatedAt),
]),
];
return {
fileName: `gateway-downstream-recovery-statuses-${formatExportTimestamp(new Date())}.csv`,
content: rows.map((row) => row.map(escapeCsvCell).join(',')).join('\n'),
total: items.length,
};
}
auditSummary(query: { tenantId?: string }) {
return this.prisma.operationLog.groupBy({
by: ['action', 'resource'],
@@ -346,6 +734,17 @@ export class OperationsService {
this.prisma.smsBatchTask.count({ where: { tenantId, auditStatus: 'pending' } }),
]).then((counts) => counts.reduce((sum, value) => sum + value, 0));
}
private gatewayDownstreamRecoveryStatusDelegate() {
return (this.prisma as PrismaService & {
gatewayDownstreamRecoveryStatus: {
findMany: (args: Record<string, unknown>) => Promise<any[]>;
count: (args: Record<string, unknown>) => Promise<number>;
findUnique: (args: Record<string, unknown>) => Promise<any | null>;
groupBy: (args: Record<string, unknown>) => Promise<any[]>;
};
}).gatewayDownstreamRecoveryStatus;
}
}
function messageWhere(query: MessageQuery): Prisma.SmsMessageRecordWhereInput {
@@ -390,6 +789,68 @@ function createdAtRange(range?: string): Prisma.DateTimeFilter | undefined {
return { gte: date };
}
function downstreamAlertPendingMinutes() {
const value = Number(process.env.CMPP_DOWNSTREAM_ALERT_PENDING_MINUTES ?? 10);
return Number.isFinite(value) && value > 0 ? value : 10;
}
function downstreamAlertRecentFailedHours() {
const value = Number(process.env.CMPP_DOWNSTREAM_ALERT_RECENT_FAILED_HOURS ?? 1);
return Number.isFinite(value) && value > 0 ? value : 1;
}
function downstreamDeliveryScopedWhere(query: DownstreamDeliveryDashboardQuery): Prisma.CmppDownstreamDeliveryWhereInput {
return {
tenantId: query.tenantId,
applicationId: query.applicationId,
deliveryType: query.deliveryType && query.deliveryType !== 'all' ? query.deliveryType : undefined,
};
}
function downstreamRecoveryStatusWhere(query: DownstreamRecoveryStatusQuery) {
return {
tenantId: query.tenantId,
applicationId: query.applicationId,
state: query.state && query.state !== 'all' ? query.state : undefined,
failureCategory: query.failureCategory && query.failureCategory !== 'all' ? query.failureCategory : undefined,
OR: query.keyword ? [
{ account: { contains: query.keyword } },
{ gatewayInstanceId: { contains: query.keyword } },
{ lastError: { contains: query.keyword } },
{ lastSkipReason: { contains: query.keyword } },
{ tenant: { name: { contains: query.keyword } } },
{ application: { name: { contains: query.keyword } } },
] : undefined,
};
}
function escapeCsvCell(value: string) {
const normalized = value.replace(/\r\n/g, '\n').replace(/\r/g, '\n');
if (normalized.includes(',') || normalized.includes('"') || normalized.includes('\n')) {
return `"${normalized.replace(/"/g, '""')}"`;
}
return normalized;
}
function formatCsvDate(value?: Date | string | null) {
if (!value) {
return '';
}
return value instanceof Date ? value.toISOString() : value;
}
function formatExportTimestamp(date: Date) {
const parts = [
date.getFullYear(),
String(date.getMonth() + 1).padStart(2, '0'),
String(date.getDate()).padStart(2, '0'),
String(date.getHours()).padStart(2, '0'),
String(date.getMinutes()).padStart(2, '0'),
String(date.getSeconds()).padStart(2, '0'),
];
return `${parts[0]}${parts[1]}${parts[2]}-${parts[3]}${parts[4]}${parts[5]}`;
}
function summarizeMessageGroups(groups: Array<{ status: string; _count: { _all: number }; _sum: { amountCents: number | null; billingUnits: number | null } }>) {
return groups.reduce(
(summary, group) => {
@@ -410,6 +871,51 @@ function summarizeMessageGroups(groups: Array<{ status: string; _count: { _all:
);
}
function groupDownstreamByType(
groups: Array<{ deliveryType: string; status: string; _count: { _all: number } }>,
) {
return groups.reduce<Record<string, { total: number; pending: number; delivered: number; failed: number }>>((accumulator, item) => {
const current = accumulator[item.deliveryType] ?? { total: 0, pending: 0, delivered: 0, failed: 0 };
current.total += item._count._all;
if (item.status === 'pending') {
current.pending += item._count._all;
} else if (item.status === 'delivered') {
current.delivered += item._count._all;
} else if (item.status === 'failed') {
current.failed += item._count._all;
}
accumulator[item.deliveryType] = current;
return accumulator;
}, {});
}
function groupDownstreamByApplication(
groups: Array<{ applicationId: string; status: string; _count: { _all: number } }>,
applicationMap: Map<string, string>,
) {
const summaryMap = new Map<string, { applicationId: string; name: string; pending: number; failed: number; delivered: number; alertCount: number }>();
groups.forEach((item) => {
const current = summaryMap.get(item.applicationId) ?? {
applicationId: item.applicationId,
name: applicationMap.get(item.applicationId) ?? item.applicationId,
pending: 0,
failed: 0,
delivered: 0,
alertCount: 0,
};
if (item.status === 'pending') {
current.pending += item._count._all;
} else if (item.status === 'failed') {
current.failed += item._count._all;
} else if (item.status === 'delivered') {
current.delivered += item._count._all;
}
current.alertCount = current.pending + current.failed;
summaryMap.set(item.applicationId, current);
});
return [...summaryMap.values()];
}
function normalizeOperationLog(log: Prisma.OperationLogGetPayload<{ include: { tenant: true; user: true } }>) {
const detail = (log.detail ?? {}) as Record<string, unknown>;
const result = String(detail.result ?? detail.status ?? '');