hectorzhao/lislgosms
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: complete cmpp gateway delivery recovery workflows

Browse Source
This commit is contained in:
hectorzhao
2026-07-08 16:30:06 +08:00
parent cc628d0214
commit 8144f08652
60 changed files with 8901 additions and 94 deletions
Download Patch File Download Diff File Expand all files Collapse all files
api/src/operations/operations.service.ts
+508 -2
View File
@@ -1,4 +1,4 @@
import { Injectable } from '@nestjs/common';
import { Injectable, NotFoundException } from '@nestjs/common';
import { Prisma } from '@prisma/client';
import { PrismaService } from '../prisma/prisma.service';
@@ -27,6 +27,47 @@ export interface OperationLogQuery {
pageSize?: number;
}
export interface GatewaySubmitDeadLetterQuery {
tenantId?: string;
applicationId?: string;
channelId?: string;
status?: string;
keyword?: string;
page?: number;
pageSize?: number;
}
export interface DownstreamDeliveryQuery {
tenantId?: string;
applicationId?: string;
deliveryType?: string;
status?: string;
keyword?: string;
page?: number;
pageSize?: number;
}
export interface DownstreamDeliveryDashboardQuery {
tenantId?: string;
applicationId?: string;
deliveryType?: string;
}
export interface DownstreamRecoveryStatusQuery {
tenantId?: string;
applicationId?: string;
state?: string;
failureCategory?: string;
keyword?: string;
page?: number;
pageSize?: number;
}
export interface MessageSegmentAuditQuery {
messageId?: string;
messageRecordId?: string;
}
@Injectable()
export class OperationsService {
constructor(private readonly prisma: PrismaService) {}
@@ -52,7 +93,20 @@ export class OperationsService {
listUplinkMessages(query: { tenantId?: string; channelId?: string }) {
return this.prisma.smsUplinkMessage.findMany({
where: { tenantId: query.tenantId, channelId: query.channelId },
include: { tenant: true, channel: true },
include: {
tenant: true,
application: true,
channel: true,
messageRecord: { include: { application: true } },
matchCandidates: {
include: {
tenant: true,
application: true,
messageRecord: { include: { application: true, tenant: true, channel: true } },
},
orderBy: [{ status: 'asc' }, { confidence: 'desc' }, { createdAt: 'asc' }],
},
},
orderBy: { receivedAt: 'desc' },
take: 500,
});
@@ -99,6 +153,11 @@ export class OperationsService {
tenantAccounts,
recentTasks,
recentRecharges,
downstreamPendingCount,
downstreamFailedCount,
downstreamDeliveredCount,
downstreamStalledPendingCount,
downstreamRecentFailedCount,
] = await Promise.all([
this.prisma.smsBatchTask.count({ where: { tenantId: query.tenantId } }),
this.prisma.smsMessageRecord.groupBy({
@@ -152,8 +211,32 @@ export class OperationsService {
orderBy: { createdAt: 'desc' },
take: 10,
}),
this.prisma.cmppDownstreamDelivery.count({
where: { tenantId: query.tenantId, status: 'pending' },
}),
this.prisma.cmppDownstreamDelivery.count({
where: { tenantId: query.tenantId, status: 'failed' },
}),
this.prisma.cmppDownstreamDelivery.count({
where: { tenantId: query.tenantId, status: 'delivered' },
}),
this.prisma.cmppDownstreamDelivery.count({
where: {
tenantId: query.tenantId,
status: 'pending',
createdAt: { lte: new Date(Date.now() - downstreamAlertPendingMinutes() * 60_000) },
},
}),
this.prisma.cmppDownstreamDelivery.count({
where: {
tenantId: query.tenantId,
status: 'failed',
updatedAt: { gte: new Date(Date.now() - downstreamAlertRecentFailedHours() * 60 * 60_000) },
},
}),
]);
const todayTotals = summarizeMessageGroups(todayMessageGroups);
const downstreamAlertCount = downstreamStalledPendingCount + downstreamRecentFailedCount;
return {
taskCount,
messageStatus: messageGroups,
@@ -171,6 +254,14 @@ export class OperationsService {
transactions: transactionAggregate,
gatewayConnections: connectionGroups,
pendingAuditCount,
downstreamDeliverySummary: {
pending: downstreamPendingCount,
failed: downstreamFailedCount,
delivered: downstreamDeliveredCount,
stalledPending: downstreamStalledPendingCount,
recentFailed: downstreamRecentFailedCount,
alertCount: downstreamAlertCount,
},
accounts: tenantAccounts,
recentTasks,
recentRecharges,
@@ -256,6 +347,303 @@ export class OperationsService {
};
}
async listGatewaySubmitDeadLetters(query: GatewaySubmitDeadLetterQuery) {
const page = Math.max(1, Number(query.page ?? 1));
const pageSize = Math.min(100, Math.max(1, Number(query.pageSize ?? 10)));
const where: Prisma.GatewaySubmitDeadLetterWhereInput = {
tenantId: query.tenantId,
applicationId: query.applicationId,
channelId: query.channelId,
status: query.status && query.status !== 'all' ? query.status : undefined,
OR: query.keyword ? [
{ streamMessageId: { contains: query.keyword } },
{ traceId: { contains: query.keyword } },
{ messageId: { contains: query.keyword } },
{ submitId: { contains: query.keyword } },
{ failureCode: { contains: query.keyword } },
{ failureMessage: { contains: query.keyword } },
] : undefined,
};
const [items, total] = await Promise.all([
this.prisma.gatewaySubmitDeadLetter.findMany({
where,
include: { tenant: true, application: true, channel: true },
orderBy: { createdAt: 'desc' },
skip: (page - 1) * pageSize,
take: pageSize,
}),
this.prisma.gatewaySubmitDeadLetter.count({ where }),
]);
return { items, total, page, pageSize };
}
async listDownstreamDeliveries(query: DownstreamDeliveryQuery) {
const page = Math.max(1, Number(query.page ?? 1));
const pageSize = Math.min(100, Math.max(1, Number(query.pageSize ?? 10)));
const where: Prisma.CmppDownstreamDeliveryWhereInput = {
...downstreamDeliveryScopedWhere(query),
status: query.status && query.status !== 'all' ? query.status : undefined,
OR: query.keyword ? [
{ messageId: { contains: query.keyword } },
{ payload: { path: ['account'], string_contains: query.keyword } },
{ payload: { path: ['phoneNumber'], string_contains: query.keyword } },
{ lastError: { contains: query.keyword } },
] : undefined,
};
const [items, total] = await Promise.all([
this.prisma.cmppDownstreamDelivery.findMany({
where,
include: { tenant: true, application: true, messageRecord: true },
orderBy: { createdAt: 'desc' },
skip: (page - 1) * pageSize,
take: pageSize,
}),
this.prisma.cmppDownstreamDelivery.count({ where }),
]);
return { items, total, page, pageSize };
}
async downstreamDeliveryDashboard(query: DownstreamDeliveryDashboardQuery) {
const scopedWhere = downstreamDeliveryScopedWhere(query);
const stalledPendingAt = new Date(Date.now() - downstreamAlertPendingMinutes() * 60_000);
const recentFailedAt = new Date(Date.now() - downstreamAlertRecentFailedHours() * 60 * 60_000);
const [total, pending, delivered, failed, stalledPending, recentFailed, typeGroups, applicationGroups, retryZero, retryLow, retryHigh] = await Promise.all([
this.prisma.cmppDownstreamDelivery.count({ where: scopedWhere }),
this.prisma.cmppDownstreamDelivery.count({ where: { ...scopedWhere, status: 'pending' } }),
this.prisma.cmppDownstreamDelivery.count({ where: { ...scopedWhere, status: 'delivered' } }),
this.prisma.cmppDownstreamDelivery.count({ where: { ...scopedWhere, status: 'failed' } }),
this.prisma.cmppDownstreamDelivery.count({
where: {
...scopedWhere,
status: 'pending',
createdAt: { lte: stalledPendingAt },
},
}),
this.prisma.cmppDownstreamDelivery.count({
where: {
...scopedWhere,
status: 'failed',
updatedAt: { gte: recentFailedAt },
},
}),
this.prisma.cmppDownstreamDelivery.groupBy({
by: ['deliveryType', 'status'],
where: scopedWhere,
_count: { _all: true },
}),
this.prisma.cmppDownstreamDelivery.groupBy({
by: ['applicationId', 'status'],
where: scopedWhere,
_count: { _all: true },
}),
this.prisma.cmppDownstreamDelivery.count({
where: {
...scopedWhere,
status: { in: ['pending', 'failed'] },
retryCount: 0,
},
}),
this.prisma.cmppDownstreamDelivery.count({
where: {
...scopedWhere,
status: { in: ['pending', 'failed'] },
retryCount: { gte: 1, lte: 3 },
},
}),
this.prisma.cmppDownstreamDelivery.count({
where: {
...scopedWhere,
status: { in: ['pending', 'failed'] },
retryCount: { gte: 4 },
},
}),
]);
const applicationIds = [...new Set(applicationGroups.map((item) => item.applicationId).filter((value): value is string => Boolean(value)))];
const applications: Array<{ id: string; name: string }> = applicationIds.length > 0
? await this.prisma.smsApplication.findMany({
where: { id: { in: applicationIds } },
select: { id: true, name: true },
})
: [];
const applicationMap = new Map<string, string>(applications.map((item) => [item.id, item.name]));
const groupedByType = groupDownstreamByType(typeGroups);
const groupedByApplication = groupDownstreamByApplication(applicationGroups, applicationMap);
return {
summary: {
total,
pending,
delivered,
failed,
stalledPending,
recentFailed,
alertCount: stalledPending + recentFailed,
},
typeBreakdown: ['receipt', 'uplink'].map((deliveryType) => ({
deliveryType,
total: groupedByType[deliveryType]?.total ?? 0,
pending: groupedByType[deliveryType]?.pending ?? 0,
delivered: groupedByType[deliveryType]?.delivered ?? 0,
failed: groupedByType[deliveryType]?.failed ?? 0,
})),
retryBuckets: [
{ label: '0次', count: retryZero },
{ label: '1-3次', count: retryLow },
{ label: '4次及以上', count: retryHigh },
],
topApplications: groupedByApplication
.sort((left, right) => (
right.alertCount - left.alertCount
|| right.failed - left.failed
|| right.pending - left.pending
|| left.name.localeCompare(right.name, 'zh-CN')
))
.slice(0, 5),
};
}
async listDownstreamRecoveryStatuses(query: DownstreamRecoveryStatusQuery) {
const recoveryStatuses = this.gatewayDownstreamRecoveryStatusDelegate();
const page = Math.max(1, Number(query.page ?? 1));
const pageSize = Math.min(100, Math.max(1, Number(query.pageSize ?? 10)));
const where = downstreamRecoveryStatusWhere(query);
const now = new Date();
const [items, total, runningCount, successCount, failedCount, waitingConnectionCount, backoffCount, categoryGroups] = await Promise.all([
recoveryStatuses.findMany({
where,
include: { tenant: true, application: true },
orderBy: [{ updatedAt: 'desc' }, { account: 'asc' }],
skip: (page - 1) * pageSize,
take: pageSize,
}),
recoveryStatuses.count({ where }),
recoveryStatuses.count({ where: { ...where, state: 'running' } }),
recoveryStatuses.count({ where: { ...where, state: 'success' } }),
recoveryStatuses.count({ where: { ...where, state: 'failed' } }),
recoveryStatuses.count({ where: { ...where, state: 'waiting_connection' } }),
recoveryStatuses.count({
where: {
...where,
nextRetryAt: { gt: now },
},
}),
recoveryStatuses.groupBy({
by: ['failureCategory'],
where,
_count: { _all: true },
}),
]);
return {
items,
total,
page,
pageSize,
summary: {
total,
running: runningCount,
success: successCount,
failed: failedCount,
waitingConnection: waitingConnectionCount,
backoff: backoffCount,
failureCategories: categoryGroups
.filter((item) => item.failureCategory)
.map((item) => ({
category: String(item.failureCategory),
count: item._count?._all ?? 0,
}))
.sort((left, right) => right.count - left.count || left.category.localeCompare(right.category)),
},
};
}
async listMessageSegmentAudits(query: MessageSegmentAuditQuery) {
const segmentAudits = (this.prisma as PrismaService & {
smsMessageSegmentAudit: {
findMany: (args: Record<string, unknown>) => Promise<any[]>;
};
}).smsMessageSegmentAudit;
if (!query.messageId && !query.messageRecordId) {
return [];
}
return segmentAudits.findMany({
where: {
messageRecordId: query.messageRecordId,
messageRecord: query.messageId ? { messageId: query.messageId } : undefined,
},
include: { channel: true, submitRecord: true },
orderBy: [{ submitId: 'asc' }, { segmentIndex: 'asc' }],
});
}
async getDownstreamRecoveryStatus(id: string) {
const recoveryStatuses = this.gatewayDownstreamRecoveryStatusDelegate();
const item = await recoveryStatuses.findUnique({
where: { id },
include: { tenant: true, application: true },
});
if (!item) {
throw new NotFoundException('Recovery status not found');
}
return item;
}
async exportDownstreamRecoveryStatuses(query: DownstreamRecoveryStatusQuery) {
const recoveryStatuses = this.gatewayDownstreamRecoveryStatusDelegate();
const where = downstreamRecoveryStatusWhere(query);
const items = await recoveryStatuses.findMany({
where,
include: { tenant: true, application: true },
orderBy: [{ updatedAt: 'desc' }, { account: 'asc' }],
take: 5000,
});
const rows = [
[
'账号',
'企业',
'应用',
'Gateway实例',
'恢复状态',
'锁持有实例',
'锁过期时间',
'失败分类',
'尝试次数',
'最后尝试时间',
'恢复成功时间',
'恢复失败时间',
'下次恢复时间',
'最后错误',
'最后跳过原因',
'创建时间',
'更新时间',
],
...items.map((item) => [
item.account ?? '',
item.tenant?.name ?? '',
item.application?.name ?? '',
item.gatewayInstanceId ?? '',
item.state ?? '',
(item as { lockOwner?: string | null }).lockOwner ?? '',
formatCsvDate((item as { lockExpiresAt?: Date | string | null }).lockExpiresAt),
(item as { failureCategory?: string | null }).failureCategory ?? '',
String(item.attemptCount ?? 0),
formatCsvDate(item.lastAttemptAt),
formatCsvDate(item.lastSuccessAt),
formatCsvDate(item.lastFailureAt),
formatCsvDate(item.nextRetryAt),
item.lastError ?? '',
item.lastSkipReason ?? '',
formatCsvDate(item.createdAt),
formatCsvDate(item.updatedAt),
]),
];
return {
fileName: `gateway-downstream-recovery-statuses-${formatExportTimestamp(new Date())}.csv`,
content: rows.map((row) => row.map(escapeCsvCell).join(',')).join('\n'),
total: items.length,
};
}
auditSummary(query: { tenantId?: string }) {
return this.prisma.operationLog.groupBy({
by: ['action', 'resource'],
@@ -346,6 +734,17 @@ export class OperationsService {
this.prisma.smsBatchTask.count({ where: { tenantId, auditStatus: 'pending' } }),
]).then((counts) => counts.reduce((sum, value) => sum + value, 0));
}
private gatewayDownstreamRecoveryStatusDelegate() {
return (this.prisma as PrismaService & {
gatewayDownstreamRecoveryStatus: {
findMany: (args: Record<string, unknown>) => Promise<any[]>;
count: (args: Record<string, unknown>) => Promise<number>;
findUnique: (args: Record<string, unknown>) => Promise<any | null>;
groupBy: (args: Record<string, unknown>) => Promise<any[]>;
};
}).gatewayDownstreamRecoveryStatus;
}
}
function messageWhere(query: MessageQuery): Prisma.SmsMessageRecordWhereInput {
@@ -390,6 +789,68 @@ function createdAtRange(range?: string): Prisma.DateTimeFilter | undefined {
return { gte: date };
}
function downstreamAlertPendingMinutes() {
const value = Number(process.env.CMPP_DOWNSTREAM_ALERT_PENDING_MINUTES ?? 10);
return Number.isFinite(value) && value > 0 ? value : 10;
}
function downstreamAlertRecentFailedHours() {
const value = Number(process.env.CMPP_DOWNSTREAM_ALERT_RECENT_FAILED_HOURS ?? 1);
return Number.isFinite(value) && value > 0 ? value : 1;
}
function downstreamDeliveryScopedWhere(query: DownstreamDeliveryDashboardQuery): Prisma.CmppDownstreamDeliveryWhereInput {
return {
tenantId: query.tenantId,
applicationId: query.applicationId,
deliveryType: query.deliveryType && query.deliveryType !== 'all' ? query.deliveryType : undefined,
};
}
function downstreamRecoveryStatusWhere(query: DownstreamRecoveryStatusQuery) {
return {
tenantId: query.tenantId,
applicationId: query.applicationId,
state: query.state && query.state !== 'all' ? query.state : undefined,
failureCategory: query.failureCategory && query.failureCategory !== 'all' ? query.failureCategory : undefined,
OR: query.keyword ? [
{ account: { contains: query.keyword } },
{ gatewayInstanceId: { contains: query.keyword } },
{ lastError: { contains: query.keyword } },
{ lastSkipReason: { contains: query.keyword } },
{ tenant: { name: { contains: query.keyword } } },
{ application: { name: { contains: query.keyword } } },
] : undefined,
};
}
function escapeCsvCell(value: string) {
const normalized = value.replace(/\r\n/g, '\n').replace(/\r/g, '\n');
if (normalized.includes(',') || normalized.includes('"') || normalized.includes('\n')) {
return `"${normalized.replace(/"/g, '""')}"`;
}
return normalized;
}
function formatCsvDate(value?: Date | string | null) {
if (!value) {
return '';
}
return value instanceof Date ? value.toISOString() : value;
}
function formatExportTimestamp(date: Date) {
const parts = [
date.getFullYear(),
String(date.getMonth() + 1).padStart(2, '0'),
String(date.getDate()).padStart(2, '0'),
String(date.getHours()).padStart(2, '0'),
String(date.getMinutes()).padStart(2, '0'),
String(date.getSeconds()).padStart(2, '0'),
];
return `${parts[0]}${parts[1]}${parts[2]}-${parts[3]}${parts[4]}${parts[5]}`;
}
function summarizeMessageGroups(groups: Array<{ status: string; _count: { _all: number }; _sum: { amountCents: number | null; billingUnits: number | null } }>) {
return groups.reduce(
(summary, group) => {
@@ -410,6 +871,51 @@ function summarizeMessageGroups(groups: Array<{ status: string; _count: { _all:
);
}
function groupDownstreamByType(
groups: Array<{ deliveryType: string; status: string; _count: { _all: number } }>,
) {
return groups.reduce<Record<string, { total: number; pending: number; delivered: number; failed: number }>>((accumulator, item) => {
const current = accumulator[item.deliveryType] ?? { total: 0, pending: 0, delivered: 0, failed: 0 };
current.total += item._count._all;
if (item.status === 'pending') {
current.pending += item._count._all;
} else if (item.status === 'delivered') {
current.delivered += item._count._all;
} else if (item.status === 'failed') {
current.failed += item._count._all;
}
accumulator[item.deliveryType] = current;
return accumulator;
}, {});
}
function groupDownstreamByApplication(
groups: Array<{ applicationId: string; status: string; _count: { _all: number } }>,
applicationMap: Map<string, string>,
) {
const summaryMap = new Map<string, { applicationId: string; name: string; pending: number; failed: number; delivered: number; alertCount: number }>();
groups.forEach((item) => {
const current = summaryMap.get(item.applicationId) ?? {
applicationId: item.applicationId,
name: applicationMap.get(item.applicationId) ?? item.applicationId,
pending: 0,
failed: 0,
delivered: 0,
alertCount: 0,
};
if (item.status === 'pending') {
current.pending += item._count._all;
} else if (item.status === 'failed') {
current.failed += item._count._all;
} else if (item.status === 'delivered') {
current.delivered += item._count._all;
}
current.alertCount = current.pending + current.failed;
summaryMap.set(item.applicationId, current);
});
return [...summaryMap.values()];
}
function normalizeOperationLog(log: Prisma.OperationLogGetPayload<{ include: { tenant: true; user: true } }>) {
const detail = (log.detail ?? {}) as Record<string, unknown>;
const result = String(detail.result ?? detail.status ?? '');