feat: 增强下游重投与签名质量检测
This commit is contained in:
hectorzhao
@@ -0,0 +1,235 @@
|
||||
import { BadRequestException, NotFoundException } from '@nestjs/common';
|
||||
import { Prisma } from '@prisma/client';
|
||||
import { PrismaService } from '../prisma/prisma.service';
|
||||
import { parseDateBoundary } from '../operations/operations.helpers';
|
||||
|
||||
export type DownstreamRequeueFilter = {
|
||||
tenantId?: string;
|
||||
applicationId?: string;
|
||||
deliveryType?: string;
|
||||
status?: string;
|
||||
keyword?: string;
|
||||
createdAtFrom?: string;
|
||||
createdAtTo?: string;
|
||||
};
|
||||
|
||||
type RequeueFacade = { requeueDownstreamDelivery(id: string): Promise<unknown> };
|
||||
const REPLAYABLE_STATUSES = ['pending', 'failed', 'unconfirmed', 'rejected'];
|
||||
|
||||
function taskWhere(filter: DownstreamRequeueFilter, snapshotAt: Date, replayableByDefault = true): Prisma.CmppDownstreamDeliveryWhereInput {
|
||||
const from = parseDateBoundary(filter.createdAtFrom, false);
|
||||
const to = parseDateBoundary(filter.createdAtTo, true);
|
||||
return {
|
||||
tenantId: filter.tenantId && filter.tenantId !== 'all' ? filter.tenantId : undefined,
|
||||
applicationId: filter.applicationId && filter.applicationId !== 'all' ? filter.applicationId : undefined,
|
||||
deliveryType: filter.deliveryType && filter.deliveryType !== 'all' ? filter.deliveryType : undefined,
|
||||
status: filter.status && filter.status !== 'all' ? filter.status : replayableByDefault ? { in: REPLAYABLE_STATUSES } : undefined,
|
||||
createdAt: { ...(from ? { gte: from } : {}), lte: to && to < snapshotAt ? to : snapshotAt },
|
||||
OR: filter.keyword ? [
|
||||
{ messageId: { contains: filter.keyword } },
|
||||
{ payload: { path: ['account'], string_contains: filter.keyword } },
|
||||
{ payload: { path: ['phoneNumber'], string_contains: filter.keyword } },
|
||||
{ lastError: { contains: filter.keyword } },
|
||||
{ tenant: { name: { contains: filter.keyword } } },
|
||||
{ application: { name: { contains: filter.keyword } } },
|
||||
] : undefined,
|
||||
};
|
||||
}
|
||||
|
||||
export class SendDownstreamRequeueTaskService {
|
||||
constructor(private readonly prisma: PrismaService, private readonly facade: RequeueFacade) {}
|
||||
|
||||
async preview(filter: DownstreamRequeueFilter) {
|
||||
const snapshotAt = new Date();
|
||||
const base = taskWhere({ ...filter, status: 'all' }, snapshotAt, false);
|
||||
const where = taskWhere(filter, snapshotAt);
|
||||
const [matchedCount, replayableCount, statusGroups, appGroups, oldest] = await Promise.all([
|
||||
this.prisma.cmppDownstreamDelivery.count({ where: base }),
|
||||
this.prisma.cmppDownstreamDelivery.count({ where: { AND: [where, { status: { in: REPLAYABLE_STATUSES } }] } }),
|
||||
this.prisma.cmppDownstreamDelivery.groupBy({ by: ['status'], where: base, _count: { _all: true } }),
|
||||
this.prisma.cmppDownstreamDelivery.groupBy({ by: ['applicationId'], where: base, _count: { _all: true } }),
|
||||
this.prisma.cmppDownstreamDelivery.findFirst({ where: base, orderBy: { createdAt: 'asc' }, select: { createdAt: true } }),
|
||||
]);
|
||||
return {
|
||||
snapshotAt,
|
||||
matchedCount,
|
||||
replayableCount,
|
||||
skippedCount: matchedCount - replayableCount,
|
||||
applicationCount: appGroups.length,
|
||||
oldestCreatedAt: oldest?.createdAt ?? null,
|
||||
statusCounts: Object.fromEntries(statusGroups.map((item) => [item.status, item._count._all])),
|
||||
filter: { ...filter, status: filter.status ?? 'all' },
|
||||
};
|
||||
}
|
||||
|
||||
async create(data: { filter: DownstreamRequeueFilter; snapshotAt: string; reason: string; ratePerSecond?: number; consecutiveFailureLimit?: number }, createdById?: string) {
|
||||
const reason = data.reason?.trim();
|
||||
if (!reason || reason.length < 5) throw new BadRequestException('任务原因至少填写5个字');
|
||||
const snapshotAt = new Date(data.snapshotAt);
|
||||
if (Number.isNaN(snapshotAt.getTime()) || snapshotAt.getTime() > Date.now() + 10_000) throw new BadRequestException('预检快照时间无效');
|
||||
if (data.filter.status === 'delivered' || data.filter.status === 'awaiting_ack') throw new BadRequestException('第一版后台任务不支持已确认或等待ACK记录');
|
||||
const activeTask = await this.prisma.downstreamRequeueTask.findFirst({ where: {
|
||||
status: { in: ['queued', 'running', 'paused'] },
|
||||
...(data.filter.applicationId && data.filter.applicationId !== 'all'
|
||||
? { OR: [{ applicationId: data.filter.applicationId }, { applicationId: null }] }
|
||||
: {}),
|
||||
}, select: { taskNo: true } });
|
||||
if (activeTask) throw new BadRequestException(`当前应用范围已有未结束任务 ${activeTask.taskNo}`);
|
||||
const where = { AND: [taskWhere(data.filter, snapshotAt), { status: { in: REPLAYABLE_STATUSES } }] } as Prisma.CmppDownstreamDeliveryWhereInput;
|
||||
const deliveries = await this.prisma.cmppDownstreamDelivery.findMany({ where, orderBy: [{ createdAt: 'asc' }, { id: 'asc' }], take: 100001, select: { id: true, tenantId: true, applicationId: true, status: true } });
|
||||
if (!deliveries.length) throw new BadRequestException('当前筛选条件下没有可重投记录');
|
||||
if (deliveries.length > 100000) throw new BadRequestException('单个任务最多处理100000条，请缩小日期范围');
|
||||
const ratePerSecond = Math.min(50, Math.max(1, Number(data.ratePerSecond ?? 10)));
|
||||
const failureLimit = Math.min(100, Math.max(1, Number(data.consecutiveFailureLimit ?? 10)));
|
||||
const taskNo = `DRT-${Date.now()}-${Math.floor(Math.random() * 1000).toString().padStart(3, '0')}`;
|
||||
const task = await this.prisma.$transaction(async (tx) => {
|
||||
const created = await tx.downstreamRequeueTask.create({ data: {
|
||||
taskNo,
|
||||
tenantId: data.filter.tenantId && data.filter.tenantId !== 'all' ? data.filter.tenantId : null,
|
||||
applicationId: data.filter.applicationId && data.filter.applicationId !== 'all' ? data.filter.applicationId : null,
|
||||
filterSnapshot: data.filter as Prisma.InputJsonValue,
|
||||
snapshotAt, reason, ratePerSecond, consecutiveFailureLimit: failureLimit,
|
||||
totalCount: deliveries.length, createdById,
|
||||
} });
|
||||
await tx.downstreamRequeueTaskItem.createMany({ data: deliveries.map((item) => ({ taskId: created.id, deliveryId: item.id, applicationId: item.applicationId, previousStatus: item.status })) });
|
||||
await tx.operationLog.create({ data: { userId: createdById, action: 'gateway.downstream_requeue_task_created', resource: 'downstream_requeue_task', resourceId: created.id, detail: { taskNo, reason, totalCount: deliveries.length, snapshotAt, filter: data.filter, ratePerSecond } } });
|
||||
return created;
|
||||
});
|
||||
return this.get(task.id);
|
||||
}
|
||||
|
||||
async list(query: { status?: string; page?: number; pageSize?: number }) {
|
||||
const page = Math.max(1, Number(query.page ?? 1));
|
||||
const pageSize = Math.min(50, Math.max(1, Number(query.pageSize ?? 10)));
|
||||
const where = { status: query.status && query.status !== 'all' ? query.status : undefined };
|
||||
const [items, total] = await Promise.all([
|
||||
this.prisma.downstreamRequeueTask.findMany({ where, include: { tenant: true, application: true, createdBy: { select: { id: true, displayName: true, username: true } } }, orderBy: { createdAt: 'desc' }, skip: (page - 1) * pageSize, take: pageSize }),
|
||||
this.prisma.downstreamRequeueTask.count({ where }),
|
||||
]);
|
||||
return { items, total, page, pageSize };
|
||||
}
|
||||
|
||||
async get(id: string) {
|
||||
const task = await this.prisma.downstreamRequeueTask.findUnique({ where: { id }, include: { tenant: true, application: true, createdBy: { select: { id: true, displayName: true, username: true } } } });
|
||||
if (!task) throw new NotFoundException('后台重投任务不存在');
|
||||
const [itemGroups, recentItems] = await Promise.all([
|
||||
this.prisma.downstreamRequeueTaskItem.groupBy({ by: ['status'], where: { taskId: id }, _count: { _all: true } }),
|
||||
this.prisma.downstreamRequeueTaskItem.findMany({ where: { taskId: id }, include: { delivery: { select: { messageId: true, deliveryType: true, status: true, lastError: true } } }, orderBy: { updatedAt: 'desc' }, take: 50 }),
|
||||
]);
|
||||
return { ...task, itemCounts: Object.fromEntries(itemGroups.map((item) => [item.status, item._count._all])), recentItems };
|
||||
}
|
||||
|
||||
async changeStatus(id: string, action: 'pause' | 'resume' | 'terminate', operatorId?: string) {
|
||||
if (!['pause', 'resume', 'terminate'].includes(action)) throw new BadRequestException('不支持的任务操作');
|
||||
const task = await this.prisma.downstreamRequeueTask.findUnique({ where: { id } });
|
||||
if (!task) throw new NotFoundException('后台重投任务不存在');
|
||||
const allowed = action === 'pause' ? ['queued', 'running'] : action === 'resume' ? ['paused'] : ['queued', 'running', 'paused'];
|
||||
if (!allowed.includes(task.status)) throw new BadRequestException('当前任务状态不允许此操作');
|
||||
const status = action === 'pause' ? 'paused' : action === 'resume' ? 'queued' : 'terminated';
|
||||
const updated = await this.prisma.downstreamRequeueTask.update({ where: { id }, data: { status, pausedAt: status === 'paused' ? new Date() : null, finishedAt: status === 'terminated' ? new Date() : undefined } });
|
||||
if (status === 'terminated') await this.prisma.downstreamRequeueTaskItem.updateMany({ where: { taskId: id, status: 'queued' }, data: { status: 'unprocessed', skipReason: '任务已终止', completedAt: new Date() } });
|
||||
await this.prisma.operationLog.create({ data: { userId: operatorId, action: `gateway.downstream_requeue_task_${action}`, resource: 'downstream_requeue_task', resourceId: id, detail: { taskNo: task.taskNo, previousStatus: task.status, status } } });
|
||||
return updated;
|
||||
}
|
||||
|
||||
async runScan() {
|
||||
const tasks = await this.prisma.downstreamRequeueTask.findMany({ where: { status: { in: ['queued', 'running'] } }, orderBy: { createdAt: 'asc' }, take: 3 });
|
||||
for (const task of tasks) await this.processTask(task.id);
|
||||
}
|
||||
|
||||
private async processTask(taskId: string) {
|
||||
const task = await this.prisma.downstreamRequeueTask.findUnique({ where: { id: taskId } });
|
||||
if (!task || !['queued', 'running'].includes(task.status)) return;
|
||||
await this.reconcileWaiting(taskId);
|
||||
await this.prisma.downstreamRequeueTask.update({ where: { id: taskId }, data: { status: 'running', startedAt: task.startedAt ?? new Date() } });
|
||||
const items = await this.prisma.downstreamRequeueTaskItem.findMany({ where: { taskId, status: 'queued' }, orderBy: { createdAt: 'asc' }, take: Math.min(10, task.ratePerSecond), select: { id: true, deliveryId: true } });
|
||||
let consecutiveFailures = task.consecutiveFailures;
|
||||
for (const item of items) {
|
||||
const latestTask = await this.prisma.downstreamRequeueTask.findUnique({ where: { id: taskId }, select: { status: true } });
|
||||
if (latestTask?.status === 'paused' || latestTask?.status === 'terminated') break;
|
||||
const claimed = await this.prisma.downstreamRequeueTaskItem.updateMany({ where: { id: item.id, status: 'queued' }, data: { status: 'processing', claimedAt: new Date() } });
|
||||
if (!claimed.count) continue;
|
||||
try {
|
||||
const delivery = await this.prisma.cmppDownstreamDelivery.findUnique({
|
||||
where: { id: item.deliveryId },
|
||||
include: { application: { select: { status: true, interfaceEnabled: true } } },
|
||||
});
|
||||
if (!delivery) {
|
||||
await this.prisma.downstreamRequeueTaskItem.update({ where: { id: item.id }, data: { status: 'skipped', skipReason: '投递记录已不存在', completedAt: new Date() } });
|
||||
continue;
|
||||
}
|
||||
if (!REPLAYABLE_STATUSES.includes(delivery.status)) {
|
||||
if (delivery.status === 'awaiting_ack') {
|
||||
await this.prisma.downstreamRequeueTaskItem.update({ where: { id: item.id }, data: { status: 'waiting_external_ack', skipReason: null } });
|
||||
} else {
|
||||
const skipReason = delivery.status === 'delivered' ? '已被客户确认' : '执行前状态已变化';
|
||||
await this.prisma.downstreamRequeueTaskItem.update({ where: { id: item.id }, data: { status: 'skipped', skipReason, completedAt: new Date() } });
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if (delivery.application.status !== 'active' || !delivery.application.interfaceEnabled) {
|
||||
await this.prisma.downstreamRequeueTaskItem.update({ where: { id: item.id }, data: { status: 'skipped', skipReason: '应用或投递能力已停用', completedAt: new Date() } });
|
||||
continue;
|
||||
}
|
||||
if (!delivery.payload || !['receipt', 'uplink'].includes(delivery.deliveryType)) {
|
||||
await this.prisma.downstreamRequeueTaskItem.update({ where: { id: item.id }, data: { status: 'skipped', skipReason: '投递数据不完整', completedAt: new Date() } });
|
||||
continue;
|
||||
}
|
||||
const activeOther = await this.prisma.downstreamRequeueTaskItem.findFirst({ where: { deliveryId: item.deliveryId, id: { not: item.id }, status: { in: ['processing', 'waiting_ack', 'success'] } }, select: { id: true } });
|
||||
if (activeOther) {
|
||||
await this.prisma.downstreamRequeueTaskItem.update({ where: { id: item.id }, data: { status: 'skipped', skipReason: '已被其他任务处理', completedAt: new Date() } });
|
||||
continue;
|
||||
}
|
||||
const result = await this.facade.requeueDownstreamDelivery(item.deliveryId) as { status?: string; lastError?: string | null };
|
||||
if (result?.status === 'awaiting_ack' || result?.status === 'delivered') {
|
||||
await this.prisma.downstreamRequeueTaskItem.update({ where: { id: item.id }, data: { status: result.status === 'delivered' ? 'success' : 'waiting_ack', completedAt: result.status === 'delivered' ? new Date() : null } });
|
||||
consecutiveFailures = 0;
|
||||
} else {
|
||||
await this.prisma.downstreamRequeueTaskItem.update({ where: { id: item.id }, data: { status: 'failed', errorMessage: result?.lastError ?? 'Gateway未进入等待ACK状态', completedAt: new Date() } });
|
||||
consecutiveFailures += 1;
|
||||
}
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : '后台重投失败';
|
||||
const skipReason = /已被其他操作处理|状态|等待客户端确认/.test(message) ? '执行前状态已变化'
|
||||
: /payload|投递类型/.test(message) ? '投递数据不完整'
|
||||
: /Submit|Msg_Id|Sequence/.test(message) ? '缺少原Submit映射，无法安全重投'
|
||||
: null;
|
||||
await this.prisma.downstreamRequeueTaskItem.update({ where: { id: item.id }, data: { status: skipReason ? 'skipped' : 'failed', skipReason, errorMessage: skipReason ? null : message, completedAt: new Date() } });
|
||||
if (!skipReason) consecutiveFailures += 1;
|
||||
}
|
||||
await this.prisma.downstreamRequeueTask.update({ where: { id: taskId }, data: { consecutiveFailures } });
|
||||
if (consecutiveFailures >= task.consecutiveFailureLimit) {
|
||||
// Stop before claiming another delivery: a customer or Gateway outage must not become a retry flood.
|
||||
const pausedAt = new Date();
|
||||
await this.prisma.downstreamRequeueTask.update({ where: { id: taskId }, data: { status: 'paused', pausedAt, lastError: `连续失败达到安全阈值 ${task.consecutiveFailureLimit} 条，任务已自动暂停` } });
|
||||
await this.prisma.operationLog.create({ data: { action: 'gateway.downstream_requeue_task_auto_paused', resource: 'downstream_requeue_task', resourceId: taskId, detail: { taskNo: task.taskNo, consecutiveFailures, failureLimit: task.consecutiveFailureLimit } } });
|
||||
break;
|
||||
}
|
||||
}
|
||||
await this.reconcileWaiting(taskId);
|
||||
await this.refreshTask(taskId);
|
||||
}
|
||||
|
||||
private async reconcileWaiting(taskId: string) {
|
||||
const items = await this.prisma.downstreamRequeueTaskItem.findMany({ where: { taskId, status: { in: ['waiting_ack', 'waiting_external_ack'] } }, include: { delivery: { select: { status: true, ackResult: true, ackDeadlineAt: true, lastError: true } } }, take: 100 });
|
||||
const now = new Date();
|
||||
for (const item of items) {
|
||||
if (item.delivery.status === 'delivered' && item.delivery.ackResult === 0) {
|
||||
await this.prisma.downstreamRequeueTaskItem.update({ where: { id: item.id }, data: item.status === 'waiting_external_ack' ? { status: 'skipped', skipReason: '已由其他投递链路完成', completedAt: now } : { status: 'success', completedAt: now } });
|
||||
} else if (['failed', 'rejected', 'unconfirmed'].includes(item.delivery.status) || (item.delivery.ackDeadlineAt && item.delivery.ackDeadlineAt <= now)) {
|
||||
await this.prisma.downstreamRequeueTaskItem.update({ where: { id: item.id }, data: item.status === 'waiting_external_ack' ? { status: 'queued', skipReason: null, errorMessage: null, claimedAt: null } : { status: 'failed', errorMessage: item.delivery.lastError ?? '客户端ACK失败或超时', completedAt: now } });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async refreshTask(taskId: string) {
|
||||
const groups = await this.prisma.downstreamRequeueTaskItem.groupBy({ by: ['status'], where: { taskId }, _count: { _all: true } });
|
||||
const counts = new Map(groups.map((item) => [item.status, item._count._all]));
|
||||
const queued = counts.get('queued') ?? 0;
|
||||
const active = (counts.get('processing') ?? 0) + (counts.get('waiting_ack') ?? 0) + (counts.get('waiting_external_ack') ?? 0);
|
||||
const failed = counts.get('failed') ?? 0;
|
||||
const current = await this.prisma.downstreamRequeueTask.findUnique({ where: { id: taskId }, select: { status: true } });
|
||||
const status = current?.status === 'paused' || current?.status === 'terminated' ? current.status : queued + active === 0 ? (failed > 0 ? 'partial_completed' : 'completed') : 'running';
|
||||
await this.prisma.downstreamRequeueTask.update({ where: { id: taskId }, data: { status, successCount: counts.get('success') ?? 0, failedCount: failed, skippedCount: counts.get('skipped') ?? 0, waitingCount: active, ...(status === 'completed' || status === 'partial_completed' ? { finishedAt: new Date() } : {}) } });
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user