hectorzhao/lislgosms
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: 完善服务监控与下游重投

Browse Source
This commit is contained in:
hectorzhao
2026-08-14 17:21:29 +08:00
parent d30d9ea4d0
commit 1ef4380422
50 changed files with 1279 additions and 82 deletions
Download Patch File Download Diff File Expand all files Collapse all files
api/src/send-chain/send-downstream-requeue-task.service.spec.ts
+35
View File
@@ -54,6 +54,41 @@ describe('SendDownstreamRequeueTaskService', () => {
expect(mock.downstreamRequeueTaskItem.createMany).toHaveBeenCalledWith({ data: [expect.objectContaining({ deliveryId: 'd-1', applicationId: 'app-1' })] });
});
it('materializes client-confirmed deliveries from the signed preview range', async () => {
mock.cmppDownstreamDelivery.count.mockResolvedValue(1);
mock.cmppDownstreamDelivery.groupBy.mockResolvedValueOnce([{ status: 'delivered', _count: { _all: 1 } }]).mockResolvedValueOnce([{ applicationId: 'app-1', _count: { _all: 1 } }]);
mock.cmppDownstreamDelivery.findFirst.mockResolvedValue({ createdAt: new Date() });
const preview = await service.preview({ applicationId: 'app-1', status: 'delivered' }, 'user-1');
expect(preview).toEqual(expect.objectContaining({ matchedCount: 1, replayableCount: 1, skippedCount: 0 }));
mock.downstreamRequeueTask.findFirst.mockResolvedValue(null);
mock.cmppDownstreamDelivery.findMany.mockResolvedValue([{ id: 'd-1', applicationId: 'app-1', status: 'delivered' }]);
mock.downstreamRequeueTask.create.mockResolvedValue({ id: 'task-1' });
mock.downstreamRequeueTask.findUnique.mockResolvedValue({ id: 'task-1' });
mock.downstreamRequeueTaskItem.groupBy.mockResolvedValue([]);
await service.create({ previewToken: preview.previewToken, reason: '再次投递客户已确认记录' }, 'user-1');
expect(mock.downstreamRequeueTaskItem.createMany).toHaveBeenCalledWith({ data: [expect.objectContaining({ deliveryId: 'd-1', previousStatus: 'delivered' })] });
});
it('replays a delivery that was already client-confirmed in the task snapshot', async () => {
const requeue = jest.fn().mockResolvedValue({ status: 'awaiting_ack' });
service = new SendDownstreamRequeueTaskService(mock as never, { requeueDownstreamDelivery: requeue });
mock.cmppDownstreamDelivery.findUnique.mockResolvedValue({ id: 'd-1', applicationId: 'app-1', status: 'delivered', ackResult: 0, payload: {}, deliveryType: 'receipt', application: { status: 'active', interfaceEnabled: true } });
mock.downstreamRequeueTaskItem.findFirst.mockResolvedValue(null);
mock.cmppDownstreamConnection.count.mockResolvedValue(1);
await expect(service['processItem']('item-1', 'd-1', 'delivered')).resolves.toBe('waiting');
expect(requeue).toHaveBeenCalledWith('d-1');
expect(mock.downstreamRequeueTaskItem.update).toHaveBeenCalledWith(expect.objectContaining({ data: expect.objectContaining({ status: 'waiting_ack' }) }));
});
it('does not replay a record that became delivered after a non-delivered task snapshot', async () => {
const requeue = jest.fn();
service = new SendDownstreamRequeueTaskService(mock as never, { requeueDownstreamDelivery: requeue });
mock.cmppDownstreamDelivery.findUnique.mockResolvedValue({ id: 'd-1', applicationId: 'app-1', status: 'delivered', ackResult: 0, payload: {}, deliveryType: 'receipt', application: { status: 'active', interfaceEnabled: true } });
await expect(service['processItem']('item-1', 'd-1', 'failed')).resolves.toBe('skipped');
expect(requeue).not.toHaveBeenCalled();
expect(mock.downstreamRequeueTaskItem.update).toHaveBeenCalledWith(expect.objectContaining({ data: expect.objectContaining({ status: 'skipped', skipReason: '创建任务后已被客户确认' }) }));
});
it('paginates all task items with status and keyword filters', async () => {
mock.downstreamRequeueTask.findUnique.mockResolvedValue({ id: 'task-1' });
mock.downstreamRequeueTaskItem.findMany.mockResolvedValue([{ id: 'item-1' }]);
api/src/send-chain/send-downstream-requeue-task.service.ts
+12 -6
View File
@@ -15,7 +15,7 @@ export type DownstreamRequeueFilter = {
};
type RequeueFacade = { requeueDownstreamDelivery(id: string): Promise<unknown> };
const REPLAYABLE_STATUSES = ['pending', 'failed', 'unconfirmed', 'rejected'];
const REPLAYABLE_STATUSES = ['pending', 'failed', 'unconfirmed', 'rejected', 'delivered'];
const ACTIVE_TASK_STATUSES = ['queued', 'running', 'paused'];
const PROCESSING_LEASE_MS = 2 * 60_000;
const SCAN_LEASE_MS = 15_000;
@@ -119,7 +119,7 @@ export class SendDownstreamRequeueTaskService {
const preview = verifyPreview(data.previewToken, createdById);
const filter = normalizedFilter(preview.filter);
const snapshotAt = new Date(preview.snapshotAt);
if (filter.status === 'delivered' || filter.status === 'awaiting_ack') throw new BadRequestException('第一版后台任务不支持已确认或等待ACK记录');
if (filter.status === 'awaiting_ack') throw new BadRequestException('后台任务不支持正在等待ACK的记录');
const activeTask = await this.prisma.downstreamRequeueTask.findFirst({ where: {
status: { in: ACTIVE_TASK_STATUSES },
...(filter.applicationId !== 'all' ? { OR: [{ applicationId: filter.applicationId }, { applicationId: null }] } : {}),
@@ -229,14 +229,14 @@ export class SendDownstreamRequeueTaskService {
return;
}
await this.prisma.downstreamRequeueTask.update({ where: { id: taskId }, data: { status: 'running', startedAt: task.startedAt ?? new Date() } });
const items = await this.prisma.downstreamRequeueTaskItem.findMany({ where: { taskId, status: 'queued' }, orderBy: { createdAt: 'asc' }, take: Math.min(200, task.ratePerSecond * 3), select: { id: true, deliveryId: true, applicationId: true } });
const items = await this.prisma.downstreamRequeueTaskItem.findMany({ where: { taskId, status: 'queued' }, orderBy: { createdAt: 'asc' }, take: Math.min(200, task.ratePerSecond * 3), select: { id: true, deliveryId: true, applicationId: true, previousStatus: true } });
for (const item of items) {
const latestTask = await this.prisma.downstreamRequeueTask.findUnique({ where: { id: taskId }, select: { status: true } });
if (!latestTask || !['queued', 'running'].includes(latestTask.status)) break;
if (!(await this.consumeRate(item.applicationId, task.ratePerSecond))) continue;
const claimed = await this.prisma.downstreamRequeueTaskItem.updateMany({ where: { id: item.id, status: 'queued' }, data: { status: 'processing', claimedAt: new Date() } });
if (!claimed.count) continue;
const outcome = await this.processItem(item.id, item.deliveryId);
const outcome = await this.processItem(item.id, item.deliveryId, item.previousStatus);
if (outcome === 'success') failures[item.applicationId] = 0;
if (outcome === 'failed') failures[item.applicationId] = (failures[item.applicationId] ?? 0) + 1;
const maxFailures = Math.max(0, ...Object.values(failures));
@@ -257,16 +257,22 @@ export class SendDownstreamRequeueTaskService {
}
}
private async processItem(itemId: string, deliveryId: string): Promise<'success' | 'failed' | 'waiting' | 'skipped'> {
private async processItem(itemId: string, deliveryId: string, previousStatus: string): Promise<'success' | 'failed' | 'waiting' | 'skipped'> {
try {
const delivery = await this.prisma.cmppDownstreamDelivery.findUnique({
where: { id: deliveryId },
include: { application: { select: { status: true, interfaceEnabled: true } } },
});
if (!delivery) return this.finishItem(itemId, 'skipped', '投递记录已不存在');
// Only a record that was already delivered in the frozen task snapshot may be replayed as
// delivered. This preserves the operator's explicit duplicate-delivery intent while preventing
// a pending/failed record that receives a late ACK after task creation from being sent again.
if (delivery.status === 'delivered' && previousStatus !== 'delivered') {
return this.finishItem(itemId, 'skipped', '创建任务后已被客户确认');
}
if (!REPLAYABLE_STATUSES.includes(delivery.status)) {
if (delivery.status === 'awaiting_ack') { await this.prisma.downstreamRequeueTaskItem.update({ where: { id: itemId }, data: { status: 'waiting_external_ack', skipReason: null } }); return 'waiting'; }
return this.finishItem(itemId, 'skipped', delivery.status === 'delivered' ? '已被客户确认' : '执行前状态已变化');
return this.finishItem(itemId, 'skipped', '执行前状态已变化');
}
if (delivery.application.status !== 'active' || !delivery.application.interfaceEnabled) return this.finishItem(itemId, 'skipped', '应用或投递能力已停用');
if (!delivery.payload || !['receipt', 'uplink'].includes(delivery.deliveryType)) return this.finishItem(itemId, 'skipped', '投递数据不完整');