feat: harden sessions and track downstream acknowledgements

This commit is contained in:
hectorzhao
2026-07-14 14:18:43 +08:00
parent 3d37adcc9f
commit 8c03663f24
43 changed files with 1733 additions and 150 deletions
+12 -4
View File
@@ -396,9 +396,13 @@ describe('OperationsService', () => {
prisma.cmppDownstreamDelivery.count = jest.fn()
.mockResolvedValueOnce(12)
.mockResolvedValueOnce(3)
.mockResolvedValueOnce(0)
.mockResolvedValueOnce(8)
.mockResolvedValueOnce(1)
.mockResolvedValueOnce(0)
.mockResolvedValueOnce(0)
.mockResolvedValueOnce(1)
.mockResolvedValueOnce(0)
.mockResolvedValueOnce(1)
.mockResolvedValueOnce(2)
.mockResolvedValueOnce(1)
@@ -428,15 +432,19 @@ describe('OperationsService', () => {
summary: {
total: 12,
pending: 3,
awaitingAck: 0,
delivered: 8,
failed: 1,
unconfirmed: 0,
rejected: 0,
stalledPending: 1,
stalledAck: 0,
recentFailed: 1,
alertCount: 2,
},
typeBreakdown: [
{ deliveryType: 'receipt', total: 9, pending: 2, delivered: 6, failed: 1 },
{ deliveryType: 'uplink', total: 3, pending: 1, delivered: 2, failed: 0 },
{ deliveryType: 'receipt', total: 9, pending: 2, awaitingAck: 0, delivered: 6, failed: 1, unconfirmed: 0, rejected: 0 },
{ deliveryType: 'uplink', total: 3, pending: 1, awaitingAck: 0, delivered: 2, failed: 0, unconfirmed: 0, rejected: 0 },
],
retryBuckets: [
{ label: '0次', count: 2 },
@@ -444,8 +452,8 @@ describe('OperationsService', () => {
{ label: '4次及以上', count: 0 },
],
topApplications: [
{ applicationId: 'app-1', name: '应用A', pending: 2, failed: 1, delivered: 5, alertCount: 3 },
{ applicationId: 'app-2', name: '应用B', pending: 1, failed: 0, delivered: 3, alertCount: 1 },
{ applicationId: 'app-1', name: '应用A', pending: 2, awaitingAck: 0, failed: 1, unconfirmed: 0, rejected: 0, delivered: 5, alertCount: 3 },
{ applicationId: 'app-2', name: '应用B', pending: 1, awaitingAck: 0, failed: 0, unconfirmed: 0, rejected: 0, delivered: 3, alertCount: 1 },
],
});
});
+38 -10
View File
@@ -408,11 +408,14 @@ export class OperationsService {
const scopedWhere = downstreamDeliveryScopedWhere(query);
const stalledPendingAt = new Date(Date.now() - downstreamAlertPendingMinutes() * 60_000);
const recentFailedAt = new Date(Date.now() - downstreamAlertRecentFailedHours() * 60 * 60_000);
const [total, pending, delivered, failed, stalledPending, recentFailed, typeGroups, applicationGroups, retryZero, retryLow, retryHigh] = await Promise.all([
const [total, pending, awaitingAck, delivered, failed, unconfirmed, rejected, stalledPending, stalledAck, recentFailed, typeGroups, applicationGroups, retryZero, retryLow, retryHigh] = await Promise.all([
this.prisma.cmppDownstreamDelivery.count({ where: scopedWhere }),
this.prisma.cmppDownstreamDelivery.count({ where: { ...scopedWhere, status: 'pending' } }),
this.prisma.cmppDownstreamDelivery.count({ where: { ...scopedWhere, status: 'awaiting_ack' } }),
this.prisma.cmppDownstreamDelivery.count({ where: { ...scopedWhere, status: 'delivered' } }),
this.prisma.cmppDownstreamDelivery.count({ where: { ...scopedWhere, status: 'failed' } }),
this.prisma.cmppDownstreamDelivery.count({ where: { ...scopedWhere, status: 'unconfirmed' } }),
this.prisma.cmppDownstreamDelivery.count({ where: { ...scopedWhere, status: 'rejected' } }),
this.prisma.cmppDownstreamDelivery.count({
where: {
...scopedWhere,
@@ -420,10 +423,13 @@ export class OperationsService {
createdAt: { lte: stalledPendingAt },
},
}),
this.prisma.cmppDownstreamDelivery.count({
where: { ...scopedWhere, status: 'awaiting_ack', ackDeadlineAt: { lte: new Date() } },
}),
this.prisma.cmppDownstreamDelivery.count({
where: {
...scopedWhere,
status: 'failed',
status: { in: ['failed', 'unconfirmed', 'rejected'] },
updatedAt: { gte: recentFailedAt },
},
}),
@@ -440,21 +446,21 @@ export class OperationsService {
this.prisma.cmppDownstreamDelivery.count({
where: {
...scopedWhere,
status: { in: ['pending', 'failed'] },
status: { in: ['pending', 'failed', 'unconfirmed', 'rejected'] },
retryCount: 0,
},
}),
this.prisma.cmppDownstreamDelivery.count({
where: {
...scopedWhere,
status: { in: ['pending', 'failed'] },
status: { in: ['pending', 'failed', 'unconfirmed', 'rejected'] },
retryCount: { gte: 1, lte: 3 },
},
}),
this.prisma.cmppDownstreamDelivery.count({
where: {
...scopedWhere,
status: { in: ['pending', 'failed'] },
status: { in: ['pending', 'failed', 'unconfirmed', 'rejected'] },
retryCount: { gte: 4 },
},
}),
@@ -474,18 +480,25 @@ export class OperationsService {
summary: {
total,
pending,
awaitingAck,
delivered,
failed,
unconfirmed,
rejected,
stalledPending,
stalledAck,
recentFailed,
alertCount: stalledPending + recentFailed,
alertCount: stalledPending + stalledAck + recentFailed,
},
typeBreakdown: ['receipt', 'uplink'].map((deliveryType) => ({
deliveryType,
total: groupedByType[deliveryType]?.total ?? 0,
pending: groupedByType[deliveryType]?.pending ?? 0,
awaitingAck: groupedByType[deliveryType]?.awaitingAck ?? 0,
delivered: groupedByType[deliveryType]?.delivered ?? 0,
failed: groupedByType[deliveryType]?.failed ?? 0,
unconfirmed: groupedByType[deliveryType]?.unconfirmed ?? 0,
rejected: groupedByType[deliveryType]?.rejected ?? 0,
})),
retryBuckets: [
{ label: '0次', count: retryZero },
@@ -899,15 +912,21 @@ function summarizeMessageGroups(groups: Array<{ status: string; _count: { _all:
function groupDownstreamByType(
groups: Array<{ deliveryType: string; status: string; _count: { _all: number } }>,
) {
return groups.reduce<Record<string, { total: number; pending: number; delivered: number; failed: number }>>((accumulator, item) => {
const current = accumulator[item.deliveryType] ?? { total: 0, pending: 0, delivered: 0, failed: 0 };
return groups.reduce<Record<string, { total: number; pending: number; awaitingAck: number; delivered: number; failed: number; unconfirmed: number; rejected: number }>>((accumulator, item) => {
const current = accumulator[item.deliveryType] ?? { total: 0, pending: 0, awaitingAck: 0, delivered: 0, failed: 0, unconfirmed: 0, rejected: 0 };
current.total += item._count._all;
if (item.status === 'pending') {
current.pending += item._count._all;
} else if (item.status === 'awaiting_ack') {
current.awaitingAck += item._count._all;
} else if (item.status === 'delivered') {
current.delivered += item._count._all;
} else if (item.status === 'failed') {
current.failed += item._count._all;
} else if (item.status === 'unconfirmed') {
current.unconfirmed += item._count._all;
} else if (item.status === 'rejected') {
current.rejected += item._count._all;
}
accumulator[item.deliveryType] = current;
return accumulator;
@@ -918,24 +937,33 @@ function groupDownstreamByApplication(
groups: Array<{ applicationId: string; status: string; _count: { _all: number } }>,
applicationMap: Map<string, string>,
) {
const summaryMap = new Map<string, { applicationId: string; name: string; pending: number; failed: number; delivered: number; alertCount: number }>();
const summaryMap = new Map<string, { applicationId: string; name: string; pending: number; awaitingAck: number; failed: number; unconfirmed: number; rejected: number; delivered: number; alertCount: number }>();
groups.forEach((item) => {
const current = summaryMap.get(item.applicationId) ?? {
applicationId: item.applicationId,
name: applicationMap.get(item.applicationId) ?? item.applicationId,
pending: 0,
awaitingAck: 0,
failed: 0,
unconfirmed: 0,
rejected: 0,
delivered: 0,
alertCount: 0,
};
if (item.status === 'pending') {
current.pending += item._count._all;
} else if (item.status === 'awaiting_ack') {
current.awaitingAck += item._count._all;
} else if (item.status === 'failed') {
current.failed += item._count._all;
} else if (item.status === 'unconfirmed') {
current.unconfirmed += item._count._all;
} else if (item.status === 'rejected') {
current.rejected += item._count._all;
} else if (item.status === 'delivered') {
current.delivered += item._count._all;
}
current.alertCount = current.pending + current.failed;
current.alertCount = current.pending + current.failed + current.unconfirmed + current.rejected;
summaryMap.set(item.applicationId, current);
});
return [...summaryMap.values()];