feat: complete cmpp gateway delivery recovery workflows

2026-07-08 16:30:06 +08:00
parent cc628d0214
commit 8144f08652
60 changed files with 8901 additions and 94 deletions
@@ -40,6 +40,12 @@ function createPrismaMock() {
    enterpriseCertification: {
      count: jest.fn().mockResolvedValue(1),
    },
+    smsApplication: {
+      findMany: jest.fn().mockResolvedValue([
+        { id: 'app-1', name: '应用A' },
+        { id: 'app-2', name: '应用B' },
+      ]),
+    },
    cmppConnectionState: {
      groupBy: jest.fn().mockResolvedValue([{ status: 'connected', _count: { _all: 1 }, _sum: { currentConnections: 2, desiredConnections: 2 } }]),
    },
@@ -58,6 +64,101 @@ function createPrismaMock() {
      count: jest.fn().mockResolvedValue(1),
      groupBy: jest.fn().mockResolvedValue([{ resource: 'recharge_order', _count: { _all: 1 } }]),
    },
+    gatewaySubmitDeadLetter: {
+      findMany: jest.fn().mockResolvedValue([{
+        id: 'dead-1',
+        streamMessageId: '1710000000000-0',
+        status: 'pending',
+        failureCode: 'SUBMIT_PROCESSING_FAILED',
+        failureMessage: 'network down',
+        tenant: { name: '租户A' },
+        application: { name: '应用A' },
+        channel: { code: 'CMPP-A' },
+      }]),
+      count: jest.fn().mockResolvedValue(1),
+    },
+    gatewayDownstreamRecoveryStatus: {
+      findMany: jest.fn().mockResolvedValue([{
+        id: 'recover-1',
+        account: '100001',
+        state: 'waiting_connection',
+        lockOwner: 'gateway-a',
+        lockExpiresAt: new Date('2026-07-08T12:00:30.000Z'),
+        attemptCount: 2,
+        failureCategory: 'client_disconnected',
+        nextRetryAt: new Date('2026-07-08T12:10:00.000Z'),
+        lastError: 'downstream client is not connected',
+        tenant: { name: '租户A' },
+        application: { name: '应用A' },
+      }]),
+      findUnique: jest.fn().mockResolvedValue({
+        id: 'recover-1',
+        account: '100001',
+        state: 'waiting_connection',
+        lockOwner: 'gateway-a',
+        lockExpiresAt: new Date('2026-07-08T12:00:30.000Z'),
+        attemptCount: 2,
+        failureCategory: 'client_disconnected',
+        nextRetryAt: new Date('2026-07-08T12:10:00.000Z'),
+        lastAttemptAt: new Date('2026-07-08T12:00:00.000Z'),
+        lastSuccessAt: null,
+        lastFailureAt: new Date('2026-07-08T11:58:00.000Z'),
+        lastError: 'downstream client is not connected',
+        lastSkipReason: 'waiting for reconnect',
+        gatewayInstanceId: 'gw-01',
+        createdAt: new Date('2026-07-08T11:50:00.000Z'),
+        updatedAt: new Date('2026-07-08T12:00:00.000Z'),
+        tenant: { name: '租户A' },
+        application: { name: '应用A' },
+      }),
+      count: jest.fn().mockResolvedValue(1),
+      groupBy: jest.fn().mockResolvedValue([
+        { failureCategory: 'client_disconnected', _count: { _all: 1 } },
+      ]),
+    },
+    smsMessageSegmentAudit: {
+      findMany: jest.fn().mockResolvedValue([{
+        id: 'segment-1',
+        messageRecordId: 'record-1',
+        submitId: 'SUB-1',
+        segmentTotal: 2,
+        segmentIndex: 1,
+        sequenceId: 7,
+        gatewayMessageId: 'GW-1-A',
+        submitStatus: 'accepted',
+        receiptStatus: 'delivered',
+        channel: { name: '通道A' },
+      }]),
+    },
+    cmppDownstreamDelivery: {
+      findMany: jest.fn().mockResolvedValue([{
+        id: 'delivery-1',
+        tenantId: 'tenant-1',
+        applicationId: 'app-1',
+        messageId: 'MSG-1',
+        deliveryType: 'receipt',
+        status: 'failed',
+        payload: { account: '100001', phoneNumber: '13800000001' },
+        retryCount: 10,
+        lastError: 'client offline',
+        tenant: { name: '租户A' },
+        application: { name: '应用A' },
+        messageRecord: { messageId: 'MSG-1' },
+      }]),
+      count: jest.fn().mockResolvedValue(1),
+      groupBy: jest.fn().mockResolvedValue([
+        { deliveryType: 'receipt', status: 'pending', _count: { _all: 2 } },
+        { deliveryType: 'receipt', status: 'failed', _count: { _all: 1 } },
+        { deliveryType: 'receipt', status: 'delivered', _count: { _all: 6 } },
+        { deliveryType: 'uplink', status: 'pending', _count: { _all: 1 } },
+        { deliveryType: 'uplink', status: 'delivered', _count: { _all: 2 } },
+        { applicationId: 'app-1', status: 'pending', _count: { _all: 2 } },
+        { applicationId: 'app-1', status: 'failed', _count: { _all: 1 } },
+        { applicationId: 'app-1', status: 'delivered', _count: { _all: 5 } },
+        { applicationId: 'app-2', status: 'pending', _count: { _all: 1 } },
+        { applicationId: 'app-2', status: 'delivered', _count: { _all: 3 } },
+      ]),
+    },
  };
 }

@@ -100,7 +201,20 @@ describe('OperationsService', () => {

    expect(prisma.smsUplinkMessage.findMany).toHaveBeenCalledWith({
      where: { tenantId: 'tenant-1', channelId: 'channel-1' },
-      include: { tenant: true, channel: true },
+      include: {
+        tenant: true,
+        application: true,
+        channel: true,
+        messageRecord: { include: { application: true } },
+        matchCandidates: {
+          include: {
+            tenant: true,
+            application: true,
+            messageRecord: { include: { application: true, tenant: true, channel: true } },
+          },
+          orderBy: [{ status: 'asc' }, { confidence: 'desc' }, { createdAt: 'asc' }],
+        },
+      },
      orderBy: { receivedAt: 'desc' },
      take: 500,
    });
@@ -108,6 +222,12 @@ describe('OperationsService', () => {

  it('builds dashboard and statistics aggregates', async () => {
    const prisma = createPrismaMock();
+    prisma.cmppDownstreamDelivery.count = jest.fn()
+      .mockResolvedValueOnce(3)
+      .mockResolvedValueOnce(2)
+      .mockResolvedValueOnce(8)
+      .mockResolvedValueOnce(1)
+      .mockResolvedValueOnce(2);
    const service = new OperationsService(prisma as never);

    await expect(service.dashboard({ tenantId: 'tenant-1' })).resolves.toEqual(
@@ -116,6 +236,14 @@ describe('OperationsService', () => {
        uplinkCount: 1,
        pendingAuditCount: 6,
        gatewayConnections: [{ status: 'connected', _count: { _all: 1 }, _sum: { currentConnections: 2, desiredConnections: 2 } }],
+        downstreamDeliverySummary: expect.objectContaining({
+          pending: 3,
+          failed: 2,
+          delivered: 8,
+          stalledPending: 1,
+          recentFailed: 2,
+          alertCount: 3,
+        }),
      }),
    );
    await service.statistics({ tenantId: 'tenant-1', groupBy: 'application' });
@@ -169,4 +297,221 @@ describe('OperationsService', () => {
      }),
    );
  });
+
+  it('returns paginated gateway submit dead letters', async () => {
+    const prisma = createPrismaMock();
+    const service = new OperationsService(prisma as never);
+
+    await expect(service.listGatewaySubmitDeadLetters({
+      tenantId: 'tenant-1',
+      status: 'pending',
+      keyword: 'SUBMIT',
+      page: 1,
+      pageSize: 10,
+    })).resolves.toEqual({
+      items: [expect.objectContaining({ id: 'dead-1', status: 'pending' })],
+      total: 1,
+      page: 1,
+      pageSize: 10,
+    });
+
+    expect(prisma.gatewaySubmitDeadLetter.findMany).toHaveBeenCalledWith({
+      where: expect.objectContaining({
+        tenantId: 'tenant-1',
+        status: 'pending',
+      }),
+      include: { tenant: true, application: true, channel: true },
+      orderBy: { createdAt: 'desc' },
+      skip: 0,
+      take: 10,
+    });
+  });
+
+  it('returns paginated downstream deliveries', async () => {
+    const prisma = createPrismaMock();
+    const service = new OperationsService(prisma as never);
+
+    await expect(service.listDownstreamDeliveries({
+      tenantId: 'tenant-1',
+      deliveryType: 'receipt',
+      status: 'failed',
+      keyword: '1380',
+      page: 1,
+      pageSize: 10,
+    })).resolves.toEqual({
+      items: [expect.objectContaining({ id: 'delivery-1', status: 'failed' })],
+      total: 1,
+      page: 1,
+      pageSize: 10,
+    });
+
+    expect(prisma.cmppDownstreamDelivery.findMany).toHaveBeenCalledWith({
+      where: expect.objectContaining({
+        tenantId: 'tenant-1',
+        deliveryType: 'receipt',
+        status: 'failed',
+      }),
+      include: { tenant: true, application: true, messageRecord: true },
+      orderBy: { createdAt: 'desc' },
+      skip: 0,
+      take: 10,
+    });
+  });
+
+  it('builds downstream delivery dashboard aggregates', async () => {
+    const prisma = createPrismaMock();
+    prisma.cmppDownstreamDelivery.count = jest.fn()
+      .mockResolvedValueOnce(12)
+      .mockResolvedValueOnce(3)
+      .mockResolvedValueOnce(8)
+      .mockResolvedValueOnce(1)
+      .mockResolvedValueOnce(1)
+      .mockResolvedValueOnce(1)
+      .mockResolvedValueOnce(2)
+      .mockResolvedValueOnce(1)
+      .mockResolvedValueOnce(0);
+    prisma.cmppDownstreamDelivery.groupBy = jest.fn()
+      .mockResolvedValueOnce([
+        { deliveryType: 'receipt', status: 'pending', _count: { _all: 2 } },
+        { deliveryType: 'receipt', status: 'failed', _count: { _all: 1 } },
+        { deliveryType: 'receipt', status: 'delivered', _count: { _all: 6 } },
+        { deliveryType: 'uplink', status: 'pending', _count: { _all: 1 } },
+        { deliveryType: 'uplink', status: 'delivered', _count: { _all: 2 } },
+      ])
+      .mockResolvedValueOnce([
+        { applicationId: 'app-1', status: 'pending', _count: { _all: 2 } },
+        { applicationId: 'app-1', status: 'failed', _count: { _all: 1 } },
+        { applicationId: 'app-1', status: 'delivered', _count: { _all: 5 } },
+        { applicationId: 'app-2', status: 'pending', _count: { _all: 1 } },
+        { applicationId: 'app-2', status: 'delivered', _count: { _all: 3 } },
+      ]);
+    const service = new OperationsService(prisma as never);
+
+    await expect(service.downstreamDeliveryDashboard({
+      tenantId: 'tenant-1',
+      applicationId: 'app-1',
+      deliveryType: 'all',
+    })).resolves.toEqual({
+      summary: {
+        total: 12,
+        pending: 3,
+        delivered: 8,
+        failed: 1,
+        stalledPending: 1,
+        recentFailed: 1,
+        alertCount: 2,
+      },
+      typeBreakdown: [
+        { deliveryType: 'receipt', total: 9, pending: 2, delivered: 6, failed: 1 },
+        { deliveryType: 'uplink', total: 3, pending: 1, delivered: 2, failed: 0 },
+      ],
+      retryBuckets: [
+        { label: '0次', count: 2 },
+        { label: '1-3次', count: 1 },
+        { label: '4次及以上', count: 0 },
+      ],
+      topApplications: [
+        { applicationId: 'app-1', name: '应用A', pending: 2, failed: 1, delivered: 5, alertCount: 3 },
+        { applicationId: 'app-2', name: '应用B', pending: 1, failed: 0, delivered: 3, alertCount: 1 },
+      ],
+    });
+  });
+
+  it('returns paginated downstream recovery statuses', async () => {
+    const prisma = createPrismaMock();
+    prisma.gatewayDownstreamRecoveryStatus.count = jest.fn()
+      .mockResolvedValueOnce(1)
+      .mockResolvedValueOnce(0)
+      .mockResolvedValueOnce(0)
+      .mockResolvedValueOnce(0)
+      .mockResolvedValueOnce(1)
+      .mockResolvedValueOnce(1);
+    const service = new OperationsService(prisma as never);
+
+    await expect(service.listDownstreamRecoveryStatuses({
+      tenantId: 'tenant-1',
+      applicationId: 'app-1',
+      state: 'waiting_connection',
+      failureCategory: 'client_disconnected',
+      keyword: '100001',
+      page: 1,
+      pageSize: 10,
+    })).resolves.toEqual({
+      items: [expect.objectContaining({ id: 'recover-1', account: '100001', state: 'waiting_connection' })],
+      total: 1,
+      page: 1,
+      pageSize: 10,
+      summary: {
+        total: 1,
+        running: 0,
+        success: 0,
+        failed: 0,
+        waitingConnection: 1,
+        backoff: 1,
+        failureCategories: [
+          { category: 'client_disconnected', count: 1 },
+        ],
+      },
+    });
+  });
+
+  it('returns downstream recovery status detail', async () => {
+    const prisma = createPrismaMock();
+    const service = new OperationsService(prisma as never);
+
+    await expect(service.getDownstreamRecoveryStatus('recover-1')).resolves.toEqual(
+      expect.objectContaining({
+        id: 'recover-1',
+        account: '100001',
+        gatewayInstanceId: 'gw-01',
+      }),
+    );
+
+    expect(prisma.gatewayDownstreamRecoveryStatus.findUnique).toHaveBeenCalledWith({
+      where: { id: 'recover-1' },
+      include: { tenant: true, application: true },
+    });
+  });
+
+  it('exports downstream recovery statuses as csv rows', async () => {
+    const prisma = createPrismaMock();
+    const service = new OperationsService(prisma as never);
+
+    await expect(service.exportDownstreamRecoveryStatuses({
+      tenantId: 'tenant-1',
+      state: 'waiting_connection',
+      keyword: '100001',
+    })).resolves.toEqual(expect.objectContaining({
+      total: 1,
+      fileName: expect.stringMatching(/^gateway-downstream-recovery-statuses-\d{8}-\d{6}\.csv$/),
+      content: expect.stringContaining('100001'),
+    }));
+    expect(prisma.gatewayDownstreamRecoveryStatus.findMany).toHaveBeenCalledWith(expect.objectContaining({
+      where: expect.objectContaining({
+        failureCategory: undefined,
+      }),
+    }));
+  });
+
+  it('returns message segment audit rows', async () => {
+    const prisma = createPrismaMock();
+    const service = new OperationsService(prisma as never);
+
+    await expect(service.listMessageSegmentAudits({ messageRecordId: 'record-1' })).resolves.toEqual([
+      expect.objectContaining({
+        id: 'segment-1',
+        segmentIndex: 1,
+        gatewayMessageId: 'GW-1-A',
+      }),
+    ]);
+
+    expect(prisma.smsMessageSegmentAudit.findMany).toHaveBeenCalledWith({
+      where: {
+        messageRecordId: 'record-1',
+        messageRecord: undefined,
+      },
+      include: { channel: true, submitRecord: true },
+      orderBy: [{ submitId: 'asc' }, { segmentIndex: 'asc' }],
+    });
+  });
 });