feat: improve channel resilience and operations

2026-07-24 08:17:33 +08:00
parent 2f781ebb8a
commit afd3c96070
43 changed files with 1969 additions and 299 deletions
@@ -1,6 +1,7 @@
 import { ChannelsService } from './channels.service';

-const mockQueueAdd = jest.fn().mockResolvedValue(undefined);
+const mockJobRemove = jest.fn().mockResolvedValue(undefined);
+const mockQueueAdd = jest.fn().mockImplementation(async () => ({ id: 'job-1', remove: mockJobRemove }));
 const mockQueueClose = jest.fn().mockResolvedValue(undefined);
 const mockFetch = jest.fn().mockResolvedValue({
  ok: true,
@@ -8,6 +9,8 @@ const mockFetch = jest.fn().mockResolvedValue({
  text: jest.fn().mockResolvedValue(''),
 });
 const mockRedisXadd = jest.fn().mockResolvedValue('1710000000000-0');
+const mockRedisSet = jest.fn().mockResolvedValue('OK');
+const mockRedisEval = jest.fn().mockResolvedValue(1);
 const mockRedisDisconnect = jest.fn();

 jest.mock('bullmq', () => ({
@@ -19,6 +22,8 @@ jest.mock('bullmq', () => ({

 jest.mock('ioredis', () => jest.fn().mockImplementation(() => ({
  xadd: mockRedisXadd,
+  set: mockRedisSet,
+  eval: mockRedisEval,
  disconnect: mockRedisDisconnect,
 })));

@@ -70,7 +75,7 @@ function createPrismaMock() {
      findMany: jest.fn(),
      create: jest.fn().mockImplementation(({ data }) => Promise.resolve({ id: 'channel-1', ...data })),
      findUnique: jest.fn().mockResolvedValue(channel),
-      update: jest.fn().mockImplementation(({ data }) => Promise.resolve({ id: 'channel-1', ...data })),
+      update: jest.fn().mockImplementation(({ data }) => Promise.resolve({ ...channel, ...data })),
    },
    channelHealthMetric: { findMany: jest.fn() },
    smsChannelGroup: {
@@ -272,8 +277,11 @@ describe('ChannelsService', () => {
  beforeEach(() => {
    mockQueueAdd.mockClear();
    mockQueueClose.mockClear();
+    mockJobRemove.mockClear();
    mockFetch.mockClear();
    mockRedisXadd.mockClear();
+    mockRedisSet.mockClear();
+    mockRedisEval.mockClear();
    mockRedisDisconnect.mockClear();
    global.fetch = mockFetch as never;
  });
@@ -300,12 +308,60 @@ describe('ChannelsService', () => {
      channelId: 'channel-1',
      reason: 'gateway_restarted',
      channel: expect.objectContaining({ rateLimitPerSecond: 100 }),
-    }), { jobId: 'channel-1:primary:connect' });
+    }), expect.objectContaining({
+      jobId: expect.stringMatching(/^gateway-connect-channel-1-/),
+      removeOnComplete: 1000,
+      removeOnFail: 1000,
+    }));
    expect(mockFetch).toHaveBeenCalledWith('http://127.0.0.1:8090/connections/connect', expect.objectContaining({
      body: expect.stringContaining('"reason":"gateway_restarted"'),
    }));
  });

+  it('uses the direct Gateway control path when the Redis marker queue is unavailable', async () => {
+    const prisma = createPrismaMock();
+    mockQueueAdd.mockRejectedValueOnce(new Error('redis unavailable'));
+    const service = new ChannelsService(prisma as never);
+
+    await expect(service.createChannel({
+      code: 'CMPP-DIRECT',
+      name: '直连控制测试',
+      gatewayHost: '127.0.0.1',
+      gatewayPort: 17890,
+      account: 'sp',
+      passwordCipher: 'secret',
+      srcId: '10690000',
+      status: 'active',
+    })).resolves.toEqual(expect.objectContaining({ id: 'channel-1' }));
+    expect(mockFetch).toHaveBeenCalledWith('http://127.0.0.1:8090/connections/connect', expect.objectContaining({
+      method: 'POST',
+    }));
+  });
+
+  it('reuses a supplier state created concurrently by another API instance', async () => {
+    const prisma = createPrismaMock();
+    prisma.cmppConnectionState.findFirst
+      .mockResolvedValueOnce(null)
+      .mockResolvedValueOnce({ id: 'state-concurrent', applicationId: null });
+    prisma.cmppConnectionState.create.mockRejectedValueOnce({ code: 'P2002' });
+    const service = new ChannelsService(prisma as never);
+
+    await expect(service.createChannel({
+      code: 'CMPP-CONCURRENT',
+      name: '并发状态测试',
+      gatewayHost: '127.0.0.1',
+      gatewayPort: 17890,
+      account: 'sp',
+      passwordCipher: 'secret',
+      srcId: '10690000',
+      status: 'active',
+    })).resolves.toEqual(expect.objectContaining({ id: 'channel-1' }));
+    expect(prisma.cmppConnectionState.update).toHaveBeenCalledWith({
+      where: { id: 'state-concurrent' },
+      data: expect.objectContaining({ status: 'connecting' }),
+    });
+  });
+
  it('creates CMPP channels and route rules with first-version defaults', async () => {
    const prisma = createPrismaMock();
    const service = new ChannelsService(prisma as never);
@@ -333,7 +389,7 @@ describe('ChannelsService', () => {
        rateLimitPerSecond: 750,
        sendRegion: '全国',
        status: 'active',
-        config: expect.objectContaining({ desiredConnections: 2, windowSize: 32, extensionDigits: 4 }),
+        config: expect.objectContaining({ desiredConnections: 2, windowSize: 32, extensionDigits: 4, serviceId: 'SMS' }),
      }),
    });
    expect(prisma.cmppConnectionState.create).toHaveBeenCalledWith({
@@ -351,7 +407,11 @@ describe('ChannelsService', () => {
      connectionId: 'channel-1:primary',
      reason: 'channel_created',
      channel: expect.objectContaining({ cmppVersion: '2.0' }),
-    }), { jobId: 'channel-1:primary:connect' });
+    }), expect.objectContaining({
+      jobId: expect.stringMatching(/^gateway-connect-channel-1-/),
+      removeOnComplete: 1000,
+      removeOnFail: 1000,
+    }));
    expect(mockFetch).toHaveBeenCalledWith('http://127.0.0.1:8090/connections/connect', expect.objectContaining({
      method: 'POST',
      body: expect.stringContaining('"messageType":"ConnectChannel"'),
@@ -372,6 +432,29 @@ describe('ChannelsService', () => {
    });
  });

+  it('defaults new channels to port 7890, CMPP and SMS while rejecting protocol overrides', async () => {
+    const prisma = createPrismaMock();
+    const service = new ChannelsService(prisma as never);
+
+    await service.createChannel({
+      code: 'CMPP-DEFAULT',
+      name: '默认通道',
+      gatewayHost: '127.0.0.1',
+      account: 'sp',
+      passwordCipher: 'secret',
+      srcId: '10690000',
+      protocol: 'HTTP',
+    });
+
+    expect(prisma.smsChannel.create).toHaveBeenCalledWith({
+      data: expect.objectContaining({
+        gatewayPort: 7890,
+        protocol: 'CMPP',
+        config: expect.objectContaining({ serviceId: 'SMS' }),
+      }),
+    });
+  });
+
  it('preserves explicit CMPP 3.0 and rejects unsupported CMPP versions', async () => {
    const prisma = createPrismaMock();
    const service = new ChannelsService(prisma as never);
@@ -417,6 +500,7 @@ describe('ChannelsService', () => {

    await expect(service.createChannel({ ...channel, rateLimitPerSecond: 2001 })).rejects.toThrow('rateLimitPerSecond must be between 1 and 2000');
    await expect(service.createChannel({ ...channel, config: { extensionDigits: 21 } })).rejects.toThrow('extensionDigits must be an integer between 0 and 20');
+    await expect(service.createChannel({ ...channel, config: { serviceId: '业务代码' } })).rejects.toThrow('serviceId must contain 1 to 10 ASCII characters');
  });

  it('updates CMPP channel configuration without requiring password changes', async () => {
@@ -730,11 +814,155 @@ describe('ChannelsService', () => {
      messageType: 'ConnectChannel',
      channelId: 'channel-1',
      reason: 'channel_enabled',
-    }), { jobId: 'channel-1:primary:connect' });
+    }), expect.objectContaining({
+      jobId: expect.stringMatching(/^gateway-connect-channel-1-/),
+      removeOnComplete: 1000,
+      removeOnFail: 1000,
+    }));
    expect(mockFetch).toHaveBeenCalledWith('http://127.0.0.1:8090/connections/connect', expect.objectContaining({
      method: 'POST',
      body: expect.stringContaining('"reason":"channel_enabled"'),
    }));
+    expect(mockFetch).toHaveBeenCalledWith('http://127.0.0.1:8090/connections/disconnect', expect.objectContaining({
+      method: 'POST',
+      body: expect.stringContaining('"reason":"channel_disabled"'),
+    }));
+  });
+
+  it('reconnects active failed channels and disconnects inactive live channels during reconciliation', async () => {
+    const prisma = createPrismaMock();
+    prisma.smsChannel.findMany.mockResolvedValue([
+      {
+        ...(await prisma.smsChannel.findUnique()),
+        id: 'channel-active',
+        status: 'active',
+        connectionStates: [{
+          id: 'state-active',
+          connectionId: 'channel-active:primary',
+          status: 'failed',
+          currentConnections: 0,
+          desiredConnections: 1,
+          lastHeartbeatAt: null,
+          nextReconnectAt: new Date('2026-07-23T10:00:00.000Z'),
+        }],
+      },
+      {
+        ...(await prisma.smsChannel.findUnique()),
+        id: 'channel-disabled',
+        status: 'disabled',
+        connectionStates: [{
+          id: 'state-disabled',
+          connectionId: 'channel-disabled:primary',
+          status: 'connected',
+          currentConnections: 1,
+          desiredConnections: 1,
+          lastHeartbeatAt: new Date('2026-07-23T10:59:55.000Z'),
+          nextReconnectAt: null,
+        }],
+      },
+    ]);
+    const service = new ChannelsService(prisma as never);
+
+    const result = await service.reconcileGatewayConnections(new Date('2026-07-23T11:00:00.000Z'));
+
+    expect(result).toEqual({ scanned: 2, reconnectRequested: 1, disconnectRequested: 1 });
+    expect(mockFetch).toHaveBeenCalledWith('http://127.0.0.1:8090/connections/connect', expect.objectContaining({
+      body: expect.stringContaining('"reason":"automatic_reconnect"'),
+    }));
+    expect(mockFetch).toHaveBeenCalledWith('http://127.0.0.1:8090/connections/disconnect', expect.objectContaining({
+      body: expect.stringContaining('"reason":"inactive_channel_reconcile"'),
+    }));
+    expect(mockRedisSet).toHaveBeenCalledTimes(2);
+    expect(mockRedisEval).toHaveBeenCalledTimes(2);
+  });
+
+  it('does not reconnect a fresh healthy supplier connection', async () => {
+    const prisma = createPrismaMock();
+    prisma.smsChannel.findMany.mockResolvedValue([{
+      ...(await prisma.smsChannel.findUnique()),
+      status: 'active',
+      config: { desiredConnections: 1, heartbeatIntervalSeconds: 30, heartbeatMissThreshold: 3 },
+      connectionStates: [{
+        connectionId: 'channel-1:primary',
+        status: 'connected',
+        currentConnections: 1,
+        desiredConnections: 1,
+        lastHeartbeatAt: new Date('2026-07-23T10:59:55.000Z'),
+        nextReconnectAt: null,
+      }],
+    }]);
+    const service = new ChannelsService(prisma as never);
+
+    await expect(service.reconcileGatewayConnections(new Date('2026-07-23T11:00:00.000Z')))
+      .resolves.toEqual({ scanned: 1, reconnectRequested: 0, disconnectRequested: 0 });
+    expect(mockFetch).not.toHaveBeenCalled();
+    expect(mockRedisSet).not.toHaveBeenCalled();
+  });
+
+  it('skips duplicate reconciliation when another API instance owns the Redis lease', async () => {
+    const prisma = createPrismaMock();
+    prisma.smsChannel.findMany.mockResolvedValue([{
+      ...(await prisma.smsChannel.findUnique()),
+      status: 'active',
+      connectionStates: [],
+    }]);
+    mockRedisSet.mockResolvedValueOnce(null);
+    const service = new ChannelsService(prisma as never);
+
+    await expect(service.reconcileGatewayConnections(new Date('2026-07-23T11:00:00.000Z')))
+      .resolves.toEqual({ scanned: 1, reconnectRequested: 0, disconnectRequested: 0 });
+    expect(mockFetch).not.toHaveBeenCalled();
+    expect(mockRedisEval).not.toHaveBeenCalled();
+  });
+
+  it('stores supplier heartbeat as a connected state and heartbeat audit event', async () => {
+    const prisma = createPrismaMock();
+    const service = new ChannelsService(prisma as never);
+
+    await service.upsertConnectionState({
+      channelId: 'channel-1',
+      connectionId: 'channel-1:primary',
+      status: 'heartbeat',
+      currentConnections: 1,
+      lastHeartbeatAt: '2026-07-23T11:00:00.000Z',
+    });
+
+    expect(prisma.cmppConnectionState.create).toHaveBeenCalledWith({
+      data: expect.objectContaining({
+        status: 'connected',
+        lastHeartbeatAt: new Date('2026-07-23T11:00:00.000Z'),
+      }),
+    });
+    expect(prisma.operationLog.create).toHaveBeenCalledWith({
+      data: expect.objectContaining({ action: 'cmpp_connection.heartbeat' }),
+    });
+  });
+
+  it('updates every supplier heartbeat without flooding operation logs', async () => {
+    const prisma = createPrismaMock();
+    prisma.cmppConnectionState.findFirst.mockResolvedValue({
+      id: 'state-1',
+      applicationId: null,
+      lastHeartbeatAt: new Date('2026-07-23T10:59:30.000Z'),
+    });
+    const service = new ChannelsService(prisma as never);
+
+    await service.upsertConnectionState({
+      channelId: 'channel-1',
+      connectionId: 'channel-1:primary',
+      status: 'heartbeat',
+      currentConnections: 1,
+      lastHeartbeatAt: '2026-07-23T11:00:00.000Z',
+    });
+
+    expect(prisma.cmppConnectionState.update).toHaveBeenCalledWith({
+      where: { id: 'state-1' },
+      data: expect.objectContaining({
+        status: 'connected',
+        lastHeartbeatAt: new Date('2026-07-23T11:00:00.000Z'),
+      }),
+    });
+    expect(prisma.operationLog.create).not.toHaveBeenCalled();
  });

  it('copies channels with report field configuration and report materials', async () => {