feat: improve channel resilience and operations

2026-07-24 08:17:33 +08:00
parent 2f781ebb8a
commit afd3c96070
43 changed files with 1969 additions and 299 deletions
@@ -1,6 +1,7 @@
 import { ChannelsService } from './channels.service';

-const mockQueueAdd = jest.fn().mockResolvedValue(undefined);
+const mockJobRemove = jest.fn().mockResolvedValue(undefined);
+const mockQueueAdd = jest.fn().mockImplementation(async () => ({ id: 'job-1', remove: mockJobRemove }));
 const mockQueueClose = jest.fn().mockResolvedValue(undefined);
 const mockFetch = jest.fn().mockResolvedValue({
  ok: true,
@@ -8,6 +9,8 @@ const mockFetch = jest.fn().mockResolvedValue({
  text: jest.fn().mockResolvedValue(''),
 });
 const mockRedisXadd = jest.fn().mockResolvedValue('1710000000000-0');
+const mockRedisSet = jest.fn().mockResolvedValue('OK');
+const mockRedisEval = jest.fn().mockResolvedValue(1);
 const mockRedisDisconnect = jest.fn();

 jest.mock('bullmq', () => ({
@@ -19,6 +22,8 @@ jest.mock('bullmq', () => ({

 jest.mock('ioredis', () => jest.fn().mockImplementation(() => ({
  xadd: mockRedisXadd,
+  set: mockRedisSet,
+  eval: mockRedisEval,
  disconnect: mockRedisDisconnect,
 })));

@@ -70,7 +75,7 @@ function createPrismaMock() {
      findMany: jest.fn(),
      create: jest.fn().mockImplementation(({ data }) => Promise.resolve({ id: 'channel-1', ...data })),
      findUnique: jest.fn().mockResolvedValue(channel),
-      update: jest.fn().mockImplementation(({ data }) => Promise.resolve({ id: 'channel-1', ...data })),
+      update: jest.fn().mockImplementation(({ data }) => Promise.resolve({ ...channel, ...data })),
    },
    channelHealthMetric: { findMany: jest.fn() },
    smsChannelGroup: {
@@ -272,8 +277,11 @@ describe('ChannelsService', () => {
  beforeEach(() => {
    mockQueueAdd.mockClear();
    mockQueueClose.mockClear();
+    mockJobRemove.mockClear();
    mockFetch.mockClear();
    mockRedisXadd.mockClear();
+    mockRedisSet.mockClear();
+    mockRedisEval.mockClear();
    mockRedisDisconnect.mockClear();
    global.fetch = mockFetch as never;
  });
@@ -300,12 +308,60 @@ describe('ChannelsService', () => {
      channelId: 'channel-1',
      reason: 'gateway_restarted',
      channel: expect.objectContaining({ rateLimitPerSecond: 100 }),
-    }), { jobId: 'channel-1:primary:connect' });
+    }), expect.objectContaining({
+      jobId: expect.stringMatching(/^gateway-connect-channel-1-/),
+      removeOnComplete: 1000,
+      removeOnFail: 1000,
+    }));
    expect(mockFetch).toHaveBeenCalledWith('http://127.0.0.1:8090/connections/connect', expect.objectContaining({
      body: expect.stringContaining('"reason":"gateway_restarted"'),
    }));
  });

+  it('uses the direct Gateway control path when the Redis marker queue is unavailable', async () => {
+    const prisma = createPrismaMock();
+    mockQueueAdd.mockRejectedValueOnce(new Error('redis unavailable'));
+    const service = new ChannelsService(prisma as never);
+
+    await expect(service.createChannel({
+      code: 'CMPP-DIRECT',
+      name: '直连控制测试',
+      gatewayHost: '127.0.0.1',
+      gatewayPort: 17890,
+      account: 'sp',
+      passwordCipher: 'secret',
+      srcId: '10690000',
+      status: 'active',
+    })).resolves.toEqual(expect.objectContaining({ id: 'channel-1' }));
+    expect(mockFetch).toHaveBeenCalledWith('http://127.0.0.1:8090/connections/connect', expect.objectContaining({
+      method: 'POST',
+    }));
+  });
+
+  it('reuses a supplier state created concurrently by another API instance', async () => {
+    const prisma = createPrismaMock();
+    prisma.cmppConnectionState.findFirst
+      .mockResolvedValueOnce(null)
+      .mockResolvedValueOnce({ id: 'state-concurrent', applicationId: null });
+    prisma.cmppConnectionState.create.mockRejectedValueOnce({ code: 'P2002' });
+    const service = new ChannelsService(prisma as never);
+
+    await expect(service.createChannel({
+      code: 'CMPP-CONCURRENT',
+      name: '并发状态测试',
+      gatewayHost: '127.0.0.1',
+      gatewayPort: 17890,
+      account: 'sp',
+      passwordCipher: 'secret',
+      srcId: '10690000',
+      status: 'active',
+    })).resolves.toEqual(expect.objectContaining({ id: 'channel-1' }));
+    expect(prisma.cmppConnectionState.update).toHaveBeenCalledWith({
+      where: { id: 'state-concurrent' },
+      data: expect.objectContaining({ status: 'connecting' }),
+    });
+  });
+
  it('creates CMPP channels and route rules with first-version defaults', async () => {
    const prisma = createPrismaMock();
    const service = new ChannelsService(prisma as never);
@@ -333,7 +389,7 @@ describe('ChannelsService', () => {
        rateLimitPerSecond: 750,
        sendRegion: '全国',
        status: 'active',
-        config: expect.objectContaining({ desiredConnections: 2, windowSize: 32, extensionDigits: 4 }),
+        config: expect.objectContaining({ desiredConnections: 2, windowSize: 32, extensionDigits: 4, serviceId: 'SMS' }),
      }),
    });
    expect(prisma.cmppConnectionState.create).toHaveBeenCalledWith({
@@ -351,7 +407,11 @@ describe('ChannelsService', () => {
      connectionId: 'channel-1:primary',
      reason: 'channel_created',
      channel: expect.objectContaining({ cmppVersion: '2.0' }),
-    }), { jobId: 'channel-1:primary:connect' });
+    }), expect.objectContaining({
+      jobId: expect.stringMatching(/^gateway-connect-channel-1-/),
+      removeOnComplete: 1000,
+      removeOnFail: 1000,
+    }));
    expect(mockFetch).toHaveBeenCalledWith('http://127.0.0.1:8090/connections/connect', expect.objectContaining({
      method: 'POST',
      body: expect.stringContaining('"messageType":"ConnectChannel"'),
@@ -372,6 +432,29 @@ describe('ChannelsService', () => {
    });
  });

+  it('defaults new channels to port 7890, CMPP and SMS while rejecting protocol overrides', async () => {
+    const prisma = createPrismaMock();
+    const service = new ChannelsService(prisma as never);
+
+    await service.createChannel({
+      code: 'CMPP-DEFAULT',
+      name: '默认通道',
+      gatewayHost: '127.0.0.1',
+      account: 'sp',
+      passwordCipher: 'secret',
+      srcId: '10690000',
+      protocol: 'HTTP',
+    });
+
+    expect(prisma.smsChannel.create).toHaveBeenCalledWith({
+      data: expect.objectContaining({
+        gatewayPort: 7890,
+        protocol: 'CMPP',
+        config: expect.objectContaining({ serviceId: 'SMS' }),
+      }),
+    });
+  });
+
  it('preserves explicit CMPP 3.0 and rejects unsupported CMPP versions', async () => {
    const prisma = createPrismaMock();
    const service = new ChannelsService(prisma as never);
@@ -417,6 +500,7 @@ describe('ChannelsService', () => {

    await expect(service.createChannel({ ...channel, rateLimitPerSecond: 2001 })).rejects.toThrow('rateLimitPerSecond must be between 1 and 2000');
    await expect(service.createChannel({ ...channel, config: { extensionDigits: 21 } })).rejects.toThrow('extensionDigits must be an integer between 0 and 20');
+    await expect(service.createChannel({ ...channel, config: { serviceId: '业务代码' } })).rejects.toThrow('serviceId must contain 1 to 10 ASCII characters');
  });

  it('updates CMPP channel configuration without requiring password changes', async () => {
@@ -730,11 +814,155 @@ describe('ChannelsService', () => {
      messageType: 'ConnectChannel',
      channelId: 'channel-1',
      reason: 'channel_enabled',
-    }), { jobId: 'channel-1:primary:connect' });
+    }), expect.objectContaining({
+      jobId: expect.stringMatching(/^gateway-connect-channel-1-/),
+      removeOnComplete: 1000,
+      removeOnFail: 1000,
+    }));
    expect(mockFetch).toHaveBeenCalledWith('http://127.0.0.1:8090/connections/connect', expect.objectContaining({
      method: 'POST',
      body: expect.stringContaining('"reason":"channel_enabled"'),
    }));
+    expect(mockFetch).toHaveBeenCalledWith('http://127.0.0.1:8090/connections/disconnect', expect.objectContaining({
+      method: 'POST',
+      body: expect.stringContaining('"reason":"channel_disabled"'),
+    }));
+  });
+
+  it('reconnects active failed channels and disconnects inactive live channels during reconciliation', async () => {
+    const prisma = createPrismaMock();
+    prisma.smsChannel.findMany.mockResolvedValue([
+      {
+        ...(await prisma.smsChannel.findUnique()),
+        id: 'channel-active',
+        status: 'active',
+        connectionStates: [{
+          id: 'state-active',
+          connectionId: 'channel-active:primary',
+          status: 'failed',
+          currentConnections: 0,
+          desiredConnections: 1,
+          lastHeartbeatAt: null,
+          nextReconnectAt: new Date('2026-07-23T10:00:00.000Z'),
+        }],
+      },
+      {
+        ...(await prisma.smsChannel.findUnique()),
+        id: 'channel-disabled',
+        status: 'disabled',
+        connectionStates: [{
+          id: 'state-disabled',
+          connectionId: 'channel-disabled:primary',
+          status: 'connected',
+          currentConnections: 1,
+          desiredConnections: 1,
+          lastHeartbeatAt: new Date('2026-07-23T10:59:55.000Z'),
+          nextReconnectAt: null,
+        }],
+      },
+    ]);
+    const service = new ChannelsService(prisma as never);
+
+    const result = await service.reconcileGatewayConnections(new Date('2026-07-23T11:00:00.000Z'));
+
+    expect(result).toEqual({ scanned: 2, reconnectRequested: 1, disconnectRequested: 1 });
+    expect(mockFetch).toHaveBeenCalledWith('http://127.0.0.1:8090/connections/connect', expect.objectContaining({
+      body: expect.stringContaining('"reason":"automatic_reconnect"'),
+    }));
+    expect(mockFetch).toHaveBeenCalledWith('http://127.0.0.1:8090/connections/disconnect', expect.objectContaining({
+      body: expect.stringContaining('"reason":"inactive_channel_reconcile"'),
+    }));
+    expect(mockRedisSet).toHaveBeenCalledTimes(2);
+    expect(mockRedisEval).toHaveBeenCalledTimes(2);
+  });
+
+  it('does not reconnect a fresh healthy supplier connection', async () => {
+    const prisma = createPrismaMock();
+    prisma.smsChannel.findMany.mockResolvedValue([{
+      ...(await prisma.smsChannel.findUnique()),
+      status: 'active',
+      config: { desiredConnections: 1, heartbeatIntervalSeconds: 30, heartbeatMissThreshold: 3 },
+      connectionStates: [{
+        connectionId: 'channel-1:primary',
+        status: 'connected',
+        currentConnections: 1,
+        desiredConnections: 1,
+        lastHeartbeatAt: new Date('2026-07-23T10:59:55.000Z'),
+        nextReconnectAt: null,
+      }],
+    }]);
+    const service = new ChannelsService(prisma as never);
+
+    await expect(service.reconcileGatewayConnections(new Date('2026-07-23T11:00:00.000Z')))
+      .resolves.toEqual({ scanned: 1, reconnectRequested: 0, disconnectRequested: 0 });
+    expect(mockFetch).not.toHaveBeenCalled();
+    expect(mockRedisSet).not.toHaveBeenCalled();
+  });
+
+  it('skips duplicate reconciliation when another API instance owns the Redis lease', async () => {
+    const prisma = createPrismaMock();
+    prisma.smsChannel.findMany.mockResolvedValue([{
+      ...(await prisma.smsChannel.findUnique()),
+      status: 'active',
+      connectionStates: [],
+    }]);
+    mockRedisSet.mockResolvedValueOnce(null);
+    const service = new ChannelsService(prisma as never);
+
+    await expect(service.reconcileGatewayConnections(new Date('2026-07-23T11:00:00.000Z')))
+      .resolves.toEqual({ scanned: 1, reconnectRequested: 0, disconnectRequested: 0 });
+    expect(mockFetch).not.toHaveBeenCalled();
+    expect(mockRedisEval).not.toHaveBeenCalled();
+  });
+
+  it('stores supplier heartbeat as a connected state and heartbeat audit event', async () => {
+    const prisma = createPrismaMock();
+    const service = new ChannelsService(prisma as never);
+
+    await service.upsertConnectionState({
+      channelId: 'channel-1',
+      connectionId: 'channel-1:primary',
+      status: 'heartbeat',
+      currentConnections: 1,
+      lastHeartbeatAt: '2026-07-23T11:00:00.000Z',
+    });
+
+    expect(prisma.cmppConnectionState.create).toHaveBeenCalledWith({
+      data: expect.objectContaining({
+        status: 'connected',
+        lastHeartbeatAt: new Date('2026-07-23T11:00:00.000Z'),
+      }),
+    });
+    expect(prisma.operationLog.create).toHaveBeenCalledWith({
+      data: expect.objectContaining({ action: 'cmpp_connection.heartbeat' }),
+    });
+  });
+
+  it('updates every supplier heartbeat without flooding operation logs', async () => {
+    const prisma = createPrismaMock();
+    prisma.cmppConnectionState.findFirst.mockResolvedValue({
+      id: 'state-1',
+      applicationId: null,
+      lastHeartbeatAt: new Date('2026-07-23T10:59:30.000Z'),
+    });
+    const service = new ChannelsService(prisma as never);
+
+    await service.upsertConnectionState({
+      channelId: 'channel-1',
+      connectionId: 'channel-1:primary',
+      status: 'heartbeat',
+      currentConnections: 1,
+      lastHeartbeatAt: '2026-07-23T11:00:00.000Z',
+    });
+
+    expect(prisma.cmppConnectionState.update).toHaveBeenCalledWith({
+      where: { id: 'state-1' },
+      data: expect.objectContaining({
+        status: 'connected',
+        lastHeartbeatAt: new Date('2026-07-23T11:00:00.000Z'),
+      }),
+    });
+    expect(prisma.operationLog.create).not.toHaveBeenCalled();
  });

  it('copies channels with report field configuration and report materials', async () => {
@@ -13,7 +13,7 @@ export interface CreateChannelDto {
  sendRegion?: string;
  protocol?: string;
  gatewayHost: string;
-  gatewayPort: number;
+  gatewayPort?: number;
  enterpriseCode?: string;
  account: string;
  passwordCipher: string;
@@ -24,6 +24,8 @@ export interface CreateChannelDto {
  status?: string;
  desiredConnections?: number;
  windowSize?: number;
+  heartbeatIntervalSeconds?: number;
+  heartbeatMissThreshold?: number;
  config?: Record<string, unknown>;
 }

@@ -151,6 +153,9 @@ export interface UpsertConnectionStateDto {
  lastDisconnectedAt?: string;
  lastHeartbeatAt?: string;
  reconnectCount?: number;
+  lastReconnectAttemptAt?: string;
+  nextReconnectAt?: string;
+  lastErrorCategory?: string;
  lastError?: string;
 }

@@ -182,6 +187,11 @@ const DEFAULT_CHANNEL_CONNECTION_ID = 'primary';
 const DEFAULT_CONNECTING_TIMEOUT_MS = 30_000;
 const DEFAULT_CONNECTING_TIMEOUT_SCAN_MS = 5_000;
 const DEFAULT_GATEWAY_STARTUP_RECONNECT_DELAY_MS = 1_000;
+const DEFAULT_GATEWAY_RECONCILE_INTERVAL_MS = 30_000;
+const DEFAULT_GATEWAY_CONTROL_TIMEOUT_MS = 10_000;
+const DEFAULT_HEARTBEAT_INTERVAL_SECONDS = 30;
+const DEFAULT_HEARTBEAT_MISS_THRESHOLD = 3;
+const HEARTBEAT_AUDIT_INTERVAL_MS = 5 * 60_000;
 const CONNECTING_TIMEOUT_ERROR = 'Gateway connection request timed out';
 const DEFAULT_CMPP_VERSION = '2.0';

@@ -193,6 +203,7 @@ export class ChannelsService implements OnModuleInit, OnModuleDestroy {
  private redis?: IORedis;
  private connectionTimeoutTimer?: ReturnType<typeof setInterval>;
  private gatewayStartupReconnectTimer?: ReturnType<typeof setTimeout>;
+  private gatewayReconcileTimer?: ReturnType<typeof setInterval>;

  constructor(private readonly prisma: PrismaService) {}

@@ -209,6 +220,14 @@ export class ChannelsService implements OnModuleInit, OnModuleDestroy {
      void this.reconnectActiveChannelsAfterGatewayRestart();
    }, getPositiveIntegerEnv('GATEWAY_STARTUP_RECONNECT_DELAY_MS', DEFAULT_GATEWAY_STARTUP_RECONNECT_DELAY_MS));
    this.gatewayStartupReconnectTimer.unref?.();
+    if (process.env.GATEWAY_CONNECTION_RECONCILER_DISABLED !== 'true') {
+      this.gatewayReconcileTimer = setInterval(() => {
+        void this.reconcileGatewayConnections().catch((error) => {
+          this.logger.error(`Failed to reconcile supplier connections: ${error instanceof Error ? error.message : String(error)}`);
+        });
+      }, getPositiveIntegerEnv('GATEWAY_CONNECTION_RECONCILE_INTERVAL_MS', DEFAULT_GATEWAY_RECONCILE_INTERVAL_MS));
+      this.gatewayReconcileTimer.unref?.();
+    }
  }

  async onModuleDestroy() {
@@ -218,6 +237,9 @@ export class ChannelsService implements OnModuleInit, OnModuleDestroy {
    if (this.gatewayStartupReconnectTimer) {
      clearTimeout(this.gatewayStartupReconnectTimer);
    }
+    if (this.gatewayReconcileTimer) {
+      clearInterval(this.gatewayReconcileTimer);
+    }
    await this.gatewayConnectionQueue?.close();
    await this.gatewaySubmitQueue?.close();
    this.redis?.disconnect();
@@ -232,19 +254,26 @@ export class ChannelsService implements OnModuleInit, OnModuleDestroy {

  async createChannel(data: CreateChannelDto) {
    assertMoneyUnits(data.unitPrice ?? 0, '通道单价');
-    const missingFields = ['code', 'name', 'gatewayHost', 'gatewayPort', 'account', 'passwordCipher', 'srcId'].filter((field) => {
+    const missingFields = ['code', 'name', 'gatewayHost', 'account', 'passwordCipher', 'srcId'].filter((field) => {
      const value = data[field as keyof CreateChannelDto];
      return value === undefined || value === null || value === '';
    });
    if (missingFields.length > 0) {
      throw new BadRequestException(`Missing required channel fields: ${missingFields.join(', ')}`);
    }
-    const gatewayPort = Number(data.gatewayPort);
+    const gatewayPort = Number(data.gatewayPort ?? 7890);
    if (!Number.isInteger(gatewayPort) || gatewayPort <= 0 || gatewayPort > 65535) {
      throw new BadRequestException('gatewayPort must be an integer between 1 and 65535');
    }
    const cmppVersion = normalizeCmppVersion(data.cmppVersion);
-    const config = normalizeChannelRuntimeConfig(undefined, data.config, data.desiredConnections, data.windowSize);
+    const config = normalizeChannelRuntimeConfig(
+      undefined,
+      data.config,
+      data.desiredConnections,
+      data.windowSize,
+      data.heartbeatIntervalSeconds,
+      data.heartbeatMissThreshold,
+    );
    const rateLimitPerSecond = normalizeChannelRateLimit(data.rateLimitPerSecond);
    const channel = await this.prisma.smsChannel.create({
      data: {
@@ -252,7 +281,7 @@ export class ChannelsService implements OnModuleInit, OnModuleDestroy {
        name: data.name,
        carrier: data.carrier,
        sendRegion: data.sendRegion ?? '全国',
-        protocol: data.protocol ?? 'CMPP',
+        protocol: 'CMPP',
        gatewayHost: data.gatewayHost,
        gatewayPort,
        enterpriseCode: data.enterpriseCode,
@@ -285,8 +314,19 @@ export class ChannelsService implements OnModuleInit, OnModuleDestroy {
      throw new BadRequestException('gatewayPort must be an integer between 1 and 65535');
    }
    const cmppVersion = data.cmppVersion === undefined ? undefined : normalizeCmppVersion(data.cmppVersion);
-    const config = data.config !== undefined || data.desiredConnections !== undefined || data.windowSize !== undefined
-      ? normalizeChannelRuntimeConfig(channel.config, data.config, data.desiredConnections, data.windowSize)
+    const config = data.config !== undefined
+      || data.desiredConnections !== undefined
+      || data.windowSize !== undefined
+      || data.heartbeatIntervalSeconds !== undefined
+      || data.heartbeatMissThreshold !== undefined
+      ? normalizeChannelRuntimeConfig(
+        channel.config,
+        data.config,
+        data.desiredConnections,
+        data.windowSize,
+        data.heartbeatIntervalSeconds,
+        data.heartbeatMissThreshold,
+      )
      : undefined;
    const rateLimitPerSecond = data.rateLimitPerSecond === undefined
      ? undefined
@@ -298,7 +338,7 @@ export class ChannelsService implements OnModuleInit, OnModuleDestroy {
        name: data.name,
        carrier: data.carrier,
        sendRegion: data.sendRegion,
-        protocol: data.protocol,
+        protocol: 'CMPP',
        gatewayHost: data.gatewayHost,
        gatewayPort,
        enterpriseCode: data.enterpriseCode,
@@ -334,6 +374,26 @@ export class ChannelsService implements OnModuleInit, OnModuleDestroy {
        } as Prisma.InputJsonValue,
      },
    });
+    const connectionConfigChanged = [
+      'gatewayHost',
+      'gatewayPort',
+      'account',
+      'passwordCipher',
+      'cmppVersion',
+      'rateLimitPerSecond',
+      'desiredConnections',
+      'windowSize',
+      'heartbeatIntervalSeconds',
+      'heartbeatMissThreshold',
+    ].some((key) => data[key as keyof UpdateChannelDto] !== undefined)
+      || Boolean(data.config && ['desiredConnections', 'windowSize', 'heartbeatIntervalSeconds', 'heartbeatMissThreshold']
+        .some((key) => key in data.config!));
+    const updatedStatus = data.status ?? channel.status;
+    if (updatedStatus === 'active' && (connectionConfigChanged || channel.status !== 'active')) {
+      await this.requestChannelConnection(updated, 'channel_updated');
+    } else if (updatedStatus !== 'active' && channel.status === 'active') {
+      await this.requestChannelDisconnection(updated, 'channel_disabled');
+    }
    return updated;
  }

@@ -358,6 +418,12 @@ export class ChannelsService implements OnModuleInit, OnModuleDestroy {
    });
    if (data.status === 'active') {
      await this.requestChannelConnection(updated, 'channel_enabled', data.operatorId);
+    } else if (channel.status === 'active' || data.status === 'deleted') {
+      await this.requestChannelDisconnection(
+        updated,
+        data.status === 'deleted' ? 'channel_deleted' : 'channel_disabled',
+        data.operatorId,
+      );
    }
    return updated;
  }
@@ -615,7 +681,8 @@ export class ChannelsService implements OnModuleInit, OnModuleDestroy {
  }

  async upsertConnectionState(data: UpsertConnectionStateDto) {
-    const status = normalizeGatewayConnectionStatus(data.status);
+    const rawStatus = data.status;
+    const status = normalizeGatewayConnectionStatus(rawStatus);
    if (data.applicationId) {
      const application = await this.prisma.smsApplication.findUnique({ where: { id: data.applicationId }, select: { tenantId: true } });
      if (!application) {
@@ -636,6 +703,9 @@ export class ChannelsService implements OnModuleInit, OnModuleDestroy {
      lastDisconnectedAt: data.lastDisconnectedAt ? new Date(data.lastDisconnectedAt) : undefined,
      lastHeartbeatAt: data.lastHeartbeatAt ? new Date(data.lastHeartbeatAt) : undefined,
      reconnectCount: data.reconnectCount ?? 0,
+      lastReconnectAttemptAt: data.lastReconnectAttemptAt ? new Date(data.lastReconnectAttemptAt) : undefined,
+      nextReconnectAt: data.nextReconnectAt ? new Date(data.nextReconnectAt) : status === 'connected' ? null : undefined,
+      lastErrorCategory: status === 'connected' ? null : data.lastErrorCategory,
      lastError: status === 'connected' ? null : data.lastError,
    };
    const existing = await this.prisma.cmppConnectionState.findFirst({
@@ -645,30 +715,59 @@ export class ChannelsService implements OnModuleInit, OnModuleDestroy {
        connectionId: data.connectionId,
      },
    });
-    const state = existing
-      ? await this.prisma.cmppConnectionState.update({ where: { id: existing.id }, data: payload })
-      : await this.prisma.cmppConnectionState.create({
+    let state;
+    if (existing) {
+      state = await this.prisma.cmppConnectionState.update({ where: { id: existing.id }, data: payload });
+    } else {
+      try {
+        state = await this.prisma.cmppConnectionState.create({
+          data: {
+            channelId: data.channelId,
+            connectionId: data.connectionId,
+            ...payload,
+          },
+        });
+      } catch (error) {
+        if ((error as { code?: string }).code !== 'P2002') {
+          throw error;
+        }
+        const concurrent = await this.prisma.cmppConnectionState.findFirst({
+          where: {
+            applicationId: data.applicationId ?? null,
+            channelId: data.channelId,
+            connectionId: data.connectionId,
+          },
+        });
+        if (!concurrent) {
+          throw error;
+        }
+        state = await this.prisma.cmppConnectionState.update({ where: { id: concurrent.id }, data: payload });
+      }
+    }
+    const action = normalizeConnectionAction(
+      ['heartbeat', 'active_test'].includes(rawStatus.toLowerCase()) ? rawStatus : status,
+    );
+    const heartbeatObservedAt = data.lastHeartbeatAt ? new Date(data.lastHeartbeatAt) : new Date();
+    const shouldWriteAudit = action !== 'heartbeat'
+      || !existing?.lastHeartbeatAt
+      || heartbeatObservedAt.getTime() - existing.lastHeartbeatAt.getTime() >= HEARTBEAT_AUDIT_INTERVAL_MS;
+    if (shouldWriteAudit) {
+      await this.prisma.operationLog.create({
        data: {
-          channelId: data.channelId,
-          connectionId: data.connectionId,
-          ...payload,
+          tenantId: data.tenantId,
+          action: `cmpp_connection.${action}`,
+          resource: 'cmpp_connection',
+          resourceId: `${data.channelId}:${data.connectionId}`,
+          detail: {
+            status,
+            applicationId: state.applicationId,
+            desiredConnections: state.desiredConnections,
+            currentConnections: state.currentConnections,
+            lastError: state.lastError,
+          } as Prisma.InputJsonValue,
        },
      });
-    await this.prisma.operationLog.create({
-      data: {
-        tenantId: data.tenantId,
-        action: `cmpp_connection.${normalizeConnectionAction(status)}`,
-        resource: 'cmpp_connection',
-        resourceId: `${data.channelId}:${data.connectionId}`,
-        detail: {
-          status,
-          applicationId: state.applicationId,
-          desiredConnections: state.desiredConnections,
-          currentConnections: state.currentConnections,
-          lastError: state.lastError,
-        } as Prisma.InputJsonValue,
-      },
-    });
+    }
    return state;
  }

@@ -705,6 +804,8 @@ export class ChannelsService implements OnModuleInit, OnModuleDestroy {
          status: 'failed',
          currentConnections: 0,
          lastDisconnectedAt: now,
+          nextReconnectAt: now,
+          lastErrorCategory: 'timeout',
          lastError,
        },
      });
@@ -1245,7 +1346,7 @@ export class ChannelsService implements OnModuleInit, OnModuleDestroy {
      rateLimitPerSecond: number;
      config?: Prisma.JsonValue | null;
    },
-    reason: 'channel_created' | 'channel_enabled' | 'gateway_restarted',
+    reason: 'channel_created' | 'channel_enabled' | 'channel_updated' | 'gateway_restarted' | 'automatic_reconnect',
    operatorId?: string,
  ) {
    const desiredConnections = getDesiredConnections(channel.config);
@@ -1263,16 +1364,34 @@ export class ChannelsService implements OnModuleInit, OnModuleDestroy {
      desiredConnections,
      currentConnections: 0,
      lastError: null,
+      lastReconnectAttemptAt: new Date(),
+      nextReconnectAt: new Date(Date.now() + getPositiveIntegerEnv('GATEWAY_CONNECTING_TIMEOUT_MS', DEFAULT_CONNECTING_TIMEOUT_MS)),
    };
-    const state = existing
-      ? await this.prisma.cmppConnectionState.update({ where: { id: existing.id }, data })
-      : await this.prisma.cmppConnectionState.create({
-        data: {
-          channelId: channel.id,
-          connectionId,
-          ...data,
-        },
-      });
+    let state;
+    if (existing) {
+      state = await this.prisma.cmppConnectionState.update({ where: { id: existing.id }, data });
+    } else {
+      try {
+        state = await this.prisma.cmppConnectionState.create({
+          data: {
+            channelId: channel.id,
+            connectionId,
+            ...data,
+          },
+        });
+      } catch (error) {
+        if ((error as { code?: string }).code !== 'P2002') {
+          throw error;
+        }
+        const concurrent = await this.prisma.cmppConnectionState.findFirst({
+          where: { applicationId: null, channelId: channel.id, connectionId },
+        });
+        if (!concurrent) {
+          throw error;
+        }
+        state = await this.prisma.cmppConnectionState.update({ where: { id: concurrent.id }, data });
+      }
+    }
    await this.prisma.operationLog.create({
      data: {
        userId: operatorId,
@@ -1306,10 +1425,36 @@ export class ChannelsService implements OnModuleInit, OnModuleDestroy {
        srcId: channel.srcId,
        cmppVersion: channel.cmppVersion,
        rateLimitPerSecond: channel.rateLimitPerSecond,
+        windowSize: getPositiveRuntimeInteger(getConfigValue(channel.config, 'windowSize'), 16, 'windowSize'),
+        heartbeatIntervalSeconds: getPositiveRuntimeInteger(
+          getConfigValue(channel.config, 'heartbeatIntervalSeconds'),
+          DEFAULT_HEARTBEAT_INTERVAL_SECONDS,
+          'heartbeatIntervalSeconds',
+        ),
+        heartbeatMissThreshold: getPositiveRuntimeInteger(
+          getConfigValue(channel.config, 'heartbeatMissThreshold'),
+          DEFAULT_HEARTBEAT_MISS_THRESHOLD,
+          'heartbeatMissThreshold',
+        ),
      },
    };
-    await this.getGatewayConnectionQueue().add('connect-channel', command, { jobId: `${connectionId}:connect` });
-    await this.notifyGatewayConnect(command);
+    const queuedJob = await this.getGatewayConnectionQueue().add('connect-channel', command, {
+      jobId: `gateway-connect-${channel.id}-${command.traceId}`,
+      removeOnComplete: 1000,
+      removeOnFail: 1000,
+    }).catch((error) => {
+      this.logger.warn(`Gateway connect marker enqueue failed; continuing with direct control request: ${error instanceof Error ? error.message : String(error)}`);
+      return undefined;
+    });
+    try {
+      await this.notifyGatewayConnect(command);
+    } finally {
+      if (queuedJob) {
+        await queuedJob.remove().catch((error) => {
+          this.logger.warn(`Failed to remove delivered Gateway connect marker ${queuedJob.id}: ${error instanceof Error ? error.message : String(error)}`);
+        });
+      }
+    }
    return state;
  }

@@ -1328,6 +1473,134 @@ export class ChannelsService implements OnModuleInit, OnModuleDestroy {
    });
  }

+  async reconcileGatewayConnections(now = new Date()) {
+    const channels = await this.prisma.smsChannel.findMany({
+      where: { status: { in: ['active', 'disabled', 'deleted'] } },
+      include: {
+        connectionStates: {
+          where: { applicationId: null },
+        },
+      },
+      take: 200,
+    });
+    let reconnectRequested = 0;
+    let disconnectRequested = 0;
+    for (const channel of channels) {
+      const state = channel.connectionStates.find((item) => item.connectionId === defaultChannelConnectionId(channel.id));
+      if (channel.status !== 'active') {
+        if (state && (state.currentConnections > 0 || ['connected', 'connecting', 'reconnecting'].includes(state.status))) {
+          await this.withGatewayReconcileLock(channel.id, async () => {
+            await this.requestChannelDisconnection(channel, 'inactive_channel_reconcile');
+            disconnectRequested++;
+          });
+        }
+        continue;
+      }
+      const desiredConnections = getDesiredConnections(channel.config);
+      const heartbeatIntervalSeconds = getPositiveRuntimeInteger(
+        getConfigValue(channel.config, 'heartbeatIntervalSeconds'),
+        DEFAULT_HEARTBEAT_INTERVAL_SECONDS,
+        'heartbeatIntervalSeconds',
+      );
+      const heartbeatMissThreshold = getPositiveRuntimeInteger(
+        getConfigValue(channel.config, 'heartbeatMissThreshold'),
+        DEFAULT_HEARTBEAT_MISS_THRESHOLD,
+        'heartbeatMissThreshold',
+      );
+      const heartbeatCutoff = new Date(now.getTime() - heartbeatIntervalSeconds * (heartbeatMissThreshold + 1) * 1000);
+      const connectedAndFresh = state?.status === 'connected'
+        && state.currentConnections >= desiredConnections
+        && Boolean(state.lastHeartbeatAt && state.lastHeartbeatAt > heartbeatCutoff);
+      const retryDue = !state?.nextReconnectAt || state.nextReconnectAt <= now;
+      if (!connectedAndFresh && retryDue) {
+        await this.withGatewayReconcileLock(channel.id, async () => {
+          await this.requestChannelConnection(channel, 'automatic_reconnect');
+          reconnectRequested++;
+        });
+      }
+    }
+    return { scanned: channels.length, reconnectRequested, disconnectRequested };
+  }
+
+  private async withGatewayReconcileLock(channelId: string, action: () => Promise<void>) {
+    const redis = this.getRedis();
+    const key = `cmpp:gateway:reconcile:${channelId}`;
+    const token = randomUUID();
+    const acquired = await redis.set(key, token, 'PX', DEFAULT_CONNECTING_TIMEOUT_MS, 'NX');
+    if (acquired !== 'OK') {
+      return;
+    }
+    try {
+      await action();
+    } finally {
+      await redis.eval(
+        'if redis.call("get", KEYS[1]) == ARGV[1] then return redis.call("del", KEYS[1]) else return 0 end',
+        1,
+        key,
+        token,
+      );
+    }
+  }
+
+  private async requestChannelDisconnection(
+    channel: { id: string },
+    reason: 'channel_disabled' | 'channel_deleted' | 'inactive_channel_reconcile',
+    operatorId?: string,
+  ) {
+    const connectionId = defaultChannelConnectionId(channel.id);
+    const now = new Date();
+    await this.prisma.cmppConnectionState.updateMany({
+      where: {
+        applicationId: null,
+        channelId: channel.id,
+        connectionId,
+      },
+      data: {
+        status: 'disconnected',
+        currentConnections: 0,
+        lastDisconnectedAt: now,
+        nextReconnectAt: null,
+        lastErrorCategory: null,
+        lastError: null,
+      },
+    });
+    await this.prisma.operationLog.create({
+      data: {
+        userId: operatorId,
+        action: 'cmpp_connection.disconnect_requested',
+        resource: 'cmpp_connection',
+        resourceId: `${channel.id}:${connectionId}`,
+        detail: { reason } as Prisma.InputJsonValue,
+      },
+    });
+    const command = {
+      schemaVersion: 'v1',
+      messageType: 'DisconnectChannel',
+      traceId: randomUUID(),
+      channelId: channel.id,
+      connectionId,
+      createdAt: now.toISOString(),
+      reason,
+    };
+    const queuedJob = await this.getGatewayConnectionQueue().add('disconnect-channel', command, {
+      jobId: `gateway-disconnect-${channel.id}-${command.traceId}`,
+      removeOnComplete: 1000,
+      removeOnFail: 1000,
+    }).catch((error) => {
+      this.logger.warn(`Gateway disconnect marker enqueue failed; continuing with direct control request: ${error instanceof Error ? error.message : String(error)}`);
+      return undefined;
+    });
+    try {
+      await this.notifyGatewayDisconnect(command);
+    } finally {
+      if (queuedJob) {
+        await queuedJob.remove().catch((error) => {
+          this.logger.warn(`Failed to remove delivered Gateway disconnect marker ${queuedJob.id}: ${error instanceof Error ? error.message : String(error)}`);
+        });
+      }
+    }
+  }
+
  private getGatewayConnectionQueue() {
    this.gatewayConnectionQueue ??= new Queue(GATEWAY_CONNECTION_QUEUE, { connection: bullmqConnection() });
    return this.gatewayConnectionQueue;
@@ -1366,6 +1639,7 @@ export class ChannelsService implements OnModuleInit, OnModuleDestroy {
        method: 'POST',
        headers: { 'Content-Type': 'application/json' },
        body: JSON.stringify(command),
+        signal: AbortSignal.timeout(getPositiveIntegerEnv('GATEWAY_CONTROL_TIMEOUT_MS', DEFAULT_GATEWAY_CONTROL_TIMEOUT_MS)),
      });
    } catch (error) {
      throw new BadRequestException(`Gateway connect request failed: ${error instanceof Error ? error.message : String(error)}`);
@@ -1375,6 +1649,25 @@ export class ChannelsService implements OnModuleInit, OnModuleDestroy {
      throw new BadRequestException(`Gateway connect request failed: ${response.status} ${responseText}`);
    }
  }
+
+  private async notifyGatewayDisconnect(command: Record<string, unknown>) {
+    const baseUrl = (process.env.GATEWAY_CONTROL_URL ?? DEFAULT_GATEWAY_CONTROL_URL).replace(/\/+$/, '');
+    let response: { ok: boolean; status: number; text: () => Promise<string> };
+    try {
+      response = await fetch(`${baseUrl}/connections/disconnect`, {
+        method: 'POST',
+        headers: { 'Content-Type': 'application/json' },
+        body: JSON.stringify(command),
+        signal: AbortSignal.timeout(getPositiveIntegerEnv('GATEWAY_CONTROL_TIMEOUT_MS', DEFAULT_GATEWAY_CONTROL_TIMEOUT_MS)),
+      });
+    } catch (error) {
+      throw new BadRequestException(`Gateway disconnect request failed: ${error instanceof Error ? error.message : String(error)}`);
+    }
+    if (!response.ok) {
+      const responseText = await response.text();
+      throw new BadRequestException(`Gateway disconnect request failed: ${response.status} ${responseText}`);
+    }
+  }
 }

 function normalizeTestPhones(data: TestChannelDto) {
@@ -1481,6 +1774,16 @@ function buildChannelTestSubmitCommand({
      cmppVersion: channel.cmppVersion,
      desiredConnections: getPositiveRuntimeInteger(getConfigValue(channel.config, 'desiredConnections'), 1, 'desiredConnections'),
      windowSize: getPositiveRuntimeInteger(getConfigValue(channel.config, 'windowSize'), 16, 'windowSize'),
+      heartbeatIntervalSeconds: getPositiveRuntimeInteger(
+        getConfigValue(channel.config, 'heartbeatIntervalSeconds'),
+        DEFAULT_HEARTBEAT_INTERVAL_SECONDS,
+        'heartbeatIntervalSeconds',
+      ),
+      heartbeatMissThreshold: getPositiveRuntimeInteger(
+        getConfigValue(channel.config, 'heartbeatMissThreshold'),
+        DEFAULT_HEARTBEAT_MISS_THRESHOLD,
+        'heartbeatMissThreshold',
+      ),
    },
    retry: { attempt: 0, maxAttempts: 1 },
  };
@@ -1531,7 +1834,7 @@ function normalizeCmppVersion(version?: string) {

 function normalizeGatewayConnectionStatus(status: string) {
  const normalized = status.toLowerCase();
-  if (['online', 'open', 'connected'].includes(normalized)) {
+  if (['online', 'open', 'connected', 'heartbeat', 'active_test'].includes(normalized)) {
    return 'connected';
  }
  if (['connecting', 'connect_requested'].includes(normalized)) {
@@ -1568,6 +1871,8 @@ function normalizeChannelRuntimeConfig(
  incomingConfig?: Record<string, unknown> | null,
  desiredConnections?: number,
  windowSize?: number,
+  heartbeatIntervalSeconds?: number,
+  heartbeatMissThreshold?: number,
 ) {
  const existing = existingConfig && typeof existingConfig === 'object' && !Array.isArray(existingConfig)
    ? existingConfig as Record<string, unknown>
@@ -1578,10 +1883,29 @@ function normalizeChannelRuntimeConfig(
  const base = { ...existing, ...incoming };
  base.desiredConnections = getPositiveRuntimeInteger(desiredConnections ?? base.desiredConnections, 1, 'desiredConnections');
  base.windowSize = getPositiveRuntimeInteger(windowSize ?? base.windowSize, 16, 'windowSize');
+  base.heartbeatIntervalSeconds = getPositiveRuntimeInteger(
+    heartbeatIntervalSeconds ?? base.heartbeatIntervalSeconds,
+    DEFAULT_HEARTBEAT_INTERVAL_SECONDS,
+    'heartbeatIntervalSeconds',
+  );
+  base.heartbeatMissThreshold = getPositiveRuntimeInteger(
+    heartbeatMissThreshold ?? base.heartbeatMissThreshold,
+    DEFAULT_HEARTBEAT_MISS_THRESHOLD,
+    'heartbeatMissThreshold',
+  );
  base.extensionDigits = normalizeExtensionDigits(base.extensionDigits);
+  base.serviceId = normalizeCmppServiceId(base.serviceId);
  return base;
 }

+function normalizeCmppServiceId(value: unknown) {
+  const normalized = String(value ?? 'SMS').trim() || 'SMS';
+  if (!/^[\x20-\x7E]{1,10}$/.test(normalized)) {
+    throw new BadRequestException('serviceId must contain 1 to 10 ASCII characters');
+  }
+  return normalized;
+}
+
 function normalizeChannelRateLimit(value: unknown) {
  const normalized = getPositiveRuntimeInteger(value, 100, 'rateLimitPerSecond');
  if (normalized > 2000) {