import { BadRequestException, Injectable, Logger, NotFoundException, OnModuleDestroy, OnModuleInit } from '@nestjs/common'; import { Queue } from 'bullmq'; import IORedis from 'ioredis'; import { Prisma } from '@prisma/client'; import { randomUUID } from 'crypto'; import { assertMoneyUnits, moneyToNumber } from '../common/money'; import { PrismaService } from '../prisma/prisma.service'; import type { CreateChannelDto, UpdateChannelDto, CreateChannelGroupDto, CreateChannelGroupItemDto, UpdateChannelGroupDto, CreateRouteRuleDto, CreateReportFieldDto, ReplaceReportFieldsDto, CreateReportMaterialDto, CreateReportTaskDto, ChangeReportTaskStatusesDto, CreateReportExportDto, CreateReceiptImportDto, UpsertConnectionStateDto, ChangeChannelStatusDto, CopyChannelDto, TestChannelDto } from './channels.contracts'; import { GATEWAY_CONNECTION_QUEUE, GATEWAY_SUBMIT_QUEUE, GATEWAY_SUBMIT_STREAM, DEFAULT_GATEWAY_CONTROL_URL, DEFAULT_CHANNEL_CONNECTION_ID, DEFAULT_CONNECTING_TIMEOUT_MS, DEFAULT_CONNECTING_TIMEOUT_SCAN_MS, DEFAULT_GATEWAY_STARTUP_RECONNECT_DELAY_MS, DEFAULT_GATEWAY_RECONCILE_INTERVAL_MS, DEFAULT_GATEWAY_CONTROL_TIMEOUT_MS, DEFAULT_HEARTBEAT_INTERVAL_SECONDS, DEFAULT_HEARTBEAT_MISS_THRESHOLD, HEARTBEAT_AUDIT_INTERVAL_MS, CONNECTING_TIMEOUT_ERROR, DEFAULT_CMPP_VERSION, normalizeTestPhones, normalizeTestContent, calculateBillingUnits, buildChannelTestSubmitCommand, getConfigValue, getStringConfigValue, normalizeConnectionAction, normalizeCmppVersion, normalizeGatewayConnectionStatus, defaultChannelConnectionId, getDesiredConnections, ChannelConnectionSettings, getRuntimeConfigInteger, channelConnectionSettingsChanged, channelGroupAuditSnapshot, normalizeChannelRuntimeConfig, normalizeCmppServiceId, normalizeChannelRateLimit, normalizeExtensionDigits, getPositiveRuntimeInteger, bullmqConnection, getPositiveIntegerEnv, parseReceiptContent, splitReceiptLine, stripReceiptCell, findReceiptStatusIndex, normalizeReceiptStatus, deriveReceiptStatus, ChannelReportDeliveryRow, summarizeChannelReportDelivery, sumReportDelivery, percentage, latestDate, currentShanghaiDayRange, normalizeRetryTimeLimitMinutes, normalizeSpreadsheetSize, normalizeBusinessCarrier, normalizeChannelCarrier, isChannelCarrierCompatible, normalizeRegion, isRegionCompatible, validateGroupItems, normalizeReportType, summarizeReportStatuses, normalizeLinkEvent } from './channels.helpers'; /** R5 channel domain service composed behind ChannelsService. */ export class ChannelConnectionService { private readonly logger = new Logger(ChannelConnectionService.name); private gatewayConnectionQueue?: Queue; private gatewaySubmitQueue?: Queue; private redis?: IORedis; private connectionTimeoutTimer?: ReturnType; private gatewayStartupReconnectTimer?: ReturnType; private gatewayReconcileTimer?: ReturnType; constructor(private readonly prisma: PrismaService) {} onModuleInit() { if (process.env.GATEWAY_CONNECTING_TIMEOUT_SCANNER_DISABLED !== 'true') { this.connectionTimeoutTimer = setInterval(() => { void this.markTimedOutConnectingChannels().catch((error) => { this.logger.error(`Failed to mark timed-out CMPP connections: ${error instanceof Error ? error.message : String(error)}`); }); }, getPositiveIntegerEnv('GATEWAY_CONNECTING_TIMEOUT_SCAN_MS', DEFAULT_CONNECTING_TIMEOUT_SCAN_MS)); this.connectionTimeoutTimer.unref?.(); } this.gatewayStartupReconnectTimer = setTimeout(() => { void this.reconnectActiveChannelsAfterGatewayRestart(); }, getPositiveIntegerEnv('GATEWAY_STARTUP_RECONNECT_DELAY_MS', DEFAULT_GATEWAY_STARTUP_RECONNECT_DELAY_MS)); this.gatewayStartupReconnectTimer.unref?.(); if (process.env.GATEWAY_CONNECTION_RECONCILER_DISABLED !== 'true') { this.gatewayReconcileTimer = setInterval(() => { void this.reconcileGatewayConnections().catch((error) => { this.logger.error(`Failed to reconcile supplier connections: ${error instanceof Error ? error.message : String(error)}`); }); }, getPositiveIntegerEnv('GATEWAY_CONNECTION_RECONCILE_INTERVAL_MS', DEFAULT_GATEWAY_RECONCILE_INTERVAL_MS)); this.gatewayReconcileTimer.unref?.(); } } async onModuleDestroy() { if (this.connectionTimeoutTimer) { clearInterval(this.connectionTimeoutTimer); } if (this.gatewayStartupReconnectTimer) { clearTimeout(this.gatewayStartupReconnectTimer); } if (this.gatewayReconcileTimer) { clearInterval(this.gatewayReconcileTimer); } await this.gatewayConnectionQueue?.close(); await this.gatewaySubmitQueue?.close(); this.redis?.disconnect(); } listChannelMetrics(channelId: string) { return this.prisma.channelHealthMetric.findMany({ where: { channelId }, orderBy: { windowStart: 'desc' }, take: 100, }); } listChannelConnections(channelId: string) { return this.prisma.cmppConnectionState.findMany({ where: { channelId }, orderBy: { updatedAt: 'desc' }, }); } async listChannelConnectionLogs(channelId: string) { const channel = await this.prisma.smsChannel.findUnique({ where: { id: channelId }, select: { id: true } }); if (!channel) { throw new NotFoundException('Channel not found'); } const [connectionStates, logs] = await Promise.all([ this.prisma.cmppConnectionState.findMany({ where: { channelId }, orderBy: { updatedAt: 'desc' }, take: 50, }), this.prisma.operationLog.findMany({ where: { OR: [ { resource: 'sms_channel', resourceId: channelId }, { resource: 'cmpp_connection', resourceId: { startsWith: `${channelId}:` } }, ], }, orderBy: { createdAt: 'desc' }, take: 100, }), ]); return { channelId, connectionStates, logs: logs.map((log) => ({ id: log.id, time: log.createdAt, event: normalizeLinkEvent(log.action), action: log.action, resourceId: log.resourceId, detail: log.detail, })), }; } listTenantConnections(tenantId: string) { return this.prisma.cmppConnectionState.findMany({ where: { tenantId }, include: { channel: true }, orderBy: { updatedAt: 'desc' }, }); } async upsertConnectionState(data: UpsertConnectionStateDto) { const rawStatus = data.status; const status = normalizeGatewayConnectionStatus(rawStatus); if (data.applicationId) { const application = await this.prisma.smsApplication.findUnique({ where: { id: data.applicationId }, select: { tenantId: true } }); if (!application) { throw new BadRequestException('applicationId does not reference an existing application'); } if (data.tenantId && data.tenantId !== application.tenantId) { throw new BadRequestException('applicationId does not belong to tenantId'); } data.tenantId = application.tenantId; } const payload = { tenantId: data.tenantId, applicationId: data.applicationId, status, desiredConnections: data.desiredConnections ?? 1, currentConnections: data.currentConnections ?? (status === 'connected' ? 1 : 0), lastConnectedAt: data.lastConnectedAt ? new Date(data.lastConnectedAt) : undefined, lastDisconnectedAt: data.lastDisconnectedAt ? new Date(data.lastDisconnectedAt) : undefined, lastHeartbeatAt: data.lastHeartbeatAt ? new Date(data.lastHeartbeatAt) : undefined, reconnectCount: data.reconnectCount ?? 0, lastReconnectAttemptAt: data.lastReconnectAttemptAt ? new Date(data.lastReconnectAttemptAt) : undefined, nextReconnectAt: data.nextReconnectAt ? new Date(data.nextReconnectAt) : status === 'connected' ? null : undefined, lastErrorCategory: status === 'connected' ? null : data.lastErrorCategory, lastError: status === 'connected' ? null : data.lastError, }; const existing = await this.prisma.cmppConnectionState.findFirst({ where: { applicationId: data.applicationId ?? null, channelId: data.channelId, connectionId: data.connectionId, }, }); let state; if (existing) { state = await this.prisma.cmppConnectionState.update({ where: { id: existing.id }, data: payload }); } else { try { state = await this.prisma.cmppConnectionState.create({ data: { channelId: data.channelId, connectionId: data.connectionId, ...payload, }, }); } catch (error) { if ((error as { code?: string }).code !== 'P2002') { throw error; } const concurrent = await this.prisma.cmppConnectionState.findFirst({ where: { applicationId: data.applicationId ?? null, channelId: data.channelId, connectionId: data.connectionId, }, }); if (!concurrent) { throw error; } state = await this.prisma.cmppConnectionState.update({ where: { id: concurrent.id }, data: payload }); } } const action = normalizeConnectionAction( ['heartbeat', 'active_test'].includes(rawStatus.toLowerCase()) ? rawStatus : status, ); const heartbeatObservedAt = data.lastHeartbeatAt ? new Date(data.lastHeartbeatAt) : new Date(); const shouldWriteAudit = action !== 'heartbeat' || !existing?.lastHeartbeatAt || heartbeatObservedAt.getTime() - existing.lastHeartbeatAt.getTime() >= HEARTBEAT_AUDIT_INTERVAL_MS; if (shouldWriteAudit) { await this.prisma.operationLog.create({ data: { tenantId: data.tenantId, action: `cmpp_connection.${action}`, resource: 'cmpp_connection', resourceId: `${data.channelId}:${data.connectionId}`, detail: { status, applicationId: state.applicationId, desiredConnections: state.desiredConnections, currentConnections: state.currentConnections, lastError: state.lastError, } as Prisma.InputJsonValue, }, }); } return state; } async markTimedOutConnectingChannels(now = new Date()) { const timeoutMs = getPositiveIntegerEnv('GATEWAY_CONNECTING_TIMEOUT_MS', DEFAULT_CONNECTING_TIMEOUT_MS); const cutoff = new Date(now.getTime() - timeoutMs); const lastError = `${CONNECTING_TIMEOUT_ERROR} after ${Math.round(timeoutMs / 1000)} seconds`; const states = await this.prisma.cmppConnectionState.findMany({ where: { status: 'connecting', updatedAt: { lte: cutoff }, }, select: { id: true, tenantId: true, applicationId: true, channelId: true, connectionId: true, desiredConnections: true, currentConnections: true, updatedAt: true, }, take: 100, }); let failed = 0; for (const state of states) { const result = await this.prisma.cmppConnectionState.updateMany({ where: { id: state.id, status: 'connecting', updatedAt: { lte: cutoff }, }, data: { status: 'failed', currentConnections: 0, lastDisconnectedAt: now, nextReconnectAt: now, lastErrorCategory: 'timeout', lastError, }, }); if (result.count === 0) { continue; } failed += result.count; await this.prisma.operationLog.create({ data: { tenantId: state.tenantId, action: 'cmpp_connection.failed', resource: 'cmpp_connection', resourceId: `${state.channelId}:${state.connectionId}`, detail: { reason: 'connect_timeout', applicationId: state.applicationId, status: 'failed', previousStatus: 'connecting', desiredConnections: state.desiredConnections, currentConnectionsBefore: state.currentConnections, currentConnections: 0, timeoutMs, lastError, } as Prisma.InputJsonValue, }, }); } return { checked: states.length, failed }; } async requestChannelConnection( channel: { id: string; code: string; name: string; gatewayHost: string; gatewayPort: number; account: string; passwordCipher: string; srcId: string; cmppVersion: string; rateLimitPerSecond: number; config?: Prisma.JsonValue | null; }, reason: 'channel_created' | 'channel_enabled' | 'channel_updated' | 'gateway_restarted' | 'automatic_reconnect', operatorId?: string, ) { const desiredConnections = getDesiredConnections(channel.config); const connectionId = defaultChannelConnectionId(channel.id); const existing = await this.prisma.cmppConnectionState.findFirst({ where: { applicationId: null, channelId: channel.id, connectionId, }, }); const data = { applicationId: null, status: 'connecting', desiredConnections, currentConnections: 0, lastError: null, lastReconnectAttemptAt: new Date(), nextReconnectAt: new Date(Date.now() + getPositiveIntegerEnv('GATEWAY_CONNECTING_TIMEOUT_MS', DEFAULT_CONNECTING_TIMEOUT_MS)), }; let state; if (existing) { state = await this.prisma.cmppConnectionState.update({ where: { id: existing.id }, data }); } else { try { state = await this.prisma.cmppConnectionState.create({ data: { channelId: channel.id, connectionId, ...data, }, }); } catch (error) { if ((error as { code?: string }).code !== 'P2002') { throw error; } const concurrent = await this.prisma.cmppConnectionState.findFirst({ where: { applicationId: null, channelId: channel.id, connectionId }, }); if (!concurrent) { throw error; } state = await this.prisma.cmppConnectionState.update({ where: { id: concurrent.id }, data }); } } await this.prisma.operationLog.create({ data: { userId: operatorId, action: 'cmpp_connection.connect_requested', resource: 'cmpp_connection', resourceId: `${channel.id}:${connectionId}`, detail: { reason, status: state.status, desiredConnections: state.desiredConnections, currentConnections: state.currentConnections, } as Prisma.InputJsonValue, }, }); const command = { schemaVersion: 'v1', messageType: 'ConnectChannel', traceId: randomUUID(), channelId: channel.id, connectionId, createdAt: new Date().toISOString(), reason, desiredConnections, channel: { code: channel.code, name: channel.name, gatewayHost: channel.gatewayHost, gatewayPort: channel.gatewayPort, account: channel.account, passwordCipher: channel.passwordCipher, srcId: channel.srcId, cmppVersion: channel.cmppVersion, rateLimitPerSecond: channel.rateLimitPerSecond, windowSize: getPositiveRuntimeInteger(getConfigValue(channel.config, 'windowSize'), 16, 'windowSize'), connectionWarmupSeconds: Number(getConfigValue(channel.config, 'connectionWarmupSeconds') ?? 30), connectionDrainTimeoutSeconds: getPositiveRuntimeInteger(getConfigValue(channel.config, 'connectionDrainTimeoutSeconds'), 60, 'connectionDrainTimeoutSeconds'), submitResponseTimeoutSeconds: getPositiveRuntimeInteger(getConfigValue(channel.config, 'submitResponseTimeoutSeconds'), 60, 'submitResponseTimeoutSeconds'), connectionFailureCooldownSeconds: getPositiveRuntimeInteger(getConfigValue(channel.config, 'connectionFailureCooldownSeconds'), 30, 'connectionFailureCooldownSeconds'), heartbeatIntervalSeconds: getPositiveRuntimeInteger( getConfigValue(channel.config, 'heartbeatIntervalSeconds'), DEFAULT_HEARTBEAT_INTERVAL_SECONDS, 'heartbeatIntervalSeconds', ), heartbeatMissThreshold: getPositiveRuntimeInteger( getConfigValue(channel.config, 'heartbeatMissThreshold'), DEFAULT_HEARTBEAT_MISS_THRESHOLD, 'heartbeatMissThreshold', ), }, }; const queuedJob = await this.getGatewayConnectionQueue().add('connect-channel', command, { jobId: `gateway-connect-${channel.id}-${command.traceId}`, removeOnComplete: 1000, removeOnFail: 1000, }).catch((error) => { this.logger.warn(`Gateway connect marker enqueue failed; continuing with direct control request: ${error instanceof Error ? error.message : String(error)}`); return undefined; }); try { await this.notifyGatewayConnect(command); } finally { if (queuedJob) { await queuedJob.remove().catch((error) => { this.logger.warn(`Failed to remove delivered Gateway connect marker ${queuedJob.id}: ${error instanceof Error ? error.message : String(error)}`); }); } } return state; } async reconnectActiveChannelsAfterGatewayRestart() { const channels = await this.prisma.smsChannel.findMany({ where: { status: 'active' } }); const results = await Promise.allSettled( channels.map((channel) => this.requestChannelConnection(channel, 'gateway_restarted')), ); results.forEach((result, index) => { if (result.status === 'rejected') { const channel = channels[index]; this.logger.error( `Failed to restore active CMPP channel ${channel?.code ?? channel?.id ?? index}: ${result.reason instanceof Error ? result.reason.message : String(result.reason)}`, ); } }); } async reconcileGatewayConnections(now = new Date()) { const channels = await this.prisma.smsChannel.findMany({ where: { status: { in: ['active', 'disabled', 'deleted'] } }, include: { connectionStates: { where: { applicationId: null }, }, }, take: 200, }); let reconnectRequested = 0; let disconnectRequested = 0; for (const channel of channels) { const state = channel.connectionStates.find((item) => item.connectionId === defaultChannelConnectionId(channel.id)); if (channel.status !== 'active') { if (state && (state.currentConnections > 0 || ['connected', 'connecting', 'reconnecting'].includes(state.status))) { await this.withGatewayReconcileLock(channel.id, async () => { await this.requestChannelDisconnection(channel, 'inactive_channel_reconcile'); disconnectRequested++; }); } continue; } const desiredConnections = getDesiredConnections(channel.config); const heartbeatIntervalSeconds = getPositiveRuntimeInteger( getConfigValue(channel.config, 'heartbeatIntervalSeconds'), DEFAULT_HEARTBEAT_INTERVAL_SECONDS, 'heartbeatIntervalSeconds', ); const heartbeatMissThreshold = getPositiveRuntimeInteger( getConfigValue(channel.config, 'heartbeatMissThreshold'), DEFAULT_HEARTBEAT_MISS_THRESHOLD, 'heartbeatMissThreshold', ); const heartbeatCutoff = new Date(now.getTime() - heartbeatIntervalSeconds * (heartbeatMissThreshold + 1) * 1000); const connectedAndFresh = state?.status === 'connected' && state.currentConnections >= desiredConnections && Boolean(state.lastHeartbeatAt && state.lastHeartbeatAt > heartbeatCutoff); const retryDue = !state?.nextReconnectAt || state.nextReconnectAt <= now; if (!connectedAndFresh && retryDue) { await this.withGatewayReconcileLock(channel.id, async () => { await this.requestChannelConnection(channel, 'automatic_reconnect'); reconnectRequested++; }); } } return { scanned: channels.length, reconnectRequested, disconnectRequested }; } async withGatewayReconcileLock(channelId: string, action: () => Promise) { const redis = this.getRedis(); const key = `cmpp:gateway:reconcile:${channelId}`; const token = randomUUID(); const acquired = await redis.set(key, token, 'PX', DEFAULT_CONNECTING_TIMEOUT_MS, 'NX'); if (acquired !== 'OK') { return; } try { await action(); } finally { await redis.eval( 'if redis.call("get", KEYS[1]) == ARGV[1] then return redis.call("del", KEYS[1]) else return 0 end', 1, key, token, ); } } async requestChannelDisconnection( channel: { id: string }, reason: 'channel_disabled' | 'channel_deleted' | 'inactive_channel_reconcile', operatorId?: string, ) { const connectionId = defaultChannelConnectionId(channel.id); const now = new Date(); await this.prisma.cmppConnectionState.updateMany({ where: { applicationId: null, channelId: channel.id, connectionId, }, data: { status: 'disconnected', currentConnections: 0, lastDisconnectedAt: now, nextReconnectAt: null, lastErrorCategory: null, lastError: null, }, }); await this.prisma.operationLog.create({ data: { userId: operatorId, action: 'cmpp_connection.disconnect_requested', resource: 'cmpp_connection', resourceId: `${channel.id}:${connectionId}`, detail: { reason } as Prisma.InputJsonValue, }, }); const command = { schemaVersion: 'v1', messageType: 'DisconnectChannel', traceId: randomUUID(), channelId: channel.id, connectionId, createdAt: now.toISOString(), reason, }; const queuedJob = await this.getGatewayConnectionQueue().add('disconnect-channel', command, { jobId: `gateway-disconnect-${channel.id}-${command.traceId}`, removeOnComplete: 1000, removeOnFail: 1000, }).catch((error) => { this.logger.warn(`Gateway disconnect marker enqueue failed; continuing with direct control request: ${error instanceof Error ? error.message : String(error)}`); return undefined; }); try { await this.notifyGatewayDisconnect(command); } finally { if (queuedJob) { await queuedJob.remove().catch((error) => { this.logger.warn(`Failed to remove delivered Gateway disconnect marker ${queuedJob.id}: ${error instanceof Error ? error.message : String(error)}`); }); } } } getGatewayConnectionQueue() { this.gatewayConnectionQueue ??= new Queue(GATEWAY_CONNECTION_QUEUE, { connection: bullmqConnection() }); return this.gatewayConnectionQueue; } getGatewaySubmitQueue() { this.gatewaySubmitQueue ??= new Queue(GATEWAY_SUBMIT_QUEUE, { connection: bullmqConnection() }); return this.gatewaySubmitQueue; } getRedis() { if (!this.redis) { this.redis = new IORedis(process.env.REDIS_URL ?? 'redis://127.0.0.1:6379', { maxRetriesPerRequest: null, }); } return this.redis; } async publishGatewaySubmitCommand(command: unknown) { return this.getRedis().xadd( process.env.GATEWAY_SUBMIT_STREAM ?? GATEWAY_SUBMIT_STREAM, '*', 'messageType', 'SubmitCommand', 'data', JSON.stringify(command), ); } async notifyGatewayConnect(command: Record) { const baseUrl = (process.env.GATEWAY_CONTROL_URL ?? DEFAULT_GATEWAY_CONTROL_URL).replace(/\/+$/, ''); let response: { ok: boolean; status: number; text: () => Promise }; try { response = await fetch(`${baseUrl}/connections/connect`, { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(command), signal: AbortSignal.timeout(getPositiveIntegerEnv('GATEWAY_CONTROL_TIMEOUT_MS', DEFAULT_GATEWAY_CONTROL_TIMEOUT_MS)), }); } catch (error) { throw new BadRequestException(`Gateway connect request failed: ${error instanceof Error ? error.message : String(error)}`); } if (!response.ok) { const responseText = await response.text(); throw new BadRequestException(`Gateway connect request failed: ${response.status} ${responseText}`); } } async notifyGatewayDisconnect(command: Record) { const baseUrl = (process.env.GATEWAY_CONTROL_URL ?? DEFAULT_GATEWAY_CONTROL_URL).replace(/\/+$/, ''); let response: { ok: boolean; status: number; text: () => Promise }; try { response = await fetch(`${baseUrl}/connections/disconnect`, { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(command), signal: AbortSignal.timeout(getPositiveIntegerEnv('GATEWAY_CONTROL_TIMEOUT_MS', DEFAULT_GATEWAY_CONTROL_TIMEOUT_MS)), }); } catch (error) { throw new BadRequestException(`Gateway disconnect request failed: ${error instanceof Error ? error.message : String(error)}`); } if (!response.ok) { const responseText = await response.text(); throw new BadRequestException(`Gateway disconnect request failed: ${response.status} ${responseText}`); } } }