import { BadRequestException, HttpException, HttpStatus, Logger, NotFoundException } from '@nestjs/common'; import { Prisma } from '@prisma/client'; import { createHash } from 'node:crypto'; import { BillingService } from '../billing/billing.service'; import { moneyToNumber } from '../common/money'; import type { OpenApiService } from '../open-api/open-api.service'; import { PrismaService } from '../prisma/prisma.service'; import type { GatewaySubmitResultDto, GatewaySubmitSegmentResultDto, GatewayReceiptEventDto, GatewayUplinkEventDto, GatewayPendingDeliveryQueryDto, GatewayDownstreamSentDto, GatewayDownstreamAcknowledgedDto, GatewayDownstreamFailureType, GatewayControlDeliveryResult, GatewaySubmitDeadLetterDto, RequeueGatewaySubmitExceptionDto, GatewayDownstreamRecoveryStatusDto, TimeoutUnknownDto } from './send-chain.contracts'; import { positiveInteger, normalizeReceiptStatus, normalizeCarrier, DEFAULT_DOWNSTREAM_RETRY_DELAY_MS, DEFAULT_DOWNSTREAM_RETRY_MAX_DELAY_MS, DEFAULT_DOWNSTREAM_MAX_RETRIES, DEFAULT_DOWNSTREAM_PENDING_TIMEOUT_HOURS, DEFAULT_RECEIPT_TIMEOUT_HOURS, DEFAULT_GATEWAY_SUBMIT_REQUEUE_STALE_MS, DEFAULT_DOWNSTREAM_MANUAL_REQUEUE_STALE_MS, DEFAULT_UPSTREAM_RECEIPT_INBOX_PROCESSING_STALE_MS, DEFAULT_UPSTREAM_RECEIPT_INBOX_MAX_ATTEMPTS, DEFAULT_UPSTREAM_RECEIPT_INBOX_MAX_AGE_HOURS, gatewaySubmitRequeueKey, isObjectRecord, asDateOrNull, downstreamRetryDelayMs, downstreamAckTimeoutMs, downstreamRetryBaseDelayMs, downstreamRetryMaxDelayMs, downstreamMaxRetries, downstreamPendingTimeoutHours, downstreamControlFailureMessage, normalizeSubmitStatus, downstreamDeliveryAttemptKey, hasRecoveryAuditStateChanged, normalizeRecoveryFailureCategory, aggregateReceiptSegmentState, isSameUpstreamEndpointIdentity, receiptEventKey } from './send-chain.helpers'; import type { SendSubmissionService } from './send-submission.service'; import type { SendCompletionCallbacks, SendCompletionFacade } from './send-completion.service'; /** * R10 downstreamState implementation. * Cross-domain calls return through the stable SendChainService seam to preserve locking and test observability. */ export class SendDownstreamStateService { private readonly logger = new Logger('SendChainService'); constructor( private readonly prisma: PrismaService, private readonly billing: BillingService, private readonly openApi: OpenApiService | undefined, private readonly facade: SendCompletionFacade, private readonly callbacks: SendCompletionCallbacks, ) {} async listPendingDownstreamDeliveries(data: GatewayPendingDeliveryQueryDto) { const application = await this.facade.findInboundApplication(data.account); if (!application) { throw new BadRequestException('CMPP account is invalid'); } const expiredAcknowledgements = await this.prisma.cmppDownstreamDelivery.findMany({ where: { applicationId: application.id, status: 'awaiting_ack', ackDeadlineAt: { lte: new Date() } }, select: { id: true }, take: 500, }); for (const expired of expiredAcknowledgements) { await this.facade.markDownstreamDeliveryFailed(expired.id, 'CMPP_DELIVER_RESP timeout recovered after Gateway restart', 'ack_timeout'); } return this.prisma.cmppDownstreamDelivery.findMany({ where: { applicationId: application.id, status: 'pending', OR: [{ nextRetryAt: null }, { nextRetryAt: { lte: new Date() } }], }, orderBy: { createdAt: 'asc' }, take: Math.min(Math.max(data.limit ?? 100, 1), 500), }); } async markDownstreamDeliveryDelivered(id: string) { return this.prisma.cmppDownstreamDelivery.update({ where: { id }, data: { status: 'delivered', deliveredAt: new Date(), lastError: null, }, }); } async markDownstreamDeliverySent(data: GatewayDownstreamSentDto) { const sentAt = asDateOrNull(data.sentAt) ?? new Date(); const ackDeadlineAt = asDateOrNull(data.ackDeadlineAt) ?? new Date(sentAt.getTime() + downstreamAckTimeoutMs()); const delivery = await this.prisma.cmppDownstreamDelivery.findUnique({ where: { id: data.id } }); if (!delivery) { throw new NotFoundException('Downstream delivery not found'); } const attemptKey = downstreamDeliveryAttemptKey(data); await this.prisma.cmppDownstreamDeliveryAttempt.upsert({ where: { attemptKey }, update: { connectionId: data.connectionId, sequenceId: data.sequenceId, messageId: data.messageId, sentAt, ackDeadlineAt, }, create: { deliveryId: data.id, attemptKey, attemptNo: delivery.retryCount + 1, connectionId: data.connectionId, sequenceId: data.sequenceId, messageId: data.messageId, status: 'awaiting_ack', sentAt, ackDeadlineAt, }, }); await this.prisma.cmppDownstreamDelivery.updateMany({ where: { id: data.id, status: { not: 'delivered' } }, data: { status: 'awaiting_ack', sentAt, ackDeadlineAt, ackSequenceId: data.sequenceId, ackMessageId: data.messageId, connectionId: data.connectionId, nextRetryAt: null, lastError: null, }, }); return this.prisma.cmppDownstreamDelivery.findUnique({ where: { id: data.id } }); } async acknowledgeDownstreamDelivery(data: GatewayDownstreamAcknowledgedDto) { const acknowledgedAt = asDateOrNull(data.acknowledgedAt) ?? new Date(); const acknowledgedMessageId = String(data.messageId ?? '').trim(); const delivery = await this.prisma.cmppDownstreamDelivery.findUnique({ where: { id: data.id } }); if (!delivery) { throw new NotFoundException('Downstream delivery not found'); } const attemptKey = downstreamDeliveryAttemptKey(data); const acknowledgementAccepted = data.result === 0 && acknowledgedMessageId !== '' && acknowledgedMessageId !== '0'; await this.prisma.cmppDownstreamDeliveryAttempt.upsert({ where: { attemptKey }, update: { status: acknowledgementAccepted ? 'acknowledged' : 'rejected', connectionId: data.connectionId, sequenceId: data.sequenceId, messageId: data.messageId, acknowledgedAt, ackResult: data.result, ackDeadlineAt: null, failureType: acknowledgementAccepted ? null : data.result === 0 ? 'ack_invalid' : 'ack_rejected', errorMessage: acknowledgementAccepted ? null : data.result === 0 ? 'CMPP_DELIVER_RESP Msg_Id=0' : `CMPP_DELIVER_RESP result=${data.result}`, }, create: { deliveryId: data.id, attemptKey, attemptNo: delivery.retryCount + 1, connectionId: data.connectionId, sequenceId: data.sequenceId, messageId: data.messageId, status: acknowledgementAccepted ? 'acknowledged' : 'rejected', acknowledgedAt, ackResult: data.result, failureType: acknowledgementAccepted ? null : data.result === 0 ? 'ack_invalid' : 'ack_rejected', errorMessage: acknowledgementAccepted ? null : data.result === 0 ? 'CMPP_DELIVER_RESP Msg_Id=0' : `CMPP_DELIVER_RESP result=${data.result}`, }, }); if (acknowledgementAccepted) { await this.prisma.cmppDownstreamDelivery.updateMany({ where: { id: data.id, status: { not: 'delivered' } }, data: { status: 'delivered', acknowledgedAt, deliveredAt: acknowledgedAt, ackDeadlineAt: null, ackResult: data.result, ackSequenceId: data.sequenceId, ackMessageId: data.messageId, connectionId: data.connectionId, nextRetryAt: null, lastError: null, }, }); return this.prisma.cmppDownstreamDelivery.findUnique({ where: { id: data.id } }); } await this.prisma.cmppDownstreamDelivery.updateMany({ where: { id: data.id, status: { not: 'delivered' } }, data: { acknowledgedAt, ackResult: data.result, ackSequenceId: data.sequenceId, ackMessageId: data.messageId, connectionId: data.connectionId, }, }); if (data.result === 0) { return this.facade.markDownstreamDeliveryFailed(data.id, 'CMPP_DELIVER_RESP Msg_Id=0,客户端仅确认协议收包,无法关联原短信', 'ack_invalid'); } return this.facade.markDownstreamDeliveryFailed(data.id, `downstream CMPP_DELIVER_RESP result=${data.result}`, 'ack_rejected'); } async markDownstreamDeliveryFailed( id: string, errorMessage?: string, failureType: GatewayDownstreamFailureType = 'send_failed', attempt?: GatewayDownstreamSentDto, ) { const delivery = await this.prisma.cmppDownstreamDelivery.findUnique({ where: { id } }); if (!delivery) { throw new NotFoundException('Downstream delivery not found'); } if (delivery.status === 'delivered') { return delivery; } if (failureType === 'queue_timeout' && delivery.status !== 'pending') { return delivery; } const retryCount = (delivery.retryCount ?? 0) + 1; const acknowledgementFailure = failureType === 'ack_timeout' || failureType === 'ack_rejected' || failureType === 'ack_invalid' || failureType === 'connection_lost'; const retryAllowed = !acknowledgementFailure || delivery.retryEnabled !== false; const nonRetryableFailure = failureType === 'unrecoverable' || failureType === 'queue_timeout'; const finalFailure = nonRetryableFailure || !retryAllowed || retryCount >= downstreamMaxRetries(); const finalStatus = failureType === 'ack_rejected' ? 'rejected' : acknowledgementFailure ? 'unconfirmed' : 'failed'; if (attempt && (attempt.connectionId || attempt.sequenceId || attempt.messageId)) { const attemptKey = downstreamDeliveryAttemptKey({ ...attempt, id }); await this.prisma.cmppDownstreamDeliveryAttempt.upsert({ where: { attemptKey }, update: { status: 'failed', connectionId: attempt.connectionId, sequenceId: attempt.sequenceId, messageId: attempt.messageId, failureType, errorMessage: errorMessage ?? 'downstream delivery failed', ackDeadlineAt: null, }, create: { deliveryId: id, attemptKey, attemptNo: delivery.retryCount + 1, connectionId: attempt.connectionId, sequenceId: attempt.sequenceId, messageId: attempt.messageId, status: 'failed', sentAt: asDateOrNull(attempt.sentAt), failureType, errorMessage: errorMessage ?? 'downstream delivery failed', }, }); } const updated = await this.prisma.cmppDownstreamDelivery.update({ where: { id }, data: { status: finalFailure ? finalStatus : 'pending', retryCount, nextRetryAt: finalFailure ? null : new Date(Date.now() + downstreamRetryDelayMs(retryCount)), ackDeadlineAt: null, lastError: errorMessage ?? 'downstream delivery failed', }, }); if (finalFailure) { await this.prisma.operationLog.create({ data: { tenantId: updated.tenantId, action: 'gateway.downstream_delivery_failed', resource: 'cmpp_downstream_delivery', resourceId: updated.id, detail: { deliveryType: updated.deliveryType, applicationId: updated.applicationId, messageId: updated.messageId, retryCount, failureType, retryEnabled: updated.retryEnabled, errorMessage: updated.lastError, }, }, }); } return updated; } async recordGatewayDownstreamRecoveryStatus(data: GatewayDownstreamRecoveryStatusDto) { const account = String(data.account ?? '').trim(); if (!account) { throw new BadRequestException('account is required'); } const recoveryStatuses = (this.prisma as PrismaService & { gatewayDownstreamRecoveryStatus: { findUnique: (args: Record) => Promise; upsert: (args: Record) => Promise; }; }).gatewayDownstreamRecoveryStatus; const previous = await recoveryStatuses.findUnique({ where: { account }, select: { state: true, gatewayInstanceId: true, lockOwner: true, failureCategory: true, lastError: true, lastSkipReason: true, }, }); const application = await this.prisma.smsApplication.findUnique({ where: { cmppAccount: account }, select: { id: true, tenantId: true, name: true }, }); const failureCategory = normalizeRecoveryFailureCategory(data); const updated = await recoveryStatuses.upsert({ where: { account }, update: { tenantId: application?.tenantId ?? null, applicationId: application?.id ?? null, gatewayInstanceId: data.gatewayInstanceId ?? null, state: data.state, lockOwner: data.lockOwner ?? null, lockExpiresAt: asDateOrNull(data.lockExpiresAt), lastAttemptAt: asDateOrNull(data.lastAttemptAt), lastSuccessAt: asDateOrNull(data.lastSuccessAt), lastFailureAt: asDateOrNull(data.lastFailureAt), nextRetryAt: asDateOrNull(data.nextRetryAt), attemptCount: Number.isFinite(Number(data.attemptCount)) ? Number(data.attemptCount) : 0, failureCategory, lastError: data.lastError ?? null, lastSkipReason: data.lastSkipReason ?? null, }, create: { account, tenantId: application?.tenantId, applicationId: application?.id, gatewayInstanceId: data.gatewayInstanceId, state: data.state, lockOwner: data.lockOwner, lockExpiresAt: asDateOrNull(data.lockExpiresAt), lastAttemptAt: asDateOrNull(data.lastAttemptAt), lastSuccessAt: asDateOrNull(data.lastSuccessAt), lastFailureAt: asDateOrNull(data.lastFailureAt), nextRetryAt: asDateOrNull(data.nextRetryAt), attemptCount: Number.isFinite(Number(data.attemptCount)) ? Number(data.attemptCount) : 0, failureCategory, lastError: data.lastError, lastSkipReason: data.lastSkipReason, }, include: { tenant: true, application: true, }, }); const normalizedUpdated = updated as typeof updated & { failureCategory?: string | null; lockOwner?: string | null; lockExpiresAt?: Date | null; }; if (hasRecoveryAuditStateChanged(previous, updated)) { await this.prisma.operationLog.create({ data: { tenantId: updated.tenantId ?? undefined, action: 'gateway.downstream_recovery_status_changed', resource: 'gateway_downstream_recovery_status', resourceId: updated.id, detail: { account, previousState: previous?.state ?? null, state: updated.state, gatewayInstanceId: updated.gatewayInstanceId, lockOwner: normalizedUpdated.lockOwner, attemptCount: updated.attemptCount, nextRetryAt: updated.nextRetryAt, failureCategory: normalizedUpdated.failureCategory, applicationId: updated.applicationId, applicationName: application?.name, lastError: updated.lastError, lastSkipReason: updated.lastSkipReason, }, }, }); } return updated; } async requeueDownstreamDelivery(id: string) { const delivery = await this.prisma.cmppDownstreamDelivery.findUnique({ where: { id }, include: { application: { select: { cmppAccount: true } } }, }); if (!delivery) { throw new NotFoundException('Downstream delivery not found'); } if (delivery.status === 'awaiting_ack') { throw new BadRequestException('该记录正在等待客户端确认,不允许并发重投'); } const payload = isObjectRecord(delivery.payload) ? { ...delivery.payload } : null; if (!payload) { throw new BadRequestException('下游投递记录缺少可重放 payload'); } const path = delivery.deliveryType === 'receipt' ? '/downstream/receipt' : delivery.deliveryType === 'uplink' ? '/downstream/uplink' : null; if (!path) { throw new BadRequestException(`Unsupported downstream delivery type ${delivery.deliveryType}`); } const requestPayload = { deliveryId: delivery.id, account: String(payload.account ?? delivery.application?.cmppAccount ?? ''), ...payload, }; const retriedAt = new Date(); const claimed = await this.prisma.cmppDownstreamDelivery.updateMany({ where: { id: delivery.id, status: delivery.status, updatedAt: delivery.updatedAt, }, data: { status: 'manual_requeueing', retryCount: 0, manualRetryCount: { increment: 1 }, lastRetriedAt: retriedAt, nextRetryAt: null, sentAt: null, acknowledgedAt: null, ackDeadlineAt: null, ackResult: null, ackSequenceId: null, ackMessageId: null, connectionId: null, deliveredAt: null, lastError: null, }, }); if (claimed.count !== 1) { throw new BadRequestException('该下游投递记录已被其他操作处理,请刷新后重试'); } await this.prisma.operationLog.create({ data: { tenantId: delivery.tenantId, action: 'gateway.downstream_delivery_requeue', resource: 'cmpp_downstream_delivery', resourceId: delivery.id, detail: { deliveryType: delivery.deliveryType, applicationId: delivery.applicationId, messageId: delivery.messageId, previousStatus: delivery.status, previousRetryCount: delivery.retryCount, manualRetryCount: (delivery.manualRetryCount ?? 0) + 1, lastRetriedAt: retriedAt, }, }, }); try { const result = await this.facade.postGatewayControl(path, requestPayload) as GatewayControlDeliveryResult; if (result.sent || result.delivered) { return this.facade.markDownstreamDeliverySent({ id: delivery.id, ...result }); } return this.facade.markDownstreamDeliveryFailed( delivery.id, downstreamControlFailureMessage(result), result.retryable === false ? 'unrecoverable' : 'send_failed', ); } catch (error) { return this.facade.markDownstreamDeliveryFailed( delivery.id, error instanceof Error ? error.message : 'Gateway control delivery failed', ); } } async recoverStaleDownstreamManualRequeues(now = new Date()) { const staleCutoff = new Date(now.getTime() - positiveInteger( process.env.CMPP_DOWNSTREAM_MANUAL_REQUEUE_STALE_MS, DEFAULT_DOWNSTREAM_MANUAL_REQUEUE_STALE_MS, )); const stale = await this.prisma.cmppDownstreamDelivery.findMany({ where: { status: 'manual_requeueing', updatedAt: { lt: staleCutoff } }, select: { id: true, updatedAt: true }, orderBy: { updatedAt: 'asc' }, take: 500, }); let recovered = 0; for (const delivery of stale) { const updated = await this.prisma.cmppDownstreamDelivery.updateMany({ where: { id: delivery.id, status: 'manual_requeueing', updatedAt: delivery.updatedAt }, data: { status: 'pending', nextRetryAt: null, lastError: '人工重投进程中断,已恢复为待投递', }, }); recovered += updated.count; } return { recovered }; } async batchRequeueDownstreamDeliveries(ids: string[]) { const uniqueIds = [...new Set(ids.filter(Boolean))]; if (uniqueIds.length === 0) { throw new BadRequestException('请选择至少一条下游投递记录'); } const results: Array<{ id: string; status: 'success' | 'failed'; errorMessage?: string }> = []; for (const id of uniqueIds) { try { await this.facade.requeueDownstreamDelivery(id); results.push({ id, status: 'success' }); } catch (error) { results.push({ id, status: 'failed', errorMessage: error instanceof Error ? error.message : '批量重投失败', }); } } return { total: uniqueIds.length, successCount: results.filter((item) => item.status === 'success').length, failedCount: results.filter((item) => item.status === 'failed').length, results, }; } }