1203 lines
52 KiB
TypeScript
1203 lines
52 KiB
TypeScript
import { BadRequestException, HttpException, HttpStatus, Logger, NotFoundException } from '@nestjs/common';
|
|
import { Prisma } from '@prisma/client';
|
|
import { Queue, Worker } from 'bullmq';
|
|
import IORedis from 'ioredis';
|
|
import { createHash, randomUUID } from 'node:crypto';
|
|
import { setTimeout as sleep } from 'node:timers/promises';
|
|
import { BillingService } from '../billing/billing.service';
|
|
import { isIpAllowed } from '../common/ip-allowlist';
|
|
import { moneyToNumber } from '../common/money';
|
|
import { PhoneRoutingLookupService } from '../dictionaries/phone-routing-lookup.service';
|
|
import { MetricsService, SendWorkerQueueState, SendWorkerStage } from '../metrics/metrics.service';
|
|
import { PrismaService } from '../prisma/prisma.service';
|
|
import { RiskReviewService } from '../risk-review/risk-review.service';
|
|
import { PhoneFrequencyService } from '../risk-review/phone-frequency.service';
|
|
import type { CreateBatchTaskDto, CreateHttpBatchTaskDto, GatewayInboundAuthDto, GatewayInboundSubmitDto, GatewayInboundSingleSubmitResult, ImportPreviewDto, ConfirmImportDto, SendJob, QueuePriority, RoutedChannel } from './send-chain.contracts';
|
|
import { SEND_QUEUE, GATEWAY_SUBMIT_QUEUE, GATEWAY_SUBMIT_STREAM, DEFAULT_SCHEDULED_DISPATCH_STALE_MS, DEFAULT_INBOUND_LONG_MESSAGE_PROCESSING_STALE_SECONDS, GATEWAY_SUBMIT_REQUEUE_IDEMPOTENCY_TTL_SECONDS, BULLMQ_PRIORITY, drainageRejectionReason, statusFromRisk, parseSchedule, parseImportRows, splitImportLine, cellByHeader, normalizeCarrier, normalizeQueuePriority, getPositiveConfigInteger, getNonNegativeConfigInteger, isCarrierCompatible, matchTemplateContent, isNationalChannel, validateInboundApplicationSrcId, composeUpstreamSrcId, positiveInteger, parseOptionalSequenceId, shanghaiDateKey, bullmqConnection, matchesApplicationSecret, octetString, selectChannelCandidate } from './send-chain.helpers';
|
|
import type { SendSubmissionCallbacks, SendSubmissionService } from './send-submission.service';
|
|
|
|
type PendingSendBatchItem = {
|
|
job: SendJob;
|
|
resolve: (value: unknown) => void;
|
|
reject: (reason: unknown) => void;
|
|
};
|
|
|
|
/**
|
|
* R9 gatewaySubmit implementation. Cross-method calls return through the stable SendChainService seam.
|
|
*/
|
|
export class SendGatewaySubmitService {
|
|
private readonly logger = new Logger('SendChainService');
|
|
private redis?: IORedis;
|
|
private sendQueue?: Queue<SendJob, unknown, 'send-message'>;
|
|
private gatewayQueue?: Queue;
|
|
private worker?: Worker<SendJob>;
|
|
private sendQueueMetricsTimer?: ReturnType<typeof setInterval>;
|
|
private submitOutboxTimer?: ReturnType<typeof setInterval>;
|
|
private submitOutboxRunning = false;
|
|
private readonly submitOutboxLeaseOwner = `send-worker-${process.pid}-${randomUUID()}`;
|
|
private sendWorkerInFlight = 0;
|
|
private sendWorkerConfiguredSlots = 0;
|
|
private pendingSendBatch: PendingSendBatchItem[] = [];
|
|
private sendBatchTimer?: ReturnType<typeof setTimeout>;
|
|
private sendBatchFlushing = false;
|
|
private readonly taskProgressRefreshes = new Map<string, Promise<void>>();
|
|
private readonly dirtyTaskProgressRefreshes = new Set<string>();
|
|
private readonly openSubmitSessionIds = new Map<string, Promise<string>>();
|
|
|
|
constructor(
|
|
private readonly prisma: PrismaService,
|
|
private readonly billing: BillingService,
|
|
private readonly riskReview: RiskReviewService,
|
|
private readonly phoneFrequency: PhoneFrequencyService,
|
|
private readonly phoneRouting: PhoneRoutingLookupService,
|
|
private readonly facade: SendSubmissionService,
|
|
private readonly callbacks: SendSubmissionCallbacks,
|
|
private readonly metrics?: MetricsService,
|
|
) {}
|
|
|
|
async onModuleDestroy() {
|
|
if (this.sendQueueMetricsTimer) clearInterval(this.sendQueueMetricsTimer);
|
|
if (this.submitOutboxTimer) clearInterval(this.submitOutboxTimer);
|
|
if (this.sendBatchTimer) clearTimeout(this.sendBatchTimer);
|
|
await this.flushSendBatch();
|
|
await this.worker?.close();
|
|
await this.sendQueue?.close();
|
|
await this.gatewayQueue?.close();
|
|
this.redis?.disconnect();
|
|
}
|
|
|
|
private releaseMessageReservation(
|
|
message: { tenantId: string; batchTaskId: string; messageId: string; amountCents: number | bigint; billingUnits: number },
|
|
remark: string,
|
|
) {
|
|
return this.callbacks.releaseMessageReservation(message, remark);
|
|
}
|
|
|
|
private recordCmppFailureReceipt(
|
|
message: {
|
|
id: string;
|
|
tenantId?: string | null;
|
|
batchTaskId?: string | null;
|
|
applicationId?: string | null;
|
|
messageId: string;
|
|
phoneNumber: string;
|
|
cmppSubmitSequenceId?: string | null;
|
|
cmppSubmitGroupMessageId?: string | null;
|
|
},
|
|
errorCode: string,
|
|
reason: string,
|
|
) {
|
|
return this.callbacks.recordCmppFailureReceipt(message, errorCode, reason);
|
|
}
|
|
|
|
|
|
async enqueueBatchTask(taskId: string, preparedMessage?: { messageRecordId: string; queuePriority?: string | null }) {
|
|
if (preparedMessage) {
|
|
// CMPP内部任务在当前请求内刚完成持久化且不暴露取消入口，可安全复用已知ID；
|
|
// 普通批量任务仍走下方查询路径，以保留取消检查和多消息枚举语义。
|
|
const queuePriority = normalizeQueuePriority(preparedMessage.queuePriority);
|
|
await this.facade.getSendQueue().add('send-message', { messageRecordId: preparedMessage.messageRecordId }, {
|
|
jobId: preparedMessage.messageRecordId,
|
|
attempts: 3,
|
|
priority: BULLMQ_PRIORITY[queuePriority],
|
|
});
|
|
await this.prisma.smsBatchTask.update({ where: { id: taskId }, data: { status: 'queued' } });
|
|
return { taskId, enqueued: 1 };
|
|
}
|
|
const task = await this.prisma.smsBatchTask.findUnique({ where: { id: taskId } });
|
|
if (!task) {
|
|
throw new NotFoundException('SMS batch task not found');
|
|
}
|
|
if (task.status === 'canceled') {
|
|
throw new BadRequestException('SMS batch task is canceled');
|
|
}
|
|
const messages = await this.prisma.smsMessageRecord.findMany({
|
|
where: { batchTaskId: taskId, status: 'queued' },
|
|
select: { id: true, queuePriority: true },
|
|
take: 100000,
|
|
});
|
|
const queue = this.facade.getSendQueue();
|
|
for (const message of messages) {
|
|
const queuePriority = normalizeQueuePriority(message.queuePriority);
|
|
await queue.add('send-message', { messageRecordId: message.id }, {
|
|
jobId: message.id,
|
|
attempts: 3,
|
|
priority: BULLMQ_PRIORITY[queuePriority],
|
|
});
|
|
}
|
|
await this.prisma.smsBatchTask.update({ where: { id: taskId }, data: { status: 'queued' } });
|
|
return { taskId, enqueued: messages.length };
|
|
}
|
|
|
|
startWorker() {
|
|
if (this.worker) {
|
|
return { status: 'already_started' };
|
|
}
|
|
const connection = bullmqConnection();
|
|
const configuredConcurrency = Number(process.env.API_SEND_WORKER_CONCURRENCY ?? 20);
|
|
this.sendWorkerConfiguredSlots = Number.isInteger(configuredConcurrency) && configuredConcurrency > 0 ? configuredConcurrency : 20;
|
|
this.metrics?.setSendWorkerSlots(this.sendWorkerConfiguredSlots, this.sendWorkerInFlight);
|
|
this.worker = new Worker<SendJob>(
|
|
SEND_QUEUE,
|
|
async (job) => {
|
|
this.sendWorkerInFlight += 1;
|
|
this.metrics?.setSendWorkerSlots(this.sendWorkerConfiguredSlots, this.sendWorkerInFlight);
|
|
try {
|
|
return process.env.API_SEND_WORKER_BATCH_ENABLED === 'false'
|
|
? await this.facade.processSendJob(job.data)
|
|
: await this.enqueueSendBatch(job.data);
|
|
} finally {
|
|
this.sendWorkerInFlight = Math.max(0, this.sendWorkerInFlight - 1);
|
|
this.metrics?.setSendWorkerSlots(this.sendWorkerConfiguredSlots, this.sendWorkerInFlight);
|
|
}
|
|
},
|
|
{ connection, concurrency: this.sendWorkerConfiguredSlots },
|
|
);
|
|
void this.refreshSendQueueMetrics();
|
|
this.sendQueueMetricsTimer = setInterval(() => void this.refreshSendQueueMetrics(), 5_000);
|
|
this.sendQueueMetricsTimer.unref?.();
|
|
if (this.submitOutboxEnabled() && process.env.SEND_SUBMIT_OUTBOX_SEPARATE_PROCESS_ENABLED !== 'true') {
|
|
void this.publishSubmitOutboxBatch();
|
|
this.submitOutboxTimer = setInterval(
|
|
() => void this.publishSubmitOutboxBatch(),
|
|
getPositiveConfigInteger(process.env, 'SEND_SUBMIT_OUTBOX_POLL_INTERVAL_MS', 25),
|
|
);
|
|
this.submitOutboxTimer.unref?.();
|
|
}
|
|
return { status: 'started' };
|
|
}
|
|
|
|
startSubmitOutboxPublisher() {
|
|
if (this.submitOutboxTimer) return { status: 'already_started' };
|
|
if (!this.submitOutboxEnabled()) return { status: 'disabled' };
|
|
void this.publishSubmitOutboxBatch();
|
|
this.submitOutboxTimer = setInterval(
|
|
() => void this.publishSubmitOutboxBatch(),
|
|
getPositiveConfigInteger(process.env, 'SEND_SUBMIT_OUTBOX_POLL_INTERVAL_MS', 25),
|
|
);
|
|
this.submitOutboxTimer.unref?.();
|
|
return { status: 'started' };
|
|
}
|
|
|
|
private enqueueSendBatch(job: SendJob) {
|
|
return new Promise((resolve, reject) => {
|
|
this.pendingSendBatch.push({ job, resolve, reject });
|
|
const batchSize = Math.min(128, getPositiveConfigInteger(process.env, 'API_SEND_WORKER_BATCH_SIZE', 32));
|
|
if (this.pendingSendBatch.length >= batchSize) {
|
|
if (this.sendBatchTimer) clearTimeout(this.sendBatchTimer);
|
|
this.sendBatchTimer = undefined;
|
|
queueMicrotask(() => void this.flushSendBatch());
|
|
return;
|
|
}
|
|
if (!this.sendBatchTimer) {
|
|
this.sendBatchTimer = setTimeout(
|
|
() => {
|
|
this.sendBatchTimer = undefined;
|
|
void this.flushSendBatch();
|
|
},
|
|
Math.min(25, getPositiveConfigInteger(process.env, 'API_SEND_WORKER_BATCH_WAIT_MS', 3)),
|
|
);
|
|
this.sendBatchTimer.unref?.();
|
|
}
|
|
});
|
|
}
|
|
|
|
private async flushSendBatch() {
|
|
if (this.sendBatchFlushing) return;
|
|
this.sendBatchFlushing = true;
|
|
try {
|
|
const batchSize = Math.min(128, getPositiveConfigInteger(process.env, 'API_SEND_WORKER_BATCH_SIZE', 32));
|
|
while (this.pendingSendBatch.length > 0) {
|
|
const batch = this.pendingSendBatch.splice(0, batchSize);
|
|
try {
|
|
const results = await this.processSendJobBatch(batch.map((item) => item.job));
|
|
for (const item of batch) item.resolve(results.get(item.job.messageRecordId) ?? { skipped: true });
|
|
} catch (error) {
|
|
for (const item of batch) item.reject(error);
|
|
}
|
|
}
|
|
} finally {
|
|
this.sendBatchFlushing = false;
|
|
if (this.pendingSendBatch.length > 0) queueMicrotask(() => void this.flushSendBatch());
|
|
}
|
|
}
|
|
|
|
private async processSendJobBatch(jobs: SendJob[]) {
|
|
if (jobs.length === 1) {
|
|
return new Map([[jobs[0].messageRecordId, await this.processSendJob(jobs[0])]]);
|
|
}
|
|
const ids = [...new Set(jobs.map((job) => job.messageRecordId))];
|
|
const messages = await this.measureSendStage('message_load', () => this.prisma.smsMessageRecord.findMany({
|
|
where: { id: { in: ids } },
|
|
include: { batchTask: true, template: { include: { signature: true } }, signature: true },
|
|
}));
|
|
const messageById = new Map(messages.map((message) => [message.id, message]));
|
|
const results = new Map<string, unknown>();
|
|
const businessMessages = messages.filter((message) => {
|
|
if (message.status !== 'queued' || !message.tenantId || !message.batchTaskId) {
|
|
results.set(message.id, { skipped: true });
|
|
return false;
|
|
}
|
|
return true;
|
|
}) as Array<typeof messages[number] & { tenantId: string; batchTaskId: string }>;
|
|
for (const id of ids) if (!messageById.has(id)) results.set(id, { skipped: true });
|
|
if (businessMessages.length === 0) return results;
|
|
|
|
const { planned, failed } = await this.planRoutesBatch(businessMessages);
|
|
if (failed.length > 0) await this.failRouteBatch(failed, results);
|
|
if (planned.length === 0) return results;
|
|
|
|
await Promise.all(planned.map(({ routed }) => (
|
|
this.facade.waitForChannelRateLimit(routed.channel.id, routed.channel.rateLimitPerSecond)
|
|
)));
|
|
const sessionByChannel = new Map<string, string>();
|
|
await Promise.all([...new Set(planned.map(({ routed }) => routed.channel.id))].map(async (channelId) => {
|
|
sessionByChannel.set(channelId, await this.getOpenSubmitSessionId(channelId));
|
|
}));
|
|
const prepared = planned.map(({ message, routed }) => {
|
|
const submitId = `SUB-${randomUUID()}`;
|
|
const upstreamSrcId = composeUpstreamSrcId(routed.channel.srcId, message.applicationExtension);
|
|
return {
|
|
message,
|
|
routed,
|
|
submitId,
|
|
command: this.buildGatewaySubmitCommand(message, routed, 0, submitId, upstreamSrcId),
|
|
sessionId: sessionByChannel.get(routed.channel.id),
|
|
};
|
|
});
|
|
const writeOutbox = this.submitOutboxEnabled();
|
|
await this.measureSendStage('submit_transaction', () => this.prisma.$transaction(async (tx) => {
|
|
await tx.smsSubmitRecord.createMany({
|
|
data: prepared.map(({ message, routed, submitId, sessionId }) => ({
|
|
id: randomUUID(),
|
|
tenantId: message.tenantId,
|
|
batchTaskId: message.batchTaskId,
|
|
messageRecordId: message.id,
|
|
channelId: routed.channel.id,
|
|
channelGroupId: routed.groupId,
|
|
channelGroupName: routed.groupName,
|
|
sessionId,
|
|
submitId,
|
|
submitStatus: 'queued',
|
|
costUnitPrice: routed.channel.unitPrice ?? 0,
|
|
costAmountCents: moneyToNumber(routed.channel.unitPrice) * Math.max(1, message.billingUnits ?? 1),
|
|
})),
|
|
});
|
|
const updates = Prisma.join(prepared.map(({ message, routed, submitId }) => Prisma.sql`(
|
|
${message.id}::text, ${routed.channel.id}::text, ${routed.carrier}::text,
|
|
${routed.province ?? null}::text, ${submitId}::text
|
|
)`));
|
|
await tx.$executeRaw(Prisma.sql`
|
|
UPDATE "SmsMessageRecord" AS message
|
|
SET "channelId" = updates."channelId",
|
|
carrier = updates.carrier,
|
|
province = updates.province,
|
|
"submitId" = updates."submitId",
|
|
status = 'submit_queued',
|
|
"submitStatus" = 'queued',
|
|
"receiptStatus" = NULL,
|
|
"errorCode" = NULL,
|
|
"errorMessage" = NULL,
|
|
"updatedAt" = CURRENT_TIMESTAMP
|
|
FROM (VALUES ${updates}) AS updates(id, "channelId", carrier, province, "submitId")
|
|
WHERE message.id = updates.id AND message.status = 'queued'
|
|
`);
|
|
if (writeOutbox) {
|
|
await tx.gatewaySubmitOutbox.createMany({
|
|
data: prepared.map(({ message, routed, submitId, command }) => ({
|
|
id: randomUUID(), submitId, messageRecordId: message.id,
|
|
channelId: routed.channel.id, payload: command as Prisma.InputJsonValue,
|
|
})),
|
|
});
|
|
}
|
|
}));
|
|
if (!this.submitOutboxPublishEnabled()) {
|
|
await Promise.all(prepared.map(({ command }) => this.facade.publishGatewaySubmitCommand(command)));
|
|
}
|
|
await this.refreshTaskProgressBatch(prepared.map(({ message }) => message));
|
|
for (const { message, routed, submitId } of prepared) {
|
|
results.set(message.id, {
|
|
submitted: true, messageRecordId: message.id, channelId: routed.channel.id, attempt: 0, submitId,
|
|
});
|
|
this.metrics?.recordSendWorkerResult('completed');
|
|
}
|
|
return results;
|
|
}
|
|
|
|
private async planRoutesBatch<T extends {
|
|
id: string; tenantId: string; batchTaskId: string; applicationId?: string | null;
|
|
templateId?: string | null; signatureId?: string | null; phoneNumber: string;
|
|
carrier?: string | null; province?: string | null;
|
|
template?: { signature?: { id?: string | null } | null } | null;
|
|
signature?: { id?: string | null } | null;
|
|
}>(messages: T[]) {
|
|
const unresolvedPhones = messages.filter((message) => !message.carrier).map((message) => message.phoneNumber);
|
|
const provinces = await this.measureSendStage('phone_routing', () => this.phoneRouting.identifyProvinces(unresolvedPhones));
|
|
const routeInputs = await Promise.all(messages.map(async (message) => ({
|
|
message,
|
|
carrier: message.carrier ? normalizeCarrier(message.carrier) : normalizeCarrier(await this.phoneRouting.identifyCarrier(message.phoneNumber)),
|
|
province: message.carrier ? message.province ?? null : provinces.get(message.phoneNumber) ?? null,
|
|
signatureId: message.signatureId ?? message.template?.signature?.id ?? message.signature?.id ?? null,
|
|
})));
|
|
const valid = routeInputs.filter((input) => input.message.applicationId && input.signatureId);
|
|
const signatures = [...new Set(valid.map((input) => input.signatureId as string))];
|
|
const routes = valid.length === 0 ? [] : await this.measureSendStage('route_lookup', () => this.prisma.channelRouteRule.findMany({
|
|
where: {
|
|
status: 'active', channelId: null, province: null,
|
|
OR: valid.map((input) => ({
|
|
tenantId: input.message.tenantId,
|
|
applicationId: input.message.applicationId,
|
|
carrier: input.carrier,
|
|
})),
|
|
},
|
|
include: {
|
|
group: {
|
|
include: {
|
|
items: {
|
|
include: {
|
|
channel: {
|
|
include: {
|
|
connectionStates: { where: { status: 'connected', currentConnections: { gt: 0 } } },
|
|
reportTasks: { where: { signatureId: { in: signatures }, reportType: 'signature', status: 'approved' } },
|
|
},
|
|
},
|
|
},
|
|
orderBy: { priority: 'asc' },
|
|
},
|
|
},
|
|
},
|
|
},
|
|
orderBy: { priority: 'asc' },
|
|
}));
|
|
const routeByKey = new Map<string, typeof routes[number]>();
|
|
for (const route of routes) {
|
|
const key = `${route.tenantId}:${route.applicationId}:${normalizeCarrier(route.carrier)}`;
|
|
if (!routeByKey.has(key)) routeByKey.set(key, route);
|
|
}
|
|
const planned: Array<{ message: T; routed: RoutedChannel }> = [];
|
|
const failed: Array<{ message: T; reason: string }> = [];
|
|
for (const input of routeInputs) {
|
|
if (!input.message.applicationId) {
|
|
failed.push({ message: input.message, reason: '短信应用未配置，无法选择通道组' });
|
|
continue;
|
|
}
|
|
if (!input.signatureId) {
|
|
failed.push({ message: input.message, reason: '短信签名未配置，无法选择已报备通道' });
|
|
continue;
|
|
}
|
|
const route = routeByKey.get(`${input.message.tenantId}:${input.message.applicationId}:${input.carrier}`);
|
|
if (!route) {
|
|
failed.push({ message: input.message, reason: '企业应用未配置对应运营商通道组' });
|
|
continue;
|
|
}
|
|
if (route.group.status !== 'active' || normalizeCarrier(route.group.carrier) !== input.carrier) {
|
|
failed.push({ message: input.message, reason: '企业应用绑定的通道组已停用或运营商不一致' });
|
|
continue;
|
|
}
|
|
const approvedItems = route.group.items.filter((item) => item.channel.status === 'active'
|
|
&& item.channel.connectionStates.length > 0
|
|
&& item.channel.reportTasks.some((task) => task.signatureId === input.signatureId
|
|
&& (task.carrier === input.carrier
|
|
|| (process.env.SIGNATURE_REPORT_STRICT_CARRIER !== 'true' && task.approvalScope === 'legacy_channel'))));
|
|
const selected = selectChannelCandidate(approvedItems, {
|
|
carrier: input.carrier,
|
|
province: input.province,
|
|
excludedChannelIds: new Set(),
|
|
approvedChannelIds: new Set(approvedItems.map((item) => item.channelId)),
|
|
routingKey: input.message.id,
|
|
});
|
|
if (!selected) {
|
|
failed.push({ message: input.message, reason: '无已报备通过且在线的可用通道' });
|
|
continue;
|
|
}
|
|
planned.push({
|
|
message: input.message,
|
|
routed: {
|
|
channel: { ...selected.channel, unitPrice: moneyToNumber(selected.channel.unitPrice) },
|
|
carrier: input.carrier, province: input.province,
|
|
groupId: route.groupId, groupName: route.group.name,
|
|
routeScope: isNationalChannel(selected) ? 'national' : 'province',
|
|
},
|
|
});
|
|
}
|
|
return { planned, failed };
|
|
}
|
|
|
|
private async failRouteBatch<T extends {
|
|
id: string; tenantId: string; batchTaskId: string; applicationId?: string | null;
|
|
messageId: string; phoneNumber: string; amountCents: bigint; billingUnits: number;
|
|
cmppSubmitSequenceId?: string | null; cmppSubmitGroupMessageId?: string | null;
|
|
batchTask?: { sourceType?: string | null; phoneTotal?: number | null } | null;
|
|
}>(failed: Array<{ message: T; reason: string }>, results: Map<string, unknown>) {
|
|
const values = Prisma.join(failed.map(({ message, reason }) => Prisma.sql`(${message.id}::text, ${reason.slice(0, 1000)}::text)`));
|
|
await this.prisma.$executeRaw(Prisma.sql`
|
|
UPDATE "SmsMessageRecord" AS message
|
|
SET status = 'failed', "errorMessage" = failures.reason, "updatedAt" = CURRENT_TIMESTAMP
|
|
FROM (VALUES ${values}) AS failures(id, reason)
|
|
WHERE message.id = failures.id AND message.status = 'queued'
|
|
`);
|
|
await Promise.all(failed.map(async ({ message, reason }) => {
|
|
await this.releaseMessageReservation(message, reason);
|
|
if (message.batchTask?.sourceType === 'cmpp') await this.recordCmppFailureReceipt(message, 'ROUTE', reason);
|
|
else await this.facade.refreshTaskProgress(message.batchTaskId);
|
|
results.set(message.id, { submitted: false, messageRecordId: message.id, status: 'failed', reason });
|
|
this.metrics?.recordSendWorkerResult('failed');
|
|
}));
|
|
}
|
|
|
|
private async refreshTaskProgressBatch(messages: Array<{
|
|
batchTaskId: string; batchTask?: { sourceType?: string | null; phoneTotal?: number | null } | null;
|
|
}>) {
|
|
const singleCmppIds = [...new Set(messages
|
|
.filter((message) => message.batchTask?.sourceType === 'cmpp' && message.batchTask.phoneTotal === 1)
|
|
.map((message) => message.batchTaskId))];
|
|
if (singleCmppIds.length > 0) {
|
|
await this.prisma.smsBatchTask.updateMany({
|
|
where: { id: { in: singleCmppIds }, sourceType: 'cmpp', phoneTotal: 1 },
|
|
data: singleMessageTaskProgress('submit_queued'),
|
|
});
|
|
}
|
|
const otherTaskIds = [...new Set(messages
|
|
.filter((message) => !singleCmppIds.includes(message.batchTaskId))
|
|
.map((message) => message.batchTaskId))];
|
|
await Promise.all(otherTaskIds.map((taskId) => this.facade.refreshTaskProgress(taskId)));
|
|
}
|
|
|
|
async processSendJob(job: SendJob) {
|
|
const totalStartedAt = this.metrics?.beginSendWorkerStage();
|
|
let totalFinished = false;
|
|
const finish = (result: 'completed' | 'failed' | 'skipped') => {
|
|
if (totalFinished) return;
|
|
totalFinished = true;
|
|
if (totalStartedAt != null) {
|
|
this.metrics?.finishSendWorkerStage(totalStartedAt, 'total', result === 'completed' ? 'success' : result === 'skipped' ? 'skipped' : 'error');
|
|
}
|
|
this.metrics?.recordSendWorkerResult(result);
|
|
};
|
|
try {
|
|
const message = await this.measureSendStage('message_load', () => this.prisma.smsMessageRecord.findUnique({
|
|
where: { id: job.messageRecordId },
|
|
include: { batchTask: true, template: { include: { signature: true } }, signature: true },
|
|
}));
|
|
if (!message || message.status !== 'queued') {
|
|
finish('skipped');
|
|
return { skipped: true };
|
|
}
|
|
if (!message.tenantId || !message.batchTaskId) {
|
|
finish('skipped');
|
|
return { skipped: true, reason: 'standalone channel test message' };
|
|
}
|
|
const businessMessage = message as typeof message & { tenantId: string; batchTaskId: string };
|
|
try {
|
|
const routed = await this.facade.selectChannelForMessage(businessMessage);
|
|
const result = await this.facade.submitMessageToGateway(businessMessage, routed, 0);
|
|
finish(result.submitted ? 'completed' : 'skipped');
|
|
return result;
|
|
} catch (error) {
|
|
const reason = error instanceof Error ? error.message : '无可用通道组或通道';
|
|
await this.prisma.smsMessageRecord.update({
|
|
where: { id: message.id },
|
|
data: { status: 'failed', errorMessage: reason },
|
|
});
|
|
await this.releaseMessageReservation(businessMessage, reason);
|
|
if (message.batchTask?.sourceType === 'cmpp') {
|
|
await this.recordCmppFailureReceipt(businessMessage, 'ROUTE', reason);
|
|
} else {
|
|
await this.facade.refreshTaskProgress(
|
|
businessMessage.batchTaskId,
|
|
message.batchTask?.sourceType === 'cmpp' && message.batchTask.phoneTotal === 1 ? 'failed' : undefined,
|
|
);
|
|
}
|
|
finish('failed');
|
|
return { submitted: false, messageRecordId: message.id, status: 'failed', reason };
|
|
}
|
|
} catch (error) {
|
|
finish('failed');
|
|
throw error;
|
|
}
|
|
}
|
|
|
|
async submitMessageToGateway(
|
|
message: {
|
|
id: string;
|
|
tenantId: string;
|
|
batchTaskId: string;
|
|
applicationId?: string | null;
|
|
templateId?: string | null;
|
|
signatureId?: string | null;
|
|
submitId?: string | null;
|
|
messageId: string;
|
|
phoneNumber: string;
|
|
content: string;
|
|
billingUnits: number;
|
|
queuePriority?: string | null;
|
|
clientSrcId?: string | null;
|
|
applicationExtension?: string | null;
|
|
template?: { signature?: { id?: string | null; name?: string | null } | null } | null;
|
|
signature?: { id?: string | null; name?: string | null } | null;
|
|
batchTask?: { sourceType?: string | null; phoneTotal?: number | null } | null;
|
|
},
|
|
routed: RoutedChannel,
|
|
attempt: number,
|
|
retryOfSubmitRecordId?: string,
|
|
) {
|
|
const channel = routed.channel;
|
|
const upstreamSrcId = composeUpstreamSrcId(channel.srcId, message.applicationExtension);
|
|
await this.measureSendStage('rate_limit', () => this.facade.waitForChannelRateLimit(channel.id, channel.rateLimitPerSecond));
|
|
const submitId = `SUB-${randomUUID()}`;
|
|
const sessionId = await this.getOpenSubmitSessionId(channel.id);
|
|
const command = this.buildGatewaySubmitCommand(message, routed, attempt, submitId, upstreamSrcId);
|
|
const writeOutbox = this.submitOutboxEnabled();
|
|
try {
|
|
await this.measureSendStage('submit_transaction', () => this.prisma.$transaction(async (tx) => {
|
|
await tx.smsSubmitRecord.create({
|
|
data: {
|
|
tenantId: message.tenantId,
|
|
batchTaskId: message.batchTaskId,
|
|
messageRecordId: message.id,
|
|
channelId: channel.id,
|
|
channelGroupId: routed.groupId,
|
|
channelGroupName: routed.groupName,
|
|
sessionId,
|
|
retryOfSubmitRecordId,
|
|
submitId,
|
|
submitStatus: 'queued',
|
|
costUnitPrice: channel.unitPrice ?? 0,
|
|
costAmountCents: moneyToNumber(channel.unitPrice) * Math.max(1, message.billingUnits ?? 1),
|
|
},
|
|
});
|
|
await tx.smsMessageRecord.update({
|
|
where: { id: message.id },
|
|
data: {
|
|
channelId: channel.id,
|
|
carrier: routed.carrier,
|
|
province: routed.province,
|
|
submitId,
|
|
status: 'submit_queued',
|
|
submitStatus: 'queued',
|
|
receiptStatus: null,
|
|
errorCode: null,
|
|
errorMessage: attempt > 0 ? `第 ${attempt + 1} 次提交，路由至${routed.routeScope === 'national' ? '全国' : '省网'}通道` : undefined,
|
|
},
|
|
});
|
|
if (writeOutbox) {
|
|
await tx.gatewaySubmitOutbox.create({
|
|
data: {
|
|
submitId,
|
|
messageRecordId: message.id,
|
|
channelId: channel.id,
|
|
payload: command as Prisma.InputJsonValue,
|
|
},
|
|
});
|
|
}
|
|
}));
|
|
if (retryOfSubmitRecordId) {
|
|
this.logger.log(`sms_retry_claim_acquired ${JSON.stringify({
|
|
messageId: message.messageId,
|
|
messageRecordId: message.id,
|
|
retryOfSubmitRecordId,
|
|
submitId,
|
|
channelId: channel.id,
|
|
})}`);
|
|
}
|
|
} catch (error) {
|
|
if (
|
|
retryOfSubmitRecordId
|
|
&& error instanceof Prisma.PrismaClientKnownRequestError
|
|
&& error.code === 'P2002'
|
|
) {
|
|
const existingRetry = await this.prisma.smsSubmitRecord.findUnique({
|
|
where: { retryOfSubmitRecordId },
|
|
});
|
|
if (existingRetry) {
|
|
this.logger.warn(`sms_retry_claim_reused ${JSON.stringify({
|
|
messageId: message.messageId,
|
|
messageRecordId: message.id,
|
|
retryOfSubmitRecordId,
|
|
submitId: existingRetry.submitId,
|
|
channelId: existingRetry.channelId,
|
|
})}`);
|
|
return {
|
|
submitted: false,
|
|
duplicateRetry: true,
|
|
messageRecordId: message.id,
|
|
channelId: existingRetry.channelId,
|
|
attempt,
|
|
submitId: existingRetry.submitId,
|
|
};
|
|
}
|
|
}
|
|
throw error;
|
|
}
|
|
if (!this.submitOutboxPublishEnabled()) {
|
|
await this.measureSendStage('gateway_stream_publish', () => this.facade.publishGatewaySubmitCommand(command));
|
|
}
|
|
await this.measureSendStage('task_progress', () => this.facade.refreshTaskProgress(
|
|
message.batchTaskId,
|
|
message.batchTask?.sourceType === 'cmpp' && message.batchTask.phoneTotal === 1 ? 'submit_queued' : undefined,
|
|
));
|
|
return { submitted: true, messageRecordId: message.id, channelId: channel.id, attempt };
|
|
}
|
|
|
|
private buildGatewaySubmitCommand(
|
|
message: {
|
|
id: string; tenantId: string; batchTaskId: string; applicationId?: string | null;
|
|
templateId?: string | null; messageId: string; phoneNumber: string; content: string;
|
|
billingUnits: number; queuePriority?: string | null; applicationExtension?: string | null;
|
|
template?: { signature?: { name?: string | null } | null } | null;
|
|
signature?: { name?: string | null } | null;
|
|
},
|
|
routed: RoutedChannel,
|
|
attempt: number,
|
|
submitId: string,
|
|
upstreamSrcId: string,
|
|
) {
|
|
const channel = routed.channel;
|
|
return {
|
|
schemaVersion: 'v1',
|
|
messageType: 'SubmitCommand',
|
|
traceId: randomUUID(),
|
|
messageId: message.messageId,
|
|
channelId: channel.id,
|
|
createdAt: new Date().toISOString(),
|
|
tenantId: message.tenantId,
|
|
applicationId: message.applicationId ?? 'unknown',
|
|
taskId: message.batchTaskId,
|
|
submitId,
|
|
queuePriority: normalizeQueuePriority(message.queuePriority),
|
|
phoneNumber: message.phoneNumber,
|
|
content: message.content,
|
|
signature: message.template?.signature?.name ?? message.signature?.name ?? 'SMS',
|
|
templateId: message.templateId ?? 'unknown',
|
|
billingUnits: message.billingUnits,
|
|
route: {
|
|
channelCode: channel.code,
|
|
cmppAccountCode: channel.account,
|
|
priority: attempt,
|
|
rateLimitPerSecond: channel.rateLimitPerSecond,
|
|
carrier: routed.carrier,
|
|
province: routed.province ?? undefined,
|
|
scope: routed.routeScope,
|
|
groupId: routed.groupId,
|
|
},
|
|
cmpp: {
|
|
serviceId: channel.config && typeof channel.config === 'object' && 'serviceId' in channel.config
|
|
? String(channel.config.serviceId)
|
|
: 'SMS',
|
|
srcId: upstreamSrcId,
|
|
extensionDigits: getNonNegativeConfigInteger(channel.config, 'extensionDigits', 0),
|
|
registeredDelivery: 1,
|
|
msgFmt: 8,
|
|
},
|
|
upstream: {
|
|
gatewayHost: channel.gatewayHost,
|
|
gatewayPort: channel.gatewayPort,
|
|
account: channel.account,
|
|
passwordCipher: channel.passwordCipher,
|
|
cmppVersion: channel.cmppVersion,
|
|
desiredConnections: getPositiveConfigInteger(channel.config, 'desiredConnections', 1),
|
|
windowSize: getPositiveConfigInteger(channel.config, 'windowSize', 16),
|
|
heartbeatIntervalSeconds: getPositiveConfigInteger(channel.config, 'heartbeatIntervalSeconds', 30),
|
|
heartbeatMissThreshold: getPositiveConfigInteger(channel.config, 'heartbeatMissThreshold', 3),
|
|
connectionWarmupSeconds: getNonNegativeConfigInteger(channel.config, 'connectionWarmupSeconds', 30),
|
|
connectionDrainTimeoutSeconds: getPositiveConfigInteger(channel.config, 'connectionDrainTimeoutSeconds', 60),
|
|
submitResponseTimeoutSeconds: getPositiveConfigInteger(channel.config, 'submitResponseTimeoutSeconds', 60),
|
|
connectionFailureCooldownSeconds: getPositiveConfigInteger(channel.config, 'connectionFailureCooldownSeconds', 30),
|
|
},
|
|
retry: { attempt, maxAttempts: 1 },
|
|
};
|
|
}
|
|
|
|
private submitOutboxEnabled() {
|
|
return process.env.SEND_SUBMIT_OUTBOX_SHADOW_ENABLED === 'true'
|
|
|| process.env.SEND_SUBMIT_OUTBOX_PUBLISH_ENABLED === 'true';
|
|
}
|
|
|
|
private submitOutboxPublishEnabled() {
|
|
return process.env.SEND_SUBMIT_OUTBOX_PUBLISH_ENABLED === 'true';
|
|
}
|
|
|
|
private async publishSubmitOutboxBatch() {
|
|
if (this.submitOutboxRunning || !this.submitOutboxEnabled()) return;
|
|
this.submitOutboxRunning = true;
|
|
try {
|
|
const batchSize = Math.min(500, getPositiveConfigInteger(process.env, 'SEND_SUBMIT_OUTBOX_BATCH_SIZE', 64));
|
|
const leaseSeconds = Math.min(300, getPositiveConfigInteger(process.env, 'SEND_SUBMIT_OUTBOX_LEASE_SECONDS', 30));
|
|
const rows = await this.prisma.$queryRaw<Array<{ id: string; submitId: string; payload: Prisma.JsonValue }>>(Prisma.sql`
|
|
WITH candidates AS (
|
|
SELECT id
|
|
FROM "GatewaySubmitOutbox"
|
|
WHERE (
|
|
(status = 'pending' AND "nextAttemptAt" <= CURRENT_TIMESTAMP)
|
|
OR (status = 'publishing' AND "leaseExpiresAt" < CURRENT_TIMESTAMP)
|
|
)
|
|
ORDER BY "createdAt", id
|
|
LIMIT ${batchSize}
|
|
FOR UPDATE SKIP LOCKED
|
|
)
|
|
UPDATE "GatewaySubmitOutbox" AS outbox
|
|
SET status = 'publishing',
|
|
"leaseOwner" = ${this.submitOutboxLeaseOwner},
|
|
"leaseExpiresAt" = CURRENT_TIMESTAMP + (${leaseSeconds} * INTERVAL '1 second'),
|
|
"attemptCount" = outbox."attemptCount" + 1,
|
|
"updatedAt" = CURRENT_TIMESTAMP
|
|
FROM candidates
|
|
WHERE outbox.id = candidates.id
|
|
RETURNING outbox.id, outbox."submitId", outbox.payload
|
|
`);
|
|
if (rows.length === 0) return;
|
|
const results = this.submitOutboxPublishEnabled()
|
|
? await this.publishGatewaySubmitCommandBatch(rows)
|
|
: rows.map((row) => ({ row, streamEntryId: `shadow:${row.submitId}` }));
|
|
const succeeded = results.filter((result): result is { row: typeof rows[number]; streamEntryId: string } => 'streamEntryId' in result);
|
|
if (succeeded.length > 0) {
|
|
const values = Prisma.join(succeeded.map(({ row, streamEntryId }) => Prisma.sql`(${row.id}, ${streamEntryId})`));
|
|
await this.prisma.$executeRaw(Prisma.sql`
|
|
UPDATE "GatewaySubmitOutbox" AS outbox
|
|
SET status = 'published',
|
|
"streamEntryId" = published."streamEntryId",
|
|
"publishedAt" = CURRENT_TIMESTAMP,
|
|
"leaseOwner" = NULL,
|
|
"leaseExpiresAt" = NULL,
|
|
"lastError" = NULL,
|
|
"updatedAt" = CURRENT_TIMESTAMP
|
|
FROM (VALUES ${values}) AS published(id, "streamEntryId")
|
|
WHERE outbox.id = published.id
|
|
AND outbox.status = 'publishing'
|
|
AND outbox."leaseOwner" = ${this.submitOutboxLeaseOwner}
|
|
`);
|
|
}
|
|
for (const result of results) {
|
|
if ('streamEntryId' in result) continue;
|
|
const { row, error } = result;
|
|
try {
|
|
const message = error instanceof Error ? error.message : String(error);
|
|
await this.prisma.$executeRaw(Prisma.sql`
|
|
UPDATE "GatewaySubmitOutbox"
|
|
SET status = CASE WHEN "attemptCount" >= 10 THEN 'dead' ELSE 'pending' END,
|
|
"nextAttemptAt" = CURRENT_TIMESTAMP + (LEAST(60, POWER(2, LEAST("attemptCount", 6))) * INTERVAL '1 second'),
|
|
"leaseOwner" = NULL,
|
|
"leaseExpiresAt" = NULL,
|
|
"lastError" = ${message.slice(0, 1000)},
|
|
"updatedAt" = CURRENT_TIMESTAMP
|
|
WHERE id = ${row.id} AND "leaseOwner" = ${this.submitOutboxLeaseOwner}
|
|
`);
|
|
} catch (recordError) {
|
|
this.logger.error(`gateway_submit_outbox_failure_record_failed ${recordError instanceof Error ? recordError.message : String(recordError)}`);
|
|
}
|
|
}
|
|
} catch (error) {
|
|
this.logger.error(`gateway_submit_outbox_publish_failed ${error instanceof Error ? error.message : String(error)}`);
|
|
} finally {
|
|
this.submitOutboxRunning = false;
|
|
}
|
|
}
|
|
|
|
private async publishGatewaySubmitCommandBatch(
|
|
rows: Array<{ id: string; submitId: string; payload: Prisma.JsonValue }>,
|
|
): Promise<Array<
|
|
{ row: typeof rows[number]; streamEntryId: string }
|
|
| { row: typeof rows[number]; error: unknown }
|
|
>> {
|
|
const redis = this.facade.getRedis();
|
|
const stream = process.env.GATEWAY_SUBMIT_STREAM ?? GATEWAY_SUBMIT_STREAM;
|
|
const script = `local existing = redis.call('GET', KEYS[2])
|
|
if existing then return existing end
|
|
local streamId = redis.call('XADD', KEYS[1], '*', 'messageType', 'SubmitCommand', 'data', ARGV[1])
|
|
redis.call('SET', KEYS[2], streamId, 'EX', ARGV[2])
|
|
return streamId`;
|
|
const pipeline = redis.pipeline();
|
|
for (const row of rows) {
|
|
pipeline.eval(
|
|
script,
|
|
2,
|
|
stream,
|
|
`gateway:submit:outbox:${row.submitId}`,
|
|
JSON.stringify(row.payload),
|
|
String(GATEWAY_SUBMIT_REQUEUE_IDEMPOTENCY_TTL_SECONDS),
|
|
);
|
|
}
|
|
const replies = await pipeline.exec();
|
|
if (!replies || replies.length !== rows.length) {
|
|
return rows.map((row) => ({ row, error: new Error('Redis Outbox pipeline result count mismatch') }));
|
|
}
|
|
return replies.map(([error, value], index) => error
|
|
? { row: rows[index], error }
|
|
: { row: rows[index], streamEntryId: typeof value === 'string' ? value : String(value ?? '') });
|
|
}
|
|
|
|
async selectChannelForMessage(
|
|
message: { id: string; tenantId: string; applicationId?: string | null; templateId?: string | null; signatureId?: string | null; phoneNumber: string; carrier?: string | null; province?: string | null; template?: { signature?: { id?: string | null } | null } | null; signature?: { id?: string | null } | null },
|
|
options: { forceNational?: boolean; excludeChannelIds?: string[] } = {},
|
|
): Promise<RoutedChannel> {
|
|
if (!message.applicationId) {
|
|
throw new BadRequestException('短信应用未配置，无法选择通道组');
|
|
}
|
|
const hasPersistedRouting = Boolean(message.carrier);
|
|
const [carrier, province] = await this.measureSendStage('phone_routing', async () => {
|
|
const resolved = hasPersistedRouting
|
|
? [normalizeCarrier(message.carrier), message.province ?? null] as const
|
|
: await Promise.all([
|
|
this.facade.identifyCarrier(message.phoneNumber),
|
|
this.facade.identifyProvince(message.phoneNumber),
|
|
]);
|
|
if (!hasPersistedRouting) {
|
|
await this.prisma.smsMessageRecord.update({
|
|
where: { id: message.id },
|
|
data: { carrier: resolved[0], province: resolved[1] },
|
|
});
|
|
}
|
|
return resolved;
|
|
});
|
|
const signatureId = await this.measureSendStage('signature_candidates', () => this.facade.resolveMessageSignatureId(message));
|
|
if (!signatureId) throw new BadRequestException('短信签名未配置，无法选择已报备通道');
|
|
const route = await this.measureSendStage('route_lookup', () => this.facade.findApplicationRoute(
|
|
message.tenantId,
|
|
message.applicationId ?? undefined,
|
|
carrier,
|
|
signatureId,
|
|
));
|
|
const excluded = new Set(options.excludeChannelIds ?? []);
|
|
const approvedChannelIds = new Set(route.group.items.map((item) => item.channelId));
|
|
const selected = selectChannelCandidate(route.group.items, {
|
|
carrier,
|
|
province,
|
|
forceNational: options.forceNational,
|
|
excludedChannelIds: excluded,
|
|
approvedChannelIds,
|
|
routingKey: message.id,
|
|
});
|
|
if (!selected) {
|
|
throw new NotFoundException('无已报备通过且在线的可用通道');
|
|
}
|
|
return {
|
|
channel: { ...selected.channel, unitPrice: moneyToNumber(selected.channel.unitPrice) },
|
|
carrier,
|
|
province,
|
|
groupId: route.groupId,
|
|
groupName: route.group.name,
|
|
routeScope: isNationalChannel(selected) ? 'national' : 'province',
|
|
};
|
|
}
|
|
|
|
private async measureSendStage<T>(stage: SendWorkerStage, operation: () => Promise<T>): Promise<T> {
|
|
const startedAt = this.metrics?.beginSendWorkerStage();
|
|
try {
|
|
const result = await operation();
|
|
if (startedAt != null) this.metrics?.finishSendWorkerStage(startedAt, stage, 'success');
|
|
return result;
|
|
} catch (error) {
|
|
if (startedAt != null) this.metrics?.finishSendWorkerStage(startedAt, stage, 'error');
|
|
throw error;
|
|
}
|
|
}
|
|
|
|
private async refreshSendQueueMetrics() {
|
|
if (!this.metrics) return;
|
|
try {
|
|
const counts = await this.facade.getSendQueue().getJobCounts('wait', 'active', 'completed', 'failed', 'delayed', 'prioritized');
|
|
const mappings: Array<[SendWorkerQueueState, number]> = [
|
|
['waiting', counts.wait ?? 0],
|
|
['active', counts.active ?? 0],
|
|
['completed', counts.completed ?? 0],
|
|
['failed', counts.failed ?? 0],
|
|
['delayed', counts.delayed ?? 0],
|
|
['prioritized', counts.prioritized ?? 0],
|
|
];
|
|
for (const [state, count] of mappings) this.metrics.setSendWorkerQueueJobs(state, count);
|
|
const pool = this.prisma.getPoolState();
|
|
for (const state of ['max', 'total', 'idle', 'waiting'] as const) {
|
|
this.metrics.setSendWorkerDatabasePool(state, pool[state]);
|
|
}
|
|
} catch (error) {
|
|
this.logger.warn(`send_queue_metrics_refresh_failed ${error instanceof Error ? error.message : String(error)}`);
|
|
}
|
|
}
|
|
|
|
async findApplicationRoute(tenantId: string, applicationId: string | undefined, carrier: string, signatureId?: string) {
|
|
const approvedChannelWhere = signatureId
|
|
? {
|
|
status: 'active',
|
|
connectionStates: { some: { status: 'connected', currentConnections: { gt: 0 } } },
|
|
reportTasks: {
|
|
some: {
|
|
signatureId,
|
|
reportType: 'signature',
|
|
status: 'approved',
|
|
OR: signatureReportApprovalScopes(carrier),
|
|
},
|
|
},
|
|
}
|
|
: undefined;
|
|
const route = await this.prisma.channelRouteRule.findFirst({
|
|
where: {
|
|
status: 'active',
|
|
tenantId,
|
|
applicationId,
|
|
carrier,
|
|
channelId: null,
|
|
province: null,
|
|
},
|
|
include: {
|
|
group: {
|
|
include: {
|
|
items: {
|
|
where: approvedChannelWhere ? { channel: approvedChannelWhere } : undefined,
|
|
include: {
|
|
channel: {
|
|
include: {
|
|
connectionStates: approvedChannelWhere
|
|
? { where: { status: 'connected', currentConnections: { gt: 0 } } }
|
|
: true,
|
|
},
|
|
},
|
|
},
|
|
orderBy: { priority: 'asc' },
|
|
},
|
|
},
|
|
},
|
|
},
|
|
orderBy: { priority: 'asc' },
|
|
});
|
|
if (!route) {
|
|
throw new NotFoundException('企业应用未配置对应运营商通道组');
|
|
}
|
|
if (route.group.status !== 'active') {
|
|
throw new BadRequestException('企业应用绑定的通道组已停用');
|
|
}
|
|
if (normalizeCarrier(route.group.carrier) !== carrier) {
|
|
throw new BadRequestException('企业应用绑定的通道组运营商与路由规则不一致');
|
|
}
|
|
return route;
|
|
}
|
|
|
|
async identifyCarrier(phoneNumber: string) {
|
|
return normalizeCarrier(await this.phoneRouting.identifyCarrier(phoneNumber));
|
|
}
|
|
|
|
async identifyProvince(phoneNumber: string) {
|
|
return this.phoneRouting.identifyProvince(phoneNumber);
|
|
}
|
|
|
|
async ensureSignatureReportedForChannel(
|
|
message: {
|
|
id: string;
|
|
templateId?: string | null;
|
|
template?: { signature?: { id?: string | null; name?: string | null } | null } | null;
|
|
signature?: { id?: string | null; name?: string | null } | null;
|
|
},
|
|
channelId: string,
|
|
carrier: string,
|
|
) {
|
|
const signatureId = await this.facade.resolveMessageSignatureId(message);
|
|
if (!signatureId) {
|
|
throw new BadRequestException('短信签名未配置，不能提交到通道');
|
|
}
|
|
const reportTask = await this.prisma.channelSignatureReportTask.findFirst({
|
|
where: {
|
|
signatureId,
|
|
channelId,
|
|
reportType: 'signature',
|
|
status: 'approved',
|
|
OR: signatureReportApprovalScopes(carrier),
|
|
},
|
|
select: { id: true },
|
|
});
|
|
if (!reportTask) {
|
|
throw new BadRequestException('短信签名未在最终通道报备通过');
|
|
}
|
|
}
|
|
|
|
async resolveMessageSignatureId(message: { templateId?: string | null; signatureId?: string | null; template?: { signature?: { id?: string | null } | null } | null; signature?: { id?: string | null } | null }) {
|
|
const direct = message.signatureId ?? message.template?.signature?.id ?? message.signature?.id ?? null;
|
|
if (direct || !message.templateId) return direct;
|
|
const template = await this.prisma.smsTemplate.findUnique({ where: { id: message.templateId }, include: { signature: true } });
|
|
return template?.signature?.id ?? null;
|
|
}
|
|
|
|
async waitForChannelRateLimit(channelId: string, tps: number) {
|
|
const redis = this.facade.getRedis();
|
|
for (;;) {
|
|
const bucket = `rate:channel:${channelId}:${Math.floor(Date.now() / 1000)}`;
|
|
const count = await redis.incr(bucket);
|
|
if (count === 1) {
|
|
await redis.expire(bucket, 2);
|
|
}
|
|
if (count <= Math.max(1, tps)) {
|
|
return;
|
|
}
|
|
await sleep(100);
|
|
}
|
|
}
|
|
|
|
async refreshTaskProgress(batchTaskId: string, knownSingleMessageStatus?: string) {
|
|
if (knownSingleMessageStatus) {
|
|
const direct = await this.prisma.smsBatchTask.updateMany({
|
|
where: { id: batchTaskId, sourceType: 'cmpp', phoneTotal: 1 },
|
|
data: singleMessageTaskProgress(knownSingleMessageStatus),
|
|
});
|
|
if (direct.count === 1) return;
|
|
} else {
|
|
// Gateway结果、回执和超时回调已经先提交了消息状态。对CMPP单号码内部任务，
|
|
// 在同一条UPDATE中读取该唯一消息的当前状态并写入精确计数，避免每次回调都
|
|
// 对一个只有一行的任务执行GROUP BY；多号码任务继续使用下方聚合路径。
|
|
const direct = await this.prisma.$executeRaw(Prisma.sql`
|
|
UPDATE "SmsBatchTask" AS task
|
|
SET
|
|
"progressTotal" = 1,
|
|
"submittedTotal" = CASE WHEN message.status IN ('submit_queued', 'submitted', 'delivered', 'failed', 'unknown', 'timeout') THEN 1 ELSE 0 END,
|
|
"successTotal" = CASE WHEN message.status = 'delivered' THEN 1 ELSE 0 END,
|
|
"failedTotal" = CASE WHEN message.status IN ('submit_failed', 'failed') THEN 1 ELSE 0 END,
|
|
"unknownTotal" = CASE WHEN message.status = 'unknown' THEN 1 ELSE 0 END,
|
|
"timeoutTotal" = CASE WHEN message.status = 'timeout' THEN 1 ELSE 0 END,
|
|
status = CASE
|
|
WHEN message.status IN ('delivered', 'submit_failed', 'failed', 'timeout') THEN 'finished'
|
|
WHEN message.status IN ('submit_queued', 'submitted', 'unknown') THEN 'sending'
|
|
ELSE 'queued'
|
|
END,
|
|
"updatedAt" = CURRENT_TIMESTAMP
|
|
FROM "SmsMessageRecord" AS message
|
|
WHERE task.id = ${batchTaskId}
|
|
AND task."sourceType" = 'cmpp'
|
|
AND task."phoneTotal" = 1
|
|
AND message."batchTaskId" = task.id
|
|
`);
|
|
if (direct === 1) return;
|
|
}
|
|
const running = this.taskProgressRefreshes.get(batchTaskId);
|
|
if (running) {
|
|
// A state transition committed after the running aggregate may not be visible
|
|
// to its snapshot. Mark one trailing pass instead of issuing another identical
|
|
// GROUP BY concurrently for every message/result/receipt callback.
|
|
this.dirtyTaskProgressRefreshes.add(batchTaskId);
|
|
return running;
|
|
}
|
|
const refresh = this.refreshTaskProgressUntilClean(batchTaskId);
|
|
this.taskProgressRefreshes.set(batchTaskId, refresh);
|
|
try {
|
|
await refresh;
|
|
} finally {
|
|
if (this.taskProgressRefreshes.get(batchTaskId) === refresh) {
|
|
this.taskProgressRefreshes.delete(batchTaskId);
|
|
}
|
|
this.dirtyTaskProgressRefreshes.delete(batchTaskId);
|
|
}
|
|
}
|
|
|
|
private async refreshTaskProgressUntilClean(batchTaskId: string) {
|
|
do {
|
|
this.dirtyTaskProgressRefreshes.delete(batchTaskId);
|
|
const groups = await this.prisma.smsMessageRecord.groupBy({
|
|
by: ['status'],
|
|
where: { batchTaskId },
|
|
_count: { _all: true },
|
|
});
|
|
const count = (statuses: string[]) =>
|
|
groups.filter((group) => statuses.includes(group.status)).reduce((sum, group) => sum + group._count._all, 0);
|
|
const progressTotal = groups.reduce((sum, group) => sum + group._count._all, 0);
|
|
const submittedTotal = count(['submit_queued', 'submitted', 'delivered', 'failed', 'unknown', 'timeout']);
|
|
const successTotal = count(['delivered']);
|
|
const failedTotal = count(['submit_failed', 'failed']);
|
|
const unknownTotal = count(['unknown']);
|
|
const timeoutTotal = count(['timeout']);
|
|
const doneTotal = successTotal + failedTotal + timeoutTotal;
|
|
const status = progressTotal > 0 && doneTotal >= progressTotal ? 'finished' : submittedTotal > 0 ? 'sending' : 'queued';
|
|
await this.prisma.smsBatchTask.update({
|
|
where: { id: batchTaskId },
|
|
data: { progressTotal, submittedTotal, successTotal, failedTotal, unknownTotal, timeoutTotal, status },
|
|
});
|
|
} while (this.dirtyTaskProgressRefreshes.has(batchTaskId));
|
|
}
|
|
|
|
getSendQueue(): Queue<SendJob, unknown, 'send-message'> {
|
|
if (!this.sendQueue) {
|
|
this.sendQueue = new Queue<SendJob, unknown, 'send-message'>(SEND_QUEUE, { connection: bullmqConnection() });
|
|
}
|
|
return this.sendQueue;
|
|
}
|
|
|
|
getGatewayQueue(): Queue {
|
|
if (!this.gatewayQueue) {
|
|
this.gatewayQueue = new Queue(GATEWAY_SUBMIT_QUEUE, { connection: bullmqConnection() });
|
|
}
|
|
return this.gatewayQueue;
|
|
}
|
|
|
|
getRedis() {
|
|
if (!this.redis) {
|
|
this.redis = new IORedis(process.env.REDIS_URL ?? 'redis://127.0.0.1:6379', {
|
|
maxRetriesPerRequest: null,
|
|
});
|
|
}
|
|
return this.redis;
|
|
}
|
|
|
|
async publishGatewaySubmitCommand(command: unknown, idempotencyKey?: string) {
|
|
const redis = this.facade.getRedis();
|
|
const stream = process.env.GATEWAY_SUBMIT_STREAM ?? GATEWAY_SUBMIT_STREAM;
|
|
const payload = JSON.stringify(command);
|
|
if (!idempotencyKey) {
|
|
return redis.xadd(stream, '*', 'messageType', 'SubmitCommand', 'data', payload);
|
|
}
|
|
const result = await redis.eval(
|
|
`local existing = redis.call('GET', KEYS[2])
|
|
if existing then return existing end
|
|
local streamId = redis.call('XADD', KEYS[1], '*', 'messageType', 'SubmitCommand', 'data', ARGV[1])
|
|
redis.call('SET', KEYS[2], streamId, 'EX', ARGV[2])
|
|
return streamId`,
|
|
2,
|
|
stream,
|
|
idempotencyKey,
|
|
payload,
|
|
String(GATEWAY_SUBMIT_REQUEUE_IDEMPOTENCY_TTL_SECONDS),
|
|
);
|
|
return typeof result === 'string' ? result : String(result ?? '');
|
|
}
|
|
|
|
private getOpenSubmitSessionId(channelId: string) {
|
|
const cached = this.openSubmitSessionIds.get(channelId);
|
|
if (cached) return cached;
|
|
const sessionNo = `OPEN-${channelId}`;
|
|
const pending = this.prisma.cmppSubmitSession.findUnique({
|
|
where: { sessionNo },
|
|
select: { id: true },
|
|
}).then((existing) => existing ?? this.prisma.cmppSubmitSession.upsert({
|
|
where: { sessionNo },
|
|
update: {},
|
|
create: { channelId, sessionNo, submitTotal: 0 },
|
|
select: { id: true },
|
|
})).then((session) => session.id).catch((error) => {
|
|
this.openSubmitSessionIds.delete(channelId);
|
|
throw error;
|
|
});
|
|
this.openSubmitSessionIds.set(channelId, pending);
|
|
return pending;
|
|
}
|
|
}
|
|
|
|
function singleMessageTaskProgress(status: string) {
|
|
const submittedTotal = ['submit_queued', 'submitted', 'delivered', 'failed', 'unknown', 'timeout'].includes(status) ? 1 : 0;
|
|
const successTotal = status === 'delivered' ? 1 : 0;
|
|
const failedTotal = ['submit_failed', 'failed'].includes(status) ? 1 : 0;
|
|
const unknownTotal = status === 'unknown' ? 1 : 0;
|
|
const timeoutTotal = status === 'timeout' ? 1 : 0;
|
|
const doneTotal = successTotal + failedTotal + timeoutTotal;
|
|
return {
|
|
progressTotal: 1,
|
|
submittedTotal,
|
|
successTotal,
|
|
failedTotal,
|
|
unknownTotal,
|
|
timeoutTotal,
|
|
status: doneTotal >= 1 ? 'finished' : submittedTotal > 0 ? 'sending' : 'queued',
|
|
};
|
|
}
|
|
|
|
function signatureReportApprovalScopes(carrier: string) {
|
|
const exact = { carrier, approvalScope: 'carrier_specific' };
|
|
// 迁移期保留双读用于平滑发布；自动转换migration完成且兼容命中清零后再切换严格口径。
|
|
return process.env.SIGNATURE_REPORT_STRICT_CARRIER === 'true'
|
|
? [exact]
|
|
: [exact, { carrier: null, approvalScope: 'legacy_channel' }];
|
|
}
|