perf: batch gateway submits and isolate callbacks
This commit is contained in:
hectorzhao
@@ -8,6 +8,7 @@ import { BillingService } from '../billing/billing.service';
|
||||
import { isIpAllowed } from '../common/ip-allowlist';
|
||||
import { moneyToNumber } from '../common/money';
|
||||
import { PhoneRoutingLookupService } from '../dictionaries/phone-routing-lookup.service';
|
||||
import { MetricsService, SendWorkerQueueState, SendWorkerStage } from '../metrics/metrics.service';
|
||||
import { PrismaService } from '../prisma/prisma.service';
|
||||
import { RiskReviewService } from '../risk-review/risk-review.service';
|
||||
import { PhoneFrequencyService } from '../risk-review/phone-frequency.service';
|
||||
@@ -15,6 +16,12 @@ import type { CreateBatchTaskDto, CreateHttpBatchTaskDto, GatewayInboundAuthDto,
|
||||
import { SEND_QUEUE, GATEWAY_SUBMIT_QUEUE, GATEWAY_SUBMIT_STREAM, DEFAULT_SCHEDULED_DISPATCH_STALE_MS, DEFAULT_INBOUND_LONG_MESSAGE_PROCESSING_STALE_SECONDS, GATEWAY_SUBMIT_REQUEUE_IDEMPOTENCY_TTL_SECONDS, BULLMQ_PRIORITY, drainageRejectionReason, statusFromRisk, parseSchedule, parseImportRows, splitImportLine, cellByHeader, normalizeCarrier, normalizeQueuePriority, getPositiveConfigInteger, getNonNegativeConfigInteger, isCarrierCompatible, matchTemplateContent, isNationalChannel, validateInboundApplicationSrcId, composeUpstreamSrcId, positiveInteger, parseOptionalSequenceId, shanghaiDateKey, bullmqConnection, matchesApplicationSecret, octetString, selectChannelCandidate } from './send-chain.helpers';
|
||||
import type { SendSubmissionCallbacks, SendSubmissionService } from './send-submission.service';
|
||||
|
||||
type PendingSendBatchItem = {
|
||||
job: SendJob;
|
||||
resolve: (value: unknown) => void;
|
||||
reject: (reason: unknown) => void;
|
||||
};
|
||||
|
||||
/**
|
||||
* R9 gatewaySubmit implementation. Cross-method calls return through the stable SendChainService seam.
|
||||
*/
|
||||
@@ -24,8 +31,18 @@ export class SendGatewaySubmitService {
|
||||
private sendQueue?: Queue<SendJob, unknown, 'send-message'>;
|
||||
private gatewayQueue?: Queue;
|
||||
private worker?: Worker<SendJob>;
|
||||
private sendQueueMetricsTimer?: ReturnType<typeof setInterval>;
|
||||
private submitOutboxTimer?: ReturnType<typeof setInterval>;
|
||||
private submitOutboxRunning = false;
|
||||
private readonly submitOutboxLeaseOwner = `send-worker-${process.pid}-${randomUUID()}`;
|
||||
private sendWorkerInFlight = 0;
|
||||
private sendWorkerConfiguredSlots = 0;
|
||||
private pendingSendBatch: PendingSendBatchItem[] = [];
|
||||
private sendBatchTimer?: ReturnType<typeof setTimeout>;
|
||||
private sendBatchFlushing = false;
|
||||
private readonly taskProgressRefreshes = new Map<string, Promise<void>>();
|
||||
private readonly dirtyTaskProgressRefreshes = new Set<string>();
|
||||
private readonly openSubmitSessionIds = new Map<string, Promise<string>>();
|
||||
|
||||
constructor(
|
||||
private readonly prisma: PrismaService,
|
||||
@@ -35,9 +52,14 @@ export class SendGatewaySubmitService {
|
||||
private readonly phoneRouting: PhoneRoutingLookupService,
|
||||
private readonly facade: SendSubmissionService,
|
||||
private readonly callbacks: SendSubmissionCallbacks,
|
||||
private readonly metrics?: MetricsService,
|
||||
) {}
|
||||
|
||||
async onModuleDestroy() {
|
||||
if (this.sendQueueMetricsTimer) clearInterval(this.sendQueueMetricsTimer);
|
||||
if (this.submitOutboxTimer) clearInterval(this.submitOutboxTimer);
|
||||
if (this.sendBatchTimer) clearTimeout(this.sendBatchTimer);
|
||||
await this.flushSendBatch();
|
||||
await this.worker?.close();
|
||||
await this.sendQueue?.close();
|
||||
await this.gatewayQueue?.close();
|
||||
@@ -112,46 +134,390 @@ startWorker() {
|
||||
return { status: 'already_started' };
|
||||
}
|
||||
const connection = bullmqConnection();
|
||||
const configuredConcurrency = Number(process.env.API_SEND_WORKER_CONCURRENCY ?? 20);
|
||||
this.sendWorkerConfiguredSlots = Number.isInteger(configuredConcurrency) && configuredConcurrency > 0 ? configuredConcurrency : 20;
|
||||
this.metrics?.setSendWorkerSlots(this.sendWorkerConfiguredSlots, this.sendWorkerInFlight);
|
||||
this.worker = new Worker<SendJob>(
|
||||
SEND_QUEUE,
|
||||
async (job) => this.facade.processSendJob(job.data),
|
||||
{ connection, concurrency: Number(process.env.API_SEND_WORKER_CONCURRENCY ?? 20) },
|
||||
async (job) => {
|
||||
this.sendWorkerInFlight += 1;
|
||||
this.metrics?.setSendWorkerSlots(this.sendWorkerConfiguredSlots, this.sendWorkerInFlight);
|
||||
try {
|
||||
return process.env.API_SEND_WORKER_BATCH_ENABLED === 'false'
|
||||
? await this.facade.processSendJob(job.data)
|
||||
: await this.enqueueSendBatch(job.data);
|
||||
} finally {
|
||||
this.sendWorkerInFlight = Math.max(0, this.sendWorkerInFlight - 1);
|
||||
this.metrics?.setSendWorkerSlots(this.sendWorkerConfiguredSlots, this.sendWorkerInFlight);
|
||||
}
|
||||
},
|
||||
{ connection, concurrency: this.sendWorkerConfiguredSlots },
|
||||
);
|
||||
void this.refreshSendQueueMetrics();
|
||||
this.sendQueueMetricsTimer = setInterval(() => void this.refreshSendQueueMetrics(), 5_000);
|
||||
this.sendQueueMetricsTimer.unref?.();
|
||||
if (this.submitOutboxEnabled() && process.env.SEND_SUBMIT_OUTBOX_SEPARATE_PROCESS_ENABLED !== 'true') {
|
||||
void this.publishSubmitOutboxBatch();
|
||||
this.submitOutboxTimer = setInterval(
|
||||
() => void this.publishSubmitOutboxBatch(),
|
||||
getPositiveConfigInteger(process.env, 'SEND_SUBMIT_OUTBOX_POLL_INTERVAL_MS', 25),
|
||||
);
|
||||
this.submitOutboxTimer.unref?.();
|
||||
}
|
||||
return { status: 'started' };
|
||||
}
|
||||
|
||||
async processSendJob(job: SendJob) {
|
||||
const message = await this.prisma.smsMessageRecord.findUnique({
|
||||
where: { id: job.messageRecordId },
|
||||
include: { batchTask: true, template: { include: { signature: true } }, signature: true },
|
||||
});
|
||||
if (!message || message.status !== 'queued') {
|
||||
return { skipped: true };
|
||||
}
|
||||
if (!message.tenantId || !message.batchTaskId) {
|
||||
return { skipped: true, reason: 'standalone channel test message' };
|
||||
}
|
||||
const businessMessage = message as typeof message & { tenantId: string; batchTaskId: string };
|
||||
try {
|
||||
const routed = await this.facade.selectChannelForMessage(businessMessage);
|
||||
return await this.facade.submitMessageToGateway(businessMessage, routed, 0);
|
||||
} catch (error) {
|
||||
const reason = error instanceof Error ? error.message : '无可用通道组或通道';
|
||||
await this.prisma.smsMessageRecord.update({
|
||||
where: { id: message.id },
|
||||
data: { status: 'failed', errorMessage: reason },
|
||||
});
|
||||
await this.releaseMessageReservation(businessMessage, reason);
|
||||
if (message.batchTask?.sourceType === 'cmpp') {
|
||||
await this.recordCmppFailureReceipt(businessMessage, 'ROUTE', reason);
|
||||
} else {
|
||||
await this.facade.refreshTaskProgress(businessMessage.batchTaskId);
|
||||
startSubmitOutboxPublisher() {
|
||||
if (this.submitOutboxTimer) return { status: 'already_started' };
|
||||
if (!this.submitOutboxEnabled()) return { status: 'disabled' };
|
||||
void this.publishSubmitOutboxBatch();
|
||||
this.submitOutboxTimer = setInterval(
|
||||
() => void this.publishSubmitOutboxBatch(),
|
||||
getPositiveConfigInteger(process.env, 'SEND_SUBMIT_OUTBOX_POLL_INTERVAL_MS', 25),
|
||||
);
|
||||
this.submitOutboxTimer.unref?.();
|
||||
return { status: 'started' };
|
||||
}
|
||||
|
||||
private enqueueSendBatch(job: SendJob) {
|
||||
return new Promise((resolve, reject) => {
|
||||
this.pendingSendBatch.push({ job, resolve, reject });
|
||||
const batchSize = Math.min(128, getPositiveConfigInteger(process.env, 'API_SEND_WORKER_BATCH_SIZE', 32));
|
||||
if (this.pendingSendBatch.length >= batchSize) {
|
||||
if (this.sendBatchTimer) clearTimeout(this.sendBatchTimer);
|
||||
this.sendBatchTimer = undefined;
|
||||
queueMicrotask(() => void this.flushSendBatch());
|
||||
return;
|
||||
}
|
||||
return { submitted: false, messageRecordId: message.id, status: 'failed', reason };
|
||||
if (!this.sendBatchTimer) {
|
||||
this.sendBatchTimer = setTimeout(
|
||||
() => {
|
||||
this.sendBatchTimer = undefined;
|
||||
void this.flushSendBatch();
|
||||
},
|
||||
Math.min(25, getPositiveConfigInteger(process.env, 'API_SEND_WORKER_BATCH_WAIT_MS', 3)),
|
||||
);
|
||||
this.sendBatchTimer.unref?.();
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
private async flushSendBatch() {
|
||||
if (this.sendBatchFlushing) return;
|
||||
this.sendBatchFlushing = true;
|
||||
try {
|
||||
const batchSize = Math.min(128, getPositiveConfigInteger(process.env, 'API_SEND_WORKER_BATCH_SIZE', 32));
|
||||
while (this.pendingSendBatch.length > 0) {
|
||||
const batch = this.pendingSendBatch.splice(0, batchSize);
|
||||
try {
|
||||
const results = await this.processSendJobBatch(batch.map((item) => item.job));
|
||||
for (const item of batch) item.resolve(results.get(item.job.messageRecordId) ?? { skipped: true });
|
||||
} catch (error) {
|
||||
for (const item of batch) item.reject(error);
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
this.sendBatchFlushing = false;
|
||||
if (this.pendingSendBatch.length > 0) queueMicrotask(() => void this.flushSendBatch());
|
||||
}
|
||||
}
|
||||
|
||||
async submitMessageToGateway(
|
||||
private async processSendJobBatch(jobs: SendJob[]) {
|
||||
if (jobs.length === 1) {
|
||||
return new Map([[jobs[0].messageRecordId, await this.processSendJob(jobs[0])]]);
|
||||
}
|
||||
const ids = [...new Set(jobs.map((job) => job.messageRecordId))];
|
||||
const messages = await this.measureSendStage('message_load', () => this.prisma.smsMessageRecord.findMany({
|
||||
where: { id: { in: ids } },
|
||||
include: { batchTask: true, template: { include: { signature: true } }, signature: true },
|
||||
}));
|
||||
const messageById = new Map(messages.map((message) => [message.id, message]));
|
||||
const results = new Map<string, unknown>();
|
||||
const businessMessages = messages.filter((message) => {
|
||||
if (message.status !== 'queued' || !message.tenantId || !message.batchTaskId) {
|
||||
results.set(message.id, { skipped: true });
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}) as Array<typeof messages[number] & { tenantId: string; batchTaskId: string }>;
|
||||
for (const id of ids) if (!messageById.has(id)) results.set(id, { skipped: true });
|
||||
if (businessMessages.length === 0) return results;
|
||||
|
||||
const { planned, failed } = await this.planRoutesBatch(businessMessages);
|
||||
if (failed.length > 0) await this.failRouteBatch(failed, results);
|
||||
if (planned.length === 0) return results;
|
||||
|
||||
await Promise.all(planned.map(({ routed }) => (
|
||||
this.facade.waitForChannelRateLimit(routed.channel.id, routed.channel.rateLimitPerSecond)
|
||||
)));
|
||||
const sessionByChannel = new Map<string, string>();
|
||||
await Promise.all([...new Set(planned.map(({ routed }) => routed.channel.id))].map(async (channelId) => {
|
||||
sessionByChannel.set(channelId, await this.getOpenSubmitSessionId(channelId));
|
||||
}));
|
||||
const prepared = planned.map(({ message, routed }) => {
|
||||
const submitId = `SUB-${randomUUID()}`;
|
||||
const upstreamSrcId = composeUpstreamSrcId(routed.channel.srcId, message.applicationExtension);
|
||||
return {
|
||||
message,
|
||||
routed,
|
||||
submitId,
|
||||
command: this.buildGatewaySubmitCommand(message, routed, 0, submitId, upstreamSrcId),
|
||||
sessionId: sessionByChannel.get(routed.channel.id),
|
||||
};
|
||||
});
|
||||
const writeOutbox = this.submitOutboxEnabled();
|
||||
await this.measureSendStage('submit_transaction', () => this.prisma.$transaction(async (tx) => {
|
||||
await tx.smsSubmitRecord.createMany({
|
||||
data: prepared.map(({ message, routed, submitId, sessionId }) => ({
|
||||
id: randomUUID(),
|
||||
tenantId: message.tenantId,
|
||||
batchTaskId: message.batchTaskId,
|
||||
messageRecordId: message.id,
|
||||
channelId: routed.channel.id,
|
||||
channelGroupId: routed.groupId,
|
||||
channelGroupName: routed.groupName,
|
||||
sessionId,
|
||||
submitId,
|
||||
submitStatus: 'queued',
|
||||
costUnitPrice: routed.channel.unitPrice ?? 0,
|
||||
costAmountCents: moneyToNumber(routed.channel.unitPrice) * Math.max(1, message.billingUnits ?? 1),
|
||||
})),
|
||||
});
|
||||
const updates = Prisma.join(prepared.map(({ message, routed, submitId }) => Prisma.sql`(
|
||||
${message.id}::text, ${routed.channel.id}::text, ${routed.carrier}::text,
|
||||
${routed.province ?? null}::text, ${submitId}::text
|
||||
)`));
|
||||
await tx.$executeRaw(Prisma.sql`
|
||||
UPDATE "SmsMessageRecord" AS message
|
||||
SET "channelId" = updates."channelId",
|
||||
carrier = updates.carrier,
|
||||
province = updates.province,
|
||||
"submitId" = updates."submitId",
|
||||
status = 'submit_queued',
|
||||
"submitStatus" = 'queued',
|
||||
"receiptStatus" = NULL,
|
||||
"errorCode" = NULL,
|
||||
"errorMessage" = NULL,
|
||||
"updatedAt" = CURRENT_TIMESTAMP
|
||||
FROM (VALUES ${updates}) AS updates(id, "channelId", carrier, province, "submitId")
|
||||
WHERE message.id = updates.id AND message.status = 'queued'
|
||||
`);
|
||||
if (writeOutbox) {
|
||||
await tx.gatewaySubmitOutbox.createMany({
|
||||
data: prepared.map(({ message, routed, submitId, command }) => ({
|
||||
id: randomUUID(), submitId, messageRecordId: message.id,
|
||||
channelId: routed.channel.id, payload: command as Prisma.InputJsonValue,
|
||||
})),
|
||||
});
|
||||
}
|
||||
}));
|
||||
if (!this.submitOutboxPublishEnabled()) {
|
||||
await Promise.all(prepared.map(({ command }) => this.facade.publishGatewaySubmitCommand(command)));
|
||||
}
|
||||
await this.refreshTaskProgressBatch(prepared.map(({ message }) => message));
|
||||
for (const { message, routed, submitId } of prepared) {
|
||||
results.set(message.id, {
|
||||
submitted: true, messageRecordId: message.id, channelId: routed.channel.id, attempt: 0, submitId,
|
||||
});
|
||||
this.metrics?.recordSendWorkerResult('completed');
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
private async planRoutesBatch<T extends {
|
||||
id: string; tenantId: string; batchTaskId: string; applicationId?: string | null;
|
||||
templateId?: string | null; signatureId?: string | null; phoneNumber: string;
|
||||
carrier?: string | null; province?: string | null;
|
||||
template?: { signature?: { id?: string | null } | null } | null;
|
||||
signature?: { id?: string | null } | null;
|
||||
}>(messages: T[]) {
|
||||
const unresolvedPhones = messages.filter((message) => !message.carrier).map((message) => message.phoneNumber);
|
||||
const provinces = await this.measureSendStage('phone_routing', () => this.phoneRouting.identifyProvinces(unresolvedPhones));
|
||||
const routeInputs = await Promise.all(messages.map(async (message) => ({
|
||||
message,
|
||||
carrier: message.carrier ? normalizeCarrier(message.carrier) : normalizeCarrier(await this.phoneRouting.identifyCarrier(message.phoneNumber)),
|
||||
province: message.carrier ? message.province ?? null : provinces.get(message.phoneNumber) ?? null,
|
||||
signatureId: message.signatureId ?? message.template?.signature?.id ?? message.signature?.id ?? null,
|
||||
})));
|
||||
const valid = routeInputs.filter((input) => input.message.applicationId && input.signatureId);
|
||||
const signatures = [...new Set(valid.map((input) => input.signatureId as string))];
|
||||
const routes = valid.length === 0 ? [] : await this.measureSendStage('route_lookup', () => this.prisma.channelRouteRule.findMany({
|
||||
where: {
|
||||
status: 'active', channelId: null, province: null,
|
||||
OR: valid.map((input) => ({
|
||||
tenantId: input.message.tenantId,
|
||||
applicationId: input.message.applicationId,
|
||||
carrier: input.carrier,
|
||||
})),
|
||||
},
|
||||
include: {
|
||||
group: {
|
||||
include: {
|
||||
items: {
|
||||
include: {
|
||||
channel: {
|
||||
include: {
|
||||
connectionStates: { where: { status: 'connected', currentConnections: { gt: 0 } } },
|
||||
reportTasks: { where: { signatureId: { in: signatures }, reportType: 'signature', status: 'approved' } },
|
||||
},
|
||||
},
|
||||
},
|
||||
orderBy: { priority: 'asc' },
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
orderBy: { priority: 'asc' },
|
||||
}));
|
||||
const routeByKey = new Map<string, typeof routes[number]>();
|
||||
for (const route of routes) {
|
||||
const key = `${route.tenantId}:${route.applicationId}:${normalizeCarrier(route.carrier)}`;
|
||||
if (!routeByKey.has(key)) routeByKey.set(key, route);
|
||||
}
|
||||
const planned: Array<{ message: T; routed: RoutedChannel }> = [];
|
||||
const failed: Array<{ message: T; reason: string }> = [];
|
||||
for (const input of routeInputs) {
|
||||
if (!input.message.applicationId) {
|
||||
failed.push({ message: input.message, reason: '短信应用未配置，无法选择通道组' });
|
||||
continue;
|
||||
}
|
||||
if (!input.signatureId) {
|
||||
failed.push({ message: input.message, reason: '短信签名未配置，无法选择已报备通道' });
|
||||
continue;
|
||||
}
|
||||
const route = routeByKey.get(`${input.message.tenantId}:${input.message.applicationId}:${input.carrier}`);
|
||||
if (!route) {
|
||||
failed.push({ message: input.message, reason: '企业应用未配置对应运营商通道组' });
|
||||
continue;
|
||||
}
|
||||
if (route.group.status !== 'active' || normalizeCarrier(route.group.carrier) !== input.carrier) {
|
||||
failed.push({ message: input.message, reason: '企业应用绑定的通道组已停用或运营商不一致' });
|
||||
continue;
|
||||
}
|
||||
const approvedItems = route.group.items.filter((item) => item.channel.status === 'active'
|
||||
&& item.channel.connectionStates.length > 0
|
||||
&& item.channel.reportTasks.some((task) => task.signatureId === input.signatureId
|
||||
&& (task.carrier === input.carrier
|
||||
|| (process.env.SIGNATURE_REPORT_STRICT_CARRIER !== 'true' && task.approvalScope === 'legacy_channel'))));
|
||||
const selected = selectChannelCandidate(approvedItems, {
|
||||
carrier: input.carrier,
|
||||
province: input.province,
|
||||
excludedChannelIds: new Set(),
|
||||
approvedChannelIds: new Set(approvedItems.map((item) => item.channelId)),
|
||||
routingKey: input.message.id,
|
||||
});
|
||||
if (!selected) {
|
||||
failed.push({ message: input.message, reason: '无已报备通过且在线的可用通道' });
|
||||
continue;
|
||||
}
|
||||
planned.push({
|
||||
message: input.message,
|
||||
routed: {
|
||||
channel: { ...selected.channel, unitPrice: moneyToNumber(selected.channel.unitPrice) },
|
||||
carrier: input.carrier, province: input.province,
|
||||
groupId: route.groupId, groupName: route.group.name,
|
||||
routeScope: isNationalChannel(selected) ? 'national' : 'province',
|
||||
},
|
||||
});
|
||||
}
|
||||
return { planned, failed };
|
||||
}
|
||||
|
||||
private async failRouteBatch<T extends {
|
||||
id: string; tenantId: string; batchTaskId: string; applicationId?: string | null;
|
||||
messageId: string; phoneNumber: string; amountCents: bigint; billingUnits: number;
|
||||
cmppSubmitSequenceId?: string | null; cmppSubmitGroupMessageId?: string | null;
|
||||
batchTask?: { sourceType?: string | null; phoneTotal?: number | null } | null;
|
||||
}>(failed: Array<{ message: T; reason: string }>, results: Map<string, unknown>) {
|
||||
const values = Prisma.join(failed.map(({ message, reason }) => Prisma.sql`(${message.id}::text, ${reason.slice(0, 1000)}::text)`));
|
||||
await this.prisma.$executeRaw(Prisma.sql`
|
||||
UPDATE "SmsMessageRecord" AS message
|
||||
SET status = 'failed', "errorMessage" = failures.reason, "updatedAt" = CURRENT_TIMESTAMP
|
||||
FROM (VALUES ${values}) AS failures(id, reason)
|
||||
WHERE message.id = failures.id AND message.status = 'queued'
|
||||
`);
|
||||
await Promise.all(failed.map(async ({ message, reason }) => {
|
||||
await this.releaseMessageReservation(message, reason);
|
||||
if (message.batchTask?.sourceType === 'cmpp') await this.recordCmppFailureReceipt(message, 'ROUTE', reason);
|
||||
else await this.facade.refreshTaskProgress(message.batchTaskId);
|
||||
results.set(message.id, { submitted: false, messageRecordId: message.id, status: 'failed', reason });
|
||||
this.metrics?.recordSendWorkerResult('failed');
|
||||
}));
|
||||
}
|
||||
|
||||
private async refreshTaskProgressBatch(messages: Array<{
|
||||
batchTaskId: string; batchTask?: { sourceType?: string | null; phoneTotal?: number | null } | null;
|
||||
}>) {
|
||||
const singleCmppIds = [...new Set(messages
|
||||
.filter((message) => message.batchTask?.sourceType === 'cmpp' && message.batchTask.phoneTotal === 1)
|
||||
.map((message) => message.batchTaskId))];
|
||||
if (singleCmppIds.length > 0) {
|
||||
await this.prisma.smsBatchTask.updateMany({
|
||||
where: { id: { in: singleCmppIds }, sourceType: 'cmpp', phoneTotal: 1 },
|
||||
data: singleMessageTaskProgress('submit_queued'),
|
||||
});
|
||||
}
|
||||
const otherTaskIds = [...new Set(messages
|
||||
.filter((message) => !singleCmppIds.includes(message.batchTaskId))
|
||||
.map((message) => message.batchTaskId))];
|
||||
await Promise.all(otherTaskIds.map((taskId) => this.facade.refreshTaskProgress(taskId)));
|
||||
}
|
||||
|
||||
async processSendJob(job: SendJob) {
|
||||
const totalStartedAt = this.metrics?.beginSendWorkerStage();
|
||||
let totalFinished = false;
|
||||
const finish = (result: 'completed' | 'failed' | 'skipped') => {
|
||||
if (totalFinished) return;
|
||||
totalFinished = true;
|
||||
if (totalStartedAt != null) {
|
||||
this.metrics?.finishSendWorkerStage(totalStartedAt, 'total', result === 'completed' ? 'success' : result === 'skipped' ? 'skipped' : 'error');
|
||||
}
|
||||
this.metrics?.recordSendWorkerResult(result);
|
||||
};
|
||||
try {
|
||||
const message = await this.measureSendStage('message_load', () => this.prisma.smsMessageRecord.findUnique({
|
||||
where: { id: job.messageRecordId },
|
||||
include: { batchTask: true, template: { include: { signature: true } }, signature: true },
|
||||
}));
|
||||
if (!message || message.status !== 'queued') {
|
||||
finish('skipped');
|
||||
return { skipped: true };
|
||||
}
|
||||
if (!message.tenantId || !message.batchTaskId) {
|
||||
finish('skipped');
|
||||
return { skipped: true, reason: 'standalone channel test message' };
|
||||
}
|
||||
const businessMessage = message as typeof message & { tenantId: string; batchTaskId: string };
|
||||
try {
|
||||
const routed = await this.facade.selectChannelForMessage(businessMessage);
|
||||
const result = await this.facade.submitMessageToGateway(businessMessage, routed, 0);
|
||||
finish(result.submitted ? 'completed' : 'skipped');
|
||||
return result;
|
||||
} catch (error) {
|
||||
const reason = error instanceof Error ? error.message : '无可用通道组或通道';
|
||||
await this.prisma.smsMessageRecord.update({
|
||||
where: { id: message.id },
|
||||
data: { status: 'failed', errorMessage: reason },
|
||||
});
|
||||
await this.releaseMessageReservation(businessMessage, reason);
|
||||
if (message.batchTask?.sourceType === 'cmpp') {
|
||||
await this.recordCmppFailureReceipt(businessMessage, 'ROUTE', reason);
|
||||
} else {
|
||||
await this.facade.refreshTaskProgress(
|
||||
businessMessage.batchTaskId,
|
||||
message.batchTask?.sourceType === 'cmpp' && message.batchTask.phoneTotal === 1 ? 'failed' : undefined,
|
||||
);
|
||||
}
|
||||
finish('failed');
|
||||
return { submitted: false, messageRecordId: message.id, status: 'failed', reason };
|
||||
}
|
||||
} catch (error) {
|
||||
finish('failed');
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async submitMessageToGateway(
|
||||
message: {
|
||||
id: string;
|
||||
tenantId: string;
|
||||
@@ -169,6 +535,7 @@ async submitMessageToGateway(
|
||||
applicationExtension?: string | null;
|
||||
template?: { signature?: { id?: string | null; name?: string | null } | null } | null;
|
||||
signature?: { id?: string | null; name?: string | null } | null;
|
||||
batchTask?: { sourceType?: string | null; phoneTotal?: number | null } | null;
|
||||
},
|
||||
routed: RoutedChannel,
|
||||
attempt: number,
|
||||
@@ -176,16 +543,13 @@ async submitMessageToGateway(
|
||||
) {
|
||||
const channel = routed.channel;
|
||||
const upstreamSrcId = composeUpstreamSrcId(channel.srcId, message.applicationExtension);
|
||||
await this.facade.ensureSignatureReportedForChannel(message, channel.id, routed.carrier);
|
||||
await this.facade.waitForChannelRateLimit(channel.id, channel.rateLimitPerSecond);
|
||||
await this.measureSendStage('rate_limit', () => this.facade.waitForChannelRateLimit(channel.id, channel.rateLimitPerSecond));
|
||||
const submitId = `SUB-${randomUUID()}`;
|
||||
const sessionId = await this.getOpenSubmitSessionId(channel.id);
|
||||
const command = this.buildGatewaySubmitCommand(message, routed, attempt, submitId, upstreamSrcId);
|
||||
const writeOutbox = this.submitOutboxEnabled();
|
||||
try {
|
||||
await this.prisma.$transaction(async (tx) => {
|
||||
const session = await tx.cmppSubmitSession.upsert({
|
||||
where: { sessionNo: `OPEN-${channel.id}` },
|
||||
update: { submitTotal: { increment: 1 } },
|
||||
create: { channelId: channel.id, sessionNo: `OPEN-${channel.id}`, submitTotal: 1 },
|
||||
});
|
||||
await this.measureSendStage('submit_transaction', () => this.prisma.$transaction(async (tx) => {
|
||||
await tx.smsSubmitRecord.create({
|
||||
data: {
|
||||
tenantId: message.tenantId,
|
||||
@@ -194,7 +558,7 @@ async submitMessageToGateway(
|
||||
channelId: channel.id,
|
||||
channelGroupId: routed.groupId,
|
||||
channelGroupName: routed.groupName,
|
||||
sessionId: session.id,
|
||||
sessionId,
|
||||
retryOfSubmitRecordId,
|
||||
submitId,
|
||||
submitStatus: 'queued',
|
||||
@@ -216,7 +580,17 @@ async submitMessageToGateway(
|
||||
errorMessage: attempt > 0 ? `第 ${attempt + 1} 次提交，路由至${routed.routeScope === 'national' ? '全国' : '省网'}通道` : undefined,
|
||||
},
|
||||
});
|
||||
});
|
||||
if (writeOutbox) {
|
||||
await tx.gatewaySubmitOutbox.create({
|
||||
data: {
|
||||
submitId,
|
||||
messageRecordId: message.id,
|
||||
channelId: channel.id,
|
||||
payload: command as Prisma.InputJsonValue,
|
||||
},
|
||||
});
|
||||
}
|
||||
}));
|
||||
if (retryOfSubmitRecordId) {
|
||||
this.logger.log(`sms_retry_claim_acquired ${JSON.stringify({
|
||||
messageId: message.messageId,
|
||||
@@ -255,7 +629,31 @@ async submitMessageToGateway(
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
const command = {
|
||||
if (!this.submitOutboxPublishEnabled()) {
|
||||
await this.measureSendStage('gateway_stream_publish', () => this.facade.publishGatewaySubmitCommand(command));
|
||||
}
|
||||
await this.measureSendStage('task_progress', () => this.facade.refreshTaskProgress(
|
||||
message.batchTaskId,
|
||||
message.batchTask?.sourceType === 'cmpp' && message.batchTask.phoneTotal === 1 ? 'submit_queued' : undefined,
|
||||
));
|
||||
return { submitted: true, messageRecordId: message.id, channelId: channel.id, attempt };
|
||||
}
|
||||
|
||||
private buildGatewaySubmitCommand(
|
||||
message: {
|
||||
id: string; tenantId: string; batchTaskId: string; applicationId?: string | null;
|
||||
templateId?: string | null; messageId: string; phoneNumber: string; content: string;
|
||||
billingUnits: number; queuePriority?: string | null; applicationExtension?: string | null;
|
||||
template?: { signature?: { name?: string | null } | null } | null;
|
||||
signature?: { name?: string | null } | null;
|
||||
},
|
||||
routed: RoutedChannel,
|
||||
attempt: number,
|
||||
submitId: string,
|
||||
upstreamSrcId: string,
|
||||
) {
|
||||
const channel = routed.channel;
|
||||
return {
|
||||
schemaVersion: 'v1',
|
||||
messageType: 'SubmitCommand',
|
||||
traceId: randomUUID(),
|
||||
@@ -304,10 +702,124 @@ async submitMessageToGateway(
|
||||
},
|
||||
retry: { attempt, maxAttempts: 1 },
|
||||
};
|
||||
await this.facade.getGatewayQueue().add('submit-command', command);
|
||||
await this.facade.publishGatewaySubmitCommand(command);
|
||||
await this.facade.refreshTaskProgress(message.batchTaskId);
|
||||
return { submitted: true, messageRecordId: message.id, channelId: channel.id, attempt };
|
||||
}
|
||||
|
||||
private submitOutboxEnabled() {
|
||||
return process.env.SEND_SUBMIT_OUTBOX_SHADOW_ENABLED === 'true'
|
||||
|| process.env.SEND_SUBMIT_OUTBOX_PUBLISH_ENABLED === 'true';
|
||||
}
|
||||
|
||||
private submitOutboxPublishEnabled() {
|
||||
return process.env.SEND_SUBMIT_OUTBOX_PUBLISH_ENABLED === 'true';
|
||||
}
|
||||
|
||||
private async publishSubmitOutboxBatch() {
|
||||
if (this.submitOutboxRunning || !this.submitOutboxEnabled()) return;
|
||||
this.submitOutboxRunning = true;
|
||||
try {
|
||||
const batchSize = Math.min(500, getPositiveConfigInteger(process.env, 'SEND_SUBMIT_OUTBOX_BATCH_SIZE', 64));
|
||||
const leaseSeconds = Math.min(300, getPositiveConfigInteger(process.env, 'SEND_SUBMIT_OUTBOX_LEASE_SECONDS', 30));
|
||||
const rows = await this.prisma.$queryRaw<Array<{ id: string; submitId: string; payload: Prisma.JsonValue }>>(Prisma.sql`
|
||||
WITH candidates AS (
|
||||
SELECT id
|
||||
FROM "GatewaySubmitOutbox"
|
||||
WHERE (
|
||||
(status = 'pending' AND "nextAttemptAt" <= CURRENT_TIMESTAMP)
|
||||
OR (status = 'publishing' AND "leaseExpiresAt" < CURRENT_TIMESTAMP)
|
||||
)
|
||||
ORDER BY "createdAt", id
|
||||
LIMIT ${batchSize}
|
||||
FOR UPDATE SKIP LOCKED
|
||||
)
|
||||
UPDATE "GatewaySubmitOutbox" AS outbox
|
||||
SET status = 'publishing',
|
||||
"leaseOwner" = ${this.submitOutboxLeaseOwner},
|
||||
"leaseExpiresAt" = CURRENT_TIMESTAMP + (${leaseSeconds} * INTERVAL '1 second'),
|
||||
"attemptCount" = outbox."attemptCount" + 1,
|
||||
"updatedAt" = CURRENT_TIMESTAMP
|
||||
FROM candidates
|
||||
WHERE outbox.id = candidates.id
|
||||
RETURNING outbox.id, outbox."submitId", outbox.payload
|
||||
`);
|
||||
if (rows.length === 0) return;
|
||||
const results = this.submitOutboxPublishEnabled()
|
||||
? await this.publishGatewaySubmitCommandBatch(rows)
|
||||
: rows.map((row) => ({ row, streamEntryId: `shadow:${row.submitId}` }));
|
||||
const succeeded = results.filter((result): result is { row: typeof rows[number]; streamEntryId: string } => 'streamEntryId' in result);
|
||||
if (succeeded.length > 0) {
|
||||
const values = Prisma.join(succeeded.map(({ row, streamEntryId }) => Prisma.sql`(${row.id}, ${streamEntryId})`));
|
||||
await this.prisma.$executeRaw(Prisma.sql`
|
||||
UPDATE "GatewaySubmitOutbox" AS outbox
|
||||
SET status = 'published',
|
||||
"streamEntryId" = published."streamEntryId",
|
||||
"publishedAt" = CURRENT_TIMESTAMP,
|
||||
"leaseOwner" = NULL,
|
||||
"leaseExpiresAt" = NULL,
|
||||
"lastError" = NULL,
|
||||
"updatedAt" = CURRENT_TIMESTAMP
|
||||
FROM (VALUES ${values}) AS published(id, "streamEntryId")
|
||||
WHERE outbox.id = published.id
|
||||
AND outbox.status = 'publishing'
|
||||
AND outbox."leaseOwner" = ${this.submitOutboxLeaseOwner}
|
||||
`);
|
||||
}
|
||||
for (const result of results) {
|
||||
if ('streamEntryId' in result) continue;
|
||||
const { row, error } = result;
|
||||
try {
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
await this.prisma.$executeRaw(Prisma.sql`
|
||||
UPDATE "GatewaySubmitOutbox"
|
||||
SET status = CASE WHEN "attemptCount" >= 10 THEN 'dead' ELSE 'pending' END,
|
||||
"nextAttemptAt" = CURRENT_TIMESTAMP + (LEAST(60, POWER(2, LEAST("attemptCount", 6))) * INTERVAL '1 second'),
|
||||
"leaseOwner" = NULL,
|
||||
"leaseExpiresAt" = NULL,
|
||||
"lastError" = ${message.slice(0, 1000)},
|
||||
"updatedAt" = CURRENT_TIMESTAMP
|
||||
WHERE id = ${row.id} AND "leaseOwner" = ${this.submitOutboxLeaseOwner}
|
||||
`);
|
||||
} catch (recordError) {
|
||||
this.logger.error(`gateway_submit_outbox_failure_record_failed ${recordError instanceof Error ? recordError.message : String(recordError)}`);
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
this.logger.error(`gateway_submit_outbox_publish_failed ${error instanceof Error ? error.message : String(error)}`);
|
||||
} finally {
|
||||
this.submitOutboxRunning = false;
|
||||
}
|
||||
}
|
||||
|
||||
private async publishGatewaySubmitCommandBatch(
|
||||
rows: Array<{ id: string; submitId: string; payload: Prisma.JsonValue }>,
|
||||
): Promise<Array<
|
||||
{ row: typeof rows[number]; streamEntryId: string }
|
||||
| { row: typeof rows[number]; error: unknown }
|
||||
>> {
|
||||
const redis = this.facade.getRedis();
|
||||
const stream = process.env.GATEWAY_SUBMIT_STREAM ?? GATEWAY_SUBMIT_STREAM;
|
||||
const script = `local existing = redis.call('GET', KEYS[2])
|
||||
if existing then return existing end
|
||||
local streamId = redis.call('XADD', KEYS[1], '*', 'messageType', 'SubmitCommand', 'data', ARGV[1])
|
||||
redis.call('SET', KEYS[2], streamId, 'EX', ARGV[2])
|
||||
return streamId`;
|
||||
const pipeline = redis.pipeline();
|
||||
for (const row of rows) {
|
||||
pipeline.eval(
|
||||
script,
|
||||
2,
|
||||
stream,
|
||||
`gateway:submit:outbox:${row.submitId}`,
|
||||
JSON.stringify(row.payload),
|
||||
String(GATEWAY_SUBMIT_REQUEUE_IDEMPOTENCY_TTL_SECONDS),
|
||||
);
|
||||
}
|
||||
const replies = await pipeline.exec();
|
||||
if (!replies || replies.length !== rows.length) {
|
||||
return rows.map((row) => ({ row, error: new Error('Redis Outbox pipeline result count mismatch') }));
|
||||
}
|
||||
return replies.map(([error, value], index) => error
|
||||
? { row: rows[index], error }
|
||||
: { row: rows[index], streamEntryId: typeof value === 'string' ? value : String(value ?? '') });
|
||||
}
|
||||
|
||||
async selectChannelForMessage(
|
||||
@@ -318,33 +830,31 @@ async selectChannelForMessage(
|
||||
throw new BadRequestException('短信应用未配置，无法选择通道组');
|
||||
}
|
||||
const hasPersistedRouting = Boolean(message.carrier);
|
||||
const [carrier, province] = hasPersistedRouting
|
||||
? [normalizeCarrier(message.carrier), message.province ?? null]
|
||||
: await Promise.all([
|
||||
this.facade.identifyCarrier(message.phoneNumber),
|
||||
this.facade.identifyProvince(message.phoneNumber),
|
||||
]);
|
||||
if (!hasPersistedRouting) {
|
||||
await this.prisma.smsMessageRecord.update({
|
||||
where: { id: message.id },
|
||||
data: { carrier, province },
|
||||
});
|
||||
}
|
||||
const route = await this.facade.findApplicationRoute(message.tenantId, message.applicationId, carrier);
|
||||
const excluded = new Set(options.excludeChannelIds ?? []);
|
||||
const signatureId = await this.facade.resolveMessageSignatureId(message);
|
||||
if (!signatureId) throw new BadRequestException('短信签名未配置，无法选择已报备通道');
|
||||
const approvedTasks = await this.prisma.channelSignatureReportTask.findMany({
|
||||
where: {
|
||||
signatureId,
|
||||
reportType: 'signature',
|
||||
status: 'approved',
|
||||
channelId: { in: route.group.items.map((item) => item.channelId) },
|
||||
OR: signatureReportApprovalScopes(carrier),
|
||||
},
|
||||
select: { channelId: true },
|
||||
const [carrier, province] = await this.measureSendStage('phone_routing', async () => {
|
||||
const resolved = hasPersistedRouting
|
||||
? [normalizeCarrier(message.carrier), message.province ?? null] as const
|
||||
: await Promise.all([
|
||||
this.facade.identifyCarrier(message.phoneNumber),
|
||||
this.facade.identifyProvince(message.phoneNumber),
|
||||
]);
|
||||
if (!hasPersistedRouting) {
|
||||
await this.prisma.smsMessageRecord.update({
|
||||
where: { id: message.id },
|
||||
data: { carrier: resolved[0], province: resolved[1] },
|
||||
});
|
||||
}
|
||||
return resolved;
|
||||
});
|
||||
const approvedChannelIds = new Set(approvedTasks.map((task) => task.channelId));
|
||||
const signatureId = await this.measureSendStage('signature_candidates', () => this.facade.resolveMessageSignatureId(message));
|
||||
if (!signatureId) throw new BadRequestException('短信签名未配置，无法选择已报备通道');
|
||||
const route = await this.measureSendStage('route_lookup', () => this.facade.findApplicationRoute(
|
||||
message.tenantId,
|
||||
message.applicationId ?? undefined,
|
||||
carrier,
|
||||
signatureId,
|
||||
));
|
||||
const excluded = new Set(options.excludeChannelIds ?? []);
|
||||
const approvedChannelIds = new Set(route.group.items.map((item) => item.channelId));
|
||||
const selected = selectChannelCandidate(route.group.items, {
|
||||
carrier,
|
||||
province,
|
||||
@@ -366,7 +876,55 @@ async selectChannelForMessage(
|
||||
};
|
||||
}
|
||||
|
||||
async findApplicationRoute(tenantId: string, applicationId: string | undefined, carrier: string) {
|
||||
private async measureSendStage<T>(stage: SendWorkerStage, operation: () => Promise<T>): Promise<T> {
|
||||
const startedAt = this.metrics?.beginSendWorkerStage();
|
||||
try {
|
||||
const result = await operation();
|
||||
if (startedAt != null) this.metrics?.finishSendWorkerStage(startedAt, stage, 'success');
|
||||
return result;
|
||||
} catch (error) {
|
||||
if (startedAt != null) this.metrics?.finishSendWorkerStage(startedAt, stage, 'error');
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
private async refreshSendQueueMetrics() {
|
||||
if (!this.metrics) return;
|
||||
try {
|
||||
const counts = await this.facade.getSendQueue().getJobCounts('wait', 'active', 'completed', 'failed', 'delayed', 'prioritized');
|
||||
const mappings: Array<[SendWorkerQueueState, number]> = [
|
||||
['waiting', counts.wait ?? 0],
|
||||
['active', counts.active ?? 0],
|
||||
['completed', counts.completed ?? 0],
|
||||
['failed', counts.failed ?? 0],
|
||||
['delayed', counts.delayed ?? 0],
|
||||
['prioritized', counts.prioritized ?? 0],
|
||||
];
|
||||
for (const [state, count] of mappings) this.metrics.setSendWorkerQueueJobs(state, count);
|
||||
const pool = this.prisma.getPoolState();
|
||||
for (const state of ['max', 'total', 'idle', 'waiting'] as const) {
|
||||
this.metrics.setSendWorkerDatabasePool(state, pool[state]);
|
||||
}
|
||||
} catch (error) {
|
||||
this.logger.warn(`send_queue_metrics_refresh_failed ${error instanceof Error ? error.message : String(error)}`);
|
||||
}
|
||||
}
|
||||
|
||||
async findApplicationRoute(tenantId: string, applicationId: string | undefined, carrier: string, signatureId?: string) {
|
||||
const approvedChannelWhere = signatureId
|
||||
? {
|
||||
status: 'active',
|
||||
connectionStates: { some: { status: 'connected', currentConnections: { gt: 0 } } },
|
||||
reportTasks: {
|
||||
some: {
|
||||
signatureId,
|
||||
reportType: 'signature',
|
||||
status: 'approved',
|
||||
OR: signatureReportApprovalScopes(carrier),
|
||||
},
|
||||
},
|
||||
}
|
||||
: undefined;
|
||||
const route = await this.prisma.channelRouteRule.findFirst({
|
||||
where: {
|
||||
status: 'active',
|
||||
@@ -376,7 +934,25 @@ async findApplicationRoute(tenantId: string, applicationId: string | undefined,
|
||||
channelId: null,
|
||||
province: null,
|
||||
},
|
||||
include: { group: { include: { items: { include: { channel: { include: { connectionStates: true } } }, orderBy: { priority: 'asc' } } } } },
|
||||
include: {
|
||||
group: {
|
||||
include: {
|
||||
items: {
|
||||
where: approvedChannelWhere ? { channel: approvedChannelWhere } : undefined,
|
||||
include: {
|
||||
channel: {
|
||||
include: {
|
||||
connectionStates: approvedChannelWhere
|
||||
? { where: { status: 'connected', currentConnections: { gt: 0 } } }
|
||||
: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
orderBy: { priority: 'asc' },
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
orderBy: { priority: 'asc' },
|
||||
});
|
||||
if (!route) {
|
||||
@@ -450,7 +1026,40 @@ async waitForChannelRateLimit(channelId: string, tps: number) {
|
||||
}
|
||||
}
|
||||
|
||||
async refreshTaskProgress(batchTaskId: string) {
|
||||
async refreshTaskProgress(batchTaskId: string, knownSingleMessageStatus?: string) {
|
||||
if (knownSingleMessageStatus) {
|
||||
const direct = await this.prisma.smsBatchTask.updateMany({
|
||||
where: { id: batchTaskId, sourceType: 'cmpp', phoneTotal: 1 },
|
||||
data: singleMessageTaskProgress(knownSingleMessageStatus),
|
||||
});
|
||||
if (direct.count === 1) return;
|
||||
} else {
|
||||
// Gateway结果、回执和超时回调已经先提交了消息状态。对CMPP单号码内部任务，
|
||||
// 在同一条UPDATE中读取该唯一消息的当前状态并写入精确计数，避免每次回调都
|
||||
// 对一个只有一行的任务执行GROUP BY；多号码任务继续使用下方聚合路径。
|
||||
const direct = await this.prisma.$executeRaw(Prisma.sql`
|
||||
UPDATE "SmsBatchTask" AS task
|
||||
SET
|
||||
"progressTotal" = 1,
|
||||
"submittedTotal" = CASE WHEN message.status IN ('submit_queued', 'submitted', 'delivered', 'failed', 'unknown', 'timeout') THEN 1 ELSE 0 END,
|
||||
"successTotal" = CASE WHEN message.status = 'delivered' THEN 1 ELSE 0 END,
|
||||
"failedTotal" = CASE WHEN message.status IN ('submit_failed', 'failed') THEN 1 ELSE 0 END,
|
||||
"unknownTotal" = CASE WHEN message.status = 'unknown' THEN 1 ELSE 0 END,
|
||||
"timeoutTotal" = CASE WHEN message.status = 'timeout' THEN 1 ELSE 0 END,
|
||||
status = CASE
|
||||
WHEN message.status IN ('delivered', 'submit_failed', 'failed', 'timeout') THEN 'finished'
|
||||
WHEN message.status IN ('submit_queued', 'submitted', 'unknown') THEN 'sending'
|
||||
ELSE 'queued'
|
||||
END,
|
||||
"updatedAt" = CURRENT_TIMESTAMP
|
||||
FROM "SmsMessageRecord" AS message
|
||||
WHERE task.id = ${batchTaskId}
|
||||
AND task."sourceType" = 'cmpp'
|
||||
AND task."phoneTotal" = 1
|
||||
AND message."batchTaskId" = task.id
|
||||
`);
|
||||
if (direct === 1) return;
|
||||
}
|
||||
const running = this.taskProgressRefreshes.get(batchTaskId);
|
||||
if (running) {
|
||||
// A state transition committed after the running aggregate may not be visible
|
||||
@@ -519,7 +1128,7 @@ getRedis() {
|
||||
return this.redis;
|
||||
}
|
||||
|
||||
async publishGatewaySubmitCommand(command: unknown, idempotencyKey?: string) {
|
||||
async publishGatewaySubmitCommand(command: unknown, idempotencyKey?: string) {
|
||||
const redis = this.facade.getRedis();
|
||||
const stream = process.env.GATEWAY_SUBMIT_STREAM ?? GATEWAY_SUBMIT_STREAM;
|
||||
const payload = JSON.stringify(command);
|
||||
@@ -540,6 +1149,44 @@ return streamId`,
|
||||
);
|
||||
return typeof result === 'string' ? result : String(result ?? '');
|
||||
}
|
||||
|
||||
private getOpenSubmitSessionId(channelId: string) {
|
||||
const cached = this.openSubmitSessionIds.get(channelId);
|
||||
if (cached) return cached;
|
||||
const sessionNo = `OPEN-${channelId}`;
|
||||
const pending = this.prisma.cmppSubmitSession.findUnique({
|
||||
where: { sessionNo },
|
||||
select: { id: true },
|
||||
}).then((existing) => existing ?? this.prisma.cmppSubmitSession.upsert({
|
||||
where: { sessionNo },
|
||||
update: {},
|
||||
create: { channelId, sessionNo, submitTotal: 0 },
|
||||
select: { id: true },
|
||||
})).then((session) => session.id).catch((error) => {
|
||||
this.openSubmitSessionIds.delete(channelId);
|
||||
throw error;
|
||||
});
|
||||
this.openSubmitSessionIds.set(channelId, pending);
|
||||
return pending;
|
||||
}
|
||||
}
|
||||
|
||||
function singleMessageTaskProgress(status: string) {
|
||||
const submittedTotal = ['submit_queued', 'submitted', 'delivered', 'failed', 'unknown', 'timeout'].includes(status) ? 1 : 0;
|
||||
const successTotal = status === 'delivered' ? 1 : 0;
|
||||
const failedTotal = ['submit_failed', 'failed'].includes(status) ? 1 : 0;
|
||||
const unknownTotal = status === 'unknown' ? 1 : 0;
|
||||
const timeoutTotal = status === 'timeout' ? 1 : 0;
|
||||
const doneTotal = successTotal + failedTotal + timeoutTotal;
|
||||
return {
|
||||
progressTotal: 1,
|
||||
submittedTotal,
|
||||
successTotal,
|
||||
failedTotal,
|
||||
unknownTotal,
|
||||
timeoutTotal,
|
||||
status: doneTotal >= 1 ? 'finished' : submittedTotal > 0 ? 'sending' : 'queued',
|
||||
};
|
||||
}
|
||||
|
||||
function signatureReportApprovalScopes(carrier: string) {
|
||||
|
||||
Reference in New Issue
Block a user