perf: batch CMPP inbound workflow processing
This commit is contained in:
hectorzhao
@@ -7,6 +7,9 @@ const CMPP_INBOUND_DURATION_BUCKETS = [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5,
|
||||
export type CmppInboundStage =
|
||||
| 'application_lookup'
|
||||
| 'inbox_persist'
|
||||
| 'worker_claim'
|
||||
| 'reference_preload'
|
||||
| 'daily_quota'
|
||||
| 'long_message_fragment'
|
||||
| 'submission_precheck'
|
||||
| 'template_match'
|
||||
|
||||
@@ -1,6 +1,37 @@
|
||||
import { PhoneFrequencyService, fixedShanghaiWindow } from './phone-frequency.service';
|
||||
|
||||
describe('PhoneFrequencyService', () => {
|
||||
it('persists independent idempotency results for a unique-phone batch in one transaction', async () => {
|
||||
const tx = {
|
||||
phoneFrequencyReservation: {
|
||||
findMany: jest.fn().mockResolvedValue([]),
|
||||
createMany: jest.fn().mockResolvedValue({ count: 2 }),
|
||||
},
|
||||
phoneFrequencyWhitelist: { findMany: jest.fn().mockResolvedValue([]) },
|
||||
};
|
||||
const prisma = {
|
||||
riskRule: { findMany: jest.fn().mockResolvedValue([]) },
|
||||
$transaction: jest.fn(async (callback: (client: typeof tx) => unknown) => callback(tx)),
|
||||
};
|
||||
const riskReview = { ensureDefaultRules: jest.fn().mockResolvedValue(undefined) };
|
||||
const service = new PhoneFrequencyService(prisma as never, riskReview as never);
|
||||
|
||||
const results = await service.reserveBatch([
|
||||
{ tenantId: 'tenant-1', applicationId: 'app-1', phoneNumber: '13800000001', reservationKey: 'inbox-1:frequency' },
|
||||
{ tenantId: 'tenant-1', applicationId: 'app-1', phoneNumber: '13800000002', reservationKey: 'inbox-2:frequency' },
|
||||
]);
|
||||
|
||||
expect(results.get('inbox-1:frequency')?.size).toBe(0);
|
||||
expect(results.get('inbox-2:frequency')?.size).toBe(0);
|
||||
expect(prisma.$transaction).toHaveBeenCalledTimes(1);
|
||||
expect(tx.phoneFrequencyReservation.createMany).toHaveBeenCalledWith({
|
||||
data: expect.arrayContaining([
|
||||
expect.objectContaining({ reservationKey: 'inbox-1:frequency', result: [] }),
|
||||
expect.objectContaining({ reservationKey: 'inbox-2:frequency', result: [] }),
|
||||
]),
|
||||
});
|
||||
});
|
||||
|
||||
it('aligns five-minute cycles and natural days in Asia/Shanghai', () => {
|
||||
const requestedAt = new Date('2026-07-30T16:07:42.000Z');
|
||||
|
||||
|
||||
@@ -63,6 +63,15 @@ export interface PhoneFrequencyRejection {
|
||||
reason: string;
|
||||
}
|
||||
|
||||
export interface PhoneFrequencyBatchReservation {
|
||||
tenantId: string;
|
||||
applicationId: string;
|
||||
phoneNumber: string;
|
||||
reservationKey: string;
|
||||
sourceType?: string;
|
||||
requestedAt?: Date;
|
||||
}
|
||||
|
||||
@Injectable()
|
||||
export class PhoneFrequencyService {
|
||||
constructor(
|
||||
@@ -180,6 +189,141 @@ export class PhoneFrequencyService {
|
||||
}, { isolationLevel: Prisma.TransactionIsolationLevel.ReadCommitted });
|
||||
}
|
||||
|
||||
/**
|
||||
* Reserve independent one-phone Inbox items in bounded database batches. The
|
||||
* reservation rows and counters commit in the same transaction, so reclaiming
|
||||
* any subset replays its original decision. Duplicate phones intentionally use
|
||||
* the established single-item path because their within-batch threshold order
|
||||
* is business-significant.
|
||||
*/
|
||||
async reserveBatch(items: PhoneFrequencyBatchReservation[]) {
|
||||
const results = new Map<string, Map<string, PhoneFrequencyRejection>>();
|
||||
if (items.length === 0) return results;
|
||||
const reservationKeys = items.map((item) => item.reservationKey.trim());
|
||||
if (reservationKeys.some((key) => !key) || new Set(reservationKeys).size !== reservationKeys.length) {
|
||||
throw new BadRequestException('号码频控批次幂等键为空或重复');
|
||||
}
|
||||
const groups = new Map<string, PhoneFrequencyBatchReservation[]>();
|
||||
for (const item of items) {
|
||||
const key = `${item.tenantId}:${item.applicationId}`;
|
||||
const group = groups.get(key) ?? [];
|
||||
group.push({ ...item, phoneNumber: item.phoneNumber.trim(), reservationKey: item.reservationKey.trim() });
|
||||
groups.set(key, group);
|
||||
}
|
||||
for (const group of groups.values()) {
|
||||
const uniquePhones = new Set(group.map((item) => item.phoneNumber));
|
||||
if (uniquePhones.size !== group.length) {
|
||||
for (const item of group) {
|
||||
results.set(item.reservationKey, await this.reserve(
|
||||
item.tenantId,
|
||||
item.applicationId,
|
||||
[item.phoneNumber],
|
||||
item.sourceType,
|
||||
item.requestedAt ?? new Date(),
|
||||
item.reservationKey,
|
||||
));
|
||||
}
|
||||
continue;
|
||||
}
|
||||
await this.riskReview.ensureDefaultRules();
|
||||
const rules = await this.effectiveRules(group[0].applicationId);
|
||||
const groupResults = await this.prisma.$transaction(async (tx) => {
|
||||
const output = new Map<string, Map<string, PhoneFrequencyRejection>>();
|
||||
const existing = await tx.phoneFrequencyReservation.findMany({
|
||||
where: { reservationKey: { in: group.map((item) => item.reservationKey) } },
|
||||
});
|
||||
const existingByKey = new Map(existing.map((item) => [item.reservationKey, item]));
|
||||
const missing: PhoneFrequencyBatchReservation[] = [];
|
||||
for (const item of group) {
|
||||
const replay = existingByKey.get(item.reservationKey);
|
||||
if (!replay) {
|
||||
missing.push(item);
|
||||
continue;
|
||||
}
|
||||
if (replay.tenantId !== item.tenantId || replay.applicationId !== item.applicationId) {
|
||||
throw new BadRequestException('号码频控幂等键已用于另一笔预留');
|
||||
}
|
||||
output.set(item.reservationKey, frequencyRejectionsFromJson(replay.result));
|
||||
}
|
||||
if (missing.length === 0) return output;
|
||||
const whitelistedPhones = await this.findActiveWhitelistedPhones(tx, missing.map((item) => item.phoneNumber));
|
||||
const controlled = missing.filter((item) => !whitelistedPhones.has(item.phoneNumber));
|
||||
const rejectedByPhone = new Map<string, PhoneFrequencyRejection>();
|
||||
for (const rule of rules) {
|
||||
const byWindow = new Map<string, { startAt: Date; endAt: Date; items: PhoneFrequencyBatchReservation[] }>();
|
||||
for (const item of controlled) {
|
||||
const window = fixedShanghaiWindow(item.requestedAt ?? new Date(), readPeriodSeconds(rule));
|
||||
const key = `${window.startAt.toISOString()}:${window.endAt.toISOString()}`;
|
||||
const bucket = byWindow.get(key) ?? { ...window, items: [] };
|
||||
bucket.items.push(item);
|
||||
byWindow.set(key, bucket);
|
||||
}
|
||||
for (const bucket of byWindow.values()) {
|
||||
const states = await this.upsertStates(tx, {
|
||||
tenantId: group[0].tenantId,
|
||||
applicationId: group[0].applicationId,
|
||||
phones: bucket.items.map((item) => item.phoneNumber),
|
||||
rule,
|
||||
window: { startAt: bucket.startAt, endAt: bucket.endAt },
|
||||
});
|
||||
const newTriggers = states.filter((state) => state.activeHitId === null && state.count > rule.thresholdValue);
|
||||
const hitByStateId = new Map<string, string>();
|
||||
if (newTriggers.length > 0) {
|
||||
await tx.phoneFrequencyHit.createMany({
|
||||
data: newTriggers.map((state) => {
|
||||
const hitId = randomUUID();
|
||||
hitByStateId.set(state.id, hitId);
|
||||
return {
|
||||
id: hitId,
|
||||
tenantId: group[0].tenantId,
|
||||
applicationId: group[0].applicationId,
|
||||
ruleId: rule.id,
|
||||
ruleCode: rule.code,
|
||||
ruleName: rule.name,
|
||||
phoneNumber: state.phoneNumber,
|
||||
thresholdValue: Math.floor(rule.thresholdValue),
|
||||
actualValue: state.count,
|
||||
windowStartedAt: state.windowStartedAt,
|
||||
windowEndsAt: state.windowEndsAt,
|
||||
generation: state.generation,
|
||||
action: 'block',
|
||||
sourceType: bucket.items[0]?.sourceType,
|
||||
};
|
||||
}),
|
||||
});
|
||||
await this.attachActiveHits(tx, hitByStateId);
|
||||
}
|
||||
for (const state of states) {
|
||||
if (state.activeHitId === null && state.count <= rule.thresholdValue) continue;
|
||||
const reason = `${rule.name}命中：本周期最多${Math.floor(rule.thresholdValue)}条，当前第${state.count}条，周期${formatWindow(state.windowStartedAt, state.windowEndsAt)}`;
|
||||
const previous = rejectedByPhone.get(state.phoneNumber);
|
||||
rejectedByPhone.set(state.phoneNumber, {
|
||||
code: 'PHONE_FREQUENCY_LIMIT',
|
||||
reason: previous ? `${previous.reason}；${reason}` : reason,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
await tx.phoneFrequencyReservation.createMany({
|
||||
data: missing.map((item) => {
|
||||
const rejection = rejectedByPhone.get(item.phoneNumber);
|
||||
const result = rejection ? [{ phoneNumber: item.phoneNumber, ...rejection }] : [];
|
||||
output.set(item.reservationKey, frequencyRejectionsFromJson(result));
|
||||
return {
|
||||
reservationKey: item.reservationKey,
|
||||
tenantId: item.tenantId,
|
||||
applicationId: item.applicationId,
|
||||
result,
|
||||
};
|
||||
}),
|
||||
});
|
||||
return output;
|
||||
}, { isolationLevel: Prisma.TransactionIsolationLevel.ReadCommitted });
|
||||
for (const [key, value] of groupResults) results.set(key, value);
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
async listHits(query: PhoneFrequencyHitQuery) {
|
||||
const page = Math.max(1, Math.floor(Number(query.page) || 1));
|
||||
const pageSize = Math.min(100, Math.max(1, Math.floor(Number(query.pageSize) || 20)));
|
||||
|
||||
@@ -27,6 +27,7 @@ function createPrismaMock(overrides: Record<string, unknown> = {}) {
|
||||
},
|
||||
smsTemplate: {
|
||||
findUnique: jest.fn().mockResolvedValue(null),
|
||||
findMany: jest.fn().mockResolvedValue([]),
|
||||
},
|
||||
smsSendTask: {
|
||||
create: jest.fn().mockImplementation(({ data }: { data: Record<string, unknown> }) =>
|
||||
@@ -62,6 +63,23 @@ function createPrismaMock(overrides: Record<string, unknown> = {}) {
|
||||
}
|
||||
|
||||
describe('RiskReviewService', () => {
|
||||
it('shares read-only rule inputs across an approved CMPP evaluation batch', async () => {
|
||||
const prisma = createPrismaMock();
|
||||
const service = new RiskReviewService(prisma as never);
|
||||
|
||||
const results = await service.evaluateTasksBatch([
|
||||
{ tenantId: 'tenant-1', applicationId: 'app-1', content: '【测试】验证码000001', phones: ['13800000001'], sourceType: 'cmpp' },
|
||||
{ tenantId: 'tenant-1', applicationId: 'app-1', content: '【测试】验证码000002', phones: ['13800000002'], sourceType: 'cmpp' },
|
||||
]);
|
||||
|
||||
expect(results).toHaveLength(2);
|
||||
expect(results.every((result) => result.status === 'approved')).toBe(true);
|
||||
expect(prisma.riskRule.findMany).toHaveBeenCalledTimes(1);
|
||||
expect(prisma.sensitiveWord.findMany).toHaveBeenCalledTimes(1);
|
||||
expect(prisma.smsBatchTask.count).not.toHaveBeenCalled();
|
||||
expect(prisma.smsSendTask.create).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('coalesces concurrent default-rule checks and reuses the short completeness cache', async () => {
|
||||
const prisma = createPrismaMock();
|
||||
let releaseCount: ((count: number) => void) | undefined;
|
||||
|
||||
@@ -433,6 +433,81 @@ export class RiskReviewService {
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Evaluate an Inbox claim batch against one database snapshot of the read-only
|
||||
* rule inputs. Approved items do not need their own template/rule/sensitive-word
|
||||
* queries; exceptional decisions still go through evaluateTask so their audit
|
||||
* task and hit rows keep the existing semantics.
|
||||
*/
|
||||
async evaluateTasksBatch(items: EvaluateSmsTaskDto[]) {
|
||||
if (items.length === 0) return [];
|
||||
await this.ensureDefaultRules();
|
||||
if (items.some((item) => item.createdById)) {
|
||||
// The CMPP worker never supplies createdById. Keep the general API honest
|
||||
// instead of silently weakening its foreign-key validation in the fast path.
|
||||
return Promise.all(items.map((item) => this.evaluateTask(item)));
|
||||
}
|
||||
const applicationIds = [...new Set(items.map((item) => item.applicationId).filter((id): id is string => Boolean(id)))];
|
||||
const templateIds = [...new Set(items.map((item) => item.templateId).filter((id): id is string => Boolean(id)))];
|
||||
const [templates, sensitiveWords, applicationInputs] = await Promise.all([
|
||||
templateIds.length
|
||||
? this.prisma.smsTemplate.findMany({ where: { id: { in: templateIds } }, include: { variables: true } })
|
||||
: Promise.resolve([]),
|
||||
this.prisma.sensitiveWord.findMany({ where: { status: 'active' }, select: { word: true, level: true } }),
|
||||
Promise.all(applicationIds.map(async (applicationId) => ({
|
||||
applicationId,
|
||||
rules: await this.effectiveRules(applicationId),
|
||||
recentTaskCount: await this.countRecentClientTasks(applicationId, 'cmpp'),
|
||||
}))),
|
||||
]);
|
||||
const templateById = new Map(templates.map((template) => [template.id, template]));
|
||||
const inputsByApplication = new Map(applicationInputs.map((entry) => [entry.applicationId, entry]));
|
||||
|
||||
return Promise.all(items.map(async (data) => {
|
||||
const phones = data.phones ?? [];
|
||||
const uniquePhones = [...new Set(phones)];
|
||||
const phoneTotal = phones.length;
|
||||
const template = data.templateId ? templateById.get(data.templateId) : undefined;
|
||||
const variableIssues = evaluateTemplateVariables(template?.variables ?? [], data.content, data.variables ?? {});
|
||||
const contentIssues = evaluateContent(data.content, sensitiveWords);
|
||||
if (variableIssues.length > 0) {
|
||||
contentIssues.push({
|
||||
ruleCode: 'TEMPLATE_VARIABLE_INVALID',
|
||||
ruleName: '模板变量校验失败',
|
||||
thresholdValue: 0,
|
||||
actualValue: variableIssues.length,
|
||||
action: 'block',
|
||||
reason: formatTemplateVariableIssueReason(variableIssues),
|
||||
});
|
||||
}
|
||||
const applicationInput = data.applicationId ? inputsByApplication.get(data.applicationId) : undefined;
|
||||
const rules = applicationInput?.rules ?? [];
|
||||
const requestedAt = data.requestedAt ? new Date(data.requestedAt) : new Date();
|
||||
const nonWorkingRule = rules.find((rule) => rule.code === 'NON_WORKING_MARKETING_BULK');
|
||||
const nonWorkingMarketingPhones = isMarketing(data.category ?? template?.category)
|
||||
&& isNonWorkingTime(requestedAt, readNonWorkingConfig(nonWorkingRule?.config))
|
||||
? phoneTotal
|
||||
: 0;
|
||||
const hits = this.evaluateRules(rules, {
|
||||
phoneTotal,
|
||||
nonWorkingMarketingPhones,
|
||||
recentTaskCount: applicationInput?.recentTaskCount ?? 0,
|
||||
});
|
||||
hits.push(...contentIssues.map(contentIssueToHit));
|
||||
const decision = decideRiskAction(hits);
|
||||
if (decision.status !== 'approved') {
|
||||
return this.evaluateTask(data);
|
||||
}
|
||||
return {
|
||||
canSubmit: true,
|
||||
status: decision.status,
|
||||
riskDecision: decision.riskDecision,
|
||||
reason: hits.length > 0 ? hits.map((hit) => hit.reason).join('; ') : null,
|
||||
task: null,
|
||||
};
|
||||
}));
|
||||
}
|
||||
|
||||
async approveTask(taskId: string, data: ReviewSmsTaskDto) {
|
||||
const task = await this.prisma.smsSendTask.findUnique({ where: { id: taskId } });
|
||||
if (!task) {
|
||||
|
||||
@@ -41,6 +41,20 @@ type PersistedInboundWorkflowRow = {
|
||||
|
||||
type InboundApplication = NonNullable<Awaited<ReturnType<SendSubmissionService['findInboundApplication']>>>;
|
||||
|
||||
type InboundTemplate = Prisma.SmsTemplateGetPayload<{ include: { signature: true } }>;
|
||||
|
||||
type InboundBatchCandidate = {
|
||||
item: ClaimedInboundWorkflow;
|
||||
payload: InboundWorkflowPayload;
|
||||
application: InboundApplication;
|
||||
phoneNumber: string;
|
||||
template?: InboundTemplate;
|
||||
signatureId: string;
|
||||
templateVariables?: Record<string, unknown>;
|
||||
drainageInfoId?: string;
|
||||
clientSrcId: string | null;
|
||||
};
|
||||
|
||||
type InboundWorkflowResponse = {
|
||||
accepted: boolean;
|
||||
tenantId: string;
|
||||
@@ -1227,7 +1241,7 @@ startInboundWorkflowWorker() {
|
||||
this.inboundWorkflowPumping = true;
|
||||
try {
|
||||
await this.refreshInboundWorkflowMetrics();
|
||||
const claimed = await this.claimInboundWorkflows(available);
|
||||
const claimed = await this.measureInboundStage('worker_claim', () => this.claimInboundWorkflows(available));
|
||||
const applications = claimed.length === 0
|
||||
? []
|
||||
: await this.prisma.smsApplication.findMany({
|
||||
@@ -1235,13 +1249,15 @@ startInboundWorkflowWorker() {
|
||||
include: { tenant: true, ipAllowlist: true },
|
||||
});
|
||||
const applicationById = new Map(applications.map((application) => [application.id, application]));
|
||||
for (const item of claimed) {
|
||||
this.inboundWorkflowInFlight += 1;
|
||||
const batchSize = positiveInteger(process.env.API_INBOUND_WORKFLOW_BATCH_SIZE, 64);
|
||||
for (let offset = 0; offset < claimed.length; offset += batchSize) {
|
||||
const batch = claimed.slice(offset, offset + batchSize);
|
||||
this.inboundWorkflowInFlight += batch.length;
|
||||
this.metrics?.setInboundWorkflowSlots(concurrency, this.inboundWorkflowInFlight);
|
||||
const task = this.processClaimedInboundWorkflow(item, applicationById.get(item.applicationId))
|
||||
.catch((error) => this.logger.error(`CMPP inbound workflow ${item.id} failed to settle: ${String(error)}`))
|
||||
const task = this.processClaimedInboundWorkflowBatch(batch, applicationById)
|
||||
.catch((error) => this.logger.error(`CMPP inbound workflow batch failed to settle: ${String(error)}`))
|
||||
.finally(() => {
|
||||
this.inboundWorkflowInFlight = Math.max(0, this.inboundWorkflowInFlight - 1);
|
||||
this.inboundWorkflowInFlight = Math.max(0, this.inboundWorkflowInFlight - batch.length);
|
||||
this.metrics?.setInboundWorkflowSlots(concurrency, this.inboundWorkflowInFlight);
|
||||
this.inboundWorkflowTasks.delete(task);
|
||||
this.scheduleInboundWorkflowPump(0);
|
||||
@@ -1294,6 +1310,333 @@ startInboundWorkflowWorker() {
|
||||
`);
|
||||
}
|
||||
|
||||
private async processClaimedInboundWorkflowBatch(
|
||||
items: ClaimedInboundWorkflow[],
|
||||
applicationById: Map<string, InboundApplication>,
|
||||
) {
|
||||
if (process.env.API_INBOUND_WORKFLOW_BATCH_ENABLED === 'false' || items.length < 2) {
|
||||
await Promise.all(items.map((item) => this.processClaimedInboundWorkflow(item, applicationById.get(item.applicationId))));
|
||||
return;
|
||||
}
|
||||
let completed = new Set<string>();
|
||||
try {
|
||||
completed = await this.processCommonInboundWorkflowBatch(items, applicationById);
|
||||
} catch (error) {
|
||||
// The common path is wholly idempotent: quota/frequency reservations have
|
||||
// stable keys, records have stable message/task keys, and BullMQ job IDs are
|
||||
// message IDs. Falling back after a partial external failure repairs rather
|
||||
// than duplicates the individual items.
|
||||
this.logger.warn(`CMPP inbound common batch fell back to individual recovery: ${String(error)}`);
|
||||
}
|
||||
await Promise.all(items
|
||||
.filter((item) => !completed.has(item.id))
|
||||
.map((item) => this.processClaimedInboundWorkflow(item, applicationById.get(item.applicationId))));
|
||||
}
|
||||
|
||||
private async processCommonInboundWorkflowBatch(
|
||||
items: ClaimedInboundWorkflow[],
|
||||
applicationById: Map<string, InboundApplication>,
|
||||
) {
|
||||
const parsed = items.flatMap((item) => {
|
||||
try {
|
||||
const payload = parseInboundWorkflowPayload(item.payload);
|
||||
const application = applicationById.get(item.applicationId);
|
||||
return payload.phoneNumbers.length === 1 && application ? [{ item, payload, application }] : [];
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
});
|
||||
if (parsed.length < 2) return new Set<string>();
|
||||
const applicationIds = [...new Set(parsed.map((entry) => entry.application.id))];
|
||||
const phones = [...new Set(parsed.map((entry) => entry.payload.phoneNumbers[0]))];
|
||||
const messageIds = parsed.flatMap((entry) => entry.payload.messageIds);
|
||||
const tenantIds = [...new Set(parsed.map((entry) => entry.application.tenantId))];
|
||||
const [templates, signatures, globalBlacklist, enterpriseBlacklist, persisted] = await this.measureInboundStage('reference_preload', () => Promise.all([
|
||||
this.prisma.smsTemplate.findMany({
|
||||
where: { applicationId: { in: applicationIds }, auditStatus: 'approved', signature: { auditStatus: 'approved' } },
|
||||
include: { signature: true },
|
||||
orderBy: { updatedAt: 'desc' },
|
||||
}),
|
||||
this.prisma.smsSignature.findMany({
|
||||
where: { applicationId: { in: applicationIds }, auditStatus: 'approved' },
|
||||
select: { id: true, applicationId: true, name: true, updatedAt: true },
|
||||
orderBy: { updatedAt: 'desc' },
|
||||
}),
|
||||
this.prisma.globalBlacklist.findMany({
|
||||
where: { phoneNumber: { in: phones }, status: 'active' },
|
||||
select: { phoneNumber: true },
|
||||
}),
|
||||
this.prisma.enterpriseBlacklist.findMany({
|
||||
where: { tenantId: { in: tenantIds }, applicationId: { in: applicationIds }, phoneNumber: { in: phones }, status: 'active' },
|
||||
select: { tenantId: true, applicationId: true, phoneNumber: true },
|
||||
}),
|
||||
this.prisma.smsMessageRecord.findMany({ where: { messageId: { in: messageIds } }, select: { messageId: true } }),
|
||||
]));
|
||||
const globalRejected = new Set(globalBlacklist.map((entry) => entry.phoneNumber));
|
||||
const enterpriseRejected = new Set(enterpriseBlacklist.map((entry) => `${entry.tenantId}:${entry.applicationId}:${entry.phoneNumber}`));
|
||||
const persistedIds = new Set(persisted.map((entry) => entry.messageId));
|
||||
const templatesByApplication = new Map<string, InboundTemplate[]>();
|
||||
for (const template of templates) {
|
||||
const values = templatesByApplication.get(template.applicationId) ?? [];
|
||||
values.push(template);
|
||||
templatesByApplication.set(template.applicationId, values);
|
||||
}
|
||||
const signaturesByApplication = new Map<string, typeof signatures>();
|
||||
for (const signature of signatures) {
|
||||
if (!signature.applicationId) continue;
|
||||
const values = signaturesByApplication.get(signature.applicationId) ?? [];
|
||||
values.push(signature);
|
||||
signaturesByApplication.set(signature.applicationId, values);
|
||||
}
|
||||
const candidates: InboundBatchCandidate[] = [];
|
||||
for (const entry of parsed) {
|
||||
const { item, payload, application } = entry;
|
||||
const data = payload.data;
|
||||
const phoneNumber = payload.phoneNumbers[0];
|
||||
if (persistedIds.has(payload.messageIds[0])
|
||||
|| application.cmppAccount !== data.account
|
||||
|| application.status !== 'active'
|
||||
|| application.tenant.status !== 'active'
|
||||
|| !application.interfaceEnabled
|
||||
|| application.tenant.certificationStatus !== 'approved'
|
||||
|| !/^1\d{10}$/.test(phoneNumber)
|
||||
|| globalRejected.has(phoneNumber)
|
||||
|| enterpriseRejected.has(`${application.tenantId}:${application.id}:${phoneNumber}`)
|
||||
// Paid messages retain the existing per-message account lock until the
|
||||
// dedicated batch-ledger migration is introduced; never weaken billing
|
||||
// correctness merely to increase the benchmark number.
|
||||
|| moneyToNumber(application.customerUnitPrice) !== 0) continue;
|
||||
try {
|
||||
if (data.remoteIp && !isIpAllowed(data.remoteIp, application.ipAllowlist.map((allowlist) => allowlist.ipCidr))) continue;
|
||||
const clientSrcId = validateInboundApplicationSrcId(data.srcId, application);
|
||||
const template = (templatesByApplication.get(application.id) ?? []).find((candidate) => (
|
||||
candidate.content === data.content || matchTemplateContent(candidate.content, data.content) !== null
|
||||
));
|
||||
const templateVariables = template ? matchTemplateContent(template.content, data.content) ?? {} : undefined;
|
||||
let signatureId = template?.signature?.id;
|
||||
if (!signatureId && application.templateMismatchMode === 'direct_send') {
|
||||
const signatureName = data.content.match(/^【[^】]+】/)?.[0];
|
||||
signatureId = (signaturesByApplication.get(application.id) ?? []).find((signature) => signature.name === signatureName)?.id;
|
||||
}
|
||||
if (!signatureId || (!template && application.templateMismatchMode !== 'direct_send')) continue;
|
||||
const finalSignatureId = signatureId;
|
||||
candidates.push({ item, payload, application, phoneNumber, template, signatureId: finalSignatureId, templateVariables, clientSrcId });
|
||||
} catch {
|
||||
// Invalid source IDs and other business rejections stay on the established
|
||||
// individual path so their exact failure receipt remains unchanged.
|
||||
}
|
||||
}
|
||||
if (candidates.length < 2) return new Set<string>();
|
||||
|
||||
const quota = await this.measureInboundStage('daily_quota', () => this.reserveDailyQuotaBatch(candidates));
|
||||
const quotaApproved = candidates.filter((candidate) => quota.get(candidate.item.requestKey)?.reserved);
|
||||
if (quotaApproved.length < 2) return new Set<string>();
|
||||
const riskResults = await this.measureInboundStage('risk_frequency', () => this.riskReview.evaluateTasksBatch(quotaApproved.map((candidate) => ({
|
||||
tenantId: candidate.application.tenantId,
|
||||
applicationId: candidate.application.id,
|
||||
templateId: candidate.template?.id,
|
||||
content: candidate.payload.data.content,
|
||||
variables: candidate.templateVariables,
|
||||
phones: [candidate.phoneNumber],
|
||||
sourceType: 'cmpp' as const,
|
||||
}))));
|
||||
const riskApproved = quotaApproved.filter((_, index) => riskResults[index]?.status === 'approved');
|
||||
if (riskApproved.length < 2) return new Set<string>();
|
||||
const frequency = await this.measureInboundStage('risk_frequency', () => this.phoneFrequency.reserveBatch(riskApproved.map((candidate) => ({
|
||||
tenantId: candidate.application.tenantId,
|
||||
applicationId: candidate.application.id,
|
||||
phoneNumber: candidate.phoneNumber,
|
||||
sourceType: 'cmpp',
|
||||
reservationKey: `${candidate.item.requestKey}:message:0:frequency`,
|
||||
}))));
|
||||
const approved = riskApproved.filter((candidate) => (
|
||||
(frequency.get(`${candidate.item.requestKey}:message:0:frequency`)?.size ?? 0) === 0
|
||||
));
|
||||
if (approved.length < 2) return new Set<string>();
|
||||
|
||||
const drainageRows = await this.prisma.smsDrainageInfo.findMany({
|
||||
where: { signatureId: { in: [...new Set(approved.map((candidate) => candidate.signatureId))] }, auditStatus: { not: 'deleted' } },
|
||||
select: { id: true, signatureId: true, url: true, updatedAt: true },
|
||||
orderBy: [{ updatedAt: 'desc' }, { id: 'asc' }],
|
||||
});
|
||||
const finalCandidates = approved.filter((candidate) => {
|
||||
const matches = drainageRows
|
||||
.filter((row) => row.signatureId === candidate.signatureId && row.url.trim() && candidate.payload.data.content.includes(row.url.trim()))
|
||||
.sort((left, right) => right.url.trim().length - left.url.trim().length || right.updatedAt.getTime() - left.updatedAt.getTime());
|
||||
if (matches.length > 1 && matches[0].url.trim().length === matches[1].url.trim().length) return false;
|
||||
candidate.drainageInfoId = matches[0]?.id;
|
||||
return true;
|
||||
});
|
||||
if (finalCandidates.length < 2) return new Set<string>();
|
||||
return this.persistCommonInboundWorkflowBatch(finalCandidates);
|
||||
}
|
||||
|
||||
private async reserveDailyQuotaBatch(candidates: InboundBatchCandidate[]) {
|
||||
const usageDate = shanghaiDateKey();
|
||||
const usageDateValue = new Date(`${usageDate}T00:00:00.000Z`);
|
||||
return this.prisma.$transaction(async (tx) => {
|
||||
const applicationIds = [...new Set(candidates.map((candidate) => candidate.application.id))].sort();
|
||||
await tx.smsApplicationDailyUsage.createMany({
|
||||
data: applicationIds.map((applicationId) => ({ id: randomUUID(), applicationId, usageDate: usageDateValue, usedCount: 0 })),
|
||||
skipDuplicates: true,
|
||||
});
|
||||
const usage = await tx.$queryRaw<Array<{ applicationId: string; tenantId: string; dailyLimit: number; usedCount: number }>>(Prisma.sql`
|
||||
SELECT usage."applicationId", application."tenantId",
|
||||
COALESCE(application."dailyLimit", 100000)::integer AS "dailyLimit",
|
||||
usage."usedCount"
|
||||
FROM "SmsApplicationDailyUsage" usage
|
||||
JOIN "SmsApplication" application ON application.id = usage."applicationId"
|
||||
WHERE usage."usageDate" = ${usageDate}::date
|
||||
AND usage."applicationId" IN (${Prisma.join(applicationIds)})
|
||||
ORDER BY usage."applicationId"
|
||||
FOR UPDATE OF usage
|
||||
`);
|
||||
const state = new Map(usage.map((row) => [row.applicationId, { ...row }]));
|
||||
const keys = candidates.map((candidate) => `${candidate.item.requestKey}:daily-quota`);
|
||||
const existing = await tx.smsApplicationDailyReservation.findMany({ where: { reservationKey: { in: keys } } });
|
||||
const existingByKey = new Map(existing.map((row) => [row.reservationKey, row]));
|
||||
const output = new Map<string, { reserved: boolean; dailyLimit: number; usedCount: number | null }>();
|
||||
const inserts: Prisma.SmsApplicationDailyReservationCreateManyInput[] = [];
|
||||
for (const candidate of candidates) {
|
||||
const reservationKey = `${candidate.item.requestKey}:daily-quota`;
|
||||
const replay = existingByKey.get(reservationKey);
|
||||
if (replay) {
|
||||
if (replay.applicationId !== candidate.application.id || replay.requestedCount !== 1) {
|
||||
throw new Error('CMPP daily quota idempotency key conflicts with another reservation');
|
||||
}
|
||||
output.set(candidate.item.requestKey, { reserved: replay.reserved, dailyLimit: replay.dailyLimit, usedCount: replay.usedCount });
|
||||
continue;
|
||||
}
|
||||
const current = state.get(candidate.application.id);
|
||||
if (!current) throw new Error(`CMPP daily quota application ${candidate.application.id} disappeared`);
|
||||
const reserved = current.usedCount + 1 <= current.dailyLimit;
|
||||
if (reserved) current.usedCount += 1;
|
||||
inserts.push({
|
||||
id: randomUUID(), reservationKey, tenantId: candidate.application.tenantId,
|
||||
applicationId: candidate.application.id, usageDate: usageDateValue, requestedCount: 1,
|
||||
dailyLimit: current.dailyLimit, usedCount: reserved ? current.usedCount : null, reserved,
|
||||
});
|
||||
output.set(candidate.item.requestKey, { reserved, dailyLimit: current.dailyLimit, usedCount: reserved ? current.usedCount : null });
|
||||
}
|
||||
for (const row of state.values()) {
|
||||
await tx.smsApplicationDailyUsage.update({
|
||||
where: { applicationId_usageDate: { applicationId: row.applicationId, usageDate: usageDateValue } },
|
||||
data: { usedCount: row.usedCount },
|
||||
});
|
||||
}
|
||||
if (inserts.length) await tx.smsApplicationDailyReservation.createMany({ data: inserts });
|
||||
return output;
|
||||
}, { isolationLevel: Prisma.TransactionIsolationLevel.ReadCommitted });
|
||||
}
|
||||
|
||||
private async persistCommonInboundWorkflowBatch(candidates: InboundBatchCandidate[]) {
|
||||
const prepared = await Promise.all(candidates.map(async (candidate) => {
|
||||
const workflowDigest = createHash('sha256').update(`${candidate.item.requestKey}:message:0`).digest('hex').slice(0, 32);
|
||||
const taskId = randomUUID();
|
||||
const messageRecordId = randomUUID();
|
||||
const content = candidate.payload.data.content;
|
||||
const drainageDetection = await detectDrainageContent(this.prisma, content);
|
||||
return { candidate, workflowDigest, taskId, messageRecordId, content, drainageDetection };
|
||||
}));
|
||||
await this.measureInboundStage('message_persist', () => this.prisma.$transaction(async (tx) => {
|
||||
await tx.smsBatchTask.createMany({
|
||||
data: prepared.map(({ candidate, workflowDigest, taskId, content }) => ({
|
||||
id: taskId,
|
||||
tenantId: candidate.application.tenantId,
|
||||
applicationId: candidate.application.id,
|
||||
templateId: candidate.template?.id,
|
||||
taskNo: `BT-IN-${workflowDigest}`,
|
||||
sourceType: 'cmpp',
|
||||
content,
|
||||
phoneTotal: 1,
|
||||
status: 'ready',
|
||||
auditStatus: 'approved',
|
||||
progressTotal: 1,
|
||||
})),
|
||||
});
|
||||
await tx.smsApiRequest.createMany({
|
||||
data: prepared.map(({ candidate, workflowDigest, taskId, content }) => ({
|
||||
id: randomUUID(), tenantId: candidate.application.tenantId, batchTaskId: taskId,
|
||||
requestId: `REQ-IN-${workflowDigest}`, sourceIp: candidate.payload.data.remoteIp,
|
||||
userAgent: 'cmpp-gateway', payloadSummary: { phoneTotal: 1, contentLength: [...content].length, account: candidate.payload.data.account },
|
||||
status: 'accepted',
|
||||
})),
|
||||
});
|
||||
await tx.smsMessageRecord.createMany({
|
||||
data: prepared.map(({ candidate, taskId, messageRecordId, content, drainageDetection }) => ({
|
||||
id: messageRecordId,
|
||||
tenantId: candidate.application.tenantId,
|
||||
batchTaskId: taskId,
|
||||
applicationId: candidate.application.id,
|
||||
templateId: candidate.template?.id,
|
||||
signatureId: candidate.signatureId,
|
||||
drainageInfoId: candidate.drainageInfoId,
|
||||
messageId: candidate.payload.messageIds[0],
|
||||
phoneNumber: candidate.phoneNumber,
|
||||
content,
|
||||
...drainageDetection,
|
||||
billingUnits: this.billing.estimateSmsCost({
|
||||
tenantId: candidate.application.tenantId, applicationId: candidate.application.id,
|
||||
content, phoneCount: 1, unitPrice: 0,
|
||||
}).billingUnitsPerMessage,
|
||||
unitPrice: 0,
|
||||
amountCents: 0,
|
||||
queuePriority: normalizeQueuePriority(candidate.application.queuePriority),
|
||||
cmppSubmitSequenceId: candidate.payload.data.sequenceId == null ? null : String(candidate.payload.data.sequenceId),
|
||||
cmppSubmitGroupMessageId: candidate.payload.submitGroupMessageId,
|
||||
cmppRegisteredDelivery: candidate.payload.data.registeredDelivery !== 0,
|
||||
clientSrcId: candidate.clientSrcId,
|
||||
applicationExtension: candidate.application.cmppApplicationExtension,
|
||||
status: 'queued',
|
||||
})),
|
||||
});
|
||||
}));
|
||||
await this.measureInboundStage('queue_publish', () => this.facade.getSendQueue().addBulk(prepared.map(({ candidate, messageRecordId }) => ({
|
||||
name: 'send-message' as const,
|
||||
data: { messageRecordId },
|
||||
opts: {
|
||||
jobId: messageRecordId,
|
||||
attempts: 3,
|
||||
priority: BULLMQ_PRIORITY[normalizeQueuePriority(candidate.application.queuePriority)],
|
||||
},
|
||||
}))));
|
||||
await this.prisma.smsBatchTask.updateMany({
|
||||
where: { id: { in: prepared.map((entry) => entry.taskId) }, status: 'ready' },
|
||||
data: { status: 'queued' },
|
||||
});
|
||||
const results = prepared.map(({ candidate, taskId, messageRecordId }) => ({
|
||||
accepted: true,
|
||||
tenantId: candidate.application.tenantId,
|
||||
applicationId: candidate.application.id,
|
||||
taskId,
|
||||
messageId: candidate.payload.messageIds[0],
|
||||
messageRecordId,
|
||||
status: 'accepted',
|
||||
phoneCount: 1,
|
||||
messages: [{
|
||||
phoneNumber: candidate.phoneNumber,
|
||||
messageId: candidate.payload.messageIds[0],
|
||||
messageRecordId,
|
||||
taskId,
|
||||
status: 'accepted',
|
||||
}],
|
||||
}));
|
||||
const values = prepared.map((entry, index) => Prisma.sql`(${entry.candidate.item.id}, ${JSON.stringify(results[index])}::jsonb)`);
|
||||
const settled = await this.prisma.$queryRaw<Array<{ id: string }>>(Prisma.sql`
|
||||
UPDATE "CmppInboundSubmissionInbox" inbox
|
||||
SET status = 'completed', result = updates.result, "completedAt" = (NOW() AT TIME ZONE 'UTC'),
|
||||
"lockedAt" = NULL, "lockedBy" = NULL, "lastError" = NULL, "updatedAt" = (NOW() AT TIME ZONE 'UTC')
|
||||
FROM (VALUES ${Prisma.join(values)}) AS updates(id, result)
|
||||
WHERE inbox.id = updates.id
|
||||
AND inbox.status = 'processing'
|
||||
AND inbox."lockedBy" = ${this.inboundWorkflowWorkerId}
|
||||
RETURNING inbox.id
|
||||
`);
|
||||
const settledIds = new Set(settled.map((row) => row.id));
|
||||
for (let index = 0; index < settled.length; index += 1) this.metrics?.recordInboundWorkflowResult('completed');
|
||||
return settledIds;
|
||||
}
|
||||
|
||||
private async processClaimedInboundWorkflow(item: ClaimedInboundWorkflow, application?: InboundApplication) {
|
||||
try {
|
||||
const payload = parseInboundWorkflowPayload(item.payload);
|
||||
|
||||
Reference in New Issue
Block a user