perf(cmpp): add durable inbound fast path
This commit is contained in:
@@ -12,6 +12,14 @@ HTTP_API_MASTER_KEY=replace-with-at-least-32-random-characters
|
|||||||
HTTP_API_PUBLIC_ORIGIN=https://api.example.com
|
HTTP_API_PUBLIC_ORIGIN=https://api.example.com
|
||||||
API_ENABLE_SEND_WORKER=true
|
API_ENABLE_SEND_WORKER=true
|
||||||
API_SEND_WORKER_CONCURRENCY=50
|
API_SEND_WORKER_CONCURRENCY=50
|
||||||
|
API_WORKER_DATABASE_URL=
|
||||||
|
API_WORKER_METRICS_HOST=127.0.0.1
|
||||||
|
API_WORKER_METRICS_PORT=9465
|
||||||
|
CMPP_INBOUND_FAST_PATH_ENABLED=true
|
||||||
|
CMPP_INBOUND_WORKFLOW_WORKER_ENABLED=true
|
||||||
|
API_INBOUND_WORKFLOW_CONCURRENCY=32
|
||||||
|
API_INBOUND_WORKFLOW_POLL_INTERVAL_MS=100
|
||||||
|
API_INBOUND_WORKFLOW_STALE_SECONDS=300
|
||||||
ADMIN_SESSION_IDLE_TIMEOUT_MS=3600000
|
ADMIN_SESSION_IDLE_TIMEOUT_MS=3600000
|
||||||
CLIENT_SESSION_IDLE_TIMEOUT_MS=7200000
|
CLIENT_SESSION_IDLE_TIMEOUT_MS=7200000
|
||||||
SESSION_LOCK_RECOVERY_MS=14400000
|
SESSION_LOCK_RECOVERY_MS=14400000
|
||||||
|
|||||||
@@ -0,0 +1,91 @@
|
|||||||
|
CREATE TABLE "CmppInboundSubmissionInbox" (
|
||||||
|
"id" TEXT NOT NULL,
|
||||||
|
"requestKey" TEXT NOT NULL,
|
||||||
|
"payloadHash" TEXT NOT NULL,
|
||||||
|
"tenantId" TEXT NOT NULL,
|
||||||
|
"applicationId" TEXT NOT NULL,
|
||||||
|
"queuePriority" TEXT NOT NULL DEFAULT 'normal',
|
||||||
|
"payload" JSONB NOT NULL,
|
||||||
|
"response" JSONB NOT NULL,
|
||||||
|
"status" TEXT NOT NULL DEFAULT 'pending',
|
||||||
|
"attempts" INTEGER NOT NULL DEFAULT 0,
|
||||||
|
"nextAttemptAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||||
|
"lockedAt" TIMESTAMP(3),
|
||||||
|
"lockedBy" TEXT,
|
||||||
|
"lastError" TEXT,
|
||||||
|
"result" JSONB,
|
||||||
|
"completedAt" TIMESTAMP(3),
|
||||||
|
"createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||||
|
"updatedAt" TIMESTAMP(3) NOT NULL,
|
||||||
|
|
||||||
|
CONSTRAINT "CmppInboundSubmissionInbox_pkey" PRIMARY KEY ("id")
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE UNIQUE INDEX "CmppInboundSubmissionInbox_requestKey_key"
|
||||||
|
ON "CmppInboundSubmissionInbox"("requestKey");
|
||||||
|
|
||||||
|
CREATE INDEX "CmppInboundSubmissionInbox_status_nextAttemptAt_createdAt_idx"
|
||||||
|
ON "CmppInboundSubmissionInbox"("status", "nextAttemptAt", "createdAt");
|
||||||
|
|
||||||
|
CREATE INDEX "CmppInboundSubmissionInbox_status_lockedAt_idx"
|
||||||
|
ON "CmppInboundSubmissionInbox"("status", "lockedAt");
|
||||||
|
|
||||||
|
CREATE INDEX "CmppInboundSubmissionInbox_applicationId_createdAt_idx"
|
||||||
|
ON "CmppInboundSubmissionInbox"("applicationId", "createdAt");
|
||||||
|
|
||||||
|
CREATE INDEX "CmppInboundSubmissionInbox_queuePriority_status_createdAt_idx"
|
||||||
|
ON "CmppInboundSubmissionInbox"("queuePriority", "status", "createdAt");
|
||||||
|
|
||||||
|
ALTER TABLE "CmppInboundSubmissionInbox"
|
||||||
|
ADD CONSTRAINT "CmppInboundSubmissionInbox_tenantId_fkey"
|
||||||
|
FOREIGN KEY ("tenantId") REFERENCES "Tenant"("id") ON DELETE RESTRICT ON UPDATE CASCADE;
|
||||||
|
|
||||||
|
ALTER TABLE "CmppInboundSubmissionInbox"
|
||||||
|
ADD CONSTRAINT "CmppInboundSubmissionInbox_applicationId_fkey"
|
||||||
|
FOREIGN KEY ("applicationId") REFERENCES "SmsApplication"("id") ON DELETE RESTRICT ON UPDATE CASCADE;
|
||||||
|
|
||||||
|
CREATE TABLE "SmsApplicationDailyReservation" (
|
||||||
|
"id" TEXT NOT NULL,
|
||||||
|
"reservationKey" TEXT NOT NULL,
|
||||||
|
"tenantId" TEXT NOT NULL,
|
||||||
|
"applicationId" TEXT NOT NULL,
|
||||||
|
"usageDate" DATE NOT NULL,
|
||||||
|
"requestedCount" INTEGER NOT NULL,
|
||||||
|
"dailyLimit" INTEGER NOT NULL,
|
||||||
|
"usedCount" INTEGER,
|
||||||
|
"reserved" BOOLEAN NOT NULL,
|
||||||
|
"createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||||
|
CONSTRAINT "SmsApplicationDailyReservation_pkey" PRIMARY KEY ("id")
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE UNIQUE INDEX "SmsApplicationDailyReservation_reservationKey_key"
|
||||||
|
ON "SmsApplicationDailyReservation"("reservationKey");
|
||||||
|
CREATE INDEX "SmsApplicationDailyReservation_applicationId_usageDate_idx"
|
||||||
|
ON "SmsApplicationDailyReservation"("applicationId", "usageDate");
|
||||||
|
CREATE INDEX "SmsApplicationDailyReservation_tenantId_createdAt_idx"
|
||||||
|
ON "SmsApplicationDailyReservation"("tenantId", "createdAt");
|
||||||
|
ALTER TABLE "SmsApplicationDailyReservation" ADD CONSTRAINT "SmsApplicationDailyReservation_tenantId_fkey"
|
||||||
|
FOREIGN KEY ("tenantId") REFERENCES "Tenant"("id") ON DELETE RESTRICT ON UPDATE CASCADE;
|
||||||
|
ALTER TABLE "SmsApplicationDailyReservation" ADD CONSTRAINT "SmsApplicationDailyReservation_applicationId_fkey"
|
||||||
|
FOREIGN KEY ("applicationId") REFERENCES "SmsApplication"("id") ON DELETE RESTRICT ON UPDATE CASCADE;
|
||||||
|
|
||||||
|
CREATE TABLE "PhoneFrequencyReservation" (
|
||||||
|
"id" TEXT NOT NULL,
|
||||||
|
"reservationKey" TEXT NOT NULL,
|
||||||
|
"tenantId" TEXT NOT NULL,
|
||||||
|
"applicationId" TEXT NOT NULL,
|
||||||
|
"result" JSONB NOT NULL,
|
||||||
|
"createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||||
|
CONSTRAINT "PhoneFrequencyReservation_pkey" PRIMARY KEY ("id")
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE UNIQUE INDEX "PhoneFrequencyReservation_reservationKey_key"
|
||||||
|
ON "PhoneFrequencyReservation"("reservationKey");
|
||||||
|
CREATE INDEX "PhoneFrequencyReservation_applicationId_createdAt_idx"
|
||||||
|
ON "PhoneFrequencyReservation"("applicationId", "createdAt");
|
||||||
|
CREATE INDEX "PhoneFrequencyReservation_tenantId_createdAt_idx"
|
||||||
|
ON "PhoneFrequencyReservation"("tenantId", "createdAt");
|
||||||
|
ALTER TABLE "PhoneFrequencyReservation" ADD CONSTRAINT "PhoneFrequencyReservation_tenantId_fkey"
|
||||||
|
FOREIGN KEY ("tenantId") REFERENCES "Tenant"("id") ON DELETE RESTRICT ON UPDATE CASCADE;
|
||||||
|
ALTER TABLE "PhoneFrequencyReservation" ADD CONSTRAINT "PhoneFrequencyReservation_applicationId_fkey"
|
||||||
|
FOREIGN KEY ("applicationId") REFERENCES "SmsApplication"("id") ON DELETE RESTRICT ON UPDATE CASCADE;
|
||||||
@@ -52,6 +52,9 @@ model Tenant {
|
|||||||
openApiRequests OpenApiRequest[]
|
openApiRequests OpenApiRequest[]
|
||||||
httpWebhookEvents HttpWebhookEvent[]
|
httpWebhookEvents HttpWebhookEvent[]
|
||||||
cmppInboundLongMessages CmppInboundLongMessage[]
|
cmppInboundLongMessages CmppInboundLongMessage[]
|
||||||
|
cmppInboundSubmissionInboxes CmppInboundSubmissionInbox[]
|
||||||
|
smsApplicationDailyReservations SmsApplicationDailyReservation[]
|
||||||
|
phoneFrequencyReservations PhoneFrequencyReservation[]
|
||||||
}
|
}
|
||||||
|
|
||||||
model EnterpriseCertification {
|
model EnterpriseCertification {
|
||||||
@@ -467,10 +470,13 @@ model SmsApplication {
|
|||||||
httpWebhookEndpoints HttpWebhookEndpoint[]
|
httpWebhookEndpoints HttpWebhookEndpoint[]
|
||||||
httpWebhookEvents HttpWebhookEvent[]
|
httpWebhookEvents HttpWebhookEvent[]
|
||||||
dailyUsages SmsApplicationDailyUsage[]
|
dailyUsages SmsApplicationDailyUsage[]
|
||||||
|
dailyReservations SmsApplicationDailyReservation[]
|
||||||
inboundLongMessages CmppInboundLongMessage[]
|
inboundLongMessages CmppInboundLongMessage[]
|
||||||
|
inboundSubmissionInboxes CmppInboundSubmissionInbox[]
|
||||||
riskRules RiskRule[]
|
riskRules RiskRule[]
|
||||||
phoneFrequencyStates PhoneFrequencyState[]
|
phoneFrequencyStates PhoneFrequencyState[]
|
||||||
phoneFrequencyHits PhoneFrequencyHit[]
|
phoneFrequencyHits PhoneFrequencyHit[]
|
||||||
|
phoneFrequencyReservations PhoneFrequencyReservation[]
|
||||||
|
|
||||||
@@index([tenantId, status])
|
@@index([tenantId, status])
|
||||||
@@index([status, createdAt])
|
@@index([status, createdAt])
|
||||||
@@ -533,6 +539,25 @@ model SmsApplicationDailyUsage {
|
|||||||
@@index([usageDate])
|
@@index([usageDate])
|
||||||
}
|
}
|
||||||
|
|
||||||
|
model SmsApplicationDailyReservation {
|
||||||
|
id String @id @default(cuid())
|
||||||
|
reservationKey String @unique
|
||||||
|
tenantId String
|
||||||
|
applicationId String
|
||||||
|
usageDate DateTime @db.Date
|
||||||
|
requestedCount Int
|
||||||
|
dailyLimit Int
|
||||||
|
usedCount Int?
|
||||||
|
reserved Boolean
|
||||||
|
createdAt DateTime @default(now())
|
||||||
|
|
||||||
|
tenant Tenant @relation(fields: [tenantId], references: [id])
|
||||||
|
application SmsApplication @relation(fields: [applicationId], references: [id], onDelete: Restrict)
|
||||||
|
|
||||||
|
@@index([applicationId, usageDate])
|
||||||
|
@@index([tenantId, createdAt])
|
||||||
|
}
|
||||||
|
|
||||||
model SmsApplicationHttpIpAllowlist {
|
model SmsApplicationHttpIpAllowlist {
|
||||||
id String @id @default(cuid())
|
id String @id @default(cuid())
|
||||||
applicationId String
|
applicationId String
|
||||||
@@ -1578,6 +1603,21 @@ model PhoneFrequencyHit {
|
|||||||
@@index([releasedById])
|
@@index([releasedById])
|
||||||
}
|
}
|
||||||
|
|
||||||
|
model PhoneFrequencyReservation {
|
||||||
|
id String @id @default(cuid())
|
||||||
|
reservationKey String @unique
|
||||||
|
tenantId String
|
||||||
|
applicationId String
|
||||||
|
result Json
|
||||||
|
createdAt DateTime @default(now())
|
||||||
|
|
||||||
|
tenant Tenant @relation(fields: [tenantId], references: [id])
|
||||||
|
application SmsApplication @relation(fields: [applicationId], references: [id], onDelete: Restrict)
|
||||||
|
|
||||||
|
@@index([applicationId, createdAt])
|
||||||
|
@@index([tenantId, createdAt])
|
||||||
|
}
|
||||||
|
|
||||||
model PhoneFrequencyWhitelist {
|
model PhoneFrequencyWhitelist {
|
||||||
id String @id @default(cuid())
|
id String @id @default(cuid())
|
||||||
phoneNumber String @unique
|
phoneNumber String @unique
|
||||||
@@ -1952,6 +1992,35 @@ model CmppInboundLongMessageSegment {
|
|||||||
@@index([sequenceId])
|
@@index([sequenceId])
|
||||||
}
|
}
|
||||||
|
|
||||||
|
model CmppInboundSubmissionInbox {
|
||||||
|
id String @id @default(cuid())
|
||||||
|
requestKey String @unique
|
||||||
|
payloadHash String
|
||||||
|
tenantId String
|
||||||
|
applicationId String
|
||||||
|
queuePriority String @default("normal")
|
||||||
|
payload Json
|
||||||
|
response Json
|
||||||
|
status String @default("pending")
|
||||||
|
attempts Int @default(0)
|
||||||
|
nextAttemptAt DateTime @default(now())
|
||||||
|
lockedAt DateTime?
|
||||||
|
lockedBy String?
|
||||||
|
lastError String?
|
||||||
|
result Json?
|
||||||
|
completedAt DateTime?
|
||||||
|
createdAt DateTime @default(now())
|
||||||
|
updatedAt DateTime @updatedAt
|
||||||
|
|
||||||
|
tenant Tenant @relation(fields: [tenantId], references: [id])
|
||||||
|
application SmsApplication @relation(fields: [applicationId], references: [id])
|
||||||
|
|
||||||
|
@@index([status, nextAttemptAt, createdAt])
|
||||||
|
@@index([status, lockedAt])
|
||||||
|
@@index([applicationId, createdAt])
|
||||||
|
@@index([queuePriority, status, createdAt])
|
||||||
|
}
|
||||||
|
|
||||||
model SmsReceiptRecord {
|
model SmsReceiptRecord {
|
||||||
id String @id @default(cuid())
|
id String @id @default(cuid())
|
||||||
tenantId String?
|
tenantId String?
|
||||||
|
|||||||
@@ -6,6 +6,7 @@ const CMPP_INBOUND_DURATION_BUCKETS = [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5,
|
|||||||
|
|
||||||
export type CmppInboundStage =
|
export type CmppInboundStage =
|
||||||
| 'application_lookup'
|
| 'application_lookup'
|
||||||
|
| 'inbox_persist'
|
||||||
| 'long_message_fragment'
|
| 'long_message_fragment'
|
||||||
| 'submission_precheck'
|
| 'submission_precheck'
|
||||||
| 'template_match'
|
| 'template_match'
|
||||||
@@ -45,6 +46,12 @@ export class MetricsService implements OnModuleDestroy {
|
|||||||
private readonly http = new Map<string, HttpMetric>();
|
private readonly http = new Map<string, HttpMetric>();
|
||||||
private readonly cmppInbound = new Map<string, HttpMetric>();
|
private readonly cmppInbound = new Map<string, HttpMetric>();
|
||||||
private inFlight = 0;
|
private inFlight = 0;
|
||||||
|
private inboundWorkflowPending = 0;
|
||||||
|
private inboundWorkflowProcessing = 0;
|
||||||
|
private inboundWorkflowOldestPendingAgeSeconds = 0;
|
||||||
|
private inboundWorkflowConfiguredSlots = 0;
|
||||||
|
private inboundWorkflowInFlightSlots = 0;
|
||||||
|
private readonly inboundWorkflowResults = new Map<string, number>();
|
||||||
|
|
||||||
constructor() {
|
constructor() {
|
||||||
this.eventLoopDelay.enable();
|
this.eventLoopDelay.enable();
|
||||||
@@ -91,6 +98,21 @@ export class MetricsService implements OnModuleDestroy {
|
|||||||
this.cmppInbound.set(key, metric);
|
this.cmppInbound.set(key, metric);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
setInboundWorkflowState(pending: number, processing: number, oldestPendingAgeSeconds: number) {
|
||||||
|
this.inboundWorkflowPending = Math.max(0, pending);
|
||||||
|
this.inboundWorkflowProcessing = Math.max(0, processing);
|
||||||
|
this.inboundWorkflowOldestPendingAgeSeconds = Math.max(0, oldestPendingAgeSeconds);
|
||||||
|
}
|
||||||
|
|
||||||
|
setInboundWorkflowSlots(configured: number, inFlight: number) {
|
||||||
|
this.inboundWorkflowConfiguredSlots = Math.max(0, configured);
|
||||||
|
this.inboundWorkflowInFlightSlots = Math.max(0, inFlight);
|
||||||
|
}
|
||||||
|
|
||||||
|
recordInboundWorkflowResult(result: 'completed' | 'retry') {
|
||||||
|
this.inboundWorkflowResults.set(result, (this.inboundWorkflowResults.get(result) ?? 0) + 1);
|
||||||
|
}
|
||||||
|
|
||||||
render() {
|
render() {
|
||||||
const memory = process.memoryUsage();
|
const memory = process.memoryUsage();
|
||||||
const uptime = Number(process.hrtime.bigint() - this.startedAt) / 1_000_000_000;
|
const uptime = Number(process.hrtime.bigint() - this.startedAt) / 1_000_000_000;
|
||||||
@@ -119,6 +141,19 @@ export class MetricsService implements OnModuleDestroy {
|
|||||||
'# TYPE cmpp_api_http_request_duration_seconds histogram',
|
'# TYPE cmpp_api_http_request_duration_seconds histogram',
|
||||||
'# HELP cmpp_api_cmpp_inbound_stage_duration_seconds CMPP inbound processing duration by bounded stage and result.',
|
'# HELP cmpp_api_cmpp_inbound_stage_duration_seconds CMPP inbound processing duration by bounded stage and result.',
|
||||||
'# TYPE cmpp_api_cmpp_inbound_stage_duration_seconds histogram',
|
'# TYPE cmpp_api_cmpp_inbound_stage_duration_seconds histogram',
|
||||||
|
'# HELP cmpp_worker_inbound_workflow_items Current durable CMPP inbound workflow items by state.',
|
||||||
|
'# TYPE cmpp_worker_inbound_workflow_items gauge',
|
||||||
|
metricLine('cmpp_worker_inbound_workflow_items', this.inboundWorkflowPending, { state: 'pending' }),
|
||||||
|
metricLine('cmpp_worker_inbound_workflow_items', this.inboundWorkflowProcessing, { state: 'processing' }),
|
||||||
|
'# HELP cmpp_worker_inbound_workflow_slots Durable workflow worker slots by state.',
|
||||||
|
'# TYPE cmpp_worker_inbound_workflow_slots gauge',
|
||||||
|
metricLine('cmpp_worker_inbound_workflow_slots', this.inboundWorkflowConfiguredSlots, { state: 'configured' }),
|
||||||
|
metricLine('cmpp_worker_inbound_workflow_slots', this.inboundWorkflowInFlightSlots, { state: 'in_flight' }),
|
||||||
|
'# HELP cmpp_worker_inbound_workflow_oldest_pending_age_seconds Age of the oldest pending durable workflow.',
|
||||||
|
'# TYPE cmpp_worker_inbound_workflow_oldest_pending_age_seconds gauge',
|
||||||
|
metricLine('cmpp_worker_inbound_workflow_oldest_pending_age_seconds', this.inboundWorkflowOldestPendingAgeSeconds),
|
||||||
|
'# HELP cmpp_worker_inbound_workflow_results_total Durable workflow processing outcomes.',
|
||||||
|
'# TYPE cmpp_worker_inbound_workflow_results_total counter',
|
||||||
];
|
];
|
||||||
for (const [key, metric] of this.http) {
|
for (const [key, metric] of this.http) {
|
||||||
const [method, route, status] = key.split('\u0000');
|
const [method, route, status] = key.split('\u0000');
|
||||||
@@ -141,6 +176,9 @@ export class MetricsService implements OnModuleDestroy {
|
|||||||
lines.push(metricLine('cmpp_api_cmpp_inbound_stage_duration_seconds_sum', metric.durationSum, labels));
|
lines.push(metricLine('cmpp_api_cmpp_inbound_stage_duration_seconds_sum', metric.durationSum, labels));
|
||||||
lines.push(metricLine('cmpp_api_cmpp_inbound_stage_duration_seconds_count', metric.count, labels));
|
lines.push(metricLine('cmpp_api_cmpp_inbound_stage_duration_seconds_count', metric.count, labels));
|
||||||
}
|
}
|
||||||
|
for (const [result, count] of this.inboundWorkflowResults) {
|
||||||
|
lines.push(metricLine('cmpp_worker_inbound_workflow_results_total', count, { result }));
|
||||||
|
}
|
||||||
this.eventLoopDelay.reset();
|
this.eventLoopDelay.reset();
|
||||||
return `${lines.join('\n')}\n`;
|
return `${lines.join('\n')}\n`;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -6,9 +6,12 @@ import { requestContext } from '../common/request-context';
|
|||||||
@Injectable()
|
@Injectable()
|
||||||
export class PrismaService extends PrismaClient implements OnModuleDestroy {
|
export class PrismaService extends PrismaClient implements OnModuleDestroy {
|
||||||
constructor() {
|
constructor() {
|
||||||
|
const databaseUrl = process.env.CMPP_PROCESS_ROLE === 'worker'
|
||||||
|
? process.env.API_WORKER_DATABASE_URL || process.env.DATABASE_URL
|
||||||
|
: process.env.DATABASE_URL;
|
||||||
super({
|
super({
|
||||||
adapter: new PrismaPg(
|
adapter: new PrismaPg(
|
||||||
process.env.DATABASE_URL ??
|
databaseUrl ??
|
||||||
'postgresql://cmpp:cmpp_password@localhost:5432/cmpp_platform?schema=public',
|
'postgresql://cmpp:cmpp_password@localhost:5432/cmpp_platform?schema=public',
|
||||||
),
|
),
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -80,6 +80,7 @@ export class PhoneFrequencyService {
|
|||||||
phones: string[],
|
phones: string[],
|
||||||
sourceType?: string,
|
sourceType?: string,
|
||||||
requestedAt = new Date(),
|
requestedAt = new Date(),
|
||||||
|
reservationKey?: string,
|
||||||
) {
|
) {
|
||||||
if (!applicationId) return new Map<string, PhoneFrequencyRejection>();
|
if (!applicationId) return new Map<string, PhoneFrequencyRejection>();
|
||||||
const normalizedPhones = [...new Set(phones.map((phone) => phone.trim()).filter(Boolean))].sort();
|
const normalizedPhones = [...new Set(phones.map((phone) => phone.trim()).filter(Boolean))].sort();
|
||||||
@@ -87,14 +88,35 @@ export class PhoneFrequencyService {
|
|||||||
|
|
||||||
await this.riskReview.ensureDefaultRules();
|
await this.riskReview.ensureDefaultRules();
|
||||||
const rules = await this.effectiveRules(applicationId);
|
const rules = await this.effectiveRules(applicationId);
|
||||||
if (rules.length === 0) return new Map<string, PhoneFrequencyRejection>();
|
const normalizedReservationKey = reservationKey?.trim();
|
||||||
|
|
||||||
return this.prisma.$transaction(async (tx) => {
|
return this.prisma.$transaction(async (tx) => {
|
||||||
|
if (normalizedReservationKey) {
|
||||||
|
// Frequency counters and the reservation result commit together. Retrying a reclaimed
|
||||||
|
// Inbox item therefore returns the original decision without incrementing either window.
|
||||||
|
await tx.$executeRaw`SELECT pg_advisory_xact_lock(hashtextextended(${'phone-frequency:' + normalizedReservationKey}, 0))`;
|
||||||
|
const existingReservation = await tx.phoneFrequencyReservation.findUnique({
|
||||||
|
where: { reservationKey: normalizedReservationKey },
|
||||||
|
});
|
||||||
|
if (existingReservation) {
|
||||||
|
if (existingReservation.tenantId !== tenantId || existingReservation.applicationId !== applicationId) {
|
||||||
|
throw new BadRequestException('号码频控幂等键已用于另一笔预留');
|
||||||
|
}
|
||||||
|
return frequencyRejectionsFromJson(existingReservation.result);
|
||||||
|
}
|
||||||
|
}
|
||||||
const rejected = new Map<string, PhoneFrequencyRejection>();
|
const rejected = new Map<string, PhoneFrequencyRejection>();
|
||||||
// 平台级白名单只截断号码频控链路;调用 reserve 之前已执行的格式、黑名单等校验不受影响。
|
// 平台级白名单只截断号码频控链路;调用 reserve 之前已执行的格式、黑名单等校验不受影响。
|
||||||
const whitelistedPhones = await this.findActiveWhitelistedPhones(tx, normalizedPhones);
|
const whitelistedPhones = await this.findActiveWhitelistedPhones(tx, normalizedPhones);
|
||||||
const controlledPhones = normalizedPhones.filter((phone) => !whitelistedPhones.has(phone));
|
const controlledPhones = normalizedPhones.filter((phone) => !whitelistedPhones.has(phone));
|
||||||
if (controlledPhones.length === 0) return rejected;
|
if (controlledPhones.length === 0 || rules.length === 0) {
|
||||||
|
if (normalizedReservationKey) {
|
||||||
|
await tx.phoneFrequencyReservation.create({
|
||||||
|
data: { reservationKey: normalizedReservationKey, tenantId, applicationId, result: [] },
|
||||||
|
});
|
||||||
|
}
|
||||||
|
return rejected;
|
||||||
|
}
|
||||||
for (const rule of rules) {
|
for (const rule of rules) {
|
||||||
const window = fixedShanghaiWindow(requestedAt, readPeriodSeconds(rule));
|
const window = fixedShanghaiWindow(requestedAt, readPeriodSeconds(rule));
|
||||||
// 分块限制 SQL 参数数量,但两条规则的全部分块仍在同一事务中提交或回滚。
|
// 分块限制 SQL 参数数量,但两条规则的全部分块仍在同一事务中提交或回滚。
|
||||||
@@ -144,6 +166,16 @@ export class PhoneFrequencyService {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if (normalizedReservationKey) {
|
||||||
|
await tx.phoneFrequencyReservation.create({
|
||||||
|
data: {
|
||||||
|
reservationKey: normalizedReservationKey,
|
||||||
|
tenantId,
|
||||||
|
applicationId,
|
||||||
|
result: [...rejected.entries()].map(([phoneNumber, rejection]) => ({ phoneNumber, ...rejection })),
|
||||||
|
},
|
||||||
|
});
|
||||||
|
}
|
||||||
return rejected;
|
return rejected;
|
||||||
}, { isolationLevel: Prisma.TransactionIsolationLevel.ReadCommitted });
|
}, { isolationLevel: Prisma.TransactionIsolationLevel.ReadCommitted });
|
||||||
}
|
}
|
||||||
@@ -566,6 +598,18 @@ export class PhoneFrequencyService {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function frequencyRejectionsFromJson(value: Prisma.JsonValue) {
|
||||||
|
const result = new Map<string, PhoneFrequencyRejection>();
|
||||||
|
if (!Array.isArray(value)) return result;
|
||||||
|
for (const item of value) {
|
||||||
|
if (!item || typeof item !== 'object' || Array.isArray(item)) continue;
|
||||||
|
const phoneNumber = typeof item.phoneNumber === 'string' ? item.phoneNumber : '';
|
||||||
|
const reason = typeof item.reason === 'string' ? item.reason : '';
|
||||||
|
if (phoneNumber && reason) result.set(phoneNumber, { code: 'PHONE_FREQUENCY_LIMIT', reason });
|
||||||
|
}
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
const whitelistUserInclude = {
|
const whitelistUserInclude = {
|
||||||
createdBy: { select: { id: true, username: true, displayName: true } },
|
createdBy: { select: { id: true, username: true, displayName: true } },
|
||||||
updatedBy: { select: { id: true, username: true, displayName: true } },
|
updatedBy: { select: { id: true, username: true, displayName: true } },
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
import { BadRequestException, HttpException, HttpStatus, Logger, NotFoundException } from '@nestjs/common';
|
import { BadRequestException, ConflictException, HttpException, HttpStatus, Logger, NotFoundException } from '@nestjs/common';
|
||||||
import { Prisma } from '@prisma/client';
|
import { Prisma } from '@prisma/client';
|
||||||
import { Queue, Worker } from 'bullmq';
|
import { Queue, Worker } from 'bullmq';
|
||||||
import IORedis from 'ioredis';
|
import IORedis from 'ioredis';
|
||||||
@@ -525,15 +525,16 @@ async reserveDailySendQuota(applicationId: string, requestedCount: number) {
|
|||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
|
|
||||||
async tryReserveDailySendQuota(applicationId: string, requestedCount: number) {
|
async tryReserveDailySendQuota(applicationId: string, requestedCount: number, reservationKey?: string) {
|
||||||
if (!Number.isInteger(requestedCount) || requestedCount <= 0) {
|
if (!Number.isInteger(requestedCount) || requestedCount <= 0) {
|
||||||
throw new BadRequestException('发送号码数量必须为正整数');
|
throw new BadRequestException('发送号码数量必须为正整数');
|
||||||
}
|
}
|
||||||
const usageDate = shanghaiDateKey();
|
const usageDate = shanghaiDateKey();
|
||||||
const reservationId = randomUUID();
|
const reserve = (client: Pick<Prisma.TransactionClient, '$queryRaw'>) => {
|
||||||
const rows = await this.prisma.$queryRaw<Array<{ dailyLimit: number; usedCount: number | null }>>(Prisma.sql`
|
const reservationId = randomUUID();
|
||||||
|
return client.$queryRaw<Array<{ tenantId: string; dailyLimit: number; usedCount: number | null }>>(Prisma.sql`
|
||||||
WITH application_limit AS (
|
WITH application_limit AS (
|
||||||
SELECT id, COALESCE("dailyLimit", 100000)::integer AS "dailyLimit"
|
SELECT id, "tenantId", COALESCE("dailyLimit", 100000)::integer AS "dailyLimit"
|
||||||
FROM "SmsApplication"
|
FROM "SmsApplication"
|
||||||
WHERE id = ${applicationId}
|
WHERE id = ${applicationId}
|
||||||
), reservation AS (
|
), reservation AS (
|
||||||
@@ -550,10 +551,49 @@ async tryReserveDailySendQuota(applicationId: string, requestedCount: number) {
|
|||||||
<= (SELECT "dailyLimit" FROM application_limit)
|
<= (SELECT "dailyLimit" FROM application_limit)
|
||||||
RETURNING "usedCount"
|
RETURNING "usedCount"
|
||||||
)
|
)
|
||||||
SELECT application_limit."dailyLimit", reservation."usedCount"
|
SELECT application_limit."tenantId", application_limit."dailyLimit", reservation."usedCount"
|
||||||
FROM application_limit
|
FROM application_limit
|
||||||
LEFT JOIN reservation ON TRUE
|
LEFT JOIN reservation ON TRUE
|
||||||
`);
|
`);
|
||||||
|
};
|
||||||
|
const normalizedReservationKey = reservationKey?.trim();
|
||||||
|
const rows = normalizedReservationKey
|
||||||
|
? await this.prisma.$transaction(async (tx) => {
|
||||||
|
// The quota increment and its idempotency record share one short transaction. A worker
|
||||||
|
// crash can therefore neither lose a successful reservation nor increment it twice.
|
||||||
|
await tx.$executeRaw`SELECT pg_advisory_xact_lock(hashtextextended(${'daily-quota:' + normalizedReservationKey}, 0))`;
|
||||||
|
const existing = await tx.smsApplicationDailyReservation.findUnique({
|
||||||
|
where: { reservationKey: normalizedReservationKey },
|
||||||
|
});
|
||||||
|
if (existing) {
|
||||||
|
if (existing.applicationId !== applicationId || existing.requestedCount !== requestedCount) {
|
||||||
|
throw new ConflictException('日发送配额幂等键已用于另一笔预留');
|
||||||
|
}
|
||||||
|
return [{
|
||||||
|
tenantId: existing.tenantId,
|
||||||
|
dailyLimit: existing.dailyLimit,
|
||||||
|
usedCount: existing.usedCount,
|
||||||
|
}];
|
||||||
|
}
|
||||||
|
const reservedRows = await reserve(tx);
|
||||||
|
if (reservedRows.length > 0) {
|
||||||
|
const row = reservedRows[0];
|
||||||
|
await tx.smsApplicationDailyReservation.create({
|
||||||
|
data: {
|
||||||
|
reservationKey: normalizedReservationKey,
|
||||||
|
tenantId: row.tenantId,
|
||||||
|
applicationId,
|
||||||
|
usageDate,
|
||||||
|
requestedCount,
|
||||||
|
dailyLimit: Number(row.dailyLimit),
|
||||||
|
usedCount: row.usedCount == null ? null : Number(row.usedCount),
|
||||||
|
reserved: row.usedCount != null,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
}
|
||||||
|
return reservedRows;
|
||||||
|
})
|
||||||
|
: await reserve(this.prisma);
|
||||||
if (rows.length === 0) {
|
if (rows.length === 0) {
|
||||||
throw new NotFoundException('短信应用不存在');
|
throw new NotFoundException('短信应用不存在');
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -30,6 +30,7 @@ export interface GatewayInboundAuthDto {
|
|||||||
}
|
}
|
||||||
|
|
||||||
export interface GatewayInboundSubmitDto {
|
export interface GatewayInboundSubmitDto {
|
||||||
|
requestId?: string;
|
||||||
account: string;
|
account: string;
|
||||||
phoneNumber?: string;
|
phoneNumber?: string;
|
||||||
phoneNumbers?: string[];
|
phoneNumbers?: string[];
|
||||||
|
|||||||
@@ -329,6 +329,13 @@ function createPrismaMock() {
|
|||||||
lastError: 'downstream client is not connected',
|
lastError: 'downstream client is not connected',
|
||||||
}),
|
}),
|
||||||
},
|
},
|
||||||
|
cmppInboundSubmissionInbox: {
|
||||||
|
create: jest.fn().mockResolvedValue({ id: 'inbox-1' }),
|
||||||
|
findUnique: jest.fn().mockResolvedValue(null),
|
||||||
|
updateMany: jest.fn().mockResolvedValue({ count: 1 }),
|
||||||
|
count: jest.fn().mockResolvedValue(0),
|
||||||
|
findFirst: jest.fn().mockResolvedValue(null),
|
||||||
|
},
|
||||||
smsBillingRecord: {
|
smsBillingRecord: {
|
||||||
findFirst: jest.fn().mockResolvedValue(null),
|
findFirst: jest.fn().mockResolvedValue(null),
|
||||||
create: jest.fn().mockResolvedValue({ id: 'bill-1' }),
|
create: jest.fn().mockResolvedValue({ id: 'bill-1' }),
|
||||||
@@ -2098,6 +2105,95 @@ describe('SendChainService', () => {
|
|||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('returns from the durable CMPP Inbox fast path before risk, billing, or queue publication', async () => {
|
||||||
|
const previous = process.env.CMPP_INBOUND_FAST_PATH_ENABLED;
|
||||||
|
process.env.CMPP_INBOUND_FAST_PATH_ENABLED = 'true';
|
||||||
|
try {
|
||||||
|
const { service, prisma, billing, riskReview, phoneFrequency } = createService();
|
||||||
|
service.enqueueBatchTask = jest.fn();
|
||||||
|
|
||||||
|
const result = await service.submitInboundMessage({
|
||||||
|
requestId: 'cmpp-inbound:test-fast-path',
|
||||||
|
account: '100001',
|
||||||
|
phoneNumbers: ['13800000001', '13900000002'],
|
||||||
|
content: 'hello',
|
||||||
|
sequenceId: 777,
|
||||||
|
remoteIp: '127.0.0.1',
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(result).toEqual(expect.objectContaining({
|
||||||
|
accepted: true,
|
||||||
|
status: 'accepted_pending',
|
||||||
|
phoneCount: 2,
|
||||||
|
}));
|
||||||
|
expect(prisma.cmppInboundSubmissionInbox.create).toHaveBeenCalledWith({
|
||||||
|
data: expect.objectContaining({
|
||||||
|
requestKey: 'cmpp-inbound:test-fast-path',
|
||||||
|
tenantId: 'tenant-1',
|
||||||
|
applicationId: 'app-1',
|
||||||
|
}),
|
||||||
|
});
|
||||||
|
expect(prisma.smsBatchTask.create).not.toHaveBeenCalled();
|
||||||
|
expect(prisma.smsMessageRecord.create).not.toHaveBeenCalled();
|
||||||
|
expect(riskReview.evaluateTask).not.toHaveBeenCalled();
|
||||||
|
expect(phoneFrequency.reserve).not.toHaveBeenCalled();
|
||||||
|
expect(billing.freeze).not.toHaveBeenCalled();
|
||||||
|
expect(service.enqueueBatchTask).not.toHaveBeenCalled();
|
||||||
|
} finally {
|
||||||
|
if (previous == null) delete process.env.CMPP_INBOUND_FAST_PATH_ENABLED;
|
||||||
|
else process.env.CMPP_INBOUND_FAST_PATH_ENABLED = previous;
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
it('returns the stored SubmitResp for an idempotent CMPP Inbox retry', async () => {
|
||||||
|
const previous = process.env.CMPP_INBOUND_FAST_PATH_ENABLED;
|
||||||
|
process.env.CMPP_INBOUND_FAST_PATH_ENABLED = 'true';
|
||||||
|
try {
|
||||||
|
const { service, prisma } = createService();
|
||||||
|
let originalHash = '';
|
||||||
|
const storedResponse = {
|
||||||
|
accepted: true,
|
||||||
|
tenantId: 'tenant-1',
|
||||||
|
applicationId: 'app-1',
|
||||||
|
taskId: '',
|
||||||
|
messageId: 'MSG-stable',
|
||||||
|
messageRecordId: '',
|
||||||
|
status: 'accepted_pending',
|
||||||
|
phoneCount: 1,
|
||||||
|
messages: [{ phoneNumber: '13800000001', messageId: 'MSG-stable', messageRecordId: '', taskId: '', status: 'accepted_pending' }],
|
||||||
|
};
|
||||||
|
prisma.cmppInboundSubmissionInbox.create
|
||||||
|
.mockImplementationOnce(({ data }) => {
|
||||||
|
originalHash = data.payloadHash;
|
||||||
|
return Promise.resolve({ id: 'inbox-1' });
|
||||||
|
})
|
||||||
|
.mockRejectedValueOnce(new Prisma.PrismaClientKnownRequestError('duplicate request key', {
|
||||||
|
code: 'P2002',
|
||||||
|
clientVersion: '7.9.0',
|
||||||
|
}));
|
||||||
|
prisma.cmppInboundSubmissionInbox.findUnique.mockImplementation(() => Promise.resolve({
|
||||||
|
payloadHash: originalHash,
|
||||||
|
response: storedResponse,
|
||||||
|
}));
|
||||||
|
const request = {
|
||||||
|
requestId: 'cmpp-inbound:test-retry',
|
||||||
|
account: '100001',
|
||||||
|
phoneNumber: '13800000001',
|
||||||
|
content: 'hello',
|
||||||
|
sequenceId: 778,
|
||||||
|
remoteIp: '127.0.0.1',
|
||||||
|
};
|
||||||
|
|
||||||
|
await service.submitInboundMessage(request);
|
||||||
|
await expect(service.submitInboundMessage(request)).resolves.toEqual(storedResponse);
|
||||||
|
expect(prisma.cmppInboundSubmissionInbox.create).toHaveBeenCalledTimes(2);
|
||||||
|
expect(prisma.smsMessageRecord.create).not.toHaveBeenCalled();
|
||||||
|
} finally {
|
||||||
|
if (previous == null) delete process.env.CMPP_INBOUND_FAST_PATH_ENABLED;
|
||||||
|
else process.env.CMPP_INBOUND_FAST_PATH_ENABLED = previous;
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
it('enqueues a freshly persisted inbound message without querying the task and message again', async () => {
|
it('enqueues a freshly persisted inbound message without querying the task and message again', async () => {
|
||||||
const { service, prisma } = createService();
|
const { service, prisma } = createService();
|
||||||
const add = jest.fn().mockResolvedValue(undefined);
|
const add = jest.fn().mockResolvedValue(undefined);
|
||||||
|
|||||||
@@ -78,9 +78,14 @@ export class SendChainService implements OnModuleInit, OnModuleDestroy {
|
|||||||
}
|
}
|
||||||
|
|
||||||
onModuleInit() {
|
onModuleInit() {
|
||||||
|
const processRole = process.env.CMPP_PROCESS_ROLE?.trim() || 'all';
|
||||||
|
if (processRole === 'api') return;
|
||||||
if (process.env.API_ENABLE_SEND_WORKER === 'true') {
|
if (process.env.API_ENABLE_SEND_WORKER === 'true') {
|
||||||
this.startWorker();
|
this.startWorker();
|
||||||
}
|
}
|
||||||
|
if (process.env.CMPP_INBOUND_WORKFLOW_WORKER_ENABLED === 'true') {
|
||||||
|
this.submission.startInboundWorkflowWorker();
|
||||||
|
}
|
||||||
if (process.env.SMS_RECEIPT_TIMEOUT_SCAN_ENABLED !== 'false') {
|
if (process.env.SMS_RECEIPT_TIMEOUT_SCAN_ENABLED !== 'false') {
|
||||||
this.receiptTimeoutInitialTimer = setTimeout(() => void this.runReceiptTimeoutScan(), RECEIPT_TIMEOUT_INITIAL_DELAY_MS);
|
this.receiptTimeoutInitialTimer = setTimeout(() => void this.runReceiptTimeoutScan(), RECEIPT_TIMEOUT_INITIAL_DELAY_MS);
|
||||||
this.receiptTimeoutInitialTimer.unref?.();
|
this.receiptTimeoutInitialTimer.unref?.();
|
||||||
@@ -159,6 +164,7 @@ export class SendChainService implements OnModuleInit, OnModuleDestroy {
|
|||||||
await this.sendQueue?.close();
|
await this.sendQueue?.close();
|
||||||
await this.gatewayQueue?.close();
|
await this.gatewayQueue?.close();
|
||||||
this.redis?.disconnect();
|
this.redis?.disconnect();
|
||||||
|
await this.submission.onModuleDestroy();
|
||||||
}
|
}
|
||||||
|
|
||||||
async createBatchTask(data: CreateBatchTaskDto) {
|
async createBatchTask(data: CreateBatchTaskDto) {
|
||||||
@@ -579,8 +585,10 @@ export class SendChainService implements OnModuleInit, OnModuleDestroy {
|
|||||||
phoneNumbers: string[],
|
phoneNumbers: string[],
|
||||||
application: Awaited<ReturnType<SendChainService['findInboundApplication']>>,
|
application: Awaited<ReturnType<SendChainService['findInboundApplication']>>,
|
||||||
requestedGroupMessageId?: string,
|
requestedGroupMessageId?: string,
|
||||||
|
requestedMessageIds?: string[],
|
||||||
|
workflowKey?: string,
|
||||||
) {
|
) {
|
||||||
return this.submission.submitCompleteInboundMessage(data, phoneNumbers, application, requestedGroupMessageId);
|
return this.submission.submitCompleteInboundMessage(data, phoneNumbers, application, requestedGroupMessageId, requestedMessageIds, workflowKey);
|
||||||
}
|
}
|
||||||
|
|
||||||
private async collectInboundLongMessageFragment(
|
private async collectInboundLongMessageFragment(
|
||||||
@@ -602,8 +610,9 @@ export class SendChainService implements OnModuleInit, OnModuleDestroy {
|
|||||||
application: NonNullable<Awaited<ReturnType<SendChainService['findInboundApplication']>>>,
|
application: NonNullable<Awaited<ReturnType<SendChainService['findInboundApplication']>>>,
|
||||||
synchronousRejection?: { code: string; reason: string },
|
synchronousRejection?: { code: string; reason: string },
|
||||||
receiptRejection?: { code: string; reason: string },
|
receiptRejection?: { code: string; reason: string },
|
||||||
|
workflowItemKey?: string,
|
||||||
) {
|
) {
|
||||||
return this.submission.submitInboundSingleMessage(data, messageId, submitGroupMessageId, application, synchronousRejection, receiptRejection);
|
return this.submission.submitInboundSingleMessage(data, messageId, submitGroupMessageId, application, synchronousRejection, receiptRejection, workflowItemKey);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -618,8 +627,8 @@ export class SendChainService implements OnModuleInit, OnModuleDestroy {
|
|||||||
variables?: Record<string, unknown>;
|
variables?: Record<string, unknown>;
|
||||||
phoneNumber: string;
|
phoneNumber: string;
|
||||||
sourceType: 'cmpp';
|
sourceType: 'cmpp';
|
||||||
}) {
|
}, reservationKey?: string) {
|
||||||
return this.submission.evaluateRiskWithPhoneFrequency(input);
|
return this.submission.evaluateRiskWithPhoneFrequency(input, reservationKey);
|
||||||
}
|
}
|
||||||
|
|
||||||
async markUnknownTimeout(data: TimeoutUnknownDto) {
|
async markUnknownTimeout(data: TimeoutUnknownDto) {
|
||||||
@@ -774,8 +783,8 @@ export class SendChainService implements OnModuleInit, OnModuleDestroy {
|
|||||||
return this.submission.reserveDailySendQuota(applicationId, requestedCount);
|
return this.submission.reserveDailySendQuota(applicationId, requestedCount);
|
||||||
}
|
}
|
||||||
|
|
||||||
private async tryReserveDailySendQuota(applicationId: string, requestedCount: number) {
|
private async tryReserveDailySendQuota(applicationId: string, requestedCount: number, reservationKey?: string) {
|
||||||
return this.submission.tryReserveDailySendQuota(applicationId, requestedCount);
|
return this.submission.tryReserveDailySendQuota(applicationId, requestedCount, reservationKey);
|
||||||
}
|
}
|
||||||
|
|
||||||
private async chargeAcceptedMessage(message: {
|
private async chargeAcceptedMessage(message: {
|
||||||
|
|||||||
@@ -3,6 +3,7 @@ import { Prisma } from '@prisma/client';
|
|||||||
import { Queue, Worker } from 'bullmq';
|
import { Queue, Worker } from 'bullmq';
|
||||||
import IORedis from 'ioredis';
|
import IORedis from 'ioredis';
|
||||||
import { createHash, randomUUID } from 'node:crypto';
|
import { createHash, randomUUID } from 'node:crypto';
|
||||||
|
import { hostname } from 'node:os';
|
||||||
import { setTimeout as sleep } from 'node:timers/promises';
|
import { setTimeout as sleep } from 'node:timers/promises';
|
||||||
import { BillingService } from '../billing/billing.service';
|
import { BillingService } from '../billing/billing.service';
|
||||||
import { isIpAllowed } from '../common/ip-allowlist';
|
import { isIpAllowed } from '../common/ip-allowlist';
|
||||||
@@ -17,11 +18,33 @@ import { SEND_QUEUE, GATEWAY_SUBMIT_QUEUE, GATEWAY_SUBMIT_STREAM, DEFAULT_SCHEDU
|
|||||||
import type { SendSubmissionCallbacks, SendSubmissionService } from './send-submission.service';
|
import type { SendSubmissionCallbacks, SendSubmissionService } from './send-submission.service';
|
||||||
import { detectDrainageContent } from './drainage-content-detection';
|
import { detectDrainageContent } from './drainage-content-detection';
|
||||||
|
|
||||||
|
type InboundWorkflowPayload = {
|
||||||
|
data: GatewayInboundSubmitDto;
|
||||||
|
phoneNumbers: string[];
|
||||||
|
submitGroupMessageId: string;
|
||||||
|
messageIds: string[];
|
||||||
|
};
|
||||||
|
|
||||||
|
type ClaimedInboundWorkflow = {
|
||||||
|
id: string;
|
||||||
|
requestKey: string;
|
||||||
|
applicationId: string;
|
||||||
|
attempts: number;
|
||||||
|
payload: Prisma.JsonValue;
|
||||||
|
};
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* R9 inboundEntry implementation. Cross-method calls return through the stable SendChainService seam.
|
* R9 inboundEntry implementation. Cross-method calls return through the stable SendChainService seam.
|
||||||
*/
|
*/
|
||||||
export class SendInboundEntryService {
|
export class SendInboundEntryService {
|
||||||
private readonly logger = new Logger('SendChainService');
|
private readonly logger = new Logger('SendChainService');
|
||||||
|
private readonly inboundWorkflowWorkerId = `${hostname()}:${process.pid}:${randomUUID()}`;
|
||||||
|
private readonly inboundWorkflowTasks = new Set<Promise<void>>();
|
||||||
|
private inboundWorkflowTimer?: ReturnType<typeof setTimeout>;
|
||||||
|
private inboundWorkflowInFlight = 0;
|
||||||
|
private inboundWorkflowPumping = false;
|
||||||
|
private inboundWorkflowStopping = false;
|
||||||
|
private inboundWorkflowMetricsUpdatedAt = 0;
|
||||||
|
|
||||||
constructor(
|
constructor(
|
||||||
private readonly prisma: PrismaService,
|
private readonly prisma: PrismaService,
|
||||||
@@ -201,6 +224,26 @@ async submitInboundMessage(data: GatewayInboundSubmitDto) {
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
try {
|
try {
|
||||||
|
if (this.inboundFastPathEnabled()) {
|
||||||
|
const response = await this.measureInboundStage('inbox_persist', () => (
|
||||||
|
this.persistInboundWorkflow({
|
||||||
|
...data,
|
||||||
|
content: collection.content,
|
||||||
|
sequenceId: collection.sequenceId,
|
||||||
|
registeredDelivery: collection.registeredDelivery ? 1 : 0,
|
||||||
|
longMessage: undefined,
|
||||||
|
}, phoneNumbers, application, collection.messageId)
|
||||||
|
));
|
||||||
|
await this.prisma.cmppInboundLongMessage.update({
|
||||||
|
where: { id: collection.groupId },
|
||||||
|
data: {
|
||||||
|
status: 'completed',
|
||||||
|
response: JSON.parse(JSON.stringify(response)) as Prisma.InputJsonValue,
|
||||||
|
completedAt: new Date(),
|
||||||
|
},
|
||||||
|
});
|
||||||
|
return response;
|
||||||
|
}
|
||||||
const response = await this.measureInboundStage('complete_submit', async () => (
|
const response = await this.measureInboundStage('complete_submit', async () => (
|
||||||
await this.facade.recoverCompletedInboundLongMessageResponse(
|
await this.facade.recoverCompletedInboundLongMessageResponse(
|
||||||
collection.messageId,
|
collection.messageId,
|
||||||
@@ -233,12 +276,90 @@ async submitInboundMessage(data: GatewayInboundSubmitDto) {
|
|||||||
throw error;
|
throw error;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if (this.inboundFastPathEnabled()) {
|
||||||
|
return this.measureInboundStage(
|
||||||
|
'inbox_persist',
|
||||||
|
() => this.persistInboundWorkflow(data, phoneNumbers, application),
|
||||||
|
);
|
||||||
|
}
|
||||||
return this.measureInboundStage(
|
return this.measureInboundStage(
|
||||||
'complete_submit',
|
'complete_submit',
|
||||||
() => this.facade.submitCompleteInboundMessage(data, phoneNumbers, application),
|
() => this.facade.submitCompleteInboundMessage(data, phoneNumbers, application),
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private inboundFastPathEnabled() {
|
||||||
|
return process.env.CMPP_INBOUND_FAST_PATH_ENABLED === 'true';
|
||||||
|
}
|
||||||
|
|
||||||
|
private async persistInboundWorkflow(
|
||||||
|
data: GatewayInboundSubmitDto,
|
||||||
|
phoneNumbers: string[],
|
||||||
|
application: NonNullable<Awaited<ReturnType<SendSubmissionService['findInboundApplication']>>>,
|
||||||
|
requestedGroupMessageId?: string,
|
||||||
|
) {
|
||||||
|
const requestKey = data.requestId?.trim();
|
||||||
|
if (!requestKey || requestKey.length > 160) {
|
||||||
|
throw new BadRequestException('CMPP inbound requestId is required for fast-path idempotency');
|
||||||
|
}
|
||||||
|
if (data.remoteIp && !isIpAllowed(data.remoteIp, application.ipAllowlist.map((item) => item.ipCidr))) {
|
||||||
|
throw new BadRequestException('CMPP source IP is not in application allowlist');
|
||||||
|
}
|
||||||
|
validateInboundApplicationSrcId(data.srcId, application);
|
||||||
|
const submitGroupMessageId = requestedGroupMessageId ?? `MSG-${randomUUID()}`;
|
||||||
|
const messageIds = phoneNumbers.map((_, index) => index === 0 ? submitGroupMessageId : `MSG-${randomUUID()}`);
|
||||||
|
const payload: InboundWorkflowPayload = {
|
||||||
|
data: JSON.parse(JSON.stringify(data)) as GatewayInboundSubmitDto,
|
||||||
|
phoneNumbers,
|
||||||
|
submitGroupMessageId,
|
||||||
|
messageIds,
|
||||||
|
};
|
||||||
|
const payloadJson = JSON.parse(JSON.stringify(payload)) as Prisma.InputJsonValue;
|
||||||
|
const payloadHash = createHash('sha256').update(JSON.stringify({
|
||||||
|
data: payload.data,
|
||||||
|
phoneNumbers,
|
||||||
|
requestedGroupMessageId: requestedGroupMessageId ?? null,
|
||||||
|
})).digest('hex');
|
||||||
|
const response = {
|
||||||
|
accepted: true,
|
||||||
|
tenantId: application.tenantId,
|
||||||
|
applicationId: application.id,
|
||||||
|
taskId: '',
|
||||||
|
messageId: submitGroupMessageId,
|
||||||
|
messageRecordId: '',
|
||||||
|
status: 'accepted_pending',
|
||||||
|
phoneCount: phoneNumbers.length,
|
||||||
|
messages: phoneNumbers.map((phoneNumber, index) => ({
|
||||||
|
phoneNumber,
|
||||||
|
messageId: messageIds[index],
|
||||||
|
messageRecordId: '',
|
||||||
|
taskId: '',
|
||||||
|
status: 'accepted_pending',
|
||||||
|
})),
|
||||||
|
};
|
||||||
|
try {
|
||||||
|
await this.prisma.cmppInboundSubmissionInbox.create({
|
||||||
|
data: {
|
||||||
|
requestKey,
|
||||||
|
payloadHash,
|
||||||
|
tenantId: application.tenantId,
|
||||||
|
applicationId: application.id,
|
||||||
|
queuePriority: normalizeQueuePriority(application.queuePriority),
|
||||||
|
payload: payloadJson,
|
||||||
|
response: JSON.parse(JSON.stringify(response)) as Prisma.InputJsonValue,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
return response;
|
||||||
|
} catch (error) {
|
||||||
|
if (!(error instanceof Prisma.PrismaClientKnownRequestError) || error.code !== 'P2002') throw error;
|
||||||
|
const existing = await this.prisma.cmppInboundSubmissionInbox.findUnique({ where: { requestKey } });
|
||||||
|
if (!existing || existing.payloadHash !== payloadHash) {
|
||||||
|
throw new BadRequestException('CMPP inbound requestId conflicts with another payload');
|
||||||
|
}
|
||||||
|
return existing.response as typeof response;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
async recoverCompletedInboundLongMessageResponse(messageId: string, phoneNumbers: string[]) {
|
async recoverCompletedInboundLongMessageResponse(messageId: string, phoneNumbers: string[]) {
|
||||||
const existing = await this.prisma.smsMessageRecord.findMany({
|
const existing = await this.prisma.smsMessageRecord.findMany({
|
||||||
where: {
|
where: {
|
||||||
@@ -289,6 +410,8 @@ async submitCompleteInboundMessage(
|
|||||||
phoneNumbers: string[],
|
phoneNumbers: string[],
|
||||||
application: Awaited<ReturnType<SendSubmissionService['findInboundApplication']>>,
|
application: Awaited<ReturnType<SendSubmissionService['findInboundApplication']>>,
|
||||||
requestedGroupMessageId?: string,
|
requestedGroupMessageId?: string,
|
||||||
|
requestedMessageIds?: string[],
|
||||||
|
workflowKey?: string,
|
||||||
) {
|
) {
|
||||||
if (!application) {
|
if (!application) {
|
||||||
throw new BadRequestException('CMPP account is invalid');
|
throw new BadRequestException('CMPP account is invalid');
|
||||||
@@ -309,6 +432,7 @@ async submitCompleteInboundMessage(
|
|||||||
phoneNumber: true,
|
phoneNumber: true,
|
||||||
status: true,
|
status: true,
|
||||||
errorCode: true,
|
errorCode: true,
|
||||||
|
batchTask: { select: { status: true } },
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
: [];
|
: [];
|
||||||
@@ -318,7 +442,11 @@ async submitCompleteInboundMessage(
|
|||||||
!persistedByPhone.has(phoneNumber) && !phoneRejections.has(phoneNumber)
|
!persistedByPhone.has(phoneNumber) && !phoneRejections.has(phoneNumber)
|
||||||
)).length;
|
)).length;
|
||||||
const dailyQuota = missingPhoneCount > 0
|
const dailyQuota = missingPhoneCount > 0
|
||||||
? await this.facade.tryReserveDailySendQuota(application.id, missingPhoneCount)
|
? await this.facade.tryReserveDailySendQuota(
|
||||||
|
application.id,
|
||||||
|
missingPhoneCount,
|
||||||
|
workflowKey ? `${workflowKey}:daily-quota` : undefined,
|
||||||
|
)
|
||||||
: { reserved: true, dailyLimit: application.dailyLimit ?? 100000 };
|
: { reserved: true, dailyLimit: application.dailyLimit ?? 100000 };
|
||||||
return { persistedByPhone, phoneRejections, dailyQuota, missingPhoneCount };
|
return { persistedByPhone, phoneRejections, dailyQuota, missingPhoneCount };
|
||||||
});
|
});
|
||||||
@@ -336,13 +464,19 @@ async submitCompleteInboundMessage(
|
|||||||
persisted: persistedByPhone.get(phoneNumber),
|
persisted: persistedByPhone.get(phoneNumber),
|
||||||
receiptRejection: phoneRejections.get(phoneNumber),
|
receiptRejection: phoneRejections.get(phoneNumber),
|
||||||
messageId: persistedByPhone.get(phoneNumber)?.messageId
|
messageId: persistedByPhone.get(phoneNumber)?.messageId
|
||||||
|
?? requestedMessageIds?.[index]
|
||||||
?? (index === 0 ? submitGroupMessageId : `MSG-${randomUUID()}`),
|
?? (index === 0 ? submitGroupMessageId : `MSG-${randomUUID()}`),
|
||||||
|
workflowItemKey: workflowKey ? `${workflowKey}:message:${index}` : undefined,
|
||||||
}));
|
}));
|
||||||
const results: GatewayInboundSingleSubmitResult[] = [];
|
const results: GatewayInboundSingleSubmitResult[] = [];
|
||||||
const concurrency = 10;
|
const concurrency = 10;
|
||||||
for (let offset = 0; offset < submissions.length; offset += concurrency) {
|
for (let offset = 0; offset < submissions.length; offset += concurrency) {
|
||||||
const batch = submissions.slice(offset, offset + concurrency);
|
const batch = submissions.slice(offset, offset + concurrency);
|
||||||
results.push(...await Promise.all(batch.map((submission) => submission.persisted
|
results.push(...await Promise.all(batch.map((submission) => submission.persisted
|
||||||
|
&& !(workflowKey && (
|
||||||
|
submission.persisted.status === 'validating'
|
||||||
|
|| (submission.persisted.status === 'queued' && submission.persisted.batchTask?.status !== 'queued')
|
||||||
|
))
|
||||||
? Promise.resolve({
|
? Promise.resolve({
|
||||||
accepted: submission.persisted.errorCode !== 'DAILY_LIMIT',
|
accepted: submission.persisted.errorCode !== 'DAILY_LIMIT',
|
||||||
tenantId: submission.persisted.tenantId ?? application.tenantId,
|
tenantId: submission.persisted.tenantId ?? application.tenantId,
|
||||||
@@ -356,7 +490,7 @@ async submitCompleteInboundMessage(
|
|||||||
...data,
|
...data,
|
||||||
phoneNumber: submission.phoneNumber,
|
phoneNumber: submission.phoneNumber,
|
||||||
phoneNumbers: undefined,
|
phoneNumbers: undefined,
|
||||||
}, submission.messageId, submitGroupMessageId, application, submission.receiptRejection ? undefined : dailyLimitRejection, submission.receiptRejection))));
|
}, submission.messageId, submitGroupMessageId, application, submission.receiptRejection ? undefined : dailyLimitRejection, submission.receiptRejection, submission.workflowItemKey))));
|
||||||
}
|
}
|
||||||
const first = results[0];
|
const first = results[0];
|
||||||
return {
|
return {
|
||||||
@@ -553,6 +687,7 @@ async submitInboundSingleMessage(
|
|||||||
application: NonNullable<Awaited<ReturnType<SendSubmissionService['findInboundApplication']>>>,
|
application: NonNullable<Awaited<ReturnType<SendSubmissionService['findInboundApplication']>>>,
|
||||||
synchronousRejection?: { code: string; reason: string },
|
synchronousRejection?: { code: string; reason: string },
|
||||||
receiptRejection?: { code: string; reason: string },
|
receiptRejection?: { code: string; reason: string },
|
||||||
|
workflowItemKey?: string,
|
||||||
) {
|
) {
|
||||||
// 入口已按账号取得并校验同一个应用快照;复用它可避免每个目标号码再次查询应用、企业和IP白名单。
|
// 入口已按账号取得并校验同一个应用快照;复用它可避免每个目标号码再次查询应用、企业和IP白名单。
|
||||||
if (data.remoteIp && !isIpAllowed(data.remoteIp, application.ipAllowlist.map((item) => item.ipCidr))) {
|
if (data.remoteIp && !isIpAllowed(data.remoteIp, application.ipAllowlist.map((item) => item.ipCidr))) {
|
||||||
@@ -573,60 +708,101 @@ async submitInboundSingleMessage(
|
|||||||
phoneCount: 1,
|
phoneCount: 1,
|
||||||
unitPrice,
|
unitPrice,
|
||||||
});
|
});
|
||||||
const task = await this.measureInboundStage('task_persist', () => this.prisma.smsBatchTask.create({
|
|
||||||
data: {
|
|
||||||
tenantId: application.tenantId,
|
|
||||||
applicationId: application.id,
|
|
||||||
templateId: template?.id,
|
|
||||||
taskNo: `BT-${Date.now()}-${randomUUID().slice(0, 8)}`,
|
|
||||||
sourceType: 'cmpp',
|
|
||||||
content: data.content,
|
|
||||||
phoneTotal: 1,
|
|
||||||
status: synchronousRejection ? 'rejected' : 'validating',
|
|
||||||
auditStatus: synchronousRejection ? 'rejected' : undefined,
|
|
||||||
rejectReason: synchronousRejection?.reason,
|
|
||||||
progressTotal: 1,
|
|
||||||
},
|
|
||||||
}));
|
|
||||||
await this.measureInboundStage('api_request_persist', () => this.prisma.smsApiRequest.create({
|
|
||||||
data: {
|
|
||||||
tenantId: application.tenantId,
|
|
||||||
batchTaskId: task.id,
|
|
||||||
requestId: `REQ-${Date.now()}-${randomUUID().slice(0, 8)}`,
|
|
||||||
sourceIp: data.remoteIp,
|
|
||||||
userAgent: 'cmpp-gateway',
|
|
||||||
payloadSummary: { phoneTotal: 1, contentLength: [...data.content].length, account: data.account },
|
|
||||||
status: synchronousRejection ? 'rejected' : 'accepted',
|
|
||||||
},
|
|
||||||
}));
|
|
||||||
const drainageDetection = await this.measureInboundStage(
|
const drainageDetection = await this.measureInboundStage(
|
||||||
'content_detection',
|
'content_detection',
|
||||||
() => detectDrainageContent(this.prisma, data.content),
|
() => detectDrainageContent(this.prisma, data.content),
|
||||||
);
|
);
|
||||||
const message = await this.measureInboundStage('message_persist', () => this.prisma.smsMessageRecord.create({
|
const workflowDigest = workflowItemKey
|
||||||
data: {
|
? createHash('sha256').update(workflowItemKey).digest('hex').slice(0, 32)
|
||||||
tenantId: application.tenantId,
|
: undefined;
|
||||||
batchTaskId: task.id,
|
let recoveredExisting = false;
|
||||||
applicationId: application.id,
|
let persisted;
|
||||||
templateId: template?.id,
|
try {
|
||||||
messageId,
|
persisted = await this.measureInboundStage('message_persist', () => this.prisma.$transaction(async (tx) => {
|
||||||
phoneNumber: data.phoneNumber,
|
const task = await tx.smsBatchTask.create({
|
||||||
content: data.content,
|
data: {
|
||||||
...drainageDetection,
|
tenantId: application.tenantId,
|
||||||
billingUnits: billing.billingUnitsPerMessage,
|
applicationId: application.id,
|
||||||
unitPrice: receiptRejection ? 0 : billing.unitPrice,
|
templateId: template?.id,
|
||||||
amountCents: receiptRejection ? 0 : billing.amountCents,
|
taskNo: workflowDigest ? `BT-IN-${workflowDigest}` : `BT-${Date.now()}-${randomUUID().slice(0, 8)}`,
|
||||||
|
sourceType: 'cmpp',
|
||||||
|
content: data.content,
|
||||||
|
phoneTotal: 1,
|
||||||
|
status: synchronousRejection ? 'rejected' : 'validating',
|
||||||
|
auditStatus: synchronousRejection ? 'rejected' : undefined,
|
||||||
|
rejectReason: synchronousRejection?.reason,
|
||||||
|
progressTotal: 1,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
await tx.smsApiRequest.create({
|
||||||
|
data: {
|
||||||
|
tenantId: application.tenantId,
|
||||||
|
batchTaskId: task.id,
|
||||||
|
requestId: workflowDigest ? `REQ-IN-${workflowDigest}` : `REQ-${Date.now()}-${randomUUID().slice(0, 8)}`,
|
||||||
|
sourceIp: data.remoteIp,
|
||||||
|
userAgent: 'cmpp-gateway',
|
||||||
|
payloadSummary: { phoneTotal: 1, contentLength: [...data.content].length, account: data.account },
|
||||||
|
status: synchronousRejection ? 'rejected' : 'accepted',
|
||||||
|
},
|
||||||
|
});
|
||||||
|
const message = await tx.smsMessageRecord.create({
|
||||||
|
data: {
|
||||||
|
tenantId: application.tenantId,
|
||||||
|
batchTaskId: task.id,
|
||||||
|
applicationId: application.id,
|
||||||
|
templateId: template?.id,
|
||||||
|
messageId,
|
||||||
|
phoneNumber: data.phoneNumber,
|
||||||
|
content: data.content,
|
||||||
|
...drainageDetection,
|
||||||
|
billingUnits: billing.billingUnitsPerMessage,
|
||||||
|
unitPrice: receiptRejection ? 0 : billing.unitPrice,
|
||||||
|
amountCents: receiptRejection ? 0 : billing.amountCents,
|
||||||
|
queuePriority,
|
||||||
|
cmppSubmitSequenceId: data.sequenceId == null ? null : String(data.sequenceId),
|
||||||
|
cmppSubmitGroupMessageId: submitGroupMessageId,
|
||||||
|
cmppRegisteredDelivery: data.registeredDelivery !== 0,
|
||||||
|
clientSrcId,
|
||||||
|
applicationExtension: application.cmppApplicationExtension,
|
||||||
|
status: synchronousRejection ? 'rejected' : 'validating',
|
||||||
|
errorCode: synchronousRejection?.code,
|
||||||
|
errorMessage: synchronousRejection?.reason,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
return { task, message };
|
||||||
|
}));
|
||||||
|
} catch (error) {
|
||||||
|
if (!workflowItemKey || !(error instanceof Prisma.PrismaClientKnownRequestError) || error.code !== 'P2002') throw error;
|
||||||
|
const existing = await this.prisma.smsMessageRecord.findUnique({
|
||||||
|
where: { messageId },
|
||||||
|
include: { batchTask: true },
|
||||||
|
});
|
||||||
|
if (!existing?.batchTask || existing.cmppSubmitGroupMessageId !== submitGroupMessageId
|
||||||
|
|| existing.phoneNumber !== data.phoneNumber || existing.applicationId !== application.id) {
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
recoveredExisting = true;
|
||||||
|
persisted = { task: existing.batchTask, message: existing };
|
||||||
|
}
|
||||||
|
const { task, message } = persisted;
|
||||||
|
|
||||||
|
if (recoveredExisting && message.status === 'queued' && task.status !== 'queued') {
|
||||||
|
await this.facade.enqueueBatchTask(task.id, {
|
||||||
|
messageRecordId: message.id,
|
||||||
queuePriority,
|
queuePriority,
|
||||||
cmppSubmitSequenceId: data.sequenceId == null ? null : String(data.sequenceId),
|
});
|
||||||
cmppSubmitGroupMessageId: submitGroupMessageId,
|
}
|
||||||
cmppRegisteredDelivery: data.registeredDelivery !== 0,
|
if (recoveredExisting && message.status !== 'validating') {
|
||||||
clientSrcId,
|
return {
|
||||||
applicationExtension: application.cmppApplicationExtension,
|
accepted: message.status !== 'rejected' && message.status !== 'failed',
|
||||||
status: synchronousRejection ? 'rejected' : 'validating',
|
tenantId: application.tenantId,
|
||||||
errorCode: synchronousRejection?.code,
|
applicationId: application.id,
|
||||||
errorMessage: synchronousRejection?.reason,
|
taskId: task.id,
|
||||||
},
|
messageId: message.messageId,
|
||||||
}));
|
messageRecordId: message.id,
|
||||||
|
status: message.status,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
if (synchronousRejection) {
|
if (synchronousRejection) {
|
||||||
return {
|
return {
|
||||||
@@ -658,7 +834,7 @@ async submitInboundSingleMessage(
|
|||||||
variables: options.templateId ? templateVariables : undefined,
|
variables: options.templateId ? templateVariables : undefined,
|
||||||
phoneNumber: data.phoneNumber,
|
phoneNumber: data.phoneNumber,
|
||||||
sourceType: 'cmpp',
|
sourceType: 'cmpp',
|
||||||
});
|
}, workflowItemKey ? `${workflowItemKey}:frequency` : undefined);
|
||||||
return { drainageInfoId: drainage?.id, risk: evaluatedRisk };
|
return { drainageInfoId: drainage?.id, risk: evaluatedRisk };
|
||||||
});
|
});
|
||||||
if (risk.status === 'rejected') {
|
if (risk.status === 'rejected') {
|
||||||
@@ -693,6 +869,7 @@ async submitInboundSingleMessage(
|
|||||||
relatedType: 'sms_batch_task',
|
relatedType: 'sms_batch_task',
|
||||||
relatedId: task.id,
|
relatedId: task.id,
|
||||||
remark: 'CMPP 入站短信冻结',
|
remark: 'CMPP 入站短信冻结',
|
||||||
|
idempotencyKey: workflowItemKey ? `${workflowItemKey}:freeze` : undefined,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
return check;
|
return check;
|
||||||
@@ -736,7 +913,7 @@ async submitInboundSingleMessage(
|
|||||||
content: data.content,
|
content: data.content,
|
||||||
phoneNumber: data.phoneNumber,
|
phoneNumber: data.phoneNumber,
|
||||||
sourceType: 'cmpp',
|
sourceType: 'cmpp',
|
||||||
});
|
}, workflowItemKey ? `${workflowItemKey}:frequency` : undefined);
|
||||||
if (risk.status === 'rejected') {
|
if (risk.status === 'rejected') {
|
||||||
await reject('RISK', risk.reason || '短信被风控拒绝');
|
await reject('RISK', risk.reason || '短信被风控拒绝');
|
||||||
} else {
|
} else {
|
||||||
@@ -754,6 +931,7 @@ async submitInboundSingleMessage(
|
|||||||
relatedType: 'sms_batch_task',
|
relatedType: 'sms_batch_task',
|
||||||
relatedId: task.id,
|
relatedId: task.id,
|
||||||
remark: 'CMPP 模板不匹配待审核短信冻结',
|
remark: 'CMPP 模板不匹配待审核短信冻结',
|
||||||
|
idempotencyKey: workflowItemKey ? `${workflowItemKey}:freeze` : undefined,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
const reviewTask = risk.status === 'pending_review' && risk.task
|
const reviewTask = risk.status === 'pending_review' && risk.task
|
||||||
@@ -813,7 +991,7 @@ async evaluateRiskWithPhoneFrequency(input: {
|
|||||||
variables?: Record<string, unknown>;
|
variables?: Record<string, unknown>;
|
||||||
phoneNumber: string;
|
phoneNumber: string;
|
||||||
sourceType: 'cmpp';
|
sourceType: 'cmpp';
|
||||||
}) {
|
}, reservationKey?: string) {
|
||||||
const risk = await this.riskReview.evaluateTask({
|
const risk = await this.riskReview.evaluateTask({
|
||||||
tenantId: input.tenantId,
|
tenantId: input.tenantId,
|
||||||
applicationId: input.applicationId,
|
applicationId: input.applicationId,
|
||||||
@@ -829,6 +1007,8 @@ async evaluateRiskWithPhoneFrequency(input: {
|
|||||||
input.applicationId,
|
input.applicationId,
|
||||||
[input.phoneNumber],
|
[input.phoneNumber],
|
||||||
input.sourceType,
|
input.sourceType,
|
||||||
|
new Date(),
|
||||||
|
reservationKey,
|
||||||
);
|
);
|
||||||
const rejection = frequencyRejections.get(input.phoneNumber);
|
const rejection = frequencyRejections.get(input.phoneNumber);
|
||||||
return rejection
|
return rejection
|
||||||
@@ -836,6 +1016,170 @@ async evaluateRiskWithPhoneFrequency(input: {
|
|||||||
: risk;
|
: risk;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
startInboundWorkflowWorker() {
|
||||||
|
if (this.inboundWorkflowTimer || this.inboundWorkflowPumping || this.inboundWorkflowTasks.size > 0) {
|
||||||
|
return { status: 'already_started' };
|
||||||
|
}
|
||||||
|
this.inboundWorkflowStopping = false;
|
||||||
|
this.scheduleInboundWorkflowPump(0);
|
||||||
|
return { status: 'started' };
|
||||||
|
}
|
||||||
|
|
||||||
|
async stopInboundWorkflowWorker() {
|
||||||
|
this.inboundWorkflowStopping = true;
|
||||||
|
if (this.inboundWorkflowTimer) clearTimeout(this.inboundWorkflowTimer);
|
||||||
|
this.inboundWorkflowTimer = undefined;
|
||||||
|
await Promise.allSettled([...this.inboundWorkflowTasks]);
|
||||||
|
}
|
||||||
|
|
||||||
|
private scheduleInboundWorkflowPump(delayMs: number) {
|
||||||
|
if (this.inboundWorkflowStopping || this.inboundWorkflowTimer) return;
|
||||||
|
this.inboundWorkflowTimer = setTimeout(() => {
|
||||||
|
this.inboundWorkflowTimer = undefined;
|
||||||
|
void this.pumpInboundWorkflow();
|
||||||
|
}, delayMs);
|
||||||
|
this.inboundWorkflowTimer.unref?.();
|
||||||
|
}
|
||||||
|
|
||||||
|
private async pumpInboundWorkflow() {
|
||||||
|
if (this.inboundWorkflowStopping || this.inboundWorkflowPumping) return;
|
||||||
|
const concurrency = positiveInteger(process.env.API_INBOUND_WORKFLOW_CONCURRENCY, 32);
|
||||||
|
this.metrics?.setInboundWorkflowSlots(concurrency, this.inboundWorkflowInFlight);
|
||||||
|
const available = Math.max(0, concurrency - this.inboundWorkflowInFlight);
|
||||||
|
if (available === 0) return;
|
||||||
|
this.inboundWorkflowPumping = true;
|
||||||
|
try {
|
||||||
|
await this.refreshInboundWorkflowMetrics();
|
||||||
|
const claimed = await this.claimInboundWorkflows(available);
|
||||||
|
for (const item of claimed) {
|
||||||
|
this.inboundWorkflowInFlight += 1;
|
||||||
|
this.metrics?.setInboundWorkflowSlots(concurrency, this.inboundWorkflowInFlight);
|
||||||
|
const task = this.processClaimedInboundWorkflow(item)
|
||||||
|
.catch((error) => this.logger.error(`CMPP inbound workflow ${item.id} failed to settle: ${String(error)}`))
|
||||||
|
.finally(() => {
|
||||||
|
this.inboundWorkflowInFlight = Math.max(0, this.inboundWorkflowInFlight - 1);
|
||||||
|
this.metrics?.setInboundWorkflowSlots(concurrency, this.inboundWorkflowInFlight);
|
||||||
|
this.inboundWorkflowTasks.delete(task);
|
||||||
|
this.scheduleInboundWorkflowPump(0);
|
||||||
|
});
|
||||||
|
this.inboundWorkflowTasks.add(task);
|
||||||
|
}
|
||||||
|
if (claimed.length === 0) {
|
||||||
|
this.scheduleInboundWorkflowPump(positiveInteger(process.env.API_INBOUND_WORKFLOW_POLL_INTERVAL_MS, 100));
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
this.logger.error(`Failed to claim CMPP inbound workflow: ${String(error)}`);
|
||||||
|
this.scheduleInboundWorkflowPump(1000);
|
||||||
|
} finally {
|
||||||
|
this.inboundWorkflowPumping = false;
|
||||||
|
if (this.inboundWorkflowInFlight < concurrency && !this.inboundWorkflowTimer) {
|
||||||
|
this.scheduleInboundWorkflowPump(0);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private claimInboundWorkflows(limit: number) {
|
||||||
|
const staleSeconds = positiveInteger(process.env.API_INBOUND_WORKFLOW_STALE_SECONDS, 300);
|
||||||
|
return this.prisma.$queryRaw<ClaimedInboundWorkflow[]>(Prisma.sql`
|
||||||
|
WITH candidates AS (
|
||||||
|
SELECT id
|
||||||
|
FROM "CmppInboundSubmissionInbox"
|
||||||
|
WHERE (
|
||||||
|
status = 'pending'
|
||||||
|
AND "nextAttemptAt" <= NOW()
|
||||||
|
) OR (
|
||||||
|
status = 'processing'
|
||||||
|
AND "lockedAt" <= NOW() - make_interval(secs => ${staleSeconds})
|
||||||
|
)
|
||||||
|
-- Priority applications enter the same durable Inbox, but are claimed first while
|
||||||
|
-- preserving FIFO within each class. This keeps the V5 priority contract effective
|
||||||
|
-- before BullMQ without adding another non-durable queue.
|
||||||
|
ORDER BY CASE WHEN "queuePriority" = 'priority' THEN 0 ELSE 1 END, "createdAt" ASC
|
||||||
|
LIMIT ${limit}
|
||||||
|
FOR UPDATE SKIP LOCKED
|
||||||
|
)
|
||||||
|
UPDATE "CmppInboundSubmissionInbox" AS inbox
|
||||||
|
SET status = 'processing',
|
||||||
|
attempts = inbox.attempts + 1,
|
||||||
|
"lockedAt" = NOW(),
|
||||||
|
"lockedBy" = ${this.inboundWorkflowWorkerId},
|
||||||
|
"updatedAt" = NOW()
|
||||||
|
FROM candidates
|
||||||
|
WHERE inbox.id = candidates.id
|
||||||
|
RETURNING inbox.id, inbox."requestKey", inbox."applicationId", inbox.attempts, inbox.payload
|
||||||
|
`);
|
||||||
|
}
|
||||||
|
|
||||||
|
private async processClaimedInboundWorkflow(item: ClaimedInboundWorkflow) {
|
||||||
|
try {
|
||||||
|
const payload = parseInboundWorkflowPayload(item.payload);
|
||||||
|
const application = await this.facade.findInboundApplication(payload.data.account);
|
||||||
|
if (!application || application.id !== item.applicationId) {
|
||||||
|
throw new Error('CMPP inbound application no longer matches persisted workflow');
|
||||||
|
}
|
||||||
|
const result = await this.facade.submitCompleteInboundMessage(
|
||||||
|
payload.data,
|
||||||
|
payload.phoneNumbers,
|
||||||
|
application,
|
||||||
|
payload.submitGroupMessageId,
|
||||||
|
payload.messageIds,
|
||||||
|
item.requestKey,
|
||||||
|
);
|
||||||
|
const settled = await this.prisma.cmppInboundSubmissionInbox.updateMany({
|
||||||
|
where: { id: item.id, status: 'processing', lockedBy: this.inboundWorkflowWorkerId },
|
||||||
|
data: {
|
||||||
|
status: 'completed',
|
||||||
|
result: JSON.parse(JSON.stringify(result)) as Prisma.InputJsonValue,
|
||||||
|
completedAt: new Date(),
|
||||||
|
lockedAt: null,
|
||||||
|
lockedBy: null,
|
||||||
|
lastError: null,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
if (settled.count !== 1) throw new Error('CMPP inbound workflow lease was lost before completion');
|
||||||
|
this.metrics?.recordInboundWorkflowResult('completed');
|
||||||
|
} catch (error) {
|
||||||
|
const reason = (error instanceof Error ? error.message : String(error)).slice(0, 2000);
|
||||||
|
const delayMs = Math.min(60_000, 250 * 2 ** Math.min(8, Math.max(0, item.attempts - 1)));
|
||||||
|
const released = await this.prisma.cmppInboundSubmissionInbox.updateMany({
|
||||||
|
where: { id: item.id, status: 'processing', lockedBy: this.inboundWorkflowWorkerId },
|
||||||
|
data: {
|
||||||
|
status: 'pending',
|
||||||
|
nextAttemptAt: new Date(Date.now() + delayMs),
|
||||||
|
lockedAt: null,
|
||||||
|
lockedBy: null,
|
||||||
|
lastError: reason,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
if (released.count === 1) {
|
||||||
|
this.metrics?.recordInboundWorkflowResult('retry');
|
||||||
|
this.logger.warn(`CMPP inbound workflow ${item.id} will retry after attempt ${item.attempts}: ${reason}`);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private async refreshInboundWorkflowMetrics() {
|
||||||
|
const now = Date.now();
|
||||||
|
if (now - this.inboundWorkflowMetricsUpdatedAt < 5_000) return;
|
||||||
|
this.inboundWorkflowMetricsUpdatedAt = now;
|
||||||
|
const [pending, processing, oldest] = await Promise.all([
|
||||||
|
this.prisma.cmppInboundSubmissionInbox.count({ where: { status: 'pending' } }),
|
||||||
|
this.prisma.cmppInboundSubmissionInbox.count({ where: { status: 'processing' } }),
|
||||||
|
this.prisma.cmppInboundSubmissionInbox.findFirst({
|
||||||
|
where: { status: 'pending' },
|
||||||
|
select: { createdAt: true },
|
||||||
|
orderBy: { createdAt: 'asc' },
|
||||||
|
}),
|
||||||
|
]);
|
||||||
|
this.metrics?.setInboundWorkflowState(
|
||||||
|
pending,
|
||||||
|
processing,
|
||||||
|
oldest ? Math.max(0, (now - oldest.createdAt.getTime()) / 1000) : 0,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
findInboundApplication(account: string) {
|
findInboundApplication(account: string) {
|
||||||
return this.prisma.smsApplication.findFirst({
|
return this.prisma.smsApplication.findFirst({
|
||||||
where: { cmppAccount: account },
|
where: { cmppAccount: account },
|
||||||
@@ -917,3 +1261,24 @@ async attachMessageToReviewTask(reviewTaskId: string, messageRecordId: string, s
|
|||||||
return this.prisma.smsSendTask.findUnique({ where: { id: reviewTaskId } });
|
return this.prisma.smsSendTask.findUnique({ where: { id: reviewTaskId } });
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function parseInboundWorkflowPayload(value: Prisma.JsonValue): InboundWorkflowPayload {
|
||||||
|
if (!value || typeof value !== 'object' || Array.isArray(value)) throw new Error('CMPP inbound workflow payload is invalid');
|
||||||
|
const data = value.data;
|
||||||
|
const phoneNumbers = value.phoneNumbers;
|
||||||
|
const submitGroupMessageId = value.submitGroupMessageId;
|
||||||
|
const messageIds = value.messageIds;
|
||||||
|
if (!data || typeof data !== 'object' || Array.isArray(data)
|
||||||
|
|| !Array.isArray(phoneNumbers) || phoneNumbers.some((item) => typeof item !== 'string')
|
||||||
|
|| typeof submitGroupMessageId !== 'string'
|
||||||
|
|| !Array.isArray(messageIds) || messageIds.some((item) => typeof item !== 'string')
|
||||||
|
|| phoneNumbers.length === 0 || phoneNumbers.length !== messageIds.length) {
|
||||||
|
throw new Error('CMPP inbound workflow payload fields are invalid');
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
data: data as unknown as GatewayInboundSubmitDto,
|
||||||
|
phoneNumbers: phoneNumbers as string[],
|
||||||
|
submitGroupMessageId,
|
||||||
|
messageIds: messageIds as string[],
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|||||||
@@ -67,7 +67,10 @@ export class SendSubmissionService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
onModuleDestroy() {
|
onModuleDestroy() {
|
||||||
return this.gatewaySubmit.onModuleDestroy();
|
return Promise.all([
|
||||||
|
this.gatewaySubmit.onModuleDestroy(),
|
||||||
|
this.inboundEntry.stopInboundWorkflowWorker(),
|
||||||
|
]);
|
||||||
}
|
}
|
||||||
|
|
||||||
async createBatchTask(data: CreateBatchTaskDto) {
|
async createBatchTask(data: CreateBatchTaskDto) {
|
||||||
@@ -123,8 +126,8 @@ async reserveDailySendQuota(applicationId: string, requestedCount: number) {
|
|||||||
return this.batchEntry.reserveDailySendQuota(applicationId, requestedCount);
|
return this.batchEntry.reserveDailySendQuota(applicationId, requestedCount);
|
||||||
}
|
}
|
||||||
|
|
||||||
async tryReserveDailySendQuota(applicationId: string, requestedCount: number) {
|
async tryReserveDailySendQuota(applicationId: string, requestedCount: number, reservationKey?: string) {
|
||||||
return this.batchEntry.tryReserveDailySendQuota(applicationId, requestedCount);
|
return this.batchEntry.tryReserveDailySendQuota(applicationId, requestedCount, reservationKey);
|
||||||
}
|
}
|
||||||
|
|
||||||
async authenticateInboundApplication(data: GatewayInboundAuthDto) {
|
async authenticateInboundApplication(data: GatewayInboundAuthDto) {
|
||||||
@@ -144,8 +147,10 @@ async submitCompleteInboundMessage(
|
|||||||
phoneNumbers: string[],
|
phoneNumbers: string[],
|
||||||
application: Awaited<ReturnType<SendSubmissionService['findInboundApplication']>>,
|
application: Awaited<ReturnType<SendSubmissionService['findInboundApplication']>>,
|
||||||
requestedGroupMessageId?: string,
|
requestedGroupMessageId?: string,
|
||||||
|
requestedMessageIds?: string[],
|
||||||
|
workflowKey?: string,
|
||||||
) {
|
) {
|
||||||
return this.inboundEntry.submitCompleteInboundMessage(data, phoneNumbers, application, requestedGroupMessageId);
|
return this.inboundEntry.submitCompleteInboundMessage(data, phoneNumbers, application, requestedGroupMessageId, requestedMessageIds, workflowKey);
|
||||||
}
|
}
|
||||||
|
|
||||||
async collectInboundLongMessageFragment(
|
async collectInboundLongMessageFragment(
|
||||||
@@ -167,8 +172,9 @@ async submitInboundSingleMessage(
|
|||||||
application: NonNullable<Awaited<ReturnType<SendSubmissionService['findInboundApplication']>>>,
|
application: NonNullable<Awaited<ReturnType<SendSubmissionService['findInboundApplication']>>>,
|
||||||
synchronousRejection?: { code: string; reason: string },
|
synchronousRejection?: { code: string; reason: string },
|
||||||
receiptRejection?: { code: string; reason: string },
|
receiptRejection?: { code: string; reason: string },
|
||||||
|
workflowItemKey?: string,
|
||||||
) {
|
) {
|
||||||
return this.inboundEntry.submitInboundSingleMessage(data, messageId, submitGroupMessageId, application, synchronousRejection, receiptRejection);
|
return this.inboundEntry.submitInboundSingleMessage(data, messageId, submitGroupMessageId, application, synchronousRejection, receiptRejection, workflowItemKey);
|
||||||
}
|
}
|
||||||
|
|
||||||
async evaluateRiskWithPhoneFrequency(input: {
|
async evaluateRiskWithPhoneFrequency(input: {
|
||||||
@@ -179,8 +185,8 @@ async evaluateRiskWithPhoneFrequency(input: {
|
|||||||
variables?: Record<string, unknown>;
|
variables?: Record<string, unknown>;
|
||||||
phoneNumber: string;
|
phoneNumber: string;
|
||||||
sourceType: 'cmpp';
|
sourceType: 'cmpp';
|
||||||
}) {
|
}, reservationKey?: string) {
|
||||||
return this.inboundEntry.evaluateRiskWithPhoneFrequency(input);
|
return this.inboundEntry.evaluateRiskWithPhoneFrequency(input, reservationKey);
|
||||||
}
|
}
|
||||||
|
|
||||||
findInboundApplication(account: string) {
|
findInboundApplication(account: string) {
|
||||||
@@ -223,6 +229,14 @@ startWorker() {
|
|||||||
return this.gatewaySubmit.startWorker();
|
return this.gatewaySubmit.startWorker();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
startInboundWorkflowWorker() {
|
||||||
|
return this.inboundEntry.startInboundWorkflowWorker();
|
||||||
|
}
|
||||||
|
|
||||||
|
stopInboundWorkflowWorker() {
|
||||||
|
return this.inboundEntry.stopInboundWorkflowWorker();
|
||||||
|
}
|
||||||
|
|
||||||
async processSendJob(job: SendJob) {
|
async processSendJob(job: SendJob) {
|
||||||
return this.gatewaySubmit.processSendJob(job);
|
return this.gatewaySubmit.processSendJob(job);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,20 @@
|
|||||||
|
import { Module } from '@nestjs/common';
|
||||||
|
import { ConfigModule } from '@nestjs/config';
|
||||||
|
import { BillingModule } from './billing/billing.module';
|
||||||
|
import { PhoneRoutingLookupService } from './dictionaries/phone-routing-lookup.service';
|
||||||
|
import { MetricsModule } from './metrics/metrics.module';
|
||||||
|
import { PrismaModule } from './prisma/prisma.module';
|
||||||
|
import { PhoneFrequencyService } from './risk-review/phone-frequency.service';
|
||||||
|
import { RiskReviewService } from './risk-review/risk-review.service';
|
||||||
|
import { SendChainService } from './send-chain/send-chain.service';
|
||||||
|
|
||||||
|
@Module({
|
||||||
|
imports: [
|
||||||
|
ConfigModule.forRoot({ isGlobal: true, envFilePath: ['.env.local', '.env'] }),
|
||||||
|
PrismaModule,
|
||||||
|
BillingModule,
|
||||||
|
MetricsModule,
|
||||||
|
],
|
||||||
|
providers: [RiskReviewService, PhoneFrequencyService, PhoneRoutingLookupService, SendChainService],
|
||||||
|
})
|
||||||
|
export class SendWorkerModule {}
|
||||||
@@ -0,0 +1,45 @@
|
|||||||
|
import 'reflect-metadata';
|
||||||
|
import { NestFactory } from '@nestjs/core';
|
||||||
|
import { createServer } from 'node:http';
|
||||||
|
import { MetricsService } from './metrics/metrics.service';
|
||||||
|
import { SendWorkerModule } from './send-worker.module';
|
||||||
|
|
||||||
|
Object.defineProperty(BigInt.prototype, 'toJSON', {
|
||||||
|
configurable: true,
|
||||||
|
value(this: bigint) {
|
||||||
|
const result = Number(this);
|
||||||
|
if (!Number.isSafeInteger(result)) throw new RangeError('金额超过 JavaScript 安全整数范围');
|
||||||
|
return result;
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
async function bootstrap() {
|
||||||
|
if ((process.env.CMPP_PROCESS_ROLE?.trim() || 'worker') !== 'worker') {
|
||||||
|
throw new Error('send-worker requires CMPP_PROCESS_ROLE=worker');
|
||||||
|
}
|
||||||
|
const app = await NestFactory.createApplicationContext(SendWorkerModule);
|
||||||
|
app.enableShutdownHooks();
|
||||||
|
const metrics = app.get(MetricsService);
|
||||||
|
const host = process.env.API_WORKER_METRICS_HOST?.trim() || '127.0.0.1';
|
||||||
|
const port = Number(process.env.API_WORKER_METRICS_PORT ?? 9465);
|
||||||
|
const metricsServer = createServer((request, response) => {
|
||||||
|
if (request.method !== 'GET' || request.url !== '/metrics') {
|
||||||
|
response.writeHead(404).end();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
response.writeHead(200, { 'Content-Type': 'text/plain; version=0.0.4; charset=utf-8', 'Cache-Control': 'no-store' });
|
||||||
|
response.end(metrics.render());
|
||||||
|
});
|
||||||
|
await new Promise<void>((resolve, reject) => {
|
||||||
|
metricsServer.once('error', reject);
|
||||||
|
metricsServer.listen(port, host, resolve);
|
||||||
|
});
|
||||||
|
const close = async () => {
|
||||||
|
await new Promise<void>((resolve) => metricsServer.close(() => resolve()));
|
||||||
|
await app.close();
|
||||||
|
};
|
||||||
|
process.once('SIGTERM', () => void close());
|
||||||
|
process.once('SIGINT', () => void close());
|
||||||
|
}
|
||||||
|
|
||||||
|
void bootstrap();
|
||||||
@@ -1155,5 +1155,7 @@ global,不能错误归入client。`client-signature-*`、发送页、企业认
|
|||||||
- V3客户入站窗口只归`gateway/internal/inbound/`与项目内受控的`third_party/gocmpp`服务循环治理:API认证只返回应用窗口,inbound会话负责收紧窗口,协议服务循环负责受限派发和断线等待;不得把客户入站槽位与`submitworker`供应商槽位或`upstream`供应商窗口合并成同一并发计数。
|
- V3客户入站窗口只归`gateway/internal/inbound/`与项目内受控的`third_party/gocmpp`服务循环治理:API认证只返回应用窗口,inbound会话负责收紧窗口,协议服务循环负责受限派发和断线等待;不得把客户入站槽位与`submitworker`供应商槽位或`upstream`供应商窗口合并成同一并发计数。
|
||||||
- V4供应商结果异步边界只归`gateway/internal/resultoutbox/`治理:`upstream`在每个真实分片SubmitResp后只调用持久化接口,`submitworker`只负责聚合结果入Outbox与命令ACK的原子边界,Outbox回调Worker独立控制API并发和PEL恢复。API的`SmsSubmitRecord.resultEventId`是跨重启持久幂等事实;不得把HTTP回调重新放回供应商连接池或Submit工作槽,也不得让Outbox承担业务计费、补发或状态机判断。
|
- V4供应商结果异步边界只归`gateway/internal/resultoutbox/`治理:`upstream`在每个真实分片SubmitResp后只调用持久化接口,`submitworker`只负责聚合结果入Outbox与命令ACK的原子边界,Outbox回调Worker独立控制API并发和PEL恢复。API的`SmsSubmitRecord.resultEventId`是跨重启持久幂等事实;不得把HTTP回调重新放回供应商连接池或Submit工作槽,也不得让Outbox承担业务计费、补发或状态机判断。
|
||||||
- V5 API数据库往返优化仍归`send-inbound-entry`、`send-gateway-submit`和`risk-review`现有边界:入口应用快照沿稳定Facade显式传递,单条快速入队只接受已持久化ID和优先级,通用批量入队保留原查询与取消校验;默认规则并发单飞属于`RiskReviewService`内部完整性保障,不得在`SendChainService`新增第二套规则缓存,也不得把余额、频控状态或实际规则决策缓存进进程内存。
|
- V5 API数据库往返优化仍归`send-inbound-entry`、`send-gateway-submit`和`risk-review`现有边界:入口应用快照沿稳定Facade显式传递,单条快速入队只接受已持久化ID和优先级,通用批量入队保留原查询与取消校验;默认规则并发单飞属于`RiskReviewService`内部完整性保障,不得在`SendChainService`新增第二套规则缓存,也不得把余额、频控状态或实际规则决策缓存进进程内存。
|
||||||
|
- 500条/秒第一阶段把接收与业务处理边界固定在`CmppInboundSubmissionInbox`:`gateway/internal/inbound`只生成连接域内稳定请求键,`send-inbound-entry`只负责最小校验、Inbox持久化和稳定响应;`api/src/send-worker.ts`在独立进程内领取并调用既有发送链。领取事务不得包住风控、计费、路由、Redis或供应商调用,业务幂等事实必须落PostgreSQL,不能依赖进程内缓存或localStorage。
|
||||||
|
- API进程角色固定为`CMPP_PROCESS_ROLE=api`,不启动发送Worker、Inbox Worker和周期扫描;`cmpp-send-worker`固定为`worker`并拥有自己的Prisma连接池和回环指标。后续扩容允许水平增加Worker实例,但不得复制HTTP控制器或绕过Inbox直接创建业务消息。
|
||||||
- `api/src/infrastructure-monitoring/`是运营端监控聚合与固定阈值应用边界:只消费代码白名单 PromQL,并通过版本化 PostgreSQL 单例、promtool 校验和原子规则热加载管理数值阈值;Exporter安装、端口隔离、固定规则模板和权限仍归`tools/monitoring/`治理。
|
- `api/src/infrastructure-monitoring/`是运营端监控聚合与固定阈值应用边界:只消费代码白名单 PromQL,并通过版本化 PostgreSQL 单例、promtool 校验和原子规则热加载管理数值阈值;Exporter安装、端口隔离、固定规则模板和权限仍归`tools/monitoring/`治理。
|
||||||
- 活动告警已读也归该边界:Prometheus保留告警事实,Prisma仅持久化逐管理员、逐触发周期的阅读状态;全局布局只消费轻量未读汇总,不复制指纹、activeAt或用户隔离逻辑。
|
- 活动告警已读也归该边界:Prometheus保留告警事实,Prisma仅持久化逐管理员、逐触发周期的阅读状态;全局布局只消费轻量未读汇总,不复制指纹、activeAt或用户隔离逻辑。
|
||||||
|
|||||||
@@ -516,7 +516,7 @@
|
|||||||
"name": "handleSubmit",
|
"name": "handleSubmit",
|
||||||
"kind": "func",
|
"kind": "func",
|
||||||
"file": "submit.go",
|
"file": "submit.go",
|
||||||
"sha256": "16a7bb939824b18ea13698293f2868a25a7cb525e318a1c6b00597f37be12466"
|
"sha256": "36b9016de0d3cf0d2286b00aad90435d67cc3720ba5fcc87a3d27e8c3db4d2c1"
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"name": "inboundLongMessageFragment",
|
"name": "inboundLongMessageFragment",
|
||||||
@@ -530,6 +530,12 @@
|
|||||||
"file": "submit.go",
|
"file": "submit.go",
|
||||||
"sha256": "adac6aed04068f3811fbcf0530ab54f7384f0ea85a303c9c70326b5a9a46ba78"
|
"sha256": "adac6aed04068f3811fbcf0530ab54f7384f0ea85a303c9c70326b5a9a46ba78"
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
"name": "inboundSubmitRequestID",
|
||||||
|
"kind": "func",
|
||||||
|
"file": "submit.go",
|
||||||
|
"sha256": "5f3515253ece00423b96d0c8b32efdefa4b4c61d88991997df66c9acccce4b7e"
|
||||||
|
},
|
||||||
{
|
{
|
||||||
"name": "messageIDFrom",
|
"name": "messageIDFrom",
|
||||||
"kind": "func",
|
"kind": "func",
|
||||||
@@ -564,7 +570,7 @@
|
|||||||
"name": "submitRequest",
|
"name": "submitRequest",
|
||||||
"kind": "type",
|
"kind": "type",
|
||||||
"file": "submit.go",
|
"file": "submit.go",
|
||||||
"sha256": "80791da6ef4e968dbafbea288ef783422b4c089013a05175b6add32bf8919df0"
|
"sha256": "7edc6ef9b65d44dd3c2f22415c3d6ceca12621cbf531b089fb500ecfea282253"
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"name": "submitResponse",
|
"name": "submitResponse",
|
||||||
|
|||||||
@@ -2103,6 +2103,10 @@
|
|||||||
- V3入站并发必须只并发Submit业务处理,连接认证保持串行先完成,心跳和Deliver ACK不得被长耗时Submit阻塞;每个SubmitResp继续使用原请求Sequence_Id关联,允许按实际完成顺序返回。同一连接关闭时必须先等待已接受的在途处理收尾,再清理会话和回执映射,避免迟到处理重新注册已断开的连接。`cmpp_gateway_inbound_submit_slots{state=configured|in_flight}`只暴露全部在线连接的聚合窗口与在途数量,不得增加账号、应用、连接或消息标签。
|
- V3入站并发必须只并发Submit业务处理,连接认证保持串行先完成,心跳和Deliver ACK不得被长耗时Submit阻塞;每个SubmitResp继续使用原请求Sequence_Id关联,允许按实际完成顺序返回。同一连接关闭时必须先等待已接受的在途处理收尾,再清理会话和回执映射,避免迟到处理重新注册已断开的连接。`cmpp_gateway_inbound_submit_slots{state=configured|in_flight}`只暴露全部在线连接的聚合窗口与在途数量,不得增加账号、应用、连接或消息标签。
|
||||||
- V4结果Outbox必须暴露独立回调Worker的configured/in_flight槽位和Outbox pending/lag,仍只使用固定状态标签。`api_callback`从V4起只在Outbox回调Worker计时,不再混入供应商Submit工作槽;压测结束必须同时核对命令Stream和结果Outbox均`pending=0/lag=0`,并证明API回调故障时供应商槽继续释放、结果不丢失且恢复后只处理一次。
|
- V4结果Outbox必须暴露独立回调Worker的configured/in_flight槽位和Outbox pending/lag,仍只使用固定状态标签。`api_callback`从V4起只在Outbox回调Worker计时,不再混入供应商Submit工作槽;压测结束必须同时核对命令Stream和结果Outbox均`pending=0/lag=0`,并证明API回调故障时供应商槽继续释放、结果不丢失且恢复后只处理一次。
|
||||||
- V5 API入站数据库往返治理不得改变SubmitResp、余额冻结、号码频控、模板/签名、路由、队列或失败回执语义。同一Submit入口已取得的应用、企业和IP白名单快照必须复用于其全部目标号码,不得逐号码重复查询;任务风控与号码频控共用的默认规则完整性检查必须使用短TTL并发单飞,正常完整状态最多执行一次聚合数据库检查,实际生效规则仍逐次读取;刚持久化的单条CMPP内部任务和消息可凭已知ID、优先级直接入队,不得为入队重新查询同一任务和消息。缓存只覆盖默认规则“是否齐全”,检查失败必须立即失效,规则缺失须在短TTL到期后自动恢复;不得缓存余额、频控计数、应用启停或实际生效规则。
|
- V5 API入站数据库往返治理不得改变SubmitResp、余额冻结、号码频控、模板/签名、路由、队列或失败回执语义。同一Submit入口已取得的应用、企业和IP白名单快照必须复用于其全部目标号码,不得逐号码重复查询;任务风控与号码频控共用的默认规则完整性检查必须使用短TTL并发单飞,正常完整状态最多执行一次聚合数据库检查,实际生效规则仍逐次读取;刚持久化的单条CMPP内部任务和消息可凭已知ID、优先级直接入队,不得为入队重新查询同一任务和消息。缓存只覆盖默认规则“是否齐全”,检查失败必须立即失效,规则缺失须在短TTL到期后自动恢复;不得缓存余额、频控计数、应用启停或实际生效规则。
|
||||||
|
- 面向完整处理500条/秒目标的第一阶段,CMPP SubmitResp语义调整为“已完成最小协议/账号/IP/Src_Id校验且已持久化接收事实”,不再同步等待模板、风控、频控、日限、计费、路由和入队。Gateway必须为同一连接内同一Submit重试生成稳定请求键;PostgreSQL Inbox以唯一请求键和载荷摘要幂等保存原请求、稳定内部MessageId及响应。相同键相同载荷返回原响应,相同键不同载荷必须拒绝。
|
||||||
|
- Inbox Worker必须作为独立非root进程运行,使用独立数据库连接池和持续有界工作池;领取使用短事务、`FOR UPDATE SKIP LOCKED`和过期租约回收,实际业务处理在领取事务外执行。成功逐条完成,失败以有上限退避回到pending且不得静默丢弃。日发送配额、号码频控及余额冻结必须使用由Inbox请求派生的持久幂等键,确保进程崩溃或租约回收不会重复扣量、重复计频或重复冻结。
|
||||||
|
- 优先应用和普通应用共享同一耐久Inbox,但领取顺序必须保留优先级并在同一优先级内FIFO;进入BullMQ后继续沿用priority=1、normal=100。快路径成功只代表平台已可靠接收,异步业务拒绝仍必须落真实消息/任务状态并按既有CMPP失败回执链路通知客户,不得伪装为供应商最终送达。
|
||||||
|
- 第一阶段的验收是入口可持续接收、Inbox不丢不重且最终可排空,并为后续500条/秒全链路扩容建立解耦边界;不能仅凭SubmitResp吞吐宣称完整500条/秒。压测必须同时报告SubmitResp成功率/延迟、Inbox pending/processing/最老等待、异步完成速率和排空时间,以及命令Stream、结果Outbox和数据库最终对账。
|
||||||
- 系统监控标题说明必须明确标注数据来自 Prometheus;“服务关键指标”位于趋势/核心服务区域之后、活动告警之前,并提供统一的“告警阈值设置”入口。安全检测页不重复渲染大号标题,说明文字必须明确标注使用 Fail2ban。
|
- 系统监控标题说明必须明确标注数据来自 Prometheus;“服务关键指标”位于趋势/核心服务区域之后、活动告警之前,并提供统一的“告警阈值设置”入口。安全检测页不重复渲染大号标题,说明文字必须明确标注使用 Fail2ban。
|
||||||
- 告警阈值仅开放固定指标的警告/严重数值,不允许前端提交 PromQL、标签、文件路径或持续时间;必须满足警告值小于严重值。配置以 PostgreSQL 保存版本、期望值、生效值和应用状态,经 `promtool check rules` 校验、同目录原子替换和 Prometheus 热加载成功后才标记生效,失败保留上一生效规则并展示原因。
|
- 告警阈值仅开放固定指标的警告/严重数值,不允许前端提交 PromQL、标签、文件路径或持续时间;必须满足警告值小于严重值。配置以 PostgreSQL 保存版本、期望值、生效值和应用状态,经 `promtool check rules` 校验、同目录原子替换和 Prometheus 热加载成功后才标记生效,失败保留上一生效规则并展示原因。
|
||||||
- 右上角预警中心增加“系统监控告警”,通过独立轻量接口统计 Prometheus 当前 firing/pending 告警及严重数,跳转系统监控活动告警区;任一预警域失败不得清空其他域。
|
- 右上角预警中心增加“系统监控告警”,通过独立轻量接口统计 Prometheus 当前 firing/pending 告警及严重数,跳转系统监控活动告警区;任一预警域失败不得清空其他域。
|
||||||
|
|||||||
@@ -152,6 +152,9 @@ curl http://127.0.0.1:12026/
|
|||||||
redis-cli -h 127.0.0.1 -p 6379 ping
|
redis-cli -h 127.0.0.1 -p 6379 ping
|
||||||
pg_isready -d "$(grep '^DATABASE_URL=' /etc/cmpp-platform/cmpp-platform.env | cut -d= -f2-)"
|
pg_isready -d "$(grep '^DATABASE_URL=' /etc/cmpp-platform/cmpp-platform.env | cut -d= -f2-)"
|
||||||
grep -E '^(API_ENABLE_SEND_WORKER|API_SEND_WORKER_CONCURRENCY|GATEWAY_SUBMIT_WORKER_CONCURRENCY|GATEWAY_SUBMIT_RESULT_WORKER_CONCURRENCY|GATEWAY_CMPP_INBOUND_MAX_CONCURRENCY)=' /etc/cmpp-platform/cmpp-platform.env
|
grep -E '^(API_ENABLE_SEND_WORKER|API_SEND_WORKER_CONCURRENCY|GATEWAY_SUBMIT_WORKER_CONCURRENCY|GATEWAY_SUBMIT_RESULT_WORKER_CONCURRENCY|GATEWAY_CMPP_INBOUND_MAX_CONCURRENCY)=' /etc/cmpp-platform/cmpp-platform.env
|
||||||
|
grep -E '^(CMPP_INBOUND_FAST_PATH_ENABLED|CMPP_INBOUND_WORKFLOW_WORKER_ENABLED|API_INBOUND_WORKFLOW_CONCURRENCY|API_INBOUND_WORKFLOW_POLL_INTERVAL_MS|API_INBOUND_WORKFLOW_STALE_SECONDS|API_WORKER_METRICS_PORT)=' /etc/cmpp-platform/cmpp-platform.env
|
||||||
|
systemctl is-active cmpp-api cmpp-send-worker cmpp-gateway
|
||||||
|
curl -fsS http://127.0.0.1:9465/metrics | grep '^cmpp_worker_inbound_workflow_'
|
||||||
redis-cli --scan --pattern 'rate:gateway:channel:*'
|
redis-cli --scan --pattern 'rate:gateway:channel:*'
|
||||||
redis-cli XINFO GROUPS gateway.submit.commands
|
redis-cli XINFO GROUPS gateway.submit.commands
|
||||||
redis-cli XINFO GROUPS gateway.submit.results
|
redis-cli XINFO GROUPS gateway.submit.results
|
||||||
@@ -174,3 +177,10 @@ bash tools/deploy/production-deploy.sh
|
|||||||
```
|
```
|
||||||
|
|
||||||
3. 如迁移造成不可兼容故障,先停服务,再恢复数据库备份。
|
3. 如迁移造成不可兼容故障,先停服务,再恢复数据库备份。
|
||||||
|
|
||||||
|
## CMPP耐久Inbox与独立Worker发布门禁(2026-08-20)
|
||||||
|
|
||||||
|
- 发布前恢复资产除PostgreSQL、运行源码和环境文件外,必须包含`cmpp-api.service`、`cmpp-send-worker.service`及其drop-in;逐项校验`pg_restore --list`、tar可读性和SHA-256。数据库回滚与运行代码必须成套执行,禁止只回退代码后让旧Prisma Client访问新状态机。
|
||||||
|
- 环境必须显式启用`CMPP_INBOUND_FAST_PATH_ENABLED=true`、`CMPP_INBOUND_WORKFLOW_WORKER_ENABLED=true`,并给出正整数`API_INBOUND_WORKFLOW_CONCURRENCY`;推荐初始值32、轮询100ms、租约300秒。API systemd角色必须是`api`,Worker角色必须是`worker`;Worker可通过`API_WORKER_DATABASE_URL`使用独立连接上限,未配置时仍使用同一数据库地址但保持独立进程连接池。
|
||||||
|
- Worker日志目录归`cmpp-api:cmpp-security`且仅服务可写,9465只监听回环并加入Prometheus `cmpp-send-worker` target。发布后必须验证两进程均为非root、API/Gateway health、Worker metrics、PostgreSQL/Redis,以及Inbox pending/processing/最老等待可观测。
|
||||||
|
- 回滚前先停止Gateway、API和Worker,保留故障现场Inbox及日志;如恢复旧数据库备份,必须同时恢复对应源码和环境/systemd资产。不得在回滚时删除pending Inbox或重投真实短信。
|
||||||
|
|||||||
@@ -4726,3 +4726,18 @@ npm run verify:phase8
|
|||||||
| TC-INFRA-MON-037 | 已读用户隔离 | 管理员A标记已读后由管理员B查看同一告警 | 管理员B仍显示未读且铃铛数量不减少,管理员A的状态保持已读 |
|
| TC-INFRA-MON-037 | 已读用户隔离 | 管理员A标记已读后由管理员B查看同一告警 | 管理员B仍显示未读且铃铛数量不减少,管理员A的状态保持已读 |
|
||||||
| TC-INFRA-MON-038 | 同告警重新触发 | 标记已读后让告警恢复,再以相同标签重新触发并产生新 activeAt | 新触发记录重新显示“标记已读”,计入预警中心;旧 activeAt 不会永久屏蔽同指纹告警 |
|
| TC-INFRA-MON-038 | 同告警重新触发 | 标记已读后让告警恢复,再以相同标签重新触发并产生新 activeAt | 新触发记录重新显示“标记已读”,计入预警中心;旧 activeAt 不会永久屏蔽同指纹告警 |
|
||||||
| TC-INFRA-MON-039 | 过期与幂等 | 重复提交同一活动告警,再提交已恢复或 activeAt 不匹配的请求 | 同一次告警重复提交幂等;过期/不匹配请求返回404且不生成虚假已读记录;操作日志可追溯 |
|
| TC-INFRA-MON-039 | 过期与幂等 | 重复提交同一活动告警,再提交已恢复或 activeAt 不匹配的请求 | 同一次告警重复提交幂等;过期/不匹配请求返回404且不生成虚假已读记录;操作日志可追溯 |
|
||||||
|
|
||||||
|
## CMPP 500条/秒第一阶段:耐久Inbox快路径(2026-08-20)
|
||||||
|
|
||||||
|
| 用例ID | 场景 | 步骤 | 预期 |
|
||||||
|
| --- | --- | --- | --- |
|
||||||
|
| TC-CMPP-500-P1-001 | 快速耐久受理 | 开启快路径提交合法短消息,并在风控/计费/队列依赖可观测时检查调用顺序 | SubmitResp在一条Inbox事实提交后返回`accepted_pending`;响应前不调用风控、频控、计费或BullMQ |
|
||||||
|
| TC-CMPP-500-P1-002 | 请求幂等 | 在同一已鉴权连接重投相同Sequence_Id与载荷 | Gateway请求键稳定,数据库只保留一条Inbox;两次返回相同MessageId,不重复创建任务、消息或冻结 |
|
||||||
|
| TC-CMPP-500-P1-003 | 幂等冲突 | 使用同一请求键提交不同载荷 | API拒绝冲突且不覆盖原Inbox/响应 |
|
||||||
|
| TC-CMPP-500-P1-004 | 多号码与长短信 | 分别提交多号码Submit和完整长短信分片 | Inbox保留全部号码及稳定子MessageId;长短信Worker处理的是完整重组正文,不是最后一片正文 |
|
||||||
|
| TC-CMPP-500-P1-005 | Worker领取与逐条完成 | 启动两个Worker并制造至少一个批次积压 | `SKIP LOCKED`领取不重复;每条独立完成,处理逻辑不占用领取事务 |
|
||||||
|
| TC-CMPP-500-P1-006 | 崩溃恢复 | 在领取后终止Worker,超过租约后重启 | processing记录被回收并完成;日限、频控、冻结、任务和消息均不重复 |
|
||||||
|
| TC-CMPP-500-P1-007 | 异步业务拒绝 | 让已耐久受理短信命中真实模板/风控/余额拒绝 | SubmitResp仍表示已接收;后台生成真实失败状态和失败回执,不进入供应商发送 |
|
||||||
|
| TC-CMPP-500-P1-008 | 优先级 | 在普通Inbox积压期间持续混入priority应用 | priority先领取且类内FIFO;普通队列最终可排空;同时记录两类等待分位数 |
|
||||||
|
| TC-CMPP-500-P1-009 | 进程隔离 | 检查systemd、进程、连接和指标端口 | API角色不运行发送后台任务;`cmpp-send-worker`独立非root运行,指标仅监听127.0.0.1:9465 |
|
||||||
|
| TC-CMPP-500-P1-010 | 阶梯压测与对账 | 隔离供应商环境按既定同口径阶梯执行,压后等待全链排空 | 逐档报告SubmitResp、Inbox、两条Stream、最终数据库计数和排空时间;任何丢响应、重复、错误或未排空均失败,不发送真实短信 |
|
||||||
|
|||||||
@@ -3782,3 +3782,12 @@ git diff --check
|
|||||||
- Prometheus按各60秒窗口附近75秒区间计算的API阶段平均值显示,10/20/30/40/50档`application_lookup`约为`1.42/51.04/65.54/108.64/247.65ms`,`risk_frequency`约`4.34/115.10/143.61/229.05/519.32ms`,`queue_publish`约`4.71/70.79/101.17/163.46/383.60ms`,`complete_submit`约`18.99/479.58/610.03/992.49/2285.77ms`。V5减少固定数据库往返后40条/秒P95继续下降,但50条/秒时同步风控、入队和持久化仍随数据库并发竞争放大,下一阶段应针对这些阶段继续做查询合并/事务缩短,而不是把50条/秒宣称为安全容量。
|
- Prometheus按各60秒窗口附近75秒区间计算的API阶段平均值显示,10/20/30/40/50档`application_lookup`约为`1.42/51.04/65.54/108.64/247.65ms`,`risk_frequency`约`4.34/115.10/143.61/229.05/519.32ms`,`queue_publish`约`4.71/70.79/101.17/163.46/383.60ms`,`complete_submit`约`18.99/479.58/610.03/992.49/2285.77ms`。V5减少固定数据库往返后40条/秒P95继续下降,但50条/秒时同步风控、入队和持久化仍随数据库并发竞争放大,下一阶段应针对这些阶段继续做查询合并/事务缩短,而不是把50条/秒宣称为安全容量。
|
||||||
- 优先队列在真实积压下通过验收。BullMQ配置为priority=1、normal=100;30条/秒时优先任务`queuedAt`至首条`SmsSubmitRecord.createdAt`的P50/P95=`0.786/0.988s`,普通任务为`14.678/26.282s`;40条/秒分别为`2.996/3.916s`与`45.017/58.536s`。低负载10/20条/秒两类接近,符合无积压时无需插队的预期;高负载差异证明优先任务能够越过普通积压持续取得发送Worker。SubmitResp按优先/普通拆分在40条/秒P95为`1727/1741ms`,说明上游受理没有饿死普通连接。50条/秒优先任务也优于普通任务,但两类均出现几十秒下游等待,不能用优先级掩盖总容量过载。
|
- 优先队列在真实积压下通过验收。BullMQ配置为priority=1、normal=100;30条/秒时优先任务`queuedAt`至首条`SmsSubmitRecord.createdAt`的P50/P95=`0.786/0.988s`,普通任务为`14.678/26.282s`;40条/秒分别为`2.996/3.916s`与`45.017/58.536s`。低负载10/20条/秒两类接近,符合无积压时无需插队的预期;高负载差异证明优先任务能够越过普通积压持续取得发送Worker。SubmitResp按优先/普通拆分在40条/秒P95为`1727/1741ms`,说明上游受理没有饿死普通连接。50条/秒优先任务也优于普通任务,但两类均出现几十秒下游等待,不能用优先级掩盖总容量过载。
|
||||||
- 本轮优先级结论只覆盖3个priority应用与7个normal应用混合、移动号段和现有六个隔离供应商账号;联通、电信号段及六通道跨运营商容量/优先级隔离仍未完成,不外推为全运营商结论。完整报告和原始`events.jsonl/summary.json`保存在短信平台测试项目。没有发送真实短信,没有修改通道账号、密码、启停状态、企业余额或客户连接;受保护的`api/tsconfig.build.tsbuildinfo`、`tsconfig.tsbuildinfo`、`outputs/`、`pnpm-lock.yaml`和空文件`=`继续排除提交、不删除、不归因。
|
- 本轮优先级结论只覆盖3个priority应用与7个normal应用混合、移动号段和现有六个隔离供应商账号;联通、电信号段及六通道跨运营商容量/优先级隔离仍未完成,不外推为全运营商结论。完整报告和原始`events.jsonl/summary.json`保存在短信平台测试项目。没有发送真实短信,没有修改通道账号、密码、启停状态、企业余额或客户连接;受保护的`api/tsconfig.build.tsbuildinfo`、`tsconfig.tsbuildinfo`、`outputs/`、`pnpm-lock.yaml`和空文件`=`继续排除提交、不删除、不归因。
|
||||||
|
|
||||||
|
# 2026-08-20 完整处理500条/秒第一阶段:PostgreSQL耐久Inbox快路径(提交前验证)
|
||||||
|
|
||||||
|
- 第一阶段把CMPP Submit同步路径收敛为账号/状态/IP/Src_Id/协议最小校验和一条`CmppInboundSubmissionInbox`持久化;Gateway按已鉴权连接、Sequence_Id和载荷生成稳定请求键。重复相同Submit返回原持久化MessageId,相同请求键的冲突载荷拒绝。多号码保留稳定子MessageId,长短信在完整重组后才写Inbox,Worker处理完整正文。
|
||||||
|
- 独立`cmpp-send-worker`进程持续以默认32槽领取Inbox;短事务使用`FOR UPDATE SKIP LOCKED`和300秒过期租约,处理在领取事务外完成,失败按有上限指数退避回到pending。priority应用在领取阶段优先且类内FIFO,BullMQ继续使用priority=1、normal=100。API角色不再运行发送Worker或周期扫描;Worker拥有独立Prisma连接池入口和127.0.0.1:9465指标。
|
||||||
|
- 新增持久幂等预留:日发送配额与`SmsApplicationDailyUsage`同事务写`SmsApplicationDailyReservation`,号码频控计数与`PhoneFrequencyReservation`同事务提交,余额冻结继续使用业务键;任务、API请求和短信主记录合并为一个短事务并使用确定性任务号/请求号。租约回收或进程重启不会重复计量、计频、冻结或创建业务主记录。
|
||||||
|
- Prisma新增第92条migration `20260820170000_add_cmpp_inbound_submission_inbox`,包含Inbox、日配额预留和号码频控预留三张真实PostgreSQL表及领取/租约/应用索引。发布脚本增加快路径/Worker强制门禁、systemd分进程、安全drop-in、Worker日志目录、健康检查和Prometheus target/告警;需求、系统用例、模块路线图和生产发布文档同步更新。
|
||||||
|
- 提交前验证:Prisma generate/validate、API TypeScript正式构建、前端TypeScript与Vite生产构建通过;API全量42套489项通过,Gateway全量`go test ./... -count=1`和`go vet ./...`通过,5份Stream契约、依赖/安全/部署门禁、R0/R6/R7/R10及`git diff --check`通过。R6契约只新增/更新本轮`submit.go`的稳定请求键声明。R9仍先被HEAD既有`dispatchDueScheduledTasks`哈希漂移阻断,与本轮文件和既有V5记录一致,未为通过本任务错误吸收该并行历史。Jest仍用`--forceExit`收尾仓库既有开放句柄,Vite仅保留既有大chunk告警。
|
||||||
|
- 当前尚未提交、部署或压测;下一步在排除受保护文件后提交,随后仅对`100.93.204.60`测试环境建立并校验PostgreSQL、运行源码、环境/systemd恢复资产,应用migration和独立Worker,再用隔离供应商执行同口径阶梯压测。预生产不发布、不回退、不压测;不发送、补发或重投真实短信,不修改真实通道账号、密码、启停状态、企业余额或客户连接。
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ import (
|
|||||||
"cmpp-platform/gateway/internal/metrics"
|
"cmpp-platform/gateway/internal/metrics"
|
||||||
"context"
|
"context"
|
||||||
"crypto/md5"
|
"crypto/md5"
|
||||||
|
"crypto/sha256"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
cmpp "github.com/bigwhite/gocmpp"
|
cmpp "github.com/bigwhite/gocmpp"
|
||||||
@@ -19,6 +20,7 @@ import (
|
|||||||
// one internal message mapping per destination while CMPP receives one response.
|
// one internal message mapping per destination while CMPP receives one response.
|
||||||
|
|
||||||
type submitRequest struct {
|
type submitRequest struct {
|
||||||
|
RequestID string `json:"requestId,omitempty"`
|
||||||
Account string `json:"account"`
|
Account string `json:"account"`
|
||||||
PhoneNumber string `json:"phoneNumber,omitempty"`
|
PhoneNumber string `json:"phoneNumber,omitempty"`
|
||||||
PhoneNumbers []string `json:"phoneNumbers,omitempty"`
|
PhoneNumbers []string `json:"phoneNumbers,omitempty"`
|
||||||
@@ -117,6 +119,7 @@ func (s Server) handleSubmit(response *cmpp.Response, packet *cmpp.Packet, logge
|
|||||||
releaseSubmitBarrier := beginDownstreamSubmitBarrier(packet.Conn)
|
releaseSubmitBarrier := beginDownstreamSubmitBarrier(packet.Conn)
|
||||||
apiStartedAt := time.Now()
|
apiStartedAt := time.Now()
|
||||||
result, err := s.submit(remote, submitRequest{
|
result, err := s.submit(remote, submitRequest{
|
||||||
|
RequestID: inboundSubmitRequestID(session.connectionID, req.sequenceID, phones, content, req.srcID, longMessage),
|
||||||
Account: account,
|
Account: account,
|
||||||
PhoneNumber: phone,
|
PhoneNumber: phone,
|
||||||
PhoneNumbers: phones,
|
PhoneNumbers: phones,
|
||||||
@@ -222,6 +225,14 @@ func (s Server) handleSubmit(response *cmpp.Response, packet *cmpp.Packet, logge
|
|||||||
return false, nil
|
return false, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func inboundSubmitRequestID(connectionID string, sequenceID uint32, phones []string, content string, srcID string, longMessage *inboundLongMessageFragment) string {
|
||||||
|
// The key is stable for an API retry of the same packet but scoped to the authenticated
|
||||||
|
// connection, so a later client session may intentionally reuse the CMPP Sequence_Id.
|
||||||
|
payload := fmt.Sprintf("%s\x00%d\x00%s\x00%s\x00%s\x00%v", connectionID, sequenceID, strings.Join(phones, ","), strings.TrimSpace(srcID), content, longMessage)
|
||||||
|
digest := sha256.Sum256([]byte(payload))
|
||||||
|
return fmt.Sprintf("cmpp-inbound:%x", digest[:])
|
||||||
|
}
|
||||||
|
|
||||||
func observeInboundSubmitResponse(handlerStartedAt time.Time, responseReadyAt time.Time, accepted bool, next func(error)) func(error) {
|
func observeInboundSubmitResponse(handlerStartedAt time.Time, responseReadyAt time.Time, accepted bool, next func(error)) func(error) {
|
||||||
return func(sendErr error) {
|
return func(sendErr error) {
|
||||||
metrics.ObserveInboundStage("response_write", sendErr == nil, time.Since(responseReadyAt))
|
metrics.ObserveInboundStage("response_write", sendErr == nil, time.Since(responseReadyAt))
|
||||||
|
|||||||
@@ -0,0 +1,27 @@
|
|||||||
|
package inbound
|
||||||
|
|
||||||
|
import "testing"
|
||||||
|
|
||||||
|
func TestInboundSubmitRequestIDIsStableForRetry(t *testing.T) {
|
||||||
|
fragment := &inboundLongMessageFragment{Reference: 7, Total: 2, Index: 1, Format: 8}
|
||||||
|
first := inboundSubmitRequestID("connection-1", 42, []string{"13800000001"}, "【测试】内容", "1069", fragment)
|
||||||
|
second := inboundSubmitRequestID("connection-1", 42, []string{"13800000001"}, "【测试】内容", "1069", fragment)
|
||||||
|
if first != second {
|
||||||
|
t.Fatalf("retry key changed: %q != %q", first, second)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestInboundSubmitRequestIDSeparatesPayloadAndSession(t *testing.T) {
|
||||||
|
base := inboundSubmitRequestID("connection-1", 42, []string{"13800000001"}, "content", "1069", nil)
|
||||||
|
cases := []string{
|
||||||
|
inboundSubmitRequestID("connection-2", 42, []string{"13800000001"}, "content", "1069", nil),
|
||||||
|
inboundSubmitRequestID("connection-1", 43, []string{"13800000001"}, "content", "1069", nil),
|
||||||
|
inboundSubmitRequestID("connection-1", 42, []string{"13800000002"}, "content", "1069", nil),
|
||||||
|
inboundSubmitRequestID("connection-1", 42, []string{"13800000001"}, "other", "1069", nil),
|
||||||
|
}
|
||||||
|
for _, candidate := range cases {
|
||||||
|
if candidate == base {
|
||||||
|
t.Fatalf("different submit produced duplicate key %q", base)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -11,6 +11,13 @@ API_METRICS_HOST="${API_METRICS_HOST:-127.0.0.1}"
|
|||||||
API_METRICS_PORT="${API_METRICS_PORT:-9464}"
|
API_METRICS_PORT="${API_METRICS_PORT:-9464}"
|
||||||
API_ENABLE_SEND_WORKER="${API_ENABLE_SEND_WORKER:-true}"
|
API_ENABLE_SEND_WORKER="${API_ENABLE_SEND_WORKER:-true}"
|
||||||
API_SEND_WORKER_CONCURRENCY="${API_SEND_WORKER_CONCURRENCY:-50}"
|
API_SEND_WORKER_CONCURRENCY="${API_SEND_WORKER_CONCURRENCY:-50}"
|
||||||
|
API_WORKER_METRICS_HOST="${API_WORKER_METRICS_HOST:-127.0.0.1}"
|
||||||
|
API_WORKER_METRICS_PORT="${API_WORKER_METRICS_PORT:-9465}"
|
||||||
|
CMPP_INBOUND_FAST_PATH_ENABLED="${CMPP_INBOUND_FAST_PATH_ENABLED:-true}"
|
||||||
|
CMPP_INBOUND_WORKFLOW_WORKER_ENABLED="${CMPP_INBOUND_WORKFLOW_WORKER_ENABLED:-true}"
|
||||||
|
API_INBOUND_WORKFLOW_CONCURRENCY="${API_INBOUND_WORKFLOW_CONCURRENCY:-32}"
|
||||||
|
API_INBOUND_WORKFLOW_POLL_INTERVAL_MS="${API_INBOUND_WORKFLOW_POLL_INTERVAL_MS:-100}"
|
||||||
|
API_INBOUND_WORKFLOW_STALE_SECONDS="${API_INBOUND_WORKFLOW_STALE_SECONDS:-300}"
|
||||||
GATEWAY_CONTROL_ADDR="${GATEWAY_CONTROL_ADDR:-127.0.0.1:8090}"
|
GATEWAY_CONTROL_ADDR="${GATEWAY_CONTROL_ADDR:-127.0.0.1:8090}"
|
||||||
GATEWAY_CMPP_ADDR="${GATEWAY_CMPP_ADDR:-0.0.0.0:17890}"
|
GATEWAY_CMPP_ADDR="${GATEWAY_CMPP_ADDR:-0.0.0.0:17890}"
|
||||||
CMPP_PUBLIC_HOST="${CMPP_PUBLIC_HOST:-8.160.169.106}"
|
CMPP_PUBLIC_HOST="${CMPP_PUBLIC_HOST:-8.160.169.106}"
|
||||||
@@ -182,7 +189,7 @@ SQL
|
|||||||
|
|
||||||
write_env() {
|
write_env() {
|
||||||
log "Writing production environment"
|
log "Writing production environment"
|
||||||
mkdir -p /etc/cmpp-platform "$APP_DIR" "$APP_DIR/logs/api" "$APP_DIR/logs/gateway" "$APP_DIR/backups" "$OBJECT_STORAGE_LOCAL_ROOT"
|
mkdir -p /etc/cmpp-platform "$APP_DIR" "$APP_DIR/logs/api" "$APP_DIR/logs/send-worker" "$APP_DIR/logs/gateway" "$APP_DIR/backups" "$OBJECT_STORAGE_LOCAL_ROOT"
|
||||||
cat >/etc/cmpp-platform/cmpp-platform.env <<EOF
|
cat >/etc/cmpp-platform/cmpp-platform.env <<EOF
|
||||||
NODE_ENV=production
|
NODE_ENV=production
|
||||||
API_PORT=${API_PORT}
|
API_PORT=${API_PORT}
|
||||||
@@ -191,6 +198,13 @@ API_METRICS_HOST=${API_METRICS_HOST}
|
|||||||
API_METRICS_PORT=${API_METRICS_PORT}
|
API_METRICS_PORT=${API_METRICS_PORT}
|
||||||
API_ENABLE_SEND_WORKER=${API_ENABLE_SEND_WORKER}
|
API_ENABLE_SEND_WORKER=${API_ENABLE_SEND_WORKER}
|
||||||
API_SEND_WORKER_CONCURRENCY=${API_SEND_WORKER_CONCURRENCY}
|
API_SEND_WORKER_CONCURRENCY=${API_SEND_WORKER_CONCURRENCY}
|
||||||
|
API_WORKER_METRICS_HOST=${API_WORKER_METRICS_HOST}
|
||||||
|
API_WORKER_METRICS_PORT=${API_WORKER_METRICS_PORT}
|
||||||
|
CMPP_INBOUND_FAST_PATH_ENABLED=${CMPP_INBOUND_FAST_PATH_ENABLED}
|
||||||
|
CMPP_INBOUND_WORKFLOW_WORKER_ENABLED=${CMPP_INBOUND_WORKFLOW_WORKER_ENABLED}
|
||||||
|
API_INBOUND_WORKFLOW_CONCURRENCY=${API_INBOUND_WORKFLOW_CONCURRENCY}
|
||||||
|
API_INBOUND_WORKFLOW_POLL_INTERVAL_MS=${API_INBOUND_WORKFLOW_POLL_INTERVAL_MS}
|
||||||
|
API_INBOUND_WORKFLOW_STALE_SECONDS=${API_INBOUND_WORKFLOW_STALE_SECONDS}
|
||||||
DATABASE_URL=postgresql://${DB_USER}:${DB_PASSWORD}@127.0.0.1:5432/${DB_NAME}?schema=public
|
DATABASE_URL=postgresql://${DB_USER}:${DB_PASSWORD}@127.0.0.1:5432/${DB_NAME}?schema=public
|
||||||
REDIS_HOST=127.0.0.1
|
REDIS_HOST=127.0.0.1
|
||||||
REDIS_PORT=6379
|
REDIS_PORT=6379
|
||||||
@@ -265,12 +279,34 @@ User=cmpp-api
|
|||||||
Group=cmpp-security
|
Group=cmpp-security
|
||||||
WorkingDirectory=${APP_DIR}/api
|
WorkingDirectory=${APP_DIR}/api
|
||||||
EnvironmentFile=/etc/cmpp-platform/cmpp-platform.env
|
EnvironmentFile=/etc/cmpp-platform/cmpp-platform.env
|
||||||
|
Environment=CMPP_PROCESS_ROLE=api
|
||||||
ExecStart=${node_bin} dist/main.js
|
ExecStart=${node_bin} dist/main.js
|
||||||
Restart=always
|
Restart=always
|
||||||
RestartSec=5
|
RestartSec=5
|
||||||
StandardOutput=append:${APP_DIR}/logs/api/stdout.log
|
StandardOutput=append:${APP_DIR}/logs/api/stdout.log
|
||||||
StandardError=append:${APP_DIR}/logs/api/stderr.log
|
StandardError=append:${APP_DIR}/logs/api/stderr.log
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=multi-user.target
|
||||||
|
EOF
|
||||||
|
|
||||||
|
cat >/etc/systemd/system/cmpp-send-worker.service <<EOF
|
||||||
|
[Unit]
|
||||||
|
Description=CMPP durable send workflow worker
|
||||||
|
After=network.target postgresql.service redis.service cmpp-minio.service
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
User=cmpp-api
|
||||||
|
Group=cmpp-security
|
||||||
|
WorkingDirectory=${APP_DIR}/api
|
||||||
|
EnvironmentFile=/etc/cmpp-platform/cmpp-platform.env
|
||||||
|
Environment=CMPP_PROCESS_ROLE=worker
|
||||||
|
ExecStart=${node_bin} dist/send-worker.js
|
||||||
|
Restart=always
|
||||||
|
RestartSec=5
|
||||||
|
StandardOutput=append:${APP_DIR}/logs/send-worker/stdout.log
|
||||||
|
StandardError=append:${APP_DIR}/logs/send-worker/stderr.log
|
||||||
|
|
||||||
[Install]
|
[Install]
|
||||||
WantedBy=multi-user.target
|
WantedBy=multi-user.target
|
||||||
EOF
|
EOF
|
||||||
|
|||||||
@@ -29,6 +29,16 @@ if [[ ! "${API_SEND_WORKER_CONCURRENCY:-}" =~ ^[1-9][0-9]*$ ]]; then
|
|||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
if [[ "${CMPP_INBOUND_FAST_PATH_ENABLED:-}" != "true" || "${CMPP_INBOUND_WORKFLOW_WORKER_ENABLED:-}" != "true" ]]; then
|
||||||
|
echo "CMPP_INBOUND_FAST_PATH_ENABLED=true and CMPP_INBOUND_WORKFLOW_WORKER_ENABLED=true are required in $ENV_FILE." >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [[ ! "${API_INBOUND_WORKFLOW_CONCURRENCY:-}" =~ ^[1-9][0-9]*$ ]]; then
|
||||||
|
echo "API_INBOUND_WORKFLOW_CONCURRENCY must be a positive integer in $ENV_FILE." >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
if [[ -z "${CMPP_PUBLIC_HOST:-}" || ! "${CMPP_PUBLIC_PORT:-}" =~ ^[1-9][0-9]*$ ]]; then
|
if [[ -z "${CMPP_PUBLIC_HOST:-}" || ! "${CMPP_PUBLIC_PORT:-}" =~ ^[1-9][0-9]*$ ]]; then
|
||||||
echo "CMPP_PUBLIC_HOST and a positive CMPP_PUBLIC_PORT are required in $ENV_FILE; these are the customer-facing CMPP endpoint." >&2
|
echo "CMPP_PUBLIC_HOST and a positive CMPP_PUBLIC_PORT are required in $ENV_FILE; these are the customer-facing CMPP endpoint." >&2
|
||||||
exit 1
|
exit 1
|
||||||
@@ -64,7 +74,35 @@ PROD_ADMIN_CREDENTIAL_FILE="$ADMIN_CREDENTIAL_FILE" node tools/deploy/ensure-pro
|
|||||||
chmod 600 "$ADMIN_CREDENTIAL_FILE" || true
|
chmod 600 "$ADMIN_CREDENTIAL_FILE" || true
|
||||||
|
|
||||||
echo "[deploy] Ensuring runtime log directories"
|
echo "[deploy] Ensuring runtime log directories"
|
||||||
install -d -m 0755 "$APP_DIR/logs/api" "$APP_DIR/logs/gateway"
|
install -d -m 0755 "$APP_DIR/logs/api" "$APP_DIR/logs/send-worker" "$APP_DIR/logs/gateway"
|
||||||
|
|
||||||
|
echo "[deploy] Installing split API and send-worker services"
|
||||||
|
node_bin="$(command -v node)"
|
||||||
|
install -d -m 0755 /etc/systemd/system/cmpp-api.service.d
|
||||||
|
cat >/etc/systemd/system/cmpp-api.service.d/process-role.conf <<'EOF'
|
||||||
|
[Service]
|
||||||
|
Environment=CMPP_PROCESS_ROLE=api
|
||||||
|
EOF
|
||||||
|
cat >/etc/systemd/system/cmpp-send-worker.service <<EOF
|
||||||
|
[Unit]
|
||||||
|
Description=CMPP durable send workflow worker
|
||||||
|
After=network.target postgresql.service redis.service cmpp-minio.service
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
User=cmpp-api
|
||||||
|
Group=cmpp-security
|
||||||
|
WorkingDirectory=$APP_DIR/api
|
||||||
|
EnvironmentFile=$ENV_FILE
|
||||||
|
Environment=CMPP_PROCESS_ROLE=worker
|
||||||
|
ExecStart=$node_bin dist/send-worker.js
|
||||||
|
Restart=always
|
||||||
|
RestartSec=5
|
||||||
|
StandardOutput=append:$APP_DIR/logs/send-worker/stdout.log
|
||||||
|
StandardError=append:$APP_DIR/logs/send-worker/stderr.log
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=multi-user.target
|
||||||
|
EOF
|
||||||
|
|
||||||
echo "[deploy] Installing restricted security boundary"
|
echo "[deploy] Installing restricted security boundary"
|
||||||
bash "$APP_DIR/tools/security/install-security-agent.sh"
|
bash "$APP_DIR/tools/security/install-security-agent.sh"
|
||||||
@@ -90,15 +128,16 @@ echo "[deploy] Restarting services"
|
|||||||
systemctl daemon-reload
|
systemctl daemon-reload
|
||||||
if [[ "${OBJECT_STORAGE_DRIVER:-minio}" == "local" ]]; then
|
if [[ "${OBJECT_STORAGE_DRIVER:-minio}" == "local" ]]; then
|
||||||
systemctl disable --now cmpp-minio 2>/dev/null || true
|
systemctl disable --now cmpp-minio 2>/dev/null || true
|
||||||
systemctl enable --now cmpp-api cmpp-gateway nginx
|
systemctl enable --now cmpp-api cmpp-send-worker cmpp-gateway nginx
|
||||||
else
|
else
|
||||||
systemctl enable --now cmpp-minio
|
systemctl enable --now cmpp-minio
|
||||||
systemctl restart cmpp-minio
|
systemctl restart cmpp-minio
|
||||||
systemctl enable --now cmpp-api cmpp-gateway nginx
|
systemctl enable --now cmpp-api cmpp-send-worker cmpp-gateway nginx
|
||||||
fi
|
fi
|
||||||
systemctl restart cmpp-gateway
|
systemctl restart cmpp-gateway
|
||||||
systemctl restart cmpp-security-agent
|
systemctl restart cmpp-security-agent
|
||||||
systemctl restart cmpp-api
|
systemctl restart cmpp-api
|
||||||
|
systemctl restart cmpp-send-worker
|
||||||
systemctl restart nginx
|
systemctl restart nginx
|
||||||
|
|
||||||
echo "[deploy] Health checks"
|
echo "[deploy] Health checks"
|
||||||
@@ -117,6 +156,7 @@ wait_for_http() {
|
|||||||
return 1
|
return 1
|
||||||
}
|
}
|
||||||
wait_for_http "API" "http://127.0.0.1:${API_PORT:-3000}/api/health"
|
wait_for_http "API" "http://127.0.0.1:${API_PORT:-3000}/api/health"
|
||||||
|
wait_for_http "Send worker metrics" "http://127.0.0.1:${API_WORKER_METRICS_PORT:-9465}/metrics"
|
||||||
wait_for_http "Gateway" "http://127.0.0.1:8090/health"
|
wait_for_http "Gateway" "http://127.0.0.1:8090/health"
|
||||||
redis-cli -h "${REDIS_HOST:-127.0.0.1}" -p "${REDIS_PORT:-6379}" ping >/dev/null
|
redis-cli -h "${REDIS_HOST:-127.0.0.1}" -p "${REDIS_PORT:-6379}" ping >/dev/null
|
||||||
pg_isready -d "${DATABASE_URL%%\?*}" >/dev/null
|
pg_isready -d "${DATABASE_URL%%\?*}" >/dev/null
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ import { resolve } from 'node:path';
|
|||||||
const deploy = readFileSync(resolve(import.meta.dirname, 'production-deploy.sh'), 'utf8');
|
const deploy = readFileSync(resolve(import.meta.dirname, 'production-deploy.sh'), 'utf8');
|
||||||
const bootstrap = readFileSync(resolve(import.meta.dirname, 'production-bootstrap.sh'), 'utf8');
|
const bootstrap = readFileSync(resolve(import.meta.dirname, 'production-bootstrap.sh'), 'utf8');
|
||||||
const apiMain = readFileSync(resolve(import.meta.dirname, '../../api/src/main.ts'), 'utf8');
|
const apiMain = readFileSync(resolve(import.meta.dirname, '../../api/src/main.ts'), 'utf8');
|
||||||
|
const workerMain = readFileSync(resolve(import.meta.dirname, '../../api/src/send-worker.ts'), 'utf8');
|
||||||
const required = [
|
const required = [
|
||||||
'compression_config=/etc/nginx/conf.d/cmpp-compression.conf',
|
'compression_config=/etc/nginx/conf.d/cmpp-compression.conf',
|
||||||
': >"$compression_config"',
|
': >"$compression_config"',
|
||||||
@@ -26,4 +27,22 @@ if (!apiMain.includes("process.env.API_HOST?.trim() || '127.0.0.1'") || !apiMain
|
|||||||
throw new Error('NestJS API must bind to API_HOST and default to loopback');
|
throw new Error('NestJS API must bind to API_HOST and default to loopback');
|
||||||
}
|
}
|
||||||
|
|
||||||
console.log('Production deployment verified: Nginx compression is idempotent and NestJS defaults to loopback.');
|
for (const marker of [
|
||||||
|
'CMPP_INBOUND_FAST_PATH_ENABLED=true',
|
||||||
|
'CMPP_INBOUND_WORKFLOW_WORKER_ENABLED=true',
|
||||||
|
'API_INBOUND_WORKFLOW_CONCURRENCY',
|
||||||
|
'cmpp-send-worker.service',
|
||||||
|
'Environment=CMPP_PROCESS_ROLE=api',
|
||||||
|
'Environment=CMPP_PROCESS_ROLE=worker',
|
||||||
|
'dist/send-worker.js',
|
||||||
|
'API_WORKER_METRICS_PORT',
|
||||||
|
]) {
|
||||||
|
if (!`${deploy}\n${bootstrap}`.includes(marker)) {
|
||||||
|
throw new Error(`production deployment is missing the durable inbound worker contract: ${marker}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (!workerMain.includes("process.env.API_WORKER_METRICS_HOST?.trim() || '127.0.0.1'")) {
|
||||||
|
throw new Error('send worker metrics must default to loopback');
|
||||||
|
}
|
||||||
|
|
||||||
|
console.log('Production deployment verified: Nginx/API guards and the split durable send worker contract are present.');
|
||||||
|
|||||||
@@ -182,7 +182,7 @@ groups:
|
|||||||
- name: cmpp-core-services
|
- name: cmpp-core-services
|
||||||
rules:
|
rules:
|
||||||
- alert: CmppCoreServiceInactive
|
- alert: CmppCoreServiceInactive
|
||||||
expr: node_systemd_unit_state{name=~"cmpp-api\\.service|cmpp-gateway\\.service|postgresql\\.service|redis(-server)?\\.service|cmpp-minio\\.service|nginx\\.service",state="active"} == 0
|
expr: node_systemd_unit_state{name=~"cmpp-api\\.service|cmpp-send-worker\\.service|cmpp-gateway\\.service|postgresql\\.service|redis(-server)?\\.service|cmpp-minio\\.service|nginx\\.service",state="active"} == 0
|
||||||
for: 2m
|
for: 2m
|
||||||
labels:
|
labels:
|
||||||
severity: critical
|
severity: critical
|
||||||
@@ -200,6 +200,16 @@ groups:
|
|||||||
for: 2m
|
for: 2m
|
||||||
labels: { severity: critical, service: api }
|
labels: { severity: critical, service: api }
|
||||||
annotations: { summary: "API指标采集不可用", description: "Prometheus连续2分钟无法读取API内部指标端点。", currentValue: "{{ $value }}", threshold: "up = 1" }
|
annotations: { summary: "API指标采集不可用", description: "Prometheus连续2分钟无法读取API内部指标端点。", currentValue: "{{ $value }}", threshold: "up = 1" }
|
||||||
|
- alert: CmppSendWorkerMetricsDown
|
||||||
|
expr: up{job="cmpp-send-worker"} == 0
|
||||||
|
for: 2m
|
||||||
|
labels: { severity: critical, service: send-worker }
|
||||||
|
annotations: { summary: "发送Worker指标采集不可用", description: "Prometheus连续2分钟无法读取独立发送Worker指标端点。", currentValue: "{{ $value }}", threshold: "up = 1" }
|
||||||
|
- alert: CmppInboundWorkflowBacklogCritical
|
||||||
|
expr: cmpp_worker_inbound_workflow_oldest_pending_age_seconds > 120
|
||||||
|
for: 2m
|
||||||
|
labels: { severity: critical, service: send-worker }
|
||||||
|
annotations: { summary: "CMPP耐久Inbox严重积压", description: "最旧待处理CMPP Inbox连续2分钟超过120秒。", currentValue: "{{ printf \"%.0f\" $value }}s", threshold: "120s" }
|
||||||
- alert: CmppApiHttpErrorRateWarning
|
- alert: CmppApiHttpErrorRateWarning
|
||||||
expr: (sum(rate(cmpp_api_http_requests_total{status=~"5.."}[5m])) / clamp_min(sum(rate(cmpp_api_http_requests_total[5m])), 0.001) > 0.01) and (sum(rate(cmpp_api_http_requests_total{status=~"5.."}[5m])) / clamp_min(sum(rate(cmpp_api_http_requests_total[5m])), 0.001) <= 0.05) and sum(increase(cmpp_api_http_requests_total{status=~"5.."}[5m])) >= 5
|
expr: (sum(rate(cmpp_api_http_requests_total{status=~"5.."}[5m])) / clamp_min(sum(rate(cmpp_api_http_requests_total[5m])), 0.001) > 0.01) and (sum(rate(cmpp_api_http_requests_total{status=~"5.."}[5m])) / clamp_min(sum(rate(cmpp_api_http_requests_total[5m])), 0.001) <= 0.05) and sum(increase(cmpp_api_http_requests_total{status=~"5.."}[5m])) >= 5
|
||||||
for: 5m
|
for: 5m
|
||||||
|
|||||||
@@ -68,7 +68,7 @@ EOF
|
|||||||
cat >/etc/systemd/system/prometheus-node-exporter.service.d/cmpp-monitoring.conf <<EOF
|
cat >/etc/systemd/system/prometheus-node-exporter.service.d/cmpp-monitoring.conf <<EOF
|
||||||
[Service]
|
[Service]
|
||||||
ExecStart=
|
ExecStart=
|
||||||
ExecStart=${node_exporter_bin} --web.listen-address=127.0.0.1:9100 --collector.systemd --collector.systemd.unit-include='cmpp-api\\.service|cmpp-gateway\\.service|postgresql\\.service|redis(-server)?\\.service|cmpp-minio\\.service|nginx\\.service' --collector.filesystem.mount-points-exclude='^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/)'
|
ExecStart=${node_exporter_bin} --web.listen-address=127.0.0.1:9100 --collector.systemd --collector.systemd.unit-include='cmpp-api\\.service|cmpp-send-worker\\.service|cmpp-gateway\\.service|postgresql\\.service|redis(-server)?\\.service|cmpp-minio\\.service|nginx\\.service' --collector.filesystem.mount-points-exclude='^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/)'
|
||||||
EOF
|
EOF
|
||||||
|
|
||||||
log "Validating Prometheus configuration before restart"
|
log "Validating Prometheus configuration before restart"
|
||||||
|
|||||||
@@ -24,6 +24,10 @@ scrape_configs:
|
|||||||
static_configs:
|
static_configs:
|
||||||
- targets: [127.0.0.1:9464]
|
- targets: [127.0.0.1:9464]
|
||||||
|
|
||||||
|
- job_name: cmpp-send-worker
|
||||||
|
static_configs:
|
||||||
|
- targets: [127.0.0.1:9465]
|
||||||
|
|
||||||
- job_name: cmpp-gateway
|
- job_name: cmpp-gateway
|
||||||
metrics_path: /metrics
|
metrics_path: /metrics
|
||||||
static_configs:
|
static_configs:
|
||||||
|
|||||||
@@ -13,6 +13,7 @@ usermod -a -G cmpp-security cmpp-api
|
|||||||
install -d -o root -g cmpp-security -m 0770 /run/cmpp-security-agent
|
install -d -o root -g cmpp-security -m 0770 /run/cmpp-security-agent
|
||||||
install -d -o root -g cmpp-security -m 0750 /var/lib/cmpp-security-agent
|
install -d -o root -g cmpp-security -m 0750 /var/lib/cmpp-security-agent
|
||||||
install -d -o cmpp-api -g cmpp-security -m 0750 "$APP_DIR/logs/api"
|
install -d -o cmpp-api -g cmpp-security -m 0750 "$APP_DIR/logs/api"
|
||||||
|
install -d -o cmpp-api -g cmpp-security -m 0750 "$APP_DIR/logs/send-worker"
|
||||||
[[ -d /var/lib/cmpp-platform/object-storage ]] && chown -R cmpp-api:cmpp-security /var/lib/cmpp-platform/object-storage
|
[[ -d /var/lib/cmpp-platform/object-storage ]] && chown -R cmpp-api:cmpp-security /var/lib/cmpp-platform/object-storage
|
||||||
|
|
||||||
sed "s#@CMPP_SECURITY_AGENT_BIN@#$agent_binary#g" "$APP_DIR/deploy/security/cmpp-report-only.conf" >/etc/fail2ban/action.d/cmpp-report-only.conf
|
sed "s#@CMPP_SECURITY_AGENT_BIN@#$agent_binary#g" "$APP_DIR/deploy/security/cmpp-report-only.conf" >/etc/fail2ban/action.d/cmpp-report-only.conf
|
||||||
@@ -29,6 +30,18 @@ table inet cmpp_security {
|
|||||||
chain input { type filter hook input priority -10; policy accept; ip saddr @blocked_ipv4 drop; ip6 saddr @blocked_ipv6 drop; }
|
chain input { type filter hook input priority -10; policy accept; ip saddr @blocked_ipv4 drop; ip6 saddr @blocked_ipv6 drop; }
|
||||||
}
|
}
|
||||||
EOF
|
EOF
|
||||||
|
|
||||||
|
install -d -m 0755 /etc/systemd/system/cmpp-send-worker.service.d
|
||||||
|
cat >/etc/systemd/system/cmpp-send-worker.service.d/security-boundary.conf <<EOF
|
||||||
|
[Service]
|
||||||
|
User=cmpp-api
|
||||||
|
Group=cmpp-security
|
||||||
|
NoNewPrivileges=true
|
||||||
|
PrivateTmp=true
|
||||||
|
ProtectHome=true
|
||||||
|
ProtectSystem=true
|
||||||
|
ReadWritePaths=$APP_DIR/logs/send-worker /var/lib/cmpp-platform/object-storage
|
||||||
|
EOF
|
||||||
grep -q 'cmpp-security.nft' /etc/nftables.conf || printf '\ninclude "/etc/nftables.d/cmpp-security.nft"\n' >>/etc/nftables.conf
|
grep -q 'cmpp-security.nft' /etc/nftables.conf || printf '\ninclude "/etc/nftables.d/cmpp-security.nft"\n' >>/etc/nftables.conf
|
||||||
nft -c -f /etc/nftables.conf
|
nft -c -f /etc/nftables.conf
|
||||||
nft list table inet cmpp_security >/dev/null 2>&1 || nft -f /etc/nftables.d/cmpp-security.nft
|
nft list table inet cmpp_security >/dev/null 2>&1 || nft -f /etc/nftables.d/cmpp-security.nft
|
||||||
@@ -54,4 +67,4 @@ fail2ban-client -t
|
|||||||
nginx -t
|
nginx -t
|
||||||
systemctl daemon-reload
|
systemctl daemon-reload
|
||||||
systemctl enable cmpp-security-agent
|
systemctl enable cmpp-security-agent
|
||||||
echo "Security boundary installed. Restart cmpp-security-agent and cmpp-api only in the approved release window."
|
echo "Security boundary installed. Restart cmpp-security-agent, cmpp-api and cmpp-send-worker only in the approved release window."
|
||||||
|
|||||||
Reference in New Issue
Block a user