Compare commits
24
Commits
c4f36fc50d
...
c6f11014d6
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
c6f11014d6 | ||
|
|
03f72c87de | ||
|
|
2b5256d4a1 | ||
|
|
90345fba22 | ||
|
|
3c6f1beed1 | ||
|
|
fcf6d3e6b4 | ||
|
|
487b5282a6 | ||
|
|
0176aa6952 | ||
|
|
52028b9bbd | ||
|
|
57192b7586 | ||
|
|
f4560479d3 | ||
|
|
99fb346566 | ||
|
|
633e7a7055 | ||
|
|
229a0b28fd | ||
|
|
6708f1f7c5 | ||
|
|
53073461e9 | ||
|
|
0b63bcd74e | ||
|
|
26ef67fb6a | ||
|
|
14f993c1f8 | ||
|
|
67b760a599 | ||
|
|
485af688d2 | ||
|
|
e9c73333b3 | ||
|
|
0757a699ff | ||
|
|
b9a71fe0b9 |
@@ -12,6 +12,16 @@ HTTP_API_MASTER_KEY=replace-with-at-least-32-random-characters
|
||||
HTTP_API_PUBLIC_ORIGIN=https://api.example.com
|
||||
API_ENABLE_SEND_WORKER=true
|
||||
API_SEND_WORKER_CONCURRENCY=50
|
||||
API_WORKER_DATABASE_URL=
|
||||
API_WORKER_METRICS_HOST=127.0.0.1
|
||||
API_WORKER_METRICS_PORT=9465
|
||||
CMPP_INBOUND_FAST_PATH_ENABLED=true
|
||||
CMPP_INBOUND_WORKFLOW_WORKER_ENABLED=true
|
||||
API_INBOUND_WORKFLOW_CONCURRENCY=32
|
||||
API_INBOUND_WORKFLOW_BATCH_ENABLED=true
|
||||
API_INBOUND_WORKFLOW_BATCH_SIZE=64
|
||||
API_INBOUND_WORKFLOW_POLL_INTERVAL_MS=100
|
||||
API_INBOUND_WORKFLOW_STALE_SECONDS=300
|
||||
ADMIN_SESSION_IDLE_TIMEOUT_MS=3600000
|
||||
CLIENT_SESSION_IDLE_TIMEOUT_MS=7200000
|
||||
SESSION_LOCK_RECOVERY_MS=14400000
|
||||
@@ -42,4 +52,10 @@ GATEWAY_CMPP_VERSION=3.0
|
||||
GATEWAY_CMPP_ADDR=127.0.0.1:7890
|
||||
GATEWAY_CMPP_USER=900001
|
||||
GATEWAY_CMPP_PASSWORD=888888
|
||||
GATEWAY_SUBMIT_WORKER_CONCURRENCY=64
|
||||
GATEWAY_SUBMIT_RESULT_STREAM=gateway.submit.results
|
||||
GATEWAY_SUBMIT_RESULT_GROUP=cmpp-api-callback
|
||||
GATEWAY_SUBMIT_RESULT_CONSUMER=gateway-1
|
||||
GATEWAY_SUBMIT_RESULT_WORKER_CONCURRENCY=8
|
||||
GATEWAY_CMPP_INBOUND_MAX_CONCURRENCY=64
|
||||
CMPP_DOWNSTREAM_ACK_TIMEOUT_SECONDS=30
|
||||
|
||||
@@ -0,0 +1,6 @@
|
||||
ALTER TABLE "SmsSubmitRecord"
|
||||
ADD COLUMN "resultEventId" TEXT,
|
||||
ADD COLUMN "resultProcessedAt" TIMESTAMP(3);
|
||||
|
||||
CREATE UNIQUE INDEX "SmsSubmitRecord_resultEventId_key"
|
||||
ON "SmsSubmitRecord"("resultEventId");
|
||||
@@ -0,0 +1,91 @@
|
||||
CREATE TABLE "CmppInboundSubmissionInbox" (
|
||||
"id" TEXT NOT NULL,
|
||||
"requestKey" TEXT NOT NULL,
|
||||
"payloadHash" TEXT NOT NULL,
|
||||
"tenantId" TEXT NOT NULL,
|
||||
"applicationId" TEXT NOT NULL,
|
||||
"queuePriority" TEXT NOT NULL DEFAULT 'normal',
|
||||
"payload" JSONB NOT NULL,
|
||||
"response" JSONB NOT NULL,
|
||||
"status" TEXT NOT NULL DEFAULT 'pending',
|
||||
"attempts" INTEGER NOT NULL DEFAULT 0,
|
||||
"nextAttemptAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
"lockedAt" TIMESTAMP(3),
|
||||
"lockedBy" TEXT,
|
||||
"lastError" TEXT,
|
||||
"result" JSONB,
|
||||
"completedAt" TIMESTAMP(3),
|
||||
"createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
"updatedAt" TIMESTAMP(3) NOT NULL,
|
||||
|
||||
CONSTRAINT "CmppInboundSubmissionInbox_pkey" PRIMARY KEY ("id")
|
||||
);
|
||||
|
||||
CREATE UNIQUE INDEX "CmppInboundSubmissionInbox_requestKey_key"
|
||||
ON "CmppInboundSubmissionInbox"("requestKey");
|
||||
|
||||
CREATE INDEX "CmppInboundSubmissionInbox_status_nextAttemptAt_createdAt_idx"
|
||||
ON "CmppInboundSubmissionInbox"("status", "nextAttemptAt", "createdAt");
|
||||
|
||||
CREATE INDEX "CmppInboundSubmissionInbox_status_lockedAt_idx"
|
||||
ON "CmppInboundSubmissionInbox"("status", "lockedAt");
|
||||
|
||||
CREATE INDEX "CmppInboundSubmissionInbox_applicationId_createdAt_idx"
|
||||
ON "CmppInboundSubmissionInbox"("applicationId", "createdAt");
|
||||
|
||||
CREATE INDEX "CmppInboundSubmissionInbox_queuePriority_status_createdAt_idx"
|
||||
ON "CmppInboundSubmissionInbox"("queuePriority", "status", "createdAt");
|
||||
|
||||
ALTER TABLE "CmppInboundSubmissionInbox"
|
||||
ADD CONSTRAINT "CmppInboundSubmissionInbox_tenantId_fkey"
|
||||
FOREIGN KEY ("tenantId") REFERENCES "Tenant"("id") ON DELETE RESTRICT ON UPDATE CASCADE;
|
||||
|
||||
ALTER TABLE "CmppInboundSubmissionInbox"
|
||||
ADD CONSTRAINT "CmppInboundSubmissionInbox_applicationId_fkey"
|
||||
FOREIGN KEY ("applicationId") REFERENCES "SmsApplication"("id") ON DELETE RESTRICT ON UPDATE CASCADE;
|
||||
|
||||
CREATE TABLE "SmsApplicationDailyReservation" (
|
||||
"id" TEXT NOT NULL,
|
||||
"reservationKey" TEXT NOT NULL,
|
||||
"tenantId" TEXT NOT NULL,
|
||||
"applicationId" TEXT NOT NULL,
|
||||
"usageDate" DATE NOT NULL,
|
||||
"requestedCount" INTEGER NOT NULL,
|
||||
"dailyLimit" INTEGER NOT NULL,
|
||||
"usedCount" INTEGER,
|
||||
"reserved" BOOLEAN NOT NULL,
|
||||
"createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
CONSTRAINT "SmsApplicationDailyReservation_pkey" PRIMARY KEY ("id")
|
||||
);
|
||||
|
||||
CREATE UNIQUE INDEX "SmsApplicationDailyReservation_reservationKey_key"
|
||||
ON "SmsApplicationDailyReservation"("reservationKey");
|
||||
CREATE INDEX "SmsApplicationDailyReservation_applicationId_usageDate_idx"
|
||||
ON "SmsApplicationDailyReservation"("applicationId", "usageDate");
|
||||
CREATE INDEX "SmsApplicationDailyReservation_tenantId_createdAt_idx"
|
||||
ON "SmsApplicationDailyReservation"("tenantId", "createdAt");
|
||||
ALTER TABLE "SmsApplicationDailyReservation" ADD CONSTRAINT "SmsApplicationDailyReservation_tenantId_fkey"
|
||||
FOREIGN KEY ("tenantId") REFERENCES "Tenant"("id") ON DELETE RESTRICT ON UPDATE CASCADE;
|
||||
ALTER TABLE "SmsApplicationDailyReservation" ADD CONSTRAINT "SmsApplicationDailyReservation_applicationId_fkey"
|
||||
FOREIGN KEY ("applicationId") REFERENCES "SmsApplication"("id") ON DELETE RESTRICT ON UPDATE CASCADE;
|
||||
|
||||
CREATE TABLE "PhoneFrequencyReservation" (
|
||||
"id" TEXT NOT NULL,
|
||||
"reservationKey" TEXT NOT NULL,
|
||||
"tenantId" TEXT NOT NULL,
|
||||
"applicationId" TEXT NOT NULL,
|
||||
"result" JSONB NOT NULL,
|
||||
"createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
CONSTRAINT "PhoneFrequencyReservation_pkey" PRIMARY KEY ("id")
|
||||
);
|
||||
|
||||
CREATE UNIQUE INDEX "PhoneFrequencyReservation_reservationKey_key"
|
||||
ON "PhoneFrequencyReservation"("reservationKey");
|
||||
CREATE INDEX "PhoneFrequencyReservation_applicationId_createdAt_idx"
|
||||
ON "PhoneFrequencyReservation"("applicationId", "createdAt");
|
||||
CREATE INDEX "PhoneFrequencyReservation_tenantId_createdAt_idx"
|
||||
ON "PhoneFrequencyReservation"("tenantId", "createdAt");
|
||||
ALTER TABLE "PhoneFrequencyReservation" ADD CONSTRAINT "PhoneFrequencyReservation_tenantId_fkey"
|
||||
FOREIGN KEY ("tenantId") REFERENCES "Tenant"("id") ON DELETE RESTRICT ON UPDATE CASCADE;
|
||||
ALTER TABLE "PhoneFrequencyReservation" ADD CONSTRAINT "PhoneFrequencyReservation_applicationId_fkey"
|
||||
FOREIGN KEY ("applicationId") REFERENCES "SmsApplication"("id") ON DELETE RESTRICT ON UPDATE CASCADE;
|
||||
@@ -52,6 +52,9 @@ model Tenant {
|
||||
openApiRequests OpenApiRequest[]
|
||||
httpWebhookEvents HttpWebhookEvent[]
|
||||
cmppInboundLongMessages CmppInboundLongMessage[]
|
||||
cmppInboundSubmissionInboxes CmppInboundSubmissionInbox[]
|
||||
smsApplicationDailyReservations SmsApplicationDailyReservation[]
|
||||
phoneFrequencyReservations PhoneFrequencyReservation[]
|
||||
}
|
||||
|
||||
model EnterpriseCertification {
|
||||
@@ -467,10 +470,13 @@ model SmsApplication {
|
||||
httpWebhookEndpoints HttpWebhookEndpoint[]
|
||||
httpWebhookEvents HttpWebhookEvent[]
|
||||
dailyUsages SmsApplicationDailyUsage[]
|
||||
dailyReservations SmsApplicationDailyReservation[]
|
||||
inboundLongMessages CmppInboundLongMessage[]
|
||||
inboundSubmissionInboxes CmppInboundSubmissionInbox[]
|
||||
riskRules RiskRule[]
|
||||
phoneFrequencyStates PhoneFrequencyState[]
|
||||
phoneFrequencyHits PhoneFrequencyHit[]
|
||||
phoneFrequencyReservations PhoneFrequencyReservation[]
|
||||
|
||||
@@index([tenantId, status])
|
||||
@@index([status, createdAt])
|
||||
@@ -533,6 +539,25 @@ model SmsApplicationDailyUsage {
|
||||
@@index([usageDate])
|
||||
}
|
||||
|
||||
model SmsApplicationDailyReservation {
|
||||
id String @id @default(cuid())
|
||||
reservationKey String @unique
|
||||
tenantId String
|
||||
applicationId String
|
||||
usageDate DateTime @db.Date
|
||||
requestedCount Int
|
||||
dailyLimit Int
|
||||
usedCount Int?
|
||||
reserved Boolean
|
||||
createdAt DateTime @default(now())
|
||||
|
||||
tenant Tenant @relation(fields: [tenantId], references: [id])
|
||||
application SmsApplication @relation(fields: [applicationId], references: [id], onDelete: Restrict)
|
||||
|
||||
@@index([applicationId, usageDate])
|
||||
@@index([tenantId, createdAt])
|
||||
}
|
||||
|
||||
model SmsApplicationHttpIpAllowlist {
|
||||
id String @id @default(cuid())
|
||||
applicationId String
|
||||
@@ -1578,6 +1603,21 @@ model PhoneFrequencyHit {
|
||||
@@index([releasedById])
|
||||
}
|
||||
|
||||
model PhoneFrequencyReservation {
|
||||
id String @id @default(cuid())
|
||||
reservationKey String @unique
|
||||
tenantId String
|
||||
applicationId String
|
||||
result Json
|
||||
createdAt DateTime @default(now())
|
||||
|
||||
tenant Tenant @relation(fields: [tenantId], references: [id])
|
||||
application SmsApplication @relation(fields: [applicationId], references: [id], onDelete: Restrict)
|
||||
|
||||
@@index([applicationId, createdAt])
|
||||
@@index([tenantId, createdAt])
|
||||
}
|
||||
|
||||
model PhoneFrequencyWhitelist {
|
||||
id String @id @default(cuid())
|
||||
phoneNumber String @unique
|
||||
@@ -1760,6 +1800,8 @@ model SmsSubmitRecord {
|
||||
sequenceId Int?
|
||||
gatewayMessageId String?
|
||||
submitStatus String @default("queued")
|
||||
resultEventId String? @unique
|
||||
resultProcessedAt DateTime?
|
||||
costUnitPrice BigInt @default(0)
|
||||
costAmountCents BigInt @default(0)
|
||||
errorCode String?
|
||||
@@ -1950,6 +1992,35 @@ model CmppInboundLongMessageSegment {
|
||||
@@index([sequenceId])
|
||||
}
|
||||
|
||||
model CmppInboundSubmissionInbox {
|
||||
id String @id @default(cuid())
|
||||
requestKey String @unique
|
||||
payloadHash String
|
||||
tenantId String
|
||||
applicationId String
|
||||
queuePriority String @default("normal")
|
||||
payload Json
|
||||
response Json
|
||||
status String @default("pending")
|
||||
attempts Int @default(0)
|
||||
nextAttemptAt DateTime @default(now())
|
||||
lockedAt DateTime?
|
||||
lockedBy String?
|
||||
lastError String?
|
||||
result Json?
|
||||
completedAt DateTime?
|
||||
createdAt DateTime @default(now())
|
||||
updatedAt DateTime @updatedAt
|
||||
|
||||
tenant Tenant @relation(fields: [tenantId], references: [id])
|
||||
application SmsApplication @relation(fields: [applicationId], references: [id])
|
||||
|
||||
@@index([status, nextAttemptAt, createdAt])
|
||||
@@index([status, lockedAt])
|
||||
@@index([applicationId, createdAt])
|
||||
@@index([queuePriority, status, createdAt])
|
||||
}
|
||||
|
||||
model SmsReceiptRecord {
|
||||
id String @id @default(cuid())
|
||||
tenantId String?
|
||||
|
||||
@@ -11,12 +11,14 @@ function createPrismaMock() {
|
||||
findMany: jest.fn().mockResolvedValue([]),
|
||||
},
|
||||
tenantAccount: {
|
||||
findMany: jest.fn(),
|
||||
findMany: jest.fn().mockResolvedValue([]),
|
||||
findUnique: jest.fn().mockImplementation(() => Promise.resolve({ ...accountState })),
|
||||
findUniqueOrThrow: jest.fn().mockImplementation(() => Promise.resolve({ ...accountState })),
|
||||
create: jest.fn(),
|
||||
upsert: jest.fn().mockImplementation(() => Promise.resolve({ ...accountState })),
|
||||
updateMany: jest.fn().mockImplementation(({ data }) => {
|
||||
if (data.balanceCents?.increment !== undefined) accountState.balanceCents += data.balanceCents.increment;
|
||||
else if (data.balanceCents?.decrement !== undefined) accountState.balanceCents -= data.balanceCents.decrement;
|
||||
accountState.updatedAt = new Date(accountState.updatedAt.getTime() + 1);
|
||||
return Promise.resolve({ count: 1 });
|
||||
}),
|
||||
@@ -28,10 +30,12 @@ function createPrismaMock() {
|
||||
}),
|
||||
},
|
||||
accountTransaction: {
|
||||
findMany: jest.fn(),
|
||||
findMany: jest.fn().mockResolvedValue([]),
|
||||
findFirst: jest.fn(),
|
||||
findUnique: jest.fn().mockResolvedValue(null),
|
||||
findUniqueOrThrow: jest.fn().mockImplementation(({ where }) => Promise.resolve({ id: 'tx-charged', idempotencyKey: where.idempotencyKey })),
|
||||
create: jest.fn().mockImplementation(({ data }) => Promise.resolve({ id: `tx-${data.transactionType}`, ...data })),
|
||||
createMany: jest.fn().mockResolvedValue({ count: 2 }),
|
||||
},
|
||||
rechargeOrder: {
|
||||
findMany: jest.fn(),
|
||||
@@ -51,6 +55,7 @@ function createPrismaMock() {
|
||||
create: jest.fn().mockResolvedValue({ id: 'operation-1' }),
|
||||
},
|
||||
$executeRaw: jest.fn(),
|
||||
$queryRaw: jest.fn(),
|
||||
};
|
||||
return Object.assign(prisma, {
|
||||
$transaction: jest.fn((callback: (client: typeof prisma) => unknown) => callback(prisma)),
|
||||
@@ -94,7 +99,7 @@ describe('BillingService', () => {
|
||||
);
|
||||
});
|
||||
|
||||
it('allows sending only when cash balance plus credit is greater than zero', async () => {
|
||||
it('allows sending only when cash balance plus credit covers the required amount', async () => {
|
||||
const prisma = createPrismaMock();
|
||||
const service = new BillingService(prisma as never);
|
||||
|
||||
@@ -102,7 +107,7 @@ describe('BillingService', () => {
|
||||
expect.objectContaining({ availableAmount: 1000, canSend: true }),
|
||||
);
|
||||
await expect(service.checkAccount({ tenantId: 'tenant-1', amountCents: 1001 })).resolves.toEqual(
|
||||
expect.objectContaining({ availableAmount: 1000, canSend: true }),
|
||||
expect.objectContaining({ availableAmount: 1000, canSend: false }),
|
||||
);
|
||||
await service.updateCreditLimit('tenant-1', { creditCents: -1000, operatorId: 'admin-1' });
|
||||
await expect(service.checkAccount({ tenantId: 'tenant-1', amountCents: 1 })).resolves.toEqual(
|
||||
@@ -110,7 +115,7 @@ describe('BillingService', () => {
|
||||
);
|
||||
await service.updateCreditLimit('tenant-1', { creditCents: 500, operatorId: 'admin-1' });
|
||||
await expect(service.checkAccount({ tenantId: 'tenant-1', amountCents: 999999 })).resolves.toEqual(
|
||||
expect.objectContaining({ availableAmount: 1500, creditCents: 500, canSend: true }),
|
||||
expect.objectContaining({ availableAmount: 1500, creditCents: 500, canSend: false }),
|
||||
);
|
||||
await expect(service.updateCreditLimit('tenant-1', { creditCents: 1.5 })).rejects.toThrow('授信额度最多支持人民币小数点后 4 位');
|
||||
expect(prisma.operationLog.create).toHaveBeenCalledWith({
|
||||
@@ -312,6 +317,28 @@ describe('BillingService', () => {
|
||||
expect(prisma.accountState).toEqual(expect.objectContaining({ balanceCents: 890 }));
|
||||
});
|
||||
|
||||
it('settles a frozen SMS charge with one account lock and an idempotent ledger pair', async () => {
|
||||
const prisma = createPrismaMock();
|
||||
const rows = new Map<string, Record<string, unknown>>();
|
||||
prisma.accountTransaction.findMany.mockImplementation(() => Promise.resolve([...rows.values()]));
|
||||
prisma.$queryRaw.mockImplementation(() => {
|
||||
const charged = { id: 'tx-charged', idempotencyKey: 'sms-charge:msg-paid-1', transactionType: 'charged', amountCents: -325 };
|
||||
rows.set(String(charged.idempotencyKey), charged);
|
||||
return Promise.resolve([charged]);
|
||||
});
|
||||
const service = new BillingService(prisma as never);
|
||||
const input = { tenantId: 'tenant-1', amountCents: 325, messageId: 'msg-paid-1', taskId: 'task-paid-1' };
|
||||
|
||||
const first = await service.settleFrozenCharge(input);
|
||||
const replay = await service.settleFrozenCharge(input);
|
||||
|
||||
expect(first).toEqual(expect.objectContaining({ transactionType: 'charged', amountCents: -325 }));
|
||||
expect(replay).toEqual(first);
|
||||
expect(prisma.$queryRaw).toHaveBeenCalledTimes(1);
|
||||
expect(prisma.$executeRaw).not.toHaveBeenCalled();
|
||||
expect(prisma.tenantAccount.update).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('serializes and replays concurrent refunds with one balance mutation', async () => {
|
||||
const prisma = createPrismaMock();
|
||||
let transactionChain = Promise.resolve<unknown>(undefined);
|
||||
|
||||
@@ -416,7 +416,7 @@ export class BillingService {
|
||||
availableAmount,
|
||||
balanceCents,
|
||||
creditCents,
|
||||
canSend: availableAmount > 0,
|
||||
canSend: availableAmount >= requiredAmount,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -436,6 +436,56 @@ export class BillingService {
|
||||
});
|
||||
}
|
||||
|
||||
async settleFrozenCharge(data: { tenantId: string; amountCents: number; messageId: string; taskId: string; remark?: string }) {
|
||||
const amountCents = data.amountCents ?? 0;
|
||||
if (amountCents <= 0) return null;
|
||||
const releaseKey = `sms-charge-release:${data.messageId}`;
|
||||
const chargeKey = `sms-charge:${data.messageId}`;
|
||||
const existing = await this.prisma.accountTransaction.findMany({
|
||||
where: { idempotencyKey: { in: [releaseKey, chargeKey] } },
|
||||
});
|
||||
const charge = existing.find((row) => row.idempotencyKey === chargeKey);
|
||||
if (charge) return charge;
|
||||
const release = existing.find((row) => row.idempotencyKey === releaseKey);
|
||||
if (release) {
|
||||
// Recover the legacy two-transaction boundary: a crash may have committed
|
||||
// release before charge, so this path must perform the missing balance debit.
|
||||
return this.applyAccountDelta({
|
||||
tenantId: data.tenantId, transactionType: 'charged', idempotencyKey: chargeKey,
|
||||
amountCents: -amountCents, relatedType: 'sms_message_record', relatedId: data.messageId,
|
||||
remark: '提交成功扣费(恢复既有已释放冻结)',
|
||||
});
|
||||
}
|
||||
const rows = await this.prisma.$queryRaw<Array<{ id: string; idempotencyKey: string }>>(Prisma.sql`
|
||||
WITH account AS (
|
||||
SELECT "balanceCents" FROM "TenantAccount" WHERE "tenantId" = ${data.tenantId}
|
||||
), inserted AS (
|
||||
INSERT INTO "AccountTransaction" (
|
||||
id, "tenantId", "transactionType", "idempotencyKey", "amountCents",
|
||||
"balanceAfter", "relatedType", "relatedId", remark, "createdAt"
|
||||
)
|
||||
SELECT gen_random_uuid()::text, ${data.tenantId}, ledger."transactionType", ledger."idempotencyKey",
|
||||
ledger."amountCents", account."balanceCents" + ledger."balanceDelta",
|
||||
ledger."relatedType", ledger."relatedId", ledger.remark, (NOW() AT TIME ZONE 'UTC')
|
||||
FROM account
|
||||
CROSS JOIN (VALUES
|
||||
('released', ${releaseKey}, ${amountCents}::bigint, ${amountCents}::bigint, 'sms_batch_task', ${data.taskId}, ${data.remark ?? null}),
|
||||
('charged', ${chargeKey}, ${-amountCents}::bigint, 0::bigint, 'sms_message_record', ${data.messageId}, '提交成功扣费')
|
||||
) AS ledger("transactionType", "idempotencyKey", "amountCents", "balanceDelta", "relatedType", "relatedId", remark)
|
||||
ON CONFLICT ("idempotencyKey") DO NOTHING
|
||||
RETURNING id, "idempotencyKey"
|
||||
)
|
||||
SELECT id, "idempotencyKey" FROM inserted WHERE "idempotencyKey" = ${chargeKey}
|
||||
UNION ALL
|
||||
SELECT id, "idempotencyKey" FROM "AccountTransaction" WHERE "idempotencyKey" = ${chargeKey}
|
||||
LIMIT 1
|
||||
`);
|
||||
if (rows[0]) return rows[0];
|
||||
// A concurrent identical callback can win ON CONFLICT while remaining
|
||||
// invisible to this statement's snapshot; one read repairs that MVCC edge.
|
||||
return this.prisma.accountTransaction.findUniqueOrThrow({ where: { idempotencyKey: chargeKey } });
|
||||
}
|
||||
|
||||
release(data: BillingActionDto) {
|
||||
return this.applyAccountDelta({
|
||||
...data,
|
||||
|
||||
@@ -5,11 +5,14 @@ describe('MetricsService', () => {
|
||||
const service = new MetricsService();
|
||||
const startedAt = service.beginRequest();
|
||||
service.finishRequest(startedAt, 'GET', '/api/admin/tenants/:id', 200);
|
||||
const inboundStartedAt = service.beginCmppInboundStage();
|
||||
service.finishCmppInboundStage(inboundStartedAt, 'application_lookup', 'success');
|
||||
const output = service.render();
|
||||
|
||||
expect(output).toContain('cmpp_api_process_resident_memory_bytes');
|
||||
expect(output).toContain('cmpp_api_http_requests_total{method="GET",route="/api/admin/tenants/:id",status="200"} 1');
|
||||
expect(output).toContain('cmpp_api_http_request_duration_seconds_bucket');
|
||||
expect(output).toContain('cmpp_api_cmpp_inbound_stage_duration_seconds_count{stage="application_lookup",result="success"} 1');
|
||||
expect(output).not.toContain('phone_number');
|
||||
service.onModuleDestroy();
|
||||
});
|
||||
|
||||
@@ -2,6 +2,28 @@ import { Injectable, OnModuleDestroy } from '@nestjs/common';
|
||||
import { monitorEventLoopDelay } from 'node:perf_hooks';
|
||||
|
||||
const HTTP_DURATION_BUCKETS = [0.05, 0.1, 0.25, 0.5, 1, 3, 10] as const;
|
||||
const CMPP_INBOUND_DURATION_BUCKETS = [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 3, 10] as const;
|
||||
|
||||
export type CmppInboundStage =
|
||||
| 'application_lookup'
|
||||
| 'inbox_persist'
|
||||
| 'worker_claim'
|
||||
| 'reference_preload'
|
||||
| 'daily_quota'
|
||||
| 'long_message_fragment'
|
||||
| 'submission_precheck'
|
||||
| 'template_match'
|
||||
| 'task_persist'
|
||||
| 'api_request_persist'
|
||||
| 'content_detection'
|
||||
| 'message_persist'
|
||||
| 'risk_frequency'
|
||||
| 'billing'
|
||||
| 'queue_publish'
|
||||
| 'complete_submit'
|
||||
| 'total';
|
||||
|
||||
export type CmppInboundStageResult = 'success' | 'error';
|
||||
|
||||
type HttpMetric = {
|
||||
count: number;
|
||||
@@ -25,7 +47,14 @@ export class MetricsService implements OnModuleDestroy {
|
||||
private readonly startedAt = process.hrtime.bigint();
|
||||
private readonly eventLoopDelay = monitorEventLoopDelay({ resolution: 20 });
|
||||
private readonly http = new Map<string, HttpMetric>();
|
||||
private readonly cmppInbound = new Map<string, HttpMetric>();
|
||||
private inFlight = 0;
|
||||
private inboundWorkflowPending = 0;
|
||||
private inboundWorkflowProcessing = 0;
|
||||
private inboundWorkflowOldestPendingAgeSeconds = 0;
|
||||
private inboundWorkflowConfiguredSlots = 0;
|
||||
private inboundWorkflowInFlightSlots = 0;
|
||||
private readonly inboundWorkflowResults = new Map<string, number>();
|
||||
|
||||
constructor() {
|
||||
this.eventLoopDelay.enable();
|
||||
@@ -52,6 +81,41 @@ export class MetricsService implements OnModuleDestroy {
|
||||
this.http.set(key, metric);
|
||||
}
|
||||
|
||||
beginCmppInboundStage() {
|
||||
return process.hrtime.bigint();
|
||||
}
|
||||
|
||||
finishCmppInboundStage(startedAt: bigint, stage: CmppInboundStage, result: CmppInboundStageResult) {
|
||||
const key = `${stage}\u0000${result}`;
|
||||
const metric = this.cmppInbound.get(key) ?? {
|
||||
count: 0,
|
||||
durationSum: 0,
|
||||
buckets: CMPP_INBOUND_DURATION_BUCKETS.map(() => 0),
|
||||
};
|
||||
const durationSeconds = Number(process.hrtime.bigint() - startedAt) / 1_000_000_000;
|
||||
metric.count += 1;
|
||||
metric.durationSum += durationSeconds;
|
||||
CMPP_INBOUND_DURATION_BUCKETS.forEach((bucket, index) => {
|
||||
if (durationSeconds <= bucket) metric.buckets[index] += 1;
|
||||
});
|
||||
this.cmppInbound.set(key, metric);
|
||||
}
|
||||
|
||||
setInboundWorkflowState(pending: number, processing: number, oldestPendingAgeSeconds: number) {
|
||||
this.inboundWorkflowPending = Math.max(0, pending);
|
||||
this.inboundWorkflowProcessing = Math.max(0, processing);
|
||||
this.inboundWorkflowOldestPendingAgeSeconds = Math.max(0, oldestPendingAgeSeconds);
|
||||
}
|
||||
|
||||
setInboundWorkflowSlots(configured: number, inFlight: number) {
|
||||
this.inboundWorkflowConfiguredSlots = Math.max(0, configured);
|
||||
this.inboundWorkflowInFlightSlots = Math.max(0, inFlight);
|
||||
}
|
||||
|
||||
recordInboundWorkflowResult(result: 'completed' | 'retry') {
|
||||
this.inboundWorkflowResults.set(result, (this.inboundWorkflowResults.get(result) ?? 0) + 1);
|
||||
}
|
||||
|
||||
render() {
|
||||
const memory = process.memoryUsage();
|
||||
const uptime = Number(process.hrtime.bigint() - this.startedAt) / 1_000_000_000;
|
||||
@@ -78,6 +142,21 @@ export class MetricsService implements OnModuleDestroy {
|
||||
'# TYPE cmpp_api_http_requests_total counter',
|
||||
'# HELP cmpp_api_http_request_duration_seconds API request duration.',
|
||||
'# TYPE cmpp_api_http_request_duration_seconds histogram',
|
||||
'# HELP cmpp_api_cmpp_inbound_stage_duration_seconds CMPP inbound processing duration by bounded stage and result.',
|
||||
'# TYPE cmpp_api_cmpp_inbound_stage_duration_seconds histogram',
|
||||
'# HELP cmpp_worker_inbound_workflow_items Current durable CMPP inbound workflow items by state.',
|
||||
'# TYPE cmpp_worker_inbound_workflow_items gauge',
|
||||
metricLine('cmpp_worker_inbound_workflow_items', this.inboundWorkflowPending, { state: 'pending' }),
|
||||
metricLine('cmpp_worker_inbound_workflow_items', this.inboundWorkflowProcessing, { state: 'processing' }),
|
||||
'# HELP cmpp_worker_inbound_workflow_slots Durable workflow worker slots by state.',
|
||||
'# TYPE cmpp_worker_inbound_workflow_slots gauge',
|
||||
metricLine('cmpp_worker_inbound_workflow_slots', this.inboundWorkflowConfiguredSlots, { state: 'configured' }),
|
||||
metricLine('cmpp_worker_inbound_workflow_slots', this.inboundWorkflowInFlightSlots, { state: 'in_flight' }),
|
||||
'# HELP cmpp_worker_inbound_workflow_oldest_pending_age_seconds Age of the oldest pending durable workflow.',
|
||||
'# TYPE cmpp_worker_inbound_workflow_oldest_pending_age_seconds gauge',
|
||||
metricLine('cmpp_worker_inbound_workflow_oldest_pending_age_seconds', this.inboundWorkflowOldestPendingAgeSeconds),
|
||||
'# HELP cmpp_worker_inbound_workflow_results_total Durable workflow processing outcomes.',
|
||||
'# TYPE cmpp_worker_inbound_workflow_results_total counter',
|
||||
];
|
||||
for (const [key, metric] of this.http) {
|
||||
const [method, route, status] = key.split('\u0000');
|
||||
@@ -90,6 +169,19 @@ export class MetricsService implements OnModuleDestroy {
|
||||
lines.push(metricLine('cmpp_api_http_request_duration_seconds_sum', metric.durationSum, labels));
|
||||
lines.push(metricLine('cmpp_api_http_request_duration_seconds_count', metric.count, labels));
|
||||
}
|
||||
for (const [key, metric] of this.cmppInbound) {
|
||||
const [stage, result] = key.split('\u0000');
|
||||
const labels = { stage, result };
|
||||
CMPP_INBOUND_DURATION_BUCKETS.forEach((bucket, index) => {
|
||||
lines.push(metricLine('cmpp_api_cmpp_inbound_stage_duration_seconds_bucket', metric.buckets[index], { ...labels, le: String(bucket) }));
|
||||
});
|
||||
lines.push(metricLine('cmpp_api_cmpp_inbound_stage_duration_seconds_bucket', metric.count, { ...labels, le: '+Inf' }));
|
||||
lines.push(metricLine('cmpp_api_cmpp_inbound_stage_duration_seconds_sum', metric.durationSum, labels));
|
||||
lines.push(metricLine('cmpp_api_cmpp_inbound_stage_duration_seconds_count', metric.count, labels));
|
||||
}
|
||||
for (const [result, count] of this.inboundWorkflowResults) {
|
||||
lines.push(metricLine('cmpp_worker_inbound_workflow_results_total', count, { result }));
|
||||
}
|
||||
this.eventLoopDelay.reset();
|
||||
return `${lines.join('\n')}\n`;
|
||||
}
|
||||
|
||||
@@ -6,11 +6,24 @@ import { requestContext } from '../common/request-context';
|
||||
@Injectable()
|
||||
export class PrismaService extends PrismaClient implements OnModuleDestroy {
|
||||
constructor() {
|
||||
const workerRole = process.env.CMPP_PROCESS_ROLE === 'worker';
|
||||
const databaseUrl = workerRole
|
||||
? process.env.API_WORKER_DATABASE_URL || process.env.DATABASE_URL
|
||||
: process.env.DATABASE_URL;
|
||||
const configuredPoolMax = Number(workerRole
|
||||
? process.env.API_WORKER_DB_POOL_MAX ?? 8
|
||||
: process.env.API_DB_POOL_MAX ?? 32);
|
||||
const poolMax = Number.isInteger(configuredPoolMax) && configuredPoolMax > 0
|
||||
? configuredPoolMax
|
||||
: workerRole ? 8 : 32;
|
||||
super({
|
||||
adapter: new PrismaPg(
|
||||
process.env.DATABASE_URL ??
|
||||
'postgresql://cmpp:cmpp_password@localhost:5432/cmpp_platform?schema=public',
|
||||
),
|
||||
adapter: new PrismaPg({
|
||||
connectionString: databaseUrl
|
||||
?? 'postgresql://cmpp:cmpp_password@localhost:5432/cmpp_platform?schema=public',
|
||||
// API capacity must be reserved independently from the heavier Worker
|
||||
// transactions; explicit bounds also protect PostgreSQL max_connections.
|
||||
max: poolMax,
|
||||
}),
|
||||
});
|
||||
const operationLog = this.operationLog;
|
||||
Object.defineProperty(this, 'operationLog', {
|
||||
|
||||
@@ -1,6 +1,37 @@
|
||||
import { PhoneFrequencyService, fixedShanghaiWindow } from './phone-frequency.service';
|
||||
|
||||
describe('PhoneFrequencyService', () => {
|
||||
it('persists independent idempotency results for a unique-phone batch in one transaction', async () => {
|
||||
const tx = {
|
||||
phoneFrequencyReservation: {
|
||||
findMany: jest.fn().mockResolvedValue([]),
|
||||
createMany: jest.fn().mockResolvedValue({ count: 2 }),
|
||||
},
|
||||
phoneFrequencyWhitelist: { findMany: jest.fn().mockResolvedValue([]) },
|
||||
};
|
||||
const prisma = {
|
||||
riskRule: { findMany: jest.fn().mockResolvedValue([]) },
|
||||
$transaction: jest.fn(async (callback: (client: typeof tx) => unknown) => callback(tx)),
|
||||
};
|
||||
const riskReview = { ensureDefaultRules: jest.fn().mockResolvedValue(undefined) };
|
||||
const service = new PhoneFrequencyService(prisma as never, riskReview as never);
|
||||
|
||||
const results = await service.reserveBatch([
|
||||
{ tenantId: 'tenant-1', applicationId: 'app-1', phoneNumber: '13800000001', reservationKey: 'inbox-1:frequency' },
|
||||
{ tenantId: 'tenant-1', applicationId: 'app-1', phoneNumber: '13800000002', reservationKey: 'inbox-2:frequency' },
|
||||
]);
|
||||
|
||||
expect(results.get('inbox-1:frequency')?.size).toBe(0);
|
||||
expect(results.get('inbox-2:frequency')?.size).toBe(0);
|
||||
expect(prisma.$transaction).toHaveBeenCalledTimes(1);
|
||||
expect(tx.phoneFrequencyReservation.createMany).toHaveBeenCalledWith({
|
||||
data: expect.arrayContaining([
|
||||
expect.objectContaining({ reservationKey: 'inbox-1:frequency', result: [] }),
|
||||
expect.objectContaining({ reservationKey: 'inbox-2:frequency', result: [] }),
|
||||
]),
|
||||
});
|
||||
});
|
||||
|
||||
it('aligns five-minute cycles and natural days in Asia/Shanghai', () => {
|
||||
const requestedAt = new Date('2026-07-30T16:07:42.000Z');
|
||||
|
||||
|
||||
@@ -63,6 +63,15 @@ export interface PhoneFrequencyRejection {
|
||||
reason: string;
|
||||
}
|
||||
|
||||
export interface PhoneFrequencyBatchReservation {
|
||||
tenantId: string;
|
||||
applicationId: string;
|
||||
phoneNumber: string;
|
||||
reservationKey: string;
|
||||
sourceType?: string;
|
||||
requestedAt?: Date;
|
||||
}
|
||||
|
||||
@Injectable()
|
||||
export class PhoneFrequencyService {
|
||||
constructor(
|
||||
@@ -80,6 +89,7 @@ export class PhoneFrequencyService {
|
||||
phones: string[],
|
||||
sourceType?: string,
|
||||
requestedAt = new Date(),
|
||||
reservationKey?: string,
|
||||
) {
|
||||
if (!applicationId) return new Map<string, PhoneFrequencyRejection>();
|
||||
const normalizedPhones = [...new Set(phones.map((phone) => phone.trim()).filter(Boolean))].sort();
|
||||
@@ -87,14 +97,35 @@ export class PhoneFrequencyService {
|
||||
|
||||
await this.riskReview.ensureDefaultRules();
|
||||
const rules = await this.effectiveRules(applicationId);
|
||||
if (rules.length === 0) return new Map<string, PhoneFrequencyRejection>();
|
||||
const normalizedReservationKey = reservationKey?.trim();
|
||||
|
||||
return this.prisma.$transaction(async (tx) => {
|
||||
if (normalizedReservationKey) {
|
||||
// Frequency counters and the reservation result commit together. Retrying a reclaimed
|
||||
// Inbox item therefore returns the original decision without incrementing either window.
|
||||
await tx.$executeRaw`SELECT pg_advisory_xact_lock(hashtextextended(${'phone-frequency:' + normalizedReservationKey}, 0))`;
|
||||
const existingReservation = await tx.phoneFrequencyReservation.findUnique({
|
||||
where: { reservationKey: normalizedReservationKey },
|
||||
});
|
||||
if (existingReservation) {
|
||||
if (existingReservation.tenantId !== tenantId || existingReservation.applicationId !== applicationId) {
|
||||
throw new BadRequestException('号码频控幂等键已用于另一笔预留');
|
||||
}
|
||||
return frequencyRejectionsFromJson(existingReservation.result);
|
||||
}
|
||||
}
|
||||
const rejected = new Map<string, PhoneFrequencyRejection>();
|
||||
// 平台级白名单只截断号码频控链路;调用 reserve 之前已执行的格式、黑名单等校验不受影响。
|
||||
const whitelistedPhones = await this.findActiveWhitelistedPhones(tx, normalizedPhones);
|
||||
const controlledPhones = normalizedPhones.filter((phone) => !whitelistedPhones.has(phone));
|
||||
if (controlledPhones.length === 0) return rejected;
|
||||
if (controlledPhones.length === 0 || rules.length === 0) {
|
||||
if (normalizedReservationKey) {
|
||||
await tx.phoneFrequencyReservation.create({
|
||||
data: { reservationKey: normalizedReservationKey, tenantId, applicationId, result: [] },
|
||||
});
|
||||
}
|
||||
return rejected;
|
||||
}
|
||||
for (const rule of rules) {
|
||||
const window = fixedShanghaiWindow(requestedAt, readPeriodSeconds(rule));
|
||||
// 分块限制 SQL 参数数量,但两条规则的全部分块仍在同一事务中提交或回滚。
|
||||
@@ -144,10 +175,155 @@ export class PhoneFrequencyService {
|
||||
}
|
||||
}
|
||||
}
|
||||
if (normalizedReservationKey) {
|
||||
await tx.phoneFrequencyReservation.create({
|
||||
data: {
|
||||
reservationKey: normalizedReservationKey,
|
||||
tenantId,
|
||||
applicationId,
|
||||
result: [...rejected.entries()].map(([phoneNumber, rejection]) => ({ phoneNumber, ...rejection })),
|
||||
},
|
||||
});
|
||||
}
|
||||
return rejected;
|
||||
}, { isolationLevel: Prisma.TransactionIsolationLevel.ReadCommitted });
|
||||
}
|
||||
|
||||
/**
|
||||
* Reserve independent one-phone Inbox items in bounded database batches. The
|
||||
* reservation rows and counters commit in the same transaction, so reclaiming
|
||||
* any subset replays its original decision. Duplicate phones intentionally use
|
||||
* the established single-item path because their within-batch threshold order
|
||||
* is business-significant.
|
||||
*/
|
||||
async reserveBatch(items: PhoneFrequencyBatchReservation[]) {
|
||||
const results = new Map<string, Map<string, PhoneFrequencyRejection>>();
|
||||
if (items.length === 0) return results;
|
||||
const reservationKeys = items.map((item) => item.reservationKey.trim());
|
||||
if (reservationKeys.some((key) => !key) || new Set(reservationKeys).size !== reservationKeys.length) {
|
||||
throw new BadRequestException('号码频控批次幂等键为空或重复');
|
||||
}
|
||||
const groups = new Map<string, PhoneFrequencyBatchReservation[]>();
|
||||
for (const item of items) {
|
||||
const key = `${item.tenantId}:${item.applicationId}`;
|
||||
const group = groups.get(key) ?? [];
|
||||
group.push({ ...item, phoneNumber: item.phoneNumber.trim(), reservationKey: item.reservationKey.trim() });
|
||||
groups.set(key, group);
|
||||
}
|
||||
for (const group of groups.values()) {
|
||||
const uniquePhones = new Set(group.map((item) => item.phoneNumber));
|
||||
if (uniquePhones.size !== group.length) {
|
||||
for (const item of group) {
|
||||
results.set(item.reservationKey, await this.reserve(
|
||||
item.tenantId,
|
||||
item.applicationId,
|
||||
[item.phoneNumber],
|
||||
item.sourceType,
|
||||
item.requestedAt ?? new Date(),
|
||||
item.reservationKey,
|
||||
));
|
||||
}
|
||||
continue;
|
||||
}
|
||||
await this.riskReview.ensureDefaultRules();
|
||||
const rules = await this.effectiveRules(group[0].applicationId);
|
||||
const groupResults = await this.prisma.$transaction(async (tx) => {
|
||||
const output = new Map<string, Map<string, PhoneFrequencyRejection>>();
|
||||
const existing = await tx.phoneFrequencyReservation.findMany({
|
||||
where: { reservationKey: { in: group.map((item) => item.reservationKey) } },
|
||||
});
|
||||
const existingByKey = new Map(existing.map((item) => [item.reservationKey, item]));
|
||||
const missing: PhoneFrequencyBatchReservation[] = [];
|
||||
for (const item of group) {
|
||||
const replay = existingByKey.get(item.reservationKey);
|
||||
if (!replay) {
|
||||
missing.push(item);
|
||||
continue;
|
||||
}
|
||||
if (replay.tenantId !== item.tenantId || replay.applicationId !== item.applicationId) {
|
||||
throw new BadRequestException('号码频控幂等键已用于另一笔预留');
|
||||
}
|
||||
output.set(item.reservationKey, frequencyRejectionsFromJson(replay.result));
|
||||
}
|
||||
if (missing.length === 0) return output;
|
||||
const whitelistedPhones = await this.findActiveWhitelistedPhones(tx, missing.map((item) => item.phoneNumber));
|
||||
const controlled = missing.filter((item) => !whitelistedPhones.has(item.phoneNumber));
|
||||
const rejectedByPhone = new Map<string, PhoneFrequencyRejection>();
|
||||
for (const rule of rules) {
|
||||
const byWindow = new Map<string, { startAt: Date; endAt: Date; items: PhoneFrequencyBatchReservation[] }>();
|
||||
for (const item of controlled) {
|
||||
const window = fixedShanghaiWindow(item.requestedAt ?? new Date(), readPeriodSeconds(rule));
|
||||
const key = `${window.startAt.toISOString()}:${window.endAt.toISOString()}`;
|
||||
const bucket = byWindow.get(key) ?? { ...window, items: [] };
|
||||
bucket.items.push(item);
|
||||
byWindow.set(key, bucket);
|
||||
}
|
||||
for (const bucket of byWindow.values()) {
|
||||
const states = await this.upsertStates(tx, {
|
||||
tenantId: group[0].tenantId,
|
||||
applicationId: group[0].applicationId,
|
||||
phones: bucket.items.map((item) => item.phoneNumber),
|
||||
rule,
|
||||
window: { startAt: bucket.startAt, endAt: bucket.endAt },
|
||||
});
|
||||
const newTriggers = states.filter((state) => state.activeHitId === null && state.count > rule.thresholdValue);
|
||||
const hitByStateId = new Map<string, string>();
|
||||
if (newTriggers.length > 0) {
|
||||
await tx.phoneFrequencyHit.createMany({
|
||||
data: newTriggers.map((state) => {
|
||||
const hitId = randomUUID();
|
||||
hitByStateId.set(state.id, hitId);
|
||||
return {
|
||||
id: hitId,
|
||||
tenantId: group[0].tenantId,
|
||||
applicationId: group[0].applicationId,
|
||||
ruleId: rule.id,
|
||||
ruleCode: rule.code,
|
||||
ruleName: rule.name,
|
||||
phoneNumber: state.phoneNumber,
|
||||
thresholdValue: Math.floor(rule.thresholdValue),
|
||||
actualValue: state.count,
|
||||
windowStartedAt: state.windowStartedAt,
|
||||
windowEndsAt: state.windowEndsAt,
|
||||
generation: state.generation,
|
||||
action: 'block',
|
||||
sourceType: bucket.items[0]?.sourceType,
|
||||
};
|
||||
}),
|
||||
});
|
||||
await this.attachActiveHits(tx, hitByStateId);
|
||||
}
|
||||
for (const state of states) {
|
||||
if (state.activeHitId === null && state.count <= rule.thresholdValue) continue;
|
||||
const reason = `${rule.name}命中:本周期最多${Math.floor(rule.thresholdValue)}条,当前第${state.count}条,周期${formatWindow(state.windowStartedAt, state.windowEndsAt)}`;
|
||||
const previous = rejectedByPhone.get(state.phoneNumber);
|
||||
rejectedByPhone.set(state.phoneNumber, {
|
||||
code: 'PHONE_FREQUENCY_LIMIT',
|
||||
reason: previous ? `${previous.reason};${reason}` : reason,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
await tx.phoneFrequencyReservation.createMany({
|
||||
data: missing.map((item) => {
|
||||
const rejection = rejectedByPhone.get(item.phoneNumber);
|
||||
const result = rejection ? [{ phoneNumber: item.phoneNumber, ...rejection }] : [];
|
||||
output.set(item.reservationKey, frequencyRejectionsFromJson(result));
|
||||
return {
|
||||
reservationKey: item.reservationKey,
|
||||
tenantId: item.tenantId,
|
||||
applicationId: item.applicationId,
|
||||
result,
|
||||
};
|
||||
}),
|
||||
});
|
||||
return output;
|
||||
}, { isolationLevel: Prisma.TransactionIsolationLevel.ReadCommitted });
|
||||
for (const [key, value] of groupResults) results.set(key, value);
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
async listHits(query: PhoneFrequencyHitQuery) {
|
||||
const page = Math.max(1, Math.floor(Number(query.page) || 1));
|
||||
const pageSize = Math.min(100, Math.max(1, Math.floor(Number(query.pageSize) || 20)));
|
||||
@@ -566,6 +742,18 @@ export class PhoneFrequencyService {
|
||||
}
|
||||
}
|
||||
|
||||
function frequencyRejectionsFromJson(value: Prisma.JsonValue) {
|
||||
const result = new Map<string, PhoneFrequencyRejection>();
|
||||
if (!Array.isArray(value)) return result;
|
||||
for (const item of value) {
|
||||
if (!item || typeof item !== 'object' || Array.isArray(item)) continue;
|
||||
const phoneNumber = typeof item.phoneNumber === 'string' ? item.phoneNumber : '';
|
||||
const reason = typeof item.reason === 'string' ? item.reason : '';
|
||||
if (phoneNumber && reason) result.set(phoneNumber, { code: 'PHONE_FREQUENCY_LIMIT', reason });
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
const whitelistUserInclude = {
|
||||
createdBy: { select: { id: true, username: true, displayName: true } },
|
||||
updatedBy: { select: { id: true, username: true, displayName: true } },
|
||||
|
||||
@@ -3,6 +3,7 @@ import { RiskReviewService } from './risk-review.service';
|
||||
function createPrismaMock(overrides: Record<string, unknown> = {}) {
|
||||
return {
|
||||
riskRule: {
|
||||
count: jest.fn().mockResolvedValue(5),
|
||||
findFirst: jest.fn().mockResolvedValue({ id: 'default-rule' }),
|
||||
findUnique: jest.fn(),
|
||||
create: jest.fn(),
|
||||
@@ -26,6 +27,7 @@ function createPrismaMock(overrides: Record<string, unknown> = {}) {
|
||||
},
|
||||
smsTemplate: {
|
||||
findUnique: jest.fn().mockResolvedValue(null),
|
||||
findMany: jest.fn().mockResolvedValue([]),
|
||||
},
|
||||
smsSendTask: {
|
||||
create: jest.fn().mockImplementation(({ data }: { data: Record<string, unknown> }) =>
|
||||
@@ -61,6 +63,68 @@ function createPrismaMock(overrides: Record<string, unknown> = {}) {
|
||||
}
|
||||
|
||||
describe('RiskReviewService', () => {
|
||||
it('shares read-only rule inputs across an approved CMPP evaluation batch', async () => {
|
||||
const prisma = createPrismaMock();
|
||||
const service = new RiskReviewService(prisma as never);
|
||||
|
||||
const results = await service.evaluateTasksBatch([
|
||||
{ tenantId: 'tenant-1', applicationId: 'app-1', content: '【测试】验证码000001', phones: ['13800000001'], sourceType: 'cmpp' },
|
||||
{ tenantId: 'tenant-1', applicationId: 'app-1', content: '【测试】验证码000002', phones: ['13800000002'], sourceType: 'cmpp' },
|
||||
]);
|
||||
|
||||
expect(results).toHaveLength(2);
|
||||
expect(results.every((result) => result.status === 'approved')).toBe(true);
|
||||
expect(prisma.riskRule.findMany).toHaveBeenCalledTimes(1);
|
||||
expect(prisma.sensitiveWord.findMany).toHaveBeenCalledTimes(1);
|
||||
expect(prisma.smsBatchTask.count).not.toHaveBeenCalled();
|
||||
expect(prisma.smsSendTask.create).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('coalesces concurrent default-rule checks and reuses the short completeness cache', async () => {
|
||||
const prisma = createPrismaMock();
|
||||
let releaseCount: ((count: number) => void) | undefined;
|
||||
prisma.riskRule.count.mockReturnValue(new Promise((resolve) => { releaseCount = resolve; }));
|
||||
const service = new RiskReviewService(prisma as never);
|
||||
|
||||
const first = service.ensureDefaultRules();
|
||||
const second = service.ensureDefaultRules();
|
||||
releaseCount?.(5);
|
||||
await Promise.all([first, second]);
|
||||
await service.ensureDefaultRules();
|
||||
|
||||
expect(prisma.riskRule.count).toHaveBeenCalledTimes(1);
|
||||
expect(prisma.riskRule.findFirst).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('clears a failed default-rule check so the next request can retry', async () => {
|
||||
const prisma = createPrismaMock();
|
||||
prisma.riskRule.count
|
||||
.mockRejectedValueOnce(new Error('database unavailable'))
|
||||
.mockResolvedValueOnce(5);
|
||||
const service = new RiskReviewService(prisma as never);
|
||||
|
||||
await expect(service.ensureDefaultRules()).rejects.toThrow('database unavailable');
|
||||
await expect(service.ensureDefaultRules()).resolves.toBeUndefined();
|
||||
|
||||
expect(prisma.riskRule.count).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it('falls back to per-rule recovery when the completeness count finds a missing default', async () => {
|
||||
const prisma = createPrismaMock();
|
||||
prisma.riskRule.count.mockResolvedValue(4);
|
||||
prisma.riskRule.findFirst.mockImplementation(({ where }: { where: { code: string } }) => (
|
||||
Promise.resolve(where.code === 'PHONE_FREQUENCY_5M' ? null : { id: `rule-${where.code}` })
|
||||
));
|
||||
const service = new RiskReviewService(prisma as never);
|
||||
service.createRule = jest.fn().mockResolvedValue({ id: 'restored-rule' }) as never;
|
||||
|
||||
await service.ensureDefaultRules();
|
||||
|
||||
expect(prisma.riskRule.findFirst).toHaveBeenCalledTimes(5);
|
||||
expect(service.createRule).toHaveBeenCalledTimes(1);
|
||||
expect(service.createRule).toHaveBeenCalledWith(expect.objectContaining({ code: 'PHONE_FREQUENCY_5M' }));
|
||||
});
|
||||
|
||||
it('keeps phone-frequency periods fixed and rejects manual-review actions', () => {
|
||||
const service = new RiskReviewService(createPrismaMock() as never);
|
||||
|
||||
|
||||
@@ -114,6 +114,8 @@ const RULE_DEFINITIONS = new Map(DEFAULT_RULES.map((rule) => [rule.code, rule]))
|
||||
|
||||
@Injectable()
|
||||
export class RiskReviewService {
|
||||
private defaultRulesCheck?: { expiresAt: number; promise: Promise<void> };
|
||||
|
||||
constructor(private readonly prisma: PrismaService) {}
|
||||
|
||||
async listRules(applicationId?: string) {
|
||||
@@ -431,6 +433,81 @@ export class RiskReviewService {
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Evaluate an Inbox claim batch against one database snapshot of the read-only
|
||||
* rule inputs. Approved items do not need their own template/rule/sensitive-word
|
||||
* queries; exceptional decisions still go through evaluateTask so their audit
|
||||
* task and hit rows keep the existing semantics.
|
||||
*/
|
||||
async evaluateTasksBatch(items: EvaluateSmsTaskDto[]) {
|
||||
if (items.length === 0) return [];
|
||||
await this.ensureDefaultRules();
|
||||
if (items.some((item) => item.createdById)) {
|
||||
// The CMPP worker never supplies createdById. Keep the general API honest
|
||||
// instead of silently weakening its foreign-key validation in the fast path.
|
||||
return Promise.all(items.map((item) => this.evaluateTask(item)));
|
||||
}
|
||||
const applicationIds = [...new Set(items.map((item) => item.applicationId).filter((id): id is string => Boolean(id)))];
|
||||
const templateIds = [...new Set(items.map((item) => item.templateId).filter((id): id is string => Boolean(id)))];
|
||||
const [templates, sensitiveWords, applicationInputs] = await Promise.all([
|
||||
templateIds.length
|
||||
? this.prisma.smsTemplate.findMany({ where: { id: { in: templateIds } }, include: { variables: true } })
|
||||
: Promise.resolve([]),
|
||||
this.prisma.sensitiveWord.findMany({ where: { status: 'active' }, select: { word: true, level: true } }),
|
||||
Promise.all(applicationIds.map(async (applicationId) => ({
|
||||
applicationId,
|
||||
rules: await this.effectiveRules(applicationId),
|
||||
recentTaskCount: await this.countRecentClientTasks(applicationId, 'cmpp'),
|
||||
}))),
|
||||
]);
|
||||
const templateById = new Map(templates.map((template) => [template.id, template]));
|
||||
const inputsByApplication = new Map(applicationInputs.map((entry) => [entry.applicationId, entry]));
|
||||
|
||||
return Promise.all(items.map(async (data) => {
|
||||
const phones = data.phones ?? [];
|
||||
const uniquePhones = [...new Set(phones)];
|
||||
const phoneTotal = phones.length;
|
||||
const template = data.templateId ? templateById.get(data.templateId) : undefined;
|
||||
const variableIssues = evaluateTemplateVariables(template?.variables ?? [], data.content, data.variables ?? {});
|
||||
const contentIssues = evaluateContent(data.content, sensitiveWords);
|
||||
if (variableIssues.length > 0) {
|
||||
contentIssues.push({
|
||||
ruleCode: 'TEMPLATE_VARIABLE_INVALID',
|
||||
ruleName: '模板变量校验失败',
|
||||
thresholdValue: 0,
|
||||
actualValue: variableIssues.length,
|
||||
action: 'block',
|
||||
reason: formatTemplateVariableIssueReason(variableIssues),
|
||||
});
|
||||
}
|
||||
const applicationInput = data.applicationId ? inputsByApplication.get(data.applicationId) : undefined;
|
||||
const rules = applicationInput?.rules ?? [];
|
||||
const requestedAt = data.requestedAt ? new Date(data.requestedAt) : new Date();
|
||||
const nonWorkingRule = rules.find((rule) => rule.code === 'NON_WORKING_MARKETING_BULK');
|
||||
const nonWorkingMarketingPhones = isMarketing(data.category ?? template?.category)
|
||||
&& isNonWorkingTime(requestedAt, readNonWorkingConfig(nonWorkingRule?.config))
|
||||
? phoneTotal
|
||||
: 0;
|
||||
const hits = this.evaluateRules(rules, {
|
||||
phoneTotal,
|
||||
nonWorkingMarketingPhones,
|
||||
recentTaskCount: applicationInput?.recentTaskCount ?? 0,
|
||||
});
|
||||
hits.push(...contentIssues.map(contentIssueToHit));
|
||||
const decision = decideRiskAction(hits);
|
||||
if (decision.status !== 'approved') {
|
||||
return this.evaluateTask(data);
|
||||
}
|
||||
return {
|
||||
canSubmit: true,
|
||||
status: decision.status,
|
||||
riskDecision: decision.riskDecision,
|
||||
reason: hits.length > 0 ? hits.map((hit) => hit.reason).join('; ') : null,
|
||||
task: null,
|
||||
};
|
||||
}));
|
||||
}
|
||||
|
||||
async approveTask(taskId: string, data: ReviewSmsTaskDto) {
|
||||
const task = await this.prisma.smsSendTask.findUnique({ where: { id: taskId } });
|
||||
if (!task) {
|
||||
@@ -491,14 +568,39 @@ export class RiskReviewService {
|
||||
}
|
||||
|
||||
async ensureDefaultRules() {
|
||||
const now = Date.now();
|
||||
if (this.defaultRulesCheck && this.defaultRulesCheck.expiresAt > now) {
|
||||
return this.defaultRulesCheck.promise;
|
||||
}
|
||||
|
||||
// Submit高并发时,任务风控和号码频控都会确认默认规则。短TTL只缓存“规则是否齐全”,
|
||||
// 实际生效规则仍逐次查询;并发单飞避免每条短信重复执行5次存在性SQL,同时允许删除后自动恢复。
|
||||
const promise = this.ensureDefaultRulesFromDatabase();
|
||||
this.defaultRulesCheck = { expiresAt: now + 30_000, promise };
|
||||
try {
|
||||
await promise;
|
||||
} catch (error) {
|
||||
if (this.defaultRulesCheck?.promise === promise) this.defaultRulesCheck = undefined;
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
private async ensureDefaultRulesFromDatabase() {
|
||||
const existingCount = await this.prisma.riskRule.count({
|
||||
where: {
|
||||
applicationId: null,
|
||||
code: { in: DEFAULT_RULES.map((rule) => rule.code) },
|
||||
status: { not: 'deleted' },
|
||||
},
|
||||
});
|
||||
if (existingCount === DEFAULT_RULES.length) return;
|
||||
|
||||
for (const rule of DEFAULT_RULES) {
|
||||
const exists = await this.prisma.riskRule.findFirst({
|
||||
where: { applicationId: null, code: rule.code, status: { not: 'deleted' } },
|
||||
select: { id: true },
|
||||
});
|
||||
if (!exists) {
|
||||
await this.createRule(rule);
|
||||
}
|
||||
if (!exists) await this.createRule(rule);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -11,7 +11,17 @@ describe('GatewayEventsController protocol logging', () => {
|
||||
const protocolLogs = {
|
||||
record: jest.fn(),
|
||||
};
|
||||
const controller = new GatewayEventsController(sendChain as never, {} as never, protocolLogs as never, { recordEvent: jest.fn() } as never);
|
||||
const metrics = {
|
||||
beginCmppInboundStage: jest.fn().mockReturnValue(1n),
|
||||
finishCmppInboundStage: jest.fn(),
|
||||
};
|
||||
const controller = new GatewayEventsController(
|
||||
sendChain as never,
|
||||
{} as never,
|
||||
protocolLogs as never,
|
||||
{ recordEvent: jest.fn() } as never,
|
||||
metrics as never,
|
||||
);
|
||||
|
||||
beforeEach(() => {
|
||||
jest.clearAllMocks();
|
||||
@@ -77,6 +87,20 @@ describe('GatewayEventsController protocol logging', () => {
|
||||
}));
|
||||
});
|
||||
|
||||
it('records a failed inbound total without swallowing the service error', async () => {
|
||||
const failure = new Error('inbound failed');
|
||||
(sendChain as Record<string, jest.Mock>).submitInboundMessage = jest.fn().mockRejectedValue(failure);
|
||||
|
||||
await expect(controller.submitInbound({
|
||||
account: '607532',
|
||||
phoneNumber: '13127620092',
|
||||
content: 'failed',
|
||||
sequenceId: 142,
|
||||
})).rejects.toBe(failure);
|
||||
|
||||
expect(metrics.finishCmppInboundStage).toHaveBeenCalledWith(1n, 'total', 'error');
|
||||
});
|
||||
|
||||
it('accepts only safe outbound Gateway packet events', () => {
|
||||
expect(controller.protocolLog({
|
||||
protocol: 'cmpp',
|
||||
@@ -155,6 +179,7 @@ describe('GatewayEventsController protocol logging', () => {
|
||||
messageId: 'MSG-1',
|
||||
status: 'success',
|
||||
}));
|
||||
expect(metrics.finishCmppInboundStage).toHaveBeenCalledWith(1n, 'total', 'success');
|
||||
});
|
||||
|
||||
it('replaces a fallback receipt identifier with the resolved main message identifier', async () => {
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { BadRequestException, Body, Controller, Post } from '@nestjs/common';
|
||||
import { BadRequestException, Body, Controller, Optional, Post } from '@nestjs/common';
|
||||
import { ApiTags } from '@nestjs/swagger';
|
||||
import {
|
||||
GatewayInboundAuthDto,
|
||||
@@ -19,6 +19,7 @@ import { GatewayDownstreamConnectionEventDto } from '../sms-config/sms-config.co
|
||||
import { SmsConfigService } from '../sms-config/sms-config.service';
|
||||
import { ProtocolLogsService, type ProtocolLogInput } from '../protocol-logs/protocol-logs.service';
|
||||
import { SecurityDetectionService } from '../security-detection/security-detection.service';
|
||||
import { MetricsService } from '../metrics/metrics.service';
|
||||
|
||||
@ApiTags('gateway-events')
|
||||
@Controller('gateway/events')
|
||||
@@ -28,6 +29,7 @@ export class GatewayEventsController {
|
||||
private readonly smsConfig: SmsConfigService,
|
||||
private readonly protocolLogs: ProtocolLogsService,
|
||||
private readonly security: SecurityDetectionService,
|
||||
@Optional() private readonly metrics?: MetricsService,
|
||||
) {}
|
||||
|
||||
@Post('submit-result')
|
||||
@@ -139,6 +141,9 @@ export class GatewayEventsController {
|
||||
direction: ProtocolLogInput['direction'] = 'channel_to_platform',
|
||||
) {
|
||||
const startedAt = Date.now();
|
||||
const inboundMetricStartedAt = eventType === 'submit' && direction === 'client_to_platform'
|
||||
? this.metrics?.beginCmppInboundStage()
|
||||
: undefined;
|
||||
const value = body as Record<string, unknown>;
|
||||
const common: Omit<ProtocolLogInput, 'status'> = {
|
||||
protocol: 'cmpp',
|
||||
@@ -173,6 +178,9 @@ export class GatewayEventsController {
|
||||
status: 'success',
|
||||
durationMs: Date.now() - startedAt,
|
||||
});
|
||||
if (inboundMetricStartedAt != null) {
|
||||
this.metrics?.finishCmppInboundStage(inboundMetricStartedAt, 'total', 'success');
|
||||
}
|
||||
return result;
|
||||
} catch (error) {
|
||||
this.protocolLogs.record({
|
||||
@@ -181,6 +189,9 @@ export class GatewayEventsController {
|
||||
durationMs: Date.now() - startedAt,
|
||||
detail: { error: error instanceof Error ? error.message : 'unknown error' },
|
||||
});
|
||||
if (inboundMetricStartedAt != null) {
|
||||
this.metrics?.finishCmppInboundStage(inboundMetricStartedAt, 'total', 'error');
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -44,24 +44,13 @@ export class SendAccountingService {
|
||||
if (exists?.billingStatus === 'charged') {
|
||||
return;
|
||||
}
|
||||
if (amountCents > 0) {
|
||||
await this.billing.release({
|
||||
tenantId: message.tenantId,
|
||||
amountCents,
|
||||
idempotencyKey: `sms-charge-release:${message.messageId}`,
|
||||
relatedType: 'sms_batch_task',
|
||||
relatedId: message.batchTaskId,
|
||||
remark: `短信 ${message.messageId} 提交成功释放冻结并转扣费`,
|
||||
});
|
||||
}
|
||||
const transaction = await this.billing.charge({
|
||||
const transaction = amountCents > 0 ? await this.billing.settleFrozenCharge({
|
||||
tenantId: message.tenantId,
|
||||
amountCents,
|
||||
idempotencyKey: `sms-charge:${message.messageId}`,
|
||||
relatedType: 'sms_message_record',
|
||||
relatedId: message.messageId,
|
||||
remark: '提交成功扣费',
|
||||
});
|
||||
taskId: message.batchTaskId,
|
||||
messageId: message.messageId,
|
||||
remark: `短信 ${message.messageId} 提交成功释放冻结并转扣费`,
|
||||
}) : null;
|
||||
const data = {
|
||||
tenantId: message.tenantId,
|
||||
applicationId: message.applicationId ?? undefined,
|
||||
@@ -73,7 +62,7 @@ export class SendAccountingService {
|
||||
unitPrice,
|
||||
amountCents,
|
||||
billingStatus: 'charged',
|
||||
transactionId: transaction.id,
|
||||
transactionId: transaction?.id,
|
||||
};
|
||||
if (exists) {
|
||||
await this.prisma.smsBillingRecord.update({ where: { id: exists.id }, data });
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { BadRequestException, HttpException, HttpStatus, Logger, NotFoundException } from '@nestjs/common';
|
||||
import { BadRequestException, ConflictException, HttpException, HttpStatus, Logger, NotFoundException } from '@nestjs/common';
|
||||
import { Prisma } from '@prisma/client';
|
||||
import { Queue, Worker } from 'bullmq';
|
||||
import IORedis from 'ioredis';
|
||||
@@ -525,15 +525,17 @@ async reserveDailySendQuota(applicationId: string, requestedCount: number) {
|
||||
return result;
|
||||
}
|
||||
|
||||
async tryReserveDailySendQuota(applicationId: string, requestedCount: number) {
|
||||
async tryReserveDailySendQuota(applicationId: string, requestedCount: number, reservationKey?: string) {
|
||||
if (!Number.isInteger(requestedCount) || requestedCount <= 0) {
|
||||
throw new BadRequestException('发送号码数量必须为正整数');
|
||||
}
|
||||
const usageDate = shanghaiDateKey();
|
||||
const reservationId = randomUUID();
|
||||
const rows = await this.prisma.$queryRaw<Array<{ dailyLimit: number; usedCount: number | null }>>(Prisma.sql`
|
||||
const usageDateValue = new Date(`${usageDate}T00:00:00.000Z`);
|
||||
const reserve = (client: Pick<Prisma.TransactionClient, '$queryRaw'>) => {
|
||||
const reservationId = randomUUID();
|
||||
return client.$queryRaw<Array<{ tenantId: string; dailyLimit: number; usedCount: number | null }>>(Prisma.sql`
|
||||
WITH application_limit AS (
|
||||
SELECT id, COALESCE("dailyLimit", 100000)::integer AS "dailyLimit"
|
||||
SELECT id, "tenantId", COALESCE("dailyLimit", 100000)::integer AS "dailyLimit"
|
||||
FROM "SmsApplication"
|
||||
WHERE id = ${applicationId}
|
||||
), reservation AS (
|
||||
@@ -550,10 +552,49 @@ async tryReserveDailySendQuota(applicationId: string, requestedCount: number) {
|
||||
<= (SELECT "dailyLimit" FROM application_limit)
|
||||
RETURNING "usedCount"
|
||||
)
|
||||
SELECT application_limit."dailyLimit", reservation."usedCount"
|
||||
SELECT application_limit."tenantId", application_limit."dailyLimit", reservation."usedCount"
|
||||
FROM application_limit
|
||||
LEFT JOIN reservation ON TRUE
|
||||
`);
|
||||
`);
|
||||
};
|
||||
const normalizedReservationKey = reservationKey?.trim();
|
||||
const rows = normalizedReservationKey
|
||||
? await this.prisma.$transaction(async (tx) => {
|
||||
// The quota increment and its idempotency record share one short transaction. A worker
|
||||
// crash can therefore neither lose a successful reservation nor increment it twice.
|
||||
await tx.$executeRaw`SELECT pg_advisory_xact_lock(hashtextextended(${'daily-quota:' + normalizedReservationKey}, 0))`;
|
||||
const existing = await tx.smsApplicationDailyReservation.findUnique({
|
||||
where: { reservationKey: normalizedReservationKey },
|
||||
});
|
||||
if (existing) {
|
||||
if (existing.applicationId !== applicationId || existing.requestedCount !== requestedCount) {
|
||||
throw new ConflictException('日发送配额幂等键已用于另一笔预留');
|
||||
}
|
||||
return [{
|
||||
tenantId: existing.tenantId,
|
||||
dailyLimit: existing.dailyLimit,
|
||||
usedCount: existing.usedCount,
|
||||
}];
|
||||
}
|
||||
const reservedRows = await reserve(tx);
|
||||
if (reservedRows.length > 0) {
|
||||
const row = reservedRows[0];
|
||||
await tx.smsApplicationDailyReservation.create({
|
||||
data: {
|
||||
reservationKey: normalizedReservationKey,
|
||||
tenantId: row.tenantId,
|
||||
applicationId,
|
||||
usageDate: usageDateValue,
|
||||
requestedCount,
|
||||
dailyLimit: Number(row.dailyLimit),
|
||||
usedCount: row.usedCount == null ? null : Number(row.usedCount),
|
||||
reserved: row.usedCount != null,
|
||||
},
|
||||
});
|
||||
}
|
||||
return reservedRows;
|
||||
})
|
||||
: await reserve(this.prisma);
|
||||
if (rows.length === 0) {
|
||||
throw new NotFoundException('短信应用不存在');
|
||||
}
|
||||
|
||||
@@ -30,6 +30,7 @@ export interface GatewayInboundAuthDto {
|
||||
}
|
||||
|
||||
export interface GatewayInboundSubmitDto {
|
||||
requestId?: string;
|
||||
account: string;
|
||||
phoneNumber?: string;
|
||||
phoneNumbers?: string[];
|
||||
@@ -58,6 +59,7 @@ export interface GatewayInboundSingleSubmitResult {
|
||||
}
|
||||
|
||||
export interface GatewaySubmitResultDto {
|
||||
eventId?: string;
|
||||
traceId?: string;
|
||||
messageId: string;
|
||||
channelId: string;
|
||||
@@ -81,6 +83,7 @@ export interface GatewaySubmitResultDto {
|
||||
}
|
||||
|
||||
export interface GatewaySubmitSegmentResultDto {
|
||||
eventId?: string;
|
||||
traceId?: string;
|
||||
messageId: string;
|
||||
channelId: string;
|
||||
|
||||
@@ -49,6 +49,20 @@ describe('send-chain pure policies', () => {
|
||||
})).toBeUndefined();
|
||||
});
|
||||
|
||||
it('uses a stable weighted choice among equal-priority online primary channels', () => {
|
||||
const items = [
|
||||
{ channelId: 'primary-a', carrier: 'mobile', province: null, priority: 1, weight: 1, isBackup: false, channel: { carrier: 'mobile', sendRegion: '全国', status: 'active', connectionStates: connected } },
|
||||
{ channelId: 'primary-b', carrier: 'mobile', province: null, priority: 1, weight: 1, isBackup: false, channel: { carrier: 'mobile', sendRegion: '全国', status: 'active', connectionStates: connected } },
|
||||
{ channelId: 'backup', carrier: 'mobile', province: null, priority: 2, weight: 100, isBackup: true, channel: { carrier: 'mobile', sendRegion: '全国', status: 'active', connectionStates: connected } },
|
||||
];
|
||||
const selected = new Set(Array.from({ length: 100 }, (_, index) => selectChannelCandidate(items, {
|
||||
carrier: 'mobile', routingKey: `message-${index}`,
|
||||
excludedChannelIds: new Set(), approvedChannelIds: new Set(items.map((item) => item.channelId)),
|
||||
})?.channelId));
|
||||
|
||||
expect(selected).toEqual(new Set(['primary-a', 'primary-b']));
|
||||
});
|
||||
|
||||
it('keeps a segmented message non-terminal until all receipts arrive', () => {
|
||||
const result = aggregateReceiptSegmentState(
|
||||
[{ segmentTotal: 2, receiptStatus: 'delivered', deliveredAt: new Date('2026-07-31T00:00:00Z') }],
|
||||
|
||||
@@ -447,6 +447,9 @@ export type ChannelCandidate = {
|
||||
channelId: string;
|
||||
carrier?: string | null;
|
||||
province?: string | null;
|
||||
priority?: number | null;
|
||||
weight?: number | null;
|
||||
isBackup?: boolean | null;
|
||||
channel: {
|
||||
carrier?: string | null;
|
||||
carriers?: string[] | null;
|
||||
@@ -477,6 +480,7 @@ export function selectChannelCandidate<T extends ChannelCandidate>(
|
||||
forceNational?: boolean;
|
||||
excludedChannelIds: ReadonlySet<string>;
|
||||
approvedChannelIds: ReadonlySet<string>;
|
||||
routingKey?: string;
|
||||
},
|
||||
) {
|
||||
const eligible = items.filter((item) =>
|
||||
@@ -489,7 +493,28 @@ export function selectChannelCandidate<T extends ChannelCandidate>(
|
||||
? []
|
||||
: eligible.filter((item) => isProvinceChannel(item, options.province));
|
||||
const nationalCandidates = eligible.filter((item) => isNationalChannel(item));
|
||||
return [...provinceCandidates, ...nationalCandidates].find((item) => isChannelSendAvailable(item.channel));
|
||||
const scope = provinceCandidates.some((item) => isChannelSendAvailable(item.channel))
|
||||
? provinceCandidates
|
||||
: nationalCandidates;
|
||||
const available = scope.filter((item) => isChannelSendAvailable(item.channel));
|
||||
if (available.length === 0) return undefined;
|
||||
const priority = Math.min(...available.map((item) => item.priority ?? 100));
|
||||
const priorityPool = available.filter((item) => (item.priority ?? 100) === priority);
|
||||
const primaryPool = priorityPool.filter((item) => !item.isBackup);
|
||||
const pool = primaryPool.length > 0 ? primaryPool : priorityPool;
|
||||
if (pool.length === 1 || !options.routingKey) return pool[0];
|
||||
const totalWeight = pool.reduce((sum, item) => sum + Math.max(1, item.weight ?? 1), 0);
|
||||
let hash = 2166136261;
|
||||
for (const character of options.routingKey) {
|
||||
hash ^= character.charCodeAt(0);
|
||||
hash = Math.imul(hash, 16777619) >>> 0;
|
||||
}
|
||||
let slot = hash % totalWeight;
|
||||
for (const item of pool) {
|
||||
slot -= Math.max(1, item.weight ?? 1);
|
||||
if (slot < 0) return item;
|
||||
}
|
||||
return pool[0];
|
||||
}
|
||||
|
||||
export type ReceiptSegmentAudit = {
|
||||
|
||||
@@ -81,6 +81,7 @@ function createPrismaMock() {
|
||||
status: 'active',
|
||||
interfaceEnabled: true,
|
||||
cmppMaxConnections: 2,
|
||||
cmppWindowSize: 32,
|
||||
queuePriority: 'normal',
|
||||
ipAllowlist: [{ ipCidr: '127.0.0.1/32' }],
|
||||
tenant: { id: 'tenant-1', status: 'active', certificationStatus: 'approved' },
|
||||
@@ -328,6 +329,21 @@ function createPrismaMock() {
|
||||
lastError: 'downstream client is not connected',
|
||||
}),
|
||||
},
|
||||
cmppInboundSubmissionInbox: {
|
||||
create: jest.fn().mockResolvedValue({ id: 'inbox-1' }),
|
||||
findUnique: jest.fn().mockResolvedValue(null),
|
||||
updateMany: jest.fn().mockResolvedValue({ count: 1 }),
|
||||
count: jest.fn().mockResolvedValue(0),
|
||||
findFirst: jest.fn().mockResolvedValue(null),
|
||||
},
|
||||
smsApplicationDailyReservation: {
|
||||
findUnique: jest.fn().mockResolvedValue(null),
|
||||
create: jest.fn().mockResolvedValue({ id: 'daily-reservation-1' }),
|
||||
},
|
||||
phoneFrequencyReservation: {
|
||||
findUnique: jest.fn().mockResolvedValue(null),
|
||||
create: jest.fn().mockResolvedValue({ id: 'frequency-reservation-1' }),
|
||||
},
|
||||
smsBillingRecord: {
|
||||
findFirst: jest.fn().mockResolvedValue(null),
|
||||
create: jest.fn().mockResolvedValue({ id: 'bill-1' }),
|
||||
@@ -371,6 +387,7 @@ function createService(
|
||||
freeze: jest.fn().mockResolvedValue({ id: 'tx-freeze' }),
|
||||
release: jest.fn().mockResolvedValue({ id: 'tx-release' }),
|
||||
charge: jest.fn().mockResolvedValue({ id: 'tx-charge' }),
|
||||
settleFrozenCharge: jest.fn().mockResolvedValue({ id: 'tx-charge' }),
|
||||
refund: jest.fn().mockResolvedValue({ id: 'tx-refund' }),
|
||||
} as unknown as BillingService;
|
||||
const riskReview = {
|
||||
@@ -396,6 +413,7 @@ function createService(
|
||||
);
|
||||
service['postGatewayControl'] = jest.fn().mockResolvedValue({ delivered: true });
|
||||
service['publishGatewaySubmitCommand'] = jest.fn().mockResolvedValue(undefined);
|
||||
service['getSendQueue'] = jest.fn().mockReturnValue({ add: jest.fn().mockResolvedValue(undefined) });
|
||||
return { service, prisma, billing, riskReview, phoneFrequency };
|
||||
}
|
||||
|
||||
@@ -906,6 +924,28 @@ describe('SendChainService', () => {
|
||||
}));
|
||||
});
|
||||
|
||||
it('coalesces concurrent batch progress refreshes and keeps a trailing refresh', async () => {
|
||||
const { service, prisma } = createService();
|
||||
let resolveFirst: ((value: Array<{ status: string; _count: { _all: number } }>) => void) | undefined;
|
||||
prisma.smsMessageRecord.groupBy
|
||||
.mockImplementationOnce(() => new Promise((resolve) => { resolveFirst = resolve; }))
|
||||
.mockResolvedValue([{ status: 'delivered', _count: { _all: 2 } }]);
|
||||
|
||||
const first = service['submission'].refreshTaskProgress('task-1');
|
||||
await Promise.resolve();
|
||||
const second = service['submission'].refreshTaskProgress('task-1');
|
||||
|
||||
expect(prisma.smsMessageRecord.groupBy).toHaveBeenCalledTimes(1);
|
||||
resolveFirst?.([{ status: 'delivered', _count: { _all: 1 } }]);
|
||||
await Promise.all([first, second]);
|
||||
|
||||
expect(prisma.smsMessageRecord.groupBy).toHaveBeenCalledTimes(2);
|
||||
expect(prisma.smsBatchTask.update).toHaveBeenLastCalledWith({
|
||||
where: { id: 'task-1' },
|
||||
data: expect.objectContaining({ progressTotal: 2, successTotal: 2, status: 'finished' }),
|
||||
});
|
||||
});
|
||||
|
||||
it('does not expose CMPP internal tasks through client task detail or messages', async () => {
|
||||
const { service, prisma } = createService();
|
||||
prisma.smsBatchTask.findFirst.mockResolvedValue(null);
|
||||
@@ -1057,6 +1097,7 @@ describe('SendChainService', () => {
|
||||
account: '100001',
|
||||
enterpriseCode: 'SP0001',
|
||||
maxConnections: 2,
|
||||
windowSize: 32,
|
||||
status: 'authenticated',
|
||||
}));
|
||||
expect(prisma.operationLog.create).toHaveBeenCalledWith({
|
||||
@@ -1625,6 +1666,7 @@ describe('SendChainService', () => {
|
||||
})).resolves.toEqual(expect.objectContaining({ accepted: true, phoneCount: 2 }));
|
||||
|
||||
expect(prisma.smsMessageRecord.create).toHaveBeenCalledTimes(2);
|
||||
expect(prisma.smsApplication.findFirst).toHaveBeenCalledTimes(1);
|
||||
expect(prisma.smsMessageRecord.update).toHaveBeenCalledWith({
|
||||
where: { id: 'record-2' },
|
||||
data: expect.objectContaining({
|
||||
@@ -1764,7 +1806,10 @@ describe('SendChainService', () => {
|
||||
templateId: 'tpl-code',
|
||||
variables: { code: '715021' },
|
||||
}));
|
||||
expect(service.enqueueBatchTask).toHaveBeenCalledWith('task-1');
|
||||
expect(service.enqueueBatchTask).toHaveBeenCalledWith('task-1', {
|
||||
messageRecordId: 'record-1',
|
||||
queuePriority: 'normal',
|
||||
});
|
||||
expect(prisma.smsReceiptRecord.create).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
@@ -1812,7 +1857,10 @@ describe('SendChainService', () => {
|
||||
where: { id: 'record-1' },
|
||||
data: { status: 'queued', signatureId: 'sig-1' },
|
||||
});
|
||||
expect(service.enqueueBatchTask).toHaveBeenCalledWith('task-1');
|
||||
expect(service.enqueueBatchTask).toHaveBeenCalledWith('task-1', {
|
||||
messageRecordId: 'record-1',
|
||||
queuePriority: 'normal',
|
||||
});
|
||||
expect(prisma.smsReceiptRecord.create).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
@@ -2088,6 +2136,207 @@ describe('SendChainService', () => {
|
||||
});
|
||||
});
|
||||
|
||||
it('returns from the durable CMPP Inbox fast path before risk, billing, or queue publication', async () => {
|
||||
const previous = process.env.CMPP_INBOUND_FAST_PATH_ENABLED;
|
||||
process.env.CMPP_INBOUND_FAST_PATH_ENABLED = 'true';
|
||||
try {
|
||||
const { service, prisma, billing, riskReview, phoneFrequency } = createService();
|
||||
service.enqueueBatchTask = jest.fn();
|
||||
prisma.$queryRaw.mockImplementationOnce((query) => {
|
||||
const payloadHash = query.values.find((value: unknown) => typeof value === 'string' && /^[a-f0-9]{64}$/.test(value));
|
||||
return Promise.resolve([{
|
||||
validationError: null,
|
||||
payloadHash,
|
||||
response: {
|
||||
accepted: true,
|
||||
tenantId: 'tenant-1',
|
||||
applicationId: 'app-1',
|
||||
taskId: '',
|
||||
messageId: 'MSG-fast',
|
||||
messageRecordId: '',
|
||||
status: 'accepted_pending',
|
||||
phoneCount: 2,
|
||||
messages: [],
|
||||
},
|
||||
}]);
|
||||
});
|
||||
|
||||
const result = await service.submitInboundMessage({
|
||||
requestId: 'cmpp-inbound:test-fast-path',
|
||||
account: '100001',
|
||||
phoneNumbers: ['13800000001', '13900000002'],
|
||||
content: 'hello',
|
||||
sequenceId: 777,
|
||||
remoteIp: '127.0.0.1',
|
||||
});
|
||||
|
||||
expect(result).toEqual(expect.objectContaining({
|
||||
accepted: true,
|
||||
status: 'accepted_pending',
|
||||
phoneCount: 2,
|
||||
}));
|
||||
expect(prisma.smsApplication.findFirst).not.toHaveBeenCalled();
|
||||
expect(prisma.$queryRaw).toHaveBeenCalledTimes(1);
|
||||
const sql = prisma.$queryRaw.mock.calls[0][0].strings.join(' ');
|
||||
expect(sql).toContain('INSERT INTO "CmppInboundSubmissionInbox"');
|
||||
expect(sql).toContain('JOIN "Tenant"');
|
||||
expect(sql).toContain('ON CONFLICT ("requestKey") DO NOTHING');
|
||||
expect(sql).toContain("'messageId', ");
|
||||
expect(sql).toContain('::text');
|
||||
expect(sql).toContain("'phoneCount', ");
|
||||
expect(sql).toContain('::integer');
|
||||
expect(prisma.smsBatchTask.create).not.toHaveBeenCalled();
|
||||
expect(prisma.smsMessageRecord.create).not.toHaveBeenCalled();
|
||||
expect(riskReview.evaluateTask).not.toHaveBeenCalled();
|
||||
expect(phoneFrequency.reserve).not.toHaveBeenCalled();
|
||||
expect(billing.freeze).not.toHaveBeenCalled();
|
||||
expect(service.enqueueBatchTask).not.toHaveBeenCalled();
|
||||
} finally {
|
||||
if (previous == null) delete process.env.CMPP_INBOUND_FAST_PATH_ENABLED;
|
||||
else process.env.CMPP_INBOUND_FAST_PATH_ENABLED = previous;
|
||||
}
|
||||
});
|
||||
|
||||
it('returns the stored SubmitResp for an idempotent CMPP Inbox retry', async () => {
|
||||
const previous = process.env.CMPP_INBOUND_FAST_PATH_ENABLED;
|
||||
process.env.CMPP_INBOUND_FAST_PATH_ENABLED = 'true';
|
||||
try {
|
||||
const { service, prisma } = createService();
|
||||
const storedResponse = {
|
||||
accepted: true,
|
||||
tenantId: 'tenant-1',
|
||||
applicationId: 'app-1',
|
||||
taskId: '',
|
||||
messageId: 'MSG-stable',
|
||||
messageRecordId: '',
|
||||
status: 'accepted_pending',
|
||||
phoneCount: 1,
|
||||
messages: [{ phoneNumber: '13800000001', messageId: 'MSG-stable', messageRecordId: '', taskId: '', status: 'accepted_pending' }],
|
||||
};
|
||||
prisma.$queryRaw.mockImplementation((query) => {
|
||||
const payloadHash = query.values.find((value: unknown) => typeof value === 'string' && /^[a-f0-9]{64}$/.test(value));
|
||||
return Promise.resolve([{ validationError: null, payloadHash, response: storedResponse }]);
|
||||
});
|
||||
const request = {
|
||||
requestId: 'cmpp-inbound:test-retry',
|
||||
account: '100001',
|
||||
phoneNumber: '13800000001',
|
||||
content: 'hello',
|
||||
sequenceId: 778,
|
||||
remoteIp: '127.0.0.1',
|
||||
};
|
||||
|
||||
await service.submitInboundMessage(request);
|
||||
await expect(service.submitInboundMessage(request)).resolves.toEqual(storedResponse);
|
||||
expect(prisma.$queryRaw).toHaveBeenCalledTimes(2);
|
||||
expect(prisma.cmppInboundSubmissionInbox.findUnique).not.toHaveBeenCalled();
|
||||
expect(prisma.smsMessageRecord.create).not.toHaveBeenCalled();
|
||||
} finally {
|
||||
if (previous == null) delete process.env.CMPP_INBOUND_FAST_PATH_ENABLED;
|
||||
else process.env.CMPP_INBOUND_FAST_PATH_ENABLED = previous;
|
||||
}
|
||||
});
|
||||
|
||||
it('rejects a disabled application before the merged Inbox statement can insert', async () => {
|
||||
const previous = process.env.CMPP_INBOUND_FAST_PATH_ENABLED;
|
||||
process.env.CMPP_INBOUND_FAST_PATH_ENABLED = 'true';
|
||||
try {
|
||||
const { service, prisma } = createService();
|
||||
prisma.$queryRaw.mockResolvedValueOnce([{
|
||||
validationError: 'CMPP account is disabled for new submissions',
|
||||
payloadHash: null,
|
||||
response: null,
|
||||
}]);
|
||||
|
||||
await expect(service.submitInboundMessage({
|
||||
requestId: 'cmpp-inbound:disabled',
|
||||
account: '100001',
|
||||
phoneNumber: '13800000001',
|
||||
content: 'hello',
|
||||
})).rejects.toThrow('CMPP account is disabled for new submissions');
|
||||
expect(prisma.cmppInboundSubmissionInbox.findUnique).not.toHaveBeenCalled();
|
||||
} finally {
|
||||
if (previous == null) delete process.env.CMPP_INBOUND_FAST_PATH_ENABLED;
|
||||
else process.env.CMPP_INBOUND_FAST_PATH_ENABLED = previous;
|
||||
}
|
||||
});
|
||||
|
||||
it('uses a read-only recovery only for a concurrent Inbox insert outside the CTE snapshot', async () => {
|
||||
const previous = process.env.CMPP_INBOUND_FAST_PATH_ENABLED;
|
||||
process.env.CMPP_INBOUND_FAST_PATH_ENABLED = 'true';
|
||||
try {
|
||||
const { service, prisma } = createService();
|
||||
let payloadHash = '';
|
||||
prisma.$queryRaw.mockImplementationOnce((query) => {
|
||||
payloadHash = query.values.find((value: unknown) => typeof value === 'string' && /^[a-f0-9]{64}$/.test(value));
|
||||
return Promise.resolve([{ validationError: null, payloadHash: null, response: null }]);
|
||||
});
|
||||
prisma.cmppInboundSubmissionInbox.findUnique.mockImplementationOnce(() => Promise.resolve({
|
||||
payloadHash,
|
||||
response: { accepted: true, messageId: 'MSG-concurrent', status: 'accepted_pending' },
|
||||
}));
|
||||
|
||||
await expect(service.submitInboundMessage({
|
||||
requestId: 'cmpp-inbound:concurrent',
|
||||
account: '100001',
|
||||
phoneNumber: '13800000001',
|
||||
content: 'hello',
|
||||
})).resolves.toEqual(expect.objectContaining({ messageId: 'MSG-concurrent' }));
|
||||
expect(prisma.cmppInboundSubmissionInbox.findUnique).toHaveBeenCalledTimes(1);
|
||||
} finally {
|
||||
if (previous == null) delete process.env.CMPP_INBOUND_FAST_PATH_ENABLED;
|
||||
else process.env.CMPP_INBOUND_FAST_PATH_ENABLED = previous;
|
||||
}
|
||||
});
|
||||
|
||||
it('persists an idempotent daily quota reservation with a Prisma Date value', async () => {
|
||||
const { service, prisma } = createService();
|
||||
prisma.$queryRaw.mockResolvedValueOnce([{ tenantId: 'tenant-1', dailyLimit: 100000, usedCount: 1 }]);
|
||||
|
||||
await expect((service as any).tryReserveDailySendQuota('app-1', 1, 'workflow-1:daily-quota'))
|
||||
.resolves.toEqual({ dailyLimit: 100000, usedCount: 1, reserved: true });
|
||||
|
||||
expect(prisma.smsApplicationDailyReservation.create).toHaveBeenCalledWith({
|
||||
data: expect.objectContaining({
|
||||
reservationKey: 'workflow-1:daily-quota',
|
||||
usageDate: expect.any(Date),
|
||||
}),
|
||||
});
|
||||
expect(prisma.smsApplicationDailyReservation.create.mock.calls[0][0].data.usageDate.toISOString())
|
||||
.toMatch(/^\d{4}-\d{2}-\d{2}T00:00:00\.000Z$/);
|
||||
});
|
||||
|
||||
it('claims Inbox leases against UTC for timestamp-without-time-zone columns', async () => {
|
||||
const { service, prisma } = createService();
|
||||
prisma.$queryRaw.mockResolvedValueOnce([]);
|
||||
|
||||
await (service as any).submission.inboundEntry.claimInboundWorkflows(5);
|
||||
|
||||
const sql = prisma.$queryRaw.mock.calls[0][0].strings.join(' ');
|
||||
expect(sql).toContain("NOW() AT TIME ZONE 'UTC'");
|
||||
expect(sql).toContain('FOR UPDATE SKIP LOCKED');
|
||||
});
|
||||
|
||||
it('enqueues a freshly persisted inbound message without querying the task and message again', async () => {
|
||||
const { service, prisma } = createService();
|
||||
const add = jest.fn().mockResolvedValue(undefined);
|
||||
service['getSendQueue'] = jest.fn().mockReturnValue({ add });
|
||||
|
||||
await expect(service.enqueueBatchTask('task-1', {
|
||||
messageRecordId: 'record-1',
|
||||
queuePriority: 'priority',
|
||||
})).resolves.toEqual({ taskId: 'task-1', enqueued: 1 });
|
||||
|
||||
expect(prisma.smsBatchTask.findUnique).not.toHaveBeenCalled();
|
||||
expect(prisma.smsMessageRecord.findMany).not.toHaveBeenCalled();
|
||||
expect(add).toHaveBeenCalledWith('send-message', { messageRecordId: 'record-1' }, {
|
||||
jobId: 'record-1',
|
||||
attempts: 3,
|
||||
priority: 1,
|
||||
});
|
||||
expect(prisma.smsBatchTask.update).toHaveBeenCalledWith({ where: { id: 'task-1' }, data: { status: 'queued' } });
|
||||
});
|
||||
|
||||
it('reuses persisted carrier and province without querying routing dictionaries again', async () => {
|
||||
const { service, prisma } = createService();
|
||||
service['identifyCarrier'] = jest.fn();
|
||||
@@ -2147,8 +2396,7 @@ describe('SendChainService', () => {
|
||||
where: { id: 'record-1', status: { notIn: ['delivered', 'failed', 'unknown'] } },
|
||||
data: expect.objectContaining({ gatewayMessageId: 'GW-1', status: 'submitted', submitStatus: 'accepted' }),
|
||||
});
|
||||
expect(billing.release).toHaveBeenCalledWith(expect.objectContaining({ amountCents: 3, relatedId: 'task-1' }));
|
||||
expect(billing.charge).toHaveBeenCalledWith(expect.objectContaining({ amountCents: 3, relatedId: 'MSG-1' }));
|
||||
expect(billing.settleFrozenCharge).toHaveBeenCalledWith(expect.objectContaining({ amountCents: 3, taskId: 'task-1', messageId: 'MSG-1' }));
|
||||
expect(prisma.smsBillingRecord.create).toHaveBeenCalledWith({
|
||||
data: expect.objectContaining({ messageId: 'MSG-1', amountCents: 3, billingStatus: 'charged', transactionId: 'tx-charge' }),
|
||||
});
|
||||
@@ -2214,6 +2462,45 @@ describe('SendChainService', () => {
|
||||
});
|
||||
});
|
||||
|
||||
it('treats a redelivered Outbox aggregate event as idempotent', async () => {
|
||||
const { service, prisma, billing } = createService();
|
||||
const baseSubmit = {
|
||||
id: 'submit-1',
|
||||
messageRecordId: 'record-1',
|
||||
channelId: 'channel-1',
|
||||
submitId: 'SUB-1',
|
||||
submitStatus: 'accepted',
|
||||
};
|
||||
prisma.smsSubmitRecord.findUnique
|
||||
.mockResolvedValueOnce({ ...baseSubmit, resultEventId: null })
|
||||
.mockResolvedValueOnce({ ...baseSubmit, resultEventId: 'submit:SUB-1:aggregate' });
|
||||
const event = {
|
||||
eventId: 'submit:SUB-1:aggregate',
|
||||
messageId: 'MSG-1',
|
||||
channelId: 'channel-1',
|
||||
submitId: 'SUB-1',
|
||||
sequenceId: 7,
|
||||
gatewayMessageId: 'GW-1',
|
||||
submitStatus: 'accepted' as const,
|
||||
submittedAt: '2026-07-01T10:00:00.000Z',
|
||||
};
|
||||
|
||||
await service.handleSubmitResult(event);
|
||||
await service.handleSubmitResult(event);
|
||||
|
||||
expect(billing.settleFrozenCharge).toHaveBeenCalledTimes(1);
|
||||
expect(prisma.smsSubmitRecord.updateMany).toHaveBeenCalledWith({
|
||||
where: {
|
||||
id: 'submit-1',
|
||||
OR: [{ resultEventId: null }, { resultEventId: 'submit:SUB-1:aggregate' }],
|
||||
},
|
||||
data: {
|
||||
resultEventId: 'submit:SUB-1:aggregate',
|
||||
resultProcessedAt: new Date('2026-07-01T10:00:00.000Z'),
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it('rejects a legacy aggregate SubmitResult when it cannot match one submit attempt uniquely', async () => {
|
||||
const { service, prisma } = createService();
|
||||
prisma.smsSubmitRecord.findMany.mockResolvedValue([
|
||||
|
||||
@@ -12,6 +12,7 @@ import { PhoneRoutingLookupService } from '../dictionaries/phone-routing-lookup.
|
||||
import { PrismaService } from '../prisma/prisma.service';
|
||||
import { RiskReviewService } from '../risk-review/risk-review.service';
|
||||
import { PhoneFrequencyService } from '../risk-review/phone-frequency.service';
|
||||
import { MetricsService } from '../metrics/metrics.service';
|
||||
import { OpenApiService } from '../open-api/open-api.service';
|
||||
import type { CreateBatchTaskDto, CreateHttpBatchTaskDto, GatewayInboundAuthDto, GatewayInboundSubmitDto, GatewayInboundSingleSubmitResult, GatewaySubmitResultDto, GatewaySubmitSegmentResultDto, GatewayReceiptEventDto, GatewayUplinkEventDto, UplinkMatchCandidateInput, GatewayPendingDeliveryQueryDto, GatewayDownstreamSentDto, GatewayDownstreamAcknowledgedDto, GatewayDownstreamFailureType, GatewayControlDeliveryResult, GatewaySubmitDeadLetterDto, RequeueGatewaySubmitExceptionDto, GatewayDownstreamRecoveryStatusDto, TimeoutUnknownDto, ImportPreviewDto, ConfirmImportDto, SendJob, QueuePriority, RoutedChannel } from './send-chain.contracts';
|
||||
import { SEND_QUEUE, GATEWAY_SUBMIT_QUEUE, GATEWAY_SUBMIT_STREAM, DEFAULT_DOWNSTREAM_RETRY_DELAY_MS, DEFAULT_DOWNSTREAM_RETRY_MAX_DELAY_MS, DEFAULT_DOWNSTREAM_MAX_RETRIES, DEFAULT_DOWNSTREAM_PENDING_TIMEOUT_HOURS, DEFAULT_RECEIPT_TIMEOUT_HOURS, DEFAULT_RECEIPT_TIMEOUT_SCAN_INTERVAL_MS, RECEIPT_TIMEOUT_INITIAL_DELAY_MS, DEFAULT_SCHEDULED_DISPATCH_SCAN_INTERVAL_MS, DEFAULT_SCHEDULED_DISPATCH_STALE_MS, SCHEDULED_DISPATCH_INITIAL_DELAY_MS, DEFAULT_GATEWAY_SUBMIT_REQUEUE_STALE_MS, DEFAULT_DOWNSTREAM_MANUAL_REQUEUE_STALE_MS, DEFAULT_INBOUND_LONG_MESSAGE_SCAN_INTERVAL_MS, INBOUND_LONG_MESSAGE_SCAN_INITIAL_DELAY_MS, DEFAULT_INBOUND_LONG_MESSAGE_PROCESSING_STALE_SECONDS, DEFAULT_UPSTREAM_RECEIPT_INBOX_SCAN_INTERVAL_MS, UPSTREAM_RECEIPT_INBOX_INITIAL_DELAY_MS, DEFAULT_UPSTREAM_RECEIPT_INBOX_PROCESSING_STALE_MS, DEFAULT_UPSTREAM_RECEIPT_INBOX_MAX_ATTEMPTS, DEFAULT_UPSTREAM_RECEIPT_INBOX_MAX_AGE_HOURS, GATEWAY_SUBMIT_REQUEUE_IDEMPOTENCY_TTL_SECONDS, BULLMQ_PRIORITY, gatewaySubmitRequeueKey, drainageRejectionReason, statusFromRisk, parseSchedule, isObjectRecord, asDateOrNull, downstreamRetryDelayMs, downstreamAckTimeoutMs, downstreamRetryBaseDelayMs, downstreamRetryMaxDelayMs, downstreamMaxRetries, downstreamPendingTimeoutHours, downstreamControlFailureMessage, parseImportRows, splitImportLine, cellByHeader, normalizeCarrier, normalizeQueuePriority, getPositiveConfigInteger, getNonNegativeConfigInteger, isCarrierCompatible, normalizeRegion, matchTemplateContent, escapeRegularExpression, isNationalChannel, isProvinceChannel, validateInboundApplicationSrcId, composeUpstreamSrcId, positiveInteger, parseOptionalSequenceId, normalizeSubmitStatus, normalizeReceiptStatus, downstreamDeliveryAttemptKey, shanghaiDateKey, bullmqConnection, matchesApplicationSecret, octetString, hasRecoveryAuditStateChanged, normalizeRecoveryFailureCategory } from './send-chain.helpers';
|
||||
@@ -50,6 +51,7 @@ export class SendChainService implements OnModuleInit, OnModuleDestroy {
|
||||
private readonly phoneFrequency: PhoneFrequencyService,
|
||||
@Optional() @Inject(forwardRef(() => OpenApiService)) private readonly openApi?: OpenApiService,
|
||||
@Optional() phoneRouting?: PhoneRoutingLookupService,
|
||||
@Optional() metrics?: MetricsService,
|
||||
) {
|
||||
const resolvedPhoneRouting = phoneRouting ?? new PhoneRoutingLookupService(prisma);
|
||||
this.submission = new SendSubmissionService(
|
||||
@@ -64,6 +66,7 @@ export class SendChainService implements OnModuleInit, OnModuleDestroy {
|
||||
recordCmppFailureReceipt: (message, errorCode, reason) =>
|
||||
this.recordCmppFailureReceipt(message, errorCode, reason),
|
||||
},
|
||||
metrics,
|
||||
);
|
||||
this.completion = new SendCompletionService(
|
||||
prisma,
|
||||
@@ -75,9 +78,14 @@ export class SendChainService implements OnModuleInit, OnModuleDestroy {
|
||||
}
|
||||
|
||||
onModuleInit() {
|
||||
const processRole = process.env.CMPP_PROCESS_ROLE?.trim() || 'all';
|
||||
if (processRole === 'api') return;
|
||||
if (process.env.API_ENABLE_SEND_WORKER === 'true') {
|
||||
this.startWorker();
|
||||
}
|
||||
if (process.env.CMPP_INBOUND_WORKFLOW_WORKER_ENABLED === 'true') {
|
||||
this.submission.startInboundWorkflowWorker();
|
||||
}
|
||||
if (process.env.SMS_RECEIPT_TIMEOUT_SCAN_ENABLED !== 'false') {
|
||||
this.receiptTimeoutInitialTimer = setTimeout(() => void this.runReceiptTimeoutScan(), RECEIPT_TIMEOUT_INITIAL_DELAY_MS);
|
||||
this.receiptTimeoutInitialTimer.unref?.();
|
||||
@@ -156,6 +164,7 @@ export class SendChainService implements OnModuleInit, OnModuleDestroy {
|
||||
await this.sendQueue?.close();
|
||||
await this.gatewayQueue?.close();
|
||||
this.redis?.disconnect();
|
||||
await this.submission.onModuleDestroy();
|
||||
}
|
||||
|
||||
async createBatchTask(data: CreateBatchTaskDto) {
|
||||
@@ -349,8 +358,8 @@ export class SendChainService implements OnModuleInit, OnModuleDestroy {
|
||||
return this.submission.confirmImport(data);
|
||||
}
|
||||
|
||||
async enqueueBatchTask(taskId: string) {
|
||||
return this.submission.enqueueBatchTask(taskId);
|
||||
async enqueueBatchTask(taskId: string, preparedMessage?: { messageRecordId: string; queuePriority?: string | null }) {
|
||||
return this.submission.enqueueBatchTask(taskId, preparedMessage);
|
||||
}
|
||||
|
||||
async cancelBatchTask(taskId: string, tenantId?: string, sourceType = 'client') {
|
||||
@@ -576,8 +585,10 @@ export class SendChainService implements OnModuleInit, OnModuleDestroy {
|
||||
phoneNumbers: string[],
|
||||
application: Awaited<ReturnType<SendChainService['findInboundApplication']>>,
|
||||
requestedGroupMessageId?: string,
|
||||
requestedMessageIds?: string[],
|
||||
workflowKey?: string,
|
||||
) {
|
||||
return this.submission.submitCompleteInboundMessage(data, phoneNumbers, application, requestedGroupMessageId);
|
||||
return this.submission.submitCompleteInboundMessage(data, phoneNumbers, application, requestedGroupMessageId, requestedMessageIds, workflowKey);
|
||||
}
|
||||
|
||||
private async collectInboundLongMessageFragment(
|
||||
@@ -596,10 +607,12 @@ export class SendChainService implements OnModuleInit, OnModuleDestroy {
|
||||
data: GatewayInboundSubmitDto & { phoneNumber: string },
|
||||
messageId: string,
|
||||
submitGroupMessageId: string,
|
||||
application: NonNullable<Awaited<ReturnType<SendChainService['findInboundApplication']>>>,
|
||||
synchronousRejection?: { code: string; reason: string },
|
||||
receiptRejection?: { code: string; reason: string },
|
||||
workflowItemKey?: string,
|
||||
) {
|
||||
return this.submission.submitInboundSingleMessage(data, messageId, submitGroupMessageId, synchronousRejection, receiptRejection);
|
||||
return this.submission.submitInboundSingleMessage(data, messageId, submitGroupMessageId, application, synchronousRejection, receiptRejection, workflowItemKey);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -614,8 +627,8 @@ export class SendChainService implements OnModuleInit, OnModuleDestroy {
|
||||
variables?: Record<string, unknown>;
|
||||
phoneNumber: string;
|
||||
sourceType: 'cmpp';
|
||||
}) {
|
||||
return this.submission.evaluateRiskWithPhoneFrequency(input);
|
||||
}, reservationKey?: string) {
|
||||
return this.submission.evaluateRiskWithPhoneFrequency(input, reservationKey);
|
||||
}
|
||||
|
||||
async markUnknownTimeout(data: TimeoutUnknownDto) {
|
||||
@@ -770,8 +783,8 @@ export class SendChainService implements OnModuleInit, OnModuleDestroy {
|
||||
return this.submission.reserveDailySendQuota(applicationId, requestedCount);
|
||||
}
|
||||
|
||||
private async tryReserveDailySendQuota(applicationId: string, requestedCount: number) {
|
||||
return this.submission.tryReserveDailySendQuota(applicationId, requestedCount);
|
||||
private async tryReserveDailySendQuota(applicationId: string, requestedCount: number, reservationKey?: string) {
|
||||
return this.submission.tryReserveDailySendQuota(applicationId, requestedCount, reservationKey);
|
||||
}
|
||||
|
||||
private async chargeAcceptedMessage(message: {
|
||||
|
||||
@@ -128,6 +128,12 @@ export class SendGatewayResultService {
|
||||
async handleSubmitResult(data: GatewaySubmitResultDto) {
|
||||
const message = await this.facade.requireMessageByGatewayEvent(data.messageId, data.gatewayMessageId);
|
||||
const submitRecord = await this.facade.resolveSubmitRecordForGatewayResult(message.id, data);
|
||||
if (data.eventId && submitRecord.resultEventId) {
|
||||
if (submitRecord.resultEventId !== data.eventId) {
|
||||
throw new BadRequestException('Gateway SubmitResult eventId conflicts with the submit attempt');
|
||||
}
|
||||
return this.prisma.smsMessageRecord.findUnique({ where: { id: message.id } });
|
||||
}
|
||||
const effectiveData = { ...data, submitId: submitRecord.submitId };
|
||||
const batchTask = message.batchTaskId
|
||||
? await this.prisma.smsBatchTask.findUnique({ where: { id: message.batchTaskId }, select: { sourceType: true } })
|
||||
@@ -147,6 +153,7 @@ export class SendGatewayResultService {
|
||||
await this.facade.recordSubmitSegments(message, effectiveData, submittedAt);
|
||||
const isStandaloneChannelTest = !message.tenantId && !message.batchTaskId;
|
||||
if (message.submitId && effectiveData.submitId !== message.submitId) {
|
||||
await this.markSubmitResultProcessed(submitRecord.id, data.eventId, submittedAt);
|
||||
return this.prisma.smsMessageRecord.findUnique({ where: { id: message.id } });
|
||||
}
|
||||
const status = data.submitStatus === 'accepted' ? 'submitted' : data.submitStatus === 'timeout' ? 'timeout' : 'submit_failed';
|
||||
@@ -166,6 +173,7 @@ export class SendGatewayResultService {
|
||||
);
|
||||
if (retried) {
|
||||
await this.facade.refreshTaskProgress(businessMessage.batchTaskId);
|
||||
await this.markSubmitResultProcessed(submitRecord.id, data.eventId, submittedAt);
|
||||
return retried;
|
||||
}
|
||||
await this.facade.releaseMessageReservation(businessMessage, data.submitStatus === 'timeout' ? '提交超时释放冻结' : '提交失败释放冻结');
|
||||
@@ -218,9 +226,29 @@ export class SendGatewayResultService {
|
||||
if (message.batchTaskId) {
|
||||
await this.facade.refreshTaskProgress(message.batchTaskId);
|
||||
}
|
||||
await this.markSubmitResultProcessed(submitRecord.id, data.eventId, submittedAt);
|
||||
return this.prisma.smsMessageRecord.findUnique({ where: { id: message.id } });
|
||||
}
|
||||
|
||||
private async markSubmitResultProcessed(submitRecordId: string, eventId: string | undefined, processedAt: Date) {
|
||||
if (!eventId) {
|
||||
return;
|
||||
}
|
||||
const updated = await this.prisma.smsSubmitRecord.updateMany({
|
||||
where: {
|
||||
id: submitRecordId,
|
||||
OR: [{ resultEventId: null }, { resultEventId: eventId }],
|
||||
},
|
||||
data: {
|
||||
resultEventId: eventId,
|
||||
resultProcessedAt: processedAt,
|
||||
},
|
||||
});
|
||||
if (updated.count === 0) {
|
||||
throw new BadRequestException('Gateway SubmitResult eventId conflicts with the submit attempt');
|
||||
}
|
||||
}
|
||||
|
||||
async resolveSubmitRecordForGatewayResult(messageRecordId: string, data: GatewaySubmitResultDto) {
|
||||
if (data.submitId) {
|
||||
const exact = await this.prisma.smsSubmitRecord.findUnique({ where: { submitId: data.submitId } });
|
||||
|
||||
@@ -24,6 +24,8 @@ export class SendGatewaySubmitService {
|
||||
private sendQueue?: Queue<SendJob, unknown, 'send-message'>;
|
||||
private gatewayQueue?: Queue;
|
||||
private worker?: Worker<SendJob>;
|
||||
private readonly taskProgressRefreshes = new Map<string, Promise<void>>();
|
||||
private readonly dirtyTaskProgressRefreshes = new Set<string>();
|
||||
|
||||
constructor(
|
||||
private readonly prisma: PrismaService,
|
||||
@@ -67,7 +69,19 @@ export class SendGatewaySubmitService {
|
||||
}
|
||||
|
||||
|
||||
async enqueueBatchTask(taskId: string) {
|
||||
async enqueueBatchTask(taskId: string, preparedMessage?: { messageRecordId: string; queuePriority?: string | null }) {
|
||||
if (preparedMessage) {
|
||||
// CMPP内部任务在当前请求内刚完成持久化且不暴露取消入口,可安全复用已知ID;
|
||||
// 普通批量任务仍走下方查询路径,以保留取消检查和多消息枚举语义。
|
||||
const queuePriority = normalizeQueuePriority(preparedMessage.queuePriority);
|
||||
await this.facade.getSendQueue().add('send-message', { messageRecordId: preparedMessage.messageRecordId }, {
|
||||
jobId: preparedMessage.messageRecordId,
|
||||
attempts: 3,
|
||||
priority: BULLMQ_PRIORITY[queuePriority],
|
||||
});
|
||||
await this.prisma.smsBatchTask.update({ where: { id: taskId }, data: { status: 'queued' } });
|
||||
return { taskId, enqueued: 1 };
|
||||
}
|
||||
const task = await this.prisma.smsBatchTask.findUnique({ where: { id: taskId } });
|
||||
if (!task) {
|
||||
throw new NotFoundException('SMS batch task not found');
|
||||
@@ -337,6 +351,7 @@ async selectChannelForMessage(
|
||||
forceNational: options.forceNational,
|
||||
excludedChannelIds: excluded,
|
||||
approvedChannelIds,
|
||||
routingKey: message.id,
|
||||
});
|
||||
if (!selected) {
|
||||
throw new NotFoundException('无已报备通过且在线的可用通道');
|
||||
@@ -436,6 +451,29 @@ async waitForChannelRateLimit(channelId: string, tps: number) {
|
||||
}
|
||||
|
||||
async refreshTaskProgress(batchTaskId: string) {
|
||||
const running = this.taskProgressRefreshes.get(batchTaskId);
|
||||
if (running) {
|
||||
// A state transition committed after the running aggregate may not be visible
|
||||
// to its snapshot. Mark one trailing pass instead of issuing another identical
|
||||
// GROUP BY concurrently for every message/result/receipt callback.
|
||||
this.dirtyTaskProgressRefreshes.add(batchTaskId);
|
||||
return running;
|
||||
}
|
||||
const refresh = this.refreshTaskProgressUntilClean(batchTaskId);
|
||||
this.taskProgressRefreshes.set(batchTaskId, refresh);
|
||||
try {
|
||||
await refresh;
|
||||
} finally {
|
||||
if (this.taskProgressRefreshes.get(batchTaskId) === refresh) {
|
||||
this.taskProgressRefreshes.delete(batchTaskId);
|
||||
}
|
||||
this.dirtyTaskProgressRefreshes.delete(batchTaskId);
|
||||
}
|
||||
}
|
||||
|
||||
private async refreshTaskProgressUntilClean(batchTaskId: string) {
|
||||
do {
|
||||
this.dirtyTaskProgressRefreshes.delete(batchTaskId);
|
||||
const groups = await this.prisma.smsMessageRecord.groupBy({
|
||||
by: ['status'],
|
||||
where: { batchTaskId },
|
||||
@@ -455,6 +493,7 @@ async refreshTaskProgress(batchTaskId: string) {
|
||||
where: { id: batchTaskId },
|
||||
data: { progressTotal, submittedTotal, successTotal, failedTotal, unknownTotal, timeoutTotal, status },
|
||||
});
|
||||
} while (this.dirtyTaskProgressRefreshes.has(batchTaskId));
|
||||
}
|
||||
|
||||
getSendQueue(): Queue<SendJob, unknown, 'send-message'> {
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -4,6 +4,7 @@ import { PhoneRoutingLookupService } from '../dictionaries/phone-routing-lookup.
|
||||
import { PrismaService } from '../prisma/prisma.service';
|
||||
import { RiskReviewService } from '../risk-review/risk-review.service';
|
||||
import { PhoneFrequencyService } from '../risk-review/phone-frequency.service';
|
||||
import { MetricsService } from '../metrics/metrics.service';
|
||||
import type { CreateBatchTaskDto, CreateHttpBatchTaskDto, GatewayInboundAuthDto, GatewayInboundSubmitDto, ImportPreviewDto, ConfirmImportDto, SendJob, QueuePriority, RoutedChannel } from './send-chain.contracts';
|
||||
import { SendBatchEntryService } from './send-batch-entry.service';
|
||||
import { SendGatewaySubmitService } from './send-gateway-submit.service';
|
||||
@@ -56,16 +57,20 @@ export class SendSubmissionService {
|
||||
phoneRouting: PhoneRoutingLookupService,
|
||||
facade: SendSubmissionService,
|
||||
callbacks: SendSubmissionCallbacks,
|
||||
metrics?: MetricsService,
|
||||
) {
|
||||
this.batchEntry = new SendBatchEntryService(prisma, billing, riskReview, phoneFrequency, phoneRouting, facade, callbacks);
|
||||
this.inboundEntry = new SendInboundEntryService(prisma, billing, riskReview, phoneFrequency, phoneRouting, facade, callbacks);
|
||||
this.inboundEntry = new SendInboundEntryService(prisma, billing, riskReview, phoneFrequency, phoneRouting, facade, callbacks, metrics);
|
||||
this.reviewContinuation = new SendReviewContinuationService(prisma, billing, riskReview, phoneFrequency, phoneRouting, facade, callbacks);
|
||||
this.scheduledDispatch = new SendScheduledDispatchService(prisma, billing, riskReview, phoneFrequency, phoneRouting, facade, callbacks);
|
||||
this.gatewaySubmit = new SendGatewaySubmitService(prisma, billing, riskReview, phoneFrequency, phoneRouting, facade, callbacks);
|
||||
}
|
||||
|
||||
onModuleDestroy() {
|
||||
return this.gatewaySubmit.onModuleDestroy();
|
||||
return Promise.all([
|
||||
this.gatewaySubmit.onModuleDestroy(),
|
||||
this.inboundEntry.stopInboundWorkflowWorker(),
|
||||
]);
|
||||
}
|
||||
|
||||
async createBatchTask(data: CreateBatchTaskDto) {
|
||||
@@ -121,8 +126,8 @@ async reserveDailySendQuota(applicationId: string, requestedCount: number) {
|
||||
return this.batchEntry.reserveDailySendQuota(applicationId, requestedCount);
|
||||
}
|
||||
|
||||
async tryReserveDailySendQuota(applicationId: string, requestedCount: number) {
|
||||
return this.batchEntry.tryReserveDailySendQuota(applicationId, requestedCount);
|
||||
async tryReserveDailySendQuota(applicationId: string, requestedCount: number, reservationKey?: string) {
|
||||
return this.batchEntry.tryReserveDailySendQuota(applicationId, requestedCount, reservationKey);
|
||||
}
|
||||
|
||||
async authenticateInboundApplication(data: GatewayInboundAuthDto) {
|
||||
@@ -142,8 +147,10 @@ async submitCompleteInboundMessage(
|
||||
phoneNumbers: string[],
|
||||
application: Awaited<ReturnType<SendSubmissionService['findInboundApplication']>>,
|
||||
requestedGroupMessageId?: string,
|
||||
requestedMessageIds?: string[],
|
||||
workflowKey?: string,
|
||||
) {
|
||||
return this.inboundEntry.submitCompleteInboundMessage(data, phoneNumbers, application, requestedGroupMessageId);
|
||||
return this.inboundEntry.submitCompleteInboundMessage(data, phoneNumbers, application, requestedGroupMessageId, requestedMessageIds, workflowKey);
|
||||
}
|
||||
|
||||
async collectInboundLongMessageFragment(
|
||||
@@ -162,10 +169,12 @@ async submitInboundSingleMessage(
|
||||
data: GatewayInboundSubmitDto & { phoneNumber: string },
|
||||
messageId: string,
|
||||
submitGroupMessageId: string,
|
||||
application: NonNullable<Awaited<ReturnType<SendSubmissionService['findInboundApplication']>>>,
|
||||
synchronousRejection?: { code: string; reason: string },
|
||||
receiptRejection?: { code: string; reason: string },
|
||||
workflowItemKey?: string,
|
||||
) {
|
||||
return this.inboundEntry.submitInboundSingleMessage(data, messageId, submitGroupMessageId, synchronousRejection, receiptRejection);
|
||||
return this.inboundEntry.submitInboundSingleMessage(data, messageId, submitGroupMessageId, application, synchronousRejection, receiptRejection, workflowItemKey);
|
||||
}
|
||||
|
||||
async evaluateRiskWithPhoneFrequency(input: {
|
||||
@@ -176,8 +185,8 @@ async evaluateRiskWithPhoneFrequency(input: {
|
||||
variables?: Record<string, unknown>;
|
||||
phoneNumber: string;
|
||||
sourceType: 'cmpp';
|
||||
}) {
|
||||
return this.inboundEntry.evaluateRiskWithPhoneFrequency(input);
|
||||
}, reservationKey?: string) {
|
||||
return this.inboundEntry.evaluateRiskWithPhoneFrequency(input, reservationKey);
|
||||
}
|
||||
|
||||
findInboundApplication(account: string) {
|
||||
@@ -212,14 +221,22 @@ async runScheduledDispatchScan() {
|
||||
return this.scheduledDispatch.runScheduledDispatchScan();
|
||||
}
|
||||
|
||||
async enqueueBatchTask(taskId: string) {
|
||||
return this.gatewaySubmit.enqueueBatchTask(taskId);
|
||||
async enqueueBatchTask(taskId: string, preparedMessage?: { messageRecordId: string; queuePriority?: string | null }) {
|
||||
return this.gatewaySubmit.enqueueBatchTask(taskId, preparedMessage);
|
||||
}
|
||||
|
||||
startWorker() {
|
||||
return this.gatewaySubmit.startWorker();
|
||||
}
|
||||
|
||||
startInboundWorkflowWorker() {
|
||||
return this.inboundEntry.startInboundWorkflowWorker();
|
||||
}
|
||||
|
||||
stopInboundWorkflowWorker() {
|
||||
return this.inboundEntry.stopInboundWorkflowWorker();
|
||||
}
|
||||
|
||||
async processSendJob(job: SendJob) {
|
||||
return this.gatewaySubmit.processSendJob(job);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,20 @@
|
||||
import { Module } from '@nestjs/common';
|
||||
import { ConfigModule } from '@nestjs/config';
|
||||
import { BillingModule } from './billing/billing.module';
|
||||
import { PhoneRoutingLookupService } from './dictionaries/phone-routing-lookup.service';
|
||||
import { MetricsModule } from './metrics/metrics.module';
|
||||
import { PrismaModule } from './prisma/prisma.module';
|
||||
import { PhoneFrequencyService } from './risk-review/phone-frequency.service';
|
||||
import { RiskReviewService } from './risk-review/risk-review.service';
|
||||
import { SendChainService } from './send-chain/send-chain.service';
|
||||
|
||||
@Module({
|
||||
imports: [
|
||||
ConfigModule.forRoot({ isGlobal: true, envFilePath: ['.env.local', '.env'] }),
|
||||
PrismaModule,
|
||||
BillingModule,
|
||||
MetricsModule,
|
||||
],
|
||||
providers: [RiskReviewService, PhoneFrequencyService, PhoneRoutingLookupService, SendChainService],
|
||||
})
|
||||
export class SendWorkerModule {}
|
||||
@@ -0,0 +1,45 @@
|
||||
import 'reflect-metadata';
|
||||
import { NestFactory } from '@nestjs/core';
|
||||
import { createServer } from 'node:http';
|
||||
import { MetricsService } from './metrics/metrics.service';
|
||||
import { SendWorkerModule } from './send-worker.module';
|
||||
|
||||
Object.defineProperty(BigInt.prototype, 'toJSON', {
|
||||
configurable: true,
|
||||
value(this: bigint) {
|
||||
const result = Number(this);
|
||||
if (!Number.isSafeInteger(result)) throw new RangeError('金额超过 JavaScript 安全整数范围');
|
||||
return result;
|
||||
},
|
||||
});
|
||||
|
||||
async function bootstrap() {
|
||||
if ((process.env.CMPP_PROCESS_ROLE?.trim() || 'worker') !== 'worker') {
|
||||
throw new Error('send-worker requires CMPP_PROCESS_ROLE=worker');
|
||||
}
|
||||
const app = await NestFactory.createApplicationContext(SendWorkerModule);
|
||||
app.enableShutdownHooks();
|
||||
const metrics = app.get(MetricsService);
|
||||
const host = process.env.API_WORKER_METRICS_HOST?.trim() || '127.0.0.1';
|
||||
const port = Number(process.env.API_WORKER_METRICS_PORT ?? 9465);
|
||||
const metricsServer = createServer((request, response) => {
|
||||
if (request.method !== 'GET' || request.url !== '/metrics') {
|
||||
response.writeHead(404).end();
|
||||
return;
|
||||
}
|
||||
response.writeHead(200, { 'Content-Type': 'text/plain; version=0.0.4; charset=utf-8', 'Cache-Control': 'no-store' });
|
||||
response.end(metrics.render());
|
||||
});
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
metricsServer.once('error', reject);
|
||||
metricsServer.listen(port, host, resolve);
|
||||
});
|
||||
const close = async () => {
|
||||
await new Promise<void>((resolve) => metricsServer.close(() => resolve()));
|
||||
await app.close();
|
||||
};
|
||||
process.once('SIGTERM', () => void close());
|
||||
process.once('SIGINT', () => void close());
|
||||
}
|
||||
|
||||
void bootstrap();
|
||||
@@ -1150,5 +1150,26 @@ global,不能错误归入client。`client-signature-*`、发送页、企业认
|
||||
|
||||
- `api/src/metrics/`只负责API进程和HTTP路由模板的低基数计数/直方图,不依赖Prisma、Redis或业务Service;独立回环监听由`api/src/main.ts`组装。
|
||||
- `gateway/internal/metrics/`只负责线程安全聚合和Prometheus文本暴露;上游连接池、下游Session和Submit Worker仅提供总数或有界结果,不把实体ID或凭据交给监控模块。
|
||||
- CMPP性能分段沿用上述边界:业务模块只在原调用边界提交固定阶段名、成功标志和单调时钟耗时;指标模块拒绝未知阶段。`supplier_rtt`在供应商连接调用结束时立即停止,API结果回写另计`api_callback`,防止后续优化依据混杂总耗时。
|
||||
- V2提交工作池只归`gateway/internal/submitworker/`治理:Redis领取、全局槽位、在途消息ID和逐条ACK不能渗入上游连接池;`gateway/internal/upstream/`继续只负责通道连接、窗口和供应商协议往返。这样Worker吞吐调优不会改写CMPP连接状态机,连接池也不能自行确认Redis消息。
|
||||
- V3客户入站窗口只归`gateway/internal/inbound/`与项目内受控的`third_party/gocmpp`服务循环治理:API认证只返回应用窗口,inbound会话负责收紧窗口,协议服务循环负责受限派发和断线等待;不得把客户入站槽位与`submitworker`供应商槽位或`upstream`供应商窗口合并成同一并发计数。
|
||||
- V4供应商结果异步边界只归`gateway/internal/resultoutbox/`治理:`upstream`在每个真实分片SubmitResp后只调用持久化接口,`submitworker`只负责聚合结果入Outbox与命令ACK的原子边界,Outbox回调Worker独立控制API并发和PEL恢复。API的`SmsSubmitRecord.resultEventId`是跨重启持久幂等事实;不得把HTTP回调重新放回供应商连接池或Submit工作槽,也不得让Outbox承担业务计费、补发或状态机判断。
|
||||
- V5 API数据库往返优化仍归`send-inbound-entry`、`send-gateway-submit`和`risk-review`现有边界:入口应用快照沿稳定Facade显式传递,单条快速入队只接受已持久化ID和优先级,通用批量入队保留原查询与取消校验;默认规则并发单飞属于`RiskReviewService`内部完整性保障,不得在`SendChainService`新增第二套规则缓存,也不得把余额、频控状态或实际规则决策缓存进进程内存。
|
||||
- 500条/秒第一阶段把接收与业务处理边界固定在`CmppInboundSubmissionInbox`:`gateway/internal/inbound`只生成连接域内稳定请求键,`send-inbound-entry`只负责最小校验、Inbox持久化和稳定响应;`api/src/send-worker.ts`在独立进程内领取并调用既有发送链。领取事务不得包住风控、计费、路由、Redis或供应商调用,业务幂等事实必须落PostgreSQL,不能依赖进程内缓存或localStorage。
|
||||
- API进程角色固定为`CMPP_PROCESS_ROLE=api`,不启动发送Worker、Inbox Worker和周期扫描;`cmpp-send-worker`固定为`worker`并拥有自己的Prisma连接池和回环指标。后续扩容允许水平增加Worker实例,但不得复制HTTP控制器或绕过Inbox直接创建业务消息。
|
||||
- 进程隔离同时包含容量隔离:`PrismaService`按进程角色选择独立、有界的连接池上限;Gateway inbound分别持有与受限Submit窗口匹配的Submit API Transport和小型后台Transport,协议日志/回执流量不得占用Submit连接。连接复用、池上限和超时只属于传输/基础设施边界,不得渗入风控、计费、路由或消息状态机。
|
||||
- `api/src/infrastructure-monitoring/`是运营端监控聚合与固定阈值应用边界:只消费代码白名单 PromQL,并通过版本化 PostgreSQL 单例、promtool 校验和原子规则热加载管理数值阈值;Exporter安装、端口隔离、固定规则模板和权限仍归`tools/monitoring/`治理。
|
||||
- 活动告警已读也归该边界:Prometheus保留告警事实,Prisma仅持久化逐管理员、逐触发周期的阅读状态;全局布局只消费轻量未读汇总,不复制指纹、activeAt或用户隔离逻辑。
|
||||
- 500条/秒第二阶段仍限制在`send-inbound-entry`和既有进程容量边界:短短信用一个PostgreSQL CTE完成当前应用校验和Inbox写入;长短信继续走既有分片域。Worker可按一次领取批次预取应用快照,但不得引入跨批应用状态缓存,也不得把模板、风控、计费、路由状态塞回Gateway同步入口。
|
||||
- 500条/秒第三阶段继续留在`send-inbound-entry`编排边界,但将批次只读风控快照下沉到`RiskReviewService.evaluateTasksBatch`,将频控原子批量预留下沉到`PhoneFrequencyService.reserveBatch`。Inbox编排层只负责批次分组、日限额锁顺序、正常三表批量持久化、幂等入队与逐条租约结算;付费及异常状态机仍委托原单条路径,避免形成第二套计费/审核领域模型。
|
||||
- 批次锁顺序固定为:领取事务只锁Inbox后立即提交;日限额事务按applicationId排序锁`SmsApplicationDailyUsage`;频控按tenant/application分组且规则优先级顺序更新号码状态;三表事务不持有前两类锁,Redis/BullMQ发布永不位于数据库事务内。该顺序用于限制死锁面并允许单批失败后按每条稳定幂等键恢复。
|
||||
- 容量参数继续归进程组装层:Inbox业务槽、BullMQ发送槽、Gateway供应商槽和结果Outbox槽分别有界、分别观测。代码模块不得假定测试环境参数就是生产默认值;生产调优必须重新基于PostgreSQL连接预算、六通道TPS/窗口和回调承载证据。
|
||||
|
||||
## 2026-08-21 第四阶段模块边界
|
||||
|
||||
- `send-inbound-entry`负责编排正价批次短事务;账户固定锁序、总额覆盖、逐短信冻结幂等键留在PostgreSQL一致性边界,不迁入Redis。
|
||||
- `billing.service`集中维护余额锁和流水;`settleFrozenCharge`利用冻结已完成扣减、释放与扣费净变化为零的事实,以单个幂等SQL写成流水对且不争抢账户锁,只有历史半完成恢复才进入带锁余额补扣;`send-accounting`只编排消息计费状态。
|
||||
- `send-chain.helpers`提供无副作用的优先级、主备和稳定weight选择;提交服务只提供消息稳定键与真实在线/报备候选,不保存进程内轮询状态。
|
||||
- Gateway既有Submit池与结果Outbox继续独立有界;先以正价六通道实测定位容量,只有证据证明单条回调仍触发停止线时才扩展批量回调协议。
|
||||
- Gateway上游模块对单分片使用聚合结果作为唯一Outbox边界,对多分片保留逐片持久化;结果Outbox不承担业务去重猜测,事件数量由上游已知分片数确定。
|
||||
- 批次任务进度聚合仍归`send-gateway-submit`统一实现;同进程、同批次并发调用使用单飞与尾随刷新收敛重复`GROUP BY`,不改变消息状态写入、跨进程幂等或PostgreSQL最终事实。
|
||||
|
||||
@@ -0,0 +1,24 @@
|
||||
{
|
||||
"schemaVersion": "v1",
|
||||
"eventId": "submit:submit-20260820-0001:aggregate",
|
||||
"eventType": "submit_result",
|
||||
"path": "/gateway/events/submit-result",
|
||||
"traceId": "trace-20260820-0001",
|
||||
"messageId": "message-20260820-0001",
|
||||
"channelId": "channel-test-1",
|
||||
"submitId": "submit-20260820-0001",
|
||||
"payload": {
|
||||
"eventId": "submit:submit-20260820-0001:aggregate",
|
||||
"schemaVersion": "v1",
|
||||
"messageType": "SubmitResult",
|
||||
"traceId": "trace-20260820-0001",
|
||||
"messageId": "message-20260820-0001",
|
||||
"channelId": "channel-test-1",
|
||||
"createdAt": "2026-08-20T05:00:00Z",
|
||||
"sequenceId": 1,
|
||||
"gatewayMessageId": "1000000000000001",
|
||||
"submitStatus": "accepted",
|
||||
"submittedAt": "2026-08-20T05:00:00Z"
|
||||
},
|
||||
"createdAt": "2026-08-20T05:00:00Z"
|
||||
}
|
||||
@@ -6,7 +6,8 @@
|
||||
{ "$ref": "#/$defs/SubmitCommand" },
|
||||
{ "$ref": "#/$defs/SubmitResult" },
|
||||
{ "$ref": "#/$defs/ReceiptEvent" },
|
||||
{ "$ref": "#/$defs/UplinkEvent" }
|
||||
{ "$ref": "#/$defs/UplinkEvent" },
|
||||
{ "$ref": "#/$defs/SubmitResultOutboxEvent" }
|
||||
],
|
||||
"$defs": {
|
||||
"Envelope": {
|
||||
@@ -139,6 +140,26 @@
|
||||
}
|
||||
]
|
||||
},
|
||||
"SubmitResultOutboxEvent": {
|
||||
"type": "object",
|
||||
"required": ["schemaVersion", "eventId", "eventType", "path", "messageId", "channelId", "submitId", "payload", "createdAt"],
|
||||
"properties": {
|
||||
"schemaVersion": { "const": "v1" },
|
||||
"eventId": { "type": "string", "pattern": "^submit:.+:(aggregate|segment:[1-9][0-9]*)$" },
|
||||
"eventType": { "enum": ["submit_result", "submit_segment_result"] },
|
||||
"path": { "enum": ["/gateway/events/submit-result", "/gateway/events/submit-segment-result"] },
|
||||
"traceId": { "type": "string" },
|
||||
"messageId": { "type": "string", "minLength": 1 },
|
||||
"channelId": { "type": "string", "minLength": 1 },
|
||||
"submitId": { "type": "string", "minLength": 1 },
|
||||
"payload": {
|
||||
"type": "object",
|
||||
"required": ["eventId"],
|
||||
"properties": { "eventId": { "type": "string", "minLength": 1 } }
|
||||
},
|
||||
"createdAt": { "type": "string", "format": "date-time" }
|
||||
}
|
||||
},
|
||||
"ReceiptEvent": {
|
||||
"allOf": [
|
||||
{ "$ref": "#/$defs/Envelope" },
|
||||
|
||||
@@ -78,19 +78,19 @@
|
||||
"name": "authRequest",
|
||||
"kind": "type",
|
||||
"file": "authentication.go",
|
||||
"sha256": "0d3e6ab7ec4f418fefb88d2930b01c23eaff926097474c50ba678d553a1aa5a1"
|
||||
"sha256": "742c3df9a2743741b679a5e1524a8826168385e2af5442fc9019ffc8139ea159"
|
||||
},
|
||||
{
|
||||
"name": "authResponse",
|
||||
"kind": "type",
|
||||
"file": "authentication.go",
|
||||
"sha256": "387ff32b3067a0eb342d09e1e5d3308f8359415171a2045f10dc56ca01205727"
|
||||
"sha256": "049a0b49b97fca25e6093948732f33c9896020c8f3ef8ca4303b05796047ff2b"
|
||||
},
|
||||
{
|
||||
"name": "authenticate",
|
||||
"kind": "func",
|
||||
"file": "authentication.go",
|
||||
"sha256": "3dba30340070be6b78dcb1de963fde7a567701925684279c5b675ee9b197ec2c"
|
||||
"sha256": "fbd2a9aab7116a66f4a9cdc4538d1ccd71f822dda37f3f92c08d62417c1f4418"
|
||||
},
|
||||
{
|
||||
"name": "cmppVersionName",
|
||||
@@ -102,7 +102,13 @@
|
||||
"name": "handleLogin",
|
||||
"kind": "func",
|
||||
"file": "authentication.go",
|
||||
"sha256": "bf3bdbcda7ddfd9f0e2ff53b436e151c94247b4ed40bcf60ec2f78fdb2559e64"
|
||||
"sha256": "4d38524f2e4efc3bd19d7e319268b611c306341e18cb8b92f693922c1a21eee0"
|
||||
},
|
||||
{
|
||||
"name": "reportSecurityEvent",
|
||||
"kind": "func",
|
||||
"file": "authentication.go",
|
||||
"sha256": "8d20e1b5b80f1a8a7c8948bc5dcab283f4c14d9d516008afdfc061b3f74cbaeb"
|
||||
},
|
||||
{
|
||||
"name": "setInboundConnectResponse",
|
||||
@@ -182,6 +188,12 @@
|
||||
"file": "delivery.go",
|
||||
"sha256": "4acf5485100e1bc6bb19a8004a3b1b7796fd8a6192fe2899e163abed7d134b69"
|
||||
},
|
||||
{
|
||||
"name": "downstreamReceiptMessageID",
|
||||
"kind": "func",
|
||||
"file": "delivery.go",
|
||||
"sha256": "8b9d214452acdaebfb193c79bd810219fc2400a4c10784f1c3e3322e70a7ad9c"
|
||||
},
|
||||
{
|
||||
"name": "errorMessageWithCode",
|
||||
"kind": "func",
|
||||
@@ -342,13 +354,19 @@
|
||||
"name": "ListenAndServe",
|
||||
"kind": "func",
|
||||
"file": "server.go",
|
||||
"sha256": "c57e5d860f6bdbb7d47a5414e0d186fe34cf61c42f94fc2b1ab5739b6272c272"
|
||||
"sha256": "8415f1a682d28c5a64f3fbe26972b1cab07cd04c43c74d7e9da1065255a06e5d"
|
||||
},
|
||||
{
|
||||
"name": "Server",
|
||||
"kind": "type",
|
||||
"file": "server.go",
|
||||
"sha256": "9dacd52458fb19e19752e201de3b9c9b6e8fd9ba93afda454e7d7444b17148f5"
|
||||
"sha256": "405e6bca394e2990f7fca11eda9ba45f1915a33f64e1321a4fa63f292e191b88"
|
||||
},
|
||||
{
|
||||
"name": "boundedSubmitWindow",
|
||||
"kind": "func",
|
||||
"file": "server.go",
|
||||
"sha256": "b5ef665e15e9c4b6bad345c94cdc19c28cce4037fb9f484a9d8ca7319c8418c3"
|
||||
},
|
||||
{
|
||||
"name": "defaultHTTPTimeout",
|
||||
@@ -356,12 +374,30 @@
|
||||
"file": "server.go",
|
||||
"sha256": "247aaf1070a6886f01059677dad739e69cb9dd10bd55d6397ba5fb9d385fe39a"
|
||||
},
|
||||
{
|
||||
"name": "ActiveConnectionCount",
|
||||
"kind": "func",
|
||||
"file": "sessions.go",
|
||||
"sha256": "3df8a36ca047e9b0d2fb6e2cc4f54b5623b54edacb28397c7e49d2452ee40450"
|
||||
},
|
||||
{
|
||||
"name": "DisconnectAccount",
|
||||
"kind": "func",
|
||||
"file": "sessions.go",
|
||||
"sha256": "306c822ca618b5d47dd002595e0896b87d86c26a0b35ef7ea6b032df8cefe66b"
|
||||
},
|
||||
{
|
||||
"name": "SubmitSlotSnapshot",
|
||||
"kind": "func",
|
||||
"file": "sessions.go",
|
||||
"sha256": "4d80321cc1634c4c15880f335144c4c4927cb94df0b4390da1499aa649feb317"
|
||||
},
|
||||
{
|
||||
"name": "beginInboundSubmit",
|
||||
"kind": "func",
|
||||
"file": "sessions.go",
|
||||
"sha256": "9a25ff062c976af7bddd07b2ace5b0e0de3a562a5e26f8df03fb0030a7ac470b"
|
||||
},
|
||||
{
|
||||
"name": "downstreamConnectionEvent",
|
||||
"kind": "type",
|
||||
@@ -378,7 +414,7 @@
|
||||
"name": "downstreamSession",
|
||||
"kind": "type",
|
||||
"file": "sessions.go",
|
||||
"sha256": "36d85eab72c673e4e4d47a2b60e68d96561ecb93b0a851de850280706e6779bc"
|
||||
"sha256": "3f4383f1c5406906df5065f32dc38f3a1be84f6fd678a2533101ab35d90f5d68"
|
||||
},
|
||||
{
|
||||
"name": "findSessionByConn",
|
||||
@@ -452,6 +488,12 @@
|
||||
"file": "sessions.go",
|
||||
"sha256": "61cf0ec3ac4278be5ccbd112d196aec662ae20f6aebd6079ac7102e83b1e7008"
|
||||
},
|
||||
{
|
||||
"name": "submitWindowByConn",
|
||||
"kind": "func",
|
||||
"file": "sessions.go",
|
||||
"sha256": "d17509dc36a19c5d1910bb8588ef875fd4f6225675643f994886d0f48aa15f08"
|
||||
},
|
||||
{
|
||||
"name": "touchPresence",
|
||||
"kind": "func",
|
||||
@@ -474,7 +516,7 @@
|
||||
"name": "handleSubmit",
|
||||
"kind": "func",
|
||||
"file": "submit.go",
|
||||
"sha256": "61e3b8235ab9121a82651e07ce53e38cb650561d80c5f1e4fe6bf4b101b4240a"
|
||||
"sha256": "36b9016de0d3cf0d2286b00aad90435d67cc3720ba5fcc87a3d27e8c3db4d2c1"
|
||||
},
|
||||
{
|
||||
"name": "inboundLongMessageFragment",
|
||||
@@ -488,6 +530,12 @@
|
||||
"file": "submit.go",
|
||||
"sha256": "adac6aed04068f3811fbcf0530ab54f7384f0ea85a303c9c70326b5a9a46ba78"
|
||||
},
|
||||
{
|
||||
"name": "inboundSubmitRequestID",
|
||||
"kind": "func",
|
||||
"file": "submit.go",
|
||||
"sha256": "5f3515253ece00423b96d0c8b32efdefa4b4c61d88991997df66c9acccce4b7e"
|
||||
},
|
||||
{
|
||||
"name": "messageIDFrom",
|
||||
"kind": "func",
|
||||
@@ -500,6 +548,12 @@
|
||||
"file": "submit.go",
|
||||
"sha256": "977d81193fdc2adfd402d6b1ddb142ea2fdf44f7c8f8d2d00ca8cf20d2cf0830"
|
||||
},
|
||||
{
|
||||
"name": "observeInboundSubmitResponse",
|
||||
"kind": "func",
|
||||
"file": "submit.go",
|
||||
"sha256": "645537f6f6e6eb69296f756fb3cb291ccecf2b82d48ce0bea427bb898dff6188"
|
||||
},
|
||||
{
|
||||
"name": "setInboundSubmitResponse",
|
||||
"kind": "func",
|
||||
@@ -510,13 +564,13 @@
|
||||
"name": "submit",
|
||||
"kind": "func",
|
||||
"file": "submit.go",
|
||||
"sha256": "b541d2c1d7e5592a6b8ad213d0cdbb81fcc98702023f1d5a4d35207476767ae7"
|
||||
"sha256": "84c47d59444bf475d416dc9d1cb556b4257b8bd6d453773e01215ab568dc4963"
|
||||
},
|
||||
{
|
||||
"name": "submitRequest",
|
||||
"kind": "type",
|
||||
"file": "submit.go",
|
||||
"sha256": "80791da6ef4e968dbafbea288ef783422b4c089013a05175b6add32bf8919df0"
|
||||
"sha256": "7edc6ef9b65d44dd3c2f22415c3d6ceca12621cbf531b089fb500ecfea282253"
|
||||
},
|
||||
{
|
||||
"name": "submitResponse",
|
||||
@@ -548,11 +602,23 @@
|
||||
"file": "transport.go",
|
||||
"sha256": "fd069087ad8497197f55a98ecfb9e5b1234ce641a0fb0a9161f8c320f436dd37"
|
||||
},
|
||||
{
|
||||
"name": "maxAPIResponseBodyBytes",
|
||||
"kind": "const",
|
||||
"file": "transport.go",
|
||||
"sha256": "3b238a26101aa6a3b50d6651479502adc2be2fb98b338c933cbfd4c90510995f"
|
||||
},
|
||||
{
|
||||
"name": "post",
|
||||
"kind": "func",
|
||||
"file": "transport.go",
|
||||
"sha256": "10fac868d1fe77fe2c261f14692245b6a08b573ee460cde0c7a77f6ee3ca77b1"
|
||||
"sha256": "e4debafbce512419600840f4b8e6ddf8b32ad54b674e3a86f42aea43ea195efc"
|
||||
},
|
||||
{
|
||||
"name": "postWithClient",
|
||||
"kind": "func",
|
||||
"file": "transport.go",
|
||||
"sha256": "0d98d43b9d8ebdb9cf0e0adf726f7af70fd771b2fbfdf7f6c47ebb0c529a4116"
|
||||
},
|
||||
{
|
||||
"name": "remoteIP",
|
||||
|
||||
@@ -17,7 +17,7 @@
|
||||
{
|
||||
"name": "handleSubmitResult",
|
||||
"file": "send-gateway-result.service.ts",
|
||||
"bodySha256": "6184170748eb3934496b060ea3b6704f1d1a831e7ffde7e7f8b73a7c223a869c"
|
||||
"bodySha256": "9414f8d4e5e1c3cf6d0747e47ce01e101e142772af4f56900932bc6d4ac11ec7"
|
||||
},
|
||||
{
|
||||
"name": "resolveSubmitRecordForGatewayResult",
|
||||
|
||||
@@ -31,7 +31,7 @@
|
||||
},
|
||||
{
|
||||
"name": "enqueueBatchTask",
|
||||
"bodySha256": "909972eb0accfb6e31b9fd6118750005c4a6671577ef5b5c9222a341dac59800",
|
||||
"bodySha256": "081fab5913288d7325a8c4d15cea5e7d37af1c6614154f2963eb635bbcc0f40e",
|
||||
"file": "send-gateway-submit.service.ts"
|
||||
},
|
||||
{
|
||||
@@ -76,7 +76,7 @@
|
||||
},
|
||||
{
|
||||
"name": "submitCompleteInboundMessage",
|
||||
"bodySha256": "10e2140580e4bae2055853981de65f4adf82c228b647dbb56fce7c39469acaf3",
|
||||
"bodySha256": "14bd1ffd41b3e38d9ceeabd7f0e19d6bb356b4ff2b281acc31a8477c8bc18c13",
|
||||
"file": "send-inbound-entry.service.ts"
|
||||
},
|
||||
{
|
||||
@@ -91,7 +91,7 @@
|
||||
},
|
||||
{
|
||||
"name": "submitInboundSingleMessage",
|
||||
"bodySha256": "6fa4c577848eb06f635a8f186d1e1510b1213f0ba6d439d032ea08c593725d14",
|
||||
"bodySha256": "6246c081fa2433ef5b3ffc00a2da7cfe32da9d02c310ef757b904800ffbd6732",
|
||||
"file": "send-inbound-entry.service.ts"
|
||||
},
|
||||
{
|
||||
|
||||
@@ -173,7 +173,7 @@
|
||||
"name": "Manager",
|
||||
"kind": "type",
|
||||
"file": "manager.go",
|
||||
"sha256": "c7ead8b037bc87ad9800650ce191f5fcee9648b8805a133b9506743c762ea608"
|
||||
"sha256": "b08726201c34da87ae735abd7fc56d2b904104d4b7f6a6a6365263ec5b0a43bc"
|
||||
},
|
||||
{
|
||||
"name": "defaultChannelConnectionID",
|
||||
@@ -389,7 +389,7 @@
|
||||
"kind": "func",
|
||||
"receiver": "Manager",
|
||||
"file": "submit.go",
|
||||
"sha256": "81d3554f05c38d91cde37b4fa918c2d30d7f4e6da2265c2fe0f9b499650375f3"
|
||||
"sha256": "9be92b0d32626ce93d0b2d721bf008305c4b9e5919e287d49b91c6f2474ddeb1"
|
||||
},
|
||||
{
|
||||
"name": "submitPart",
|
||||
@@ -410,7 +410,7 @@
|
||||
"kind": "func",
|
||||
"receiver": "connectionPool",
|
||||
"file": "submit.go",
|
||||
"sha256": "9efe1c726215a512d7039c852fdca06544f603ed810daae3fe291cc51038315b"
|
||||
"sha256": "577b7f942f4b95ea3868d7cf9aadae800df8edda0e7ca94a205b4124be58e9e3"
|
||||
},
|
||||
{
|
||||
"name": "defaultInt",
|
||||
|
||||
@@ -124,7 +124,7 @@
|
||||
9. 短信应用必须有应用级客户侧企业代码 `cmppEnterpriseCode`,运营端添加/编辑应用时可自定义;不得从上游通道 `SmsChannel.enterpriseCode` 透传。
|
||||
10. 短信应用接口密码 `passwordCipher` 新建时默认随机生成 16 位 UUID 片段,运营端可手工修改;编辑时留空不覆盖原密码。
|
||||
11. 应用 `AppID` 是平台内部应用标识,用于页面展示、复制参数和工单定位,不作为 CMPP bind/login 认证参数。
|
||||
12. 短信应用必须可配置客户侧 CMPP 最大连接数 `cmppMaxConnections`;客户侧提交窗口 `cmppWindowSize` 后端保留默认值,当前第一版不在运营端展示或要求运营配置,待 Gateway 入站侧按应用窗口真正限流后再开放为高级配置。
|
||||
12. 短信应用必须可配置客户侧 CMPP 最大连接数 `cmppMaxConnections`;客户侧提交窗口 `cmppWindowSize` 由真实后端保存并在认证成功时下发 Gateway。V3 起 Gateway 对同一已认证连接最多并发处理该窗口数量的 Submit,超过窗口的报文通过停止继续读取形成 TCP 背压,不得无界创建协程;Gateway 全局保护上限由 `GATEWAY_CMPP_INBOUND_MAX_CONCURRENCY` 控制,默认 64、最大 1024,应用窗口只能进一步收紧而不能放大全局上限。
|
||||
13. 短信应用必须恢复设计基线中的“短信接口”开关,字段为 `interfaceEnabled`,默认开通;关闭后客户端/API 发送链路、客户侧 CMPP Gateway bind/login 和 submit 都必须被真实后端拒绝,不允许只在前端隐藏入口。
|
||||
14. CMPP 协议类型当前第一版仅允许 `CMPP2.0`,字段保持 `interfaceType=cmpp20`;HTTP 不写入该字段,而是通过独立的 `SmsApplicationHttpConfig` 总开关和子能力配置开通。前后端仍必须拒绝把 `interfaceType` 直接改成 `http` 等无效协议值。
|
||||
|
||||
@@ -238,8 +238,9 @@
|
||||
7. Gateway 必须实现真实 CMPP Submit,包括短信内容编码、长短信拆分、RegisteredDelivery、serviceId、srcId、destTerminalId、msgFmt、feeType/feeCode 等字段映射。
|
||||
8. Gateway 必须消费 NestJS 投递的 `SubmitCommand` 队列或等价内部接口;提交成功、提交失败、超时均必须回传 `SubmitResult`,不得只停留在 API 侧入队。
|
||||
9. Gateway 必须按通道连接和窗口容量控制并发,处理窗口满、SMSC 慢响应、sequence 回绕、连接断开时的在途消息状态。
|
||||
10. Gateway 必须对每个物理通道执行 Redis 分布式 TPS 限速。`SmsChannel.rateLimitPerSecond` 是单通道上限,不是平台总上限;同一通道被多个通道组或多个 Gateway 实例使用时共享同一额度,不同通道独立计数。通道连接命令下发的配置值是最终上限,提交命令携带的值只能进一步降低、不能放大该上限。超流速消息必须继续保留在 Redis Stream pending 中等待可用时隙,不能因等待直接标记发送失败;Gateway 重启后仍可由 consumer group 恢复。worker 必须并发处理一个读取批次,让不同通道独立等待,不能因低 TPS 通道造成其他通道队头阻塞;单通道实际并发仍由 Redis 限速和 CMPP 窗口共同约束。
|
||||
10. Gateway 必须对每个物理通道执行 Redis 分布式 TPS 限速。`SmsChannel.rateLimitPerSecond` 是单通道上限,不是平台总上限;同一通道被多个通道组或多个 Gateway 实例使用时共享同一额度,不同通道独立计数。通道连接命令下发的配置值是最终上限,提交命令携带的值只能进一步降低、不能放大该上限。超流速消息必须继续保留在 Redis Stream pending 中等待可用时隙,不能因等待直接标记发送失败;Gateway 重启后仍可由 consumer group 恢复。worker 必须使用持续补位的全局有界工作池,任一任务完成后立即领取后续消息,不得以“读取10条、等待整批结束”形成批次屏障;每条消息只在自身提交结果已持久回传或完成死信处理后独立`XACK`。全局并发默认64、可由`GATEWAY_SUBMIT_WORKER_CONCURRENCY`配置且上限1024;单通道实际并发仍由 Redis 限速、连接数和 CMPP 窗口共同约束。恢复pending时必须防止超过`MinIdle`的在途消息被同一进程重复提交。
|
||||
11. Gateway 不承担业务审核、计费、签名报备、通道组路由、黑名单或敏感词判断;这些由 NestJS 完成,Gateway 只执行已授权通道提交与协议事件回传。
|
||||
12. 供应商SubmitResp及长短信分片结果必须先进入共享Redis的幂等Outbox,再由独立有界回调Worker异步上报API;供应商Submit工作槽只等待连接窗口、限速和真实供应商往返,不得等待API结果回写。每个分片结果必须在继续发送下一分片前写入Outbox;聚合结果入Outbox与原`gateway.submit.commands`消息`XACK`必须原子完成。事件ID按submitId和分片序号确定生成,重复发布不得产生第二个事件;API必须按事件ID持久幂等,重复回调不得重复扣费、释放余额、补发或推进状态。回调失败保留在Outbox PEL并可在进程重启后恢复,成功后逐条ACK并删除;去重键和并发必须有界配置,不得形成无限内存或Stream归档。
|
||||
|
||||
#### 4.8.2 下游客户 CMPP 接入能力
|
||||
|
||||
@@ -2096,8 +2097,44 @@
|
||||
- 运营端展示API请求/错误/延迟/事件循环、Gateway Submit/队列/连接、PostgreSQL连接/死锁、Redis内存/连接/淘汰、Nginx连接/请求和MinIO可用性;指标缺失显示“待采集”,不以0伪装。
|
||||
- 告警必须使用持续窗口和最低样本量;默认阈值、收敛关系、标签禁止项和性能预算以`docs/prometheus-system-monitoring-design-20260814.md`第9节为准。
|
||||
- 指标不得包含手机号、短信正文、短信/CMPP/任务ID、密钥、完整URL或SQL原文;不得把时序指标高频写入业务PostgreSQL。
|
||||
- CMPP入站性能优化第一步只增加观测,不改变SubmitResp、持久化、风控、计费、路由、入队或回执语义。API必须用固定低基数阶段记录`application_lookup/long_message_fragment/submission_precheck/template_match/task_persist/api_request_persist/content_detection/message_persist/risk_frequency/billing/queue_publish/complete_submit/total`耗时及成功/失败;Gateway入站必须拆分`decode/api_roundtrip/response_write/handler_total`。
|
||||
- Gateway供应商下发必须分别记录`stream_wait/rate_limit_wait/connection_wait/supplier_rtt/api_callback`;其中`supplier_rtt`只覆盖供应商连接上的Submit请求与SubmitResp往返,不得包含结果回写API的耗时。阶段名和结果值必须使用代码固定白名单,不得添加手机号、企业、应用、通道、消息、任务或连接ID标签。
|
||||
- V2有界工作池必须暴露配置槽位数和当前在途槽位数,使用固定`state=configured|in_flight`标签;该指标用于区分Worker容量耗尽与供应商窗口/限速等待,不得增加通道或消息标签。
|
||||
- V3入站并发必须只并发Submit业务处理,连接认证保持串行先完成,心跳和Deliver ACK不得被长耗时Submit阻塞;每个SubmitResp继续使用原请求Sequence_Id关联,允许按实际完成顺序返回。同一连接关闭时必须先等待已接受的在途处理收尾,再清理会话和回执映射,避免迟到处理重新注册已断开的连接。`cmpp_gateway_inbound_submit_slots{state=configured|in_flight}`只暴露全部在线连接的聚合窗口与在途数量,不得增加账号、应用、连接或消息标签。
|
||||
- V4结果Outbox必须暴露独立回调Worker的configured/in_flight槽位和Outbox pending/lag,仍只使用固定状态标签。`api_callback`从V4起只在Outbox回调Worker计时,不再混入供应商Submit工作槽;压测结束必须同时核对命令Stream和结果Outbox均`pending=0/lag=0`,并证明API回调故障时供应商槽继续释放、结果不丢失且恢复后只处理一次。
|
||||
- V5 API入站数据库往返治理不得改变SubmitResp、余额冻结、号码频控、模板/签名、路由、队列或失败回执语义。同一Submit入口已取得的应用、企业和IP白名单快照必须复用于其全部目标号码,不得逐号码重复查询;任务风控与号码频控共用的默认规则完整性检查必须使用短TTL并发单飞,正常完整状态最多执行一次聚合数据库检查,实际生效规则仍逐次读取;刚持久化的单条CMPP内部任务和消息可凭已知ID、优先级直接入队,不得为入队重新查询同一任务和消息。缓存只覆盖默认规则“是否齐全”,检查失败必须立即失效,规则缺失须在短TTL到期后自动恢复;不得缓存余额、频控计数、应用启停或实际生效规则。
|
||||
- 面向完整处理500条/秒目标的第一阶段,CMPP SubmitResp语义调整为“已完成最小协议/账号/IP/Src_Id校验且已持久化接收事实”,不再同步等待模板、风控、频控、日限、计费、路由和入队。Gateway必须为同一连接内同一Submit重试生成稳定请求键;PostgreSQL Inbox以唯一请求键和载荷摘要幂等保存原请求、稳定内部MessageId及响应。相同键相同载荷返回原响应,相同键不同载荷必须拒绝。
|
||||
- Inbox Worker必须作为独立非root进程运行,使用独立数据库连接池和持续有界工作池;领取使用短事务、`FOR UPDATE SKIP LOCKED`和过期租约回收,实际业务处理在领取事务外执行。成功逐条完成,失败以有上限退避回到pending且不得静默丢弃。日发送配额、号码频控及余额冻结必须使用由Inbox请求派生的持久幂等键,确保进程崩溃或租约回收不会重复扣量、重复计频或重复冻结。
|
||||
- Inbox的`DateTime`列沿用平台UTC无时区存储口径,领取、租约和退避SQL必须显式使用UTC时钟比较,不能受数据库会话或宿主机Asia/Shanghai时区影响;日期型日配额预留写Prisma时必须传合法Date对象,不能把`YYYY-MM-DD`字符串当作DateTime。
|
||||
- 优先应用和普通应用共享同一耐久Inbox,但领取顺序必须保留优先级并在同一优先级内FIFO;进入BullMQ后继续沿用priority=1、normal=100。快路径成功只代表平台已可靠接收,异步业务拒绝仍必须落真实消息/任务状态并按既有CMPP失败回执链路通知客户,不得伪装为供应商最终送达。
|
||||
- 第一阶段的验收是入口可持续接收、Inbox不丢不重且最终可排空,并为后续500条/秒全链路扩容建立解耦边界;不能仅凭SubmitResp吞吐宣称完整500条/秒。压测必须同时报告SubmitResp成功率/延迟、Inbox pending/processing/最老等待、异步完成速率和排空时间,以及命令Stream、结果Outbox和数据库最终对账。
|
||||
- Gateway到本机API的Submit专用HTTP连接池必须按`GATEWAY_CMPP_INBOUND_MAX_CONCURRENCY`设置相同的每主机连接上限与空闲连接上限,复用长连接并继续保留10秒请求超时;协议日志、回执恢复和连接状态使用独立的16连接后台池,不得与Submit竞争传输槽,也不得让Go默认每主机仅保留2条空闲连接在高窗口下制造连接抖动。API与Worker数据库连接池必须分别以`API_DB_POOL_MAX`和`API_WORKER_DB_POOL_MAX`显式有界,默认32/8;API槽位优先保障Inbox受理,Worker不得以默认大池抢占入口连接,所有上限之和必须低于PostgreSQL`max_connections`并为运维连接留余量。
|
||||
- 系统监控标题说明必须明确标注数据来自 Prometheus;“服务关键指标”位于趋势/核心服务区域之后、活动告警之前,并提供统一的“告警阈值设置”入口。安全检测页不重复渲染大号标题,说明文字必须明确标注使用 Fail2ban。
|
||||
- 告警阈值仅开放固定指标的警告/严重数值,不允许前端提交 PromQL、标签、文件路径或持续时间;必须满足警告值小于严重值。配置以 PostgreSQL 保存版本、期望值、生效值和应用状态,经 `promtool check rules` 校验、同目录原子替换和 Prometheus 热加载成功后才标记生效,失败保留上一生效规则并展示原因。
|
||||
- 右上角预警中心增加“系统监控告警”,通过独立轻量接口统计 Prometheus 当前 firing/pending 告警及严重数,跳转系统监控活动告警区;任一预警域失败不得清空其他域。
|
||||
- 活动告警列表必须提供逐条“标记已读”。已读状态按管理员和“告警指纹 + 本次 activeAt”持久化到 PostgreSQL;仅从当前管理员的预警中心数量中扣减,不改变 Prometheus firing/pending 状态,也不减少页面活动告警总数。同标签告警恢复后再次触发时必须重新成为未读。
|
||||
- 服务端只能确认 Prometheus 当前仍存在且 activeAt 一致的告警,过期、已恢复或已重新触发的请求必须拒绝;重复点击同一次告警应幂等,并写操作日志。阈值设置弹窗只保留通用 Modal 外层滚动,不得嵌套第二个独立滚动区域。
|
||||
|
||||
## 完整处理500条/秒第二阶段:合并入口SQL与有界全链扩容(2026-08-20)
|
||||
|
||||
- 普通短短信快路径必须把应用、企业当前启停状态、接口开关、IP白名单、Src_Id校验与Inbox幂等写入合并为同一个PostgreSQL语句和同一MVCC快照;正常新请求只允许一次数据库往返,不得先查应用再写Inbox。应用账号唯一索引和Inbox请求键唯一索引继续作为查询与幂等边界,不增加无证据索引。
|
||||
- 合并语句不得缓存应用启停或白名单;应用不存在、已停用、接口关闭、IP或Src_Id不匹配时不得产生Inbox。相同请求键同载荷返回原响应,不同载荷拒绝;并发唯一键竞争因快照不可见时只允许一次只读恢复,不得通过无意义`ON CONFLICT DO UPDATE`制造写放大。
|
||||
- 长短信仍在完整重组并完成既有校验后写Inbox,不为追求吞吐改写分片状态机。Inbox Worker每次领取后应按本批应用ID一次读取当前应用/企业/白名单快照,不能对同一批每条消息重复查应用;领取、实际业务处理和逐条完成仍保持原短事务与租约边界。
|
||||
- 测试环境可在PostgreSQL连接总预算内分别提高Worker业务槽、BullMQ发送槽、Gateway供应商槽及结果Outbox槽,但每个池必须显式有界,且总数据库连接须为运维保留余量。供应商真实TPS、连接数和窗口仍是硬上限,禁止用并发参数绕过单通道限速。
|
||||
- 第二阶段仍以“完整处理”验收:除500条/秒SubmitResp零拒绝、零缺失和延迟停止线外,还必须等待Inbox、BullMQ、命令Stream、结果Outbox与回执链排空,并按数据库业务记录、唯一MessageId、供应商尝试/接受和最终状态对账。入口达到500而全链排空速率不足500条/秒时必须明确判失败。
|
||||
|
||||
## 完整处理500条/秒第三阶段:Inbox业务批处理与数据库竞争治理(2026-08-21)
|
||||
|
||||
- Worker以`API_INBOUND_WORKFLOW_BATCH_SIZE`控制单次有界批次,仍用优先级/FIFO和`FOR UPDATE SKIP LOCKED`短事务领取;批次只共享当次读取的应用、模板、签名、黑名单、风控规则和敏感词快照,不得跨批缓存应用状态、余额、频控计数或日限额。
|
||||
- 正常单号码、零计费短短信允许走批量快路径:日限额按应用锁定并为每个Inbox请求写独立幂等预留,风控只共享只读输入,号码频控在同一事务批量更新状态并逐请求保存决定,任务/API请求/消息三表在一个短事务批量创建,BullMQ使用消息ID作为幂等Job ID批量入队。
|
||||
- 付费短信、重复号码、黑名单/格式拒绝、模板人工审核、引流匹配歧义、既有消息恢复及其他异常分支必须回到原逐条状态机;不得为压测降低计费、风控、频控、模板、签名或失败回执规则。批量路径在数据库提交后、队列发布前崩溃时,逐条恢复必须利用稳定任务号、请求号、MessageId和预留键补齐,不得重复计量或创建消息。
|
||||
- 批量Worker指标增加`worker_claim/reference_preload/daily_quota`固定低基数阶段,并继续记录`risk_frequency/message_persist/queue_publish`;配置必须显式启用批处理并使用正整数批次大小。验收仍以真实PostgreSQL、Redis、BullMQ和隔离供应商证据为准,不能用零计费压测结果外推付费链路吞吐。
|
||||
|
||||
## 完整处理500条/秒第四阶段:正价计费与供应商并行(2026-08-21)
|
||||
|
||||
- 普通CMPP短短信批量路径必须支持正单价:按企业固定锁序一次校验批次总金额、一次更新余额,并为每短信写独立幂等冻结流水;任务、请求、消息与冻结流水同属一个短事务。余额加授信必须覆盖所需金额,不能只判断大于零。
|
||||
- 供应商接受后,释放冻结与正式扣费以一个原子SQL写入两条幂等审计流水;二者净余额变化为零,不得争抢企业账户锁或重复更新账户。只有恢复历史“已释放但未扣费”半完成状态时才取得账户锁补扣。零金额不创建AccountTransaction;拒绝、超时、重投和最终失败继续沿用逐短信幂等释放、扣费和退款。
|
||||
- 同通道组内在线、已报备、同地域、同最低优先级且非备用的通道按weight和稳定消息键分流;备用、离线及更低优先级不抢占。只有显式配置为同优先级非备用的通道才参与主动双活。
|
||||
- 单分片短短信的聚合SubmitResult已经携带完整分片信息,只发布聚合Outbox事件;不得再发布内容相同的分片事件造成双倍API回调和数据库写入。多分片长短信仍须在发送下一片前持久化当前分片事件,保持崩溃恢复边界。
|
||||
- 同一批次并发提交结果、最终回执和失败处理触发任务进度刷新时,进程内只允许一个PostgreSQL聚合查询执行;并发触发合并为一次尾随刷新,既避免逐消息并发扫描整个批次,也必须覆盖运行中查询快照之后已经提交的状态变化。消息状态、计费和幂等事实仍以PostgreSQL为准,不得缓存聚合结果替代最终刷新。
|
||||
- 压测使用隔离测试企业、真实PostgreSQL账务、单价`0.0325元/计费条`、三运营商混合号码和六个模拟供应商账号;逐档核对消息、冻结/释放/扣费/退款、SmsBillingRecord、通道分布、队列和数据库。临时单价与主动双活配置须先快照、测试后恢复。
|
||||
|
||||
@@ -0,0 +1,65 @@
|
||||
# CMPP 第四阶段后续性能优化方案(暂停实施)
|
||||
|
||||
更新日期:2026-08-21
|
||||
当前决定:本方案仅作为后续工作依据,本轮不再修改性能代码、不再部署性能补丁、不再继续阶梯压测。
|
||||
|
||||
## 1. 当前结论
|
||||
|
||||
测试环境以客户单价 `0.0325 元/计费条`、三运营商混合号码和六个隔离供应商账号执行最终 100 条/秒档。入口 2999/2999 受理,P50/P95/P99 为 `30/63/96ms`,但从首条消息入队到最后一次供应商提交耗时 `167.991 秒`,完整供应商提交吞吐约 `17.85 条/秒`。负载期一度有 1651 条消息停留在 `submit_queued`,说明瓶颈位于 BullMQ 发送 Worker 到 Gateway 之间的逐消息业务处理,不是六条供应商连接的协议容量。
|
||||
|
||||
## 2. 根因说明
|
||||
|
||||
当前发送 Worker 对每条消息依次执行完整消息查询、运营商/省份识别及回写、应用路由与通道连接查询、签名报备候选查询、最终通道签名二次确认、Redis 通道限速、会话累计更新、提交记录创建、消息状态更新、Gateway BullMQ 写入、Redis Stream 发布和批次进度聚合。
|
||||
|
||||
主要放大点:
|
||||
|
||||
1. 一条客户 CMPP 短信对应一个内部 `batchTaskId`,按任务键进行的进度单飞无法跨 2999 个内部任务合并。
|
||||
2. 路由选择已读取签名报备候选,提交前又对最终通道执行一次报备查询。
|
||||
3. 每条短信都在事务内更新所属通道的 `CmppSubmitSession.submitTotal`;六通道形成六个高频热点行。
|
||||
4. 每条短信分别创建提交记录、更新消息并刷新内部任务,数据库往返数量随消息数线性增长。
|
||||
5. 同一个 Gateway 命令当前同步写 BullMQ 和 Redis Stream;两者的恢复职责需要重新确认,不能未经验证直接删除任一边界。
|
||||
6. Worker 并发高于数据库有效并发时只会增加连接等待;单纯继续提高 `API_SEND_WORKER_CONCURRENCY` 不能解决问题。
|
||||
|
||||
## 3. 后续优化顺序
|
||||
|
||||
### P0:补齐可观测证据
|
||||
|
||||
- 在测试环境启用 `pg_stat_statements`,或为发送热路径增加固定低基数耗时指标。
|
||||
- 分别记录消息加载、号码识别、路由、签名报备、限速、提交事务、Gateway 发布和任务进度耗时。
|
||||
- 同时记录 BullMQ waiting/active/completed、数据库连接池等待、事务耗时和六通道命令供给速率。
|
||||
|
||||
### P1:低风险数据库往返收敛
|
||||
|
||||
1. CMPP 单消息内部任务使用已知状态直接更新计数,不再执行整批 `GROUP BY`。
|
||||
2. 将活跃路由、在线通道和签名报备通过条件合并为一次受数据库事实约束的候选查询,取消第二次等价查询;不使用长期进程缓存替代实时停用/撤销状态。
|
||||
3. 提交事务只复用已存在的开放会话 ID;`submitTotal`改为异步批量累计或从提交记录聚合,不让统计字段锁住真实发送。
|
||||
4. 确认 Gateway BullMQ 与 Redis Stream 的消费、重放和死信职责;若存在重复同步发布,改为单一持久入口加可恢复 Outbox。
|
||||
|
||||
### P2:发送微批处理
|
||||
|
||||
- Worker 在 5~10ms 窗口内领取最多 32 或 64 条任务。
|
||||
- 使用 `WHERE id IN (...)` 批量加载消息,按应用、运营商、签名分组预取静态路由候选。
|
||||
- 在线状态、停用状态和最终报备条件在提交批次内重新核验,不建立跨批长期缓存。
|
||||
- 一次短事务批量创建独立 `SmsSubmitRecord`、更新独立 `SmsMessageRecord`,保留每条消息唯一 `submitId`、幂等键、补发和计费关联。
|
||||
- Gateway 命令使用 Redis pipeline/批量发布;任一部分失败必须能根据 PostgreSQL 事实安全补发,不能重复提交或重复计费。
|
||||
|
||||
### P3:容量参数复核
|
||||
|
||||
- 完成 P1/P2 后再让 Worker 并发与数据库连接预算匹配,初始建议按 24~32 个有效数据库槽验证,不直接扩大到更高并发。
|
||||
- 按 PostgreSQL `max_connections` 为 API、Worker、Gateway 回调和运维连接保留独立余量。
|
||||
- 通道 TPS、窗口和六连接继续独立限速,平台业务吞吐不得绕过供应商配置。
|
||||
|
||||
## 4. 正确性约束
|
||||
|
||||
- 正价短信继续执行真实冻结、接受后扣费、拒绝释放和最终失败退款;不得用单价 0 结果外推计费性能。
|
||||
- 计费、报备、路由、消息状态、重试认领和回执幂等必须保留 PostgreSQL 稳定事实。
|
||||
- 不得用进程内缓存保存余额、号码频控、在线状态或最终报备决策。
|
||||
- 多分片长短信继续保留逐片持久化和聚合结果边界;不能为吞吐恢复重复单分片回调。
|
||||
- 任何非向后兼容迁移必须同时提供 PostgreSQL 与运行源码恢复路径。
|
||||
|
||||
## 5. 后续验收方式
|
||||
|
||||
每轮变更均执行:自动化回归 → 正价 smoke → 50 → 100 → 200 → 300 → 500 条/秒。每档必须核对入口受理、Inbox、BullMQ、Gateway Stream、供应商提交、回执、上行、消息终态、冻结/释放/扣费/退款、数据库锁和连接池。出现丢失、重复、账务不一致、队列持续增长或数据库持续不稳定时立即停止升档。
|
||||
|
||||
500 条/秒只有在入口和完整供应商提交均持续达到目标、队列可在限定时间稳定排空且零丢重、账务恒等式成立时才判定通过。
|
||||
|
||||
@@ -33,6 +33,8 @@ API_PORT=3000
|
||||
API_HOST=127.0.0.1
|
||||
API_METRICS_HOST=127.0.0.1
|
||||
API_METRICS_PORT=9464
|
||||
API_DB_POOL_MAX=32
|
||||
API_WORKER_DB_POOL_MAX=8
|
||||
HTTP_API_MASTER_KEY=<至少32位随机值,用于AES-256-GCM加密HTTP访问凭据和Webhook密钥>
|
||||
HTTP_API_PUBLIC_ORIGIN=https://api.lisglo.com
|
||||
API_ENABLE_SEND_WORKER=true
|
||||
@@ -60,6 +62,12 @@ GATEWAY_CMPP_ADDR=0.0.0.0:17890
|
||||
CMPP_PUBLIC_HOST=8.160.169.106
|
||||
CMPP_PUBLIC_PORT=17890
|
||||
GATEWAY_STARTUP_RECONNECT_DELAY_MS=1000
|
||||
GATEWAY_SUBMIT_WORKER_CONCURRENCY=64
|
||||
GATEWAY_SUBMIT_RESULT_STREAM=gateway.submit.results
|
||||
GATEWAY_SUBMIT_RESULT_GROUP=cmpp-api-callback
|
||||
GATEWAY_SUBMIT_RESULT_CONSUMER=gateway-1
|
||||
GATEWAY_SUBMIT_RESULT_WORKER_CONCURRENCY=8
|
||||
GATEWAY_CMPP_INBOUND_MAX_CONCURRENCY=64
|
||||
OBJECT_STORAGE_DRIVER=minio
|
||||
OBJECT_STORAGE_LOCAL_ROOT=/var/lib/cmpp-platform/object-storage
|
||||
PROD_ADMIN_EMAIL=admin@example.com
|
||||
@@ -79,7 +87,11 @@ PROD_ADMIN_PASSWORD='change-me'
|
||||
|
||||
HTTP容量边界固定为:NestJS普通JSON/URL-encoded请求体`2 MiB`,仅`/api/client/send/imports/*`使用`25 MiB` JSON解析上限,原始CSV/TSV正文继续由业务层限制为`20 MiB`;Gateway读取NestJS API响应最多`4 MiB`且超限必须明确报错。客户文件导入走`sms.lisglo.com`私有API,因此该虚拟主机的`client_max_body_size`必须不低于`30m`,标准bootstrap配置为`50m`。`api.lisglo.com`只承载单条公网HTTP API、Swagger和健康检查,不承载客户文件导入;不要为导入需求开放私有路由或把NestJS所有JSON接口统一放宽到25MiB。发布前使用`nginx -T`确认最终生效值,不能只检查仓库模板。
|
||||
|
||||
Gateway 的最终 TPS 防线依赖与 API 相同的 Redis。通道连接时会写入 `rate:gateway:channel:config:<channelId>` 权威上限,实际预约使用 `rate:gateway:channel:<channelId>`;这些 key 不应在正常发布时清理。多 Gateway 实例必须指向同一 Redis,才能共享单通道额度。超速的 `gateway.submit.commands` 消息会保持在 consumer group pending 中等待,不应通过手工 `XACK` 或删除 Stream 处理积压;先检查通道配置、Redis key、consumer group 和 Gateway 日志。
|
||||
Gateway 的最终 TPS 防线依赖与 API 相同的 Redis。通道连接时会写入 `rate:gateway:channel:config:<channelId>` 权威上限,实际预约使用 `rate:gateway:channel:<channelId>`;这些 key 不应在正常发布时清理。多 Gateway 实例必须指向同一 Redis,才能共享单通道额度。超速的 `gateway.submit.commands` 消息会保持在 consumer group pending 中等待,不应通过手工 `XACK` 或删除 Stream 处理积压;先检查通道配置、Redis key、consumer group 和 Gateway 日志。V2起Submit Worker使用持续补位有界池并逐条ACK,`GATEWAY_SUBMIT_WORKER_CONCURRENCY`缺省64、最大1024;调整前必须同时核对供应商连接数、窗口、TPS限制、Gateway RSS和`cmpp_gateway_submit_worker_slots`,不能用放大并发绕过通道限速。
|
||||
|
||||
V3起客户CMPP入站Submit按应用`cmppWindowSize`在单连接内受限并发,全局上限`GATEWAY_CMPP_INBOUND_MAX_CONCURRENCY`缺省64、最大1024。发布后必须先以已认证测试连接确认`cmpp_gateway_inbound_submit_slots{state="configured"}`等于应用有效窗口,再执行阶梯压测;不得通过调高全局值绕过应用窗口,也不得在未验证Sequence_Id关联、心跳/ACK活性和断线清理时直接提高生产并发。
|
||||
|
||||
V4起供应商分片和聚合结果写入`gateway.submit.results`幂等Outbox,由默认8槽、最大1024的独立回调Worker上报API。聚合结果XADD与原命令XACK由Redis Lua原子执行;回调成功后结果事件XACK+XDEL,失败事件留在PEL。发布时必须保留同一Redis数据和AOF,不得清理`gateway.submit.results`、其consumer group或`gateway.submit.results:dedupe:*`;调整`GATEWAY_SUBMIT_RESULT_WORKER_CONCURRENCY`前须核对API/PostgreSQL承载能力。第91条向前兼容migration增加`SmsSubmitRecord.resultEventId/resultProcessedAt`,用于回调跨重启幂等,不删除历史字段或数据。
|
||||
|
||||
服务重启顺序必须是 Gateway 在前、API 在后。API 启动后等待 `GATEWAY_STARTUP_RECONNECT_DELAY_MS`(默认 1 秒),从 PostgreSQL 读取全部 active 通道并重新下发真实连接命令,同时恢复 Gateway 内存连接池和 Redis 权威 TPS key;禁止沿用数据库中重启前的 connected 状态冒充当前连接。
|
||||
|
||||
@@ -141,9 +153,13 @@ curl http://127.0.0.1:8090/health
|
||||
curl http://127.0.0.1:12026/
|
||||
redis-cli -h 127.0.0.1 -p 6379 ping
|
||||
pg_isready -d "$(grep '^DATABASE_URL=' /etc/cmpp-platform/cmpp-platform.env | cut -d= -f2-)"
|
||||
grep -E '^(API_ENABLE_SEND_WORKER|API_SEND_WORKER_CONCURRENCY)=' /etc/cmpp-platform/cmpp-platform.env
|
||||
grep -E '^(API_ENABLE_SEND_WORKER|API_SEND_WORKER_CONCURRENCY|GATEWAY_SUBMIT_WORKER_CONCURRENCY|GATEWAY_SUBMIT_RESULT_WORKER_CONCURRENCY|GATEWAY_CMPP_INBOUND_MAX_CONCURRENCY)=' /etc/cmpp-platform/cmpp-platform.env
|
||||
grep -E '^(CMPP_INBOUND_FAST_PATH_ENABLED|CMPP_INBOUND_WORKFLOW_WORKER_ENABLED|API_INBOUND_WORKFLOW_CONCURRENCY|API_INBOUND_WORKFLOW_BATCH_ENABLED|API_INBOUND_WORKFLOW_BATCH_SIZE|API_INBOUND_WORKFLOW_POLL_INTERVAL_MS|API_INBOUND_WORKFLOW_STALE_SECONDS|API_DB_POOL_MAX|API_WORKER_DB_POOL_MAX|API_WORKER_METRICS_PORT)=' /etc/cmpp-platform/cmpp-platform.env
|
||||
systemctl is-active cmpp-api cmpp-send-worker cmpp-gateway
|
||||
curl -fsS http://127.0.0.1:9465/metrics | grep '^cmpp_worker_inbound_workflow_'
|
||||
redis-cli --scan --pattern 'rate:gateway:channel:*'
|
||||
redis-cli XINFO GROUPS gateway.submit.commands
|
||||
redis-cli XINFO GROUPS gateway.submit.results
|
||||
```
|
||||
|
||||
## 回滚
|
||||
@@ -163,3 +179,11 @@ bash tools/deploy/production-deploy.sh
|
||||
```
|
||||
|
||||
3. 如迁移造成不可兼容故障,先停服务,再恢复数据库备份。
|
||||
|
||||
## CMPP耐久Inbox与独立Worker发布门禁(2026-08-20)
|
||||
|
||||
- 发布前恢复资产除PostgreSQL、运行源码和环境文件外,必须包含`cmpp-api.service`、`cmpp-send-worker.service`及其drop-in;逐项校验`pg_restore --list`、tar可读性和SHA-256。数据库回滚与运行代码必须成套执行,禁止只回退代码后让旧Prisma Client访问新状态机。
|
||||
- 环境必须显式启用`CMPP_INBOUND_FAST_PATH_ENABLED=true`、`CMPP_INBOUND_WORKFLOW_WORKER_ENABLED=true`,并给出正整数`API_INBOUND_WORKFLOW_CONCURRENCY`、`API_DB_POOL_MAX`和`API_WORKER_DB_POOL_MAX`;推荐初始值分别为32槽、API池32、Worker池8、轮询100ms、租约300秒。API systemd角色必须是`api`,Worker角色必须是`worker`;Worker可通过`API_WORKER_DATABASE_URL`使用独立地址,未配置时仍使用同一数据库地址但保持独立进程和有界连接池。发布前必须核对两池之和、其他服务连接与运维余量不超过PostgreSQL`max_connections`。
|
||||
- Worker日志目录归`cmpp-api:cmpp-security`且仅服务可写,9465只监听回环并加入Prometheus `cmpp-send-worker` target。发布后必须验证两进程均为非root、API/Gateway health、Worker metrics、PostgreSQL/Redis,以及Inbox pending/processing/最老等待可观测。
|
||||
- 回滚前先停止Gateway、API和Worker,保留故障现场Inbox及日志;如恢复旧数据库备份,必须同时恢复对应源码和环境/systemd资产。不得在回滚时删除pending Inbox或重投真实短信。
|
||||
- 第三阶段要求环境显式设置`API_INBOUND_WORKFLOW_BATCH_ENABLED=true`和正整数`API_INBOUND_WORKFLOW_BATCH_SIZE`,初始建议64且不得大于Worker业务槽的可解释倍数。批次增大前必须核对PostgreSQL参数数量、单事务持续时间、Worker RSS和租约时长;付费短信仍走逐条账务锁,不能用零计费批量结果替代付费链路验收。
|
||||
|
||||
@@ -4664,10 +4664,57 @@ npm run verify:phase8
|
||||
| TC-INFRA-MON-026 | 高基数和敏感字段防护 | 检查API/Gateway/Exporter全量metrics文本及Prometheus label names/values | 不存在手机号、短信正文、message/submit/task/channel实体ID、凭据、原始URL或SQL文本 |
|
||||
| TC-INFRA-MON-027 | Recording Rules查询收敛 | 刷新系统监控页并检查Prometheus请求 | 服务卡片只读取`cmpp:service_*`固定聚合,不按卡片开放任意PromQL;缺失指标显示“待采集” |
|
||||
| TC-INFRA-MON-028 | 监控开销对比 | 在同等请求压力下对比开启前后API/Gateway CPU、RSS、P95和吞吐 | 无高基数增长、无业务PostgreSQL高频写入;开销超出预算时暂停发布并调整采集/桶配置 |
|
||||
| TC-CMPP-PERF-OBS-001 | API入站分段耗时 | 在隔离测试环境提交覆盖成功、同步拒绝、长短信分片和异常的CMPP Submit,抓取API回环metrics | 输出固定`cmpp_api_cmpp_inbound_stage_duration_seconds`直方图;查询、分片、预检、模板、各持久化、检测、风控频次、计费、入队、完整提交及总耗时按实际路径增长,结果仅为`success/error` |
|
||||
| TC-CMPP-PERF-OBS-002 | Gateway入站分段耗时 | 在隔离测试环境发送合法与非法Submit并抓取Gateway回环metrics | `decode/api_roundtrip/response_write/handler_total`分别增长;失败路径也记录对应阶段,不因指标异常吞掉原协议错误 |
|
||||
| TC-CMPP-PERF-OBS-003 | Gateway供应商下发分段耗时 | 在隔离测试环境构造Stream等待、限速等待、连接窗口等待、供应商慢响应和API慢回调 | `stream_wait/rate_limit_wait/connection_wait/supplier_rtt/api_callback`可独立区分;`supplier_rtt`在API回调变慢时不等量增长 |
|
||||
| TC-CMPP-PERF-OBS-004 | 观测标签边界 | 检查API/Gateway新增指标文本和Prometheus时序标签 | 只出现固定`stage/result/le`;不得出现手机号、企业/应用/通道/连接/消息/Submit/任务ID、短信正文或凭据,未知阶段不生成时序 |
|
||||
| TC-CMPP-PERF-OBS-005 | 纯观测语义回归 | 对比启用埋点前后的同一组CMPP Submit结果、数据库记录、扣费冻结、队列命令及回执 | SubmitResp状态、Msg_Id、多号码独立记录、同步拒绝、异步回执、幂等键和业务调用顺序均不改变;埋点不写PostgreSQL/Redis |
|
||||
| TC-CMPP-PERF-V2-001 | 持续补位无批次屏障 | 工作池并发设为2,先投递一个阻塞任务和一个快速任务,再投递第三个任务 | 快速任务结束后第三个任务立即开始,不等待第一个慢任务结束;读取批次不形成整批`Wait`屏障 |
|
||||
| TC-CMPP-PERF-V2-002 | 单消息独立ACK | 同一批次投递一快一慢两条消息,慢任务保持在供应商等待 | 快任务完成后Redis PEL立即只剩慢任务;不得等慢任务结束后整批ACK,也不得在供应商结果回传前提前ACK |
|
||||
| TC-CMPP-PERF-V2-003 | 全局并发边界 | 分别配置并发1、64、1024和大于1024的值,持续投递超过槽位数的消息 | 同时处理数不超过有效配置;缺省为64,大于1024按1024执行,空闲槽位持续补充 |
|
||||
| TC-CMPP-PERF-V2-004 | Pending恢复去重 | 让一个供应商调用超过`MinIdle`,同时触发`XAUTOCLAIM`扫描 | 同一进程检测到相同Stream消息ID仍在处理时不重复Submit;原任务结束后按自身结果ACK或进入既有失败/死信流程 |
|
||||
| TC-CMPP-PERF-V2-005 | Worker槽位指标 | 工作池空闲、部分占用和满载时抓取Gateway metrics | `cmpp_gateway_submit_worker_slots`的`configured/in_flight`与真实配置和在途数一致,不包含通道、消息或客户标识 |
|
||||
| TC-CMPP-PERF-V2-006 | 失败、死信和重启兼容 | 构造提交失败至最大次数、畸形命令、进程重启后的pending恢复 | 失败次数、死信上报、独立ACK和failure hash清理保持既有语义;重启不丢消息、不把未完成消息误报成功 |
|
||||
| TC-CMPP-PERF-V2-007 | 隔离环境阶梯持续压测 | 在供应商模拟器、真实API/PostgreSQL/Redis/Gateway链路中,先做100条受控突发,再依次执行10、20、30、40、50条/秒各60秒;每档等待Stream排空并核对数据库、模拟器和Prometheus | 每档SubmitResp拒绝和连接错误为0,Stream最终`pending=0/lag=0`,无死信或服务异常;任一档SubmitResp P95超过5秒、Stream持续增长或服务异常时立即停止升档并保留该档证据,不把未执行档位记为通过 |
|
||||
| TC-CMPP-PERF-V3-001 | 同连接窗口内并发 | 应用窗口设为2,让第1个Submit的API处理阻塞,再发送第2个Submit | 第2个请求无需等待第1个完成即可进入API并按自身Sequence_Id先返回SubmitResp;释放第1个后仍返回其原Sequence_Id |
|
||||
| TC-CMPP-PERF-V3-002 | 应用窗口与全局上限 | 分别设置应用窗口1、32、2048及Gateway全局上限16、64 | 窗口1保持串行;有效并发为`min(应用窗口, Gateway上限, 1024)`,超过窗口时停止继续读取形成背压,不产生无界协程 |
|
||||
| TC-CMPP-PERF-V3-003 | 非Submit协议活性 | 在多个慢Submit在途时发送ActiveTest并接收Deliver ACK | 心跳和ACK仍可被读取和处理,不因业务Submit串行处理而超时;登录必须在任何并发Submit前串行完成 |
|
||||
| TC-CMPP-PERF-V3-004 | 断线在途清理 | Submit进入API后由客户端断开TCP,随后让API处理完成 | Gateway等待已接受处理收尾后再执行连接关闭回调;会话、Submit barrier和消息映射最终清理,不重新出现幽灵连接,不发生panic |
|
||||
| TC-CMPP-PERF-V3-005 | 入站槽位聚合指标 | 建立不同窗口的测试连接并制造部分在途Submit后抓取metrics | `cmpp_gateway_inbound_submit_slots`的configured等于在线连接有效窗口合计、in_flight等于当前业务处理数;只有固定state标签 |
|
||||
| TC-CMPP-PERF-V3-006 | V3阶梯容量复测 | 在与V2相同的隔离真实后端/数据库/Redis/供应商模拟器中执行10、20、30、40、50条/秒各60秒 | 与V2按同口径比较SubmitResp P50/P95/P99、API阶段、Stream pending/lag、供应商吞吐和资源;遇P95超过5秒、持续积压或服务异常立即停止,未执行档位不记为通过 |
|
||||
| TC-CMPP-PERF-V4-001 | 分片结果幂等入Outbox | 对同一submitId和segmentIndex重复发布两次分片结果 | `gateway.submit.results`只新增一个确定性eventId事件;下一分片仅在前一分片Outbox写入成功后发送 |
|
||||
| TC-CMPP-PERF-V4-002 | 聚合结果与命令ACK原子性 | 让供应商返回成功,在聚合结果写入与命令ACK边界注入Redis失败并重启Gateway | 不存在“命令已ACK但结果Outbox缺失”状态;重试同一发布脚本不会产生重复聚合事件 |
|
||||
| TC-CMPP-PERF-V4-003 | API回调不占供应商槽 | API submit-result接口延迟10秒,同时持续让供应商快速返回SubmitResp | 供应商Worker槽在结果写入Outbox后立即释放;API延迟只增加独立回调Worker和Outbox积压,不降低供应商Submit槽可继续补位的能力 |
|
||||
| TC-CMPP-PERF-V4-004 | 回调失败恢复与逐条ACK | 结果回调第一次返回503,随后恢复201并重启回调Worker | 失败事件保留PEL且未删除;恢复后重新投递,成功时单事件原子ACK+删除,其他事件不受整批等待 |
|
||||
| TC-CMPP-PERF-V4-005 | API持久幂等 | 对同一聚合eventId重复回调两次,并检查提交记录、计费、余额、重试和任务进度 | `SmsSubmitRecord.resultEventId`只记录一次;第二次直接返回当前结果,不重复扣费、释放、补发或推进状态;不同eventId占用同一submit尝试时拒绝 |
|
||||
| TC-CMPP-PERF-V4-006 | Outbox有界指标 | 制造回调在途和积压后抓取Gateway metrics | 回调Worker configured/in_flight与真实槽位一致,Outbox pending/lag与Redis consumer group一致,指标不含手机号、消息、submit、企业、应用或通道标签 |
|
||||
| TC-CMPP-PERF-V4-007 | 双Stream发布后排空 | 完成一档隔离压测并等待异步处理结束 | `gateway.submit.commands`和`gateway.submit.results`均为`pending=0/lag=0`,结果Outbox成功事件已删除,无Gateway Submit死信;数据库业务数与客户端完全一致 |
|
||||
| TC-CMPP-PERF-V4-008 | V4阶梯容量复测 | 在V3相同隔离供应商模拟器与真实API/PostgreSQL/Redis/Gateway中依次执行10、20、30、40、50条/秒各60秒 | 对比V3的SubmitResp分位、供应商RTT、命令Stream、结果Outbox、API阶段和资源;任一档出现拒绝、连接错误、P95超过5秒、双Stream持续增长或服务异常立即停止升档 |
|
||||
| TC-CMPP-PERF-V5-001 | 入站应用快照复用 | 分别提交单号码、多号码和完整长短信,并统计`SmsApplication`查询 | 每次Submit入口只查询一次账号对应应用、企业和IP白名单;全部目标号码复用该已校验快照,应用/IP/模板/风控业务结果不变 |
|
||||
| TC-CMPP-PERF-V5-002 | 默认规则完整性检查单飞 | 同一API实例并发触发任务风控和号码频控,再在30秒内重复提交 | 并发调用共用一个检查Promise;默认5条规则完整时只执行一次聚合count,不再逐消息产生两轮各5次存在性查询;实际生效规则和频控状态仍逐消息读取 |
|
||||
| TC-CMPP-PERF-V5-003 | 默认规则缓存失效与恢复 | 让聚合检查失败后重试;另在缓存期后模拟缺失一条默认规则 | 检查失败立即清除缓存并允许下次重试;短TTL内只缓存完整性,TTL后发现缺失规则并按既有创建校验恢复,不缓存应用规则内容或业务判断 |
|
||||
| TC-CMPP-PERF-V5-004 | 已持久化单消息快速入队 | 创建CMPP单号码内部任务和消息,记录已知taskId、messageRecordId、queuePriority后进入队列 | BullMQ jobId仍为messageRecordId、attempts和优先级不变;不再回查刚创建的任务和消息,任务最终更新为queued;普通批量任务原通用入队路径保持兼容 |
|
||||
| TC-CMPP-PERF-V5-005 | V5隔离环境阶梯复测 | 发布到虚拟机测试环境后,以V4相同模拟器、连接数、8槽Outbox和10/20/30/40/50条每秒阶梯执行 | 对比V4的SubmitResp分位及`application_lookup/risk_frequency/queue_publish/complete_submit`阶段;数据库业务数、冻结/计费、双Stream排空和回执幂等保持一致,遇拒绝、连接错误、P95超过5秒或持续积压立即停止 |
|
||||
|
||||
执行记录(2026-08-20):`TC-CMPP-PERF-V5-001`至`004`通过本地API全量回归;`005`在`100.93.204.60`隔离测试环境完成。10/20/30/40条每秒均零拒绝、零连接错误且双Stream排空,判定通过;50条每秒缺8个SubmitResp且结束时命令Stream仍有`pending=64/lag=1407`,判定失败并停止升压。30与40条每秒真实积压下,优先任务排队P95分别为0.988秒、3.916秒,普通任务为26.282秒、58.536秒,优先级隔离通过;该结论仅覆盖移动号段,联通/电信六通道仍待P1复测。
|
||||
| TC-GLOBAL-ALERT-001 | 铃铛分域预警菜单 | 准备签名清退未读消息和安全待处置告警后点击右上角铃铛 | 弹层分开显示“签名清退预警”和“安全检测与封禁”,分别展示真实数量和摘要,角标等于两项之和 |
|
||||
| TC-GLOBAL-ALERT-002 | 预警菜单跳转 | 分别点击铃铛中的两个菜单项 | 签名项跳转`/admin/signature-retirement`,安全项跳转`/admin/security-detection`,弹层关闭且对应页面读取真实后端数据 |
|
||||
| TC-GLOBAL-ALERT-003 | 域间故障隔离与轻量轮询 | 分别让一个汇总接口失败并观察30秒轮询请求 | 失败域显示0且另一域数据保留;安全预警使用专用汇总接口,不调用完整overview、规则、代理状态或告警大列表 |
|
||||
| TC-DEPLOY-NET-001 | API回环监听边界 | 使用标准生产环境启动API,执行`ss -lnt`并从LAN/Tailscale探测3000端口,同时经Nginx业务入口请求健康接口 | API仅监听`127.0.0.1:3000`,外部不能直连3000;Nginx入口仍正常返回真实API健康结果;部署静态门禁校验`API_HOST`默认值与启动参数一致 |
|
||||
|
||||
## CMPP第三阶段业务批处理专项(2026-08-21)
|
||||
|
||||
| 用例ID | 场景 | 步骤 | 预期 |
|
||||
| --- | --- | --- | --- |
|
||||
| TC-CMPP-PERF-P3-001 | 有界批次领取 | 制造priority/normal混合Inbox并并发启动两个Worker | 每次领取不超过配置批次/槽位,使用SKIP LOCKED,无重复领取;priority先于normal且类内FIFO |
|
||||
| TC-CMPP-PERF-P3-002 | 批次只读预加载 | 同批放入多应用、多内容正常短短信并统计SQL | 应用、模板、签名、黑名单、风控规则和敏感词按批读取,不逐短信重复;下一批重新读取应用状态 |
|
||||
| TC-CMPP-PERF-P3-003 | 日限额批量原子性 | 在剩余额度边界并发提交并重放相同请求键 | 使用量不超过上限;每条预留决定独立持久化,重放不重复递增,拒绝项走既有失败回执 |
|
||||
| TC-CMPP-PERF-P3-004 | 号码频控批量原子性 | 唯一号码批量提交、同号重复提交并模拟Worker崩溃重领 | 唯一号码批量更新;同号保留顺序语义并逐条处理;重领返回原决定,无穿透、重复计频或重复命中 |
|
||||
| TC-CMPP-PERF-P3-005 | 三表与队列崩溃恢复 | 在三表提交后、BullMQ发布前注入失败并重领 | 任务/API请求/消息不重复,稳定MessageId对应唯一消息;以MessageId Job ID补入队后Inbox独立完成 |
|
||||
| TC-CMPP-PERF-P3-006 | 异常与付费回退 | 覆盖正单价、黑名单、非法号、人工审核、引流歧义和已有消息 | 全部走原逐条状态机,余额冻结和回执语义不变;不得进入零计费批量快路径 |
|
||||
| TC-CMPP-PERF-P3-007 | 500条/秒阶梯 | 在100.93.204.60依次执行smoke、100/200/300/500并逐级检查数据库、Redis和日志 | 任一级拒绝、缺响应、持续积压、锁等待或对账不一致立即停止;分别报告入口、Inbox完成和完整供应商链速率,不用积压冒充吞吐 |
|
||||
|
||||
执行记录(2026-08-21):`TC-CMPP-PERF-P3-001/002/003/004/005`已通过本地专项和测试机正常批次对账;`006`保留既有逐条回归通过,正单价吞吐未外推;`007`的smoke通过,100条/秒入口及Inbox对账通过,但命令Stream在注入结束时`pending=128/lag=922`且完整下游排空约87秒,按停止线判完整链失败并停止200/300/500档。
|
||||
## Fail2ban 安全检测与人工封禁测试矩阵(2026-08-14)
|
||||
|
||||
- 本模块必须执行 `docs/fail2ban-assisted-blocking-test-cases-20260814.md` 中 TC-F2B 全量用例,专项用例是本平台功能测试的组成部分,不是可选附录。
|
||||
@@ -4693,3 +4740,73 @@ npm run verify:phase8
|
||||
| TC-INFRA-MON-037 | 已读用户隔离 | 管理员A标记已读后由管理员B查看同一告警 | 管理员B仍显示未读且铃铛数量不减少,管理员A的状态保持已读 |
|
||||
| TC-INFRA-MON-038 | 同告警重新触发 | 标记已读后让告警恢复,再以相同标签重新触发并产生新 activeAt | 新触发记录重新显示“标记已读”,计入预警中心;旧 activeAt 不会永久屏蔽同指纹告警 |
|
||||
| TC-INFRA-MON-039 | 过期与幂等 | 重复提交同一活动告警,再提交已恢复或 activeAt 不匹配的请求 | 同一次告警重复提交幂等;过期/不匹配请求返回404且不生成虚假已读记录;操作日志可追溯 |
|
||||
|
||||
## CMPP 500条/秒第一阶段:耐久Inbox快路径(2026-08-20)
|
||||
|
||||
| 用例ID | 场景 | 步骤 | 预期 |
|
||||
| --- | --- | --- | --- |
|
||||
| TC-CMPP-500-P1-001 | 快速耐久受理 | 开启快路径提交合法短消息,并在风控/计费/队列依赖可观测时检查调用顺序 | SubmitResp在一条Inbox事实提交后返回`accepted_pending`;响应前不调用风控、频控、计费或BullMQ |
|
||||
| TC-CMPP-500-P1-002 | 请求幂等 | 在同一已鉴权连接重投相同Sequence_Id与载荷 | Gateway请求键稳定,数据库只保留一条Inbox;两次返回相同MessageId,不重复创建任务、消息或冻结 |
|
||||
| TC-CMPP-500-P1-003 | 幂等冲突 | 使用同一请求键提交不同载荷 | API拒绝冲突且不覆盖原Inbox/响应 |
|
||||
| TC-CMPP-500-P1-004 | 多号码与长短信 | 分别提交多号码Submit和完整长短信分片 | Inbox保留全部号码及稳定子MessageId;长短信Worker处理的是完整重组正文,不是最后一片正文 |
|
||||
| TC-CMPP-500-P1-005 | Worker领取与逐条完成 | 启动两个Worker并制造至少一个批次积压 | `SKIP LOCKED`领取不重复;每条独立完成,处理逻辑不占用领取事务 |
|
||||
| TC-CMPP-500-P1-006 | 崩溃恢复与时区 | 数据库会话使用Asia/Shanghai,在领取后终止Worker,超过租约后重启 | UTC无时区列按UTC时钟比较,退避不会立即重领;processing记录被回收并完成,日限Date值合法,频控、冻结、任务和消息均不重复 |
|
||||
| TC-CMPP-500-P1-007 | 异步业务拒绝 | 让已耐久受理短信命中真实模板/风控/余额拒绝 | SubmitResp仍表示已接收;后台生成真实失败状态和失败回执,不进入供应商发送 |
|
||||
| TC-CMPP-500-P1-008 | 优先级 | 在普通Inbox积压期间持续混入priority应用 | priority先领取且类内FIFO;普通队列最终可排空;同时记录两类等待分位数 |
|
||||
| TC-CMPP-500-P1-009 | 进程隔离 | 检查systemd、进程、连接和指标端口 | API角色不运行发送后台任务;`cmpp-send-worker`独立非root运行,指标仅监听127.0.0.1:9465 |
|
||||
| TC-CMPP-500-P1-010 | 阶梯压测与对账 | 隔离供应商环境按既定同口径阶梯执行,压后等待全链排空 | 逐档报告SubmitResp、Inbox、两条Stream、最终数据库计数和排空时间;任何丢响应、重复、错误或未排空均失败,不发送真实短信 |
|
||||
| TC-CMPP-500-P1-011 | Gateway API连接复用与流量隔离 | 将入站全局窗口设为48,后台协议日志持续写入,并构造Submit专用HTTP客户端 | Submit池每主机最大连接与空闲连接均为48、后台池独立16连接,HTTP总超时仍为10秒;后台日志/回执不得占用Submit连接或造成10秒API超时 |
|
||||
| TC-CMPP-500-P1-012 | API/Worker数据库池隔离 | API与Worker分别配置32/8连接并在Worker积压时持续提交 | 两进程使用各自有界连接池,API受理连接不被Worker抢占;总连接数不超过PostgreSQL上限且压后无`idle in transaction`泄漏 |
|
||||
|
||||
执行记录(2026-08-20,测试环境):P1-001至009已由API/Gateway自动化、真实PostgreSQL迁移和独立Worker恢复验证覆盖;P1-011专用Submit传输隔离后100、200条/秒分别2999/2999、3998/3998成功,零拒绝、零节流、零连接错误。P1-010在500条/秒档失败:测试环境池调优后10秒仅2979条,实际297.9条/秒且节流611次;全部2979条最终完成并排空,但完整链仅约24条/秒。故第一阶段不通过500条/秒总目标,不执行“完整500条/秒已达标”的结论。
|
||||
|
||||
### 完整处理500条/秒第二阶段
|
||||
|
||||
| 编号 | 场景 | 操作 | 预期 |
|
||||
|---|---|---|---|
|
||||
| TC-CMPP-500-P2-001 | 合并校验与Inbox写入 | 普通短短信走快路径并检查SQL与阶段指标 | 使用账号唯一索引在同一SQL校验应用/企业/接口/IP/Src_Id并幂等插入;正常请求仅一次DB往返且不再单独记录`application_lookup` |
|
||||
| TC-CMPP-500-P2-002 | 合并SQL拒绝安全 | 分别停用应用、停用企业、关闭接口、使用错误IP和Src_Id | 返回对应拒绝且Inbox没有新增;不读取或缓存旧应用快照 |
|
||||
| TC-CMPP-500-P2-003 | 幂等与并发唯一键竞争 | 同载荷重试、冲突载荷重试,并并发提交相同请求键 | 同载荷返回原MessageId;冲突载荷拒绝;并发竞争最多执行一次只读恢复,不发生重复Inbox或无意义更新 |
|
||||
| TC-CMPP-500-P2-004 | Worker批量应用快照 | 同批领取多个相同及不同应用Inbox | 每个领取批次按唯一应用ID一次查询,逐条继续使用匹配快照;应用不匹配时安全退避,不跨租约持有事务 |
|
||||
| TC-CMPP-500-P2-005 | 有界容量配置 | 在测试环境提高Worker、BullMQ、Gateway Submit和Outbox并发并观测连接 | 各池配置值与在途数可观测,PostgreSQL连接低于`max_connections`并无长事务;单通道限速不被绕过 |
|
||||
| TC-CMPP-500-P2-006 | 100→200→500完整链阶梯 | 使用全新测试号段、隔离六通道模拟器,逐档注入并等待全部队列排空 | 每档报告入口实际速率/分位、Inbox完成、各队列峰值与排空、供应商阶段和数据库对账;500档只有入口及完整链均达到500条/秒且零丢重才通过 |
|
||||
|
||||
执行记录(2026-08-20,测试环境):P2-001至004由121项SendChain专项、真实PostgreSQL修复后低负载9/9及100条/秒2998/2998受理覆盖;首次真实执行发现并修复`jsonb_build_object`参数类型错误,该无效轮未写Inbox。P2-005使用API/Worker池48/32及96/96/128/32四类有界业务槽,数据库压后无idle-in-transaction。P2-006在100条/秒入口通过但完整链失败:Inbox约57.5条/秒、双Stream完整排空约22.0条/秒,且后台API回调超时;按停止线未继续200/500,因此第二阶段仍不通过完整500条/秒目标。
|
||||
|
||||
## TC-CMPP-500-P4 正价计费与六通道并行
|
||||
|
||||
| 用例 | 场景 | 预期 |
|
||||
| --- | --- | --- |
|
||||
| TC-CMPP-500-P4-001 | 同企业正价短短信批量入站并重放 | 一次账户余额更新;每短信唯一`:freeze`流水;任务/请求/消息金额正确;重放不重复冻结 |
|
||||
| TC-CMPP-500-P4-002 | 余额加授信小于批次或单条金额 | 不得透支;批次安全回退且只发送余额覆盖的短信 |
|
||||
| TC-CMPP-500-P4-003 | 正价短信Submit接受及Outbox重放 | 一个原子SQL生成released/charged且不取得账户锁;余额净值不重复变化;SmsBillingRecord唯一charged;历史半完成状态仍可锁定恢复 |
|
||||
| TC-CMPP-500-P4-004 | 零价短信Submit接受 | 保留业务结果但无0金额AccountTransaction |
|
||||
| TC-CMPP-500-P4-005 | 两个同优先级非备用通道与一个备用通道 | 稳定weighted分流只覆盖两个主动通道;排除已尝试通道后可安全切换;备用不抢占 |
|
||||
| TC-CMPP-500-P4-006 | 三运营商、六通道、325金额单位阶梯压测 | 每档入口/Inbox/供应商/回执/账务对账一致,双Stream和数据库最终稳定排空;失败即停止升档 |
|
||||
| TC-CMPP-500-P4-007 | priority与normal并发积压 | priority保持明确服务能力且normal最终不饿死 |
|
||||
| TC-CMPP-500-P4-008 | 测试配置治理 | 单价、账户与组项目先快照;临时主动双活和单价测试后恢复;预生产和凭据不变 |
|
||||
| TC-CMPP-500-P4-009 | 单分片与多分片供应商Submit | 单分片只产生一个含segments的聚合Outbox事件;多分片逐片持久化并另有聚合事件,任何分片不丢失 |
|
||||
| TC-CMPP-500-P4-010 | 同一批次并发任务进度刷新 | 并发调用共享一个运行中聚合查询,并在其间有新状态提交时只补一次尾随聚合;最终任务计数与消息终态一致 |
|
||||
|
||||
执行记录(2026-08-21):`P4-001/003/004/005/008/009/010`已由自动化、真实PostgreSQL账务、六通道隔离模拟器及配置回读覆盖;正价smoke通过。`P4-006`在100条/秒档2999/2999入口受理,但最后供应商提交耗时167.991秒,完整链约17.85条/秒,判定失败并按停止线未升200/300/500。`P4-002/007`本轮未追加专项压力场景,保留待测;临时单价、主备和号段规则已恢复,财务流水保留审计。
|
||||
|
||||
主流程回归记录(2026-08-21):停止继续性能优化后,以2条单价325的隔离CMPP短信验证接收、发送、供应商SubmitResp、最终回执和计费,Inbox/Submit/Receipt/Message/Billing均2条闭合,冻结/释放/扣费金额均650。另以实际MessageId注入1条测试机内部Gateway上行事件,上行精确匹配且客户CMPP普通Deliver/ACK最终delivered;Gateway原始CMPP上行解析由全量Go测试和vet通过补充覆盖。临时单价已恢复为0,队列和数据库稳定,允许进入Git发布检查。
|
||||
|
||||
## CMPP发送准入与通道路由专项(2026-08-21)
|
||||
|
||||
| 用例ID | 场景 | 步骤 | 预期 |
|
||||
| --- | --- | --- | --- |
|
||||
| TC-CMPP-GUARD-001 | 签名未审核 | 将隔离应用签名改为待审核后提交1条带该签名短信 | 消息以`SIGNATURE`失败,生成客户失败回执,供应商提交0条 |
|
||||
| TC-CMPP-GUARD-002 | 模板未报备 | 应用启用模板强校验且不存在匹配的已审核模板时提交1条 | 消息以`TEMPLATE`失败,供应商提交0条;`direct_send`应用不应误套用此断言 |
|
||||
| TC-CMPP-GUARD-003 | 余额不足 | 设置正单价并使余额加授信小于本条金额后提交 | 消息以`BALANCE`失败,不冻结成负数、不进入供应商提交 |
|
||||
| TC-CMPP-GUARD-004 | 应用接口关闭 | 关闭企业应用接口并发起CMPP登录/提交 | CMPP登录被拒绝,不能新增入站消息或供应商提交 |
|
||||
| TC-CMPP-GUARD-005 | 应用禁用或删除 | 分别将应用状态置为`inactive`、`deleted`后登录 | 两种状态均在认证阶段拒绝,恢复后可重新登录 |
|
||||
| TC-CMPP-GUARD-006 | 企业禁用或删除 | 分别将企业状态置为`inactive`、`deleted`后由其应用登录 | 两种状态均在认证阶段拒绝,不影响恢复后的应用配置 |
|
||||
| TC-CMPP-GUARD-007 | 单号码频次 | 配置应用级5分钟阈值1并连续向同号提交2条 | 第1条正常发送,第2条以`RISK`拦截;命中记录阈值/实际值正确且第2条供应商提交0条 |
|
||||
| TC-CMPP-GUARD-008 | 签名未在候选通道报备 | 将签名在路由组所有候选通道的运营商报备改为未通过后提交 | 消息以`ROUTE`失败,供应商提交0条;任一已报备在线候选仍存在时不得误拦截 |
|
||||
| TC-CMPP-GUARD-009 | 主通道禁用 | 禁用通道组主通道、保留已报备且在线的备通道后提交 | 新消息不选禁用主通道,自动选择备通道并可最终送达 |
|
||||
| TC-CMPP-GUARD-010 | 通道组全部通道禁用 | 同时禁用组内主备通道后提交 | 消息以`ROUTE`失败,供应商提交0条,不向已禁用连接发送 |
|
||||
| TC-CMPP-GUARD-011 | 主通道拒绝后组内补发 | 让主通道返回非0 Submit结果,组开启补发且备通道在线/已报备 | 首次提交`rejected`;第二次提交指向未尝试备通道并关联`retryOfSubmitRecordId`,接受后按真实回执进入终态 |
|
||||
| TC-CMPP-GUARD-012 | 配置恢复审计 | 每项测试后回读企业、应用、余额、签名、报备、通道、连接和临时规则 | 所有临时配置恢复原值,服务健康、队列无异常状态,操作和测试证据可追溯 |
|
||||
|
||||
执行记录(2026-08-21,测试环境):`TC-CMPP-GUARD-001`至`012`全部通过。入口采用耐久异步受理,因此业务拦截用例的SubmitResp仍可为0;最终结论以本次MessageId对应的消息错误码、供应商提交数及客户失败回执为准。通道组补发实测主通道结果码8、备通道accepted、消息最终delivered;全部临时配置已恢复。
|
||||
|
||||
+188
-1
@@ -1,6 +1,6 @@
|
||||
# 第一版系统化测试进度
|
||||
|
||||
> 环境命名:当前 `8.160.169.106:12026`(Web/API)和 `8.160.169.106:17890`(CMPP 入站)实例统一定义为“预发布环境”。历史记录中涉及该实例的验证、部署和业务页面均按预发布环境理解;`production-deploy.sh`、`NODE_ENV=production` 及正式生产安全/备份规范保留原有技术语义,不代表该实例为正式生产。
|
||||
> 环境命名:`8.160.169.106:12026`(Web/API)和 `8.160.169.106:17890`(CMPP 入站)实例统一定义为“预生产环境”;`100.93.204.60`统一定义为“虚拟机测试环境”或“测试机”。“虚拟机”不得再用于指代预生产。历史记录中涉及这两个实例的验证和部署按其明确IP归属理解;`production-deploy.sh`、`NODE_ENV=production`及正式生产安全/备份规范保留原有技术语义,不代表测试机或预生产为正式生产。
|
||||
|
||||
## 2026-08-12 企业签名弹窗、充值回执、通道列表与金额显示优化(已提交、已部署)
|
||||
|
||||
@@ -3687,3 +3687,190 @@ git diff --check
|
||||
- 监控专项 2 套 8 项、Prisma format/generate、前后端 TypeScript、API 正式编译、Vite 生产构建和 `git diff --check` 通过;Vite 仅保留既有大 chunk 提示。本地真实 PostgreSQL 不可用,`prisma migrate deploy` 返回 Schema engine error,因此没有伪造依赖或把本地 migration 记为通过,migration 已在测试机真实 PostgreSQL 验证。
|
||||
- 最终浏览器控制会话没有可接管的现有标签页,未绕过图形验证码或另行创建登录会话,因此登录后弹窗滚动和按钮视觉点击未伪报为通过;代码样式契约、真实接口、数据库、Prometheus 与测试机部署均已验收,用户刷新测试机现有登录页面即可查看。
|
||||
- 本轮没有发送、补发或重投短信,没有创建后台重投任务,没有修改通道账号、密码、启停状态、企业余额或客户连接;`api/tsconfig.build.tsbuildinfo`、`tsconfig.tsbuildinfo`、`outputs/`和空文件 `=` 继续作为受保护项排除提交。
|
||||
|
||||
# 2026-08-17 CMPP压测优化第一步:纯观测分段(本地未部署)
|
||||
|
||||
- 按首轮压测瓶颈方案先实施V0纯观测,不调整SubmitResp快路径、并发窗口、Stream Worker模型、风控、计费、路由、幂等、事务或供应商回调状态机。API入站增加固定阶段直方图,覆盖应用查询、长短信分片、提交预检、模板匹配、任务/API请求/消息持久化、内容检测、风控与频次、计费、入队、完整提交和总耗时。
|
||||
- Gateway入站增加`decode/api_roundtrip/response_write/handler_total`,供应商下发增加`stream_wait/rate_limit_wait/connection_wait/supplier_rtt/api_callback`。供应商RTT只围绕真实连接Submit往返计时,API回写单独计时;阶段和结果均为代码白名单,指标不含手机号、企业、应用、通道、连接、消息、Submit或任务ID。
|
||||
- API TypeScript正式构建通过;`metrics.service.spec.ts`与`gateway-events.controller.spec.ts`共8项通过。Gateway全量`go test ./... -count=1`及`go vet ./...`通过,4份Redis Stream消息契约与SendChain R10结构门禁通过;上游R7契约按本轮`Manager.Submit/connectionPool.submit`观测边界同步后通过。
|
||||
- 入站R6契约已同步本轮`handleSubmit`实现哈希,但门禁首先被开始前已存在且源码未修改的`authentication.go/authRequest`哈希漂移阻塞;没有为通过本轮门禁而重写或归因该认证声明。`git diff --check`通过,仅输出工作区既有的LF/CRLF提示。
|
||||
- SendChain专项大套件执行120项,其中115项通过;5项走真实BullMQ连接时因本机`127.0.0.1:6379`拒绝连接而超过5秒,测试进程同时留下Redis重连句柄。未启动或伪造Redis,因此本轮不把该套件记为全通过;该阻塞不影响两个独立指标专项和TypeScript正式构建证据,后续应在具备真实Redis的隔离测试环境补跑语义回归。
|
||||
- 本轮未连接预生产或测试虚拟机、未发起压力流量,未发送、补发或重投短信,未修改通道账号、密码、启停状态、企业余额或客户连接;未提交、未推送、未部署。开始前已有的`api/tsconfig.build.tsbuildinfo`、`tsconfig.tsbuildinfo`、`outputs/`及空文件`=`继续保护,不归因于本轮。
|
||||
|
||||
# 2026-08-20 CMPP压测优化V2:持续有界Submit Worker(测试环境已发布;预生产保持现状)
|
||||
|
||||
- 根据首轮压测报告中供应商提交峰值约14.24次/秒、Redis Stream lag峰值684、API平均耗时82.5ms且VM资源未饱和的证据,V2只重构Gateway供应商Submit Worker,不提前实施入站SubmitResp快路径或V3/V4异步Outbox。
|
||||
- 原实现每次`XREADGROUP Count=10`后启动协程并等待整批全部完成,慢任务形成批次屏障。V2改为默认64槽位、最大1024的持续有界工作池:任一任务结束立即按空闲槽位继续领取;成功、终态拒绝或死信均保持单消息独立ACK,失败消息继续留在PEL按既有`MinIdle/MaxFailures`恢复。
|
||||
- Pending恢复对本进程在途消息ID去重,并在`XAUTOCLAIM`返回后回查真实PEL,避免原任务恰好ACK时的竞态重复Submit;该处注释解释了为什么必须同时检查内存活动集合和Redis事实。新增`GATEWAY_SUBMIT_WORKER_CONCURRENCY`及`cmpp_gateway_submit_worker_slots{state=configured|in_flight}`,不增加实体标签。
|
||||
- Worker专项连续20轮通过;Gateway全量`go test ./... -count=1`、`go vet ./...`、4份Stream契约、R6/R7与SendChain R10结构门禁通过。R6额外将当前HEAD中未被本轮修改的认证/Server/HTTP声明哈希与契约重新对齐,不归因于V2业务修改。
|
||||
- 使用仅监听`127.0.0.1:16379`的本地真实Redis 8.8补跑API观测与SendChain回归,3套121项全部通过;测试后临时Redis已确认停止。Jest仍按仓库既有`--forceExit`口径收尾异步句柄。
|
||||
- 2026-08-20发布前核对:本地`HEAD=origin/main=c4f36fc50d7906dfb2f97c881e9ea43c6a64c370`;本段操作目标实际为预生产`8.160.169.106`,其`/opt/cmpp-platform/.deployed-commit=433b2ee56f6016ad8afff1bac73f510b8fd53083`。此前记录中将该目标写成“虚拟机”属于环境称谓错误,现明确更正为“预生产”;API/Gateway/PostgreSQL/Redis/MinIO/Nginx均active。用户要求预生产保持当前状态,不执行回退或后续测试环境发布动作。
|
||||
- 预生产发布前恢复资产位于`/opt/cmpp-platform-backups/releases/20260820-100554-before-v2-worker`,包含PostgreSQL、运行源码、环境文件和平台配置;`SHA256SUMS`、数据库gzip及源码/配置tar均验证通过。精确发布包`outputs/cmpp-v2-worker-20260820-101239.tar.gz`共923项、2854923字节,本地及预生产服务器SHA-256均为`cd7bb8d05e7bf9a77ced4522948e0f00b8f25f6b5477eb93037e2a0000b7da5d`,排除了`.env`、`node_modules`、`outputs`、`*.tsbuildinfo`和空文件`=`。
|
||||
- 标准全量发布完成前端/API/Gateway构建并将数据库从87条推进到90条migration,但在任何服务重启前被系统安全代理安装闸门阻断:Aliyun Linux 4当前启用仓库没有`fail2ban`包。未擅自增加第三方系统源;随后从恢复资产重新构建并恢复原`433b2ee`的API与前端运行产物,仅重启Gateway启用V2。因此API/前端仍为原运行版本,数据库保留3张向前兼容的监控/安全增量migration,当前运行标识明确写为`433b2ee56f6016ad8afff1bac73f510b8fd53083+gateway-v2.cd7bb8d05e7b`,不伪装为全量工作区已发布。
|
||||
- Gateway于北京时间10:27:17重启成功,新PID监听17890,回环健康检查通过;`cmpp_gateway_submit_worker_slots{state="configured"}=64`、`in_flight=0`,`gateway.submit.commands`消费者1、`pending=0`、`lag=0`。原4条真实下游连接受90秒旧心跳租约影响,首次重连被连接上限拒绝并出现2次连接协程`close of closed channel` panic;Gateway进程未退出,旧租约自动过期后4条连接全部恢复,PostgreSQL心跳持续更新。该重启恢复现象作为后续独立稳定性缺陷保留,不通过手工修改客户连接规避。
|
||||
- 预生产发布后观察到1条既有真实Stream命令由新Worker接受:总提交耗时68.288ms、Stream等待0.927ms、限速等待0.168ms;供应商RTT样本与API回调样本也已按新指标拆分,处理后PEL和lag均为0。随后30秒被动窗口没有新Stream命令,无法形成V2容量/TPS结论;本轮没有主动发压、发送、补发或重投短信。预生产6项核心服务保持active,Gateway RSS约18MB;完整容量复测必须在`100.93.204.60`虚拟机测试环境使用已确认隔离的供应商模拟器执行。
|
||||
- 测试环境发布结果:用户明确指定`100.93.204.60`为V2发布目标,并再次确认该地址才是“虚拟机测试环境”。取得用户提供的`hector`账号授权后完成只读预检:Ubuntu 24.04、原运行标识`482f7ac1ae4c219e47aeaac8735c0584b7d120f2`、90条migration、6/6测试供应商连接、下游连接0、Stream `pending=0/lag=0`。发布前恢复资产位于`/opt/cmpp-platform-backups/releases/20260820-104529-before-v2-worker-test`,包含PostgreSQL、运行源码、环境及平台配置,全部SHA-256、gzip和tar校验通过。
|
||||
- 测试机使用同一精确归档`cmpp-v2-worker-20260820-101239.tar.gz`,SHA-256再次核对为`cd7bb8d05e7bf9a77ced4522948e0f00b8f25f6b5477eb93037e2a0000b7da5d`;标准`production-deploy.sh`完成两套依赖安装、安全缓解门禁、Prisma generate/migrate、前端/API/Gateway与安全代理构建、Fail2ban配置测试、Nginx校验、服务重启及健康检查,90条migration无待应用项。测试机运行标识为`c4f36fc50d7906dfb2f97c881e9ea43c6a64c370+workspace.v2.cd7bb8d05e7b`。
|
||||
- 发布后API、Gateway、安全代理、PostgreSQL、Redis、MinIO、Nginx、Prometheus及四类Exporter均active;API/Gateway健康、前端HTTP 200,3000/8090/9464继续只监听回环,17890按测试CMPP入口监听。Prometheus真实返回8个target为`up`;Gateway测试供应商连接6/6、下游连接0,`cmpp_gateway_submit_worker_slots{state="configured"}=64`、`in_flight=0`,Stream消费者1、`pending=0`、`lag=0`。发布时间窗API/Gateway/安全代理journal无warning,API/Gateway文件日志无新增ERROR/Exception/panic/fatal。本次只发布和只读验证,没有主动发送、补发或重投短信,也没有修改测试通道账号、密码、启停状态、余额或客户连接。
|
||||
- 代码保持未提交、未推送。开始前已有的`api/tsconfig.build.tsbuildinfo`、`tsconfig.tsbuildinfo`、`outputs/`及空文件`=`继续保护;本轮生成的发布包位于既有`outputs/`目录,不扩大提交范围。
|
||||
|
||||
# 2026-08-20 CMPP压测优化V2:测试环境阶梯复测
|
||||
|
||||
- 仅对`100.93.204.60`虚拟机测试环境执行隔离压测;供应商端为本地CMPP模拟器`100.91.249.119:17900`,没有连接或改变预生产`8.160.169.106`,没有发送真实短信,也没有修改通道账号、密码、启停状态、企业余额或客户连接。测试环境运行标识复核为`c4f36fc50d7906dfb2f97c881e9ea43c6a64c370+workspace.v2.cd7bb8d05e7b`,API、Gateway、PostgreSQL和Redis最终均为active。
|
||||
- 按停止线先执行100条受控突发,再执行10条/秒和20条/秒各60秒。100条突发全部收到成功SubmitResp、无拒绝和连接错误,但P50/P95/P99为2972/5555/5808ms,因此不直接跳到高档;10条/秒实际599条,SubmitResp 599/599成功、无拒绝和连接错误,P50/P95/P99为35/268/1317ms;20条/秒实际1199条,SubmitResp 1199/1199成功、无拒绝和连接错误,P50/P95/P99升至1110/5580/7767ms。20条/秒P95超过5秒停止线,因此没有执行30/40/50条/秒,未把未执行档位记为通过。
|
||||
- V2 Worker目标已获得正向证据:10条/秒阶段Stream最大pending 15、lag 0、最大在途15,结束后排空;20条/秒阶段最大pending 43、lag 0、最大在途43,结束后同样`pending=0/lag=0`。20条/秒窗口供应商Submit尝试约1290次,约21.5次/秒,明显高于首轮旧实现约14.24次/秒且没有重现lag 684;供应商阶段平均`stream_wait≈3.55ms`、`rate_limit_wait≈0.15ms`、`connection_wait≈0.006ms`、`supplier_rtt≈149.16ms`。测试窗口主机CPU峰值约53.87%、内存使用峰值约27.49%,不是资源饱和。
|
||||
- 新瓶颈位于客户入站SubmitResp路径而不是Redis Stream Worker。1898条业务消息的API分段平均总耗时约256.20ms,其中`risk_frequency≈95.73ms`最大,其后为`queue_publish≈37.69ms`、两次`application_lookup`合计约32.62ms、`template_match≈28.91ms`、`submission_precheck≈17.30ms`;`complete_submit`平均约239.41ms。20条/秒时客户端P95达到5.58秒而API平均仍为0.256秒,结合单连接窗口表明入站连接串行处理/排队仍在放大尾延迟,下一步应实施受窗口约束的连接内并发,或把SubmitResp收敛为“最小校验+幂等持久化”后异步执行风控、计费和路由。
|
||||
- 数据库按实际首条`queuedAt=2026-08-20 03:01:47.980`对账,恰好新增1898条:最终delivered 1859、failed 11、submitted 28;28条submitted与模拟器配置的28次不回执一致。供应商模拟器同期收到2048次Submit尝试,其中2013次接受、35次拒绝,发送并收到ACK的回执均为1985次,错误0;Gateway重试解释了尝试数高于业务消息数。客户端进程在收集窗口内看到的receipt数量包含异步到达,不能直接替代数据库和模拟器最终对账。
|
||||
- 本轮所有1898条记录仍识别为`mobile`,未覆盖联通、电信,优先级在20条/秒下也未表现出隔离优势:priority P95约5991ms,normal P95约4791ms。因此“修复号段/运营商识别后验证移动、联通、电信六通道容量及优先级隔离”继续保持P1未完成。
|
||||
- 本轮新增测试辅助脚本`lg-cmpp-stress-lab/scripts/run-v2-stage.ps1`,只根据档位和持续时间生成一次性客户端配置并调用既有真实CMPP压测客户端,不引入mock、静态结果或localStorage。完整复测报告及原始结果保存在短信平台测试项目;代码仍未提交、未推送,预生产保持原状。
|
||||
|
||||
# 2026-08-20 CMPP压测优化V3:受窗口约束的连接内并发(开发中)
|
||||
|
||||
- V0/V2已按用户授权提交为`b9a71fe`,提交范围为入站/供应商阶段指标、持续有界Submit Worker、契约和同步文档;`*.tsbuildinfo`、`outputs/`、自动产生的`pnpm-lock.yaml`及空文件`=`继续排除并保护,未推送。提交前Gateway全量测试、`go vet`、API TypeScript正式编译和`git diff --check`通过。
|
||||
- V3保持API风控、计费、路由、持久化和SubmitResp业务结果语义不变。认证接口新增返回真实`cmppWindowSize`;Gateway对同一认证连接按`min(应用窗口, GATEWAY_CMPP_INBOUND_MAX_CONCURRENCY, 1024)`并发处理Submit,窗口满时停止继续读取形成TCP背压。登录、心跳和Deliver ACK不纳入Submit槽位,SubmitResp继续以原Sequence_Id关联并允许按完成顺序返回。
|
||||
- 项目内gocmpp服务循环只对CMPP2/3 Submit启用并发;连接退出前等待已接受的在途处理完成后才调用会话清理,代码注释解释了这是为了防止迟到API处理重新注册已关闭连接。新增聚合指标`cmpp_gateway_inbound_submit_slots{state=configured|in_flight}`,不带账号、应用、连接或消息标签。
|
||||
- 专项集成测试已证明窗口2时,第1个Submit被API阻塞后,第2个Submit可进入API并以自身Sequence_Id先返回;释放后第1个响应仍按原Sequence_Id返回。Gateway全量`go test ./... -count=1`和`go vet ./...`、项目内gocmpp测试/vet、API TypeScript正式编译、真实隔离Redis上的SendChain 113/113项、4份Stream契约、R6的102声明/14项关键测试、R7及SendChain R10门禁和`git diff --check`通过。Windows本机`go test -race`因Go工具链未启用CGO而无法运行,未伪报通过;将在Linux测试环境部署前补跑。测试环境恢复资产、部署和阶梯压测待后续补记。
|
||||
- 当前只修改本地代码,没有连接或改变预生产运行版本;预生产只读标识仍为`433b2ee56f6016ad8afff1bac73f510b8fd53083+gateway-v2.cd7bb8d05e7b`。V3只允许发布到`100.93.204.60`虚拟机测试环境。
|
||||
- V3首次测试环境发布后10条/秒为599/599成功、P50/P95/P99=36/78/206ms;20条/秒为1199/1199成功,但P50/P95/P99=1143/5294/7107ms,仍触发5秒停止线,因此没有继续30/40/50。Prometheus同时显示认证阶段10个连接合计窗口320,但压测期入站in_flight始终为0;代码复核发现首次消息完成后`rememberDownstream`用消息级快照覆盖`byConn`时没有继承连接的`windowSize/submitInFlight`,使后续请求退回串行。该次20条/秒结果不作为修复后V3容量结论。
|
||||
- 最小修复让消息级快照共享原连接窗口和原子在途计数器,并增加“消息注册后窗口32及聚合槽位仍保持”的回归断言;需重新完成Gateway/API回归、R6契约、提交、测试环境发布与10/20阶梯复测后再判断V3效果。
|
||||
|
||||
# 2026-08-20 CMPP压测优化V3:修复后测试环境发布与阶梯复测
|
||||
|
||||
- V3主实现提交为`0757a69`,消息注册后连接窗口继承修复提交为`e9c7333`;均只保存在本地`main`,未推送。最终发布目标仅为`100.93.204.60`虚拟机测试环境,运行标识为`e9c73333b37746dd8a5f35628e4fe8fdfe02237c+workspace.v3fix.467865d78510`。预生产`8.160.169.106`未发布、未回退、未压测。
|
||||
- 发布前分别建立`/opt/cmpp-platform-backups/releases/20260820-114900-before-v3-inbound-test`和`/opt/cmpp-platform-backups/releases/20260820-121100-before-v3-window-fix-test`两套恢复资产,均包含PostgreSQL自定义格式备份、运行源码和平台环境配置;`SHA256SUMS`、`pg_restore --list`和tar可读性校验通过。最终修复包`outputs/cmpp-v3-inbound-fix-20260820-121000.tar.gz`共900项,SHA-256=`467865d785109f47869f6ea44e2c1f9b09c6368a812bc859ae56989caf086899`,排除了`.env`、依赖、`outputs`、`*.tsbuildinfo`、`pnpm-lock.yaml`和空文件`=`。
|
||||
- 标准发布流程完成依赖安装、安全/发布门禁、Prisma检查、前端/API/Gateway构建、Nginx校验和服务重启;90条migration无待应用项。API、Gateway、Nginx、PostgreSQL、Redis、MinIO和监控服务最终均active,测试供应商连接6/6,Stream消费者1且最终`pending=0/lag=0`,本轮没有Gateway Submit死信。
|
||||
- 修复后阶梯结果:10条/秒实际599条,599/599受理,P50/P95/P99=`38/66/197ms`;20条/秒实际1199条,1199/1199受理,`942/1427/1641ms`;30条/秒实际1799条,1799/1799受理,`1266/2033/2085ms`;40条/秒实际2399条,2397条SubmitResp成功、2条返回result=9,`2637/3912/5201ms`。两次失败均为优先应用sequence=6等待API响应头满10秒后超时;因此40条/秒不满足零拒绝停止线,没有继续50条/秒。
|
||||
- V3获得直接正向证据:20条/秒P95由V2的5580ms降至1427ms,压测期入站并发最大21;30条/秒仍无拒绝、无连接错误,入站并发最大55。当前可确认安全档位由10条/秒提升到30条/秒,拐点位于30至40条/秒。40条/秒时入站并发最大150,说明连接内窗口确实生效,不再退回串行。
|
||||
- 新瓶颈转移到供应商工作池及同步回写链路:20/30/40条/秒时64个Worker槽均打满,Stream lag峰值分别约48/642/1212,测试后均排空。全窗口供应商尝试6569次,其中成功6331、失败238;成功供应商RTT平均约1040ms,成功连接等待平均约519ms,成功API回调单次平均约337ms。40条/秒最终触发两个上游API 10秒超时,V4应优先把供应商结果改为幂等Outbox/异步回调,释放Worker槽位,并保留可恢复重试和逐条ACK语义。
|
||||
- 5996条业务消息均由API返回201并真实持久化,与四档客户端业务数完全一致;40条/秒的两次Gateway超时发生在API完成持久化之后,不能重投。最终状态为delivered 5768、failed 57、submitted 171,优先/普通分别为1861/4135条,全部仍识别为mobile;`GatewaySubmitDeadLetter`新增0。测试客户端有限回执收集窗口不替代数据库最终状态。
|
||||
- API全窗口平均`total≈1647.72ms`、`complete_submit≈1550.89ms`、`risk_frequency≈677.38ms`、`queue_publish≈242.43ms`、两次`application_lookup`合计约191.44ms、`template_match≈189.90ms`;并发消除了连接串行放大,但这些同步阶段随负载明显增长。主机CPU峰值约55.56%、内存使用峰值约32.55%,不是CPU或内存饱和。
|
||||
- 优先级隔离仍未通过:40条/秒priority P95约3912ms、normal P95约3911ms,且两次SubmitResp拒绝都发生在priority连接;全部5996条均为mobile,联通/电信及六通道容量仍待号段识别修复后测试。原始目录名继续沿用既有`lg-v2-*`脚本标签,但被测运行标识和代码均为V3修复版,正式结论以运行标识为准。
|
||||
- V3回归已通过Gateway全量`go test ./... -count=1`、`go vet ./...`、项目内gocmpp测试/vet、API TypeScript正式编译、真实隔离Redis上的SendChain 113/113、4份Stream契约、R6 102声明/14项关键测试、R7、SendChain R10及`git diff --check`。Linux测试机补跑`-race`时因网络无法下载仅供测试的`miniredis`和`golang.org/x/text`依赖而阻塞,未伪报通过,也未伪造外部依赖。
|
||||
- 本轮只使用隔离供应商模拟器,没有发送、补发或重投真实短信,没有修改通道账号、密码、启停状态、企业余额或客户连接。`api/tsconfig.build.tsbuildinfo`、`tsconfig.tsbuildinfo`、`outputs/`、`pnpm-lock.yaml`和空文件`=`继续作为受保护项排除提交。
|
||||
|
||||
# 2026-08-20 CMPP压测优化V4:供应商结果幂等Outbox、测试环境发布与阶梯复测
|
||||
|
||||
- V4将供应商分片结果和聚合结果先写入Redis Stream `gateway.submit.results`,API回调由独立持续有界工作池执行,供应商Submit工作槽不再等待API往返。分片事件ID为`submit:<submitId>:segment:<index>`,聚合事件ID为`submit:<submitId>:aggregate`;Lua脚本保证分片去重键与XADD原子、聚合XADD与原命令XACK原子、成功回调XACK与XDEL原子。失败事件保留在PEL并由`XAUTOCLAIM`恢复,回收阈值30秒高于10秒HTTP超时,避免多Gateway实例在回调仍执行时并发重领。
|
||||
- 第91条向前兼容migration `20260820130000_add_submit_result_idempotency`为`SmsSubmitRecord`增加唯一可空`resultEventId`和`resultProcessedAt`。API重复收到同一已完成事件直接返回当前消息,冲突事件ID拒绝;测试覆盖重复回调不重复计费、重试或下游业务副作用。新增第5类Gateway队列契约示例,Outbox指标覆盖回调Worker槽位和结果Stream pending/lag。
|
||||
- 本地验证通过:Gateway全量`go test ./... -count=1`及`go vet ./...`,API 42套/483项断言全部通过,API与前端TypeScript正式编译、Vite生产构建、5份队列契约、R0/R6/R7及SendChain R10结构门禁、`git diff --check`均通过。全量Jest在断言完成后仍因仓库既有异步句柄不自行退出,本轮未把人工终止后的进程伪报为完整退出码0;使用本机真实Redis补跑,不伪造外部依赖。
|
||||
- 仅发布到`100.93.204.60`虚拟机测试环境;预生产`8.160.169.106`只读复核标识保持`433b2ee56f6016ad8afff1bac73f510b8fd53083+gateway-v2.cd7bb8d05e7b`,未发布、未回退、未压测。测试环境发布前恢复资产位于`/opt/cmpp-platform-backups/v4-20260820T052241Z`,包含PostgreSQL自定义格式备份、运行源码、环境和systemd配置;`pg_restore --list`、源码tar可读性及`SHA256SUMS`全部通过。
|
||||
- 最终运行包`outputs/cmpp-v4-runtime-final-20260820-141323.tar.gz`共827项、1641927字节,本地与测试机SHA-256均为`a63885439ed53d53a3a012048d5fd1a9aa67772503fd778ec510ec6a38a9944f`,排除依赖、构建产物、`outputs`、`*.tsbuildinfo`、`pnpm-lock.yaml`和空文件`=`。测试环境运行标识为`485af688d21ee0bdb98281f4c20d151d69f7889f+workspace.v4.a63885439ed5`;第91条migration仅应用一次,API、Gateway、安全代理、MinIO、PostgreSQL和Redis健康,最终供应商连接6/6,两条Stream均`pending=0/lag=0`。
|
||||
- 首轮默认32槽10条/秒恰逢重启积压回执回放,599/599受理但回执2536、P95=3294ms;积压排空后再测仍为P95=671ms。把测试配置收敛为8槽后,10条/秒599/599、P50/P95/P99=`39/146/304ms`、回执603。基于该对照,代码、示例环境和发布文档的默认值同步改为8;32槽不作为推荐配置。
|
||||
- 8槽正式阶梯结果:20条/秒1199/1199受理、P50/P95/P99=`45/1129/1259ms`;30条/秒1799/1799受理、`1081/1626/1885ms`;40条/秒2399/2399受理、`2049/2330/2386ms`。三档均无连接错误;相对V3,20/30/40条/秒P95分别由1427/2033/3912ms降至1129/1626/2330ms,且40条/秒从2条10秒超时改进为零拒绝,确认安全档位由30提升到40条/秒。
|
||||
- 解耦证据:20条/秒命令Stream未投递lag峰值0,结果Outbox峰值`pending=8/lag=242`后归零;30条/秒峰值约`pending=8/lag=1076`;40条/秒峰值约`pending=9/lag=1639`,压测结束后约36秒归零并连续保持0/0。40条/秒时供应商命令与结果回调已分开排队,结果回调积压不再占用供应商Submit槽;但Outbox回落时间已成为容量判定的一部分,不能只看客户SubmitResp。
|
||||
- 50条/秒触发停止线:因客户端背压只生成2790条而非约2999条,2787条成功、3条等待API满10秒后返回result=9,P50/P95/P99=`6765/7236/7789ms`,throttled ticks=3184。结束后命令Stream一度`pending=64/lag=489`、结果Outbox`pending=8/lag=1690`;命令约1分钟内、结果随后约20秒内归零。日志显示API饱和时协议遥测和连接状态回调超时,并暴露既有`CmppDownstreamConnection`创建/更新的P2002/P2025竞态;未出现`resultEventId`唯一键冲突或Outbox事件失败。停止后未继续上探。
|
||||
- 整个窗口客户侧共9984条业务提交,真实PostgreSQL按`queuedAt`精确新增9984条;窗口内7922条实际供应商提交记录全部具有非空且互不重复的`resultEventId`,重复事件组0、Gateway Submit死信0。提交结果为accepted 7392、rejected 147、timeout 383;客户有限回执收集窗口和重连积压回执不替代数据库/Stream对账。最终确认40条/秒为当前测试环境安全档,50条/秒瓶颈转为API/数据库同步入站及遥测争用,后续应继续V1的重复查询/零散写入合并和P1分阶段指标分析。
|
||||
- 本轮只连接隔离供应商模拟器`100.91.249.119:17900`,没有发送、补发或重投真实短信,没有修改真实通道账号、密码、启停状态、企业余额或客户连接。压测原始目录沿用既有`lg-v2-*`名称,但被测运行标识与结论均为V4;完整V4报告保存在短信平台测试项目。受保护的`api/tsconfig.build.tsbuildinfo`、`tsconfig.tsbuildinfo`、`outputs/`、`pnpm-lock.yaml`和空文件`=`继续排除提交、不删除、不归因。
|
||||
|
||||
# 2026-08-20 CMPP压测优化V5:API入站数据库往返治理(本地完成、待授权发布复测)
|
||||
|
||||
- 接管复核确认本地`HEAD=67b760a5992d7ae10d7889fe6765e895d238ab3d`、`origin/main=c4f36fc50d7906dfb2f97c881e9ea43c6a64c370`,本地领先5个提交;测试环境只读标识仍为`485af688d21ee0bdb98281f4c20d151d69f7889f+workspace.v4.a63885439ed5`,预生产只读标识仍为`433b2ee56f6016ad8afff1bac73f510b8fd53083+gateway-v2.cd7bb8d05e7b`。本轮没有发布、回退、提交或推送。
|
||||
- V4高负载证据显示两次`application_lookup`合计约191.44ms、`risk_frequency`约677.38ms、`queue_publish`约242.43ms。代码复核确认每个目标号码会重复查询入口已取得的应用/企业/IP白名单;任务风控与号码频控各自调用默认规则检查,旧实现每次按5条规则逐条查询,单条短信可产生两轮共约10次存在性SQL;通用入队又回查刚创建的任务和消息。
|
||||
- V5复用Submit入口已经校验的应用快照并显式传入每个目标号码,删除第二次`application_lookup`;默认规则完整性改为30秒短TTL和并发单飞,正常完整状态使用一次聚合count,失败立即清除缓存,实际生效规则、应用规则和号码频控状态仍逐消息读取;缺失时继续走既有逐规则确认和创建逻辑。刚持久化的单条CMPP内部消息使用已知taskId、messageRecordId和queuePriority直接添加BullMQ任务,不再回查任务和消息;普通批量任务继续使用原通用入队路径。
|
||||
- 只读测试环境PostgreSQL `EXPLAIN (ANALYZE, BUFFERS)`确认应用查询使用`SmsApplication_cmppAccount_key`,执行约0.046ms;默认规则聚合检查使用`RiskRule_applicationId_code_key`并命中5行,执行约0.047ms。现有索引已经匹配查询,因此本轮不增加猜测性索引或migration;测试环境未安装`pg_stat_statements`,没有为诊断修改数据库扩展或配置。
|
||||
- 本地API TypeScript正式编译通过;API全量42套487项全部通过,其中`RiskReviewService`与`SendChainService`专项覆盖默认规则并发单飞/短缓存/失败重试/缺失恢复、单Submit只查一次应用、已知消息直接入队且不回查任务/消息,以及既有多号码、模板、频控、计费、补发和回执回归。5份Gateway队列契约、R0、R6、R7、R10和`git diff --check`通过;R9门禁在本轮已更新的`enqueueBatchTask`哈希通过后,被HEAD既有且本轮未修改的`dispatchDueScheduledTasks`哈希漂移阻断(契约期望`8fd154...`、当前方法`37db96...`),未为通过本需求门禁而改写或归因该并行历史。Jest使用`--forceExit`收尾仓库既有异步句柄;首次从仓库根目录误启动时扫描受保护`outputs/`并使用错误转换配置,未修改或删除其中任何文件,随后在`api`目录按项目配置重跑通过。
|
||||
- 当前尚未部署虚拟机测试环境或执行V5阶梯压测;`TC-CMPP-PERF-V5-005`需在取得明确发布授权、建立PostgreSQL/运行源码/环境恢复资产后执行。没有发送、补发或重投真实短信,没有修改通道账号、密码、启停状态、企业余额或客户连接;`api/tsconfig.build.tsbuildinfo`、`tsconfig.tsbuildinfo`、`outputs/`、`pnpm-lock.yaml`和空文件`=`继续保护、不归因。
|
||||
|
||||
# 2026-08-20 CMPP压测优化V5:测试环境发布、阶梯复测与优先队列验收
|
||||
|
||||
- V5代码提交为`14f993c1f8e4ef5075a16c64ee8e46987054e21e`,未推送。仅发布到`100.93.204.60`虚拟机测试环境,运行标识为`14f993c1f8e4ef5075a16c64ee8e46987054e21e+workspace.v5.e38b3598c8de`;预生产`8.160.169.106`只读标识复核仍为`433b2ee56f6016ad8afff1bac73f510b8fd53083+gateway-v2.cd7bb8d05e7b`,未发布、未回退、未压测。
|
||||
- 发布前恢复目录为`/opt/cmpp-platform-backups/v5-20260820T073310Z`,包含PostgreSQL自定义格式备份、运行源码、环境/systemd配置和原运行目录Git元数据。`pg_restore --list`、tar可读性和`SHA256SUMS`均通过;数据库、源码、环境资产SHA-256分别为`1d72b72b8702b66494f6c63384ba11fb7414982aac3b3dba4f2bbcf32691c6d5`、`cfa4679ffeef64d0a5d13b198bac272b53b9b01a4887371503d76591cca8d44e`、`f8aa632d622556d709d1ddaba0610e5eeeb604075ad92b8f81aa945b16c6f251`。V5运行包`outputs/cmpp-v5-runtime-20260820-153020.tar.gz`为2407264字节,本地与测试机SHA-256均为`e38b3598c8debc00d32c336831d172b492119199a7d8c843dfb774c16711fd22`。
|
||||
- 标准发布完成依赖、安全/发布门禁、Prisma生成与迁移、前端/API/Gateway/安全代理构建、Nginx校验、重启和健康检查;91条migration无待应用项。API、Gateway、安全代理、MinIO、PostgreSQL、Redis、Nginx、Prometheus最终均active,隔离供应商连接6/6,命令与结果Stream最终均`pending=0/lag=0`。首次构建因部署目录旧Git元数据不能完成Go VCS stamping而在服务重启前停止;该元数据移入恢复目录并校验后,以`GOFLAGS=-buildvcs=false`按同一标准脚本重新发布成功,没有删除恢复资产。
|
||||
- 原压测脚本重复使用历史`1380000xxxx`号码池,首轮10条/秒有8条被真实24小时号码频控拒绝,因此该轮明确作废、未计入V5容量。没有清除或篡改Redis/PostgreSQL频控状态;测试脚本增加可配置手机号前缀,正式五档分别使用尚无频控记录的`1390000/1370000/1360000/1350000/1340000`号码池,全部经真实API、PostgreSQL、Redis、Gateway和隔离供应商链路,且数据库均识别为mobile。
|
||||
- V5正式阶梯结果:10条/秒599/599受理,P50/P95/P99=`30/105/1736ms`;20条/秒1199/1199,`552/735/808ms`;30条/秒1799/1799,`685/890/1464ms`;40条/秒2399/2399,`1016/1748/3111ms`。四档均为0拒绝、0连接错误,双Stream均在下一档前排空;相对V4,10/20/30/40条/秒P95由`146/1129/1626/2330ms`降至`105/735/890/1748ms`,40条/秒继续为安全档。
|
||||
- 50条/秒生成2999条,但只收到2991个SubmitResp,缺8个响应,因此按零丢响应停止线判定失败;已收到响应P50/P95/P99=`2586/2896/3895ms`,0显式拒绝、0连接错误。结束时命令Stream为`pending=64/lag=1407`,约75秒后排空;结果Outbox随后也恢复0/0,所有服务保持active且API/Gateway错误日志筛查为空。该档属于可恢复过载,不作为安全容量;当前结论仍为40条/秒健康、50条/秒不健康。
|
||||
- Prometheus按各60秒窗口附近75秒区间计算的API阶段平均值显示,10/20/30/40/50档`application_lookup`约为`1.42/51.04/65.54/108.64/247.65ms`,`risk_frequency`约`4.34/115.10/143.61/229.05/519.32ms`,`queue_publish`约`4.71/70.79/101.17/163.46/383.60ms`,`complete_submit`约`18.99/479.58/610.03/992.49/2285.77ms`。V5减少固定数据库往返后40条/秒P95继续下降,但50条/秒时同步风控、入队和持久化仍随数据库并发竞争放大,下一阶段应针对这些阶段继续做查询合并/事务缩短,而不是把50条/秒宣称为安全容量。
|
||||
- 优先队列在真实积压下通过验收。BullMQ配置为priority=1、normal=100;30条/秒时优先任务`queuedAt`至首条`SmsSubmitRecord.createdAt`的P50/P95=`0.786/0.988s`,普通任务为`14.678/26.282s`;40条/秒分别为`2.996/3.916s`与`45.017/58.536s`。低负载10/20条/秒两类接近,符合无积压时无需插队的预期;高负载差异证明优先任务能够越过普通积压持续取得发送Worker。SubmitResp按优先/普通拆分在40条/秒P95为`1727/1741ms`,说明上游受理没有饿死普通连接。50条/秒优先任务也优于普通任务,但两类均出现几十秒下游等待,不能用优先级掩盖总容量过载。
|
||||
- 本轮优先级结论只覆盖3个priority应用与7个normal应用混合、移动号段和现有六个隔离供应商账号;联通、电信号段及六通道跨运营商容量/优先级隔离仍未完成,不外推为全运营商结论。完整报告和原始`events.jsonl/summary.json`保存在短信平台测试项目。没有发送真实短信,没有修改通道账号、密码、启停状态、企业余额或客户连接;受保护的`api/tsconfig.build.tsbuildinfo`、`tsconfig.tsbuildinfo`、`outputs/`、`pnpm-lock.yaml`和空文件`=`继续排除提交、不删除、不归因。
|
||||
|
||||
# 2026-08-20 完整处理500条/秒第一阶段:PostgreSQL耐久Inbox快路径(提交前验证)
|
||||
|
||||
- 第一阶段把CMPP Submit同步路径收敛为账号/状态/IP/Src_Id/协议最小校验和一条`CmppInboundSubmissionInbox`持久化;Gateway按已鉴权连接、Sequence_Id和载荷生成稳定请求键。重复相同Submit返回原持久化MessageId,相同请求键的冲突载荷拒绝。多号码保留稳定子MessageId,长短信在完整重组后才写Inbox,Worker处理完整正文。
|
||||
- 独立`cmpp-send-worker`进程持续以默认32槽领取Inbox;短事务使用`FOR UPDATE SKIP LOCKED`和300秒过期租约,处理在领取事务外完成,失败按有上限指数退避回到pending。priority应用在领取阶段优先且类内FIFO,BullMQ继续使用priority=1、normal=100。API角色不再运行发送Worker或周期扫描;Worker拥有独立Prisma连接池入口和127.0.0.1:9465指标。
|
||||
- 新增持久幂等预留:日发送配额与`SmsApplicationDailyUsage`同事务写`SmsApplicationDailyReservation`,号码频控计数与`PhoneFrequencyReservation`同事务提交,余额冻结继续使用业务键;任务、API请求和短信主记录合并为一个短事务并使用确定性任务号/请求号。租约回收或进程重启不会重复计量、计频、冻结或创建业务主记录。
|
||||
- Prisma新增第92条migration `20260820170000_add_cmpp_inbound_submission_inbox`,包含Inbox、日配额预留和号码频控预留三张真实PostgreSQL表及领取/租约/应用索引。发布脚本增加快路径/Worker强制门禁、systemd分进程、安全drop-in、Worker日志目录、健康检查和Prometheus target/告警;需求、系统用例、模块路线图和生产发布文档同步更新。
|
||||
- 提交前验证:Prisma generate/validate、API TypeScript正式构建、前端TypeScript与Vite生产构建通过;API全量42套489项通过,Gateway全量`go test ./... -count=1`和`go vet ./...`通过,5份Stream契约、依赖/安全/部署门禁、R0/R6/R7/R10及`git diff --check`通过。R6契约只新增/更新本轮`submit.go`的稳定请求键声明。R9仍先被HEAD既有`dispatchDueScheduledTasks`哈希漂移阻断,与本轮文件和既有V5记录一致,未为通过本任务错误吸收该并行历史。Jest仍用`--forceExit`收尾仓库既有开放句柄,Vite仅保留既有大chunk告警。
|
||||
- 当前尚未提交、部署或压测;下一步在排除受保护文件后提交,随后仅对`100.93.204.60`测试环境建立并校验PostgreSQL、运行源码、环境/systemd恢复资产,应用migration和独立Worker,再用隔离供应商执行同口径阶梯压测。预生产不发布、不回退、不压测;不发送、补发或重投真实短信,不修改真实通道账号、密码、启停状态、企业余额或客户连接。
|
||||
|
||||
## 2026-08-20 第一阶段测试环境首轮部署、UTC修复与连接池容量补丁
|
||||
|
||||
- 第一阶段提交`0b63bcd74e8f8be8b85aaa5f58a5e8ea7fdc636c`已部署到`100.93.204.60`测试环境。发布前恢复资产位于`/opt/cmpp-platform-backups/p1-20260820T173800-before-durable-inbox`,包含可由`pg_restore --list`读取的PostgreSQL custom dump、运行源码、环境/systemd、发布包与`SHA256SUMS`,tar和摘要均通过校验。第92条migration仅在测试数据库应用,API、Gateway、独立Worker和监控均健康;预生产未修改。
|
||||
- 首轮50条/秒30秒测试写入1499条Inbox,但Worker因把`YYYY-MM-DD`字符串传给Prisma DateTime以及Asia/Shanghai会话中用`NOW()`比较UTC无时区租约而反复重领。发现后立即停止Worker,未清理Inbox、未重投客户提交。修复提交`53073461e9b991978a3063a691d454b68c20ced9`改为合法UTC Date对象并在领取/退避SQL显式使用`NOW() AT TIME ZONE 'UTC'`;修复前第二套恢复资产位于`/opt/cmpp-platform-backups/p1fix-20260820T175000-before-utc-fix`且全部校验通过。重启后原1499条Inbox无需客户重发即全部完成,日限/频控预留和短信主记录均1499条,双Stream最终0/0。
|
||||
- UTC修复后的第二轮50条/秒生成1499帧,收到1492个SubmitResp,其中1483成功、9拒绝、7缺响应,P50/P95/P99=`519/3385/3909ms`,按零拒绝/零丢响应停止线仍失败。数据库时间窗内1487条新Inbox最终全部completed。Gateway累计阶段证明1489次API往返成功、10次恰好10秒超时;API自身1493个`POST /api/gateway/events/inbound/submit`全部在1秒内完成,说明未归因尾延迟位于Gateway到本机API的HTTP传输,而不是SubmitResp写Socket。
|
||||
- 根因补充为Go默认Transport每主机只保留2条空闲连接,无法匹配64槽入站窗口;同时PrismaPg API/Worker仍使用默认连接池上限,无法为入口和重业务事务建立容量隔离。最小补丁新增Gateway专用API Transport,最大/空闲连接数均受`GATEWAY_CMPP_INBOUND_MAX_CONCURRENCY`约束并保留10秒总超时;Prisma按角色显式使用`API_DB_POOL_MAX=32`和`API_WORKER_DB_POOL_MAX=8`。该补丁不改变最小校验、Inbox幂等、风控、计费、路由或状态机,待提交、建立新恢复资产、部署并从50条/秒重新阶梯验证。
|
||||
- 容量补丁`6708f1f7c540401d1c92083407c3750693bcbb4b`发布前恢复资产位于`/opt/cmpp-platform-backups/p1pool-20260820T101000-before-capacity-isolation`,PostgreSQL 55MB、运行源码187MB、环境/systemd和发布包均经`pg_restore --list`、tar读取及SHA-256校验。测试环境配置API/Worker池32/8,部署后服务、9464/9465回环指标和数据库连接正常。
|
||||
- 补丁后50条/秒30秒生成1499帧,1499/1499成功、零拒绝、零连接错误,P50/P95/P99=`1391/3460/4205ms`,1499条Inbox全部completed,双Stream在注入结束约62秒后归零,入口停止线通过。100条/秒档只生成2719帧(实际90.6条/秒),虽2719/2719成功且P95=4277ms,但客户端窗口被压满1346次,未达到100条/秒目标并停止升档;此时API受理2719个请求全部小于250ms,而Gateway API往返平均3.305秒。
|
||||
- 100档同时存在大量供应商Submit、回执和通讯日志;`inbound.Server`此前让Submit、协议日志、回执恢复和连接事件共享同一64连接Transport,后台流量仍能占满Submit传输槽。第二个最小补丁将Submit池按入站窗口独立,后台池固定16连接;保留10秒超时和全部日志/回执功能,不靠删除遥测通过压测。该补丁待专项验证、提交和再次建立恢复资产后发布复测。
|
||||
- 第二个容量补丁`229a0b28fd8b84d910c359ff9ac442fc3e843cb4`经Gateway全量测试/vet、R6 104声明契约和部署门禁通过。发布前恢复资产`/opt/cmpp-platform-backups/p1submitpool-20260820T102400-before-dedicated-submit`包含PostgreSQL、运行源码、环境/systemd和发布包,均经列表、tar和SHA-256校验;测试环境标记为`229a0b28...+workspace.p1submitpool.f50505ad9ee9`,服务与启动日志正常,预生产未修改。
|
||||
- 专用Submit池后100条/秒30秒发满2999条,2999/2999成功、零拒绝/节流/连接错误,P50/P95/P99=`42/147/173ms`,API平均约37ms、Gateway API往返平均约54ms;2999条Inbox全部completed,完整供应商链从开始到双Stream排空约132秒,折算约22.7条/秒。200条/秒20秒发满3998条,3998/3998成功、零拒绝/节流/连接错误,分位=`134/370/837ms`;Inbox在开始后约75秒全部完成,双Stream在开始后约166秒排空,完整链折算约24.1条/秒。
|
||||
- 默认API池32、Submit池64下首次500条/秒10秒只生成1716条,1716/1716成功但节流615次,P95=5217ms,实际171.6条/秒,失败停止。基于PostgreSQL`max_connections=100`和进程连接证据,仅在测试环境把`API_DB_POOL_MAX`调为48、`GATEWAY_CMPP_INBOUND_MAX_CONCURRENCY`调为128,Worker仍为8;复测生成2979条,2979/2979成功、零拒绝/连接错误,P50/P95/P99=`942/1162/3221ms`,但仍节流611次,实际297.9条/秒,500入口仍失败。10个测试连接各窗口32形成320总在途,Gateway平均往返约965ms,实测上限与窗口理论值一致;API平均约322ms,继续只加连接会放大PostgreSQL竞争。
|
||||
- 最终数据库按四个新号码前缀对账:`1390005/6/7/8`的Inbox completed分别为`2999/3998/1716/2979`,对应`SmsMessageRecord`及唯一MessageId数量完全相同,非completed Inbox为0,命令Stream和结果Outbox均`pending=0/lag=0`,相关服务日志无panic/fatal/Prisma错误或Inbox retry。第一阶段结论为200条/秒入口可靠受理通过、500条/秒入口未通过、完整异步链约24条/秒;不得宣称已完整处理500条/秒。下一阶段必须合并每Submit应用查询与Inbox写入、提高测试连接/窗口总量,并把Worker业务事务、供应商六通道总TPS及结果/回执写入水平扩容,不能再靠单机连接池参数解决。
|
||||
|
||||
## 2026-08-20 完整处理500条/秒第二阶段:合并入口SQL与全链有界扩容(实施中)
|
||||
|
||||
- 本轮重新核对本地`HEAD=633e7a705539cc706aa0a6dbabfd47cc61265e7f`、`origin/main=c4f36fc50d7906dfb2f97c881e9ea43c6a64c370`;测试环境运行标识为`229a0b28fd8b84d910c359ff9ac442fc3e843cb4+workspace.p1submitpool.f50505ad9ee9`,预生产仍为`433b2ee56f6016ad8afff1bac73f510b8fd53083+gateway-v2.cd7bb8d05e7b`。受保护的构建产物、`outputs/`、`pnpm-lock.yaml`和空文件`=`保持原状,不归因本轮。
|
||||
- Worker累计真实指标4695条显示:预检、模板、消息持久化、风控频次、计费和入队平均约`83.9/74.5/28.3/92.2/25.2/64.7ms`,与上一阶段完整链约24条/秒一致。该证据确认不能把入口吞吐冒充完整处理能力。
|
||||
- 普通短短信快路径已改为单个PostgreSQL CTE:使用账号唯一索引读取应用和企业当前状态,在同一快照校验接口、IP白名单及Src_Id,并以请求键`ON CONFLICT DO NOTHING`幂等写Inbox和稳定响应。正常路径不再先执行Prisma应用查询;仅并发唯一键竞争且当前快照看不到胜者时执行一次只读恢复,避免用自更新制造表膨胀。长短信分片状态机保持不变。
|
||||
- Inbox Worker领取后改为按本批唯一`applicationId`一次查询应用、企业和白名单,再逐条处理;不做跨批应用状态缓存,领取事务仍不包住风控、计费、Redis或供应商调用。专项SendChain 121项、API全量42套493项、API与前端TypeScript、Vite生产构建、Prisma validate、Gateway全量测试/vet、5份Stream契约、R0/R6/R7/R10、安全/部署门禁和`git diff --check`均通过;R9仍只被HEAD既有`dispatchDueScheduledTasks`哈希漂移阻断,未改写该方法。提交、恢复资产、测试环境发布和100→200→500阶梯结果待后续补记。
|
||||
- 主提交`99fb346c125b0f620641562c50ad517a7682a10f`发布到测试环境后,100条/秒首轮2998个请求全部收到业务拒绝且API返回500,立即按停止线终止升档。日志给出PostgreSQL `42P18 could not determine data type of parameter $11`;参数定位为`jsonb_build_object`的多态value位置没有为MessageId/phoneCount声明类型。该轮没有新增Inbox或业务短信,不计容量结果;修复为显式`text/integer`类型后须重新走门禁、提交和发布验证。
|
||||
- 类型修复提交为`f4560479d3bcb9f1b1a6fc9b1b2664a997cdf851`,测试环境最终标识为`f4560479...+workspace.p2fix.51553e0d5555`,未推送。首次发布恢复资产`/opt/cmpp-platform-backups/p2-20260820T105600Z`和修复发布恢复资产`/opt/cmpp-platform-backups/p2fix-20260820T110500Z`均包含PostgreSQL custom dump、完整运行源码、环境和systemd,分别通过`pg_restore --list`、tar读取与SHA-256校验。最终修复包大小2431905字节,本地/测试机SHA-256均为`51553e0d5555ee6a6f3317cc1a4b99410d2c0cd717ad37eac0fc839e91baa2fd`;92条migration无待执行项,API/Gateway/Worker/PostgreSQL/Redis/Nginx/Prometheus均active。预生产只读标识仍为`433b2ee56f6016ad8afff1bac73f510b8fd53083+gateway-v2.cd7bb8d05e7b`,未发布、未回退、未压测。
|
||||
- 仅在测试环境连接总预算内设置API/Worker池48/32、Inbox业务槽96、BullMQ发送槽96、Gateway Submit槽128、结果Outbox槽32和客户入站上限128;代码默认值未改变,PostgreSQL`max_connections=100`仍保留至少20条非业务池余量。修复后1条/秒低负载9/9受理、P95=41ms,Inbox与短信主记录均9条且MessageId唯一,双Stream归零。
|
||||
- 正式100条/秒30秒档生成2998条,2998/2998成功,零拒绝、零节流、零连接错误,P50/P95/P99=`34/79/128ms`;相较第一阶段100条/秒P95=147ms,合并SQL在更高Worker竞争下仍降低入口尾延迟。数据库2998条Inbox全部completed、2998条短信主记录及唯一MessageId完全对账,Inbox从首条创建至末条完成约52.1秒,折算约57.5条/秒。
|
||||
- 该档未通过“完整处理100条/秒”停止线:全部为移动号码并只走`LGST-M-P`主通道,最终产生3863次供应商尝试,其中accepted3118、rejected78、timeout667;回退通道接受846次。命令Stream和结果Outbox从首条接收至最终0/0约136秒,按2998条业务短信折算约22.0条/秒;最终消息delivered2797、failed70、submitted131。高峰期间API后台协议日志、连接状态和待投递查询出现超时,说明提高Worker/Outbox槽后共享API/数据库回调仍受争用。因100档完整链已经失败,按阶梯停止线没有继续200/500档,不能宣称完整500条/秒达标。
|
||||
|
||||
## 2026-08-21 完整处理500条/秒第三阶段:业务Worker批处理与数据库竞争治理(实施中)
|
||||
|
||||
- 接管复核:本地`HEAD=57192b7586b1e2c14f2edf0633d2d35936352879`、`origin/main=c4f36fc50d7906dfb2f97c881e9ea43c6a64c370`,ahead 15;测试环境标记`f4560479...+workspace.p2fix.51553e0d5555`,预生产标记`433b2ee5...+gateway-v2.cd7bb8d05e7b`。测试机API/Worker/Gateway/PostgreSQL/Redis/Nginx/Prometheus均active,92条migration、数据库6连接/无idle in transaction/无未授予锁,Inbox和双Stream均排空;预生产仅只读核验,未修改。
|
||||
- 当前实现把Worker按默认64条组成有界业务批次:批次共享应用、模板、签名、黑名单、风控规则和敏感词只读快照;日限额按applicationId固定顺序锁定并为每条请求写独立预留;唯一号码频控按应用在短事务批量更新并逐条持久化决定;正常零计费单号码的任务、API请求和消息三表在一个短事务批量创建,再以MessageId作为BullMQ Job ID批量入队和逐条结算Inbox。
|
||||
- 正单价、同号重复、业务拒绝、人工审核、歧义匹配、既有消息和其他异常保留原逐条状态机。批量路径任一步失败会使用稳定日限/频控键、确定性任务号/请求号、MessageId和队列Job ID逐条恢复;Redis发布不包在数据库事务中。新增固定阶段`worker_claim/reference_preload/daily_quota`并保留`risk_frequency/message_persist/queue_publish`。
|
||||
- 当前本地API正式TypeScript编译通过;RiskReview、PhoneFrequency、SendChain专项3套145项通过。尚未提交、建立本次恢复资产、部署或压测;后续结果必须按smoke→100→200→300→500停止线补记,且零计费批量结果不得外推为付费链路吞吐。
|
||||
- 第三阶段代码提交`52028b9bbdea20f0cd02efe792e91b558ba50be9`,未推送。API全量42套495项、Prisma validate/generate、API与前端正式构建、Gateway全量test/vet、5份Stream契约、安全/部署门禁、R0/R6/R7/R10、Bash语法和`git diff --check`通过;R8/R9分别被HEAD既有且本轮未修改的`GatewayInboundSubmitDto`与`dispatchDueScheduledTasks`契约漂移阻断,没有为通过门禁吸收无关历史。
|
||||
- 本次测试环境恢复资产位于`/opt/cmpp-platform-backups/p3-20260821T014531Z-before-batch-worker`:PostgreSQL custom dump 96897546字节、运行源码189193980字节、环境/systemd/Nginx/Prometheus配置包23127字节,全部0600;`pg_restore --list`、两份tar读取和`SHA256SUMS`全部通过,恢复说明明确Gateway→API→Worker启动顺序。精确发布包2441194字节,本地/服务器SHA-256均为`7c14cd5f197a0fa049748bea44b9a99b28f2ca1b5a23c4d6bce4ec8f22f1f9eb`。
|
||||
- 首次发布在Prisma输出Unicode勾号时,本地Windows GBK日志转发进程异常关闭SSH通道,远端失败保护自动恢复原运行目录、环境和服务;恢复后标记仍为`f456047...p2fix`且三服务/健康检查正常,失败目录保留为`/opt/cmpp-platform.failed-p3-20260821T014701Z`。改用UTF-8二进制日志后复用同一已校验包成功发布;当前标记`52028b9b...+workspace.p3.7c14cd5f197a`,上一运行目录`/opt/cmpp-platform.previous-p3-20260821T014803Z`,92条migration无待应用项。
|
||||
- 发布后API/Worker/Gateway/PostgreSQL/Redis/Nginx/Prometheus全部active,批处理显式启用、批次64、Worker槽96、API/Worker池48/32;数据库7连接、无未授予锁/idle in transaction,Inbox和双Stream初始排空。隔离供应商模拟器健康且6条连接在线,未连接或修改预生产。
|
||||
- smoke使用新前缀`1380010`:1条/秒10秒生成9条,9/9受理、零拒绝/节流/连接错误,P50/P95/P99=`30/71/71ms`;数据库9条消息、9个唯一MessageId,最终9条delivered,BullMQ与双Stream排空,Worker无warning,数据库无锁等待或idle in transaction。
|
||||
- 100条/秒30秒使用新前缀`1380011`:生成2998条,2998/2998受理,零拒绝、零节流、零连接错误,P50/P95/P99=`28/71/181ms`。2998条Inbox全部completed,消息/唯一MessageId/任务/API请求/日限预留/频控预留均2998且任务号、请求号无重复;创建窗口29.954秒,最后一条创建后1.418秒完成,完成分布32个秒桶、平均93.69条/秒、P50/P95=`96.5/121`、最大124条/秒。批量阶段累计180次reference preload/日限批次、216次三表持久化/批量入队,证明不是逐条伪装。
|
||||
- 100档注入结束时命令Stream`pending=128/lag=922`、结果Stream`pending=32/lag=31`,数据库瞬时`idle in transaction=3/waiting active=12/not-granted locks=7`;约25秒后数据库恢复0/0/0、双Stream恢复0/0。自开始至最终排空约87秒,按2998条折算完整下游约34.5条/秒,低于100条/秒;最终消息delivered2770、failed64、submitted164,隔离供应商本档累计提交尝试3281、accepted3226、rejected55、无回执69,错误0。
|
||||
- 因100档下游队列在负载期持续增长且完整链不足100条/秒,严格按停止线没有执行200/300/500档。第三阶段结论:入口100条/秒和业务Inbox接近实时排空通过、相较第二阶段约57.5条/秒显著改善,但业务Worker500条/秒与完整链500条/秒均未证明;第四阶段仍需供应商六通道真实并行、发送链/回调独立工作池后再升档。测试应用单价为0,本结果不得外推正单价批量计费吞吐;正单价继续走已回归的逐条原子账务路径。
|
||||
- 全程仅使用`100.93.204.60`测试环境和`100.91.249.119:17900`隔离供应商模拟器,没有发送、补发或重投真实短信,没有修改通道账号、密码、启停状态、企业余额或客户连接。预生产只读标记保持`433b2ee5...+gateway-v2.cd7bb8d05e7b`,未发布、未回退、未压测;受保护的`tsbuildinfo`、`outputs/`、`pnpm-lock.yaml`和空文件`=`继续排除提交、不删除、不归因。
|
||||
|
||||
## 2026-08-21 完整处理500条/秒第四阶段(实施中)
|
||||
|
||||
- 接管复核:本地`HEAD=0176aa6952f1e71a50a8f49eeaa64e307da02370`、`origin/main=c4f36fc50d7906dfb2f97c881e9ea43c6a64c370`;测试环境仍为`52028b9b...+workspace.p3.7c14cd5f197a`,预生产只读标记仍为`433b2ee5...+gateway-v2.cd7bb8d05e7b`。测试机六连接在线、双Stream 0/0、数据库无等待锁和idle in transaction。
|
||||
- 10个隔离应用仍为单价0、同企业余额1/授信0;移动主备各200条/秒,联通/电信主备各150条/秒,窗口32。组项目为主10/备用20,默认只能三主主动承载,六连接在线不等于六通道并行。
|
||||
- 已实现正价Worker批次、覆盖本次金额的余额判断、单事务释放/扣费、零价免账户流水,以及同最低优先级非备用通道的稳定weighted分流;默认主备语义不变。API正式编译和Billing/纯策略/SendChain三套143项通过。
|
||||
- 部署、恢复资产、测试充值、临时六通道主动双活、smoke及阶梯结果待补记;所有临时配置只允许在测试环境先快照后变更并在测试后恢复。
|
||||
- 首轮正价三运营商100条/秒生成2999条,2999/2999 SubmitResp成功、P50/P95/P99=`32/68/130ms`,Inbox与唯一MessageId均2999;移动/联通/电信分别998/996/1005,六通道最终各承载488至514条,证明主动双活与跨运营商分流有效。注入结束时仅388条完成扣费,命令Stream`128/2131`、结果Outbox`32/192`、等待锁/idle in transaction各5,按停止线不升200档。
|
||||
- 该轮最终双Stream归零,冻结/释放各2999、正式扣费2987、退款30、12条供应商最终拒绝只释放不扣费;SmsBillingRecord为charged2957/refunded30,余额恒等式精确成立,六通道与数据库最终无等待锁/idle in transaction。证据确认瓶颈是每个正价Submit结果在企业账户锁上串行,而不是单价0时可见的通道容量。
|
||||
- 后续最小修复将正常“释放冻结+正式扣费”改为单个幂等SQL:冻结已在入口扣减,流水对净余额为零,因此正常结算不再取得账户锁或更新账户;历史已释放未扣费状态仍用原带锁路径补扣,并保留并发`ON CONFLICT`后一次MVCC恢复读。专项134项和TypeScript已通过,完整门禁、修复提交、新恢复资产与复测待补记。
|
||||
- 无账户锁修复后100条/秒复测仍在注入结束出现命令Stream`128/1687`、结果Outbox`32/452`;相同观测点完成扣费从388提高至637、等待锁降为0,但单分片短信同时发布分片与聚合两个结果事件,仍造成约双倍API回调和分片审计写入。新增最小Gateway修复:单分片只发布已携带完整segments的聚合事件,多分片仍逐片先持久化,待新恢复资产和同口径复测。
|
||||
- 单分片回调收敛后第三轮100条/秒生成2999条,2999/2999受理,P50/P95/P99=`31/71/191ms`,三运营商`998/996/1005`、六通道`480至518`。注入期命令/结果Stream已排空但一度有2227条`submit_queued`,最终BullMQ wait/active/failed/delayed均0,消息2910 delivered、34 failed、55 submitted;冻结/释放各2999、charged 2987、refunded 21,SmsBillingRecord charged 2966/refunded 21,余额恒等式准确,数据库0等待锁/0 idle in transaction。该证据把剩余瓶颈定位到BullMQ发送Worker的逐消息数据库热路径,仍未达到全链100条/秒,因此没有升200档。
|
||||
- 热路径确认每条提交结果和回执都会并发执行整批`SmsMessageRecord GROUP BY status`并更新任务。最小修复对同进程同批次刷新做单飞合并,并在运行中查询后出现新状态时保留一次尾随刷新,不缓存业务结果。新增`TC-CMPP-500-P4-010`;SendChain专项122项与API正式TypeScript编译通过,部署和同口径100档复测待补记。
|
||||
- 单飞修复提交`90345fba22e3ae183e1edb4fdae718fb7cb2d963`;部署前新恢复资产`/opt/cmpp-platform-backups/p4-20260821T030748Z-before-paid-six-channel`包含121MB PostgreSQL dump、181MB运行源码、24KB系统配置,`pg_restore --list`、两个tar目录及SHA256全部通过。首次发布因本地SSH输出端GBK无法编码Prisma成功符号而中断,远端部署脚本自动回滚至`3c6f1be...`且所有服务健康;修正本地UTF-8输出后使用同一哈希校验包重新发布成功,测试标记为`90345fba...+workspace.p4progress.d82933c344ec`,92个migration无待执行项。
|
||||
- 发布后正价smoke使用`1380027/1300027/1890027`:9/9受理并delivered,P50/P95/P99=`35/250/250ms`,冻结/释放/扣费各9笔且金额均2925,余额精确减少2925,双Stream排空、数据库0锁等待/0 idle in transaction。
|
||||
- 最终100条/秒30秒使用`1380028/1300028/1890028`:2999/2999受理、零拒绝/节流/连接错误,P50/P95/P99=`30/63/96ms`;移动/联通/电信`998/996/1005`,六通道各承载479至517。中间审计完成扣费1391笔,较修复前同口径805笔显著改善,但仍有1651条`submit_queued`及命令Stream`128/1322`;最终消息2815 delivered、43 failed、141 submitted,冻结/释放各2999、charged 2988、refunded 32,SmsBillingRecord charged 2956/refunded 32,双Stream和数据库最终0/0。首条入队到最后供应商提交167.991秒,完整提交约17.85条/秒,100档仍失败,故未执行200/300/500。
|
||||
- 第四阶段全部隔离正价样本共12040条,金额单价均325;冻结/释放各12040笔、总额3913000,charged 11992笔/3897400,refunded 125笔/40625。SmsBillingRecord最终charged 11867/refunded 125;测试账户从初始1经审计充值20000000、扣费与退款后余额16143226,恒等式`1+20000000-3897400+40625=16143226`成立。
|
||||
- 测试结束已恢复配置:同企业11个应用单价均0;三主通道priority10/非备用、三备priority20/备用,weight均100;3条临时运营商规则已删除并重启Worker。真实充值、扣费、退款和操作日志保留作为审计事实;六连接在线,双Stream 0/0,数据库0等待锁/0 idle in transaction。预生产未发布、未压测、未写入;本地未push,受保护工作区文件未纳入提交或删除。
|
||||
|
||||
## 2026-08-21 第四阶段停止优化后的主流程回归
|
||||
|
||||
- 按用户决定停止继续性能优化,新增`docs/phase-4-send-worker-optimization-plan.md`只记录后续诊断、P0至P3实施顺序、正确性约束和验收停止线。本轮不再修改或部署性能代码,不再执行阶梯压测。
|
||||
- 测试环境标记保持`90345fba...+workspace.p4progress.d82933c344ec`。回归前六个隔离供应商连接在线,API、Worker、Gateway、Nginx、PostgreSQL、Redis均active且健康,数据库无等待锁。仅将测试应用`920001`单价从0临时改为325金额单位,保留前后快照和操作日志。
|
||||
- 客户CMPP最小真实样本生成2条:2/2 SubmitResp成功,P50/P95/P99=`18/20/20ms`。按本轮开始时间和专用号段`1380031`直接核对真实数据库,Inbox completed 2、SmsSubmitRecord accepted 2、SmsReceiptRecord delivered 2、消息delivered 2,客户侧状态回执下游投递delivered 2。
|
||||
- 正价计费闭环通过:两条消息单价325、总金额650;冻结/释放/正式扣费各2笔且金额分别为`-650/+650/-650`,SmsBillingRecord charged 2,无重复、无退款;测试账户余额从16143226精确降至16142576。
|
||||
- 在客户CMPP连接保持在线期间,经测试机内部Gateway事件入口注入一条携带实际`messageId/channelId/phoneNumber`的上行事件。`SmsUplinkMessage`按messageId精确匹配原应用和下发记录,内容“主流程回归上行短信”;客户端捕获`registered=0`普通Deliver并返回ACK,`CmppDownstreamDelivery`最终uplink delivered 1。该步骤验证Gateway事件入口至API入库、匹配和客户下行Deliver/ACK;供应商到Gateway的原始CMPP Deliver解包由随后Gateway全量`go test ./... -count=1`和`go vet ./...`覆盖,本轮未伪称为真实供应商上行报文注入。
|
||||
- 客户连接上线时还收到历史pending回执补投,因此客户端汇总`receipts=283`不能作为本轮回执数量;本轮严格使用开始时间、专用号段、MessageId和数据库外键隔离,新增回执事实为2条、上行为1条。该现象是既有待投递恢复行为,不归因到本次新增短信。
|
||||
- 回归结束已把`920001`单价恢复为0并保存after快照;双Stream pending/lag均0,数据库0等待锁/0 idle in transaction,API/Worker/Gateway均active。真实消息、计费、回执、上行和操作日志保留审计,预生产未变更。
|
||||
|
||||
## 2026-08-21 CMPP发送准入、频控、通道禁用与通道组补发专项复测
|
||||
|
||||
- 本轮只在`100.93.204.60`测试环境、`qa-bell-alerts-tenant`隔离企业及`910001`至`920007`压测应用执行;供应商为`100.91.249.119:17900`本地模拟器,没有触达预生产或真实短信。每个用例仅发送1条,号码频控用例发送同号2条;判定采用本次MessageId对应的PostgreSQL消息、提交、频控和回执事实,不采用客户端重连时收到的历史积压回执数。
|
||||
- 签名审核拦截通过:`920002`签名临时从`approved`改为`pending`后,消息`MSG-eedf1831-7c2c-425c-98be-8a4befcc8446`最终为`failed/SIGNATURE`,错误为“短信内容未识别到已审核通过的签名”,供应商提交0条。签名状态已恢复。
|
||||
- 模板强校验通过:压测应用默认没有模板且`templateMismatchMode=direct_send`,因此先将`920003`临时切为`reject`模拟必须报备模板;消息`MSG-3d3bea6a-3de8-4052-8aae-3ed613736b2f`最终为`failed/TEMPLATE`,错误为“短信内容未匹配到已报备模板”,供应商提交0条。应用模式已恢复`direct_send`。
|
||||
- 余额不足通过:`920004`临时单价325、企业可用余额临时置0后,消息`MSG-aaeb23b1-b18b-4f6b-8b41-0e610d244e9b`按325计算并最终为`failed/BALANCE`,供应商提交0条。应用单价已恢复0,账户余额/授信已恢复`16142576/0`。
|
||||
- 企业应用禁用/删除通过:分别关闭`interfaceEnabled`、把应用状态改为`inactive`和`deleted`时,客户CMPP登录均返回状态码3并断开,未进入提交阶段;恢复后`920005`为`active/interfaceEnabled=true`。企业禁用/删除通过:企业状态分别为`inactive`和`deleted`时,`920006`登录均返回状态码3;企业已恢复`active`。
|
||||
- 号码频次通过:为`920007`临时建立5分钟1条的应用级规则,同号`13800310000`第1条真实提交并`delivered`,第2条`MSG-40004fe5-21e0-41f2-8bc4-388d00ec6b2d`最终为`failed/RISK`、供应商提交0条;命中记录为阈值1、实际2。临时规则已删除,命中历史作为测试审计保留。
|
||||
- 通道签名报备拦截通过:将`910001`签名在移动主备两通道的报备任务临时改为`pending`后,消息`MSG-2c52fdc8-bc42-4287-a325-6780781ee2a3`最终为`failed/ROUTE`,错误为“无已报备通过且在线的可用通道”,供应商提交0条;2条报备任务均已恢复`approved`。
|
||||
- 通道禁用行为通过:仅禁用`LGST-M-P`时,消息`MSG-fc4a6384-3dfd-4252-bc5b-855e4e881a1a`自动选择`LGST-M-B`并`delivered`;主备同时禁用时,消息`MSG-5bf5960f-cfc1-4409-a6bc-80e6969ad827`最终为`failed/ROUTE`且供应商提交0条。两通道均已恢复`active`。
|
||||
- 通道组补发通过:模拟`LGST-M-P/SUP001`对消息`MSG-3c2ad6ea-9b45-435c-8833-7dbf7bf28ebd`返回Submit结果码8,首条提交记录为`rejected`;平台随后在`LGST-M-B`创建带`retryOfSubmitRecordId`的第二条提交,状态`accepted`,消息最终`delivered`。模拟器已恢复普通配置,6条通道连接均为`connected`。
|
||||
- 最终恢复审计:API及API/Worker/Gateway/PostgreSQL/Redis/MinIO服务健康;10个隔离应用全部`active`、接口全部开启、单价均0、模板模式均`direct_send`;10个签名审核/汇总报备均通过,60条签名通道报备任务全部`approved`;6个LGST通道全部`active/connected`;临时风险规则0条;Inbox仅有`completed`状态;专项窗口Worker严重错误筛查0条。
|
||||
|
||||
@@ -11,6 +11,9 @@
|
||||
- 维护 `messageId -> sequenceId -> gatewayMessageId` 映射。
|
||||
- 支持断线重连和后续消息继续消费。
|
||||
- 暴露健康检查和最小指标。
|
||||
- Redis Stream Submit Worker 使用持续补位的有界工作池并逐条 ACK;默认并发64,可用`GATEWAY_SUBMIT_WORKER_CONCURRENCY`调整,最大1024。供应商通道的真实上限仍由TPS限速、连接数和CMPP窗口共同决定。
|
||||
- 客户CMPP入站Submit按认证接口返回的应用`cmppWindowSize`在单连接内并发,Gateway再以`GATEWAY_CMPP_INBOUND_MAX_CONCURRENCY`实施默认64、最大1024的全局单连接保护。登录保持串行,心跳和Deliver ACK不等待慢Submit;SubmitResp依靠Sequence_Id关联,允许按完成顺序返回。
|
||||
- 供应商SubmitResp先写入Redis Stream幂等Outbox`gateway.submit.results`,聚合结果入Outbox与原Submit命令ACK使用同一Lua脚本;独立默认8槽回调Worker再调用API,供应商工作槽不等待API。每个事件使用确定性`eventId`和7天Redis去重键,API在`SmsSubmitRecord.resultEventId`完成持久幂等;成功回调后Outbox事件原子ACK+删除,失败事件留在PEL恢复。
|
||||
|
||||
## 建议骨架
|
||||
|
||||
|
||||
@@ -13,7 +13,9 @@ import (
|
||||
"cmpp-platform/gateway/internal/health"
|
||||
"cmpp-platform/gateway/internal/inbound"
|
||||
platformmetrics "cmpp-platform/gateway/internal/metrics"
|
||||
"cmpp-platform/gateway/internal/queue"
|
||||
"cmpp-platform/gateway/internal/ratelimit"
|
||||
"cmpp-platform/gateway/internal/resultoutbox"
|
||||
"cmpp-platform/gateway/internal/submitworker"
|
||||
"cmpp-platform/gateway/internal/upstream"
|
||||
|
||||
@@ -32,6 +34,7 @@ func main() {
|
||||
apiBaseURL := os.Getenv("API_BASE_URL")
|
||||
upstreamManager := &upstream.Manager{APIBaseURL: apiBaseURL}
|
||||
var worker *submitworker.Worker
|
||||
var resultOutbox *resultoutbox.Outbox
|
||||
channelLimiter, err := ratelimit.New(os.Getenv("REDIS_URL"))
|
||||
if err != nil {
|
||||
log.Fatalf("gateway channel rate limiter init failed: %v", err)
|
||||
@@ -47,13 +50,17 @@ func main() {
|
||||
|
||||
go func() {
|
||||
log.Printf("cmpp gateway inbound server listening on %s", cmppAddr)
|
||||
inboundConcurrency := positiveEnvInt("GATEWAY_CMPP_INBOUND_MAX_CONCURRENCY", 64)
|
||||
if err := (inbound.Server{
|
||||
Addr: cmppAddr,
|
||||
APIBaseURL: apiBaseURL,
|
||||
PresenceStore: presenceStore,
|
||||
RecoveryStore: recoveryStore,
|
||||
GatewayInstanceID: getenv("GATEWAY_INSTANCE_ID", hostname()),
|
||||
SecurityEventToken: os.Getenv("SECURITY_EVENT_TOKEN"),
|
||||
Addr: cmppAddr,
|
||||
APIBaseURL: apiBaseURL,
|
||||
HTTPClient: inbound.NewAPIHTTPClient(16),
|
||||
SubmitHTTPClient: inbound.NewAPIHTTPClient(inboundConcurrency),
|
||||
PresenceStore: presenceStore,
|
||||
RecoveryStore: recoveryStore,
|
||||
GatewayInstanceID: getenv("GATEWAY_INSTANCE_ID", hostname()),
|
||||
MaxSubmitConcurrency: inboundConcurrency,
|
||||
SecurityEventToken: os.Getenv("SECURITY_EVENT_TOKEN"),
|
||||
}).ListenAndServe(); err != nil {
|
||||
log.Fatalf("gateway inbound server stopped: %v", err)
|
||||
}
|
||||
@@ -67,13 +74,28 @@ func main() {
|
||||
worker.Stream = getenv("GATEWAY_SUBMIT_STREAM", "gateway.submit.commands")
|
||||
worker.Group = getenv("GATEWAY_SUBMIT_GROUP", "cmpp-gateway")
|
||||
worker.Consumer = getenv("GATEWAY_SUBMIT_CONSUMER", "gateway-1")
|
||||
worker.Concurrency = positiveEnvInt("GATEWAY_SUBMIT_WORKER_CONCURRENCY", 64)
|
||||
worker.APIBaseURL = apiBaseURL
|
||||
resultOutbox = resultoutbox.New(worker.Redis)
|
||||
resultOutbox.Stream = getenv("GATEWAY_SUBMIT_RESULT_STREAM", "gateway.submit.results")
|
||||
resultOutbox.Group = getenv("GATEWAY_SUBMIT_RESULT_GROUP", "cmpp-api-callback")
|
||||
resultOutbox.Consumer = getenv("GATEWAY_SUBMIT_RESULT_CONSUMER", "gateway-1")
|
||||
resultOutbox.APIBaseURL = apiBaseURL
|
||||
resultOutbox.Concurrency = positiveEnvInt("GATEWAY_SUBMIT_RESULT_WORKER_CONCURRENCY", 8)
|
||||
worker.ResultOutbox = resultOutbox
|
||||
upstreamManager.SubmitSegmentPublisher = resultOutbox
|
||||
go func() {
|
||||
log.Printf("cmpp gateway submit worker consuming stream=%s group=%s consumer=%s", worker.Stream, worker.Group, worker.Consumer)
|
||||
if err := worker.Run(context.Background()); err != nil {
|
||||
log.Printf("gateway submit worker stopped: %v", err)
|
||||
}
|
||||
}()
|
||||
go func() {
|
||||
log.Printf("cmpp gateway result Outbox consuming stream=%s group=%s consumer=%s", resultOutbox.StreamName(), resultOutbox.GroupName(), resultOutbox.Consumer)
|
||||
if err := resultOutbox.Run(context.Background()); err != nil {
|
||||
log.Printf("gateway result Outbox worker stopped: %v", err)
|
||||
}
|
||||
}()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -85,6 +107,16 @@ func main() {
|
||||
UpstreamDesired: desired, UpstreamConnected: connected,
|
||||
DownstreamConnected: inbound.ActiveConnectionCount(), SubmitWorkerUp: worker != nil,
|
||||
}
|
||||
if worker != nil {
|
||||
snapshot.SubmitWorkerConcurrency = worker.ConfiguredConcurrency()
|
||||
snapshot.SubmitWorkerInFlight = worker.InFlight()
|
||||
}
|
||||
if resultOutbox != nil {
|
||||
snapshot.ResultWorkerUp = true
|
||||
snapshot.ResultWorkerConcurrency = resultOutbox.ConfiguredConcurrency()
|
||||
snapshot.ResultWorkerInFlight = resultOutbox.InFlight()
|
||||
}
|
||||
snapshot.InboundSubmitConcurrency, snapshot.InboundSubmitInFlight = inbound.SubmitSlotSnapshot()
|
||||
if worker == nil || worker.Redis == nil {
|
||||
return snapshot
|
||||
}
|
||||
@@ -110,12 +142,38 @@ func main() {
|
||||
snapshot.QueueOldestAgeSeconds = max(0, time.Since(time.UnixMilli(milliseconds)).Seconds())
|
||||
}
|
||||
}
|
||||
if resultOutbox != nil {
|
||||
resultPending, resultErr := worker.Redis.XPending(ctx, resultOutbox.StreamName(), resultOutbox.GroupName()).Result()
|
||||
if resultErr == nil {
|
||||
snapshot.ResultQueueAvailable = true
|
||||
snapshot.ResultQueuePending = resultPending.Count
|
||||
}
|
||||
resultGroups, resultErr := worker.Redis.XInfoGroups(ctx, resultOutbox.StreamName()).Result()
|
||||
if resultErr == nil {
|
||||
for _, group := range resultGroups {
|
||||
if group.Name == resultOutbox.GroupName() {
|
||||
snapshot.ResultQueueLag = group.Lag
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return snapshot
|
||||
}))
|
||||
control.Register(mux, control.Server{
|
||||
APIBaseURL: apiBaseURL,
|
||||
Upstream: upstreamManager,
|
||||
Limiter: channelLimiter,
|
||||
Submit: func(ctx context.Context, command queue.SubmitCommand) (queue.SubmitResult, error) {
|
||||
result, submitErr := upstreamManager.Submit(ctx, command)
|
||||
if resultOutbox == nil {
|
||||
return result, submitErr
|
||||
}
|
||||
if publishErr := resultOutbox.PublishSubmitResult(ctx, command, result); publishErr != nil {
|
||||
return result, publishErr
|
||||
}
|
||||
return result, submitErr
|
||||
},
|
||||
RecoveryCandidates: func(ctx context.Context) ([]inbound.DownstreamPresence, error) {
|
||||
return inbound.ListRecoveryCandidates(ctx, presenceStore)
|
||||
},
|
||||
@@ -141,6 +199,14 @@ func getenv(key string, fallback string) string {
|
||||
return value
|
||||
}
|
||||
|
||||
func positiveEnvInt(key string, fallback int) int {
|
||||
value, err := strconv.Atoi(os.Getenv(key))
|
||||
if err != nil || value <= 0 {
|
||||
return fallback
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
func hostname() string {
|
||||
name, err := os.Hostname()
|
||||
if err != nil || name == "" {
|
||||
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
"net"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
)
|
||||
|
||||
@@ -28,6 +29,7 @@ type authResponse struct {
|
||||
Account string `json:"account"`
|
||||
EnterpriseCode string `json:"enterpriseCode"`
|
||||
MaxConnections int `json:"maxConnections"`
|
||||
WindowSize int `json:"windowSize"`
|
||||
}
|
||||
|
||||
func (s Server) handleLogin(response *cmpp.Response, packet *cmpp.Packet, logger *log.Logger) (bool, error) {
|
||||
@@ -59,6 +61,8 @@ func (s Server) handleLogin(response *cmpp.Response, packet *cmpp.Packet, logger
|
||||
applicationID: strings.TrimSpace(auth.ApplicationID),
|
||||
enterpriseCode: strings.TrimSpace(auth.EnterpriseCode),
|
||||
protocol: cmppVersionName(req.Version),
|
||||
windowSize: boundedSubmitWindow(auth.WindowSize, s.MaxSubmitConcurrency),
|
||||
submitInFlight: &atomic.Int64{},
|
||||
srcID: strings.TrimSpace(auth.Account),
|
||||
remoteIP: remoteIP(packet.Conn.Conn.RemoteAddr()),
|
||||
connectedAt: now,
|
||||
|
||||
@@ -0,0 +1,26 @@
|
||||
package inbound
|
||||
|
||||
import (
|
||||
"net"
|
||||
"net/http"
|
||||
"time"
|
||||
)
|
||||
|
||||
// NewAPIHTTPClient keeps enough loopback connections warm for the bounded CMPP
|
||||
// Submit window. Go's default of two idle connections per host otherwise causes
|
||||
// connection churn exactly when SubmitResp latency matters most.
|
||||
func NewAPIHTTPClient(maxConcurrency int) *http.Client {
|
||||
if maxConcurrency < 1 {
|
||||
maxConcurrency = 64
|
||||
}
|
||||
transport := http.DefaultTransport.(*http.Transport).Clone()
|
||||
transport.MaxIdleConns = maxConcurrency + 16
|
||||
transport.MaxIdleConnsPerHost = maxConcurrency
|
||||
transport.MaxConnsPerHost = maxConcurrency
|
||||
transport.IdleConnTimeout = 90 * time.Second
|
||||
transport.DialContext = (&net.Dialer{
|
||||
Timeout: 3 * time.Second,
|
||||
KeepAlive: 30 * time.Second,
|
||||
}).DialContext
|
||||
return &http.Client{Transport: transport, Timeout: defaultHTTPTimeout}
|
||||
}
|
||||
@@ -0,0 +1,60 @@
|
||||
package inbound
|
||||
|
||||
import (
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestNewAPIHTTPClientMatchesBoundedSubmitConcurrency(t *testing.T) {
|
||||
client := NewAPIHTTPClient(48)
|
||||
transport, ok := client.Transport.(*http.Transport)
|
||||
if !ok {
|
||||
t.Fatalf("expected *http.Transport, got %T", client.Transport)
|
||||
}
|
||||
if transport.MaxConnsPerHost != 48 || transport.MaxIdleConnsPerHost != 48 {
|
||||
t.Fatalf("unexpected host connection bounds: max=%d idle=%d", transport.MaxConnsPerHost, transport.MaxIdleConnsPerHost)
|
||||
}
|
||||
if client.Timeout != defaultHTTPTimeout {
|
||||
t.Fatalf("unexpected client timeout: %s", client.Timeout)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewAPIHTTPClientUsesSafeDefault(t *testing.T) {
|
||||
client := NewAPIHTTPClient(0)
|
||||
transport := client.Transport.(*http.Transport)
|
||||
if transport.MaxConnsPerHost != 64 {
|
||||
t.Fatalf("expected default max connections 64, got %d", transport.MaxConnsPerHost)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSubmitUsesDedicatedHTTPClient(t *testing.T) {
|
||||
background := &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
|
||||
t.Fatal("background client must not carry inbound Submit")
|
||||
return nil, nil
|
||||
})}
|
||||
submit := &http.Client{Transport: roundTripFunc(func(request *http.Request) (*http.Response, error) {
|
||||
if request.URL.Path != "/api/gateway/events/inbound/submit" {
|
||||
t.Fatalf("unexpected submit path %s", request.URL.Path)
|
||||
}
|
||||
return &http.Response{
|
||||
StatusCode: http.StatusCreated,
|
||||
Status: "201 Created",
|
||||
Header: make(http.Header),
|
||||
Body: io.NopCloser(strings.NewReader(`{"accepted":true,"messageId":"MSG-1"}`)),
|
||||
}, nil
|
||||
})}
|
||||
server := Server{APIBaseURL: "http://api.test/api", HTTPClient: background, SubmitHTTPClient: submit}
|
||||
result, err := server.submit(&net.TCPAddr{IP: net.ParseIP("127.0.0.1"), Port: 12000}, submitRequest{Account: "test"})
|
||||
if err != nil || !result.Accepted {
|
||||
t.Fatalf("expected dedicated submit success, result=%+v err=%v", result, err)
|
||||
}
|
||||
}
|
||||
|
||||
type roundTripFunc func(*http.Request) (*http.Response, error)
|
||||
|
||||
func (fn roundTripFunc) RoundTrip(request *http.Request) (*http.Response, error) {
|
||||
return fn(request)
|
||||
}
|
||||
@@ -15,11 +15,13 @@ type Server struct {
|
||||
APIBaseURL string
|
||||
SecurityEventToken string
|
||||
HTTPClient *http.Client
|
||||
SubmitHTTPClient *http.Client
|
||||
LogWriter io.Writer
|
||||
PendingFlushInterval time.Duration
|
||||
PresenceStore PresenceStore
|
||||
RecoveryStore RecoveryStore
|
||||
GatewayInstanceID string
|
||||
MaxSubmitConcurrency int
|
||||
}
|
||||
|
||||
func (s Server) ListenAndServe() error {
|
||||
@@ -30,9 +32,19 @@ func (s Server) ListenAndServe() error {
|
||||
s.logRecoveryCandidates(log.Default())
|
||||
go s.recoverPendingCandidates(log.Default())
|
||||
go s.runPendingFlusher(log.Default())
|
||||
return cmpp.ListenAndServeWithClose(addr, cmpp.V30, 30*time.Second, 3, s.LogWriter, s.handleConnectionClosed,
|
||||
return cmpp.ListenAndServeWithCloseAndSubmitWindow(addr, cmpp.V30, 30*time.Second, 3, s.LogWriter, s.handleConnectionClosed, submitWindowByConn,
|
||||
cmpp.HandlerFunc(s.handleLogin),
|
||||
cmpp.HandlerFunc(s.handleSubmit),
|
||||
cmpp.HandlerFunc(s.handleActivity),
|
||||
)
|
||||
}
|
||||
|
||||
func boundedSubmitWindow(applicationWindow int, gatewayMaximum int) int {
|
||||
if applicationWindow < 1 {
|
||||
applicationWindow = 1
|
||||
}
|
||||
if gatewayMaximum < 1 {
|
||||
gatewayMaximum = 64
|
||||
}
|
||||
return min(applicationWindow, min(gatewayMaximum, 1024))
|
||||
}
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"context"
|
||||
"encoding/binary"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log"
|
||||
"net"
|
||||
"net/http"
|
||||
@@ -237,6 +238,131 @@ func TestInboundServerAuthenticatesAndSubmits(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestInboundServerProcessesSubmitWithinAuthenticatedConnectionWindow(t *testing.T) {
|
||||
resetDownstreamRegistry()
|
||||
defer resetDownstreamRegistry()
|
||||
account := "100020"
|
||||
password := "window-secret"
|
||||
firstStarted := make(chan struct{})
|
||||
secondStarted := make(chan struct{})
|
||||
releaseFirst := make(chan struct{})
|
||||
var releaseOnce sync.Once
|
||||
defer releaseOnce.Do(func() { close(releaseFirst) })
|
||||
var calls atomic.Int32
|
||||
api := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch r.URL.Path {
|
||||
case "/api/gateway/events/inbound/authenticate":
|
||||
_ = json.NewEncoder(w).Encode(authResponse{
|
||||
PasswordCipher: password, Account: account, EnterpriseCode: account, WindowSize: 2,
|
||||
})
|
||||
case "/api/gateway/events/inbound/submit":
|
||||
call := calls.Add(1)
|
||||
if call == 1 {
|
||||
close(firstStarted)
|
||||
<-releaseFirst
|
||||
} else {
|
||||
close(secondStarted)
|
||||
}
|
||||
_ = json.NewEncoder(w).Encode(submitResponse{Accepted: true, MessageID: fmt.Sprintf("MSG-WINDOW-%d", call)})
|
||||
case "/api/gateway/events/inbound/connection", "/api/gateway/events/protocol-log":
|
||||
w.WriteHeader(http.StatusOK)
|
||||
case "/api/gateway/events/downstream/pending":
|
||||
_ = json.NewEncoder(w).Encode([]pendingDelivery{})
|
||||
default:
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}
|
||||
}))
|
||||
defer api.Close()
|
||||
|
||||
addr := reserveTCPAddr(t)
|
||||
go func() {
|
||||
_ = (Server{Addr: addr, APIBaseURL: api.URL + "/api", MaxSubmitConcurrency: 2}).ListenAndServe()
|
||||
}()
|
||||
time.Sleep(300 * time.Millisecond)
|
||||
|
||||
client := cmpp.NewClient(cmpp.V30)
|
||||
defer client.Disconnect()
|
||||
if err := client.Connect(addr, account, password, 2*time.Second); err != nil {
|
||||
t.Fatalf("connect inbound cmpp: %v", err)
|
||||
}
|
||||
packet := func(phone string) *cmpp.Cmpp3SubmitReqPkt {
|
||||
return &cmpp.Cmpp3SubmitReqPkt{
|
||||
PkTotal: 1, PkNumber: 1, RegisteredDelivery: 1, MsgLevel: 1,
|
||||
ServiceId: "cmpp", FeeUserType: 2, FeeTerminalId: phone, MsgFmt: 0,
|
||||
MsgSrc: account, FeeType: "02", FeeCode: "0", SrcId: "10690000",
|
||||
DestUsrTl: 1, DestTerminalId: []string{phone}, MsgLength: 6, MsgContent: "window",
|
||||
}
|
||||
}
|
||||
firstSequence, err := client.SendReqPkt(packet("13800000001"))
|
||||
if err != nil {
|
||||
t.Fatalf("send first submit: %v", err)
|
||||
}
|
||||
<-firstStarted
|
||||
secondSequence, err := client.SendReqPkt(packet("13800000002"))
|
||||
if err != nil {
|
||||
t.Fatalf("send second submit: %v", err)
|
||||
}
|
||||
select {
|
||||
case <-secondStarted:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("second Submit did not enter the authenticated connection window")
|
||||
}
|
||||
response := recvSubmitRsp(t, client)
|
||||
if response.SeqId != secondSequence {
|
||||
t.Fatalf("first completed response sequence=%d, want second sequence=%d (first=%d)", response.SeqId, secondSequence, firstSequence)
|
||||
}
|
||||
releaseOnce.Do(func() { close(releaseFirst) })
|
||||
response = recvSubmitRsp(t, client)
|
||||
if response.SeqId != firstSequence {
|
||||
t.Fatalf("released response sequence=%d, want first sequence=%d", response.SeqId, firstSequence)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBoundedSubmitWindowAndAggregateSlotSnapshot(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
application int
|
||||
maximum int
|
||||
want int
|
||||
}{
|
||||
{application: 0, maximum: 0, want: 1},
|
||||
{application: 32, maximum: 64, want: 32},
|
||||
{application: 32, maximum: 16, want: 16},
|
||||
{application: 2048, maximum: 2048, want: 1024},
|
||||
} {
|
||||
if got := boundedSubmitWindow(test.application, test.maximum); got != test.want {
|
||||
t.Fatalf("boundedSubmitWindow(%d, %d)=%d, want %d", test.application, test.maximum, got, test.want)
|
||||
}
|
||||
}
|
||||
|
||||
resetDownstreamRegistry()
|
||||
defer resetDownstreamRegistry()
|
||||
firstCounter := &atomic.Int64{}
|
||||
secondCounter := &atomic.Int64{}
|
||||
firstCounter.Store(3)
|
||||
secondCounter.Store(1)
|
||||
downstreamRegistry.byConn[&cmpp.Conn{}] = &downstreamSession{windowSize: 32, submitInFlight: firstCounter}
|
||||
downstreamRegistry.byConn[&cmpp.Conn{}] = &downstreamSession{windowSize: 16, submitInFlight: secondCounter}
|
||||
configured, inFlight := SubmitSlotSnapshot()
|
||||
if configured != 48 || inFlight != 4 {
|
||||
t.Fatalf("slot snapshot configured=%d in_flight=%d, want 48/4", configured, inFlight)
|
||||
}
|
||||
|
||||
conn := &cmpp.Conn{}
|
||||
counter := &atomic.Int64{}
|
||||
counter.Store(2)
|
||||
rememberDownstream(downstreamSession{
|
||||
messageID: "MSG-WINDOW-SNAPSHOT", account: "100030", conn: conn,
|
||||
windowSize: 32, submitInFlight: counter,
|
||||
})
|
||||
if got := submitWindowByConn(conn); got != 32 {
|
||||
t.Fatalf("message registration replaced connection window with %d, want 32", got)
|
||||
}
|
||||
configured, inFlight = SubmitSlotSnapshot()
|
||||
if configured != 80 || inFlight != 6 {
|
||||
t.Fatalf("post-message slot snapshot configured=%d in_flight=%d, want 80/6", configured, inFlight)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecodeInboundLongMessageStripsConcatUDHBeforeUCS2Decode(t *testing.T) {
|
||||
payload, err := cmpputils.Utf8ToUcs2("【深圳市合正物业服务有限公司】第一片正文")
|
||||
if err != nil {
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"log"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
)
|
||||
|
||||
@@ -30,6 +31,8 @@ type downstreamSession struct {
|
||||
applicationID string
|
||||
enterpriseCode string
|
||||
protocol string
|
||||
windowSize int
|
||||
submitInFlight *atomic.Int64
|
||||
srcID string
|
||||
phoneNumber string
|
||||
gatewayMsgID uint64
|
||||
@@ -123,6 +126,39 @@ func findSessionByConn(conn *cmpp.Conn) *downstreamSession {
|
||||
return downstreamRegistry.byConn[conn]
|
||||
}
|
||||
|
||||
func submitWindowByConn(conn *cmpp.Conn) int {
|
||||
session := findSessionByConn(conn)
|
||||
if session == nil || session.windowSize < 1 {
|
||||
return 1
|
||||
}
|
||||
return session.windowSize
|
||||
}
|
||||
|
||||
func beginInboundSubmit(session *downstreamSession) func() {
|
||||
if session == nil || session.submitInFlight == nil {
|
||||
return func() {}
|
||||
}
|
||||
session.submitInFlight.Add(1)
|
||||
return func() { session.submitInFlight.Add(-1) }
|
||||
}
|
||||
|
||||
func SubmitSlotSnapshot() (int, int64) {
|
||||
downstreamRegistry.RLock()
|
||||
defer downstreamRegistry.RUnlock()
|
||||
configured := 0
|
||||
var inFlight int64
|
||||
for _, session := range downstreamRegistry.byConn {
|
||||
if session == nil {
|
||||
continue
|
||||
}
|
||||
configured += max(1, session.windowSize)
|
||||
if session.submitInFlight != nil {
|
||||
inFlight += session.submitInFlight.Load()
|
||||
}
|
||||
}
|
||||
return configured, inFlight
|
||||
}
|
||||
|
||||
func rememberDownstream(session downstreamSession) {
|
||||
if session.messageID == "" || session.conn == nil {
|
||||
return
|
||||
|
||||
@@ -1,8 +1,10 @@
|
||||
package inbound
|
||||
|
||||
import (
|
||||
"cmpp-platform/gateway/internal/metrics"
|
||||
"context"
|
||||
"crypto/md5"
|
||||
"crypto/sha256"
|
||||
"errors"
|
||||
"fmt"
|
||||
cmpp "github.com/bigwhite/gocmpp"
|
||||
@@ -18,6 +20,7 @@ import (
|
||||
// one internal message mapping per destination while CMPP receives one response.
|
||||
|
||||
type submitRequest struct {
|
||||
RequestID string `json:"requestId,omitempty"`
|
||||
Account string `json:"account"`
|
||||
PhoneNumber string `json:"phoneNumber,omitempty"`
|
||||
PhoneNumbers []string `json:"phoneNumbers,omitempty"`
|
||||
@@ -56,6 +59,7 @@ func (s Server) handleSubmit(response *cmpp.Response, packet *cmpp.Packet, logge
|
||||
if !ok {
|
||||
return true, nil
|
||||
}
|
||||
handlerStartedAt := time.Now()
|
||||
session := findSessionByConn(packet.Conn)
|
||||
if session == nil || strings.TrimSpace(session.account) == "" {
|
||||
logger.Printf(
|
||||
@@ -63,9 +67,12 @@ func (s Server) handleSubmit(response *cmpp.Response, packet *cmpp.Packet, logge
|
||||
req.protocol, req.protocol, packet.Conn.Conn.RemoteAddr(), req.sequenceID, "authenticated connection session not found",
|
||||
)
|
||||
setInboundSubmitResponse(response.Packer, 0, 9)
|
||||
response.AfterSend = s.submitResponseProtocolLogger("", req.protocol, req.sequenceID, "", "", 0, 9)
|
||||
responseReadyAt := time.Now()
|
||||
response.AfterSend = observeInboundSubmitResponse(handlerStartedAt, responseReadyAt, false, s.submitResponseProtocolLogger("", req.protocol, req.sequenceID, "", "", 0, 9))
|
||||
return false, nil
|
||||
}
|
||||
releaseInboundSlot := beginInboundSubmit(session)
|
||||
defer releaseInboundSlot()
|
||||
account := session.account
|
||||
enterpriseCode := strings.TrimRight(req.msgSrc, "\x00")
|
||||
if session.enterpriseCode != "" && enterpriseCode != session.enterpriseCode {
|
||||
@@ -75,7 +82,8 @@ func (s Server) handleSubmit(response *cmpp.Response, packet *cmpp.Packet, logge
|
||||
fmt.Sprintf("enterprise code mismatch: expected %s", session.enterpriseCode),
|
||||
)
|
||||
setInboundSubmitResponse(response.Packer, 0, 9)
|
||||
response.AfterSend = s.submitResponseProtocolLogger(account, defaultString(session.protocol, req.protocol), req.sequenceID, "", "", 0, 9)
|
||||
responseReadyAt := time.Now()
|
||||
response.AfterSend = observeInboundSubmitResponse(handlerStartedAt, responseReadyAt, false, s.submitResponseProtocolLogger(account, defaultString(session.protocol, req.protocol), req.sequenceID, "", "", 0, 9))
|
||||
return false, nil
|
||||
}
|
||||
phones := make([]string, len(req.destTerminalIDs))
|
||||
@@ -93,20 +101,25 @@ func (s Server) handleSubmit(response *cmpp.Response, packet *cmpp.Packet, logge
|
||||
clientProtocol, req.protocol, account, enterpriseCode, remote, req.sequenceID, phone, strings.TrimSpace(req.srcID), req.msgFmt,
|
||||
req.pkNumber, req.pkTotal, len(req.destTerminalIDs), len(req.msgContent),
|
||||
)
|
||||
decodeStartedAt := time.Now()
|
||||
content, longMessage, err := decodeInboundSubmitContent(req)
|
||||
metrics.ObserveInboundStage("decode", err == nil, time.Since(decodeStartedAt))
|
||||
if err != nil {
|
||||
logger.Printf(
|
||||
"cmpp inbound event=submit_rejected protocol=%s packet_type=%s account=%s remote=%s seq=%d phone=%s result=9 stage=decode reason=%q",
|
||||
clientProtocol, req.protocol, account, remote, req.sequenceID, phone, err,
|
||||
)
|
||||
setInboundSubmitResponse(response.Packer, 0, 9)
|
||||
response.AfterSend = s.submitResponseProtocolLogger(account, clientProtocol, req.sequenceID, phone, "", 0, 9)
|
||||
responseReadyAt := time.Now()
|
||||
response.AfterSend = observeInboundSubmitResponse(handlerStartedAt, responseReadyAt, false, s.submitResponseProtocolLogger(account, clientProtocol, req.sequenceID, phone, "", 0, 9))
|
||||
return false, nil
|
||||
}
|
||||
contentHash := fmt.Sprintf("%x", md5.Sum([]byte(content)))
|
||||
startedAt := time.Now()
|
||||
releaseSubmitBarrier := beginDownstreamSubmitBarrier(packet.Conn)
|
||||
apiStartedAt := time.Now()
|
||||
result, err := s.submit(remote, submitRequest{
|
||||
RequestID: inboundSubmitRequestID(session.connectionID, req.sequenceID, phones, content, req.srcID, longMessage),
|
||||
Account: account,
|
||||
PhoneNumber: phone,
|
||||
PhoneNumbers: phones,
|
||||
@@ -118,6 +131,7 @@ func (s Server) handleSubmit(response *cmpp.Response, packet *cmpp.Packet, logge
|
||||
RemoteIP: remoteIP(remote),
|
||||
LongMessage: longMessage,
|
||||
})
|
||||
metrics.ObserveInboundStage("api_roundtrip", err == nil, time.Since(apiStartedAt))
|
||||
if err != nil || !result.Accepted {
|
||||
reason := "api returned accepted=false"
|
||||
if err != nil {
|
||||
@@ -133,10 +147,11 @@ func (s Server) handleSubmit(response *cmpp.Response, packet *cmpp.Packet, logge
|
||||
)
|
||||
setInboundSubmitResponse(response.Packer, 0, responseResult)
|
||||
protocolLogger := s.submitResponseProtocolLogger(account, clientProtocol, req.sequenceID, phone, result.MessageID, 0, responseResult)
|
||||
response.AfterSend = func(sendErr error) {
|
||||
responseReadyAt := time.Now()
|
||||
response.AfterSend = observeInboundSubmitResponse(handlerStartedAt, responseReadyAt, false, func(sendErr error) {
|
||||
releaseSubmitBarrier()
|
||||
protocolLogger(sendErr)
|
||||
}
|
||||
})
|
||||
return false, nil
|
||||
}
|
||||
gatewayMsgID := messageIDFrom(result.MessageID, req.sequenceID)
|
||||
@@ -157,6 +172,8 @@ func (s Server) handleSubmit(response *cmpp.Response, packet *cmpp.Packet, logge
|
||||
applicationID: result.ApplicationID,
|
||||
enterpriseCode: session.enterpriseCode,
|
||||
protocol: clientProtocol,
|
||||
windowSize: session.windowSize,
|
||||
submitInFlight: session.submitInFlight,
|
||||
srcID: strings.TrimSpace(req.srcID),
|
||||
phoneNumber: acceptedPhone,
|
||||
gatewayMsgID: gatewayMsgID,
|
||||
@@ -175,7 +192,8 @@ func (s Server) handleSubmit(response *cmpp.Response, packet *cmpp.Packet, logge
|
||||
if current := findSessionByConn(packet.Conn); current != nil && current.report != nil {
|
||||
go current.report(current, "submit", "")
|
||||
}
|
||||
response.AfterSend = func(sendErr error) {
|
||||
responseReadyAt := time.Now()
|
||||
response.AfterSend = observeInboundSubmitResponse(handlerStartedAt, responseReadyAt, true, func(sendErr error) {
|
||||
releaseSubmitBarrier()
|
||||
s.emitProtocolLog(protocolLogEvent{
|
||||
Protocol: "cmpp",
|
||||
@@ -199,7 +217,7 @@ func (s Server) handleSubmit(response *cmpp.Response, packet *cmpp.Packet, logge
|
||||
logger.Printf("cmpp inbound event=post_submit_pending_flush_failed account=%s message_id=%s error=%q", account, result.MessageID, err.Error())
|
||||
}
|
||||
}()
|
||||
}
|
||||
})
|
||||
logger.Printf(
|
||||
"cmpp inbound event=submit_accepted protocol=%s packet_type=%s account=%s remote=%s seq=%d phone=%s dest_count=%d accepted_count=%d result=0 message_id=%s gateway_message_id=%d duration_ms=%d content_chars=%d content_hash=%s",
|
||||
clientProtocol, req.protocol, account, remote, req.sequenceID, phone, len(phones), len(responseMessages), result.MessageID, gatewayMsgID, time.Since(startedAt).Milliseconds(), len([]rune(content)), contentHash,
|
||||
@@ -207,6 +225,24 @@ func (s Server) handleSubmit(response *cmpp.Response, packet *cmpp.Packet, logge
|
||||
return false, nil
|
||||
}
|
||||
|
||||
func inboundSubmitRequestID(connectionID string, sequenceID uint32, phones []string, content string, srcID string, longMessage *inboundLongMessageFragment) string {
|
||||
// The key is stable for an API retry of the same packet but scoped to the authenticated
|
||||
// connection, so a later client session may intentionally reuse the CMPP Sequence_Id.
|
||||
payload := fmt.Sprintf("%s\x00%d\x00%s\x00%s\x00%s\x00%v", connectionID, sequenceID, strings.Join(phones, ","), strings.TrimSpace(srcID), content, longMessage)
|
||||
digest := sha256.Sum256([]byte(payload))
|
||||
return fmt.Sprintf("cmpp-inbound:%x", digest[:])
|
||||
}
|
||||
|
||||
func observeInboundSubmitResponse(handlerStartedAt time.Time, responseReadyAt time.Time, accepted bool, next func(error)) func(error) {
|
||||
return func(sendErr error) {
|
||||
metrics.ObserveInboundStage("response_write", sendErr == nil, time.Since(responseReadyAt))
|
||||
metrics.ObserveInboundStage("handler_total", accepted && sendErr == nil, time.Since(handlerStartedAt))
|
||||
if next != nil {
|
||||
next(sendErr)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type inboundSubmitPacket struct {
|
||||
protocol string
|
||||
pkTotal uint8
|
||||
@@ -254,7 +290,13 @@ func setInboundSubmitResponse(packet any, messageID uint64, result uint32) {
|
||||
func (s Server) submit(remote net.Addr, payload submitRequest) (submitResponse, error) {
|
||||
payload.RemoteIP = remoteIP(remote)
|
||||
var result submitResponse
|
||||
err := s.post(context.Background(), "/gateway/events/inbound/submit", payload, &result)
|
||||
client := s.SubmitHTTPClient
|
||||
if client == nil {
|
||||
client = s.HTTPClient
|
||||
}
|
||||
// Submit has a dedicated transport so protocol logs, receipt recovery and
|
||||
// presence traffic cannot occupy the connections needed for SubmitResp.
|
||||
err := s.postWithClient(context.Background(), client, "/gateway/events/inbound/submit", payload, &result)
|
||||
return result, err
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,27 @@
|
||||
package inbound
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestInboundSubmitRequestIDIsStableForRetry(t *testing.T) {
|
||||
fragment := &inboundLongMessageFragment{Reference: 7, Total: 2, Index: 1, Format: 8}
|
||||
first := inboundSubmitRequestID("connection-1", 42, []string{"13800000001"}, "【测试】内容", "1069", fragment)
|
||||
second := inboundSubmitRequestID("connection-1", 42, []string{"13800000001"}, "【测试】内容", "1069", fragment)
|
||||
if first != second {
|
||||
t.Fatalf("retry key changed: %q != %q", first, second)
|
||||
}
|
||||
}
|
||||
|
||||
func TestInboundSubmitRequestIDSeparatesPayloadAndSession(t *testing.T) {
|
||||
base := inboundSubmitRequestID("connection-1", 42, []string{"13800000001"}, "content", "1069", nil)
|
||||
cases := []string{
|
||||
inboundSubmitRequestID("connection-2", 42, []string{"13800000001"}, "content", "1069", nil),
|
||||
inboundSubmitRequestID("connection-1", 43, []string{"13800000001"}, "content", "1069", nil),
|
||||
inboundSubmitRequestID("connection-1", 42, []string{"13800000002"}, "content", "1069", nil),
|
||||
inboundSubmitRequestID("connection-1", 42, []string{"13800000001"}, "other", "1069", nil),
|
||||
}
|
||||
for _, candidate := range cases {
|
||||
if candidate == base {
|
||||
t.Fatalf("different submit produced duplicate key %q", base)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -15,7 +15,10 @@ import (
|
||||
const maxAPIResponseBodyBytes int64 = 4 * 1024 * 1024
|
||||
|
||||
func (s Server) post(ctx context.Context, path string, payload any, result any) error {
|
||||
client := s.HTTPClient
|
||||
return s.postWithClient(ctx, s.HTTPClient, path, payload, result)
|
||||
}
|
||||
|
||||
func (s Server) postWithClient(ctx context.Context, client *http.Client, path string, payload any, result any) error {
|
||||
if client == nil {
|
||||
client = &http.Client{Timeout: defaultHTTPTimeout}
|
||||
}
|
||||
|
||||
@@ -14,15 +14,47 @@ var submitAccepted atomic.Uint64
|
||||
var submitFailed atomic.Uint64
|
||||
var submitDurationNanoseconds atomic.Uint64
|
||||
|
||||
var durationBuckets = [...]time.Duration{
|
||||
5 * time.Millisecond,
|
||||
10 * time.Millisecond,
|
||||
25 * time.Millisecond,
|
||||
50 * time.Millisecond,
|
||||
100 * time.Millisecond,
|
||||
250 * time.Millisecond,
|
||||
500 * time.Millisecond,
|
||||
time.Second,
|
||||
3 * time.Second,
|
||||
10 * time.Second,
|
||||
}
|
||||
|
||||
type durationHistogram struct {
|
||||
count atomic.Uint64
|
||||
sumNano atomic.Uint64
|
||||
buckets [len(durationBuckets)]atomic.Uint64
|
||||
}
|
||||
|
||||
var inboundStageHistograms [4][2]durationHistogram
|
||||
var submitStageHistograms [5][2]durationHistogram
|
||||
|
||||
type Snapshot struct {
|
||||
UpstreamDesired int
|
||||
UpstreamConnected int
|
||||
DownstreamConnected int
|
||||
SubmitWorkerUp bool
|
||||
QueueAvailable bool
|
||||
QueuePending int64
|
||||
QueueLag int64
|
||||
QueueOldestAgeSeconds float64
|
||||
UpstreamDesired int
|
||||
UpstreamConnected int
|
||||
DownstreamConnected int
|
||||
SubmitWorkerUp bool
|
||||
SubmitWorkerConcurrency int
|
||||
SubmitWorkerInFlight int64
|
||||
ResultWorkerUp bool
|
||||
ResultWorkerConcurrency int
|
||||
ResultWorkerInFlight int64
|
||||
InboundSubmitConcurrency int
|
||||
InboundSubmitInFlight int64
|
||||
QueueAvailable bool
|
||||
QueuePending int64
|
||||
QueueLag int64
|
||||
QueueOldestAgeSeconds float64
|
||||
ResultQueueAvailable bool
|
||||
ResultQueuePending int64
|
||||
ResultQueueLag int64
|
||||
}
|
||||
|
||||
type SnapshotFunc func(context.Context) Snapshot
|
||||
@@ -36,6 +68,25 @@ func ObserveSubmit(accepted bool, duration time.Duration) {
|
||||
submitDurationNanoseconds.Add(uint64(max(duration, 0)))
|
||||
}
|
||||
|
||||
// ObserveInboundStage deliberately accepts only a fixed stage/result vocabulary.
|
||||
// Entity identifiers would create unbounded Prometheus series during high-volume traffic.
|
||||
func ObserveInboundStage(stage string, success bool, duration time.Duration) {
|
||||
stageIndex := inboundStageIndex(stage)
|
||||
if stageIndex < 0 {
|
||||
return
|
||||
}
|
||||
observeDuration(&inboundStageHistograms[stageIndex][boolIndex(success)], duration)
|
||||
}
|
||||
|
||||
// ObserveSubmitStage separates queueing, limiting, connection-window, supplier and API callback time.
|
||||
func ObserveSubmitStage(stage string, success bool, duration time.Duration) {
|
||||
stageIndex := submitStageIndex(stage)
|
||||
if stageIndex < 0 {
|
||||
return
|
||||
}
|
||||
observeDuration(&submitStageHistograms[stageIndex][boolIndex(success)], duration)
|
||||
}
|
||||
|
||||
func Handler(load SnapshotFunc) http.Handler {
|
||||
return http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) {
|
||||
if request.Method != http.MethodGet || request.URL.Path != "/metrics" {
|
||||
@@ -61,17 +112,103 @@ func Handler(load SnapshotFunc) http.Handler {
|
||||
fmt.Fprintf(response, "# HELP cmpp_gateway_submit_total Upstream submit attempts by bounded result.\n# TYPE cmpp_gateway_submit_total counter\ncmpp_gateway_submit_total{result=\"accepted\"} %d\ncmpp_gateway_submit_total{result=\"failed\"} %d\n", accepted, failed)
|
||||
fmt.Fprintf(response, "# HELP cmpp_gateway_submit_duration_seconds_sum Total upstream submit duration.\n# TYPE cmpp_gateway_submit_duration_seconds_sum counter\ncmpp_gateway_submit_duration_seconds_sum %f\n", float64(submitDurationNanoseconds.Load())/float64(time.Second))
|
||||
fmt.Fprintf(response, "# HELP cmpp_gateway_submit_duration_seconds_count Total measured upstream submits.\n# TYPE cmpp_gateway_submit_duration_seconds_count counter\ncmpp_gateway_submit_duration_seconds_count %d\n", count)
|
||||
fmt.Fprint(response, "# HELP cmpp_gateway_inbound_stage_duration_seconds CMPP inbound handler duration by bounded stage and result.\n# TYPE cmpp_gateway_inbound_stage_duration_seconds histogram\n")
|
||||
for index, stage := range []string{"decode", "api_roundtrip", "response_write", "handler_total"} {
|
||||
writeDurationHistogram(response, "cmpp_gateway_inbound_stage_duration_seconds", stage, &inboundStageHistograms[index])
|
||||
}
|
||||
fmt.Fprint(response, "# HELP cmpp_gateway_submit_stage_duration_seconds Gateway submit duration by bounded stage and result.\n# TYPE cmpp_gateway_submit_stage_duration_seconds histogram\n")
|
||||
for index, stage := range []string{"stream_wait", "rate_limit_wait", "connection_wait", "supplier_rtt", "api_callback"} {
|
||||
writeDurationHistogram(response, "cmpp_gateway_submit_stage_duration_seconds", stage, &submitStageHistograms[index])
|
||||
}
|
||||
fmt.Fprintf(response, "# HELP cmpp_gateway_upstream_connections Desired and live supplier connections.\n# TYPE cmpp_gateway_upstream_connections gauge\ncmpp_gateway_upstream_connections{state=\"desired\"} %d\ncmpp_gateway_upstream_connections{state=\"connected\"} %d\n", snapshot.UpstreamDesired, snapshot.UpstreamConnected)
|
||||
fmt.Fprintf(response, "# HELP cmpp_gateway_downstream_connections Authenticated client connections.\n# TYPE cmpp_gateway_downstream_connections gauge\ncmpp_gateway_downstream_connections %d\n", snapshot.DownstreamConnected)
|
||||
fmt.Fprintf(response, "# HELP cmpp_gateway_submit_worker_up Whether the submit worker was initialized.\n# TYPE cmpp_gateway_submit_worker_up gauge\ncmpp_gateway_submit_worker_up %d\n", boolNumber(snapshot.SubmitWorkerUp))
|
||||
fmt.Fprintf(response, "# HELP cmpp_gateway_submit_worker_slots Configured and active bounded submit worker slots.\n# TYPE cmpp_gateway_submit_worker_slots gauge\ncmpp_gateway_submit_worker_slots{state=\"configured\"} %d\ncmpp_gateway_submit_worker_slots{state=\"in_flight\"} %d\n", snapshot.SubmitWorkerConcurrency, snapshot.SubmitWorkerInFlight)
|
||||
fmt.Fprintf(response, "# HELP cmpp_gateway_result_callback_worker_up Whether the asynchronous result callback worker was initialized.\n# TYPE cmpp_gateway_result_callback_worker_up gauge\ncmpp_gateway_result_callback_worker_up %d\n", boolNumber(snapshot.ResultWorkerUp))
|
||||
fmt.Fprintf(response, "# HELP cmpp_gateway_result_callback_worker_slots Configured and active result callback worker slots.\n# TYPE cmpp_gateway_result_callback_worker_slots gauge\ncmpp_gateway_result_callback_worker_slots{state=\"configured\"} %d\ncmpp_gateway_result_callback_worker_slots{state=\"in_flight\"} %d\n", snapshot.ResultWorkerConcurrency, snapshot.ResultWorkerInFlight)
|
||||
fmt.Fprintf(response, "# HELP cmpp_gateway_inbound_submit_slots Configured and active authenticated client Submit slots.\n# TYPE cmpp_gateway_inbound_submit_slots gauge\ncmpp_gateway_inbound_submit_slots{state=\"configured\"} %d\ncmpp_gateway_inbound_submit_slots{state=\"in_flight\"} %d\n", snapshot.InboundSubmitConcurrency, snapshot.InboundSubmitInFlight)
|
||||
if snapshot.QueueAvailable {
|
||||
fmt.Fprintf(response, "# HELP cmpp_gateway_submit_queue_pending Pending entries owned by the consumer group.\n# TYPE cmpp_gateway_submit_queue_pending gauge\ncmpp_gateway_submit_queue_pending %d\n", snapshot.QueuePending)
|
||||
fmt.Fprintf(response, "# HELP cmpp_gateway_submit_queue_lag Undelivered entries for the consumer group.\n# TYPE cmpp_gateway_submit_queue_lag gauge\ncmpp_gateway_submit_queue_lag %d\n", snapshot.QueueLag)
|
||||
fmt.Fprintf(response, "# HELP cmpp_gateway_submit_queue_oldest_pending_age_seconds Age of the oldest pending entry.\n# TYPE cmpp_gateway_submit_queue_oldest_pending_age_seconds gauge\ncmpp_gateway_submit_queue_oldest_pending_age_seconds %f\n", snapshot.QueueOldestAgeSeconds)
|
||||
}
|
||||
if snapshot.ResultQueueAvailable {
|
||||
fmt.Fprintf(response, "# HELP cmpp_gateway_result_outbox_pending Pending result callbacks owned by the consumer group.\n# TYPE cmpp_gateway_result_outbox_pending gauge\ncmpp_gateway_result_outbox_pending %d\n", snapshot.ResultQueuePending)
|
||||
fmt.Fprintf(response, "# HELP cmpp_gateway_result_outbox_lag Undelivered result callbacks for the consumer group.\n# TYPE cmpp_gateway_result_outbox_lag gauge\ncmpp_gateway_result_outbox_lag %d\n", snapshot.ResultQueueLag)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func observeDuration(histogram *durationHistogram, duration time.Duration) {
|
||||
duration = max(duration, 0)
|
||||
histogram.count.Add(1)
|
||||
histogram.sumNano.Add(uint64(duration))
|
||||
for index, bucket := range durationBuckets {
|
||||
if duration <= bucket {
|
||||
histogram.buckets[index].Add(1)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func writeDurationHistogram(response http.ResponseWriter, metricName string, stage string, histograms *[2]durationHistogram) {
|
||||
for resultIndex, result := range []string{"failed", "success"} {
|
||||
histogram := &histograms[resultIndex]
|
||||
count := histogram.count.Load()
|
||||
if count == 0 {
|
||||
continue
|
||||
}
|
||||
for index, bucket := range durationBuckets {
|
||||
fmt.Fprintf(response, "%s_bucket{stage=%q,result=%q,le=%q} %d\n", metricName, stage, result, durationBucketLabel(bucket), histogram.buckets[index].Load())
|
||||
}
|
||||
fmt.Fprintf(response, "%s_bucket{stage=%q,result=%q,le=\"+Inf\"} %d\n", metricName, stage, result, count)
|
||||
fmt.Fprintf(response, "%s_sum{stage=%q,result=%q} %f\n", metricName, stage, result, float64(histogram.sumNano.Load())/float64(time.Second))
|
||||
fmt.Fprintf(response, "%s_count{stage=%q,result=%q} %d\n", metricName, stage, result, count)
|
||||
}
|
||||
}
|
||||
|
||||
func durationBucketLabel(bucket time.Duration) string {
|
||||
return fmt.Sprintf("%g", bucket.Seconds())
|
||||
}
|
||||
|
||||
func boolIndex(value bool) int {
|
||||
if value {
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func inboundStageIndex(stage string) int {
|
||||
switch stage {
|
||||
case "decode":
|
||||
return 0
|
||||
case "api_roundtrip":
|
||||
return 1
|
||||
case "response_write":
|
||||
return 2
|
||||
case "handler_total":
|
||||
return 3
|
||||
default:
|
||||
return -1
|
||||
}
|
||||
}
|
||||
|
||||
func submitStageIndex(stage string) int {
|
||||
switch stage {
|
||||
case "stream_wait":
|
||||
return 0
|
||||
case "rate_limit_wait":
|
||||
return 1
|
||||
case "connection_wait":
|
||||
return 2
|
||||
case "supplier_rtt":
|
||||
return 3
|
||||
case "api_callback":
|
||||
return 4
|
||||
default:
|
||||
return -1
|
||||
}
|
||||
}
|
||||
|
||||
func boolNumber(value bool) int {
|
||||
if value {
|
||||
return 1
|
||||
|
||||
@@ -11,16 +11,34 @@ import (
|
||||
|
||||
func TestMetricsHandlerExportsOnlyAggregateGatewayState(t *testing.T) {
|
||||
ObserveSubmit(true, 20*time.Millisecond)
|
||||
ObserveInboundStage("api_roundtrip", true, 30*time.Millisecond)
|
||||
ObserveSubmitStage("rate_limit_wait", true, 15*time.Millisecond)
|
||||
request := httptest.NewRequest(http.MethodGet, "/metrics", nil)
|
||||
response := httptest.NewRecorder()
|
||||
Handler(func(context.Context) Snapshot {
|
||||
return Snapshot{UpstreamDesired: 2, UpstreamConnected: 1, DownstreamConnected: 3, SubmitWorkerUp: true, QueueAvailable: true, QueuePending: 4, QueueLag: 5, QueueOldestAgeSeconds: 12}
|
||||
return Snapshot{UpstreamDesired: 2, UpstreamConnected: 1, DownstreamConnected: 3, SubmitWorkerUp: true, SubmitWorkerConcurrency: 64, SubmitWorkerInFlight: 7, ResultWorkerUp: true, ResultWorkerConcurrency: 32, ResultWorkerInFlight: 5, InboundSubmitConcurrency: 96, InboundSubmitInFlight: 9, QueueAvailable: true, QueuePending: 4, QueueLag: 5, QueueOldestAgeSeconds: 12, ResultQueueAvailable: true, ResultQueuePending: 6, ResultQueueLag: 8}
|
||||
}).ServeHTTP(response, request)
|
||||
|
||||
body := response.Body.String()
|
||||
if response.Code != http.StatusOK || !strings.Contains(body, "cmpp_gateway_submit_queue_pending 4") || !strings.Contains(body, "cmpp_gateway_upstream_connections{state=\"connected\"} 1") {
|
||||
t.Fatalf("unexpected metrics response: code=%d body=%s", response.Code, body)
|
||||
}
|
||||
for _, expected := range []string{
|
||||
`cmpp_gateway_inbound_stage_duration_seconds_count{stage="api_roundtrip",result="success"} 1`,
|
||||
`cmpp_gateway_submit_stage_duration_seconds_count{stage="rate_limit_wait",result="success"} 1`,
|
||||
`cmpp_gateway_submit_worker_slots{state="configured"} 64`,
|
||||
`cmpp_gateway_submit_worker_slots{state="in_flight"} 7`,
|
||||
`cmpp_gateway_result_callback_worker_slots{state="configured"} 32`,
|
||||
`cmpp_gateway_result_callback_worker_slots{state="in_flight"} 5`,
|
||||
`cmpp_gateway_result_outbox_pending 6`,
|
||||
`cmpp_gateway_result_outbox_lag 8`,
|
||||
`cmpp_gateway_inbound_submit_slots{state="configured"} 96`,
|
||||
`cmpp_gateway_inbound_submit_slots{state="in_flight"} 9`,
|
||||
} {
|
||||
if !strings.Contains(body, expected) {
|
||||
t.Fatalf("metrics response is missing %q: %s", expected, body)
|
||||
}
|
||||
}
|
||||
for _, forbidden := range []string{"phone_number", "message_id", "channel_id"} {
|
||||
if strings.Contains(body, forbidden) {
|
||||
t.Fatalf("metrics expose forbidden label %q", forbidden)
|
||||
|
||||
@@ -0,0 +1,260 @@
|
||||
package resultoutbox
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"cmpp-platform/gateway/internal/queue"
|
||||
|
||||
"github.com/redis/go-redis/v9"
|
||||
)
|
||||
|
||||
const (
|
||||
defaultStream = "gateway.submit.results"
|
||||
defaultGroup = "cmpp-api-callback"
|
||||
defaultConsumer = "gateway-1"
|
||||
defaultDedupeTTL = 7 * 24 * time.Hour
|
||||
)
|
||||
|
||||
var publishScript = redis.NewScript(`
|
||||
local inserted = redis.call('SET', KEYS[2], '1', 'NX', 'EX', ARGV[1])
|
||||
if inserted then
|
||||
redis.call('XADD', KEYS[1], '*', 'data', ARGV[2])
|
||||
return 1
|
||||
end
|
||||
return 0
|
||||
`)
|
||||
|
||||
var publishAndAckScript = redis.NewScript(`
|
||||
local inserted = redis.call('SET', KEYS[2], '1', 'NX', 'EX', ARGV[1])
|
||||
if inserted then
|
||||
redis.call('XADD', KEYS[1], '*', 'data', ARGV[2])
|
||||
end
|
||||
redis.call('XACK', KEYS[3], ARGV[3], ARGV[4])
|
||||
if inserted then return 1 end
|
||||
return 0
|
||||
`)
|
||||
|
||||
type Event struct {
|
||||
SchemaVersion string `json:"schemaVersion"`
|
||||
EventID string `json:"eventId"`
|
||||
EventType string `json:"eventType"`
|
||||
Path string `json:"path"`
|
||||
TraceID string `json:"traceId,omitempty"`
|
||||
MessageID string `json:"messageId"`
|
||||
ChannelID string `json:"channelId"`
|
||||
SubmitID string `json:"submitId"`
|
||||
Payload json.RawMessage `json:"payload"`
|
||||
CreatedAt time.Time `json:"createdAt"`
|
||||
}
|
||||
|
||||
type Outbox struct {
|
||||
Redis *redis.Client
|
||||
Stream string
|
||||
Group string
|
||||
Consumer string
|
||||
DedupeTTL time.Duration
|
||||
APIBaseURL string
|
||||
HTTPTimeout time.Duration
|
||||
Concurrency int
|
||||
MinIdle time.Duration
|
||||
inFlight atomic.Int64
|
||||
}
|
||||
|
||||
func New(client *redis.Client) *Outbox {
|
||||
return &Outbox{Redis: client}
|
||||
}
|
||||
|
||||
func (o *Outbox) PublishSubmitSegment(ctx context.Context, command queue.SubmitCommand, segment queue.SubmitSegmentResult) error {
|
||||
payload := struct {
|
||||
queue.Envelope
|
||||
SubmitID string `json:"submitId,omitempty"`
|
||||
queue.SubmitSegmentResult
|
||||
}{
|
||||
Envelope: command.Envelope,
|
||||
SubmitID: command.SubmitID,
|
||||
SubmitSegmentResult: segment,
|
||||
}
|
||||
event, err := newEvent(
|
||||
fmt.Sprintf("submit:%s:segment:%d", command.SubmitID, segment.SegmentIndex),
|
||||
"submit_segment_result",
|
||||
"/gateway/events/submit-segment-result",
|
||||
command,
|
||||
payload,
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return o.publish(ctx, event)
|
||||
}
|
||||
|
||||
func (o *Outbox) PublishSubmitResultAndAck(
|
||||
ctx context.Context,
|
||||
commandStream string,
|
||||
commandGroup string,
|
||||
commandMessageID string,
|
||||
command queue.SubmitCommand,
|
||||
result queue.SubmitResult,
|
||||
) error {
|
||||
if o.Redis == nil {
|
||||
return fmt.Errorf("result Outbox Redis client is required")
|
||||
}
|
||||
event, err := newEvent(
|
||||
fmt.Sprintf("submit:%s:aggregate", command.SubmitID),
|
||||
"submit_result",
|
||||
"/gateway/events/submit-result",
|
||||
command,
|
||||
result,
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
data, err := json.Marshal(event)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// XADD and command XACK share one Redis script so a process crash cannot leave
|
||||
// an acknowledged supplier command without its aggregate result in the Outbox.
|
||||
_, err = publishAndAckScript.Run(
|
||||
ctx,
|
||||
o.Redis,
|
||||
[]string{o.stream(), o.dedupeKey(event.EventID), commandStream},
|
||||
int64(o.dedupeTTL().Seconds()),
|
||||
string(data),
|
||||
commandGroup,
|
||||
commandMessageID,
|
||||
).Result()
|
||||
return err
|
||||
}
|
||||
|
||||
func (o *Outbox) PublishSubmitResult(ctx context.Context, command queue.SubmitCommand, result queue.SubmitResult) error {
|
||||
event, err := newEvent(
|
||||
fmt.Sprintf("submit:%s:aggregate", command.SubmitID),
|
||||
"submit_result",
|
||||
"/gateway/events/submit-result",
|
||||
command,
|
||||
result,
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return o.publish(ctx, event)
|
||||
}
|
||||
|
||||
func (o *Outbox) publish(ctx context.Context, event Event) error {
|
||||
if o.Redis == nil {
|
||||
return fmt.Errorf("result Outbox Redis client is required")
|
||||
}
|
||||
data, err := json.Marshal(event)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = publishScript.Run(
|
||||
ctx,
|
||||
o.Redis,
|
||||
[]string{o.stream(), o.dedupeKey(event.EventID)},
|
||||
int64(o.dedupeTTL().Seconds()),
|
||||
string(data),
|
||||
).Result()
|
||||
return err
|
||||
}
|
||||
|
||||
func newEvent(eventID string, eventType string, path string, command queue.SubmitCommand, payload any) (Event, error) {
|
||||
data, err := json.Marshal(payload)
|
||||
if err != nil {
|
||||
return Event{}, err
|
||||
}
|
||||
var object map[string]interface{}
|
||||
if err := json.Unmarshal(data, &object); err != nil {
|
||||
return Event{}, err
|
||||
}
|
||||
// The API stores the deterministic ID on the submit attempt after successful
|
||||
// processing, making callback redelivery idempotent across Gateway restarts.
|
||||
object["eventId"] = eventID
|
||||
data, err = json.Marshal(object)
|
||||
if err != nil {
|
||||
return Event{}, err
|
||||
}
|
||||
return Event{
|
||||
SchemaVersion: queue.SchemaVersion,
|
||||
EventID: eventID,
|
||||
EventType: eventType,
|
||||
Path: path,
|
||||
TraceID: command.TraceID,
|
||||
MessageID: command.MessageID,
|
||||
ChannelID: command.ChannelID,
|
||||
SubmitID: command.SubmitID,
|
||||
Payload: data,
|
||||
CreatedAt: time.Now().UTC(),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func EventFromStreamValues(values map[string]interface{}) (Event, error) {
|
||||
raw, ok := values["data"]
|
||||
if !ok {
|
||||
return Event{}, fmt.Errorf("result Outbox data field is required")
|
||||
}
|
||||
var data string
|
||||
switch value := raw.(type) {
|
||||
case string:
|
||||
data = value
|
||||
case []byte:
|
||||
data = string(value)
|
||||
default:
|
||||
data = fmt.Sprint(value)
|
||||
}
|
||||
var event Event
|
||||
if err := json.Unmarshal([]byte(data), &event); err != nil {
|
||||
return Event{}, err
|
||||
}
|
||||
if event.SchemaVersion != queue.SchemaVersion || event.EventID == "" || event.MessageID == "" || event.SubmitID == "" {
|
||||
return Event{}, fmt.Errorf("invalid result Outbox envelope")
|
||||
}
|
||||
if event.Path != "/gateway/events/submit-result" && event.Path != "/gateway/events/submit-segment-result" {
|
||||
return Event{}, fmt.Errorf("unsupported result Outbox path %q", event.Path)
|
||||
}
|
||||
if len(event.Payload) == 0 {
|
||||
return Event{}, fmt.Errorf("result Outbox payload is required")
|
||||
}
|
||||
return event, nil
|
||||
}
|
||||
|
||||
func (o *Outbox) stream() string {
|
||||
if strings.TrimSpace(o.Stream) != "" {
|
||||
return o.Stream
|
||||
}
|
||||
return defaultStream
|
||||
}
|
||||
|
||||
func (o *Outbox) group() string {
|
||||
if strings.TrimSpace(o.Group) != "" {
|
||||
return o.Group
|
||||
}
|
||||
return defaultGroup
|
||||
}
|
||||
|
||||
func (o *Outbox) consumer() string {
|
||||
if strings.TrimSpace(o.Consumer) != "" {
|
||||
return o.Consumer
|
||||
}
|
||||
return defaultConsumer
|
||||
}
|
||||
|
||||
func (o *Outbox) dedupeTTL() time.Duration {
|
||||
if o.DedupeTTL > 0 {
|
||||
return o.DedupeTTL
|
||||
}
|
||||
return defaultDedupeTTL
|
||||
}
|
||||
|
||||
func (o *Outbox) dedupeKey(eventID string) string {
|
||||
return o.stream() + ":dedupe:" + eventID
|
||||
}
|
||||
|
||||
func (o *Outbox) StreamName() string { return o.stream() }
|
||||
func (o *Outbox) GroupName() string { return o.group() }
|
||||
func (o *Outbox) InFlight() int64 { return o.inFlight.Load() }
|
||||
@@ -0,0 +1,129 @@
|
||||
package resultoutbox
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"cmpp-platform/gateway/internal/queue"
|
||||
|
||||
"github.com/alicebob/miniredis/v2"
|
||||
"github.com/redis/go-redis/v9"
|
||||
)
|
||||
|
||||
func TestPublishSubmitSegmentIsIdempotent(t *testing.T) {
|
||||
mr := miniredis.RunT(t)
|
||||
client := redis.NewClient(&redis.Options{Addr: mr.Addr()})
|
||||
outbox := New(client)
|
||||
command := testCommand()
|
||||
segment := queue.SubmitSegmentResult{SegmentTotal: 1, SegmentIndex: 1, SequenceID: 7, GatewayMessageID: "88", SubmitStatus: "accepted"}
|
||||
|
||||
if err := outbox.PublishSubmitSegment(context.Background(), command, segment); err != nil {
|
||||
t.Fatalf("first publish: %v", err)
|
||||
}
|
||||
if err := outbox.PublishSubmitSegment(context.Background(), command, segment); err != nil {
|
||||
t.Fatalf("duplicate publish: %v", err)
|
||||
}
|
||||
if got := client.XLen(context.Background(), outbox.StreamName()).Val(); got != 1 {
|
||||
t.Fatalf("stream length = %d, want 1", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPublishAggregateAndCommandAckAreAtomicAndIdempotent(t *testing.T) {
|
||||
mr := miniredis.RunT(t)
|
||||
client := redis.NewClient(&redis.Options{Addr: mr.Addr()})
|
||||
ctx := context.Background()
|
||||
commandStream := "gateway.submit.commands"
|
||||
commandGroup := "cmpp-gateway"
|
||||
if err := client.XGroupCreateMkStream(ctx, commandStream, commandGroup, "0").Err(); err != nil {
|
||||
t.Fatalf("create command group: %v", err)
|
||||
}
|
||||
commandID, err := client.XAdd(ctx, &redis.XAddArgs{Stream: commandStream, Values: map[string]interface{}{"data": "command"}}).Result()
|
||||
if err != nil {
|
||||
t.Fatalf("add command: %v", err)
|
||||
}
|
||||
if _, err := client.XReadGroup(ctx, &redis.XReadGroupArgs{Group: commandGroup, Consumer: "gateway-1", Streams: []string{commandStream, ">"}, Count: 1}).Result(); err != nil {
|
||||
t.Fatalf("claim command: %v", err)
|
||||
}
|
||||
outbox := New(client)
|
||||
command := testCommand()
|
||||
result := queue.SubmitResult{Envelope: command.Envelope, SubmitID: command.SubmitID, GatewayMessageID: "99", SubmitStatus: "accepted"}
|
||||
|
||||
for attempt := 0; attempt < 2; attempt++ {
|
||||
if err := outbox.PublishSubmitResultAndAck(ctx, commandStream, commandGroup, commandID, command, result); err != nil {
|
||||
t.Fatalf("publish attempt %d: %v", attempt+1, err)
|
||||
}
|
||||
}
|
||||
pending, err := client.XPending(ctx, commandStream, commandGroup).Result()
|
||||
if err != nil {
|
||||
t.Fatalf("command pending: %v", err)
|
||||
}
|
||||
if pending.Count != 0 {
|
||||
t.Fatalf("command pending = %d, want 0", pending.Count)
|
||||
}
|
||||
if got := client.XLen(ctx, outbox.StreamName()).Val(); got != 1 {
|
||||
t.Fatalf("result stream length = %d, want 1", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCallbackWorkerRetriesAndOnlyDeletesAfterSuccess(t *testing.T) {
|
||||
mr := miniredis.RunT(t)
|
||||
client := redis.NewClient(&redis.Options{Addr: mr.Addr()})
|
||||
var calls atomic.Int32
|
||||
server := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) {
|
||||
if request.Header.Get("X-CMPP-Result-Event-ID") == "" {
|
||||
t.Error("missing result event id header")
|
||||
}
|
||||
if calls.Add(1) == 1 {
|
||||
http.Error(response, "temporary failure", http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
response.WriteHeader(http.StatusCreated)
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
outbox := New(client)
|
||||
outbox.APIBaseURL = server.URL
|
||||
outbox.MinIdle = 10 * time.Millisecond
|
||||
outbox.Concurrency = 1
|
||||
command := testCommand()
|
||||
if err := outbox.PublishSubmitSegment(context.Background(), command, queue.SubmitSegmentResult{
|
||||
SegmentTotal: 1, SegmentIndex: 1, SequenceID: 7, GatewayMessageID: "88", SubmitStatus: "accepted",
|
||||
}); err != nil {
|
||||
t.Fatalf("publish segment: %v", err)
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- outbox.Run(ctx) }()
|
||||
deadline := time.Now().Add(5 * time.Second)
|
||||
for calls.Load() < 2 || client.XLen(context.Background(), outbox.StreamName()).Val() != 0 {
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatalf("calls=%d streamLength=%d", calls.Load(), client.XLen(context.Background(), outbox.StreamName()).Val())
|
||||
}
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
cancel()
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(3 * time.Second):
|
||||
t.Fatal("worker did not stop")
|
||||
}
|
||||
}
|
||||
|
||||
func testCommand() queue.SubmitCommand {
|
||||
return queue.SubmitCommand{
|
||||
Envelope: queue.Envelope{
|
||||
SchemaVersion: queue.SchemaVersion,
|
||||
MessageType: queue.MessageTypeSubmitCommand,
|
||||
TraceID: "trace-1",
|
||||
MessageID: "message-1",
|
||||
ChannelID: "channel-1",
|
||||
CreatedAt: time.Now().UTC(),
|
||||
},
|
||||
SubmitID: "submit-1",
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,275 @@
|
||||
package resultoutbox
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"net/http"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"cmpp-platform/gateway/internal/metrics"
|
||||
|
||||
"github.com/redis/go-redis/v9"
|
||||
)
|
||||
|
||||
const (
|
||||
// Eight callbacks matched the test VM's API/PostgreSQL capacity through
|
||||
// 40 rps. A larger default caused callback bursts to contend with inbound
|
||||
// persistence; operators can still raise it after measuring both queues.
|
||||
defaultConcurrency = 8
|
||||
// Keep the reclaim threshold above the callback timeout. Otherwise another
|
||||
// Gateway replica could reclaim a still-running callback and execute the same
|
||||
// business transition concurrently before the API stores its idempotency ID.
|
||||
defaultMinIdle = 30 * time.Second
|
||||
defaultBlock = 2 * time.Second
|
||||
defaultHTTPTimeout = 10 * time.Second
|
||||
)
|
||||
|
||||
var acknowledgeAndDeleteScript = redis.NewScript(`
|
||||
redis.call('XACK', KEYS[1], ARGV[1], ARGV[2])
|
||||
redis.call('XDEL', KEYS[1], ARGV[2])
|
||||
return 1
|
||||
`)
|
||||
|
||||
func (o *Outbox) Run(ctx context.Context) error {
|
||||
if o.Redis == nil {
|
||||
return fmt.Errorf("result Outbox Redis client is required")
|
||||
}
|
||||
if strings.TrimSpace(o.APIBaseURL) == "" {
|
||||
return fmt.Errorf("result Outbox API base URL is required")
|
||||
}
|
||||
if err := o.ensureGroup(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
pool := newCallbackPool(ctx, o, o.concurrency())
|
||||
defer pool.wait()
|
||||
for {
|
||||
if err := o.recoverPending(ctx, pool); err != nil && ctx.Err() == nil {
|
||||
log.Printf("gateway result Outbox pending recovery failed: %v", err)
|
||||
sleep(ctx, time.Second)
|
||||
continue
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
return ctx.Err()
|
||||
}
|
||||
if err := o.consumeOnce(ctx, pool); err != nil && ctx.Err() == nil {
|
||||
log.Printf("gateway result Outbox consume failed: %v", err)
|
||||
sleep(ctx, time.Second)
|
||||
continue
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
return ctx.Err()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (o *Outbox) ensureGroup(ctx context.Context) error {
|
||||
err := o.Redis.XGroupCreateMkStream(ctx, o.stream(), o.group(), "0").Err()
|
||||
if err == nil || strings.Contains(err.Error(), "BUSYGROUP") {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func (o *Outbox) consumeOnce(ctx context.Context, pool *callbackPool) error {
|
||||
available, err := pool.waitForCapacity(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
streams, err := o.Redis.XReadGroup(ctx, &redis.XReadGroupArgs{
|
||||
Group: o.group(), Consumer: o.consumer(), Streams: []string{o.stream(), ">"},
|
||||
Count: int64(available), Block: defaultBlock,
|
||||
}).Result()
|
||||
if errors.Is(err, redis.Nil) {
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, stream := range streams {
|
||||
for _, message := range stream.Messages {
|
||||
if !pool.dispatch(message) {
|
||||
return fmt.Errorf("gateway result Outbox capacity accounting mismatch")
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (o *Outbox) recoverPending(ctx context.Context, pool *callbackPool) error {
|
||||
if pool.available() == 0 {
|
||||
return nil
|
||||
}
|
||||
messages, _, err := o.Redis.XAutoClaim(ctx, &redis.XAutoClaimArgs{
|
||||
Stream: o.stream(), Group: o.group(), Consumer: o.consumer(), MinIdle: o.minIdle(),
|
||||
Start: "0-0", Count: int64(pool.available()),
|
||||
}).Result()
|
||||
if errors.Is(err, redis.Nil) {
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, message := range messages {
|
||||
if !pool.dispatch(message) {
|
||||
return fmt.Errorf("gateway result Outbox recovery capacity accounting mismatch")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (o *Outbox) processMessage(ctx context.Context, message redis.XMessage) error {
|
||||
event, err := EventFromStreamValues(message.Values)
|
||||
if err != nil {
|
||||
// Malformed internal events cannot be delivered. Keep them pending for operator
|
||||
// evidence instead of ACKing and silently losing a supplier result.
|
||||
return err
|
||||
}
|
||||
startedAt := time.Now()
|
||||
err = o.post(ctx, event)
|
||||
metrics.ObserveSubmitStage("api_callback", err == nil, time.Since(startedAt))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// Result events have a bounded dedupe key, so successful callbacks can be
|
||||
// ACKed and deleted atomically instead of turning the Outbox into an archive.
|
||||
return acknowledgeAndDeleteScript.Run(
|
||||
ctx,
|
||||
o.Redis,
|
||||
[]string{o.stream()},
|
||||
o.group(),
|
||||
message.ID,
|
||||
).Err()
|
||||
}
|
||||
|
||||
func (o *Outbox) post(ctx context.Context, event Event) error {
|
||||
client := &http.Client{Timeout: o.httpTimeout()}
|
||||
req, err := http.NewRequestWithContext(
|
||||
ctx,
|
||||
http.MethodPost,
|
||||
strings.TrimRight(o.APIBaseURL, "/")+event.Path,
|
||||
bytes.NewReader(event.Payload),
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.Header.Set("X-CMPP-Result-Event-ID", event.EventID)
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices {
|
||||
body, _ := io.ReadAll(io.LimitReader(resp.Body, 512))
|
||||
return fmt.Errorf("result callback returned %d: %s", resp.StatusCode, strings.TrimSpace(string(body)))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type callbackPool struct {
|
||||
ctx context.Context
|
||||
outbox *Outbox
|
||||
slots chan struct{}
|
||||
completed chan struct{}
|
||||
group sync.WaitGroup
|
||||
mu sync.Mutex
|
||||
active map[string]struct{}
|
||||
}
|
||||
|
||||
func newCallbackPool(ctx context.Context, outbox *Outbox, concurrency int) *callbackPool {
|
||||
return &callbackPool{
|
||||
ctx: ctx, outbox: outbox, slots: make(chan struct{}, concurrency), completed: make(chan struct{}, concurrency), active: make(map[string]struct{}),
|
||||
}
|
||||
}
|
||||
|
||||
func (p *callbackPool) available() int { return cap(p.slots) - len(p.slots) }
|
||||
|
||||
func (p *callbackPool) waitForCapacity(ctx context.Context) (int, error) {
|
||||
for p.available() == 0 {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return 0, ctx.Err()
|
||||
case <-p.completed:
|
||||
}
|
||||
}
|
||||
return p.available(), nil
|
||||
}
|
||||
|
||||
func (p *callbackPool) dispatch(message redis.XMessage) bool {
|
||||
p.mu.Lock()
|
||||
if _, exists := p.active[message.ID]; exists {
|
||||
p.mu.Unlock()
|
||||
return true
|
||||
}
|
||||
select {
|
||||
case p.slots <- struct{}{}:
|
||||
p.active[message.ID] = struct{}{}
|
||||
p.outbox.inFlight.Add(1)
|
||||
p.group.Add(1)
|
||||
p.mu.Unlock()
|
||||
case <-p.ctx.Done():
|
||||
p.mu.Unlock()
|
||||
return false
|
||||
default:
|
||||
p.mu.Unlock()
|
||||
return false
|
||||
}
|
||||
go func() {
|
||||
defer func() {
|
||||
p.mu.Lock()
|
||||
delete(p.active, message.ID)
|
||||
p.mu.Unlock()
|
||||
<-p.slots
|
||||
p.outbox.inFlight.Add(-1)
|
||||
select {
|
||||
case p.completed <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
p.group.Done()
|
||||
}()
|
||||
if err := p.outbox.processMessage(p.ctx, message); err != nil {
|
||||
log.Printf("gateway result Outbox event %s failed: %v", message.ID, err)
|
||||
}
|
||||
}()
|
||||
return true
|
||||
}
|
||||
|
||||
func (p *callbackPool) wait() { p.group.Wait() }
|
||||
|
||||
func (o *Outbox) concurrency() int {
|
||||
if o.Concurrency > 0 {
|
||||
return min(o.Concurrency, 1024)
|
||||
}
|
||||
return defaultConcurrency
|
||||
}
|
||||
|
||||
func (o *Outbox) minIdle() time.Duration {
|
||||
if o.MinIdle > 0 {
|
||||
return o.MinIdle
|
||||
}
|
||||
return defaultMinIdle
|
||||
}
|
||||
|
||||
func (o *Outbox) httpTimeout() time.Duration {
|
||||
if o.HTTPTimeout > 0 {
|
||||
return o.HTTPTimeout
|
||||
}
|
||||
return defaultHTTPTimeout
|
||||
}
|
||||
|
||||
func (o *Outbox) ConfiguredConcurrency() int { return o.concurrency() }
|
||||
|
||||
func sleep(ctx context.Context, duration time.Duration) {
|
||||
timer := time.NewTimer(duration)
|
||||
defer timer.Stop()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
case <-timer.C:
|
||||
}
|
||||
}
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
"net/http"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"cmpp-platform/gateway/internal/metrics"
|
||||
@@ -21,11 +22,12 @@ import (
|
||||
)
|
||||
|
||||
const (
|
||||
defaultStream = "gateway.submit.commands"
|
||||
defaultGroup = "cmpp-gateway"
|
||||
defaultConsumer = "gateway-1"
|
||||
defaultMinIdle = 30 * time.Second
|
||||
defaultMaxFails = 3
|
||||
defaultStream = "gateway.submit.commands"
|
||||
defaultGroup = "cmpp-gateway"
|
||||
defaultConsumer = "gateway-1"
|
||||
defaultMinIdle = 30 * time.Second
|
||||
defaultMaxFails = 3
|
||||
defaultConcurrency = 64
|
||||
)
|
||||
|
||||
type Worker struct {
|
||||
@@ -34,16 +36,30 @@ type Worker struct {
|
||||
Limiter ratelimit.Limiter
|
||||
Submit func(context.Context, queue.SubmitCommand) (queue.SubmitResult, error)
|
||||
ReportDeadLetter func(context.Context, DeadLetterEvent) error
|
||||
ResultOutbox SubmitResultOutbox
|
||||
Stream string
|
||||
Group string
|
||||
Consumer string
|
||||
Block time.Duration
|
||||
Count int64
|
||||
Concurrency int
|
||||
MinIdle time.Duration
|
||||
MaxFailures int
|
||||
APIBaseURL string
|
||||
HTTPClient *http.Client
|
||||
Logger *log.Logger
|
||||
inFlight atomic.Int64
|
||||
}
|
||||
|
||||
type SubmitResultOutbox interface {
|
||||
PublishSubmitResultAndAck(
|
||||
context.Context,
|
||||
string,
|
||||
string,
|
||||
string,
|
||||
queue.SubmitCommand,
|
||||
queue.SubmitResult,
|
||||
) error
|
||||
}
|
||||
|
||||
type DeadLetterEvent struct {
|
||||
@@ -78,6 +94,11 @@ func (w *Worker) Run(ctx context.Context) error {
|
||||
if w.Upstream == nil {
|
||||
return fmt.Errorf("upstream manager is required")
|
||||
}
|
||||
if w.ResultOutbox == nil {
|
||||
return fmt.Errorf("submit result Outbox is required")
|
||||
}
|
||||
pool := newMessageWorkPool(ctx, w, w.concurrency())
|
||||
defer pool.wait()
|
||||
for {
|
||||
if err := w.ensureGroup(ctx); err != nil {
|
||||
if ctx.Err() != nil {
|
||||
@@ -87,7 +108,7 @@ func (w *Worker) Run(ctx context.Context) error {
|
||||
sleep(ctx, 3*time.Second)
|
||||
continue
|
||||
}
|
||||
if err := w.recoverPending(ctx); err != nil {
|
||||
if err := w.recoverPending(ctx, pool); err != nil {
|
||||
if ctx.Err() != nil {
|
||||
return ctx.Err()
|
||||
}
|
||||
@@ -95,7 +116,7 @@ func (w *Worker) Run(ctx context.Context) error {
|
||||
sleep(ctx, time.Second)
|
||||
continue
|
||||
}
|
||||
if err := w.consumeOnce(ctx); err != nil {
|
||||
if err := w.consumeOnce(ctx, pool); err != nil {
|
||||
if ctx.Err() != nil {
|
||||
return ctx.Err()
|
||||
}
|
||||
@@ -113,12 +134,16 @@ func (w *Worker) ensureGroup(ctx context.Context) error {
|
||||
return err
|
||||
}
|
||||
|
||||
func (w *Worker) consumeOnce(ctx context.Context) error {
|
||||
func (w *Worker) consumeOnce(ctx context.Context, pool *messageWorkPool) error {
|
||||
available, err := pool.waitForCapacity(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
streams, err := w.Redis.XReadGroup(ctx, &redis.XReadGroupArgs{
|
||||
Group: w.group(),
|
||||
Consumer: w.consumer(),
|
||||
Streams: []string{w.stream(), ">"},
|
||||
Count: w.count(),
|
||||
Count: min(w.count(), int64(available)),
|
||||
Block: w.block(),
|
||||
}).Result()
|
||||
if errors.Is(err, redis.Nil) {
|
||||
@@ -128,23 +153,29 @@ func (w *Worker) consumeOnce(ctx context.Context) error {
|
||||
return err
|
||||
}
|
||||
for _, stream := range streams {
|
||||
if err := w.processMessages(ctx, stream.Messages); err != nil {
|
||||
return err
|
||||
for _, message := range stream.Messages {
|
||||
if !pool.dispatch(message) {
|
||||
return fmt.Errorf("gateway submit worker capacity accounting mismatch")
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (w *Worker) recoverPending(ctx context.Context) error {
|
||||
func (w *Worker) recoverPending(ctx context.Context, pool *messageWorkPool) error {
|
||||
start := "0-0"
|
||||
for {
|
||||
available := pool.available()
|
||||
if available == 0 {
|
||||
return nil
|
||||
}
|
||||
messages, next, err := w.Redis.XAutoClaim(ctx, &redis.XAutoClaimArgs{
|
||||
Stream: w.stream(),
|
||||
Group: w.group(),
|
||||
Consumer: w.consumer(),
|
||||
MinIdle: w.minIdle(),
|
||||
Start: start,
|
||||
Count: w.count(),
|
||||
Count: min(w.count(), int64(available)),
|
||||
}).Result()
|
||||
if errors.Is(err, redis.Nil) {
|
||||
return nil
|
||||
@@ -156,8 +187,13 @@ func (w *Worker) recoverPending(ctx context.Context) error {
|
||||
return nil
|
||||
}
|
||||
w.logf("gateway submit worker reclaimed %d pending message(s)", len(messages))
|
||||
if err := w.processMessages(ctx, messages); err != nil {
|
||||
return err
|
||||
for _, message := range messages {
|
||||
// An in-flight command can legitimately exceed MinIdle while waiting on a supplier.
|
||||
// Rechecking both the local active set and Redis PEL closes the race where the
|
||||
// original attempt ACKs between XAUTOCLAIM returning and local dispatch.
|
||||
if err := pool.dispatchRecovered(ctx, message); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
start = next
|
||||
if next == "0-0" {
|
||||
@@ -166,28 +202,113 @@ func (w *Worker) recoverPending(ctx context.Context) error {
|
||||
}
|
||||
}
|
||||
|
||||
func (w *Worker) processMessages(ctx context.Context, messages []redis.XMessage) error {
|
||||
var group sync.WaitGroup
|
||||
for _, message := range messages {
|
||||
message := message
|
||||
group.Add(1)
|
||||
go func() {
|
||||
defer group.Done()
|
||||
if err := w.processMessage(ctx, message); err != nil {
|
||||
w.logf("gateway submit worker message %s failed: %v", message.ID, err)
|
||||
}
|
||||
}()
|
||||
type messageWorkPool struct {
|
||||
ctx context.Context
|
||||
worker *Worker
|
||||
slots chan struct{}
|
||||
completed chan struct{}
|
||||
group sync.WaitGroup
|
||||
mu sync.Mutex
|
||||
active map[string]struct{}
|
||||
}
|
||||
|
||||
func newMessageWorkPool(ctx context.Context, worker *Worker, concurrency int) *messageWorkPool {
|
||||
return &messageWorkPool{
|
||||
ctx: ctx, worker: worker, slots: make(chan struct{}, concurrency),
|
||||
completed: make(chan struct{}, concurrency), active: make(map[string]struct{}),
|
||||
}
|
||||
}
|
||||
|
||||
func (p *messageWorkPool) available() int {
|
||||
return cap(p.slots) - len(p.slots)
|
||||
}
|
||||
|
||||
func (p *messageWorkPool) waitForCapacity(ctx context.Context) (int, error) {
|
||||
for p.available() == 0 {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return 0, ctx.Err()
|
||||
case <-p.completed:
|
||||
}
|
||||
}
|
||||
return p.available(), nil
|
||||
}
|
||||
|
||||
func (p *messageWorkPool) dispatch(message redis.XMessage) bool {
|
||||
p.mu.Lock()
|
||||
if _, exists := p.active[message.ID]; exists {
|
||||
p.mu.Unlock()
|
||||
return true
|
||||
}
|
||||
select {
|
||||
case p.slots <- struct{}{}:
|
||||
p.active[message.ID] = struct{}{}
|
||||
p.worker.inFlight.Add(1)
|
||||
p.group.Add(1)
|
||||
p.mu.Unlock()
|
||||
case <-p.ctx.Done():
|
||||
p.mu.Unlock()
|
||||
return false
|
||||
default:
|
||||
p.mu.Unlock()
|
||||
return false
|
||||
}
|
||||
go func() {
|
||||
defer func() {
|
||||
p.mu.Lock()
|
||||
delete(p.active, message.ID)
|
||||
p.mu.Unlock()
|
||||
<-p.slots
|
||||
p.worker.inFlight.Add(-1)
|
||||
select {
|
||||
case p.completed <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
p.group.Done()
|
||||
}()
|
||||
if err := p.worker.processMessage(p.ctx, message); err != nil {
|
||||
p.worker.logf("gateway submit worker message %s failed: %v", message.ID, err)
|
||||
}
|
||||
}()
|
||||
return true
|
||||
}
|
||||
|
||||
func (p *messageWorkPool) dispatchRecovered(ctx context.Context, message redis.XMessage) error {
|
||||
p.mu.Lock()
|
||||
_, active := p.active[message.ID]
|
||||
p.mu.Unlock()
|
||||
if active {
|
||||
return nil
|
||||
}
|
||||
pending, err := p.worker.Redis.XPendingExt(ctx, &redis.XPendingExtArgs{
|
||||
Stream: p.worker.stream(), Group: p.worker.group(), Start: message.ID, End: message.ID, Count: 1,
|
||||
}).Result()
|
||||
if err != nil && !errors.Is(err, redis.Nil) {
|
||||
return err
|
||||
}
|
||||
if len(pending) == 0 {
|
||||
return nil
|
||||
}
|
||||
if !p.dispatch(message) {
|
||||
return fmt.Errorf("gateway submit worker recovery capacity accounting mismatch")
|
||||
}
|
||||
group.Wait()
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *messageWorkPool) wait() {
|
||||
p.group.Wait()
|
||||
}
|
||||
|
||||
func (w *Worker) processMessage(ctx context.Context, message redis.XMessage) error {
|
||||
command, err := CommandFromStreamValues(message.Values)
|
||||
if err != nil {
|
||||
return w.deadLetterMalformedMessage(ctx, message, err)
|
||||
}
|
||||
if err := w.handleCommand(ctx, command); err != nil {
|
||||
if !command.CreatedAt.IsZero() {
|
||||
metrics.ObserveSubmitStage("stream_wait", true, time.Since(command.CreatedAt))
|
||||
}
|
||||
result, err := w.executeCommand(ctx, command)
|
||||
if err != nil && (result.SubmitStatus == "" || result.SubmitStatus == "accepted") {
|
||||
if ctx.Err() != nil {
|
||||
return ctx.Err()
|
||||
}
|
||||
@@ -203,26 +324,50 @@ func (w *Worker) processMessage(ctx context.Context, message redis.XMessage) err
|
||||
}
|
||||
return err
|
||||
}
|
||||
return w.ackAndClearFailure(ctx, message.ID)
|
||||
if err := w.ResultOutbox.PublishSubmitResultAndAck(
|
||||
ctx,
|
||||
w.stream(),
|
||||
w.group(),
|
||||
message.ID,
|
||||
command,
|
||||
result,
|
||||
); err != nil {
|
||||
return fmt.Errorf("persist aggregate submit result: %w", err)
|
||||
}
|
||||
// The command was ACKed atomically with the aggregate result event. Clearing the
|
||||
// auxiliary retry counter may lag without affecting delivery correctness.
|
||||
w.clearFailure(ctx, message.ID)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (w *Worker) handleCommand(ctx context.Context, command queue.SubmitCommand) error {
|
||||
func (w *Worker) executeCommand(ctx context.Context, command queue.SubmitCommand) (queue.SubmitResult, error) {
|
||||
startedAt := time.Now()
|
||||
limitStartedAt := time.Now()
|
||||
if w.Limiter != nil {
|
||||
if _, err := w.Limiter.Wait(ctx, command.ChannelID, command.Route.RateLimitPerSecond); err != nil {
|
||||
return err
|
||||
metrics.ObserveSubmitStage("rate_limit_wait", false, time.Since(limitStartedAt))
|
||||
return queue.SubmitResult{}, err
|
||||
}
|
||||
}
|
||||
metrics.ObserveSubmitStage("rate_limit_wait", true, time.Since(limitStartedAt))
|
||||
submit := w.Submit
|
||||
if submit == nil {
|
||||
if w.Upstream == nil {
|
||||
return fmt.Errorf("upstream manager is required")
|
||||
return queue.SubmitResult{}, fmt.Errorf("upstream manager is required")
|
||||
}
|
||||
submit = w.Upstream.Submit
|
||||
}
|
||||
result, err := submit(ctx, command)
|
||||
accepted := err == nil && (result.SubmitStatus == "" || result.SubmitStatus == "accepted")
|
||||
metrics.ObserveSubmit(accepted, time.Since(startedAt))
|
||||
return result, err
|
||||
}
|
||||
|
||||
// handleCommand remains a narrow compatibility seam for focused tests and the
|
||||
// control path. Stream consumption uses executeCommand so it can persist the
|
||||
// exact terminal result before acknowledging the command.
|
||||
func (w *Worker) handleCommand(ctx context.Context, command queue.SubmitCommand) error {
|
||||
result, err := w.executeCommand(ctx, command)
|
||||
if err != nil && (result.SubmitStatus == "" || result.SubmitStatus == "accepted") {
|
||||
return err
|
||||
}
|
||||
@@ -348,10 +493,14 @@ func (w *Worker) ackAndClearFailure(ctx context.Context, messageID string) error
|
||||
if err := w.Redis.XAck(ctx, w.stream(), w.group(), messageID).Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
w.clearFailure(ctx, messageID)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (w *Worker) clearFailure(ctx context.Context, messageID string) {
|
||||
if err := w.Redis.HDel(ctx, w.failureAttemptsKey(), messageID).Err(); err != nil {
|
||||
w.logf("gateway submit worker clear failure %s failed: %v", messageID, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (w *Worker) stream() string {
|
||||
@@ -389,6 +538,21 @@ func (w *Worker) count() int64 {
|
||||
return 10
|
||||
}
|
||||
|
||||
func (w *Worker) concurrency() int {
|
||||
if w.Concurrency > 0 {
|
||||
return min(w.Concurrency, 1024)
|
||||
}
|
||||
return defaultConcurrency
|
||||
}
|
||||
|
||||
func (w *Worker) ConfiguredConcurrency() int {
|
||||
return w.concurrency()
|
||||
}
|
||||
|
||||
func (w *Worker) InFlight() int64 {
|
||||
return w.inFlight.Load()
|
||||
}
|
||||
|
||||
func (w *Worker) minIdle() time.Duration {
|
||||
if w.MinIdle > 0 {
|
||||
return w.MinIdle
|
||||
|
||||
@@ -2,6 +2,7 @@ package submitworker
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -16,6 +17,24 @@ type recordingLimiter struct {
|
||||
called bool
|
||||
}
|
||||
|
||||
type acknowledgingResultOutbox struct {
|
||||
client *redis.Client
|
||||
}
|
||||
|
||||
func (o acknowledgingResultOutbox) PublishSubmitResultAndAck(
|
||||
ctx context.Context,
|
||||
commandStream string,
|
||||
commandGroup string,
|
||||
commandMessageID string,
|
||||
_ queue.SubmitCommand,
|
||||
_ queue.SubmitResult,
|
||||
) error {
|
||||
if o.client == nil {
|
||||
return nil
|
||||
}
|
||||
return o.client.XAck(ctx, commandStream, commandGroup, commandMessageID).Err()
|
||||
}
|
||||
|
||||
func (l *recordingLimiter) Wait(_ context.Context, channelID string, rate int) (time.Duration, error) {
|
||||
l.called = true
|
||||
l.channelID = channelID
|
||||
@@ -121,14 +140,25 @@ func TestMinIdleDefaultsToThirtySeconds(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessMessagesDoesNotLetOneChannelBlockAnother(t *testing.T) {
|
||||
func TestConcurrencyUsesBoundedDefaultAndMaximum(t *testing.T) {
|
||||
if got := (&Worker{}).ConfiguredConcurrency(); got != defaultConcurrency {
|
||||
t.Fatalf("default concurrency = %d, want %d", got, defaultConcurrency)
|
||||
}
|
||||
if got := (&Worker{Concurrency: 2048}).ConfiguredConcurrency(); got != 1024 {
|
||||
t.Fatalf("capped concurrency = %d, want 1024", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMessageWorkPoolContinuouslyRefillsWithoutWaitingForSlowSibling(t *testing.T) {
|
||||
mr := miniredis.RunT(t)
|
||||
client := redis.NewClient(&redis.Options{Addr: mr.Addr()})
|
||||
startedA := make(chan struct{})
|
||||
startedB := make(chan struct{})
|
||||
startedC := make(chan struct{})
|
||||
releaseA := make(chan struct{})
|
||||
worker := &Worker{
|
||||
Redis: client,
|
||||
Redis: client,
|
||||
ResultOutbox: acknowledgingResultOutbox{client: client},
|
||||
Submit: func(_ context.Context, command queue.SubmitCommand) (queue.SubmitResult, error) {
|
||||
switch command.ChannelID {
|
||||
case "channel-a":
|
||||
@@ -136,19 +166,17 @@ func TestProcessMessagesDoesNotLetOneChannelBlockAnother(t *testing.T) {
|
||||
<-releaseA
|
||||
case "channel-b":
|
||||
close(startedB)
|
||||
case "channel-c":
|
||||
close(startedC)
|
||||
}
|
||||
return queue.SubmitResult{SubmitStatus: "accepted"}, nil
|
||||
},
|
||||
}
|
||||
messages := []redis.XMessage{
|
||||
{ID: "1-0", Values: submitCommandValues("message-a", "channel-a")},
|
||||
{ID: "2-0", Values: submitCommandValues("message-b", "channel-b")},
|
||||
pool := newMessageWorkPool(context.Background(), worker, 2)
|
||||
if !pool.dispatch(redis.XMessage{ID: "1-0", Values: submitCommandValues("message-a", "channel-a")}) ||
|
||||
!pool.dispatch(redis.XMessage{ID: "2-0", Values: submitCommandValues("message-b", "channel-b")}) {
|
||||
t.Fatal("initial messages were not dispatched")
|
||||
}
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
_ = worker.processMessages(context.Background(), messages)
|
||||
close(done)
|
||||
}()
|
||||
select {
|
||||
case <-startedA:
|
||||
case <-time.After(time.Second):
|
||||
@@ -159,11 +187,125 @@ func TestProcessMessagesDoesNotLetOneChannelBlockAnother(t *testing.T) {
|
||||
case <-time.After(200 * time.Millisecond):
|
||||
t.Fatal("channel-b was blocked by channel-a")
|
||||
}
|
||||
close(releaseA)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
|
||||
defer cancel()
|
||||
if _, err := pool.waitForCapacity(ctx); err != nil {
|
||||
t.Fatalf("wait for refill capacity: %v", err)
|
||||
}
|
||||
if !pool.dispatch(redis.XMessage{ID: "3-0", Values: submitCommandValues("message-c", "channel-c")}) {
|
||||
t.Fatal("refill message was not dispatched")
|
||||
}
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("message batch did not complete")
|
||||
case <-startedC:
|
||||
case <-time.After(200 * time.Millisecond):
|
||||
t.Fatal("pool waited for the slow sibling instead of refilling its free slot")
|
||||
}
|
||||
close(releaseA)
|
||||
pool.wait()
|
||||
}
|
||||
|
||||
func TestMessageWorkPoolAcknowledgesFastMessageBeforeSlowSiblingCompletes(t *testing.T) {
|
||||
mr := miniredis.RunT(t)
|
||||
client := redis.NewClient(&redis.Options{Addr: mr.Addr()})
|
||||
worker := &Worker{Redis: client, Stream: "gateway.submit.commands", Group: "cmpp-gateway", ResultOutbox: acknowledgingResultOutbox{client: client}}
|
||||
ctx := context.Background()
|
||||
if err := worker.ensureGroup(ctx); err != nil {
|
||||
t.Fatalf("ensureGroup: %v", err)
|
||||
}
|
||||
for _, entry := range []struct{ id, messageID, channelID string }{
|
||||
{"1-0", "message-slow", "channel-slow"},
|
||||
{"2-0", "message-fast", "channel-fast"},
|
||||
} {
|
||||
if err := client.XAdd(ctx, &redis.XAddArgs{Stream: worker.stream(), ID: entry.id, Values: submitCommandValues(entry.messageID, entry.channelID)}).Err(); err != nil {
|
||||
t.Fatalf("xadd %s: %v", entry.id, err)
|
||||
}
|
||||
}
|
||||
streams, err := client.XReadGroup(ctx, &redis.XReadGroupArgs{Group: worker.group(), Consumer: worker.consumer(), Streams: []string{worker.stream(), ">"}, Count: 2}).Result()
|
||||
if err != nil || len(streams) != 1 || len(streams[0].Messages) != 2 {
|
||||
t.Fatalf("xreadgroup: streams=%+v err=%v", streams, err)
|
||||
}
|
||||
slowStarted := make(chan struct{})
|
||||
fastReturned := make(chan struct{})
|
||||
releaseSlow := make(chan struct{})
|
||||
worker.Submit = func(_ context.Context, command queue.SubmitCommand) (queue.SubmitResult, error) {
|
||||
if command.ChannelID == "channel-slow" {
|
||||
close(slowStarted)
|
||||
<-releaseSlow
|
||||
} else {
|
||||
close(fastReturned)
|
||||
}
|
||||
return queue.SubmitResult{SubmitStatus: "accepted"}, nil
|
||||
}
|
||||
pool := newMessageWorkPool(ctx, worker, 2)
|
||||
for _, message := range streams[0].Messages {
|
||||
if !pool.dispatch(message) {
|
||||
t.Fatalf("message %s was not dispatched", message.ID)
|
||||
}
|
||||
}
|
||||
<-slowStarted
|
||||
<-fastReturned
|
||||
deadline := time.Now().Add(time.Second)
|
||||
for {
|
||||
pending, pendingErr := client.XPending(ctx, worker.stream(), worker.group()).Result()
|
||||
if pendingErr != nil {
|
||||
t.Fatalf("xpending: %v", pendingErr)
|
||||
}
|
||||
if pending.Count == 1 {
|
||||
break
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatalf("pending count = %d, want 1 while slow sibling is still running", pending.Count)
|
||||
}
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
close(releaseSlow)
|
||||
pool.wait()
|
||||
}
|
||||
|
||||
func TestPendingRecoveryDoesNotDuplicateAnActiveOrAlreadyAcknowledgedMessage(t *testing.T) {
|
||||
mr := miniredis.RunT(t)
|
||||
client := redis.NewClient(&redis.Options{Addr: mr.Addr()})
|
||||
worker := &Worker{Redis: client, Stream: "gateway.submit.commands", Group: "cmpp-gateway", MinIdle: time.Millisecond, ResultOutbox: acknowledgingResultOutbox{client: client}}
|
||||
ctx := context.Background()
|
||||
if err := worker.ensureGroup(ctx); err != nil {
|
||||
t.Fatalf("ensureGroup: %v", err)
|
||||
}
|
||||
if err := client.XAdd(ctx, &redis.XAddArgs{Stream: worker.stream(), ID: "3-0", Values: submitCommandValues("message-active", "channel-active")}).Err(); err != nil {
|
||||
t.Fatalf("xadd: %v", err)
|
||||
}
|
||||
streams, err := client.XReadGroup(ctx, &redis.XReadGroupArgs{Group: worker.group(), Consumer: worker.consumer(), Streams: []string{worker.stream(), ">"}, Count: 1}).Result()
|
||||
if err != nil || len(streams) != 1 || len(streams[0].Messages) != 1 {
|
||||
t.Fatalf("xreadgroup: streams=%+v err=%v", streams, err)
|
||||
}
|
||||
message := streams[0].Messages[0]
|
||||
started := make(chan struct{})
|
||||
release := make(chan struct{})
|
||||
var submits atomic.Int32
|
||||
worker.Submit = func(_ context.Context, _ queue.SubmitCommand) (queue.SubmitResult, error) {
|
||||
submits.Add(1)
|
||||
close(started)
|
||||
<-release
|
||||
return queue.SubmitResult{SubmitStatus: "accepted"}, nil
|
||||
}
|
||||
pool := newMessageWorkPool(ctx, worker, 2)
|
||||
if !pool.dispatch(message) {
|
||||
t.Fatal("active message was not dispatched")
|
||||
}
|
||||
<-started
|
||||
time.Sleep(5 * time.Millisecond)
|
||||
if err := worker.recoverPending(ctx, pool); err != nil {
|
||||
t.Fatalf("recoverPending: %v", err)
|
||||
}
|
||||
if got := submits.Load(); got != 1 {
|
||||
t.Fatalf("active message submit count = %d, want 1", got)
|
||||
}
|
||||
close(release)
|
||||
pool.wait()
|
||||
if err := pool.dispatchRecovered(ctx, message); err != nil {
|
||||
t.Fatalf("dispatch acknowledged recovery: %v", err)
|
||||
}
|
||||
if got := submits.Load(); got != 1 {
|
||||
t.Fatalf("acknowledged message submit count = %d, want 1", got)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -22,13 +22,21 @@ const (
|
||||
)
|
||||
|
||||
type Manager struct {
|
||||
APIBaseURL string
|
||||
HTTPClient *http.Client
|
||||
APIBaseURL string
|
||||
HTTPClient *http.Client
|
||||
SubmitSegmentPublisher SubmitSegmentPublisher
|
||||
|
||||
mu sync.Mutex
|
||||
conns map[string]*connectionPool
|
||||
}
|
||||
|
||||
// SubmitSegmentPublisher persists each supplier response before the next long-message
|
||||
// segment is sent. The boundary is intentionally storage-only: HTTP callbacks belong to
|
||||
// the result Outbox worker and must not consume a supplier Submit window slot.
|
||||
type SubmitSegmentPublisher interface {
|
||||
PublishSubmitSegment(context.Context, queue.SubmitCommand, queue.SubmitSegmentResult) error
|
||||
}
|
||||
|
||||
type ConnectionState struct {
|
||||
ChannelID string `json:"channelId"`
|
||||
ConnectionID string `json:"connectionId"`
|
||||
|
||||
@@ -1,12 +1,12 @@
|
||||
package upstream
|
||||
|
||||
import (
|
||||
"cmpp-platform/gateway/internal/metrics"
|
||||
"cmpp-platform/gateway/internal/queue"
|
||||
"context"
|
||||
"fmt"
|
||||
cmpp "github.com/bigwhite/gocmpp"
|
||||
cmpputils "github.com/bigwhite/gocmpp/utils"
|
||||
"log"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
@@ -16,58 +16,30 @@ import (
|
||||
|
||||
func (m *Manager) Submit(ctx context.Context, cmd queue.SubmitCommand) (queue.SubmitResult, error) {
|
||||
if err := validateSubmitCommand(cmd); err != nil {
|
||||
result := submitResult(cmd, 0, "", "rejected", "INVALID_COMMAND", err.Error())
|
||||
if postErr := m.post(ctx, "/gateway/events/submit-result", result); postErr != nil {
|
||||
return result, postErr
|
||||
}
|
||||
return result, err
|
||||
return submitResult(cmd, 0, "", "rejected", "INVALID_COMMAND", err.Error()), err
|
||||
}
|
||||
|
||||
pool, err := m.connectionFor(cmd)
|
||||
if err != nil {
|
||||
result := submitResult(cmd, 0, "", "rejected", "CONNECT_FAILED", err.Error())
|
||||
if postErr := m.post(ctx, "/gateway/events/submit-result", result); postErr != nil {
|
||||
return result, postErr
|
||||
}
|
||||
return result, err
|
||||
return submitResult(cmd, 0, "", "rejected", "CONNECT_FAILED", err.Error()), err
|
||||
}
|
||||
|
||||
result, err := pool.submit(ctx, cmd, func(segment queue.SubmitSegmentResult) {
|
||||
payload := struct {
|
||||
queue.Envelope
|
||||
SubmitID string `json:"submitId,omitempty"`
|
||||
queue.SubmitSegmentResult
|
||||
}{
|
||||
Envelope: cmd.Envelope,
|
||||
SubmitID: cmd.SubmitID,
|
||||
SubmitSegmentResult: segment,
|
||||
}
|
||||
callbackCtx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
|
||||
postErr := m.post(callbackCtx, "/gateway/events/submit-segment-result", payload)
|
||||
cancel()
|
||||
if postErr != nil {
|
||||
log.Printf(
|
||||
"protocol_event protocol=cmpp direction=gateway_to_api event=submit_segment_result status=forward_failed channel_id=%s message_id=%s segment=%d/%d error=%q",
|
||||
cmd.ChannelID, cmd.MessageID, segment.SegmentIndex, segment.SegmentTotal, postErr,
|
||||
)
|
||||
result, err := pool.submit(ctx, cmd, func(segment queue.SubmitSegmentResult) error {
|
||||
if m.SubmitSegmentPublisher == nil {
|
||||
return fmt.Errorf("submit segment result publisher is required")
|
||||
}
|
||||
// A segment result must reach durable local storage before the next segment.
|
||||
// This cannot make the supplier/Redis boundary globally atomic, but it avoids
|
||||
// holding the supplier slot for an API round trip and minimizes untracked sends.
|
||||
return m.SubmitSegmentPublisher.PublishSubmitSegment(ctx, cmd, segment)
|
||||
})
|
||||
if err != nil {
|
||||
if postErr := m.post(ctx, "/gateway/events/submit-result", result); postErr != nil {
|
||||
return result, postErr
|
||||
}
|
||||
return result, err
|
||||
}
|
||||
if err := m.post(ctx, "/gateway/events/submit-result", result); err != nil {
|
||||
return result, err
|
||||
}
|
||||
return result, nil
|
||||
return result, err
|
||||
}
|
||||
|
||||
func (p *connectionPool) submit(
|
||||
ctx context.Context,
|
||||
cmd queue.SubmitCommand,
|
||||
onSegment func(queue.SubmitSegmentResult),
|
||||
onSegment func(queue.SubmitSegmentResult) error,
|
||||
) (queue.SubmitResult, error) {
|
||||
parts, err := splitSubmitContent(cmd.CMPP.MsgFmt, cmd.Content)
|
||||
if err != nil {
|
||||
@@ -79,18 +51,29 @@ func (p *connectionPool) submit(
|
||||
var firstGatewayMessageID string
|
||||
segments := make([]queue.SubmitSegmentResult, 0, len(parts))
|
||||
for _, part := range parts {
|
||||
connectionStartedAt := time.Now()
|
||||
conn, release, err := p.acquireConnection(ctx)
|
||||
metrics.ObserveSubmitStage("connection_wait", err == nil, time.Since(connectionStartedAt))
|
||||
if err != nil {
|
||||
result := submitResult(cmd, 0, "", "timeout", "WINDOW_TIMEOUT", err.Error())
|
||||
result.Segments = segments
|
||||
return result, err
|
||||
}
|
||||
supplierStartedAt := time.Now()
|
||||
seq, gatewayMessageID, result, err := conn.submitPart(ctx, cmd, part)
|
||||
metrics.ObserveSubmitStage("supplier_rtt", err == nil, time.Since(supplierStartedAt))
|
||||
release()
|
||||
segment := submitSegmentResult(part, seq, gatewayMessageID, result)
|
||||
segments = append(segments, segment)
|
||||
if onSegment != nil {
|
||||
onSegment(segment)
|
||||
// A one-part Submit is fully represented by the aggregate event below;
|
||||
// publishing an identical segment event doubles API callbacks and database
|
||||
// writes without adding crash-recovery evidence. Multi-part messages still
|
||||
// persist every segment before advancing to the next supplier Submit.
|
||||
if onSegment != nil && len(parts) > 1 {
|
||||
if publishErr := onSegment(segment); publishErr != nil {
|
||||
result.Segments = segments
|
||||
return result, publishErr
|
||||
}
|
||||
}
|
||||
if firstSequence == 0 {
|
||||
firstSequence = seq
|
||||
|
||||
Vendored
+89
-18
@@ -20,6 +20,7 @@ import (
|
||||
"log"
|
||||
"net"
|
||||
"os"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
)
|
||||
@@ -80,6 +81,9 @@ type Server struct {
|
||||
// standard logger.
|
||||
ErrorLog *log.Logger
|
||||
OnClose func(*Conn)
|
||||
// SubmitWindow resolves the authenticated client's allowed in-flight Submit
|
||||
// count. A nil resolver, or a value below two, preserves serial handling.
|
||||
SubmitWindow func(*Conn) int
|
||||
}
|
||||
|
||||
// A conn represents the server side of a Cmpp connection.
|
||||
@@ -394,7 +398,14 @@ func (c *conn) serve() {
|
||||
}
|
||||
}()
|
||||
|
||||
var submitGroup sync.WaitGroup
|
||||
var submitSlots chan struct{}
|
||||
fatal := make(chan error, 1)
|
||||
defer func() {
|
||||
// Why wait: a handler may persist the message and register receipt routing
|
||||
// after the peer disconnects. Session cleanup must run after every accepted
|
||||
// in-flight request finishes, otherwise a late handler can recreate stale state.
|
||||
submitGroup.Wait()
|
||||
c.close()
|
||||
if c.server.OnClose != nil {
|
||||
c.server.OnClose(c.Conn)
|
||||
@@ -408,6 +419,8 @@ func (c *conn) serve() {
|
||||
select {
|
||||
case <-c.exceed:
|
||||
return // close the connection.
|
||||
case <-fatal:
|
||||
return
|
||||
default:
|
||||
}
|
||||
|
||||
@@ -426,29 +439,81 @@ func (c *conn) serve() {
|
||||
break
|
||||
}
|
||||
|
||||
_, err = c.server.Handler.ServeCmpp(r, r.Packet, c.server.ErrorLog)
|
||||
err1 := c.finishPacket(r)
|
||||
if r.AfterSend != nil {
|
||||
r.AfterSend(err1)
|
||||
if isSubmitPacket(r.Packet.Packer) && c.submitWindow() > 1 {
|
||||
if submitSlots == nil {
|
||||
submitSlots = make(chan struct{}, c.submitWindow())
|
||||
}
|
||||
select {
|
||||
case submitSlots <- struct{}{}:
|
||||
case <-c.exceed:
|
||||
return
|
||||
case <-fatal:
|
||||
return
|
||||
}
|
||||
submitGroup.Add(1)
|
||||
go func(response *Response) {
|
||||
defer submitGroup.Done()
|
||||
defer func() { <-submitSlots }()
|
||||
if handleErr := c.handlePacket(response); handleErr != nil {
|
||||
select {
|
||||
case fatal <- handleErr:
|
||||
default:
|
||||
}
|
||||
}
|
||||
}(r)
|
||||
continue
|
||||
}
|
||||
if err1 != nil {
|
||||
c.server.ErrorLog.Printf(
|
||||
"send response packet failed remote=%v protocol=%s packet_type=%T seq=%d err_type=%T err=%v",
|
||||
c.Conn.RemoteAddr(), c.Conn.Typ, r.Packer, r.SeqId, err1, err1,
|
||||
)
|
||||
break
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
c.server.ErrorLog.Printf(
|
||||
"handler failed remote=%v protocol=%s packet_type=%T seq=%d err_type=%T err=%v",
|
||||
c.Conn.RemoteAddr(), c.Conn.Typ, r.Packet.Packer, r.SeqId, err, err,
|
||||
)
|
||||
if err = c.handlePacket(r); err != nil {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (c *conn) submitWindow() int {
|
||||
if c.server.SubmitWindow == nil {
|
||||
return 1
|
||||
}
|
||||
window := c.server.SubmitWindow(c.Conn)
|
||||
if window < 1 {
|
||||
return 1
|
||||
}
|
||||
if window > 1024 {
|
||||
return 1024
|
||||
}
|
||||
return window
|
||||
}
|
||||
|
||||
func isSubmitPacket(packet Packer) bool {
|
||||
switch packet.(type) {
|
||||
case *Cmpp2SubmitReqPkt, *Cmpp3SubmitReqPkt:
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func (c *conn) handlePacket(r *Response) error {
|
||||
_, handlerErr := c.server.Handler.ServeCmpp(r, r.Packet, c.server.ErrorLog)
|
||||
sendErr := c.finishPacket(r)
|
||||
if r.AfterSend != nil {
|
||||
r.AfterSend(sendErr)
|
||||
}
|
||||
if sendErr != nil {
|
||||
c.server.ErrorLog.Printf(
|
||||
"send response packet failed remote=%v protocol=%s packet_type=%T seq=%d err_type=%T err=%v",
|
||||
c.Conn.RemoteAddr(), c.Conn.Typ, r.Packer, r.SeqId, sendErr, sendErr,
|
||||
)
|
||||
return sendErr
|
||||
}
|
||||
if handlerErr != nil {
|
||||
c.server.ErrorLog.Printf(
|
||||
"handler failed remote=%v protocol=%s packet_type=%T seq=%d err_type=%T err=%v",
|
||||
c.Conn.RemoteAddr(), c.Conn.Typ, r.Packet.Packer, r.SeqId, handlerErr, handlerErr,
|
||||
)
|
||||
}
|
||||
return handlerErr
|
||||
}
|
||||
|
||||
// Create new connection from rwc.
|
||||
func (srv *Server) newConn(rwc net.Conn) (c *conn, err error) {
|
||||
c = new(conn)
|
||||
@@ -480,6 +545,12 @@ func ListenAndServe(addr string, typ Type, t time.Duration, n int32, logWriter i
|
||||
// ListenAndServeWithClose behaves like ListenAndServe and invokes onClose once
|
||||
// after an accepted client connection ends, including abrupt TCP disconnects.
|
||||
func ListenAndServeWithClose(addr string, typ Type, t time.Duration, n int32, logWriter io.Writer, onClose func(*Conn), handlers ...Handler) error {
|
||||
return ListenAndServeWithCloseAndSubmitWindow(addr, typ, t, n, logWriter, onClose, nil, handlers...)
|
||||
}
|
||||
|
||||
// ListenAndServeWithCloseAndSubmitWindow adds bounded per-connection Submit
|
||||
// concurrency while keeping login, heartbeat and acknowledgement handling serial.
|
||||
func ListenAndServeWithCloseAndSubmitWindow(addr string, typ Type, t time.Duration, n int32, logWriter io.Writer, onClose func(*Conn), submitWindow func(*Conn) int, handlers ...Handler) error {
|
||||
if addr == "" {
|
||||
return ErrEmptyServerAddr
|
||||
}
|
||||
@@ -504,7 +575,7 @@ func ListenAndServeWithClose(addr string, typ Type, t time.Duration, n int32, lo
|
||||
}
|
||||
server := &Server{Addr: addr, Handler: handler, Typ: typ,
|
||||
T: t, N: n,
|
||||
ErrorLog: log.New(logWriter, "cmppserver: ", log.LstdFlags), OnClose: onClose}
|
||||
ErrorLog: log.New(logWriter, "cmppserver: ", log.LstdFlags), OnClose: onClose, SubmitWindow: submitWindow}
|
||||
return server.listenAndServe()
|
||||
}
|
||||
|
||||
|
||||
@@ -11,6 +11,17 @@ API_METRICS_HOST="${API_METRICS_HOST:-127.0.0.1}"
|
||||
API_METRICS_PORT="${API_METRICS_PORT:-9464}"
|
||||
API_ENABLE_SEND_WORKER="${API_ENABLE_SEND_WORKER:-true}"
|
||||
API_SEND_WORKER_CONCURRENCY="${API_SEND_WORKER_CONCURRENCY:-50}"
|
||||
API_WORKER_METRICS_HOST="${API_WORKER_METRICS_HOST:-127.0.0.1}"
|
||||
API_WORKER_METRICS_PORT="${API_WORKER_METRICS_PORT:-9465}"
|
||||
CMPP_INBOUND_FAST_PATH_ENABLED="${CMPP_INBOUND_FAST_PATH_ENABLED:-true}"
|
||||
CMPP_INBOUND_WORKFLOW_WORKER_ENABLED="${CMPP_INBOUND_WORKFLOW_WORKER_ENABLED:-true}"
|
||||
API_INBOUND_WORKFLOW_CONCURRENCY="${API_INBOUND_WORKFLOW_CONCURRENCY:-32}"
|
||||
API_INBOUND_WORKFLOW_BATCH_ENABLED="${API_INBOUND_WORKFLOW_BATCH_ENABLED:-true}"
|
||||
API_INBOUND_WORKFLOW_BATCH_SIZE="${API_INBOUND_WORKFLOW_BATCH_SIZE:-64}"
|
||||
API_INBOUND_WORKFLOW_POLL_INTERVAL_MS="${API_INBOUND_WORKFLOW_POLL_INTERVAL_MS:-100}"
|
||||
API_INBOUND_WORKFLOW_STALE_SECONDS="${API_INBOUND_WORKFLOW_STALE_SECONDS:-300}"
|
||||
API_DB_POOL_MAX="${API_DB_POOL_MAX:-32}"
|
||||
API_WORKER_DB_POOL_MAX="${API_WORKER_DB_POOL_MAX:-8}"
|
||||
GATEWAY_CONTROL_ADDR="${GATEWAY_CONTROL_ADDR:-127.0.0.1:8090}"
|
||||
GATEWAY_CMPP_ADDR="${GATEWAY_CMPP_ADDR:-0.0.0.0:17890}"
|
||||
CMPP_PUBLIC_HOST="${CMPP_PUBLIC_HOST:-8.160.169.106}"
|
||||
@@ -182,7 +193,7 @@ SQL
|
||||
|
||||
write_env() {
|
||||
log "Writing production environment"
|
||||
mkdir -p /etc/cmpp-platform "$APP_DIR" "$APP_DIR/logs/api" "$APP_DIR/logs/gateway" "$APP_DIR/backups" "$OBJECT_STORAGE_LOCAL_ROOT"
|
||||
mkdir -p /etc/cmpp-platform "$APP_DIR" "$APP_DIR/logs/api" "$APP_DIR/logs/send-worker" "$APP_DIR/logs/gateway" "$APP_DIR/backups" "$OBJECT_STORAGE_LOCAL_ROOT"
|
||||
cat >/etc/cmpp-platform/cmpp-platform.env <<EOF
|
||||
NODE_ENV=production
|
||||
API_PORT=${API_PORT}
|
||||
@@ -191,6 +202,17 @@ API_METRICS_HOST=${API_METRICS_HOST}
|
||||
API_METRICS_PORT=${API_METRICS_PORT}
|
||||
API_ENABLE_SEND_WORKER=${API_ENABLE_SEND_WORKER}
|
||||
API_SEND_WORKER_CONCURRENCY=${API_SEND_WORKER_CONCURRENCY}
|
||||
API_WORKER_METRICS_HOST=${API_WORKER_METRICS_HOST}
|
||||
API_WORKER_METRICS_PORT=${API_WORKER_METRICS_PORT}
|
||||
CMPP_INBOUND_FAST_PATH_ENABLED=${CMPP_INBOUND_FAST_PATH_ENABLED}
|
||||
CMPP_INBOUND_WORKFLOW_WORKER_ENABLED=${CMPP_INBOUND_WORKFLOW_WORKER_ENABLED}
|
||||
API_INBOUND_WORKFLOW_CONCURRENCY=${API_INBOUND_WORKFLOW_CONCURRENCY}
|
||||
API_INBOUND_WORKFLOW_BATCH_ENABLED=${API_INBOUND_WORKFLOW_BATCH_ENABLED}
|
||||
API_INBOUND_WORKFLOW_BATCH_SIZE=${API_INBOUND_WORKFLOW_BATCH_SIZE}
|
||||
API_INBOUND_WORKFLOW_POLL_INTERVAL_MS=${API_INBOUND_WORKFLOW_POLL_INTERVAL_MS}
|
||||
API_INBOUND_WORKFLOW_STALE_SECONDS=${API_INBOUND_WORKFLOW_STALE_SECONDS}
|
||||
API_DB_POOL_MAX=${API_DB_POOL_MAX}
|
||||
API_WORKER_DB_POOL_MAX=${API_WORKER_DB_POOL_MAX}
|
||||
DATABASE_URL=postgresql://${DB_USER}:${DB_PASSWORD}@127.0.0.1:5432/${DB_NAME}?schema=public
|
||||
REDIS_HOST=127.0.0.1
|
||||
REDIS_PORT=6379
|
||||
@@ -265,12 +287,34 @@ User=cmpp-api
|
||||
Group=cmpp-security
|
||||
WorkingDirectory=${APP_DIR}/api
|
||||
EnvironmentFile=/etc/cmpp-platform/cmpp-platform.env
|
||||
Environment=CMPP_PROCESS_ROLE=api
|
||||
ExecStart=${node_bin} dist/main.js
|
||||
Restart=always
|
||||
RestartSec=5
|
||||
StandardOutput=append:${APP_DIR}/logs/api/stdout.log
|
||||
StandardError=append:${APP_DIR}/logs/api/stderr.log
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
EOF
|
||||
|
||||
cat >/etc/systemd/system/cmpp-send-worker.service <<EOF
|
||||
[Unit]
|
||||
Description=CMPP durable send workflow worker
|
||||
After=network.target postgresql.service redis.service cmpp-minio.service
|
||||
|
||||
[Service]
|
||||
User=cmpp-api
|
||||
Group=cmpp-security
|
||||
WorkingDirectory=${APP_DIR}/api
|
||||
EnvironmentFile=/etc/cmpp-platform/cmpp-platform.env
|
||||
Environment=CMPP_PROCESS_ROLE=worker
|
||||
ExecStart=${node_bin} dist/send-worker.js
|
||||
Restart=always
|
||||
RestartSec=5
|
||||
StandardOutput=append:${APP_DIR}/logs/send-worker/stdout.log
|
||||
StandardError=append:${APP_DIR}/logs/send-worker/stderr.log
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
EOF
|
||||
|
||||
@@ -29,6 +29,24 @@ if [[ ! "${API_SEND_WORKER_CONCURRENCY:-}" =~ ^[1-9][0-9]*$ ]]; then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ "${CMPP_INBOUND_FAST_PATH_ENABLED:-}" != "true" || "${CMPP_INBOUND_WORKFLOW_WORKER_ENABLED:-}" != "true" ]]; then
|
||||
echo "CMPP_INBOUND_FAST_PATH_ENABLED=true and CMPP_INBOUND_WORKFLOW_WORKER_ENABLED=true are required in $ENV_FILE." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ ! "${API_INBOUND_WORKFLOW_CONCURRENCY:-}" =~ ^[1-9][0-9]*$ ]]; then
|
||||
echo "API_INBOUND_WORKFLOW_CONCURRENCY must be a positive integer in $ENV_FILE." >&2
|
||||
exit 1
|
||||
fi
|
||||
if [[ "${API_INBOUND_WORKFLOW_BATCH_ENABLED:-}" != "true" || ! "${API_INBOUND_WORKFLOW_BATCH_SIZE:-}" =~ ^[1-9][0-9]*$ ]]; then
|
||||
echo "API_INBOUND_WORKFLOW_BATCH_ENABLED=true and a positive API_INBOUND_WORKFLOW_BATCH_SIZE are required in $ENV_FILE." >&2
|
||||
exit 1
|
||||
fi
|
||||
if [[ ! "${API_DB_POOL_MAX:-}" =~ ^[1-9][0-9]*$ || ! "${API_WORKER_DB_POOL_MAX:-}" =~ ^[1-9][0-9]*$ ]]; then
|
||||
echo "API_DB_POOL_MAX and API_WORKER_DB_POOL_MAX must be positive integers in $ENV_FILE." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ -z "${CMPP_PUBLIC_HOST:-}" || ! "${CMPP_PUBLIC_PORT:-}" =~ ^[1-9][0-9]*$ ]]; then
|
||||
echo "CMPP_PUBLIC_HOST and a positive CMPP_PUBLIC_PORT are required in $ENV_FILE; these are the customer-facing CMPP endpoint." >&2
|
||||
exit 1
|
||||
@@ -64,7 +82,35 @@ PROD_ADMIN_CREDENTIAL_FILE="$ADMIN_CREDENTIAL_FILE" node tools/deploy/ensure-pro
|
||||
chmod 600 "$ADMIN_CREDENTIAL_FILE" || true
|
||||
|
||||
echo "[deploy] Ensuring runtime log directories"
|
||||
install -d -m 0755 "$APP_DIR/logs/api" "$APP_DIR/logs/gateway"
|
||||
install -d -m 0755 "$APP_DIR/logs/api" "$APP_DIR/logs/send-worker" "$APP_DIR/logs/gateway"
|
||||
|
||||
echo "[deploy] Installing split API and send-worker services"
|
||||
node_bin="$(command -v node)"
|
||||
install -d -m 0755 /etc/systemd/system/cmpp-api.service.d
|
||||
cat >/etc/systemd/system/cmpp-api.service.d/process-role.conf <<'EOF'
|
||||
[Service]
|
||||
Environment=CMPP_PROCESS_ROLE=api
|
||||
EOF
|
||||
cat >/etc/systemd/system/cmpp-send-worker.service <<EOF
|
||||
[Unit]
|
||||
Description=CMPP durable send workflow worker
|
||||
After=network.target postgresql.service redis.service cmpp-minio.service
|
||||
|
||||
[Service]
|
||||
User=cmpp-api
|
||||
Group=cmpp-security
|
||||
WorkingDirectory=$APP_DIR/api
|
||||
EnvironmentFile=$ENV_FILE
|
||||
Environment=CMPP_PROCESS_ROLE=worker
|
||||
ExecStart=$node_bin dist/send-worker.js
|
||||
Restart=always
|
||||
RestartSec=5
|
||||
StandardOutput=append:$APP_DIR/logs/send-worker/stdout.log
|
||||
StandardError=append:$APP_DIR/logs/send-worker/stderr.log
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
EOF
|
||||
|
||||
echo "[deploy] Installing restricted security boundary"
|
||||
bash "$APP_DIR/tools/security/install-security-agent.sh"
|
||||
@@ -90,15 +136,16 @@ echo "[deploy] Restarting services"
|
||||
systemctl daemon-reload
|
||||
if [[ "${OBJECT_STORAGE_DRIVER:-minio}" == "local" ]]; then
|
||||
systemctl disable --now cmpp-minio 2>/dev/null || true
|
||||
systemctl enable --now cmpp-api cmpp-gateway nginx
|
||||
systemctl enable --now cmpp-api cmpp-send-worker cmpp-gateway nginx
|
||||
else
|
||||
systemctl enable --now cmpp-minio
|
||||
systemctl restart cmpp-minio
|
||||
systemctl enable --now cmpp-api cmpp-gateway nginx
|
||||
systemctl enable --now cmpp-api cmpp-send-worker cmpp-gateway nginx
|
||||
fi
|
||||
systemctl restart cmpp-gateway
|
||||
systemctl restart cmpp-security-agent
|
||||
systemctl restart cmpp-api
|
||||
systemctl restart cmpp-send-worker
|
||||
systemctl restart nginx
|
||||
|
||||
echo "[deploy] Health checks"
|
||||
@@ -117,6 +164,7 @@ wait_for_http() {
|
||||
return 1
|
||||
}
|
||||
wait_for_http "API" "http://127.0.0.1:${API_PORT:-3000}/api/health"
|
||||
wait_for_http "Send worker metrics" "http://127.0.0.1:${API_WORKER_METRICS_PORT:-9465}/metrics"
|
||||
wait_for_http "Gateway" "http://127.0.0.1:8090/health"
|
||||
redis-cli -h "${REDIS_HOST:-127.0.0.1}" -p "${REDIS_PORT:-6379}" ping >/dev/null
|
||||
pg_isready -d "${DATABASE_URL%%\?*}" >/dev/null
|
||||
|
||||
@@ -4,6 +4,7 @@ import { resolve } from 'node:path';
|
||||
const deploy = readFileSync(resolve(import.meta.dirname, 'production-deploy.sh'), 'utf8');
|
||||
const bootstrap = readFileSync(resolve(import.meta.dirname, 'production-bootstrap.sh'), 'utf8');
|
||||
const apiMain = readFileSync(resolve(import.meta.dirname, '../../api/src/main.ts'), 'utf8');
|
||||
const workerMain = readFileSync(resolve(import.meta.dirname, '../../api/src/send-worker.ts'), 'utf8');
|
||||
const required = [
|
||||
'compression_config=/etc/nginx/conf.d/cmpp-compression.conf',
|
||||
': >"$compression_config"',
|
||||
@@ -26,4 +27,24 @@ if (!apiMain.includes("process.env.API_HOST?.trim() || '127.0.0.1'") || !apiMain
|
||||
throw new Error('NestJS API must bind to API_HOST and default to loopback');
|
||||
}
|
||||
|
||||
console.log('Production deployment verified: Nginx compression is idempotent and NestJS defaults to loopback.');
|
||||
for (const marker of [
|
||||
'CMPP_INBOUND_FAST_PATH_ENABLED=true',
|
||||
'CMPP_INBOUND_WORKFLOW_WORKER_ENABLED=true',
|
||||
'API_INBOUND_WORKFLOW_CONCURRENCY',
|
||||
'API_INBOUND_WORKFLOW_BATCH_ENABLED=true',
|
||||
'API_INBOUND_WORKFLOW_BATCH_SIZE',
|
||||
'cmpp-send-worker.service',
|
||||
'Environment=CMPP_PROCESS_ROLE=api',
|
||||
'Environment=CMPP_PROCESS_ROLE=worker',
|
||||
'dist/send-worker.js',
|
||||
'API_WORKER_METRICS_PORT',
|
||||
]) {
|
||||
if (!`${deploy}\n${bootstrap}`.includes(marker)) {
|
||||
throw new Error(`production deployment is missing the durable inbound worker contract: ${marker}`);
|
||||
}
|
||||
}
|
||||
if (!workerMain.includes("process.env.API_WORKER_METRICS_HOST?.trim() || '127.0.0.1'")) {
|
||||
throw new Error('send worker metrics must default to loopback');
|
||||
}
|
||||
|
||||
console.log('Production deployment verified: Nginx/API guards and the split durable send worker contract are present.');
|
||||
|
||||
@@ -182,7 +182,7 @@ groups:
|
||||
- name: cmpp-core-services
|
||||
rules:
|
||||
- alert: CmppCoreServiceInactive
|
||||
expr: node_systemd_unit_state{name=~"cmpp-api\\.service|cmpp-gateway\\.service|postgresql\\.service|redis(-server)?\\.service|cmpp-minio\\.service|nginx\\.service",state="active"} == 0
|
||||
expr: node_systemd_unit_state{name=~"cmpp-api\\.service|cmpp-send-worker\\.service|cmpp-gateway\\.service|postgresql\\.service|redis(-server)?\\.service|cmpp-minio\\.service|nginx\\.service",state="active"} == 0
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
@@ -200,6 +200,16 @@ groups:
|
||||
for: 2m
|
||||
labels: { severity: critical, service: api }
|
||||
annotations: { summary: "API指标采集不可用", description: "Prometheus连续2分钟无法读取API内部指标端点。", currentValue: "{{ $value }}", threshold: "up = 1" }
|
||||
- alert: CmppSendWorkerMetricsDown
|
||||
expr: up{job="cmpp-send-worker"} == 0
|
||||
for: 2m
|
||||
labels: { severity: critical, service: send-worker }
|
||||
annotations: { summary: "发送Worker指标采集不可用", description: "Prometheus连续2分钟无法读取独立发送Worker指标端点。", currentValue: "{{ $value }}", threshold: "up = 1" }
|
||||
- alert: CmppInboundWorkflowBacklogCritical
|
||||
expr: cmpp_worker_inbound_workflow_oldest_pending_age_seconds > 120
|
||||
for: 2m
|
||||
labels: { severity: critical, service: send-worker }
|
||||
annotations: { summary: "CMPP耐久Inbox严重积压", description: "最旧待处理CMPP Inbox连续2分钟超过120秒。", currentValue: "{{ printf \"%.0f\" $value }}s", threshold: "120s" }
|
||||
- alert: CmppApiHttpErrorRateWarning
|
||||
expr: (sum(rate(cmpp_api_http_requests_total{status=~"5.."}[5m])) / clamp_min(sum(rate(cmpp_api_http_requests_total[5m])), 0.001) > 0.01) and (sum(rate(cmpp_api_http_requests_total{status=~"5.."}[5m])) / clamp_min(sum(rate(cmpp_api_http_requests_total[5m])), 0.001) <= 0.05) and sum(increase(cmpp_api_http_requests_total{status=~"5.."}[5m])) >= 5
|
||||
for: 5m
|
||||
|
||||
@@ -68,7 +68,7 @@ EOF
|
||||
cat >/etc/systemd/system/prometheus-node-exporter.service.d/cmpp-monitoring.conf <<EOF
|
||||
[Service]
|
||||
ExecStart=
|
||||
ExecStart=${node_exporter_bin} --web.listen-address=127.0.0.1:9100 --collector.systemd --collector.systemd.unit-include='cmpp-api\\.service|cmpp-gateway\\.service|postgresql\\.service|redis(-server)?\\.service|cmpp-minio\\.service|nginx\\.service' --collector.filesystem.mount-points-exclude='^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/)'
|
||||
ExecStart=${node_exporter_bin} --web.listen-address=127.0.0.1:9100 --collector.systemd --collector.systemd.unit-include='cmpp-api\\.service|cmpp-send-worker\\.service|cmpp-gateway\\.service|postgresql\\.service|redis(-server)?\\.service|cmpp-minio\\.service|nginx\\.service' --collector.filesystem.mount-points-exclude='^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/)'
|
||||
EOF
|
||||
|
||||
log "Validating Prometheus configuration before restart"
|
||||
|
||||
@@ -24,6 +24,10 @@ scrape_configs:
|
||||
static_configs:
|
||||
- targets: [127.0.0.1:9464]
|
||||
|
||||
- job_name: cmpp-send-worker
|
||||
static_configs:
|
||||
- targets: [127.0.0.1:9465]
|
||||
|
||||
- job_name: cmpp-gateway
|
||||
metrics_path: /metrics
|
||||
static_configs:
|
||||
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
"go/token"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
@@ -41,6 +42,8 @@ var requiredFiles = []string{
|
||||
|
||||
var requiredTests = []string{
|
||||
"TestInboundServerAuthenticatesAndSubmits",
|
||||
"TestInboundServerProcessesSubmitWithinAuthenticatedConnectionWindow",
|
||||
"TestBoundedSubmitWindowAndAggregateSlotSnapshot",
|
||||
"TestInboundServerForwardsLongMessageFragmentsWithoutUDHAndAcknowledgesEachSubmit",
|
||||
"TestSubmitResponsePrecedesQueuedFailureReceipt",
|
||||
"TestDailyLimitRejectsSubmitSynchronouslyWithoutPendingReceipt",
|
||||
@@ -83,6 +86,25 @@ func main() {
|
||||
actual[key] = item
|
||||
}
|
||||
}
|
||||
if os.Getenv("UPDATE_INBOUND_R6_CONTRACT") == "1" {
|
||||
contract.Declarations = contract.Declarations[:0]
|
||||
for _, item := range actual {
|
||||
contract.Declarations = append(contract.Declarations, item)
|
||||
}
|
||||
sort.Slice(contract.Declarations, func(i, j int) bool {
|
||||
if contract.Declarations[i].File != contract.Declarations[j].File {
|
||||
return contract.Declarations[i].File < contract.Declarations[j].File
|
||||
}
|
||||
if contract.Declarations[i].Name == contract.Declarations[j].Name {
|
||||
return contract.Declarations[i].Kind < contract.Declarations[j].Kind
|
||||
}
|
||||
return contract.Declarations[i].Name < contract.Declarations[j].Name
|
||||
})
|
||||
updated, err := json.MarshalIndent(contract, "", " ")
|
||||
must(err)
|
||||
must(os.WriteFile(manifestPath, append(updated, '\n'), 0o644))
|
||||
fmt.Printf("R6 inbound contract updated with %d declarations.\n", len(contract.Declarations))
|
||||
}
|
||||
|
||||
for _, expected := range contract.Declarations {
|
||||
key := expected.Kind + ":" + expected.Name
|
||||
|
||||
@@ -13,6 +13,7 @@ usermod -a -G cmpp-security cmpp-api
|
||||
install -d -o root -g cmpp-security -m 0770 /run/cmpp-security-agent
|
||||
install -d -o root -g cmpp-security -m 0750 /var/lib/cmpp-security-agent
|
||||
install -d -o cmpp-api -g cmpp-security -m 0750 "$APP_DIR/logs/api"
|
||||
install -d -o cmpp-api -g cmpp-security -m 0750 "$APP_DIR/logs/send-worker"
|
||||
[[ -d /var/lib/cmpp-platform/object-storage ]] && chown -R cmpp-api:cmpp-security /var/lib/cmpp-platform/object-storage
|
||||
|
||||
sed "s#@CMPP_SECURITY_AGENT_BIN@#$agent_binary#g" "$APP_DIR/deploy/security/cmpp-report-only.conf" >/etc/fail2ban/action.d/cmpp-report-only.conf
|
||||
@@ -29,6 +30,18 @@ table inet cmpp_security {
|
||||
chain input { type filter hook input priority -10; policy accept; ip saddr @blocked_ipv4 drop; ip6 saddr @blocked_ipv6 drop; }
|
||||
}
|
||||
EOF
|
||||
|
||||
install -d -m 0755 /etc/systemd/system/cmpp-send-worker.service.d
|
||||
cat >/etc/systemd/system/cmpp-send-worker.service.d/security-boundary.conf <<EOF
|
||||
[Service]
|
||||
User=cmpp-api
|
||||
Group=cmpp-security
|
||||
NoNewPrivileges=true
|
||||
PrivateTmp=true
|
||||
ProtectHome=true
|
||||
ProtectSystem=true
|
||||
ReadWritePaths=$APP_DIR/logs/send-worker /var/lib/cmpp-platform/object-storage
|
||||
EOF
|
||||
grep -q 'cmpp-security.nft' /etc/nftables.conf || printf '\ninclude "/etc/nftables.d/cmpp-security.nft"\n' >>/etc/nftables.conf
|
||||
nft -c -f /etc/nftables.conf
|
||||
nft list table inet cmpp_security >/dev/null 2>&1 || nft -f /etc/nftables.d/cmpp-security.nft
|
||||
@@ -54,4 +67,4 @@ fail2ban-client -t
|
||||
nginx -t
|
||||
systemctl daemon-reload
|
||||
systemctl enable cmpp-security-agent
|
||||
echo "Security boundary installed. Restart cmpp-security-agent and cmpp-api only in the approved release window."
|
||||
echo "Security boundary installed. Restart cmpp-security-agent, cmpp-api and cmpp-send-worker only in the approved release window."
|
||||
|
||||
@@ -40,6 +40,17 @@ function assertIsoDateTime(value, field, file) {
|
||||
function validateExample(fileName) {
|
||||
const fullPath = join(examplesDir, fileName);
|
||||
const payload = JSON.parse(readFileSync(fullPath, 'utf8'));
|
||||
if (fileName === 'submit-result-outbox-event.json') {
|
||||
for (const field of ['schemaVersion', 'eventId', 'eventType', 'path', 'messageId', 'channelId', 'submitId', 'payload', 'createdAt']) {
|
||||
assert(Object.hasOwn(payload, field), `${fileName}: missing ${field}`);
|
||||
}
|
||||
assert(payload.schemaVersion === 'v1', `${fileName}: schemaVersion must be v1`);
|
||||
assert(['submit_result', 'submit_segment_result'].includes(payload.eventType), `${fileName}: unsupported eventType`);
|
||||
assert(['/gateway/events/submit-result', '/gateway/events/submit-segment-result'].includes(payload.path), `${fileName}: unsupported path`);
|
||||
assert(payload.payload?.eventId === payload.eventId, `${fileName}: payload.eventId must match eventId`);
|
||||
assertIsoDateTime(payload.createdAt, 'createdAt', fileName);
|
||||
return `${fileName}: ${payload.eventType} ok`;
|
||||
}
|
||||
const type = payload.messageType;
|
||||
|
||||
assert(validTypes.has(type), `${fileName}: unsupported messageType ${type}`);
|
||||
|
||||
Reference in New Issue
Block a user