import assert from 'node:assert/strict'; import { createRequire } from 'node:module'; import { randomUUID } from 'node:crypto'; import { spawn } from 'node:child_process'; import { fileURLToPath } from 'node:url'; const require = createRequire(new URL('../../api/package.json', import.meta.url)); const url = new URL(process.env.COMPLETION_TEST_DATABASE_URL || ''); assert(['localhost', '127.0.0.1'].includes(url.hostname) && url.pathname.startsWith('/cmpp_qa_')); process.env.DATABASE_URL = url.toString(); process.env.NODE_ENV = 'test'; require('reflect-metadata'); const { PrismaService } = require('./dist/prisma/prisma.service'); const { AttemptCompletion } = require('./dist/send-chain/attempt-completion'); const { completionContext, completionDatabase, CompletionRouteRequired, } = require('./dist/send-chain/completion-context'); const { retainAlerts, retainedAlerts, clearRetainedAlert, } = require('./dist/infrastructure-monitoring/persistent-alerts'); const { SendChainService } = require('./dist/send-chain/send-chain.service'); const { BillingService } = require('./dist/billing/billing.service'); const { OpenApiService } = require('./dist/open-api/open-api.service'); const db = new PrismaService(); const scoped = completionDatabase(db); const prefix = randomUUID(); const pass = (name) => console.log('PASS', name); const fixture = async () => { const message = await db.smsMessageRecord.create({ data: { messageId: randomUUID(), phoneNumber: '13800138000', content: '隔离并发验收'.repeat(30), billingUnits: 3 }, }); return message; }; if (process.argv[2] === '--worker') { try { const worker = new AttemptCompletion( db, async (_kind, payload) => { await scoped.operationLog.create({ data: { action: 'qa.process', resource: process.argv[4], resourceId: payload.id }, }); }, async () => {}, ); await worker.process(process.argv[3]); } finally { await db.onModuleDestroy(); } process.exit(0); } try { const message = await fixture(); let active = 0; let maximum = 0; const execute = async (_kind, payload) => { active++; maximum = Math.max(maximum, active); try { await scoped.$transaction(async (tx) => { assert.equal(tx, completionContext.getStore().tx); await tx.operationLog.create({ data: { action: 'qa.completion', resource: prefix, resourceId: payload.id } }); }); } finally { active--; } }; const workers = [ new AttemptCompletion(db, execute, async () => {}), new AttemptCompletion(db, execute, async () => {}), ]; await Promise.all( Array.from({ length: 24 }, (_, i) => workers[i % 2].enqueue(message.id, undefined, 'receipt', { id: `segment-${i % 12}` }), ), ); await Promise.all(workers.map((worker) => worker.scan())); const work = await db.smsAttemptCompletionWork.findUniqueOrThrow({ where: { workKey: `message:${message.id}` } }); assert.equal(maximum, 1); assert.equal(work.revision, 12); assert.equal(work.processedRevision, 12); assert.equal(work.state, 'idle'); assert.equal(await db.operationLog.count({ where: { resource: prefix } }), 12); pass('two consumers, 24 concurrent receipts, 12 unique facts, exactly one owner'); const processMessage = await fixture(); const processWork = await db.smsAttemptCompletionWork.create({ data: { workKey: `message:${processMessage.id}`, messageRecordId: processMessage.id, revision: 20, nextAttemptAt: new Date(0), }, }); await db.smsCompletionEvent.createMany({ data: Array.from({ length: 20 }, (_, i) => ({ workId: processWork.id, eventKey: randomUUID(), kind: 'receipt', payload: { id: `process-${i}` }, })), }); const child = () => new Promise((resolve, reject) => { const processWorker = spawn( process.execPath, [fileURLToPath(import.meta.url), '--worker', processWork.id, prefix], { env: process.env, windowsHide: true, stdio: 'pipe', timeout: 20000 }, ); processWorker.once('error', reject); processWorker.once('exit', (code) => (code === 0 ? resolve() : reject(new Error(`child worker exit ${code}`)))); }); await Promise.all([child(), child()]); assert.equal(await db.operationLog.count({ where: { action: 'qa.process', resource: prefix } }), 20); assert.equal((await db.smsAttemptCompletionWork.findUniqueOrThrow({ where: { id: processWork.id } })).state, 'idle'); pass('two independent OS processes consume one durable work item without duplicate effects'); const crashMessage = await fixture(); let crash = true; const crashWorker = new AttemptCompletion( db, async () => { await scoped.smsMessageRecord.update({ where: { id: crashMessage.id }, data: { status: 'delivered' } }); await scoped.operationLog.create({ data: { action: 'qa.atomic', resource: prefix, resourceId: crashMessage.id }, }); if (crash) throw new Error('injected'); }, async () => {}, ); await crashWorker.enqueue(crashMessage.id, undefined, 'receipt', { id: 'crash' }); let crashWork = await db.smsAttemptCompletionWork.findUniqueOrThrow({ where: { workKey: `message:${crashMessage.id}` }, }); assert.equal(crashWork.state, 'retry_wait'); assert.equal((await db.smsMessageRecord.findUniqueOrThrow({ where: { id: crashMessage.id } })).status, 'queued'); assert.equal(await db.operationLog.count({ where: { resourceId: crashMessage.id } }), 0); crash = false; await db.smsAttemptCompletionWork.update({ where: { id: crashWork.id }, data: { nextAttemptAt: new Date(0) } }); await crashWorker.process(crashWork.id); crashWork = await db.smsAttemptCompletionWork.findUniqueOrThrow({ where: { id: crashWork.id } }); assert.equal(crashWork.state, 'idle'); assert.equal(await db.operationLog.count({ where: { resourceId: crashMessage.id } }), 1); pass('failure rolls back all effects; durable retry commits once'); const routeMessage = await fixture(); let selected = 0; let waited = 0; const routed = { channel: { id: 'qa' } }; const routeWorker = new AttemptCompletion( db, async () => { await scoped.smsMessageRecord.update({ where: { id: routeMessage.id }, data: { status: 'delivered' } }); if (!completionContext.getStore().routePlanned) throw new CompletionRouteRequired(async () => { assert.equal(completionContext.getStore(), undefined); assert.equal( (await db.smsMessageRecord.findUniqueOrThrow({ where: { id: routeMessage.id } })).status, 'queued', ); selected++; return routed; }); assert.equal(completionContext.getStore().route.channel.id, 'qa'); }, async () => { assert.equal(completionContext.getStore(), undefined); waited++; }, ); await routeWorker.enqueue(routeMessage.id, undefined, 'receipt', { id: 'route' }); assert.equal(selected, 1); assert.equal(waited, 1); assert.equal((await db.smsMessageRecord.findUniqueOrThrow({ where: { id: routeMessage.id } })).status, 'delivered'); pass('route planning and rate limiting run outside rolled-back transaction'); const stalledMessage = await fixture(); const stalled = await db.smsAttemptCompletionWork.create({ data: { workKey: `message:${stalledMessage.id}`, messageRecordId: stalledMessage.id, state: 'processing', leaseOwner: 'dead-process', leaseUntil: new Date(0), fenceVersion: 3, revision: 1, attempts: 1, }, }); await db.smsCompletionEvent.create({ data: { workId: stalled.id, eventKey: randomUUID(), kind: 'receipt', payload: { id: 'takeover' } }, }); await workers[0].process(stalled.id); const recovered = await db.smsAttemptCompletionWork.findUniqueOrThrow({ where: { id: stalled.id } }); assert.equal(recovered.state, 'idle'); assert.equal(recovered.fenceVersion, 4); assert.equal( ( await db.smsAttemptCompletionWork.updateMany({ where: { id: stalled.id, fenceVersion: 3, leaseOwner: 'dead-process' }, data: { state: 'processing' }, }) ).count, 0, ); pass('expired process lease is taken over; old owner cannot write'); const fencedMessage = await fixture(); let entered; let releaseOld; const enteredRoute = new Promise((resolve) => { entered = resolve; }); const holdRoute = new Promise((resolve) => { releaseOld = resolve; }); const oldWorker = new AttemptCompletion( db, async () => { if (!completionContext.getStore().routePlanned) throw new CompletionRouteRequired(async () => { entered(); await holdRoute; return routed; }); await scoped.operationLog.create({ data: { action: 'qa.fenced', resource: prefix, resourceId: fencedMessage.id }, }); }, async () => {}, ); const oldAttempt = oldWorker.enqueue(fencedMessage.id, undefined, 'receipt', { id: 'fenced' }); await enteredRoute; const fencedWork = await db.smsAttemptCompletionWork.findUniqueOrThrow({ where: { workKey: `message:${fencedMessage.id}` }, }); await db.smsAttemptCompletionWork.update({ where: { id: fencedWork.id }, data: { leaseUntil: new Date(0) } }); const newWorker = new AttemptCompletion( db, async () => { await scoped.operationLog.create({ data: { action: 'qa.fenced', resource: prefix, resourceId: fencedMessage.id }, }); }, async () => {}, ); await newWorker.process(fencedWork.id); releaseOld(); await oldAttempt; assert.equal(await db.operationLog.count({ where: { action: 'qa.fenced', resourceId: fencedMessage.id } }), 1); assert.equal((await db.smsAttemptCompletionWork.findUniqueOrThrow({ where: { id: fencedWork.id } })).state, 'idle'); pass('suspended old consumer resumes after takeover and is fenced before any side effect'); const tenant = await db.tenant.create({ data: { name: '并发回执隔离验收', code: randomUUID() } }); const app = await db.smsApplication.create({ data: { tenantId: tenant.id, name: '隔离验收', cmppAccount: randomUUID(), cmppEnterpriseCode: '000001', secretHash: 'disabled', httpConfig: { create: { enabled: true, receiptWebhookEnabled: true } }, }, }); await db.httpWebhookEndpoint.create({ data: { applicationId: app.id, eventType: 'receipt', url: 'https://example.invalid/never-send', secretEncrypted: 'unused', secretLast4: 'none', }, }); const channel = await db.smsChannel.create({ data: { name: '隔离不联网', code: randomUUID(), gatewayHost: '127.0.0.1', gatewayPort: 1, account: randomUUID(), passwordCipher: 'unused', srcId: '1069', status: 'disabled', carriers: ['mobile'], }, }); const task = await db.smsBatchTask.create({ data: { tenantId: tenant.id, applicationId: app.id, taskNo: randomUUID(), sourceType: 'cmpp', content: '测试'.repeat(100), phoneTotal: 1, }, }); const long = await db.smsMessageRecord.create({ data: { tenantId: tenant.id, applicationId: app.id, batchTaskId: task.id, messageId: randomUUID(), phoneNumber: '13800138000', content: task.content, billingUnits: 3, status: 'submitted', channelId: channel.id, submitId: randomUUID(), cmppSubmitSequenceId: '42', cmppRegisteredDelivery: true, }, }); const submit = await db.smsSubmitRecord.create({ data: { tenantId: tenant.id, messageRecordId: long.id, channelId: channel.id, submitId: long.submitId, submitStatus: 'accepted', }, }); await db.cmppInboundLongMessage.create({ data: { tenantId: tenant.id, applicationId: app.id, groupKey: randomUUID(), account: app.cmppAccount, phoneNumbers: [long.phoneNumber], concatReference: 7, segmentTotal: 3, msgFmt: 8, messageId: long.messageId, expiresAt: new Date(Date.now() + 60000), status: 'completed', segments: { create: [1, 2, 3].map((index) => ({ segmentIndex: index, sequenceId: String(40 + index), registeredDelivery: index !== 3, content: '测试', contentHash: randomUUID(), })), }, }, }); await db.smsMessageRecord.update({ where: { id: long.id }, data: { cmppSubmitGroupMessageId: long.messageId } }); const billing = new BillingService(db); const api = new OpenApiService(db, undefined); // No worker or network publisher is started: this stage verifies real business // persistence. Gateway/HTTP delivery is a separate environment acceptance stage. const chain = new SendChainService(db, billing, {}, {}, api); for (let index = 1; index <= 3; index++) await chain.handleSubmitSegmentResult({ messageId: long.messageId, channelId: channel.id, submitId: submit.submitId, gatewayMessageId: `${prefix}-${index}`, sequenceId: index, segmentIndex: index, segmentTotal: 3, submitStatus: 'accepted', }); const receipt = (index) => ({ messageId: long.messageId, channelId: channel.id, gatewayMessageId: `${prefix}-${index}`, phoneNumber: long.phoneNumber, receiptStatus: 'delivered', rawStatus: 'DELIVRD', deliveredAt: '2026-09-16T00:00:00.000Z', }); await chain.handleReceipt(receipt(1)); assert.notEqual((await db.smsMessageRecord.findUniqueOrThrow({ where: { id: long.id } })).status, 'delivered'); await Promise.all([ chain.handleReceipt(receipt(2)), chain.handleReceipt(receipt(3)), chain.handleReceipt(receipt(2)), ]); await chain.attemptCompletion.scan(); assert.equal((await db.smsMessageRecord.findUniqueOrThrow({ where: { id: long.id } })).status, 'delivered'); assert.equal(await db.smsReceiptRecord.count({ where: { messageRecordId: long.id } }), 3); assert.equal(await db.cmppDownstreamDelivery.count({ where: { messageRecordId: long.id } }), 2); assert.equal(await db.httpWebhookEvent.count({ where: { messageRecordId: long.id } }), 1); assert.equal((await db.smsBatchTask.findUniqueOrThrow({ where: { id: task.id } })).successTotal, 1); pass( 'real SendChain: all three receipts required; two requested client fragments and one HTTP notification, unrequested fragment omitted', ); await db.tenantAccount.create({ data: { tenantId: tenant.id, balanceCents: 100000 } }); const moneyMessage = await db.smsMessageRecord.create({ data: { tenantId: tenant.id, applicationId: app.id, batchTaskId: task.id, messageId: randomUUID(), phoneNumber: '13800138000', content: task.content, billingUnits: 3, unitPrice: 325, amountCents: 975, status: 'submitted', channelId: channel.id, submitId: randomUUID(), submittedAt: new Date(Date.now() - 73 * 3600_000), }, }); await db.smsSubmitRecord.create({ data: { tenantId: tenant.id, messageRecordId: moneyMessage.id, channelId: channel.id, submitId: moneyMessage.submitId, submitStatus: 'accepted', }, }); await billing.freeze({ tenantId: tenant.id, amountCents: 975, idempotencyKey: `qa-freeze:${moneyMessage.id}` }); await chain.handleSubmitResult({ messageId: moneyMessage.messageId, channelId: channel.id, submitId: moneyMessage.submitId, gatewayMessageId: `${prefix}-money`, submitStatus: 'accepted', submittedAt: moneyMessage.submittedAt.toISOString(), eventId: randomUUID(), }); assert.equal( (await db.smsBillingRecord.findFirstOrThrow({ where: { messageId: moneyMessage.messageId } })).billingStatus, 'charged', ); await Promise.all([ chain.markUnknownTimeout({ olderThanHours: 72 }), chain.markUnknownTimeout({ olderThanHours: 72 }), ]); await chain.attemptCompletion.scan(); assert.equal((await db.smsMessageRecord.findUniqueOrThrow({ where: { id: moneyMessage.id } })).status, 'timeout'); assert.equal((await db.tenantAccount.findUniqueOrThrow({ where: { tenantId: tenant.id } })).balanceCents, 100000n); assert.equal( await db.accountTransaction.count({ where: { idempotencyKey: `sms-refund:${moneyMessage.messageId}` } }), 1, ); await chain.handleReceipt({ messageId: moneyMessage.messageId, channelId: channel.id, gatewayMessageId: `${prefix}-money`, phoneNumber: moneyMessage.phoneNumber, receiptStatus: 'delivered', rawStatus: 'DELIVRD', }); assert.equal((await db.smsMessageRecord.findUniqueOrThrow({ where: { id: moneyMessage.id } })).status, 'timeout'); assert.equal( (await db.smsBillingRecord.findFirstOrThrow({ where: { messageId: moneyMessage.messageId } })).billingStatus, 'refunded', ); pass('nonzero charge, concurrent timeout, one refund and late receipt do not change terminal accounting'); const retryGroup = await db.smsChannelGroup.create({ data: { code: randomUUID(), name: '事务补发隔离', retryEnabled: true }, }); const retryMessage = await db.smsMessageRecord.create({ data: { tenantId: tenant.id, applicationId: app.id, batchTaskId: task.id, messageId: randomUUID(), phoneNumber: '13800138000', content: task.content, carrier: 'mobile', billingUnits: 3, status: 'submitted', channelId: channel.id, submitId: randomUUID(), }, }); const retrySource = await db.smsSubmitRecord.create({ data: { tenantId: tenant.id, messageRecordId: retryMessage.id, channelId: channel.id, submitId: retryMessage.submitId, submitStatus: 'accepted', }, }); // Routing/rate limiting are deliberately isolated here. Production routing and // Redis must additionally pass the authorized simulator acceptance. chain.findApplicationRoute = async () => ({ group: retryGroup, groupId: retryGroup.id }); chain.submission.selectChannelForMessage = async () => ({ channel, groupId: retryGroup.id, groupName: retryGroup.name, carrier: 'mobile', routeScope: 'national', }); chain.submission.waitForChannelRateLimit = async () => {}; for (let index = 1; index <= 3; index++) await chain.handleSubmitSegmentResult({ messageId: retryMessage.messageId, channelId: channel.id, submitId: retrySource.submitId, gatewayMessageId: `${prefix}-fail-${index}`, sequenceId: index, segmentIndex: index, segmentTotal: 3, submitStatus: 'accepted', }); await Promise.all( [1, 2, 3].map((index) => chain.handleReceipt({ messageId: retryMessage.messageId, channelId: channel.id, gatewayMessageId: `${prefix}-fail-${index}`, phoneNumber: retryMessage.phoneNumber, receiptStatus: 'undelivered', rawStatus: 'UNDELIV', }), ), ); await chain.attemptCompletion.scan(); const retries = await db.smsSubmitRecord.findMany({ where: { retryOfSubmitRecordId: retrySource.id } }); assert.equal(retries.length, 1); assert.equal(await db.gatewaySubmitOutbox.count({ where: { submitId: retries[0].submitId } }), 1); assert.equal(await db.httpWebhookEvent.count({ where: { messageRecordId: retryMessage.id } }), 0); assert.equal( (await db.smsMessageRecord.findUniqueOrThrow({ where: { id: retryMessage.id } })).submitId, retries[0].submitId, ); pass('three failure segments atomically create one successor Submit and Outbox without final failure notification'); const poisonMessage = await fixture(); const poisonWork = await db.smsAttemptCompletionWork.create({ data: { workKey: `message:${poisonMessage.id}`, messageRecordId: poisonMessage.id, revision: 1, attempts: 11, }, }); await db.smsCompletionEvent.create({ data: { workId: poisonWork.id, eventKey: randomUUID(), kind: 'receipt', payload: {} }, }); const poisonWorker = new AttemptCompletion( db, async () => { throw new Error('injected unavailable database'); }, async () => {}, ); await poisonWorker.process(poisonWork.id); assert.equal( (await db.smsAttemptCompletionWork.findUniqueOrThrow({ where: { id: poisonWork.id } })).state, 'needs_review', ); const durableAlerts = await retainAlerts(db, [], new Date()); assert(durableAlerts.some((item) => item.name === 'SmsCompletionNeedsReview' && item.status === 'firing')); await db.smsAttemptCompletionWork.update({ where: { id: poisonWork.id }, data: { state: 'pending', nextAttemptAt: new Date(0) }, }); await workers[0].process(poisonWork.id); assert.equal((await db.smsAttemptCompletionWork.findUniqueOrThrow({ where: { id: poisonWork.id } })).state, 'idle'); const recoveredAlerts = await retainAlerts(db, [], new Date(Date.now() + 1)); assert(recoveredAlerts.some((item) => item.name === 'SmsCompletionNeedsReview' && item.status === 'resolved')); pass( 'twelfth failure stops in needs_review and is a durable visible alert; isolated database recovery retains the resolved alert', ); const at = Date.now() + 10; const alert = { fingerprint: prefix.replaceAll('-', '').slice(0, 24), startedAt: new Date(at).toISOString(), name: 'QA', severity: 'warning', status: 'firing', acknowledged: false, }; const first = new Date(at + 1); const second = new Date(at + 2); await retainAlerts(db, [alert], first); const resolved = await retainAlerts(db, [], second); assert.equal(resolved.find((item) => item.fingerprint === alert.fingerprint)?.status, 'resolved'); await retainAlerts(db, [alert], first); assert.equal((await retainedAlerts(db)).find((item) => item.fingerprint === alert.fingerprint).status, 'resolved'); const operator = await db.user.create({ data: { username: randomUUID(), displayName: 'isolated QA', passwordHash: 'disabled', status: 'disabled' }, }); await Promise.all([ clearRetainedAlert(db, alert.fingerprint, alert.startedAt, operator.id), clearRetainedAlert(db, alert.fingerprint, alert.startedAt, operator.id), ]); assert.equal((await retainedAlerts(db)).filter((item) => item.fingerprint === alert.fingerprint).length, 0); await retainAlerts(db, [alert], new Date(at + 3)); assert.equal((await retainedAlerts(db)).filter((item) => item.fingerprint === alert.fingerprint).length, 0); await retainAlerts(db, [{ ...alert, startedAt: new Date(at + 4).toISOString() }], new Date(at + 5)); assert.equal((await retainedAlerts(db)).filter((item) => item.fingerprint === alert.fingerprint).length, 1); pass('recovery retains alert, stale snapshots ignored, manual clear idempotent, new occurrence visible'); } catch (error) { console.error(error instanceof Error ? error.message : String(error)); process.exitCode = 1; } finally { await db.onModuleDestroy(); }