603 lines
22 KiB
JavaScript
603 lines
22 KiB
JavaScript
import assert from 'node:assert/strict';
|
|
import { createRequire } from 'node:module';
|
|
import { randomUUID } from 'node:crypto';
|
|
import { spawn } from 'node:child_process';
|
|
import { fileURLToPath } from 'node:url';
|
|
const require = createRequire(new URL('../../api/package.json', import.meta.url));
|
|
const url = new URL(process.env.COMPLETION_TEST_DATABASE_URL || '');
|
|
assert(['localhost', '127.0.0.1'].includes(url.hostname) && url.pathname.startsWith('/cmpp_qa_'));
|
|
process.env.DATABASE_URL = url.toString();
|
|
process.env.NODE_ENV = 'test';
|
|
require('reflect-metadata');
|
|
const { PrismaService } = require('./dist/prisma/prisma.service');
|
|
const { AttemptCompletion } = require('./dist/send-chain/attempt-completion');
|
|
const {
|
|
completionContext,
|
|
completionDatabase,
|
|
CompletionRouteRequired,
|
|
} = require('./dist/send-chain/completion-context');
|
|
const {
|
|
retainAlerts,
|
|
retainedAlerts,
|
|
clearRetainedAlert,
|
|
} = require('./dist/infrastructure-monitoring/persistent-alerts');
|
|
const { SendChainService } = require('./dist/send-chain/send-chain.service');
|
|
const { BillingService } = require('./dist/billing/billing.service');
|
|
const { OpenApiService } = require('./dist/open-api/open-api.service');
|
|
const db = new PrismaService();
|
|
const scoped = completionDatabase(db);
|
|
const prefix = randomUUID();
|
|
const pass = (name) => console.log('PASS', name);
|
|
const fixture = async () => {
|
|
const message = await db.smsMessageRecord.create({
|
|
data: { messageId: randomUUID(), phoneNumber: '13800138000', content: '隔离并发验收'.repeat(30), billingUnits: 3 },
|
|
});
|
|
return message;
|
|
};
|
|
if (process.argv[2] === '--worker') {
|
|
try {
|
|
const worker = new AttemptCompletion(
|
|
db,
|
|
async (_kind, payload) => {
|
|
await scoped.operationLog.create({
|
|
data: { action: 'qa.process', resource: process.argv[4], resourceId: payload.id },
|
|
});
|
|
},
|
|
async () => {},
|
|
);
|
|
await worker.process(process.argv[3]);
|
|
} finally {
|
|
await db.onModuleDestroy();
|
|
}
|
|
process.exit(0);
|
|
}
|
|
try {
|
|
const message = await fixture();
|
|
let active = 0;
|
|
let maximum = 0;
|
|
const execute = async (_kind, payload) => {
|
|
active++;
|
|
maximum = Math.max(maximum, active);
|
|
try {
|
|
await scoped.$transaction(async (tx) => {
|
|
assert.equal(tx, completionContext.getStore().tx);
|
|
await tx.operationLog.create({ data: { action: 'qa.completion', resource: prefix, resourceId: payload.id } });
|
|
});
|
|
} finally {
|
|
active--;
|
|
}
|
|
};
|
|
const workers = [
|
|
new AttemptCompletion(db, execute, async () => {}),
|
|
new AttemptCompletion(db, execute, async () => {}),
|
|
];
|
|
await Promise.all(
|
|
Array.from({ length: 24 }, (_, i) =>
|
|
workers[i % 2].enqueue(message.id, undefined, 'receipt', { id: `segment-${i % 12}` }),
|
|
),
|
|
);
|
|
await Promise.all(workers.map((worker) => worker.scan()));
|
|
const work = await db.smsAttemptCompletionWork.findUniqueOrThrow({ where: { workKey: `message:${message.id}` } });
|
|
assert.equal(maximum, 1);
|
|
assert.equal(work.revision, 12);
|
|
assert.equal(work.processedRevision, 12);
|
|
assert.equal(work.state, 'idle');
|
|
assert.equal(await db.operationLog.count({ where: { resource: prefix } }), 12);
|
|
pass('two consumers, 24 concurrent receipts, 12 unique facts, exactly one owner');
|
|
|
|
const processMessage = await fixture();
|
|
const processWork = await db.smsAttemptCompletionWork.create({
|
|
data: {
|
|
workKey: `message:${processMessage.id}`,
|
|
messageRecordId: processMessage.id,
|
|
revision: 20,
|
|
nextAttemptAt: new Date(0),
|
|
},
|
|
});
|
|
await db.smsCompletionEvent.createMany({
|
|
data: Array.from({ length: 20 }, (_, i) => ({
|
|
workId: processWork.id,
|
|
eventKey: randomUUID(),
|
|
kind: 'receipt',
|
|
payload: { id: `process-${i}` },
|
|
})),
|
|
});
|
|
const child = () =>
|
|
new Promise((resolve, reject) => {
|
|
const processWorker = spawn(
|
|
process.execPath,
|
|
[fileURLToPath(import.meta.url), '--worker', processWork.id, prefix],
|
|
{ env: process.env, windowsHide: true, stdio: 'pipe', timeout: 20000 },
|
|
);
|
|
processWorker.once('error', reject);
|
|
processWorker.once('exit', (code) => (code === 0 ? resolve() : reject(new Error(`child worker exit ${code}`))));
|
|
});
|
|
await Promise.all([child(), child()]);
|
|
assert.equal(await db.operationLog.count({ where: { action: 'qa.process', resource: prefix } }), 20);
|
|
assert.equal((await db.smsAttemptCompletionWork.findUniqueOrThrow({ where: { id: processWork.id } })).state, 'idle');
|
|
pass('two independent OS processes consume one durable work item without duplicate effects');
|
|
|
|
const crashMessage = await fixture();
|
|
let crash = true;
|
|
const crashWorker = new AttemptCompletion(
|
|
db,
|
|
async () => {
|
|
await scoped.smsMessageRecord.update({ where: { id: crashMessage.id }, data: { status: 'delivered' } });
|
|
await scoped.operationLog.create({
|
|
data: { action: 'qa.atomic', resource: prefix, resourceId: crashMessage.id },
|
|
});
|
|
if (crash) throw new Error('injected');
|
|
},
|
|
async () => {},
|
|
);
|
|
await crashWorker.enqueue(crashMessage.id, undefined, 'receipt', { id: 'crash' });
|
|
let crashWork = await db.smsAttemptCompletionWork.findUniqueOrThrow({
|
|
where: { workKey: `message:${crashMessage.id}` },
|
|
});
|
|
assert.equal(crashWork.state, 'retry_wait');
|
|
assert.equal((await db.smsMessageRecord.findUniqueOrThrow({ where: { id: crashMessage.id } })).status, 'queued');
|
|
assert.equal(await db.operationLog.count({ where: { resourceId: crashMessage.id } }), 0);
|
|
crash = false;
|
|
await db.smsAttemptCompletionWork.update({ where: { id: crashWork.id }, data: { nextAttemptAt: new Date(0) } });
|
|
await crashWorker.process(crashWork.id);
|
|
crashWork = await db.smsAttemptCompletionWork.findUniqueOrThrow({ where: { id: crashWork.id } });
|
|
assert.equal(crashWork.state, 'idle');
|
|
assert.equal(await db.operationLog.count({ where: { resourceId: crashMessage.id } }), 1);
|
|
pass('failure rolls back all effects; durable retry commits once');
|
|
|
|
const routeMessage = await fixture();
|
|
let selected = 0;
|
|
let waited = 0;
|
|
const routed = { channel: { id: 'qa' } };
|
|
const routeWorker = new AttemptCompletion(
|
|
db,
|
|
async () => {
|
|
await scoped.smsMessageRecord.update({ where: { id: routeMessage.id }, data: { status: 'delivered' } });
|
|
if (!completionContext.getStore().routePlanned)
|
|
throw new CompletionRouteRequired(async () => {
|
|
assert.equal(completionContext.getStore(), undefined);
|
|
assert.equal(
|
|
(await db.smsMessageRecord.findUniqueOrThrow({ where: { id: routeMessage.id } })).status,
|
|
'queued',
|
|
);
|
|
selected++;
|
|
return routed;
|
|
});
|
|
assert.equal(completionContext.getStore().route.channel.id, 'qa');
|
|
},
|
|
async () => {
|
|
assert.equal(completionContext.getStore(), undefined);
|
|
waited++;
|
|
},
|
|
);
|
|
await routeWorker.enqueue(routeMessage.id, undefined, 'receipt', { id: 'route' });
|
|
assert.equal(selected, 1);
|
|
assert.equal(waited, 1);
|
|
assert.equal((await db.smsMessageRecord.findUniqueOrThrow({ where: { id: routeMessage.id } })).status, 'delivered');
|
|
pass('route planning and rate limiting run outside rolled-back transaction');
|
|
|
|
const stalledMessage = await fixture();
|
|
const stalled = await db.smsAttemptCompletionWork.create({
|
|
data: {
|
|
workKey: `message:${stalledMessage.id}`,
|
|
messageRecordId: stalledMessage.id,
|
|
state: 'processing',
|
|
leaseOwner: 'dead-process',
|
|
leaseUntil: new Date(0),
|
|
fenceVersion: 3,
|
|
revision: 1,
|
|
attempts: 1,
|
|
},
|
|
});
|
|
await db.smsCompletionEvent.create({
|
|
data: { workId: stalled.id, eventKey: randomUUID(), kind: 'receipt', payload: { id: 'takeover' } },
|
|
});
|
|
await workers[0].process(stalled.id);
|
|
const recovered = await db.smsAttemptCompletionWork.findUniqueOrThrow({ where: { id: stalled.id } });
|
|
assert.equal(recovered.state, 'idle');
|
|
assert.equal(recovered.fenceVersion, 4);
|
|
assert.equal(
|
|
(
|
|
await db.smsAttemptCompletionWork.updateMany({
|
|
where: { id: stalled.id, fenceVersion: 3, leaseOwner: 'dead-process' },
|
|
data: { state: 'processing' },
|
|
})
|
|
).count,
|
|
0,
|
|
);
|
|
pass('expired process lease is taken over; old owner cannot write');
|
|
|
|
const fencedMessage = await fixture();
|
|
let entered;
|
|
let releaseOld;
|
|
const enteredRoute = new Promise((resolve) => {
|
|
entered = resolve;
|
|
});
|
|
const holdRoute = new Promise((resolve) => {
|
|
releaseOld = resolve;
|
|
});
|
|
const oldWorker = new AttemptCompletion(
|
|
db,
|
|
async () => {
|
|
if (!completionContext.getStore().routePlanned)
|
|
throw new CompletionRouteRequired(async () => {
|
|
entered();
|
|
await holdRoute;
|
|
return routed;
|
|
});
|
|
await scoped.operationLog.create({
|
|
data: { action: 'qa.fenced', resource: prefix, resourceId: fencedMessage.id },
|
|
});
|
|
},
|
|
async () => {},
|
|
);
|
|
const oldAttempt = oldWorker.enqueue(fencedMessage.id, undefined, 'receipt', { id: 'fenced' });
|
|
await enteredRoute;
|
|
const fencedWork = await db.smsAttemptCompletionWork.findUniqueOrThrow({
|
|
where: { workKey: `message:${fencedMessage.id}` },
|
|
});
|
|
await db.smsAttemptCompletionWork.update({ where: { id: fencedWork.id }, data: { leaseUntil: new Date(0) } });
|
|
const newWorker = new AttemptCompletion(
|
|
db,
|
|
async () => {
|
|
await scoped.operationLog.create({
|
|
data: { action: 'qa.fenced', resource: prefix, resourceId: fencedMessage.id },
|
|
});
|
|
},
|
|
async () => {},
|
|
);
|
|
await newWorker.process(fencedWork.id);
|
|
releaseOld();
|
|
await oldAttempt;
|
|
assert.equal(await db.operationLog.count({ where: { action: 'qa.fenced', resourceId: fencedMessage.id } }), 1);
|
|
assert.equal((await db.smsAttemptCompletionWork.findUniqueOrThrow({ where: { id: fencedWork.id } })).state, 'idle');
|
|
pass('suspended old consumer resumes after takeover and is fenced before any side effect');
|
|
|
|
const tenant = await db.tenant.create({ data: { name: '并发回执隔离验收', code: randomUUID() } });
|
|
const app = await db.smsApplication.create({
|
|
data: {
|
|
tenantId: tenant.id,
|
|
name: '隔离验收',
|
|
cmppAccount: randomUUID(),
|
|
cmppEnterpriseCode: '000001',
|
|
secretHash: 'disabled',
|
|
httpConfig: { create: { enabled: true, receiptWebhookEnabled: true } },
|
|
},
|
|
});
|
|
await db.httpWebhookEndpoint.create({
|
|
data: {
|
|
applicationId: app.id,
|
|
eventType: 'receipt',
|
|
url: 'https://example.invalid/never-send',
|
|
secretEncrypted: 'unused',
|
|
secretLast4: 'none',
|
|
},
|
|
});
|
|
const channel = await db.smsChannel.create({
|
|
data: {
|
|
name: '隔离不联网',
|
|
code: randomUUID(),
|
|
gatewayHost: '127.0.0.1',
|
|
gatewayPort: 1,
|
|
account: randomUUID(),
|
|
passwordCipher: 'unused',
|
|
srcId: '1069',
|
|
status: 'disabled',
|
|
carriers: ['mobile'],
|
|
},
|
|
});
|
|
const task = await db.smsBatchTask.create({
|
|
data: {
|
|
tenantId: tenant.id,
|
|
applicationId: app.id,
|
|
taskNo: randomUUID(),
|
|
sourceType: 'cmpp',
|
|
content: '测试'.repeat(100),
|
|
phoneTotal: 1,
|
|
},
|
|
});
|
|
const long = await db.smsMessageRecord.create({
|
|
data: {
|
|
tenantId: tenant.id,
|
|
applicationId: app.id,
|
|
batchTaskId: task.id,
|
|
messageId: randomUUID(),
|
|
phoneNumber: '13800138000',
|
|
content: task.content,
|
|
billingUnits: 3,
|
|
status: 'submitted',
|
|
channelId: channel.id,
|
|
submitId: randomUUID(),
|
|
cmppSubmitSequenceId: '42',
|
|
cmppRegisteredDelivery: true,
|
|
},
|
|
});
|
|
const submit = await db.smsSubmitRecord.create({
|
|
data: {
|
|
tenantId: tenant.id,
|
|
messageRecordId: long.id,
|
|
channelId: channel.id,
|
|
submitId: long.submitId,
|
|
submitStatus: 'accepted',
|
|
},
|
|
});
|
|
await db.cmppInboundLongMessage.create({
|
|
data: {
|
|
tenantId: tenant.id,
|
|
applicationId: app.id,
|
|
groupKey: randomUUID(),
|
|
account: app.cmppAccount,
|
|
phoneNumbers: [long.phoneNumber],
|
|
concatReference: 7,
|
|
segmentTotal: 3,
|
|
msgFmt: 8,
|
|
messageId: long.messageId,
|
|
expiresAt: new Date(Date.now() + 60000),
|
|
status: 'completed',
|
|
segments: {
|
|
create: [1, 2, 3].map((index) => ({
|
|
segmentIndex: index,
|
|
sequenceId: String(40 + index),
|
|
registeredDelivery: index !== 3,
|
|
content: '测试',
|
|
contentHash: randomUUID(),
|
|
})),
|
|
},
|
|
},
|
|
});
|
|
await db.smsMessageRecord.update({ where: { id: long.id }, data: { cmppSubmitGroupMessageId: long.messageId } });
|
|
const billing = new BillingService(db);
|
|
const api = new OpenApiService(db, undefined);
|
|
// No worker or network publisher is started: this stage verifies real business
|
|
// persistence. Gateway/HTTP delivery is a separate environment acceptance stage.
|
|
const chain = new SendChainService(db, billing, {}, {}, api);
|
|
for (let index = 1; index <= 3; index++)
|
|
await chain.handleSubmitSegmentResult({
|
|
messageId: long.messageId,
|
|
channelId: channel.id,
|
|
submitId: submit.submitId,
|
|
gatewayMessageId: `${prefix}-${index}`,
|
|
sequenceId: index,
|
|
segmentIndex: index,
|
|
segmentTotal: 3,
|
|
submitStatus: 'accepted',
|
|
});
|
|
const receipt = (index) => ({
|
|
messageId: long.messageId,
|
|
channelId: channel.id,
|
|
gatewayMessageId: `${prefix}-${index}`,
|
|
phoneNumber: long.phoneNumber,
|
|
receiptStatus: 'delivered',
|
|
rawStatus: 'DELIVRD',
|
|
deliveredAt: '2026-09-16T00:00:00.000Z',
|
|
});
|
|
await chain.handleReceipt(receipt(1));
|
|
assert.notEqual((await db.smsMessageRecord.findUniqueOrThrow({ where: { id: long.id } })).status, 'delivered');
|
|
await Promise.all([
|
|
chain.handleReceipt(receipt(2)),
|
|
chain.handleReceipt(receipt(3)),
|
|
chain.handleReceipt(receipt(2)),
|
|
]);
|
|
await chain.attemptCompletion.scan();
|
|
assert.equal((await db.smsMessageRecord.findUniqueOrThrow({ where: { id: long.id } })).status, 'delivered');
|
|
assert.equal(await db.smsReceiptRecord.count({ where: { messageRecordId: long.id } }), 3);
|
|
assert.equal(await db.cmppDownstreamDelivery.count({ where: { messageRecordId: long.id } }), 2);
|
|
assert.equal(await db.httpWebhookEvent.count({ where: { messageRecordId: long.id } }), 1);
|
|
assert.equal((await db.smsBatchTask.findUniqueOrThrow({ where: { id: task.id } })).successTotal, 1);
|
|
pass(
|
|
'real SendChain: all three receipts required; two requested client fragments and one HTTP notification, unrequested fragment omitted',
|
|
);
|
|
|
|
await db.tenantAccount.create({ data: { tenantId: tenant.id, balanceCents: 100000 } });
|
|
const moneyMessage = await db.smsMessageRecord.create({
|
|
data: {
|
|
tenantId: tenant.id,
|
|
applicationId: app.id,
|
|
batchTaskId: task.id,
|
|
messageId: randomUUID(),
|
|
phoneNumber: '13800138000',
|
|
content: task.content,
|
|
billingUnits: 3,
|
|
unitPrice: 325,
|
|
amountCents: 975,
|
|
status: 'submitted',
|
|
channelId: channel.id,
|
|
submitId: randomUUID(),
|
|
submittedAt: new Date(Date.now() - 73 * 3600_000),
|
|
},
|
|
});
|
|
await db.smsSubmitRecord.create({
|
|
data: {
|
|
tenantId: tenant.id,
|
|
messageRecordId: moneyMessage.id,
|
|
channelId: channel.id,
|
|
submitId: moneyMessage.submitId,
|
|
submitStatus: 'accepted',
|
|
},
|
|
});
|
|
await billing.freeze({ tenantId: tenant.id, amountCents: 975, idempotencyKey: `qa-freeze:${moneyMessage.id}` });
|
|
await chain.handleSubmitResult({
|
|
messageId: moneyMessage.messageId,
|
|
channelId: channel.id,
|
|
submitId: moneyMessage.submitId,
|
|
gatewayMessageId: `${prefix}-money`,
|
|
submitStatus: 'accepted',
|
|
submittedAt: moneyMessage.submittedAt.toISOString(),
|
|
eventId: randomUUID(),
|
|
});
|
|
assert.equal(
|
|
(await db.smsBillingRecord.findFirstOrThrow({ where: { messageId: moneyMessage.messageId } })).billingStatus,
|
|
'charged',
|
|
);
|
|
await Promise.all([
|
|
chain.markUnknownTimeout({ olderThanHours: 72 }),
|
|
chain.markUnknownTimeout({ olderThanHours: 72 }),
|
|
]);
|
|
await chain.attemptCompletion.scan();
|
|
assert.equal((await db.smsMessageRecord.findUniqueOrThrow({ where: { id: moneyMessage.id } })).status, 'timeout');
|
|
assert.equal((await db.tenantAccount.findUniqueOrThrow({ where: { tenantId: tenant.id } })).balanceCents, 100000n);
|
|
assert.equal(
|
|
await db.accountTransaction.count({ where: { idempotencyKey: `sms-refund:${moneyMessage.messageId}` } }),
|
|
1,
|
|
);
|
|
await chain.handleReceipt({
|
|
messageId: moneyMessage.messageId,
|
|
channelId: channel.id,
|
|
gatewayMessageId: `${prefix}-money`,
|
|
phoneNumber: moneyMessage.phoneNumber,
|
|
receiptStatus: 'delivered',
|
|
rawStatus: 'DELIVRD',
|
|
});
|
|
assert.equal((await db.smsMessageRecord.findUniqueOrThrow({ where: { id: moneyMessage.id } })).status, 'timeout');
|
|
assert.equal(
|
|
(await db.smsBillingRecord.findFirstOrThrow({ where: { messageId: moneyMessage.messageId } })).billingStatus,
|
|
'refunded',
|
|
);
|
|
pass('nonzero charge, concurrent timeout, one refund and late receipt do not change terminal accounting');
|
|
|
|
const retryGroup = await db.smsChannelGroup.create({
|
|
data: { code: randomUUID(), name: '事务补发隔离', retryEnabled: true },
|
|
});
|
|
const retryMessage = await db.smsMessageRecord.create({
|
|
data: {
|
|
tenantId: tenant.id,
|
|
applicationId: app.id,
|
|
batchTaskId: task.id,
|
|
messageId: randomUUID(),
|
|
phoneNumber: '13800138000',
|
|
content: task.content,
|
|
carrier: 'mobile',
|
|
billingUnits: 3,
|
|
status: 'submitted',
|
|
channelId: channel.id,
|
|
submitId: randomUUID(),
|
|
},
|
|
});
|
|
const retrySource = await db.smsSubmitRecord.create({
|
|
data: {
|
|
tenantId: tenant.id,
|
|
messageRecordId: retryMessage.id,
|
|
channelId: channel.id,
|
|
submitId: retryMessage.submitId,
|
|
submitStatus: 'accepted',
|
|
},
|
|
});
|
|
// Routing/rate limiting are deliberately isolated here. Production routing and
|
|
// Redis must additionally pass the authorized simulator acceptance.
|
|
chain.findApplicationRoute = async () => ({ group: retryGroup, groupId: retryGroup.id });
|
|
chain.submission.selectChannelForMessage = async () => ({
|
|
channel,
|
|
groupId: retryGroup.id,
|
|
groupName: retryGroup.name,
|
|
carrier: 'mobile',
|
|
routeScope: 'national',
|
|
});
|
|
chain.submission.waitForChannelRateLimit = async () => {};
|
|
for (let index = 1; index <= 3; index++)
|
|
await chain.handleSubmitSegmentResult({
|
|
messageId: retryMessage.messageId,
|
|
channelId: channel.id,
|
|
submitId: retrySource.submitId,
|
|
gatewayMessageId: `${prefix}-fail-${index}`,
|
|
sequenceId: index,
|
|
segmentIndex: index,
|
|
segmentTotal: 3,
|
|
submitStatus: 'accepted',
|
|
});
|
|
await Promise.all(
|
|
[1, 2, 3].map((index) =>
|
|
chain.handleReceipt({
|
|
messageId: retryMessage.messageId,
|
|
channelId: channel.id,
|
|
gatewayMessageId: `${prefix}-fail-${index}`,
|
|
phoneNumber: retryMessage.phoneNumber,
|
|
receiptStatus: 'undelivered',
|
|
rawStatus: 'UNDELIV',
|
|
}),
|
|
),
|
|
);
|
|
await chain.attemptCompletion.scan();
|
|
const retries = await db.smsSubmitRecord.findMany({ where: { retryOfSubmitRecordId: retrySource.id } });
|
|
assert.equal(retries.length, 1);
|
|
assert.equal(await db.gatewaySubmitOutbox.count({ where: { submitId: retries[0].submitId } }), 1);
|
|
assert.equal(await db.httpWebhookEvent.count({ where: { messageRecordId: retryMessage.id } }), 0);
|
|
assert.equal(
|
|
(await db.smsMessageRecord.findUniqueOrThrow({ where: { id: retryMessage.id } })).submitId,
|
|
retries[0].submitId,
|
|
);
|
|
pass('three failure segments atomically create one successor Submit and Outbox without final failure notification');
|
|
|
|
const poisonMessage = await fixture();
|
|
const poisonWork = await db.smsAttemptCompletionWork.create({
|
|
data: {
|
|
workKey: `message:${poisonMessage.id}`,
|
|
messageRecordId: poisonMessage.id,
|
|
revision: 1,
|
|
attempts: 11,
|
|
},
|
|
});
|
|
await db.smsCompletionEvent.create({
|
|
data: { workId: poisonWork.id, eventKey: randomUUID(), kind: 'receipt', payload: {} },
|
|
});
|
|
const poisonWorker = new AttemptCompletion(
|
|
db,
|
|
async () => {
|
|
throw new Error('injected unavailable database');
|
|
},
|
|
async () => {},
|
|
);
|
|
await poisonWorker.process(poisonWork.id);
|
|
assert.equal(
|
|
(await db.smsAttemptCompletionWork.findUniqueOrThrow({ where: { id: poisonWork.id } })).state,
|
|
'needs_review',
|
|
);
|
|
const durableAlerts = await retainAlerts(db, [], new Date());
|
|
assert(durableAlerts.some((item) => item.name === 'SmsCompletionNeedsReview' && item.status === 'firing'));
|
|
await db.smsAttemptCompletionWork.update({
|
|
where: { id: poisonWork.id },
|
|
data: { state: 'pending', nextAttemptAt: new Date(0) },
|
|
});
|
|
await workers[0].process(poisonWork.id);
|
|
assert.equal((await db.smsAttemptCompletionWork.findUniqueOrThrow({ where: { id: poisonWork.id } })).state, 'idle');
|
|
const recoveredAlerts = await retainAlerts(db, [], new Date(Date.now() + 1));
|
|
assert(recoveredAlerts.some((item) => item.name === 'SmsCompletionNeedsReview' && item.status === 'resolved'));
|
|
pass(
|
|
'twelfth failure stops in needs_review and is a durable visible alert; audited recovery retains the resolved alert',
|
|
);
|
|
|
|
const at = Date.now() + 10;
|
|
const alert = {
|
|
fingerprint: prefix.replaceAll('-', '').slice(0, 24),
|
|
startedAt: new Date(at).toISOString(),
|
|
name: 'QA',
|
|
severity: 'warning',
|
|
status: 'firing',
|
|
acknowledged: false,
|
|
};
|
|
const first = new Date(at + 1);
|
|
const second = new Date(at + 2);
|
|
await retainAlerts(db, [alert], first);
|
|
const resolved = await retainAlerts(db, [], second);
|
|
assert.equal(resolved.find((item) => item.fingerprint === alert.fingerprint)?.status, 'resolved');
|
|
await retainAlerts(db, [alert], first);
|
|
assert.equal((await retainedAlerts(db)).find((item) => item.fingerprint === alert.fingerprint).status, 'resolved');
|
|
const operator = await db.user.create({
|
|
data: { username: randomUUID(), displayName: 'isolated QA', passwordHash: 'disabled', status: 'disabled' },
|
|
});
|
|
await Promise.all([
|
|
clearRetainedAlert(db, alert.fingerprint, alert.startedAt, operator.id),
|
|
clearRetainedAlert(db, alert.fingerprint, alert.startedAt, operator.id),
|
|
]);
|
|
assert.equal((await retainedAlerts(db)).filter((item) => item.fingerprint === alert.fingerprint).length, 0);
|
|
await retainAlerts(db, [alert], new Date(at + 3));
|
|
assert.equal((await retainedAlerts(db)).filter((item) => item.fingerprint === alert.fingerprint).length, 0);
|
|
await retainAlerts(db, [{ ...alert, startedAt: new Date(at + 4).toISOString() }], new Date(at + 5));
|
|
assert.equal((await retainedAlerts(db)).filter((item) => item.fingerprint === alert.fingerprint).length, 1);
|
|
pass('recovery retains alert, stale snapshots ignored, manual clear idempotent, new occurrence visible');
|
|
} catch (error) {
|
|
console.error(error instanceof Error ? error.message : String(error));
|
|
process.exitCode = 1;
|
|
} finally {
|
|
await db.onModuleDestroy();
|
|
}
|