fix: coordinate SMS completion and improve operations diagnostics

tools/testing/verify-attempt-completion.mjs

@@ -0,0 +1,602 @@
+import assert from 'node:assert/strict';
+import { createRequire } from 'node:module';
+import { randomUUID } from 'node:crypto';
+import { spawn } from 'node:child_process';
+import { fileURLToPath } from 'node:url';
+const require = createRequire(new URL('../../api/package.json', import.meta.url));
+const url = new URL(process.env.COMPLETION_TEST_DATABASE_URL || '');
+assert(['localhost', '127.0.0.1'].includes(url.hostname) && url.pathname.startsWith('/cmpp_qa_'));
+process.env.DATABASE_URL = url.toString();
+process.env.NODE_ENV = 'test';
+require('reflect-metadata');
+const { PrismaService } = require('./dist/prisma/prisma.service');
+const { AttemptCompletion } = require('./dist/send-chain/attempt-completion');
+const {
+  completionContext,
+  completionDatabase,
+  CompletionRouteRequired,
+} = require('./dist/send-chain/completion-context');
+const {
+  retainAlerts,
+  retainedAlerts,
+  clearRetainedAlert,
+} = require('./dist/infrastructure-monitoring/persistent-alerts');
+const { SendChainService } = require('./dist/send-chain/send-chain.service');
+const { BillingService } = require('./dist/billing/billing.service');
+const { OpenApiService } = require('./dist/open-api/open-api.service');
+const db = new PrismaService();
+const scoped = completionDatabase(db);
+const prefix = randomUUID();
+const pass = (name) => console.log('PASS', name);
+const fixture = async () => {
+  const message = await db.smsMessageRecord.create({
+    data: { messageId: randomUUID(), phoneNumber: '13800138000', content: '隔离并发验收'.repeat(30), billingUnits: 3 },
+  });
+  return message;
+};
+if (process.argv[2] === '--worker') {
+  try {
+    const worker = new AttemptCompletion(
+      db,
+      async (_kind, payload) => {
+        await scoped.operationLog.create({
+          data: { action: 'qa.process', resource: process.argv[4], resourceId: payload.id },
+        });
+      },
+      async () => {},
+    );
+    await worker.process(process.argv[3]);
+  } finally {
+    await db.onModuleDestroy();
+  }
+  process.exit(0);
+}
+try {
+  const message = await fixture();
+  let active = 0;
+  let maximum = 0;
+  const execute = async (_kind, payload) => {
+    active++;
+    maximum = Math.max(maximum, active);
+    try {
+      await scoped.$transaction(async (tx) => {
+        assert.equal(tx, completionContext.getStore().tx);
+        await tx.operationLog.create({ data: { action: 'qa.completion', resource: prefix, resourceId: payload.id } });
+      });
+    } finally {
+      active--;
+    }
+  };
+  const workers = [
+    new AttemptCompletion(db, execute, async () => {}),
+    new AttemptCompletion(db, execute, async () => {}),
+  ];
+  await Promise.all(
+    Array.from({ length: 24 }, (_, i) =>
+      workers[i % 2].enqueue(message.id, undefined, 'receipt', { id: `segment-${i % 12}` }),
+    ),
+  );
+  await Promise.all(workers.map((worker) => worker.scan()));
+  const work = await db.smsAttemptCompletionWork.findUniqueOrThrow({ where: { workKey: `message:${message.id}` } });
+  assert.equal(maximum, 1);
+  assert.equal(work.revision, 12);
+  assert.equal(work.processedRevision, 12);
+  assert.equal(work.state, 'idle');
+  assert.equal(await db.operationLog.count({ where: { resource: prefix } }), 12);
+  pass('two consumers, 24 concurrent receipts, 12 unique facts, exactly one owner');
+
+  const processMessage = await fixture();
+  const processWork = await db.smsAttemptCompletionWork.create({
+    data: {
+      workKey: `message:${processMessage.id}`,
+      messageRecordId: processMessage.id,
+      revision: 20,
+      nextAttemptAt: new Date(0),
+    },
+  });
+  await db.smsCompletionEvent.createMany({
+    data: Array.from({ length: 20 }, (_, i) => ({
+      workId: processWork.id,
+      eventKey: randomUUID(),
+      kind: 'receipt',
+      payload: { id: `process-${i}` },
+    })),
+  });
+  const child = () =>
+    new Promise((resolve, reject) => {
+      const processWorker = spawn(
+        process.execPath,
+        [fileURLToPath(import.meta.url), '--worker', processWork.id, prefix],
+        { env: process.env, windowsHide: true, stdio: 'pipe', timeout: 20000 },
+      );
+      processWorker.once('error', reject);
+      processWorker.once('exit', (code) => (code === 0 ? resolve() : reject(new Error(`child worker exit ${code}`))));
+    });
+  await Promise.all([child(), child()]);
+  assert.equal(await db.operationLog.count({ where: { action: 'qa.process', resource: prefix } }), 20);
+  assert.equal((await db.smsAttemptCompletionWork.findUniqueOrThrow({ where: { id: processWork.id } })).state, 'idle');
+  pass('two independent OS processes consume one durable work item without duplicate effects');
+
+  const crashMessage = await fixture();
+  let crash = true;
+  const crashWorker = new AttemptCompletion(
+    db,
+    async () => {
+      await scoped.smsMessageRecord.update({ where: { id: crashMessage.id }, data: { status: 'delivered' } });
+      await scoped.operationLog.create({
+        data: { action: 'qa.atomic', resource: prefix, resourceId: crashMessage.id },
+      });
+      if (crash) throw new Error('injected');
+    },
+    async () => {},
+  );
+  await crashWorker.enqueue(crashMessage.id, undefined, 'receipt', { id: 'crash' });
+  let crashWork = await db.smsAttemptCompletionWork.findUniqueOrThrow({
+    where: { workKey: `message:${crashMessage.id}` },
+  });
+  assert.equal(crashWork.state, 'retry_wait');
+  assert.equal((await db.smsMessageRecord.findUniqueOrThrow({ where: { id: crashMessage.id } })).status, 'queued');
+  assert.equal(await db.operationLog.count({ where: { resourceId: crashMessage.id } }), 0);
+  crash = false;
+  await db.smsAttemptCompletionWork.update({ where: { id: crashWork.id }, data: { nextAttemptAt: new Date(0) } });
+  await crashWorker.process(crashWork.id);
+  crashWork = await db.smsAttemptCompletionWork.findUniqueOrThrow({ where: { id: crashWork.id } });
+  assert.equal(crashWork.state, 'idle');
+  assert.equal(await db.operationLog.count({ where: { resourceId: crashMessage.id } }), 1);
+  pass('failure rolls back all effects; durable retry commits once');
+
+  const routeMessage = await fixture();
+  let selected = 0;
+  let waited = 0;
+  const routed = { channel: { id: 'qa' } };
+  const routeWorker = new AttemptCompletion(
+    db,
+    async () => {
+      await scoped.smsMessageRecord.update({ where: { id: routeMessage.id }, data: { status: 'delivered' } });
+      if (!completionContext.getStore().routePlanned)
+        throw new CompletionRouteRequired(async () => {
+          assert.equal(completionContext.getStore(), undefined);
+          assert.equal(
+            (await db.smsMessageRecord.findUniqueOrThrow({ where: { id: routeMessage.id } })).status,
+            'queued',
+          );
+          selected++;
+          return routed;
+        });
+      assert.equal(completionContext.getStore().route.channel.id, 'qa');
+    },
+    async () => {
+      assert.equal(completionContext.getStore(), undefined);
+      waited++;
+    },
+  );
+  await routeWorker.enqueue(routeMessage.id, undefined, 'receipt', { id: 'route' });
+  assert.equal(selected, 1);
+  assert.equal(waited, 1);
+  assert.equal((await db.smsMessageRecord.findUniqueOrThrow({ where: { id: routeMessage.id } })).status, 'delivered');
+  pass('route planning and rate limiting run outside rolled-back transaction');
+
+  const stalledMessage = await fixture();
+  const stalled = await db.smsAttemptCompletionWork.create({
+    data: {
+      workKey: `message:${stalledMessage.id}`,
+      messageRecordId: stalledMessage.id,
+      state: 'processing',
+      leaseOwner: 'dead-process',
+      leaseUntil: new Date(0),
+      fenceVersion: 3,
+      revision: 1,
+      attempts: 1,
+    },
+  });
+  await db.smsCompletionEvent.create({
+    data: { workId: stalled.id, eventKey: randomUUID(), kind: 'receipt', payload: { id: 'takeover' } },
+  });
+  await workers[0].process(stalled.id);
+  const recovered = await db.smsAttemptCompletionWork.findUniqueOrThrow({ where: { id: stalled.id } });
+  assert.equal(recovered.state, 'idle');
+  assert.equal(recovered.fenceVersion, 4);
+  assert.equal(
+    (
+      await db.smsAttemptCompletionWork.updateMany({
+        where: { id: stalled.id, fenceVersion: 3, leaseOwner: 'dead-process' },
+        data: { state: 'processing' },
+      })
+    ).count,
+    0,
+  );
+  pass('expired process lease is taken over; old owner cannot write');
+
+  const fencedMessage = await fixture();
+  let entered;
+  let releaseOld;
+  const enteredRoute = new Promise((resolve) => {
+    entered = resolve;
+  });
+  const holdRoute = new Promise((resolve) => {
+    releaseOld = resolve;
+  });
+  const oldWorker = new AttemptCompletion(
+    db,
+    async () => {
+      if (!completionContext.getStore().routePlanned)
+        throw new CompletionRouteRequired(async () => {
+          entered();
+          await holdRoute;
+          return routed;
+        });
+      await scoped.operationLog.create({
+        data: { action: 'qa.fenced', resource: prefix, resourceId: fencedMessage.id },
+      });
+    },
+    async () => {},
+  );
+  const oldAttempt = oldWorker.enqueue(fencedMessage.id, undefined, 'receipt', { id: 'fenced' });
+  await enteredRoute;
+  const fencedWork = await db.smsAttemptCompletionWork.findUniqueOrThrow({
+    where: { workKey: `message:${fencedMessage.id}` },
+  });
+  await db.smsAttemptCompletionWork.update({ where: { id: fencedWork.id }, data: { leaseUntil: new Date(0) } });
+  const newWorker = new AttemptCompletion(
+    db,
+    async () => {
+      await scoped.operationLog.create({
+        data: { action: 'qa.fenced', resource: prefix, resourceId: fencedMessage.id },
+      });
+    },
+    async () => {},
+  );
+  await newWorker.process(fencedWork.id);
+  releaseOld();
+  await oldAttempt;
+  assert.equal(await db.operationLog.count({ where: { action: 'qa.fenced', resourceId: fencedMessage.id } }), 1);
+  assert.equal((await db.smsAttemptCompletionWork.findUniqueOrThrow({ where: { id: fencedWork.id } })).state, 'idle');
+  pass('suspended old consumer resumes after takeover and is fenced before any side effect');
+
+  const tenant = await db.tenant.create({ data: { name: '并发回执隔离验收', code: randomUUID() } });
+  const app = await db.smsApplication.create({
+    data: {
+      tenantId: tenant.id,
+      name: '隔离验收',
+      cmppAccount: randomUUID(),
+      cmppEnterpriseCode: '000001',
+      secretHash: 'disabled',
+      httpConfig: { create: { enabled: true, receiptWebhookEnabled: true } },
+    },
+  });
+  await db.httpWebhookEndpoint.create({
+    data: {
+      applicationId: app.id,
+      eventType: 'receipt',
+      url: 'https://example.invalid/never-send',
+      secretEncrypted: 'unused',
+      secretLast4: 'none',
+    },
+  });
+  const channel = await db.smsChannel.create({
+    data: {
+      name: '隔离不联网',
+      code: randomUUID(),
+      gatewayHost: '127.0.0.1',
+      gatewayPort: 1,
+      account: randomUUID(),
+      passwordCipher: 'unused',
+      srcId: '1069',
+      status: 'disabled',
+      carriers: ['mobile'],
+    },
+  });
+  const task = await db.smsBatchTask.create({
+    data: {
+      tenantId: tenant.id,
+      applicationId: app.id,
+      taskNo: randomUUID(),
+      sourceType: 'cmpp',
+      content: '测试'.repeat(100),
+      phoneTotal: 1,
+    },
+  });
+  const long = await db.smsMessageRecord.create({
+    data: {
+      tenantId: tenant.id,
+      applicationId: app.id,
+      batchTaskId: task.id,
+      messageId: randomUUID(),
+      phoneNumber: '13800138000',
+      content: task.content,
+      billingUnits: 3,
+      status: 'submitted',
+      channelId: channel.id,
+      submitId: randomUUID(),
+      cmppSubmitSequenceId: '42',
+      cmppRegisteredDelivery: true,
+    },
+  });
+  const submit = await db.smsSubmitRecord.create({
+    data: {
+      tenantId: tenant.id,
+      messageRecordId: long.id,
+      channelId: channel.id,
+      submitId: long.submitId,
+      submitStatus: 'accepted',
+    },
+  });
+  await db.cmppInboundLongMessage.create({
+    data: {
+      tenantId: tenant.id,
+      applicationId: app.id,
+      groupKey: randomUUID(),
+      account: app.cmppAccount,
+      phoneNumbers: [long.phoneNumber],
+      concatReference: 7,
+      segmentTotal: 3,
+      msgFmt: 8,
+      messageId: long.messageId,
+      expiresAt: new Date(Date.now() + 60000),
+      status: 'completed',
+      segments: {
+        create: [1, 2, 3].map((index) => ({
+          segmentIndex: index,
+          sequenceId: String(40 + index),
+          registeredDelivery: index !== 3,
+          content: '测试',
+          contentHash: randomUUID(),
+        })),
+      },
+    },
+  });
+  await db.smsMessageRecord.update({ where: { id: long.id }, data: { cmppSubmitGroupMessageId: long.messageId } });
+  const billing = new BillingService(db);
+  const api = new OpenApiService(db, undefined);
+  // No worker or network publisher is started: this stage verifies real business
+  // persistence. Gateway/HTTP delivery is a separate environment acceptance stage.
+  const chain = new SendChainService(db, billing, {}, {}, api);
+  for (let index = 1; index <= 3; index++)
+    await chain.handleSubmitSegmentResult({
+      messageId: long.messageId,
+      channelId: channel.id,
+      submitId: submit.submitId,
+      gatewayMessageId: `${prefix}-${index}`,
+      sequenceId: index,
+      segmentIndex: index,
+      segmentTotal: 3,
+      submitStatus: 'accepted',
+    });
+  const receipt = (index) => ({
+    messageId: long.messageId,
+    channelId: channel.id,
+    gatewayMessageId: `${prefix}-${index}`,
+    phoneNumber: long.phoneNumber,
+    receiptStatus: 'delivered',
+    rawStatus: 'DELIVRD',
+    deliveredAt: '2026-09-16T00:00:00.000Z',
+  });
+  await chain.handleReceipt(receipt(1));
+  assert.notEqual((await db.smsMessageRecord.findUniqueOrThrow({ where: { id: long.id } })).status, 'delivered');
+  await Promise.all([
+    chain.handleReceipt(receipt(2)),
+    chain.handleReceipt(receipt(3)),
+    chain.handleReceipt(receipt(2)),
+  ]);
+  await chain.attemptCompletion.scan();
+  assert.equal((await db.smsMessageRecord.findUniqueOrThrow({ where: { id: long.id } })).status, 'delivered');
+  assert.equal(await db.smsReceiptRecord.count({ where: { messageRecordId: long.id } }), 3);
+  assert.equal(await db.cmppDownstreamDelivery.count({ where: { messageRecordId: long.id } }), 2);
+  assert.equal(await db.httpWebhookEvent.count({ where: { messageRecordId: long.id } }), 1);
+  assert.equal((await db.smsBatchTask.findUniqueOrThrow({ where: { id: task.id } })).successTotal, 1);
+  pass(
+    'real SendChain: all three receipts required; two requested client fragments and one HTTP notification, unrequested fragment omitted',
+  );
+
+  await db.tenantAccount.create({ data: { tenantId: tenant.id, balanceCents: 100000 } });
+  const moneyMessage = await db.smsMessageRecord.create({
+    data: {
+      tenantId: tenant.id,
+      applicationId: app.id,
+      batchTaskId: task.id,
+      messageId: randomUUID(),
+      phoneNumber: '13800138000',
+      content: task.content,
+      billingUnits: 3,
+      unitPrice: 325,
+      amountCents: 975,
+      status: 'submitted',
+      channelId: channel.id,
+      submitId: randomUUID(),
+      submittedAt: new Date(Date.now() - 73 * 3600_000),
+    },
+  });
+  await db.smsSubmitRecord.create({
+    data: {
+      tenantId: tenant.id,
+      messageRecordId: moneyMessage.id,
+      channelId: channel.id,
+      submitId: moneyMessage.submitId,
+      submitStatus: 'accepted',
+    },
+  });
+  await billing.freeze({ tenantId: tenant.id, amountCents: 975, idempotencyKey: `qa-freeze:${moneyMessage.id}` });
+  await chain.handleSubmitResult({
+    messageId: moneyMessage.messageId,
+    channelId: channel.id,
+    submitId: moneyMessage.submitId,
+    gatewayMessageId: `${prefix}-money`,
+    submitStatus: 'accepted',
+    submittedAt: moneyMessage.submittedAt.toISOString(),
+    eventId: randomUUID(),
+  });
+  assert.equal(
+    (await db.smsBillingRecord.findFirstOrThrow({ where: { messageId: moneyMessage.messageId } })).billingStatus,
+    'charged',
+  );
+  await Promise.all([
+    chain.markUnknownTimeout({ olderThanHours: 72 }),
+    chain.markUnknownTimeout({ olderThanHours: 72 }),
+  ]);
+  await chain.attemptCompletion.scan();
+  assert.equal((await db.smsMessageRecord.findUniqueOrThrow({ where: { id: moneyMessage.id } })).status, 'timeout');
+  assert.equal((await db.tenantAccount.findUniqueOrThrow({ where: { tenantId: tenant.id } })).balanceCents, 100000n);
+  assert.equal(
+    await db.accountTransaction.count({ where: { idempotencyKey: `sms-refund:${moneyMessage.messageId}` } }),
+    1,
+  );
+  await chain.handleReceipt({
+    messageId: moneyMessage.messageId,
+    channelId: channel.id,
+    gatewayMessageId: `${prefix}-money`,
+    phoneNumber: moneyMessage.phoneNumber,
+    receiptStatus: 'delivered',
+    rawStatus: 'DELIVRD',
+  });
+  assert.equal((await db.smsMessageRecord.findUniqueOrThrow({ where: { id: moneyMessage.id } })).status, 'timeout');
+  assert.equal(
+    (await db.smsBillingRecord.findFirstOrThrow({ where: { messageId: moneyMessage.messageId } })).billingStatus,
+    'refunded',
+  );
+  pass('nonzero charge, concurrent timeout, one refund and late receipt do not change terminal accounting');
+
+  const retryGroup = await db.smsChannelGroup.create({
+    data: { code: randomUUID(), name: '事务补发隔离', retryEnabled: true },
+  });
+  const retryMessage = await db.smsMessageRecord.create({
+    data: {
+      tenantId: tenant.id,
+      applicationId: app.id,
+      batchTaskId: task.id,
+      messageId: randomUUID(),
+      phoneNumber: '13800138000',
+      content: task.content,
+      carrier: 'mobile',
+      billingUnits: 3,
+      status: 'submitted',
+      channelId: channel.id,
+      submitId: randomUUID(),
+    },
+  });
+  const retrySource = await db.smsSubmitRecord.create({
+    data: {
+      tenantId: tenant.id,
+      messageRecordId: retryMessage.id,
+      channelId: channel.id,
+      submitId: retryMessage.submitId,
+      submitStatus: 'accepted',
+    },
+  });
+  // Routing/rate limiting are deliberately isolated here. Production routing and
+  // Redis must additionally pass the authorized simulator acceptance.
+  chain.findApplicationRoute = async () => ({ group: retryGroup, groupId: retryGroup.id });
+  chain.submission.selectChannelForMessage = async () => ({
+    channel,
+    groupId: retryGroup.id,
+    groupName: retryGroup.name,
+    carrier: 'mobile',
+    routeScope: 'national',
+  });
+  chain.submission.waitForChannelRateLimit = async () => {};
+  for (let index = 1; index <= 3; index++)
+    await chain.handleSubmitSegmentResult({
+      messageId: retryMessage.messageId,
+      channelId: channel.id,
+      submitId: retrySource.submitId,
+      gatewayMessageId: `${prefix}-fail-${index}`,
+      sequenceId: index,
+      segmentIndex: index,
+      segmentTotal: 3,
+      submitStatus: 'accepted',
+    });
+  await Promise.all(
+    [1, 2, 3].map((index) =>
+      chain.handleReceipt({
+        messageId: retryMessage.messageId,
+        channelId: channel.id,
+        gatewayMessageId: `${prefix}-fail-${index}`,
+        phoneNumber: retryMessage.phoneNumber,
+        receiptStatus: 'undelivered',
+        rawStatus: 'UNDELIV',
+      }),
+    ),
+  );
+  await chain.attemptCompletion.scan();
+  const retries = await db.smsSubmitRecord.findMany({ where: { retryOfSubmitRecordId: retrySource.id } });
+  assert.equal(retries.length, 1);
+  assert.equal(await db.gatewaySubmitOutbox.count({ where: { submitId: retries[0].submitId } }), 1);
+  assert.equal(await db.httpWebhookEvent.count({ where: { messageRecordId: retryMessage.id } }), 0);
+  assert.equal(
+    (await db.smsMessageRecord.findUniqueOrThrow({ where: { id: retryMessage.id } })).submitId,
+    retries[0].submitId,
+  );
+  pass('three failure segments atomically create one successor Submit and Outbox without final failure notification');
+
+  const poisonMessage = await fixture();
+  const poisonWork = await db.smsAttemptCompletionWork.create({
+    data: {
+      workKey: `message:${poisonMessage.id}`,
+      messageRecordId: poisonMessage.id,
+      revision: 1,
+      attempts: 11,
+    },
+  });
+  await db.smsCompletionEvent.create({
+    data: { workId: poisonWork.id, eventKey: randomUUID(), kind: 'receipt', payload: {} },
+  });
+  const poisonWorker = new AttemptCompletion(
+    db,
+    async () => {
+      throw new Error('injected unavailable database');
+    },
+    async () => {},
+  );
+  await poisonWorker.process(poisonWork.id);
+  assert.equal(
+    (await db.smsAttemptCompletionWork.findUniqueOrThrow({ where: { id: poisonWork.id } })).state,
+    'needs_review',
+  );
+  const durableAlerts = await retainAlerts(db, [], new Date());
+  assert(durableAlerts.some((item) => item.name === 'SmsCompletionNeedsReview' && item.status === 'firing'));
+  await db.smsAttemptCompletionWork.update({
+    where: { id: poisonWork.id },
+    data: { state: 'pending', nextAttemptAt: new Date(0) },
+  });
+  await workers[0].process(poisonWork.id);
+  assert.equal((await db.smsAttemptCompletionWork.findUniqueOrThrow({ where: { id: poisonWork.id } })).state, 'idle');
+  const recoveredAlerts = await retainAlerts(db, [], new Date(Date.now() + 1));
+  assert(recoveredAlerts.some((item) => item.name === 'SmsCompletionNeedsReview' && item.status === 'resolved'));
+  pass(
+    'twelfth failure stops in needs_review and is a durable visible alert; audited recovery retains the resolved alert',
+  );
+
+  const at = Date.now() + 10;
+  const alert = {
+    fingerprint: prefix.replaceAll('-', '').slice(0, 24),
+    startedAt: new Date(at).toISOString(),
+    name: 'QA',
+    severity: 'warning',
+    status: 'firing',
+    acknowledged: false,
+  };
+  const first = new Date(at + 1);
+  const second = new Date(at + 2);
+  await retainAlerts(db, [alert], first);
+  const resolved = await retainAlerts(db, [], second);
+  assert.equal(resolved.find((item) => item.fingerprint === alert.fingerprint)?.status, 'resolved');
+  await retainAlerts(db, [alert], first);
+  assert.equal((await retainedAlerts(db)).find((item) => item.fingerprint === alert.fingerprint).status, 'resolved');
+  const operator = await db.user.create({
+    data: { username: randomUUID(), displayName: 'isolated QA', passwordHash: 'disabled', status: 'disabled' },
+  });
+  await Promise.all([
+    clearRetainedAlert(db, alert.fingerprint, alert.startedAt, operator.id),
+    clearRetainedAlert(db, alert.fingerprint, alert.startedAt, operator.id),
+  ]);
+  assert.equal((await retainedAlerts(db)).filter((item) => item.fingerprint === alert.fingerprint).length, 0);
+  await retainAlerts(db, [alert], new Date(at + 3));
+  assert.equal((await retainedAlerts(db)).filter((item) => item.fingerprint === alert.fingerprint).length, 0);
+  await retainAlerts(db, [{ ...alert, startedAt: new Date(at + 4).toISOString() }], new Date(at + 5));
+  assert.equal((await retainedAlerts(db)).filter((item) => item.fingerprint === alert.fingerprint).length, 1);
+  pass('recovery retains alert, stale snapshots ignored, manual clear idempotent, new occurrence visible');
+} catch (error) {
+  console.error(error instanceof Error ? error.message : String(error));
+  process.exitCode = 1;
+} finally {
+  await db.onModuleDestroy();
+}