fix: coordinate SMS completion and improve operations diagnostics

2026-09-16 18:27:29 +08:00
parent a0209f93bc
commit a350aca883
40 changed files with 2599 additions and 366 deletions
@@ -0,0 +1,143 @@
+import { BadRequestException, NotFoundException } from '@nestjs/common';
+import { Prisma } from '@prisma/client';
+import { createHash } from 'node:crypto';
+import { PrismaService } from '../prisma/prisma.service';
+import type { InfrastructureAlert } from './infrastructure-monitoring.contracts';
+
+export async function retainAlerts(prisma: PrismaService, alerts: InfrastructureAlert[], observedAt: Date) {
+  await prisma.$transaction(async (tx) => {
+    // Serialize snapshots across API processes; timestamps reject late HTTP results.
+    await tx.$executeRaw`SELECT pg_advisory_xact_lock(160916, 1)`;
+    const previous = await tx.infrastructureAlertCollection.findUnique({ where: { id: 'prometheus' } });
+    if (previous && previous.observedAt >= observedAt) return;
+    // Read durable work directly: application releases do not install Prometheus rules.
+    // Keep one occurrence identity until the condition really recovers, even after manual clear.
+    const reviewCount = await tx.smsAttemptCompletionWork.count({ where: { state: 'needs_review' } });
+    const oldest = await tx.smsCompletionEvent.findFirst({
+      where: { processedAt: null, work: { state: { in: ['pending', 'processing', 'retry_wait'] } } },
+      orderBy: { createdAt: 'asc' },
+      select: { createdAt: true },
+    });
+    const age = oldest ? Math.max(0, (observedAt.getTime() - oldest.createdAt.getTime()) / 1000) : 0;
+    alerts = [...alerts];
+    for (const condition of [
+      {
+        name: 'SmsCompletionNeedsReview',
+        active: reviewCount > 0,
+        severity: 'critical' as const,
+        summary: '短信收尾工作需要人工排查',
+        value: String(reviewCount),
+        threshold: '0',
+      },
+      {
+        name: 'SmsCompletionBacklog',
+        active: age > 300,
+        severity: 'warning' as const,
+        summary: '短信收尾工作等待超过5分钟',
+        value: `${Math.floor(age)}秒`,
+        threshold: '300秒',
+      },
+    ]) {
+      if (!condition.active) continue;
+      const fingerprint = createHash('sha256').update(`durable:${condition.name}`).digest('hex').slice(0, 24);
+      const occurrence = await tx.infrastructureAlertEvent.findFirst({
+        where: { fingerprint, recoveredAt: null },
+        orderBy: { activeAt: 'desc' },
+      });
+      alerts.push({
+        fingerprint,
+        name: condition.name,
+        severity: condition.severity,
+        status: 'firing',
+        startedAt: (occurrence?.activeAt ?? observedAt).toISOString(),
+        summary: condition.summary,
+        currentValue: condition.value,
+        threshold: condition.threshold,
+        service: '短信收尾',
+        acknowledged: false,
+      });
+    }
+    await tx.infrastructureAlertCollection.upsert({
+      where: { id: 'prometheus' },
+      create: { id: 'prometheus', observedAt },
+      update: { observedAt },
+    });
+    for (const alert of alerts) {
+      const activeAt = new Date(alert.startedAt);
+      const payload = JSON.parse(JSON.stringify(alert)) as Prisma.InputJsonValue;
+      await tx.infrastructureAlertEvent.createMany({
+        data: [{ fingerprint: alert.fingerprint, activeAt, payload, lastObservedAt: observedAt }],
+        skipDuplicates: true,
+      });
+      await tx.infrastructureAlertEvent.updateMany({
+        where: { fingerprint: alert.fingerprint, activeAt, lastObservedAt: { lte: observedAt } },
+        data: { payload, lastObservedAt: observedAt, recoveredAt: null },
+      });
+    }
+    await tx.infrastructureAlertEvent.updateMany({
+      where: {
+        recoveredAt: null,
+        lastObservedAt: { lt: observedAt },
+        ...(alerts.length
+          ? {
+              NOT: {
+                OR: alerts.map((alert) => ({ fingerprint: alert.fingerprint, activeAt: new Date(alert.startedAt) })),
+              },
+            }
+          : {}),
+      },
+      data: { recoveredAt: observedAt },
+    });
+  });
+  return retainedAlerts(prisma);
+}
+
+export async function retainedAlerts(prisma: PrismaService): Promise<InfrastructureAlert[]> {
+  const records = await prisma.infrastructureAlertEvent.findMany({
+    where: { clearedAt: null },
+    orderBy: [{ activeAt: 'desc' }, { id: 'asc' }],
+  });
+  return records.map((record) => ({
+    ...(record.payload as unknown as InfrastructureAlert),
+    ...(record.recoveredAt ? { status: 'resolved' } : {}),
+    acknowledged: false,
+  }));
+}
+
+export async function clearRetainedAlert(
+  prisma: PrismaService,
+  fingerprint: string,
+  rawActiveAt: unknown,
+  userId: string,
+) {
+  const activeAt = new Date(String(rawActiveAt ?? ''));
+  if (!/^[a-f0-9]{24}$/.test(fingerprint) || !Number.isFinite(activeAt.getTime()))
+    throw new BadRequestException('告警标识无效');
+  return prisma.$transaction(async (tx) => {
+    const record = await tx.infrastructureAlertEvent.findUnique({
+      where: { fingerprint_activeAt: { fingerprint, activeAt } },
+    });
+    if (!record) throw new NotFoundException('告警记录不存在');
+    const clearedAt = new Date();
+    const result = await tx.infrastructureAlertEvent.updateMany({
+      where: { id: record.id, clearedAt: null },
+      data: { clearedAt, clearedBy: userId },
+    });
+    if (result.count)
+      await tx.operationLog.create({
+        data: {
+          userId,
+          action: 'monitoring.alert_cleared',
+          resource: 'infrastructure_alert',
+          resourceId: record.id,
+          detail: { fingerprint, activeAt: activeAt.toISOString() },
+        },
+      });
+    return {
+      fingerprint,
+      activeAt: activeAt.toISOString(),
+      cleared: true,
+      clearedAt: (record.clearedAt ?? clearedAt).toISOString(),
+    };
+  });
+}