feat: 完善服务监控与下游重投

2026-08-14 17:21:29 +08:00
parent d30d9ea4d0
commit 1ef4380422
50 changed files with 1279 additions and 82 deletions
@@ -26,6 +26,7 @@ import { TenantsModule } from './tenants/tenants.module';
 import { UsersModule } from './users/users.module';
 import { SignatureRetirementModule } from './signature-retirement/signature-retirement.module';
 import { SecurityDetectionModule } from './security-detection/security-detection.module';
+import { MetricsModule } from './metrics/metrics.module';

@Module({
  imports: [
@@ -55,6 +56,7 @@ import { SecurityDetectionModule } from './security-detection/security-detection
    OpenApiModule,
    SignatureRetirementModule,
    SecurityDetectionModule,
+    MetricsModule,
  ],
  controllers: [HealthController],
  providers: [RequestContextMiddleware, SessionValidationMiddleware, ManualOperationAuditMiddleware],
@@ -12,6 +12,18 @@ export type InfrastructureServiceStatus = {
  status: 'healthy' | 'unhealthy' | 'unknown';
 };

+export type InfrastructureServiceMetricGroup = {
+  key: string;
+  name: string;
+  available: boolean;
+  metrics: Array<{
+    key: string;
+    label: string;
+    value: number | null;
+    unit: 'percent' | 'seconds' | 'count' | 'per_second' | 'bytes';
+  }>;
+};
+
 export type InfrastructureAlert = {
  fingerprint: string;
  name: string;
@@ -61,5 +73,6 @@ export type InfrastructureMonitoringOverview = {
    networkTransmitBytesPerSecond: InfrastructureMetricPoint[];
  };
  services: InfrastructureServiceStatus[];
+  serviceMetrics: InfrastructureServiceMetricGroup[];
  alerts: InfrastructureAlert[];
 };
@@ -55,6 +55,13 @@ describe('InfrastructureMonitoringService', () => {
          { metric: { name: 'nginx.service' }, value: [1_765_000_060, '1'] },
        ] });
      }
+      if (query.includes('cmpp:service_.*')) {
+        return success({ result: [
+          { metric: { __name__: 'cmpp:service_api:requests_per_second' }, value: [1_765_000_060, '12.5'] },
+          { metric: { __name__: 'cmpp:service_api:error_percent' }, value: [1_765_000_060, '0.2'] },
+          { metric: { __name__: 'cmpp:service_gateway:queue_pending' }, value: [1_765_000_060, '3'] },
+        ] });
+      }
      if (query.includes('timestamp(node_uname_info)')) {
        return success({ result: [{ metric: {}, value: [1_765_000_060, '1765000060'] }] });
      }
@@ -69,9 +76,13 @@ describe('InfrastructureMonitoringService', () => {
    expect(result.trends.cpuUsagePercent).toHaveLength(2);
    expect(result.summary).toMatchObject({ overallStatus: 'warning', serviceHealthy: 6, warningAlerts: 1 });
    expect(result.services.find((item) => item.key === 'redis')).toMatchObject({ unit: 'redis-server.service', status: 'healthy' });
+    expect(result.serviceMetrics.find((item) => item.key === 'api')).toMatchObject({ available: true });
+    expect(result.serviceMetrics.find((item) => item.key === 'gateway')?.metrics.find((item) => item.key === 'queuePending')?.value).toBe(3);
    expect(result.alerts[0]).toMatchObject({ name: 'HostCpuHigh', severity: 'warning', currentValue: '88.2' });
    expect(requestedUrls.filter((url) => url.pathname.endsWith('/query_range'))).toHaveLength(5);
    expect(requestedUrls.filter((url) => url.pathname.endsWith('/query_range')).every((url) => url.searchParams.get('step') === '60')).toBe(true);
+    expect(requestedUrls.find((url) => url.searchParams.get('query')?.includes('node_systemd_unit_state'))?.searchParams.get('query'))
+      .toContain('cmpp-api\\\\.service');
  });

  it('returns an explicit unavailable payload without stale metrics when Prometheus fails', async () => {
@@ -84,6 +95,7 @@ describe('InfrastructureMonitoringService', () => {
    expect(result.summary.overallStatus).toBe('unknown');
    expect(result.metrics.cpuUsagePercent).toBeNull();
    expect(result.trends.cpuUsagePercent).toEqual([]);
+    expect(result.serviceMetrics.every((item) => item.available === false)).toBe(true);
    expect(result.error).not.toContain('ECONNREFUSED');
  });
 });
@@ -1,4 +1,4 @@
-import { BadRequestException, Injectable } from '@nestjs/common';
+import { BadRequestException, Injectable, Logger } from '@nestjs/common';
 import { ConfigService } from '@nestjs/config';
 import { createHash } from 'node:crypto';
 import type {
@@ -7,6 +7,7 @@ import type {
  InfrastructureMonitoringOverview,
  InfrastructureMonitoringRange,
  InfrastructureServiceStatus,
+  InfrastructureServiceMetricGroup,
 } from './infrastructure-monitoring.contracts';

 type PrometheusSample = [number, string];
@@ -52,7 +53,8 @@ const QUERIES = {
  load1: 'node_load1',
  uptimeSeconds: 'time() - node_boot_time_seconds',
  lastSampleAt: 'max(timestamp(node_uname_info))',
-  services: 'max by (name) (node_systemd_unit_state{name=~"cmpp-api\\.service|cmpp-gateway\\.service|postgresql\\.service|redis(-server)?\\.service|cmpp-minio\\.service|nginx\\.service",state="active"})',
+  // PromQL字符串本身需要两个反斜杠才能把正则的“\.”传给RE2；TypeScript字面量因此需要写四个。
+  services: 'max by (name) (node_systemd_unit_state{name=~"cmpp-api\\\\.service|cmpp-gateway\\\\.service|postgresql\\\\.service|redis(-server)?\\\\.service|cmpp-minio\\\\.service|nginx\\\\.service",state="active"})',
 } as const;

 const SERVICE_DEFINITIONS = [
@@ -64,6 +66,43 @@ const SERVICE_DEFINITIONS = [
  { key: 'nginx', name: 'Nginx', units: ['nginx.service'] },
 ] as const;

+const SERVICE_METRIC_DEFINITIONS = [
+  { key: 'api', name: 'API服务', metrics: [
+    ['requestsPerSecond', '请求速率', 'cmpp:service_api:requests_per_second', 'per_second'],
+    ['errorPercent', '5xx错误率', 'cmpp:service_api:error_percent', 'percent'],
+    ['latencyP95', 'P95响应', 'cmpp:service_api:latency_p95_seconds', 'seconds'],
+    ['eventLoopP99', '事件循环P99', 'cmpp:service_api:event_loop_p99_seconds', 'seconds'],
+  ] },
+  { key: 'gateway', name: 'Gateway服务', metrics: [
+    ['submitsPerSecond', '提交速率', 'cmpp:service_gateway:submits_per_second', 'per_second'],
+    ['failurePercent', '提交失败率', 'cmpp:service_gateway:failure_percent', 'percent'],
+    ['queuePending', 'Stream pending', 'cmpp:service_gateway:queue_pending', 'count'],
+    ['queueOldestSeconds', '最旧pending', 'cmpp:service_gateway:queue_oldest_seconds', 'seconds'],
+  ] },
+  { key: 'postgresql', name: 'PostgreSQL', metrics: [
+    ['connectionPercent', '连接使用率', 'cmpp:service_postgresql:connection_percent', 'percent'],
+    ['deadlocks15m', '15分钟死锁', 'cmpp:service_postgresql:deadlocks_15m', 'count'],
+  ] },
+  { key: 'redis', name: 'Redis', metrics: [
+    ['memoryPercent', '内存使用率', 'cmpp:service_redis:memory_percent', 'percent'],
+    ['memoryUsedBytes', '已用内存', 'cmpp:service_redis:memory_used_bytes', 'bytes'],
+    ['connectedClients', '客户端连接', 'cmpp:service_redis:connected_clients', 'count'],
+    ['evictions5m', '5分钟淘汰', 'cmpp:service_redis:evictions_5m', 'count'],
+  ] },
+  { key: 'minio', name: 'MinIO', metrics: [
+    ['capacityPercent', '存储容量使用率', 'cmpp:service_minio:capacity_percent', 'percent'],
+    ['usageBytes', '对象数据量', 'cmpp:service_minio:usage_bytes', 'bytes'],
+    ['objects', '对象数', 'cmpp:service_minio:objects', 'count'],
+    ['drivesOffline', '离线存储盘', 'cmpp:service_minio:drives_offline', 'count'],
+  ] },
+  { key: 'nginx', name: 'Nginx', metrics: [
+    ['connectionsActive', '活跃连接', 'cmpp:service_nginx:connections_active', 'count'],
+    ['requestsPerSecond', '请求速率', 'cmpp:service_nginx:requests_per_second', 'per_second'],
+  ] },
+] as const;
+
+const SERVICE_METRICS_QUERY = '{__name__=~"cmpp:service_.*"}';
+
 function finiteNumber(value: string | number | undefined): number | null {
  const parsed = Number(value);
  return Number.isFinite(parsed) ? parsed : null;
@@ -119,6 +158,7 @@ function emptyTrends(): InfrastructureMonitoringOverview['trends'] {

@Injectable()
 export class InfrastructureMonitoringService {
+  private readonly logger = new Logger(InfrastructureMonitoringService.name);
  private readonly prometheusUrl: string;
  private readonly queryTimeoutMs: number;

@@ -131,13 +171,15 @@ export class InfrastructureMonitoringService {
    const range = this.parseRange(rawRange);
    const collectedAt = new Date().toISOString();
    try {
-      const [instant, trends, serviceResponse, alertResponse] = await Promise.all([
+      const [instant, trends, serviceResponse, serviceMetricResponse, alertResponse] = await Promise.all([
        this.loadInstantMetrics(),
        this.loadTrends(range),
        this.query(QUERIES.services),
+        this.query(SERVICE_METRICS_QUERY),
        this.getJson<PrometheusAlertResponse>('/api/v1/alerts'),
      ]);
      const services = this.parseServices(serviceResponse);
+      const serviceMetrics = this.parseServiceMetrics(serviceMetricResponse);
      const alerts = this.parseAlerts(alertResponse);
      const warningAlerts = alerts.filter((item) => item.severity === 'warning').length;
      const criticalAlerts = alerts.filter((item) => item.severity === 'critical').length;
@@ -158,9 +200,12 @@ export class InfrastructureMonitoringService {
        metrics: instant.metrics,
        trends,
        services,
+        serviceMetrics,
        alerts,
      };
-    } catch {
+    } catch (error) {
+      // 页面必须整体清空陈旧指标，但服务端仍需留下不含PromQL/地址/凭据的根因摘要便于运维诊断。
+      this.logger.warn(`Prometheus monitoring overview unavailable: ${error instanceof Error ? error.message : 'unknown error'}`);
      return this.unavailable(range, collectedAt);
    }
  }
@@ -228,6 +273,21 @@ export class InfrastructureMonitoringService {
      });
  }

+  private parseServiceMetrics(response: PrometheusQueryResponse): InfrastructureServiceMetricGroup[] {
+    const values = new Map<string, number>();
+    for (const item of response.data?.result ?? []) {
+      const metricName = item.metric.__name__;
+      const value = vectorValue({ status: 'success', data: { result: [item] } });
+      if (metricName && value !== null) values.set(metricName, value);
+    }
+    return SERVICE_METRIC_DEFINITIONS.map((group) => ({
+      key: group.key,
+      name: group.name,
+      available: group.metrics.some((metric) => values.has(metric[2])),
+      metrics: group.metrics.map(([key, label, metricName, unit]) => ({ key, label, value: values.get(metricName) ?? null, unit })),
+    }));
+  }
+
  private unavailable(range: InfrastructureMonitoringRange, collectedAt: string): InfrastructureMonitoringOverview {
    const services = SERVICE_DEFINITIONS.map((item) => ({ key: item.key, name: item.name, unit: item.units[0], status: 'unknown' as const }));
    return {
@@ -240,6 +300,7 @@ export class InfrastructureMonitoringService {
      metrics: emptyMetrics(),
      trends: emptyTrends(),
      services,
+      serviceMetrics: SERVICE_METRIC_DEFINITIONS.map((group) => ({ key: group.key, name: group.name, available: false, metrics: [] })),
      alerts: [],
    };
  }
@@ -1,8 +1,10 @@
 import 'reflect-metadata';
 import { NestFactory } from '@nestjs/core';
+import { createServer } from 'node:http';
 import type { NestExpressApplication } from '@nestjs/platform-express';
 import { DocumentBuilder, SwaggerModule } from '@nestjs/swagger';
 import { AppModule } from './app.module';
+import { MetricsService } from './metrics/metrics.service';
 import { OpenApiModule } from './open-api/open-api.module';
 import { configureHttpBodyParsers } from './http-body-limits';

@@ -39,7 +41,26 @@ async function bootstrap() {
  SwaggerModule.setup('api/client-docs', app, clientDocument);

  const port = Number(process.env.API_PORT ?? 3000);
-  await app.listen(port);
+  // 生产环境只允许 Nginx 访问管理 API；显式绑定回环，避免默认的全网卡监听绕过入口鉴权与限流。
+  const host = process.env.API_HOST?.trim() || '127.0.0.1';
+  await app.listen(port, host);
+
+  const metrics = app.get(MetricsService);
+  const metricsHost = process.env.API_METRICS_HOST?.trim() || '127.0.0.1';
+  const metricsPort = Number(process.env.API_METRICS_PORT ?? 9464);
+  const metricsServer = createServer((request, response) => {
+    if (request.method !== 'GET' || request.url !== '/metrics') {
+      response.writeHead(404).end();
+      return;
+    }
+    response.writeHead(200, { 'Content-Type': 'text/plain; version=0.0.4; charset=utf-8', 'Cache-Control': 'no-store' });
+    response.end(metrics.render());
+  });
+  // Metrics use a dedicated loopback listener so Nginx cannot accidentally expose them through /api/.
+  await new Promise<void>((resolve, reject) => {
+    metricsServer.once('error', reject);
+    metricsServer.listen(metricsPort, metricsHost, resolve);
+  });
 }

 void bootstrap();
@@ -0,0 +1,24 @@
+import { CallHandler, ExecutionContext, Injectable, NestInterceptor } from '@nestjs/common';
+import type { Observable } from 'rxjs';
+import { finalize } from 'rxjs/operators';
+import { MetricsService } from './metrics.service';
+
+type RequestLike = { method?: string; baseUrl?: string; route?: { path?: string } };
+type ResponseLike = { statusCode?: number };
+
+@Injectable()
+export class MetricsInterceptor implements NestInterceptor {
+  constructor(private readonly metrics: MetricsService) {}
+
+  intercept(context: ExecutionContext, next: CallHandler): Observable<unknown> {
+    if (context.getType() !== 'http') return next.handle();
+    const http = context.switchToHttp();
+    const request = http.getRequest<RequestLike>();
+    const response = http.getResponse<ResponseLike>();
+    const startedAt = this.metrics.beginRequest();
+    return next.handle().pipe(finalize(() => {
+      const route = `${request.baseUrl ?? ''}${request.route?.path ?? '/unmatched'}`;
+      this.metrics.finishRequest(startedAt, request.method ?? 'UNKNOWN', route, response.statusCode ?? 500);
+    }));
+  }
+}
@@ -0,0 +1,11 @@
+import { Global, Module } from '@nestjs/common';
+import { APP_INTERCEPTOR } from '@nestjs/core';
+import { MetricsInterceptor } from './metrics.interceptor';
+import { MetricsService } from './metrics.service';
+
+@Global()
+@Module({
+  providers: [MetricsService, { provide: APP_INTERCEPTOR, useClass: MetricsInterceptor }],
+  exports: [MetricsService],
+})
+export class MetricsModule {}
@@ -0,0 +1,16 @@
+import { MetricsService } from './metrics.service';
+
+describe('MetricsService', () => {
+  it('exports bounded API process and HTTP metrics without raw identifiers', () => {
+    const service = new MetricsService();
+    const startedAt = service.beginRequest();
+    service.finishRequest(startedAt, 'GET', '/api/admin/tenants/:id', 200);
+    const output = service.render();
+
+    expect(output).toContain('cmpp_api_process_resident_memory_bytes');
+    expect(output).toContain('cmpp_api_http_requests_total{method="GET",route="/api/admin/tenants/:id",status="200"} 1');
+    expect(output).toContain('cmpp_api_http_request_duration_seconds_bucket');
+    expect(output).not.toContain('phone_number');
+    service.onModuleDestroy();
+  });
+});
@@ -0,0 +1,100 @@
+import { Injectable, OnModuleDestroy } from '@nestjs/common';
+import { monitorEventLoopDelay } from 'node:perf_hooks';
+
+const HTTP_DURATION_BUCKETS = [0.05, 0.1, 0.25, 0.5, 1, 3, 10] as const;
+
+type HttpMetric = {
+  count: number;
+  durationSum: number;
+  buckets: number[];
+};
+
+function escapeLabel(value: string) {
+  return value.replace(/\\/g, '\\\\').replace(/\n/g, '\\n').replace(/"/g, '\\"');
+}
+
+function metricLine(name: string, value: number, labels?: Record<string, string>) {
+  const suffix = labels
+    ? `{${Object.entries(labels).map(([key, item]) => `${key}="${escapeLabel(item)}"`).join(',')}}`
+    : '';
+  return `${name}${suffix} ${Number.isFinite(value) ? value : 0}`;
+}
+
+@Injectable()
+export class MetricsService implements OnModuleDestroy {
+  private readonly startedAt = process.hrtime.bigint();
+  private readonly eventLoopDelay = monitorEventLoopDelay({ resolution: 20 });
+  private readonly http = new Map<string, HttpMetric>();
+  private inFlight = 0;
+
+  constructor() {
+    this.eventLoopDelay.enable();
+  }
+
+  beginRequest() {
+    this.inFlight += 1;
+    return process.hrtime.bigint();
+  }
+
+  finishRequest(startedAt: bigint, method: string, route: string, statusCode: number) {
+    this.inFlight = Math.max(0, this.inFlight - 1);
+    // Only route templates enter labels. Raw URLs, IDs, phone numbers and query strings would create unbounded time series.
+    const normalizedRoute = route.startsWith('/') ? route : `/${route}`;
+    const labels = [method.toUpperCase(), normalizedRoute, String(statusCode)];
+    const key = labels.join('\u0000');
+    const metric = this.http.get(key) ?? { count: 0, durationSum: 0, buckets: HTTP_DURATION_BUCKETS.map(() => 0) };
+    const durationSeconds = Number(process.hrtime.bigint() - startedAt) / 1_000_000_000;
+    metric.count += 1;
+    metric.durationSum += durationSeconds;
+    HTTP_DURATION_BUCKETS.forEach((bucket, index) => {
+      if (durationSeconds <= bucket) metric.buckets[index] += 1;
+    });
+    this.http.set(key, metric);
+  }
+
+  render() {
+    const memory = process.memoryUsage();
+    const uptime = Number(process.hrtime.bigint() - this.startedAt) / 1_000_000_000;
+    const lines = [
+      '# HELP cmpp_api_process_uptime_seconds API process uptime.',
+      '# TYPE cmpp_api_process_uptime_seconds gauge',
+      metricLine('cmpp_api_process_uptime_seconds', uptime),
+      '# HELP cmpp_api_process_resident_memory_bytes API resident memory.',
+      '# TYPE cmpp_api_process_resident_memory_bytes gauge',
+      metricLine('cmpp_api_process_resident_memory_bytes', memory.rss),
+      '# HELP cmpp_api_nodejs_heap_used_bytes Node.js heap currently used.',
+      '# TYPE cmpp_api_nodejs_heap_used_bytes gauge',
+      metricLine('cmpp_api_nodejs_heap_used_bytes', memory.heapUsed),
+      '# HELP cmpp_api_nodejs_heap_total_bytes Node.js allocated heap.',
+      '# TYPE cmpp_api_nodejs_heap_total_bytes gauge',
+      metricLine('cmpp_api_nodejs_heap_total_bytes', memory.heapTotal),
+      '# HELP cmpp_api_nodejs_event_loop_lag_p99_seconds Event loop delay p99 since the previous scrape.',
+      '# TYPE cmpp_api_nodejs_event_loop_lag_p99_seconds gauge',
+      metricLine('cmpp_api_nodejs_event_loop_lag_p99_seconds', this.eventLoopDelay.count ? this.eventLoopDelay.percentile(99) / 1_000_000_000 : 0),
+      '# HELP cmpp_api_http_requests_in_flight Current API requests in flight.',
+      '# TYPE cmpp_api_http_requests_in_flight gauge',
+      metricLine('cmpp_api_http_requests_in_flight', this.inFlight),
+      '# HELP cmpp_api_http_requests_total API requests grouped by bounded route templates.',
+      '# TYPE cmpp_api_http_requests_total counter',
+      '# HELP cmpp_api_http_request_duration_seconds API request duration.',
+      '# TYPE cmpp_api_http_request_duration_seconds histogram',
+    ];
+    for (const [key, metric] of this.http) {
+      const [method, route, status] = key.split('\u0000');
+      const labels = { method, route, status };
+      lines.push(metricLine('cmpp_api_http_requests_total', metric.count, labels));
+      HTTP_DURATION_BUCKETS.forEach((bucket, index) => {
+        lines.push(metricLine('cmpp_api_http_request_duration_seconds_bucket', metric.buckets[index], { ...labels, le: String(bucket) }));
+      });
+      lines.push(metricLine('cmpp_api_http_request_duration_seconds_bucket', metric.count, { ...labels, le: '+Inf' }));
+      lines.push(metricLine('cmpp_api_http_request_duration_seconds_sum', metric.durationSum, labels));
+      lines.push(metricLine('cmpp_api_http_request_duration_seconds_count', metric.count, labels));
+    }
+    this.eventLoopDelay.reset();
+    return `${lines.join('\n')}\n`;
+  }
+
+  onModuleDestroy() {
+    this.eventLoopDelay.disable();
+  }
+}
@@ -9,6 +9,7 @@ import { SecurityDetectionService } from './security-detection.service';
 export class SecurityDetectionController {
  constructor(private readonly security: SecurityDetectionService) {}
  @Get('overview') overview(@Query('range') range?: string) { return this.security.overview(range); }
+  @Get('notification-summary') notificationSummary() { return this.security.notificationSummary(); }
  @Get('alerts') alerts(@Query() query: Record<string, string>) { return this.security.listAlerts(query); }
  @Get('rules') rules() { return this.security.listRules(); }
  @Put('rules/:id') @RequireRecentAuthentication() updateRule(@Param('id') id: string, @Body() body: Record<string, unknown>, @CurrentSessionUserId() userId: string) { return this.security.updateRule(id, body, userId); }
@@ -11,7 +11,7 @@ function createPrisma() {
  const prisma = {
    securityDetectionRule: { findUnique: jest.fn(), findMany: jest.fn() },
    securityDetectionEvent: { count: jest.fn() },
-    securityAlert: { findUnique: jest.fn(), update: jest.fn() },
+    securityAlert: { findUnique: jest.fn(), update: jest.fn(), count: jest.fn() },
    securityBlock: { create: jest.fn(), update: jest.fn() },
    securityProtectedNetwork: { findMany: jest.fn().mockResolvedValue([]) },
    operationLog: { create: jest.fn() },
@@ -21,6 +21,16 @@ function createPrisma() {
 }

 describe('SecurityDetectionService', () => {
+  it('returns an independent active and critical alert summary for the global bell', async () => {
+    const { prisma } = createPrisma();
+    prisma.securityAlert.count.mockResolvedValueOnce(4).mockResolvedValueOnce(2);
+    const service = new SecurityDetectionService(prisma as never, {} as never);
+
+    await expect(service.notificationSummary()).resolves.toEqual({ count: 4, criticalCount: 2 });
+    expect(prisma.securityAlert.count).toHaveBeenNthCalledWith(1, { where: { status: { in: ['open', 'acknowledged', 'block_failed'] } } });
+    expect(prisma.securityAlert.count).toHaveBeenNthCalledWith(2, { where: { status: { in: ['open', 'acknowledged', 'block_failed'] }, severity: 'critical' } });
+  });
+
  it('keeps a below-threshold event without creating a false alert', async () => {
    const { prisma, tx } = createPrisma();
    prisma.securityDetectionRule.findUnique.mockResolvedValue({ id: 'rule-1', enabled: true, threshold: 3, windowSeconds: 60, cooldownSeconds: 60, severity: 'high' });
@@ -94,6 +94,15 @@ export class SecurityDetectionService {
    };
  }

+  async notificationSummary() {
+    const activeStatuses = ['open', 'acknowledged', 'block_failed'];
+    const [count, criticalCount] = await Promise.all([
+      this.prisma.securityAlert.count({ where: { status: { in: activeStatuses } } }),
+      this.prisma.securityAlert.count({ where: { status: { in: activeStatuses }, severity: 'critical' } }),
+    ]);
+    return { count, criticalCount };
+  }
+
  listAlerts(query: { status?: string; ruleCode?: string; sourceIp?: string; page?: string; pageSize?: string }) {
    const page = positiveInt(query.page, 1, 100000);
    const pageSize = positiveInt(query.pageSize, 20, 100);
@@ -54,6 +54,41 @@ describe('SendDownstreamRequeueTaskService', () => {
    expect(mock.downstreamRequeueTaskItem.createMany).toHaveBeenCalledWith({ data: [expect.objectContaining({ deliveryId: 'd-1', applicationId: 'app-1' })] });
  });

+  it('materializes client-confirmed deliveries from the signed preview range', async () => {
+    mock.cmppDownstreamDelivery.count.mockResolvedValue(1);
+    mock.cmppDownstreamDelivery.groupBy.mockResolvedValueOnce([{ status: 'delivered', _count: { _all: 1 } }]).mockResolvedValueOnce([{ applicationId: 'app-1', _count: { _all: 1 } }]);
+    mock.cmppDownstreamDelivery.findFirst.mockResolvedValue({ createdAt: new Date() });
+    const preview = await service.preview({ applicationId: 'app-1', status: 'delivered' }, 'user-1');
+    expect(preview).toEqual(expect.objectContaining({ matchedCount: 1, replayableCount: 1, skippedCount: 0 }));
+    mock.downstreamRequeueTask.findFirst.mockResolvedValue(null);
+    mock.cmppDownstreamDelivery.findMany.mockResolvedValue([{ id: 'd-1', applicationId: 'app-1', status: 'delivered' }]);
+    mock.downstreamRequeueTask.create.mockResolvedValue({ id: 'task-1' });
+    mock.downstreamRequeueTask.findUnique.mockResolvedValue({ id: 'task-1' });
+    mock.downstreamRequeueTaskItem.groupBy.mockResolvedValue([]);
+    await service.create({ previewToken: preview.previewToken, reason: '再次投递客户已确认记录' }, 'user-1');
+    expect(mock.downstreamRequeueTaskItem.createMany).toHaveBeenCalledWith({ data: [expect.objectContaining({ deliveryId: 'd-1', previousStatus: 'delivered' })] });
+  });
+
+  it('replays a delivery that was already client-confirmed in the task snapshot', async () => {
+    const requeue = jest.fn().mockResolvedValue({ status: 'awaiting_ack' });
+    service = new SendDownstreamRequeueTaskService(mock as never, { requeueDownstreamDelivery: requeue });
+    mock.cmppDownstreamDelivery.findUnique.mockResolvedValue({ id: 'd-1', applicationId: 'app-1', status: 'delivered', ackResult: 0, payload: {}, deliveryType: 'receipt', application: { status: 'active', interfaceEnabled: true } });
+    mock.downstreamRequeueTaskItem.findFirst.mockResolvedValue(null);
+    mock.cmppDownstreamConnection.count.mockResolvedValue(1);
+    await expect(service['processItem']('item-1', 'd-1', 'delivered')).resolves.toBe('waiting');
+    expect(requeue).toHaveBeenCalledWith('d-1');
+    expect(mock.downstreamRequeueTaskItem.update).toHaveBeenCalledWith(expect.objectContaining({ data: expect.objectContaining({ status: 'waiting_ack' }) }));
+  });
+
+  it('does not replay a record that became delivered after a non-delivered task snapshot', async () => {
+    const requeue = jest.fn();
+    service = new SendDownstreamRequeueTaskService(mock as never, { requeueDownstreamDelivery: requeue });
+    mock.cmppDownstreamDelivery.findUnique.mockResolvedValue({ id: 'd-1', applicationId: 'app-1', status: 'delivered', ackResult: 0, payload: {}, deliveryType: 'receipt', application: { status: 'active', interfaceEnabled: true } });
+    await expect(service['processItem']('item-1', 'd-1', 'failed')).resolves.toBe('skipped');
+    expect(requeue).not.toHaveBeenCalled();
+    expect(mock.downstreamRequeueTaskItem.update).toHaveBeenCalledWith(expect.objectContaining({ data: expect.objectContaining({ status: 'skipped', skipReason: '创建任务后已被客户确认' }) }));
+  });
+
  it('paginates all task items with status and keyword filters', async () => {
    mock.downstreamRequeueTask.findUnique.mockResolvedValue({ id: 'task-1' });
    mock.downstreamRequeueTaskItem.findMany.mockResolvedValue([{ id: 'item-1' }]);
@@ -15,7 +15,7 @@ export type DownstreamRequeueFilter = {
 };

 type RequeueFacade = { requeueDownstreamDelivery(id: string): Promise<unknown> };
-const REPLAYABLE_STATUSES = ['pending', 'failed', 'unconfirmed', 'rejected'];
+const REPLAYABLE_STATUSES = ['pending', 'failed', 'unconfirmed', 'rejected', 'delivered'];
 const ACTIVE_TASK_STATUSES = ['queued', 'running', 'paused'];
 const PROCESSING_LEASE_MS = 2 * 60_000;
 const SCAN_LEASE_MS = 15_000;
@@ -119,7 +119,7 @@ export class SendDownstreamRequeueTaskService {
    const preview = verifyPreview(data.previewToken, createdById);
    const filter = normalizedFilter(preview.filter);
    const snapshotAt = new Date(preview.snapshotAt);
-    if (filter.status === 'delivered' || filter.status === 'awaiting_ack') throw new BadRequestException('第一版后台任务不支持已确认或等待ACK记录');
+    if (filter.status === 'awaiting_ack') throw new BadRequestException('后台任务不支持正在等待ACK的记录');
    const activeTask = await this.prisma.downstreamRequeueTask.findFirst({ where: {
      status: { in: ACTIVE_TASK_STATUSES },
      ...(filter.applicationId !== 'all' ? { OR: [{ applicationId: filter.applicationId }, { applicationId: null }] } : {}),
@@ -229,14 +229,14 @@ export class SendDownstreamRequeueTaskService {
        return;
      }
      await this.prisma.downstreamRequeueTask.update({ where: { id: taskId }, data: { status: 'running', startedAt: task.startedAt ?? new Date() } });
-      const items = await this.prisma.downstreamRequeueTaskItem.findMany({ where: { taskId, status: 'queued' }, orderBy: { createdAt: 'asc' }, take: Math.min(200, task.ratePerSecond * 3), select: { id: true, deliveryId: true, applicationId: true } });
+      const items = await this.prisma.downstreamRequeueTaskItem.findMany({ where: { taskId, status: 'queued' }, orderBy: { createdAt: 'asc' }, take: Math.min(200, task.ratePerSecond * 3), select: { id: true, deliveryId: true, applicationId: true, previousStatus: true } });
      for (const item of items) {
        const latestTask = await this.prisma.downstreamRequeueTask.findUnique({ where: { id: taskId }, select: { status: true } });
        if (!latestTask || !['queued', 'running'].includes(latestTask.status)) break;
        if (!(await this.consumeRate(item.applicationId, task.ratePerSecond))) continue;
        const claimed = await this.prisma.downstreamRequeueTaskItem.updateMany({ where: { id: item.id, status: 'queued' }, data: { status: 'processing', claimedAt: new Date() } });
        if (!claimed.count) continue;
-        const outcome = await this.processItem(item.id, item.deliveryId);
+        const outcome = await this.processItem(item.id, item.deliveryId, item.previousStatus);
        if (outcome === 'success') failures[item.applicationId] = 0;
        if (outcome === 'failed') failures[item.applicationId] = (failures[item.applicationId] ?? 0) + 1;
        const maxFailures = Math.max(0, ...Object.values(failures));
@@ -257,16 +257,22 @@ export class SendDownstreamRequeueTaskService {
    }
  }

-  private async processItem(itemId: string, deliveryId: string): Promise<'success' | 'failed' | 'waiting' | 'skipped'> {
+  private async processItem(itemId: string, deliveryId: string, previousStatus: string): Promise<'success' | 'failed' | 'waiting' | 'skipped'> {
    try {
      const delivery = await this.prisma.cmppDownstreamDelivery.findUnique({
        where: { id: deliveryId },
        include: { application: { select: { status: true, interfaceEnabled: true } } },
      });
      if (!delivery) return this.finishItem(itemId, 'skipped', '投递记录已不存在');
+      // Only a record that was already delivered in the frozen task snapshot may be replayed as
+      // delivered. This preserves the operator's explicit duplicate-delivery intent while preventing
+      // a pending/failed record that receives a late ACK after task creation from being sent again.
+      if (delivery.status === 'delivered' && previousStatus !== 'delivered') {
+        return this.finishItem(itemId, 'skipped', '创建任务后已被客户确认');
+      }
      if (!REPLAYABLE_STATUSES.includes(delivery.status)) {
        if (delivery.status === 'awaiting_ack') { await this.prisma.downstreamRequeueTaskItem.update({ where: { id: itemId }, data: { status: 'waiting_external_ack', skipReason: null } }); return 'waiting'; }
-        return this.finishItem(itemId, 'skipped', delivery.status === 'delivered' ? '已被客户确认' : '执行前状态已变化');
+        return this.finishItem(itemId, 'skipped', '执行前状态已变化');
      }
      if (delivery.application.status !== 'active' || !delivery.application.interfaceEnabled) return this.finishItem(itemId, 'skipped', '应用或投递能力已停用');
      if (!delivery.payload || !['receipt', 'uplink'].includes(delivery.deliveryType)) return this.finishItem(itemId, 'skipped', '投递数据不完整');