feat: 完善服务监控与下游重投
This commit is contained in:
@@ -26,6 +26,7 @@ import { TenantsModule } from './tenants/tenants.module';
|
|||||||
import { UsersModule } from './users/users.module';
|
import { UsersModule } from './users/users.module';
|
||||||
import { SignatureRetirementModule } from './signature-retirement/signature-retirement.module';
|
import { SignatureRetirementModule } from './signature-retirement/signature-retirement.module';
|
||||||
import { SecurityDetectionModule } from './security-detection/security-detection.module';
|
import { SecurityDetectionModule } from './security-detection/security-detection.module';
|
||||||
|
import { MetricsModule } from './metrics/metrics.module';
|
||||||
|
|
||||||
@Module({
|
@Module({
|
||||||
imports: [
|
imports: [
|
||||||
@@ -55,6 +56,7 @@ import { SecurityDetectionModule } from './security-detection/security-detection
|
|||||||
OpenApiModule,
|
OpenApiModule,
|
||||||
SignatureRetirementModule,
|
SignatureRetirementModule,
|
||||||
SecurityDetectionModule,
|
SecurityDetectionModule,
|
||||||
|
MetricsModule,
|
||||||
],
|
],
|
||||||
controllers: [HealthController],
|
controllers: [HealthController],
|
||||||
providers: [RequestContextMiddleware, SessionValidationMiddleware, ManualOperationAuditMiddleware],
|
providers: [RequestContextMiddleware, SessionValidationMiddleware, ManualOperationAuditMiddleware],
|
||||||
|
|||||||
@@ -12,6 +12,18 @@ export type InfrastructureServiceStatus = {
|
|||||||
status: 'healthy' | 'unhealthy' | 'unknown';
|
status: 'healthy' | 'unhealthy' | 'unknown';
|
||||||
};
|
};
|
||||||
|
|
||||||
|
export type InfrastructureServiceMetricGroup = {
|
||||||
|
key: string;
|
||||||
|
name: string;
|
||||||
|
available: boolean;
|
||||||
|
metrics: Array<{
|
||||||
|
key: string;
|
||||||
|
label: string;
|
||||||
|
value: number | null;
|
||||||
|
unit: 'percent' | 'seconds' | 'count' | 'per_second' | 'bytes';
|
||||||
|
}>;
|
||||||
|
};
|
||||||
|
|
||||||
export type InfrastructureAlert = {
|
export type InfrastructureAlert = {
|
||||||
fingerprint: string;
|
fingerprint: string;
|
||||||
name: string;
|
name: string;
|
||||||
@@ -61,5 +73,6 @@ export type InfrastructureMonitoringOverview = {
|
|||||||
networkTransmitBytesPerSecond: InfrastructureMetricPoint[];
|
networkTransmitBytesPerSecond: InfrastructureMetricPoint[];
|
||||||
};
|
};
|
||||||
services: InfrastructureServiceStatus[];
|
services: InfrastructureServiceStatus[];
|
||||||
|
serviceMetrics: InfrastructureServiceMetricGroup[];
|
||||||
alerts: InfrastructureAlert[];
|
alerts: InfrastructureAlert[];
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -55,6 +55,13 @@ describe('InfrastructureMonitoringService', () => {
|
|||||||
{ metric: { name: 'nginx.service' }, value: [1_765_000_060, '1'] },
|
{ metric: { name: 'nginx.service' }, value: [1_765_000_060, '1'] },
|
||||||
] });
|
] });
|
||||||
}
|
}
|
||||||
|
if (query.includes('cmpp:service_.*')) {
|
||||||
|
return success({ result: [
|
||||||
|
{ metric: { __name__: 'cmpp:service_api:requests_per_second' }, value: [1_765_000_060, '12.5'] },
|
||||||
|
{ metric: { __name__: 'cmpp:service_api:error_percent' }, value: [1_765_000_060, '0.2'] },
|
||||||
|
{ metric: { __name__: 'cmpp:service_gateway:queue_pending' }, value: [1_765_000_060, '3'] },
|
||||||
|
] });
|
||||||
|
}
|
||||||
if (query.includes('timestamp(node_uname_info)')) {
|
if (query.includes('timestamp(node_uname_info)')) {
|
||||||
return success({ result: [{ metric: {}, value: [1_765_000_060, '1765000060'] }] });
|
return success({ result: [{ metric: {}, value: [1_765_000_060, '1765000060'] }] });
|
||||||
}
|
}
|
||||||
@@ -69,9 +76,13 @@ describe('InfrastructureMonitoringService', () => {
|
|||||||
expect(result.trends.cpuUsagePercent).toHaveLength(2);
|
expect(result.trends.cpuUsagePercent).toHaveLength(2);
|
||||||
expect(result.summary).toMatchObject({ overallStatus: 'warning', serviceHealthy: 6, warningAlerts: 1 });
|
expect(result.summary).toMatchObject({ overallStatus: 'warning', serviceHealthy: 6, warningAlerts: 1 });
|
||||||
expect(result.services.find((item) => item.key === 'redis')).toMatchObject({ unit: 'redis-server.service', status: 'healthy' });
|
expect(result.services.find((item) => item.key === 'redis')).toMatchObject({ unit: 'redis-server.service', status: 'healthy' });
|
||||||
|
expect(result.serviceMetrics.find((item) => item.key === 'api')).toMatchObject({ available: true });
|
||||||
|
expect(result.serviceMetrics.find((item) => item.key === 'gateway')?.metrics.find((item) => item.key === 'queuePending')?.value).toBe(3);
|
||||||
expect(result.alerts[0]).toMatchObject({ name: 'HostCpuHigh', severity: 'warning', currentValue: '88.2' });
|
expect(result.alerts[0]).toMatchObject({ name: 'HostCpuHigh', severity: 'warning', currentValue: '88.2' });
|
||||||
expect(requestedUrls.filter((url) => url.pathname.endsWith('/query_range'))).toHaveLength(5);
|
expect(requestedUrls.filter((url) => url.pathname.endsWith('/query_range'))).toHaveLength(5);
|
||||||
expect(requestedUrls.filter((url) => url.pathname.endsWith('/query_range')).every((url) => url.searchParams.get('step') === '60')).toBe(true);
|
expect(requestedUrls.filter((url) => url.pathname.endsWith('/query_range')).every((url) => url.searchParams.get('step') === '60')).toBe(true);
|
||||||
|
expect(requestedUrls.find((url) => url.searchParams.get('query')?.includes('node_systemd_unit_state'))?.searchParams.get('query'))
|
||||||
|
.toContain('cmpp-api\\\\.service');
|
||||||
});
|
});
|
||||||
|
|
||||||
it('returns an explicit unavailable payload without stale metrics when Prometheus fails', async () => {
|
it('returns an explicit unavailable payload without stale metrics when Prometheus fails', async () => {
|
||||||
@@ -84,6 +95,7 @@ describe('InfrastructureMonitoringService', () => {
|
|||||||
expect(result.summary.overallStatus).toBe('unknown');
|
expect(result.summary.overallStatus).toBe('unknown');
|
||||||
expect(result.metrics.cpuUsagePercent).toBeNull();
|
expect(result.metrics.cpuUsagePercent).toBeNull();
|
||||||
expect(result.trends.cpuUsagePercent).toEqual([]);
|
expect(result.trends.cpuUsagePercent).toEqual([]);
|
||||||
|
expect(result.serviceMetrics.every((item) => item.available === false)).toBe(true);
|
||||||
expect(result.error).not.toContain('ECONNREFUSED');
|
expect(result.error).not.toContain('ECONNREFUSED');
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
import { BadRequestException, Injectable } from '@nestjs/common';
|
import { BadRequestException, Injectable, Logger } from '@nestjs/common';
|
||||||
import { ConfigService } from '@nestjs/config';
|
import { ConfigService } from '@nestjs/config';
|
||||||
import { createHash } from 'node:crypto';
|
import { createHash } from 'node:crypto';
|
||||||
import type {
|
import type {
|
||||||
@@ -7,6 +7,7 @@ import type {
|
|||||||
InfrastructureMonitoringOverview,
|
InfrastructureMonitoringOverview,
|
||||||
InfrastructureMonitoringRange,
|
InfrastructureMonitoringRange,
|
||||||
InfrastructureServiceStatus,
|
InfrastructureServiceStatus,
|
||||||
|
InfrastructureServiceMetricGroup,
|
||||||
} from './infrastructure-monitoring.contracts';
|
} from './infrastructure-monitoring.contracts';
|
||||||
|
|
||||||
type PrometheusSample = [number, string];
|
type PrometheusSample = [number, string];
|
||||||
@@ -52,7 +53,8 @@ const QUERIES = {
|
|||||||
load1: 'node_load1',
|
load1: 'node_load1',
|
||||||
uptimeSeconds: 'time() - node_boot_time_seconds',
|
uptimeSeconds: 'time() - node_boot_time_seconds',
|
||||||
lastSampleAt: 'max(timestamp(node_uname_info))',
|
lastSampleAt: 'max(timestamp(node_uname_info))',
|
||||||
services: 'max by (name) (node_systemd_unit_state{name=~"cmpp-api\\.service|cmpp-gateway\\.service|postgresql\\.service|redis(-server)?\\.service|cmpp-minio\\.service|nginx\\.service",state="active"})',
|
// PromQL字符串本身需要两个反斜杠才能把正则的“\.”传给RE2;TypeScript字面量因此需要写四个。
|
||||||
|
services: 'max by (name) (node_systemd_unit_state{name=~"cmpp-api\\\\.service|cmpp-gateway\\\\.service|postgresql\\\\.service|redis(-server)?\\\\.service|cmpp-minio\\\\.service|nginx\\\\.service",state="active"})',
|
||||||
} as const;
|
} as const;
|
||||||
|
|
||||||
const SERVICE_DEFINITIONS = [
|
const SERVICE_DEFINITIONS = [
|
||||||
@@ -64,6 +66,43 @@ const SERVICE_DEFINITIONS = [
|
|||||||
{ key: 'nginx', name: 'Nginx', units: ['nginx.service'] },
|
{ key: 'nginx', name: 'Nginx', units: ['nginx.service'] },
|
||||||
] as const;
|
] as const;
|
||||||
|
|
||||||
|
const SERVICE_METRIC_DEFINITIONS = [
|
||||||
|
{ key: 'api', name: 'API服务', metrics: [
|
||||||
|
['requestsPerSecond', '请求速率', 'cmpp:service_api:requests_per_second', 'per_second'],
|
||||||
|
['errorPercent', '5xx错误率', 'cmpp:service_api:error_percent', 'percent'],
|
||||||
|
['latencyP95', 'P95响应', 'cmpp:service_api:latency_p95_seconds', 'seconds'],
|
||||||
|
['eventLoopP99', '事件循环P99', 'cmpp:service_api:event_loop_p99_seconds', 'seconds'],
|
||||||
|
] },
|
||||||
|
{ key: 'gateway', name: 'Gateway服务', metrics: [
|
||||||
|
['submitsPerSecond', '提交速率', 'cmpp:service_gateway:submits_per_second', 'per_second'],
|
||||||
|
['failurePercent', '提交失败率', 'cmpp:service_gateway:failure_percent', 'percent'],
|
||||||
|
['queuePending', 'Stream pending', 'cmpp:service_gateway:queue_pending', 'count'],
|
||||||
|
['queueOldestSeconds', '最旧pending', 'cmpp:service_gateway:queue_oldest_seconds', 'seconds'],
|
||||||
|
] },
|
||||||
|
{ key: 'postgresql', name: 'PostgreSQL', metrics: [
|
||||||
|
['connectionPercent', '连接使用率', 'cmpp:service_postgresql:connection_percent', 'percent'],
|
||||||
|
['deadlocks15m', '15分钟死锁', 'cmpp:service_postgresql:deadlocks_15m', 'count'],
|
||||||
|
] },
|
||||||
|
{ key: 'redis', name: 'Redis', metrics: [
|
||||||
|
['memoryPercent', '内存使用率', 'cmpp:service_redis:memory_percent', 'percent'],
|
||||||
|
['memoryUsedBytes', '已用内存', 'cmpp:service_redis:memory_used_bytes', 'bytes'],
|
||||||
|
['connectedClients', '客户端连接', 'cmpp:service_redis:connected_clients', 'count'],
|
||||||
|
['evictions5m', '5分钟淘汰', 'cmpp:service_redis:evictions_5m', 'count'],
|
||||||
|
] },
|
||||||
|
{ key: 'minio', name: 'MinIO', metrics: [
|
||||||
|
['capacityPercent', '存储容量使用率', 'cmpp:service_minio:capacity_percent', 'percent'],
|
||||||
|
['usageBytes', '对象数据量', 'cmpp:service_minio:usage_bytes', 'bytes'],
|
||||||
|
['objects', '对象数', 'cmpp:service_minio:objects', 'count'],
|
||||||
|
['drivesOffline', '离线存储盘', 'cmpp:service_minio:drives_offline', 'count'],
|
||||||
|
] },
|
||||||
|
{ key: 'nginx', name: 'Nginx', metrics: [
|
||||||
|
['connectionsActive', '活跃连接', 'cmpp:service_nginx:connections_active', 'count'],
|
||||||
|
['requestsPerSecond', '请求速率', 'cmpp:service_nginx:requests_per_second', 'per_second'],
|
||||||
|
] },
|
||||||
|
] as const;
|
||||||
|
|
||||||
|
const SERVICE_METRICS_QUERY = '{__name__=~"cmpp:service_.*"}';
|
||||||
|
|
||||||
function finiteNumber(value: string | number | undefined): number | null {
|
function finiteNumber(value: string | number | undefined): number | null {
|
||||||
const parsed = Number(value);
|
const parsed = Number(value);
|
||||||
return Number.isFinite(parsed) ? parsed : null;
|
return Number.isFinite(parsed) ? parsed : null;
|
||||||
@@ -119,6 +158,7 @@ function emptyTrends(): InfrastructureMonitoringOverview['trends'] {
|
|||||||
|
|
||||||
@Injectable()
|
@Injectable()
|
||||||
export class InfrastructureMonitoringService {
|
export class InfrastructureMonitoringService {
|
||||||
|
private readonly logger = new Logger(InfrastructureMonitoringService.name);
|
||||||
private readonly prometheusUrl: string;
|
private readonly prometheusUrl: string;
|
||||||
private readonly queryTimeoutMs: number;
|
private readonly queryTimeoutMs: number;
|
||||||
|
|
||||||
@@ -131,13 +171,15 @@ export class InfrastructureMonitoringService {
|
|||||||
const range = this.parseRange(rawRange);
|
const range = this.parseRange(rawRange);
|
||||||
const collectedAt = new Date().toISOString();
|
const collectedAt = new Date().toISOString();
|
||||||
try {
|
try {
|
||||||
const [instant, trends, serviceResponse, alertResponse] = await Promise.all([
|
const [instant, trends, serviceResponse, serviceMetricResponse, alertResponse] = await Promise.all([
|
||||||
this.loadInstantMetrics(),
|
this.loadInstantMetrics(),
|
||||||
this.loadTrends(range),
|
this.loadTrends(range),
|
||||||
this.query(QUERIES.services),
|
this.query(QUERIES.services),
|
||||||
|
this.query(SERVICE_METRICS_QUERY),
|
||||||
this.getJson<PrometheusAlertResponse>('/api/v1/alerts'),
|
this.getJson<PrometheusAlertResponse>('/api/v1/alerts'),
|
||||||
]);
|
]);
|
||||||
const services = this.parseServices(serviceResponse);
|
const services = this.parseServices(serviceResponse);
|
||||||
|
const serviceMetrics = this.parseServiceMetrics(serviceMetricResponse);
|
||||||
const alerts = this.parseAlerts(alertResponse);
|
const alerts = this.parseAlerts(alertResponse);
|
||||||
const warningAlerts = alerts.filter((item) => item.severity === 'warning').length;
|
const warningAlerts = alerts.filter((item) => item.severity === 'warning').length;
|
||||||
const criticalAlerts = alerts.filter((item) => item.severity === 'critical').length;
|
const criticalAlerts = alerts.filter((item) => item.severity === 'critical').length;
|
||||||
@@ -158,9 +200,12 @@ export class InfrastructureMonitoringService {
|
|||||||
metrics: instant.metrics,
|
metrics: instant.metrics,
|
||||||
trends,
|
trends,
|
||||||
services,
|
services,
|
||||||
|
serviceMetrics,
|
||||||
alerts,
|
alerts,
|
||||||
};
|
};
|
||||||
} catch {
|
} catch (error) {
|
||||||
|
// 页面必须整体清空陈旧指标,但服务端仍需留下不含PromQL/地址/凭据的根因摘要便于运维诊断。
|
||||||
|
this.logger.warn(`Prometheus monitoring overview unavailable: ${error instanceof Error ? error.message : 'unknown error'}`);
|
||||||
return this.unavailable(range, collectedAt);
|
return this.unavailable(range, collectedAt);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -228,6 +273,21 @@ export class InfrastructureMonitoringService {
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private parseServiceMetrics(response: PrometheusQueryResponse): InfrastructureServiceMetricGroup[] {
|
||||||
|
const values = new Map<string, number>();
|
||||||
|
for (const item of response.data?.result ?? []) {
|
||||||
|
const metricName = item.metric.__name__;
|
||||||
|
const value = vectorValue({ status: 'success', data: { result: [item] } });
|
||||||
|
if (metricName && value !== null) values.set(metricName, value);
|
||||||
|
}
|
||||||
|
return SERVICE_METRIC_DEFINITIONS.map((group) => ({
|
||||||
|
key: group.key,
|
||||||
|
name: group.name,
|
||||||
|
available: group.metrics.some((metric) => values.has(metric[2])),
|
||||||
|
metrics: group.metrics.map(([key, label, metricName, unit]) => ({ key, label, value: values.get(metricName) ?? null, unit })),
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
|
||||||
private unavailable(range: InfrastructureMonitoringRange, collectedAt: string): InfrastructureMonitoringOverview {
|
private unavailable(range: InfrastructureMonitoringRange, collectedAt: string): InfrastructureMonitoringOverview {
|
||||||
const services = SERVICE_DEFINITIONS.map((item) => ({ key: item.key, name: item.name, unit: item.units[0], status: 'unknown' as const }));
|
const services = SERVICE_DEFINITIONS.map((item) => ({ key: item.key, name: item.name, unit: item.units[0], status: 'unknown' as const }));
|
||||||
return {
|
return {
|
||||||
@@ -240,6 +300,7 @@ export class InfrastructureMonitoringService {
|
|||||||
metrics: emptyMetrics(),
|
metrics: emptyMetrics(),
|
||||||
trends: emptyTrends(),
|
trends: emptyTrends(),
|
||||||
services,
|
services,
|
||||||
|
serviceMetrics: SERVICE_METRIC_DEFINITIONS.map((group) => ({ key: group.key, name: group.name, available: false, metrics: [] })),
|
||||||
alerts: [],
|
alerts: [],
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|||||||
+22
-1
@@ -1,8 +1,10 @@
|
|||||||
import 'reflect-metadata';
|
import 'reflect-metadata';
|
||||||
import { NestFactory } from '@nestjs/core';
|
import { NestFactory } from '@nestjs/core';
|
||||||
|
import { createServer } from 'node:http';
|
||||||
import type { NestExpressApplication } from '@nestjs/platform-express';
|
import type { NestExpressApplication } from '@nestjs/platform-express';
|
||||||
import { DocumentBuilder, SwaggerModule } from '@nestjs/swagger';
|
import { DocumentBuilder, SwaggerModule } from '@nestjs/swagger';
|
||||||
import { AppModule } from './app.module';
|
import { AppModule } from './app.module';
|
||||||
|
import { MetricsService } from './metrics/metrics.service';
|
||||||
import { OpenApiModule } from './open-api/open-api.module';
|
import { OpenApiModule } from './open-api/open-api.module';
|
||||||
import { configureHttpBodyParsers } from './http-body-limits';
|
import { configureHttpBodyParsers } from './http-body-limits';
|
||||||
|
|
||||||
@@ -39,7 +41,26 @@ async function bootstrap() {
|
|||||||
SwaggerModule.setup('api/client-docs', app, clientDocument);
|
SwaggerModule.setup('api/client-docs', app, clientDocument);
|
||||||
|
|
||||||
const port = Number(process.env.API_PORT ?? 3000);
|
const port = Number(process.env.API_PORT ?? 3000);
|
||||||
await app.listen(port);
|
// 生产环境只允许 Nginx 访问管理 API;显式绑定回环,避免默认的全网卡监听绕过入口鉴权与限流。
|
||||||
|
const host = process.env.API_HOST?.trim() || '127.0.0.1';
|
||||||
|
await app.listen(port, host);
|
||||||
|
|
||||||
|
const metrics = app.get(MetricsService);
|
||||||
|
const metricsHost = process.env.API_METRICS_HOST?.trim() || '127.0.0.1';
|
||||||
|
const metricsPort = Number(process.env.API_METRICS_PORT ?? 9464);
|
||||||
|
const metricsServer = createServer((request, response) => {
|
||||||
|
if (request.method !== 'GET' || request.url !== '/metrics') {
|
||||||
|
response.writeHead(404).end();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
response.writeHead(200, { 'Content-Type': 'text/plain; version=0.0.4; charset=utf-8', 'Cache-Control': 'no-store' });
|
||||||
|
response.end(metrics.render());
|
||||||
|
});
|
||||||
|
// Metrics use a dedicated loopback listener so Nginx cannot accidentally expose them through /api/.
|
||||||
|
await new Promise<void>((resolve, reject) => {
|
||||||
|
metricsServer.once('error', reject);
|
||||||
|
metricsServer.listen(metricsPort, metricsHost, resolve);
|
||||||
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
void bootstrap();
|
void bootstrap();
|
||||||
|
|||||||
@@ -0,0 +1,24 @@
|
|||||||
|
import { CallHandler, ExecutionContext, Injectable, NestInterceptor } from '@nestjs/common';
|
||||||
|
import type { Observable } from 'rxjs';
|
||||||
|
import { finalize } from 'rxjs/operators';
|
||||||
|
import { MetricsService } from './metrics.service';
|
||||||
|
|
||||||
|
type RequestLike = { method?: string; baseUrl?: string; route?: { path?: string } };
|
||||||
|
type ResponseLike = { statusCode?: number };
|
||||||
|
|
||||||
|
@Injectable()
|
||||||
|
export class MetricsInterceptor implements NestInterceptor {
|
||||||
|
constructor(private readonly metrics: MetricsService) {}
|
||||||
|
|
||||||
|
intercept(context: ExecutionContext, next: CallHandler): Observable<unknown> {
|
||||||
|
if (context.getType() !== 'http') return next.handle();
|
||||||
|
const http = context.switchToHttp();
|
||||||
|
const request = http.getRequest<RequestLike>();
|
||||||
|
const response = http.getResponse<ResponseLike>();
|
||||||
|
const startedAt = this.metrics.beginRequest();
|
||||||
|
return next.handle().pipe(finalize(() => {
|
||||||
|
const route = `${request.baseUrl ?? ''}${request.route?.path ?? '/unmatched'}`;
|
||||||
|
this.metrics.finishRequest(startedAt, request.method ?? 'UNKNOWN', route, response.statusCode ?? 500);
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,11 @@
|
|||||||
|
import { Global, Module } from '@nestjs/common';
|
||||||
|
import { APP_INTERCEPTOR } from '@nestjs/core';
|
||||||
|
import { MetricsInterceptor } from './metrics.interceptor';
|
||||||
|
import { MetricsService } from './metrics.service';
|
||||||
|
|
||||||
|
@Global()
|
||||||
|
@Module({
|
||||||
|
providers: [MetricsService, { provide: APP_INTERCEPTOR, useClass: MetricsInterceptor }],
|
||||||
|
exports: [MetricsService],
|
||||||
|
})
|
||||||
|
export class MetricsModule {}
|
||||||
@@ -0,0 +1,16 @@
|
|||||||
|
import { MetricsService } from './metrics.service';
|
||||||
|
|
||||||
|
describe('MetricsService', () => {
|
||||||
|
it('exports bounded API process and HTTP metrics without raw identifiers', () => {
|
||||||
|
const service = new MetricsService();
|
||||||
|
const startedAt = service.beginRequest();
|
||||||
|
service.finishRequest(startedAt, 'GET', '/api/admin/tenants/:id', 200);
|
||||||
|
const output = service.render();
|
||||||
|
|
||||||
|
expect(output).toContain('cmpp_api_process_resident_memory_bytes');
|
||||||
|
expect(output).toContain('cmpp_api_http_requests_total{method="GET",route="/api/admin/tenants/:id",status="200"} 1');
|
||||||
|
expect(output).toContain('cmpp_api_http_request_duration_seconds_bucket');
|
||||||
|
expect(output).not.toContain('phone_number');
|
||||||
|
service.onModuleDestroy();
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -0,0 +1,100 @@
|
|||||||
|
import { Injectable, OnModuleDestroy } from '@nestjs/common';
|
||||||
|
import { monitorEventLoopDelay } from 'node:perf_hooks';
|
||||||
|
|
||||||
|
const HTTP_DURATION_BUCKETS = [0.05, 0.1, 0.25, 0.5, 1, 3, 10] as const;
|
||||||
|
|
||||||
|
type HttpMetric = {
|
||||||
|
count: number;
|
||||||
|
durationSum: number;
|
||||||
|
buckets: number[];
|
||||||
|
};
|
||||||
|
|
||||||
|
function escapeLabel(value: string) {
|
||||||
|
return value.replace(/\\/g, '\\\\').replace(/\n/g, '\\n').replace(/"/g, '\\"');
|
||||||
|
}
|
||||||
|
|
||||||
|
function metricLine(name: string, value: number, labels?: Record<string, string>) {
|
||||||
|
const suffix = labels
|
||||||
|
? `{${Object.entries(labels).map(([key, item]) => `${key}="${escapeLabel(item)}"`).join(',')}}`
|
||||||
|
: '';
|
||||||
|
return `${name}${suffix} ${Number.isFinite(value) ? value : 0}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Injectable()
|
||||||
|
export class MetricsService implements OnModuleDestroy {
|
||||||
|
private readonly startedAt = process.hrtime.bigint();
|
||||||
|
private readonly eventLoopDelay = monitorEventLoopDelay({ resolution: 20 });
|
||||||
|
private readonly http = new Map<string, HttpMetric>();
|
||||||
|
private inFlight = 0;
|
||||||
|
|
||||||
|
constructor() {
|
||||||
|
this.eventLoopDelay.enable();
|
||||||
|
}
|
||||||
|
|
||||||
|
beginRequest() {
|
||||||
|
this.inFlight += 1;
|
||||||
|
return process.hrtime.bigint();
|
||||||
|
}
|
||||||
|
|
||||||
|
finishRequest(startedAt: bigint, method: string, route: string, statusCode: number) {
|
||||||
|
this.inFlight = Math.max(0, this.inFlight - 1);
|
||||||
|
// Only route templates enter labels. Raw URLs, IDs, phone numbers and query strings would create unbounded time series.
|
||||||
|
const normalizedRoute = route.startsWith('/') ? route : `/${route}`;
|
||||||
|
const labels = [method.toUpperCase(), normalizedRoute, String(statusCode)];
|
||||||
|
const key = labels.join('\u0000');
|
||||||
|
const metric = this.http.get(key) ?? { count: 0, durationSum: 0, buckets: HTTP_DURATION_BUCKETS.map(() => 0) };
|
||||||
|
const durationSeconds = Number(process.hrtime.bigint() - startedAt) / 1_000_000_000;
|
||||||
|
metric.count += 1;
|
||||||
|
metric.durationSum += durationSeconds;
|
||||||
|
HTTP_DURATION_BUCKETS.forEach((bucket, index) => {
|
||||||
|
if (durationSeconds <= bucket) metric.buckets[index] += 1;
|
||||||
|
});
|
||||||
|
this.http.set(key, metric);
|
||||||
|
}
|
||||||
|
|
||||||
|
render() {
|
||||||
|
const memory = process.memoryUsage();
|
||||||
|
const uptime = Number(process.hrtime.bigint() - this.startedAt) / 1_000_000_000;
|
||||||
|
const lines = [
|
||||||
|
'# HELP cmpp_api_process_uptime_seconds API process uptime.',
|
||||||
|
'# TYPE cmpp_api_process_uptime_seconds gauge',
|
||||||
|
metricLine('cmpp_api_process_uptime_seconds', uptime),
|
||||||
|
'# HELP cmpp_api_process_resident_memory_bytes API resident memory.',
|
||||||
|
'# TYPE cmpp_api_process_resident_memory_bytes gauge',
|
||||||
|
metricLine('cmpp_api_process_resident_memory_bytes', memory.rss),
|
||||||
|
'# HELP cmpp_api_nodejs_heap_used_bytes Node.js heap currently used.',
|
||||||
|
'# TYPE cmpp_api_nodejs_heap_used_bytes gauge',
|
||||||
|
metricLine('cmpp_api_nodejs_heap_used_bytes', memory.heapUsed),
|
||||||
|
'# HELP cmpp_api_nodejs_heap_total_bytes Node.js allocated heap.',
|
||||||
|
'# TYPE cmpp_api_nodejs_heap_total_bytes gauge',
|
||||||
|
metricLine('cmpp_api_nodejs_heap_total_bytes', memory.heapTotal),
|
||||||
|
'# HELP cmpp_api_nodejs_event_loop_lag_p99_seconds Event loop delay p99 since the previous scrape.',
|
||||||
|
'# TYPE cmpp_api_nodejs_event_loop_lag_p99_seconds gauge',
|
||||||
|
metricLine('cmpp_api_nodejs_event_loop_lag_p99_seconds', this.eventLoopDelay.count ? this.eventLoopDelay.percentile(99) / 1_000_000_000 : 0),
|
||||||
|
'# HELP cmpp_api_http_requests_in_flight Current API requests in flight.',
|
||||||
|
'# TYPE cmpp_api_http_requests_in_flight gauge',
|
||||||
|
metricLine('cmpp_api_http_requests_in_flight', this.inFlight),
|
||||||
|
'# HELP cmpp_api_http_requests_total API requests grouped by bounded route templates.',
|
||||||
|
'# TYPE cmpp_api_http_requests_total counter',
|
||||||
|
'# HELP cmpp_api_http_request_duration_seconds API request duration.',
|
||||||
|
'# TYPE cmpp_api_http_request_duration_seconds histogram',
|
||||||
|
];
|
||||||
|
for (const [key, metric] of this.http) {
|
||||||
|
const [method, route, status] = key.split('\u0000');
|
||||||
|
const labels = { method, route, status };
|
||||||
|
lines.push(metricLine('cmpp_api_http_requests_total', metric.count, labels));
|
||||||
|
HTTP_DURATION_BUCKETS.forEach((bucket, index) => {
|
||||||
|
lines.push(metricLine('cmpp_api_http_request_duration_seconds_bucket', metric.buckets[index], { ...labels, le: String(bucket) }));
|
||||||
|
});
|
||||||
|
lines.push(metricLine('cmpp_api_http_request_duration_seconds_bucket', metric.count, { ...labels, le: '+Inf' }));
|
||||||
|
lines.push(metricLine('cmpp_api_http_request_duration_seconds_sum', metric.durationSum, labels));
|
||||||
|
lines.push(metricLine('cmpp_api_http_request_duration_seconds_count', metric.count, labels));
|
||||||
|
}
|
||||||
|
this.eventLoopDelay.reset();
|
||||||
|
return `${lines.join('\n')}\n`;
|
||||||
|
}
|
||||||
|
|
||||||
|
onModuleDestroy() {
|
||||||
|
this.eventLoopDelay.disable();
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -9,6 +9,7 @@ import { SecurityDetectionService } from './security-detection.service';
|
|||||||
export class SecurityDetectionController {
|
export class SecurityDetectionController {
|
||||||
constructor(private readonly security: SecurityDetectionService) {}
|
constructor(private readonly security: SecurityDetectionService) {}
|
||||||
@Get('overview') overview(@Query('range') range?: string) { return this.security.overview(range); }
|
@Get('overview') overview(@Query('range') range?: string) { return this.security.overview(range); }
|
||||||
|
@Get('notification-summary') notificationSummary() { return this.security.notificationSummary(); }
|
||||||
@Get('alerts') alerts(@Query() query: Record<string, string>) { return this.security.listAlerts(query); }
|
@Get('alerts') alerts(@Query() query: Record<string, string>) { return this.security.listAlerts(query); }
|
||||||
@Get('rules') rules() { return this.security.listRules(); }
|
@Get('rules') rules() { return this.security.listRules(); }
|
||||||
@Put('rules/:id') @RequireRecentAuthentication() updateRule(@Param('id') id: string, @Body() body: Record<string, unknown>, @CurrentSessionUserId() userId: string) { return this.security.updateRule(id, body, userId); }
|
@Put('rules/:id') @RequireRecentAuthentication() updateRule(@Param('id') id: string, @Body() body: Record<string, unknown>, @CurrentSessionUserId() userId: string) { return this.security.updateRule(id, body, userId); }
|
||||||
|
|||||||
@@ -11,7 +11,7 @@ function createPrisma() {
|
|||||||
const prisma = {
|
const prisma = {
|
||||||
securityDetectionRule: { findUnique: jest.fn(), findMany: jest.fn() },
|
securityDetectionRule: { findUnique: jest.fn(), findMany: jest.fn() },
|
||||||
securityDetectionEvent: { count: jest.fn() },
|
securityDetectionEvent: { count: jest.fn() },
|
||||||
securityAlert: { findUnique: jest.fn(), update: jest.fn() },
|
securityAlert: { findUnique: jest.fn(), update: jest.fn(), count: jest.fn() },
|
||||||
securityBlock: { create: jest.fn(), update: jest.fn() },
|
securityBlock: { create: jest.fn(), update: jest.fn() },
|
||||||
securityProtectedNetwork: { findMany: jest.fn().mockResolvedValue([]) },
|
securityProtectedNetwork: { findMany: jest.fn().mockResolvedValue([]) },
|
||||||
operationLog: { create: jest.fn() },
|
operationLog: { create: jest.fn() },
|
||||||
@@ -21,6 +21,16 @@ function createPrisma() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
describe('SecurityDetectionService', () => {
|
describe('SecurityDetectionService', () => {
|
||||||
|
it('returns an independent active and critical alert summary for the global bell', async () => {
|
||||||
|
const { prisma } = createPrisma();
|
||||||
|
prisma.securityAlert.count.mockResolvedValueOnce(4).mockResolvedValueOnce(2);
|
||||||
|
const service = new SecurityDetectionService(prisma as never, {} as never);
|
||||||
|
|
||||||
|
await expect(service.notificationSummary()).resolves.toEqual({ count: 4, criticalCount: 2 });
|
||||||
|
expect(prisma.securityAlert.count).toHaveBeenNthCalledWith(1, { where: { status: { in: ['open', 'acknowledged', 'block_failed'] } } });
|
||||||
|
expect(prisma.securityAlert.count).toHaveBeenNthCalledWith(2, { where: { status: { in: ['open', 'acknowledged', 'block_failed'] }, severity: 'critical' } });
|
||||||
|
});
|
||||||
|
|
||||||
it('keeps a below-threshold event without creating a false alert', async () => {
|
it('keeps a below-threshold event without creating a false alert', async () => {
|
||||||
const { prisma, tx } = createPrisma();
|
const { prisma, tx } = createPrisma();
|
||||||
prisma.securityDetectionRule.findUnique.mockResolvedValue({ id: 'rule-1', enabled: true, threshold: 3, windowSeconds: 60, cooldownSeconds: 60, severity: 'high' });
|
prisma.securityDetectionRule.findUnique.mockResolvedValue({ id: 'rule-1', enabled: true, threshold: 3, windowSeconds: 60, cooldownSeconds: 60, severity: 'high' });
|
||||||
|
|||||||
@@ -94,6 +94,15 @@ export class SecurityDetectionService {
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async notificationSummary() {
|
||||||
|
const activeStatuses = ['open', 'acknowledged', 'block_failed'];
|
||||||
|
const [count, criticalCount] = await Promise.all([
|
||||||
|
this.prisma.securityAlert.count({ where: { status: { in: activeStatuses } } }),
|
||||||
|
this.prisma.securityAlert.count({ where: { status: { in: activeStatuses }, severity: 'critical' } }),
|
||||||
|
]);
|
||||||
|
return { count, criticalCount };
|
||||||
|
}
|
||||||
|
|
||||||
listAlerts(query: { status?: string; ruleCode?: string; sourceIp?: string; page?: string; pageSize?: string }) {
|
listAlerts(query: { status?: string; ruleCode?: string; sourceIp?: string; page?: string; pageSize?: string }) {
|
||||||
const page = positiveInt(query.page, 1, 100000);
|
const page = positiveInt(query.page, 1, 100000);
|
||||||
const pageSize = positiveInt(query.pageSize, 20, 100);
|
const pageSize = positiveInt(query.pageSize, 20, 100);
|
||||||
|
|||||||
@@ -54,6 +54,41 @@ describe('SendDownstreamRequeueTaskService', () => {
|
|||||||
expect(mock.downstreamRequeueTaskItem.createMany).toHaveBeenCalledWith({ data: [expect.objectContaining({ deliveryId: 'd-1', applicationId: 'app-1' })] });
|
expect(mock.downstreamRequeueTaskItem.createMany).toHaveBeenCalledWith({ data: [expect.objectContaining({ deliveryId: 'd-1', applicationId: 'app-1' })] });
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('materializes client-confirmed deliveries from the signed preview range', async () => {
|
||||||
|
mock.cmppDownstreamDelivery.count.mockResolvedValue(1);
|
||||||
|
mock.cmppDownstreamDelivery.groupBy.mockResolvedValueOnce([{ status: 'delivered', _count: { _all: 1 } }]).mockResolvedValueOnce([{ applicationId: 'app-1', _count: { _all: 1 } }]);
|
||||||
|
mock.cmppDownstreamDelivery.findFirst.mockResolvedValue({ createdAt: new Date() });
|
||||||
|
const preview = await service.preview({ applicationId: 'app-1', status: 'delivered' }, 'user-1');
|
||||||
|
expect(preview).toEqual(expect.objectContaining({ matchedCount: 1, replayableCount: 1, skippedCount: 0 }));
|
||||||
|
mock.downstreamRequeueTask.findFirst.mockResolvedValue(null);
|
||||||
|
mock.cmppDownstreamDelivery.findMany.mockResolvedValue([{ id: 'd-1', applicationId: 'app-1', status: 'delivered' }]);
|
||||||
|
mock.downstreamRequeueTask.create.mockResolvedValue({ id: 'task-1' });
|
||||||
|
mock.downstreamRequeueTask.findUnique.mockResolvedValue({ id: 'task-1' });
|
||||||
|
mock.downstreamRequeueTaskItem.groupBy.mockResolvedValue([]);
|
||||||
|
await service.create({ previewToken: preview.previewToken, reason: '再次投递客户已确认记录' }, 'user-1');
|
||||||
|
expect(mock.downstreamRequeueTaskItem.createMany).toHaveBeenCalledWith({ data: [expect.objectContaining({ deliveryId: 'd-1', previousStatus: 'delivered' })] });
|
||||||
|
});
|
||||||
|
|
||||||
|
it('replays a delivery that was already client-confirmed in the task snapshot', async () => {
|
||||||
|
const requeue = jest.fn().mockResolvedValue({ status: 'awaiting_ack' });
|
||||||
|
service = new SendDownstreamRequeueTaskService(mock as never, { requeueDownstreamDelivery: requeue });
|
||||||
|
mock.cmppDownstreamDelivery.findUnique.mockResolvedValue({ id: 'd-1', applicationId: 'app-1', status: 'delivered', ackResult: 0, payload: {}, deliveryType: 'receipt', application: { status: 'active', interfaceEnabled: true } });
|
||||||
|
mock.downstreamRequeueTaskItem.findFirst.mockResolvedValue(null);
|
||||||
|
mock.cmppDownstreamConnection.count.mockResolvedValue(1);
|
||||||
|
await expect(service['processItem']('item-1', 'd-1', 'delivered')).resolves.toBe('waiting');
|
||||||
|
expect(requeue).toHaveBeenCalledWith('d-1');
|
||||||
|
expect(mock.downstreamRequeueTaskItem.update).toHaveBeenCalledWith(expect.objectContaining({ data: expect.objectContaining({ status: 'waiting_ack' }) }));
|
||||||
|
});
|
||||||
|
|
||||||
|
it('does not replay a record that became delivered after a non-delivered task snapshot', async () => {
|
||||||
|
const requeue = jest.fn();
|
||||||
|
service = new SendDownstreamRequeueTaskService(mock as never, { requeueDownstreamDelivery: requeue });
|
||||||
|
mock.cmppDownstreamDelivery.findUnique.mockResolvedValue({ id: 'd-1', applicationId: 'app-1', status: 'delivered', ackResult: 0, payload: {}, deliveryType: 'receipt', application: { status: 'active', interfaceEnabled: true } });
|
||||||
|
await expect(service['processItem']('item-1', 'd-1', 'failed')).resolves.toBe('skipped');
|
||||||
|
expect(requeue).not.toHaveBeenCalled();
|
||||||
|
expect(mock.downstreamRequeueTaskItem.update).toHaveBeenCalledWith(expect.objectContaining({ data: expect.objectContaining({ status: 'skipped', skipReason: '创建任务后已被客户确认' }) }));
|
||||||
|
});
|
||||||
|
|
||||||
it('paginates all task items with status and keyword filters', async () => {
|
it('paginates all task items with status and keyword filters', async () => {
|
||||||
mock.downstreamRequeueTask.findUnique.mockResolvedValue({ id: 'task-1' });
|
mock.downstreamRequeueTask.findUnique.mockResolvedValue({ id: 'task-1' });
|
||||||
mock.downstreamRequeueTaskItem.findMany.mockResolvedValue([{ id: 'item-1' }]);
|
mock.downstreamRequeueTaskItem.findMany.mockResolvedValue([{ id: 'item-1' }]);
|
||||||
|
|||||||
@@ -15,7 +15,7 @@ export type DownstreamRequeueFilter = {
|
|||||||
};
|
};
|
||||||
|
|
||||||
type RequeueFacade = { requeueDownstreamDelivery(id: string): Promise<unknown> };
|
type RequeueFacade = { requeueDownstreamDelivery(id: string): Promise<unknown> };
|
||||||
const REPLAYABLE_STATUSES = ['pending', 'failed', 'unconfirmed', 'rejected'];
|
const REPLAYABLE_STATUSES = ['pending', 'failed', 'unconfirmed', 'rejected', 'delivered'];
|
||||||
const ACTIVE_TASK_STATUSES = ['queued', 'running', 'paused'];
|
const ACTIVE_TASK_STATUSES = ['queued', 'running', 'paused'];
|
||||||
const PROCESSING_LEASE_MS = 2 * 60_000;
|
const PROCESSING_LEASE_MS = 2 * 60_000;
|
||||||
const SCAN_LEASE_MS = 15_000;
|
const SCAN_LEASE_MS = 15_000;
|
||||||
@@ -119,7 +119,7 @@ export class SendDownstreamRequeueTaskService {
|
|||||||
const preview = verifyPreview(data.previewToken, createdById);
|
const preview = verifyPreview(data.previewToken, createdById);
|
||||||
const filter = normalizedFilter(preview.filter);
|
const filter = normalizedFilter(preview.filter);
|
||||||
const snapshotAt = new Date(preview.snapshotAt);
|
const snapshotAt = new Date(preview.snapshotAt);
|
||||||
if (filter.status === 'delivered' || filter.status === 'awaiting_ack') throw new BadRequestException('第一版后台任务不支持已确认或等待ACK记录');
|
if (filter.status === 'awaiting_ack') throw new BadRequestException('后台任务不支持正在等待ACK的记录');
|
||||||
const activeTask = await this.prisma.downstreamRequeueTask.findFirst({ where: {
|
const activeTask = await this.prisma.downstreamRequeueTask.findFirst({ where: {
|
||||||
status: { in: ACTIVE_TASK_STATUSES },
|
status: { in: ACTIVE_TASK_STATUSES },
|
||||||
...(filter.applicationId !== 'all' ? { OR: [{ applicationId: filter.applicationId }, { applicationId: null }] } : {}),
|
...(filter.applicationId !== 'all' ? { OR: [{ applicationId: filter.applicationId }, { applicationId: null }] } : {}),
|
||||||
@@ -229,14 +229,14 @@ export class SendDownstreamRequeueTaskService {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
await this.prisma.downstreamRequeueTask.update({ where: { id: taskId }, data: { status: 'running', startedAt: task.startedAt ?? new Date() } });
|
await this.prisma.downstreamRequeueTask.update({ where: { id: taskId }, data: { status: 'running', startedAt: task.startedAt ?? new Date() } });
|
||||||
const items = await this.prisma.downstreamRequeueTaskItem.findMany({ where: { taskId, status: 'queued' }, orderBy: { createdAt: 'asc' }, take: Math.min(200, task.ratePerSecond * 3), select: { id: true, deliveryId: true, applicationId: true } });
|
const items = await this.prisma.downstreamRequeueTaskItem.findMany({ where: { taskId, status: 'queued' }, orderBy: { createdAt: 'asc' }, take: Math.min(200, task.ratePerSecond * 3), select: { id: true, deliveryId: true, applicationId: true, previousStatus: true } });
|
||||||
for (const item of items) {
|
for (const item of items) {
|
||||||
const latestTask = await this.prisma.downstreamRequeueTask.findUnique({ where: { id: taskId }, select: { status: true } });
|
const latestTask = await this.prisma.downstreamRequeueTask.findUnique({ where: { id: taskId }, select: { status: true } });
|
||||||
if (!latestTask || !['queued', 'running'].includes(latestTask.status)) break;
|
if (!latestTask || !['queued', 'running'].includes(latestTask.status)) break;
|
||||||
if (!(await this.consumeRate(item.applicationId, task.ratePerSecond))) continue;
|
if (!(await this.consumeRate(item.applicationId, task.ratePerSecond))) continue;
|
||||||
const claimed = await this.prisma.downstreamRequeueTaskItem.updateMany({ where: { id: item.id, status: 'queued' }, data: { status: 'processing', claimedAt: new Date() } });
|
const claimed = await this.prisma.downstreamRequeueTaskItem.updateMany({ where: { id: item.id, status: 'queued' }, data: { status: 'processing', claimedAt: new Date() } });
|
||||||
if (!claimed.count) continue;
|
if (!claimed.count) continue;
|
||||||
const outcome = await this.processItem(item.id, item.deliveryId);
|
const outcome = await this.processItem(item.id, item.deliveryId, item.previousStatus);
|
||||||
if (outcome === 'success') failures[item.applicationId] = 0;
|
if (outcome === 'success') failures[item.applicationId] = 0;
|
||||||
if (outcome === 'failed') failures[item.applicationId] = (failures[item.applicationId] ?? 0) + 1;
|
if (outcome === 'failed') failures[item.applicationId] = (failures[item.applicationId] ?? 0) + 1;
|
||||||
const maxFailures = Math.max(0, ...Object.values(failures));
|
const maxFailures = Math.max(0, ...Object.values(failures));
|
||||||
@@ -257,16 +257,22 @@ export class SendDownstreamRequeueTaskService {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private async processItem(itemId: string, deliveryId: string): Promise<'success' | 'failed' | 'waiting' | 'skipped'> {
|
private async processItem(itemId: string, deliveryId: string, previousStatus: string): Promise<'success' | 'failed' | 'waiting' | 'skipped'> {
|
||||||
try {
|
try {
|
||||||
const delivery = await this.prisma.cmppDownstreamDelivery.findUnique({
|
const delivery = await this.prisma.cmppDownstreamDelivery.findUnique({
|
||||||
where: { id: deliveryId },
|
where: { id: deliveryId },
|
||||||
include: { application: { select: { status: true, interfaceEnabled: true } } },
|
include: { application: { select: { status: true, interfaceEnabled: true } } },
|
||||||
});
|
});
|
||||||
if (!delivery) return this.finishItem(itemId, 'skipped', '投递记录已不存在');
|
if (!delivery) return this.finishItem(itemId, 'skipped', '投递记录已不存在');
|
||||||
|
// Only a record that was already delivered in the frozen task snapshot may be replayed as
|
||||||
|
// delivered. This preserves the operator's explicit duplicate-delivery intent while preventing
|
||||||
|
// a pending/failed record that receives a late ACK after task creation from being sent again.
|
||||||
|
if (delivery.status === 'delivered' && previousStatus !== 'delivered') {
|
||||||
|
return this.finishItem(itemId, 'skipped', '创建任务后已被客户确认');
|
||||||
|
}
|
||||||
if (!REPLAYABLE_STATUSES.includes(delivery.status)) {
|
if (!REPLAYABLE_STATUSES.includes(delivery.status)) {
|
||||||
if (delivery.status === 'awaiting_ack') { await this.prisma.downstreamRequeueTaskItem.update({ where: { id: itemId }, data: { status: 'waiting_external_ack', skipReason: null } }); return 'waiting'; }
|
if (delivery.status === 'awaiting_ack') { await this.prisma.downstreamRequeueTaskItem.update({ where: { id: itemId }, data: { status: 'waiting_external_ack', skipReason: null } }); return 'waiting'; }
|
||||||
return this.finishItem(itemId, 'skipped', delivery.status === 'delivered' ? '已被客户确认' : '执行前状态已变化');
|
return this.finishItem(itemId, 'skipped', '执行前状态已变化');
|
||||||
}
|
}
|
||||||
if (delivery.application.status !== 'active' || !delivery.application.interfaceEnabled) return this.finishItem(itemId, 'skipped', '应用或投递能力已停用');
|
if (delivery.application.status !== 'active' || !delivery.application.interfaceEnabled) return this.finishItem(itemId, 'skipped', '应用或投递能力已停用');
|
||||||
if (!delivery.payload || !['receipt', 'uplink'].includes(delivery.deliveryType)) return this.finishItem(itemId, 'skipped', '投递数据不完整');
|
if (!delivery.payload || !['receipt', 'uplink'].includes(delivery.deliveryType)) return this.finishItem(itemId, 'skipped', '投递数据不完整');
|
||||||
|
|||||||
@@ -4,5 +4,5 @@
|
|||||||
actionstart =
|
actionstart =
|
||||||
actionstop =
|
actionstop =
|
||||||
actioncheck =
|
actioncheck =
|
||||||
actionban = /opt/cmpp-platform/current/bin/cmpp-security-agent report <name> <ip>
|
actionban = @CMPP_SECURITY_AGENT_BIN@ report <name> <ip>
|
||||||
actionunban =
|
actionunban =
|
||||||
|
|||||||
@@ -7,7 +7,7 @@ Type=simple
|
|||||||
User=root
|
User=root
|
||||||
Group=cmpp-security
|
Group=cmpp-security
|
||||||
EnvironmentFile=/etc/cmpp-platform/cmpp-platform.env
|
EnvironmentFile=/etc/cmpp-platform/cmpp-platform.env
|
||||||
ExecStart=/opt/cmpp-platform/current/bin/cmpp-security-agent
|
ExecStart=@CMPP_SECURITY_AGENT_BIN@
|
||||||
Restart=on-failure
|
Restart=on-failure
|
||||||
RestartSec=3
|
RestartSec=3
|
||||||
NoNewPrivileges=true
|
NoNewPrivileges=true
|
||||||
|
|||||||
@@ -1142,4 +1142,12 @@ global,不能错误归入client。`client-signature-*`、发送页、企业认
|
|||||||
|
|
||||||
- `api/src/security-detection/` 是安全事件、聚合告警、规则版本和人工封禁编排的唯一业务边界;登录、OpenAPI 和 Gateway 只上报固定类型的结构化事件,不复制聚合或封禁逻辑。
|
- `api/src/security-detection/` 是安全事件、聚合告警、规则版本和人工封禁编排的唯一业务边界;登录、OpenAPI 和 Gateway 只上报固定类型的结构化事件,不复制聚合或封禁逻辑。
|
||||||
- `gateway/cmd/security-agent/` 是最小特权执行边界,不依赖 NestJS Service,不接受任意命令、路径、jail、action 或 shell 参数。该二进制与 `deploy/security/`、`tools/security/install-security-agent.sh` 作为同一发布单元评审。
|
- `gateway/cmd/security-agent/` 是最小特权执行边界,不依赖 NestJS Service,不接受任意命令、路径、jail、action 或 shell 参数。该二进制与 `deploy/security/`、`tools/security/install-security-agent.sh` 作为同一发布单元评审。
|
||||||
|
- 安全代理可执行文件位置只由安装器的`agent_binary=$APP_DIR/dist/cmpp-security-agent`定义;systemd和Fail2ban模板使用同一占位符渲染,不能各自维护易漂移的绝对路径。
|
||||||
- 前端 `src/apps/admin/security-detection/` 通过 `src/api/admin/security-detection.api.ts` 访问稳定门面,不直接访问 Fail2ban、Nginx、nftables 或安全代理。
|
- 前端 `src/apps/admin/security-detection/` 通过 `src/api/admin/security-detection.api.ts` 访问稳定门面,不直接访问 Fail2ban、Nginx、nftables 或安全代理。
|
||||||
|
- NestJS进程边界由`api/src/main.ts`和生产环境`API_HOST`共同固定为回环监听,外部HTTP入口统一归Nginx模块治理;后续拆分不得让业务模块自行新增外部监听或绕过反向代理边界。
|
||||||
|
|
||||||
|
## 基础设施指标领域边界补充(2026-08-14)
|
||||||
|
|
||||||
|
- `api/src/metrics/`只负责API进程和HTTP路由模板的低基数计数/直方图,不依赖Prisma、Redis或业务Service;独立回环监听由`api/src/main.ts`组装。
|
||||||
|
- `gateway/internal/metrics/`只负责线程安全聚合和Prometheus文本暴露;上游连接池、下游Session和Submit Worker仅提供总数或有界结果,不把实体ID或凭据交给监控模块。
|
||||||
|
- `api/src/infrastructure-monitoring/`仍是运营端只读聚合门面,只消费固定PromQL和Recording Rules;Exporter安装、端口隔离和阈值归`tools/monitoring/`治理。
|
||||||
|
|||||||
@@ -1,8 +1,8 @@
|
|||||||
# 下游投递后台批量重投任务设计与实现符合性审计
|
# 下游投递后台批量重投任务设计与实现符合性审计
|
||||||
|
|
||||||
> 版本:V1.0<br>
|
> 版本:V1.1<br>
|
||||||
> 需求确认日期:2026-08-12<br>
|
> 需求确认日期:2026-08-12<br>
|
||||||
> 文档整理日期:2026-08-13<br>
|
> 文档整理日期:2026-08-13;业务口径更新:2026-08-14<br>
|
||||||
> 适用页面:运营端 → 下游投递记录<br>
|
> 适用页面:运营端 → 下游投递记录<br>
|
||||||
> 审计基线:当前工作区 `HEAD=67fee216162e638ba21004fcf87e7711facefb91`;本功能相关文件相对 HEAD 无未提交修改<br>
|
> 审计基线:当前工作区 `HEAD=67fee216162e638ba21004fcf87e7711facefb91`;本功能相关文件相对 HEAD 无未提交修改<br>
|
||||||
> 本文目的:还原 2026-08-12 已确认的设计口径,并将当前实现逐条映射到设计,不能以“已有代码”代替“符合设计”的结论。
|
> 本文目的:还原 2026-08-12 已确认的设计口径,并将当前实现逐条映射到设计,不能以“已有代码”代替“符合设计”的结论。
|
||||||
@@ -27,9 +27,9 @@
|
|||||||
|
|
||||||
1. 保留单条重投、当前页勾选批量重投,新增“按筛选条件重投”;投递记录分页支持每页 10/25/50 条。
|
1. 保留单条重投、当前页勾选批量重投,新增“按筛选条件重投”;投递记录分页支持每页 10/25/50 条。
|
||||||
2. 后台任务使用企业、应用、投递类型、状态、创建日期、关键词组成的筛选快照;页码和每页条数不属于任务范围。预检生成 `snapshotAt`,创建任务后产生的新记录不进入该任务。
|
2. 后台任务使用企业、应用、投递类型、状态、创建日期、关键词组成的筛选快照;页码和每页条数不属于任务范围。预检生成 `snapshotAt`,创建任务后产生的新记录不进入该任务。
|
||||||
3. 第一版后台任务只允许 `pending`、`failed`、`unconfirmed`、`rejected`。不得批量重投客户端已确认的 `delivered`;处于 `awaiting_ack` 的记录不得并发重投。创建前展示真实命中数、可重投数、规则跳过数、状态分布,任务原因必填。
|
3. 后台任务允许 `pending`、`failed`、`unconfirmed`、`rejected`、`delivered`。客户端已确认的 `delivered` 可按筛选快照再次投递,但必须醒目提示可能造成客户端重复处理;处于 `awaiting_ack` 的记录不得并发重投。创建前展示真实命中数、可重投数、规则跳过数、状态分布,任务原因必填。
|
||||||
4. 任务按企业应用分批执行,默认每个应用 10 条/秒。单条失败不阻断整批;连续失败达到 10 条,或 ACK 超时/拒绝达到安全阈值时,自动暂停。客户离线、已有链路等待 ACK 属于“等待”,不能记作“跳过”。
|
4. 任务按企业应用分批执行,默认每个应用 10 条/秒。单条失败不阻断整批;连续失败达到 10 条,或 ACK 超时/拒绝达到安全阈值时,自动暂停。客户离线、已有链路等待 ACK 属于“等待”,不能记作“跳过”。
|
||||||
5. “跳过”严格表示本任务没有调用 Gateway。第一版跳过原因包括:执行前状态变化、已被客户确认、已被其他任务处理、本任务已成功处理、不属于任务快照、应用或投递能力已停用、投递数据不完整、缺少原 Submit 映射。
|
5. “跳过”严格表示本任务没有调用 Gateway。跳过原因包括:执行前状态变化、创建任务后才被客户确认、已被其他任务处理、本任务已成功处理、不属于任务快照、应用或投递能力已停用、投递数据不完整、缺少原 Submit 映射。只有任务项冻结的原状态已是 `delivered` 时,才允许按已确认记录重投;其他状态在执行前收到迟到成功 ACK 时必须跳过。
|
||||||
6. 任务支持列表、详情、暂停、继续、终止。终止只影响尚未发送的任务项。任务项以 `taskId + deliveryId` 幂等;执行前原子认领并复核当前状态;API 重启后继续执行;已获得成功 ACK 的任务项不得再次发送。
|
6. 任务支持列表、详情、暂停、继续、终止。终止只影响尚未发送的任务项。任务项以 `taskId + deliveryId` 幂等;执行前原子认领并复核当前状态;API 重启后继续执行;已获得成功 ACK 的任务项不得再次发送。
|
||||||
7. 创建、暂停、继续、终止、自动暂停都必须写操作日志。任务必须使用真实 PostgreSQL、Gateway 和客户 ACK,不得使用 mock、静态数据或 localStorage。
|
7. 创建、暂停、继续、终止、自动暂停都必须写操作日志。任务必须使用真实 PostgreSQL、Gateway 和客户 ACK,不得使用 mock、静态数据或 localStorage。
|
||||||
|
|
||||||
@@ -54,7 +54,7 @@
|
|||||||
|
|
||||||
1. 用户设置筛选条件,点击“按筛选条件重投”。
|
1. 用户设置筛选条件,点击“按筛选条件重投”。
|
||||||
2. 后端在同一时点生成预检快照,返回:筛选条件、`snapshotAt`、筛选命中数、可重投数、规则跳过数、状态分布、涉及应用数、最早记录时间。
|
2. 后端在同一时点生成预检快照,返回:筛选条件、`snapshotAt`、筛选命中数、可重投数、规则跳过数、状态分布、涉及应用数、最早记录时间。
|
||||||
3. 弹窗明确告知第一版允许和禁止的状态。
|
3. 弹窗明确告知允许 `pending/failed/unconfirmed/rejected/delivered`、禁止 `awaiting_ack`,并提示已确认记录再次投递可能造成客户端重复处理。
|
||||||
4. 用户选择执行速度,填写不少于 5 个字的事故原因、工单号或处理说明。
|
4. 用户选择执行速度,填写不少于 5 个字的事故原因、工单号或处理说明。
|
||||||
5. 用户确认后,后端必须重新按预检的筛选快照和 `snapshotAt` 物化任务项,而不是使用前端传入的记录 ID 列表。
|
5. 用户确认后,后端必须重新按预检的筛选快照和 `snapshotAt` 物化任务项,而不是使用前端传入的记录 ID 列表。
|
||||||
6. 创建成功后关闭弹窗,任务出现在任务列表,状态为“排队中”。
|
6. 创建成功后关闭弹窗,任务出现在任务列表,状态为“排队中”。
|
||||||
@@ -160,8 +160,8 @@
|
|||||||
|
|
||||||
| 原因 | 判定 |
|
| 原因 | 判定 |
|
||||||
| --- | --- |
|
| --- | --- |
|
||||||
| 执行前状态已变化 | 已不属于允许重投状态,且不是等待 ACK/已确认 |
|
| 执行前状态已变化 | 已不属于允许重投状态,且不是等待 ACK |
|
||||||
| 已被客户确认 | 当前投递已是 `delivered` 且有效 ACK |
|
| 创建任务后才被客户确认 | 任务项冻结原状态不是 `delivered`,执行前收到有效成功 ACK;避免把迟到确认变成未授权重复投递 |
|
||||||
| 已被其他任务处理 | 其他任务已认领、等待 ACK 或成功 |
|
| 已被其他任务处理 | 其他任务已认领、等待 ACK 或成功 |
|
||||||
| 本任务已成功处理 | 同任务项已有成功结果,重复扫描不得再调用 |
|
| 本任务已成功处理 | 同任务项已有成功结果,重复扫描不得再调用 |
|
||||||
| 不属于任务快照 | 创建时间晚于 `snapshotAt` 或不再满足冻结范围 |
|
| 不属于任务快照 | 创建时间晚于 `snapshotAt` 或不再满足冻结范围 |
|
||||||
@@ -222,7 +222,7 @@
|
|||||||
| --- | --- | --- |
|
| --- | --- | --- |
|
||||||
| DRQ-001 | 筛选快照 | 企业、应用、类型、状态、日期、关键词均生效;分页无关;快照后新记录不进入 |
|
| DRQ-001 | 筛选快照 | 企业、应用、类型、状态、日期、关键词均生效;分页无关;快照后新记录不进入 |
|
||||||
| DRQ-002 | 预检口径 | 命中、可重投、跳过和状态分布严格基于当前筛选条件 |
|
| DRQ-002 | 预检口径 | 命中、可重投、跳过和状态分布严格基于当前筛选条件 |
|
||||||
| DRQ-003 | 状态白名单 | 只物化 `pending/failed/unconfirmed/rejected`;拒绝 `delivered/awaiting_ack` |
|
| DRQ-003 | 状态白名单 | 物化 `pending/failed/unconfirmed/rejected/delivered`;拒绝 `awaiting_ack`;仅冻结原状态为 `delivered` 的任务项可按已确认记录重投 |
|
||||||
| DRQ-004 | 每应用限速 | 多应用任务中每个应用独立达到配置速度,任意扫描重叠都不超速 |
|
| DRQ-004 | 每应用限速 | 多应用任务中每个应用独立达到配置速度,任意扫描重叠都不超速 |
|
||||||
| DRQ-005 | 客户离线 | 进入等待连接,不记失败或跳过,恢复连接后继续 |
|
| DRQ-005 | 客户离线 | 进入等待连接,不记失败或跳过,恢复连接后继续 |
|
||||||
| DRQ-006 | ACK 闭环 | 写出只进入等待;有效 ACK 成功;超时、拒绝、无效 Msg_Id 失败 |
|
| DRQ-006 | ACK 闭环 | 写出只进入等待;有效 ACK 成功;超时、拒绝、无效 Msg_Id 失败 |
|
||||||
@@ -250,13 +250,13 @@
|
|||||||
| --- | --- | --- |
|
| --- | --- | --- |
|
||||||
| 真实持久化 | 符合 | 已有任务表、任务项表和 migration,不使用前端本地状态代替任务 |
|
| 真实持久化 | 符合 | 已有任务表、任务项表和 migration,不使用前端本地状态代替任务 |
|
||||||
| 快照时间上限 | 基本符合 | 创建时按 `snapshotAt` 限制 `createdAt`,快照后记录不物化 |
|
| 快照时间上限 | 基本符合 | 创建时按 `snapshotAt` 限制 `createdAt`,快照后记录不物化 |
|
||||||
| 后台状态白名单 | 符合 | `REPLAYABLE_STATUSES` 为 `pending/failed/unconfirmed/rejected` |
|
| 后台状态白名单 | 符合 | `REPLAYABLE_STATUSES` 为 `pending/failed/unconfirmed/rejected/delivered` |
|
||||||
| 禁止后台任务处理已确认/等待 ACK 筛选 | 符合 | 创建接口明确拒绝 `delivered/awaiting_ack` |
|
| 禁止后台任务处理等待 ACK 筛选 | 符合 | 创建接口明确拒绝 `awaiting_ack`;`delivered` 按已确认重复投递风险口径放行 |
|
||||||
| 原因校验 | 符合 | 少于 5 个字拒绝 |
|
| 原因校验 | 符合 | 少于 5 个字拒绝 |
|
||||||
| 任务项幂等 | 符合 | 数据库唯一约束 `taskId + deliveryId` |
|
| 任务项幂等 | 符合 | 数据库唯一约束 `taskId + deliveryId` |
|
||||||
| 执行前任务项认领 | 基本符合 | 通过 `status=queued` 的条件更新认领为 `processing` |
|
| 执行前任务项认领 | 基本符合 | 通过 `status=queued` 的条件更新认领为 `processing` |
|
||||||
| 执行前复核 | 基本符合 | 重新查询投递、应用、payload 和其他任务状态 |
|
| 执行前复核 | 基本符合 | 重新查询投递、应用、payload 和其他任务状态 |
|
||||||
| 成功 ACK 不再发送 | 基本符合 | 已确认投递会跳过,其他任务 `success` 也会阻止调用 |
|
| 成功 ACK 不再误发送 | 基本符合 | 非 `delivered` 快照项执行前才收到成功 ACK 时跳过;其他任务 `success` 也会阻止调用;冻结原状态为 `delivered` 的项目属于运营明确授权的再次投递 |
|
||||||
| 人工控制 | 基本符合 | 已有暂停、继续、终止接口和页面按钮 |
|
| 人工控制 | 基本符合 | 已有暂停、继续、终止接口和页面按钮 |
|
||||||
| 核心操作日志 | 基本符合 | 创建、暂停、继续、终止、自动暂停均写日志 |
|
| 核心操作日志 | 基本符合 | 创建、暂停、继续、终止、自动暂停均写日志 |
|
||||||
| 投递记录分页 | 符合 | 页面支持每页 10/25/50 条并回到第一页 |
|
| 投递记录分页 | 符合 | 页面支持每页 10/25/50 条并回到第一页 |
|
||||||
@@ -279,7 +279,7 @@
|
|||||||
| P1 | 自动化覆盖远低于设计风险 | 专项仅4个测试:预检计数、参数拒绝、活动任务冲突、已确认跳过 | 未覆盖限速、多应用、离线等待、ACK阈值、重启恢复、并发扫描、完整分页及所有审计动作 |
|
| P1 | 自动化覆盖远低于设计风险 | 专项仅4个测试:预检计数、参数拒绝、活动任务冲突、已确认跳过 | 未覆盖限速、多应用、离线等待、ACK阈值、重启恢复、并发扫描、完整分页及所有审计动作 |
|
||||||
| P2 | 状态和详情表达偏内部化 | 任务列表/详情直接展示英文状态;任务详情只展示汇总和最近项 | 运营人员不易区分等待连接、等待外部ACK、任务写出等待ACK等状态 |
|
| P2 | 状态和详情表达偏内部化 | 任务列表/详情直接展示英文状态;任务详情只展示汇总和最近项 | 运营人员不易区分等待连接、等待外部ACK、任务写出等待ACK等状态 |
|
||||||
| P2 | 终止后的未处理数未进入常规汇总 | 终止把`queued`改为`unprocessed`,但任务汇总只保存成功/失败/跳过/等待 | 任务进度分子可能小于总数,页面没有单独解释未处理数量 |
|
| P2 | 终止后的未处理数未进入常规汇总 | 终止把`queued`改为`unprocessed`,但任务汇总只保存成功/失败/跳过/等待 | 任务进度分子可能小于总数,页面没有单独解释未处理数量 |
|
||||||
| P2 | 已确认跳过原因存在口径混用 | `waiting_external_ack`最终由其他链路成功后记为“跳过:已由其他投递链路完成” | 可以接受为“本任务未调用Gateway”,但详情必须明确这是外部链路成功,不应让用户误以为业务未处理 |
|
| P2 | 外部 ACK 跳过原因存在口径混用 | `waiting_external_ack`最终由其他链路成功后记为“跳过:已由其他投递链路完成” | 可以接受为“本任务未调用Gateway”,但详情必须明确这是外部链路成功,不应让用户误以为业务未处理;这与冻结原状态为 `delivered` 的主动再次投递是两种情形 |
|
||||||
|
|
||||||
### 11.3 综合结论
|
### 11.3 综合结论
|
||||||
|
|
||||||
|
|||||||
@@ -345,6 +345,7 @@ GET /api/admin/security-detection/health
|
|||||||
4. 验证 Cloudflare 可信 IP 网段、`real_ip_header` 和源站绕过防护。
|
4. 验证 Cloudflare 可信 IP 网段、`real_ip_header` 和源站绕过防护。
|
||||||
5. 建立 root security agent、Unix Socket 权限、systemd 加固和固定协议。
|
5. 建立 root security agent、Unix Socket 权限、systemd 加固和固定协议。
|
||||||
6. 发布前备份 PostgreSQL、运行源码、环境文件、Fail2ban/Nginx 平台生成配置和 nftables 当前规则。
|
6. 发布前备份 PostgreSQL、运行源码、环境文件、Fail2ban/Nginx 平台生成配置和 nftables 当前规则。
|
||||||
|
7. 安装器必须将systemd `ExecStart`与Fail2ban report-only `actionban`从同一占位符渲染为实际构建产物`$APP_DIR/dist/cmpp-security-agent`,并在旧路径或未替换占位符残留时失败关闭。
|
||||||
|
|
||||||
## 15. 分阶段实施建议
|
## 15. 分阶段实施建议
|
||||||
|
|
||||||
|
|||||||
@@ -151,6 +151,7 @@
|
|||||||
| TC-F2B-OPS-005 | P1 | 执行30天事件、180天告警/审计保留任务 | 只删除到期且不受保护数据;活动封禁、未完成处置和审计期数据不删除 |
|
| TC-F2B-OPS-005 | P1 | 执行30天事件、180天告警/审计保留任务 | 只删除到期且不受保护数据;活动封禁、未完成处置和审计期数据不删除 |
|
||||||
| TC-F2B-OPS-006 | P1 | 大量扫描事件压测 | Collector 有界处理,API与Gateway业务不被阻塞;告警聚合避免写放大 |
|
| TC-F2B-OPS-006 | P1 | 大量扫描事件压测 | Collector 有界处理,API与Gateway业务不被阻塞;告警聚合避免写放大 |
|
||||||
| TC-F2B-OPS-007 | P0 | 对比执行器、数据库和页面 | 当前封禁集合、到期时间和执行器类型一致;差异进入异常状态和告警 |
|
| TC-F2B-OPS-007 | P0 | 对比执行器、数据库和页面 | 当前封禁集合、到期时间和执行器类型一致;差异进入异常状态和告警 |
|
||||||
|
| TC-F2B-OPS-008 | P0 | 在非默认`APP_DIR`构建并运行安全安装器,检查systemd与Fail2ban action的执行路径 | 两者均指向同一个真实可执行的`$APP_DIR/dist/cmpp-security-agent`;残留占位符、旧`current/bin`路径或不可执行目标时安装失败 |
|
||||||
|
|
||||||
## 13. 发布验收证据
|
## 13. 发布验收证据
|
||||||
|
|
||||||
|
|||||||
@@ -1390,6 +1390,7 @@
|
|||||||
- 数据统计的“通道占比”默认统计北京时间当天,可选择单个历史日期重新查询;图表使用真实通道名称和该日期的通道提交量。
|
- 数据统计的“通道占比”默认统计北京时间当天,可选择单个历史日期重新查询;图表使用真实通道名称和该日期的通道提交量。
|
||||||
- 输出 500 条/秒压测报告。
|
- 输出 500 条/秒压测报告。
|
||||||
- 输出 Linux 部署方案,至少覆盖 Docker Compose 或 systemd 部署、环境变量、数据库迁移、日志目录、备份恢复、服务健康检查和回滚步骤。
|
- 输出 Linux 部署方案,至少覆盖 Docker Compose 或 systemd 部署、环境变量、数据库迁移、日志目录、备份恢复、服务健康检查和回滚步骤。
|
||||||
|
- 标准发布脚本配置Nginx压缩时必须兼容发行版已有的HTTP级`gzip on`:先排除自身生成文件检查现有有效配置,已有时复用,没有时才创建平台级配置;重复发布不得因重复指令使`nginx -t`失败。
|
||||||
|
|
||||||
## 16. 新 Codex 会话提示词
|
## 16. 新 Codex 会话提示词
|
||||||
|
|
||||||
@@ -2035,14 +2036,22 @@
|
|||||||
- 支持近1小时、近24小时和近7天固定范围,步长分别为60秒、300秒和1800秒。页面可见时每30秒刷新,隐藏或卸载后停止;手动刷新保留当前范围。
|
- 支持近1小时、近24小时和近7天固定范围,步长分别为60秒、300秒和1800秒。页面可见时每30秒刷新,隐藏或卸载后停止;手动刷新保留当前范围。
|
||||||
- 页面展示Prometheus当前firing/pending告警,严重性使用`info/warning/critical`。告警阈值由Prometheus规则计算,前端不重复判断;第一版仅只读展示,不提供缺少审计模型的确认、备注、静默或关闭操作。
|
- 页面展示Prometheus当前firing/pending告警,严重性使用`info/warning/critical`。告警阈值由Prometheus规则计算,前端不重复判断;第一版仅只读展示,不提供缺少审计模型的确认、备注、静默或关闭操作。
|
||||||
- Prometheus不可用、查询超时、响应非法时接口返回`available=false`及安全错误摘要,页面清空陈旧指标并展示监控不可用;不能继续显示上一次数据造成误判。
|
- Prometheus不可用、查询超时、响应非法时接口返回`available=false`及安全错误摘要,页面清空陈旧指标并展示监控不可用;不能继续显示上一次数据造成误判。
|
||||||
|
- 固定PromQL必须按Prometheus字符串与RE2正则两层语义正确转义,并由专项契约锁定systemd单元正则;任一查询导致整页降级时,API必须记录不含PromQL、地址或凭据的安全错误摘要,不能只向页面返回笼统不可用而没有服务端诊断证据。
|
||||||
|
- 系统监控页只保留平台通用页头中的页面名称;内容区不得再次显示大号“系统监控”标题,可保留“服务器资源、核心服务与活动告警”说明、综合状态和时间范围操作。
|
||||||
- 完整采集、查询、安全、响应契约、视觉规格和验收口径见`docs/prometheus-system-monitoring-design-20260814.md`。
|
- 完整采集、查询、安全、响应契约、视觉规格和验收口径见`docs/prometheus-system-monitoring-design-20260814.md`。
|
||||||
|
|
||||||
|
## 全局预警通知菜单(2026-08-14)
|
||||||
|
|
||||||
|
- 运营端右上角铃铛统一作为预警入口,点击后必须分开显示“签名清退预警”和“安全检测与封禁”两个菜单项;审核待办继续使用独立审核图标和菜单,不得把审核数与预警数混合。
|
||||||
|
- 签名清退项展示今日未读且未抑制的真实消息数并跳转`/admin/signature-retirement`;安全检测项展示`open/acknowledged/block_failed`真实待处置告警总数、严重告警摘要并跳转`/admin/security-detection`。
|
||||||
|
- 铃铛角标为两个预警域数量之和;任一域接口失败时只把该域降级为0,不能影响另一域或审核待办。全局轮询使用专用轻量汇总接口,不得每30秒调用安全检测完整总览、代理回读或大列表。
|
||||||
# 下游投递后台重投任务(2026-08-12)
|
# 下游投递后台重投任务(2026-08-12)
|
||||||
|
|
||||||
## 完整设计口径与安全整改(2026-08-13)
|
## 完整设计口径与安全整改(2026-08-13)
|
||||||
|
|
||||||
1. 后台任务按企业、应用、投递类型、状态、北京时间创建日期和关键词冻结筛选快照,分页、每页条数和当前页勾选不属于范围。预检的命中数、可重投数、规则跳过数、状态分布、涉及应用和最早记录必须严格基于当前筛选条件,不得把单一状态擅自扩为全部状态。
|
1. 后台任务按企业、应用、投递类型、状态、北京时间创建日期和关键词冻结筛选快照,分页、每页条数和当前页勾选不属于范围。预检的命中数、可重投数、规则跳过数、状态分布、涉及应用和最早记录必须严格基于当前筛选条件,不得把单一状态擅自扩为全部状态。
|
||||||
2. 预检返回短期有效且绑定当前操作人、筛选条件和 `snapshotAt` 的服务端签名凭证;创建接口只接受该凭证、原因、速度和安全阈值,不再信任前端重传的范围。创建时后端按签名快照重新物化真实 PostgreSQL 任务项。
|
2. 预检返回短期有效且绑定当前操作人、筛选条件和 `snapshotAt` 的服务端签名凭证;创建接口只接受该凭证、原因、速度和安全阈值,不再信任前端重传的范围。创建时后端按签名快照重新物化真实 PostgreSQL 任务项。
|
||||||
3. 第一版后台任务仅处理 `pending/failed/unconfirmed/rejected`;`delivered/awaiting_ack` 不得进入批量任务。跳过严格表示本任务没有调用 Gateway;客户离线进入 `waiting_connection`,其他链路等待 ACK 进入 `waiting_external_ack`,本任务写出后进入 `waiting_ack`,三类等待均不计失败或跳过。
|
3. 后台任务允许处理 `pending/failed/unconfirmed/rejected/delivered`;客户端已确认的 `delivered` 必须在预检、创建弹窗和任务项原状态中明确呈现,运营确认后可以再次投递,但页面必须提示可能造成客户端重复处理。`awaiting_ack` 不得进入批量任务。若任务项创建时不是 `delivered`、执行前才收到成功 ACK,则本任务必须跳过,不能把迟到确认变成未授权重复投递。跳过严格表示本任务没有调用 Gateway;客户离线进入 `waiting_connection`,其他链路等待 ACK 进入 `waiting_external_ack`,本任务写出后进入 `waiting_ack`,三类等待均不计失败或跳过。
|
||||||
4. 限速以企业应用为维度,使用数据库原子秒级窗口在多实例、扫描重叠和执行耗时变化下保持每应用不超过配置速度。任务扫描使用数据库租约;`processing` 项使用认领租约,API 中断后过期恢复为 `queued` 并重新复核,已成功 ACK 的项目不得重放。
|
4. 限速以企业应用为维度,使用数据库原子秒级窗口在多实例、扫描重叠和执行耗时变化下保持每应用不超过配置速度。任务扫描使用数据库租约;`processing` 项使用认领租约,API 中断后过期恢复为 `queued` 并重新复核,已成功 ACK 的项目不得重放。
|
||||||
5. 连续失败按应用隔离统计。Gateway 立即失败、ACK 超时、ACK 拒绝和无法安全关联均计入阈值;只有有效 ACK 成功才清零。达到阈值前原子暂停整个任务,记录触发应用、失败数、阈值和暂停时间,人工继续后从未完成项恢复。
|
5. 连续失败按应用隔离统计。Gateway 立即失败、ACK 超时、ACK 拒绝和无法安全关联均计入阈值;只有有效 ACK 成功才清零。达到阈值前原子暂停整个任务,记录触发应用、失败数、阈值和暂停时间,人工继续后从未完成项恢复。
|
||||||
6. 任务列表支持状态筛选和真实分页,展示任务号、创建时间、企业/应用、原因、中文状态、总数、成功、失败、跳过、等待、创建人及进度。任务详情展示筛选快照、时间、安全参数、完整结果汇总和任务项分页;任务项可按中文结果、消息 ID、错误或跳过原因查询,不得仅返回最近 50 项。
|
6. 任务列表支持状态筛选和真实分页,展示任务号、创建时间、企业/应用、原因、中文状态、总数、成功、失败、跳过、等待、创建人及进度。任务详情展示筛选快照、时间、安全参数、完整结果汇总和任务项分页;任务项可按中文结果、消息 ID、错误或跳过原因查询,不得仅返回最近 50 项。
|
||||||
@@ -2051,9 +2060,9 @@
|
|||||||
|
|
||||||
1. 运营端“下游投递记录”必须同时保留单条重投、当前页勾选批量重投,并新增“按筛选条件重投”;分页支持每页 `10/25/50` 条,切换后回到第一页并重新查询真实后端。
|
1. 运营端“下游投递记录”必须同时保留单条重投、当前页勾选批量重投,并新增“按筛选条件重投”;分页支持每页 `10/25/50` 条,切换后回到第一页并重新查询真实后端。
|
||||||
2. 后台任务使用当前企业、应用、投递类型、状态、创建日期和关键词的后端筛选快照,分页不属于任务范围;任务创建时固定 `snapshotAt`,之后产生的记录不得被卷入。
|
2. 后台任务使用当前企业、应用、投递类型、状态、创建日期和关键词的后端筛选快照,分页不属于任务范围;任务创建时固定 `snapshotAt`,之后产生的记录不得被卷入。
|
||||||
3. 第一版只允许 `pending/failed/unconfirmed/rejected`,不支持批量重投客户端已确认的 `delivered`,`awaiting_ack` 不得并发重投。创建前必须真实预检命中、可重投、跳过和状态分布,原因必填。
|
3. 批量任务允许 `pending/failed/unconfirmed/rejected/delivered`,其中 `delivered` 会再次发送并可能造成客户端重复处理,创建弹窗必须醒目提示;`awaiting_ack` 不得并发重投。创建前必须真实预检命中、可重投、跳过和状态分布,原因必填。执行器仅允许重投在冻结快照中原状态已经是 `delivered` 的已确认任务项;任务建立后才变成 `delivered` 的项目必须跳过。
|
||||||
4. 任务按应用分批执行,默认每秒 10 条;单条失败不阻断整批,连续失败达到 10 条或 ACK 超时/拒绝达到安全阈值时自动暂停。客户离线、等待 ACK 属于等待状态,不得误记为跳过。
|
4. 任务按应用分批执行,默认每秒 10 条;单条失败不阻断整批,连续失败达到 10 条或 ACK 超时/拒绝达到安全阈值时自动暂停。客户离线、等待 ACK 属于等待状态,不得误记为跳过。
|
||||||
5. 跳过只表示未调用 Gateway,第一版原因包括:状态已变化、已被客户确认、已被其他任务处理、本任务已成功处理、不属于任务快照、应用或投递能力已停用、投递数据不完整、缺少原 Submit 映射。
|
5. 跳过只表示未调用 Gateway,原因包括:状态已变化、创建任务后才被客户确认、已被其他任务处理、本任务已成功处理、不属于任务快照、应用或投递能力已停用、投递数据不完整、缺少原 Submit 映射。
|
||||||
6. 任务必须支持列表、详情、暂停、继续和终止;终止只影响尚未发送的记录。任务项以 `taskId + deliveryId` 幂等,执行前原子认领并复核状态,API 重启后可继续,成功 ACK 的项目不得再次发送。
|
6. 任务必须支持列表、详情、暂停、继续和终止;终止只影响尚未发送的记录。任务项以 `taskId + deliveryId` 幂等,执行前原子认领并复核状态,API 重启后可继续,成功 ACK 的项目不得再次发送。
|
||||||
7. 所有创建、暂停、继续、终止和自动暂停均写操作日志;任务使用真实 PostgreSQL、Gateway 和客户 ACK,不得使用 mock、静态数据或 localStorage。
|
7. 所有创建、暂停、继续、终止和自动暂停均写操作日志;任务使用真实 PostgreSQL、Gateway 和客户 ACK,不得使用 mock、静态数据或 localStorage。
|
||||||
# 2026-08-13 HTTP 请求与 Gateway API 响应容量边界
|
# 2026-08-13 HTTP 请求与 Gateway API 响应容量边界
|
||||||
@@ -2078,3 +2087,12 @@
|
|||||||
- NestJS 必须以专用非 root 用户运行,不得获得通用 sudo、任意 shell、直接编辑 `/etc/fail2ban/*` 或防火墙的能力。独立 root security agent 只监听本机 Unix Socket、接受固定 JSON 动作、使用参数数组执行固定操作,并在真实 nftables 或 Nginx deny 回读成功后才允许数据库标记 `blocked`。
|
- NestJS 必须以专用非 root 用户运行,不得获得通用 sudo、任意 shell、直接编辑 `/etc/fail2ban/*` 或防火墙的能力。独立 root security agent 只监听本机 Unix Socket、接受固定 JSON 动作、使用参数数组执行固定操作,并在真实 nftables 或 Nginx deny 回读成功后才允许数据库标记 `blocked`。
|
||||||
- 运营端/客户端的 Cloudflare 入口使用 Nginx real-IP deny;直连 HTTP API、SSH 与 CMPP 使用 nftables。只有可信代理 TCP 来源可以提供访客 IP;系统回环、私网、链路本地、组播和配置的运维/健康检查网段必须内置保护。
|
- 运营端/客户端的 Cloudflare 入口使用 Nginx real-IP deny;直连 HTTP API、SSH 与 CMPP 使用 nftables。只有可信代理 TCP 来源可以提供访客 IP;系统回环、私网、链路本地、组播和配置的运维/健康检查网段必须内置保护。
|
||||||
- 规则更新先保存待应用版本,由安全代理生成固定 Fail2ban 配置、执行语法校验并 reload;失败保留上一生效值并展示失败原因,不得显示为已生效。完整架构、状态机、字段、接口和安全边界以 `docs/fail2ban-assisted-blocking-design-20260814.md` 为准。
|
- 规则更新先保存待应用版本,由安全代理生成固定 Fail2ban 配置、执行语法校验并 reload;失败保留上一生效值并展示失败原因,不得显示为已生效。完整架构、状态机、字段、接口和安全边界以 `docs/fail2ban-assisted-blocking-design-20260814.md` 为准。
|
||||||
|
- 安全代理的systemd单元与Fail2ban report-only action必须由安装器从同一受控占位符渲染,并共同指向发布脚本实际生成的`$APP_DIR/dist/cmpp-security-agent`;安装后残留占位符、旧`current/bin`路径或不可执行目标时必须终止发布,禁止出现Agent服务可启动但Fail2ban action静默失效的分叉配置。
|
||||||
|
- NestJS API生产进程必须显式绑定`127.0.0.1`,仅由Nginx受控入口反向代理;不得依赖框架默认的全网卡监听而把3000端口直接暴露到LAN、Tailscale或公网。非生产环境如确需其他地址,只能通过明确的`API_HOST`配置覆盖。
|
||||||
|
|
||||||
|
## 服务内部指标与阈值(2026-08-14)
|
||||||
|
|
||||||
|
- API、Gateway必须以回环端点暴露低基数运行指标;PostgreSQL、Redis、Nginx使用发行版Exporter,MinIO使用原生指标。任何指标端口都不得经Nginx或安全组对公网暴露。
|
||||||
|
- 运营端展示API请求/错误/延迟/事件循环、Gateway Submit/队列/连接、PostgreSQL连接/死锁、Redis内存/连接/淘汰、Nginx连接/请求和MinIO可用性;指标缺失显示“待采集”,不以0伪装。
|
||||||
|
- 告警必须使用持续窗口和最低样本量;默认阈值、收敛关系、标签禁止项和性能预算以`docs/prometheus-system-monitoring-design-20260814.md`第9节为准。
|
||||||
|
- 指标不得包含手机号、短信正文、短信/CMPP/任务ID、密钥、完整URL或SQL原文;不得把时序指标高频写入业务PostgreSQL。
|
||||||
|
|||||||
@@ -30,6 +30,9 @@ REPO_URL=http://175.27.255.91:3000/hectorzhao/lislgosms.git
|
|||||||
BRANCH=main
|
BRANCH=main
|
||||||
PUBLIC_HTTP_PORT=12026
|
PUBLIC_HTTP_PORT=12026
|
||||||
API_PORT=3000
|
API_PORT=3000
|
||||||
|
API_HOST=127.0.0.1
|
||||||
|
API_METRICS_HOST=127.0.0.1
|
||||||
|
API_METRICS_PORT=9464
|
||||||
HTTP_API_MASTER_KEY=<至少32位随机值,用于AES-256-GCM加密HTTP访问凭据和Webhook密钥>
|
HTTP_API_MASTER_KEY=<至少32位随机值,用于AES-256-GCM加密HTTP访问凭据和Webhook密钥>
|
||||||
HTTP_API_PUBLIC_ORIGIN=https://api.lisglo.com
|
HTTP_API_PUBLIC_ORIGIN=https://api.lisglo.com
|
||||||
API_ENABLE_SEND_WORKER=true
|
API_ENABLE_SEND_WORKER=true
|
||||||
@@ -64,7 +67,7 @@ PROD_ADMIN_USERNAME=prod_admin
|
|||||||
PROD_ADMIN_PASSWORD='change-me'
|
PROD_ADMIN_PASSWORD='change-me'
|
||||||
```
|
```
|
||||||
|
|
||||||
系统监控的Prometheus和Node Exporter需要在发布前单独安装,配置和规则位于`tools/monitoring/`。在Debian/Ubuntu服务器执行`bash tools/monitoring/install-prometheus-monitoring.sh`;脚本会先备份现有Prometheus配置并运行`promtool`校验,只重启Prometheus和Node Exporter,不重启CMPP业务服务。9090和9100必须只监听`127.0.0.1`,不得加入Nginx公网反向代理或安全组放行。安装完成且确认`curl http://127.0.0.1:9090/-/ready`成功后,才可在正常发布窗口重启API使系统监控接口生效;详细口径见`docs/prometheus-system-monitoring-design-20260814.md`。
|
系统监控需要在发布前单独安装,配置和规则位于`tools/monitoring/`。在Debian/Ubuntu服务器依次执行`bash tools/monitoring/install-prometheus-monitoring.sh`和`bash tools/monitoring/install-service-exporters.sh`;前者安装Prometheus/Node Exporter,后者安装PostgreSQL/Redis/Nginx Exporter并开启MinIO回环原生指标。脚本必须先备份现有配置并运行`promtool`校验;9090、9100、9187、9121、9113、9464和API/Gateway控制端口必须只监听`127.0.0.1`,不得加入Nginx公网反向代理或安全组放行。安装完成且确认全部target为up后,才可完成发布;详细口径见`docs/prometheus-system-monitoring-design-20260814.md`。
|
||||||
|
|
||||||
安全会话使用 HttpOnly Cookie,正式生产必须先为页面和管理 API 配置 HTTPS,并保持 `SESSION_COOKIE_SECURE=true`;纯 HTTP 的 `IP:12026` 不作为受支持的登录入口,即使切换期仍保留其监听,也只允许用于非登录的兼容检查并应尽快下线。`sms.lisglo.com` 只允许 Cloudflare 回源,`api.lisglo.com` 通过独立 Nginx SNI 虚拟主机只开放客户接口、客户 Swagger 和健康检查;Let’s Encrypt 使用 DNS-01 自动续期,不依赖开放 80 端口。
|
安全会话使用 HttpOnly Cookie,正式生产必须先为页面和管理 API 配置 HTTPS,并保持 `SESSION_COOKIE_SECURE=true`;纯 HTTP 的 `IP:12026` 不作为受支持的登录入口,即使切换期仍保留其监听,也只允许用于非登录的兼容检查并应尽快下线。`sms.lisglo.com` 只允许 Cloudflare 回源,`api.lisglo.com` 通过独立 Nginx SNI 虚拟主机只开放客户接口、客户 Swagger 和健康检查;Let’s Encrypt 使用 DNS-01 自动续期,不依赖开放 80 端口。
|
||||||
|
|
||||||
@@ -111,6 +114,8 @@ SECURITY_BUILTIN_PROTECTED_NETWORKS=<运维出口CIDR,健康检查CIDR,源站公
|
|||||||
|
|
||||||
部署脚本重启 Gateway、API 和 Nginx 后,会分别对 Gateway、API 健康接口执行最多 60 秒的逐秒就绪检查。Nest 初始化、活动通道恢复或生产数据量增加可能使 API 启动超过固定数秒;发布流程不得用单次固定延时把正常慢启动误判为失败。超过 60 秒仍不健康时才终止发布,并结合 systemd journal 和发布前数据库、源码、环境备份判断回滚方式。
|
部署脚本重启 Gateway、API 和 Nginx 后,会分别对 Gateway、API 健康接口执行最多 60 秒的逐秒就绪检查。Nest 初始化、活动通道恢复或生产数据量增加可能使 API 启动超过固定数秒;发布流程不得用单次固定延时把正常慢启动误判为失败。超过 60 秒仍不健康时才终止发布,并结合 systemd journal 和发布前数据库、源码、环境备份判断回滚方式。
|
||||||
|
|
||||||
|
标准发布会先清空自身管理的`/etc/nginx/conf.d/cmpp-compression.conf`,再排除该文件检查Nginx现有配置。发行版或既有虚拟主机已经启用`gzip on`时直接复用;完全未启用时才写入平台级压缩配置。不得无条件叠加第二个HTTP级`gzip on`,每次重启前必须以`nginx -t`为准。
|
||||||
|
|
||||||
## 账号和密钥
|
## 账号和密钥
|
||||||
|
|
||||||
- 生产管理员账号写入 `/root/cmpp-platform-admin.txt`。
|
- 生产管理员账号写入 `/root/cmpp-platform-admin.txt`。
|
||||||
|
|||||||
@@ -192,3 +192,48 @@ type InfrastructureOverview = {
|
|||||||
8. Prometheus/Exporter 只监听本机,公网不能连接 9090/9100。
|
8. Prometheus/Exporter 只监听本机,公网不能连接 9090/9100。
|
||||||
9. 页面在桌面和移动宽度下无重叠、截断和横向溢出,控制台无相关错误。
|
9. 页面在桌面和移动宽度下无重叠、截断和横向溢出,控制台无相关错误。
|
||||||
10. TypeScript、API专项测试、生产构建、配置校验和 `git diff --check`通过。
|
10. TypeScript、API专项测试、生产构建、配置校验和 `git diff --check`通过。
|
||||||
|
|
||||||
|
## 9. 服务内部指标扩展(V1.1)
|
||||||
|
|
||||||
|
### 9.1 采集边界
|
||||||
|
|
||||||
|
- API使用独立回环端口`127.0.0.1:9464/metrics`,采集进程内存、堆内存、事件循环P99、请求量、状态码和延迟直方图。
|
||||||
|
- Gateway在已有回环控制端口`127.0.0.1:8090/metrics`暴露Go运行时、上下游连接总数、Submit成败和耗时、Redis Stream pending/lag/最旧年龄。
|
||||||
|
- PostgreSQL、Redis、Nginx使用发行版Exporter;MinIO使用原生Prometheus端点。全部Exporter只监听回环地址。
|
||||||
|
- 指标标签只允许方法、路由模板、HTTP状态、结果类别和固定服务名。禁止手机号、短信ID、CMPP Msg_Id、任务ID、通道凭据、短信正文、原始URL和SQL文本进入标签。
|
||||||
|
- 监控页只查询`cmpp:service_*`固定Recording Rules,不为每张卡片执行一条高代价PromQL。
|
||||||
|
|
||||||
|
### 9.2 默认阈值
|
||||||
|
|
||||||
|
| 领域 | Warning | Critical | 持续窗口 |
|
||||||
|
|---|---:|---:|---:|
|
||||||
|
| CPU | >80% | >90% | 10m / 5m |
|
||||||
|
| 内存 | >85% | >95% | 10m / 5m |
|
||||||
|
| 磁盘或inode | >80% | >90% | 15m / 5m |
|
||||||
|
| API 5xx | >1%且至少5次 | >5%且至少5次 | 5m |
|
||||||
|
| API P95 | >1s | >3s | 10m / 5m |
|
||||||
|
| API事件循环P99 | >200ms | >1s | 10m / 5m |
|
||||||
|
| Gateway提交队列最旧pending | >30s | >120s | 2m |
|
||||||
|
| Gateway实际上游连接 | — | 少于期朖2m | 2m |
|
||||||
|
| PostgreSQL连接使用率 | >70% | >85% | 10m / 5m |
|
||||||
|
| PostgreSQL死锁 | 15m内>0 | 15m内重复出现 | 1m |
|
||||||
|
| Redis内存/maxmemory | >70% | >85% | 10m / 5m |
|
||||||
|
| Redis淘汰或拒绝连接 | — | 5m内>0 | 1m |
|
||||||
|
| 任一核心Exporter失联 | — | >2m | 2m |
|
||||||
|
| Prometheus采集耗时/周期 | >80% | >100% | 5m |
|
||||||
|
| Prometheus规则计算失败 | — | 5m内>0 | 1m |
|
||||||
|
|
||||||
|
比例告警必须带最低样本量,不得把单次失败误报为100%错误率。短信最终回执可由运营商延迟数小时,不纳入Gateway基础设施短窗口Critical,仍由72小时业务终结机制和短信质量看板处理。
|
||||||
|
|
||||||
|
### 9.3 告警收敛与校准
|
||||||
|
|
||||||
|
- 主机或Node Exporter失联时,抑制其CPU、内存和磁盘派生告警;API/Gateway指标端点失联时,抑制对应延迟和错误率告警。
|
||||||
|
- Critical表示需立即处理;Warning表示当日检查;趋势指标未达到可操作条件时只展示,不生成告警。
|
||||||
|
- 上线后保留7至14天基线,核对业务高峰P95/P99、正常连接数和队列年龄。阈值调整必须同步修改设计、用例、规则和进度记录。
|
||||||
|
|
||||||
|
### 9.4 性能预算
|
||||||
|
|
||||||
|
- 全局采集周期保持15秒,无排障需求不降到1秒。
|
||||||
|
- API和Gateway请求路径只做内存计数、有界直方图和原子计数,不在业务请求中写PostgreSQL或Redis。
|
||||||
|
- PostgreSQL Exporter只使用发行版默认低代价查询,不采集SQL原文或扫描业务大表。
|
||||||
|
- 时序仍受30天和8GB双重上限约束;时序增长时先缩短实际保留期,不允许无界占满业务盘。
|
||||||
|
|||||||
@@ -4567,6 +4567,7 @@ npm run verify:phase8
|
|||||||
| TC-SIGNATURE-RETIREMENT-027 | 在抑制管理点击“取消抑制”,填写或不填写原因 | 只出现平台自研弹窗;未填原因不能确认,填写后调用真实取消接口并刷新消息及抑制列表,不出现浏览器`prompt/confirm` |
|
| TC-SIGNATURE-RETIREMENT-027 | 在抑制管理点击“取消抑制”,填写或不填写原因 | 只出现平台自研弹窗;未填原因不能确认,填写后调用真实取消接口并刷新消息及抑制列表,不出现浏览器`prompt/confirm` |
|
||||||
| TC-REPORT-RECORD-LAYOUT-001 | 在报备记录页面查看长备注和短备注 | 备注列桌面宽度不小于320px,使用统一长文本换行样式;宽表允许内部横向滚动,备注不被其他固定列挤成窄竖列,详情仍展示全文 |
|
| TC-REPORT-RECORD-LAYOUT-001 | 在报备记录页面查看长备注和短备注 | 备注列桌面宽度不小于320px,使用统一长文本换行样式;宽表允许内部横向滚动,备注不被其他固定列挤成窄竖列,详情仍展示全文 |
|
||||||
| TC-DEPLOY-HEALTH-001 | 发布重启后模拟API初始化超过3秒但在60秒内恢复,并分别模拟API或Gateway持续60秒不可用 | 前者由部署脚本逐秒重试并正常完成,不触发误回滚;后者在60秒后明确失败并保留发布前数据库、源码和环境恢复资产,不把端口尚未就绪当作构建或migration失败 |
|
| TC-DEPLOY-HEALTH-001 | 发布重启后模拟API初始化超过3秒但在60秒内恢复,并分别模拟API或Gateway持续60秒不可用 | 前者由部署脚本逐秒重试并正常完成,不触发误回滚;后者在60秒后明确失败并保留发布前数据库、源码和环境恢复资产,不把端口尚未就绪当作构建或migration失败 |
|
||||||
|
| TC-DEPLOY-NGINX-001 | 分别在Ubuntu默认`nginx.conf`已有全局`gzip on`和完全没有gzip配置的环境执行两次标准发布 | 已有配置时平台生成文件保持为空并复用发行版配置;缺失时写入平台配置;两种环境连续执行两次`nginx -t`均通过且不存在重复gzip指令 |
|
||||||
|
|
||||||
### 2026-08-10 本地执行状态
|
### 2026-08-10 本地执行状态
|
||||||
|
|
||||||
@@ -4588,11 +4589,11 @@ npm run verify:phase8
|
|||||||
| 编号 | 场景 | 预期 |
|
| 编号 | 场景 | 预期 |
|
||||||
|---|---|---|
|
|---|---|---|
|
||||||
| TC-DOWNSTREAM-REQUEUE-TASK-001 | 当前筛选条件预检 | 后端按企业、应用、类型、状态、日期、关键词和 `snapshotAt` 返回真实命中、可重投、跳过及状态分布;分页不影响数量。 |
|
| TC-DOWNSTREAM-REQUEUE-TASK-001 | 当前筛选条件预检 | 后端按企业、应用、类型、状态、日期、关键词和 `snapshotAt` 返回真实命中、可重投、跳过及状态分布;分页不影响数量。 |
|
||||||
| TC-DOWNSTREAM-REQUEUE-TASK-002 | 创建任务 | 原因少于 5 字拒绝;仅物化 `pending/failed/unconfirmed/rejected`;`delivered/awaiting_ack` 不进入执行。 |
|
| TC-DOWNSTREAM-REQUEUE-TASK-002 | 创建任务 | 原因少于 5 字拒绝;物化 `pending/failed/unconfirmed/rejected/delivered`;客户端已确认记录在预检中计为可重投并保留 `previousStatus=delivered`,`awaiting_ack` 不进入执行。 |
|
||||||
| TC-DOWNSTREAM-REQUEUE-TASK-003 | 快照边界 | 创建任务后新增或筛选条件外记录不进入任务。 |
|
| TC-DOWNSTREAM-REQUEUE-TASK-003 | 快照边界 | 创建任务后新增或筛选条件外记录不进入任务。 |
|
||||||
| TC-DOWNSTREAM-REQUEUE-TASK-004 | 并发与幂等 | `taskId+deliveryId` 唯一;重复扫描、API 重启及并发任务不会重复调用 Gateway。 |
|
| TC-DOWNSTREAM-REQUEUE-TASK-004 | 并发与幂等 | `taskId+deliveryId` 唯一;重复扫描、API 重启及并发任务不会重复调用 Gateway。 |
|
||||||
| TC-DOWNSTREAM-REQUEUE-TASK-005 | ACK 闭环 | Gateway 写出后项目进入等待 ACK;`Result=0` 成功,拒绝/超时计失败并按阈值自动暂停。 |
|
| TC-DOWNSTREAM-REQUEUE-TASK-005 | ACK 闭环 | Gateway 写出后项目进入等待 ACK;`Result=0` 成功,拒绝/超时计失败并按阈值自动暂停。 |
|
||||||
| TC-DOWNSTREAM-REQUEUE-TASK-006 | 状态变化跳过 | 执行前状态变化、已确认或被其他操作认领时不调用 Gateway,记录明确跳过原因。 |
|
| TC-DOWNSTREAM-REQUEUE-TASK-006 | 状态变化跳过 | 创建任务时不是 `delivered`、执行前才收到成功 ACK 的记录,或已被其他操作认领时不调用 Gateway,记录明确跳过原因;快照原状态就是 `delivered` 的记录允许调用 Gateway。 |
|
||||||
| TC-DOWNSTREAM-REQUEUE-TASK-007 | 任务控制 | 待执行/执行中任务可暂停、继续、终止;终止不撤回已写出消息。 |
|
| TC-DOWNSTREAM-REQUEUE-TASK-007 | 任务控制 | 待执行/执行中任务可暂停、继续、终止;终止不撤回已写出消息。 |
|
||||||
| TC-DOWNSTREAM-REQUEUE-TASK-008 | 审计 | 创建、暂停、继续、终止和自动暂停记录操作人、筛选快照、原因和结果。 |
|
| TC-DOWNSTREAM-REQUEUE-TASK-008 | 审计 | 创建、暂停、继续、终止和自动暂停记录操作人、筛选快照、原因和结果。 |
|
||||||
| TC-DOWNSTREAM-PAGE-SIZE-001 | 分页数量 | 可选 10/25/50;切换回第一页,后端返回对应条数,总数和筛选条件保持一致。 |
|
| TC-DOWNSTREAM-PAGE-SIZE-001 | 分页数量 | 可选 10/25/50;切换回第一页,后端返回对应条数,总数和筛选条件保持一致。 |
|
||||||
@@ -4610,6 +4611,8 @@ npm run verify:phase8
|
|||||||
| TC-DOWNSTREAM-REQUEUE-TASK-015 | 列表完整分页 | 任务列表支持状态和分页,第 11 条以后可访问,中文状态、创建人、原因、进度和各结果数准确。 |
|
| TC-DOWNSTREAM-REQUEUE-TASK-015 | 列表完整分页 | 任务列表支持状态和分页,第 11 条以后可访问,中文状态、创建人、原因、进度和各结果数准确。 |
|
||||||
| TC-DOWNSTREAM-REQUEUE-TASK-016 | 完整任务项查询 | 任务项支持分页、结果及关键词查询,等待连接/外部 ACK/本任务 ACK、跳过、失败和未处理均中文展示并保留原因。 |
|
| TC-DOWNSTREAM-REQUEUE-TASK-016 | 完整任务项查询 | 任务项支持分页、结果及关键词查询,等待连接/外部 ACK/本任务 ACK、跳过、失败和未处理均中文展示并保留原因。 |
|
||||||
| TC-DOWNSTREAM-REQUEUE-TASK-017 | 终止并发边界 | 终止后未认领和等待连接项置为未处理;处理中或已写出项不撤回;执行器不再认领新项。 |
|
| TC-DOWNSTREAM-REQUEUE-TASK-017 | 终止并发边界 | 终止后未认领和等待连接项置为未处理;处理中或已写出项不撤回;执行器不再认领新项。 |
|
||||||
|
| TC-DOWNSTREAM-REQUEUE-TASK-018 | 已确认记录批量重投 | 以状态 `delivered` 预检并创建任务,核对任务项原状态后执行;页面显示重复投递风险,真实任务项进入等待 ACK/成功闭环;同一任务已成功项不重复调用 Gateway。 |
|
||||||
|
| TC-DOWNSTREAM-REQUEUE-TASK-019 | 创建弹窗与列表留白 | 桌面及窄屏打开创建弹窗和后台任务列表,输入不足5字及合法原因 | 原因使用统一多行输入组件,必填、错误、说明、字数和焦点态清晰;任务列表与卡片边缘保持设计间距,行内容不贴边、不裁切,移动端留白同步收敛。 |
|
||||||
# 2026-08-13 HTTP 与 Gateway 报文容量专项用例
|
# 2026-08-13 HTTP 与 Gateway 报文容量专项用例
|
||||||
|
|
||||||
| 用例编号 | 优先级 | 验证内容 | 预期结果 |
|
| 用例编号 | 优先级 | 验证内容 | 预期结果 |
|
||||||
@@ -4650,9 +4653,25 @@ npm run verify:phase8
|
|||||||
| TC-INFRA-MON-015 | 响应式与无障碍 | 在1536×1024、1280×800和390×844打开页面,操作范围和刷新按钮 | 桌面信息层级符合设计稿;窄屏无内容重叠和页面横向溢出;按钮有可读名称,活动范围和告警严重性不只依赖颜色表达 |
|
| TC-INFRA-MON-015 | 响应式与无障碍 | 在1536×1024、1280×800和390×844打开页面,操作范围和刷新按钮 | 桌面信息层级符合设计稿;窄屏无内容重叠和页面横向溢出;按钮有可读名称,活动范围和告警严重性不只依赖颜色表达 |
|
||||||
| TC-INFRA-MON-016 | 配置和部署幂等 | 在测试服务器重复执行监控安装脚本和配置校验 | 不重复创建系统用户,不开放公网端口;配置通过`promtool check config/rules`,服务保持active,CMPP API/Gateway不因安装被重启 |
|
| TC-INFRA-MON-016 | 配置和部署幂等 | 在测试服务器重复执行监控安装脚本和配置校验 | 不重复创建系统用户,不开放公网端口;配置通过`promtool check config/rules`,服务保持active,CMPP API/Gateway不因安装被重启 |
|
||||||
| TC-INFRA-MON-017 | 业务数据隔离 | 运行监控24小时并检查PostgreSQL业务库和指标标签 | 监控时序只保存在Prometheus TSDB,业务PostgreSQL无高频指标写入;标签、日志和API响应不含手机号、短信正文、账号或密钥 |
|
| TC-INFRA-MON-017 | 业务数据隔离 | 运行监控24小时并检查PostgreSQL业务库和指标标签 | 监控时序只保存在Prometheus TSDB,业务PostgreSQL无高频指标写入;标签、日志和API响应不含手机号、短信正文、账号或密钥 |
|
||||||
|
| TC-INFRA-MON-018 | systemd PromQL转义兼容 | 使用真实Prometheus执行API生成的服务状态查询,并检查自动化请求参数 | 查询文本向Prometheus传递双反斜杠转义的`\\.`正则,API返回`available=true`及真实服务状态;不得因`unknown escape sequence`把整页降级 |
|
||||||
|
| TC-INFRA-MON-019 | 页面标题去重 | 打开系统监控页并检查平台页头和内容区 | 平台通用页头保留“系统监控”,内容区不再出现重复大号标题;说明、状态、范围和刷新操作完整可用 |
|
||||||
|
| TC-INFRA-MON-020 | API内部指标 | 回环请求API metrics,再发起成功与失败的固定路由请求 | 请求量、状态码、延迟桶、堆内存和事件循环指标变化;route为路由模板,不含实体ID或查询串 |
|
||||||
|
| TC-INFRA-MON-021 | Gateway内部指标 | 请求`127.0.0.1:8090/metrics`,交叉核对连接池和Redis Stream | 上下游连接、Submit计数/耗时、worker up、pending、lag和最旧pending年龄与真实状态一致 |
|
||||||
|
| TC-INFRA-MON-022 | 服务Exporter目标 | 安装PostgreSQL、Redis、Nginx Exporter并开启MinIO原生指标 | Prometheus六个新服务target均up;数据库、Redis、MinIO、Nginx指标与各服务本地命令在采样误差内一致 |
|
||||||
|
| TC-INFRA-MON-023 | Exporter端口隔离 | 执行`ss -lnt`并从LAN/公网探测9464、9187、9121、9113、9090、9100 | 全部只监听127.0.0.1或::1,Nginx业务站点不代理metrics端点 |
|
||||||
|
| TC-INFRA-MON-024 | 阈值持续窗口 | 在隔离节点分别制造瞬时和持续的CPU/API错误/队列延迟 | 瞬时尖峰不告警;达到阈值与`for`窗口后进入pending/firing,恢复后移除 |
|
||||||
|
| TC-INFRA-MON-025 | 低流量错误率保护 | 5分钟内只产生1次API请求且返回500 | 因未达至5次错误的最低样本量,不产生5xx比例告警 |
|
||||||
|
| TC-INFRA-MON-026 | 高基数和敏感字段防护 | 检查API/Gateway/Exporter全量metrics文本及Prometheus label names/values | 不存在手机号、短信正文、message/submit/task/channel实体ID、凭据、原始URL或SQL文本 |
|
||||||
|
| TC-INFRA-MON-027 | Recording Rules查询收敛 | 刷新系统监控页并检查Prometheus请求 | 服务卡片只读取`cmpp:service_*`固定聚合,不按卡片开放任意PromQL;缺失指标显示“待采集” |
|
||||||
|
| TC-INFRA-MON-028 | 监控开销对比 | 在同等请求压力下对比开启前后API/Gateway CPU、RSS、P95和吞吐 | 无高基数增长、无业务PostgreSQL高频写入;开销超出预算时暂停发布并调整采集/桶配置 |
|
||||||
|
| TC-GLOBAL-ALERT-001 | 铃铛分域预警菜单 | 准备签名清退未读消息和安全待处置告警后点击右上角铃铛 | 弹层分开显示“签名清退预警”和“安全检测与封禁”,分别展示真实数量和摘要,角标等于两项之和 |
|
||||||
|
| TC-GLOBAL-ALERT-002 | 预警菜单跳转 | 分别点击铃铛中的两个菜单项 | 签名项跳转`/admin/signature-retirement`,安全项跳转`/admin/security-detection`,弹层关闭且对应页面读取真实后端数据 |
|
||||||
|
| TC-GLOBAL-ALERT-003 | 域间故障隔离与轻量轮询 | 分别让一个汇总接口失败并观察30秒轮询请求 | 失败域显示0且另一域数据保留;安全预警使用专用汇总接口,不调用完整overview、规则、代理状态或告警大列表 |
|
||||||
|
| TC-DEPLOY-NET-001 | API回环监听边界 | 使用标准生产环境启动API,执行`ss -lnt`并从LAN/Tailscale探测3000端口,同时经Nginx业务入口请求健康接口 | API仅监听`127.0.0.1:3000`,外部不能直连3000;Nginx入口仍正常返回真实API健康结果;部署静态门禁校验`API_HOST`默认值与启动参数一致 |
|
||||||
## Fail2ban 安全检测与人工封禁测试矩阵(2026-08-14)
|
## Fail2ban 安全检测与人工封禁测试矩阵(2026-08-14)
|
||||||
|
|
||||||
- 本模块必须执行 `docs/fail2ban-assisted-blocking-test-cases-20260814.md` 中 TC-F2B 全量用例,专项用例是本平台功能测试的组成部分,不是可选附录。
|
- 本模块必须执行 `docs/fail2ban-assisted-blocking-test-cases-20260814.md` 中 TC-F2B 全量用例,专项用例是本平台功能测试的组成部分,不是可选附录。
|
||||||
- P0 门禁至少覆盖:九类规则真实 PostgreSQL 默认值与版本冲突、阈值边界、规则应用失败保留旧生效值、登录/HTTP/CMPP/SSH/Nginx 真实事件脱敏、事件键幂等、窗口聚合并发、可信代理 IP、Cloudflare 与直连入口执行器映射、系统和人工保护网段、近期重新认证、重复封禁原子认领、代理超时/失败、真实执行器回读、非 root NestJS 及任意命令/参数注入拒绝。
|
- P0 门禁至少覆盖:九类规则真实 PostgreSQL 默认值与版本冲突、阈值边界、规则应用失败保留旧生效值、登录/HTTP/CMPP/SSH/Nginx 真实事件脱敏、事件键幂等、窗口聚合并发、可信代理 IP、Cloudflare 与直连入口执行器映射、系统和人工保护网段、近期重新认证、重复封禁原子认领、代理超时/失败、真实执行器回读、非 root NestJS 及任意命令/参数注入拒绝。
|
||||||
- 集成验收必须在隔离测试节点或网络 namespace 使用文档保留 IP;不得封禁预生产运维出口、Cloudflare 节点或真实客户 IP。未安装真实 Fail2ban/nftables/Nginx 资产时,只能把相关用例标记阻塞,不得用 Mock 通过代替。
|
- 集成验收必须在隔离测试节点或网络 namespace 使用文档保留 IP;不得封禁预生产运维出口、Cloudflare 节点或真实客户 IP。未安装真实 Fail2ban/nftables/Nginx 资产时,只能把相关用例标记阻塞,不得用 Mock 通过代替。
|
||||||
- UI 验收覆盖桌面与窄屏的总览、告警、规则、封禁记录、保护名单、加载、空数据、失败和规则未生效状态;所有数字与操作结果必须能从 API、数据库、agent 与执行器证据交叉验证。
|
- UI 验收覆盖桌面与窄屏的总览、告警、规则、封禁记录、保护名单、加载、空数据、失败和规则未生效状态;所有数字与操作结果必须能从 API、数据库、agent 与执行器证据交叉验证。
|
||||||
|
- `TC-F2B-OPS-008`:在非默认`APP_DIR`构建安全代理后执行安装器,核对systemd `ExecStart`与Fail2ban `actionban`均指向同一个真实可执行的`$APP_DIR/dist/cmpp-security-agent`;任一文件残留占位符、旧`current/bin`路径或目标不可执行时,安装/发布必须失败。
|
||||||
|
|||||||
@@ -3591,3 +3591,70 @@ git diff --check
|
|||||||
- Prisma schema validate、API与前端TypeScript、API全量40套/472项、Fail2ban专项和Gateway控制器3套/11项、Gateway全量`go test ./...`、Vite 8.1.5生产构建、3个Shell脚本语法和`git diff --check`通过;Vite仅有既有约2.11MiB单chunk提示,全量Jest仍使用`--forceExit`收尾既有异步句柄。
|
- Prisma schema validate、API与前端TypeScript、API全量40套/472项、Fail2ban专项和Gateway控制器3套/11项、Gateway全量`go test ./...`、Vite 8.1.5生产构建、3个Shell脚本语法和`git diff --check`通过;Vite仅有既有约2.11MiB单chunk提示,全量Jest仍使用`--forceExit`收尾既有异步句柄。
|
||||||
- 浏览器优先接管本地路由,`/admin/security-detection`在无有效Session时正确跳转运营端登录并保留返回地址,控制台error/warn为0;未读取、重置或猜测账号,登录后页面视觉与交互验收尚未完成。真实Fail2ban、Nginx、nftables、Unix Socket和第88条migration未在本机数据库或预生产安装/执行,必须在具备恢复资产和文档保留测试IP的授权发布窗口完成,当前不以Mock替代集成验收。
|
- 浏览器优先接管本地路由,`/admin/security-detection`在无有效Session时正确跳转运营端登录并保留返回地址,控制台error/warn为0;未读取、重置或猜测账号,登录后页面视觉与交互验收尚未完成。真实Fail2ban、Nginx、nftables、Unix Socket和第88条migration未在本机数据库或预生产安装/执行,必须在具备恢复资产和文档保留测试IP的授权发布窗口完成,当前不以Mock替代集成验收。
|
||||||
- 本轮未发送、补发或重投短信,未修改通道账号、密码、启停状态、企业余额、客户连接或预生产数据;`api/tsconfig.build.tsbuildinfo`、`tsconfig.tsbuildinfo`、`outputs/`和空文件`=`继续作为受保护项排除提交。
|
- 本轮未发送、补发或重投短信,未修改通道账号、密码、启停状态、企业余额、客户连接或预生产数据;`api/tsconfig.build.tsbuildinfo`、`tsconfig.tsbuildinfo`、`outputs/`和空文件`=`继续作为受保护项排除提交。
|
||||||
|
|
||||||
|
# 2026-08-14 Prometheus 与 Fail2ban 两次提交本地部署复核(未推送、未发布)
|
||||||
|
|
||||||
|
- 从侧边会话已经落到本地 `main` 的两个提交开始复核:`b78faa1aa24a89830892b511a6cb17cc5a5fbe68`(Prometheus 系统监控)和 `d30d9ea4d0ec8a46154dfb1e39a9bba2156ec455`(Fail2ban 安全检测)。复核时本地 `HEAD=d30d9ea`,`origin/main=96e475d`,本地领先2个提交;本轮没有再次提交、推送或发布。
|
||||||
|
- migration 前已将本地真实 PostgreSQL 备份到 `C:\cmpp-platform-local\backups\cmpp-platform-before-d30d9ea-20260814-111254.dump`,637967字节,SHA-256=`523ecc82b55b5575ebe78fc4d253c5ec44932a76a51209130442f619e1971089`。Prisma generate、validate、migrate deploy/status均通过;`20260814150000_add_security_detection`真实应用一次,本地数据库由87条升级为88/88条migration,九类默认安全规则均存在且配置版本为1。
|
||||||
|
- 本地API以正式构建产物运行在3000端口,前端Vite生产预览运行在4173端口,API health与前端HTTP均为200。为避免本地Gateway连接真实供应商,本轮只编译和测试Gateway,没有启动8090;因此API日志中的活动通道恢复和供应商状态对账失败是预期的本地Gateway离线结果,未修改任何通道参数,也未触发短信提交。
|
||||||
|
- API全量40套/472项、API TypeScript正式构建、前端TypeScript与Vite 8.1.5生产构建、Gateway `go test ./... -count=1`及`go vet ./...`全部通过;Vite仅保留既有大chunk提示。Prometheus、Fail2ban、nftables和Linux systemd在Windows本机不可用,三个部署Shell及标准发布/初始化Shell均通过Bash语法检查,但不以语法检查冒充Linux真实安装验收。
|
||||||
|
- 正式服务层连接真实本地PostgreSQL和真实不可达的 `127.0.0.1:9090` 验证:监控概览返回 `available=false`,所有指标为`null`、趋势为空,非法范围`30d`返回400;安全检测写入保留测试地址`203.0.113.10`的一条 `http_invalid_api_key` 事件,证据中的访问密钥已存为`[REDACTED]`,同一`eventKey=local-qa-d30d9ea-http-invalid-api-key`第二次上报命中幂等去重且未达到告警阈值。该真实本地测试事件保留作审计证据,没有创建封禁或调用安全代理。
|
||||||
|
- 未认证HTTP请求访问监控总览、安全总览和规则列表均返回401。经用户明确授权,只临时替换本地专用 `codex_local_admin` 的密码哈希完成图形验证码登录;登录后立即恢复原密码哈希、失败计数和锁定时间,未新建账号、未变更session版本。浏览器桌面验收确认监控页切换到近1小时并刷新后仍明确显示Prometheus不可用且没有Mock/缓存数值;安全页显示24小时事件1条、安全代理不可用,并从真实数据库展示9条规则及`1/1`版本。390×844窄屏下两页核心内容和交互可用,控制台warning/error为0;安全页五个页签在窄屏中文字换行偏碎,记为非阻断视觉问题。
|
||||||
|
- 19份既有结构门禁中11份通过、8份失败;失败集中在后台重投/异常处置API、通道/发送质量哈希、报备导出、运营商集合、定时调度、全局长文本样式和签名查询等既有契约漂移,与这两个提交新增文件无交集,本轮不顺带改写其他模块契约。`git diff --check`通过。
|
||||||
|
- 本轮没有发送、补发或重投真实短信,没有修改真实通道账号、密码、启停状态、企业余额或客户连接。既有`api/tsconfig.build.tsbuildinfo`、`tsconfig.tsbuildinfo`、`outputs/`和空文件`=`继续保护;本段测试记录是本轮新增的唯一业务工作区修改。
|
||||||
|
|
||||||
|
# 2026-08-14 Prometheus 与 Fail2ban 本地服务器部署及真实集成验收
|
||||||
|
|
||||||
|
- 经用户明确授权,将本地`HEAD=d30d9ea4d0ec8a46154dfb1e39a9bba2156ec455`及当前工作区的部署修复安装到全新Ubuntu 24.04测试节点`100.93.204.60`;未把该节点当作预生产。部署前只读盘点确认4核、7.8GiB内存、根盘约98GiB且无既有平台目录/数据库/环境文件,恢复基线位于`/opt/cmpp-platform-backups/releases/20260814-135221-before-d30d9ea-localserver`,API回环修复前增量备份位于`/opt/cmpp-platform-backups/releases/20260814-144709-before-api-loopback`。
|
||||||
|
- 基础源码归档SHA-256=`c9e0b4ba5895b0d434a3335dcc6044f291779337841bb19b636bbed5d91b1b96`,首轮修复overlay=`24c5720dd0da6fab1b745d3c976cfc70d1a2f1b49da13443157a0ae89eac1bad`,API回环修复overlay=`3d6cdd45a87e12fcad50a10d18862a1527c6a36fae2b18391142c0fba61e717e`;运行标识写为`d30d9ea4d0ec8a46154dfb1e39a9bba2156ec455+localfix.3d6cdd45a87e`。生成的本地管理员凭据以0600权限保存在服务器`/home/hector/cmpp-platform-admin.txt`,未写入仓库或测试记录。
|
||||||
|
- 已安装Node 22.21.1、Go 1.26.0、PostgreSQL 16.14、Redis 7.0.15、Nginx 1.24、Fail2ban 1.0.2、Prometheus 2.45.3及Node Exporter 1.7。由于该LAN的DNS/代理返回不可路由的fake-IP,安装期间备份原DNS、hosts和APT源后改用清华Ubuntu镜像并为必要下载域名写入临时hosts固定解析;这些固定解析仅为安装绕行,网络恢复后应按恢复资产移除。首次部署因MinIO官方二进制跳转GitHub后不可达,依照部署文档临时使用真实服务器文件存储驱动`local`,没有用Mock或localStorage伪造对象存储;后续已按用户提供的MinIO二进制完成正式切换,证据见下方补充记录。
|
||||||
|
- 第88条migration真实应用,Prisma报告88 migrations且schema up to date;`SecurityDetectionRule`真实9条,`configVersion/effectiveVersion`均为1。该全新节点的通道、短信记录、下游重投任务和安全封禁记录均为0,未连接真实供应商或客户。
|
||||||
|
- 真实Linux集成检查通过:API、Gateway、安全代理、PostgreSQL、Redis、Nginx、Prometheus、Node Exporter及Fail2ban均active;API/Gateway/Prometheus健康、Redis PONG、PostgreSQL ready、Nginx语法和Fail2ban配置通过。Prometheus两个target均为`up`,配置有效且告警文件12条规则通过`promtool`;9090/9100仅监听127.0.0.1。安全代理Unix Socket为0660 root:cmpp-security,NestJS用户`cmpp-api`无sudo,systemd与Fail2ban action共同指向`/opt/cmpp-platform/dist/cmpp-security-agent`,nftables专用IPv4/IPv6 timeout set存在,Fail2ban sshd jail运行。
|
||||||
|
- 部署中发现并修复三项真实Fresh-Install缺陷:安全代理systemd与Fail2ban action旧路径漂移;Ubuntu默认gzip与发布脚本重复声明导致`nginx -t`失败;NestJS文档要求回环但代码默认监听全网卡。前两项由安装/发布静态门禁保护,第三项新增`API_HOST`并默认127.0.0.1。修复后服务器`ss`确认3000/5432/6379/8090/9090/9100均为回环,外部探测12026和17890可连、3000/9090/9100不可连,Nginx入口`/api/health`返回200。专项部署门禁、API TypeScript正式构建、Shell语法和`git diff --check`通过。
|
||||||
|
- 内置浏览器两次导航该Tailscale地址均在页面加载阶段超时;用户随后明确要求改用系统Chrome,Chrome控制扩展同样在新建页导航阶段超时,而同机PowerShell对同URL返回HTTP 200,判定为浏览器控制链路到Tailscale HTTP地址的环境阻塞。两次尝试均未到验证码或登录提交;管理员密码哈希、失败次数和锁定时间已从临时数据库备份恢复,临时备份表已删除。未绕过图形验证码,也未把登录后桌面/窄屏视觉验收伪报为通过。页面构建与真实后端/基础设施证据已通过,登录后视觉及范围切换仍需用户在本机Chrome手动打开页面,或共享已打开的具体标签页后补验。
|
||||||
|
- 本轮没有发送、补发、重投短信或创建重投任务,没有修改真实通道账号、密码、启停状态、企业余额或客户连接。工作区修复与文档尚未提交、推送;`api/tsconfig.build.tsbuildinfo`、`tsconfig.tsbuildinfo`、`outputs/`及空文件`=`继续作为受保护项,不删除、不提交、不归因。
|
||||||
|
|
||||||
|
## MinIO 正式切换补充(2026-08-14)
|
||||||
|
|
||||||
|
- 用户提供`D:\迅雷下载\minio.linux-amd64.RELEASE.2025-09-07T16-13-09Z`,本地与服务器SHA-256均为`7c5bd8512c6e966455b1d198209358b2d191c77a83ab377c4073281065fb855f`;服务器`file`确认其为静态链接Linux x86-64 ELF,运行版本为`RELEASE.2025-09-07T16-13-09Z`、Go 1.24.6。
|
||||||
|
- 切换前确认`/var/lib/cmpp-platform/object-storage`文件数为0,并将环境、MinIO环境、systemd单元和本地对象存储目录备份到`/opt/cmpp-platform-backups/releases/20260814-151145-before-minio`。安装后`cmpp-minio`与API均active,API使用`OBJECT_STORAGE_DRIVER=minio`;9000/9001只监听127.0.0.1/::1,外部探测均不可连接。
|
||||||
|
- 使用API同款MinIO Node客户端真实创建`cmpp-platform` bucket,并执行测试对象写入、读取内容比对和删除,三步均成功且测试对象已清理。API健康、Gateway、Prometheus、Node Exporter及Fail2ban继续active;未创建业务附件记录、短信、重投任务或通道连接。
|
||||||
|
|
||||||
|
# 2026-08-14 系统监控恢复、标题去重与全局预警菜单(测试服务器已部署)
|
||||||
|
|
||||||
|
- 用户在`100.93.204.60`真实页面看到系统监控整体降级。只读诊断确认Prometheus、Node Exporter、API、MinIO均active,两个采集target均为`up`,CPU、内存、磁盘、网络、负载、运行时长和systemd查询单独执行都成功;直接运行实际`InfrastructureMonitoringService`后捕获到systemd查询HTTP 400,Prometheus明确返回`unknown escape sequence '.'`。根因是TypeScript字符串只向PromQL传递单反斜杠`\.`,而Prometheus字符串层要求双反斜杠后再交给RE2。
|
||||||
|
- systemd固定PromQL改为TypeScript四反斜杠字面量,实际查询文本正确传递双反斜杠;专项测试锁定请求参数。整页降级行为继续清空陈旧数据,但新增仅含错误摘要的服务端warning,不记录PromQL、地址或凭据。部署后真实服务返回`available=true`、综合状态`healthy`、核心服务6/6、1小时CPU趋势56点,API发布后无新增监控不可用warning。
|
||||||
|
- 系统监控内容区删除重复的大号`<h1>系统监控</h1>`,保留平台通用页头、说明、状态、时间范围和刷新操作。构建产物与部署源码静态核对确认重复标题不存在。
|
||||||
|
- 右上角铃铛由签名清退直接链接改为“预警中心”弹层,分开显示“签名清退预警”和“安全检测与封禁”;审核待办继续使用独立图标和菜单。签名项读取今日未读且未抑制消息数,安全项新增轻量`GET /api/admin/security-detection/notification-summary`,只统计`open/acknowledged/block_failed`总数与严重数,不轮询完整总览、规则或安全代理。任一域失败由`Promise.allSettled`独立降级,不清空另一域。
|
||||||
|
- 本地监控/安全专项2套8项、API全量40套473项、前后端TypeScript和Vite生产构建全部通过;全量Jest仍因既有异步句柄使用`--forceExit`收尾,Vite只保留既有大chunk提示,`git diff --check`通过。
|
||||||
|
- 发布归档`outputs/cmpp-monitor-alert-fix-20260814-152652.tar.gz`及服务器副本SHA-256均为`8a00a02714ced5311b25fbc3e1cfc17f3d1a38759f1499db78c52089bc0b60b7`。发布前PostgreSQL、环境和运行源码恢复资产位于`/opt/cmpp-platform-backups/releases/20260814-152719-before-monitor-alert-fix`,三项SHA校验、源码tar目录和pg_restore清单均通过;运行标识为`d30d9ea4d0ec8a46154dfb1e39a9bba2156ec455+localfix.monitor-alert.8a00a02714ce`。
|
||||||
|
- 经用户明确要求,在该空白测试服务器真实PostgreSQL创建可清理的`qa-bell-alerts-*`预警验收数据:2条签名清退未读消息,关联一个`interfaceEnabled=false/status=disabled`的QA应用和两条QA签名;3条安全告警通过真实内部安全事件接口按规则阈值触发,使用文档保留IP`203.0.113.101-103`,其中critical 1条。铃铛真实汇总为2+3=5;封禁记录0、短信记录0,没有调用安全代理封禁、Gateway提交或供应商连接。
|
||||||
|
- Chrome中已存在登录后的测试服务器页面,但浏览器控制扩展在接管该标签页阶段持续超时,因此未伪报点击和响应式视觉验收通过。服务器真实服务、数据库、接口服务层、构建产物和静态契约均已验收;用户刷新页面即可查看,后续以用户截图继续视觉核对。
|
||||||
|
- 本轮没有发送、补发或重投短信,没有修改真实通道账号、密码、余额或客户连接。代码和文档尚未提交、推送;受保护的`*.tsbuildinfo`、`outputs/`及空文件`=`继续保留,不归因或提交。
|
||||||
|
|
||||||
|
# 2026-08-14 后台重投支持客户端已确认记录与创建/列表样式优化(本地未部署)
|
||||||
|
|
||||||
|
- 后台任务可重投状态由`pending/failed/unconfirmed/rejected`扩展为`pending/failed/unconfirmed/rejected/delivered`;状态筛选为`delivered`时,预检将客户端已确认记录计入可重投数并在真实PostgreSQL任务项保存`previousStatus=delivered`。`awaiting_ack`继续由预检计为不可重投且创建接口明确拒绝,避免确认窗口内并发写出。
|
||||||
|
- 执行器不是简单放开当前`delivered`状态:只有冻结快照原状态已经是`delivered`的任务项才允许再次调用Gateway;任务创建时为失败/待投递等状态、执行前才收到迟到成功ACK的项目会以“创建任务后已被客户确认”跳过。该判断用于保留运营明确选择已确认记录时的重复投递能力,同时防止其他任务范围意外扩大。
|
||||||
|
- 创建弹窗将原生无统一样式的`textarea`替换为平台`Textarea`,增加必填标识、少于5字错误、说明、200字计数、统一焦点态和重复投递风险提示。后台任务列表使用独立内容边框和圆角,桌面卡片边缘保留24px、行内保留20px,窄屏收敛为16px;标题区和分页同步使用设计间距。
|
||||||
|
- 已同步`docs/downstream-requeue-task-design-20260812.md` V1.1、平台需求和系统测试用例,新增已确认批量重投、迟到ACK保护及桌面/窄屏视觉用例。专项Jest 10/10、API全量41套/477项、前后端TypeScript、API正式编译、Vite 8.1.5生产构建(2549 modules)、SendChain R10结构契约和`git diff --check`通过;Vite仅保留既有约2.11MiB单chunk提示。Operations R2结构门禁仍因本轮开始前已有的`sendQuality`查询哈希漂移失败,与本次下游重投文件无交集,未为通过门禁改写其他会话业务契约。
|
||||||
|
- 本地真实PostgreSQL为88/88条migration且schema up to date。使用现有失败投递在单一数据库事务中临时改为`delivered`,真实验证状态白名单命中1条并成功物化`previousStatus=delivered`任务项1条,随后强制回滚;事务后原投递恢复`failed`且测试任务持久化数为0。该验证没有启动任务扫描、调用Gateway或发送短信。
|
||||||
|
- 本轮尚未部署测试服务器或预生产,因当前需求只授权修改且交接约束要求部署另行明确授权;没有创建真实重投任务、发送/补发/重投短信、修改通道账号、密码、启停状态、企业余额或客户连接。既有监控/预警工作区修改及`*.tsbuildinfo`、`outputs/`、空文件`=`继续保护,不归因于本次改动。
|
||||||
|
|
||||||
|
# 2026-08-14 服务内部Prometheus指标、阈值与测试机部署
|
||||||
|
|
||||||
|
- 在现有工作区上增量实施,未reset/checkout或覆盖其他会话改动。API新增独立`127.0.0.1:9464/metrics`,仅使用路由模板、HTTP方法和状态的低基数标签;Gateway回环`8090/metrics`新增Go运行时、上下游连接、Submit结果/耗时及Redis Stream pending/lag/最旧年龄。两者均只在内存计数,不写业务PostgreSQL或Redis。
|
||||||
|
- 测试机`100.93.204.60`安装Ubuntu发行版`prometheus-postgres-exporter 0.15.0`、`prometheus-redis-exporter 1.54.0`和`prometheus-nginx-exporter 1.1.0`,开启MinIO回环原生指标;Prometheus现实际采集`prometheus/node/cmpp-api/cmpp-gateway/postgresql/redis/minio/nginx`8个target,全部`up`。
|
||||||
|
- 规则扩展至61条,覆盖主机资源、API 5xx/P95/事件循环、Gateway worker/连接/队列年龄、PostgreSQL连接/死锁、Redis内存/淘汰/拒绝连接、MinIO容量/离线盘、Nginx可用性和Prometheus自监控。比例告警有最低错误样本量,Warning/Critical范围不重叠;`promtool check rules/config`通过,8个规则组计算失败计数全为0,部署后无活动告警。
|
||||||
|
- Recording Rules真实返回API P95约0.048s、Gateway pending/lag均0、PostgreSQL连接使用率3%、Redis连接10/已用约1.78MB、MinIO容量使用率约20.8%/离线盘0、Nginx活跃连接4。系统监控页新增六组“服务关键指标”卡片,只读取`cmpp:service_*`聚合;缺失指标显示破折号/待采集,不以0伪造。
|
||||||
|
- 外部真实TCP探测确认仅业务端口12026可连接;3000、8090、9000、9090、9100、9113、9121、9187、9464全部从LAN不可连接。各监控进程当时CPU合计约0.4%,Prometheus RSS约101MB、三个新Exporter RSS合计约57MB;各8个target抓取耗时0.0008至0.037s,明显低于15s周期,TSDB当时5790条series。
|
||||||
|
- 本地API专项2套5项、API全量41套/477项、API/前端TypeScript、Gateway全量`go test ./... -count=1`和`go vet ./...`通过;测试机Vite 8.0.16生产构建、API/Gateway构建和健康检查通过,仅保留既有大chunk提示。API/Gateway/Prometheus发布后warning级journal为0。
|
||||||
|
- 完整恢复资产位于`/opt/cmpp-platform-backups/releases/20260814-164653-before-service-metrics`,包含PostgreSQL、运行源码、环境/监控/Nginx/systemd配置,SHA-256和gzip校验通过。`20260814-164636-before-service-metrics`是因`pg_dump`不接受Prisma `schema` URL参数而立即停止的不完整目录,不可用于恢复;未删除以保留证据。当前运行标识为`d30d9ea4d0ec8a46154dfb1e39a9bba2156ec455+workspace.service-metrics.28575bc8a7d8.minio`。
|
||||||
|
- Edge现有测试机页面会话未登录,访问系统监控被正常引导到带图形验证码的登录页;未代填或绕过验证码,因此登录后页面视觉验收留待用户使用已有测试账号查看。本轮未发送、补发或重投短信,未修改通道账号/密码/启停、余额或客户连接;代码未提交、未推送、未发布预生产。
|
||||||
|
|
||||||
|
# 2026-08-14 跨会话工作区合并复核
|
||||||
|
|
||||||
|
- 合并复核覆盖系统监控恢复与预警菜单、服务内部Prometheus指标/Exporter/阈值、Fail2ban部署路径修复,以及后台重投支持客户端已确认记录和创建/列表样式优化。Git不存在未合并文件或冲突标记;共同修改的API模块、全局布局/样式、部署脚本、平台需求、系统用例和进度记录均已逐项核对,没有发现状态口径、路由、样式选择器或部署时间线互相覆盖。
|
||||||
|
- 测试机静态包只读核对显示当前已部署“服务关键指标”,但尚未包含“重复投递风险”和已确认重投的新表单文案,证明服务指标会话使用定向发布,没有把尚未授权部署的下游重投改动意外带入测试机;两项发布记录保持一致。
|
||||||
|
- 合并后统一回归通过:API全量41套/477项、前后端TypeScript、API正式编译、Vite 8.1.5生产构建(2549 modules)、Gateway全量`go test ./... -count=1`与`go vet ./...`、SendChain R10、生产部署和安全代理静态契约、5个Shell脚本语法、真实本地PostgreSQL 88/88 migration状态及`git diff --check`。Vite仅保留既有约2.11MiB单chunk提示;Operations R2仍因本轮开始前已有的`sendQuality`查询哈希漂移失败,与本次合并文件无交集。
|
||||||
|
- 本次合并没有发送、补发或重投短信,没有创建后台重投任务,没有修改通道账号、密码、启停状态、企业余额或客户连接;提交范围继续排除`api/tsconfig.build.tsbuildinfo`、`tsconfig.tsbuildinfo`、`outputs/`和空文件`=`。
|
||||||
|
|||||||
@@ -5,13 +5,19 @@ import (
|
|||||||
"log"
|
"log"
|
||||||
"net/http"
|
"net/http"
|
||||||
"os"
|
"os"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
"cmpp-platform/gateway/internal/control"
|
"cmpp-platform/gateway/internal/control"
|
||||||
"cmpp-platform/gateway/internal/health"
|
"cmpp-platform/gateway/internal/health"
|
||||||
"cmpp-platform/gateway/internal/inbound"
|
"cmpp-platform/gateway/internal/inbound"
|
||||||
|
platformmetrics "cmpp-platform/gateway/internal/metrics"
|
||||||
"cmpp-platform/gateway/internal/ratelimit"
|
"cmpp-platform/gateway/internal/ratelimit"
|
||||||
"cmpp-platform/gateway/internal/submitworker"
|
"cmpp-platform/gateway/internal/submitworker"
|
||||||
"cmpp-platform/gateway/internal/upstream"
|
"cmpp-platform/gateway/internal/upstream"
|
||||||
|
|
||||||
|
"github.com/redis/go-redis/v9"
|
||||||
)
|
)
|
||||||
|
|
||||||
func main() {
|
func main() {
|
||||||
@@ -25,6 +31,7 @@ func main() {
|
|||||||
}
|
}
|
||||||
apiBaseURL := os.Getenv("API_BASE_URL")
|
apiBaseURL := os.Getenv("API_BASE_URL")
|
||||||
upstreamManager := &upstream.Manager{APIBaseURL: apiBaseURL}
|
upstreamManager := &upstream.Manager{APIBaseURL: apiBaseURL}
|
||||||
|
var worker *submitworker.Worker
|
||||||
channelLimiter, err := ratelimit.New(os.Getenv("REDIS_URL"))
|
channelLimiter, err := ratelimit.New(os.Getenv("REDIS_URL"))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Fatalf("gateway channel rate limiter init failed: %v", err)
|
log.Fatalf("gateway channel rate limiter init failed: %v", err)
|
||||||
@@ -53,7 +60,7 @@ func main() {
|
|||||||
}()
|
}()
|
||||||
|
|
||||||
if os.Getenv("GATEWAY_SUBMIT_WORKER_DISABLED") != "true" {
|
if os.Getenv("GATEWAY_SUBMIT_WORKER_DISABLED") != "true" {
|
||||||
worker, err := submitworker.New(os.Getenv("REDIS_URL"), upstreamManager)
|
worker, err = submitworker.New(os.Getenv("REDIS_URL"), upstreamManager)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Printf("gateway submit worker init failed: %v", err)
|
log.Printf("gateway submit worker init failed: %v", err)
|
||||||
} else {
|
} else {
|
||||||
@@ -72,6 +79,39 @@ func main() {
|
|||||||
|
|
||||||
mux := http.NewServeMux()
|
mux := http.NewServeMux()
|
||||||
mux.Handle("/health", health.Handler())
|
mux.Handle("/health", health.Handler())
|
||||||
|
mux.Handle("/metrics", platformmetrics.Handler(func(ctx context.Context) platformmetrics.Snapshot {
|
||||||
|
desired, connected := upstreamManager.ConnectionCounts()
|
||||||
|
snapshot := platformmetrics.Snapshot{
|
||||||
|
UpstreamDesired: desired, UpstreamConnected: connected,
|
||||||
|
DownstreamConnected: inbound.ActiveConnectionCount(), SubmitWorkerUp: worker != nil,
|
||||||
|
}
|
||||||
|
if worker == nil || worker.Redis == nil {
|
||||||
|
return snapshot
|
||||||
|
}
|
||||||
|
pending, err := worker.Redis.XPending(ctx, worker.Stream, worker.Group).Result()
|
||||||
|
if err != nil {
|
||||||
|
return snapshot
|
||||||
|
}
|
||||||
|
snapshot.QueueAvailable = true
|
||||||
|
snapshot.QueuePending = pending.Count
|
||||||
|
groups, err := worker.Redis.XInfoGroups(ctx, worker.Stream).Result()
|
||||||
|
if err == nil {
|
||||||
|
for _, group := range groups {
|
||||||
|
if group.Name == worker.Group {
|
||||||
|
snapshot.QueueLag = group.Lag
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
entries, err := worker.Redis.XPendingExt(ctx, &redis.XPendingExtArgs{Stream: worker.Stream, Group: worker.Group, Start: "-", End: "+", Count: 1}).Result()
|
||||||
|
if err == nil && len(entries) > 0 {
|
||||||
|
milliseconds, parseErr := strconv.ParseInt(strings.SplitN(entries[0].ID, "-", 2)[0], 10, 64)
|
||||||
|
if parseErr == nil {
|
||||||
|
snapshot.QueueOldestAgeSeconds = max(0, time.Since(time.UnixMilli(milliseconds)).Seconds())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return snapshot
|
||||||
|
}))
|
||||||
control.Register(mux, control.Server{
|
control.Register(mux, control.Server{
|
||||||
APIBaseURL: apiBaseURL,
|
APIBaseURL: apiBaseURL,
|
||||||
Upstream: upstreamManager,
|
Upstream: upstreamManager,
|
||||||
|
|||||||
@@ -201,6 +201,13 @@ func onlineAccounts() []string {
|
|||||||
return accounts
|
return accounts
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ActiveConnectionCount exposes only an aggregate gauge; account and remote-IP labels are intentionally excluded.
|
||||||
|
func ActiveConnectionCount() int {
|
||||||
|
downstreamRegistry.RLock()
|
||||||
|
defer downstreamRegistry.RUnlock()
|
||||||
|
return len(downstreamRegistry.byConn)
|
||||||
|
}
|
||||||
|
|
||||||
// DisconnectAccount closes every live downstream CMPP session for an
|
// DisconnectAccount closes every live downstream CMPP session for an
|
||||||
// application account. The normal connection-close callback removes registry
|
// application account. The normal connection-close callback removes registry
|
||||||
// and presence state and reports the disconnect to the API.
|
// and presence state and reports the disconnect to the API.
|
||||||
|
|||||||
@@ -0,0 +1,80 @@
|
|||||||
|
package metrics
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"net/http"
|
||||||
|
"runtime"
|
||||||
|
"sync/atomic"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
var startedAt = time.Now()
|
||||||
|
var submitAccepted atomic.Uint64
|
||||||
|
var submitFailed atomic.Uint64
|
||||||
|
var submitDurationNanoseconds atomic.Uint64
|
||||||
|
|
||||||
|
type Snapshot struct {
|
||||||
|
UpstreamDesired int
|
||||||
|
UpstreamConnected int
|
||||||
|
DownstreamConnected int
|
||||||
|
SubmitWorkerUp bool
|
||||||
|
QueueAvailable bool
|
||||||
|
QueuePending int64
|
||||||
|
QueueLag int64
|
||||||
|
QueueOldestAgeSeconds float64
|
||||||
|
}
|
||||||
|
|
||||||
|
type SnapshotFunc func(context.Context) Snapshot
|
||||||
|
|
||||||
|
func ObserveSubmit(accepted bool, duration time.Duration) {
|
||||||
|
if accepted {
|
||||||
|
submitAccepted.Add(1)
|
||||||
|
} else {
|
||||||
|
submitFailed.Add(1)
|
||||||
|
}
|
||||||
|
submitDurationNanoseconds.Add(uint64(max(duration, 0)))
|
||||||
|
}
|
||||||
|
|
||||||
|
func Handler(load SnapshotFunc) http.Handler {
|
||||||
|
return http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) {
|
||||||
|
if request.Method != http.MethodGet || request.URL.Path != "/metrics" {
|
||||||
|
response.WriteHeader(http.StatusNotFound)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
ctx, cancel := context.WithTimeout(request.Context(), time.Second)
|
||||||
|
defer cancel()
|
||||||
|
snapshot := Snapshot{}
|
||||||
|
if load != nil {
|
||||||
|
snapshot = load(ctx)
|
||||||
|
}
|
||||||
|
var memory runtime.MemStats
|
||||||
|
runtime.ReadMemStats(&memory)
|
||||||
|
accepted := submitAccepted.Load()
|
||||||
|
failed := submitFailed.Load()
|
||||||
|
count := accepted + failed
|
||||||
|
response.Header().Set("Content-Type", "text/plain; version=0.0.4; charset=utf-8")
|
||||||
|
response.Header().Set("Cache-Control", "no-store")
|
||||||
|
fmt.Fprintf(response, "# HELP cmpp_gateway_process_uptime_seconds Gateway process uptime.\n# TYPE cmpp_gateway_process_uptime_seconds gauge\ncmpp_gateway_process_uptime_seconds %f\n", time.Since(startedAt).Seconds())
|
||||||
|
fmt.Fprintf(response, "# HELP cmpp_gateway_go_goroutines Current goroutine count.\n# TYPE cmpp_gateway_go_goroutines gauge\ncmpp_gateway_go_goroutines %d\n", runtime.NumGoroutine())
|
||||||
|
fmt.Fprintf(response, "# HELP cmpp_gateway_go_heap_alloc_bytes Current Go heap allocation.\n# TYPE cmpp_gateway_go_heap_alloc_bytes gauge\ncmpp_gateway_go_heap_alloc_bytes %d\n", memory.HeapAlloc)
|
||||||
|
fmt.Fprintf(response, "# HELP cmpp_gateway_submit_total Upstream submit attempts by bounded result.\n# TYPE cmpp_gateway_submit_total counter\ncmpp_gateway_submit_total{result=\"accepted\"} %d\ncmpp_gateway_submit_total{result=\"failed\"} %d\n", accepted, failed)
|
||||||
|
fmt.Fprintf(response, "# HELP cmpp_gateway_submit_duration_seconds_sum Total upstream submit duration.\n# TYPE cmpp_gateway_submit_duration_seconds_sum counter\ncmpp_gateway_submit_duration_seconds_sum %f\n", float64(submitDurationNanoseconds.Load())/float64(time.Second))
|
||||||
|
fmt.Fprintf(response, "# HELP cmpp_gateway_submit_duration_seconds_count Total measured upstream submits.\n# TYPE cmpp_gateway_submit_duration_seconds_count counter\ncmpp_gateway_submit_duration_seconds_count %d\n", count)
|
||||||
|
fmt.Fprintf(response, "# HELP cmpp_gateway_upstream_connections Desired and live supplier connections.\n# TYPE cmpp_gateway_upstream_connections gauge\ncmpp_gateway_upstream_connections{state=\"desired\"} %d\ncmpp_gateway_upstream_connections{state=\"connected\"} %d\n", snapshot.UpstreamDesired, snapshot.UpstreamConnected)
|
||||||
|
fmt.Fprintf(response, "# HELP cmpp_gateway_downstream_connections Authenticated client connections.\n# TYPE cmpp_gateway_downstream_connections gauge\ncmpp_gateway_downstream_connections %d\n", snapshot.DownstreamConnected)
|
||||||
|
fmt.Fprintf(response, "# HELP cmpp_gateway_submit_worker_up Whether the submit worker was initialized.\n# TYPE cmpp_gateway_submit_worker_up gauge\ncmpp_gateway_submit_worker_up %d\n", boolNumber(snapshot.SubmitWorkerUp))
|
||||||
|
if snapshot.QueueAvailable {
|
||||||
|
fmt.Fprintf(response, "# HELP cmpp_gateway_submit_queue_pending Pending entries owned by the consumer group.\n# TYPE cmpp_gateway_submit_queue_pending gauge\ncmpp_gateway_submit_queue_pending %d\n", snapshot.QueuePending)
|
||||||
|
fmt.Fprintf(response, "# HELP cmpp_gateway_submit_queue_lag Undelivered entries for the consumer group.\n# TYPE cmpp_gateway_submit_queue_lag gauge\ncmpp_gateway_submit_queue_lag %d\n", snapshot.QueueLag)
|
||||||
|
fmt.Fprintf(response, "# HELP cmpp_gateway_submit_queue_oldest_pending_age_seconds Age of the oldest pending entry.\n# TYPE cmpp_gateway_submit_queue_oldest_pending_age_seconds gauge\ncmpp_gateway_submit_queue_oldest_pending_age_seconds %f\n", snapshot.QueueOldestAgeSeconds)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func boolNumber(value bool) int {
|
||||||
|
if value {
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
return 0
|
||||||
|
}
|
||||||
@@ -0,0 +1,29 @@
|
|||||||
|
package metrics
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestMetricsHandlerExportsOnlyAggregateGatewayState(t *testing.T) {
|
||||||
|
ObserveSubmit(true, 20*time.Millisecond)
|
||||||
|
request := httptest.NewRequest(http.MethodGet, "/metrics", nil)
|
||||||
|
response := httptest.NewRecorder()
|
||||||
|
Handler(func(context.Context) Snapshot {
|
||||||
|
return Snapshot{UpstreamDesired: 2, UpstreamConnected: 1, DownstreamConnected: 3, SubmitWorkerUp: true, QueueAvailable: true, QueuePending: 4, QueueLag: 5, QueueOldestAgeSeconds: 12}
|
||||||
|
}).ServeHTTP(response, request)
|
||||||
|
|
||||||
|
body := response.Body.String()
|
||||||
|
if response.Code != http.StatusOK || !strings.Contains(body, "cmpp_gateway_submit_queue_pending 4") || !strings.Contains(body, "cmpp_gateway_upstream_connections{state=\"connected\"} 1") {
|
||||||
|
t.Fatalf("unexpected metrics response: code=%d body=%s", response.Code, body)
|
||||||
|
}
|
||||||
|
for _, forbidden := range []string{"phone_number", "message_id", "channel_id"} {
|
||||||
|
if strings.Contains(body, forbidden) {
|
||||||
|
t.Fatalf("metrics expose forbidden label %q", forbidden)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -12,6 +12,7 @@ import (
|
|||||||
"sync"
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"cmpp-platform/gateway/internal/metrics"
|
||||||
"cmpp-platform/gateway/internal/queue"
|
"cmpp-platform/gateway/internal/queue"
|
||||||
"cmpp-platform/gateway/internal/ratelimit"
|
"cmpp-platform/gateway/internal/ratelimit"
|
||||||
"cmpp-platform/gateway/internal/upstream"
|
"cmpp-platform/gateway/internal/upstream"
|
||||||
@@ -206,6 +207,7 @@ func (w *Worker) processMessage(ctx context.Context, message redis.XMessage) err
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (w *Worker) handleCommand(ctx context.Context, command queue.SubmitCommand) error {
|
func (w *Worker) handleCommand(ctx context.Context, command queue.SubmitCommand) error {
|
||||||
|
startedAt := time.Now()
|
||||||
if w.Limiter != nil {
|
if w.Limiter != nil {
|
||||||
if _, err := w.Limiter.Wait(ctx, command.ChannelID, command.Route.RateLimitPerSecond); err != nil {
|
if _, err := w.Limiter.Wait(ctx, command.ChannelID, command.Route.RateLimitPerSecond); err != nil {
|
||||||
return err
|
return err
|
||||||
@@ -219,6 +221,8 @@ func (w *Worker) handleCommand(ctx context.Context, command queue.SubmitCommand)
|
|||||||
submit = w.Upstream.Submit
|
submit = w.Upstream.Submit
|
||||||
}
|
}
|
||||||
result, err := submit(ctx, command)
|
result, err := submit(ctx, command)
|
||||||
|
accepted := err == nil && (result.SubmitStatus == "" || result.SubmitStatus == "accepted")
|
||||||
|
metrics.ObserveSubmit(accepted, time.Since(startedAt))
|
||||||
if err != nil && (result.SubmitStatus == "" || result.SubmitStatus == "accepted") {
|
if err != nil && (result.SubmitStatus == "" || result.SubmitStatus == "accepted") {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -146,6 +146,21 @@ func (m *Manager) ensureDefaultsLocked() {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ConnectionCounts returns bounded platform totals without channel identifiers to prevent time-series cardinality growth.
|
||||||
|
func (m *Manager) ConnectionCounts() (desired int, connected int) {
|
||||||
|
m.mu.Lock()
|
||||||
|
pools := make([]*connectionPool, 0, len(m.conns))
|
||||||
|
for _, pool := range m.conns {
|
||||||
|
pools = append(pools, pool)
|
||||||
|
}
|
||||||
|
m.mu.Unlock()
|
||||||
|
for _, pool := range pools {
|
||||||
|
desired += max(pool.config.DesiredConnections, 1)
|
||||||
|
connected += pool.countActiveConnections()
|
||||||
|
}
|
||||||
|
return desired, connected
|
||||||
|
}
|
||||||
|
|
||||||
func (m *Manager) newConnectionPool(channelID string, connectionID string, config queue.UpstreamConfig) *connectionPool {
|
func (m *Manager) newConnectionPool(channelID string, connectionID string, config queue.UpstreamConfig) *connectionPool {
|
||||||
return &connectionPool{
|
return &connectionPool{
|
||||||
channelID: channelID,
|
channelID: channelID,
|
||||||
|
|||||||
+2
-1
@@ -11,7 +11,8 @@
|
|||||||
"start:local": "powershell -NoProfile -ExecutionPolicy Bypass -File tools/start-local.ps1",
|
"start:local": "powershell -NoProfile -ExecutionPolicy Bypass -File tools/start-local.ps1",
|
||||||
"start:local:minio": "powershell -NoProfile -ExecutionPolicy Bypass -File tools/start-local.ps1 -OnlyMinio",
|
"start:local:minio": "powershell -NoProfile -ExecutionPolicy Bypass -File tools/start-local.ps1 -OnlyMinio",
|
||||||
"prisma:generate": "npm --prefix api run prisma:generate",
|
"prisma:generate": "npm --prefix api run prisma:generate",
|
||||||
"security:verify": "node tools/security/verify-dependency-mitigations.mjs",
|
"security:verify": "node tools/security/verify-dependency-mitigations.mjs && node tools/security/verify-security-deployment.mjs",
|
||||||
|
"deploy:verify": "node tools/deploy/verify-production-deployment.mjs",
|
||||||
"spike:contracts": "node tools/spike/validate-gateway-queue-contract.mjs",
|
"spike:contracts": "node tools/spike/validate-gateway-queue-contract.mjs",
|
||||||
"spike:gateway": "powershell -NoProfile -ExecutionPolicy Bypass -Command \"$env:Path='C:\\Program Files\\Go\\bin;'+$env:Path; Push-Location gateway; go test ./...; Pop-Location\"",
|
"spike:gateway": "powershell -NoProfile -ExecutionPolicy Bypass -Command \"$env:Path='C:\\Program Files\\Go\\bin;'+$env:Path; Push-Location gateway; go test ./...; Pop-Location\"",
|
||||||
"spike:bullmq": "node api/src/spike/bullmq-link-spike.mjs",
|
"spike:bullmq": "node api/src/spike/bullmq-link-spike.mjs",
|
||||||
|
|||||||
@@ -3,6 +3,7 @@ import type { SecurityAlert, SecurityBlock, SecurityOverview, SecurityProtectedN
|
|||||||
|
|
||||||
export const adminSecurityDetectionApi = {
|
export const adminSecurityDetectionApi = {
|
||||||
getSecurityOverview: (range = '24h') => request<SecurityOverview>(withQuery('/admin/security-detection/overview', { range })),
|
getSecurityOverview: (range = '24h') => request<SecurityOverview>(withQuery('/admin/security-detection/overview', { range })),
|
||||||
|
getSecurityNotificationSummary: () => request<{ count: number; criticalCount: number }>('/admin/security-detection/notification-summary'),
|
||||||
listSecurityAlerts: (query: Record<string, string | number | undefined> = {}) => request<{ items: SecurityAlert[]; total: number }> (withQuery('/admin/security-detection/alerts', query)),
|
listSecurityAlerts: (query: Record<string, string | number | undefined> = {}) => request<{ items: SecurityAlert[]; total: number }> (withQuery('/admin/security-detection/alerts', query)),
|
||||||
listSecurityRules: () => request<SecurityRule[]>('/admin/security-detection/rules'),
|
listSecurityRules: () => request<SecurityRule[]>('/admin/security-detection/rules'),
|
||||||
updateSecurityRule: (id: string, body: Partial<SecurityRule>) => request<SecurityRule>(`/admin/security-detection/rules/${id}`, { method: 'PUT', body: JSON.stringify(body) }),
|
updateSecurityRule: (id: string, body: Partial<SecurityRule>) => request<SecurityRule>(`/admin/security-detection/rules/${id}`, { method: 'PUT', body: JSON.stringify(body) }),
|
||||||
|
|||||||
@@ -12,6 +12,18 @@ export type InfrastructureServiceStatus = {
|
|||||||
status: 'healthy' | 'unhealthy' | 'unknown';
|
status: 'healthy' | 'unhealthy' | 'unknown';
|
||||||
};
|
};
|
||||||
|
|
||||||
|
export type InfrastructureServiceMetricGroup = {
|
||||||
|
key: string;
|
||||||
|
name: string;
|
||||||
|
available: boolean;
|
||||||
|
metrics: Array<{
|
||||||
|
key: string;
|
||||||
|
label: string;
|
||||||
|
value: number | null;
|
||||||
|
unit: 'percent' | 'seconds' | 'count' | 'per_second' | 'bytes';
|
||||||
|
}>;
|
||||||
|
};
|
||||||
|
|
||||||
export type InfrastructureAlert = {
|
export type InfrastructureAlert = {
|
||||||
fingerprint: string;
|
fingerprint: string;
|
||||||
name: string;
|
name: string;
|
||||||
@@ -61,5 +73,6 @@ export type InfrastructureMonitoringOverview = {
|
|||||||
networkTransmitBytesPerSecond: InfrastructureMetricPoint[];
|
networkTransmitBytesPerSecond: InfrastructureMetricPoint[];
|
||||||
};
|
};
|
||||||
services: InfrastructureServiceStatus[];
|
services: InfrastructureServiceStatus[];
|
||||||
|
serviceMetrics: InfrastructureServiceMetricGroup[];
|
||||||
alerts: InfrastructureAlert[];
|
alerts: InfrastructureAlert[];
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
import { useCallback, useEffect, useMemo, useRef, useState } from 'react';
|
import { useCallback, useEffect, useMemo, useRef, useState } from 'react';
|
||||||
import { AlertTriangle, BarChart3, CheckCircle2, Eye, RefreshCw, Search, TimerReset } from 'lucide-react';
|
import { AlertTriangle, BarChart3, CheckCircle2, Eye, RefreshCw, Search, TimerReset } from 'lucide-react';
|
||||||
import { adminApi, type DownstreamDeliveryDashboard, type DownstreamDeliveryRecord, type DownstreamRequeuePreview, type DownstreamRequeueTask, type DownstreamRequeueTaskItem, type EnterpriseApplication, type TenantOption } from '@/api/adminApi';
|
import { adminApi, type DownstreamDeliveryDashboard, type DownstreamDeliveryRecord, type DownstreamRequeuePreview, type DownstreamRequeueTask, type DownstreamRequeueTaskItem, type EnterpriseApplication, type TenantOption } from '@/api/adminApi';
|
||||||
import { Breadcrumb, Button, DateRangeInput, Input, Modal, Pagination, Select, Tag, type DateRangeValue } from '@/components/ui';
|
import { Breadcrumb, Button, DateRangeInput, Input, Modal, Pagination, Select, Tag, Textarea, type DateRangeValue } from '@/components/ui';
|
||||||
import { formatDateTime } from '@/utils/dateTime';
|
import { formatDateTime } from '@/utils/dateTime';
|
||||||
|
|
||||||
const statusTone: Record<string, 'neutral' | 'info' | 'success' | 'warning' | 'danger'> = {
|
const statusTone: Record<string, 'neutral' | 'info' | 'success' | 'warning' | 'danger'> = {
|
||||||
@@ -677,7 +677,7 @@ export function AdminDownstreamDeliveriesPage() {
|
|||||||
/>
|
/>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<div className="surface admin-task-table-card report-task-table-card">
|
<div className="surface admin-task-table-card report-task-table-card downstream-requeue-task-card">
|
||||||
<div className="section-heading downstream-requeue-task-heading">
|
<div className="section-heading downstream-requeue-task-heading">
|
||||||
<div><h2>后台重投任务</h2><p className="muted">按筛选快照安全恢复,支持暂停、继续、终止和完整结果追踪。</p></div>
|
<div><h2>后台重投任务</h2><p className="muted">按筛选快照安全恢复,支持暂停、继续、终止和完整结果追踪。</p></div>
|
||||||
<div className="downstream-requeue-task-heading__actions">
|
<div className="downstream-requeue-task-heading__actions">
|
||||||
@@ -741,8 +741,20 @@ export function AdminDownstreamDeliveriesPage() {
|
|||||||
</div>
|
</div>
|
||||||
<div className="downstream-requeue-preview__distribution"><span>状态分布</span><div>{Object.entries(taskPreview.statusCounts).map(([key, value]) => <Tag key={key} tone={statusTone[key] ?? 'neutral'}>{statusLabel[key] ?? key} {value}</Tag>)}</div></div>
|
<div className="downstream-requeue-preview__distribution"><span>状态分布</span><div>{Object.entries(taskPreview.statusCounts).map(([key, value]) => <Tag key={key} tone={statusTone[key] ?? 'neutral'}>{statusLabel[key] ?? key} {value}</Tag>)}</div></div>
|
||||||
<Select label="执行速度" value={String(taskRate)} onChange={(event) => setTaskRate(Number(event.target.value))} options={[{ label: '平稳(每应用10条/秒)', value: '10' }, { label: '快速(每应用20条/秒)', value: '20' }, { label: '低速(每应用5条/秒)', value: '5' }]} />
|
<Select label="执行速度" value={String(taskRate)} onChange={(event) => setTaskRate(Number(event.target.value))} options={[{ label: '平稳(每应用10条/秒)', value: '10' }, { label: '快速(每应用20条/秒)', value: '20' }, { label: '低速(每应用5条/秒)', value: '5' }]} />
|
||||||
<label className="field"><span>任务原因 *</span><textarea value={taskReason} onChange={(event) => setTaskReason(event.target.value)} placeholder="请填写事故原因、工单号或处理说明(至少5个字)" rows={3} /></label>
|
<Textarea
|
||||||
<div className="downstream-requeue-warning"><AlertTriangle size={20} /><div><strong>安全边界</strong><p>仅处理待投递、失败、未确认和拒绝记录;不会批量重投客户端已确认或正在等待 ACK 的记录。客户离线时进入等待,不计失败或跳过。</p></div></div>
|
className="downstream-requeue-reason"
|
||||||
|
error={taskReason.length > 0 && taskReason.trim().length < 5 ? '任务原因至少填写 5 个字' : undefined}
|
||||||
|
hint={`请填写事故原因、工单号或处理说明 · ${taskReason.length}/200`}
|
||||||
|
id="downstream-requeue-task-reason"
|
||||||
|
label="任务原因"
|
||||||
|
maxLength={200}
|
||||||
|
onChange={(event) => setTaskReason(event.target.value)}
|
||||||
|
placeholder="例如:工单 INC-20260814,重新投递客户已确认的历史回执"
|
||||||
|
required
|
||||||
|
rows={4}
|
||||||
|
value={taskReason}
|
||||||
|
/>
|
||||||
|
<div className="downstream-requeue-warning"><AlertTriangle size={20} /><div><strong>重复投递风险</strong><p>任务支持待投递、失败、未确认、拒绝和客户端已确认记录;已确认记录会再次发送,可能导致客户端重复处理。正在等待 ACK 的记录仍不会并发重投,客户离线时进入等待。</p></div></div>
|
||||||
</div>
|
</div>
|
||||||
</Modal>
|
</Modal>
|
||||||
) : null}
|
) : null}
|
||||||
|
|||||||
@@ -11,20 +11,13 @@
|
|||||||
align-items: center;
|
align-items: center;
|
||||||
display: flex;
|
display: flex;
|
||||||
gap: 12px;
|
gap: 12px;
|
||||||
margin-top: 10px;
|
margin-top: 8px;
|
||||||
}
|
|
||||||
|
|
||||||
.system-monitoring-title-row h1 {
|
|
||||||
color: var(--color-text-strong);
|
|
||||||
font-size: 24px;
|
|
||||||
line-height: 1.25;
|
|
||||||
margin: 0;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
.system-monitoring-title-row p {
|
.system-monitoring-title-row p {
|
||||||
color: var(--color-text-muted);
|
color: var(--color-text-muted);
|
||||||
font-size: 13px;
|
font-size: 13px;
|
||||||
margin: 4px 0 0;
|
margin: 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
.system-monitoring-controls,
|
.system-monitoring-controls,
|
||||||
@@ -191,6 +184,24 @@
|
|||||||
grid-template-columns: minmax(0, 1fr) 300px;
|
grid-template-columns: minmax(0, 1fr) 300px;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
.system-monitoring-service-metrics { padding: 18px; }
|
||||||
|
.system-monitoring-service-metrics > header {
|
||||||
|
align-items: center;
|
||||||
|
display: flex;
|
||||||
|
justify-content: space-between;
|
||||||
|
margin-bottom: 14px;
|
||||||
|
}
|
||||||
|
.system-monitoring-service-metrics > header > div { align-items: center; color: var(--color-text-strong); display: flex; gap: 8px; }
|
||||||
|
.system-monitoring-service-metrics > header > span { color: var(--color-text-muted); font-size: 12px; }
|
||||||
|
.system-monitoring-service-metric-grid { display: grid; gap: 12px; grid-template-columns: repeat(3, minmax(0, 1fr)); }
|
||||||
|
.system-monitoring-service-metric-grid article { background: var(--color-bg-subtle); border: 1px solid var(--color-border); border-radius: var(--radius-md); padding: 14px; }
|
||||||
|
.system-monitoring-service-metric-title { align-items: center; display: flex; justify-content: space-between; margin-bottom: 9px; }
|
||||||
|
.system-monitoring-service-metric-title > strong { color: var(--color-text-strong); font-size: 14px; }
|
||||||
|
.system-monitoring-service-metric-row { align-items: center; border-top: 1px solid var(--color-border); display: flex; justify-content: space-between; min-height: 34px; }
|
||||||
|
.system-monitoring-service-metric-row span { color: var(--color-text-muted); font-size: 12px; }
|
||||||
|
.system-monitoring-service-metric-row strong { color: var(--color-text); font-size: 13px; }
|
||||||
|
.system-monitoring-service-metric-empty { color: var(--color-text-subtle); font-size: 12px; line-height: 1.55; padding-top: 8px; }
|
||||||
|
|
||||||
.system-monitoring-chart-stack {
|
.system-monitoring-chart-stack {
|
||||||
display: grid;
|
display: grid;
|
||||||
gap: 16px;
|
gap: 16px;
|
||||||
@@ -285,6 +296,7 @@
|
|||||||
.system-monitoring-main-grid { grid-template-columns: minmax(0, 1fr); }
|
.system-monitoring-main-grid { grid-template-columns: minmax(0, 1fr); }
|
||||||
.system-monitoring-services { order: -1; }
|
.system-monitoring-services { order: -1; }
|
||||||
.system-monitoring-service-list { grid-template-columns: repeat(3, minmax(0, 1fr)); }
|
.system-monitoring-service-list { grid-template-columns: repeat(3, minmax(0, 1fr)); }
|
||||||
|
.system-monitoring-service-metric-grid { grid-template-columns: repeat(2, minmax(0, 1fr)); }
|
||||||
.system-monitoring-service { border-bottom: 0; border-right: 1px solid var(--color-border); padding: 11px 12px; }
|
.system-monitoring-service { border-bottom: 0; border-right: 1px solid var(--color-border); padding: 11px 12px; }
|
||||||
.system-monitoring-service:nth-child(3n) { border-right: 0; }
|
.system-monitoring-service:nth-child(3n) { border-right: 0; }
|
||||||
}
|
}
|
||||||
@@ -299,7 +311,9 @@
|
|||||||
.system-monitoring-health__fact { border-left: 0; border-top: 1px solid var(--color-border); padding: 12px 0 0; }
|
.system-monitoring-health__fact { border-left: 0; border-top: 1px solid var(--color-border); padding: 12px 0 0; }
|
||||||
.system-monitoring-metrics,
|
.system-monitoring-metrics,
|
||||||
.system-monitoring-chart-stack,
|
.system-monitoring-chart-stack,
|
||||||
.system-monitoring-service-list { grid-template-columns: minmax(0, 1fr); }
|
.system-monitoring-service-list,
|
||||||
|
.system-monitoring-service-metric-grid { grid-template-columns: minmax(0, 1fr); }
|
||||||
|
.system-monitoring-service-metrics > header { align-items: flex-start; flex-direction: column; gap: 6px; }
|
||||||
.system-monitoring-service { border-bottom: 1px solid var(--color-border); border-right: 0; padding: 13px 0; }
|
.system-monitoring-service { border-bottom: 1px solid var(--color-border); border-right: 0; padding: 13px 0; }
|
||||||
.system-monitoring-chart-card { padding: 14px 10px; }
|
.system-monitoring-chart-card { padding: 14px 10px; }
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -69,6 +69,15 @@ function formatUptime(value: number | null) {
|
|||||||
return days > 0 ? `${days}天 ${hours}小时` : `${hours}小时`;
|
return days > 0 ? `${days}天 ${hours}小时` : `${hours}小时`;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function formatServiceMetric(value: number | null, unit: 'percent' | 'seconds' | 'count' | 'per_second' | 'bytes') {
|
||||||
|
if (value === null) return '—';
|
||||||
|
if (unit === 'percent') return `${value.toFixed(2)}%`;
|
||||||
|
if (unit === 'seconds') return value < 1 ? `${Math.round(value * 1000)} ms` : `${value.toFixed(1)} s`;
|
||||||
|
if (unit === 'per_second') return `${value.toFixed(value < 10 ? 2 : 1)}/s`;
|
||||||
|
if (unit === 'bytes') return formatBytes(value);
|
||||||
|
return Math.round(value).toLocaleString('zh-CN');
|
||||||
|
}
|
||||||
|
|
||||||
function formatTime(value: string | null) {
|
function formatTime(value: string | null) {
|
||||||
if (!value) return '暂无采样';
|
if (!value) return '暂无采样';
|
||||||
return new Intl.DateTimeFormat('zh-CN', {
|
return new Intl.DateTimeFormat('zh-CN', {
|
||||||
@@ -223,10 +232,7 @@ export function AdminSystemMonitoringPage() {
|
|||||||
<div>
|
<div>
|
||||||
<Breadcrumb items={['系统管理', '系统监控']} />
|
<Breadcrumb items={['系统管理', '系统监控']} />
|
||||||
<div className="system-monitoring-title-row">
|
<div className="system-monitoring-title-row">
|
||||||
<div>
|
|
||||||
<h1>系统监控</h1>
|
|
||||||
<p>服务器资源、核心服务与活动告警</p>
|
<p>服务器资源、核心服务与活动告警</p>
|
||||||
</div>
|
|
||||||
<Tag tone={status.tone}>{status.label}</Tag>
|
<Tag tone={status.tone}>{status.label}</Tag>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
@@ -277,6 +283,29 @@ export function AdminSystemMonitoringPage() {
|
|||||||
<article className="surface system-monitoring-metric"><div className="system-monitoring-metric__icon is-green"><Network size={19} /></div><div><span>网络吞吐</span><strong>{formatRate(totalNetworkRate(metrics?.networkReceiveBytesPerSecond, metrics?.networkTransmitBytesPerSecond))}</strong><small>接收 {formatRate(metrics?.networkReceiveBytesPerSecond ?? null)} · 发送 {formatRate(metrics?.networkTransmitBytesPerSecond ?? null)}</small></div></article>
|
<article className="surface system-monitoring-metric"><div className="system-monitoring-metric__icon is-green"><Network size={19} /></div><div><span>网络吞吐</span><strong>{formatRate(totalNetworkRate(metrics?.networkReceiveBytesPerSecond, metrics?.networkTransmitBytesPerSecond))}</strong><small>接收 {formatRate(metrics?.networkReceiveBytesPerSecond ?? null)} · 发送 {formatRate(metrics?.networkTransmitBytesPerSecond ?? null)}</small></div></article>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
<section className="surface system-monitoring-service-metrics">
|
||||||
|
<header>
|
||||||
|
<div><Database size={18} /><strong>服务关键指标</strong></div>
|
||||||
|
<span>固定低基数聚合,不含手机号、短信ID或SQL文本</span>
|
||||||
|
</header>
|
||||||
|
<div className="system-monitoring-service-metric-grid">
|
||||||
|
{(overview?.serviceMetrics ?? []).map((group) => (
|
||||||
|
<article key={group.key}>
|
||||||
|
<div className="system-monitoring-service-metric-title">
|
||||||
|
<strong>{group.name}</strong>
|
||||||
|
<Tag tone={group.available ? 'success' : 'neutral'}>{group.available ? '已采集' : '待采集'}</Tag>
|
||||||
|
</div>
|
||||||
|
{group.metrics.length ? group.metrics.map((metric) => (
|
||||||
|
<div className="system-monitoring-service-metric-row" key={metric.key}>
|
||||||
|
<span>{metric.label}</span>
|
||||||
|
<strong>{formatServiceMetric(metric.value, metric.unit)}</strong>
|
||||||
|
</div>
|
||||||
|
)) : <div className="system-monitoring-service-metric-empty">已监控服务可用性,待原生容量指标接入</div>}
|
||||||
|
</article>
|
||||||
|
))}
|
||||||
|
</div>
|
||||||
|
</section>
|
||||||
|
|
||||||
<div className="system-monitoring-main-grid">
|
<div className="system-monitoring-main-grid">
|
||||||
<div className="system-monitoring-chart-stack">
|
<div className="system-monitoring-chart-stack">
|
||||||
<article className="surface system-monitoring-chart-card"><header><div><Cpu size={17} /><strong>CPU 趋势</strong></div><span>{formatPercent(metrics?.cpuUsagePercent ?? null)}</span></header>{overview?.trends.cpuUsagePercent.length ? <Chart height={230} option={cpuOption} /> : <EmptyChart />}</article>
|
<article className="surface system-monitoring-chart-card"><header><div><Cpu size={17} /><strong>CPU 趋势</strong></div><span>{formatPercent(metrics?.cpuUsagePercent ?? null)}</span></header>{overview?.trends.cpuUsagePercent.length ? <Chart height={230} option={cpuOption} /> : <EmptyChart />}</article>
|
||||||
|
|||||||
@@ -48,6 +48,7 @@ export function AdminLayout() {
|
|||||||
function AdminAuthenticatedLayout({ session }: { session: LoginSession }) {
|
function AdminAuthenticatedLayout({ session }: { session: LoginSession }) {
|
||||||
const [pendingAudits, setPendingAudits] = useState(EMPTY_PENDING_AUDITS);
|
const [pendingAudits, setPendingAudits] = useState(EMPTY_PENDING_AUDITS);
|
||||||
const [retirementUnreadCount, setRetirementUnreadCount] = useState(0);
|
const [retirementUnreadCount, setRetirementUnreadCount] = useState(0);
|
||||||
|
const [securityAlertSummary, setSecurityAlertSummary] = useState({ count: 0, criticalCount: 0 });
|
||||||
const [sessionLocked, setSessionLocked] = useState(Boolean(session.locked));
|
const [sessionLocked, setSessionLocked] = useState(Boolean(session.locked));
|
||||||
const loadPendingAuditCount = useCallback(() => {
|
const loadPendingAuditCount = useCallback(() => {
|
||||||
const currentSession = readSession('admin');
|
const currentSession = readSession('admin');
|
||||||
@@ -56,14 +57,16 @@ function AdminAuthenticatedLayout({ session }: { session: LoginSession }) {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
// This runs globally and on a timer, so it must not fan out through the full dashboard aggregation.
|
// This runs globally and on a timer, so it must not fan out through the full dashboard aggregation.
|
||||||
Promise.allSettled([adminApi.getPendingAudits(), adminApi.getSignatureRetirementUnreadCount()])
|
Promise.allSettled([adminApi.getPendingAudits(), adminApi.getSignatureRetirementUnreadCount(), adminApi.getSecurityNotificationSummary()])
|
||||||
.then(([audits, retirement]) => {
|
.then(([audits, retirement, security]) => {
|
||||||
setPendingAudits(audits.status === 'fulfilled' ? audits.value : EMPTY_PENDING_AUDITS);
|
setPendingAudits(audits.status === 'fulfilled' ? audits.value : EMPTY_PENDING_AUDITS);
|
||||||
setRetirementUnreadCount(retirement.status === 'fulfilled' ? retirement.value.count : 0);
|
setRetirementUnreadCount(retirement.status === 'fulfilled' ? retirement.value.count : 0);
|
||||||
|
setSecurityAlertSummary(security.status === 'fulfilled' ? security.value : { count: 0, criticalCount: 0 });
|
||||||
})
|
})
|
||||||
.catch(() => {
|
.catch(() => {
|
||||||
setPendingAudits(EMPTY_PENDING_AUDITS);
|
setPendingAudits(EMPTY_PENDING_AUDITS);
|
||||||
setRetirementUnreadCount(0);
|
setRetirementUnreadCount(0);
|
||||||
|
setSecurityAlertSummary({ count: 0, criticalCount: 0 });
|
||||||
});
|
});
|
||||||
}, []);
|
}, []);
|
||||||
|
|
||||||
@@ -78,11 +81,13 @@ function AdminAuthenticatedLayout({ session }: { session: LoginSession }) {
|
|||||||
window.addEventListener('focus', onFocus);
|
window.addEventListener('focus', onFocus);
|
||||||
window.addEventListener('cmpp-audit-count-refresh', onAuditRefresh);
|
window.addEventListener('cmpp-audit-count-refresh', onAuditRefresh);
|
||||||
window.addEventListener('cmpp-retirement-count-refresh', onAuditRefresh);
|
window.addEventListener('cmpp-retirement-count-refresh', onAuditRefresh);
|
||||||
|
window.addEventListener('cmpp-security-alert-count-refresh', onAuditRefresh);
|
||||||
return () => {
|
return () => {
|
||||||
window.clearInterval(timer);
|
window.clearInterval(timer);
|
||||||
window.removeEventListener('focus', onFocus);
|
window.removeEventListener('focus', onFocus);
|
||||||
window.removeEventListener('cmpp-audit-count-refresh', onAuditRefresh);
|
window.removeEventListener('cmpp-audit-count-refresh', onAuditRefresh);
|
||||||
window.removeEventListener('cmpp-retirement-count-refresh', onAuditRefresh);
|
window.removeEventListener('cmpp-retirement-count-refresh', onAuditRefresh);
|
||||||
|
window.removeEventListener('cmpp-security-alert-count-refresh', onAuditRefresh);
|
||||||
};
|
};
|
||||||
}, [loadPendingAuditCount, session.portal, sessionLocked]);
|
}, [loadPendingAuditCount, session.portal, sessionLocked]);
|
||||||
|
|
||||||
@@ -96,7 +101,10 @@ function AdminAuthenticatedLayout({ session }: { session: LoginSession }) {
|
|||||||
userName={session.user.displayName}
|
userName={session.user.displayName}
|
||||||
userRole="平台管理员"
|
userRole="平台管理员"
|
||||||
onSessionLockedChange={setSessionLocked}
|
onSessionLockedChange={setSessionLocked}
|
||||||
retirementAlert={{ count: retirementUnreadCount, to: '/admin/signature-retirement' }}
|
alertNotifications={[
|
||||||
|
{ label: '签名清退预警', count: retirementUnreadCount, description: '今日未读且未抑制', to: '/admin/signature-retirement' },
|
||||||
|
{ label: '安全检测与封禁', count: securityAlertSummary.count, description: securityAlertSummary.criticalCount > 0 ? `${securityAlertSummary.criticalCount} 条严重告警待处置` : '待处置安全告警', to: '/admin/security-detection' },
|
||||||
|
]}
|
||||||
auditNotifications={[
|
auditNotifications={[
|
||||||
{ label: '企业认证待审', count: pendingAudits.enterpriseCertifications, to: '/admin/enterprise-audit' },
|
{ label: '企业认证待审', count: pendingAudits.enterpriseCertifications, to: '/admin/enterprise-audit' },
|
||||||
{ label: '短信审核待审', count: pendingAudits.smsAudits, to: '/admin/sms-audit' },
|
{ label: '短信审核待审', count: pendingAudits.smsAudits, to: '/admin/sms-audit' },
|
||||||
|
|||||||
+38
-11
@@ -50,6 +50,10 @@ export type AuditNotificationItem = {
|
|||||||
to: string;
|
to: string;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
export type AlertNotificationItem = AuditNotificationItem & {
|
||||||
|
description: string;
|
||||||
|
};
|
||||||
|
|
||||||
type AppShellProps = {
|
type AppShellProps = {
|
||||||
title: string;
|
title: string;
|
||||||
subtitle: string;
|
subtitle: string;
|
||||||
@@ -60,7 +64,7 @@ type AppShellProps = {
|
|||||||
userRole: string;
|
userRole: string;
|
||||||
navSections: ShellNavSection[];
|
navSections: ShellNavSection[];
|
||||||
auditNotifications?: AuditNotificationItem[];
|
auditNotifications?: AuditNotificationItem[];
|
||||||
retirementAlert?: { count: number; to: string };
|
alertNotifications?: AlertNotificationItem[];
|
||||||
onSessionLockedChange?: (locked: boolean) => void;
|
onSessionLockedChange?: (locked: boolean) => void;
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -73,7 +77,7 @@ export function AppShell({
|
|||||||
userRole,
|
userRole,
|
||||||
navSections,
|
navSections,
|
||||||
auditNotifications = [],
|
auditNotifications = [],
|
||||||
retirementAlert,
|
alertNotifications = [],
|
||||||
onSessionLockedChange,
|
onSessionLockedChange,
|
||||||
}: AppShellProps) {
|
}: AppShellProps) {
|
||||||
const [collapsed, setCollapsed] = useState(false);
|
const [collapsed, setCollapsed] = useState(false);
|
||||||
@@ -81,6 +85,7 @@ export function AppShell({
|
|||||||
const [closedSections, setClosedSections] = useState<Record<string, boolean>>({});
|
const [closedSections, setClosedSections] = useState<Record<string, boolean>>({});
|
||||||
const [userMenuOpen, setUserMenuOpen] = useState(false);
|
const [userMenuOpen, setUserMenuOpen] = useState(false);
|
||||||
const [noticeOpen, setNoticeOpen] = useState(false);
|
const [noticeOpen, setNoticeOpen] = useState(false);
|
||||||
|
const [alertNoticeOpen, setAlertNoticeOpen] = useState(false);
|
||||||
const [passwordModalOpen, setPasswordModalOpen] = useState(false);
|
const [passwordModalOpen, setPasswordModalOpen] = useState(false);
|
||||||
const [currentPassword, setCurrentPassword] = useState('');
|
const [currentPassword, setCurrentPassword] = useState('');
|
||||||
const [newPassword, setNewPassword] = useState('');
|
const [newPassword, setNewPassword] = useState('');
|
||||||
@@ -107,6 +112,10 @@ export function AppShell({
|
|||||||
() => auditNotifications.reduce((sum, item) => sum + item.count, 0),
|
() => auditNotifications.reduce((sum, item) => sum + item.count, 0),
|
||||||
[auditNotifications],
|
[auditNotifications],
|
||||||
);
|
);
|
||||||
|
const alertTotal = useMemo(
|
||||||
|
() => alertNotifications.reduce((sum, item) => sum + item.count, 0),
|
||||||
|
[alertNotifications],
|
||||||
|
);
|
||||||
|
|
||||||
async function changeOwnPassword() {
|
async function changeOwnPassword() {
|
||||||
if (!currentPassword || newPassword.length < 6) {
|
if (!currentPassword || newPassword.length < 6) {
|
||||||
@@ -417,23 +426,41 @@ export function AppShell({
|
|||||||
<button className="icon-button topbar-help" type="button" aria-label="帮助中心">
|
<button className="icon-button topbar-help" type="button" aria-label="帮助中心">
|
||||||
<CircleHelp size={18} />
|
<CircleHelp size={18} />
|
||||||
</button>
|
</button>
|
||||||
{retirementAlert ? (
|
{alertNotifications.length ? (
|
||||||
<Link
|
<div className="notice-menu-wrap">
|
||||||
aria-label={`今日未读且未抑制签名清退预警 ${retirementAlert.count} 条`}
|
<button
|
||||||
className={['icon-button', retirementAlert.count > 0 ? 'has-dot' : ''].filter(Boolean).join(' ')}
|
aria-expanded={alertNoticeOpen}
|
||||||
title="今日未读且未抑制签名清退预警"
|
aria-haspopup="menu"
|
||||||
to={retirementAlert.to}
|
aria-label="预警通知"
|
||||||
|
className={['icon-button', alertTotal > 0 ? 'has-dot' : ''].filter(Boolean).join(' ')}
|
||||||
|
onClick={() => { setAlertNoticeOpen((open) => !open); setNoticeOpen(false); }}
|
||||||
|
type="button"
|
||||||
>
|
>
|
||||||
<Bell size={18} />
|
<Bell size={18} />
|
||||||
{retirementAlert.count > 0 ? <span className="notice-count">{retirementAlert.count}</span> : null}
|
{alertTotal > 0 ? <span className="notice-count">{alertTotal}</span> : null}
|
||||||
</Link>
|
</button>
|
||||||
|
{alertNoticeOpen ? (
|
||||||
|
<div className="notice-popover notice-popover--alerts" role="menu">
|
||||||
|
<div className="notice-popover__header">
|
||||||
|
<strong>预警中心</strong>
|
||||||
|
<span className={alertTotal === 0 ? 'is-zero' : ''}>{alertTotal} 条</span>
|
||||||
|
</div>
|
||||||
|
{alertNotifications.map((item) => (
|
||||||
|
<NavLink key={item.to} onClick={() => setAlertNoticeOpen(false)} role="menuitem" to={item.to}>
|
||||||
|
<span className="notice-popover__copy"><b>{item.label}</b><small>{item.description}</small></span>
|
||||||
|
<strong className={item.count === 0 ? 'is-zero' : ''}>{item.count}</strong>
|
||||||
|
</NavLink>
|
||||||
|
))}
|
||||||
|
</div>
|
||||||
|
) : null}
|
||||||
|
</div>
|
||||||
) : null}
|
) : null}
|
||||||
<div className="notice-menu-wrap">
|
<div className="notice-menu-wrap">
|
||||||
<button
|
<button
|
||||||
aria-expanded={noticeOpen}
|
aria-expanded={noticeOpen}
|
||||||
aria-haspopup="menu"
|
aria-haspopup="menu"
|
||||||
className={['icon-button', auditTotal > 0 ? 'has-dot' : ''].filter(Boolean).join(' ')}
|
className={['icon-button', auditTotal > 0 ? 'has-dot' : ''].filter(Boolean).join(' ')}
|
||||||
onClick={() => setNoticeOpen((open) => !open)}
|
onClick={() => { setNoticeOpen((open) => !open); setAlertNoticeOpen(false); }}
|
||||||
type="button"
|
type="button"
|
||||||
aria-label="通知"
|
aria-label="通知"
|
||||||
>
|
>
|
||||||
|
|||||||
@@ -9424,10 +9424,14 @@
|
|||||||
.downstream-page-size { display: inline-flex; align-items: center; gap: 8px; color: var(--text-muted); font-size: 13px; }
|
.downstream-page-size { display: inline-flex; align-items: center; gap: 8px; color: var(--text-muted); font-size: 13px; }
|
||||||
.downstream-page-size select { min-width: 76px; height: 36px; border: 1px solid var(--border); border-radius: 8px; background: var(--surface); color: var(--text); padding: 0 10px; }
|
.downstream-page-size select { min-width: 76px; height: 36px; border: 1px solid var(--border); border-radius: 8px; background: var(--surface); color: var(--text); padding: 0 10px; }
|
||||||
.downstream-requeue-task-heading { gap: 20px; }
|
.downstream-requeue-task-heading { gap: 20px; }
|
||||||
|
.downstream-requeue-task-card > .downstream-requeue-task-heading { padding: var(--space-6) var(--space-6) var(--space-5); }
|
||||||
.downstream-requeue-task-heading__actions { display: flex; align-items: center; gap: 8px; }
|
.downstream-requeue-task-heading__actions { display: flex; align-items: center; gap: 8px; }
|
||||||
.downstream-requeue-task-heading__actions .ui-field { min-width: 150px; }
|
.downstream-requeue-task-heading__actions .ui-field { min-width: 150px; }
|
||||||
.downstream-requeue-task-list { display: grid; border-top: 1px solid var(--border); }
|
.downstream-requeue-task-list { display: grid; margin: 0 var(--space-6); overflow: hidden; border: 1px solid var(--border); border-radius: var(--radius-lg); background: var(--color-surface); }
|
||||||
.downstream-requeue-task-list article { display: grid; grid-template-columns: minmax(190px, .9fr) minmax(260px, 1.35fr) minmax(250px, 1fr) auto; gap: 22px; align-items: center; padding: 18px 4px; border-bottom: 1px solid var(--border); }
|
.downstream-requeue-task-list article { display: grid; grid-template-columns: minmax(190px, .9fr) minmax(260px, 1.35fr) minmax(250px, 1fr) auto; gap: 22px; align-items: center; padding: 18px var(--space-5); border-bottom: 1px solid var(--border); }
|
||||||
|
.downstream-requeue-task-list article:last-of-type { border-bottom: 0; }
|
||||||
|
.downstream-requeue-task-list > .muted { margin: 0; padding: var(--space-6); text-align: center; }
|
||||||
|
.downstream-requeue-task-card > .ui-pagination { padding: var(--space-5) var(--space-6) var(--space-6); }
|
||||||
.downstream-requeue-task-list article > div { min-width: 0; }
|
.downstream-requeue-task-list article > div { min-width: 0; }
|
||||||
.downstream-requeue-task-list__identity, .downstream-requeue-task-list__scope, .downstream-requeue-task-list__progress { display: grid; gap: 5px; }
|
.downstream-requeue-task-list__identity, .downstream-requeue-task-list__scope, .downstream-requeue-task-list__progress { display: grid; gap: 5px; }
|
||||||
.downstream-requeue-task-list__identity > strong { color: var(--text-strong); font-size: 14px; font-variant-numeric: tabular-nums; }
|
.downstream-requeue-task-list__identity > strong { color: var(--text-strong); font-size: 14px; font-variant-numeric: tabular-nums; }
|
||||||
@@ -9447,6 +9451,8 @@
|
|||||||
.downstream-requeue-preview__distribution { display: grid; gap: 8px; }
|
.downstream-requeue-preview__distribution { display: grid; gap: 8px; }
|
||||||
.downstream-requeue-preview__distribution > span { color: var(--text-muted); font-size: 13px; }
|
.downstream-requeue-preview__distribution > span { color: var(--text-muted); font-size: 13px; }
|
||||||
.downstream-requeue-preview__distribution > div { display: flex; flex-wrap: wrap; gap: 8px; }
|
.downstream-requeue-preview__distribution > div { display: flex; flex-wrap: wrap; gap: 8px; }
|
||||||
|
.downstream-requeue-reason { padding: 16px; border: 1px solid var(--border); border-radius: var(--radius-lg); background: var(--surface-subtle, #f8fafc); }
|
||||||
|
.downstream-requeue-reason .ui-textarea { min-height: 124px; background: var(--color-surface); line-height: 1.65; }
|
||||||
.downstream-requeue-warning { display: flex; align-items: flex-start; gap: 10px; padding: 14px; border: 1px solid #fed7aa; border-radius: 10px; background: var(--warning-soft, #fff7ed); color: var(--warning-text, #9a3412); }
|
.downstream-requeue-warning { display: flex; align-items: flex-start; gap: 10px; padding: 14px; border: 1px solid #fed7aa; border-radius: 10px; background: var(--warning-soft, #fff7ed); color: var(--warning-text, #9a3412); }
|
||||||
.downstream-requeue-warning > div { display: grid; gap: 4px; }
|
.downstream-requeue-warning > div { display: grid; gap: 4px; }
|
||||||
.downstream-requeue-warning p { margin: 0; color: inherit; line-height: 1.6; }
|
.downstream-requeue-warning p { margin: 0; color: inherit; line-height: 1.6; }
|
||||||
@@ -9460,4 +9466,4 @@
|
|||||||
.downstream-requeue-detail-items__head { background: var(--surface-subtle, #f8fafc); color: var(--text-muted); font-size: 12px; font-weight: 600; }
|
.downstream-requeue-detail-items__head { background: var(--surface-subtle, #f8fafc); color: var(--text-muted); font-size: 12px; font-weight: 600; }
|
||||||
.downstream-requeue-detail-items time { color: var(--text-muted); font-size: 12px; }
|
.downstream-requeue-detail-items time { color: var(--text-muted); font-size: 12px; }
|
||||||
@media (max-width: 1100px) { .downstream-requeue-task-list article { grid-template-columns: 1fr 1.4fr; } .downstream-requeue-task-list__actions { justify-content: flex-start; } .downstream-requeue-preview__summary { grid-template-columns: repeat(2, 1fr); } }
|
@media (max-width: 1100px) { .downstream-requeue-task-list article { grid-template-columns: 1fr 1.4fr; } .downstream-requeue-task-list__actions { justify-content: flex-start; } .downstream-requeue-preview__summary { grid-template-columns: repeat(2, 1fr); } }
|
||||||
@media (max-width: 780px) { .downstream-requeue-task-heading, .downstream-requeue-task-heading__actions { align-items: stretch; flex-direction: column; } .downstream-requeue-task-list article, .downstream-requeue-preview__summary, .downstream-requeue-detail-filter { grid-template-columns: 1fr; } .downstream-requeue-detail-items { border: 0; overflow: visible; gap: 10px; } .downstream-requeue-detail-items__head { display: none !important; } .downstream-requeue-detail-items > div { grid-template-columns: 1fr; gap: 6px; padding: 14px; border: 1px solid var(--border); border-radius: 10px; } }
|
@media (max-width: 780px) { .downstream-requeue-task-card > .downstream-requeue-task-heading { padding: var(--space-5) var(--space-4) var(--space-4); } .downstream-requeue-task-heading, .downstream-requeue-task-heading__actions { align-items: stretch; flex-direction: column; } .downstream-requeue-task-list { margin: 0 var(--space-4); } .downstream-requeue-task-list article, .downstream-requeue-preview__summary, .downstream-requeue-detail-filter { grid-template-columns: 1fr; } .downstream-requeue-task-card > .ui-pagination { padding: var(--space-4); } .downstream-requeue-detail-items { border: 0; overflow: visible; gap: 10px; } .downstream-requeue-detail-items__head { display: none !important; } .downstream-requeue-detail-items > div { grid-template-columns: 1fr; gap: 6px; padding: 14px; border: 1px solid var(--border); border-radius: 10px; } }
|
||||||
|
|||||||
@@ -571,6 +571,31 @@
|
|||||||
text-align: right;
|
text-align: right;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
.notice-popover--alerts {
|
||||||
|
min-width: 292px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.notice-popover--alerts a {
|
||||||
|
gap: var(--space-4);
|
||||||
|
min-height: 58px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.notice-popover__copy {
|
||||||
|
align-items: flex-start;
|
||||||
|
display: grid;
|
||||||
|
gap: 2px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.notice-popover__copy b {
|
||||||
|
color: var(--color-text-strong);
|
||||||
|
font-weight: var(--font-weight-semibold);
|
||||||
|
}
|
||||||
|
|
||||||
|
.notice-popover__copy small {
|
||||||
|
color: var(--color-text-muted);
|
||||||
|
font-size: 11px;
|
||||||
|
}
|
||||||
|
|
||||||
.page-heading__actions {
|
.page-heading__actions {
|
||||||
align-items: center;
|
align-items: center;
|
||||||
display: flex;
|
display: flex;
|
||||||
|
|||||||
@@ -6,6 +6,9 @@ REPO_URL="${REPO_URL:-http://175.27.255.91:3000/hectorzhao/lislgosms.git}"
|
|||||||
BRANCH="${BRANCH:-main}"
|
BRANCH="${BRANCH:-main}"
|
||||||
PUBLIC_HTTP_PORT="${PUBLIC_HTTP_PORT:-12026}"
|
PUBLIC_HTTP_PORT="${PUBLIC_HTTP_PORT:-12026}"
|
||||||
API_PORT="${API_PORT:-3000}"
|
API_PORT="${API_PORT:-3000}"
|
||||||
|
API_HOST="${API_HOST:-127.0.0.1}"
|
||||||
|
API_METRICS_HOST="${API_METRICS_HOST:-127.0.0.1}"
|
||||||
|
API_METRICS_PORT="${API_METRICS_PORT:-9464}"
|
||||||
API_ENABLE_SEND_WORKER="${API_ENABLE_SEND_WORKER:-true}"
|
API_ENABLE_SEND_WORKER="${API_ENABLE_SEND_WORKER:-true}"
|
||||||
API_SEND_WORKER_CONCURRENCY="${API_SEND_WORKER_CONCURRENCY:-50}"
|
API_SEND_WORKER_CONCURRENCY="${API_SEND_WORKER_CONCURRENCY:-50}"
|
||||||
GATEWAY_CONTROL_ADDR="${GATEWAY_CONTROL_ADDR:-127.0.0.1:8090}"
|
GATEWAY_CONTROL_ADDR="${GATEWAY_CONTROL_ADDR:-127.0.0.1:8090}"
|
||||||
@@ -183,6 +186,9 @@ write_env() {
|
|||||||
cat >/etc/cmpp-platform/cmpp-platform.env <<EOF
|
cat >/etc/cmpp-platform/cmpp-platform.env <<EOF
|
||||||
NODE_ENV=production
|
NODE_ENV=production
|
||||||
API_PORT=${API_PORT}
|
API_PORT=${API_PORT}
|
||||||
|
API_HOST=${API_HOST}
|
||||||
|
API_METRICS_HOST=${API_METRICS_HOST}
|
||||||
|
API_METRICS_PORT=${API_METRICS_PORT}
|
||||||
API_ENABLE_SEND_WORKER=${API_ENABLE_SEND_WORKER}
|
API_ENABLE_SEND_WORKER=${API_ENABLE_SEND_WORKER}
|
||||||
API_SEND_WORKER_CONCURRENCY=${API_SEND_WORKER_CONCURRENCY}
|
API_SEND_WORKER_CONCURRENCY=${API_SEND_WORKER_CONCURRENCY}
|
||||||
DATABASE_URL=postgresql://${DB_USER}:${DB_PASSWORD}@127.0.0.1:5432/${DB_NAME}?schema=public
|
DATABASE_URL=postgresql://${DB_USER}:${DB_PASSWORD}@127.0.0.1:5432/${DB_NAME}?schema=public
|
||||||
|
|||||||
@@ -42,6 +42,7 @@ npm --prefix api ci --include=dev
|
|||||||
|
|
||||||
echo "[deploy] Verifying dependency security mitigations"
|
echo "[deploy] Verifying dependency security mitigations"
|
||||||
npm run security:verify
|
npm run security:verify
|
||||||
|
npm run deploy:verify
|
||||||
|
|
||||||
echo "[deploy] Generating Prisma client and applying migrations"
|
echo "[deploy] Generating Prisma client and applying migrations"
|
||||||
npm --prefix api run prisma:generate
|
npm --prefix api run prisma:generate
|
||||||
@@ -69,13 +70,20 @@ echo "[deploy] Installing restricted security boundary"
|
|||||||
bash "$APP_DIR/tools/security/install-security-agent.sh"
|
bash "$APP_DIR/tools/security/install-security-agent.sh"
|
||||||
|
|
||||||
echo "[deploy] Ensuring HTTP response compression"
|
echo "[deploy] Ensuring HTTP response compression"
|
||||||
cat >/etc/nginx/conf.d/cmpp-compression.conf <<'EOF'
|
compression_config=/etc/nginx/conf.d/cmpp-compression.conf
|
||||||
|
: >"$compression_config"
|
||||||
|
if grep -RqsE --exclude='cmpp-compression.conf' '^[[:space:]]*gzip[[:space:]]+on;' \
|
||||||
|
/etc/nginx/nginx.conf /etc/nginx/conf.d /etc/nginx/sites-enabled 2>/dev/null; then
|
||||||
|
echo "[deploy] Reusing existing Nginx gzip configuration"
|
||||||
|
else
|
||||||
|
cat >"$compression_config" <<'EOF'
|
||||||
gzip on;
|
gzip on;
|
||||||
gzip_vary on;
|
gzip_vary on;
|
||||||
gzip_min_length 1024;
|
gzip_min_length 1024;
|
||||||
gzip_comp_level 5;
|
gzip_comp_level 5;
|
||||||
gzip_types application/json application/javascript text/javascript text/css text/plain text/csv image/svg+xml;
|
gzip_types application/json application/javascript text/javascript text/css text/plain text/csv image/svg+xml;
|
||||||
EOF
|
EOF
|
||||||
|
fi
|
||||||
nginx -t
|
nginx -t
|
||||||
|
|
||||||
echo "[deploy] Restarting services"
|
echo "[deploy] Restarting services"
|
||||||
|
|||||||
@@ -0,0 +1,29 @@
|
|||||||
|
import { readFileSync } from 'node:fs';
|
||||||
|
import { resolve } from 'node:path';
|
||||||
|
|
||||||
|
const deploy = readFileSync(resolve(import.meta.dirname, 'production-deploy.sh'), 'utf8');
|
||||||
|
const bootstrap = readFileSync(resolve(import.meta.dirname, 'production-bootstrap.sh'), 'utf8');
|
||||||
|
const apiMain = readFileSync(resolve(import.meta.dirname, '../../api/src/main.ts'), 'utf8');
|
||||||
|
const required = [
|
||||||
|
'compression_config=/etc/nginx/conf.d/cmpp-compression.conf',
|
||||||
|
': >"$compression_config"',
|
||||||
|
"--exclude='cmpp-compression.conf'",
|
||||||
|
"'^[[:space:]]*gzip[[:space:]]+on;'",
|
||||||
|
'Reusing existing Nginx gzip configuration',
|
||||||
|
'cat >"$compression_config"',
|
||||||
|
];
|
||||||
|
for (const marker of required) {
|
||||||
|
if (!deploy.includes(marker)) throw new Error(`production deploy is missing the idempotent Nginx compression guard: ${marker}`);
|
||||||
|
}
|
||||||
|
if (deploy.includes("cat >/etc/nginx/conf.d/cmpp-compression.conf <<'EOF'")) {
|
||||||
|
throw new Error('production deploy still writes a duplicate global gzip directive unconditionally');
|
||||||
|
}
|
||||||
|
|
||||||
|
for (const marker of ['API_HOST="${API_HOST:-127.0.0.1}"', 'API_HOST=${API_HOST}']) {
|
||||||
|
if (!bootstrap.includes(marker)) throw new Error(`production bootstrap is missing the loopback API binding: ${marker}`);
|
||||||
|
}
|
||||||
|
if (!apiMain.includes("process.env.API_HOST?.trim() || '127.0.0.1'") || !apiMain.includes('app.listen(port, host)')) {
|
||||||
|
throw new Error('NestJS API must bind to API_HOST and default to loopback');
|
||||||
|
}
|
||||||
|
|
||||||
|
console.log('Production deployment verified: Nginx compression is idempotent and NestJS defaults to loopback.');
|
||||||
@@ -7,9 +7,10 @@
|
|||||||
```bash
|
```bash
|
||||||
cd /opt/cmpp-platform
|
cd /opt/cmpp-platform
|
||||||
bash tools/monitoring/install-prometheus-monitoring.sh
|
bash tools/monitoring/install-prometheus-monitoring.sh
|
||||||
|
bash tools/monitoring/install-service-exporters.sh
|
||||||
```
|
```
|
||||||
|
|
||||||
脚本会安装Prometheus与Node Exporter、备份已有Prometheus配置及本脚本曾写入的systemd override、校验规则、写入新override,并仅重启这两个监控服务。备份目录会在脚本结束时打印。它不会重启API、Gateway、数据库、Redis、MinIO或Nginx。9090和9100固定监听`127.0.0.1`。
|
第一个脚本安装Prometheus与Node Exporter,备份已有配置并校验规则;第二个脚本安装PostgreSQL、Redis和Nginx Exporter,开启MinIO回环原生指标。API指标仅监听`127.0.0.1:9464`,Gateway指标复用回环控制端口`8090`。9090、9100、9187、9121、9113和9464均不得对公网开放。两个脚本均会打印恢复资产路径。
|
||||||
|
|
||||||
安装后将下列配置写入`/etc/cmpp-platform/cmpp-platform.env`,再按正常发布窗口重启API:
|
安装后将下列配置写入`/etc/cmpp-platform/cmpp-platform.env`,再按正常发布窗口重启API:
|
||||||
|
|
||||||
@@ -26,6 +27,8 @@ promtool check rules /etc/prometheus/cmpp-alerts.yml
|
|||||||
curl -fsS http://127.0.0.1:9090/-/ready
|
curl -fsS http://127.0.0.1:9090/-/ready
|
||||||
curl -fsS 'http://127.0.0.1:9090/api/v1/query?query=up'
|
curl -fsS 'http://127.0.0.1:9090/api/v1/query?query=up'
|
||||||
ss -lnt | grep -E ':(9090|9100)'
|
ss -lnt | grep -E ':(9090|9100)'
|
||||||
|
curl -fsS http://127.0.0.1:9464/metrics
|
||||||
|
curl -fsS http://127.0.0.1:8090/metrics
|
||||||
```
|
```
|
||||||
|
|
||||||
完整架构、PromQL口径、故障语义和验收标准见`docs/prometheus-system-monitoring-design-20260814.md`。
|
完整架构、PromQL口径、故障语义和验收标准见`docs/prometheus-system-monitoring-design-20260814.md`。
|
||||||
|
|||||||
@@ -1,4 +1,50 @@
|
|||||||
groups:
|
groups:
|
||||||
|
- name: cmpp-service-recording
|
||||||
|
interval: 15s
|
||||||
|
rules:
|
||||||
|
- record: cmpp:service_api:requests_per_second
|
||||||
|
expr: sum(rate(cmpp_api_http_requests_total[5m]))
|
||||||
|
- record: cmpp:service_api:error_percent
|
||||||
|
expr: 100 * sum(rate(cmpp_api_http_requests_total{status=~"5.."}[5m])) / clamp_min(sum(rate(cmpp_api_http_requests_total[5m])), 0.001)
|
||||||
|
- record: cmpp:service_api:latency_p95_seconds
|
||||||
|
expr: histogram_quantile(0.95, sum by (le) (rate(cmpp_api_http_request_duration_seconds_bucket[5m])))
|
||||||
|
- record: cmpp:service_api:event_loop_p99_seconds
|
||||||
|
expr: cmpp_api_nodejs_event_loop_lag_p99_seconds
|
||||||
|
- record: cmpp:service_gateway:submits_per_second
|
||||||
|
expr: sum(rate(cmpp_gateway_submit_total[5m]))
|
||||||
|
- record: cmpp:service_gateway:failure_percent
|
||||||
|
expr: 100 * sum(rate(cmpp_gateway_submit_total{result="failed"}[5m])) / clamp_min(sum(rate(cmpp_gateway_submit_total[5m])), 0.001)
|
||||||
|
- record: cmpp:service_gateway:queue_pending
|
||||||
|
expr: cmpp_gateway_submit_queue_pending
|
||||||
|
- record: cmpp:service_gateway:queue_lag
|
||||||
|
expr: cmpp_gateway_submit_queue_lag
|
||||||
|
- record: cmpp:service_gateway:queue_oldest_seconds
|
||||||
|
expr: cmpp_gateway_submit_queue_oldest_pending_age_seconds
|
||||||
|
- record: cmpp:service_postgresql:connection_percent
|
||||||
|
expr: 100 * sum(pg_stat_activity_count) / clamp_min(max(pg_settings_max_connections), 1)
|
||||||
|
- record: cmpp:service_postgresql:deadlocks_15m
|
||||||
|
expr: sum(increase(pg_stat_database_deadlocks[15m]))
|
||||||
|
- record: cmpp:service_redis:memory_percent
|
||||||
|
expr: (100 * redis_memory_used_bytes / redis_memory_max_bytes) and on(instance) (redis_memory_max_bytes > 0)
|
||||||
|
- record: cmpp:service_redis:memory_used_bytes
|
||||||
|
expr: redis_memory_used_bytes
|
||||||
|
- record: cmpp:service_redis:evictions_5m
|
||||||
|
expr: increase(redis_evicted_keys_total[5m])
|
||||||
|
- record: cmpp:service_redis:connected_clients
|
||||||
|
expr: redis_connected_clients
|
||||||
|
- record: cmpp:service_nginx:connections_active
|
||||||
|
expr: nginx_connections_active
|
||||||
|
- record: cmpp:service_nginx:requests_per_second
|
||||||
|
expr: rate(nginx_http_requests_total[5m])
|
||||||
|
- record: cmpp:service_minio:capacity_percent
|
||||||
|
expr: 100 * (1 - minio_cluster_capacity_usable_free_bytes / minio_cluster_capacity_usable_total_bytes)
|
||||||
|
- record: cmpp:service_minio:usage_bytes
|
||||||
|
expr: minio_cluster_usage_total_bytes
|
||||||
|
- record: cmpp:service_minio:objects
|
||||||
|
expr: minio_cluster_usage_object_total
|
||||||
|
- record: cmpp:service_minio:drives_offline
|
||||||
|
expr: minio_cluster_drive_offline_total
|
||||||
|
|
||||||
- name: cmpp-host-resources
|
- name: cmpp-host-resources
|
||||||
rules:
|
rules:
|
||||||
- alert: NodeExporterDown
|
- alert: NodeExporterDown
|
||||||
@@ -14,28 +60,28 @@ groups:
|
|||||||
threshold: "up = 1"
|
threshold: "up = 1"
|
||||||
|
|
||||||
- alert: HostCpuUsageWarning
|
- alert: HostCpuUsageWarning
|
||||||
expr: (100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 85) and (100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) <= 95)
|
expr: (100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 80) and (100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) <= 90)
|
||||||
for: 10m
|
for: 10m
|
||||||
labels:
|
labels:
|
||||||
severity: warning
|
severity: warning
|
||||||
service: host
|
service: host
|
||||||
annotations:
|
annotations:
|
||||||
summary: CPU使用率持续偏高
|
summary: CPU使用率持续偏高
|
||||||
description: 主机CPU使用率连续10分钟高于85%。
|
description: 主机CPU使用率连续10分钟高于80%。
|
||||||
currentValue: "{{ printf \"%.1f\" $value }}%"
|
currentValue: "{{ printf \"%.1f\" $value }}%"
|
||||||
threshold: "85%"
|
threshold: "80%"
|
||||||
|
|
||||||
- alert: HostCpuUsageCritical
|
- alert: HostCpuUsageCritical
|
||||||
expr: 100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 95
|
expr: 100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 90
|
||||||
for: 5m
|
for: 5m
|
||||||
labels:
|
labels:
|
||||||
severity: critical
|
severity: critical
|
||||||
service: host
|
service: host
|
||||||
annotations:
|
annotations:
|
||||||
summary: CPU使用率严重超限
|
summary: CPU使用率严重超限
|
||||||
description: 主机CPU使用率连续5分钟高于95%。
|
description: 主机CPU使用率连续5分钟高于90%。
|
||||||
currentValue: "{{ printf \"%.1f\" $value }}%"
|
currentValue: "{{ printf \"%.1f\" $value }}%"
|
||||||
threshold: "95%"
|
threshold: "90%"
|
||||||
|
|
||||||
- alert: HostMemoryUsageWarning
|
- alert: HostMemoryUsageWarning
|
||||||
expr: ((1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100 > 85) and ((1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100 <= 95)
|
expr: ((1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100 > 85) and ((1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100 <= 95)
|
||||||
@@ -146,3 +192,158 @@ groups:
|
|||||||
description: "systemd服务 {{ $labels.name }} 连续2分钟未处于active状态。"
|
description: "systemd服务 {{ $labels.name }} 连续2分钟未处于active状态。"
|
||||||
currentValue: "{{ $value }}"
|
currentValue: "{{ $value }}"
|
||||||
threshold: "active = 1"
|
threshold: "active = 1"
|
||||||
|
|
||||||
|
- name: cmpp-api-runtime
|
||||||
|
rules:
|
||||||
|
- alert: CmppApiMetricsDown
|
||||||
|
expr: up{job="cmpp-api"} == 0
|
||||||
|
for: 2m
|
||||||
|
labels: { severity: critical, service: api }
|
||||||
|
annotations: { summary: "API指标采集不可用", description: "Prometheus连续2分钟无法读取API内部指标端点。", currentValue: "{{ $value }}", threshold: "up = 1" }
|
||||||
|
- alert: CmppApiHttpErrorRateWarning
|
||||||
|
expr: (sum(rate(cmpp_api_http_requests_total{status=~"5.."}[5m])) / clamp_min(sum(rate(cmpp_api_http_requests_total[5m])), 0.001) > 0.01) and (sum(rate(cmpp_api_http_requests_total{status=~"5.."}[5m])) / clamp_min(sum(rate(cmpp_api_http_requests_total[5m])), 0.001) <= 0.05) and sum(increase(cmpp_api_http_requests_total{status=~"5.."}[5m])) >= 5
|
||||||
|
for: 5m
|
||||||
|
labels: { severity: warning, service: api }
|
||||||
|
annotations: { summary: "API 5xx错误率偏高", description: "API 5xx错误率连续5分钟高于1%,且窗口内至少5次错误。", currentValue: "{{ printf \"%.2f\" $value }}", threshold: "1%" }
|
||||||
|
- alert: CmppApiHttpErrorRateCritical
|
||||||
|
expr: (sum(rate(cmpp_api_http_requests_total{status=~"5.."}[5m])) / clamp_min(sum(rate(cmpp_api_http_requests_total[5m])), 0.001) > 0.05) and sum(increase(cmpp_api_http_requests_total{status=~"5.."}[5m])) >= 5
|
||||||
|
for: 5m
|
||||||
|
labels: { severity: critical, service: api }
|
||||||
|
annotations: { summary: "API 5xx错误率严重超限", description: "API 5xx错误率连续5分钟高于5%。", currentValue: "{{ printf \"%.2f\" $value }}", threshold: "5%" }
|
||||||
|
- alert: CmppApiLatencyWarning
|
||||||
|
expr: (histogram_quantile(0.95, sum by (le) (rate(cmpp_api_http_request_duration_seconds_bucket[10m]))) > 1) and (histogram_quantile(0.95, sum by (le) (rate(cmpp_api_http_request_duration_seconds_bucket[10m]))) <= 3)
|
||||||
|
for: 10m
|
||||||
|
labels: { severity: warning, service: api }
|
||||||
|
annotations: { summary: "API P95响应偏慢", description: "API P95响应时间连续10分钟超过1秒。", currentValue: "{{ printf \"%.3f\" $value }}s", threshold: "1s" }
|
||||||
|
- alert: CmppApiLatencyCritical
|
||||||
|
expr: histogram_quantile(0.95, sum by (le) (rate(cmpp_api_http_request_duration_seconds_bucket[5m]))) > 3
|
||||||
|
for: 5m
|
||||||
|
labels: { severity: critical, service: api }
|
||||||
|
annotations: { summary: "API P95响应严重超时", description: "API P95响应时间连续5分钟超过3秒。", currentValue: "{{ printf \"%.3f\" $value }}s", threshold: "3s" }
|
||||||
|
- alert: CmppApiEventLoopLagWarning
|
||||||
|
expr: (cmpp_api_nodejs_event_loop_lag_p99_seconds > 0.2) and (cmpp_api_nodejs_event_loop_lag_p99_seconds <= 1)
|
||||||
|
for: 10m
|
||||||
|
labels: { severity: warning, service: api }
|
||||||
|
annotations: { summary: "API事件循环延迟偏高", description: "Node.js事件循环P99延迟连续10分钟超过200ms。", currentValue: "{{ printf \"%.3f\" $value }}s", threshold: "0.2s" }
|
||||||
|
- alert: CmppApiEventLoopLagCritical
|
||||||
|
expr: cmpp_api_nodejs_event_loop_lag_p99_seconds > 1
|
||||||
|
for: 5m
|
||||||
|
labels: { severity: critical, service: api }
|
||||||
|
annotations: { summary: "API事件循环严重阻塞", description: "Node.js事件循环P99延迟连续5分钟超过1秒。", currentValue: "{{ printf \"%.3f\" $value }}s", threshold: "1s" }
|
||||||
|
|
||||||
|
- name: cmpp-gateway-runtime
|
||||||
|
rules:
|
||||||
|
- alert: CmppGatewayMetricsDown
|
||||||
|
expr: up{job="cmpp-gateway"} == 0
|
||||||
|
for: 2m
|
||||||
|
labels: { severity: critical, service: gateway }
|
||||||
|
annotations: { summary: "Gateway指标采集不可用", description: "Prometheus连续2分钟无法读取Gateway指标。", currentValue: "{{ $value }}", threshold: "up = 1" }
|
||||||
|
- alert: CmppGatewaySubmitWorkerDown
|
||||||
|
expr: cmpp_gateway_submit_worker_up == 0
|
||||||
|
for: 1m
|
||||||
|
labels: { severity: critical, service: gateway }
|
||||||
|
annotations: { summary: "Gateway提交消费者未运行", description: "Gateway进程存活,但提交消费者未成功初始化。", currentValue: "{{ $value }}", threshold: "1" }
|
||||||
|
- alert: CmppGatewayUpstreamConnectionShortage
|
||||||
|
expr: cmpp_gateway_upstream_connections{state="connected"} < cmpp_gateway_upstream_connections{state="desired"}
|
||||||
|
for: 2m
|
||||||
|
labels: { severity: critical, service: gateway }
|
||||||
|
annotations: { summary: "Gateway上游连接不足", description: "实际上游CMPP连接数连续2分钟低于期望数。", currentValue: "{{ $value }}", threshold: "connected = desired" }
|
||||||
|
- alert: CmppGatewayQueueDelayedWarning
|
||||||
|
expr: (cmpp_gateway_submit_queue_oldest_pending_age_seconds > 30) and (cmpp_gateway_submit_queue_oldest_pending_age_seconds <= 120)
|
||||||
|
for: 2m
|
||||||
|
labels: { severity: warning, service: gateway }
|
||||||
|
annotations: { summary: "Gateway提交队列开始延迟", description: "Redis Stream最旧pending消息等待超过30秒。", currentValue: "{{ printf \"%.0f\" $value }}s", threshold: "30s" }
|
||||||
|
- alert: CmppGatewayQueueDelayedCritical
|
||||||
|
expr: cmpp_gateway_submit_queue_oldest_pending_age_seconds > 120
|
||||||
|
for: 2m
|
||||||
|
labels: { severity: critical, service: gateway }
|
||||||
|
annotations: { summary: "Gateway提交队列严重延迟", description: "Redis Stream最旧pending消息等待超过120秒。", currentValue: "{{ printf \"%.0f\" $value }}s", threshold: "120s" }
|
||||||
|
|
||||||
|
- name: cmpp-data-services
|
||||||
|
rules:
|
||||||
|
- alert: PostgresExporterDown
|
||||||
|
expr: up{job="postgresql"} == 0 or pg_up == 0
|
||||||
|
for: 2m
|
||||||
|
labels: { severity: critical, service: postgresql }
|
||||||
|
annotations: { summary: "PostgreSQL指标或数据库不可用", description: "PostgreSQL Exporter或其数据库连接连续2分钟不可用。", currentValue: "{{ $value }}", threshold: "up = 1" }
|
||||||
|
- alert: PostgresConnectionsWarning
|
||||||
|
expr: (sum(pg_stat_activity_count) / clamp_min(max(pg_settings_max_connections), 1) > 0.70) and (sum(pg_stat_activity_count) / clamp_min(max(pg_settings_max_connections), 1) <= 0.85)
|
||||||
|
for: 10m
|
||||||
|
labels: { severity: warning, service: postgresql }
|
||||||
|
annotations: { summary: "PostgreSQL连接使用率偏高", description: "数据库连接数连续10分钟超过上限的70%。", currentValue: "{{ printf \"%.2f\" $value }}", threshold: "70%" }
|
||||||
|
- alert: PostgresConnectionsCritical
|
||||||
|
expr: sum(pg_stat_activity_count) / clamp_min(max(pg_settings_max_connections), 1) > 0.85
|
||||||
|
for: 5m
|
||||||
|
labels: { severity: critical, service: postgresql }
|
||||||
|
annotations: { summary: "PostgreSQL连接即将耗尽", description: "数据库连接数连续5分钟超过上限的85%。", currentValue: "{{ printf \"%.2f\" $value }}", threshold: "85%" }
|
||||||
|
- alert: PostgresDeadlocksDetected
|
||||||
|
expr: sum(increase(pg_stat_database_deadlocks[15m])) > 0
|
||||||
|
for: 1m
|
||||||
|
labels: { severity: warning, service: postgresql }
|
||||||
|
annotations: { summary: "PostgreSQL发生死锁", description: "15分钟窗口内检测到数据库死锁。", currentValue: "{{ $value }}", threshold: "0" }
|
||||||
|
- alert: RedisExporterDown
|
||||||
|
expr: up{job="redis"} == 0 or redis_up == 0
|
||||||
|
for: 2m
|
||||||
|
labels: { severity: critical, service: redis }
|
||||||
|
annotations: { summary: "Redis指标或服务不可用", description: "Redis Exporter或Redis连接连续2分钟不可用。", currentValue: "{{ $value }}", threshold: "up = 1" }
|
||||||
|
- alert: RedisMemoryWarning
|
||||||
|
expr: (redis_memory_max_bytes > 0) and (redis_memory_used_bytes / redis_memory_max_bytes > 0.70) and (redis_memory_used_bytes / redis_memory_max_bytes <= 0.85)
|
||||||
|
for: 10m
|
||||||
|
labels: { severity: warning, service: redis }
|
||||||
|
annotations: { summary: "Redis内存使用率偏高", description: "Redis内存连续10分钟超过maxmemory的70%。", currentValue: "{{ printf \"%.2f\" $value }}", threshold: "70%" }
|
||||||
|
- alert: RedisMemoryCritical
|
||||||
|
expr: (redis_memory_max_bytes > 0) and (redis_memory_used_bytes / redis_memory_max_bytes > 0.85)
|
||||||
|
for: 5m
|
||||||
|
labels: { severity: critical, service: redis }
|
||||||
|
annotations: { summary: "Redis内存即将耗尽", description: "Redis内存连续5分钟超过maxmemory的85%。", currentValue: "{{ printf \"%.2f\" $value }}", threshold: "85%" }
|
||||||
|
- alert: RedisUnexpectedEvictions
|
||||||
|
expr: increase(redis_evicted_keys_total[5m]) > 0
|
||||||
|
for: 1m
|
||||||
|
labels: { severity: critical, service: redis }
|
||||||
|
annotations: { summary: "Redis发生Key淘汰", description: "Redis承载队列和运行状态,5分钟内不应出现淘汰。", currentValue: "{{ $value }}", threshold: "0" }
|
||||||
|
- alert: RedisRejectedConnections
|
||||||
|
expr: increase(redis_rejected_connections_total[5m]) > 0
|
||||||
|
for: 1m
|
||||||
|
labels: { severity: critical, service: redis }
|
||||||
|
annotations: { summary: "Redis拒绝连接", description: "5分钟内Redis出现被拒绝连接。", currentValue: "{{ $value }}", threshold: "0" }
|
||||||
|
|
||||||
|
- name: cmpp-storage-and-edge
|
||||||
|
rules:
|
||||||
|
- alert: MinioMetricsDown
|
||||||
|
expr: up{job="minio"} == 0
|
||||||
|
for: 2m
|
||||||
|
labels: { severity: critical, service: minio }
|
||||||
|
annotations: { summary: "MinIO指标采集不可用", description: "Prometheus连续2分钟无法读取MinIO原生指标。", currentValue: "{{ $value }}", threshold: "up = 1" }
|
||||||
|
- alert: MinioCapacityWarning
|
||||||
|
expr: (100 * (1 - minio_cluster_capacity_usable_free_bytes / minio_cluster_capacity_usable_total_bytes) > 80) and (100 * (1 - minio_cluster_capacity_usable_free_bytes / minio_cluster_capacity_usable_total_bytes) <= 90)
|
||||||
|
for: 15m
|
||||||
|
labels: { severity: warning, service: minio }
|
||||||
|
annotations: { summary: "MinIO存储容量偏高", description: "MinIO可用容量使用率连续15分钟超过80%。", currentValue: "{{ printf \"%.1f\" $value }}%", threshold: "80%" }
|
||||||
|
- alert: MinioCapacityCritical
|
||||||
|
expr: 100 * (1 - minio_cluster_capacity_usable_free_bytes / minio_cluster_capacity_usable_total_bytes) > 90
|
||||||
|
for: 5m
|
||||||
|
labels: { severity: critical, service: minio }
|
||||||
|
annotations: { summary: "MinIO存储容量即将耗尽", description: "MinIO可用容量使用率连续5分钟超过90%。", currentValue: "{{ printf \"%.1f\" $value }}%", threshold: "90%" }
|
||||||
|
- alert: MinioDriveOffline
|
||||||
|
expr: minio_cluster_drive_offline_total > 0
|
||||||
|
for: 1m
|
||||||
|
labels: { severity: critical, service: minio }
|
||||||
|
annotations: { summary: "MinIO存储盘离线", description: "MinIO检测到离线存储盘。", currentValue: "{{ $value }}", threshold: "0" }
|
||||||
|
- alert: NginxExporterDown
|
||||||
|
expr: up{job="nginx"} == 0 or nginx_up == 0
|
||||||
|
for: 2m
|
||||||
|
labels: { severity: critical, service: nginx }
|
||||||
|
annotations: { summary: "Nginx指标或状态页不可用", description: "Nginx Exporter或回环stub_status连续2分钟不可用。", currentValue: "{{ $value }}", threshold: "up = 1" }
|
||||||
|
|
||||||
|
- name: cmpp-monitoring-self
|
||||||
|
rules:
|
||||||
|
- alert: PrometheusScrapeSlow
|
||||||
|
expr: scrape_duration_seconds / scrape_interval_seconds > 0.8
|
||||||
|
for: 5m
|
||||||
|
labels: { severity: warning, service: prometheus }
|
||||||
|
annotations: { summary: "Prometheus采集接近超时", description: "采集耗时连续5分钟超过采集周期的80%。", currentValue: "{{ printf \"%.2f\" $value }}", threshold: "80%" }
|
||||||
|
- alert: PrometheusRuleEvaluationFailures
|
||||||
|
expr: increase(prometheus_rule_evaluation_failures_total[5m]) > 0
|
||||||
|
for: 1m
|
||||||
|
labels: { severity: critical, service: prometheus }
|
||||||
|
annotations: { summary: "Prometheus告警规则计算失败", description: "5分钟内出现告警规则计算失败。", currentValue: "{{ $value }}", threshold: "0" }
|
||||||
|
|||||||
@@ -0,0 +1,92 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
set -Eeuo pipefail
|
||||||
|
|
||||||
|
if [[ "$(id -u)" -ne 0 ]]; then echo "Run as root." >&2; exit 1; fi
|
||||||
|
for command_name in apt-get systemctl nginx curl; do command -v "$command_name" >/dev/null || { echo "Missing command: $command_name" >&2; exit 1; }; done
|
||||||
|
|
||||||
|
log() { printf '\n[%s] %s\n' "$(date '+%F %T')" "$*"; }
|
||||||
|
backup_dir="/etc/prometheus/cmpp-backups/$(date '+%Y%m%d-%H%M%S')-service-exporters"
|
||||||
|
mkdir -p "$backup_dir"
|
||||||
|
|
||||||
|
for config_file in /etc/cmpp-platform/minio.env /etc/nginx/conf.d/cmpp-monitoring-status.conf /etc/cmpp-platform/monitoring-exporters.env; do
|
||||||
|
[[ -f "$config_file" ]] && cp --preserve=mode,timestamps "$config_file" "$backup_dir/$(basename "$config_file")"
|
||||||
|
done
|
||||||
|
|
||||||
|
log "Installing bounded service exporters"
|
||||||
|
apt-get update
|
||||||
|
DEBIAN_FRONTEND=noninteractive apt-get install -y prometheus-postgres-exporter prometheus-redis-exporter prometheus-nginx-exporter
|
||||||
|
|
||||||
|
[[ -f /etc/cmpp-platform/cmpp-platform.env ]] || { echo "Missing platform environment file." >&2; exit 1; }
|
||||||
|
set -a
|
||||||
|
# shellcheck disable=SC1091
|
||||||
|
. /etc/cmpp-platform/cmpp-platform.env
|
||||||
|
set +a
|
||||||
|
database_base="${DATABASE_URL%%\?*}"
|
||||||
|
cat >/etc/cmpp-platform/monitoring-exporters.env <<EOF
|
||||||
|
DATA_SOURCE_NAME=${database_base}?sslmode=disable
|
||||||
|
REDIS_ADDR=redis://${REDIS_HOST:-127.0.0.1}:${REDIS_PORT:-6379}
|
||||||
|
EOF
|
||||||
|
chmod 0600 /etc/cmpp-platform/monitoring-exporters.env
|
||||||
|
|
||||||
|
write_override() {
|
||||||
|
local unit="$1" executable="$2" arguments="$3"
|
||||||
|
local directory="/etc/systemd/system/${unit}.service.d"
|
||||||
|
mkdir -p "$directory"
|
||||||
|
[[ -f "$directory/cmpp-monitoring.conf" ]] && cp --preserve=mode,timestamps "$directory/cmpp-monitoring.conf" "$backup_dir/${unit}-override.conf"
|
||||||
|
cat >"$directory/cmpp-monitoring.conf" <<EOF
|
||||||
|
[Service]
|
||||||
|
EnvironmentFile=/etc/cmpp-platform/monitoring-exporters.env
|
||||||
|
ExecStart=
|
||||||
|
ExecStart=${executable} ${arguments}
|
||||||
|
EOF
|
||||||
|
}
|
||||||
|
|
||||||
|
postgres_exporter="$(command -v prometheus-postgres-exporter)"
|
||||||
|
redis_exporter="$(command -v prometheus-redis-exporter)"
|
||||||
|
nginx_exporter="$(command -v prometheus-nginx-exporter)"
|
||||||
|
write_override prometheus-postgres-exporter "$postgres_exporter" '--web.listen-address=127.0.0.1:9187'
|
||||||
|
write_override prometheus-redis-exporter "$redis_exporter" '--web.listen-address=127.0.0.1:9121'
|
||||||
|
|
||||||
|
cat >/etc/nginx/conf.d/cmpp-monitoring-status.conf <<'EOF'
|
||||||
|
server {
|
||||||
|
listen 127.0.0.1:8088;
|
||||||
|
server_name localhost;
|
||||||
|
access_log off;
|
||||||
|
location = /stub_status {
|
||||||
|
stub_status;
|
||||||
|
allow 127.0.0.1;
|
||||||
|
allow ::1;
|
||||||
|
deny all;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
EOF
|
||||||
|
write_override prometheus-nginx-exporter "$nginx_exporter" '--web.listen-address=127.0.0.1:9113 --nginx.scrape-uri=http://127.0.0.1:8088/stub_status'
|
||||||
|
|
||||||
|
# MinIO exposes only operational aggregates and listens on loopback; public auth here does not expose objects or credentials.
|
||||||
|
grep -q '^MINIO_PROMETHEUS_AUTH_TYPE=' /etc/cmpp-platform/minio.env \
|
||||||
|
&& sed -i 's/^MINIO_PROMETHEUS_AUTH_TYPE=.*/MINIO_PROMETHEUS_AUTH_TYPE=public/' /etc/cmpp-platform/minio.env \
|
||||||
|
|| printf '\nMINIO_PROMETHEUS_AUTH_TYPE=public\n' >>/etc/cmpp-platform/minio.env
|
||||||
|
|
||||||
|
nginx -t
|
||||||
|
systemctl daemon-reload
|
||||||
|
systemctl enable prometheus-postgres-exporter prometheus-redis-exporter prometheus-nginx-exporter
|
||||||
|
systemctl restart prometheus-postgres-exporter prometheus-redis-exporter prometheus-nginx-exporter
|
||||||
|
systemctl restart cmpp-minio
|
||||||
|
systemctl reload nginx
|
||||||
|
|
||||||
|
wait_for_http() {
|
||||||
|
local endpoint="$1" attempt
|
||||||
|
for attempt in $(seq 1 30); do
|
||||||
|
curl -fsS "$endpoint" >/dev/null 2>&1 && return 0
|
||||||
|
sleep 1
|
||||||
|
done
|
||||||
|
echo "Monitoring endpoint did not become ready: $endpoint" >&2
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
for endpoint in 127.0.0.1:9187 127.0.0.1:9121 127.0.0.1:9113; do wait_for_http "http://${endpoint}/metrics"; done
|
||||||
|
wait_for_http http://127.0.0.1:9000/minio/v2/metrics/cluster
|
||||||
|
if ss -lnt | grep -Eq '(^|[[:space:]])(0\.0\.0\.0|\[::\]):(9187|9121|9113)([[:space:]]|$)'; then
|
||||||
|
echo "A service exporter unexpectedly listens on a wildcard address." >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
log "Service exporters are ready on loopback only; backup: $backup_dir"
|
||||||
@@ -18,3 +18,29 @@ scrape_configs:
|
|||||||
- targets: [127.0.0.1:9100]
|
- targets: [127.0.0.1:9100]
|
||||||
labels:
|
labels:
|
||||||
host: cmpp-primary
|
host: cmpp-primary
|
||||||
|
|
||||||
|
- job_name: cmpp-api
|
||||||
|
static_configs:
|
||||||
|
- targets: [127.0.0.1:9464]
|
||||||
|
|
||||||
|
- job_name: cmpp-gateway
|
||||||
|
metrics_path: /metrics
|
||||||
|
static_configs:
|
||||||
|
- targets: [127.0.0.1:8090]
|
||||||
|
|
||||||
|
- job_name: postgresql
|
||||||
|
static_configs:
|
||||||
|
- targets: [127.0.0.1:9187]
|
||||||
|
|
||||||
|
- job_name: redis
|
||||||
|
static_configs:
|
||||||
|
- targets: [127.0.0.1:9121]
|
||||||
|
|
||||||
|
- job_name: minio
|
||||||
|
metrics_path: /minio/v2/metrics/cluster
|
||||||
|
static_configs:
|
||||||
|
- targets: [127.0.0.1:9000]
|
||||||
|
|
||||||
|
- job_name: nginx
|
||||||
|
static_configs:
|
||||||
|
- targets: [127.0.0.1:9113]
|
||||||
|
|||||||
@@ -4,7 +4,8 @@ set -Eeuo pipefail
|
|||||||
APP_DIR="${APP_DIR:-/opt/cmpp-platform}"
|
APP_DIR="${APP_DIR:-/opt/cmpp-platform}"
|
||||||
if [[ "$(id -u)" -ne 0 ]]; then echo "Run as root." >&2; exit 1; fi
|
if [[ "$(id -u)" -ne 0 ]]; then echo "Run as root." >&2; exit 1; fi
|
||||||
for command_name in fail2ban-client nft nginx systemctl; do command -v "$command_name" >/dev/null || { echo "Missing command: $command_name" >&2; exit 1; }; done
|
for command_name in fail2ban-client nft nginx systemctl; do command -v "$command_name" >/dev/null || { echo "Missing command: $command_name" >&2; exit 1; }; done
|
||||||
[[ -x "$APP_DIR/dist/cmpp-security-agent" ]] || { echo "Missing built security agent" >&2; exit 1; }
|
agent_binary="$APP_DIR/dist/cmpp-security-agent"
|
||||||
|
[[ -x "$agent_binary" ]] || { echo "Missing built security agent: $agent_binary" >&2; exit 1; }
|
||||||
|
|
||||||
getent group cmpp-security >/dev/null || groupadd --system cmpp-security
|
getent group cmpp-security >/dev/null || groupadd --system cmpp-security
|
||||||
id cmpp-api >/dev/null 2>&1 || useradd --system --home-dir /nonexistent --shell /usr/sbin/nologin cmpp-api
|
id cmpp-api >/dev/null 2>&1 || useradd --system --home-dir /nonexistent --shell /usr/sbin/nologin cmpp-api
|
||||||
@@ -14,7 +15,8 @@ install -d -o root -g cmpp-security -m 0750 /var/lib/cmpp-security-agent
|
|||||||
install -d -o cmpp-api -g cmpp-security -m 0750 "$APP_DIR/logs/api"
|
install -d -o cmpp-api -g cmpp-security -m 0750 "$APP_DIR/logs/api"
|
||||||
[[ -d /var/lib/cmpp-platform/object-storage ]] && chown -R cmpp-api:cmpp-security /var/lib/cmpp-platform/object-storage
|
[[ -d /var/lib/cmpp-platform/object-storage ]] && chown -R cmpp-api:cmpp-security /var/lib/cmpp-platform/object-storage
|
||||||
|
|
||||||
install -m 0640 "$APP_DIR/deploy/security/cmpp-report-only.conf" /etc/fail2ban/action.d/cmpp-report-only.conf
|
sed "s#@CMPP_SECURITY_AGENT_BIN@#$agent_binary#g" "$APP_DIR/deploy/security/cmpp-report-only.conf" >/etc/fail2ban/action.d/cmpp-report-only.conf
|
||||||
|
chmod 0640 /etc/fail2ban/action.d/cmpp-report-only.conf
|
||||||
install -m 0640 "$APP_DIR/deploy/security/cmpp-http-scan.conf" /etc/fail2ban/filter.d/cmpp-http-scan.conf
|
install -m 0640 "$APP_DIR/deploy/security/cmpp-http-scan.conf" /etc/fail2ban/filter.d/cmpp-http-scan.conf
|
||||||
install -d -m 0750 /etc/nginx/snippets /etc/nftables.d
|
install -d -m 0750 /etc/nginx/snippets /etc/nftables.d
|
||||||
touch /etc/nginx/snippets/cmpp-security-deny.conf
|
touch /etc/nginx/snippets/cmpp-security-deny.conf
|
||||||
@@ -31,7 +33,11 @@ grep -q 'cmpp-security.nft' /etc/nftables.conf || printf '\ninclude "/etc/nftabl
|
|||||||
nft -c -f /etc/nftables.conf
|
nft -c -f /etc/nftables.conf
|
||||||
nft list table inet cmpp_security >/dev/null 2>&1 || nft -f /etc/nftables.d/cmpp-security.nft
|
nft list table inet cmpp_security >/dev/null 2>&1 || nft -f /etc/nftables.d/cmpp-security.nft
|
||||||
|
|
||||||
sed "s#/opt/cmpp-platform/current#$APP_DIR#g" "$APP_DIR/deploy/security/cmpp-security-agent.service" >/etc/systemd/system/cmpp-security-agent.service
|
sed "s#@CMPP_SECURITY_AGENT_BIN@#$agent_binary#g" "$APP_DIR/deploy/security/cmpp-security-agent.service" >/etc/systemd/system/cmpp-security-agent.service
|
||||||
|
if grep -Rqs '@CMPP_SECURITY_AGENT_BIN@' /etc/systemd/system/cmpp-security-agent.service /etc/fail2ban/action.d/cmpp-report-only.conf; then
|
||||||
|
echo "Security agent executable placeholder was not rendered." >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
install -d -m 0755 /etc/systemd/system/cmpp-api.service.d
|
install -d -m 0755 /etc/systemd/system/cmpp-api.service.d
|
||||||
cat >/etc/systemd/system/cmpp-api.service.d/security-boundary.conf <<EOF
|
cat >/etc/systemd/system/cmpp-api.service.d/security-boundary.conf <<EOF
|
||||||
[Service]
|
[Service]
|
||||||
|
|||||||
@@ -0,0 +1,30 @@
|
|||||||
|
import { readFileSync } from 'node:fs';
|
||||||
|
import { resolve } from 'node:path';
|
||||||
|
|
||||||
|
const root = resolve(import.meta.dirname, '..', '..');
|
||||||
|
const read = (relativePath) => readFileSync(resolve(root, relativePath), 'utf8');
|
||||||
|
const installer = read('tools/security/install-security-agent.sh');
|
||||||
|
const service = read('deploy/security/cmpp-security-agent.service');
|
||||||
|
const action = read('deploy/security/cmpp-report-only.conf');
|
||||||
|
const placeholder = '@CMPP_SECURITY_AGENT_BIN@';
|
||||||
|
|
||||||
|
for (const [label, source] of [['systemd service', service], ['Fail2ban action', action]]) {
|
||||||
|
if (!source.includes(placeholder)) throw new Error(`${label} is missing the security-agent executable placeholder`);
|
||||||
|
if (source.includes('/opt/cmpp-platform/current/bin/cmpp-security-agent')) {
|
||||||
|
throw new Error(`${label} still references the removed current/bin deployment layout`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!installer.includes('agent_binary="$APP_DIR/dist/cmpp-security-agent"')) {
|
||||||
|
throw new Error('installer does not bind the security agent to the built dist executable');
|
||||||
|
}
|
||||||
|
for (const target of ['cmpp-security-agent.service', 'cmpp-report-only.conf']) {
|
||||||
|
if (!installer.includes(`sed "s#${placeholder}#$agent_binary#g"`) || !installer.includes(target)) {
|
||||||
|
throw new Error(`installer does not render ${target} with the built security-agent executable`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (!installer.includes("grep -Rqs '@CMPP_SECURITY_AGENT_BIN@'")) {
|
||||||
|
throw new Error('installer does not fail closed when an executable placeholder remains');
|
||||||
|
}
|
||||||
|
|
||||||
|
console.log('Security deployment verified: systemd and Fail2ban use the built dist security-agent executable.');
|
||||||
Reference in New Issue
Block a user