9 Commits
86 changed files with 5553 additions and 68 deletions
+3
View File
@@ -25,6 +25,9 @@ OPERATION_LOG_ARCHIVE_INTERVAL_MS=86400000
SMS_RECEIPT_TIMEOUT_SCAN_ENABLED=true SMS_RECEIPT_TIMEOUT_SCAN_ENABLED=true
SMS_RECEIPT_TIMEOUT_HOURS=72 SMS_RECEIPT_TIMEOUT_HOURS=72
SMS_RECEIPT_TIMEOUT_SCAN_INTERVAL_MS=300000 SMS_RECEIPT_TIMEOUT_SCAN_INTERVAL_MS=300000
# System monitoring reads only fixed queries from a loopback Prometheus instance.
PROMETHEUS_URL=http://127.0.0.1:9090
PROMETHEUS_QUERY_TIMEOUT_MS=5000
# Local HTTP development only. Production must use HTTPS and true. # Local HTTP development only. Production must use HTTPS and true.
SESSION_COOKIE_SECURE=false SESSION_COOKIE_SECURE=false
MINIO_ENDPOINT=localhost:9000 MINIO_ENDPOINT=localhost:9000
@@ -0,0 +1,80 @@
CREATE TABLE "SecurityDetectionRule" (
"id" TEXT NOT NULL,
"code" TEXT NOT NULL,
"name" TEXT NOT NULL,
"sourceType" TEXT NOT NULL,
"enabled" BOOLEAN NOT NULL DEFAULT true,
"threshold" INTEGER NOT NULL,
"windowSeconds" INTEGER NOT NULL,
"cooldownSeconds" INTEGER NOT NULL,
"severity" TEXT NOT NULL,
"defaultBlockSeconds" INTEGER NOT NULL,
"maximumBlockSeconds" INTEGER NOT NULL,
"configVersion" INTEGER NOT NULL DEFAULT 1,
"effectiveVersion" INTEGER NOT NULL DEFAULT 0,
"applyStatus" TEXT NOT NULL DEFAULT 'pending',
"lastApplyError" TEXT,
"pendingConfig" JSONB,
"createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
"updatedAt" TIMESTAMP(3) NOT NULL,
CONSTRAINT "SecurityDetectionRule_pkey" PRIMARY KEY ("id")
);
CREATE TABLE "SecurityDetectionEvent" (
"id" TEXT NOT NULL, "eventKey" TEXT NOT NULL, "ruleId" TEXT NOT NULL,
"sourceIp" TEXT NOT NULL, "sourcePort" INTEGER, "accountHash" TEXT,
"path" TEXT, "protocol" TEXT, "resultCode" TEXT, "evidence" JSONB,
"occurredAt" TIMESTAMP(3) NOT NULL, "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
CONSTRAINT "SecurityDetectionEvent_pkey" PRIMARY KEY ("id")
);
CREATE TABLE "SecurityAlert" (
"id" TEXT NOT NULL, "fingerprint" TEXT NOT NULL, "ruleId" TEXT NOT NULL,
"sourceIp" TEXT NOT NULL, "severity" TEXT NOT NULL, "status" TEXT NOT NULL DEFAULT 'open',
"eventCount" INTEGER NOT NULL DEFAULT 0, "windowStartedAt" TIMESTAMP(3) NOT NULL,
"firstOccurredAt" TIMESTAMP(3) NOT NULL, "lastOccurredAt" TIMESTAMP(3) NOT NULL,
"acknowledgedAt" TIMESTAMP(3), "acknowledgedById" TEXT, "ignoredAt" TIMESTAMP(3),
"ignoredById" TEXT, "ignoreReason" TEXT, "blockId" TEXT,
"createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, "updatedAt" TIMESTAMP(3) NOT NULL,
CONSTRAINT "SecurityAlert_pkey" PRIMARY KEY ("id")
);
CREATE TABLE "SecurityBlock" (
"id" TEXT NOT NULL, "operationKey" TEXT NOT NULL, "alertId" TEXT, "sourceIp" TEXT NOT NULL,
"executor" TEXT NOT NULL, "status" TEXT NOT NULL DEFAULT 'requested', "durationSeconds" INTEGER NOT NULL,
"reason" TEXT NOT NULL, "requestedById" TEXT NOT NULL, "requestedAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
"appliedAt" TIMESTAMP(3), "expiresAt" TIMESTAMP(3), "releasedAt" TIMESTAMP(3), "releasedById" TEXT,
"executorReference" TEXT, "lastError" TEXT, "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
"updatedAt" TIMESTAMP(3) NOT NULL, CONSTRAINT "SecurityBlock_pkey" PRIMARY KEY ("id")
);
CREATE TABLE "SecurityProtectedNetwork" (
"id" TEXT NOT NULL, "network" TEXT NOT NULL, "name" TEXT NOT NULL, "reason" TEXT NOT NULL,
"enabled" BOOLEAN NOT NULL DEFAULT true, "createdById" TEXT NOT NULL,
"createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, "updatedAt" TIMESTAMP(3) NOT NULL,
CONSTRAINT "SecurityProtectedNetwork_pkey" PRIMARY KEY ("id")
);
CREATE UNIQUE INDEX "SecurityDetectionRule_code_key" ON "SecurityDetectionRule"("code");
CREATE INDEX "SecurityDetectionRule_enabled_sourceType_idx" ON "SecurityDetectionRule"("enabled", "sourceType");
CREATE UNIQUE INDEX "SecurityDetectionEvent_eventKey_key" ON "SecurityDetectionEvent"("eventKey");
CREATE INDEX "SecurityDetectionEvent_ruleId_occurredAt_idx" ON "SecurityDetectionEvent"("ruleId", "occurredAt");
CREATE INDEX "SecurityDetectionEvent_sourceIp_occurredAt_idx" ON "SecurityDetectionEvent"("sourceIp", "occurredAt");
CREATE UNIQUE INDEX "SecurityAlert_fingerprint_key" ON "SecurityAlert"("fingerprint");
CREATE INDEX "SecurityAlert_status_severity_lastOccurredAt_idx" ON "SecurityAlert"("status", "severity", "lastOccurredAt");
CREATE INDEX "SecurityAlert_sourceIp_status_lastOccurredAt_idx" ON "SecurityAlert"("sourceIp", "status", "lastOccurredAt");
CREATE INDEX "SecurityAlert_ruleId_status_lastOccurredAt_idx" ON "SecurityAlert"("ruleId", "status", "lastOccurredAt");
CREATE UNIQUE INDEX "SecurityBlock_operationKey_key" ON "SecurityBlock"("operationKey");
CREATE INDEX "SecurityBlock_status_expiresAt_idx" ON "SecurityBlock"("status", "expiresAt");
CREATE INDEX "SecurityBlock_sourceIp_status_requestedAt_idx" ON "SecurityBlock"("sourceIp", "status", "requestedAt");
CREATE INDEX "SecurityBlock_alertId_idx" ON "SecurityBlock"("alertId");
CREATE UNIQUE INDEX "SecurityProtectedNetwork_network_key" ON "SecurityProtectedNetwork"("network");
CREATE INDEX "SecurityProtectedNetwork_enabled_createdAt_idx" ON "SecurityProtectedNetwork"("enabled", "createdAt");
ALTER TABLE "SecurityDetectionEvent" ADD CONSTRAINT "SecurityDetectionEvent_ruleId_fkey" FOREIGN KEY ("ruleId") REFERENCES "SecurityDetectionRule"("id") ON DELETE RESTRICT ON UPDATE CASCADE;
ALTER TABLE "SecurityAlert" ADD CONSTRAINT "SecurityAlert_ruleId_fkey" FOREIGN KEY ("ruleId") REFERENCES "SecurityDetectionRule"("id") ON DELETE RESTRICT ON UPDATE CASCADE;
INSERT INTO "SecurityDetectionRule" ("id", "code", "name", "sourceType", "threshold", "windowSeconds", "cooldownSeconds", "severity", "defaultBlockSeconds", "maximumBlockSeconds", "configVersion", "effectiveVersion", "applyStatus", "updatedAt") VALUES
('sec_admin_login', 'admin_login_failure', '运营端登录失败', 'application', 8, 600, 900, 'medium', 3600, 604800, 1, 1, 'effective', CURRENT_TIMESTAMP),
('sec_client_login', 'client_login_failure', '客户端登录失败', 'application', 8, 600, 900, 'medium', 3600, 604800, 1, 1, 'effective', CURRENT_TIMESTAMP),
('sec_ssh_auth', 'ssh_auth_failure', 'SSH认证失败', 'fail2ban', 6, 600, 1800, 'high', 86400, 604800, 1, 1, 'effective', CURRENT_TIMESTAMP),
('sec_cmpp_auth', 'cmpp_auth_failure', 'CMPP认证失败', 'gateway', 5, 300, 900, 'high', 3600, 604800, 1, 1, 'effective', CURRENT_TIMESTAMP),
('sec_cmpp_abuse', 'cmpp_protocol_abuse', 'CMPP协议滥用', 'gateway', 20, 60, 900, 'critical', 3600, 604800, 1, 1, 'effective', CURRENT_TIMESTAMP),
('sec_http_key', 'http_invalid_api_key', 'HTTP错误密钥', 'application', 10, 300, 900, 'high', 3600, 604800, 1, 1, 'effective', CURRENT_TIMESTAMP),
('sec_http_sign', 'http_signature_failure', 'HTTP签名错误', 'application', 10, 300, 900, 'high', 3600, 604800, 1, 1, 'effective', CURRENT_TIMESTAMP),
('sec_http_replay', 'http_replay_attempt', 'HTTP重放尝试', 'application', 3, 600, 1800, 'critical', 86400, 604800, 1, 1, 'effective', CURRENT_TIMESTAMP),
('sec_http_scan', 'http_malicious_scan', 'HTTP恶意扫描', 'fail2ban', 20, 60, 900, 'high', 3600, 604800, 1, 1, 'effective', CURRENT_TIMESTAMP);
@@ -0,0 +1,23 @@
CREATE TABLE "InfrastructureAlertSetting" (
"id" TEXT NOT NULL DEFAULT 'global',
"configVersion" INTEGER NOT NULL DEFAULT 1,
"effectiveVersion" INTEGER NOT NULL DEFAULT 1,
"thresholds" JSONB NOT NULL,
"effectiveThresholds" JSONB NOT NULL,
"applyStatus" TEXT NOT NULL DEFAULT 'effective',
"lastError" TEXT,
"updatedById" TEXT,
"appliedAt" TIMESTAMP(3),
"createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
"updatedAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
CONSTRAINT "InfrastructureAlertSetting_pkey" PRIMARY KEY ("id")
);
INSERT INTO "InfrastructureAlertSetting" (
"id", "thresholds", "effectiveThresholds", "appliedAt"
) VALUES (
'global',
'{"hostCpu":{"warning":80,"critical":90},"hostMemory":{"warning":85,"critical":95},"hostDisk":{"warning":80,"critical":90},"apiError":{"warning":1,"critical":5},"apiLatency":{"warning":1,"critical":3},"apiEventLoop":{"warning":0.2,"critical":1},"gatewayQueue":{"warning":30,"critical":120},"postgresConnections":{"warning":70,"critical":85},"redisMemory":{"warning":70,"critical":85},"minioCapacity":{"warning":80,"critical":90}}'::jsonb,
'{"hostCpu":{"warning":80,"critical":90},"hostMemory":{"warning":85,"critical":95},"hostDisk":{"warning":80,"critical":90},"apiError":{"warning":1,"critical":5},"apiLatency":{"warning":1,"critical":3},"apiEventLoop":{"warning":0.2,"critical":1},"gatewayQueue":{"warning":30,"critical":120},"postgresConnections":{"warning":70,"critical":85},"redisMemory":{"warning":70,"critical":85},"minioCapacity":{"warning":80,"critical":90}}'::jsonb,
CURRENT_TIMESTAMP
);
@@ -0,0 +1,17 @@
CREATE TABLE "InfrastructureAlertRead" (
"id" TEXT NOT NULL,
"fingerprint" TEXT NOT NULL,
"activeAt" TIMESTAMP(3) NOT NULL,
"userId" TEXT NOT NULL,
"readAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
"createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
"updatedAt" TIMESTAMP(3) NOT NULL,
CONSTRAINT "InfrastructureAlertRead_pkey" PRIMARY KEY ("id"),
CONSTRAINT "InfrastructureAlertRead_userId_fkey" FOREIGN KEY ("userId") REFERENCES "User"("id") ON DELETE CASCADE ON UPDATE CASCADE
);
CREATE UNIQUE INDEX "InfrastructureAlertRead_fingerprint_userId_key"
ON "InfrastructureAlertRead"("fingerprint", "userId");
CREATE INDEX "InfrastructureAlertRead_userId_readAt_idx"
ON "InfrastructureAlertRead"("userId", "readAt");
+141
View File
@@ -103,6 +103,7 @@ model User {
releasedPhoneFrequencyHits PhoneFrequencyHit[] @relation("PhoneFrequencyHitReleaser") releasedPhoneFrequencyHits PhoneFrequencyHit[] @relation("PhoneFrequencyHitReleaser")
createdPhoneFrequencyWhitelistEntries PhoneFrequencyWhitelist[] @relation("PhoneFrequencyWhitelistCreator") createdPhoneFrequencyWhitelistEntries PhoneFrequencyWhitelist[] @relation("PhoneFrequencyWhitelistCreator")
updatedPhoneFrequencyWhitelistEntries PhoneFrequencyWhitelist[] @relation("PhoneFrequencyWhitelistUpdater") updatedPhoneFrequencyWhitelistEntries PhoneFrequencyWhitelist[] @relation("PhoneFrequencyWhitelistUpdater")
infrastructureAlertReads InfrastructureAlertRead[]
} }
model Role { model Role {
@@ -2321,3 +2322,143 @@ model GatewayDownstreamRecoveryStatus {
@@index([state, updatedAt]) @@index([state, updatedAt])
@@index([nextRetryAt]) @@index([nextRetryAt])
} }
model SecurityDetectionRule {
id String @id @default(cuid())
code String @unique
name String
sourceType String
enabled Boolean @default(true)
threshold Int
windowSeconds Int
cooldownSeconds Int
severity String
defaultBlockSeconds Int
maximumBlockSeconds Int
configVersion Int @default(1)
effectiveVersion Int @default(0)
applyStatus String @default("pending")
lastApplyError String?
pendingConfig Json?
createdAt DateTime @default(now())
updatedAt DateTime @updatedAt
events SecurityDetectionEvent[]
alerts SecurityAlert[]
@@index([enabled, sourceType])
}
model SecurityDetectionEvent {
id String @id @default(cuid())
eventKey String @unique
ruleId String
sourceIp String
sourcePort Int?
accountHash String?
path String?
protocol String?
resultCode String?
evidence Json?
occurredAt DateTime
createdAt DateTime @default(now())
rule SecurityDetectionRule @relation(fields: [ruleId], references: [id], onDelete: Restrict)
@@index([ruleId, occurredAt])
@@index([sourceIp, occurredAt])
}
model SecurityAlert {
id String @id @default(cuid())
fingerprint String @unique
ruleId String
sourceIp String
severity String
status String @default("open")
eventCount Int @default(0)
windowStartedAt DateTime
firstOccurredAt DateTime
lastOccurredAt DateTime
acknowledgedAt DateTime?
acknowledgedById String?
ignoredAt DateTime?
ignoredById String?
ignoreReason String?
blockId String?
createdAt DateTime @default(now())
updatedAt DateTime @updatedAt
rule SecurityDetectionRule @relation(fields: [ruleId], references: [id], onDelete: Restrict)
@@index([status, severity, lastOccurredAt])
@@index([sourceIp, status, lastOccurredAt])
@@index([ruleId, status, lastOccurredAt])
}
model SecurityBlock {
id String @id @default(cuid())
operationKey String @unique
alertId String?
sourceIp String
executor String
status String @default("requested")
durationSeconds Int
reason String
requestedById String
requestedAt DateTime @default(now())
appliedAt DateTime?
expiresAt DateTime?
releasedAt DateTime?
releasedById String?
executorReference String?
lastError String?
createdAt DateTime @default(now())
updatedAt DateTime @updatedAt
@@index([status, expiresAt])
@@index([sourceIp, status, requestedAt])
@@index([alertId])
}
model SecurityProtectedNetwork {
id String @id @default(cuid())
network String @unique
name String
reason String
enabled Boolean @default(true)
createdById String
createdAt DateTime @default(now())
updatedAt DateTime @updatedAt
@@index([enabled, createdAt])
}
model InfrastructureAlertSetting {
id String @id @default("global")
configVersion Int @default(1)
effectiveVersion Int @default(1)
thresholds Json
effectiveThresholds Json
applyStatus String @default("effective")
lastError String?
updatedById String?
appliedAt DateTime?
createdAt DateTime @default(now())
updatedAt DateTime @updatedAt
}
model InfrastructureAlertRead {
id String @id @default(cuid())
fingerprint String
activeAt DateTime
userId String
readAt DateTime @default(now())
createdAt DateTime @default(now())
updatedAt DateTime @updatedAt
user User @relation(fields: [userId], references: [id], onDelete: Cascade)
@@unique([fingerprint, userId])
@@index([userId, readAt])
}
+6
View File
@@ -12,6 +12,7 @@ import { DictionariesModule } from './dictionaries/dictionaries.module';
import { DeletionGovernanceModule } from './deletion-governance/deletion-governance.module'; import { DeletionGovernanceModule } from './deletion-governance/deletion-governance.module';
import { FilesModule } from './files/files.module'; import { FilesModule } from './files/files.module';
import { HealthController } from './health.controller'; import { HealthController } from './health.controller';
import { InfrastructureMonitoringModule } from './infrastructure-monitoring/infrastructure-monitoring.module';
import { OperationsModule } from './operations/operations.module'; import { OperationsModule } from './operations/operations.module';
import { OpenApiModule } from './open-api/open-api.module'; import { OpenApiModule } from './open-api/open-api.module';
import { PrismaModule } from './prisma/prisma.module'; import { PrismaModule } from './prisma/prisma.module';
@@ -24,6 +25,8 @@ import { SmsConfigModule } from './sms-config/sms-config.module';
import { TenantsModule } from './tenants/tenants.module'; import { TenantsModule } from './tenants/tenants.module';
import { UsersModule } from './users/users.module'; import { UsersModule } from './users/users.module';
import { SignatureRetirementModule } from './signature-retirement/signature-retirement.module'; import { SignatureRetirementModule } from './signature-retirement/signature-retirement.module';
import { SecurityDetectionModule } from './security-detection/security-detection.module';
import { MetricsModule } from './metrics/metrics.module';
@Module({ @Module({
imports: [ imports: [
@@ -49,8 +52,11 @@ import { SignatureRetirementModule } from './signature-retirement/signature-reti
ReportMaterialsModule, ReportMaterialsModule,
SendChainModule, SendChainModule,
OperationsModule, OperationsModule,
InfrastructureMonitoringModule,
OpenApiModule, OpenApiModule,
SignatureRetirementModule, SignatureRetirementModule,
SecurityDetectionModule,
MetricsModule,
], ],
controllers: [HealthController], controllers: [HealthController],
providers: [RequestContextMiddleware, SessionValidationMiddleware, ManualOperationAuditMiddleware], providers: [RequestContextMiddleware, SessionValidationMiddleware, ManualOperationAuditMiddleware],
+30 -3
View File
@@ -7,6 +7,8 @@ import type { SessionRequest } from './session-validation.middleware';
import { UsersService } from '../users/users.service'; import { UsersService } from '../users/users.service';
import { PrismaService } from '../prisma/prisma.service'; import { PrismaService } from '../prisma/prisma.service';
import { Prisma } from '@prisma/client'; import { Prisma } from '@prisma/client';
import { requestContext } from '../common/request-context';
import { SecurityDetectionService } from '../security-detection/security-detection.service';
type CookieResponse = { type CookieResponse = {
cookie(name: string, value: string, options: Record<string, unknown>): void; cookie(name: string, value: string, options: Record<string, unknown>): void;
@@ -16,7 +18,7 @@ type CookieResponse = {
@ApiTags('auth') @ApiTags('auth')
@Controller() @Controller()
export class AuthController { export class AuthController {
constructor(private readonly auth: AuthService, private readonly users: UsersService, private readonly sessions: SessionService, private readonly prisma: PrismaService) {} constructor(private readonly auth: AuthService, private readonly users: UsersService, private readonly sessions: SessionService, private readonly prisma: PrismaService, private readonly security: SecurityDetectionService) {}
@Get('admin/auth/captcha') @Get('admin/auth/captcha')
adminCaptcha() { adminCaptcha() {
@@ -25,7 +27,14 @@ export class AuthController {
@Post('admin/auth/login') @Post('admin/auth/login')
async adminLogin(@Body() body: LoginDto, @Req() request: SessionRequest, @Res({ passthrough: true }) response: CookieResponse) { async adminLogin(@Body() body: LoginDto, @Req() request: SessionRequest, @Res({ passthrough: true }) response: CookieResponse) {
return this.finishLogin(await this.auth.login(body, 'admin'), request, response); let result: Awaited<ReturnType<AuthService['login']>>;
try {
result = await this.auth.login(body, 'admin');
} catch (error) {
await this.recordLoginFailure('admin_login_failure', body.login, request).catch(() => undefined);
throw error;
}
return this.finishLogin(result, request, response);
} }
@Get('client/auth/captcha') @Get('client/auth/captcha')
@@ -35,7 +44,14 @@ export class AuthController {
@Post('client/auth/login') @Post('client/auth/login')
async clientLogin(@Body() body: LoginDto, @Req() request: SessionRequest, @Res({ passthrough: true }) response: CookieResponse) { async clientLogin(@Body() body: LoginDto, @Req() request: SessionRequest, @Res({ passthrough: true }) response: CookieResponse) {
return this.finishLogin(await this.auth.login(body, 'client'), request, response); let result: Awaited<ReturnType<AuthService['login']>>;
try {
result = await this.auth.login(body, 'client');
} catch (error) {
await this.recordLoginFailure('client_login_failure', body.login, request).catch(() => undefined);
throw error;
}
return this.finishLogin(result, request, response);
} }
@Get(['admin/auth/session', 'client/auth/session']) @Get(['admin/auth/session', 'client/auth/session'])
@@ -121,6 +137,17 @@ export class AuthController {
return publicResult; return publicResult;
} }
private recordLoginFailure(ruleCode: 'admin_login_failure' | 'client_login_failure', account: string, request: SessionRequest) {
return this.security.recordEvent({
ruleCode,
sourceIp: requestContext.getStore()?.ipAddress ?? '127.0.0.1',
account,
protocol: 'http',
path: ruleCode === 'admin_login_failure' ? '/admin/auth/login' : '/client/auth/login',
evidence: { userAgent: request.header('user-agent')?.slice(0, 256) },
});
}
private writeLog(request: SessionRequest, action: string, detail: Record<string, unknown>) { private writeLog(request: SessionRequest, action: string, detail: Record<string, unknown>) {
return this.prisma.operationLog.create({ return this.prisma.operationLog.create({
data: { userId: request.sessionUserId, action, resource: 'auth_session', userAgent: request.header('user-agent'), detail: JSON.parse(JSON.stringify(detail)) as Prisma.InputJsonValue }, data: { userId: request.sessionUserId, action, resource: 'auth_session', userAgent: request.header('user-agent'), detail: JSON.parse(JSON.stringify(detail)) as Prisma.InputJsonValue },
+2 -1
View File
@@ -5,9 +5,10 @@ import { AuthController } from './auth.controller';
import { AuthService } from './auth.service'; import { AuthService } from './auth.service';
import { RecentAuthenticationGuard } from './recent-authentication.guard'; import { RecentAuthenticationGuard } from './recent-authentication.guard';
import { SessionService } from './session.service'; import { SessionService } from './session.service';
import { SecurityDetectionModule } from '../security-detection/security-detection.module';
@Module({ @Module({
imports: [UsersModule], imports: [UsersModule, SecurityDetectionModule],
controllers: [AuthController], controllers: [AuthController],
providers: [ providers: [
AuthService, AuthService,
+4 -1
View File
@@ -8,7 +8,10 @@ export class RequestContextMiddleware implements NestMiddleware {
use(request: RequestLike, _response: unknown, next: () => void) { use(request: RequestLike, _response: unknown, next: () => void) {
const forwarded = request.headers['x-forwarded-for']; const forwarded = request.headers['x-forwarded-for'];
const firstForwarded = Array.isArray(forwarded) ? forwarded[0] : forwarded?.split(',')[0]; const firstForwarded = Array.isArray(forwarded) ? forwarded[0] : forwarded?.split(',')[0];
const ipAddress = (firstForwarded ?? request.socket?.remoteAddress)?.trim().replace(/^::ffff:/, ''); const remoteAddress = request.socket?.remoteAddress?.trim().replace(/^::ffff:/, '');
const trustedProxies = new Set((process.env.TRUSTED_PROXY_IPS ?? '127.0.0.1,::1').split(',').map((item) => item.trim()).filter(Boolean));
// 仅可信反向代理可以声明客户端地址,防止攻击者伪造 X-Forwarded-For 绕过保护名单或嫁祸他人。
const ipAddress = (remoteAddress && trustedProxies.has(remoteAddress) ? firstForwarded : remoteAddress)?.trim().replace(/^::ffff:/, '');
requestContext.run({ ipAddress }, next); requestContext.run({ ipAddress }, next);
} }
} }
@@ -0,0 +1,21 @@
import { BadRequestException } from '@nestjs/common';
import { DEFAULT_ALERT_THRESHOLDS, InfrastructureAlertSettingsService } from './infrastructure-alert-settings.service';
describe('InfrastructureAlertSettingsService', () => {
const service = new InfrastructureAlertSettingsService({} as never, { get: () => undefined } as never);
it('accepts the fixed threshold whitelist and renders managed rules', () => {
const validated = (service as unknown as { validate(value: unknown): unknown }).validate(DEFAULT_ALERT_THRESHOLDS);
const rules = (service as unknown as { renderRules(value: unknown): string }).renderRules(validated);
expect(rules).toContain('HostCpuUsageWarning');
expect(rules).toContain('CmppGatewayQueueDelayedCritical');
expect(rules).toContain('threshold: "120秒"');
expect(rules).toContain('redis_memory_max_bytes > 0');
expect(rules).toContain('sum(increase(cmpp_api_http_requests_total');
});
it('rejects unknown keys and warning thresholds that are not below critical', () => {
expect(() => (service as unknown as { validate(value: unknown): unknown }).validate({ ...DEFAULT_ALERT_THRESHOLDS, promql: { warning: 1, critical: 2 } })).toThrow(BadRequestException);
expect(() => (service as unknown as { validate(value: unknown): unknown }).validate({ ...DEFAULT_ALERT_THRESHOLDS, hostCpu: { warning: 90, critical: 90 } })).toThrow(BadRequestException);
});
});
@@ -0,0 +1,144 @@
import { BadRequestException, ConflictException, Injectable, Logger, ServiceUnavailableException } from '@nestjs/common';
import { ConfigService } from '@nestjs/config';
import { Prisma } from '@prisma/client';
import { execFile } from 'node:child_process';
import { mkdir, readFile, rename, rm, writeFile } from 'node:fs/promises';
import { dirname } from 'node:path';
import { promisify } from 'node:util';
import { PrismaService } from '../prisma/prisma.service';
import type { InfrastructureAlertSettings, InfrastructureAlertThresholds } from './infrastructure-monitoring.contracts';
const execFileAsync = promisify(execFile);
export const ALERT_THRESHOLD_DEFINITIONS = [
{ key: 'hostCpu', label: '主机 CPU 使用率', unit: '%', min: 1, max: 100, step: 1, warning: 80, critical: 90, expr: '100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100)', names: ['HostCpuUsageWarning', 'HostCpuUsageCritical'], service: 'host', durations: ['10m', '5m'] },
{ key: 'hostMemory', label: '主机内存使用率', unit: '%', min: 1, max: 100, step: 1, warning: 85, critical: 95, expr: '(1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100', names: ['HostMemoryUsageWarning', 'HostMemoryUsageCritical'], service: 'host', durations: ['10m', '5m'] },
{ key: 'hostDisk', label: '根磁盘使用率', unit: '%', min: 1, max: 100, step: 1, warning: 80, critical: 90, expr: '(1 - node_filesystem_avail_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"} / node_filesystem_size_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"}) * 100', names: ['HostRootDiskUsageWarning', 'HostRootDiskUsageCritical'], service: 'host', durations: ['15m', '5m'] },
{ key: 'apiError', label: 'API 5xx 错误率', unit: '%', min: 0.1, max: 100, step: 0.1, warning: 1, critical: 5, expr: '100 * sum(rate(cmpp_api_http_requests_total{status=~"5.."}[5m])) / clamp_min(sum(rate(cmpp_api_http_requests_total[5m])), 0.001)', guard: 'sum(increase(cmpp_api_http_requests_total{status=~"5.."}[5m])) >= 5', names: ['CmppApiHttpErrorRateWarning', 'CmppApiHttpErrorRateCritical'], service: 'api', durations: ['5m', '5m'] },
{ key: 'apiLatency', label: 'API P95 响应时间', unit: '秒', min: 0.1, max: 60, step: 0.1, warning: 1, critical: 3, expr: 'histogram_quantile(0.95, sum by (le) (rate(cmpp_api_http_request_duration_seconds_bucket[10m])))', names: ['CmppApiLatencyWarning', 'CmppApiLatencyCritical'], service: 'api', durations: ['10m', '5m'] },
{ key: 'apiEventLoop', label: 'API 事件循环 P99', unit: '秒', min: 0.01, max: 10, step: 0.01, warning: 0.2, critical: 1, expr: 'cmpp_api_nodejs_event_loop_lag_p99_seconds', names: ['CmppApiEventLoopLagWarning', 'CmppApiEventLoopLagCritical'], service: 'api', durations: ['10m', '5m'] },
{ key: 'gatewayQueue', label: 'Gateway 最旧 pending', unit: '秒', min: 1, max: 3600, step: 1, warning: 30, critical: 120, expr: 'cmpp_gateway_submit_queue_oldest_pending_age_seconds', names: ['CmppGatewayQueueDelayedWarning', 'CmppGatewayQueueDelayedCritical'], service: 'gateway', durations: ['2m', '2m'] },
{ key: 'postgresConnections', label: 'PostgreSQL 连接使用率', unit: '%', min: 1, max: 100, step: 1, warning: 70, critical: 85, expr: '100 * sum(pg_stat_activity_count) / clamp_min(max(pg_settings_max_connections), 1)', names: ['PostgresConnectionsWarning', 'PostgresConnectionsCritical'], service: 'postgresql', durations: ['10m', '5m'] },
{ key: 'redisMemory', label: 'Redis 内存使用率', unit: '%', min: 1, max: 100, step: 1, warning: 70, critical: 85, expr: '100 * redis_memory_used_bytes / redis_memory_max_bytes', guard: 'redis_memory_max_bytes > 0', names: ['RedisMemoryWarning', 'RedisMemoryCritical'], service: 'redis', durations: ['10m', '5m'] },
{ key: 'minioCapacity', label: 'MinIO 容量使用率', unit: '%', min: 1, max: 100, step: 1, warning: 80, critical: 90, expr: '100 * (1 - minio_cluster_capacity_usable_free_bytes / minio_cluster_capacity_usable_total_bytes)', names: ['MinioCapacityWarning', 'MinioCapacityCritical'], service: 'minio', durations: ['15m', '5m'] },
] as const;
export const DEFAULT_ALERT_THRESHOLDS: InfrastructureAlertThresholds = Object.fromEntries(
ALERT_THRESHOLD_DEFINITIONS.map((item) => [item.key, { warning: item.warning, critical: item.critical }]),
);
@Injectable()
export class InfrastructureAlertSettingsService {
private readonly logger = new Logger(InfrastructureAlertSettingsService.name);
private readonly rulesPath: string;
private readonly promtoolPath: string;
private readonly reloadUrl: string;
constructor(private readonly prisma: PrismaService, config: ConfigService) {
this.rulesPath = String(config.get('PROMETHEUS_MANAGED_RULES_PATH') ?? '/var/lib/cmpp-platform/monitoring/cmpp-managed-alerts.yml');
this.promtoolPath = String(config.get('PROMTOOL_PATH') ?? '/usr/bin/promtool');
this.reloadUrl = String(config.get('PROMETHEUS_RELOAD_URL') ?? 'http://127.0.0.1:9090/-/reload');
}
async get(): Promise<InfrastructureAlertSettings> {
const row = await this.prisma.infrastructureAlertSetting.findUnique({ where: { id: 'global' } });
const thresholds = this.asThresholds(row?.thresholds) ?? DEFAULT_ALERT_THRESHOLDS;
const effective = this.asThresholds(row?.effectiveThresholds) ?? thresholds;
return {
configVersion: row?.configVersion ?? 1,
effectiveVersion: row?.effectiveVersion ?? 1,
applyStatus: (row?.applyStatus as InfrastructureAlertSettings['applyStatus']) ?? 'effective',
lastError: row?.lastError ?? null,
appliedAt: row?.appliedAt?.toISOString() ?? null,
thresholds,
effectiveThresholds: effective,
definitions: ALERT_THRESHOLD_DEFINITIONS.map(({ key, label, unit, min, max, step }) => ({ key, label, unit, min, max, step })),
};
}
async update(body: { configVersion?: number; thresholds?: unknown }, operatorId?: string) {
const expectedVersion = Number(body.configVersion);
if (!Number.isInteger(expectedVersion) || expectedVersion < 1) throw new BadRequestException('配置版本无效');
const thresholds = this.validate(body.thresholds);
const claimed = await this.prisma.infrastructureAlertSetting.updateMany({
where: { id: 'global', configVersion: expectedVersion },
data: { configVersion: { increment: 1 }, thresholds: thresholds as Prisma.InputJsonValue, applyStatus: 'applying', lastError: null, updatedById: operatorId },
});
// 版本条件更新是跨进程的写锁;避免两个 API 实例同时覆盖规则文件并把旧配置误标成已生效。
if (claimed.count !== 1) throw new ConflictException('告警阈值已被其他管理员修改,请刷新后重试');
const nextVersion = expectedVersion + 1;
try {
await this.applyRules(thresholds);
await this.prisma.$transaction([
this.prisma.infrastructureAlertSetting.update({ where: { id: 'global' }, data: { effectiveVersion: nextVersion, effectiveThresholds: thresholds as Prisma.InputJsonValue, applyStatus: 'effective', lastError: null, appliedAt: new Date() } }),
this.prisma.operationLog.create({ data: { userId: operatorId, action: 'monitoring.alert_thresholds_updated', resource: 'infrastructure_alert_setting', resourceId: 'global', detail: { configVersion: nextVersion, thresholds } } }),
]);
} catch (error) {
const message = error instanceof Error ? error.message.slice(0, 500) : 'unknown error';
await this.prisma.infrastructureAlertSetting.update({ where: { id: 'global' }, data: { applyStatus: 'failed', lastError: message } });
this.logger.error(`Prometheus managed rules apply failed: ${message}`);
throw new ServiceUnavailableException('阈值已保存但 Prometheus 应用失败,原生效规则已保留');
}
return this.get();
}
private validate(value: unknown): InfrastructureAlertThresholds {
if (!value || typeof value !== 'object' || Array.isArray(value)) throw new BadRequestException('告警阈值格式无效');
const input = value as Record<string, unknown>;
if (Object.keys(input).some((key) => !ALERT_THRESHOLD_DEFINITIONS.some((item) => item.key === key))) throw new BadRequestException('存在不允许配置的告警指标');
const result: InfrastructureAlertThresholds = {};
for (const definition of ALERT_THRESHOLD_DEFINITIONS) {
const pair = input[definition.key] as { warning?: unknown; critical?: unknown } | undefined;
const warning = Number(pair?.warning);
const critical = Number(pair?.critical);
if (!Number.isFinite(warning) || !Number.isFinite(critical) || warning < definition.min || critical > definition.max || warning >= critical) {
throw new BadRequestException(`${definition.label}必须满足最小值 ≤ 警告阈值 < 严重阈值 ≤ 最大值`);
}
result[definition.key] = { warning, critical };
}
return result;
}
private asThresholds(value: unknown) {
try { return this.validate(value); } catch { return null; }
}
private renderRules(thresholds: InfrastructureAlertThresholds) {
const lines = ['groups:', ' - name: cmpp-managed-thresholds', ' rules:'];
for (const definition of ALERT_THRESHOLD_DEFINITIONS) {
const pair = thresholds[definition.key];
const values = [pair.warning, pair.critical];
for (let index = 0; index < 2; index += 1) {
const isWarning = index === 0;
// 低样本量与“未配置容量上限”必须继续作为固定保护条件,避免单次错误或除零结果触发伪告警。
const guard = 'guard' in definition ? ` and (${definition.guard})` : '';
const expr = isWarning ? `(${definition.expr} > ${values[0]}) and (${definition.expr} <= ${values[1]})${guard}` : `(${definition.expr} > ${values[1]})${guard}`;
lines.push(` - alert: ${definition.names[index]}`, ` expr: ${expr}`, ` for: ${definition.durations[index]}`, ' labels:', ` severity: ${isWarning ? 'warning' : 'critical'}`, ` service: ${definition.service}`, ' annotations:', ` summary: "${definition.label}${isWarning ? '达到警告阈值' : '达到严重阈值'}"`, ` description: "${definition.label}持续超过${values[index]}${definition.unit}。"`, ' currentValue: "{{ $value }}"', ` threshold: "${values[index]}${definition.unit}"`);
}
}
return `${lines.join('\n')}\n`;
}
private async applyRules(thresholds: InfrastructureAlertThresholds) {
const directory = dirname(this.rulesPath);
const temporary = `${this.rulesPath}.${process.pid}.${Date.now()}.tmp`;
await mkdir(directory, { recursive: true });
const previous = await readFile(this.rulesPath).catch(() => null);
try {
await writeFile(temporary, this.renderRules(thresholds), { mode: 0o640 });
await execFileAsync(this.promtoolPath, ['check', 'rules', temporary], { timeout: 10_000 });
await rename(temporary, this.rulesPath);
const response = await fetch(this.reloadUrl, { method: 'POST', signal: AbortSignal.timeout(5_000) });
if (!response.ok) throw new Error(`Prometheus reload HTTP ${response.status}`);
} catch (error) {
await rm(temporary, { force: true });
// 规则替换和 reload 不是一个事务,失败时必须恢复旧文件并再次 reload,避免数据库状态与实际告警漂移。
if (previous) {
await writeFile(temporary, previous, { mode: 0o640 });
await rename(temporary, this.rulesPath);
await fetch(this.reloadUrl, { method: 'POST', signal: AbortSignal.timeout(5_000) }).catch(() => undefined);
}
throw error;
}
}
}
@@ -0,0 +1,93 @@
export type InfrastructureMonitoringRange = '1h' | '24h' | '7d';
export type InfrastructureMetricPoint = {
timestamp: string;
value: number;
};
export type InfrastructureServiceStatus = {
key: string;
name: string;
unit: string;
status: 'healthy' | 'unhealthy' | 'unknown';
};
export type InfrastructureServiceMetricGroup = {
key: string;
name: string;
available: boolean;
metrics: Array<{
key: string;
label: string;
value: number | null;
unit: 'percent' | 'seconds' | 'count' | 'per_second' | 'bytes';
}>;
};
export type InfrastructureAlert = {
fingerprint: string;
name: string;
severity: 'info' | 'warning' | 'critical';
status: string;
startedAt: string;
summary: string;
description?: string;
currentValue?: string;
threshold?: string;
service?: string;
instance?: string;
acknowledged: boolean;
acknowledgedAt?: string;
};
export type InfrastructureMonitoringOverview = {
available: boolean;
range: InfrastructureMonitoringRange;
collectedAt: string;
lastSampleAt: string | null;
error?: string;
summary: {
overallStatus: 'healthy' | 'warning' | 'critical' | 'unknown';
serviceTotal: number;
serviceHealthy: number;
warningAlerts: number;
criticalAlerts: number;
activeAlerts: number;
};
metrics: {
cpuUsagePercent: number | null;
memoryUsagePercent: number | null;
memoryTotalBytes: number | null;
memoryAvailableBytes: number | null;
diskUsagePercent: number | null;
diskTotalBytes: number | null;
diskAvailableBytes: number | null;
networkReceiveBytesPerSecond: number | null;
networkTransmitBytesPerSecond: number | null;
load1: number | null;
uptimeSeconds: number | null;
};
trends: {
cpuUsagePercent: InfrastructureMetricPoint[];
memoryUsagePercent: InfrastructureMetricPoint[];
diskUsagePercent: InfrastructureMetricPoint[];
networkReceiveBytesPerSecond: InfrastructureMetricPoint[];
networkTransmitBytesPerSecond: InfrastructureMetricPoint[];
};
services: InfrastructureServiceStatus[];
serviceMetrics: InfrastructureServiceMetricGroup[];
alerts: InfrastructureAlert[];
};
export type InfrastructureAlertThresholds = Record<string, { warning: number; critical: number }>;
export type InfrastructureAlertSettings = {
configVersion: number;
effectiveVersion: number;
applyStatus: 'effective' | 'applying' | 'failed';
lastError: string | null;
appliedAt: string | null;
thresholds: InfrastructureAlertThresholds;
effectiveThresholds: InfrastructureAlertThresholds;
definitions: Array<{ key: string; label: string; unit: string; min: number; max: number; step: number }>;
};
@@ -0,0 +1,34 @@
import { Body, Controller, Get, Param, Post, Put, Query } from '@nestjs/common';
import { ApiTags } from '@nestjs/swagger';
import { CurrentSessionUserId } from '../auth/current-session-user.decorator';
import { RequireRecentAuthentication } from '../auth/require-recent-authentication.decorator';
import { InfrastructureAlertSettingsService } from './infrastructure-alert-settings.service';
import { InfrastructureMonitoringService } from './infrastructure-monitoring.service';
@ApiTags('infrastructure-monitoring')
@Controller('admin/infrastructure-monitoring')
export class InfrastructureMonitoringController {
constructor(private readonly monitoring: InfrastructureMonitoringService, private readonly settings: InfrastructureAlertSettingsService) {}
@Get('overview')
overview(@Query('range') range?: string, @CurrentSessionUserId() userId?: string) {
return this.monitoring.overview(range, userId);
}
@Get('notification-summary')
notificationSummary(@CurrentSessionUserId() userId?: string) { return this.monitoring.notificationSummary(userId); }
@Post('alerts/:fingerprint/read')
markAlertRead(@Param('fingerprint') fingerprint: string, @Body('activeAt') activeAt: unknown, @CurrentSessionUserId() userId: string) {
return this.monitoring.markAlertRead(fingerprint, activeAt, userId);
}
@Get('alert-thresholds')
alertThresholds() { return this.settings.get(); }
@Put('alert-thresholds')
@RequireRecentAuthentication()
updateAlertThresholds(@Body() body: { configVersion?: number; thresholds?: unknown }, @CurrentSessionUserId() operatorId?: string) {
return this.settings.update(body, operatorId);
}
}
@@ -0,0 +1,10 @@
import { Module } from '@nestjs/common';
import { InfrastructureMonitoringController } from './infrastructure-monitoring.controller';
import { InfrastructureMonitoringService } from './infrastructure-monitoring.service';
import { InfrastructureAlertSettingsService } from './infrastructure-alert-settings.service';
@Module({
controllers: [InfrastructureMonitoringController],
providers: [InfrastructureMonitoringService, InfrastructureAlertSettingsService],
})
export class InfrastructureMonitoringModule {}
@@ -0,0 +1,137 @@
import { BadRequestException } from '@nestjs/common';
import { ConfigService } from '@nestjs/config';
import { createHash } from 'node:crypto';
import { InfrastructureMonitoringService } from './infrastructure-monitoring.service';
function success(data: unknown) {
return {
ok: true,
status: 200,
json: async () => ({ status: 'success', data }),
} as Response;
}
describe('InfrastructureMonitoringService', () => {
const prisma = {
infrastructureAlertRead: { findMany: jest.fn().mockResolvedValue([]), create: jest.fn(), update: jest.fn(), findUniqueOrThrow: jest.fn() },
operationLog: { create: jest.fn() },
$transaction: jest.fn(),
};
afterEach(() => {
jest.restoreAllMocks();
jest.clearAllMocks();
prisma.infrastructureAlertRead.findMany.mockResolvedValue([]);
});
it('rejects ranges outside the fixed whitelist before querying Prometheus', async () => {
const fetchSpy = jest.spyOn(global, 'fetch');
const service = new InfrastructureMonitoringService(new ConfigService(), prisma as never);
await expect(service.overview('30d')).rejects.toBeInstanceOf(BadRequestException);
expect(fetchSpy).not.toHaveBeenCalled();
});
it('rejects credential-bearing or remote plaintext Prometheus endpoints at startup', () => {
expect(() => new InfrastructureMonitoringService(new ConfigService({ PROMETHEUS_URL: 'http://user:secret@127.0.0.1:9090' }), prisma as never)).toThrow('must not contain credentials');
expect(() => new InfrastructureMonitoringService(new ConfigService({ PROMETHEUS_URL: 'http://monitor.example.com:9090' }), prisma as never)).toThrow('must use HTTPS');
expect(() => new InfrastructureMonitoringService(new ConfigService({ PROMETHEUS_URL: 'https://monitor.example.com' }), prisma as never)).not.toThrow();
});
it('loads real Prometheus vectors, ranges, services and active alerts', async () => {
const requestedUrls: URL[] = [];
jest.spyOn(global, 'fetch').mockImplementation(async (input) => {
const url = new URL(String(input));
requestedUrls.push(url);
if (url.pathname.endsWith('/alerts')) {
return success({ alerts: [{
labels: { alertname: 'HostCpuHigh', severity: 'warning', instance: '127.0.0.1:9100' },
annotations: { summary: 'CPU持续偏高', threshold: '85%' },
state: 'firing',
activeAt: '2026-08-14T03:00:00.000Z',
value: '88.2',
}] });
}
const query = url.searchParams.get('query') ?? '';
if (url.pathname.endsWith('/query_range')) {
return success({ result: [{ metric: {}, values: [[1_765_000_000, '12.5'], [1_765_000_060, '14.5']] }] });
}
if (query.includes('node_systemd_unit_state')) {
return success({ result: [
{ metric: { name: 'cmpp-api.service' }, value: [1_765_000_060, '1'] },
{ metric: { name: 'cmpp-gateway.service' }, value: [1_765_000_060, '1'] },
{ metric: { name: 'postgresql.service' }, value: [1_765_000_060, '1'] },
{ metric: { name: 'redis-server.service' }, value: [1_765_000_060, '1'] },
{ metric: { name: 'cmpp-minio.service' }, value: [1_765_000_060, '1'] },
{ metric: { name: 'nginx.service' }, value: [1_765_000_060, '1'] },
] });
}
if (query.includes('cmpp:service_.*')) {
return success({ result: [
{ metric: { __name__: 'cmpp:service_api:requests_per_second' }, value: [1_765_000_060, '12.5'] },
{ metric: { __name__: 'cmpp:service_api:error_percent' }, value: [1_765_000_060, '0.2'] },
{ metric: { __name__: 'cmpp:service_gateway:queue_pending' }, value: [1_765_000_060, '3'] },
] });
}
if (query.includes('timestamp(node_uname_info)')) {
return success({ result: [{ metric: {}, value: [1_765_000_060, '1765000060'] }] });
}
return success({ result: [{ metric: {}, value: [1_765_000_060, '25'] }] });
});
const service = new InfrastructureMonitoringService(new ConfigService(), prisma as never);
const result = await service.overview('1h');
expect(result.available).toBe(true);
expect(result.metrics.cpuUsagePercent).toBe(25);
expect(result.trends.cpuUsagePercent).toHaveLength(2);
expect(result.summary).toMatchObject({ overallStatus: 'warning', serviceHealthy: 6, warningAlerts: 1 });
expect(result.services.find((item) => item.key === 'redis')).toMatchObject({ unit: 'redis-server.service', status: 'healthy' });
expect(result.serviceMetrics.find((item) => item.key === 'api')).toMatchObject({ available: true });
expect(result.serviceMetrics.find((item) => item.key === 'gateway')?.metrics.find((item) => item.key === 'queuePending')?.value).toBe(3);
expect(result.alerts[0]).toMatchObject({ name: 'HostCpuHigh', severity: 'warning', currentValue: '88.2' });
expect(requestedUrls.filter((url) => url.pathname.endsWith('/query_range'))).toHaveLength(5);
expect(requestedUrls.filter((url) => url.pathname.endsWith('/query_range')).every((url) => url.searchParams.get('step') === '60')).toBe(true);
expect(requestedUrls.find((url) => url.searchParams.get('query')?.includes('node_systemd_unit_state'))?.searchParams.get('query'))
.toContain('cmpp-api\\\\.service');
});
it('returns an explicit unavailable payload without stale metrics when Prometheus fails', async () => {
jest.spyOn(global, 'fetch').mockRejectedValue(new Error('ECONNREFUSED'));
const service = new InfrastructureMonitoringService(new ConfigService(), prisma as never);
const result = await service.overview('24h');
expect(result.available).toBe(false);
expect(result.summary.overallStatus).toBe('unknown');
expect(result.metrics.cpuUsagePercent).toBeNull();
expect(result.trends.cpuUsagePercent).toEqual([]);
expect(result.serviceMetrics.every((item) => item.available === false)).toBe(true);
expect(result.error).not.toContain('ECONNREFUSED');
});
it('excludes only the current alert occurrence after the current administrator marks it read', async () => {
const labels = { alertname: 'QaWarning', severity: 'warning', service: 'qa-preview' };
const fingerprint = createHash('sha256').update(JSON.stringify(Object.entries(labels).sort(([left], [right]) => left.localeCompare(right)))).digest('hex').slice(0, 24);
jest.spyOn(global, 'fetch').mockResolvedValue(success({ alerts: [{ labels, annotations: { summary: '演示预警' }, state: 'firing', activeAt: '2026-08-16T01:00:00.000Z' }] }));
prisma.infrastructureAlertRead.findMany.mockResolvedValue([{ fingerprint, activeAt: new Date('2026-08-16T01:00:00.000Z'), readAt: new Date('2026-08-16T01:01:00.000Z') }]);
const service = new InfrastructureMonitoringService(new ConfigService(), prisma as never);
await expect(service.notificationSummary('admin-1')).resolves.toEqual({ count: 0, criticalCount: 0 });
prisma.infrastructureAlertRead.findMany.mockResolvedValue([{ fingerprint, activeAt: new Date('2026-08-15T01:00:00.000Z'), readAt: new Date('2026-08-15T01:01:00.000Z') }]);
await expect(service.notificationSummary('admin-1')).resolves.toEqual({ count: 1, criticalCount: 0 });
});
it('upserts an idempotent per-user read record only for a currently active occurrence', async () => {
const labels = { alertname: 'QaCritical', severity: 'critical', service: 'qa-preview' };
const fingerprint = createHash('sha256').update(JSON.stringify(Object.entries(labels).sort(([left], [right]) => left.localeCompare(right)))).digest('hex').slice(0, 24);
jest.spyOn(global, 'fetch').mockResolvedValue(success({ alerts: [{ labels, annotations: { summary: '演示严重告警' }, state: 'firing', activeAt: '2026-08-16T02:00:00.000Z' }] }));
prisma.$transaction.mockResolvedValue([{ activeAt: new Date('2026-08-16T02:00:00.000Z'), readAt: new Date('2026-08-16T02:01:00.000Z') }, {}]);
const service = new InfrastructureMonitoringService(new ConfigService(), prisma as never);
await expect(service.markAlertRead(fingerprint, '2026-08-16T02:00:00.000Z', 'admin-1')).resolves.toMatchObject({ fingerprint, acknowledged: true });
expect(prisma.infrastructureAlertRead.create).toHaveBeenCalledWith(expect.objectContaining({ data: expect.objectContaining({ fingerprint, userId: 'admin-1' }) }));
await expect(service.markAlertRead(fingerprint, '2026-08-15T02:00:00.000Z', 'admin-1')).rejects.toThrow('已结束或已重新触发');
});
});
@@ -0,0 +1,383 @@
import { BadRequestException, Injectable, Logger, NotFoundException, ServiceUnavailableException } from '@nestjs/common';
import { ConfigService } from '@nestjs/config';
import { Prisma } from '@prisma/client';
import { createHash } from 'node:crypto';
import { PrismaService } from '../prisma/prisma.service';
import type {
InfrastructureAlert,
InfrastructureMetricPoint,
InfrastructureMonitoringOverview,
InfrastructureMonitoringRange,
InfrastructureServiceStatus,
InfrastructureServiceMetricGroup,
} from './infrastructure-monitoring.contracts';
type PrometheusSample = [number, string];
type PrometheusSeries = {
metric: Record<string, string>;
value?: PrometheusSample;
values?: PrometheusSample[];
};
type PrometheusQueryResponse = {
status: 'success' | 'error';
data?: { result?: PrometheusSeries[] };
error?: string;
};
type PrometheusAlertResponse = {
status: 'success' | 'error';
data?: {
alerts?: Array<{
labels?: Record<string, string>;
annotations?: Record<string, string>;
state?: string;
activeAt?: string;
value?: string;
}>;
};
};
const RANGE_CONFIG: Record<InfrastructureMonitoringRange, { seconds: number; step: number }> = {
'1h': { seconds: 60 * 60, step: 60 },
'24h': { seconds: 24 * 60 * 60, step: 300 },
'7d': { seconds: 7 * 24 * 60 * 60, step: 1800 },
};
const QUERIES = {
cpuUsagePercent: '100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100)',
memoryUsagePercent: '(1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) * 100',
memoryTotalBytes: 'node_memory_MemTotal_bytes',
memoryAvailableBytes: 'node_memory_MemAvailable_bytes',
diskUsagePercent: '(1 - (node_filesystem_avail_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"} / node_filesystem_size_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"})) * 100',
diskTotalBytes: 'node_filesystem_size_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"}',
diskAvailableBytes: 'node_filesystem_avail_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"}',
networkReceiveBytesPerSecond: 'sum(rate(node_network_receive_bytes_total{device!~"lo"}[5m]))',
networkTransmitBytesPerSecond: 'sum(rate(node_network_transmit_bytes_total{device!~"lo"}[5m]))',
load1: 'node_load1',
uptimeSeconds: 'time() - node_boot_time_seconds',
lastSampleAt: 'max(timestamp(node_uname_info))',
// PromQL字符串本身需要两个反斜杠才能把正则的“\.”传给RE2;TypeScript字面量因此需要写四个。
services: 'max by (name) (node_systemd_unit_state{name=~"cmpp-api\\\\.service|cmpp-gateway\\\\.service|postgresql\\\\.service|redis(-server)?\\\\.service|cmpp-minio\\\\.service|nginx\\\\.service",state="active"})',
} as const;
const SERVICE_DEFINITIONS = [
{ key: 'api', name: 'API服务', units: ['cmpp-api.service'] },
{ key: 'gateway', name: 'Gateway服务', units: ['cmpp-gateway.service'] },
{ key: 'postgresql', name: 'PostgreSQL', units: ['postgresql.service'] },
{ key: 'redis', name: 'Redis', units: ['redis.service', 'redis-server.service'] },
{ key: 'minio', name: 'MinIO', units: ['cmpp-minio.service'] },
{ key: 'nginx', name: 'Nginx', units: ['nginx.service'] },
] as const;
const SERVICE_METRIC_DEFINITIONS = [
{ key: 'api', name: 'API服务', metrics: [
['requestsPerSecond', '请求速率', 'cmpp:service_api:requests_per_second', 'per_second'],
['errorPercent', '5xx错误率', 'cmpp:service_api:error_percent', 'percent'],
['latencyP95', 'P95响应', 'cmpp:service_api:latency_p95_seconds', 'seconds'],
['eventLoopP99', '事件循环P99', 'cmpp:service_api:event_loop_p99_seconds', 'seconds'],
] },
{ key: 'gateway', name: 'Gateway服务', metrics: [
['submitsPerSecond', '提交速率', 'cmpp:service_gateway:submits_per_second', 'per_second'],
['failurePercent', '提交失败率', 'cmpp:service_gateway:failure_percent', 'percent'],
['queuePending', 'Stream pending', 'cmpp:service_gateway:queue_pending', 'count'],
['queueOldestSeconds', '最旧pending', 'cmpp:service_gateway:queue_oldest_seconds', 'seconds'],
] },
{ key: 'postgresql', name: 'PostgreSQL', metrics: [
['connectionPercent', '连接使用率', 'cmpp:service_postgresql:connection_percent', 'percent'],
['deadlocks15m', '15分钟死锁', 'cmpp:service_postgresql:deadlocks_15m', 'count'],
] },
{ key: 'redis', name: 'Redis', metrics: [
['memoryPercent', '内存使用率', 'cmpp:service_redis:memory_percent', 'percent'],
['memoryUsedBytes', '已用内存', 'cmpp:service_redis:memory_used_bytes', 'bytes'],
['connectedClients', '客户端连接', 'cmpp:service_redis:connected_clients', 'count'],
['evictions5m', '5分钟淘汰', 'cmpp:service_redis:evictions_5m', 'count'],
] },
{ key: 'minio', name: 'MinIO', metrics: [
['capacityPercent', '存储容量使用率', 'cmpp:service_minio:capacity_percent', 'percent'],
['usageBytes', '对象数据量', 'cmpp:service_minio:usage_bytes', 'bytes'],
['objects', '对象数', 'cmpp:service_minio:objects', 'count'],
['drivesOffline', '离线存储盘', 'cmpp:service_minio:drives_offline', 'count'],
] },
{ key: 'nginx', name: 'Nginx', metrics: [
['connectionsActive', '活跃连接', 'cmpp:service_nginx:connections_active', 'count'],
['requestsPerSecond', '请求速率', 'cmpp:service_nginx:requests_per_second', 'per_second'],
] },
] as const;
const SERVICE_METRICS_QUERY = '{__name__=~"cmpp:service_.*"}';
function finiteNumber(value: string | number | undefined): number | null {
const parsed = Number(value);
return Number.isFinite(parsed) ? parsed : null;
}
function normalizePrometheusUrl(rawValue: unknown) {
const url = new URL(String(rawValue ?? 'http://127.0.0.1:9090'));
if (url.protocol !== 'http:' && url.protocol !== 'https:') throw new Error('PROMETHEUS_URL must use HTTP or HTTPS');
if (url.username || url.password) throw new Error('PROMETHEUS_URL must not contain credentials');
const privateIpv4 = /^(10\.|192\.168\.|172\.(1[6-9]|2\d|3[01])\.)/.test(url.hostname);
const loopback = url.hostname === '127.0.0.1' || url.hostname === 'localhost' || url.hostname === '[::1]';
// Plain HTTP is only safe on loopback or an explicit RFC1918 address; named remote endpoints must use HTTPS.
if (url.protocol === 'http:' && !loopback && !privateIpv4) throw new Error('Remote PROMETHEUS_URL must use HTTPS');
return url.toString().replace(/\/$/, '');
}
function vectorValue(response: PrometheusQueryResponse): number | null {
return finiteNumber(response.data?.result?.[0]?.value?.[1]);
}
function matrixValues(response: PrometheusQueryResponse): InfrastructureMetricPoint[] {
return (response.data?.result?.[0]?.values ?? []).flatMap(([timestamp, value]) => {
const parsed = finiteNumber(value);
return parsed === null ? [] : [{ timestamp: new Date(timestamp * 1000).toISOString(), value: parsed }];
});
}
function emptyMetrics(): InfrastructureMonitoringOverview['metrics'] {
return {
cpuUsagePercent: null,
memoryUsagePercent: null,
memoryTotalBytes: null,
memoryAvailableBytes: null,
diskUsagePercent: null,
diskTotalBytes: null,
diskAvailableBytes: null,
networkReceiveBytesPerSecond: null,
networkTransmitBytesPerSecond: null,
load1: null,
uptimeSeconds: null,
};
}
function emptyTrends(): InfrastructureMonitoringOverview['trends'] {
return {
cpuUsagePercent: [],
memoryUsagePercent: [],
diskUsagePercent: [],
networkReceiveBytesPerSecond: [],
networkTransmitBytesPerSecond: [],
};
}
@Injectable()
export class InfrastructureMonitoringService {
private readonly logger = new Logger(InfrastructureMonitoringService.name);
private readonly prometheusUrl: string;
private readonly queryTimeoutMs: number;
constructor(config: ConfigService, private readonly prisma: PrismaService) {
this.prometheusUrl = normalizePrometheusUrl(config.get('PROMETHEUS_URL'));
this.queryTimeoutMs = Math.min(15_000, Math.max(1_000, Number(config.get('PROMETHEUS_QUERY_TIMEOUT_MS') ?? 5_000)));
}
async overview(rawRange?: string, userId?: string): Promise<InfrastructureMonitoringOverview> {
const range = this.parseRange(rawRange);
const collectedAt = new Date().toISOString();
try {
const [instant, trends, serviceResponse, serviceMetricResponse, alertResponse] = await Promise.all([
this.loadInstantMetrics(),
this.loadTrends(range),
this.query(QUERIES.services),
this.query(SERVICE_METRICS_QUERY),
this.getJson<PrometheusAlertResponse>('/api/v1/alerts'),
]);
const services = this.parseServices(serviceResponse);
const serviceMetrics = this.parseServiceMetrics(serviceMetricResponse);
const alerts = await this.attachReadState(this.parseAlerts(alertResponse), userId);
const warningAlerts = alerts.filter((item) => item.severity === 'warning').length;
const criticalAlerts = alerts.filter((item) => item.severity === 'critical').length;
const overallStatus = criticalAlerts > 0 ? 'critical' : warningAlerts > 0 ? 'warning' : 'healthy';
return {
available: true,
range,
collectedAt,
lastSampleAt: instant.lastSampleAt === null ? null : new Date(instant.lastSampleAt * 1000).toISOString(),
summary: {
overallStatus,
serviceTotal: services.length,
serviceHealthy: services.filter((item) => item.status === 'healthy').length,
warningAlerts,
criticalAlerts,
activeAlerts: alerts.length,
},
metrics: instant.metrics,
trends,
services,
serviceMetrics,
alerts,
};
} catch (error) {
// 页面必须整体清空陈旧指标,但服务端仍需留下不含PromQL/地址/凭据的根因摘要便于运维诊断。
this.logger.warn(`Prometheus monitoring overview unavailable: ${error instanceof Error ? error.message : 'unknown error'}`);
return this.unavailable(range, collectedAt);
}
}
async notificationSummary(userId?: string) {
try {
const alerts = await this.attachReadState(this.parseAlerts(await this.getJson<PrometheusAlertResponse>('/api/v1/alerts')), userId);
const unreadAlerts = alerts.filter((item) => !item.acknowledged);
return { count: unreadAlerts.length, criticalCount: unreadAlerts.filter((item) => item.severity === 'critical').length };
} catch (error) {
this.logger.warn(`Prometheus notification summary unavailable: ${error instanceof Error ? error.message : 'unknown error'}`);
throw new ServiceUnavailableException('Prometheus活动告警当前不可用');
}
}
async markAlertRead(fingerprint: string, rawActiveAt: unknown, userId: string) {
if (!/^[a-f0-9]{24}$/.test(fingerprint)) throw new BadRequestException('告警指纹无效');
const activeAt = new Date(String(rawActiveAt ?? ''));
if (!Number.isFinite(activeAt.getTime())) throw new BadRequestException('告警开始时间无效');
const activeAlerts = this.parseAlerts(await this.getJson<PrometheusAlertResponse>('/api/v1/alerts'));
const current = activeAlerts.find((item) => item.fingerprint === fingerprint && Date.parse(item.startedAt) === activeAt.getTime());
if (!current) throw new NotFoundException('该次活动告警已结束或已重新触发,请刷新后重试');
const readAt = new Date();
const log = () => this.prisma.operationLog.create({
data: { userId, action: 'monitoring.alert_marked_read', resource: 'infrastructure_alert', resourceId: fingerprint, detail: { activeAt: activeAt.toISOString(), alertName: current.name, severity: current.severity } },
});
let read;
try {
[read] = await this.prisma.$transaction([
this.prisma.infrastructureAlertRead.create({ data: { fingerprint, activeAt, userId, readAt } }),
log(),
]);
} catch (error) {
if (!(error instanceof Prisma.PrismaClientKnownRequestError) || error.code !== 'P2002') throw error;
const existing = await this.prisma.infrastructureAlertRead.findUniqueOrThrow({ where: { fingerprint_userId: { fingerprint, userId } } });
// 同一次触发重复点击不更新readAt也不重复写日志;activeAt变化才代表同指纹的新触发周期。
if (existing.activeAt.getTime() === activeAt.getTime()) read = existing;
else [read] = await this.prisma.$transaction([
this.prisma.infrastructureAlertRead.update({ where: { fingerprint_userId: { fingerprint, userId } }, data: { activeAt, readAt } }),
log(),
]);
}
return { fingerprint, activeAt: read.activeAt.toISOString(), acknowledged: true, acknowledgedAt: read.readAt.toISOString() };
}
private parseRange(value?: string): InfrastructureMonitoringRange {
const range = value || '24h';
if (!(range in RANGE_CONFIG)) throw new BadRequestException('监控时间范围只支持1h、24h或7d');
return range as InfrastructureMonitoringRange;
}
private async loadInstantMetrics() {
const keys = Object.keys(emptyMetrics()) as Array<keyof InfrastructureMonitoringOverview['metrics']>;
const responses = await Promise.all([...keys.map((key) => this.query(QUERIES[key])), this.query(QUERIES.lastSampleAt)]);
const metrics = emptyMetrics();
keys.forEach((key, index) => { metrics[key] = vectorValue(responses[index]); });
return { metrics, lastSampleAt: vectorValue(responses[responses.length - 1]) };
}
private async loadTrends(range: InfrastructureMonitoringRange) {
const config = RANGE_CONFIG[range];
const end = Math.floor(Date.now() / 1000);
const start = end - config.seconds;
const keys = Object.keys(emptyTrends()) as Array<keyof InfrastructureMonitoringOverview['trends']>;
const responses = await Promise.all(keys.map((key) => this.queryRange(QUERIES[key], start, end, config.step)));
return Object.fromEntries(keys.map((key, index) => [key, matrixValues(responses[index])])) as InfrastructureMonitoringOverview['trends'];
}
private parseServices(response: PrometheusQueryResponse): InfrastructureServiceStatus[] {
const values = new Map<string, number>();
for (const item of response.data?.result ?? []) {
if (item.metric.name) values.set(item.metric.name, vectorValue({ status: 'success', data: { result: [item] } }) ?? 0);
}
return SERVICE_DEFINITIONS.map((definition) => {
const present = definition.units.filter((unit) => values.has(unit));
const status = present.length === 0 ? 'unknown' : present.some((unit) => (values.get(unit) ?? 0) >= 1) ? 'healthy' : 'unhealthy';
return { key: definition.key, name: definition.name, unit: present[0] ?? definition.units[0], status };
});
}
private parseAlerts(response: PrometheusAlertResponse): InfrastructureAlert[] {
return (response.data?.alerts ?? [])
.filter((item) => item.state === 'firing' || item.state === 'pending')
.map<InfrastructureAlert>((item) => {
const labels = item.labels ?? {};
const annotations = item.annotations ?? {};
const severity: InfrastructureAlert['severity'] = labels.severity === 'critical' ? 'critical' : labels.severity === 'warning' ? 'warning' : 'info';
const identity = JSON.stringify(Object.entries(labels).sort(([left], [right]) => left.localeCompare(right)));
return {
fingerprint: createHash('sha256').update(identity).digest('hex').slice(0, 24),
name: labels.alertname || '未命名告警',
severity,
status: item.state || 'unknown',
startedAt: item.activeAt || new Date().toISOString(),
summary: annotations.summary || annotations.description || labels.alertname || '监控告警',
description: annotations.description,
currentValue: annotations.currentValue || item.value,
threshold: annotations.threshold,
service: labels.service,
instance: labels.instance,
acknowledged: false,
};
})
.sort((left, right) => {
const priority: Record<InfrastructureAlert['severity'], number> = { critical: 0, warning: 1, info: 2 };
return priority[left.severity] - priority[right.severity] || Date.parse(left.startedAt) - Date.parse(right.startedAt);
});
}
private async attachReadState(alerts: InfrastructureAlert[], userId?: string) {
if (!userId || alerts.length === 0) return alerts;
const reads = await this.prisma.infrastructureAlertRead.findMany({
where: { userId, fingerprint: { in: alerts.map((item) => item.fingerprint) } },
select: { fingerprint: true, activeAt: true, readAt: true },
});
const byFingerprint = new Map(reads.map((item) => [item.fingerprint, item]));
return alerts.map((alert) => {
const read = byFingerprint.get(alert.fingerprint);
const acknowledged = Boolean(read && read.activeAt.getTime() === Date.parse(alert.startedAt));
return { ...alert, acknowledged, acknowledgedAt: acknowledged ? read?.readAt.toISOString() : undefined };
});
}
private parseServiceMetrics(response: PrometheusQueryResponse): InfrastructureServiceMetricGroup[] {
const values = new Map<string, number>();
for (const item of response.data?.result ?? []) {
const metricName = item.metric.__name__;
const value = vectorValue({ status: 'success', data: { result: [item] } });
if (metricName && value !== null) values.set(metricName, value);
}
return SERVICE_METRIC_DEFINITIONS.map((group) => ({
key: group.key,
name: group.name,
available: group.metrics.some((metric) => values.has(metric[2])),
metrics: group.metrics.map(([key, label, metricName, unit]) => ({ key, label, value: values.get(metricName) ?? null, unit })),
}));
}
private unavailable(range: InfrastructureMonitoringRange, collectedAt: string): InfrastructureMonitoringOverview {
const services = SERVICE_DEFINITIONS.map((item) => ({ key: item.key, name: item.name, unit: item.units[0], status: 'unknown' as const }));
return {
available: false,
range,
collectedAt,
lastSampleAt: null,
error: 'Prometheus监控数据当前不可用,请检查采集与服务状态',
summary: { overallStatus: 'unknown', serviceTotal: services.length, serviceHealthy: 0, warningAlerts: 0, criticalAlerts: 0, activeAlerts: 0 },
metrics: emptyMetrics(),
trends: emptyTrends(),
services,
serviceMetrics: SERVICE_METRIC_DEFINITIONS.map((group) => ({ key: group.key, name: group.name, available: false, metrics: [] })),
alerts: [],
};
}
private query(query: string) {
return this.getJson<PrometheusQueryResponse>('/api/v1/query', { query });
}
private queryRange(query: string, start: number, end: number, step: number) {
return this.getJson<PrometheusQueryResponse>('/api/v1/query_range', { query, start: String(start), end: String(end), step: String(step) });
}
private async getJson<T extends { status: 'success' | 'error'; error?: string }>(path: string, params: Record<string, string> = {}): Promise<T> {
const url = new URL(`${this.prometheusUrl}${path}`);
Object.entries(params).forEach(([key, value]) => url.searchParams.set(key, value));
const response = await fetch(url, { headers: { Accept: 'application/json' }, signal: AbortSignal.timeout(this.queryTimeoutMs) });
if (!response.ok) throw new Error(`Prometheus HTTP ${response.status}`);
const result = await response.json() as T;
if (result.status !== 'success') throw new Error('Prometheus query failed');
return result;
}
}
+22 -1
View File
@@ -1,8 +1,10 @@
import 'reflect-metadata'; import 'reflect-metadata';
import { NestFactory } from '@nestjs/core'; import { NestFactory } from '@nestjs/core';
import { createServer } from 'node:http';
import type { NestExpressApplication } from '@nestjs/platform-express'; import type { NestExpressApplication } from '@nestjs/platform-express';
import { DocumentBuilder, SwaggerModule } from '@nestjs/swagger'; import { DocumentBuilder, SwaggerModule } from '@nestjs/swagger';
import { AppModule } from './app.module'; import { AppModule } from './app.module';
import { MetricsService } from './metrics/metrics.service';
import { OpenApiModule } from './open-api/open-api.module'; import { OpenApiModule } from './open-api/open-api.module';
import { configureHttpBodyParsers } from './http-body-limits'; import { configureHttpBodyParsers } from './http-body-limits';
@@ -39,7 +41,26 @@ async function bootstrap() {
SwaggerModule.setup('api/client-docs', app, clientDocument); SwaggerModule.setup('api/client-docs', app, clientDocument);
const port = Number(process.env.API_PORT ?? 3000); const port = Number(process.env.API_PORT ?? 3000);
await app.listen(port); // 生产环境只允许 Nginx 访问管理 API;显式绑定回环,避免默认的全网卡监听绕过入口鉴权与限流。
const host = process.env.API_HOST?.trim() || '127.0.0.1';
await app.listen(port, host);
const metrics = app.get(MetricsService);
const metricsHost = process.env.API_METRICS_HOST?.trim() || '127.0.0.1';
const metricsPort = Number(process.env.API_METRICS_PORT ?? 9464);
const metricsServer = createServer((request, response) => {
if (request.method !== 'GET' || request.url !== '/metrics') {
response.writeHead(404).end();
return;
}
response.writeHead(200, { 'Content-Type': 'text/plain; version=0.0.4; charset=utf-8', 'Cache-Control': 'no-store' });
response.end(metrics.render());
});
// Metrics use a dedicated loopback listener so Nginx cannot accidentally expose them through /api/.
await new Promise<void>((resolve, reject) => {
metricsServer.once('error', reject);
metricsServer.listen(metricsPort, metricsHost, resolve);
});
} }
void bootstrap(); void bootstrap();
+24
View File
@@ -0,0 +1,24 @@
import { CallHandler, ExecutionContext, Injectable, NestInterceptor } from '@nestjs/common';
import type { Observable } from 'rxjs';
import { finalize } from 'rxjs/operators';
import { MetricsService } from './metrics.service';
type RequestLike = { method?: string; baseUrl?: string; route?: { path?: string } };
type ResponseLike = { statusCode?: number };
@Injectable()
export class MetricsInterceptor implements NestInterceptor {
constructor(private readonly metrics: MetricsService) {}
intercept(context: ExecutionContext, next: CallHandler): Observable<unknown> {
if (context.getType() !== 'http') return next.handle();
const http = context.switchToHttp();
const request = http.getRequest<RequestLike>();
const response = http.getResponse<ResponseLike>();
const startedAt = this.metrics.beginRequest();
return next.handle().pipe(finalize(() => {
const route = `${request.baseUrl ?? ''}${request.route?.path ?? '/unmatched'}`;
this.metrics.finishRequest(startedAt, request.method ?? 'UNKNOWN', route, response.statusCode ?? 500);
}));
}
}
+11
View File
@@ -0,0 +1,11 @@
import { Global, Module } from '@nestjs/common';
import { APP_INTERCEPTOR } from '@nestjs/core';
import { MetricsInterceptor } from './metrics.interceptor';
import { MetricsService } from './metrics.service';
@Global()
@Module({
providers: [MetricsService, { provide: APP_INTERCEPTOR, useClass: MetricsInterceptor }],
exports: [MetricsService],
})
export class MetricsModule {}
+16
View File
@@ -0,0 +1,16 @@
import { MetricsService } from './metrics.service';
describe('MetricsService', () => {
it('exports bounded API process and HTTP metrics without raw identifiers', () => {
const service = new MetricsService();
const startedAt = service.beginRequest();
service.finishRequest(startedAt, 'GET', '/api/admin/tenants/:id', 200);
const output = service.render();
expect(output).toContain('cmpp_api_process_resident_memory_bytes');
expect(output).toContain('cmpp_api_http_requests_total{method="GET",route="/api/admin/tenants/:id",status="200"} 1');
expect(output).toContain('cmpp_api_http_request_duration_seconds_bucket');
expect(output).not.toContain('phone_number');
service.onModuleDestroy();
});
});
+100
View File
@@ -0,0 +1,100 @@
import { Injectable, OnModuleDestroy } from '@nestjs/common';
import { monitorEventLoopDelay } from 'node:perf_hooks';
const HTTP_DURATION_BUCKETS = [0.05, 0.1, 0.25, 0.5, 1, 3, 10] as const;
type HttpMetric = {
count: number;
durationSum: number;
buckets: number[];
};
function escapeLabel(value: string) {
return value.replace(/\\/g, '\\\\').replace(/\n/g, '\\n').replace(/"/g, '\\"');
}
function metricLine(name: string, value: number, labels?: Record<string, string>) {
const suffix = labels
? `{${Object.entries(labels).map(([key, item]) => `${key}="${escapeLabel(item)}"`).join(',')}}`
: '';
return `${name}${suffix} ${Number.isFinite(value) ? value : 0}`;
}
@Injectable()
export class MetricsService implements OnModuleDestroy {
private readonly startedAt = process.hrtime.bigint();
private readonly eventLoopDelay = monitorEventLoopDelay({ resolution: 20 });
private readonly http = new Map<string, HttpMetric>();
private inFlight = 0;
constructor() {
this.eventLoopDelay.enable();
}
beginRequest() {
this.inFlight += 1;
return process.hrtime.bigint();
}
finishRequest(startedAt: bigint, method: string, route: string, statusCode: number) {
this.inFlight = Math.max(0, this.inFlight - 1);
// Only route templates enter labels. Raw URLs, IDs, phone numbers and query strings would create unbounded time series.
const normalizedRoute = route.startsWith('/') ? route : `/${route}`;
const labels = [method.toUpperCase(), normalizedRoute, String(statusCode)];
const key = labels.join('\u0000');
const metric = this.http.get(key) ?? { count: 0, durationSum: 0, buckets: HTTP_DURATION_BUCKETS.map(() => 0) };
const durationSeconds = Number(process.hrtime.bigint() - startedAt) / 1_000_000_000;
metric.count += 1;
metric.durationSum += durationSeconds;
HTTP_DURATION_BUCKETS.forEach((bucket, index) => {
if (durationSeconds <= bucket) metric.buckets[index] += 1;
});
this.http.set(key, metric);
}
render() {
const memory = process.memoryUsage();
const uptime = Number(process.hrtime.bigint() - this.startedAt) / 1_000_000_000;
const lines = [
'# HELP cmpp_api_process_uptime_seconds API process uptime.',
'# TYPE cmpp_api_process_uptime_seconds gauge',
metricLine('cmpp_api_process_uptime_seconds', uptime),
'# HELP cmpp_api_process_resident_memory_bytes API resident memory.',
'# TYPE cmpp_api_process_resident_memory_bytes gauge',
metricLine('cmpp_api_process_resident_memory_bytes', memory.rss),
'# HELP cmpp_api_nodejs_heap_used_bytes Node.js heap currently used.',
'# TYPE cmpp_api_nodejs_heap_used_bytes gauge',
metricLine('cmpp_api_nodejs_heap_used_bytes', memory.heapUsed),
'# HELP cmpp_api_nodejs_heap_total_bytes Node.js allocated heap.',
'# TYPE cmpp_api_nodejs_heap_total_bytes gauge',
metricLine('cmpp_api_nodejs_heap_total_bytes', memory.heapTotal),
'# HELP cmpp_api_nodejs_event_loop_lag_p99_seconds Event loop delay p99 since the previous scrape.',
'# TYPE cmpp_api_nodejs_event_loop_lag_p99_seconds gauge',
metricLine('cmpp_api_nodejs_event_loop_lag_p99_seconds', this.eventLoopDelay.count ? this.eventLoopDelay.percentile(99) / 1_000_000_000 : 0),
'# HELP cmpp_api_http_requests_in_flight Current API requests in flight.',
'# TYPE cmpp_api_http_requests_in_flight gauge',
metricLine('cmpp_api_http_requests_in_flight', this.inFlight),
'# HELP cmpp_api_http_requests_total API requests grouped by bounded route templates.',
'# TYPE cmpp_api_http_requests_total counter',
'# HELP cmpp_api_http_request_duration_seconds API request duration.',
'# TYPE cmpp_api_http_request_duration_seconds histogram',
];
for (const [key, metric] of this.http) {
const [method, route, status] = key.split('\u0000');
const labels = { method, route, status };
lines.push(metricLine('cmpp_api_http_requests_total', metric.count, labels));
HTTP_DURATION_BUCKETS.forEach((bucket, index) => {
lines.push(metricLine('cmpp_api_http_request_duration_seconds_bucket', metric.buckets[index], { ...labels, le: String(bucket) }));
});
lines.push(metricLine('cmpp_api_http_request_duration_seconds_bucket', metric.count, { ...labels, le: '+Inf' }));
lines.push(metricLine('cmpp_api_http_request_duration_seconds_sum', metric.durationSum, labels));
lines.push(metricLine('cmpp_api_http_request_duration_seconds_count', metric.count, labels));
}
this.eventLoopDelay.reset();
return `${lines.join('\n')}\n`;
}
onModuleDestroy() {
this.eventLoopDelay.disable();
}
}
+18 -2
View File
@@ -5,12 +5,13 @@ import IORedis from 'ioredis';
import { PrismaService } from '../prisma/prisma.service'; import { PrismaService } from '../prisma/prisma.service';
import { decryptSecret } from './open-api.crypto'; import { decryptSecret } from './open-api.crypto';
import type { OpenApiRequestLike } from './open-api.types'; import type { OpenApiRequestLike } from './open-api.types';
import { SecurityDetectionService } from '../security-detection/security-detection.service';
@Injectable() @Injectable()
export class OpenApiAuthGuard implements CanActivate, OnModuleDestroy { export class OpenApiAuthGuard implements CanActivate, OnModuleDestroy {
private redis?: IORedis; private redis?: IORedis;
constructor(private readonly prisma: PrismaService) {} constructor(private readonly prisma: PrismaService, private readonly security: SecurityDetectionService) {}
async canActivate(context: ExecutionContext) { async canActivate(context: ExecutionContext) {
const request = context.switchToHttp().getRequest<OpenApiRequestLike>(); const request = context.switchToHttp().getRequest<OpenApiRequestLike>();
@@ -19,9 +20,11 @@ export class OpenApiAuthGuard implements CanActivate, OnModuleDestroy {
const nonce = header(request, 'x-nonce'); const nonce = header(request, 'x-nonce');
const suppliedSignature = header(request, 'x-signature')?.replace(/^sha256=/i, ''); const suppliedSignature = header(request, 'x-signature')?.replace(/^sha256=/i, '');
if (!accessKey || !timestampText || !nonce || !suppliedSignature) { if (!accessKey || !timestampText || !nonce || !suppliedSignature) {
await this.recordFailure('http_signature_failure', request, undefined, 'AUTH_HEADERS_MISSING');
throw new UnauthorizedException({ code: 'AUTH_HEADERS_MISSING', message: '缺少HTTP接口鉴权请求头' }); throw new UnauthorizedException({ code: 'AUTH_HEADERS_MISSING', message: '缺少HTTP接口鉴权请求头' });
} }
if (!/^[A-Za-z0-9_-]{8,128}$/.test(nonce)) { if (!/^[A-Za-z0-9_-]{8,128}$/.test(nonce)) {
await this.recordFailure('http_signature_failure', request, accessKey, 'NONCE_INVALID');
throw new UnauthorizedException({ code: 'NONCE_INVALID', message: 'X-Nonce 格式非法' }); throw new UnauthorizedException({ code: 'NONCE_INVALID', message: 'X-Nonce 格式非法' });
} }
const credential = await this.prisma.httpApiCredential.findUnique({ const credential = await this.prisma.httpApiCredential.findUnique({
@@ -29,6 +32,7 @@ export class OpenApiAuthGuard implements CanActivate, OnModuleDestroy {
include: { application: { include: { httpConfig: true, httpIpAllowlist: true } } }, include: { application: { include: { httpConfig: true, httpIpAllowlist: true } } },
}); });
if (!credential || credential.status !== 'active' || (credential.expiresAt && credential.expiresAt <= new Date())) { if (!credential || credential.status !== 'active' || (credential.expiresAt && credential.expiresAt <= new Date())) {
await this.recordFailure('http_invalid_api_key', request, accessKey, 'CREDENTIAL_INVALID');
throw new UnauthorizedException({ code: 'CREDENTIAL_INVALID', message: '访问凭据无效或已失效' }); throw new UnauthorizedException({ code: 'CREDENTIAL_INVALID', message: '访问凭据无效或已失效' });
} }
const config = credential.application.httpConfig; const config = credential.application.httpConfig;
@@ -37,6 +41,7 @@ export class OpenApiAuthGuard implements CanActivate, OnModuleDestroy {
} }
const timestamp = Number(timestampText); const timestamp = Number(timestampText);
if (!Number.isFinite(timestamp) || Math.abs(Date.now() - timestamp * 1000) > config.timestampToleranceSeconds * 1000) { if (!Number.isFinite(timestamp) || Math.abs(Date.now() - timestamp * 1000) > config.timestampToleranceSeconds * 1000) {
await this.recordFailure('http_signature_failure', request, accessKey, 'TIMESTAMP_EXPIRED');
throw new UnauthorizedException({ code: 'TIMESTAMP_EXPIRED', message: '请求时间戳已过期' }); throw new UnauthorizedException({ code: 'TIMESTAMP_EXPIRED', message: '请求时间戳已过期' });
} }
const sourceIp = requestIp(request); const sourceIp = requestIp(request);
@@ -50,11 +55,13 @@ export class OpenApiAuthGuard implements CanActivate, OnModuleDestroy {
const expectedBuffer = Buffer.from(expected, 'hex'); const expectedBuffer = Buffer.from(expected, 'hex');
const suppliedBuffer = /^[0-9a-f]{64}$/i.test(suppliedSignature) ? Buffer.from(suppliedSignature, 'hex') : Buffer.alloc(0); const suppliedBuffer = /^[0-9a-f]{64}$/i.test(suppliedSignature) ? Buffer.from(suppliedSignature, 'hex') : Buffer.alloc(0);
if (expectedBuffer.length !== suppliedBuffer.length || !timingSafeEqual(expectedBuffer, suppliedBuffer)) { if (expectedBuffer.length !== suppliedBuffer.length || !timingSafeEqual(expectedBuffer, suppliedBuffer)) {
await this.recordFailure('http_signature_failure', request, accessKey, 'SIGNATURE_INVALID');
throw new UnauthorizedException({ code: 'SIGNATURE_INVALID', message: '请求签名校验失败' }); throw new UnauthorizedException({ code: 'SIGNATURE_INVALID', message: '请求签名校验失败' });
} }
const redis = this.getRedis(); const redis = this.getRedis();
const nonceAccepted = await redis.set(`openapi:nonce:${credential.id}:${nonce}`, '1', 'EX', config.timestampToleranceSeconds * 2, 'NX'); const nonceAccepted = await redis.set(`openapi:nonce:${credential.id}:${nonce}`, '1', 'EX', config.timestampToleranceSeconds * 2, 'NX');
if (nonceAccepted !== 'OK') { if (nonceAccepted !== 'OK') {
await this.recordFailure('http_replay_attempt', request, accessKey, 'NONCE_REPLAYED');
throw new UnauthorizedException({ code: 'NONCE_REPLAYED', message: 'X-Nonce 已使用' }); throw new UnauthorizedException({ code: 'NONCE_REPLAYED', message: 'X-Nonce 已使用' });
} }
const second = Math.floor(Date.now() / 1000); const second = Math.floor(Date.now() / 1000);
@@ -81,6 +88,13 @@ export class OpenApiAuthGuard implements CanActivate, OnModuleDestroy {
this.redis ??= new IORedis(process.env.REDIS_URL ?? 'redis://127.0.0.1:6379', { maxRetriesPerRequest: 1 }); this.redis ??= new IORedis(process.env.REDIS_URL ?? 'redis://127.0.0.1:6379', { maxRetriesPerRequest: 1 });
return this.redis; return this.redis;
} }
private async recordFailure(ruleCode: 'http_invalid_api_key' | 'http_signature_failure' | 'http_replay_attempt', request: OpenApiRequestLike, account: string | undefined, resultCode: string) {
const sourceIp = requestIp(request);
if (!sourceIp) return;
// 检测记录失败不能改变原鉴权响应,避免安全辅助链路放大为业务可用性事故。
await this.security.recordEvent({ ruleCode, sourceIp, account, resultCode, protocol: 'http', path: (request.originalUrl ?? request.url ?? '').split('?')[0] }).catch(() => undefined);
}
} }
function header(request: OpenApiRequestLike, name: string) { function header(request: OpenApiRequestLike, name: string) {
@@ -90,7 +104,9 @@ function header(request: OpenApiRequestLike, name: string) {
function requestIp(request: OpenApiRequestLike) { function requestIp(request: OpenApiRequestLike) {
const forwarded = header(request, 'x-forwarded-for')?.split(',')[0]?.trim(); const forwarded = header(request, 'x-forwarded-for')?.split(',')[0]?.trim();
return (forwarded ?? request.socket?.remoteAddress)?.replace(/^::ffff:/, ''); const remoteAddress = request.socket?.remoteAddress?.replace(/^::ffff:/, '');
const trustedProxies = new Set((process.env.TRUSTED_PROXY_IPS ?? '127.0.0.1,::1').split(',').map((item) => item.trim()).filter(Boolean));
return (remoteAddress && trustedProxies.has(remoteAddress) ? forwarded : remoteAddress)?.replace(/^::ffff:/, '');
} }
function ipMatches(ip: string, rule: string) { function ipMatches(ip: string, rule: string) {
+2 -1
View File
@@ -6,9 +6,10 @@ import { ClientOpenApiController } from './client-open-api.controller';
import { OpenApiAuthGuard } from './open-api-auth.guard'; import { OpenApiAuthGuard } from './open-api-auth.guard';
import { OpenApiController } from './open-api.controller'; import { OpenApiController } from './open-api.controller';
import { OpenApiService } from './open-api.service'; import { OpenApiService } from './open-api.service';
import { SecurityDetectionModule } from '../security-detection/security-detection.module';
@Module({ @Module({
imports: [PrismaModule, forwardRef(() => SendChainModule)], imports: [PrismaModule, forwardRef(() => SendChainModule), SecurityDetectionModule],
controllers: [OpenApiController, AdminOpenApiController, ClientOpenApiController], controllers: [OpenApiController, AdminOpenApiController, ClientOpenApiController],
providers: [OpenApiService, OpenApiAuthGuard], providers: [OpenApiService, OpenApiAuthGuard],
exports: [OpenApiService], exports: [OpenApiService],
@@ -0,0 +1,58 @@
import { Injectable } from '@nestjs/common';
import { ConfigService } from '@nestjs/config';
import { createConnection } from 'node:net';
type AgentResponse = { ok: boolean; reference?: string; blocked?: boolean; active?: boolean; error?: string };
@Injectable()
export class SecurityAgentClient {
constructor(private readonly config: ConfigService) {}
block(input: { operationKey: string; sourceIp: string; executor: string; durationSeconds: number }) {
return this.call({ action: 'block', ...input });
}
unblock(input: { operationKey: string; sourceIp: string; executor: string }) {
return this.call({ action: 'unblock', ...input });
}
status(sourceIp?: string, executor?: string) {
return this.call({ action: 'status', sourceIp, executor });
}
applyRules(version: number, rules: Array<Record<string, unknown>>) {
return this.call({ action: 'apply_rules', version, rules });
}
private call(payload: Record<string, unknown>): Promise<AgentResponse> {
const socketPath = this.config.get<string>('SECURITY_AGENT_SOCKET') ?? '/run/cmpp-security-agent/agent.sock';
const timeoutMs = Number(this.config.get<string>('SECURITY_AGENT_TIMEOUT_MS') ?? 3000);
return new Promise((resolve, reject) => {
const socket = createConnection(socketPath);
let settled = false;
let response = '';
const finish = (error?: Error) => {
if (settled) return;
settled = true;
socket.destroy();
if (error) reject(error);
};
socket.setTimeout(timeoutMs, () => finish(new Error('安全执行代理响应超时')));
socket.on('error', (error) => finish(new Error(`安全执行代理不可用: ${error.message}`)));
socket.on('connect', () => socket.write(`${JSON.stringify(payload)}\n`));
socket.on('data', (chunk) => {
response += chunk.toString('utf8');
const lineEnd = response.indexOf('\n');
if (lineEnd < 0) return;
try {
const parsed = JSON.parse(response.slice(0, lineEnd)) as AgentResponse;
settled = true;
socket.end();
resolve(parsed);
} catch {
finish(new Error('安全执行代理返回了非法响应'));
}
});
});
}
}
@@ -0,0 +1,10 @@
export const SECURITY_RULE_CODES = [
'admin_login_failure', 'client_login_failure', 'ssh_auth_failure', 'cmpp_auth_failure',
'cmpp_protocol_abuse', 'http_invalid_api_key', 'http_signature_failure',
'http_replay_attempt', 'http_malicious_scan',
] as const;
export type SecurityRuleCode = typeof SECURITY_RULE_CODES[number];
export const SECURITY_RULE_CODE_SET = new Set<string>(SECURITY_RULE_CODES);
export const SECURITY_SEVERITIES = new Set(['low', 'medium', 'high', 'critical']);
export const SECURITY_BLOCK_DURATIONS = new Set([600, 3600, 86400, 604800]);
@@ -0,0 +1,22 @@
import { Body, Controller, Get, Param, Post, Put, Query } from '@nestjs/common';
import { ApiTags } from '@nestjs/swagger';
import { CurrentSessionUserId } from '../auth/current-session-user.decorator';
import { RequireRecentAuthentication } from '../auth/require-recent-authentication.decorator';
import { SecurityDetectionService } from './security-detection.service';
@ApiTags('security-detection')
@Controller('admin/security-detection')
export class SecurityDetectionController {
constructor(private readonly security: SecurityDetectionService) {}
@Get('overview') overview(@Query('range') range?: string) { return this.security.overview(range); }
@Get('notification-summary') notificationSummary() { return this.security.notificationSummary(); }
@Get('alerts') alerts(@Query() query: Record<string, string>) { return this.security.listAlerts(query); }
@Get('rules') rules() { return this.security.listRules(); }
@Put('rules/:id') @RequireRecentAuthentication() updateRule(@Param('id') id: string, @Body() body: Record<string, unknown>, @CurrentSessionUserId() userId: string) { return this.security.updateRule(id, body, userId); }
@Post('alerts/:id/block') @RequireRecentAuthentication() block(@Param('id') id: string, @Body() body: { durationSeconds?: number; reason?: string }, @CurrentSessionUserId() userId: string) { return this.security.block(id, body, userId); }
@Post('alerts/:id/ignore') @RequireRecentAuthentication() ignore(@Param('id') id: string, @Body('reason') reason: string, @CurrentSessionUserId() userId: string) { return this.security.ignore(id, reason ?? '', userId); }
@Get('blocks') blocks() { return this.security.listBlocks(); }
@Post('blocks/:id/unblock') @RequireRecentAuthentication() unblock(@Param('id') id: string, @Body('reason') reason: string, @CurrentSessionUserId() userId: string) { return this.security.unblock(id, reason ?? '', userId); }
@Get('protected-networks') protectedNetworks() { return this.security.listProtectedNetworks(); }
@Post('protected-networks') @RequireRecentAuthentication() addProtectedNetwork(@Body() body: { network?: string; name?: string; reason?: string }, @CurrentSessionUserId() userId: string) { return this.security.addProtectedNetwork(body, userId); }
}
@@ -0,0 +1,8 @@
import { Module } from '@nestjs/common';
import { SecurityAgentClient } from './security-agent.client';
import { SecurityDetectionController } from './security-detection.controller';
import { SecurityEventController } from './security-event.controller';
import { SecurityDetectionService } from './security-detection.service';
@Module({ controllers: [SecurityDetectionController, SecurityEventController], providers: [SecurityAgentClient, SecurityDetectionService], exports: [SecurityDetectionService] })
export class SecurityDetectionModule {}
@@ -0,0 +1,75 @@
import { ConflictException } from '@nestjs/common';
import { SecurityDetectionService } from './security-detection.service';
function createPrisma() {
const tx = {
$executeRaw: jest.fn(),
securityDetectionEvent: { create: jest.fn(), count: jest.fn() },
securityAlert: { findFirst: jest.fn(), create: jest.fn(), update: jest.fn(), updateMany: jest.fn() },
securityBlock: { create: jest.fn(), update: jest.fn() },
};
const prisma = {
securityDetectionRule: { findUnique: jest.fn(), findMany: jest.fn() },
securityDetectionEvent: { count: jest.fn() },
securityAlert: { findUnique: jest.fn(), update: jest.fn(), count: jest.fn() },
securityBlock: { create: jest.fn(), update: jest.fn() },
securityProtectedNetwork: { findMany: jest.fn().mockResolvedValue([]) },
operationLog: { create: jest.fn() },
$transaction: jest.fn(async (value: unknown) => typeof value === 'function' ? value(tx) : Promise.all(value as Promise<unknown>[])),
};
return { prisma, tx };
}
describe('SecurityDetectionService', () => {
it('returns an independent active and critical alert summary for the global bell', async () => {
const { prisma } = createPrisma();
prisma.securityAlert.count.mockResolvedValueOnce(4).mockResolvedValueOnce(2);
const service = new SecurityDetectionService(prisma as never, {} as never);
await expect(service.notificationSummary()).resolves.toEqual({ count: 4, criticalCount: 2 });
expect(prisma.securityAlert.count).toHaveBeenNthCalledWith(1, { where: { status: { in: ['open', 'acknowledged', 'block_failed'] } } });
expect(prisma.securityAlert.count).toHaveBeenNthCalledWith(2, { where: { status: { in: ['open', 'acknowledged', 'block_failed'] }, severity: 'critical' } });
});
it('keeps a below-threshold event without creating a false alert', async () => {
const { prisma, tx } = createPrisma();
prisma.securityDetectionRule.findUnique.mockResolvedValue({ id: 'rule-1', enabled: true, threshold: 3, windowSeconds: 60, cooldownSeconds: 60, severity: 'high' });
tx.securityDetectionEvent.create.mockResolvedValue({ id: 'event-1' });
tx.securityDetectionEvent.count.mockResolvedValue(2);
const service = new SecurityDetectionService(prisma as never, {} as never);
await expect(service.recordEvent({ eventKey: 'event-key-1', ruleCode: 'http_signature_failure', sourceIp: '203.0.113.5' })).resolves.toEqual({ accepted: true, duplicate: false, alertId: null });
expect(tx.securityAlert.create).not.toHaveBeenCalled();
});
it('refuses built-in protected addresses before calling the privileged agent', async () => {
const { prisma } = createPrisma();
prisma.securityAlert.findUnique.mockResolvedValue({
id: 'alert-1', sourceIp: '127.0.0.1', status: 'open',
rule: { code: 'ssh_auth_failure', defaultBlockSeconds: 600, maximumBlockSeconds: 604800 },
});
const agent = { block: jest.fn(), status: jest.fn() };
const service = new SecurityDetectionService(prisma as never, agent as never);
await expect(service.block('alert-1', { durationSeconds: 600, reason: '隔离测试封禁' }, 'operator-1')).rejects.toBeInstanceOf(ConflictException);
expect(agent.block).not.toHaveBeenCalled();
});
it('maps an admin-login alert to nginx and marks blocked only after readback', async () => {
const { prisma, tx } = createPrisma();
prisma.securityAlert.findUnique.mockResolvedValue({
id: 'alert-1', sourceIp: '203.0.113.8', status: 'open',
rule: { code: 'admin_login_failure', defaultBlockSeconds: 600, maximumBlockSeconds: 604800 },
});
tx.securityAlert.updateMany.mockResolvedValue({ count: 1 });
tx.securityBlock.create.mockResolvedValue({ id: 'block-1' });
tx.securityBlock.update.mockResolvedValue({ id: 'block-1', status: 'blocked' });
tx.securityAlert.update.mockResolvedValue({ id: 'alert-1', status: 'blocked' });
const agent = { block: jest.fn().mockResolvedValue({ ok: true, reference: 'op-1' }), status: jest.fn().mockResolvedValue({ ok: true, blocked: true }) };
const service = new SecurityDetectionService(prisma as never, agent as never);
await expect(service.block('alert-1', { durationSeconds: 600, reason: '确认恶意登录扫描' }, 'operator-1')).resolves.toEqual(expect.objectContaining({ status: 'blocked' }));
expect(agent.block).toHaveBeenCalledWith(expect.objectContaining({ executor: 'nginx_real_ip', sourceIp: '203.0.113.8' }));
expect(agent.status).toHaveBeenCalledWith('203.0.113.8', 'nginx_real_ip');
});
});
@@ -0,0 +1,255 @@
import { BadRequestException, ConflictException, Injectable, NotFoundException } from '@nestjs/common';
import { createHash, randomUUID } from 'node:crypto';
import { isIP } from 'node:net';
import { Prisma } from '@prisma/client';
import { PrismaService } from '../prisma/prisma.service';
import { SecurityAgentClient } from './security-agent.client';
import { SECURITY_BLOCK_DURATIONS, SECURITY_RULE_CODE_SET, SECURITY_SEVERITIES, type SecurityRuleCode } from './security-detection.constants';
export type SecurityEventInput = {
eventKey?: string; ruleCode: SecurityRuleCode; sourceIp: string; sourcePort?: number;
account?: string; path?: string; protocol?: string; resultCode?: string;
evidence?: Record<string, unknown>; occurredAt?: string | Date;
};
@Injectable()
export class SecurityDetectionService {
constructor(private readonly prisma: PrismaService, private readonly agent: SecurityAgentClient) {}
async recordEvent(input: SecurityEventInput) {
if (!SECURITY_RULE_CODE_SET.has(input.ruleCode)) throw new BadRequestException('不支持的安全检测类型');
const sourceIp = normalizeIp(input.sourceIp);
const occurredAt = input.occurredAt ? new Date(input.occurredAt) : new Date();
if (!Number.isFinite(occurredAt.getTime())) throw new BadRequestException('安全事件时间无效');
const eventKey = input.eventKey ?? createHash('sha256').update(JSON.stringify([
input.ruleCode, sourceIp, input.sourcePort, input.account, input.path, input.resultCode,
occurredAt.toISOString(), input.evidence,
])).digest('hex');
const rule = await this.prisma.securityDetectionRule.findUnique({ where: { code: input.ruleCode } });
if (!rule) throw new NotFoundException('安全检测规则不存在');
return this.prisma.$transaction(async (tx) => {
// 同一来源和规则串行聚合,避免并发计数跨过阈值时创建多个告警。
await tx.$executeRaw`SELECT pg_advisory_xact_lock(hashtext(${`${rule.id}:${sourceIp}`}))`;
try {
await tx.securityDetectionEvent.create({ data: {
eventKey, ruleId: rule.id, sourceIp, sourcePort: input.sourcePort,
accountHash: input.account ? createHash('sha256').update(input.account).digest('hex') : undefined,
path: input.path?.slice(0, 512), protocol: input.protocol?.slice(0, 32), resultCode: input.resultCode?.slice(0, 128),
evidence: sanitizeEvidence(input.evidence), occurredAt,
} });
} catch (error) {
if (isUniqueViolation(error)) return { accepted: true, duplicate: true, alertId: null };
throw error;
}
if (!rule.enabled) return { accepted: true, duplicate: false, alertId: null };
const windowStartedAt = new Date(occurredAt.getTime() - rule.windowSeconds * 1000);
const storedEventCount = await tx.securityDetectionEvent.count({
where: { ruleId: rule.id, sourceIp, occurredAt: { gte: windowStartedAt, lte: occurredAt } },
});
// Fail2ban上报代表其自身窗口已经达到maxretry;应用事件则逐条在数据库窗口内计数。
const eventCount = rule.sourceType === 'fail2ban' ? Math.max(storedEventCount, rule.threshold) : storedEventCount;
if (eventCount < rule.threshold) return { accepted: true, duplicate: false, alertId: null };
const cooldownStart = new Date(occurredAt.getTime() - rule.cooldownSeconds * 1000);
const active = await tx.securityAlert.findFirst({
where: { ruleId: rule.id, sourceIp, status: { in: ['open', 'acknowledged', 'block_failed', 'blocked'] }, lastOccurredAt: { gte: cooldownStart } },
orderBy: { lastOccurredAt: 'desc' },
});
if (active) {
const updated = await tx.securityAlert.update({ where: { id: active.id }, data: { eventCount, lastOccurredAt: occurredAt } });
return { accepted: true, duplicate: false, alertId: updated.id };
}
const fingerprint = createHash('sha256').update(`${rule.id}:${sourceIp}:${occurredAt.toISOString()}`).digest('hex');
const alert = await tx.securityAlert.create({ data: {
fingerprint, ruleId: rule.id, sourceIp, severity: rule.severity, eventCount,
windowStartedAt, firstOccurredAt: occurredAt, lastOccurredAt: occurredAt,
} });
return { accepted: true, duplicate: false, alertId: alert.id };
}, { isolationLevel: Prisma.TransactionIsolationLevel.ReadCommitted });
}
async overview(range = '24h') {
if (!['1h', '24h', '7d'].includes(range)) throw new BadRequestException('仅支持1h、24h或7d安全检测范围');
const hours = range === '1h' ? 1 : range === '7d' ? 168 : 24;
const since = new Date(Date.now() - hours * 3600_000);
const activeStatuses = ['open', 'acknowledged', 'block_failed'];
const [alerts, totalEvents, activeBlocks, rules, activeAlerts, criticalAlerts, distribution, agentStatus] = await Promise.all([
this.prisma.securityAlert.findMany({ where: { lastOccurredAt: { gte: since } }, include: { rule: true }, orderBy: { lastOccurredAt: 'desc' }, take: 12 }),
this.prisma.securityDetectionEvent.count({ where: { occurredAt: { gte: since } } }),
this.prisma.securityBlock.count({ where: { status: 'blocked', expiresAt: { gt: new Date() } } }),
this.prisma.securityDetectionRule.findMany({ orderBy: { name: 'asc' } }),
this.prisma.securityAlert.count({ where: { status: { in: activeStatuses } } }),
this.prisma.securityAlert.count({ where: { status: { in: activeStatuses }, severity: 'critical' } }),
this.prisma.securityAlert.groupBy({ by: ['ruleId'], where: { lastOccurredAt: { gte: since } }, _sum: { eventCount: true } }),
this.agent.status().catch((error: Error) => ({ ok: false, active: false, error: error.message })),
]);
const ruleNames = new Map(rules.map((rule) => [rule.id, rule.name]));
return {
range, collectedAt: new Date().toISOString(), totalEvents, activeAlerts, criticalAlerts, activeBlocks,
health: { agent: agentStatus.ok && agentStatus.active ? 'healthy' : 'unavailable', agentError: agentStatus.error, rulesEffective: rules.filter((rule) => rule.applyStatus === 'effective').length, rulesTotal: rules.length },
sourceDistribution: distribution.map((item) => ({ name: ruleNames.get(item.ruleId) ?? item.ruleId, value: item._sum.eventCount ?? 0 })),
alerts,
};
}
async notificationSummary() {
const activeStatuses = ['open', 'acknowledged', 'block_failed'];
const [count, criticalCount] = await Promise.all([
this.prisma.securityAlert.count({ where: { status: { in: activeStatuses } } }),
this.prisma.securityAlert.count({ where: { status: { in: activeStatuses }, severity: 'critical' } }),
]);
return { count, criticalCount };
}
listAlerts(query: { status?: string; ruleCode?: string; sourceIp?: string; page?: string; pageSize?: string }) {
const page = positiveInt(query.page, 1, 100000);
const pageSize = positiveInt(query.pageSize, 20, 100);
const where: Prisma.SecurityAlertWhereInput = {
...(query.status ? { status: query.status } : {}),
...(query.ruleCode ? { rule: { code: query.ruleCode } } : {}),
...(query.sourceIp ? { sourceIp: normalizeIp(query.sourceIp) } : {}),
};
return Promise.all([
this.prisma.securityAlert.findMany({ where, include: { rule: true }, orderBy: { lastOccurredAt: 'desc' }, skip: (page - 1) * pageSize, take: pageSize }),
this.prisma.securityAlert.count({ where }),
]).then(([items, total]) => ({ items, total, page, pageSize }));
}
listRules() { return this.prisma.securityDetectionRule.findMany({ orderBy: [{ sourceType: 'asc' }, { name: 'asc' }] }); }
async updateRule(id: string, input: Record<string, unknown>, operatorId: string) {
assertAllowedKeys(input, ['configVersion', 'enabled', 'threshold', 'windowSeconds', 'cooldownSeconds', 'severity', 'defaultBlockSeconds', 'maximumBlockSeconds']);
if (typeof input.enabled !== 'boolean') throw new BadRequestException('启用状态必须为布尔值');
const current = await this.prisma.securityDetectionRule.findUnique({ where: { id } });
if (!current) throw new NotFoundException('规则不存在');
if (Number(input.configVersion) !== current.configVersion) throw new ConflictException('规则已被其他管理员修改,请刷新后重试');
const threshold = boundedInt(input.threshold, 1, 100000, '触发次数');
const windowSeconds = boundedInt(input.windowSeconds, 10, 86400, '检测窗口');
const cooldownSeconds = boundedInt(input.cooldownSeconds, 0, 604800, '告警冷却');
const defaultBlockSeconds = boundedInt(input.defaultBlockSeconds, 600, 604800, '默认封禁时长');
const maximumBlockSeconds = boundedInt(input.maximumBlockSeconds, defaultBlockSeconds, 604800, '最大封禁时长');
const severity = String(input.severity ?? '');
if (!SECURITY_SEVERITIES.has(severity)) throw new BadRequestException('告警级别无效');
const version = current.configVersion + 1;
const nextConfig = { enabled: Boolean(input.enabled), threshold, windowSeconds, cooldownSeconds, severity, defaultBlockSeconds, maximumBlockSeconds };
await this.prisma.securityDetectionRule.update({ where: { id }, data: {
configVersion: version, applyStatus: 'applying', lastApplyError: null, pendingConfig: nextConfig,
} });
await this.prisma.operationLog.create({ data: { userId: operatorId, action: 'security.rule_updated', resource: 'security_detection_rule', resourceId: id, detail: { version, beforeVersion: current.configVersion } } });
try {
const response = await this.agent.applyRules(version, (await this.listRules()).map((rule) => rule.id === id
? { code: rule.code, enabled: nextConfig.enabled, threshold: nextConfig.threshold, windowSeconds: nextConfig.windowSeconds, cooldownSeconds: nextConfig.cooldownSeconds }
: { code: rule.code, enabled: rule.enabled, threshold: rule.threshold, windowSeconds: rule.windowSeconds, cooldownSeconds: rule.cooldownSeconds }));
if (!response.ok) throw new Error(response.error ?? '安全代理拒绝应用规则');
return this.prisma.securityDetectionRule.update({ where: { id }, data: { ...nextConfig, effectiveVersion: version, applyStatus: 'effective', pendingConfig: Prisma.JsonNull } });
} catch (error) {
const message = error instanceof Error ? error.message : '规则应用失败';
await this.prisma.securityDetectionRule.update({ where: { id }, data: { applyStatus: 'failed', lastApplyError: message } });
throw new ConflictException({ code: 'SECURITY_RULE_APPLY_FAILED', message });
}
}
async block(alertId: string, input: { durationSeconds?: number; reason?: string }, operatorId: string) {
assertAllowedKeys(input as Record<string, unknown>, ['durationSeconds', 'reason']);
const alert = await this.prisma.securityAlert.findUnique({ where: { id: alertId }, include: { rule: true } });
if (!alert) throw new NotFoundException('告警不存在');
if (!['open', 'acknowledged', 'block_failed'].includes(alert.status)) throw new ConflictException('该告警当前不可封禁');
const durationSeconds = Number(input.durationSeconds ?? alert.rule.defaultBlockSeconds);
if (!SECURITY_BLOCK_DURATIONS.has(durationSeconds) || durationSeconds > alert.rule.maximumBlockSeconds) throw new BadRequestException('封禁时长不在允许范围内');
const reason = String(input.reason ?? '').trim();
if (reason.length < 5 || reason.length > 500) throw new BadRequestException('封禁原因需为5至500个字符');
if (isSystemProtected(alert.sourceIp) || await this.isProtected(alert.sourceIp)) throw new ConflictException({ code: 'PROTECTED_NETWORK', message: '该地址属于系统或人工保护名单,禁止封禁' });
// 执行器由可信的规则入口固定映射,绝不接受浏览器指定,避免把Cloudflare访客IP错误交给nftables。
const executor = ['admin_login_failure', 'client_login_failure'].includes(alert.rule.code) ? 'nginx_real_ip' : 'nftables';
const operationKey = randomUUID();
const block = await this.prisma.$transaction(async (tx) => {
const claimed = await tx.securityAlert.updateMany({ where: { id: alert.id, status: { in: ['open', 'acknowledged', 'block_failed'] } }, data: { status: 'block_requested' } });
if (!claimed.count) throw new ConflictException('告警已由其他管理员处理,请刷新后重试');
return tx.securityBlock.create({ data: { operationKey, alertId, sourceIp: alert.sourceIp, executor, durationSeconds, reason, requestedById: operatorId } });
});
await this.prisma.operationLog.create({ data: { userId: operatorId, action: 'security.block_requested', resource: 'security_block', resourceId: block.id, detail: { alertId, sourceIp: alert.sourceIp, executor, durationSeconds, reason } } });
try {
const applied = await this.agent.block({ operationKey, sourceIp: alert.sourceIp, executor, durationSeconds });
if (!applied.ok) throw new Error(applied.error ?? '安全代理拒绝封禁');
const readback = await this.agent.status(alert.sourceIp, executor);
if (!readback.ok || !readback.blocked) throw new Error(readback.error ?? '执行后未读到真实封禁状态');
const expiresAt = new Date(Date.now() + durationSeconds * 1000);
const result = await this.prisma.$transaction(async (tx) => {
const updated = await tx.securityBlock.update({ where: { id: block.id }, data: { status: 'blocked', appliedAt: new Date(), expiresAt, executorReference: applied.reference } });
await tx.securityAlert.update({ where: { id: alert.id }, data: { status: 'blocked', blockId: block.id } });
return updated;
});
return result;
} catch (error) {
const message = error instanceof Error ? error.message : '封禁执行失败';
await this.prisma.$transaction([
this.prisma.securityBlock.update({ where: { id: block.id }, data: { status: 'failed', lastError: message } }),
this.prisma.securityAlert.update({ where: { id: alert.id }, data: { status: 'block_failed' } }),
]);
throw new ConflictException({ code: 'SECURITY_BLOCK_FAILED', message });
}
}
async ignore(alertId: string, reason: string, operatorId: string) {
if (reason.trim().length < 5) throw new BadRequestException('忽略原因至少5个字符');
const updated = await this.prisma.securityAlert.updateMany({ where: { id: alertId, status: { in: ['open', 'acknowledged', 'block_failed'] } }, data: { status: 'ignored', ignoredAt: new Date(), ignoredById: operatorId, ignoreReason: reason.trim() } });
if (!updated.count) throw new ConflictException('告警状态已变化,请刷新后重试');
await this.prisma.operationLog.create({ data: { userId: operatorId, action: 'security.alert_ignored', resource: 'security_alert', resourceId: alertId, detail: { reason: reason.trim() } } });
return { success: true };
}
listBlocks() { return this.prisma.securityBlock.findMany({ orderBy: { requestedAt: 'desc' }, take: 200 }); }
async unblock(blockId: string, reason: string, operatorId: string) {
if (reason.trim().length < 5) throw new BadRequestException('解封原因至少5个字符');
const block = await this.prisma.securityBlock.findUnique({ where: { id: blockId } });
if (!block) throw new NotFoundException('封禁记录不存在');
if (block.status !== 'blocked') throw new ConflictException('该记录当前不可解封');
const claimed = await this.prisma.securityBlock.updateMany({ where: { id: blockId, status: 'blocked' }, data: { status: 'unblock_requested' } });
if (!claimed.count) throw new ConflictException('封禁状态已变化,请刷新后重试');
try {
const result = await this.agent.unblock({ operationKey: randomUUID(), sourceIp: block.sourceIp, executor: block.executor });
if (!result.ok) throw new Error(result.error ?? '安全代理拒绝解封');
const readback = await this.agent.status(block.sourceIp, block.executor);
if (!readback.ok || readback.blocked) throw new Error(readback.error ?? '执行后仍读到封禁规则');
const updated = await this.prisma.securityBlock.update({ where: { id: block.id }, data: { status: 'released', releasedAt: new Date(), releasedById: operatorId } });
if (block.alertId) await this.prisma.securityAlert.updateMany({ where: { id: block.alertId, blockId: block.id }, data: { status: 'unblocked' } });
await this.prisma.operationLog.create({ data: { userId: operatorId, action: 'security.block_released', resource: 'security_block', resourceId: block.id, detail: { sourceIp: block.sourceIp, executor: block.executor, reason: reason.trim() } } });
return updated;
} catch (error) {
const message = error instanceof Error ? error.message : '解封失败';
await this.prisma.securityBlock.update({ where: { id: block.id }, data: { status: 'blocked', lastError: message } });
throw new ConflictException({ code: 'SECURITY_UNBLOCK_FAILED', message });
}
}
listProtectedNetworks() { return this.prisma.securityProtectedNetwork.findMany({ orderBy: { createdAt: 'desc' } }); }
async addProtectedNetwork(input: { network?: string; name?: string; reason?: string }, operatorId: string) {
const network = normalizeNetwork(String(input.network ?? ''));
if (!input.name?.trim() || !input.reason?.trim()) throw new BadRequestException('名称和保护原因不能为空');
const result = await this.prisma.securityProtectedNetwork.create({ data: { network, name: input.name.trim(), reason: input.reason.trim(), createdById: operatorId } });
await this.prisma.operationLog.create({ data: { userId: operatorId, action: 'security.protected_network_created', resource: 'security_protected_network', resourceId: result.id, detail: { network } } });
return result;
}
private async isProtected(ip: string) {
const entries = await this.prisma.securityProtectedNetwork.findMany({ where: { enabled: true }, select: { network: true } });
return entries.some((entry) => networkContains(entry.network, ip));
}
}
function normalizeIp(value: string) { const normalized = value?.trim().replace(/^::ffff:/, ''); if (!isIP(normalized)) throw new BadRequestException('来源IP无效'); return normalized; }
function normalizeNetwork(value: string) { const [address, prefix] = value.trim().split('/'); const family = isIP(address); if (!family) throw new BadRequestException('保护网段无效'); if (prefix === undefined) return address; const bits = Number(prefix); const max = family === 4 ? 32 : 128; if (!Number.isInteger(bits) || bits < 0 || bits > max) throw new BadRequestException('保护网段前缀无效'); return `${address}/${bits}`; }
function networkContains(network: string, ip: string) { const [address, prefixText] = network.split('/'); if (isIP(address) !== isIP(ip)) return false; if (prefixText === undefined) return address === ip; const bits = Number(prefixText); return (addressToBigInt(address) >> BigInt((isIP(address) === 4 ? 32 : 128) - bits)) === (addressToBigInt(ip) >> BigInt((isIP(ip) === 4 ? 32 : 128) - bits)); }
function addressToBigInt(value: string) { if (isIP(value) === 4) return value.split('.').reduce((total, part) => (total << 8n) + BigInt(part), 0n); const [left, right = ''] = value.toLowerCase().split('::'); const leftParts = left ? left.split(':') : []; const rightParts = right ? right.split(':') : []; const parts = [...leftParts, ...Array(Math.max(0, 8 - leftParts.length - rightParts.length)).fill('0'), ...rightParts]; return parts.reduce((total, part) => (total << 16n) + BigInt(`0x${part || '0'}`), 0n); }
function positiveInt(value: string | undefined, fallback: number, max: number) { const parsed = Number(value ?? fallback); return Number.isInteger(parsed) && parsed > 0 ? Math.min(parsed, max) : fallback; }
function boundedInt(value: unknown, min: number, max: number, label: string) { const parsed = Number(value); if (!Number.isInteger(parsed) || parsed < min || parsed > max) throw new BadRequestException(`${label}必须在${min}${max}之间`); return parsed; }
function sanitizeEvidence(value?: Record<string, unknown>) { if (!value) return undefined; const sanitized = JSON.parse(JSON.stringify(value, (key, item) => /password|secret|token|signature|access.?key/i.test(key) ? '[REDACTED]' : item)); return sanitized as Prisma.InputJsonValue; }
function isUniqueViolation(error: unknown) { return error instanceof Prisma.PrismaClientKnownRequestError && error.code === 'P2002'; }
function assertAllowedKeys(input: Record<string, unknown>, allowed: string[]) { const unknown = Object.keys(input).filter((key) => !allowed.includes(key)); if (unknown.length) throw new BadRequestException(`不支持的字段: ${unknown.join(', ')}`); }
function isSystemProtected(ip: string) {
const builtIns = ['0.0.0.0/8', '10.0.0.0/8', '100.64.0.0/10', '127.0.0.0/8', '169.254.0.0/16', '172.16.0.0/12', '192.168.0.0/16', '224.0.0.0/4', '::/128', '::1/128', 'fc00::/7', 'fe80::/10', ...(process.env.SECURITY_BUILTIN_PROTECTED_NETWORKS ?? '').split(',').map((item) => item.trim()).filter(Boolean)];
return builtIns.some((network) => networkContains(network, ip));
}
@@ -0,0 +1,20 @@
import { UnauthorizedException } from '@nestjs/common';
import { SecurityEventController } from './security-event.controller';
describe('SecurityEventController', () => {
const security = { recordEvent: jest.fn().mockResolvedValue({ accepted: true }) };
const config = { get: jest.fn().mockReturnValue('internal-token-0123456789') };
const controller = new SecurityEventController(security as never, config as never);
beforeEach(() => jest.clearAllMocks());
it('rejects a public event injection without the internal token', () => {
expect(() => controller.record({ ruleCode: 'ssh_auth_failure', sourceIp: '203.0.113.9' }, undefined)).toThrow(UnauthorizedException);
expect(security.recordEvent).not.toHaveBeenCalled();
});
it('accepts a fixed event from an authenticated local producer', async () => {
await expect(controller.record({ ruleCode: 'ssh_auth_failure', sourceIp: '203.0.113.9' }, 'internal-token-0123456789')).resolves.toEqual({ accepted: true });
expect(security.recordEvent).toHaveBeenCalledWith(expect.objectContaining({ ruleCode: 'ssh_auth_failure' }));
});
});
@@ -0,0 +1,18 @@
import { Body, Controller, Headers, Post, UnauthorizedException } from '@nestjs/common';
import { ConfigService } from '@nestjs/config';
import { timingSafeEqual } from 'node:crypto';
import { ApiTags } from '@nestjs/swagger';
import { SecurityDetectionService, type SecurityEventInput } from './security-detection.service';
@ApiTags('gateway-security-events')
@Controller('gateway/events/security-detection')
export class SecurityEventController {
constructor(private readonly security: SecurityDetectionService, private readonly config: ConfigService) {}
@Post() record(@Body() body: SecurityEventInput, @Headers('x-security-event-token') supplied?: string) {
const expected = this.config.get<string>('SECURITY_EVENT_TOKEN');
if (!expected || !supplied || !safeEqual(expected, supplied)) throw new UnauthorizedException('安全事件来源认证失败');
return this.security.recordEvent(body);
}
}
function safeEqual(left: string, right: string) { const a = Buffer.from(left); const b = Buffer.from(right); return a.length === b.length && timingSafeEqual(a, b); }
@@ -11,7 +11,7 @@ describe('GatewayEventsController protocol logging', () => {
const protocolLogs = { const protocolLogs = {
record: jest.fn(), record: jest.fn(),
}; };
const controller = new GatewayEventsController(sendChain as never, {} as never, protocolLogs as never); const controller = new GatewayEventsController(sendChain as never, {} as never, protocolLogs as never, { recordEvent: jest.fn() } as never);
beforeEach(() => { beforeEach(() => {
jest.clearAllMocks(); jest.clearAllMocks();
@@ -18,6 +18,7 @@ import { SendChainService } from './send-chain.service';
import { GatewayDownstreamConnectionEventDto } from '../sms-config/sms-config.contracts'; import { GatewayDownstreamConnectionEventDto } from '../sms-config/sms-config.contracts';
import { SmsConfigService } from '../sms-config/sms-config.service'; import { SmsConfigService } from '../sms-config/sms-config.service';
import { ProtocolLogsService, type ProtocolLogInput } from '../protocol-logs/protocol-logs.service'; import { ProtocolLogsService, type ProtocolLogInput } from '../protocol-logs/protocol-logs.service';
import { SecurityDetectionService } from '../security-detection/security-detection.service';
@ApiTags('gateway-events') @ApiTags('gateway-events')
@Controller('gateway/events') @Controller('gateway/events')
@@ -26,6 +27,7 @@ export class GatewayEventsController {
private readonly sendChain: SendChainService, private readonly sendChain: SendChainService,
private readonly smsConfig: SmsConfigService, private readonly smsConfig: SmsConfigService,
private readonly protocolLogs: ProtocolLogsService, private readonly protocolLogs: ProtocolLogsService,
private readonly security: SecurityDetectionService,
) {} ) {}
@Post('submit-result') @Post('submit-result')
@@ -81,8 +83,13 @@ export class GatewayEventsController {
} }
@Post('inbound/authenticate') @Post('inbound/authenticate')
authenticateInbound(@Body() body: GatewayInboundAuthDto) { async authenticateInbound(@Body() body: GatewayInboundAuthDto) {
return this.trackGatewayEvent('connect', body, () => this.sendChain.authenticateInboundApplication(body), 'client_to_platform'); try {
return await this.trackGatewayEvent('connect', body, () => this.sendChain.authenticateInboundApplication(body), 'client_to_platform');
} catch (error) {
if (body.remoteIp) await this.security.recordEvent({ ruleCode: 'cmpp_auth_failure', sourceIp: body.remoteIp, account: body.account, protocol: body.version ?? 'cmpp', resultCode: error instanceof Error ? error.name : 'AUTH_FAILED' }).catch(() => undefined);
throw error;
}
} }
@Post('inbound/submit') @Post('inbound/submit')
+2 -1
View File
@@ -9,9 +9,10 @@ import { AdminSendChainController } from './admin-send-chain.controller';
import { ClientSendChainController } from './client-send-chain.controller'; import { ClientSendChainController } from './client-send-chain.controller';
import { GatewayEventsController } from './gateway-events.controller'; import { GatewayEventsController } from './gateway-events.controller';
import { SendChainService } from './send-chain.service'; import { SendChainService } from './send-chain.service';
import { SecurityDetectionModule } from '../security-detection/security-detection.module';
@Module({ @Module({
imports: [PrismaModule, BillingModule, DictionariesModule, forwardRef(() => RiskReviewModule), SmsConfigModule, forwardRef(() => OpenApiModule)], imports: [PrismaModule, BillingModule, DictionariesModule, forwardRef(() => RiskReviewModule), SmsConfigModule, forwardRef(() => OpenApiModule), SecurityDetectionModule],
controllers: [AdminSendChainController, ClientSendChainController, GatewayEventsController], controllers: [AdminSendChainController, ClientSendChainController, GatewayEventsController],
providers: [SendChainService], providers: [SendChainService],
exports: [SendChainService], exports: [SendChainService],
@@ -54,6 +54,41 @@ describe('SendDownstreamRequeueTaskService', () => {
expect(mock.downstreamRequeueTaskItem.createMany).toHaveBeenCalledWith({ data: [expect.objectContaining({ deliveryId: 'd-1', applicationId: 'app-1' })] }); expect(mock.downstreamRequeueTaskItem.createMany).toHaveBeenCalledWith({ data: [expect.objectContaining({ deliveryId: 'd-1', applicationId: 'app-1' })] });
}); });
it('materializes client-confirmed deliveries from the signed preview range', async () => {
mock.cmppDownstreamDelivery.count.mockResolvedValue(1);
mock.cmppDownstreamDelivery.groupBy.mockResolvedValueOnce([{ status: 'delivered', _count: { _all: 1 } }]).mockResolvedValueOnce([{ applicationId: 'app-1', _count: { _all: 1 } }]);
mock.cmppDownstreamDelivery.findFirst.mockResolvedValue({ createdAt: new Date() });
const preview = await service.preview({ applicationId: 'app-1', status: 'delivered' }, 'user-1');
expect(preview).toEqual(expect.objectContaining({ matchedCount: 1, replayableCount: 1, skippedCount: 0 }));
mock.downstreamRequeueTask.findFirst.mockResolvedValue(null);
mock.cmppDownstreamDelivery.findMany.mockResolvedValue([{ id: 'd-1', applicationId: 'app-1', status: 'delivered' }]);
mock.downstreamRequeueTask.create.mockResolvedValue({ id: 'task-1' });
mock.downstreamRequeueTask.findUnique.mockResolvedValue({ id: 'task-1' });
mock.downstreamRequeueTaskItem.groupBy.mockResolvedValue([]);
await service.create({ previewToken: preview.previewToken, reason: '再次投递客户已确认记录' }, 'user-1');
expect(mock.downstreamRequeueTaskItem.createMany).toHaveBeenCalledWith({ data: [expect.objectContaining({ deliveryId: 'd-1', previousStatus: 'delivered' })] });
});
it('replays a delivery that was already client-confirmed in the task snapshot', async () => {
const requeue = jest.fn().mockResolvedValue({ status: 'awaiting_ack' });
service = new SendDownstreamRequeueTaskService(mock as never, { requeueDownstreamDelivery: requeue });
mock.cmppDownstreamDelivery.findUnique.mockResolvedValue({ id: 'd-1', applicationId: 'app-1', status: 'delivered', ackResult: 0, payload: {}, deliveryType: 'receipt', application: { status: 'active', interfaceEnabled: true } });
mock.downstreamRequeueTaskItem.findFirst.mockResolvedValue(null);
mock.cmppDownstreamConnection.count.mockResolvedValue(1);
await expect(service['processItem']('item-1', 'd-1', 'delivered')).resolves.toBe('waiting');
expect(requeue).toHaveBeenCalledWith('d-1');
expect(mock.downstreamRequeueTaskItem.update).toHaveBeenCalledWith(expect.objectContaining({ data: expect.objectContaining({ status: 'waiting_ack' }) }));
});
it('does not replay a record that became delivered after a non-delivered task snapshot', async () => {
const requeue = jest.fn();
service = new SendDownstreamRequeueTaskService(mock as never, { requeueDownstreamDelivery: requeue });
mock.cmppDownstreamDelivery.findUnique.mockResolvedValue({ id: 'd-1', applicationId: 'app-1', status: 'delivered', ackResult: 0, payload: {}, deliveryType: 'receipt', application: { status: 'active', interfaceEnabled: true } });
await expect(service['processItem']('item-1', 'd-1', 'failed')).resolves.toBe('skipped');
expect(requeue).not.toHaveBeenCalled();
expect(mock.downstreamRequeueTaskItem.update).toHaveBeenCalledWith(expect.objectContaining({ data: expect.objectContaining({ status: 'skipped', skipReason: '创建任务后已被客户确认' }) }));
});
it('paginates all task items with status and keyword filters', async () => { it('paginates all task items with status and keyword filters', async () => {
mock.downstreamRequeueTask.findUnique.mockResolvedValue({ id: 'task-1' }); mock.downstreamRequeueTask.findUnique.mockResolvedValue({ id: 'task-1' });
mock.downstreamRequeueTaskItem.findMany.mockResolvedValue([{ id: 'item-1' }]); mock.downstreamRequeueTaskItem.findMany.mockResolvedValue([{ id: 'item-1' }]);
@@ -15,7 +15,7 @@ export type DownstreamRequeueFilter = {
}; };
type RequeueFacade = { requeueDownstreamDelivery(id: string): Promise<unknown> }; type RequeueFacade = { requeueDownstreamDelivery(id: string): Promise<unknown> };
const REPLAYABLE_STATUSES = ['pending', 'failed', 'unconfirmed', 'rejected']; const REPLAYABLE_STATUSES = ['pending', 'failed', 'unconfirmed', 'rejected', 'delivered'];
const ACTIVE_TASK_STATUSES = ['queued', 'running', 'paused']; const ACTIVE_TASK_STATUSES = ['queued', 'running', 'paused'];
const PROCESSING_LEASE_MS = 2 * 60_000; const PROCESSING_LEASE_MS = 2 * 60_000;
const SCAN_LEASE_MS = 15_000; const SCAN_LEASE_MS = 15_000;
@@ -119,7 +119,7 @@ export class SendDownstreamRequeueTaskService {
const preview = verifyPreview(data.previewToken, createdById); const preview = verifyPreview(data.previewToken, createdById);
const filter = normalizedFilter(preview.filter); const filter = normalizedFilter(preview.filter);
const snapshotAt = new Date(preview.snapshotAt); const snapshotAt = new Date(preview.snapshotAt);
if (filter.status === 'delivered' || filter.status === 'awaiting_ack') throw new BadRequestException('第一版后台任务不支持已确认或等待ACK记录'); if (filter.status === 'awaiting_ack') throw new BadRequestException('后台任务不支持正在等待ACK记录');
const activeTask = await this.prisma.downstreamRequeueTask.findFirst({ where: { const activeTask = await this.prisma.downstreamRequeueTask.findFirst({ where: {
status: { in: ACTIVE_TASK_STATUSES }, status: { in: ACTIVE_TASK_STATUSES },
...(filter.applicationId !== 'all' ? { OR: [{ applicationId: filter.applicationId }, { applicationId: null }] } : {}), ...(filter.applicationId !== 'all' ? { OR: [{ applicationId: filter.applicationId }, { applicationId: null }] } : {}),
@@ -229,14 +229,14 @@ export class SendDownstreamRequeueTaskService {
return; return;
} }
await this.prisma.downstreamRequeueTask.update({ where: { id: taskId }, data: { status: 'running', startedAt: task.startedAt ?? new Date() } }); await this.prisma.downstreamRequeueTask.update({ where: { id: taskId }, data: { status: 'running', startedAt: task.startedAt ?? new Date() } });
const items = await this.prisma.downstreamRequeueTaskItem.findMany({ where: { taskId, status: 'queued' }, orderBy: { createdAt: 'asc' }, take: Math.min(200, task.ratePerSecond * 3), select: { id: true, deliveryId: true, applicationId: true } }); const items = await this.prisma.downstreamRequeueTaskItem.findMany({ where: { taskId, status: 'queued' }, orderBy: { createdAt: 'asc' }, take: Math.min(200, task.ratePerSecond * 3), select: { id: true, deliveryId: true, applicationId: true, previousStatus: true } });
for (const item of items) { for (const item of items) {
const latestTask = await this.prisma.downstreamRequeueTask.findUnique({ where: { id: taskId }, select: { status: true } }); const latestTask = await this.prisma.downstreamRequeueTask.findUnique({ where: { id: taskId }, select: { status: true } });
if (!latestTask || !['queued', 'running'].includes(latestTask.status)) break; if (!latestTask || !['queued', 'running'].includes(latestTask.status)) break;
if (!(await this.consumeRate(item.applicationId, task.ratePerSecond))) continue; if (!(await this.consumeRate(item.applicationId, task.ratePerSecond))) continue;
const claimed = await this.prisma.downstreamRequeueTaskItem.updateMany({ where: { id: item.id, status: 'queued' }, data: { status: 'processing', claimedAt: new Date() } }); const claimed = await this.prisma.downstreamRequeueTaskItem.updateMany({ where: { id: item.id, status: 'queued' }, data: { status: 'processing', claimedAt: new Date() } });
if (!claimed.count) continue; if (!claimed.count) continue;
const outcome = await this.processItem(item.id, item.deliveryId); const outcome = await this.processItem(item.id, item.deliveryId, item.previousStatus);
if (outcome === 'success') failures[item.applicationId] = 0; if (outcome === 'success') failures[item.applicationId] = 0;
if (outcome === 'failed') failures[item.applicationId] = (failures[item.applicationId] ?? 0) + 1; if (outcome === 'failed') failures[item.applicationId] = (failures[item.applicationId] ?? 0) + 1;
const maxFailures = Math.max(0, ...Object.values(failures)); const maxFailures = Math.max(0, ...Object.values(failures));
@@ -257,16 +257,22 @@ export class SendDownstreamRequeueTaskService {
} }
} }
private async processItem(itemId: string, deliveryId: string): Promise<'success' | 'failed' | 'waiting' | 'skipped'> { private async processItem(itemId: string, deliveryId: string, previousStatus: string): Promise<'success' | 'failed' | 'waiting' | 'skipped'> {
try { try {
const delivery = await this.prisma.cmppDownstreamDelivery.findUnique({ const delivery = await this.prisma.cmppDownstreamDelivery.findUnique({
where: { id: deliveryId }, where: { id: deliveryId },
include: { application: { select: { status: true, interfaceEnabled: true } } }, include: { application: { select: { status: true, interfaceEnabled: true } } },
}); });
if (!delivery) return this.finishItem(itemId, 'skipped', '投递记录已不存在'); if (!delivery) return this.finishItem(itemId, 'skipped', '投递记录已不存在');
// Only a record that was already delivered in the frozen task snapshot may be replayed as
// delivered. This preserves the operator's explicit duplicate-delivery intent while preventing
// a pending/failed record that receives a late ACK after task creation from being sent again.
if (delivery.status === 'delivered' && previousStatus !== 'delivered') {
return this.finishItem(itemId, 'skipped', '创建任务后已被客户确认');
}
if (!REPLAYABLE_STATUSES.includes(delivery.status)) { if (!REPLAYABLE_STATUSES.includes(delivery.status)) {
if (delivery.status === 'awaiting_ack') { await this.prisma.downstreamRequeueTaskItem.update({ where: { id: itemId }, data: { status: 'waiting_external_ack', skipReason: null } }); return 'waiting'; } if (delivery.status === 'awaiting_ack') { await this.prisma.downstreamRequeueTaskItem.update({ where: { id: itemId }, data: { status: 'waiting_external_ack', skipReason: null } }); return 'waiting'; }
return this.finishItem(itemId, 'skipped', delivery.status === 'delivered' ? '已被客户确认' : '执行前状态已变化'); return this.finishItem(itemId, 'skipped', '执行前状态已变化');
} }
if (delivery.application.status !== 'active' || !delivery.application.interfaceEnabled) return this.finishItem(itemId, 'skipped', '应用或投递能力已停用'); if (delivery.application.status !== 'active' || !delivery.application.interfaceEnabled) return this.finishItem(itemId, 'skipped', '应用或投递能力已停用');
if (!delivery.payload || !['receipt', 'uplink'].includes(delivery.deliveryType)) return this.finishItem(itemId, 'skipped', '投递数据不完整'); if (!delivery.payload || !['receipt', 'uplink'].includes(delivery.deliveryType)) return this.finishItem(itemId, 'skipped', '投递数据不完整');
+3
View File
@@ -0,0 +1,3 @@
[Definition]
failregex = ^<HOST> .* "(?:GET|POST|HEAD) /(?:\.env|\.git|wp-admin|wp-login\.php|phpmyadmin|vendor/phpunit|actuator|cgi-bin)(?:[/? ][^\"]*)?" (?:400|403|404) .*$
ignoreregex =
+8
View File
@@ -0,0 +1,8 @@
[Definition]
# Detection remains report-only. The fixed action is installed by the deployment
# script and can only forward Fail2ban's matched IP/jail values to the local collector.
actionstart =
actionstop =
actioncheck =
actionban = @CMPP_SECURITY_AGENT_BIN@ report <name> <ip>
actionunban =
@@ -0,0 +1,25 @@
[Unit]
Description=CMPP restricted security execution agent
After=network.target fail2ban.service nftables.service
[Service]
Type=simple
User=root
Group=cmpp-security
EnvironmentFile=/etc/cmpp-platform/cmpp-platform.env
ExecStart=@CMPP_SECURITY_AGENT_BIN@
Restart=on-failure
RestartSec=3
NoNewPrivileges=true
PrivateTmp=true
ProtectHome=true
ProtectSystem=strict
ReadWritePaths=/run/cmpp-security-agent /var/lib/cmpp-security-agent /etc/nginx/snippets /etc/fail2ban/jail.d
RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6 AF_NETLINK
CapabilityBoundingSet=CAP_NET_ADMIN CAP_DAC_OVERRIDE CAP_KILL
AmbientCapabilities=CAP_NET_ADMIN CAP_DAC_OVERRIDE
LockPersonality=true
MemoryDenyWriteExecute=true
[Install]
WantedBy=multi-user.target
+14
View File
@@ -1138,3 +1138,17 @@ global,不能错误归入client。`client-signature-*`、发送页、企业认
``` ```
每次开始新拆分版本时,在本路线图基础上另写该版本的短实施计划,不直接把路线图当作可执行变更清单。 每次开始新拆分版本时,在本路线图基础上另写该版本的短实施计划,不直接把路线图当作可执行变更清单。
## 安全检测领域边界补充(2026-08-14)
- `api/src/security-detection/` 是安全事件、聚合告警、规则版本和人工封禁编排的唯一业务边界;登录、OpenAPI 和 Gateway 只上报固定类型的结构化事件,不复制聚合或封禁逻辑。
- `gateway/cmd/security-agent/` 是最小特权执行边界,不依赖 NestJS Service,不接受任意命令、路径、jail、action 或 shell 参数。该二进制与 `deploy/security/``tools/security/install-security-agent.sh` 作为同一发布单元评审。
- 安全代理可执行文件位置只由安装器的`agent_binary=$APP_DIR/dist/cmpp-security-agent`定义;systemd和Fail2ban模板使用同一占位符渲染,不能各自维护易漂移的绝对路径。
- 前端 `src/apps/admin/security-detection/` 通过 `src/api/admin/security-detection.api.ts` 访问稳定门面,不直接访问 Fail2ban、Nginx、nftables 或安全代理。
- NestJS进程边界由`api/src/main.ts`和生产环境`API_HOST`共同固定为回环监听,外部HTTP入口统一归Nginx模块治理;后续拆分不得让业务模块自行新增外部监听或绕过反向代理边界。
## 基础设施指标领域边界补充(2026-08-14)
- `api/src/metrics/`只负责API进程和HTTP路由模板的低基数计数/直方图,不依赖Prisma、Redis或业务Service;独立回环监听由`api/src/main.ts`组装。
- `gateway/internal/metrics/`只负责线程安全聚合和Prometheus文本暴露;上游连接池、下游Session和Submit Worker仅提供总数或有界结果,不把实体ID或凭据交给监控模块。
- `api/src/infrastructure-monitoring/`是运营端监控聚合与固定阈值应用边界:只消费代码白名单 PromQL,并通过版本化 PostgreSQL 单例、promtool 校验和原子规则热加载管理数值阈值;Exporter安装、端口隔离、固定规则模板和权限仍归`tools/monitoring/`治理。
- 活动告警已读也归该边界:Prometheus保留告警事实,Prisma仅持久化逐管理员、逐触发周期的阅读状态;全局布局只消费轻量未读汇总,不复制指纹、activeAt或用户隔离逻辑。
+12 -12
View File
@@ -1,8 +1,8 @@
# 下游投递后台批量重投任务设计与实现符合性审计 # 下游投递后台批量重投任务设计与实现符合性审计
> 版本:V1.0<br> > 版本:V1.1<br>
> 需求确认日期:2026-08-12<br> > 需求确认日期:2026-08-12<br>
> 文档整理日期:2026-08-13<br> > 文档整理日期:2026-08-13;业务口径更新:2026-08-14<br>
> 适用页面:运营端 → 下游投递记录<br> > 适用页面:运营端 → 下游投递记录<br>
> 审计基线:当前工作区 `HEAD=67fee216162e638ba21004fcf87e7711facefb91`;本功能相关文件相对 HEAD 无未提交修改<br> > 审计基线:当前工作区 `HEAD=67fee216162e638ba21004fcf87e7711facefb91`;本功能相关文件相对 HEAD 无未提交修改<br>
> 本文目的:还原 2026-08-12 已确认的设计口径,并将当前实现逐条映射到设计,不能以“已有代码”代替“符合设计”的结论。 > 本文目的:还原 2026-08-12 已确认的设计口径,并将当前实现逐条映射到设计,不能以“已有代码”代替“符合设计”的结论。
@@ -27,9 +27,9 @@
1. 保留单条重投、当前页勾选批量重投,新增“按筛选条件重投”;投递记录分页支持每页 10/25/50 条。 1. 保留单条重投、当前页勾选批量重投,新增“按筛选条件重投”;投递记录分页支持每页 10/25/50 条。
2. 后台任务使用企业、应用、投递类型、状态、创建日期、关键词组成的筛选快照;页码和每页条数不属于任务范围。预检生成 `snapshotAt`,创建任务后产生的新记录不进入该任务。 2. 后台任务使用企业、应用、投递类型、状态、创建日期、关键词组成的筛选快照;页码和每页条数不属于任务范围。预检生成 `snapshotAt`,创建任务后产生的新记录不进入该任务。
3. 第一版后台任务允许 `pending``failed``unconfirmed``rejected`。不得批量重投客户端已确认的 `delivered`;处于 `awaiting_ack` 的记录不得并发重投。创建前展示真实命中数、可重投数、规则跳过数、状态分布,任务原因必填。 3. 后台任务允许 `pending``failed``unconfirmed``rejected``delivered`客户端已确认的 `delivered` 可按筛选快照再次投递,但必须醒目提示可能造成客户端重复处理;处于 `awaiting_ack` 的记录不得并发重投。创建前展示真实命中数、可重投数、规则跳过数、状态分布,任务原因必填。
4. 任务按企业应用分批执行,默认每个应用 10 条/秒。单条失败不阻断整批;连续失败达到 10 条,或 ACK 超时/拒绝达到安全阈值时,自动暂停。客户离线、已有链路等待 ACK 属于“等待”,不能记作“跳过”。 4. 任务按企业应用分批执行,默认每个应用 10 条/秒。单条失败不阻断整批;连续失败达到 10 条,或 ACK 超时/拒绝达到安全阈值时,自动暂停。客户离线、已有链路等待 ACK 属于“等待”,不能记作“跳过”。
5. “跳过”严格表示本任务没有调用 Gateway。第一版跳过原因包括:执行前状态变化、被客户确认、已被其他任务处理、本任务已成功处理、不属于任务快照、应用或投递能力已停用、投递数据不完整、缺少原 Submit 映射。 5. “跳过”严格表示本任务没有调用 Gateway。跳过原因包括:执行前状态变化、创建任务后才被客户确认、已被其他任务处理、本任务已成功处理、不属于任务快照、应用或投递能力已停用、投递数据不完整、缺少原 Submit 映射。只有任务项冻结的原状态已是 `delivered` 时,才允许按已确认记录重投;其他状态在执行前收到迟到成功 ACK 时必须跳过。
6. 任务支持列表、详情、暂停、继续、终止。终止只影响尚未发送的任务项。任务项以 `taskId + deliveryId` 幂等;执行前原子认领并复核当前状态;API 重启后继续执行;已获得成功 ACK 的任务项不得再次发送。 6. 任务支持列表、详情、暂停、继续、终止。终止只影响尚未发送的任务项。任务项以 `taskId + deliveryId` 幂等;执行前原子认领并复核当前状态;API 重启后继续执行;已获得成功 ACK 的任务项不得再次发送。
7. 创建、暂停、继续、终止、自动暂停都必须写操作日志。任务必须使用真实 PostgreSQL、Gateway 和客户 ACK,不得使用 mock、静态数据或 localStorage。 7. 创建、暂停、继续、终止、自动暂停都必须写操作日志。任务必须使用真实 PostgreSQL、Gateway 和客户 ACK,不得使用 mock、静态数据或 localStorage。
@@ -54,7 +54,7 @@
1. 用户设置筛选条件,点击“按筛选条件重投”。 1. 用户设置筛选条件,点击“按筛选条件重投”。
2. 后端在同一时点生成预检快照,返回:筛选条件、`snapshotAt`、筛选命中数、可重投数、规则跳过数、状态分布、涉及应用数、最早记录时间。 2. 后端在同一时点生成预检快照,返回:筛选条件、`snapshotAt`、筛选命中数、可重投数、规则跳过数、状态分布、涉及应用数、最早记录时间。
3. 弹窗明确告知第一版允许和禁止的状态 3. 弹窗明确告知允许 `pending/failed/unconfirmed/rejected/delivered`、禁止 `awaiting_ack`,并提示已确认记录再次投递可能造成客户端重复处理
4. 用户选择执行速度,填写不少于 5 个字的事故原因、工单号或处理说明。 4. 用户选择执行速度,填写不少于 5 个字的事故原因、工单号或处理说明。
5. 用户确认后,后端必须重新按预检的筛选快照和 `snapshotAt` 物化任务项,而不是使用前端传入的记录 ID 列表。 5. 用户确认后,后端必须重新按预检的筛选快照和 `snapshotAt` 物化任务项,而不是使用前端传入的记录 ID 列表。
6. 创建成功后关闭弹窗,任务出现在任务列表,状态为“排队中”。 6. 创建成功后关闭弹窗,任务出现在任务列表,状态为“排队中”。
@@ -160,8 +160,8 @@
| 原因 | 判定 | | 原因 | 判定 |
| --- | --- | | --- | --- |
| 执行前状态已变化 | 已不属于允许重投状态,且不是等待 ACK/已确认 | | 执行前状态已变化 | 已不属于允许重投状态,且不是等待 ACK |
| 被客户确认 | 当前投递已`delivered` 且有效 ACK | | 创建任务后才被客户确认 | 任务项冻结原状态不`delivered`,执行前收到有效成功 ACK;避免把迟到确认变成未授权重复投递 |
| 已被其他任务处理 | 其他任务已认领、等待 ACK 或成功 | | 已被其他任务处理 | 其他任务已认领、等待 ACK 或成功 |
| 本任务已成功处理 | 同任务项已有成功结果,重复扫描不得再调用 | | 本任务已成功处理 | 同任务项已有成功结果,重复扫描不得再调用 |
| 不属于任务快照 | 创建时间晚于 `snapshotAt` 或不再满足冻结范围 | | 不属于任务快照 | 创建时间晚于 `snapshotAt` 或不再满足冻结范围 |
@@ -222,7 +222,7 @@
| --- | --- | --- | | --- | --- | --- |
| DRQ-001 | 筛选快照 | 企业、应用、类型、状态、日期、关键词均生效;分页无关;快照后新记录不进入 | | DRQ-001 | 筛选快照 | 企业、应用、类型、状态、日期、关键词均生效;分页无关;快照后新记录不进入 |
| DRQ-002 | 预检口径 | 命中、可重投、跳过和状态分布严格基于当前筛选条件 | | DRQ-002 | 预检口径 | 命中、可重投、跳过和状态分布严格基于当前筛选条件 |
| DRQ-003 | 状态白名单 | 物化 `pending/failed/unconfirmed/rejected`;拒绝 `delivered/awaiting_ack` | | DRQ-003 | 状态白名单 | 物化 `pending/failed/unconfirmed/rejected/delivered`;拒绝 `awaiting_ack`;仅冻结原状态为 `delivered` 的任务项可按已确认记录重投 |
| DRQ-004 | 每应用限速 | 多应用任务中每个应用独立达到配置速度,任意扫描重叠都不超速 | | DRQ-004 | 每应用限速 | 多应用任务中每个应用独立达到配置速度,任意扫描重叠都不超速 |
| DRQ-005 | 客户离线 | 进入等待连接,不记失败或跳过,恢复连接后继续 | | DRQ-005 | 客户离线 | 进入等待连接,不记失败或跳过,恢复连接后继续 |
| DRQ-006 | ACK 闭环 | 写出只进入等待;有效 ACK 成功;超时、拒绝、无效 Msg_Id 失败 | | DRQ-006 | ACK 闭环 | 写出只进入等待;有效 ACK 成功;超时、拒绝、无效 Msg_Id 失败 |
@@ -250,13 +250,13 @@
| --- | --- | --- | | --- | --- | --- |
| 真实持久化 | 符合 | 已有任务表、任务项表和 migration,不使用前端本地状态代替任务 | | 真实持久化 | 符合 | 已有任务表、任务项表和 migration,不使用前端本地状态代替任务 |
| 快照时间上限 | 基本符合 | 创建时按 `snapshotAt` 限制 `createdAt`,快照后记录不物化 | | 快照时间上限 | 基本符合 | 创建时按 `snapshotAt` 限制 `createdAt`,快照后记录不物化 |
| 后台状态白名单 | 符合 | `REPLAYABLE_STATUSES``pending/failed/unconfirmed/rejected` | | 后台状态白名单 | 符合 | `REPLAYABLE_STATUSES``pending/failed/unconfirmed/rejected/delivered` |
| 禁止后台任务处理已确认/等待 ACK 筛选 | 符合 | 创建接口明确拒绝 `delivered/awaiting_ack` | | 禁止后台任务处理等待 ACK 筛选 | 符合 | 创建接口明确拒绝 `awaiting_ack``delivered` 按已确认重复投递风险口径放行 |
| 原因校验 | 符合 | 少于 5 个字拒绝 | | 原因校验 | 符合 | 少于 5 个字拒绝 |
| 任务项幂等 | 符合 | 数据库唯一约束 `taskId + deliveryId` | | 任务项幂等 | 符合 | 数据库唯一约束 `taskId + deliveryId` |
| 执行前任务项认领 | 基本符合 | 通过 `status=queued` 的条件更新认领为 `processing` | | 执行前任务项认领 | 基本符合 | 通过 `status=queued` 的条件更新认领为 `processing` |
| 执行前复核 | 基本符合 | 重新查询投递、应用、payload 和其他任务状态 | | 执行前复核 | 基本符合 | 重新查询投递、应用、payload 和其他任务状态 |
| 成功 ACK 不再发送 | 基本符合 | 已确认投递会跳过其他任务 `success` 也会阻止调用 | | 成功 ACK 不再发送 | 基本符合 | `delivered` 快照项执行前才收到成功 ACK 时跳过其他任务 `success` 也会阻止调用;冻结原状态为 `delivered` 的项目属于运营明确授权的再次投递 |
| 人工控制 | 基本符合 | 已有暂停、继续、终止接口和页面按钮 | | 人工控制 | 基本符合 | 已有暂停、继续、终止接口和页面按钮 |
| 核心操作日志 | 基本符合 | 创建、暂停、继续、终止、自动暂停均写日志 | | 核心操作日志 | 基本符合 | 创建、暂停、继续、终止、自动暂停均写日志 |
| 投递记录分页 | 符合 | 页面支持每页 10/25/50 条并回到第一页 | | 投递记录分页 | 符合 | 页面支持每页 10/25/50 条并回到第一页 |
@@ -279,7 +279,7 @@
| P1 | 自动化覆盖远低于设计风险 | 专项仅4个测试:预检计数、参数拒绝、活动任务冲突、已确认跳过 | 未覆盖限速、多应用、离线等待、ACK阈值、重启恢复、并发扫描、完整分页及所有审计动作 | | P1 | 自动化覆盖远低于设计风险 | 专项仅4个测试:预检计数、参数拒绝、活动任务冲突、已确认跳过 | 未覆盖限速、多应用、离线等待、ACK阈值、重启恢复、并发扫描、完整分页及所有审计动作 |
| P2 | 状态和详情表达偏内部化 | 任务列表/详情直接展示英文状态;任务详情只展示汇总和最近项 | 运营人员不易区分等待连接、等待外部ACK、任务写出等待ACK等状态 | | P2 | 状态和详情表达偏内部化 | 任务列表/详情直接展示英文状态;任务详情只展示汇总和最近项 | 运营人员不易区分等待连接、等待外部ACK、任务写出等待ACK等状态 |
| P2 | 终止后的未处理数未进入常规汇总 | 终止把`queued`改为`unprocessed`,但任务汇总只保存成功/失败/跳过/等待 | 任务进度分子可能小于总数,页面没有单独解释未处理数量 | | P2 | 终止后的未处理数未进入常规汇总 | 终止把`queued`改为`unprocessed`,但任务汇总只保存成功/失败/跳过/等待 | 任务进度分子可能小于总数,页面没有单独解释未处理数量 |
| P2 | 已确认跳过原因存在口径混用 | `waiting_external_ack`最终由其他链路成功后记为“跳过:已由其他投递链路完成” | 可以接受为“本任务未调用Gateway”,但详情必须明确这是外部链路成功,不应让用户误以为业务未处理 | | P2 | 外部 ACK 跳过原因存在口径混用 | `waiting_external_ack`最终由其他链路成功后记为“跳过:已由其他投递链路完成” | 可以接受为“本任务未调用Gateway”,但详情必须明确这是外部链路成功,不应让用户误以为业务未处理;这与冻结原状态为 `delivered` 的主动再次投递是两种情形 |
### 11.3 综合结论 ### 11.3 综合结论
@@ -0,0 +1,379 @@
# Fail2ban 安全检测与人工封禁平台设计方案
> 版本:V1.0(需求评审稿)<br>
> 日期:2026-08-14<br>
> 范围:运营端自研 UI、安全检测、阈值配置、告警处置、人工封禁与解封<br>
> 边界:第一版不自动封禁、不开放任意 Fail2ban/防火墙命令、不在线编辑正则表达式
## 1. 建设目标
在运营端“安全控制”下建设自研安全检测面板,统一接收 SSH、运营端登录、客户端登录、CMPP 入站和 HTTP API 的异常行为,按照可配置规则聚合为安全告警。平台管理员查看证据后,可人工执行固定时长封禁、解封、忽略或加入保护名单。
第一版采用“检测与执行分离”原则:
1. Fail2ban 和应用侧检测器只生成事件及告警,不自动修改防火墙。
2. 人工点击“封禁”后,后端重新校验告警、真实 IP、保护名单、执行目标和操作权限。
3. 只有受限安全代理可以执行系统级封禁;NestJS 不直接获得 root、通用 sudo 或任意 shell 能力。
4. PostgreSQL 保存配置、事件、告警、封禁事实和操作审计;页面不得使用 mock、静态数据或 localStorage 伪造状态。
## 2. 第一版范围
### 2.1 检测类型
| 规则编码 | 检测对象 | 事件来源 | 第一版默认建议值 | 默认风险 |
| --- | --- | --- | --- | --- |
| `admin_login_failure` | 运营端账号登录失败 | NestJS 结构化安全事件 | 同一 IP 10 分钟 8 次 | 中 |
| `client_login_failure` | 客户端账号登录失败 | NestJS 结构化安全事件 | 同一 IP 10 分钟 8 次 | 中 |
| `ssh_auth_failure` | SSH 12022 认证失败 | journald + Fail2ban filter | 同一 IP 10 分钟 6 次 | 高 |
| `cmpp_auth_failure` | CMPP 17890 未知账号、认证失败或不允许 IP | Gateway 结构化安全事件 | 同一 IP 5 分钟 5 次 | 高 |
| `cmpp_protocol_abuse` | 非法协议包、异常版本或高频无效连接 | Gateway 结构化安全事件 | 同一 IP 1 分钟 20 次 | 高 |
| `http_invalid_api_key` | HTTP API 错误或未知访问密钥 | NestJS HTTP API 鉴权事件 | 同一 IP 5 分钟 10 次 | 高 |
| `http_signature_failure` | HTTP API 签名缺失、格式错误或验签失败 | NestJS HTTP API 验签事件 | 同一 IP 5 分钟 10 次 | 高 |
| `http_replay_attempt` | nonce、时间戳或幂等凭证重放 | NestJS HTTP API 验签事件 | 同一 IP 10 分钟 3 次 | 严重 |
| `http_malicious_scan` | 扫描敏感路径、跨路径高频 404、明显漏洞探测 | Nginx 结构化日志 + Fail2ban filter | 同一 IP 1 分钟 20 次且涉及至少 8 个不同路径 | 高 |
表中数值只是首次安装默认值,必须保存到真实数据库并可在运营端配置。业务 400、正常参数校验失败、合法客户偶发时钟偏差、普通 404、供应商连接失败不得直接归类为攻击。
### 2.2 处置能力
- 查看检测总览、趋势、来源、风险等级、Top IP 和待处置告警。
- 查看告警详情、脱敏日志证据、规则快照和历史处置。
- 人工封禁固定时长:10 分钟、1 小时、24 小时、7 天。
- 人工解封、忽略告警、加入保护名单。
- 查看当前真实封禁、执行目标、到期时间和同步状态。
- 配置检测规则阈值、时间窗口、冷却时间、风险等级和启停状态。
- 查看 Fail2ban、事件采集器、安全代理、规则版本及封禁执行器健康状态。
### 2.3 第一版不做
- 自动封禁。
- 永久封禁或前端输入任意封禁秒数。
- 在线编辑 Fail2ban regex、日志路径、action、iptables/nftables 或 Nginx 配置文本。
- 从页面执行任意 shell、`fail2ban-client``systemctl` 或防火墙命令。
- 自动将外部威胁情报加入封禁。
- 删除原始告警和处置历史。
## 3. 总体架构
```text
sshd journal ── Fail2ban 检测 jail ─┐
Nginx access/error ─ Fail2ban jail ─┤
NestJS 登录/HTTP 鉴权安全事件 ──────┤
Go Gateway CMPP 安全事件 ───────────┤
Security Event Collector
PostgreSQL 事件、规则、告警、封禁
NestJS 运营端安全 API
自研运营端 UI
↓ 人工确认
Root Security Agent (Unix Socket)
↓ ↓
nftables/manual jail Nginx real-IP deny
```
### 3.1 组件职责
#### Fail2ban
- 读取 sshd 和 Nginx 等系统日志。
- 使用固定、随版本发布的 filter 识别失败行为。
- 根据已生效规则计算窗口与阈值。
- 触发“告警事件 action”,但不直接封禁。
- 不作为平台告警历史和真实封禁状态的唯一数据源。
#### 应用侧安全事件
NestJS 和 Gateway 对其掌握真实业务语义的事件直接产生结构化安全事件,避免只靠文本正则猜测:
- 登录入口、失败类型和真实请求 IP。
- HTTP API 密钥查找失败、签名失败、重放拒绝。
- CMPP 账号、AuthenticatorSource 校验、IP 白名单和协议异常。
任何安全事件不得记录明文密码、完整 API 密钥、签名密钥、AuthenticatorSource 或完整短信内容;账号、密钥标识只保存脱敏值或不可逆指纹。
#### Security Event Collector
- 通过受限 Unix Socket 或 root 写入、collector 只读的事件目录接收 Fail2ban 事件。
- 校验事件版本、规则编码、IP、时间和来源。
- 写入 PostgreSQL,并依据数据库规则做幂等聚合。
- 维护最后事件时间、丢弃数量和解析失败指标。
#### Root Security Agent
- 独立于 NestJS,以最小 root 权限运行。
- 仅监听本机 Unix Socket,不监听 TCP 公网端口。
- 只接受固定 JSON 协议:`block``unblock``status``apply_rule_version`
- 对规则编码、执行器、IPv4/IPv6、时长和幂等键做白名单校验。
- 使用无 shell 参数数组或原生库执行,不拼接命令。
- 原子生成平台专属配置文件,校验后才 reload;失败保留旧版本。
#### NestJS
- 读取真实 PostgreSQL 告警和配置。
- 执行 RBAC、近期重新认证、保护名单和状态校验。
- 创建封禁操作记录并调用本地安全代理。
- 根据代理回读结果确认真实封禁状态。
- 不以 root 运行,不直接写 `/etc/fail2ban/*` 或防火墙。
## 4. Cloudflare 与执行器选择
恢复 `CF-Connecting-IP` 只能让 Nginx/应用识别访客真实 IP,并不会改变到达服务器的 TCP 源地址。对 `sms.lisglo.com` 的 Cloudflare 橙云流量,nftables 封禁访客真实 IP 无效,误封 Cloudflare 节点反而可能中断全站。
第一版按入口固定执行器:
| 入口 | 网络形态 | 检测 IP | 人工封禁执行器 |
| --- | --- | --- | --- |
| 运营端、客户端 | Cloudflare 橙云 | 经过可信 Cloudflare 网段恢复的真实 IP | Nginx real-IP deny;未来可扩展 Cloudflare API |
| `api.lisglo.com` | 灰云直连 | TCP 源 IP | nftables/manual jail |
| SSH 12022 | 公网直连 | TCP 源 IP | nftables/manual jail |
| CMPP 17890 | 公网直连 | Gateway TCP 远端 IP | nftables/manual jail |
只有当请求 TCP 来源属于定期同步的 Cloudflare 官方网段时,才信任 `CF-Connecting-IP`。客户端直连源站时提供的同名 Header 必须忽略。配置上线前必须用真实请求证明日志、事件和页面展示 IP 一致。
## 5. 可配置规则
### 5.1 可配置字段
- 启用状态 `enabled`
- 统计窗口 `windowSeconds`
- 触发阈值 `threshold`
- 不同目标数量阈值 `distinctTargetThreshold`,仅恶意扫描等规则使用。
- 告警冷却时间 `cooldownSeconds`
- 风险等级 `severity`
- 聚合维度,只能从规则预置集合选择,例如 `ip``ip+accountFingerprint`
- 默认封禁时长和允许的最大封禁时长。
- 是否允许人工封禁;只读检测规则可以关闭封禁按钮。
### 5.2 不可由页面配置的字段
- filter 正则表达式。
- 日志文件路径和 journal unit。
- shell 命令、Fail2ban action、nftables 表/链。
- Unix Socket 路径、systemd 服务名。
- Cloudflare 可信网段来源。
- 规则到执行器的映射。
这些内容属于发布资产,必须通过代码评审、自动化测试和标准部署更新。
### 5.3 配置保护
- 每类规则设置服务端最小值、最大值和允许枚举;前端约束不能替代后端校验。
- 阈值修改使用乐观锁版本号,防止多人覆盖。
- 保存后生成新规则版本,安全代理先语法校验,再原子切换并 reload。
- reload 失败时数据库状态标记 `apply_failed`,保留旧生效版本,页面明确展示“已保存但未生效”。
- 每次变更保存操作人、原因、旧值、新值、生效版本、代理回执和时间。
- 配置修改要求 `security.rule.manage` 权限和近期重新认证。
## 6. 数据模型
### 6.1 `SecurityDetectionRule`
- `id``code`(唯一)、`name``category``description`
- `enabled``windowSeconds``threshold``distinctTargetThreshold`
- `cooldownSeconds``severity``groupingMode`
- `manualBlockAllowed``defaultBlockDurationSeconds``maxBlockDurationSeconds`
- `configVersion``effectiveVersion``applyStatus``lastApplyError`
- `updatedById``createdAt``updatedAt`
### 6.2 `SecurityDetectionEvent`
- `id``eventKey`(唯一幂等键)、`ruleCode``sourceType`
- `sourceIp``accountFingerprint``targetFingerprint``requestPathNormalized`
- `occurredAt``receivedAt``evidenceSummary``metadata`
- `collectorInstanceId``ruleVersion`
`metadata` 使用后端安全 DTO,仅保存允许字段;禁止保存密钥和完整认证材料。
### 6.3 `SecurityAlert`
- `id``alertNo``fingerprint``ruleId``ruleSnapshot`
- `sourceIp``status``severity`
- `firstOccurredAt``lastOccurredAt``eventCount``distinctTargetCount`
- `cooldownUntil``assignedToId``handledById``handledAt``handleReason`
- `blockId``createdAt``updatedAt`
同一规则、IP、聚合维度和窗口桶使用唯一 fingerprint,重复采集只增加计数,不重复创建告警。
### 6.4 `SecurityBlock`
- `id``operationKey`(唯一)、`alertId``sourceIp`
- `executorType``executorTarget``durationSeconds`
- `status``requested/applying/blocked/unblock_requested/unblocked/expired/failed`
- `startedAt``expiresAt``verifiedAt``errorMessage`
- `requestedById``requestReason``unblockedById``unblockReason`
- `agentOperationId``createdAt``updatedAt`
### 6.5 `SecurityProtectedNetwork`
- IP/CIDR、名称、类型、适用入口、启停状态、来源和备注。
- 系统内置保护项不可从页面删除,只允许通过受控发布更新。
- 人工保护项新增、修改和停用均要求重新认证和审计。
## 7. 状态机与并发控制
### 7.1 告警状态
```text
pending ──→ block_requested ──→ blocked ──→ unblocked
│ └──────────→ block_failed
├──→ ignored
├──→ whitelisted
└──→ expired
```
- `pending` 仅表示达到检测阈值,绝不代表已被防火墙封禁。
- 封禁按钮通过数据库条件更新原子认领,只有一名操作人能进入 `block_requested`
- 代理执行成功后必须回读执行器状态,确认存在真实规则才写 `blocked`
- 网络超时导致结果不确定时先查询 `operationKey`,不得盲目重复封禁。
- 忽略、保护名单和封禁互斥;状态变化后旧页面操作返回 409。
### 7.2 封禁到期
- 执行器负责真实到期解除;平台定时回读并同步 `expired`
- 平台任务只做状态对账,不能仅靠数据库时间把记录标记为已解封。
- 对账发现执行器缺失、额外规则或到期未解除时生成系统告警。
## 8. 后端接口
```text
GET /api/admin/security-detection/overview
GET /api/admin/security-detection/trends
GET /api/admin/security-detection/alerts
GET /api/admin/security-detection/alerts/:id
POST /api/admin/security-detection/alerts/:id/block
POST /api/admin/security-detection/alerts/:id/ignore
POST /api/admin/security-detection/alerts/:id/protect
GET /api/admin/security-detection/blocks
POST /api/admin/security-detection/blocks/:id/unblock
GET /api/admin/security-detection/rules
PUT /api/admin/security-detection/rules/:id
GET /api/admin/security-detection/protected-networks
GET /api/admin/security-detection/health
```
封禁请求只接受固定时长枚举和原因。IP、规则、入口和执行器全部从告警及服务端映射读取,禁止前端重传或覆盖。
## 9. 权限与审计
| 权限 | 能力 |
| --- | --- |
| `security.alert.read` | 查看面板、告警和脱敏证据 |
| `security.alert.handle` | 忽略告警、分派和填写处置说明 |
| `security.block.manage` | 人工封禁与解封 |
| `security.rule.manage` | 修改阈值、启停规则和保护名单 |
封禁、解封、修改规则和保护名单必须要求近期重新认证;所有动作写入 `OperationLog`,记录资源、操作人、IP、原因、旧值、新值、代理回执和最终结果。读取完整证据也应记录访问审计。
## 10. 自研 UI 信息架构
菜单位置:`安全控制 / 安全检测`
### 10.1 总览
- 待处置、高风险、当前真实封禁、24 小时攻击 IP、封禁失败五个指标。
- 24 小时/7 天检测事件与告警趋势。
- 按规则类型、入口和风险等级分布。
- Top 攻击 IP、Top 扫描路径、Top 被尝试账号指纹。
- Fail2ban、collector、agent、规则版本和执行器健康卡片。
### 10.2 告警中心
- 按关键词、IP、规则、风险、状态、入口和日期筛选,真实后端分页。
- 列表展示风险、IP、类型、触发数、不同目标数、首次/最近时间、状态和操作。
- 详情抽屉展示规则快照、聚合时间线、脱敏证据、关联告警和处置历史。
- 封禁确认弹窗展示执行器、影响入口、时长、保护名单结果、近期合法访问提示和必填原因。
### 10.3 规则配置
- 使用平台现有自研 Card、Table、Tag、Modal、Form、Pagination 和图表体系,不嵌入 Fail2ban 第三方面板。
- 每项配置展示当前生效值、待生效值、最后修改人和应用状态。
- 数字输入同时展示单位、允许范围和默认建议值。
- 保存前展示变更对比;应用失败不能显示成功 toast。
### 10.4 封禁与保护名单
- 独立展示真实封禁状态、执行器、到期时间、来源告警和操作人。
- 保护名单命中时封禁按钮禁用并说明原因。
- 不同执行器使用明确标签,避免把 Nginx deny 误称为防火墙封禁。
## 11. 检测准确性要求
### 11.1 HTTP API
- 错误密钥:只记录不可逆密钥指纹,不记录完整 Header 或密钥。
- 签名错误:区分缺失、格式错误、算法不支持、验签失败和时间偏差。
- 重放:只有 nonce/幂等凭证已被真实使用或时间戳明显重复时计入;正常幂等重试按既有接口语义处理。
- 恶意扫描:使用标准化路径,不保存 query 中的敏感值;规则需要“次数 + 不同路径数”双阈值,避免单个合法 404 被判攻击。
- 反向代理真实 IP 必须经过可信代理链验证,禁止直接信任客户端 Header。
### 11.2 CMPP
- 未知账号、错误 AuthenticatorSource、不允许 IP、停用企业/应用和协议异常分别分类。
- 不记录明文密码、完整 AuthenticatorSource 或平台配置密钥。
- 正常断线、心跳超时、最大连接数限制和服务重启恢复不计为恶意认证。
### 11.3 登录
- 账号锁定仍由现有账户安全逻辑负责,安全检测面板不替代账号锁定。
- 图形验证码错误、账号错误、密码错误和角色入口错误分别保存分类,但页面证据统一脱敏。
- 一个入口的登录失败不得清理另一个入口的有效会话。
## 12. 保留与隐私
- 原始检测事件默认在线保留 30 天,聚合告警、封禁记录和操作审计默认保留 180 天;最终期限在上线前由安全和运营确认。
- 清理使用小批量、可恢复任务;不得删除仍关联活动封禁、未完成处置或审计保留期内的数据。
- 页面和导出默认脱敏账号、路径参数、User-Agent 中的可识别信息。
- 第一版不提供原始日志全文导出。
## 13. 可用性与降级
- Fail2ban 不可用:页面显示检测源异常,已有告警仍可查看;相关来源不允许宣称“无攻击”。
- Collector 不可用:健康状态告警并记录事件积压;恢复后按事件键幂等补录。
- Security Agent 不可用:封禁按钮返回明确失败,不修改告警为已封禁。
- PostgreSQL 不可用:不允许执行无法审计的封禁操作。
- 规则应用失败:继续使用上一生效版本,页面显示失败版本与原因。
- Nginx 或 nftables 回读不一致:封禁状态标记异常并产生系统告警。
## 14. 部署前置条件
1. 将 `cmpp-api.service` 改为专用非 root 用户并完成文件、日志、MinIO/local storage 权限回归。
2. 安装 Fail2ban,固定版本并使用 nftables 兼容 action。
3. 为 Nginx、sshd、Gateway 和 NestJS 建立结构化、脱敏且可测试的事件格式。
4. 验证 Cloudflare 可信 IP 网段、`real_ip_header` 和源站绕过防护。
5. 建立 root security agent、Unix Socket 权限、systemd 加固和固定协议。
6. 发布前备份 PostgreSQL、运行源码、环境文件、Fail2ban/Nginx 平台生成配置和 nftables 当前规则。
7. 安装器必须将systemd `ExecStart`与Fail2ban report-only `actionban`从同一占位符渲染为实际构建产物`$APP_DIR/dist/cmpp-security-agent`,并在旧路径或未替换占位符残留时失败关闭。
## 15. 分阶段实施建议
### 阶段 A:检测与只读面板
- 数据模型、默认规则、结构化事件、collector、Fail2ban alert-only jail。
- 总览、告警列表、详情、规则只读展示和健康状态。
- 使用真实日志、真实 PostgreSQL 和真实接口验收。
### 阶段 B:阈值配置
- 规则编辑、版本、后端边界、受控配置编译、校验、原子应用和回滚。
- 配置变更对比、近期认证和审计。
### 阶段 C:人工封禁
- Security Agent、执行器、固定时长封禁、解封、保护名单、幂等和状态对账。
- Cloudflare/Nginx 与直连/nftables 分入口验收。
阶段 A、B、C 可以作为同一第一版需求连续交付,但验收必须逐阶段通过,不能为了展示按钮而跳过权限隔离和真实执行验证。
## 16. 第一版完成标准
- 九类检测全部有真实事件来源、默认规则、可配置阈值和原子测试。
- 自研 UI 通过真实 API 展示面板、告警、规则、封禁和健康数据。
- HTTP 错误密钥、签名错误、重放和恶意扫描均纳入第一版。
- NestJS 非 root,无法执行任意系统命令或编辑 Fail2ban 配置。
- 人工封禁按入口选择正确执行器,Cloudflare 场景不使用无效的访客 IP nftables 封禁。
- 保护名单、重新认证、权限、幂等、并发和操作审计全部通过。
- 只有回读执行器确认真实生效后,页面才显示“已封禁”。
- 不发送短信、不修改通道账号/密码/启停状态、企业余额或客户连接。
@@ -0,0 +1,175 @@
# Fail2ban 安全检测与人工封禁第一版测试用例
> 版本:V1.0(设计评审用例)<br>
> 日期:2026-08-14<br>
> 关联设计:`docs/fail2ban-assisted-blocking-design-20260814.md`<br>
> 原则:真实后端、真实 PostgreSQL、真实日志与真实执行器;禁止 mock、静态数据和 localStorage 作为验收证据
## 1. 测试边界
- 自动化测试使用隔离的 Fail2ban/Nginx/nftables namespace 或测试节点,不得封禁测试执行机、预生产运维 IP、Cloudflare 节点或真实客户 IP。
- 预生产验收优先使用文档保留测试 IP 和短时封禁;所有人工封禁必须先确认回滚路径。
- 不发送、补发或重投短信,不修改通道账号、密码、启停状态、企业余额或客户连接。
- HTTP 密钥、签名、CMPP AuthenticatorSource、账号和日志证据必须脱敏。
## 2. 规则配置
| 编号 | 优先级 | 测试步骤 | 预期结果 |
| --- | --- | --- | --- |
| TC-F2B-RULE-001 | P0 | 查询规则列表 | 返回九类第一版规则;数据来自 PostgreSQL;展示当前配置版本和生效版本 |
| TC-F2B-RULE-002 | P0 | 修改规则阈值、窗口、冷却时间和风险等级并保存 | 写入真实数据库,生成新版本,代理校验并应用成功;回读值一致;写操作日志 |
| TC-F2B-RULE-003 | P0 | 提交小于最小值、大于最大值、零、负数、小数、非数字或非法枚举 | 后端逐项拒绝,不生成规则版本,不依赖前端校验兜底 |
| TC-F2B-RULE-004 | P0 | 两名管理员基于同一旧版本并发保存不同阈值 | 仅一方成功;另一方返回版本冲突,不覆盖新配置 |
| TC-F2B-RULE-005 | P0 | 让新配置语法校验或 reload 失败 | 数据库标记 `apply_failed`;上一生效版本继续工作;页面不得提示已生效 |
| TC-F2B-RULE-006 | P1 | 停用一条规则后持续产生匹配日志 | 停用后不创建新告警;已有告警和历史事件保留 |
| TC-F2B-RULE-007 | P1 | 重新启用规则 | 新事件按当前版本统计,不错误合并停用期间日志 |
| TC-F2B-RULE-008 | P0 | 尝试通过接口提交 regex、日志路径、shell、action、jail 名或执行器 | DTO 不接受或后端拒绝;配置文件和系统命令不受影响 |
| TC-F2B-RULE-009 | P0 | 无 `security.rule.manage` 权限修改规则 | 返回 403,不写数据库、不调用安全代理 |
| TC-F2B-RULE-010 | P0 | 超过近期认证时间后修改规则 | 要求重新认证;重新认证成功后才能保存 |
## 3. 登录与 SSH 检测
| 编号 | 优先级 | 测试步骤 | 预期结果 |
| --- | --- | --- | --- |
| TC-F2B-AUTH-001 | P0 | 同一测试 IP 在窗口内触发达到阈值的运营端密码错误 | 只生成一条聚合告警,计数准确,状态 `pending`,未自动封禁 |
| TC-F2B-AUTH-002 | P0 | 同一测试 IP 在客户端登录达到阈值 | 产生客户端规则告警,不与运营端规则错误合并 |
| TC-F2B-AUTH-003 | P1 | 失败次数低于阈值或分散在窗口外 | 保存检测事件但不产生达到阈值告警 |
| TC-F2B-AUTH-004 | P0 | 同一浏览器已有有效运营会话时,在客户端提交错误登录 | 产生正确检测事件;运营会话不被清除、广播退出或跳转 |
| TC-F2B-AUTH-005 | P0 | 检查事件、告警详情和日志 | 不含明文密码、密码散列、完整账号或验证码答案 |
| TC-F2B-SSH-001 | P0 | 从保留测试 IP 对 12022 触发达到阈值的 SSH 失败 | Fail2ban 检测 jail 产生事件和告警,但 nftables 未自动加入封禁 |
| TC-F2B-SSH-002 | P1 | SSH 登录成功、连接中断或握手超时 | 不错误计入认证失败规则 |
| TC-F2B-SSH-003 | P0 | 重启 Fail2ban/collector 后重放同一事件 | `eventKey` 幂等,不重复增加计数或创建告警 |
## 4. CMPP 检测
| 编号 | 优先级 | 测试步骤 | 预期结果 |
| --- | --- | --- | --- |
| TC-F2B-CMPP-001 | P0 | 同一测试 IP 使用未知账号达到阈值 | 生成 `cmpp_auth_failure` 告警;保存脱敏账号指纹和真实 TCP IP |
| TC-F2B-CMPP-002 | P0 | 使用错误 AuthenticatorSource 达到阈值 | 认证继续按原协议拒绝;告警分类准确;不保存完整 AuthenticatorSource |
| TC-F2B-CMPP-003 | P0 | 从不允许 IP、停用企业或停用应用发起连接 | 原认证结果不变;事件分类可区分真实拒绝原因 |
| TC-F2B-CMPP-004 | P0 | 高频发送非法包、异常版本或短连接扫描 | 达到协议滥用规则阈值后形成告警,不污染普通认证失败统计 |
| TC-F2B-CMPP-005 | P1 | 正常断线、心跳超时、最大连接数限制或服务重启恢复 | 不作为恶意认证或协议滥用告警 |
| TC-F2B-CMPP-006 | P0 | 检查事件和页面证据 | 不包含平台密码、完整认证材料、短信正文或通道凭据 |
## 5. HTTP API 检测
| 编号 | 优先级 | 测试步骤 | 预期结果 |
| --- | --- | --- | --- |
| TC-F2B-HTTP-001 | P0 | 同一 IP 使用不存在的 HTTP API 密钥达到阈值 | 生成 `http_invalid_api_key` 告警;仅保存不可逆密钥指纹 |
| TC-F2B-HTTP-002 | P0 | 使用存在但错误的签名达到阈值 | 生成 `http_signature_failure`,不与错误密钥混淆,不保存签名密钥或完整签名 Header |
| TC-F2B-HTTP-003 | P0 | 分别触发签名缺失、格式错误、算法不支持、验签失败和时间偏差 | 内部原因分类准确;页面使用安全文案;敏感细节不泄露给调用方 |
| TC-F2B-HTTP-004 | P0 | 重复使用已消费 nonce/签名请求达到阈值 | 生成 `http_replay_attempt` 严重告警;原接口仍按既有幂等/重放规则拒绝 |
| TC-F2B-HTTP-005 | P0 | 合法客户端按接口幂等协议重试同一业务请求 | 不误判为恶意重放;响应和业务幂等结果保持原语义 |
| TC-F2B-HTTP-006 | P0 | 请求常见敏感路径和漏洞路径,达到次数及不同路径双阈值 | 生成 `http_malicious_scan`;路径已标准化且 query 敏感值不保存 |
| TC-F2B-HTTP-007 | P0 | 对同一不存在业务路径重复请求,仅满足次数、不满足不同路径数 | 不触发恶意扫描告警 |
| TC-F2B-HTTP-008 | P1 | 产生普通业务 400、字段校验失败、合法 401/403 和单次 404 | 不错误计入恶意扫描或错误密钥规则 |
| TC-F2B-HTTP-009 | P0 | 从两个 IP 分别达到一半阈值 | 不跨 IP 错误聚合;每个 IP 独立计算 |
| TC-F2B-HTTP-010 | P0 | 修改 HTTP 规则阈值后继续产生事件 | 新事件使用新生效版本,告警规则快照可追溯当时阈值 |
## 6. Cloudflare 与真实 IP
| 编号 | 优先级 | 测试步骤 | 预期结果 |
| --- | --- | --- | --- |
| TC-F2B-IP-001 | P0 | 经真实 Cloudflare 回源请求运营端和客户端 | 事件 IP 等于访客真实 IP,TCP 代理 IP 保留在安全元数据中但不作为攻击者 IP |
| TC-F2B-IP-002 | P0 | 直连源站并伪造 `CF-Connecting-IP` | Header 被忽略,事件使用真实 TCP 来源 IP |
| TC-F2B-IP-003 | P0 | 人工封禁 Cloudflare 入口告警 | 使用 Nginx real-IP deny,不调用访客 IP nftables action |
| TC-F2B-IP-004 | P0 | 人工封禁灰云 API、SSH 或 CMPP 告警 | 使用 nftables/manual jail,不修改 Nginx Cloudflare deny 列表 |
| TC-F2B-IP-005 | P0 | 尝试封禁 Cloudflare 官方节点、源站自身、回环、内网、运维和健康检查 IP | 后端和安全代理双重拒绝,记录保护名单命中,不产生真实封禁 |
| TC-F2B-IP-006 | P1 | Cloudflare 官方网段更新 | 使用受控来源更新并审计;旧配置切换原子化;失败保留旧可信网段 |
## 7. 告警聚合与状态机
| 编号 | 优先级 | 测试步骤 | 预期结果 |
| --- | --- | --- | --- |
| TC-F2B-ALERT-001 | P0 | 同规则、IP、窗口内并发写入多条事件 | 唯一 fingerprint 生效,只创建一条告警,计数无丢失 |
| TC-F2B-ALERT-002 | P0 | 冷却时间内再次达到阈值 | 更新原告警计数和最近时间,不产生告警风暴 |
| TC-F2B-ALERT-003 | P1 | 冷却结束后再次达到阈值 | 按设计创建新告警或新周期,历史告警不覆盖 |
| TC-F2B-ALERT-004 | P0 | 对 `pending` 告警执行忽略 | 状态原子变为 `ignored`,保留事件和原因,封禁按钮不可再执行 |
| TC-F2B-ALERT-005 | P0 | 两名管理员同时对同一告警点击封禁和忽略 | 仅一个状态迁移成功,另一请求返回 409 |
| TC-F2B-ALERT-006 | P1 | 告警超过可处置期限 | 状态变为 `expired`;历史仍可查询;不能用旧告警封禁 |
## 8. 人工封禁与解封
| 编号 | 优先级 | 测试步骤 | 预期结果 |
| --- | --- | --- | --- |
| TC-F2B-BLOCK-001 | P0 | 有权限管理员对待处理告警选择1小时、填写原因并确认 | 创建唯一操作记录,代理执行正确执行器,回读成功后状态才为 `blocked` |
| TC-F2B-BLOCK-002 | P0 | 前端篡改 IP、jail、action、执行器或时长 | 后端忽略未定义字段或拒绝请求;实际值只来自告警和固定映射 |
| TC-F2B-BLOCK-003 | P0 | 重复点击、网络超时重试或同一 `operationKey` 重放 | 代理和数据库幂等,仅存在一条真实封禁,不延长原期限 |
| TC-F2B-BLOCK-004 | P0 | 代理返回成功但执行器回读不存在规则 | 不标记 `blocked`;状态为失败/异常并生成系统告警 |
| TC-F2B-BLOCK-005 | P0 | Security Agent 停止或 Socket 不可用 | 明确返回封禁失败;告警不伪装已封禁;记录错误和审计 |
| TC-F2B-BLOCK-006 | P0 | PostgreSQL 不可用时点击封禁 | 拒绝无法审计的操作,不调用代理 |
| TC-F2B-BLOCK-007 | P0 | 无 `security.block.manage` 权限或近期认证过期 | 返回403或要求重新认证,不产生操作记录和系统封禁 |
| TC-F2B-BLOCK-008 | P0 | 对已封禁 IP执行解封并填写原因 | 调用原执行器解除,回读确认后变为 `unblocked`,完整记录操作人和原因 |
| TC-F2B-BLOCK-009 | P0 | 封禁自然到期 | 执行器真实解除;对账任务回读后更新 `expired`,不只依赖数据库时间 |
| TC-F2B-BLOCK-010 | P0 | 到期后执行器仍存在规则或平台记录与执行器不一致 | 标记同步异常并告警,不静默显示已解封 |
| TC-F2B-BLOCK-011 | P1 | 规则配置 `manualBlockAllowed=false` | 告警可查看,封禁按钮禁用,直接调用接口同样拒绝 |
## 9. 保护名单
| 编号 | 优先级 | 测试步骤 | 预期结果 |
| --- | --- | --- | --- |
| TC-F2B-PROTECT-001 | P0 | 查询保护名单 | 返回真实数据库和系统内置项,展示类型、范围和来源 |
| TC-F2B-PROTECT-002 | P0 | 新增合法 IPv4、IPv6 或 CIDR 人工保护项 | 后端规范化、检查重叠并保存;要求权限、重新认证、原因和审计 |
| TC-F2B-PROTECT-003 | P0 | 提交非法、过宽、重复或与系统项冲突的网段 | 后端拒绝,现有保护项不变化 |
| TC-F2B-PROTECT-004 | P0 | 尝试删除系统内置保护项 | 拒绝;只能通过受控发布变更 |
| TC-F2B-PROTECT-005 | P0 | 待处理告警 IP 后续加入保护名单 | 告警更新为 `whitelisted` 或明确标记保护命中,不允许封禁 |
## 10. 自研 UI 与真实数据
| 编号 | 优先级 | 测试步骤 | 预期结果 |
| --- | --- | --- | --- |
| TC-F2B-UI-001 | P0 | 进入“安全控制 / 安全检测” | 页面使用平台自研布局、组件和中文状态,不嵌入第三方 Fail2ban UI |
| TC-F2B-UI-002 | P0 | 对照 SQL/API 检查总览五项指标、趋势、分布和 Top IP | 页面与真实后端结果一致,无 mock、静态或 localStorage 数据 |
| TC-F2B-UI-003 | P0 | 使用 IP、规则、风险、状态、入口和日期组合筛选及分页 | 查询由后端完成;总数、页码、跨页结果准确 |
| TC-F2B-UI-004 | P0 | 打开告警详情 | 显示规则快照、聚合时间线、脱敏证据和处置历史;无密钥或认证材料泄露 |
| TC-F2B-UI-005 | P0 | 打开封禁确认弹窗 | 显示执行器、影响入口、固定时长、保护检查、风险提示和必填原因 |
| TC-F2B-UI-006 | P0 | 规则保存成功、保存但应用失败、版本冲突 | 三种状态分别准确提示;失败不得显示成功 toast |
| TC-F2B-UI-007 | P1 | 1440、1280、1024、768和375宽度验收 | 指标、图表、表格、筛选和弹窗无横向溢出;关键操作可见可用 |
| TC-F2B-UI-008 | P1 | 仅键盘和读屏操作页面 | 表单有 label,错误有关联说明,状态不只依赖颜色,弹窗焦点和按钮名称准确 |
| TC-F2B-UI-009 | P0 | 后端、collector 或 agent 不可用 | 页面显示明确降级和最后成功时间,不用空数据伪装“零攻击” |
## 11. 权限、进程与系统加固
| 编号 | 优先级 | 测试步骤 | 预期结果 |
| --- | --- | --- | --- |
| TC-F2B-SEC-001 | P0 | 检查 `cmpp-api.service` 和运行进程 | NestJS 使用专用非 root 用户;有效 UID/GID 非0 |
| TC-F2B-SEC-002 | P0 | 检查 NestJS capabilities、sudoers、可写目录 | 无通用 sudo、无 `CAP_NET_ADMIN`/`CAP_SYS_ADMIN`;只能写明确业务和日志目录 |
| TC-F2B-SEC-003 | P0 | 以 NestJS 用户尝试读取/写入 `/etc/fail2ban`、运行防火墙命令或访问代理管理文件 | 全部被操作系统权限拒绝 |
| TC-F2B-SEC-004 | P0 | 检查 Security Agent Socket | 只在本机 Unix Socket,权限和用户组符合设计,无 TCP 监听 |
| TC-F2B-SEC-005 | P0 | 向 Agent 提交未知动作、非法 JSON、超长字段、shell 字符和非白名单规则 | 全部拒绝且无命令执行;记录限量安全日志 |
| TC-F2B-SEC-006 | P0 | 检查操作日志 | 规则变更、封禁、解封、忽略、保护名单、失败和回读异常均可追溯 |
| TC-F2B-SEC-007 | P1 | 查看告警详情和完整证据 | 读取动作按设计写访问审计;低权限用户只能看到脱敏数据 |
## 12. 可用性、恢复和数据保留
| 编号 | 优先级 | 测试步骤 | 预期结果 |
| --- | --- | --- | --- |
| TC-F2B-OPS-001 | P0 | 停止 Fail2ban | 健康面板显示对应来源异常;已有告警可查;不得显示“当前无攻击” |
| TC-F2B-OPS-002 | P0 | Collector 暂停后恢复并重放积压 | 事件按幂等键补录,计数准确,无重复告警 |
| TC-F2B-OPS-003 | P0 | 重启 API、Gateway、Fail2ban、collector 和 agent | 已生效规则、活动封禁和告警状态恢复一致,不自动执行新封禁 |
| TC-F2B-OPS-004 | P0 | 发布失败触发回滚 | PostgreSQL、代码、环境、Fail2ban/Nginx生成配置和防火墙规则均有可验证恢复路径 |
| TC-F2B-OPS-005 | P1 | 执行30天事件、180天告警/审计保留任务 | 只删除到期且不受保护数据;活动封禁、未完成处置和审计期数据不删除 |
| TC-F2B-OPS-006 | P1 | 大量扫描事件压测 | Collector 有界处理,API与Gateway业务不被阻塞;告警聚合避免写放大 |
| TC-F2B-OPS-007 | P0 | 对比执行器、数据库和页面 | 当前封禁集合、到期时间和执行器类型一致;差异进入异常状态和告警 |
| TC-F2B-OPS-008 | P0 | 在非默认`APP_DIR`构建并运行安全安装器,检查systemd与Fail2ban action的执行路径 | 两者均指向同一个真实可执行的`$APP_DIR/dist/cmpp-security-agent`;残留占位符、旧`current/bin`路径或不可执行目标时安装失败 |
## 13. 发布验收证据
第一版发布前至少保留:
1. Fail2ban filter 单元样例,包含命中与不命中日志。
2. 九类规则的专项自动化结果。
3. API 权限、参数边界、并发、幂等和审计测试。
4. PostgreSQL migration 状态和表/索引/唯一约束核对。
5. NestJS 非 root、capabilities、sudoers 和文件权限证据。
6. Cloudflare 真实 IP、伪造 Header 和 Nginx 执行器真实验证。
7. 直连 API/SSH/CMPP 的隔离测试 IP nftables 封禁与解封证据。
8. Fail2ban、collector、agent、API、Gateway、Nginx、PostgreSQL 和 Redis 健康证据。
9. 自研 UI 桌面端、平板端和移动端截图以及控制台日志。
10. 发布前 PostgreSQL、运行源码、环境文件、生成配置和防火墙规则恢复资产校验。
## 14. 判定规则
- 任一 P0 失败:第一版不得发布。
- 检测达到阈值但未生成告警、未达到阈值却告警、错误 IP 聚合、密钥泄露、保护地址可被封禁、页面显示封禁而执行器未生效、NestJS 仍为 root或可执行任意系统命令,均按 P0 处理。
- 只验证 Fail2ban 命令输出、只验证前端样式或只写数据库不验证真实执行器,不能作为功能通过。
+45 -3
View File
@@ -1390,6 +1390,7 @@
- 数据统计的“通道占比”默认统计北京时间当天,可选择单个历史日期重新查询;图表使用真实通道名称和该日期的通道提交量。 - 数据统计的“通道占比”默认统计北京时间当天,可选择单个历史日期重新查询;图表使用真实通道名称和该日期的通道提交量。
- 输出 500 条/秒压测报告。 - 输出 500 条/秒压测报告。
- 输出 Linux 部署方案,至少覆盖 Docker Compose 或 systemd 部署、环境变量、数据库迁移、日志目录、备份恢复、服务健康检查和回滚步骤。 - 输出 Linux 部署方案,至少覆盖 Docker Compose 或 systemd 部署、环境变量、数据库迁移、日志目录、备份恢复、服务健康检查和回滚步骤。
- 标准发布脚本配置Nginx压缩时必须兼容发行版已有的HTTP级`gzip on`:先排除自身生成文件检查现有有效配置,已有时复用,没有时才创建平台级配置;重复发布不得因重复指令使`nginx -t`失败。
## 16. 新 Codex 会话提示词 ## 16. 新 Codex 会话提示词
@@ -2025,13 +2026,32 @@
- 运营端“短信通道组管理”在通道组名称条件之外增加“通道”筛选。选定一个通道后,只展示成员配置中真实包含该`channelId`的未删除通道组;与通道组名称同时输入时按两个条件取交集。 - 运营端“短信通道组管理”在通道组名称条件之外增加“通道”筛选。选定一个通道后,只展示成员配置中真实包含该`channelId`的未删除通道组;与通道组名称同时输入时按两个条件取交集。
- 通道选项必须来自真实通道API,不使用静态列表、Mock或localStorage。页面首次加载时通道组与通道两个独立请求并行执行;选项同时展示通道名称和编码,已删除通道明确标记“已删除”。未加入任何通道组的真实通道仍可选择,选中后结果为零而不得隐藏该选项。 - 通道选项必须来自真实通道API,不使用静态列表、Mock或localStorage。页面首次加载时通道组与通道两个独立请求并行执行;选项同时展示通道名称和编码,已删除通道明确标记“已删除”。未加入任何通道组的真实通道仍可选择,选中后结果为零而不得隐藏该选项。
- 通道选择控件必须为通用下拉可搜索控件,支持按通道名称或编码搜索;“全部通道”表示不按通道限制,点击“重置”必须同时清空通道组名称和通道条件并回到第一页。 - 通道选择控件必须为通用下拉可搜索控件,支持按通道名称或编码搜索;“全部通道”表示不按通道限制,点击“重置”必须同时清空通道组名称和通道条件并回到第一页。
## Prometheus 系统监控(2026-08-14
- 运营端“系统管理”新增“系统监控”,路由为`/admin/system-monitoring`;原“发送监控”继续负责短信通道和消息业务指标,两个页面、接口和统计口径不得混用。
- 系统监控使用Prometheus和Node Exporter作为真实指标基础设施,但全部用户界面由平台React原生实现,不嵌入Grafana、Netdata、Zabbix、Prometheus页面或第三方登录界面。
- 浏览器只请求`GET /api/admin/infrastructure-monitoring/overview?range=1h|24h|7d`。NestJS以固定PromQL模板查询Prometheus,禁止前端传任意PromQL、step、时间戳或标签选择器;9090和9100只监听本机或内网,不向公网开放。
- 第一版展示CPU、内存、根文件系统、网络收发、1分钟负载和系统运行时长,并展示API、Gateway、PostgreSQL、Redis、MinIO、Nginx六类systemd服务状态。指标缺失必须显示“暂无指标/未知”,不得用0、静态数据、Mock或localStorage冒充真实采集值。
- 支持近1小时、近24小时和近7天固定范围,步长分别为60秒、300秒和1800秒。页面可见时每30秒刷新,隐藏或卸载后停止;手动刷新保留当前范围。
- 页面展示Prometheus当前firing/pending告警,严重性使用`info/warning/critical`。告警阈值由Prometheus规则计算,前端不重复判断;第一版仅只读展示,不提供缺少审计模型的确认、备注、静默或关闭操作。
- Prometheus不可用、查询超时、响应非法时接口返回`available=false`及安全错误摘要,页面清空陈旧指标并展示监控不可用;不能继续显示上一次数据造成误判。
- 固定PromQL必须按Prometheus字符串与RE2正则两层语义正确转义,并由专项契约锁定systemd单元正则;任一查询导致整页降级时,API必须记录不含PromQL、地址或凭据的安全错误摘要,不能只向页面返回笼统不可用而没有服务端诊断证据。
- 系统监控页只保留平台通用页头中的页面名称;内容区不得再次显示大号“系统监控”标题,可保留“服务器资源、核心服务与活动告警”说明、综合状态和时间范围操作。
- 完整采集、查询、安全、响应契约、视觉规格和验收口径见`docs/prometheus-system-monitoring-design-20260814.md`
## 全局预警通知菜单(2026-08-14)
- 运营端右上角铃铛统一作为预警入口,点击后必须分开显示“签名清退预警”和“安全检测与封禁”两个菜单项;审核待办继续使用独立审核图标和菜单,不得把审核数与预警数混合。
- 签名清退项展示今日未读且未抑制的真实消息数并跳转`/admin/signature-retirement`;安全检测项展示`open/acknowledged/block_failed`真实待处置告警总数、严重告警摘要并跳转`/admin/security-detection`
- 铃铛角标为两个预警域数量之和;任一域接口失败时只把该域降级为0,不能影响另一域或审核待办。全局轮询使用专用轻量汇总接口,不得每30秒调用安全检测完整总览、代理回读或大列表。
# 下游投递后台重投任务(2026-08-12) # 下游投递后台重投任务(2026-08-12)
## 完整设计口径与安全整改(2026-08-13) ## 完整设计口径与安全整改(2026-08-13)
1. 后台任务按企业、应用、投递类型、状态、北京时间创建日期和关键词冻结筛选快照,分页、每页条数和当前页勾选不属于范围。预检的命中数、可重投数、规则跳过数、状态分布、涉及应用和最早记录必须严格基于当前筛选条件,不得把单一状态擅自扩为全部状态。 1. 后台任务按企业、应用、投递类型、状态、北京时间创建日期和关键词冻结筛选快照,分页、每页条数和当前页勾选不属于范围。预检的命中数、可重投数、规则跳过数、状态分布、涉及应用和最早记录必须严格基于当前筛选条件,不得把单一状态擅自扩为全部状态。
2. 预检返回短期有效且绑定当前操作人、筛选条件和 `snapshotAt` 的服务端签名凭证;创建接口只接受该凭证、原因、速度和安全阈值,不再信任前端重传的范围。创建时后端按签名快照重新物化真实 PostgreSQL 任务项。 2. 预检返回短期有效且绑定当前操作人、筛选条件和 `snapshotAt` 的服务端签名凭证;创建接口只接受该凭证、原因、速度和安全阈值,不再信任前端重传的范围。创建时后端按签名快照重新物化真实 PostgreSQL 任务项。
3. 第一版后台任务处理 `pending/failed/unconfirmed/rejected``delivered/awaiting_ack` 不得进入批量任务。跳过严格表示本任务没有调用 Gateway;客户离线进入 `waiting_connection`,其他链路等待 ACK 进入 `waiting_external_ack`,本任务写出后进入 `waiting_ack`,三类等待均不计失败或跳过。 3. 后台任务允许处理 `pending/failed/unconfirmed/rejected/delivered`;客户端已确认的 `delivered` 必须在预检、创建弹窗和任务项原状态中明确呈现,运营确认后可以再次投递,但页面必须提示可能造成客户端重复处理。`awaiting_ack` 不得进入批量任务。若任务项创建时不是 `delivered`、执行前才收到成功 ACK,则本任务必须跳过,不能把迟到确认变成未授权重复投递。跳过严格表示本任务没有调用 Gateway;客户离线进入 `waiting_connection`,其他链路等待 ACK 进入 `waiting_external_ack`,本任务写出后进入 `waiting_ack`,三类等待均不计失败或跳过。
4. 限速以企业应用为维度,使用数据库原子秒级窗口在多实例、扫描重叠和执行耗时变化下保持每应用不超过配置速度。任务扫描使用数据库租约;`processing` 项使用认领租约,API 中断后过期恢复为 `queued` 并重新复核,已成功 ACK 的项目不得重放。 4. 限速以企业应用为维度,使用数据库原子秒级窗口在多实例、扫描重叠和执行耗时变化下保持每应用不超过配置速度。任务扫描使用数据库租约;`processing` 项使用认领租约,API 中断后过期恢复为 `queued` 并重新复核,已成功 ACK 的项目不得重放。
5. 连续失败按应用隔离统计。Gateway 立即失败、ACK 超时、ACK 拒绝和无法安全关联均计入阈值;只有有效 ACK 成功才清零。达到阈值前原子暂停整个任务,记录触发应用、失败数、阈值和暂停时间,人工继续后从未完成项恢复。 5. 连续失败按应用隔离统计。Gateway 立即失败、ACK 超时、ACK 拒绝和无法安全关联均计入阈值;只有有效 ACK 成功才清零。达到阈值前原子暂停整个任务,记录触发应用、失败数、阈值和暂停时间,人工继续后从未完成项恢复。
6. 任务列表支持状态筛选和真实分页,展示任务号、创建时间、企业/应用、原因、中文状态、总数、成功、失败、跳过、等待、创建人及进度。任务详情展示筛选快照、时间、安全参数、完整结果汇总和任务项分页;任务项可按中文结果、消息 ID、错误或跳过原因查询,不得仅返回最近 50 项。 6. 任务列表支持状态筛选和真实分页,展示任务号、创建时间、企业/应用、原因、中文状态、总数、成功、失败、跳过、等待、创建人及进度。任务详情展示筛选快照、时间、安全参数、完整结果汇总和任务项分页;任务项可按中文结果、消息 ID、错误或跳过原因查询,不得仅返回最近 50 项。
@@ -2040,9 +2060,9 @@
1. 运营端“下游投递记录”必须同时保留单条重投、当前页勾选批量重投,并新增“按筛选条件重投”;分页支持每页 `10/25/50` 条,切换后回到第一页并重新查询真实后端。 1. 运营端“下游投递记录”必须同时保留单条重投、当前页勾选批量重投,并新增“按筛选条件重投”;分页支持每页 `10/25/50` 条,切换后回到第一页并重新查询真实后端。
2. 后台任务使用当前企业、应用、投递类型、状态、创建日期和关键词的后端筛选快照,分页不属于任务范围;任务创建时固定 `snapshotAt`,之后产生的记录不得被卷入。 2. 后台任务使用当前企业、应用、投递类型、状态、创建日期和关键词的后端筛选快照,分页不属于任务范围;任务创建时固定 `snapshotAt`,之后产生的记录不得被卷入。
3. 第一版只允许 `pending/failed/unconfirmed/rejected`,不支持批量重投客户端已确认的 `delivered``awaiting_ack` 不得并发重投。创建前必须真实预检命中、可重投、跳过和状态分布,原因必填 3. 批量任务允许 `pending/failed/unconfirmed/rejected/delivered`,其中 `delivered` 会再次发送并可能造成客户端重复处理,创建弹窗必须醒目提示;`awaiting_ack` 不得并发重投。创建前必须真实预检命中、可重投、跳过和状态分布,原因必填。执行器仅允许重投在冻结快照中原状态已经是 `delivered` 的已确认任务项;任务建立后才变成 `delivered` 的项目必须跳过
4. 任务按应用分批执行,默认每秒 10 条;单条失败不阻断整批,连续失败达到 10 条或 ACK 超时/拒绝达到安全阈值时自动暂停。客户离线、等待 ACK 属于等待状态,不得误记为跳过。 4. 任务按应用分批执行,默认每秒 10 条;单条失败不阻断整批,连续失败达到 10 条或 ACK 超时/拒绝达到安全阈值时自动暂停。客户离线、等待 ACK 属于等待状态,不得误记为跳过。
5. 跳过只表示未调用 Gateway,第一版原因包括:状态已变化、被客户确认、已被其他任务处理、本任务已成功处理、不属于任务快照、应用或投递能力已停用、投递数据不完整、缺少原 Submit 映射。 5. 跳过只表示未调用 Gateway,原因包括:状态已变化、创建任务后才被客户确认、已被其他任务处理、本任务已成功处理、不属于任务快照、应用或投递能力已停用、投递数据不完整、缺少原 Submit 映射。
6. 任务必须支持列表、详情、暂停、继续和终止;终止只影响尚未发送的记录。任务项以 `taskId + deliveryId` 幂等,执行前原子认领并复核状态,API 重启后可继续,成功 ACK 的项目不得再次发送。 6. 任务必须支持列表、详情、暂停、继续和终止;终止只影响尚未发送的记录。任务项以 `taskId + deliveryId` 幂等,执行前原子认领并复核状态,API 重启后可继续,成功 ACK 的项目不得再次发送。
7. 所有创建、暂停、继续、终止和自动暂停均写操作日志;任务使用真实 PostgreSQL、Gateway 和客户 ACK,不得使用 mock、静态数据或 localStorage。 7. 所有创建、暂停、继续、终止和自动暂停均写操作日志;任务使用真实 PostgreSQL、Gateway 和客户 ACK,不得使用 mock、静态数据或 localStorage。
# 2026-08-13 HTTP 请求与 Gateway API 响应容量边界 # 2026-08-13 HTTP 请求与 Gateway API 响应容量边界
@@ -2059,3 +2079,25 @@
2. 运营看板删除“今日签名发送统计”和“今日签名发送统计 - 含引流”两个明细模块;“今日活跃签名”指标仍使用当天真实发送聚合。今日消费金额的主数字必须与今日发送总量使用相同字号、字重和深色层级。 2. 运营看板删除“今日签名发送统计”和“今日签名发送统计 - 含引流”两个明细模块;“今日活跃签名”指标仍使用当天真实发送聚合。今日消费金额的主数字必须与今日发送总量使用相同字号、字重和深色层级。
3. 企业应用管理列表的状态、到达率、单价列在现有基础上缩窄约20%,提升大屏一次展示完整表格的概率;不得通过隐藏真实字段实现。 3. 企业应用管理列表的状态、到达率、单价列在现有基础上缩窄约20%,提升大屏一次展示完整表格的概率;不得通过隐藏真实字段实现。
4. 移动、联通、电信数据展示统一使用全局`CarrierTag`低饱和标签,包括通道与通道组、监控、报备、签名、短信审核、任务号码、发送记录及客户端发送详情等页面。筛选/表单控件的选项文案、图表图例、导出文本和业务说明仍使用纯文本,避免破坏交互、可访问性和机器可读输出;可取得运营商集合的三网通道按三个运营商标签展示,只有历史通道级字段时使用中性的“三网”标签。 4. 移动、联通、电信数据展示统一使用全局`CarrierTag`低饱和标签,包括通道与通道组、监控、报备、签名、短信审核、任务号码、发送记录及客户端发送详情等页面。筛选/表单控件的选项文案、图表图例、导出文本和业务说明仍使用纯文本,避免破坏交互、可访问性和机器可读输出;可取得运营商集合的三网通道按三个运营商标签展示,只有历史通道级字段时使用中性的“三网”标签。
## Fail2ban 安全检测与人工封禁(2026-08-14)
- 运营端“安全控制”新增“安全检测与封禁”自研页面,数据必须来自真实 NestJS API 与 PostgreSQL,包含总览、告警中心、九类规则配置、人工封禁记录和保护名单;不得嵌入第三方面板或使用 Mock、静态数据、localStorage 伪造检测状态。
- 第一版检测运营端登录失败、客户端登录失败、SSH 认证失败、CMPP 认证失败、CMPP 协议滥用、HTTP 错误密钥、HTTP 签名错误、HTTP 重放及 HTTP 恶意扫描。规则阈值、窗口、冷却、风险级别、启停状态与封禁时长边界保存在数据库,并使用配置版本防止并发覆盖。
- 检测只产生事件与人工告警,绝不自动封禁。人工封禁只允许 10 分钟、1 小时、24 小时或 7 天;操作要求近期重新认证、必填原因、保护名单校验、数据库原子认领和操作审计。执行器由服务端按可信入口固定映射,浏览器不得提交 jail、action、shell 参数或自选执行器。
- NestJS 必须以专用非 root 用户运行,不得获得通用 sudo、任意 shell、直接编辑 `/etc/fail2ban/*` 或防火墙的能力。独立 root security agent 只监听本机 Unix Socket、接受固定 JSON 动作、使用参数数组执行固定操作,并在真实 nftables 或 Nginx deny 回读成功后才允许数据库标记 `blocked`
- 运营端/客户端的 Cloudflare 入口使用 Nginx real-IP deny;直连 HTTP API、SSH 与 CMPP 使用 nftables。只有可信代理 TCP 来源可以提供访客 IP;系统回环、私网、链路本地、组播和配置的运维/健康检查网段必须内置保护。
- 规则更新先保存待应用版本,由安全代理生成固定 Fail2ban 配置、执行语法校验并 reload;失败保留上一生效值并展示失败原因,不得显示为已生效。完整架构、状态机、字段、接口和安全边界以 `docs/fail2ban-assisted-blocking-design-20260814.md` 为准。
- 安全代理的systemd单元与Fail2ban report-only action必须由安装器从同一受控占位符渲染,并共同指向发布脚本实际生成的`$APP_DIR/dist/cmpp-security-agent`;安装后残留占位符、旧`current/bin`路径或不可执行目标时必须终止发布,禁止出现Agent服务可启动但Fail2ban action静默失效的分叉配置。
- NestJS API生产进程必须显式绑定`127.0.0.1`,仅由Nginx受控入口反向代理;不得依赖框架默认的全网卡监听而把3000端口直接暴露到LAN、Tailscale或公网。非生产环境如确需其他地址,只能通过明确的`API_HOST`配置覆盖。
## 服务内部指标与阈值(2026-08-14)
- API、Gateway必须以回环端点暴露低基数运行指标;PostgreSQL、Redis、Nginx使用发行版Exporter,MinIO使用原生指标。任何指标端口都不得经Nginx或安全组对公网暴露。
- 运营端展示API请求/错误/延迟/事件循环、Gateway Submit/队列/连接、PostgreSQL连接/死锁、Redis内存/连接/淘汰、Nginx连接/请求和MinIO可用性;指标缺失显示“待采集”,不以0伪装。
- 告警必须使用持续窗口和最低样本量;默认阈值、收敛关系、标签禁止项和性能预算以`docs/prometheus-system-monitoring-design-20260814.md`第9节为准。
- 指标不得包含手机号、短信正文、短信/CMPP/任务ID、密钥、完整URL或SQL原文;不得把时序指标高频写入业务PostgreSQL。
- 系统监控标题说明必须明确标注数据来自 Prometheus;“服务关键指标”位于趋势/核心服务区域之后、活动告警之前,并提供统一的“告警阈值设置”入口。安全检测页不重复渲染大号标题,说明文字必须明确标注使用 Fail2ban。
- 告警阈值仅开放固定指标的警告/严重数值,不允许前端提交 PromQL、标签、文件路径或持续时间;必须满足警告值小于严重值。配置以 PostgreSQL 保存版本、期望值、生效值和应用状态,经 `promtool check rules` 校验、同目录原子替换和 Prometheus 热加载成功后才标记生效,失败保留上一生效规则并展示原因。
- 右上角预警中心增加“系统监控告警”,通过独立轻量接口统计 Prometheus 当前 firing/pending 告警及严重数,跳转系统监控活动告警区;任一预警域失败不得清空其他域。
- 活动告警列表必须提供逐条“标记已读”。已读状态按管理员和“告警指纹 + 本次 activeAt”持久化到 PostgreSQL;仅从当前管理员的预警中心数量中扣减,不改变 Prometheus firing/pending 状态,也不减少页面活动告警总数。同标签告警恢复后再次触发时必须重新成为未读。
- 服务端只能确认 Prometheus 当前仍存在且 activeAt 一致的告警,过期、已恢复或已重新触发的请求必须拒绝;重复点击同一次告警应幂等,并写操作日志。阈值设置弹窗只保留通用 Modal 外层滚动,不得嵌套第二个独立滚动区域。
+25
View File
@@ -30,6 +30,9 @@ REPO_URL=http://175.27.255.91:3000/hectorzhao/lislgosms.git
BRANCH=main BRANCH=main
PUBLIC_HTTP_PORT=12026 PUBLIC_HTTP_PORT=12026
API_PORT=3000 API_PORT=3000
API_HOST=127.0.0.1
API_METRICS_HOST=127.0.0.1
API_METRICS_PORT=9464
HTTP_API_MASTER_KEY=<至少32位随机值,用于AES-256-GCM加密HTTP访问凭据和Webhook密钥> HTTP_API_MASTER_KEY=<至少32位随机值,用于AES-256-GCM加密HTTP访问凭据和Webhook密钥>
HTTP_API_PUBLIC_ORIGIN=https://api.lisglo.com HTTP_API_PUBLIC_ORIGIN=https://api.lisglo.com
API_ENABLE_SEND_WORKER=true API_ENABLE_SEND_WORKER=true
@@ -48,6 +51,8 @@ OPERATION_LOG_ARCHIVE_INTERVAL_MS=86400000
SMS_RECEIPT_TIMEOUT_SCAN_ENABLED=true SMS_RECEIPT_TIMEOUT_SCAN_ENABLED=true
SMS_RECEIPT_TIMEOUT_HOURS=72 SMS_RECEIPT_TIMEOUT_HOURS=72
SMS_RECEIPT_TIMEOUT_SCAN_INTERVAL_MS=300000 SMS_RECEIPT_TIMEOUT_SCAN_INTERVAL_MS=300000
PROMETHEUS_URL=http://127.0.0.1:9090
PROMETHEUS_QUERY_TIMEOUT_MS=5000
REPORT_DAILY_REFRESH_ENABLED=true REPORT_DAILY_REFRESH_ENABLED=true
REPORT_REFRESH_INTERVAL_MS=3600000 REPORT_REFRESH_INTERVAL_MS=3600000
CMPP_DOWNSTREAM_ACK_TIMEOUT_SECONDS=30 CMPP_DOWNSTREAM_ACK_TIMEOUT_SECONDS=30
@@ -62,6 +67,8 @@ PROD_ADMIN_USERNAME=prod_admin
PROD_ADMIN_PASSWORD='change-me' PROD_ADMIN_PASSWORD='change-me'
``` ```
系统监控需要在发布前单独安装,配置和规则位于`tools/monitoring/`。在Debian/Ubuntu服务器依次执行`bash tools/monitoring/install-prometheus-monitoring.sh``bash tools/monitoring/install-service-exporters.sh`;前者安装Prometheus/Node Exporter,后者安装PostgreSQL/Redis/Nginx Exporter并开启MinIO回环原生指标。脚本必须先备份现有配置并运行`promtool`校验;9090、9100、9187、9121、9113、9464和API/Gateway控制端口必须只监听`127.0.0.1`,不得加入Nginx公网反向代理或安全组放行。安装完成且确认全部target为up后,才可完成发布;详细口径见`docs/prometheus-system-monitoring-design-20260814.md`
安全会话使用 HttpOnly Cookie,正式生产必须先为页面和管理 API 配置 HTTPS,并保持 `SESSION_COOKIE_SECURE=true`;纯 HTTP 的 `IP:12026` 不作为受支持的登录入口,即使切换期仍保留其监听,也只允许用于非登录的兼容检查并应尽快下线。`sms.lisglo.com` 只允许 Cloudflare 回源,`api.lisglo.com` 通过独立 Nginx SNI 虚拟主机只开放客户接口、客户 Swagger 和健康检查;Lets Encrypt 使用 DNS-01 自动续期,不依赖开放 80 端口。 安全会话使用 HttpOnly Cookie,正式生产必须先为页面和管理 API 配置 HTTPS,并保持 `SESSION_COOKIE_SECURE=true`;纯 HTTP 的 `IP:12026` 不作为受支持的登录入口,即使切换期仍保留其监听,也只允许用于非登录的兼容检查并应尽快下线。`sms.lisglo.com` 只允许 Cloudflare 回源,`api.lisglo.com` 通过独立 Nginx SNI 虚拟主机只开放客户接口、客户 Swagger 和健康检查;Lets Encrypt 使用 DNS-01 自动续期,不依赖开放 80 端口。
系统操作日志默认在线保留 180 天。API 每日以最多 20 个、每批 1000 条的小事务将过期记录搬入 `OperationLogArchive`,并用 `archiveMonth=YYYY-MM` 标记归档月份;归档记录不会自动删除。调整保留期或批量参数前,应先评估数据库、备份窗口和审计要求。归档表达到千万级或清理窗口不能满足要求时,再实施按 `createdAt` 的月度 PostgreSQL 分区,不在当前数据规模下提前改造主表分区。 系统操作日志默认在线保留 180 天。API 每日以最多 20 个、每批 1000 条的小事务将过期记录搬入 `OperationLogArchive`,并用 `archiveMonth=YYYY-MM` 标记归档月份;归档记录不会自动删除。调整保留期或批量参数前,应先评估数据库、备份窗口和审计要求。归档表达到千万级或清理窗口不能满足要求时,再实施按 `createdAt` 的月度 PostgreSQL 分区,不在当前数据规模下提前改造主表分区。
@@ -89,8 +96,26 @@ git reset --hard origin/main
bash tools/deploy/production-deploy.sh bash tools/deploy/production-deploy.sh
``` ```
### Fail2ban 安全检测发布前置条件
本功能包含新增 PostgreSQL migration、非 root API 身份、安全代理、Fail2ban、Nginx include 和 nftables 表,不能按普通前端热发布处理。发布前除平台标准 PostgreSQL、运行源码和环境文件恢复资产外,必须额外备份 `/etc/systemd/system/cmpp-api.service*``/etc/systemd/system/cmpp-security-agent.service``/etc/fail2ban``/etc/nginx``/etc/nftables.conf``/etc/nftables.d``/var/lib/cmpp-security-agent`,并逐项生成、复核 SHA-256。
发布脚本会构建 `cmpp-security-agent` 并运行 `tools/security/install-security-agent.sh`。安装器创建专用 `cmpp-api` 用户和 `cmpp-security` 组、写入 systemd 加固 drop-in、安装固定 Fail2ban filter/action、校验 Nginx/Fail2ban/nftables,但不会执行任何人工封禁。环境文件至少明确:
```bash
SECURITY_AGENT_SOCKET=/run/cmpp-security-agent/agent.sock
SECURITY_AGENT_TIMEOUT_MS=3000
SECURITY_EVENT_TOKEN=<至少32字节随机值,仅供Gateway和安全代理上报固定事件>
TRUSTED_PROXY_IPS=127.0.0.1,::1
SECURITY_BUILTIN_PROTECTED_NETWORKS=<运维出口CIDR,健康检查CIDR,源站公网IP>
```
正式 Nginx 的 `sms.lisglo.com` 运营端/客户端 server 块必须 `include /etc/nginx/snippets/cmpp-security-deny.conf;`;API 专用域名继续只开放客户接口。Cloudflare `real_ip_header` 及可信网段必须按官方来源单独维护和验证,禁止信任任意客户端 `CF-Connecting-IP``X-Forwarded-For`。发布后需证明 `cmpp-api` 进程用户不是 root、无 sudo 权限且无法写 `/etc/fail2ban`,安全代理 Socket 不监听 TCP,九类规则版本一致,Fail2ban 为 report-onlynftables/Nginx 回读与数据库状态一致。任何一项失败均不得开放人工封禁按钮。
部署脚本重启 Gateway、API 和 Nginx 后,会分别对 Gateway、API 健康接口执行最多 60 秒的逐秒就绪检查。Nest 初始化、活动通道恢复或生产数据量增加可能使 API 启动超过固定数秒;发布流程不得用单次固定延时把正常慢启动误判为失败。超过 60 秒仍不健康时才终止发布,并结合 systemd journal 和发布前数据库、源码、环境备份判断回滚方式。 部署脚本重启 Gateway、API 和 Nginx 后,会分别对 Gateway、API 健康接口执行最多 60 秒的逐秒就绪检查。Nest 初始化、活动通道恢复或生产数据量增加可能使 API 启动超过固定数秒;发布流程不得用单次固定延时把正常慢启动误判为失败。超过 60 秒仍不健康时才终止发布,并结合 systemd journal 和发布前数据库、源码、环境备份判断回滚方式。
标准发布会先清空自身管理的`/etc/nginx/conf.d/cmpp-compression.conf`,再排除该文件检查Nginx现有配置。发行版或既有虚拟主机已经启用`gzip on`时直接复用;完全未启用时才写入平台级压缩配置。不得无条件叠加第二个HTTP级`gzip on`,每次重启前必须以`nginx -t`为准。
## 账号和密钥 ## 账号和密钥
- 生产管理员账号写入 `/root/cmpp-platform-admin.txt` - 生产管理员账号写入 `/root/cmpp-platform-admin.txt`
@@ -0,0 +1,251 @@
# Prometheus 系统监控设计
> 版本:V1.0<br>
> 设计日期:2026-08-14<br>
> 适用范围:运营端 → 系统管理 → 系统监控<br>
> 实施边界:使用平台原生 React UI,不引入 Grafana、Netdata、Zabbix 或 Prometheus 自带 UI
## 1. 建设目标
运营人员需要在现有权限体系和视觉体系内查看服务器硬件资源、核心服务状态、历史趋势和活动告警。Prometheus 仅承担指标抓取、时序存储、PromQL 计算和告警规则执行;浏览器只访问 CMPP 平台 NestJS API,不直接访问 Prometheus、Node Exporter 或 Alertmanager。
第一版解决以下问题:
1. 统一查看 CPU、内存、根文件系统、网络流量、负载和运行时长。
2. 查看 `cmpp-api``cmpp-gateway`、PostgreSQL、Redis、MinIO、Nginx 六类核心服务状态。
3. 在近 1 小时、近 24 小时、近 7 天之间切换真实历史趋势。
4. 查看 Prometheus 当前 firing/pending 告警,不在前端伪造阈值判断。
5. Prometheus 不可用、查询超时或指标缺失时明确展示“监控不可用/暂无指标”,不得回退静态值、Mock 或 localStorage。
## 2. 非目标与边界
- 第一版不提供任意 PromQL 控制台,避免越权查询、高基数查询和资源耗尽。
- 第一版不提供告警确认、备注和关闭操作;需要审计留痕的告警处置另立需求并增加 PostgreSQL 模型。
- 第一版不采集短信正文、手机号、账号、密钥、数据库查询文本或日志原文。
- 云主机通常只能提供虚拟 CPU、内存、云盘和虚拟网卡指标;风扇、物理温度、电源、RAID 和物理硬盘 SMART 需 IPMI/Redfish 或厂商接口,未接入时不得显示伪造数据。
- Prometheus、Node Exporter 和 Alertmanager 不暴露在公网;仅 API 服务端可以访问 Prometheus。
## 3. 总体架构
```mermaid
flowchart LR
Node["Node Exporter\n主机与 systemd 指标"] --> Prom["Prometheus\n抓取、存储、PromQL、告警规则"]
API["NestJS 监控模块\n固定查询模板与响应归一化"] --> Prom
UI["运营端原生 React UI"] --> API
Prom --> Alert["Prometheus 活动告警"]
Alert --> API
```
数据流必须是 `Exporter → Prometheus → NestJS → 运营端`。前端不得直接拼接 Prometheus URL,不得保存 Prometheus Token,不得接受服务端返回的原始 PromQL。
## 4. 采集与部署设计
### 4.1 Node Exporter
Node Exporter 仅监听 `127.0.0.1:9100`,启用默认 CPU、内存、文件系统、磁盘、网络、负载和启动时间采集器,并显式启用 `systemd` 采集器。systemd 只允许采集:
- `cmpp-api.service`
- `cmpp-gateway.service`
- `postgresql.service`
- `redis.service``redis-server.service`
- `cmpp-minio.service`
- `nginx.service`
排除 `/dev``/proc``/sys``/run` 等伪文件系统和临时挂载,避免磁盘指标重复和高基数。
### 4.2 Prometheus
- 仅监听 `127.0.0.1:9090`
- 默认每 15 秒抓取一次,查询超时 5 秒。
- 第一版保留 30 天或不超过 8GB 的时序数据,达到任一边界即按 Prometheus TSDB 策略清理。
- 配置文件由仓库 `tools/monitoring/` 管理;安装脚本只安装、校验和启动监控服务,不重启 CMPP Gateway 或发送 Worker。
- 生产环境变量使用 `PROMETHEUS_URL=http://127.0.0.1:9090`API 不向响应透出该地址。
### 4.3 告警规则
规则使用持续窗口而非瞬时尖峰:
| 告警 | Warning | Critical |
|---|---:|---:|
| 主机指标失联 | — | 连续 2 分钟无数据 |
| CPU 使用率 | 连续 10 分钟 > 85% | 连续 5 分钟 > 95% |
| 内存使用率 | 连续 10 分钟 > 85% | 连续 5 分钟 > 95% |
| 根文件系统使用率 | 连续 15 分钟 > 80% | 连续 5 分钟 > 90% |
| inode 使用率 | 连续 15 分钟 > 80% | 连续 5 分钟 > 90% |
| CPU iowait | 连续 10 分钟 > 20% | 连续 10 分钟 > 35% |
| 核心 systemd 服务 | — | 非 active 2 分钟 |
告警标签至少包含 `alertname``severity``instance``service`;注解至少包含中文 `summary``description``currentValue``threshold`。敏感环境变量和凭据不得进入标签或注解。
## 5. 后端设计
### 5.1 API
```text
GET /api/admin/infrastructure-monitoring/overview?range=1h|24h|7d
```
该接口受现有运营端 Session 中间件保护,仅提供只读数据。`range` 只接受白名单;非法值返回 400。
### 5.2 固定查询
后端维护具名查询表,不接受前端 PromQL:
- CPU:非 idle CPU 秒率。
- 内存:`1 - MemAvailable / MemTotal`
- 根文件系统:`1 - available / size`
- 网络:排除 loopback 后的收发字节率。
- 负载:`node_load1`
- 运行时长:当前时间减 `node_boot_time_seconds`
- 服务:指定 unit 的 `node_systemd_unit_state{state="active"}`
- 告警:Prometheus `/api/v1/alerts`
瞬时查询和区间查询并行执行。区间步长固定为:1小时/60秒、24小时/300秒、7天/1800秒;后端最多返回约 340 个点/序列,禁止浏览器控制步长。
### 5.3 可用性与错误语义
- Prometheus 查询成功:`available=true`,返回真实指标、服务和告警。
- Prometheus 未配置、拒绝连接、超时、返回非成功状态或响应结构非法:`available=false`,返回采集时间和安全错误摘要,指标字段为 `null`、趋势为空数组。
- 单个指标不存在不影响其他指标,缺失项为 `null`;不得把缺失解释为 0。
- 服务状态使用 `healthy/unhealthy/unknown`,只有明确采集到 active 才是 healthy,明确采集到 0 才是 unhealthy,指标缺失是 unknown。
### 5.4 安全控制
- Prometheus URL 由服务端环境变量读取并限制为 `http://127.0.0.1` 或明确配置的内网 HTTPS 地址。
- 每次请求设置 5 秒 AbortSignal 超时。
- 不记录完整响应体;错误日志不得包含URL中的认证信息。
- 不允许前端传 query、step、start、end 或 Prometheus 标签选择器。
- 不将监控指标复制进业务 PostgreSQL,避免高频写入和业务库膨胀。
## 6. 前端信息架构
页面路由为 `/admin/system-monitoring`,保留原 `/admin/monitor` “发送监控”,两者业务含义不得混用。
### 6.1 页面结构
1. 页面标题、最近采集时间、刷新按钮、1小时/24小时/7天范围切换。
2. 横向健康概览:整体状态、服务总数、正常、警告、严重、活动告警。
3. CPU、内存、磁盘、网络四项资源区:当前值、辅助值和当前范围趋势。
4. 服务状态侧栏:六类服务、状态和最新采集时间。
5. 活动告警表:名称、级别、开始时间、持续时间、当前值、阈值。
6. 下方资源趋势:CPU、内存、磁盘和网络收发的完整趋势图。
### 6.2 视觉规格
- 复用现有 `surface``Button``Tag``Table``Chart` 和主题变量。
- 真白背景、低饱和蓝色主色、冷灰边框;正常/警告/严重使用既有语义色。
- 不使用第三方Logo、iframe、暗色监控皮肤、霓虹渐变或装饰性假指标。
- 桌面优先保持表格和趋势信息密度;1100px 以下两列,780px 以下单列。
- 图表缺失时显示“暂无真实指标”,不能绘制零线冒充数据。
### 6.3 交互与刷新
- 初次进入立即请求真实接口。
- 可见页面每 30 秒刷新;页面隐藏或 Session 锁定导致组件卸载后停止刷新。
- 切换时间范围立即重新查询并禁用重复点击;手动刷新不改变范围。
- 请求失败保留上一次成功数据会造成陈旧误导,因此本版失败时清空数据并展示不可用状态。
## 7. 响应契约摘要
```ts
type InfrastructureOverview = {
available: boolean;
range: '1h' | '24h' | '7d';
collectedAt: string;
error?: string;
summary: {
overallStatus: 'healthy' | 'warning' | 'critical' | 'unknown';
serviceTotal: number;
serviceHealthy: number;
warningAlerts: number;
criticalAlerts: number;
activeAlerts: number;
};
metrics: {
cpuUsagePercent: number | null;
memoryUsagePercent: number | null;
memoryTotalBytes: number | null;
memoryAvailableBytes: number | null;
diskUsagePercent: number | null;
diskTotalBytes: number | null;
diskAvailableBytes: number | null;
networkReceiveBytesPerSecond: number | null;
networkTransmitBytesPerSecond: number | null;
load1: number | null;
uptimeSeconds: number | null;
};
trends: Record<string, Array<{ timestamp: string; value: number }>>;
services: Array<{ key: string; name: string; unit: string; status: 'healthy' | 'unhealthy' | 'unknown' }>;
alerts: Array<{ fingerprint: string; name: string; severity: 'warning' | 'critical' | 'info'; status: string; startedAt: string; summary: string; currentValue?: string; threshold?: string }>;
};
```
## 8. 验收标准
1. 页面所有指标来自真实 Prometheus API,关闭 Prometheus 后页面明确不可用且没有静态回退。
2. 浏览器网络请求只访问 CMPP API,不访问 9090/9100。
3. 非法 range、任意 PromQL 和自定义 step 均无法进入后端查询。
4. CPU、内存、磁盘、网络当前值与 Prometheus 同时刻查询在允许的采样误差内一致。
5. 六类服务状态与 systemd 指标一致;指标缺失展示未知而非故障或正常。
6. 1小时、24小时、7天切换后时间轴和点数符合固定步长。
7. firing/pending 告警真实展示,告警恢复后不再出现在活动列表。
8. Prometheus/Exporter 只监听本机,公网不能连接 9090/9100。
9. 页面在桌面和移动宽度下无重叠、截断和横向溢出,控制台无相关错误。
10. TypeScript、API专项测试、生产构建、配置校验和 `git diff --check`通过。
## 9. 服务内部指标扩展(V1.1)
### 9.1 采集边界
- API使用独立回环端口`127.0.0.1:9464/metrics`,采集进程内存、堆内存、事件循环P99、请求量、状态码和延迟直方图。
- Gateway在已有回环控制端口`127.0.0.1:8090/metrics`暴露Go运行时、上下游连接总数、Submit成败和耗时、Redis Stream pending/lag/最旧年龄。
- PostgreSQL、Redis、Nginx使用发行版ExporterMinIO使用原生Prometheus端点。全部Exporter只监听回环地址。
- 指标标签只允许方法、路由模板、HTTP状态、结果类别和固定服务名。禁止手机号、短信ID、CMPP Msg_Id、任务ID、通道凭据、短信正文、原始URL和SQL文本进入标签。
- 监控页只查询`cmpp:service_*`固定Recording Rules,不为每张卡片执行一条高代价PromQL。
### 9.2 默认阈值
| 领域 | Warning | Critical | 持续窗口 |
|---|---:|---:|---:|
| CPU | >80% | >90% | 10m / 5m |
| 内存 | >85% | >95% | 10m / 5m |
| 磁盘或inode | >80% | >90% | 15m / 5m |
| API 5xx | >1%且至少5次 | >5%且至少5次 | 5m |
| API P95 | >1s | >3s | 10m / 5m |
| API事件循环P99 | >200ms | >1s | 10m / 5m |
| Gateway提交队列最旧pending | >30s | >120s | 2m |
| Gateway实际上游连接 | — | 少于期朖2m | 2m |
| PostgreSQL连接使用率 | >70% | >85% | 10m / 5m |
| PostgreSQL死锁 | 15m内>0 | 15m内重复出现 | 1m |
| Redis内存/maxmemory | >70% | >85% | 10m / 5m |
| Redis淘汰或拒绝连接 | — | 5m内>0 | 1m |
| 任一核心Exporter失联 | — | >2m | 2m |
| Prometheus采集耗时/周期 | >80% | >100% | 5m |
| Prometheus规则计算失败 | — | 5m内>0 | 1m |
比例告警必须带最低样本量,不得把单次失败误报为100%错误率。短信最终回执可由运营商延迟数小时,不纳入Gateway基础设施短窗口Critical,仍由72小时业务终结机制和短信质量看板处理。
### 9.3 告警收敛与校准
- 主机或Node Exporter失联时,抑制其CPU、内存和磁盘派生告警;API/Gateway指标端点失联时,抑制对应延迟和错误率告警。
- Critical表示需立即处理;Warning表示当日检查;趋势指标未达到可操作条件时只展示,不生成告警。
- 上线后保留7至14天基线,核对业务高峰P95/P99、正常连接数和队列年龄。阈值调整必须同步修改设计、用例、规则和进度记录。
### 9.4 性能预算
- 全局采集周期保持15秒,无排障需求不降到1秒。
- API和Gateway请求路径只做内存计数、有界直方图和原子计数,不在业务请求中写PostgreSQL或Redis。
- PostgreSQL Exporter只使用发行版默认低代价查询,不采集SQL原文或扫描业务大表。
- 时序仍受30天和8GB双重上限约束;时序增长时先缩短实际保留期,不允许无界占满业务盘。
## 10. 阈值配置与全局预警入口(2026-08-14 增补)
- 可配置范围固定为主机 CPU/内存/根磁盘、API 5xx/P95/事件循环、Gateway 最旧 pending、PostgreSQL 连接、Redis 内存和 MinIO 容量十组警告/严重数值。PromQL、持续窗口、标签和规则文件路径仍由代码固定,浏览器无权提交。
- PostgreSQL 单例记录同时保存期望阈值、生效阈值、配置版本、生效版本和 `applying/effective/failed` 状态。更新使用版本条件认领,防止多个 API 实例并发覆盖;规则先经 promtool 校验,再在同一目录原子替换并调用仅回环开放的 `/-/reload`。失败恢复旧文件并保留旧生效版本。
- 安装器把可配置规则从基础规则中剥离,托管文件归 `cmpp-api:prometheus` 且权限为 0640Prometheus 仍只监听回环。右上角铃铛只轮询轻量活动告警汇总接口,不重复加载趋势或服务指标。
## 11. 活动告警已读语义(2026-08-16 增补)
- “已读”只表示某位管理员已查看某一次 Prometheus 活动告警,不是 resolve、silence 或 acknowledge 外部告警管理器;页面活动告警总数与平台健康状态仍按 Prometheus 原始 firing/pending 计算。
- 指纹由排序后的 Prometheus labels 稳定生成,`activeAt`区分同一指纹的不同触发周期。数据库以`fingerprint + userId`唯一,upsert同时更新`activeAt/readAt`;读取时只有数据库 activeAt 与当前 Prometheus activeAt 相同才算已读。
- 标记前必须回读当前 Prometheus 告警并校验指纹和 activeAt,防止客户端伪造或把已经恢复的新周期误标已读。预警中心轻量汇总只扣减当前管理员本次已读项;数据库故障不得用 localStorage 或静态状态替代。
+67 -2
View File
@@ -4567,6 +4567,7 @@ npm run verify:phase8
| TC-SIGNATURE-RETIREMENT-027 | 在抑制管理点击“取消抑制”,填写或不填写原因 | 只出现平台自研弹窗;未填原因不能确认,填写后调用真实取消接口并刷新消息及抑制列表,不出现浏览器`prompt/confirm` | | TC-SIGNATURE-RETIREMENT-027 | 在抑制管理点击“取消抑制”,填写或不填写原因 | 只出现平台自研弹窗;未填原因不能确认,填写后调用真实取消接口并刷新消息及抑制列表,不出现浏览器`prompt/confirm` |
| TC-REPORT-RECORD-LAYOUT-001 | 在报备记录页面查看长备注和短备注 | 备注列桌面宽度不小于320px,使用统一长文本换行样式;宽表允许内部横向滚动,备注不被其他固定列挤成窄竖列,详情仍展示全文 | | TC-REPORT-RECORD-LAYOUT-001 | 在报备记录页面查看长备注和短备注 | 备注列桌面宽度不小于320px,使用统一长文本换行样式;宽表允许内部横向滚动,备注不被其他固定列挤成窄竖列,详情仍展示全文 |
| TC-DEPLOY-HEALTH-001 | 发布重启后模拟API初始化超过3秒但在60秒内恢复,并分别模拟API或Gateway持续60秒不可用 | 前者由部署脚本逐秒重试并正常完成,不触发误回滚;后者在60秒后明确失败并保留发布前数据库、源码和环境恢复资产,不把端口尚未就绪当作构建或migration失败 | | TC-DEPLOY-HEALTH-001 | 发布重启后模拟API初始化超过3秒但在60秒内恢复,并分别模拟API或Gateway持续60秒不可用 | 前者由部署脚本逐秒重试并正常完成,不触发误回滚;后者在60秒后明确失败并保留发布前数据库、源码和环境恢复资产,不把端口尚未就绪当作构建或migration失败 |
| TC-DEPLOY-NGINX-001 | 分别在Ubuntu默认`nginx.conf`已有全局`gzip on`和完全没有gzip配置的环境执行两次标准发布 | 已有配置时平台生成文件保持为空并复用发行版配置;缺失时写入平台配置;两种环境连续执行两次`nginx -t`均通过且不存在重复gzip指令 |
### 2026-08-10 本地执行状态 ### 2026-08-10 本地执行状态
@@ -4588,11 +4589,11 @@ npm run verify:phase8
| 编号 | 场景 | 预期 | | 编号 | 场景 | 预期 |
|---|---|---| |---|---|---|
| TC-DOWNSTREAM-REQUEUE-TASK-001 | 当前筛选条件预检 | 后端按企业、应用、类型、状态、日期、关键词和 `snapshotAt` 返回真实命中、可重投、跳过及状态分布;分页不影响数量。 | | TC-DOWNSTREAM-REQUEUE-TASK-001 | 当前筛选条件预检 | 后端按企业、应用、类型、状态、日期、关键词和 `snapshotAt` 返回真实命中、可重投、跳过及状态分布;分页不影响数量。 |
| TC-DOWNSTREAM-REQUEUE-TASK-002 | 创建任务 | 原因少于 5 字拒绝;物化 `pending/failed/unconfirmed/rejected``delivered/awaiting_ack` 不进入执行。 | | TC-DOWNSTREAM-REQUEUE-TASK-002 | 创建任务 | 原因少于 5 字拒绝;物化 `pending/failed/unconfirmed/rejected/delivered`;客户端已确认记录在预检中计为可重投并保留 `previousStatus=delivered``awaiting_ack` 不进入执行。 |
| TC-DOWNSTREAM-REQUEUE-TASK-003 | 快照边界 | 创建任务后新增或筛选条件外记录不进入任务。 | | TC-DOWNSTREAM-REQUEUE-TASK-003 | 快照边界 | 创建任务后新增或筛选条件外记录不进入任务。 |
| TC-DOWNSTREAM-REQUEUE-TASK-004 | 并发与幂等 | `taskId+deliveryId` 唯一;重复扫描、API 重启及并发任务不会重复调用 Gateway。 | | TC-DOWNSTREAM-REQUEUE-TASK-004 | 并发与幂等 | `taskId+deliveryId` 唯一;重复扫描、API 重启及并发任务不会重复调用 Gateway。 |
| TC-DOWNSTREAM-REQUEUE-TASK-005 | ACK 闭环 | Gateway 写出后项目进入等待 ACK;`Result=0` 成功,拒绝/超时计失败并按阈值自动暂停。 | | TC-DOWNSTREAM-REQUEUE-TASK-005 | ACK 闭环 | Gateway 写出后项目进入等待 ACK;`Result=0` 成功,拒绝/超时计失败并按阈值自动暂停。 |
| TC-DOWNSTREAM-REQUEUE-TASK-006 | 状态变化跳过 | 执行前状态变化、已确认或被其他操作认领时不调用 Gateway,记录明确跳过原因。 | | TC-DOWNSTREAM-REQUEUE-TASK-006 | 状态变化跳过 | 创建任务时不是 `delivered`、执行前才收到成功 ACK 的记录,或已被其他操作认领时不调用 Gateway,记录明确跳过原因;快照原状态就是 `delivered` 的记录允许调用 Gateway。 |
| TC-DOWNSTREAM-REQUEUE-TASK-007 | 任务控制 | 待执行/执行中任务可暂停、继续、终止;终止不撤回已写出消息。 | | TC-DOWNSTREAM-REQUEUE-TASK-007 | 任务控制 | 待执行/执行中任务可暂停、继续、终止;终止不撤回已写出消息。 |
| TC-DOWNSTREAM-REQUEUE-TASK-008 | 审计 | 创建、暂停、继续、终止和自动暂停记录操作人、筛选快照、原因和结果。 | | TC-DOWNSTREAM-REQUEUE-TASK-008 | 审计 | 创建、暂停、继续、终止和自动暂停记录操作人、筛选快照、原因和结果。 |
| TC-DOWNSTREAM-PAGE-SIZE-001 | 分页数量 | 可选 10/25/50;切换回第一页,后端返回对应条数,总数和筛选条件保持一致。 | | TC-DOWNSTREAM-PAGE-SIZE-001 | 分页数量 | 可选 10/25/50;切换回第一页,后端返回对应条数,总数和筛选条件保持一致。 |
@@ -4610,6 +4611,8 @@ npm run verify:phase8
| TC-DOWNSTREAM-REQUEUE-TASK-015 | 列表完整分页 | 任务列表支持状态和分页,第 11 条以后可访问,中文状态、创建人、原因、进度和各结果数准确。 | | TC-DOWNSTREAM-REQUEUE-TASK-015 | 列表完整分页 | 任务列表支持状态和分页,第 11 条以后可访问,中文状态、创建人、原因、进度和各结果数准确。 |
| TC-DOWNSTREAM-REQUEUE-TASK-016 | 完整任务项查询 | 任务项支持分页、结果及关键词查询,等待连接/外部 ACK/本任务 ACK、跳过、失败和未处理均中文展示并保留原因。 | | TC-DOWNSTREAM-REQUEUE-TASK-016 | 完整任务项查询 | 任务项支持分页、结果及关键词查询,等待连接/外部 ACK/本任务 ACK、跳过、失败和未处理均中文展示并保留原因。 |
| TC-DOWNSTREAM-REQUEUE-TASK-017 | 终止并发边界 | 终止后未认领和等待连接项置为未处理;处理中或已写出项不撤回;执行器不再认领新项。 | | TC-DOWNSTREAM-REQUEUE-TASK-017 | 终止并发边界 | 终止后未认领和等待连接项置为未处理;处理中或已写出项不撤回;执行器不再认领新项。 |
| TC-DOWNSTREAM-REQUEUE-TASK-018 | 已确认记录批量重投 | 以状态 `delivered` 预检并创建任务,核对任务项原状态后执行;页面显示重复投递风险,真实任务项进入等待 ACK/成功闭环;同一任务已成功项不重复调用 Gateway。 |
| TC-DOWNSTREAM-REQUEUE-TASK-019 | 创建弹窗与列表留白 | 桌面及窄屏打开创建弹窗和后台任务列表,输入不足5字及合法原因 | 原因使用统一多行输入组件,必填、错误、说明、字数和焦点态清晰;任务列表与卡片边缘保持设计间距,行内容不贴边、不裁切,移动端留白同步收敛。 |
# 2026-08-13 HTTP 与 Gateway 报文容量专项用例 # 2026-08-13 HTTP 与 Gateway 报文容量专项用例
| 用例编号 | 优先级 | 验证内容 | 预期结果 | | 用例编号 | 优先级 | 验证内容 | 预期结果 |
@@ -4628,3 +4631,65 @@ npm run verify:phase8
| TC-DASHBOARD-SIGNATURE-REMOVE-001 | 删除看板签名统计模块并统一金额样式 | 打开运营看板,检查指标卡与后续模块 | 两个“今日签名发送统计”模块均不存在;今日消费金额与今日发送总量主数字字号、字重和颜色一致;今日活跃签名仍来自真实接口 | | TC-DASHBOARD-SIGNATURE-REMOVE-001 | 删除看板签名统计模块并统一金额样式 | 打开运营看板,检查指标卡与后续模块 | 两个“今日签名发送统计”模块均不存在;今日消费金额与今日发送总量主数字字号、字重和颜色一致;今日活跃签名仍来自真实接口 |
| TC-ENTERPRISE-APP-WIDTH-001 | 企业应用关键列缩窄 | 在桌面大屏打开企业应用管理并读取表头列宽 | 状态、到达率、单价列宽均约为原宽度80%,字段与操作均未隐藏,横向滚动需求减少 | | TC-ENTERPRISE-APP-WIDTH-001 | 企业应用关键列缩窄 | 在桌面大屏打开企业应用管理并读取表头列宽 | 状态、到达率、单价列宽均约为原宽度80%,字段与操作均未隐藏,横向滚动需求减少 |
| TC-UI-CARRIER-TAG-002 | 全局运营商数据展示复用通用标签 | 抽查通道/通道组、监控、报备、企业签名、短信审核、批次号码、发送记录和客户端发送详情 | 移动、联通、电信均使用全局低饱和胶囊及统一色值;有运营商集合的三网通道显示三个标签,历史通道级字段显示中性“三网”;筛选选项、图表图例与导出文本保持纯文本 | | TC-UI-CARRIER-TAG-002 | 全局运营商数据展示复用通用标签 | 抽查通道/通道组、监控、报备、企业签名、短信审核、批次号码、发送记录和客户端发送详情 | 移动、联通、电信均使用全局低饱和胶囊及统一色值;有运营商集合的三网通道显示三个标签,历史通道级字段显示中性“三网”;筛选选项、图表图例与导出文本保持纯文本 |
## Prometheus 系统监控专项用例(2026-08-14)
| 用例编号 | 场景 | 操作 | 预期结果 |
|---|---|---|---|
| TC-INFRA-MON-001 | 运营端权限与原生页面 | 登录运营端,打开“系统管理 → 系统监控”,检查页面和浏览器网络请求 | 页面使用平台导航、组件和样式;只请求平台`/api/admin/infrastructure-monitoring/overview`,不加载iframe,不从浏览器访问9090/9100,不出现第三方Logo或登录页 |
| TC-INFRA-MON-002 | 未登录访问 | 清除运营端Session后直接访问系统监控API和页面 | API按现有Session中间件拒绝,页面进入运营端登录流程;Prometheus数据不得绕过运营端权限公开 |
| TC-INFRA-MON-003 | 当前硬件指标真实值 | 在同一采样窗口分别请求平台监控API和Prometheus固定查询 | CPU、内存、根文件系统、网络、负载、运行时长与Prometheus结果在采样误差内一致,响应不包含Prometheus地址或PromQL |
| TC-INFRA-MON-004 | 指标缺失 | 临时禁用某个Node Exporter采集器或查询一个不存在的指标后请求页面 | 对应指标为`null`并显示“暂无真实指标”,其他指标继续展示;不得显示0或生成趋势线 |
| TC-INFRA-MON-005 | Prometheus不可用 | 停止本地测试Prometheus或将测试环境指向拒绝连接端口,请求监控API | API返回`available=false`和安全错误摘要,页面显示“监控数据不可用”并清空陈旧指标,不显示Mock或上次数据 |
| TC-INFRA-MON-006 | 查询超时 | 让测试Prometheus响应超过5秒 | 请求被AbortSignal终止,接口有限时间返回不可用状态;API进程不积累悬挂请求 |
| TC-INFRA-MON-007 | 时间范围白名单 | 分别请求`1h``24h``7d``30d`和注入PromQL字符串 | 前三种成功且step分别为60/300/1800秒;非法值返回400,不能进入Prometheus查询 |
| TC-INFRA-MON-008 | 趋势切换 | 页面依次选择近1小时、近24小时、近7天 | 每次只发一个新范围请求;图表时间轴、点数和当前范围同步更新,切换期间防止重复触发 |
| TC-INFRA-MON-009 | 自动与手动刷新 | 保持页面可见超过30秒,再隐藏页面并点击手动刷新 | 可见时按30秒刷新;隐藏后停止;重新可见后立即刷新;手动刷新保留范围且不会并发重复请求 |
| TC-INFRA-MON-010 | 核心服务状态 | 核对`cmpp-api``cmpp-gateway`、PostgreSQL、Redis、MinIO、Nginx的systemd状态与页面 | active显示正常,明确0显示异常,指标不存在显示未知;Redis兼容`redis.service``redis-server.service`别名 |
| TC-INFRA-MON-011 | 活动告警 | 触发一条warning和一条critical测试规则并等待Prometheus进入firing | 页面显示真实名称、严重性、开始时间、持续时间、当前值和阈值;概览计数准确,恢复后活动列表移除 |
| TC-INFRA-MON-012 | 综合状态 | 分别构造无告警、warning、critical和Prometheus不可用状态 | 综合状态依次为正常、警告、严重、未知;critical优先于warning,不按前端瞬时指标重复计算 |
| TC-INFRA-MON-013 | 并行查询与响应上限 | 检查后端请求时序,并用7天范围请求最大趋势 | 瞬时、趋势、服务、告警查询并行;每序列约不超过340点,响应不包含原始Prometheus响应体 |
| TC-INFRA-MON-014 | 监听与公网暴露 | 在服务器执行`ss -lnt`并从外部探测9090、9100 | Prometheus和Node Exporter仅监听127.0.0.1或明确内网地址;公网9090/9100不可连接,运营端仍能经平台API读取指标 |
| TC-INFRA-MON-015 | 响应式与无障碍 | 在1536×1024、1280×800和390×844打开页面,操作范围和刷新按钮 | 桌面信息层级符合设计稿;窄屏无内容重叠和页面横向溢出;按钮有可读名称,活动范围和告警严重性不只依赖颜色表达 |
| TC-INFRA-MON-016 | 配置和部署幂等 | 在测试服务器重复执行监控安装脚本和配置校验 | 不重复创建系统用户,不开放公网端口;配置通过`promtool check config/rules`,服务保持activeCMPP API/Gateway不因安装被重启 |
| TC-INFRA-MON-017 | 业务数据隔离 | 运行监控24小时并检查PostgreSQL业务库和指标标签 | 监控时序只保存在Prometheus TSDB,业务PostgreSQL无高频指标写入;标签、日志和API响应不含手机号、短信正文、账号或密钥 |
| TC-INFRA-MON-018 | systemd PromQL转义兼容 | 使用真实Prometheus执行API生成的服务状态查询,并检查自动化请求参数 | 查询文本向Prometheus传递双反斜杠转义的`\\.`正则,API返回`available=true`及真实服务状态;不得因`unknown escape sequence`把整页降级 |
| TC-INFRA-MON-019 | 页面标题去重 | 打开系统监控页并检查平台页头和内容区 | 平台通用页头保留“系统监控”,内容区不再出现重复大号标题;说明、状态、范围和刷新操作完整可用 |
| TC-INFRA-MON-020 | API内部指标 | 回环请求API metrics,再发起成功与失败的固定路由请求 | 请求量、状态码、延迟桶、堆内存和事件循环指标变化;route为路由模板,不含实体ID或查询串 |
| TC-INFRA-MON-021 | Gateway内部指标 | 请求`127.0.0.1:8090/metrics`,交叉核对连接池和Redis Stream | 上下游连接、Submit计数/耗时、worker up、pending、lag和最旧pending年龄与真实状态一致 |
| TC-INFRA-MON-022 | 服务Exporter目标 | 安装PostgreSQL、Redis、Nginx Exporter并开启MinIO原生指标 | Prometheus六个新服务target均up;数据库、Redis、MinIO、Nginx指标与各服务本地命令在采样误差内一致 |
| TC-INFRA-MON-023 | Exporter端口隔离 | 执行`ss -lnt`并从LAN/公网探测9464、9187、9121、9113、9090、9100 | 全部只监听127.0.0.1或::1Nginx业务站点不代理metrics端点 |
| TC-INFRA-MON-024 | 阈值持续窗口 | 在隔离节点分别制造瞬时和持续的CPU/API错误/队列延迟 | 瞬时尖峰不告警;达到阈值与`for`窗口后进入pending/firing,恢复后移除 |
| TC-INFRA-MON-025 | 低流量错误率保护 | 5分钟内只产生1次API请求且返回500 | 因未达至5次错误的最低样本量,不产生5xx比例告警 |
| TC-INFRA-MON-026 | 高基数和敏感字段防护 | 检查API/Gateway/Exporter全量metrics文本及Prometheus label names/values | 不存在手机号、短信正文、message/submit/task/channel实体ID、凭据、原始URL或SQL文本 |
| TC-INFRA-MON-027 | Recording Rules查询收敛 | 刷新系统监控页并检查Prometheus请求 | 服务卡片只读取`cmpp:service_*`固定聚合,不按卡片开放任意PromQL;缺失指标显示“待采集” |
| TC-INFRA-MON-028 | 监控开销对比 | 在同等请求压力下对比开启前后API/Gateway CPU、RSS、P95和吞吐 | 无高基数增长、无业务PostgreSQL高频写入;开销超出预算时暂停发布并调整采集/桶配置 |
| TC-GLOBAL-ALERT-001 | 铃铛分域预警菜单 | 准备签名清退未读消息和安全待处置告警后点击右上角铃铛 | 弹层分开显示“签名清退预警”和“安全检测与封禁”,分别展示真实数量和摘要,角标等于两项之和 |
| TC-GLOBAL-ALERT-002 | 预警菜单跳转 | 分别点击铃铛中的两个菜单项 | 签名项跳转`/admin/signature-retirement`,安全项跳转`/admin/security-detection`,弹层关闭且对应页面读取真实后端数据 |
| TC-GLOBAL-ALERT-003 | 域间故障隔离与轻量轮询 | 分别让一个汇总接口失败并观察30秒轮询请求 | 失败域显示0且另一域数据保留;安全预警使用专用汇总接口,不调用完整overview、规则、代理状态或告警大列表 |
| TC-DEPLOY-NET-001 | API回环监听边界 | 使用标准生产环境启动API,执行`ss -lnt`并从LAN/Tailscale探测3000端口,同时经Nginx业务入口请求健康接口 | API仅监听`127.0.0.1:3000`,外部不能直连3000;Nginx入口仍正常返回真实API健康结果;部署静态门禁校验`API_HOST`默认值与启动参数一致 |
## Fail2ban 安全检测与人工封禁测试矩阵(2026-08-14)
- 本模块必须执行 `docs/fail2ban-assisted-blocking-test-cases-20260814.md` 中 TC-F2B 全量用例,专项用例是本平台功能测试的组成部分,不是可选附录。
- P0 门禁至少覆盖:九类规则真实 PostgreSQL 默认值与版本冲突、阈值边界、规则应用失败保留旧生效值、登录/HTTP/CMPP/SSH/Nginx 真实事件脱敏、事件键幂等、窗口聚合并发、可信代理 IP、Cloudflare 与直连入口执行器映射、系统和人工保护网段、近期重新认证、重复封禁原子认领、代理超时/失败、真实执行器回读、非 root NestJS 及任意命令/参数注入拒绝。
- 集成验收必须在隔离测试节点或网络 namespace 使用文档保留 IP;不得封禁预生产运维出口、Cloudflare 节点或真实客户 IP。未安装真实 Fail2ban/nftables/Nginx 资产时,只能把相关用例标记阻塞,不得用 Mock 通过代替。
- UI 验收覆盖桌面与窄屏的总览、告警、规则、封禁记录、保护名单、加载、空数据、失败和规则未生效状态;所有数字与操作结果必须能从 API、数据库、agent 与执行器证据交叉验证。
- `TC-F2B-OPS-008`:在非默认`APP_DIR`构建安全代理后执行安装器,核对systemd `ExecStart`与Fail2ban `actionban`均指向同一个真实可执行的`$APP_DIR/dist/cmpp-security-agent`;任一文件残留占位符、旧`current/bin`路径或目标不可执行时,安装/发布必须失败。
## 系统监控阈值与预警中心增量用例(2026-08-14)
| 用例ID | 场景 | 步骤 | 预期 |
| --- | --- | --- | --- |
| TC-INFRA-MON-029 | 模块顺序与技术说明 | 打开系统监控并检查标题和模块顺序 | 说明明确写明 Prometheus;服务关键指标紧邻活动告警上方,活动告警锚点可定位 |
| TC-INFRA-MON-030 | 阈值真实读取 | 打开阈值设置并核对 API 与数据库 | 十组固定指标来自 `InfrastructureAlertSetting`,不使用 Mock/localStorage,不允许编辑 PromQL |
| TC-INFRA-MON-031 | 阈值边界 | 提交警告≥严重、越界、缺项和未知指标 | API 返回 400,数据库版本和 Prometheus 规则均不改变 |
| TC-INFRA-MON-032 | 并发版本 | 两个会话以同一版本先后保存 | 仅第一个原子认领成功,后者返回 409 并提示刷新 |
| TC-INFRA-MON-033 | 规则校验与热加载 | 保存合法阈值,检查 promtool、规则文件、reload 与数据库 | 先校验再同目录原子替换,reload 成功后生效版本前进且写操作日志 |
| TC-INFRA-MON-034 | 应用失败回滚 | 令 promtool 或 reload 失败后保存 | 状态为 failed、展示原因,旧规则文件与旧生效阈值保留,不误报已生效 |
| TC-GLOBAL-ALERT-004 | 系统监控预警入口 | 准备隔离 QA Prometheus firing 告警并点击铃铛 | 第三项显示真实总数/严重数,角标计入三域总和,点击跳转系统监控活动告警区 |
| TC-SECURITY-UI-001 | Fail2ban 标识与标题规范 | 打开安全检测与封禁 | 不出现重复大号页面标题,说明明确写明使用 Fail2ban,字号遵循通用菜单标题 |
| TC-INFRA-MON-035 | 阈值弹窗单层滚动 | 在桌面和窄屏打开阈值设置,滚动到最后一组阈值 | 只有 Modal 外层内容区出现滚动条,阈值表单容器不产生第二层滚动或滚动陷阱,页头和页脚行为正常 |
| TC-INFRA-MON-036 | 活动告警逐条已读 | 使用管理员A点击一条当前活动告警的“标记已读” | PostgreSQL新增/更新管理员A与该次 activeAt 的记录;行显示“已读”,活动告警总数不变,铃铛系统监控数量减少1 |
| TC-INFRA-MON-037 | 已读用户隔离 | 管理员A标记已读后由管理员B查看同一告警 | 管理员B仍显示未读且铃铛数量不减少,管理员A的状态保持已读 |
| TC-INFRA-MON-038 | 同告警重新触发 | 标记已读后让告警恢复,再以相同标签重新触发并产生新 activeAt | 新触发记录重新显示“标记已读”,计入预警中心;旧 activeAt 不会永久屏蔽同指纹告警 |
| TC-INFRA-MON-039 | 过期与幂等 | 重复提交同一活动告警,再提交已恢复或 activeAt 不匹配的请求 | 同一次告警重复提交幂等;过期/不匹配请求返回404且不生成虚假已读记录;操作日志可追溯 |
+114
View File
@@ -3573,3 +3573,117 @@ git diff --check
- API、Gateway、Nginx、PostgreSQL、Redis与MinIO均active,内部API/Gateway/MinIO健康、Redis PONG`gateway.submit.commands`消费者1、`pending=0``lag=0`。公网运营端、客户端和API health均HTTP 200,公网CMPP 17890 TCP连通;发布时间窗API/Gateway warning级日志为0。 - API、Gateway、Nginx、PostgreSQL、Redis与MinIO均active,内部API/Gateway/MinIO健康、Redis PONG`gateway.submit.commands`消费者1、`pending=0``lag=0`。公网运营端、客户端和API health均HTTP 200,公网CMPP 17890 TCP连通;发布时间窗API/Gateway warning级日志为0。
- 首次前台SSH执行因本地等待窗口关闭而收到终止信号,包装流程按设计恢复PostgreSQL和原运行目录,运行标识回到`4c70978d`、migration回到86条、服务全部active;确认恢复资产完整后改为服务器后台日志方式重新执行并成功,未并发重复部署。 - 首次前台SSH执行因本地等待窗口关闭而收到终止信号,包装流程按设计恢复PostgreSQL和原运行目录,运行标识回到`4c70978d`、migration回到86条、服务全部active;确认恢复资产完整后改为服务器后台日志方式重新执行并成功,未并发重复部署。
- 本轮没有创建真实重投任务、调用真实Gateway重投、发送短信、修改通道配置、余额或客户连接。既有`api/tsconfig.build.tsbuildinfo``tsconfig.tsbuildinfo``outputs/`和空文件`=`继续保护,不归入业务提交。 - 本轮没有创建真实重投任务、调用真实Gateway重投、发送短信、修改通道配置、余额或客户连接。既有`api/tsconfig.build.tsbuildinfo``tsconfig.tsbuildinfo``outputs/`和空文件`=`继续保护,不归入业务提交。
# 2026-08-14 Prometheus 系统监控(本地未提交、未发布)
- 运营端“系统管理”新增独立“系统监控”页面,保留原“发送监控”业务职责。页面使用平台React、ECharts和通用组件原生实现,不嵌入Grafana、Prometheus或第三方iframe;支持近1小时、24小时、7天,展示CPU、内存、根磁盘、网络、负载、运行时长、六类核心systemd服务和Prometheus活动告警。
- 新增只读`GET /api/admin/infrastructure-monitoring/overview`。后端只接受`1h/24h/7d`白名单,按固定PromQL并行访问Prometheus5秒超时;浏览器不能提交PromQL或访问9090/9100。缺失序列返回`null`Prometheus异常返回`available=false`并清空指标和趋势,不用0、Mock、静态数据或localStorage伪装。
- 新增Debian/Ubuntu幂等安装脚本、Prometheus抓取配置和告警规则。Prometheus与Node Exporter只监听`127.0.0.1`,默认保留30天且限制8GB;脚本备份既有配置和override、运行`promtool`校验,只重启两个监控服务,不重启API、Gateway或其他业务服务。本轮未在预生产执行脚本。
- 专项Jest 1套/4项、API全量38套/467项通过,覆盖范围白名单、Prometheus HTTP响应契约解析、固定60秒step、服务别名、活动告警、不可用无陈旧数据及监控地址安全约束;全量Jest仍因仓库既有异步句柄使用`--forceExit`收尾。API正式TypeScript、前端TypeScript、Vite 8.1.5生产构建、两个Shell脚本语法及`git diff --check`通过;Vite仅保留既有约2.10MB单chunk提示。
- 浏览器优先检查现有页面:本地运营端无有效登录Session,被正常引导到带算术验证码的登录页;未绕过登录、未读取或填写验证码,因此登录后桌面与窄屏视觉验收尚未完成。真实Prometheus/Node Exporter集成和告警触发验收必须在明确授权安装的测试或预生产窗口执行,当前不以单测替代真实基础设施验收。
- 本轮代码、配置和文档保持未提交、未推送、未部署,没有发送、补发或重投短信,没有修改通道账号、密码、启停状态、企业余额、客户连接或预生产数据。既有构建缓存、`outputs/`和空文件`=`继续保护;并行会话新增的Fail2ban设计与测试文档不属于本需求,不修改、不归因。
# 2026-08-14 Fail2ban 安全检测与人工封禁第一版(本地实现完成、未发布)
- 新增真实 PostgreSQL 安全规则、事件、告警、封禁和保护网段模型及第88条 migration `20260814150000_add_security_detection`;九类默认规则覆盖运营端/客户端登录、SSH、CMPP认证与协议滥用、HTTP错误密钥/签名/重放和Nginx恶意扫描。应用事件按事件键幂等并使用PostgreSQL advisory transaction lock串行聚合;Fail2ban命中作为其自身窗口已达到阈值的聚合事件处理,不会再要求重复达到第二层阈值。
- 运营端新增“安全控制 / 安全检测与封禁”自研页面,使用真实API展示总览、告警、规则、封禁记录和保护名单;规则修改、封禁、解封、忽略和保护名单写操作要求近期重新认证。封禁时长为固定枚举,执行器由服务端按入口映射,浏览器不能传jail、action、shell或执行器参数;系统私网/回环等内置保护和人工CIDR保护在调用代理前拦截。
- 新增独立Go `cmpp-security-agent`、Unix Socket固定协议、report-only Fail2ban action/filter、Nginx real-IP deny、nftables timeout set和systemd加固。NestJS部署身份改为专用非root `cmpp-api`,安全事件入口新增独立内部令牌;agent不使用shell拼接,规则/Nginx配置校验或reload失败会恢复旧文件,只有真实执行器回读命中后数据库才标记`blocked`
- 登录、OpenAPI鉴权和Gateway已接入结构化安全事件;错误HTTP密钥只保存不可逆账号指纹,证据字段统一过滤password/secret/token/signature/access-key。反向代理地址只在TCP来源属于`TRUSTED_PROXY_IPS`时信任`X-Forwarded-For`,避免客户端伪造来源IP。
- Prisma schema validate、API与前端TypeScript、API全量40套/472项、Fail2ban专项和Gateway控制器3套/11项、Gateway全量`go test ./...`、Vite 8.1.5生产构建、3个Shell脚本语法和`git diff --check`通过;Vite仅有既有约2.11MiB单chunk提示,全量Jest仍使用`--forceExit`收尾既有异步句柄。
- 浏览器优先接管本地路由,`/admin/security-detection`在无有效Session时正确跳转运营端登录并保留返回地址,控制台error/warn为0;未读取、重置或猜测账号,登录后页面视觉与交互验收尚未完成。真实Fail2ban、Nginx、nftables、Unix Socket和第88条migration未在本机数据库或预生产安装/执行,必须在具备恢复资产和文档保留测试IP的授权发布窗口完成,当前不以Mock替代集成验收。
- 本轮未发送、补发或重投短信,未修改通道账号、密码、启停状态、企业余额、客户连接或预生产数据;`api/tsconfig.build.tsbuildinfo``tsconfig.tsbuildinfo``outputs/`和空文件`=`继续作为受保护项排除提交。
# 2026-08-14 Prometheus 与 Fail2ban 两次提交本地部署复核(未推送、未发布)
- 从侧边会话已经落到本地 `main` 的两个提交开始复核:`b78faa1aa24a89830892b511a6cb17cc5a5fbe68`Prometheus 系统监控)和 `d30d9ea4d0ec8a46154dfb1e39a9bba2156ec455`(Fail2ban 安全检测)。复核时本地 `HEAD=d30d9ea``origin/main=96e475d`,本地领先2个提交;本轮没有再次提交、推送或发布。
- migration 前已将本地真实 PostgreSQL 备份到 `C:\cmpp-platform-local\backups\cmpp-platform-before-d30d9ea-20260814-111254.dump`637967字节,SHA-256=`523ecc82b55b5575ebe78fc4d253c5ec44932a76a51209130442f619e1971089`。Prisma generate、validate、migrate deploy/status均通过;`20260814150000_add_security_detection`真实应用一次,本地数据库由87条升级为88/88条migration,九类默认安全规则均存在且配置版本为1。
- 本地API以正式构建产物运行在3000端口,前端Vite生产预览运行在4173端口,API health与前端HTTP均为200。为避免本地Gateway连接真实供应商,本轮只编译和测试Gateway,没有启动8090;因此API日志中的活动通道恢复和供应商状态对账失败是预期的本地Gateway离线结果,未修改任何通道参数,也未触发短信提交。
- API全量40套/472项、API TypeScript正式构建、前端TypeScript与Vite 8.1.5生产构建、Gateway `go test ./... -count=1``go vet ./...`全部通过;Vite仅保留既有大chunk提示。Prometheus、Fail2ban、nftables和Linux systemd在Windows本机不可用,三个部署Shell及标准发布/初始化Shell均通过Bash语法检查,但不以语法检查冒充Linux真实安装验收。
- 正式服务层连接真实本地PostgreSQL和真实不可达的 `127.0.0.1:9090` 验证:监控概览返回 `available=false`,所有指标为`null`、趋势为空,非法范围`30d`返回400;安全检测写入保留测试地址`203.0.113.10`的一条 `http_invalid_api_key` 事件,证据中的访问密钥已存为`[REDACTED]`,同一`eventKey=local-qa-d30d9ea-http-invalid-api-key`第二次上报命中幂等去重且未达到告警阈值。该真实本地测试事件保留作审计证据,没有创建封禁或调用安全代理。
- 未认证HTTP请求访问监控总览、安全总览和规则列表均返回401。经用户明确授权,只临时替换本地专用 `codex_local_admin` 的密码哈希完成图形验证码登录;登录后立即恢复原密码哈希、失败计数和锁定时间,未新建账号、未变更session版本。浏览器桌面验收确认监控页切换到近1小时并刷新后仍明确显示Prometheus不可用且没有Mock/缓存数值;安全页显示24小时事件1条、安全代理不可用,并从真实数据库展示9条规则及`1/1`版本。390×844窄屏下两页核心内容和交互可用,控制台warning/error为0;安全页五个页签在窄屏中文字换行偏碎,记为非阻断视觉问题。
- 19份既有结构门禁中11份通过、8份失败;失败集中在后台重投/异常处置API、通道/发送质量哈希、报备导出、运营商集合、定时调度、全局长文本样式和签名查询等既有契约漂移,与这两个提交新增文件无交集,本轮不顺带改写其他模块契约。`git diff --check`通过。
- 本轮没有发送、补发或重投真实短信,没有修改真实通道账号、密码、启停状态、企业余额或客户连接。既有`api/tsconfig.build.tsbuildinfo``tsconfig.tsbuildinfo``outputs/`和空文件`=`继续保护;本段测试记录是本轮新增的唯一业务工作区修改。
# 2026-08-14 Prometheus 与 Fail2ban 本地服务器部署及真实集成验收
- 经用户明确授权,将本地`HEAD=d30d9ea4d0ec8a46154dfb1e39a9bba2156ec455`及当前工作区的部署修复安装到全新Ubuntu 24.04测试节点`100.93.204.60`;未把该节点当作预生产。部署前只读盘点确认4核、7.8GiB内存、根盘约98GiB且无既有平台目录/数据库/环境文件,恢复基线位于`/opt/cmpp-platform-backups/releases/20260814-135221-before-d30d9ea-localserver`API回环修复前增量备份位于`/opt/cmpp-platform-backups/releases/20260814-144709-before-api-loopback`
- 基础源码归档SHA-256=`c9e0b4ba5895b0d434a3335dcc6044f291779337841bb19b636bbed5d91b1b96`,首轮修复overlay=`24c5720dd0da6fab1b745d3c976cfc70d1a2f1b49da13443157a0ae89eac1bad`API回环修复overlay=`3d6cdd45a87e12fcad50a10d18862a1527c6a36fae2b18391142c0fba61e717e`;运行标识写为`d30d9ea4d0ec8a46154dfb1e39a9bba2156ec455+localfix.3d6cdd45a87e`。生成的本地管理员凭据以0600权限保存在服务器`/home/hector/cmpp-platform-admin.txt`,未写入仓库或测试记录。
- 已安装Node 22.21.1、Go 1.26.0、PostgreSQL 16.14、Redis 7.0.15、Nginx 1.24、Fail2ban 1.0.2、Prometheus 2.45.3及Node Exporter 1.7。由于该LAN的DNS/代理返回不可路由的fake-IP,安装期间备份原DNS、hosts和APT源后改用清华Ubuntu镜像并为必要下载域名写入临时hosts固定解析;这些固定解析仅为安装绕行,网络恢复后应按恢复资产移除。首次部署因MinIO官方二进制跳转GitHub后不可达,依照部署文档临时使用真实服务器文件存储驱动`local`,没有用Mock或localStorage伪造对象存储;后续已按用户提供的MinIO二进制完成正式切换,证据见下方补充记录。
- 第88条migration真实应用,Prisma报告88 migrations且schema up to date`SecurityDetectionRule`真实9条,`configVersion/effectiveVersion`均为1。该全新节点的通道、短信记录、下游重投任务和安全封禁记录均为0,未连接真实供应商或客户。
- 真实Linux集成检查通过:API、Gateway、安全代理、PostgreSQL、Redis、Nginx、Prometheus、Node Exporter及Fail2ban均activeAPI/Gateway/Prometheus健康、Redis PONG、PostgreSQL ready、Nginx语法和Fail2ban配置通过。Prometheus两个target均为`up`,配置有效且告警文件12条规则通过`promtool`9090/9100仅监听127.0.0.1。安全代理Unix Socket为0660 root:cmpp-securityNestJS用户`cmpp-api`无sudosystemd与Fail2ban action共同指向`/opt/cmpp-platform/dist/cmpp-security-agent`nftables专用IPv4/IPv6 timeout set存在,Fail2ban sshd jail运行。
- 部署中发现并修复三项真实Fresh-Install缺陷:安全代理systemd与Fail2ban action旧路径漂移;Ubuntu默认gzip与发布脚本重复声明导致`nginx -t`失败;NestJS文档要求回环但代码默认监听全网卡。前两项由安装/发布静态门禁保护,第三项新增`API_HOST`并默认127.0.0.1。修复后服务器`ss`确认3000/5432/6379/8090/9090/9100均为回环,外部探测12026和17890可连、3000/9090/9100不可连,Nginx入口`/api/health`返回200。专项部署门禁、API TypeScript正式构建、Shell语法和`git diff --check`通过。
- 内置浏览器两次导航该Tailscale地址均在页面加载阶段超时;用户随后明确要求改用系统Chrome,Chrome控制扩展同样在新建页导航阶段超时,而同机PowerShell对同URL返回HTTP 200,判定为浏览器控制链路到Tailscale HTTP地址的环境阻塞。两次尝试均未到验证码或登录提交;管理员密码哈希、失败次数和锁定时间已从临时数据库备份恢复,临时备份表已删除。未绕过图形验证码,也未把登录后桌面/窄屏视觉验收伪报为通过。页面构建与真实后端/基础设施证据已通过,登录后视觉及范围切换仍需用户在本机Chrome手动打开页面,或共享已打开的具体标签页后补验。
- 本轮没有发送、补发、重投短信或创建重投任务,没有修改真实通道账号、密码、启停状态、企业余额或客户连接。工作区修复与文档尚未提交、推送;`api/tsconfig.build.tsbuildinfo``tsconfig.tsbuildinfo``outputs/`及空文件`=`继续作为受保护项,不删除、不提交、不归因。
## MinIO 正式切换补充(2026-08-14
- 用户提供`D:\迅雷下载\minio.linux-amd64.RELEASE.2025-09-07T16-13-09Z`,本地与服务器SHA-256均为`7c5bd8512c6e966455b1d198209358b2d191c77a83ab377c4073281065fb855f`;服务器`file`确认其为静态链接Linux x86-64 ELF,运行版本为`RELEASE.2025-09-07T16-13-09Z`、Go 1.24.6。
- 切换前确认`/var/lib/cmpp-platform/object-storage`文件数为0,并将环境、MinIO环境、systemd单元和本地对象存储目录备份到`/opt/cmpp-platform-backups/releases/20260814-151145-before-minio`。安装后`cmpp-minio`与API均activeAPI使用`OBJECT_STORAGE_DRIVER=minio`9000/9001只监听127.0.0.1/::1,外部探测均不可连接。
- 使用API同款MinIO Node客户端真实创建`cmpp-platform` bucket,并执行测试对象写入、读取内容比对和删除,三步均成功且测试对象已清理。API健康、Gateway、Prometheus、Node Exporter及Fail2ban继续active;未创建业务附件记录、短信、重投任务或通道连接。
# 2026-08-14 系统监控恢复、标题去重与全局预警菜单(测试服务器已部署)
- 用户在`100.93.204.60`真实页面看到系统监控整体降级。只读诊断确认Prometheus、Node Exporter、API、MinIO均active,两个采集target均为`up`,CPU、内存、磁盘、网络、负载、运行时长和systemd查询单独执行都成功;直接运行实际`InfrastructureMonitoringService`后捕获到systemd查询HTTP 400Prometheus明确返回`unknown escape sequence '.'`。根因是TypeScript字符串只向PromQL传递单反斜杠`\.`,而Prometheus字符串层要求双反斜杠后再交给RE2。
- systemd固定PromQL改为TypeScript四反斜杠字面量,实际查询文本正确传递双反斜杠;专项测试锁定请求参数。整页降级行为继续清空陈旧数据,但新增仅含错误摘要的服务端warning,不记录PromQL、地址或凭据。部署后真实服务返回`available=true`、综合状态`healthy`、核心服务6/6、1小时CPU趋势56点,API发布后无新增监控不可用warning。
- 系统监控内容区删除重复的大号`<h1>系统监控</h1>`,保留平台通用页头、说明、状态、时间范围和刷新操作。构建产物与部署源码静态核对确认重复标题不存在。
- 右上角铃铛由签名清退直接链接改为“预警中心”弹层,分开显示“签名清退预警”和“安全检测与封禁”;审核待办继续使用独立图标和菜单。签名项读取今日未读且未抑制消息数,安全项新增轻量`GET /api/admin/security-detection/notification-summary`,只统计`open/acknowledged/block_failed`总数与严重数,不轮询完整总览、规则或安全代理。任一域失败由`Promise.allSettled`独立降级,不清空另一域。
- 本地监控/安全专项2套8项、API全量40套473项、前后端TypeScript和Vite生产构建全部通过;全量Jest仍因既有异步句柄使用`--forceExit`收尾,Vite只保留既有大chunk提示,`git diff --check`通过。
- 发布归档`outputs/cmpp-monitor-alert-fix-20260814-152652.tar.gz`及服务器副本SHA-256均为`8a00a02714ced5311b25fbc3e1cfc17f3d1a38759f1499db78c52089bc0b60b7`。发布前PostgreSQL、环境和运行源码恢复资产位于`/opt/cmpp-platform-backups/releases/20260814-152719-before-monitor-alert-fix`,三项SHA校验、源码tar目录和pg_restore清单均通过;运行标识为`d30d9ea4d0ec8a46154dfb1e39a9bba2156ec455+localfix.monitor-alert.8a00a02714ce`
- 经用户明确要求,在该空白测试服务器真实PostgreSQL创建可清理的`qa-bell-alerts-*`预警验收数据:2条签名清退未读消息,关联一个`interfaceEnabled=false/status=disabled`的QA应用和两条QA签名;3条安全告警通过真实内部安全事件接口按规则阈值触发,使用文档保留IP`203.0.113.101-103`,其中critical 1条。铃铛真实汇总为2+3=5;封禁记录0、短信记录0,没有调用安全代理封禁、Gateway提交或供应商连接。
- Chrome中已存在登录后的测试服务器页面,但浏览器控制扩展在接管该标签页阶段持续超时,因此未伪报点击和响应式视觉验收通过。服务器真实服务、数据库、接口服务层、构建产物和静态契约均已验收;用户刷新页面即可查看,后续以用户截图继续视觉核对。
- 本轮没有发送、补发或重投短信,没有修改真实通道账号、密码、余额或客户连接。代码和文档尚未提交、推送;受保护的`*.tsbuildinfo``outputs/`及空文件`=`继续保留,不归因或提交。
# 2026-08-14 后台重投支持客户端已确认记录与创建/列表样式优化(本地未部署)
- 后台任务可重投状态由`pending/failed/unconfirmed/rejected`扩展为`pending/failed/unconfirmed/rejected/delivered`;状态筛选为`delivered`时,预检将客户端已确认记录计入可重投数并在真实PostgreSQL任务项保存`previousStatus=delivered``awaiting_ack`继续由预检计为不可重投且创建接口明确拒绝,避免确认窗口内并发写出。
- 执行器不是简单放开当前`delivered`状态:只有冻结快照原状态已经是`delivered`的任务项才允许再次调用Gateway;任务创建时为失败/待投递等状态、执行前才收到迟到成功ACK的项目会以“创建任务后已被客户确认”跳过。该判断用于保留运营明确选择已确认记录时的重复投递能力,同时防止其他任务范围意外扩大。
- 创建弹窗将原生无统一样式的`textarea`替换为平台`Textarea`,增加必填标识、少于5字错误、说明、200字计数、统一焦点态和重复投递风险提示。后台任务列表使用独立内容边框和圆角,桌面卡片边缘保留24px、行内保留20px,窄屏收敛为16px;标题区和分页同步使用设计间距。
- 已同步`docs/downstream-requeue-task-design-20260812.md` V1.1、平台需求和系统测试用例,新增已确认批量重投、迟到ACK保护及桌面/窄屏视觉用例。专项Jest 10/10、API全量41套/477项、前后端TypeScript、API正式编译、Vite 8.1.5生产构建(2549 modules)、SendChain R10结构契约和`git diff --check`通过;Vite仅保留既有约2.11MiB单chunk提示。Operations R2结构门禁仍因本轮开始前已有的`sendQuality`查询哈希漂移失败,与本次下游重投文件无交集,未为通过门禁改写其他会话业务契约。
- 本地真实PostgreSQL为88/88条migration且schema up to date。使用现有失败投递在单一数据库事务中临时改为`delivered`,真实验证状态白名单命中1条并成功物化`previousStatus=delivered`任务项1条,随后强制回滚;事务后原投递恢复`failed`且测试任务持久化数为0。该验证没有启动任务扫描、调用Gateway或发送短信。
- 本轮尚未部署测试服务器或预生产,因当前需求只授权修改且交接约束要求部署另行明确授权;没有创建真实重投任务、发送/补发/重投短信、修改通道账号、密码、启停状态、企业余额或客户连接。既有监控/预警工作区修改及`*.tsbuildinfo``outputs/`、空文件`=`继续保护,不归因于本次改动。
# 2026-08-14 服务内部Prometheus指标、阈值与测试机部署
- 在现有工作区上增量实施,未reset/checkout或覆盖其他会话改动。API新增独立`127.0.0.1:9464/metrics`,仅使用路由模板、HTTP方法和状态的低基数标签;Gateway回环`8090/metrics`新增Go运行时、上下游连接、Submit结果/耗时及Redis Stream pending/lag/最旧年龄。两者均只在内存计数,不写业务PostgreSQL或Redis。
- 测试机`100.93.204.60`安装Ubuntu发行版`prometheus-postgres-exporter 0.15.0``prometheus-redis-exporter 1.54.0``prometheus-nginx-exporter 1.1.0`,开启MinIO回环原生指标;Prometheus现实际采集`prometheus/node/cmpp-api/cmpp-gateway/postgresql/redis/minio/nginx`8个target,全部`up`
- 规则扩展至61条,覆盖主机资源、API 5xx/P95/事件循环、Gateway worker/连接/队列年龄、PostgreSQL连接/死锁、Redis内存/淘汰/拒绝连接、MinIO容量/离线盘、Nginx可用性和Prometheus自监控。比例告警有最低错误样本量,Warning/Critical范围不重叠;`promtool check rules/config`通过,8个规则组计算失败计数全为0,部署后无活动告警。
- Recording Rules真实返回API P95约0.048s、Gateway pending/lag均0、PostgreSQL连接使用率3%、Redis连接10/已用约1.78MB、MinIO容量使用率约20.8%/离线盘0、Nginx活跃连接4。系统监控页新增六组“服务关键指标”卡片,只读取`cmpp:service_*`聚合;缺失指标显示破折号/待采集,不以0伪造。
- 外部真实TCP探测确认仅业务端口12026可连接;3000、8090、9000、9090、9100、9113、9121、9187、9464全部从LAN不可连接。各监控进程当时CPU合计约0.4%Prometheus RSS约101MB、三个新Exporter RSS合计约57MB;各8个target抓取耗时0.0008至0.037s,明显低于15s周期,TSDB当时5790条series。
- 本地API专项2套5项、API全量41套/477项、API/前端TypeScript、Gateway全量`go test ./... -count=1``go vet ./...`通过;测试机Vite 8.0.16生产构建、API/Gateway构建和健康检查通过,仅保留既有大chunk提示。API/Gateway/Prometheus发布后warning级journal为0。
- 完整恢复资产位于`/opt/cmpp-platform-backups/releases/20260814-164653-before-service-metrics`,包含PostgreSQL、运行源码、环境/监控/Nginx/systemd配置,SHA-256和gzip校验通过。`20260814-164636-before-service-metrics`是因`pg_dump`不接受Prisma `schema` URL参数而立即停止的不完整目录,不可用于恢复;未删除以保留证据。当前运行标识为`d30d9ea4d0ec8a46154dfb1e39a9bba2156ec455+workspace.service-metrics.28575bc8a7d8.minio`
- Edge现有测试机页面会话未登录,访问系统监控被正常引导到带图形验证码的登录页;未代填或绕过验证码,因此登录后页面视觉验收留待用户使用已有测试账号查看。本轮未发送、补发或重投短信,未修改通道账号/密码/启停、余额或客户连接;代码未提交、未推送、未发布预生产。
# 2026-08-14 跨会话工作区合并复核
- 合并复核覆盖系统监控恢复与预警菜单、服务内部Prometheus指标/Exporter/阈值、Fail2ban部署路径修复,以及后台重投支持客户端已确认记录和创建/列表样式优化。Git不存在未合并文件或冲突标记;共同修改的API模块、全局布局/样式、部署脚本、平台需求、系统用例和进度记录均已逐项核对,没有发现状态口径、路由、样式选择器或部署时间线互相覆盖。
- 测试机静态包只读核对显示当前已部署“服务关键指标”,但尚未包含“重复投递风险”和已确认重投的新表单文案,证明服务指标会话使用定向发布,没有把尚未授权部署的下游重投改动意外带入测试机;两项发布记录保持一致。
- 合并后统一回归通过:API全量41套/477项、前后端TypeScript、API正式编译、Vite 8.1.5生产构建(2549 modules)、Gateway全量`go test ./... -count=1``go vet ./...`、SendChain R10、生产部署和安全代理静态契约、5个Shell脚本语法、真实本地PostgreSQL 88/88 migration状态及`git diff --check`。Vite仅保留既有约2.11MiB单chunk提示;Operations R2仍因本轮开始前已有的`sendQuality`查询哈希漂移失败,与本次合并文件无交集。
- 本次合并没有发送、补发或重投短信,没有创建后台重投任务,没有修改通道账号、密码、启停状态、企业余额或客户连接;提交范围继续排除`api/tsconfig.build.tsbuildinfo``tsconfig.tsbuildinfo``outputs/`和空文件`=`
# 2026-08-14 系统监控阈值与预警入口优化(进行中)
- 系统监控将“服务关键指标”移至活动告警正上方,标题说明明确使用 Prometheus;新增十组固定指标警告/严重阈值设置。配置真实写入 PostgreSQL,使用版本条件并发认领,promtool 校验、原子替换和回环 reload 成功后才推进生效版本,失败保留旧生效值。
- 右上角预警中心新增“系统监控告警”及数量/严重数,读取独立 Prometheus 活动告警汇总并跳转活动告警锚点;继续使用 `Promise.allSettled` 隔离签名、安全与监控域故障。安全检测页移除重复大号标题,说明明确使用 Fail2ban。
- 新增 migration `20260814173000_add_infrastructure_alert_settings`。本机 Prisma Client 生成、前后端 TypeScript 与 API 正式编译已通过;测试机恢复资产、migration、Prometheus规则校验、真实告警数据、浏览器验收和提交/部署结果待本轮后续补记。
- 首次测试机规则验收发现 Redis 未配置 `maxmemory` 时除零结果会误触发 Critical pending,立即补回 `redis_memory_max_bytes > 0` 固定保护,并补回 API 5xx 窗口至少 5 次错误的最低样本门槛;该真实验收缺陷未按通过处理。
- 第二次真实应用验证发现 API 原子 rename 后的规则文件未继承 `prometheus` 组,Prometheus reload 返回 500;数据库按设计保持旧生效版本并标记 failed。安装器将托管目录修正为 2750 SGID,确保新文件继承 `prometheus` 组,随后必须以同一真实配置重试并验证 effective。
## 发布与真实验收结果
- 功能提交 `6ccc102`、比例保护修复 `0cd0944`、规则目录权限修复 `2216d00` 已部署测试机;当前 `.deployed-commit=2216d00d511ef4daa57b44ad33cfd3e7411b913e`。API、Gateway、Prometheus、PostgreSQL、Redis、MinIO、Nginx 全部 activeAPI/Gateway 健康,发布后 API warning journal 无记录。
- 测试机已完成 89/89 migration。真实阈值应用从失败版本 2 重试到版本 3,数据库 `configVersion=3/effectiveVersion=3/applyStatus=effective`;托管目录为 2750、组 `prometheus`,原子生成文件为 0640、组 `prometheus`。promtool 验证基础 61 条、托管 20 条和 QA 2 条规则,Prometheus 8/8 targets up。
- 测试数据使用真实 Prometheus 规则 `/var/lib/cmpp-platform/monitoring/cmpp-qa-preview-alerts.yml` 创建 2 条隔离演示告警(1 Warning、1 Critical,均带 `qa_preview=true` 且文案说明不代表真实故障);轻量汇总真实返回 `count=2/criticalCount=1`,错误 Redis pending 已消失。
- 恢复资产分别为 `/opt/cmpp-platform-backups/releases/20260814-175325-before-6ccc102``/opt/cmpp-platform-backups/releases/20260814-175946-before-0cd0944``/opt/cmpp-platform-backups/releases/20260814-180307-before-2216d00`,均包含 PostgreSQL、源码、环境及监控/Nginx/systemd 配置并通过 SHA-256、pg_restore 与 gzip 校验。
- 本机监控专项 2 套 6 项、前后端 TypeScript、API 正式编译、Vite 生产构建和 git diff check 通过;API 全量回归两次分别在 120 秒和 300 秒到达执行时限,未取得完整通过证据,因此不记为通过。外部 Edge 当前停留测试机登录页,因没有现成登录会话且页面含验证码,本轮未代填或绕过验证码,登录后视觉验收由用户直接查看。
- 本轮没有发送、补发或重投短信,没有创建后台重投任务,没有修改通道账号、密码、启停状态、企业余额或客户连接;QA 数据仅为测试机 Prometheus 演示规则。
# 2026-08-16 系统监控弹窗滚动与活动告警已读(测试机已部署)
- 阈值设置表单移除自身 `max-height/overflow`,仅保留平台通用 Modal 内容区滚动,消除双层滚动区域。
- 新增逐管理员活动告警已读设计:真实 PostgreSQL 保存 `fingerprint + userId + activeAt + readAt`;服务端回读 Prometheus 校验当前触发周期后幂等 upsert。页面活动告警保持原始数量,铃铛轻量汇总只统计当前管理员未读,告警以相同标签重新触发但 activeAt 改变时重新计入未读。
- 新增 migration `20260816100000_add_infrastructure_alert_reads`、单条已读接口、操作日志及前端“标记已读”按钮。接口只接受当前仍活动且 `fingerprint + activeAt` 完全匹配的触发周期;重复请求保持原 `readAt` 且不重复写操作日志。
- 功能提交 `482f7ac1ae4c219e47aeaac8735c0584b7d120f2` 已部署测试机,当前 `.deployed-commit` 与该提交一致。发布前恢复资产位于 `/opt/cmpp-platform-backups/releases/20260816-150521-before-482f7ac`,包含 PostgreSQL、运行源码、环境及平台配置,SHA-256、`pg_restore --list` 和 gzip 校验通过。
- 测试机已完成 90/90 migrationAPI、Gateway、Prometheus、PostgreSQL、Redis、MinIO、Nginx 全部 activePrometheus 8/8 targets up,发布时间窗 API warning 日志为 0。
- 使用测试机既有隔离 QA Prometheus 规则和真实 PostgreSQL 验证:活动告警 2 条,其中 1 条标记已读后页面仍保留 2 条,当前管理员铃铛未读数降为 1,严重未读数为 1;重复标记返回相同 `readAt`,对应操作日志只有 1 条。数据库保留 1 条 QA 已读记录,便于页面同时展示“已读”和“标记已读”状态。
- 监控专项 2 套 8 项、Prisma format/generate、前后端 TypeScript、API 正式编译、Vite 生产构建和 `git diff --check` 通过;Vite 仅保留既有大 chunk 提示。本地真实 PostgreSQL 不可用,`prisma migrate deploy` 返回 Schema engine error,因此没有伪造依赖或把本地 migration 记为通过,migration 已在测试机真实 PostgreSQL 验证。
- 最终浏览器控制会话没有可接管的现有标签页,未绕过图形验证码或另行创建登录会话,因此登录后弹窗滚动和按钮视觉点击未伪报为通过;代码样式契约、真实接口、数据库、Prometheus 与测试机部署均已验收,用户刷新测试机现有登录页面即可查看。
- 本轮没有发送、补发或重投短信,没有创建后台重投任务,没有修改通道账号、密码、启停状态、企业余额或客户连接;`api/tsconfig.build.tsbuildinfo``tsconfig.tsbuildinfo``outputs/`和空文件 `=` 继续作为受保护项排除提交。
+47 -6
View File
@@ -5,13 +5,19 @@ import (
"log" "log"
"net/http" "net/http"
"os" "os"
"strconv"
"strings"
"time"
"cmpp-platform/gateway/internal/control" "cmpp-platform/gateway/internal/control"
"cmpp-platform/gateway/internal/health" "cmpp-platform/gateway/internal/health"
"cmpp-platform/gateway/internal/inbound" "cmpp-platform/gateway/internal/inbound"
platformmetrics "cmpp-platform/gateway/internal/metrics"
"cmpp-platform/gateway/internal/ratelimit" "cmpp-platform/gateway/internal/ratelimit"
"cmpp-platform/gateway/internal/submitworker" "cmpp-platform/gateway/internal/submitworker"
"cmpp-platform/gateway/internal/upstream" "cmpp-platform/gateway/internal/upstream"
"github.com/redis/go-redis/v9"
) )
func main() { func main() {
@@ -25,6 +31,7 @@ func main() {
} }
apiBaseURL := os.Getenv("API_BASE_URL") apiBaseURL := os.Getenv("API_BASE_URL")
upstreamManager := &upstream.Manager{APIBaseURL: apiBaseURL} upstreamManager := &upstream.Manager{APIBaseURL: apiBaseURL}
var worker *submitworker.Worker
channelLimiter, err := ratelimit.New(os.Getenv("REDIS_URL")) channelLimiter, err := ratelimit.New(os.Getenv("REDIS_URL"))
if err != nil { if err != nil {
log.Fatalf("gateway channel rate limiter init failed: %v", err) log.Fatalf("gateway channel rate limiter init failed: %v", err)
@@ -41,18 +48,19 @@ func main() {
go func() { go func() {
log.Printf("cmpp gateway inbound server listening on %s", cmppAddr) log.Printf("cmpp gateway inbound server listening on %s", cmppAddr)
if err := (inbound.Server{ if err := (inbound.Server{
Addr: cmppAddr, Addr: cmppAddr,
APIBaseURL: apiBaseURL, APIBaseURL: apiBaseURL,
PresenceStore: presenceStore, PresenceStore: presenceStore,
RecoveryStore: recoveryStore, RecoveryStore: recoveryStore,
GatewayInstanceID: getenv("GATEWAY_INSTANCE_ID", hostname()), GatewayInstanceID: getenv("GATEWAY_INSTANCE_ID", hostname()),
SecurityEventToken: os.Getenv("SECURITY_EVENT_TOKEN"),
}).ListenAndServe(); err != nil { }).ListenAndServe(); err != nil {
log.Fatalf("gateway inbound server stopped: %v", err) log.Fatalf("gateway inbound server stopped: %v", err)
} }
}() }()
if os.Getenv("GATEWAY_SUBMIT_WORKER_DISABLED") != "true" { if os.Getenv("GATEWAY_SUBMIT_WORKER_DISABLED") != "true" {
worker, err := submitworker.New(os.Getenv("REDIS_URL"), upstreamManager) worker, err = submitworker.New(os.Getenv("REDIS_URL"), upstreamManager)
if err != nil { if err != nil {
log.Printf("gateway submit worker init failed: %v", err) log.Printf("gateway submit worker init failed: %v", err)
} else { } else {
@@ -71,6 +79,39 @@ func main() {
mux := http.NewServeMux() mux := http.NewServeMux()
mux.Handle("/health", health.Handler()) mux.Handle("/health", health.Handler())
mux.Handle("/metrics", platformmetrics.Handler(func(ctx context.Context) platformmetrics.Snapshot {
desired, connected := upstreamManager.ConnectionCounts()
snapshot := platformmetrics.Snapshot{
UpstreamDesired: desired, UpstreamConnected: connected,
DownstreamConnected: inbound.ActiveConnectionCount(), SubmitWorkerUp: worker != nil,
}
if worker == nil || worker.Redis == nil {
return snapshot
}
pending, err := worker.Redis.XPending(ctx, worker.Stream, worker.Group).Result()
if err != nil {
return snapshot
}
snapshot.QueueAvailable = true
snapshot.QueuePending = pending.Count
groups, err := worker.Redis.XInfoGroups(ctx, worker.Stream).Result()
if err == nil {
for _, group := range groups {
if group.Name == worker.Group {
snapshot.QueueLag = group.Lag
break
}
}
}
entries, err := worker.Redis.XPendingExt(ctx, &redis.XPendingExtArgs{Stream: worker.Stream, Group: worker.Group, Start: "-", End: "+", Count: 1}).Result()
if err == nil && len(entries) > 0 {
milliseconds, parseErr := strconv.ParseInt(strings.SplitN(entries[0].ID, "-", 2)[0], 10, 64)
if parseErr == nil {
snapshot.QueueOldestAgeSeconds = max(0, time.Since(time.UnixMilli(milliseconds)).Seconds())
}
}
return snapshot
}))
control.Register(mux, control.Server{ control.Register(mux, control.Server{
APIBaseURL: apiBaseURL, APIBaseURL: apiBaseURL,
Upstream: upstreamManager, Upstream: upstreamManager,
+395
View File
@@ -0,0 +1,395 @@
// security-agent is the deliberately tiny privileged boundary for manual blocking.
// It accepts only a fixed JSON protocol over a Unix socket; it never invokes a shell
// and never accepts command, jail, action, path, or argument strings from NestJS.
package main
import (
"bufio"
"encoding/json"
"errors"
"fmt"
"log"
"net"
"net/http"
"os"
"os/exec"
"path/filepath"
"sort"
"strconv"
"strings"
"sync"
"time"
)
type request struct {
Action string `json:"action"`
OperationKey string `json:"operationKey"`
SourceIP string `json:"sourceIp"`
Executor string `json:"executor"`
DurationSeconds int `json:"durationSeconds"`
Version int `json:"version"`
Rules []rule `json:"rules"`
}
type rule struct {
Code string `json:"code"`
Enabled bool `json:"enabled"`
Threshold int `json:"threshold"`
WindowSeconds int `json:"windowSeconds"`
CooldownSeconds int `json:"cooldownSeconds"`
}
type response struct {
OK bool `json:"ok"`
Reference string `json:"reference,omitempty"`
Blocked bool `json:"blocked,omitempty"`
Active bool `json:"active,omitempty"`
Error string `json:"error,omitempty"`
}
type block struct {
OperationKey string `json:"operationKey"`
SourceIP string `json:"sourceIp"`
Executor string `json:"executor"`
ExpiresAt time.Time `json:"expiresAt"`
}
type state struct {
Blocks map[string]block `json:"blocks"`
RuleVersion int `json:"ruleVersion"`
}
var allowedDurations = map[int]bool{600: true, 3600: true, 86400: true, 604800: true}
var allowedRules = map[string]bool{"admin_login_failure": true, "client_login_failure": true, "ssh_auth_failure": true, "cmpp_auth_failure": true, "cmpp_protocol_abuse": true, "http_invalid_api_key": true, "http_signature_failure": true, "http_replay_attempt": true, "http_malicious_scan": true}
type agent struct {
mu sync.Mutex
statePath, nginxInclude, fail2banConfig string
data state
}
func main() {
if len(os.Args) == 4 && os.Args[1] == "report" {
if err := reportEvent(os.Args[2], os.Args[3]); err != nil {
log.Fatal(err)
}
return
}
socketPath := env("SECURITY_AGENT_SOCKET", "/run/cmpp-security-agent/agent.sock")
a := &agent{statePath: env("SECURITY_AGENT_STATE", "/var/lib/cmpp-security-agent/state.json"), nginxInclude: env("SECURITY_NGINX_DENY_INCLUDE", "/etc/nginx/snippets/cmpp-security-deny.conf"), fail2banConfig: env("SECURITY_FAIL2BAN_CONFIG", "/etc/fail2ban/jail.d/cmpp-platform-generated.local"), data: state{Blocks: map[string]block{}}}
if err := a.load(); err != nil {
log.Fatalf("load state: %v", err)
}
if err := os.MkdirAll(filepath.Dir(socketPath), 0750); err != nil {
log.Fatal(err)
}
_ = os.Remove(socketPath)
listener, err := net.Listen("unix", socketPath)
if err != nil {
log.Fatal(err)
}
if err := os.Chmod(socketPath, 0660); err != nil {
log.Fatal(err)
}
defer listener.Close()
log.Printf("security agent listening on %s", socketPath)
for {
connection, err := listener.Accept()
if err != nil {
log.Printf("accept: %v", err)
continue
}
go a.serve(connection)
}
}
func (a *agent) serve(connection net.Conn) {
defer connection.Close()
_ = connection.SetDeadline(time.Now().Add(5 * time.Second))
var req request
if err := json.NewDecoder(bufio.NewReader(connection)).Decode(&req); err != nil {
write(connection, response{Error: "invalid request"})
return
}
a.mu.Lock()
defer a.mu.Unlock()
a.prune()
var result response
switch req.Action {
case "block":
result = a.block(req)
case "unblock":
result = a.unblock(req)
case "status":
result = a.status(req)
case "apply_rules":
result = a.applyRules(req)
default:
result = response{Error: "unsupported action"}
}
write(connection, result)
}
func (a *agent) block(req request) response {
if net.ParseIP(req.SourceIP) == nil || !allowedDurations[req.DurationSeconds] || (req.Executor != "nftables" && req.Executor != "nginx_real_ip") || len(req.OperationKey) < 16 {
return response{Error: "invalid fixed block parameters"}
}
if existing, ok := a.data.Blocks[req.OperationKey]; ok {
return response{OK: true, Reference: req.OperationKey, Blocked: existing.ExpiresAt.After(time.Now())}
}
if req.Executor == "nftables" {
if err := nftBlock(req.SourceIP, req.DurationSeconds); err != nil {
return response{Error: err.Error()}
}
}
a.data.Blocks[req.OperationKey] = block{OperationKey: req.OperationKey, SourceIP: req.SourceIP, Executor: req.Executor, ExpiresAt: time.Now().Add(time.Duration(req.DurationSeconds) * time.Second)}
if req.Executor == "nginx_real_ip" {
if err := a.writeNginx(); err != nil {
delete(a.data.Blocks, req.OperationKey)
return response{Error: err.Error()}
}
}
if err := a.save(); err != nil {
return response{Error: err.Error()}
}
return a.status(req)
}
func (a *agent) unblock(req request) response {
if net.ParseIP(req.SourceIP) == nil || (req.Executor != "nftables" && req.Executor != "nginx_real_ip") {
return response{Error: "invalid fixed unblock parameters"}
}
if req.Executor == "nftables" {
if err := nftUnblock(req.SourceIP); err != nil {
return response{Error: err.Error()}
}
}
for key, item := range a.data.Blocks {
if item.SourceIP == req.SourceIP && item.Executor == req.Executor {
delete(a.data.Blocks, key)
}
}
if req.Executor == "nginx_real_ip" {
if err := a.writeNginx(); err != nil {
return response{Error: err.Error()}
}
}
if err := a.save(); err != nil {
return response{Error: err.Error()}
}
return response{OK: true, Reference: req.OperationKey}
}
func (a *agent) status(req request) response {
active := command("systemctl", "is-active", "--quiet", "fail2ban") == nil
if req.SourceIP == "" {
return response{OK: true, Active: active}
}
blocked := false
if req.Executor == "nftables" {
family := "blocked_ipv6"
if net.ParseIP(req.SourceIP).To4() != nil {
family = "blocked_ipv4"
}
output, err := exec.Command("nft", "list", "set", "inet", "cmpp_security", family).CombinedOutput()
if err != nil {
return response{Error: "nftables readback failed: " + strings.TrimSpace(string(output))}
}
blocked = strings.Contains(string(output), req.SourceIP+" timeout") || strings.Contains(string(output), req.SourceIP+" expires")
} else if req.Executor == "nginx_real_ip" {
content, err := os.ReadFile(a.nginxInclude)
if err != nil {
return response{Error: "nginx deny readback failed: " + err.Error()}
}
blocked = strings.Contains(string(content), "deny "+req.SourceIP+";")
} else {
return response{Error: "invalid executor"}
}
for _, item := range a.data.Blocks {
if blocked && item.SourceIP == req.SourceIP && item.Executor == req.Executor && item.ExpiresAt.After(time.Now()) {
return response{OK: true, Active: active, Blocked: true, Reference: item.OperationKey}
}
}
return response{OK: true, Active: active, Blocked: false}
}
func (a *agent) applyRules(req request) response {
if req.Version <= a.data.RuleVersion {
return response{OK: true, Reference: strconv.Itoa(a.data.RuleVersion)}
}
for _, item := range req.Rules {
if !allowedRules[item.Code] || item.Threshold < 1 || item.Threshold > 100000 || item.WindowSeconds < 10 || item.WindowSeconds > 86400 || item.CooldownSeconds < 0 || item.CooldownSeconds > 604800 {
return response{Error: "invalid fixed rule configuration"}
}
}
var ssh, scan *rule
for index := range req.Rules {
if req.Rules[index].Code == "ssh_auth_failure" {
ssh = &req.Rules[index]
}
if req.Rules[index].Code == "http_malicious_scan" {
scan = &req.Rules[index]
}
}
content := "# Generated by cmpp-security-agent. Manual changes will be overwritten.\n"
if ssh != nil {
content += jail("sshd", *ssh)
}
if scan != nil {
content += jail("cmpp-http-scan", *scan)
}
previous, previousErr := os.ReadFile(a.fail2banConfig)
if err := atomicWrite(a.fail2banConfig, []byte(content), 0640); err != nil {
return response{Error: err.Error()}
}
if err := command("fail2ban-client", "-t"); err != nil {
restore(a.fail2banConfig, previous, previousErr, 0640)
return response{Error: "fail2ban validation failed: " + err.Error()}
}
if err := command("fail2ban-client", "reload"); err != nil {
restore(a.fail2banConfig, previous, previousErr, 0640)
return response{Error: "fail2ban reload failed: " + err.Error()}
}
a.data.RuleVersion = req.Version
if err := a.save(); err != nil {
return response{Error: err.Error()}
}
return response{OK: true, Reference: strconv.Itoa(req.Version), Active: true}
}
func jail(name string, item rule) string {
enabled := "false"
if item.Enabled {
enabled = "true"
}
bantime := item.CooldownSeconds
if bantime < item.WindowSeconds {
bantime = item.WindowSeconds
}
return fmt.Sprintf("\n[%s]\nenabled = %s\nfindtime = %d\nmaxretry = %d\nbantime = %d\naction = cmpp-report-only\n", name, enabled, item.WindowSeconds, item.Threshold, bantime)
}
func nftBlock(ip string, seconds int) error {
family := "blocked_ipv6"
if net.ParseIP(ip).To4() != nil {
family = "blocked_ipv4"
}
return command("nft", "add", "element", "inet", "cmpp_security", family, fmt.Sprintf("{ %s timeout %ds }", ip, seconds))
}
func nftUnblock(ip string) error {
family := "blocked_ipv6"
if net.ParseIP(ip).To4() != nil {
family = "blocked_ipv4"
}
err := command("nft", "delete", "element", "inet", "cmpp_security", family, fmt.Sprintf("{ %s }", ip))
if err != nil && strings.Contains(err.Error(), "No such file") {
return nil
}
return err
}
func (a *agent) writeNginx() error {
ips := []string{}
for _, item := range a.data.Blocks {
if item.Executor == "nginx_real_ip" && item.ExpiresAt.After(time.Now()) {
ips = append(ips, item.SourceIP)
}
}
sort.Strings(ips)
lines := []string{"# Generated by cmpp-security-agent."}
for _, ip := range ips {
lines = append(lines, "deny "+ip+";")
}
previous, previousErr := os.ReadFile(a.nginxInclude)
if err := atomicWrite(a.nginxInclude, []byte(strings.Join(lines, "\n")+"\n"), 0640); err != nil {
return err
}
if err := command("nginx", "-t"); err != nil {
restore(a.nginxInclude, previous, previousErr, 0640)
return err
}
if err := command("systemctl", "reload", "nginx"); err != nil {
restore(a.nginxInclude, previous, previousErr, 0640)
_ = command("systemctl", "reload", "nginx")
return err
}
return nil
}
func (a *agent) prune() {
changed := false
for key, item := range a.data.Blocks {
if !item.ExpiresAt.After(time.Now()) {
delete(a.data.Blocks, key)
changed = true
}
}
if changed {
_ = a.writeNginx()
_ = a.save()
}
}
func (a *agent) load() error {
bytes, err := os.ReadFile(a.statePath)
if errors.Is(err, os.ErrNotExist) {
return nil
}
if err != nil {
return err
}
return json.Unmarshal(bytes, &a.data)
}
func (a *agent) save() error {
bytes, err := json.MarshalIndent(a.data, "", " ")
if err != nil {
return err
}
return atomicWrite(a.statePath, bytes, 0600)
}
func atomicWrite(path string, bytes []byte, mode os.FileMode) error {
if err := os.MkdirAll(filepath.Dir(path), 0750); err != nil {
return err
}
temporary := path + ".tmp"
if err := os.WriteFile(temporary, bytes, mode); err != nil {
return err
}
return os.Rename(temporary, path)
}
func restore(path string, previous []byte, previousErr error, mode os.FileMode) {
if previousErr == nil {
_ = atomicWrite(path, previous, mode)
} else if errors.Is(previousErr, os.ErrNotExist) {
_ = os.Remove(path)
}
}
func command(name string, args ...string) error {
output, err := exec.Command(name, args...).CombinedOutput()
if err != nil {
return fmt.Errorf("%s: %s", err, strings.TrimSpace(string(output)))
}
return nil
}
func write(connection net.Conn, value response) { _ = json.NewEncoder(connection).Encode(value) }
func env(name, fallback string) string {
if value := strings.TrimSpace(os.Getenv(name)); value != "" {
return value
}
return fallback
}
func reportEvent(jail, ip string) error {
ruleCode := map[string]string{"sshd": "ssh_auth_failure", "cmpp-http-scan": "http_malicious_scan"}[jail]
if ruleCode == "" || net.ParseIP(ip) == nil {
return errors.New("unsupported report event")
}
body := fmt.Sprintf(`{"ruleCode":%q,"sourceIp":%q,"protocol":%q,"resultCode":%q}`, ruleCode, ip, "fail2ban", jail)
client := &http.Client{Timeout: 3 * time.Second}
request, err := http.NewRequest(http.MethodPost, env("SECURITY_EVENT_URL", "http://127.0.0.1:3000/api/gateway/events/security-detection"), strings.NewReader(body))
if err != nil {
return err
}
request.Header.Set("Content-Type", "application/json")
request.Header.Set("X-Security-Event-Token", os.Getenv("SECURITY_EVENT_TOKEN"))
response, err := client.Do(request)
if err != nil {
return err
}
defer response.Body.Close()
if response.StatusCode < 200 || response.StatusCode >= 300 {
return fmt.Errorf("event collector returned %s", response.Status)
}
return nil
}
@@ -37,10 +37,12 @@ func (s Server) handleLogin(response *cmpp.Response, packet *cmpp.Packet, logger
} }
account := strings.TrimRight(req.SrcAddr, "\x00") account := strings.TrimRight(req.SrcAddr, "\x00")
if account == "" { if account == "" {
go s.reportSecurityEvent("cmpp_protocol_abuse", remoteIP(packet.Conn.Conn.RemoteAddr()), "EMPTY_SOURCE_ADDRESS", cmppVersionName(req.Version))
setInboundConnectResponse(response.Packer, cmpp.ErrnoConnInvalidSrcAddr, req.AuthSrc, "", req.Version) setInboundConnectResponse(response.Packer, cmpp.ErrnoConnInvalidSrcAddr, req.AuthSrc, "", req.Version)
return false, cmpp.ConnRspStatusErrMap[cmpp.ErrnoConnInvalidSrcAddr] return false, cmpp.ConnRspStatusErrMap[cmpp.ErrnoConnInvalidSrcAddr]
} }
if req.Version != cmpp.V20 && req.Version != cmpp.V21 && req.Version != cmpp.V30 { if req.Version != cmpp.V20 && req.Version != cmpp.V21 && req.Version != cmpp.V30 {
go s.reportSecurityEvent("cmpp_protocol_abuse", remoteIP(packet.Conn.Conn.RemoteAddr()), "UNSUPPORTED_VERSION", cmppVersionName(req.Version))
setInboundConnectResponse(response.Packer, cmpp.ErrnoConnVerTooHigh, req.AuthSrc, "", cmpp.V30) setInboundConnectResponse(response.Packer, cmpp.ErrnoConnVerTooHigh, req.AuthSrc, "", cmpp.V30)
return false, cmpp.ConnRspStatusErrMap[cmpp.ErrnoConnVerTooHigh] return false, cmpp.ConnRspStatusErrMap[cmpp.ErrnoConnVerTooHigh]
} }
@@ -129,3 +131,18 @@ func (s Server) authenticate(remote net.Addr, account string, authSource string,
err := s.post(context.Background(), "/gateway/events/inbound/authenticate", payload, &result) err := s.post(context.Background(), "/gateway/events/inbound/authenticate", payload, &result)
return result, err return result, err
} }
func (s Server) reportSecurityEvent(ruleCode, sourceIP, resultCode, protocol string) {
if net.ParseIP(sourceIP) == nil {
return
}
payload := map[string]any{
"ruleCode": ruleCode, "sourceIp": sourceIP, "resultCode": resultCode, "protocol": protocol,
}
var result struct {
Accepted bool `json:"accepted"`
}
if err := s.post(context.Background(), "/gateway/events/security-detection", payload, &result); err != nil {
log.Printf("security event report failed rule=%s remote=%s err=%v", ruleCode, sourceIP, err)
}
}
+1
View File
@@ -13,6 +13,7 @@ const defaultHTTPTimeout = 10 * time.Second
type Server struct { type Server struct {
Addr string Addr string
APIBaseURL string APIBaseURL string
SecurityEventToken string
HTTPClient *http.Client HTTPClient *http.Client
LogWriter io.Writer LogWriter io.Writer
PendingFlushInterval time.Duration PendingFlushInterval time.Duration
+7
View File
@@ -201,6 +201,13 @@ func onlineAccounts() []string {
return accounts return accounts
} }
// ActiveConnectionCount exposes only an aggregate gauge; account and remote-IP labels are intentionally excluded.
func ActiveConnectionCount() int {
downstreamRegistry.RLock()
defer downstreamRegistry.RUnlock()
return len(downstreamRegistry.byConn)
}
// DisconnectAccount closes every live downstream CMPP session for an // DisconnectAccount closes every live downstream CMPP session for an
// application account. The normal connection-close callback removes registry // application account. The normal connection-close callback removes registry
// and presence state and reports the disconnect to the API. // and presence state and reports the disconnect to the API.
+3
View File
@@ -28,6 +28,9 @@ func (s Server) post(ctx context.Context, path string, payload any, result any)
return err return err
} }
req.Header.Set("Content-Type", "application/json") req.Header.Set("Content-Type", "application/json")
if strings.HasSuffix(path, "/security-detection") && s.SecurityEventToken != "" {
req.Header.Set("X-Security-Event-Token", s.SecurityEventToken)
}
resp, err := client.Do(req) resp, err := client.Do(req)
if err != nil { if err != nil {
return err return err
+80
View File
@@ -0,0 +1,80 @@
package metrics
import (
"context"
"fmt"
"net/http"
"runtime"
"sync/atomic"
"time"
)
var startedAt = time.Now()
var submitAccepted atomic.Uint64
var submitFailed atomic.Uint64
var submitDurationNanoseconds atomic.Uint64
type Snapshot struct {
UpstreamDesired int
UpstreamConnected int
DownstreamConnected int
SubmitWorkerUp bool
QueueAvailable bool
QueuePending int64
QueueLag int64
QueueOldestAgeSeconds float64
}
type SnapshotFunc func(context.Context) Snapshot
func ObserveSubmit(accepted bool, duration time.Duration) {
if accepted {
submitAccepted.Add(1)
} else {
submitFailed.Add(1)
}
submitDurationNanoseconds.Add(uint64(max(duration, 0)))
}
func Handler(load SnapshotFunc) http.Handler {
return http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) {
if request.Method != http.MethodGet || request.URL.Path != "/metrics" {
response.WriteHeader(http.StatusNotFound)
return
}
ctx, cancel := context.WithTimeout(request.Context(), time.Second)
defer cancel()
snapshot := Snapshot{}
if load != nil {
snapshot = load(ctx)
}
var memory runtime.MemStats
runtime.ReadMemStats(&memory)
accepted := submitAccepted.Load()
failed := submitFailed.Load()
count := accepted + failed
response.Header().Set("Content-Type", "text/plain; version=0.0.4; charset=utf-8")
response.Header().Set("Cache-Control", "no-store")
fmt.Fprintf(response, "# HELP cmpp_gateway_process_uptime_seconds Gateway process uptime.\n# TYPE cmpp_gateway_process_uptime_seconds gauge\ncmpp_gateway_process_uptime_seconds %f\n", time.Since(startedAt).Seconds())
fmt.Fprintf(response, "# HELP cmpp_gateway_go_goroutines Current goroutine count.\n# TYPE cmpp_gateway_go_goroutines gauge\ncmpp_gateway_go_goroutines %d\n", runtime.NumGoroutine())
fmt.Fprintf(response, "# HELP cmpp_gateway_go_heap_alloc_bytes Current Go heap allocation.\n# TYPE cmpp_gateway_go_heap_alloc_bytes gauge\ncmpp_gateway_go_heap_alloc_bytes %d\n", memory.HeapAlloc)
fmt.Fprintf(response, "# HELP cmpp_gateway_submit_total Upstream submit attempts by bounded result.\n# TYPE cmpp_gateway_submit_total counter\ncmpp_gateway_submit_total{result=\"accepted\"} %d\ncmpp_gateway_submit_total{result=\"failed\"} %d\n", accepted, failed)
fmt.Fprintf(response, "# HELP cmpp_gateway_submit_duration_seconds_sum Total upstream submit duration.\n# TYPE cmpp_gateway_submit_duration_seconds_sum counter\ncmpp_gateway_submit_duration_seconds_sum %f\n", float64(submitDurationNanoseconds.Load())/float64(time.Second))
fmt.Fprintf(response, "# HELP cmpp_gateway_submit_duration_seconds_count Total measured upstream submits.\n# TYPE cmpp_gateway_submit_duration_seconds_count counter\ncmpp_gateway_submit_duration_seconds_count %d\n", count)
fmt.Fprintf(response, "# HELP cmpp_gateway_upstream_connections Desired and live supplier connections.\n# TYPE cmpp_gateway_upstream_connections gauge\ncmpp_gateway_upstream_connections{state=\"desired\"} %d\ncmpp_gateway_upstream_connections{state=\"connected\"} %d\n", snapshot.UpstreamDesired, snapshot.UpstreamConnected)
fmt.Fprintf(response, "# HELP cmpp_gateway_downstream_connections Authenticated client connections.\n# TYPE cmpp_gateway_downstream_connections gauge\ncmpp_gateway_downstream_connections %d\n", snapshot.DownstreamConnected)
fmt.Fprintf(response, "# HELP cmpp_gateway_submit_worker_up Whether the submit worker was initialized.\n# TYPE cmpp_gateway_submit_worker_up gauge\ncmpp_gateway_submit_worker_up %d\n", boolNumber(snapshot.SubmitWorkerUp))
if snapshot.QueueAvailable {
fmt.Fprintf(response, "# HELP cmpp_gateway_submit_queue_pending Pending entries owned by the consumer group.\n# TYPE cmpp_gateway_submit_queue_pending gauge\ncmpp_gateway_submit_queue_pending %d\n", snapshot.QueuePending)
fmt.Fprintf(response, "# HELP cmpp_gateway_submit_queue_lag Undelivered entries for the consumer group.\n# TYPE cmpp_gateway_submit_queue_lag gauge\ncmpp_gateway_submit_queue_lag %d\n", snapshot.QueueLag)
fmt.Fprintf(response, "# HELP cmpp_gateway_submit_queue_oldest_pending_age_seconds Age of the oldest pending entry.\n# TYPE cmpp_gateway_submit_queue_oldest_pending_age_seconds gauge\ncmpp_gateway_submit_queue_oldest_pending_age_seconds %f\n", snapshot.QueueOldestAgeSeconds)
}
})
}
func boolNumber(value bool) int {
if value {
return 1
}
return 0
}
+29
View File
@@ -0,0 +1,29 @@
package metrics
import (
"context"
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"
)
func TestMetricsHandlerExportsOnlyAggregateGatewayState(t *testing.T) {
ObserveSubmit(true, 20*time.Millisecond)
request := httptest.NewRequest(http.MethodGet, "/metrics", nil)
response := httptest.NewRecorder()
Handler(func(context.Context) Snapshot {
return Snapshot{UpstreamDesired: 2, UpstreamConnected: 1, DownstreamConnected: 3, SubmitWorkerUp: true, QueueAvailable: true, QueuePending: 4, QueueLag: 5, QueueOldestAgeSeconds: 12}
}).ServeHTTP(response, request)
body := response.Body.String()
if response.Code != http.StatusOK || !strings.Contains(body, "cmpp_gateway_submit_queue_pending 4") || !strings.Contains(body, "cmpp_gateway_upstream_connections{state=\"connected\"} 1") {
t.Fatalf("unexpected metrics response: code=%d body=%s", response.Code, body)
}
for _, forbidden := range []string{"phone_number", "message_id", "channel_id"} {
if strings.Contains(body, forbidden) {
t.Fatalf("metrics expose forbidden label %q", forbidden)
}
}
}
+4
View File
@@ -12,6 +12,7 @@ import (
"sync" "sync"
"time" "time"
"cmpp-platform/gateway/internal/metrics"
"cmpp-platform/gateway/internal/queue" "cmpp-platform/gateway/internal/queue"
"cmpp-platform/gateway/internal/ratelimit" "cmpp-platform/gateway/internal/ratelimit"
"cmpp-platform/gateway/internal/upstream" "cmpp-platform/gateway/internal/upstream"
@@ -206,6 +207,7 @@ func (w *Worker) processMessage(ctx context.Context, message redis.XMessage) err
} }
func (w *Worker) handleCommand(ctx context.Context, command queue.SubmitCommand) error { func (w *Worker) handleCommand(ctx context.Context, command queue.SubmitCommand) error {
startedAt := time.Now()
if w.Limiter != nil { if w.Limiter != nil {
if _, err := w.Limiter.Wait(ctx, command.ChannelID, command.Route.RateLimitPerSecond); err != nil { if _, err := w.Limiter.Wait(ctx, command.ChannelID, command.Route.RateLimitPerSecond); err != nil {
return err return err
@@ -219,6 +221,8 @@ func (w *Worker) handleCommand(ctx context.Context, command queue.SubmitCommand)
submit = w.Upstream.Submit submit = w.Upstream.Submit
} }
result, err := submit(ctx, command) result, err := submit(ctx, command)
accepted := err == nil && (result.SubmitStatus == "" || result.SubmitStatus == "accepted")
metrics.ObserveSubmit(accepted, time.Since(startedAt))
if err != nil && (result.SubmitStatus == "" || result.SubmitStatus == "accepted") { if err != nil && (result.SubmitStatus == "" || result.SubmitStatus == "accepted") {
return err return err
} }
+15
View File
@@ -146,6 +146,21 @@ func (m *Manager) ensureDefaultsLocked() {
} }
} }
// ConnectionCounts returns bounded platform totals without channel identifiers to prevent time-series cardinality growth.
func (m *Manager) ConnectionCounts() (desired int, connected int) {
m.mu.Lock()
pools := make([]*connectionPool, 0, len(m.conns))
for _, pool := range m.conns {
pools = append(pools, pool)
}
m.mu.Unlock()
for _, pool := range pools {
desired += max(pool.config.DesiredConnections, 1)
connected += pool.countActiveConnections()
}
return desired, connected
}
func (m *Manager) newConnectionPool(channelID string, connectionID string, config queue.UpstreamConfig) *connectionPool { func (m *Manager) newConnectionPool(channelID string, connectionID string, config queue.UpstreamConfig) *connectionPool {
return &connectionPool{ return &connectionPool{
channelID: channelID, channelID: channelID,
+2 -1
View File
@@ -11,7 +11,8 @@
"start:local": "powershell -NoProfile -ExecutionPolicy Bypass -File tools/start-local.ps1", "start:local": "powershell -NoProfile -ExecutionPolicy Bypass -File tools/start-local.ps1",
"start:local:minio": "powershell -NoProfile -ExecutionPolicy Bypass -File tools/start-local.ps1 -OnlyMinio", "start:local:minio": "powershell -NoProfile -ExecutionPolicy Bypass -File tools/start-local.ps1 -OnlyMinio",
"prisma:generate": "npm --prefix api run prisma:generate", "prisma:generate": "npm --prefix api run prisma:generate",
"security:verify": "node tools/security/verify-dependency-mitigations.mjs", "security:verify": "node tools/security/verify-dependency-mitigations.mjs && node tools/security/verify-security-deployment.mjs",
"deploy:verify": "node tools/deploy/verify-production-deployment.mjs",
"spike:contracts": "node tools/spike/validate-gateway-queue-contract.mjs", "spike:contracts": "node tools/spike/validate-gateway-queue-contract.mjs",
"spike:gateway": "powershell -NoProfile -ExecutionPolicy Bypass -Command \"$env:Path='C:\\Program Files\\Go\\bin;'+$env:Path; Push-Location gateway; go test ./...; Pop-Location\"", "spike:gateway": "powershell -NoProfile -ExecutionPolicy Bypass -Command \"$env:Path='C:\\Program Files\\Go\\bin;'+$env:Path; Push-Location gateway; go test ./...; Pop-Location\"",
"spike:bullmq": "node api/src/spike/bullmq-link-spike.mjs", "spike:bullmq": "node api/src/spike/bullmq-link-spike.mjs",
@@ -0,0 +1,11 @@
import { request, withQuery } from '../core/httpClient';
import type { InfrastructureAlertSettings, InfrastructureAlertThresholds, InfrastructureMonitoringOverview, InfrastructureMonitoringRange } from '../types';
export const adminInfrastructureMonitoringApi = {
getInfrastructureMonitoringOverview: (range: InfrastructureMonitoringRange) =>
request<InfrastructureMonitoringOverview>(withQuery('/admin/infrastructure-monitoring/overview', { range })),
getInfrastructureMonitoringNotificationSummary: () => request<{ count: number; criticalCount: number }>('/admin/infrastructure-monitoring/notification-summary'),
getInfrastructureAlertThresholds: () => request<InfrastructureAlertSettings>('/admin/infrastructure-monitoring/alert-thresholds'),
updateInfrastructureAlertThresholds: (body: { configVersion: number; thresholds: InfrastructureAlertThresholds }) => request<InfrastructureAlertSettings>('/admin/infrastructure-monitoring/alert-thresholds', { method: 'PUT', body: JSON.stringify(body) }),
markInfrastructureAlertRead: (fingerprint: string, activeAt: string) => request<{ fingerprint: string; activeAt: string; acknowledged: true; acknowledgedAt: string }>(`/admin/infrastructure-monitoring/alerts/${fingerprint}/read`, { method: 'POST', body: JSON.stringify({ activeAt }) }),
};
+16
View File
@@ -0,0 +1,16 @@
import { request, withQuery } from '../core/httpClient';
import type { SecurityAlert, SecurityBlock, SecurityOverview, SecurityProtectedNetwork, SecurityRule } from '../types';
export const adminSecurityDetectionApi = {
getSecurityOverview: (range = '24h') => request<SecurityOverview>(withQuery('/admin/security-detection/overview', { range })),
getSecurityNotificationSummary: () => request<{ count: number; criticalCount: number }>('/admin/security-detection/notification-summary'),
listSecurityAlerts: (query: Record<string, string | number | undefined> = {}) => request<{ items: SecurityAlert[]; total: number }> (withQuery('/admin/security-detection/alerts', query)),
listSecurityRules: () => request<SecurityRule[]>('/admin/security-detection/rules'),
updateSecurityRule: (id: string, body: Partial<SecurityRule>) => request<SecurityRule>(`/admin/security-detection/rules/${id}`, { method: 'PUT', body: JSON.stringify(body) }),
blockSecurityAlert: (id: string, body: { durationSeconds: number; reason: string }) => request<SecurityBlock>(`/admin/security-detection/alerts/${id}/block`, { method: 'POST', body: JSON.stringify(body) }),
ignoreSecurityAlert: (id: string, reason: string) => request<{ success: boolean }>(`/admin/security-detection/alerts/${id}/ignore`, { method: 'POST', body: JSON.stringify({ reason }) }),
listSecurityBlocks: () => request<SecurityBlock[]>('/admin/security-detection/blocks'),
unblockSecurityBlock: (id: string, reason: string) => request<SecurityBlock>(`/admin/security-detection/blocks/${id}/unblock`, { method: 'POST', body: JSON.stringify({ reason }) }),
listProtectedNetworks: () => request<SecurityProtectedNetwork[]>('/admin/security-detection/protected-networks'),
addProtectedNetwork: (body: { network: string; name: string; reason: string }) => request<SecurityProtectedNetwork>('/admin/security-detection/protected-networks', { method: 'POST', body: JSON.stringify(body) }),
};
+4
View File
@@ -10,6 +10,8 @@ import { adminOperationsApi } from './admin/operations.api';
import { adminGovernanceApi } from './admin/governance.api'; import { adminGovernanceApi } from './admin/governance.api';
import { adminFilesApi } from './admin/files.api'; import { adminFilesApi } from './admin/files.api';
import { adminSignatureRetirementApi } from './admin/signature-retirement.api'; import { adminSignatureRetirementApi } from './admin/signature-retirement.api';
import { adminInfrastructureMonitoringApi } from './admin/infrastructure-monitoring.api';
import { adminSecurityDetectionApi } from './admin/security-detection.api';
export const adminApi = { export const adminApi = {
...adminIdentityApi, ...adminIdentityApi,
@@ -18,4 +20,6 @@ export const adminApi = {
...adminGovernanceApi, ...adminGovernanceApi,
...adminFilesApi, ...adminFilesApi,
...adminSignatureRetirementApi, ...adminSignatureRetirementApi,
...adminInfrastructureMonitoringApi,
...adminSecurityDetectionApi,
}; };
+2
View File
@@ -4,3 +4,5 @@ export * from './channels-reports';
export * from './operations'; export * from './operations';
export * from './governance'; export * from './governance';
export * from './signature-retirement'; export * from './signature-retirement';
export * from './infrastructure-monitoring';
export * from './security-detection';
@@ -0,0 +1,93 @@
export type InfrastructureMonitoringRange = '1h' | '24h' | '7d';
export type InfrastructureMetricPoint = {
timestamp: string;
value: number;
};
export type InfrastructureServiceStatus = {
key: string;
name: string;
unit: string;
status: 'healthy' | 'unhealthy' | 'unknown';
};
export type InfrastructureServiceMetricGroup = {
key: string;
name: string;
available: boolean;
metrics: Array<{
key: string;
label: string;
value: number | null;
unit: 'percent' | 'seconds' | 'count' | 'per_second' | 'bytes';
}>;
};
export type InfrastructureAlert = {
fingerprint: string;
name: string;
severity: 'info' | 'warning' | 'critical';
status: string;
startedAt: string;
summary: string;
description?: string;
currentValue?: string;
threshold?: string;
service?: string;
instance?: string;
acknowledged: boolean;
acknowledgedAt?: string;
};
export type InfrastructureMonitoringOverview = {
available: boolean;
range: InfrastructureMonitoringRange;
collectedAt: string;
lastSampleAt: string | null;
error?: string;
summary: {
overallStatus: 'healthy' | 'warning' | 'critical' | 'unknown';
serviceTotal: number;
serviceHealthy: number;
warningAlerts: number;
criticalAlerts: number;
activeAlerts: number;
};
metrics: {
cpuUsagePercent: number | null;
memoryUsagePercent: number | null;
memoryTotalBytes: number | null;
memoryAvailableBytes: number | null;
diskUsagePercent: number | null;
diskTotalBytes: number | null;
diskAvailableBytes: number | null;
networkReceiveBytesPerSecond: number | null;
networkTransmitBytesPerSecond: number | null;
load1: number | null;
uptimeSeconds: number | null;
};
trends: {
cpuUsagePercent: InfrastructureMetricPoint[];
memoryUsagePercent: InfrastructureMetricPoint[];
diskUsagePercent: InfrastructureMetricPoint[];
networkReceiveBytesPerSecond: InfrastructureMetricPoint[];
networkTransmitBytesPerSecond: InfrastructureMetricPoint[];
};
services: InfrastructureServiceStatus[];
serviceMetrics: InfrastructureServiceMetricGroup[];
alerts: InfrastructureAlert[];
};
export type InfrastructureAlertThresholds = Record<string, { warning: number; critical: number }>;
export type InfrastructureAlertSettings = {
configVersion: number;
effectiveVersion: number;
applyStatus: 'effective' | 'applying' | 'failed';
lastError: string | null;
appliedAt: string | null;
thresholds: InfrastructureAlertThresholds;
effectiveThresholds: InfrastructureAlertThresholds;
definitions: Array<{ key: string; label: string; unit: string; min: number; max: number; step: number }>;
};
+18
View File
@@ -0,0 +1,18 @@
export type SecurityRule = {
id: string; code: string; name: string; sourceType: string; enabled: boolean;
threshold: number; windowSeconds: number; cooldownSeconds: number; severity: string;
defaultBlockSeconds: number; maximumBlockSeconds: number; configVersion: number;
effectiveVersion: number; applyStatus: string; lastApplyError?: string | null;
};
export type SecurityAlert = {
id: string; fingerprint: string; sourceIp: string; severity: string; status: string;
eventCount: number; firstOccurredAt: string; lastOccurredAt: string; rule: SecurityRule;
};
export type SecurityBlock = { id: string; sourceIp: string; executor: string; status: string; durationSeconds: number; reason: string; requestedAt: string; expiresAt?: string | null; lastError?: string | null };
export type SecurityProtectedNetwork = { id: string; network: string; name: string; reason: string; enabled: boolean; createdAt: string };
export type SecurityOverview = {
range: string; collectedAt: string; totalEvents: number; activeAlerts: number; criticalAlerts: number; activeBlocks: number;
health: { agent: string; agentError?: string; rulesEffective: number; rulesTotal: number };
sourceDistribution: Array<{ name: string; value: number }>;
alerts: SecurityAlert[];
};
@@ -1,7 +1,7 @@
import { useCallback, useEffect, useMemo, useRef, useState } from 'react'; import { useCallback, useEffect, useMemo, useRef, useState } from 'react';
import { AlertTriangle, BarChart3, CheckCircle2, Eye, RefreshCw, Search, TimerReset } from 'lucide-react'; import { AlertTriangle, BarChart3, CheckCircle2, Eye, RefreshCw, Search, TimerReset } from 'lucide-react';
import { adminApi, type DownstreamDeliveryDashboard, type DownstreamDeliveryRecord, type DownstreamRequeuePreview, type DownstreamRequeueTask, type DownstreamRequeueTaskItem, type EnterpriseApplication, type TenantOption } from '@/api/adminApi'; import { adminApi, type DownstreamDeliveryDashboard, type DownstreamDeliveryRecord, type DownstreamRequeuePreview, type DownstreamRequeueTask, type DownstreamRequeueTaskItem, type EnterpriseApplication, type TenantOption } from '@/api/adminApi';
import { Breadcrumb, Button, DateRangeInput, Input, Modal, Pagination, Select, Tag, type DateRangeValue } from '@/components/ui'; import { Breadcrumb, Button, DateRangeInput, Input, Modal, Pagination, Select, Tag, Textarea, type DateRangeValue } from '@/components/ui';
import { formatDateTime } from '@/utils/dateTime'; import { formatDateTime } from '@/utils/dateTime';
const statusTone: Record<string, 'neutral' | 'info' | 'success' | 'warning' | 'danger'> = { const statusTone: Record<string, 'neutral' | 'info' | 'success' | 'warning' | 'danger'> = {
@@ -677,7 +677,7 @@ export function AdminDownstreamDeliveriesPage() {
/> />
</div> </div>
<div className="surface admin-task-table-card report-task-table-card"> <div className="surface admin-task-table-card report-task-table-card downstream-requeue-task-card">
<div className="section-heading downstream-requeue-task-heading"> <div className="section-heading downstream-requeue-task-heading">
<div><h2></h2><p className="muted"></p></div> <div><h2></h2><p className="muted"></p></div>
<div className="downstream-requeue-task-heading__actions"> <div className="downstream-requeue-task-heading__actions">
@@ -741,8 +741,20 @@ export function AdminDownstreamDeliveriesPage() {
</div> </div>
<div className="downstream-requeue-preview__distribution"><span></span><div>{Object.entries(taskPreview.statusCounts).map(([key, value]) => <Tag key={key} tone={statusTone[key] ?? 'neutral'}>{statusLabel[key] ?? key} {value}</Tag>)}</div></div> <div className="downstream-requeue-preview__distribution"><span></span><div>{Object.entries(taskPreview.statusCounts).map(([key, value]) => <Tag key={key} tone={statusTone[key] ?? 'neutral'}>{statusLabel[key] ?? key} {value}</Tag>)}</div></div>
<Select label="执行速度" value={String(taskRate)} onChange={(event) => setTaskRate(Number(event.target.value))} options={[{ label: '平稳(每应用10条/秒)', value: '10' }, { label: '快速(每应用20条/秒)', value: '20' }, { label: '低速(每应用5条/秒)', value: '5' }]} /> <Select label="执行速度" value={String(taskRate)} onChange={(event) => setTaskRate(Number(event.target.value))} options={[{ label: '平稳(每应用10条/秒)', value: '10' }, { label: '快速(每应用20条/秒)', value: '20' }, { label: '低速(每应用5条/秒)', value: '5' }]} />
<label className="field"><span> *</span><textarea value={taskReason} onChange={(event) => setTaskReason(event.target.value)} placeholder="请填写事故原因、工单号或处理说明(至少5个字)" rows={3} /></label> <Textarea
<div className="downstream-requeue-warning"><AlertTriangle size={20} /><div><strong></strong><p> ACK 线</p></div></div> className="downstream-requeue-reason"
error={taskReason.length > 0 && taskReason.trim().length < 5 ? '任务原因至少填写 5 个字' : undefined}
hint={`请填写事故原因、工单号或处理说明 · ${taskReason.length}/200`}
id="downstream-requeue-task-reason"
label="任务原因"
maxLength={200}
onChange={(event) => setTaskReason(event.target.value)}
placeholder="例如:工单 INC-20260814,重新投递客户已确认的历史回执"
required
rows={4}
value={taskReason}
/>
<div className="downstream-requeue-warning"><AlertTriangle size={20} /><div><strong></strong><p> ACK 线</p></div></div>
</div> </div>
</Modal> </Modal>
) : null} ) : null}
@@ -0,0 +1,2 @@
.admin-security-page{gap:20px}.security-heading{align-items:flex-end}.security-heading h1{margin:10px 0 4px;font-size:26px}.security-heading p{margin:0;color:#64748b}.security-kpis{display:grid;grid-template-columns:repeat(4,minmax(0,1fr));gap:14px}.security-kpi{display:grid;grid-template-columns:42px 1fr;grid-template-rows:auto auto;gap:3px 12px;padding:18px}.security-kpi>div{grid-row:1/3;width:42px;height:42px;border-radius:12px;display:grid;place-items:center;background:#e8f0ff;color:#2563eb}.security-kpi>div svg{width:20px}.security-kpi span{font-size:13px;color:#64748b}.security-kpi strong{font-size:24px;line-height:1.1}.security-kpi.is-danger>div{background:#fef2f2;color:#dc2626}.security-overview-grid{display:grid;grid-template-columns:minmax(0,1fr) minmax(360px,.8fr);gap:16px}.security-chart,.security-latest,.security-table-card{padding:18px}.security-chart header,.security-latest header{display:flex;justify-content:space-between}.security-chart header span,.security-latest header span{font-size:12px;color:#94a3b8}.security-latest-row{display:grid;grid-template-columns:10px 1fr auto;gap:10px;align-items:center;padding:13px 0;border-bottom:1px solid #eef2f7}.security-latest-row div{display:grid;gap:3px}.security-latest-row small,.security-latest-row time{color:#64748b;font-size:12px}.severity-dot{width:8px;height:8px;border-radius:99px;background:#3b82f6}.severity-dot.is-high{background:#f59e0b}.severity-dot.is-critical{background:#ef4444}.security-cell{display:grid;gap:3px}.security-cell span{font-size:11px;color:#94a3b8}.security-actions{display:flex;gap:6px}.security-note,.security-section-toolbar{display:flex;align-items:center;gap:9px;margin-bottom:16px;padding:12px 14px;border-radius:10px;background:#f8fafc;color:#475569;font-size:13px}.security-section-toolbar{justify-content:space-between}.security-error{display:flex;gap:8px;align-items:center;padding:12px 14px;border:1px solid #fecaca;border-radius:10px;background:#fef2f2;color:#b91c1c}.security-empty{height:265px;display:grid;place-items:center;color:#94a3b8}.security-dialog{display:grid;gap:16px}.security-target{display:grid;gap:4px;padding:14px;border-radius:10px;background:#f8fafc}.security-target span,.security-target small{color:#64748b;font-size:12px}.security-target strong{font-family:ui-monospace,monospace;font-size:18px}.security-form-grid{display:grid;grid-template-columns:1fr 1fr;gap:16px}.security-form-grid .security-error{grid-column:1/-1}.admin-security-page code{font-size:12px;color:#334155}@media(max-width:1100px){.security-kpis{grid-template-columns:repeat(2,1fr)}.security-overview-grid{grid-template-columns:1fr}}@media(max-width:640px){.security-kpis{grid-template-columns:1fr}.security-form-grid{grid-template-columns:1fr}}
.security-heading p{margin:10px 0 0}
@@ -0,0 +1,95 @@
import { useCallback, useEffect, useMemo, useState, type ComponentProps } from 'react';
import type { EChartsOption } from 'echarts';
import { AlertTriangle, Ban, BellRing, RefreshCw, Settings2, ShieldCheck, ShieldOff } from 'lucide-react';
import { adminApi, type SecurityAlert, type SecurityBlock, type SecurityOverview, type SecurityProtectedNetwork, type SecurityRule } from '@/api/adminApi';
import { Breadcrumb, Button, Chart, Input, Modal as BaseModal, Select, Table, Tabs, Tag, Textarea, type TableColumn } from '@/components/ui';
import './AdminSecurityDetectionPage.css';
const statusLabels: Record<string, string> = { open: '待处理', acknowledged: '已确认', blocked: '已封禁', block_failed: '封禁失败', ignored: '已忽略', requested: '执行中', failed: '失败', released: '已解封' };
const severityLabels: Record<string, string> = { low: '低', medium: '中', high: '高', critical: '严重' };
const durationOptions = [{ value: '600', label: '10分钟' }, { value: '3600', label: '1小时' }, { value: '86400', label: '24小时' }, { value: '604800', label: '7天' }];
const formatTime = (value?: string | null) => value ? new Intl.DateTimeFormat('zh-CN', { month: '2-digit', day: '2-digit', hour: '2-digit', minute: '2-digit', hour12: false }).format(new Date(value)) : '—';
function Modal(props: Omit<ComponentProps<typeof BaseModal>, 'open'>) { return <BaseModal {...props} open />; }
export function AdminSecurityDetectionPage() {
const [overview, setOverview] = useState<SecurityOverview | null>(null);
const [alerts, setAlerts] = useState<SecurityAlert[]>([]);
const [rules, setRules] = useState<SecurityRule[]>([]);
const [blocks, setBlocks] = useState<SecurityBlock[]>([]);
const [networks, setNetworks] = useState<SecurityProtectedNetwork[]>([]);
const [loading, setLoading] = useState(true);
const [error, setError] = useState('');
const [blockAlert, setBlockAlert] = useState<SecurityAlert | null>(null);
const [editRule, setEditRule] = useState<SecurityRule | null>(null);
const [showNetwork, setShowNetwork] = useState(false);
const [reasonAction, setReasonAction] = useState<{ title: string; confirmLabel: string; run: (reason: string) => Promise<void> } | null>(null);
const load = useCallback(async () => {
setLoading(true); setError('');
try {
const [nextOverview, nextAlerts, nextRules, nextBlocks, nextNetworks] = await Promise.all([
adminApi.getSecurityOverview('24h'), adminApi.listSecurityAlerts({ pageSize: 100 }), adminApi.listSecurityRules(), adminApi.listSecurityBlocks(), adminApi.listProtectedNetworks(),
]);
setOverview(nextOverview); setAlerts(nextAlerts.items); setRules(nextRules); setBlocks(nextBlocks); setNetworks(nextNetworks);
} catch (reason) { setError(reason instanceof Error ? reason.message : '安全检测数据加载失败'); }
finally { setLoading(false); }
}, []);
useEffect(() => { void load(); }, [load]);
const alertColumns: Array<TableColumn<SecurityAlert>> = [
{ key: 'level', title: '级别', width: '80px', render: (item) => <Tag tone={item.severity === 'critical' ? 'danger' : item.severity === 'high' ? 'warning' : 'info'}>{severityLabels[item.severity] ?? item.severity}</Tag> },
{ key: 'rule', title: '检测类型', width: '180px', render: (item) => <div className="security-cell"><strong>{item.rule.name}</strong><span>{item.rule.code}</span></div> },
{ key: 'ip', title: '来源 IP', width: '150px', render: (item) => <code>{item.sourceIp}</code> },
{ key: 'count', title: '窗口命中', width: '100px', render: (item) => `${item.eventCount}` },
{ key: 'time', title: '最后发生', width: '140px', render: (item) => formatTime(item.lastOccurredAt) },
{ key: 'status', title: '状态', width: '96px', render: (item) => <Tag tone={item.status === 'blocked' ? 'success' : item.status === 'block_failed' ? 'danger' : 'neutral'}>{statusLabels[item.status] ?? item.status}</Tag> },
{ key: 'action', title: '人工处置', width: '190px', render: (item) => <div className="security-actions"><Button disabled={!['open', 'acknowledged', 'block_failed'].includes(item.status)} onClick={() => setBlockAlert(item)} size="sm" variant="danger"></Button><Button disabled={!['open', 'acknowledged', 'block_failed'].includes(item.status)} onClick={() => void ignore(item)} size="sm" variant="ghost"></Button></div> },
];
function ignore(item: SecurityAlert) { setReasonAction({ title: `忽略告警 · ${item.sourceIp}`, confirmLabel: '确认忽略', run: async (reason) => { await adminApi.ignoreSecurityAlert(item.id, reason); await load(); } }); }
const ruleColumns: Array<TableColumn<SecurityRule>> = [
{ key: 'name', title: '规则', width: '230px', render: (item) => <div className="security-cell"><strong>{item.name}</strong><span>{item.code}</span></div> },
{ key: 'source', title: '数据源', width: '110px', render: (item) => item.sourceType },
{ key: 'threshold', title: '阈值 / 窗口', width: '150px', render: (item) => `${item.threshold} 次 / ${item.windowSeconds}` },
{ key: 'cooldown', title: '冷却', width: '100px', render: (item) => `${item.cooldownSeconds}` },
{ key: 'version', title: '生效版本', width: '120px', render: (item) => <Tag tone={item.applyStatus === 'effective' ? 'success' : item.applyStatus === 'failed' ? 'danger' : 'warning'}>{item.effectiveVersion}/{item.configVersion}</Tag> },
{ key: 'enabled', title: '启用', width: '90px', render: (item) => <Tag tone={item.enabled ? 'success' : 'neutral'}>{item.enabled ? '启用' : '停用'}</Tag> },
{ key: 'action', title: '操作', width: '90px', render: (item) => <Button onClick={() => setEditRule(item)} size="sm" variant="ghost"></Button> },
];
const blockColumns: Array<TableColumn<SecurityBlock>> = [
{ key: 'ip', title: 'IP', width: '160px', render: (item) => <code>{item.sourceIp}</code> }, { key: 'executor', title: '执行器', width: '140px', render: (item) => item.executor },
{ key: 'status', title: '状态', width: '100px', render: (item) => <Tag tone={item.status === 'blocked' ? 'success' : item.status === 'failed' ? 'danger' : 'neutral'}>{statusLabels[item.status] ?? item.status}</Tag> },
{ key: 'reason', title: '原因', width: '260px', render: (item) => item.reason }, { key: 'expiry', title: '到期时间', width: '150px', render: (item) => formatTime(item.expiresAt) }, { key: 'error', title: '执行结果', width: '220px', render: (item) => item.lastError ?? '已由执行器回读确认' },
{ key: 'action', title: '操作', width: '90px', render: (item) => <Button disabled={item.status !== 'blocked'} onClick={() => void releaseBlock(item)} size="sm" variant="ghost"></Button> },
];
function releaseBlock(item: SecurityBlock) { setReasonAction({ title: `人工解封 · ${item.sourceIp}`, confirmLabel: '确认解封', run: async (reason) => { await adminApi.unblockSecurityBlock(item.id, reason); await load(); } }); }
const networkColumns: Array<TableColumn<SecurityProtectedNetwork>> = [
{ key: 'network', title: 'IP / 网段', width: '180px', render: (item) => <code>{item.network}</code> }, { key: 'name', title: '名称', width: '180px', render: (item) => item.name }, { key: 'reason', title: '保护原因', width: '320px', render: (item) => item.reason }, { key: 'status', title: '状态', width: '100px', render: (item) => <Tag tone={item.enabled ? 'success' : 'neutral'}>{item.enabled ? '保护中' : '已停用'}</Tag> },
];
const chartOption = useMemo<EChartsOption>(() => ({ tooltip: { trigger: 'item' }, legend: { bottom: 0, type: 'scroll' }, series: [{ type: 'pie', radius: ['52%', '76%'], center: ['50%', '43%'], label: { show: false }, data: overview?.sourceDistribution ?? [] }] }), [overview]);
const tabs = [
{ value: 'overview', label: '总览', content: <><div className="security-overview-grid"><article className="surface security-chart"><header><strong>24</strong><span></span></header>{overview?.sourceDistribution.length ? <Chart height={265} option={chartOption} /> : <div className="security-empty"></div>}</article><article className="surface security-latest"><header><strong></strong><span>{overview?.alerts.length ?? 0} </span></header>{overview?.alerts.slice(0, 6).map((item) => <div className="security-latest-row" key={item.id}><span className={`severity-dot is-${item.severity}`} /><div><strong>{item.rule.name}</strong><small>{item.sourceIp} · {item.eventCount} </small></div><time>{formatTime(item.lastOccurredAt)}</time></div>)}</article></div></> },
{ value: 'alerts', label: '告警中心', content: <section className="surface security-table-card"><Table columns={alertColumns} data={alerts} emptyText="暂无安全告警" rowKey="id" /></section> },
{ value: 'rules', label: '规则配置', content: <section className="surface security-table-card"><div className="security-note"><Settings2 size={17} /><span></span></div><Table columns={ruleColumns} data={rules} rowKey="id" /></section> },
{ value: 'blocks', label: '封禁记录', content: <section className="surface security-table-card"><Table columns={blockColumns} data={blocks} emptyText="暂无人工封禁记录" rowKey="id" /></section> },
{ value: 'protected', label: '保护名单', content: <section className="surface security-table-card"><div className="security-section-toolbar"><span></span><Button onClick={() => setShowNetwork(true)}></Button></div><Table columns={networkColumns} data={networks} emptyText="暂无保护网段" rowKey="id" /></section> },
];
return <section className="page-stack admin-security-page"><div className="page-heading security-heading"><div><Breadcrumb items={['安全控制', '安全检测与封禁']} /><p>使 Fail2ban </p></div><Button disabled={loading} icon={<RefreshCw className={loading ? 'is-spinning' : ''} size={16} />} onClick={() => void load()} variant="ghost"></Button></div>
{error ? <div className="security-error" role="alert"><AlertTriangle size={18} />{error}</div> : null}
<div className="security-kpis"><Kpi icon={<BellRing />} label="24小时检测事件" value={overview?.totalEvents ?? 0} /><Kpi icon={<AlertTriangle />} label="待处置告警" value={overview?.activeAlerts ?? 0} danger={Boolean(overview?.criticalAlerts)} /><Kpi icon={<Ban />} label="生效封禁" value={overview?.activeBlocks ?? 0} /><Kpi icon={overview?.health.agent === 'healthy' ? <ShieldCheck /> : <ShieldOff />} label="安全代理" value={overview?.health.agent === 'healthy' ? '在线' : '不可用'} danger={overview?.health.agent !== 'healthy'} /></div>
<Tabs items={tabs} />
{blockAlert ? <BlockDialog alert={blockAlert} onClose={() => setBlockAlert(null)} onSaved={async () => { setBlockAlert(null); await load(); }} /> : null}
{editRule ? <RuleDialog rule={editRule} onClose={() => setEditRule(null)} onSaved={async () => { setEditRule(null); await load(); }} /> : null}
{showNetwork ? <NetworkDialog onClose={() => setShowNetwork(false)} onSaved={async () => { setShowNetwork(false); await load(); }} /> : null}
{reasonAction ? <ActionReasonDialog action={reasonAction} onClose={() => setReasonAction(null)} /> : null}
</section>;
}
function Kpi({ icon, label, value, danger = false }: { icon: React.ReactNode; label: string; value: React.ReactNode; danger?: boolean }) { return <article className={`surface security-kpi ${danger ? 'is-danger' : ''}`}><div>{icon}</div><span>{label}</span><strong>{value}</strong></article>; }
function BlockDialog({ alert, onClose, onSaved }: { alert: SecurityAlert; onClose: () => void; onSaved: () => void }) { const [duration, setDuration] = useState(String(alert.rule.defaultBlockSeconds)); const [reason, setReason] = useState(''); const [saving, setSaving] = useState(false); const [error, setError] = useState(''); const executor = alert.rule.code.startsWith('admin_') || alert.rule.code.startsWith('client_') ? 'Nginx Real-IP deny' : 'nftables'; async function save() { setSaving(true); setError(''); try { await adminApi.blockSecurityAlert(alert.id, { durationSeconds: Number(duration), reason }); await onSaved(); } catch (reason) { setError(reason instanceof Error ? reason.message : '封禁失败'); } finally { setSaving(false); } } return <Modal title="确认人工封禁" onClose={onClose} footer={<><Button onClick={onClose} variant="ghost"></Button><Button disabled={saving || reason.trim().length < 5} onClick={() => void save()} variant="danger">{saving ? '执行并回读中' : '确认封禁'}</Button></>}><div className="security-dialog"><div className="security-target"><span> IP</span><strong>{alert.sourceIp}</strong><small>{alert.rule.name} · {alert.eventCount} </small></div><Input disabled label="服务端固定执行器" value={executor} /><Select label="封禁时长" options={durationOptions.filter((item) => Number(item.value) <= alert.rule.maximumBlockSeconds)} value={duration} onChange={(event) => setDuration(event.target.value)} /><Textarea label="封禁原因" minLength={5} onChange={(event) => setReason(event.target.value)} required value={reason} />{error ? <div className="security-error">{error}</div> : null}</div></Modal>; }
function RuleDialog({ rule, onClose, onSaved }: { rule: SecurityRule; onClose: () => void; onSaved: () => void }) { const [value, setValue] = useState(rule); const [saving, setSaving] = useState(false); const [error, setError] = useState(''); const number = (key: keyof SecurityRule) => (event: React.ChangeEvent<HTMLInputElement>) => setValue({ ...value, [key]: Number(event.target.value) }); async function save() { setSaving(true); try { await adminApi.updateSecurityRule(rule.id, value); await onSaved(); } catch (reason) { setError(reason instanceof Error ? reason.message : '规则保存失败'); } finally { setSaving(false); } } return <Modal title={`配置规则 · ${rule.name}`} onClose={onClose} footer={<><Button onClick={onClose} variant="ghost"></Button><Button disabled={saving} onClick={() => void save()}>{saving ? '验证并应用中' : '保存并应用'}</Button></>}><div className="security-form-grid"><Select label="启用状态" options={[{ value: 'true', label: '启用' }, { value: 'false', label: '停用' }]} value={String(value.enabled)} onChange={(event) => setValue({ ...value, enabled: event.target.value === 'true' })} /><Select label="告警级别" options={['low','medium','high','critical'].map((item) => ({ value: item, label: severityLabels[item] }))} value={value.severity} onChange={(event) => setValue({ ...value, severity: event.target.value })} /><Input label="触发次数" min={1} onChange={number('threshold')} type="number" value={value.threshold} /><Input label="检测窗口(秒)" min={10} onChange={number('windowSeconds')} type="number" value={value.windowSeconds} /><Input label="告警冷却(秒)" min={0} onChange={number('cooldownSeconds')} type="number" value={value.cooldownSeconds} /><Input label="默认封禁(秒)" min={600} onChange={number('defaultBlockSeconds')} type="number" value={value.defaultBlockSeconds} /><Input label="最大封禁(秒)" min={600} onChange={number('maximumBlockSeconds')} type="number" value={value.maximumBlockSeconds} />{error ? <div className="security-error">{error}</div> : null}</div></Modal>; }
function NetworkDialog({ onClose, onSaved }: { onClose: () => void; onSaved: () => void }) { const [value, setValue] = useState({ network: '', name: '', reason: '' }); const [error, setError] = useState(''); async function save() { try { await adminApi.addProtectedNetwork(value); await onSaved(); } catch (reason) { setError(reason instanceof Error ? reason.message : '新增失败'); } } return <Modal title="新增保护网段" onClose={onClose} footer={<><Button onClick={onClose} variant="ghost"></Button><Button disabled={!value.network || !value.name || !value.reason} onClick={() => void save()}></Button></>}><div className="security-dialog"><Input label="IP 或 CIDR" placeholder="例如 203.0.113.10 或 10.0.0.0/8" value={value.network} onChange={(event) => setValue({ ...value, network: event.target.value })} /><Input label="名称" value={value.name} onChange={(event) => setValue({ ...value, name: event.target.value })} /><Textarea label="保护原因" value={value.reason} onChange={(event) => setValue({ ...value, reason: event.target.value })} />{error ? <div className="security-error">{error}</div> : null}</div></Modal>; }
function ActionReasonDialog({ action, onClose }: { action: { title: string; confirmLabel: string; run: (reason: string) => Promise<void> }; onClose: () => void }) { const [reason, setReason] = useState(''); const [saving, setSaving] = useState(false); const [error, setError] = useState(''); async function save() { setSaving(true); setError(''); try { await action.run(reason); onClose(); } catch (cause) { setError(cause instanceof Error ? cause.message : '操作失败'); } finally { setSaving(false); } } return <Modal title={action.title} onClose={onClose} footer={<><Button onClick={onClose} variant="ghost"></Button><Button disabled={saving || reason.trim().length < 5} onClick={() => void save()}>{saving ? '处理中' : action.confirmLabel}</Button></>}><div className="security-dialog"><Textarea autoFocus label="操作原因" minLength={5} onChange={(event) => setReason(event.target.value)} required value={reason} />{error ? <div className="security-error">{error}</div> : null}</div></Modal>; }
@@ -0,0 +1,336 @@
.admin-system-monitoring-page {
gap: 18px;
}
.system-monitoring-heading {
align-items: flex-end;
gap: 24px;
}
.system-monitoring-title-row {
align-items: center;
display: flex;
gap: 12px;
margin-top: 8px;
}
.system-monitoring-title-row p {
color: var(--color-text-muted);
font-size: 13px;
margin: 0;
}
.system-monitoring-controls,
.system-monitoring-range {
align-items: center;
display: flex;
}
.system-monitoring-controls {
gap: 10px;
}
.system-monitoring-range {
background: var(--color-surface-muted);
border: 1px solid var(--color-border);
border-radius: var(--radius-md);
padding: 3px;
}
.system-monitoring-range button {
background: transparent;
border: 0;
border-radius: 4px;
color: var(--color-text-muted);
cursor: pointer;
font: inherit;
min-height: 30px;
padding: 0 12px;
}
.system-monitoring-range button:hover {
color: var(--color-text-strong);
}
.system-monitoring-range button:focus-visible {
box-shadow: var(--focus-ring);
outline: none;
}
.system-monitoring-range button.is-active {
background: var(--color-surface);
box-shadow: var(--shadow-xs);
color: var(--color-selected);
font-weight: var(--font-weight-semibold);
}
.system-monitoring-unavailable {
align-items: flex-start;
background: var(--color-danger-soft);
border: 1px solid color-mix(in srgb, var(--color-danger) 24%, transparent);
border-radius: var(--radius-lg);
color: var(--color-danger);
display: flex;
gap: 12px;
padding: 14px 16px;
}
.system-monitoring-unavailable div {
display: grid;
gap: 2px;
}
.system-monitoring-unavailable span {
color: var(--color-text-muted);
font-size: 13px;
}
.system-monitoring-health {
align-items: center;
display: grid;
gap: 0;
grid-template-columns: minmax(220px, 1.35fr) repeat(4, minmax(120px, 1fr));
padding: 18px 20px;
}
.system-monitoring-health__mark {
align-items: center;
border-radius: 50%;
display: flex;
height: 48px;
justify-content: center;
margin-right: 13px;
width: 48px;
}
.system-monitoring-health__mark.is-healthy { background: var(--color-success-soft); color: var(--color-success); }
.system-monitoring-health__mark.is-warning { background: var(--color-warning-soft); color: var(--color-warning); }
.system-monitoring-health__mark.is-critical { background: var(--color-danger-soft); color: var(--color-danger); }
.system-monitoring-health__mark.is-unknown { background: var(--color-surface-muted); color: var(--color-text-muted); }
.system-monitoring-health__copy {
align-items: center;
display: grid;
grid-template-columns: 48px 1fr;
grid-template-rows: repeat(3, auto);
}
.system-monitoring-health__copy .system-monitoring-health__mark { grid-row: 1 / 4; }
.system-monitoring-health__copy > span,
.system-monitoring-health__fact > span {
color: var(--color-text-muted);
font-size: 12px;
}
.system-monitoring-health__copy > strong { color: var(--color-text-strong); font-size: 17px; }
.system-monitoring-health__copy > small,
.system-monitoring-health__fact > small { color: var(--color-text-subtle); font-size: 11px; }
.system-monitoring-health__fact {
border-left: 1px solid var(--color-border);
display: grid;
gap: 3px;
padding-left: 20px;
}
.system-monitoring-health__fact strong {
color: var(--color-text-strong);
font-size: 18px;
}
.system-monitoring-metrics {
display: grid;
gap: 14px;
grid-template-columns: repeat(4, minmax(0, 1fr));
}
.system-monitoring-metric {
align-items: center;
display: flex;
gap: 14px;
min-width: 0;
padding: 18px;
}
.system-monitoring-metric__icon {
align-items: center;
border-radius: var(--radius-md);
display: flex;
flex: 0 0 40px;
height: 40px;
justify-content: center;
}
.system-monitoring-metric__icon.is-blue { background: var(--color-info-soft); color: var(--color-info); }
.system-monitoring-metric__icon.is-violet { background: #f5f3ff; color: #7c3aed; }
.system-monitoring-metric__icon.is-amber { background: var(--color-warning-soft); color: var(--color-warning); }
.system-monitoring-metric__icon.is-green { background: var(--color-success-soft); color: #0f766e; }
.system-monitoring-metric > div:last-child {
display: grid;
gap: 2px;
min-width: 0;
}
.system-monitoring-metric span,
.system-monitoring-metric small { color: var(--color-text-muted); font-size: 12px; }
.system-monitoring-metric strong { color: var(--color-text-strong); font-size: 23px; line-height: 1.25; }
.system-monitoring-metric small { overflow: hidden; text-overflow: ellipsis; white-space: nowrap; }
.system-monitoring-main-grid {
align-items: start;
display: grid;
gap: 16px;
grid-template-columns: minmax(0, 1fr) 300px;
}
.system-monitoring-service-metrics { padding: 18px; }
.system-monitoring-service-metrics > header {
align-items: center;
display: flex;
justify-content: space-between;
margin-bottom: 14px;
}
.system-monitoring-service-metrics > header > div { align-items: center; color: var(--color-text-strong); display: flex; gap: 8px; }
.system-monitoring-service-metrics > header > span { color: var(--color-text-muted); font-size: 12px; }
.system-monitoring-service-metric-grid { display: grid; gap: 12px; grid-template-columns: repeat(3, minmax(0, 1fr)); }
.system-monitoring-service-metric-grid article { background: var(--color-bg-subtle); border: 1px solid var(--color-border); border-radius: var(--radius-md); padding: 14px; }
.system-monitoring-service-metric-title { align-items: center; display: flex; justify-content: space-between; margin-bottom: 9px; }
.system-monitoring-service-metric-title > strong { color: var(--color-text-strong); font-size: 14px; }
.system-monitoring-service-metric-row { align-items: center; border-top: 1px solid var(--color-border); display: flex; justify-content: space-between; min-height: 34px; }
.system-monitoring-service-metric-row span { color: var(--color-text-muted); font-size: 12px; }
.system-monitoring-service-metric-row strong { color: var(--color-text); font-size: 13px; }
.system-monitoring-service-metric-empty { color: var(--color-text-subtle); font-size: 12px; line-height: 1.55; padding-top: 8px; }
.system-monitoring-chart-stack {
display: grid;
gap: 16px;
grid-template-columns: repeat(2, minmax(0, 1fr));
}
.system-monitoring-chart-card,
.system-monitoring-services,
.system-monitoring-alerts { padding: 18px; }
.system-monitoring-chart-card header,
.system-monitoring-services > header,
.system-monitoring-alerts > header {
align-items: center;
display: flex;
justify-content: space-between;
margin-bottom: 12px;
}
.system-monitoring-chart-card header div,
.system-monitoring-services > header div,
.system-monitoring-alerts > header div {
align-items: center;
color: var(--color-text-strong);
display: flex;
gap: 8px;
}
.system-monitoring-chart-card header > span { color: var(--color-text); font-weight: var(--font-weight-semibold); }
.system-monitoring-chart-empty {
align-items: center;
color: var(--color-text-subtle);
display: flex;
flex-direction: column;
gap: 8px;
height: 230px;
justify-content: center;
}
.system-monitoring-service-list { display: grid; }
.system-monitoring-service {
align-items: center;
border-bottom: 1px solid var(--color-border);
display: grid;
gap: 9px;
grid-template-columns: 9px 1fr auto;
padding: 13px 0;
}
.system-monitoring-service:last-child { border-bottom: 0; }
.system-monitoring-service__dot { background: var(--color-text-subtle); border-radius: 50%; height: 8px; width: 8px; }
.system-monitoring-service__dot.is-healthy { background: var(--color-success); box-shadow: 0 0 0 3px var(--color-success-soft); }
.system-monitoring-service__dot.is-unhealthy { background: var(--color-danger); box-shadow: 0 0 0 3px var(--color-danger-soft); }
.system-monitoring-service__dot.is-unknown { background: var(--color-text-subtle); box-shadow: 0 0 0 3px var(--color-surface-muted); }
.system-monitoring-service div { display: grid; gap: 1px; min-width: 0; }
.system-monitoring-service strong { color: var(--color-text); font-size: 13px; }
.system-monitoring-service small { color: var(--color-text-subtle); font-size: 11px; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; }
.system-monitoring-service > span:last-child { color: var(--color-text-muted); font-size: 12px; }
.system-monitoring-collector-note {
align-items: flex-start;
background: var(--color-bg-subtle);
border: 1px solid var(--color-border);
border-radius: var(--radius-md);
color: var(--color-text-muted);
display: flex;
font-size: 12px;
gap: 8px;
line-height: 1.55;
margin-top: 14px;
padding: 11px;
}
.system-monitoring-collector-note svg { flex: 0 0 auto; margin-top: 1px; }
.system-monitoring-alerts { overflow: hidden; }
.system-monitoring-alerts > header > span { align-items: center; color: var(--color-text-muted); display: flex; font-size: 12px; gap: 5px; }
.system-monitoring-alert-copy { display: grid; gap: 2px; }
.system-monitoring-alert-copy strong { color: var(--color-text-strong); }
.system-monitoring-alert-copy span { color: var(--color-text-muted); font-size: 12px; }
.is-spinning { animation: system-monitoring-spin 0.9s linear infinite; }
@keyframes system-monitoring-spin { to { transform: rotate(360deg); } }
@media (max-width: 1180px) {
.system-monitoring-health { grid-template-columns: repeat(3, minmax(0, 1fr)); row-gap: 18px; }
.system-monitoring-health__copy { grid-column: span 2; }
.system-monitoring-health__fact:nth-last-child(-n + 2) { border-left: 0; padding-left: 0; }
.system-monitoring-metrics { grid-template-columns: repeat(2, minmax(0, 1fr)); }
.system-monitoring-main-grid { grid-template-columns: minmax(0, 1fr); }
.system-monitoring-services { order: -1; }
.system-monitoring-service-list { grid-template-columns: repeat(3, minmax(0, 1fr)); }
.system-monitoring-service-metric-grid { grid-template-columns: repeat(2, minmax(0, 1fr)); }
.system-monitoring-service { border-bottom: 0; border-right: 1px solid var(--color-border); padding: 11px 12px; }
.system-monitoring-service:nth-child(3n) { border-right: 0; }
}
@media (max-width: 760px) {
.system-monitoring-heading { align-items: stretch; flex-direction: column; }
.system-monitoring-controls { align-items: stretch; flex-direction: column; }
.system-monitoring-range { display: grid; grid-template-columns: repeat(3, 1fr); }
.system-monitoring-title-row { align-items: flex-start; justify-content: space-between; }
.system-monitoring-health { grid-template-columns: repeat(2, minmax(0, 1fr)); }
.system-monitoring-health__copy { grid-column: 1 / -1; }
.system-monitoring-health__fact { border-left: 0; border-top: 1px solid var(--color-border); padding: 12px 0 0; }
.system-monitoring-metrics,
.system-monitoring-chart-stack,
.system-monitoring-service-list,
.system-monitoring-service-metric-grid { grid-template-columns: minmax(0, 1fr); }
.system-monitoring-service-metrics > header { align-items: flex-start; flex-direction: column; gap: 6px; }
.system-monitoring-service { border-bottom: 1px solid var(--color-border); border-right: 0; padding: 13px 0; }
.system-monitoring-chart-card { padding: 14px 10px; }
}
@media (prefers-reduced-motion: reduce) {
.is-spinning { animation: none; }
}
.system-monitoring-service-actions { align-items: center; display: flex; gap: 12px; }
.system-monitoring-threshold-dialog { display: grid; gap: 12px; }
.system-monitoring-threshold-note { align-items: flex-start; background: #eff6ff; border-radius: 10px; color: #475569; display: flex; font-size: 13px; gap: 9px; padding: 12px 14px; }
.system-monitoring-threshold-row { align-items: end; border-bottom: 1px solid #eef2f7; display: grid; gap: 14px; grid-template-columns: minmax(180px, 1fr) 150px 150px; padding: 12px 0; }
.system-monitoring-threshold-row > div:first-child { align-self: center; display: grid; gap: 4px; }
.system-monitoring-threshold-row small { color: #64748b; }
@media (max-width: 760px) {
.system-monitoring-service-actions { align-items: flex-start; flex-direction: column; }
.system-monitoring-threshold-row { grid-template-columns: 1fr 1fr; }
.system-monitoring-threshold-row > div:first-child { grid-column: 1 / -1; }
}
@@ -0,0 +1,418 @@
import { useCallback, useEffect, useMemo, useRef, useState } from 'react';
import type { EChartsOption } from 'echarts';
import {
Activity,
AlertTriangle,
CheckCircle2,
Clock3,
Cpu,
Database,
HardDrive,
MemoryStick,
Network,
RefreshCw,
Server,
Settings2,
ShieldAlert,
} from 'lucide-react';
import {
adminApi,
type InfrastructureAlert,
type InfrastructureAlertSettings,
type InfrastructureAlertThresholds,
type InfrastructureMetricPoint,
type InfrastructureMonitoringOverview,
type InfrastructureMonitoringRange,
} from '@/api/adminApi';
import { Breadcrumb, Button, Chart, Input, Modal, Table, Tag, type TableColumn } from '@/components/ui';
import './AdminSystemMonitoringPage.css';
const RANGE_OPTIONS: Array<{ value: InfrastructureMonitoringRange; label: string }> = [
{ value: '1h', label: '近1小时' },
{ value: '24h', label: '近24小时' },
{ value: '7d', label: '近7天' },
];
const STATUS_COPY = {
healthy: { label: '运行正常', tone: 'success' as const },
warning: { label: '需要关注', tone: 'warning' as const },
critical: { label: '严重告警', tone: 'danger' as const },
unknown: { label: '状态未知', tone: 'neutral' as const },
};
function formatPercent(value: number | null) {
return value === null ? '—' : `${value.toFixed(1)}%`;
}
function formatBytes(value: number | null) {
if (value === null) return '—';
const units = ['B', 'KB', 'MB', 'GB', 'TB'];
let amount = value;
let index = 0;
while (amount >= 1024 && index < units.length - 1) {
amount /= 1024;
index += 1;
}
return `${amount.toFixed(index >= 3 ? 1 : 0)} ${units[index]}`;
}
function formatRate(value: number | null) {
return value === null ? '—' : `${formatBytes(value)}/s`;
}
function totalNetworkRate(receive: number | null | undefined, transmit: number | null | undefined) {
if (receive === null || receive === undefined || transmit === null || transmit === undefined) return null;
return receive + transmit;
}
function formatUptime(value: number | null) {
if (value === null) return '—';
const days = Math.floor(value / 86400);
const hours = Math.floor((value % 86400) / 3600);
return days > 0 ? `${days}${hours}小时` : `${hours}小时`;
}
function formatServiceMetric(value: number | null, unit: 'percent' | 'seconds' | 'count' | 'per_second' | 'bytes') {
if (value === null) return '—';
if (unit === 'percent') return `${value.toFixed(2)}%`;
if (unit === 'seconds') return value < 1 ? `${Math.round(value * 1000)} ms` : `${value.toFixed(1)} s`;
if (unit === 'per_second') return `${value.toFixed(value < 10 ? 2 : 1)}/s`;
if (unit === 'bytes') return formatBytes(value);
return Math.round(value).toLocaleString('zh-CN');
}
function formatTime(value: string | null) {
if (!value) return '暂无采样';
return new Intl.DateTimeFormat('zh-CN', {
month: '2-digit', day: '2-digit', hour: '2-digit', minute: '2-digit', second: '2-digit', hour12: false,
}).format(new Date(value));
}
function formatDuration(startedAt: string) {
const milliseconds = Date.now() - Date.parse(startedAt);
if (!Number.isFinite(milliseconds) || milliseconds < 0) return '—';
const minutes = Math.floor(milliseconds / 60_000);
if (minutes < 60) return `${Math.max(minutes, 1)}分钟`;
const hours = Math.floor(minutes / 60);
return hours < 24 ? `${hours}小时 ${minutes % 60}分钟` : `${Math.floor(hours / 24)}${hours % 24}小时`;
}
function timeLabels(points: InfrastructureMetricPoint[], range: InfrastructureMonitoringRange) {
return points.map((point) => new Intl.DateTimeFormat('zh-CN', range === '7d'
? { month: '2-digit', day: '2-digit', hour: '2-digit', hour12: false }
: { hour: '2-digit', minute: '2-digit', hour12: false })
.format(new Date(point.timestamp)));
}
function makeTrendOption(params: {
range: InfrastructureMonitoringRange;
series: Array<{ name: string; points: InfrastructureMetricPoint[]; color: string }>;
suffix: string;
maximum?: number;
}): EChartsOption {
const first = params.series[0]?.points ?? [];
return {
animationDuration: 280,
color: params.series.map((item) => item.color),
grid: { left: 8, right: 16, top: 34, bottom: 4, containLabel: true },
legend: params.series.length > 1 ? { top: 0, right: 0, textStyle: { color: '#6b7280', fontSize: 12 } } : undefined,
tooltip: {
trigger: 'axis',
valueFormatter: (value) => `${Number(value).toFixed(1)}${params.suffix}`,
},
xAxis: {
type: 'category',
boundaryGap: false,
data: timeLabels(first, params.range),
axisLine: { lineStyle: { color: '#e5e7eb' } },
axisTick: { show: false },
axisLabel: { color: '#9ca3af', hideOverlap: true, margin: 12 },
},
yAxis: {
type: 'value', min: 0, max: params.maximum,
axisLabel: { color: '#9ca3af', formatter: `{value}${params.suffix}` },
splitLine: { lineStyle: { color: '#eef0f3' } },
},
series: params.series.map((item) => ({
name: item.name,
data: item.points.map((point) => point.value),
type: 'line',
smooth: true,
showSymbol: false,
lineStyle: { width: 2.5 },
areaStyle: { opacity: 0.07 },
})),
};
}
function severityTag(severity: InfrastructureAlert['severity']) {
if (severity === 'critical') return <Tag tone="danger"></Tag>;
if (severity === 'warning') return <Tag tone="warning"></Tag>;
return <Tag tone="info"></Tag>;
}
function makeAlertColumns(onMarkRead: (alert: InfrastructureAlert) => void, readingFingerprint: string): Array<TableColumn<InfrastructureAlert>> { return [
{ key: 'severity', title: '级别', width: '82px', render: (record) => severityTag(record.severity) },
{
key: 'alert', title: '告警', width: '280px', render: (record) => (
<div className="system-monitoring-alert-copy"><strong>{record.name}</strong><span>{record.summary}</span></div>
),
},
{ key: 'service', title: '服务 / 实例', width: '190px', render: (record) => record.service || record.instance || '主机资源' },
{ key: 'value', title: '当前值 / 阈值', width: '150px', render: (record) => `${record.currentValue || '—'} / ${record.threshold || '—'}` },
{ key: 'startedAt', title: '开始时间', width: '150px', render: (record) => formatTime(record.startedAt) },
{ key: 'duration', title: '持续时间', width: '120px', render: (record) => formatDuration(record.startedAt) },
{
key: 'actions', title: '操作', width: '112px', render: (record) => record.acknowledged
? <Tag tone="neutral"></Tag>
: <Button disabled={readingFingerprint === record.fingerprint} icon={<CheckCircle2 size={14} />} onClick={() => onMarkRead(record)} size="sm" variant="ghost">{readingFingerprint === record.fingerprint ? '处理中' : '标记已读'}</Button>,
},
]; }
export function AdminSystemMonitoringPage() {
const [range, setRange] = useState<InfrastructureMonitoringRange>('24h');
const [overview, setOverview] = useState<InfrastructureMonitoringOverview | null>(null);
const [loading, setLoading] = useState(true);
const [error, setError] = useState('');
const [settings, setSettings] = useState<InfrastructureAlertSettings | null>(null);
const [draftThresholds, setDraftThresholds] = useState<InfrastructureAlertThresholds>({});
const [showSettings, setShowSettings] = useState(false);
const [settingsError, setSettingsError] = useState('');
const [savingSettings, setSavingSettings] = useState(false);
const [readingFingerprint, setReadingFingerprint] = useState('');
const [readError, setReadError] = useState('');
const requestSequence = useRef(0);
const pendingRequests = useRef(0);
const loadData = useCallback(async (supersede = false) => {
if (!supersede && pendingRequests.current > 0) return;
pendingRequests.current += 1;
const sequence = ++requestSequence.current;
setLoading(true);
try {
const result = await adminApi.getInfrastructureMonitoringOverview(range);
if (sequence !== requestSequence.current) return;
setOverview(result);
setError(result.available ? '' : result.error || '监控数据当前不可用');
} catch (reason) {
if (sequence !== requestSequence.current) return;
setOverview(null);
setError(reason instanceof Error ? reason.message : '监控数据加载失败');
} finally {
if (sequence === requestSequence.current) setLoading(false);
pendingRequests.current -= 1;
}
}, [range]);
const loadSettings = useCallback(async () => {
try {
const result = await adminApi.getInfrastructureAlertThresholds();
setSettings(result);
setDraftThresholds(result.thresholds);
setSettingsError('');
} catch (reason) {
setSettingsError(reason instanceof Error ? reason.message : '告警阈值加载失败');
}
}, []);
const saveSettings = useCallback(async () => {
if (!settings) return;
setSavingSettings(true);
setSettingsError('');
try {
const result = await adminApi.updateInfrastructureAlertThresholds({ configVersion: settings.configVersion, thresholds: draftThresholds });
setSettings(result);
setDraftThresholds(result.thresholds);
setShowSettings(false);
window.dispatchEvent(new Event('cmpp-infrastructure-alert-count-refresh'));
await loadData(true);
} catch (reason) {
setSettingsError(reason instanceof Error ? reason.message : '告警阈值保存失败');
} finally {
setSavingSettings(false);
}
}, [draftThresholds, loadData, settings]);
const markAlertRead = useCallback(async (alert: InfrastructureAlert) => {
setReadingFingerprint(alert.fingerprint);
setReadError('');
try {
const result = await adminApi.markInfrastructureAlertRead(alert.fingerprint, alert.startedAt);
setOverview((current) => current ? {
...current,
alerts: current.alerts.map((item) => item.fingerprint === result.fingerprint && Date.parse(item.startedAt) === Date.parse(result.activeAt)
? { ...item, acknowledged: true, acknowledgedAt: result.acknowledgedAt }
: item),
} : current);
window.dispatchEvent(new Event('cmpp-infrastructure-alert-count-refresh'));
} catch (reason) {
setReadError(reason instanceof Error ? reason.message : '活动告警标记已读失败');
} finally {
setReadingFingerprint('');
}
}, []);
useEffect(() => {
void loadData(true);
void loadSettings();
const intervalId = window.setInterval(() => {
if (document.visibilityState === 'visible') void loadData();
}, 30_000);
const handleVisibility = () => {
if (document.visibilityState === 'visible') void loadData();
};
document.addEventListener('visibilitychange', handleVisibility);
return () => {
requestSequence.current += 1;
window.clearInterval(intervalId);
document.removeEventListener('visibilitychange', handleVisibility);
};
}, [loadData, loadSettings]);
const status = STATUS_COPY[overview?.summary.overallStatus ?? 'unknown'];
const cpuOption = useMemo(() => makeTrendOption({
range, maximum: 100, suffix: '%', series: [{ name: 'CPU', points: overview?.trends.cpuUsagePercent ?? [], color: '#2563eb' }],
}), [overview?.trends.cpuUsagePercent, range]);
const memoryOption = useMemo(() => makeTrendOption({
range, maximum: 100, suffix: '%', series: [{ name: '内存', points: overview?.trends.memoryUsagePercent ?? [], color: '#7c3aed' }],
}), [overview?.trends.memoryUsagePercent, range]);
const diskOption = useMemo(() => makeTrendOption({
range, maximum: 100, suffix: '%', series: [{ name: '根磁盘', points: overview?.trends.diskUsagePercent ?? [], color: '#d97706' }],
}), [overview?.trends.diskUsagePercent, range]);
const networkOption = useMemo(() => makeTrendOption({
range, suffix: ' B/s', series: [
{ name: '接收', points: overview?.trends.networkReceiveBytesPerSecond ?? [], color: '#0f766e' },
{ name: '发送', points: overview?.trends.networkTransmitBytesPerSecond ?? [], color: '#2563eb' },
],
}), [overview?.trends.networkReceiveBytesPerSecond, overview?.trends.networkTransmitBytesPerSecond, range]);
const metrics = overview?.metrics;
const serviceHealthy = overview?.summary.serviceHealthy ?? 0;
const serviceTotal = overview?.summary.serviceTotal ?? 6;
const alertColumns = useMemo(() => makeAlertColumns((alert) => { void markAlertRead(alert); }, readingFingerprint), [markAlertRead, readingFingerprint]);
return (
<section className="page-stack admin-system-monitoring-page">
<div className="page-heading system-monitoring-heading">
<div>
<Breadcrumb items={['系统管理', '系统监控']} />
<div className="system-monitoring-title-row">
<p> Prometheus </p>
<Tag tone={status.tone}>{status.label}</Tag>
</div>
</div>
<div className="system-monitoring-controls">
<div className="system-monitoring-range" aria-label="监控时间范围" role="group">
{RANGE_OPTIONS.map((option) => (
<button
aria-pressed={range === option.value}
className={range === option.value ? 'is-active' : ''}
key={option.value}
onClick={() => setRange(option.value)}
type="button"
>{option.label}</button>
))}
</div>
<Button disabled={loading} icon={<RefreshCw className={loading ? 'is-spinning' : ''} size={16} />} onClick={() => void loadData()} variant="ghost">
{loading ? '刷新中' : '刷新'}
</Button>
</div>
</div>
{error ? (
<div className="system-monitoring-unavailable" role="alert">
<ShieldAlert size={20} />
<div><strong></strong><span>{error}</span></div>
</div>
) : null}
<div className="system-monitoring-health surface">
<div className="system-monitoring-health__copy">
<div className={`system-monitoring-health__mark is-${overview?.summary.overallStatus ?? 'unknown'}`}>
{overview?.summary.overallStatus === 'healthy' ? <CheckCircle2 size={24} /> : <AlertTriangle size={24} />}
</div>
<span></span>
<strong>{status.label}</strong>
<small> {formatTime(overview?.lastSampleAt ?? null)}</small>
</div>
<div className="system-monitoring-health__fact"><span></span><strong>{serviceHealthy}/{serviceTotal}</strong><small></small></div>
<div className="system-monitoring-health__fact"><span></span><strong>{overview?.summary.activeAlerts ?? 0}</strong><small>{overview?.summary.criticalAlerts ?? 0} · {overview?.summary.warningAlerts ?? 0} </small></div>
<div className="system-monitoring-health__fact"><span></span><strong>{metrics?.load1 === null || metrics?.load1 === undefined ? '—' : metrics.load1.toFixed(2)}</strong><small>1</small></div>
<div className="system-monitoring-health__fact"><span></span><strong>{formatUptime(metrics?.uptimeSeconds ?? null)}</strong><small></small></div>
</div>
<div className="system-monitoring-metrics">
<article className="surface system-monitoring-metric"><div className="system-monitoring-metric__icon is-blue"><Cpu size={19} /></div><div><span>CPU 使</span><strong>{formatPercent(metrics?.cpuUsagePercent ?? null)}</strong><small>5</small></div></article>
<article className="surface system-monitoring-metric"><div className="system-monitoring-metric__icon is-violet"><MemoryStick size={19} /></div><div><span>使</span><strong>{formatPercent(metrics?.memoryUsagePercent ?? null)}</strong><small>{formatBytes(metrics?.memoryAvailableBytes ?? null)} / {formatBytes(metrics?.memoryTotalBytes ?? null)}</small></div></article>
<article className="surface system-monitoring-metric"><div className="system-monitoring-metric__icon is-amber"><HardDrive size={19} /></div><div><span>使</span><strong>{formatPercent(metrics?.diskUsagePercent ?? null)}</strong><small>{formatBytes(metrics?.diskAvailableBytes ?? null)} / {formatBytes(metrics?.diskTotalBytes ?? null)}</small></div></article>
<article className="surface system-monitoring-metric"><div className="system-monitoring-metric__icon is-green"><Network size={19} /></div><div><span></span><strong>{formatRate(totalNetworkRate(metrics?.networkReceiveBytesPerSecond, metrics?.networkTransmitBytesPerSecond))}</strong><small> {formatRate(metrics?.networkReceiveBytesPerSecond ?? null)} · {formatRate(metrics?.networkTransmitBytesPerSecond ?? null)}</small></div></article>
</div>
<div className="system-monitoring-main-grid">
<div className="system-monitoring-chart-stack">
<article className="surface system-monitoring-chart-card"><header><div><Cpu size={17} /><strong>CPU </strong></div><span>{formatPercent(metrics?.cpuUsagePercent ?? null)}</span></header>{overview?.trends.cpuUsagePercent.length ? <Chart height={230} option={cpuOption} /> : <EmptyChart />}</article>
<article className="surface system-monitoring-chart-card"><header><div><MemoryStick size={17} /><strong></strong></div><span>{formatPercent(metrics?.memoryUsagePercent ?? null)}</span></header>{overview?.trends.memoryUsagePercent.length ? <Chart height={230} option={memoryOption} /> : <EmptyChart />}</article>
<article className="surface system-monitoring-chart-card"><header><div><HardDrive size={17} /><strong></strong></div><span>{formatPercent(metrics?.diskUsagePercent ?? null)}</span></header>{overview?.trends.diskUsagePercent.length ? <Chart height={230} option={diskOption} /> : <EmptyChart />}</article>
<article className="surface system-monitoring-chart-card"><header><div><Activity size={17} /><strong></strong></div><span>{formatRate(totalNetworkRate(metrics?.networkReceiveBytesPerSecond, metrics?.networkTransmitBytesPerSecond))}</span></header>{overview?.trends.networkReceiveBytesPerSecond.length ? <Chart height={230} option={networkOption} /> : <EmptyChart />}</article>
</div>
<aside className="surface system-monitoring-services">
<header><div><Server size={18} /><strong></strong></div><Tag tone={serviceHealthy === serviceTotal && overview?.available ? 'success' : 'neutral'}>{serviceHealthy}/{serviceTotal} </Tag></header>
<div className="system-monitoring-service-list">
{(overview?.services ?? []).map((service) => (
<div className="system-monitoring-service" key={service.key}>
<span className={`system-monitoring-service__dot is-${service.status}`} />
<div><strong>{service.name}</strong><small>{service.unit}</small></div>
<span>{service.status === 'healthy' ? '正常' : service.status === 'unhealthy' ? '异常' : '未知'}</span>
</div>
))}
{!overview?.services.length ? [
['api', 'API服务'], ['gateway', 'Gateway服务'], ['postgresql', 'PostgreSQL'], ['redis', 'Redis'], ['minio', 'MinIO'], ['nginx', 'Nginx'],
].map(([key, name]) => <div className="system-monitoring-service" key={key}><span className="system-monitoring-service__dot is-unknown" /><div><strong>{name}</strong><small></small></div><span></span></div>) : null}
</div>
<div className="system-monitoring-collector-note"><Database size={16} /><span> Prometheus </span></div>
</aside>
</div>
<section className="surface system-monitoring-service-metrics">
<header>
<div><Database size={18} /><strong></strong></div>
<div className="system-monitoring-service-actions"><span>ID或SQL文本</span><Button icon={<Settings2 size={15} />} onClick={() => setShowSettings(true)} variant="ghost"></Button></div>
</header>
<div className="system-monitoring-service-metric-grid">
{(overview?.serviceMetrics ?? []).map((group) => (
<article key={group.key}>
<div className="system-monitoring-service-metric-title"><strong>{group.name}</strong><Tag tone={group.available ? 'success' : 'neutral'}>{group.available ? '已采集' : '待采集'}</Tag></div>
{group.metrics.length ? group.metrics.map((metric) => <div className="system-monitoring-service-metric-row" key={metric.key}><span>{metric.label}</span><strong>{formatServiceMetric(metric.value, metric.unit)}</strong></div>) : <div className="system-monitoring-service-metric-empty"></div>}
</article>
))}
</div>
</section>
<section className="surface system-monitoring-alerts" id="active-alerts">
<header><div><AlertTriangle size={18} /><strong></strong><Tag tone={overview?.summary.activeAlerts ? 'warning' : 'success'}>{overview?.summary.activeAlerts ?? 0}</Tag></div><span><Clock3 size={14} /> {formatTime(overview?.collectedAt ?? null)}</span></header>
{readError ? <div className="system-monitoring-unavailable" role="alert"><AlertTriangle size={18} /><div><strong></strong><span>{readError}</span></div></div> : null}
<Table columns={alertColumns} data={overview?.alerts ?? []} emptyText={overview?.available ? '当前没有活动告警' : '监控不可用,无法读取活动告警'} pagination={false} rowKey="fingerprint" />
</section>
<Modal footer={<><Button onClick={() => setShowSettings(false)} variant="ghost"></Button><Button disabled={savingSettings || !settings} onClick={() => void saveSettings()}>{savingSettings ? '验证并应用中' : '保存并应用'}</Button></>} onClose={() => setShowSettings(false)} open={showSettings} title="Prometheus 告警阈值设置">
<div className="system-monitoring-threshold-dialog">
<div className="system-monitoring-threshold-note"><ShieldAlert size={17} /><span> promtool Prometheus</span></div>
{settings?.definitions.map((definition) => (
<div className="system-monitoring-threshold-row" key={definition.key}>
<div><strong>{definition.label}</strong><small>{definition.unit}</small></div>
<Input label="警告阈值" max={definition.max} min={definition.min} onChange={(event) => setDraftThresholds((current) => ({ ...current, [definition.key]: { ...current[definition.key], warning: Number(event.target.value) } }))} step={definition.step} type="number" value={draftThresholds[definition.key]?.warning ?? ''} />
<Input label="严重阈值" max={definition.max} min={definition.min} onChange={(event) => setDraftThresholds((current) => ({ ...current, [definition.key]: { ...current[definition.key], critical: Number(event.target.value) } }))} step={definition.step} type="number" value={draftThresholds[definition.key]?.critical ?? ''} />
</div>
))}
{settings?.applyStatus === 'failed' ? <div className="system-monitoring-unavailable"><AlertTriangle size={18} /><div><strong></strong><span>{settings.lastError}</span></div></div> : null}
{settingsError ? <div className="system-monitoring-unavailable"><AlertTriangle size={18} /><div><strong></strong><span>{settingsError}</span></div></div> : null}
</div>
</Modal>
</section>
);
}
function EmptyChart() {
return <div className="system-monitoring-chart-empty"><Activity size={22} /><span></span></div>;
}
+20 -3
View File
@@ -25,6 +25,7 @@ import {
ReceiptText, ReceiptText,
ClipboardList, ClipboardList,
Send, Send,
ServerCog,
ScanSearch, ScanSearch,
Settings, Settings,
Shield, Shield,
@@ -47,6 +48,8 @@ export function AdminLayout() {
function AdminAuthenticatedLayout({ session }: { session: LoginSession }) { function AdminAuthenticatedLayout({ session }: { session: LoginSession }) {
const [pendingAudits, setPendingAudits] = useState(EMPTY_PENDING_AUDITS); const [pendingAudits, setPendingAudits] = useState(EMPTY_PENDING_AUDITS);
const [retirementUnreadCount, setRetirementUnreadCount] = useState(0); const [retirementUnreadCount, setRetirementUnreadCount] = useState(0);
const [securityAlertSummary, setSecurityAlertSummary] = useState({ count: 0, criticalCount: 0 });
const [infrastructureAlertSummary, setInfrastructureAlertSummary] = useState({ count: 0, criticalCount: 0 });
const [sessionLocked, setSessionLocked] = useState(Boolean(session.locked)); const [sessionLocked, setSessionLocked] = useState(Boolean(session.locked));
const loadPendingAuditCount = useCallback(() => { const loadPendingAuditCount = useCallback(() => {
const currentSession = readSession('admin'); const currentSession = readSession('admin');
@@ -55,14 +58,18 @@ function AdminAuthenticatedLayout({ session }: { session: LoginSession }) {
return; return;
} }
// This runs globally and on a timer, so it must not fan out through the full dashboard aggregation. // This runs globally and on a timer, so it must not fan out through the full dashboard aggregation.
Promise.allSettled([adminApi.getPendingAudits(), adminApi.getSignatureRetirementUnreadCount()]) Promise.allSettled([adminApi.getPendingAudits(), adminApi.getSignatureRetirementUnreadCount(), adminApi.getSecurityNotificationSummary(), adminApi.getInfrastructureMonitoringNotificationSummary()])
.then(([audits, retirement]) => { .then(([audits, retirement, security, infrastructure]) => {
setPendingAudits(audits.status === 'fulfilled' ? audits.value : EMPTY_PENDING_AUDITS); setPendingAudits(audits.status === 'fulfilled' ? audits.value : EMPTY_PENDING_AUDITS);
setRetirementUnreadCount(retirement.status === 'fulfilled' ? retirement.value.count : 0); setRetirementUnreadCount(retirement.status === 'fulfilled' ? retirement.value.count : 0);
setSecurityAlertSummary(security.status === 'fulfilled' ? security.value : { count: 0, criticalCount: 0 });
setInfrastructureAlertSummary(infrastructure.status === 'fulfilled' ? infrastructure.value : { count: 0, criticalCount: 0 });
}) })
.catch(() => { .catch(() => {
setPendingAudits(EMPTY_PENDING_AUDITS); setPendingAudits(EMPTY_PENDING_AUDITS);
setRetirementUnreadCount(0); setRetirementUnreadCount(0);
setSecurityAlertSummary({ count: 0, criticalCount: 0 });
setInfrastructureAlertSummary({ count: 0, criticalCount: 0 });
}); });
}, []); }, []);
@@ -77,11 +84,15 @@ function AdminAuthenticatedLayout({ session }: { session: LoginSession }) {
window.addEventListener('focus', onFocus); window.addEventListener('focus', onFocus);
window.addEventListener('cmpp-audit-count-refresh', onAuditRefresh); window.addEventListener('cmpp-audit-count-refresh', onAuditRefresh);
window.addEventListener('cmpp-retirement-count-refresh', onAuditRefresh); window.addEventListener('cmpp-retirement-count-refresh', onAuditRefresh);
window.addEventListener('cmpp-security-alert-count-refresh', onAuditRefresh);
window.addEventListener('cmpp-infrastructure-alert-count-refresh', onAuditRefresh);
return () => { return () => {
window.clearInterval(timer); window.clearInterval(timer);
window.removeEventListener('focus', onFocus); window.removeEventListener('focus', onFocus);
window.removeEventListener('cmpp-audit-count-refresh', onAuditRefresh); window.removeEventListener('cmpp-audit-count-refresh', onAuditRefresh);
window.removeEventListener('cmpp-retirement-count-refresh', onAuditRefresh); window.removeEventListener('cmpp-retirement-count-refresh', onAuditRefresh);
window.removeEventListener('cmpp-security-alert-count-refresh', onAuditRefresh);
window.removeEventListener('cmpp-infrastructure-alert-count-refresh', onAuditRefresh);
}; };
}, [loadPendingAuditCount, session.portal, sessionLocked]); }, [loadPendingAuditCount, session.portal, sessionLocked]);
@@ -95,7 +106,11 @@ function AdminAuthenticatedLayout({ session }: { session: LoginSession }) {
userName={session.user.displayName} userName={session.user.displayName}
userRole="平台管理员" userRole="平台管理员"
onSessionLockedChange={setSessionLocked} onSessionLockedChange={setSessionLocked}
retirementAlert={{ count: retirementUnreadCount, to: '/admin/signature-retirement' }} alertNotifications={[
{ label: '签名清退预警', count: retirementUnreadCount, description: '今日未读且未抑制', to: '/admin/signature-retirement' },
{ label: '安全检测与封禁', count: securityAlertSummary.count, description: securityAlertSummary.criticalCount > 0 ? `${securityAlertSummary.criticalCount} 条严重告警待处置` : '待处置安全告警', to: '/admin/security-detection' },
{ label: '系统监控告警', count: infrastructureAlertSummary.count, description: infrastructureAlertSummary.criticalCount > 0 ? `${infrastructureAlertSummary.criticalCount} 条 Prometheus 严重告警` : 'Prometheus 活动告警', to: '/admin/system-monitoring#active-alerts' },
]}
auditNotifications={[ auditNotifications={[
{ label: '企业认证待审', count: pendingAudits.enterpriseCertifications, to: '/admin/enterprise-audit' }, { label: '企业认证待审', count: pendingAudits.enterpriseCertifications, to: '/admin/enterprise-audit' },
{ label: '短信审核待审', count: pendingAudits.smsAudits, to: '/admin/sms-audit' }, { label: '短信审核待审', count: pendingAudits.smsAudits, to: '/admin/sms-audit' },
@@ -187,6 +202,7 @@ function AdminAuthenticatedLayout({ session }: { session: LoginSession }) {
icon: Shield, icon: Shield,
items: [ items: [
{ label: '风控规则', to: '/admin/risk-rules', icon: Shield }, { label: '风控规则', to: '/admin/risk-rules', icon: Shield },
{ label: '安全检测与封禁', to: '/admin/security-detection', icon: ShieldCheck },
{ label: '签名清退预警', to: '/admin/signature-retirement', icon: AlertTriangle }, { label: '签名清退预警', to: '/admin/signature-retirement', icon: AlertTriangle },
{ label: '企业黑名单', to: '/admin/enterprise-blacklist', icon: UserX }, { label: '企业黑名单', to: '/admin/enterprise-blacklist', icon: UserX },
{ label: '全局黑名单', to: '/admin/global-blacklist', icon: ShieldOff }, { label: '全局黑名单', to: '/admin/global-blacklist', icon: ShieldOff },
@@ -202,6 +218,7 @@ function AdminAuthenticatedLayout({ session }: { session: LoginSession }) {
{ label: '报备字段库', to: '/admin/drainage-fields', icon: Hash }, { label: '报备字段库', to: '/admin/drainage-fields', icon: Hash },
{ label: '引流识别规则', to: '/admin/drainage-detection-rules', icon: ScanSearch }, { label: '引流识别规则', to: '/admin/drainage-detection-rules', icon: ScanSearch },
{ label: '系统日志', to: '/admin/system-logs', icon: FileText }, { label: '系统日志', to: '/admin/system-logs', icon: FileText },
{ label: '系统监控', to: '/admin/system-monitoring', icon: ServerCog },
], ],
}, },
]} ]}
+40 -13
View File
@@ -50,6 +50,10 @@ export type AuditNotificationItem = {
to: string; to: string;
}; };
export type AlertNotificationItem = AuditNotificationItem & {
description: string;
};
type AppShellProps = { type AppShellProps = {
title: string; title: string;
subtitle: string; subtitle: string;
@@ -60,7 +64,7 @@ type AppShellProps = {
userRole: string; userRole: string;
navSections: ShellNavSection[]; navSections: ShellNavSection[];
auditNotifications?: AuditNotificationItem[]; auditNotifications?: AuditNotificationItem[];
retirementAlert?: { count: number; to: string }; alertNotifications?: AlertNotificationItem[];
onSessionLockedChange?: (locked: boolean) => void; onSessionLockedChange?: (locked: boolean) => void;
}; };
@@ -73,7 +77,7 @@ export function AppShell({
userRole, userRole,
navSections, navSections,
auditNotifications = [], auditNotifications = [],
retirementAlert, alertNotifications = [],
onSessionLockedChange, onSessionLockedChange,
}: AppShellProps) { }: AppShellProps) {
const [collapsed, setCollapsed] = useState(false); const [collapsed, setCollapsed] = useState(false);
@@ -81,6 +85,7 @@ export function AppShell({
const [closedSections, setClosedSections] = useState<Record<string, boolean>>({}); const [closedSections, setClosedSections] = useState<Record<string, boolean>>({});
const [userMenuOpen, setUserMenuOpen] = useState(false); const [userMenuOpen, setUserMenuOpen] = useState(false);
const [noticeOpen, setNoticeOpen] = useState(false); const [noticeOpen, setNoticeOpen] = useState(false);
const [alertNoticeOpen, setAlertNoticeOpen] = useState(false);
const [passwordModalOpen, setPasswordModalOpen] = useState(false); const [passwordModalOpen, setPasswordModalOpen] = useState(false);
const [currentPassword, setCurrentPassword] = useState(''); const [currentPassword, setCurrentPassword] = useState('');
const [newPassword, setNewPassword] = useState(''); const [newPassword, setNewPassword] = useState('');
@@ -107,6 +112,10 @@ export function AppShell({
() => auditNotifications.reduce((sum, item) => sum + item.count, 0), () => auditNotifications.reduce((sum, item) => sum + item.count, 0),
[auditNotifications], [auditNotifications],
); );
const alertTotal = useMemo(
() => alertNotifications.reduce((sum, item) => sum + item.count, 0),
[alertNotifications],
);
async function changeOwnPassword() { async function changeOwnPassword() {
if (!currentPassword || newPassword.length < 6) { if (!currentPassword || newPassword.length < 6) {
@@ -417,23 +426,41 @@ export function AppShell({
<button className="icon-button topbar-help" type="button" aria-label="帮助中心"> <button className="icon-button topbar-help" type="button" aria-label="帮助中心">
<CircleHelp size={18} /> <CircleHelp size={18} />
</button> </button>
{retirementAlert ? ( {alertNotifications.length ? (
<Link <div className="notice-menu-wrap">
aria-label={`今日未读且未抑制签名清退预警 ${retirementAlert.count}`} <button
className={['icon-button', retirementAlert.count > 0 ? 'has-dot' : ''].filter(Boolean).join(' ')} aria-expanded={alertNoticeOpen}
title="今日未读且未抑制签名清退预警" aria-haspopup="menu"
to={retirementAlert.to} aria-label="预警通知"
> className={['icon-button', alertTotal > 0 ? 'has-dot' : ''].filter(Boolean).join(' ')}
<Bell size={18} /> onClick={() => { setAlertNoticeOpen((open) => !open); setNoticeOpen(false); }}
{retirementAlert.count > 0 ? <span className="notice-count">{retirementAlert.count}</span> : null} type="button"
</Link> >
<Bell size={18} />
{alertTotal > 0 ? <span className="notice-count">{alertTotal}</span> : null}
</button>
{alertNoticeOpen ? (
<div className="notice-popover notice-popover--alerts" role="menu">
<div className="notice-popover__header">
<strong></strong>
<span className={alertTotal === 0 ? 'is-zero' : ''}>{alertTotal} </span>
</div>
{alertNotifications.map((item) => (
<NavLink key={item.to} onClick={() => setAlertNoticeOpen(false)} role="menuitem" to={item.to}>
<span className="notice-popover__copy"><b>{item.label}</b><small>{item.description}</small></span>
<strong className={item.count === 0 ? 'is-zero' : ''}>{item.count}</strong>
</NavLink>
))}
</div>
) : null}
</div>
) : null} ) : null}
<div className="notice-menu-wrap"> <div className="notice-menu-wrap">
<button <button
aria-expanded={noticeOpen} aria-expanded={noticeOpen}
aria-haspopup="menu" aria-haspopup="menu"
className={['icon-button', auditTotal > 0 ? 'has-dot' : ''].filter(Boolean).join(' ')} className={['icon-button', auditTotal > 0 ? 'has-dot' : ''].filter(Boolean).join(' ')}
onClick={() => setNoticeOpen((open) => !open)} onClick={() => { setNoticeOpen((open) => !open); setAlertNoticeOpen(false); }}
type="button" type="button"
aria-label="通知" aria-label="通知"
> >
+4
View File
@@ -38,6 +38,8 @@ import { AdminSignatureAuditPage } from '@/apps/admin/AdminSignatureAuditPage';
import { AdminSignatureRetirementPage } from '@/apps/admin/AdminSignatureRetirementPage'; import { AdminSignatureRetirementPage } from '@/apps/admin/AdminSignatureRetirementPage';
import { AdminDrainageAuditPage } from '@/apps/admin/AdminDrainageAuditPage'; import { AdminDrainageAuditPage } from '@/apps/admin/AdminDrainageAuditPage';
import { AdminSystemLogsPage } from '@/apps/admin/AdminSystemLogsPage'; import { AdminSystemLogsPage } from '@/apps/admin/AdminSystemLogsPage';
import { AdminSystemMonitoringPage } from '@/apps/admin/system-monitoring/AdminSystemMonitoringPage';
import { AdminSecurityDetectionPage } from '@/apps/admin/security-detection/AdminSecurityDetectionPage';
import { AdminTemplateAuditPage } from '@/apps/admin/AdminTemplateAuditPage'; import { AdminTemplateAuditPage } from '@/apps/admin/AdminTemplateAuditPage';
import { AdminUsersPage } from '@/apps/admin/AdminUsersPage'; import { AdminUsersPage } from '@/apps/admin/AdminUsersPage';
import { AdminEnterpriseAuditPage } from '@/apps/admin/AdminEnterpriseAuditPage'; import { AdminEnterpriseAuditPage } from '@/apps/admin/AdminEnterpriseAuditPage';
@@ -142,6 +144,8 @@ export function AppRoutes() {
<Route path="drainage-fields" element={<AdminDrainageFieldsPage />} /> <Route path="drainage-fields" element={<AdminDrainageFieldsPage />} />
<Route path="drainage-detection-rules" element={<AdminDrainageDetectionRulesPage />} /> <Route path="drainage-detection-rules" element={<AdminDrainageDetectionRulesPage />} />
<Route path="system-logs" element={<AdminSystemLogsPage />} /> <Route path="system-logs" element={<AdminSystemLogsPage />} />
<Route path="system-monitoring" element={<AdminSystemMonitoringPage />} />
<Route path="security-detection" element={<AdminSecurityDetectionPage />} />
<Route path="*" element={<PagePlaceholder />} /> <Route path="*" element={<PagePlaceholder />} />
</Route> </Route>
</Routes> </Routes>
+9 -3
View File
@@ -9424,10 +9424,14 @@
.downstream-page-size { display: inline-flex; align-items: center; gap: 8px; color: var(--text-muted); font-size: 13px; } .downstream-page-size { display: inline-flex; align-items: center; gap: 8px; color: var(--text-muted); font-size: 13px; }
.downstream-page-size select { min-width: 76px; height: 36px; border: 1px solid var(--border); border-radius: 8px; background: var(--surface); color: var(--text); padding: 0 10px; } .downstream-page-size select { min-width: 76px; height: 36px; border: 1px solid var(--border); border-radius: 8px; background: var(--surface); color: var(--text); padding: 0 10px; }
.downstream-requeue-task-heading { gap: 20px; } .downstream-requeue-task-heading { gap: 20px; }
.downstream-requeue-task-card > .downstream-requeue-task-heading { padding: var(--space-6) var(--space-6) var(--space-5); }
.downstream-requeue-task-heading__actions { display: flex; align-items: center; gap: 8px; } .downstream-requeue-task-heading__actions { display: flex; align-items: center; gap: 8px; }
.downstream-requeue-task-heading__actions .ui-field { min-width: 150px; } .downstream-requeue-task-heading__actions .ui-field { min-width: 150px; }
.downstream-requeue-task-list { display: grid; border-top: 1px solid var(--border); } .downstream-requeue-task-list { display: grid; margin: 0 var(--space-6); overflow: hidden; border: 1px solid var(--border); border-radius: var(--radius-lg); background: var(--color-surface); }
.downstream-requeue-task-list article { display: grid; grid-template-columns: minmax(190px, .9fr) minmax(260px, 1.35fr) minmax(250px, 1fr) auto; gap: 22px; align-items: center; padding: 18px 4px; border-bottom: 1px solid var(--border); } .downstream-requeue-task-list article { display: grid; grid-template-columns: minmax(190px, .9fr) minmax(260px, 1.35fr) minmax(250px, 1fr) auto; gap: 22px; align-items: center; padding: 18px var(--space-5); border-bottom: 1px solid var(--border); }
.downstream-requeue-task-list article:last-of-type { border-bottom: 0; }
.downstream-requeue-task-list > .muted { margin: 0; padding: var(--space-6); text-align: center; }
.downstream-requeue-task-card > .ui-pagination { padding: var(--space-5) var(--space-6) var(--space-6); }
.downstream-requeue-task-list article > div { min-width: 0; } .downstream-requeue-task-list article > div { min-width: 0; }
.downstream-requeue-task-list__identity, .downstream-requeue-task-list__scope, .downstream-requeue-task-list__progress { display: grid; gap: 5px; } .downstream-requeue-task-list__identity, .downstream-requeue-task-list__scope, .downstream-requeue-task-list__progress { display: grid; gap: 5px; }
.downstream-requeue-task-list__identity > strong { color: var(--text-strong); font-size: 14px; font-variant-numeric: tabular-nums; } .downstream-requeue-task-list__identity > strong { color: var(--text-strong); font-size: 14px; font-variant-numeric: tabular-nums; }
@@ -9447,6 +9451,8 @@
.downstream-requeue-preview__distribution { display: grid; gap: 8px; } .downstream-requeue-preview__distribution { display: grid; gap: 8px; }
.downstream-requeue-preview__distribution > span { color: var(--text-muted); font-size: 13px; } .downstream-requeue-preview__distribution > span { color: var(--text-muted); font-size: 13px; }
.downstream-requeue-preview__distribution > div { display: flex; flex-wrap: wrap; gap: 8px; } .downstream-requeue-preview__distribution > div { display: flex; flex-wrap: wrap; gap: 8px; }
.downstream-requeue-reason { padding: 16px; border: 1px solid var(--border); border-radius: var(--radius-lg); background: var(--surface-subtle, #f8fafc); }
.downstream-requeue-reason .ui-textarea { min-height: 124px; background: var(--color-surface); line-height: 1.65; }
.downstream-requeue-warning { display: flex; align-items: flex-start; gap: 10px; padding: 14px; border: 1px solid #fed7aa; border-radius: 10px; background: var(--warning-soft, #fff7ed); color: var(--warning-text, #9a3412); } .downstream-requeue-warning { display: flex; align-items: flex-start; gap: 10px; padding: 14px; border: 1px solid #fed7aa; border-radius: 10px; background: var(--warning-soft, #fff7ed); color: var(--warning-text, #9a3412); }
.downstream-requeue-warning > div { display: grid; gap: 4px; } .downstream-requeue-warning > div { display: grid; gap: 4px; }
.downstream-requeue-warning p { margin: 0; color: inherit; line-height: 1.6; } .downstream-requeue-warning p { margin: 0; color: inherit; line-height: 1.6; }
@@ -9460,4 +9466,4 @@
.downstream-requeue-detail-items__head { background: var(--surface-subtle, #f8fafc); color: var(--text-muted); font-size: 12px; font-weight: 600; } .downstream-requeue-detail-items__head { background: var(--surface-subtle, #f8fafc); color: var(--text-muted); font-size: 12px; font-weight: 600; }
.downstream-requeue-detail-items time { color: var(--text-muted); font-size: 12px; } .downstream-requeue-detail-items time { color: var(--text-muted); font-size: 12px; }
@media (max-width: 1100px) { .downstream-requeue-task-list article { grid-template-columns: 1fr 1.4fr; } .downstream-requeue-task-list__actions { justify-content: flex-start; } .downstream-requeue-preview__summary { grid-template-columns: repeat(2, 1fr); } } @media (max-width: 1100px) { .downstream-requeue-task-list article { grid-template-columns: 1fr 1.4fr; } .downstream-requeue-task-list__actions { justify-content: flex-start; } .downstream-requeue-preview__summary { grid-template-columns: repeat(2, 1fr); } }
@media (max-width: 780px) { .downstream-requeue-task-heading, .downstream-requeue-task-heading__actions { align-items: stretch; flex-direction: column; } .downstream-requeue-task-list article, .downstream-requeue-preview__summary, .downstream-requeue-detail-filter { grid-template-columns: 1fr; } .downstream-requeue-detail-items { border: 0; overflow: visible; gap: 10px; } .downstream-requeue-detail-items__head { display: none !important; } .downstream-requeue-detail-items > div { grid-template-columns: 1fr; gap: 6px; padding: 14px; border: 1px solid var(--border); border-radius: 10px; } } @media (max-width: 780px) { .downstream-requeue-task-card > .downstream-requeue-task-heading { padding: var(--space-5) var(--space-4) var(--space-4); } .downstream-requeue-task-heading, .downstream-requeue-task-heading__actions { align-items: stretch; flex-direction: column; } .downstream-requeue-task-list { margin: 0 var(--space-4); } .downstream-requeue-task-list article, .downstream-requeue-preview__summary, .downstream-requeue-detail-filter { grid-template-columns: 1fr; } .downstream-requeue-task-card > .ui-pagination { padding: var(--space-4); } .downstream-requeue-detail-items { border: 0; overflow: visible; gap: 10px; } .downstream-requeue-detail-items__head { display: none !important; } .downstream-requeue-detail-items > div { grid-template-columns: 1fr; gap: 6px; padding: 14px; border: 1px solid var(--border); border-radius: 10px; } }
+25
View File
@@ -571,6 +571,31 @@
text-align: right; text-align: right;
} }
.notice-popover--alerts {
min-width: 292px;
}
.notice-popover--alerts a {
gap: var(--space-4);
min-height: 58px;
}
.notice-popover__copy {
align-items: flex-start;
display: grid;
gap: 2px;
}
.notice-popover__copy b {
color: var(--color-text-strong);
font-weight: var(--font-weight-semibold);
}
.notice-popover__copy small {
color: var(--color-text-muted);
font-size: 11px;
}
.page-heading__actions { .page-heading__actions {
align-items: center; align-items: center;
display: flex; display: flex;
+25 -1
View File
@@ -6,6 +6,9 @@ REPO_URL="${REPO_URL:-http://175.27.255.91:3000/hectorzhao/lislgosms.git}"
BRANCH="${BRANCH:-main}" BRANCH="${BRANCH:-main}"
PUBLIC_HTTP_PORT="${PUBLIC_HTTP_PORT:-12026}" PUBLIC_HTTP_PORT="${PUBLIC_HTTP_PORT:-12026}"
API_PORT="${API_PORT:-3000}" API_PORT="${API_PORT:-3000}"
API_HOST="${API_HOST:-127.0.0.1}"
API_METRICS_HOST="${API_METRICS_HOST:-127.0.0.1}"
API_METRICS_PORT="${API_METRICS_PORT:-9464}"
API_ENABLE_SEND_WORKER="${API_ENABLE_SEND_WORKER:-true}" API_ENABLE_SEND_WORKER="${API_ENABLE_SEND_WORKER:-true}"
API_SEND_WORKER_CONCURRENCY="${API_SEND_WORKER_CONCURRENCY:-50}" API_SEND_WORKER_CONCURRENCY="${API_SEND_WORKER_CONCURRENCY:-50}"
GATEWAY_CONTROL_ADDR="${GATEWAY_CONTROL_ADDR:-127.0.0.1:8090}" GATEWAY_CONTROL_ADDR="${GATEWAY_CONTROL_ADDR:-127.0.0.1:8090}"
@@ -31,6 +34,10 @@ OPERATION_LOG_ARCHIVE_INTERVAL_MS="${OPERATION_LOG_ARCHIVE_INTERVAL_MS:-86400000
SMS_RECEIPT_TIMEOUT_SCAN_ENABLED="${SMS_RECEIPT_TIMEOUT_SCAN_ENABLED:-true}" SMS_RECEIPT_TIMEOUT_SCAN_ENABLED="${SMS_RECEIPT_TIMEOUT_SCAN_ENABLED:-true}"
SMS_RECEIPT_TIMEOUT_HOURS="${SMS_RECEIPT_TIMEOUT_HOURS:-72}" SMS_RECEIPT_TIMEOUT_HOURS="${SMS_RECEIPT_TIMEOUT_HOURS:-72}"
SMS_RECEIPT_TIMEOUT_SCAN_INTERVAL_MS="${SMS_RECEIPT_TIMEOUT_SCAN_INTERVAL_MS:-300000}" SMS_RECEIPT_TIMEOUT_SCAN_INTERVAL_MS="${SMS_RECEIPT_TIMEOUT_SCAN_INTERVAL_MS:-300000}"
PROMETHEUS_URL="${PROMETHEUS_URL:-http://127.0.0.1:9090}"
PROMETHEUS_QUERY_TIMEOUT_MS="${PROMETHEUS_QUERY_TIMEOUT_MS:-5000}"
SECURITY_EVENT_TOKEN="${SECURITY_EVENT_TOKEN:-$(openssl rand -hex 32 | tr -d '\n')}"
SECURITY_BUILTIN_PROTECTED_NETWORKS="${SECURITY_BUILTIN_PROTECTED_NETWORKS:-${CMPP_PUBLIC_HOST}/32}"
if [[ "$(id -u)" -ne 0 ]]; then if [[ "$(id -u)" -ne 0 ]]; then
echo "Run as root." >&2 echo "Run as root." >&2
@@ -43,7 +50,7 @@ install_packages() {
log "Installing OS packages" log "Installing OS packages"
if command -v apt-get >/dev/null 2>&1; then if command -v apt-get >/dev/null 2>&1; then
apt-get update apt-get update
DEBIAN_FRONTEND=noninteractive apt-get install -y ca-certificates curl gnupg git nginx redis-server postgresql postgresql-contrib build-essential tar gzip xz-utils openssl DEBIAN_FRONTEND=noninteractive apt-get install -y ca-certificates curl gnupg git nginx redis-server postgresql postgresql-contrib build-essential tar gzip xz-utils openssl fail2ban nftables
elif command -v dnf >/dev/null 2>&1; then elif command -v dnf >/dev/null 2>&1; then
dnf install -y ca-certificates curl git nginx redis postgresql-server postgresql-contrib gcc gcc-c++ make tar gzip xz openssl dnf install -y ca-certificates curl git nginx redis postgresql-server postgresql-contrib gcc gcc-c++ make tar gzip xz openssl
if [[ ! -d /var/lib/pgsql/data/base ]]; then if [[ ! -d /var/lib/pgsql/data/base ]]; then
@@ -179,6 +186,9 @@ write_env() {
cat >/etc/cmpp-platform/cmpp-platform.env <<EOF cat >/etc/cmpp-platform/cmpp-platform.env <<EOF
NODE_ENV=production NODE_ENV=production
API_PORT=${API_PORT} API_PORT=${API_PORT}
API_HOST=${API_HOST}
API_METRICS_HOST=${API_METRICS_HOST}
API_METRICS_PORT=${API_METRICS_PORT}
API_ENABLE_SEND_WORKER=${API_ENABLE_SEND_WORKER} API_ENABLE_SEND_WORKER=${API_ENABLE_SEND_WORKER}
API_SEND_WORKER_CONCURRENCY=${API_SEND_WORKER_CONCURRENCY} API_SEND_WORKER_CONCURRENCY=${API_SEND_WORKER_CONCURRENCY}
DATABASE_URL=postgresql://${DB_USER}:${DB_PASSWORD}@127.0.0.1:5432/${DB_NAME}?schema=public DATABASE_URL=postgresql://${DB_USER}:${DB_PASSWORD}@127.0.0.1:5432/${DB_NAME}?schema=public
@@ -192,6 +202,8 @@ OPERATION_LOG_ARCHIVE_INTERVAL_MS=${OPERATION_LOG_ARCHIVE_INTERVAL_MS}
SMS_RECEIPT_TIMEOUT_SCAN_ENABLED=${SMS_RECEIPT_TIMEOUT_SCAN_ENABLED} SMS_RECEIPT_TIMEOUT_SCAN_ENABLED=${SMS_RECEIPT_TIMEOUT_SCAN_ENABLED}
SMS_RECEIPT_TIMEOUT_HOURS=${SMS_RECEIPT_TIMEOUT_HOURS} SMS_RECEIPT_TIMEOUT_HOURS=${SMS_RECEIPT_TIMEOUT_HOURS}
SMS_RECEIPT_TIMEOUT_SCAN_INTERVAL_MS=${SMS_RECEIPT_TIMEOUT_SCAN_INTERVAL_MS} SMS_RECEIPT_TIMEOUT_SCAN_INTERVAL_MS=${SMS_RECEIPT_TIMEOUT_SCAN_INTERVAL_MS}
PROMETHEUS_URL=${PROMETHEUS_URL}
PROMETHEUS_QUERY_TIMEOUT_MS=${PROMETHEUS_QUERY_TIMEOUT_MS}
MINIO_ENDPOINT=127.0.0.1:9000 MINIO_ENDPOINT=127.0.0.1:9000
MINIO_ACCESS_KEY=${MINIO_ROOT_USER} MINIO_ACCESS_KEY=${MINIO_ROOT_USER}
MINIO_SECRET_KEY=${MINIO_ROOT_PASSWORD} MINIO_SECRET_KEY=${MINIO_ROOT_PASSWORD}
@@ -204,6 +216,10 @@ GATEWAY_CMPP_ADDR=${GATEWAY_CMPP_ADDR}
CMPP_PUBLIC_HOST=${CMPP_PUBLIC_HOST} CMPP_PUBLIC_HOST=${CMPP_PUBLIC_HOST}
CMPP_PUBLIC_PORT=${CMPP_PUBLIC_PORT} CMPP_PUBLIC_PORT=${CMPP_PUBLIC_PORT}
API_BASE_URL=http://127.0.0.1:${API_PORT}/api API_BASE_URL=http://127.0.0.1:${API_PORT}/api
SECURITY_EVENT_TOKEN=${SECURITY_EVENT_TOKEN}
SECURITY_AGENT_SOCKET=/run/cmpp-security-agent/agent.sock
TRUSTED_PROXY_IPS=127.0.0.1,::1
SECURITY_BUILTIN_PROTECTED_NETWORKS=${SECURITY_BUILTIN_PROTECTED_NETWORKS}
EOF EOF
chmod 600 /etc/cmpp-platform/cmpp-platform.env chmod 600 /etc/cmpp-platform/cmpp-platform.env
cat >/etc/cmpp-platform/minio.env <<EOF cat >/etc/cmpp-platform/minio.env <<EOF
@@ -217,6 +233,11 @@ write_services() {
log "Writing systemd and nginx configuration" log "Writing systemd and nginx configuration"
local node_bin local node_bin
node_bin="$(command -v node)" node_bin="$(command -v node)"
getent group cmpp-security >/dev/null || groupadd --system cmpp-security
id cmpp-api >/dev/null 2>&1 || useradd --system --home-dir /nonexistent --shell /usr/sbin/nologin --gid cmpp-security cmpp-api
install -d -m 0750 /etc/nginx/snippets
touch /etc/nginx/snippets/cmpp-security-deny.conf
chmod 0640 /etc/nginx/snippets/cmpp-security-deny.conf
cat >/etc/systemd/system/cmpp-minio.service <<'EOF' cat >/etc/systemd/system/cmpp-minio.service <<'EOF'
[Unit] [Unit]
Description=CMPP MinIO object storage Description=CMPP MinIO object storage
@@ -240,6 +261,8 @@ Description=CMPP Platform API
After=network.target postgresql.service redis.service cmpp-minio.service After=network.target postgresql.service redis.service cmpp-minio.service
[Service] [Service]
User=cmpp-api
Group=cmpp-security
WorkingDirectory=${APP_DIR}/api WorkingDirectory=${APP_DIR}/api
EnvironmentFile=/etc/cmpp-platform/cmpp-platform.env EnvironmentFile=/etc/cmpp-platform/cmpp-platform.env
ExecStart=${node_bin} dist/main.js ExecStart=${node_bin} dist/main.js
@@ -282,6 +305,7 @@ server {
gzip_min_length 1024; gzip_min_length 1024;
gzip_comp_level 5; gzip_comp_level 5;
gzip_types application/json application/javascript text/javascript text/css text/plain text/csv image/svg+xml; gzip_types application/json application/javascript text/javascript text/css text/plain text/csv image/svg+xml;
include /etc/nginx/snippets/cmpp-security-deny.conf;
location /api/ { location /api/ {
proxy_pass http://127.0.0.1:${API_PORT}/api/; proxy_pass http://127.0.0.1:${API_PORT}/api/;
+14 -1
View File
@@ -42,6 +42,7 @@ npm --prefix api ci --include=dev
echo "[deploy] Verifying dependency security mitigations" echo "[deploy] Verifying dependency security mitigations"
npm run security:verify npm run security:verify
npm run deploy:verify
echo "[deploy] Generating Prisma client and applying migrations" echo "[deploy] Generating Prisma client and applying migrations"
npm --prefix api run prisma:generate npm --prefix api run prisma:generate
@@ -52,6 +53,7 @@ rm -rf api/dist api/tsconfig.build.tsbuildinfo "$APP_DIR/dist/cmpp-gateway"
npm run build npm run build
npm --prefix api run build npm --prefix api run build
(cd gateway && GOPROXY="${GOPROXY:-https://goproxy.cn,direct}" /usr/local/bin/go build -o "$APP_DIR/dist/cmpp-gateway" ./cmd/gateway) (cd gateway && GOPROXY="${GOPROXY:-https://goproxy.cn,direct}" /usr/local/bin/go build -o "$APP_DIR/dist/cmpp-gateway" ./cmd/gateway)
(cd gateway && GOPROXY="${GOPROXY:-https://goproxy.cn,direct}" /usr/local/bin/go build -o "$APP_DIR/dist/cmpp-security-agent" ./cmd/security-agent)
chmod 755 "$APP_DIR/dist" "$APP_DIR/dist/assets" chmod 755 "$APP_DIR/dist" "$APP_DIR/dist/assets"
find "$APP_DIR/dist/assets" -type d -exec chmod 755 {} + find "$APP_DIR/dist/assets" -type d -exec chmod 755 {} +
find "$APP_DIR/dist/assets" -type f -exec chmod 644 {} + find "$APP_DIR/dist/assets" -type f -exec chmod 644 {} +
@@ -64,14 +66,24 @@ chmod 600 "$ADMIN_CREDENTIAL_FILE" || true
echo "[deploy] Ensuring runtime log directories" echo "[deploy] Ensuring runtime log directories"
install -d -m 0755 "$APP_DIR/logs/api" "$APP_DIR/logs/gateway" install -d -m 0755 "$APP_DIR/logs/api" "$APP_DIR/logs/gateway"
echo "[deploy] Installing restricted security boundary"
bash "$APP_DIR/tools/security/install-security-agent.sh"
echo "[deploy] Ensuring HTTP response compression" echo "[deploy] Ensuring HTTP response compression"
cat >/etc/nginx/conf.d/cmpp-compression.conf <<'EOF' compression_config=/etc/nginx/conf.d/cmpp-compression.conf
: >"$compression_config"
if grep -RqsE --exclude='cmpp-compression.conf' '^[[:space:]]*gzip[[:space:]]+on;' \
/etc/nginx/nginx.conf /etc/nginx/conf.d /etc/nginx/sites-enabled 2>/dev/null; then
echo "[deploy] Reusing existing Nginx gzip configuration"
else
cat >"$compression_config" <<'EOF'
gzip on; gzip on;
gzip_vary on; gzip_vary on;
gzip_min_length 1024; gzip_min_length 1024;
gzip_comp_level 5; gzip_comp_level 5;
gzip_types application/json application/javascript text/javascript text/css text/plain text/csv image/svg+xml; gzip_types application/json application/javascript text/javascript text/css text/plain text/csv image/svg+xml;
EOF EOF
fi
nginx -t nginx -t
echo "[deploy] Restarting services" echo "[deploy] Restarting services"
@@ -85,6 +97,7 @@ else
systemctl enable --now cmpp-api cmpp-gateway nginx systemctl enable --now cmpp-api cmpp-gateway nginx
fi fi
systemctl restart cmpp-gateway systemctl restart cmpp-gateway
systemctl restart cmpp-security-agent
systemctl restart cmpp-api systemctl restart cmpp-api
systemctl restart nginx systemctl restart nginx
@@ -0,0 +1,29 @@
import { readFileSync } from 'node:fs';
import { resolve } from 'node:path';
const deploy = readFileSync(resolve(import.meta.dirname, 'production-deploy.sh'), 'utf8');
const bootstrap = readFileSync(resolve(import.meta.dirname, 'production-bootstrap.sh'), 'utf8');
const apiMain = readFileSync(resolve(import.meta.dirname, '../../api/src/main.ts'), 'utf8');
const required = [
'compression_config=/etc/nginx/conf.d/cmpp-compression.conf',
': >"$compression_config"',
"--exclude='cmpp-compression.conf'",
"'^[[:space:]]*gzip[[:space:]]+on;'",
'Reusing existing Nginx gzip configuration',
'cat >"$compression_config"',
];
for (const marker of required) {
if (!deploy.includes(marker)) throw new Error(`production deploy is missing the idempotent Nginx compression guard: ${marker}`);
}
if (deploy.includes("cat >/etc/nginx/conf.d/cmpp-compression.conf <<'EOF'")) {
throw new Error('production deploy still writes a duplicate global gzip directive unconditionally');
}
for (const marker of ['API_HOST="${API_HOST:-127.0.0.1}"', 'API_HOST=${API_HOST}']) {
if (!bootstrap.includes(marker)) throw new Error(`production bootstrap is missing the loopback API binding: ${marker}`);
}
if (!apiMain.includes("process.env.API_HOST?.trim() || '127.0.0.1'") || !apiMain.includes('app.listen(port, host)')) {
throw new Error('NestJS API must bind to API_HOST and default to loopback');
}
console.log('Production deployment verified: Nginx compression is idempotent and NestJS defaults to loopback.');
+34
View File
@@ -0,0 +1,34 @@
# CMPP Prometheus 监控配置
本目录提供系统监控第一版所需的真实采集配置。运营端不嵌入Prometheus或Grafana页面;NestJS仅从本机Prometheus读取固定指标,再由平台原生UI展示。
在目标Debian/Ubuntu测试服务器上,以root执行:
```bash
cd /opt/cmpp-platform
bash tools/monitoring/install-prometheus-monitoring.sh
bash tools/monitoring/install-service-exporters.sh
```
第一个脚本安装Prometheus与Node Exporter,备份已有配置并校验规则;第二个脚本安装PostgreSQL、Redis和Nginx Exporter,开启MinIO回环原生指标。API指标仅监听`127.0.0.1:9464`Gateway指标复用回环控制端口`8090`。9090、9100、9187、9121、9113和9464均不得对公网开放。两个脚本均会打印恢复资产路径。
安装后将下列配置写入`/etc/cmpp-platform/cmpp-platform.env`,再按正常发布窗口重启API
```dotenv
PROMETHEUS_URL=http://127.0.0.1:9090
PROMETHEUS_QUERY_TIMEOUT_MS=5000
```
校验命令:
```bash
promtool check config /etc/prometheus/prometheus.yml
promtool check rules /etc/prometheus/cmpp-alerts.yml
curl -fsS http://127.0.0.1:9090/-/ready
curl -fsS 'http://127.0.0.1:9090/api/v1/query?query=up'
ss -lnt | grep -E ':(9090|9100)'
curl -fsS http://127.0.0.1:9464/metrics
curl -fsS http://127.0.0.1:8090/metrics
```
完整架构、PromQL口径、故障语义和验收标准见`docs/prometheus-system-monitoring-design-20260814.md`
+349
View File
@@ -0,0 +1,349 @@
groups:
- name: cmpp-service-recording
interval: 15s
rules:
- record: cmpp:service_api:requests_per_second
expr: sum(rate(cmpp_api_http_requests_total[5m]))
- record: cmpp:service_api:error_percent
expr: 100 * sum(rate(cmpp_api_http_requests_total{status=~"5.."}[5m])) / clamp_min(sum(rate(cmpp_api_http_requests_total[5m])), 0.001)
- record: cmpp:service_api:latency_p95_seconds
expr: histogram_quantile(0.95, sum by (le) (rate(cmpp_api_http_request_duration_seconds_bucket[5m])))
- record: cmpp:service_api:event_loop_p99_seconds
expr: cmpp_api_nodejs_event_loop_lag_p99_seconds
- record: cmpp:service_gateway:submits_per_second
expr: sum(rate(cmpp_gateway_submit_total[5m]))
- record: cmpp:service_gateway:failure_percent
expr: 100 * sum(rate(cmpp_gateway_submit_total{result="failed"}[5m])) / clamp_min(sum(rate(cmpp_gateway_submit_total[5m])), 0.001)
- record: cmpp:service_gateway:queue_pending
expr: cmpp_gateway_submit_queue_pending
- record: cmpp:service_gateway:queue_lag
expr: cmpp_gateway_submit_queue_lag
- record: cmpp:service_gateway:queue_oldest_seconds
expr: cmpp_gateway_submit_queue_oldest_pending_age_seconds
- record: cmpp:service_postgresql:connection_percent
expr: 100 * sum(pg_stat_activity_count) / clamp_min(max(pg_settings_max_connections), 1)
- record: cmpp:service_postgresql:deadlocks_15m
expr: sum(increase(pg_stat_database_deadlocks[15m]))
- record: cmpp:service_redis:memory_percent
expr: (100 * redis_memory_used_bytes / redis_memory_max_bytes) and on(instance) (redis_memory_max_bytes > 0)
- record: cmpp:service_redis:memory_used_bytes
expr: redis_memory_used_bytes
- record: cmpp:service_redis:evictions_5m
expr: increase(redis_evicted_keys_total[5m])
- record: cmpp:service_redis:connected_clients
expr: redis_connected_clients
- record: cmpp:service_nginx:connections_active
expr: nginx_connections_active
- record: cmpp:service_nginx:requests_per_second
expr: rate(nginx_http_requests_total[5m])
- record: cmpp:service_minio:capacity_percent
expr: 100 * (1 - minio_cluster_capacity_usable_free_bytes / minio_cluster_capacity_usable_total_bytes)
- record: cmpp:service_minio:usage_bytes
expr: minio_cluster_usage_total_bytes
- record: cmpp:service_minio:objects
expr: minio_cluster_usage_object_total
- record: cmpp:service_minio:drives_offline
expr: minio_cluster_drive_offline_total
- name: cmpp-host-resources
rules:
- alert: NodeExporterDown
expr: up{job="node"} == 0
for: 2m
labels:
severity: critical
service: node-exporter
annotations:
summary: 主机指标采集不可用
description: Prometheus连续2分钟无法采集Node Exporter。
currentValue: "{{ $value }}"
threshold: "up = 1"
- alert: HostCpuUsageWarning
expr: (100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 80) and (100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) <= 90)
for: 10m
labels:
severity: warning
service: host
annotations:
summary: CPU使用率持续偏高
description: 主机CPU使用率连续10分钟高于80%。
currentValue: "{{ printf \"%.1f\" $value }}%"
threshold: "80%"
- alert: HostCpuUsageCritical
expr: 100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 90
for: 5m
labels:
severity: critical
service: host
annotations:
summary: CPU使用率严重超限
description: 主机CPU使用率连续5分钟高于90%。
currentValue: "{{ printf \"%.1f\" $value }}%"
threshold: "90%"
- alert: HostMemoryUsageWarning
expr: ((1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100 > 85) and ((1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100 <= 95)
for: 10m
labels:
severity: warning
service: host
annotations:
summary: 内存使用率持续偏高
description: 主机可用内存连续10分钟低于15%。
currentValue: "{{ printf \"%.1f\" $value }}%"
threshold: "85%"
- alert: HostMemoryUsageCritical
expr: (1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100 > 95
for: 5m
labels:
severity: critical
service: host
annotations:
summary: 内存使用率严重超限
description: 主机可用内存连续5分钟低于5%。
currentValue: "{{ printf \"%.1f\" $value }}%"
threshold: "95%"
- alert: HostRootDiskUsageWarning
expr: ((1 - node_filesystem_avail_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"} / node_filesystem_size_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"}) * 100 > 80) and ((1 - node_filesystem_avail_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"} / node_filesystem_size_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"}) * 100 <= 90)
for: 15m
labels:
severity: warning
service: host
annotations:
summary: 根文件系统空间不足
description: 根文件系统使用率连续15分钟高于80%。
currentValue: "{{ printf \"%.1f\" $value }}%"
threshold: "80%"
- alert: HostRootDiskUsageCritical
expr: (1 - node_filesystem_avail_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"} / node_filesystem_size_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"}) * 100 > 90
for: 5m
labels:
severity: critical
service: host
annotations:
summary: 根文件系统空间严重不足
description: 根文件系统使用率连续5分钟高于90%。
currentValue: "{{ printf \"%.1f\" $value }}%"
threshold: "90%"
- alert: HostRootInodeUsageWarning
expr: ((1 - node_filesystem_files_free{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"} / node_filesystem_files{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"}) * 100 > 80) and ((1 - node_filesystem_files_free{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"} / node_filesystem_files{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"}) * 100 <= 90)
for: 15m
labels:
severity: warning
service: host
annotations:
summary: 根文件系统inode余量偏低
description: 根文件系统inode使用率连续15分钟高于80%。
currentValue: "{{ printf \"%.1f\" $value }}%"
threshold: "80%"
- alert: HostRootInodeUsageCritical
expr: (1 - node_filesystem_files_free{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"} / node_filesystem_files{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"}) * 100 > 90
for: 5m
labels:
severity: critical
service: host
annotations:
summary: 根文件系统inode严重不足
description: 根文件系统inode使用率连续5分钟高于90%。
currentValue: "{{ printf \"%.1f\" $value }}%"
threshold: "90%"
- alert: HostCpuIowaitWarning
expr: (avg by (instance) (rate(node_cpu_seconds_total{mode="iowait"}[5m])) * 100 > 20) and (avg by (instance) (rate(node_cpu_seconds_total{mode="iowait"}[5m])) * 100 <= 35)
for: 10m
labels:
severity: warning
service: host
annotations:
summary: CPU iowait持续偏高
description: 主机CPU iowait连续10分钟高于20%,请检查磁盘I/O。
currentValue: "{{ printf \"%.1f\" $value }}%"
threshold: "20%"
- alert: HostCpuIowaitCritical
expr: avg by (instance) (rate(node_cpu_seconds_total{mode="iowait"}[5m])) * 100 > 35
for: 10m
labels:
severity: critical
service: host
annotations:
summary: CPU iowait严重超限
description: 主机CPU iowait连续10分钟高于35%,磁盘I/O可能已成为瓶颈。
currentValue: "{{ printf \"%.1f\" $value }}%"
threshold: "35%"
- name: cmpp-core-services
rules:
- alert: CmppCoreServiceInactive
expr: node_systemd_unit_state{name=~"cmpp-api\\.service|cmpp-gateway\\.service|postgresql\\.service|redis(-server)?\\.service|cmpp-minio\\.service|nginx\\.service",state="active"} == 0
for: 2m
labels:
severity: critical
service: "{{ $labels.name }}"
annotations:
summary: CMPP核心服务未处于active状态
description: "systemd服务 {{ $labels.name }} 连续2分钟未处于active状态。"
currentValue: "{{ $value }}"
threshold: "active = 1"
- name: cmpp-api-runtime
rules:
- alert: CmppApiMetricsDown
expr: up{job="cmpp-api"} == 0
for: 2m
labels: { severity: critical, service: api }
annotations: { summary: "API指标采集不可用", description: "Prometheus连续2分钟无法读取API内部指标端点。", currentValue: "{{ $value }}", threshold: "up = 1" }
- alert: CmppApiHttpErrorRateWarning
expr: (sum(rate(cmpp_api_http_requests_total{status=~"5.."}[5m])) / clamp_min(sum(rate(cmpp_api_http_requests_total[5m])), 0.001) > 0.01) and (sum(rate(cmpp_api_http_requests_total{status=~"5.."}[5m])) / clamp_min(sum(rate(cmpp_api_http_requests_total[5m])), 0.001) <= 0.05) and sum(increase(cmpp_api_http_requests_total{status=~"5.."}[5m])) >= 5
for: 5m
labels: { severity: warning, service: api }
annotations: { summary: "API 5xx错误率偏高", description: "API 5xx错误率连续5分钟高于1%,且窗口内至少5次错误。", currentValue: "{{ printf \"%.2f\" $value }}", threshold: "1%" }
- alert: CmppApiHttpErrorRateCritical
expr: (sum(rate(cmpp_api_http_requests_total{status=~"5.."}[5m])) / clamp_min(sum(rate(cmpp_api_http_requests_total[5m])), 0.001) > 0.05) and sum(increase(cmpp_api_http_requests_total{status=~"5.."}[5m])) >= 5
for: 5m
labels: { severity: critical, service: api }
annotations: { summary: "API 5xx错误率严重超限", description: "API 5xx错误率连续5分钟高于5%。", currentValue: "{{ printf \"%.2f\" $value }}", threshold: "5%" }
- alert: CmppApiLatencyWarning
expr: (histogram_quantile(0.95, sum by (le) (rate(cmpp_api_http_request_duration_seconds_bucket[10m]))) > 1) and (histogram_quantile(0.95, sum by (le) (rate(cmpp_api_http_request_duration_seconds_bucket[10m]))) <= 3)
for: 10m
labels: { severity: warning, service: api }
annotations: { summary: "API P95响应偏慢", description: "API P95响应时间连续10分钟超过1秒。", currentValue: "{{ printf \"%.3f\" $value }}s", threshold: "1s" }
- alert: CmppApiLatencyCritical
expr: histogram_quantile(0.95, sum by (le) (rate(cmpp_api_http_request_duration_seconds_bucket[5m]))) > 3
for: 5m
labels: { severity: critical, service: api }
annotations: { summary: "API P95响应严重超时", description: "API P95响应时间连续5分钟超过3秒。", currentValue: "{{ printf \"%.3f\" $value }}s", threshold: "3s" }
- alert: CmppApiEventLoopLagWarning
expr: (cmpp_api_nodejs_event_loop_lag_p99_seconds > 0.2) and (cmpp_api_nodejs_event_loop_lag_p99_seconds <= 1)
for: 10m
labels: { severity: warning, service: api }
annotations: { summary: "API事件循环延迟偏高", description: "Node.js事件循环P99延迟连续10分钟超过200ms。", currentValue: "{{ printf \"%.3f\" $value }}s", threshold: "0.2s" }
- alert: CmppApiEventLoopLagCritical
expr: cmpp_api_nodejs_event_loop_lag_p99_seconds > 1
for: 5m
labels: { severity: critical, service: api }
annotations: { summary: "API事件循环严重阻塞", description: "Node.js事件循环P99延迟连续5分钟超过1秒。", currentValue: "{{ printf \"%.3f\" $value }}s", threshold: "1s" }
- name: cmpp-gateway-runtime
rules:
- alert: CmppGatewayMetricsDown
expr: up{job="cmpp-gateway"} == 0
for: 2m
labels: { severity: critical, service: gateway }
annotations: { summary: "Gateway指标采集不可用", description: "Prometheus连续2分钟无法读取Gateway指标。", currentValue: "{{ $value }}", threshold: "up = 1" }
- alert: CmppGatewaySubmitWorkerDown
expr: cmpp_gateway_submit_worker_up == 0
for: 1m
labels: { severity: critical, service: gateway }
annotations: { summary: "Gateway提交消费者未运行", description: "Gateway进程存活,但提交消费者未成功初始化。", currentValue: "{{ $value }}", threshold: "1" }
- alert: CmppGatewayUpstreamConnectionShortage
expr: cmpp_gateway_upstream_connections{state="connected"} < cmpp_gateway_upstream_connections{state="desired"}
for: 2m
labels: { severity: critical, service: gateway }
annotations: { summary: "Gateway上游连接不足", description: "实际上游CMPP连接数连续2分钟低于期望数。", currentValue: "{{ $value }}", threshold: "connected = desired" }
- alert: CmppGatewayQueueDelayedWarning
expr: (cmpp_gateway_submit_queue_oldest_pending_age_seconds > 30) and (cmpp_gateway_submit_queue_oldest_pending_age_seconds <= 120)
for: 2m
labels: { severity: warning, service: gateway }
annotations: { summary: "Gateway提交队列开始延迟", description: "Redis Stream最旧pending消息等待超过30秒。", currentValue: "{{ printf \"%.0f\" $value }}s", threshold: "30s" }
- alert: CmppGatewayQueueDelayedCritical
expr: cmpp_gateway_submit_queue_oldest_pending_age_seconds > 120
for: 2m
labels: { severity: critical, service: gateway }
annotations: { summary: "Gateway提交队列严重延迟", description: "Redis Stream最旧pending消息等待超过120秒。", currentValue: "{{ printf \"%.0f\" $value }}s", threshold: "120s" }
- name: cmpp-data-services
rules:
- alert: PostgresExporterDown
expr: up{job="postgresql"} == 0 or pg_up == 0
for: 2m
labels: { severity: critical, service: postgresql }
annotations: { summary: "PostgreSQL指标或数据库不可用", description: "PostgreSQL Exporter或其数据库连接连续2分钟不可用。", currentValue: "{{ $value }}", threshold: "up = 1" }
- alert: PostgresConnectionsWarning
expr: (sum(pg_stat_activity_count) / clamp_min(max(pg_settings_max_connections), 1) > 0.70) and (sum(pg_stat_activity_count) / clamp_min(max(pg_settings_max_connections), 1) <= 0.85)
for: 10m
labels: { severity: warning, service: postgresql }
annotations: { summary: "PostgreSQL连接使用率偏高", description: "数据库连接数连续10分钟超过上限的70%。", currentValue: "{{ printf \"%.2f\" $value }}", threshold: "70%" }
- alert: PostgresConnectionsCritical
expr: sum(pg_stat_activity_count) / clamp_min(max(pg_settings_max_connections), 1) > 0.85
for: 5m
labels: { severity: critical, service: postgresql }
annotations: { summary: "PostgreSQL连接即将耗尽", description: "数据库连接数连续5分钟超过上限的85%。", currentValue: "{{ printf \"%.2f\" $value }}", threshold: "85%" }
- alert: PostgresDeadlocksDetected
expr: sum(increase(pg_stat_database_deadlocks[15m])) > 0
for: 1m
labels: { severity: warning, service: postgresql }
annotations: { summary: "PostgreSQL发生死锁", description: "15分钟窗口内检测到数据库死锁。", currentValue: "{{ $value }}", threshold: "0" }
- alert: RedisExporterDown
expr: up{job="redis"} == 0 or redis_up == 0
for: 2m
labels: { severity: critical, service: redis }
annotations: { summary: "Redis指标或服务不可用", description: "Redis Exporter或Redis连接连续2分钟不可用。", currentValue: "{{ $value }}", threshold: "up = 1" }
- alert: RedisMemoryWarning
expr: (redis_memory_max_bytes > 0) and (redis_memory_used_bytes / redis_memory_max_bytes > 0.70) and (redis_memory_used_bytes / redis_memory_max_bytes <= 0.85)
for: 10m
labels: { severity: warning, service: redis }
annotations: { summary: "Redis内存使用率偏高", description: "Redis内存连续10分钟超过maxmemory的70%。", currentValue: "{{ printf \"%.2f\" $value }}", threshold: "70%" }
- alert: RedisMemoryCritical
expr: (redis_memory_max_bytes > 0) and (redis_memory_used_bytes / redis_memory_max_bytes > 0.85)
for: 5m
labels: { severity: critical, service: redis }
annotations: { summary: "Redis内存即将耗尽", description: "Redis内存连续5分钟超过maxmemory的85%。", currentValue: "{{ printf \"%.2f\" $value }}", threshold: "85%" }
- alert: RedisUnexpectedEvictions
expr: increase(redis_evicted_keys_total[5m]) > 0
for: 1m
labels: { severity: critical, service: redis }
annotations: { summary: "Redis发生Key淘汰", description: "Redis承载队列和运行状态,5分钟内不应出现淘汰。", currentValue: "{{ $value }}", threshold: "0" }
- alert: RedisRejectedConnections
expr: increase(redis_rejected_connections_total[5m]) > 0
for: 1m
labels: { severity: critical, service: redis }
annotations: { summary: "Redis拒绝连接", description: "5分钟内Redis出现被拒绝连接。", currentValue: "{{ $value }}", threshold: "0" }
- name: cmpp-storage-and-edge
rules:
- alert: MinioMetricsDown
expr: up{job="minio"} == 0
for: 2m
labels: { severity: critical, service: minio }
annotations: { summary: "MinIO指标采集不可用", description: "Prometheus连续2分钟无法读取MinIO原生指标。", currentValue: "{{ $value }}", threshold: "up = 1" }
- alert: MinioCapacityWarning
expr: (100 * (1 - minio_cluster_capacity_usable_free_bytes / minio_cluster_capacity_usable_total_bytes) > 80) and (100 * (1 - minio_cluster_capacity_usable_free_bytes / minio_cluster_capacity_usable_total_bytes) <= 90)
for: 15m
labels: { severity: warning, service: minio }
annotations: { summary: "MinIO存储容量偏高", description: "MinIO可用容量使用率连续15分钟超过80%。", currentValue: "{{ printf \"%.1f\" $value }}%", threshold: "80%" }
- alert: MinioCapacityCritical
expr: 100 * (1 - minio_cluster_capacity_usable_free_bytes / minio_cluster_capacity_usable_total_bytes) > 90
for: 5m
labels: { severity: critical, service: minio }
annotations: { summary: "MinIO存储容量即将耗尽", description: "MinIO可用容量使用率连续5分钟超过90%。", currentValue: "{{ printf \"%.1f\" $value }}%", threshold: "90%" }
- alert: MinioDriveOffline
expr: minio_cluster_drive_offline_total > 0
for: 1m
labels: { severity: critical, service: minio }
annotations: { summary: "MinIO存储盘离线", description: "MinIO检测到离线存储盘。", currentValue: "{{ $value }}", threshold: "0" }
- alert: NginxExporterDown
expr: up{job="nginx"} == 0 or nginx_up == 0
for: 2m
labels: { severity: critical, service: nginx }
annotations: { summary: "Nginx指标或状态页不可用", description: "Nginx Exporter或回环stub_status连续2分钟不可用。", currentValue: "{{ $value }}", threshold: "up = 1" }
- name: cmpp-monitoring-self
rules:
- alert: PrometheusScrapeSlow
expr: scrape_duration_seconds / scrape_interval_seconds > 0.8
for: 5m
labels: { severity: warning, service: prometheus }
annotations: { summary: "Prometheus采集接近超时", description: "采集耗时连续5分钟超过采集周期的80%。", currentValue: "{{ printf \"%.2f\" $value }}", threshold: "80%" }
- alert: PrometheusRuleEvaluationFailures
expr: increase(prometheus_rule_evaluation_failures_total[5m]) > 0
for: 1m
labels: { severity: critical, service: prometheus }
annotations: { summary: "Prometheus告警规则计算失败", description: "5分钟内出现告警规则计算失败。", currentValue: "{{ $value }}", threshold: "0" }
+103
View File
@@ -0,0 +1,103 @@
groups:
- name: cmpp-managed-thresholds
rules:
- alert: HostCpuUsageWarning
expr: (100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 80) and (100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) <= 90)
for: 10m
labels: { severity: warning, service: host }
annotations: { summary: "主机 CPU 使用率达到警告阈值", description: "主机 CPU 使用率持续超过80%。", currentValue: "{{ $value }}", threshold: "80%" }
- alert: HostCpuUsageCritical
expr: 100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 90
for: 5m
labels: { severity: critical, service: host }
annotations: { summary: "主机 CPU 使用率达到严重阈值", description: "主机 CPU 使用率持续超过90%。", currentValue: "{{ $value }}", threshold: "90%" }
- alert: HostMemoryUsageWarning
expr: ((1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100 > 85) and ((1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100 <= 95)
for: 10m
labels: { severity: warning, service: host }
annotations: { summary: "主机内存使用率达到警告阈值", description: "主机内存使用率持续超过85%。", currentValue: "{{ $value }}", threshold: "85%" }
- alert: HostMemoryUsageCritical
expr: (1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100 > 95
for: 5m
labels: { severity: critical, service: host }
annotations: { summary: "主机内存使用率达到严重阈值", description: "主机内存使用率持续超过95%。", currentValue: "{{ $value }}", threshold: "95%" }
- alert: HostRootDiskUsageWarning
expr: ((1 - node_filesystem_avail_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"} / node_filesystem_size_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"}) * 100 > 80) and ((1 - node_filesystem_avail_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"} / node_filesystem_size_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"}) * 100 <= 90)
for: 15m
labels: { severity: warning, service: host }
annotations: { summary: "根磁盘使用率达到警告阈值", description: "根磁盘使用率持续超过80%。", currentValue: "{{ $value }}", threshold: "80%" }
- alert: HostRootDiskUsageCritical
expr: (1 - node_filesystem_avail_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"} / node_filesystem_size_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"}) * 100 > 90
for: 5m
labels: { severity: critical, service: host }
annotations: { summary: "根磁盘使用率达到严重阈值", description: "根磁盘使用率持续超过90%。", currentValue: "{{ $value }}", threshold: "90%" }
- alert: CmppApiHttpErrorRateWarning
expr: (100 * sum(rate(cmpp_api_http_requests_total{status=~"5.."}[5m])) / clamp_min(sum(rate(cmpp_api_http_requests_total[5m])), 0.001) > 1) and (100 * sum(rate(cmpp_api_http_requests_total{status=~"5.."}[5m])) / clamp_min(sum(rate(cmpp_api_http_requests_total[5m])), 0.001) <= 5) and (sum(increase(cmpp_api_http_requests_total{status=~"5.."}[5m])) >= 5)
for: 5m
labels: { severity: warning, service: api }
annotations: { summary: "API 5xx 错误率达到警告阈值", description: "API 5xx 错误率持续超过1%。", currentValue: "{{ $value }}", threshold: "1%" }
- alert: CmppApiHttpErrorRateCritical
expr: (100 * sum(rate(cmpp_api_http_requests_total{status=~"5.."}[5m])) / clamp_min(sum(rate(cmpp_api_http_requests_total[5m])), 0.001) > 5) and (sum(increase(cmpp_api_http_requests_total{status=~"5.."}[5m])) >= 5)
for: 5m
labels: { severity: critical, service: api }
annotations: { summary: "API 5xx 错误率达到严重阈值", description: "API 5xx 错误率持续超过5%。", currentValue: "{{ $value }}", threshold: "5%" }
- alert: CmppApiLatencyWarning
expr: (histogram_quantile(0.95, sum by (le) (rate(cmpp_api_http_request_duration_seconds_bucket[10m]))) > 1) and (histogram_quantile(0.95, sum by (le) (rate(cmpp_api_http_request_duration_seconds_bucket[10m]))) <= 3)
for: 10m
labels: { severity: warning, service: api }
annotations: { summary: "API P95 响应时间达到警告阈值", description: "API P95 响应时间持续超过1秒。", currentValue: "{{ $value }}", threshold: "1秒" }
- alert: CmppApiLatencyCritical
expr: histogram_quantile(0.95, sum by (le) (rate(cmpp_api_http_request_duration_seconds_bucket[10m]))) > 3
for: 5m
labels: { severity: critical, service: api }
annotations: { summary: "API P95 响应时间达到严重阈值", description: "API P95 响应时间持续超过3秒。", currentValue: "{{ $value }}", threshold: "3秒" }
- alert: CmppApiEventLoopLagWarning
expr: (cmpp_api_nodejs_event_loop_lag_p99_seconds > 0.2) and (cmpp_api_nodejs_event_loop_lag_p99_seconds <= 1)
for: 10m
labels: { severity: warning, service: api }
annotations: { summary: "API 事件循环 P99 达到警告阈值", description: "API 事件循环 P99 持续超过0.2秒。", currentValue: "{{ $value }}", threshold: "0.2秒" }
- alert: CmppApiEventLoopLagCritical
expr: cmpp_api_nodejs_event_loop_lag_p99_seconds > 1
for: 5m
labels: { severity: critical, service: api }
annotations: { summary: "API 事件循环 P99 达到严重阈值", description: "API 事件循环 P99 持续超过1秒。", currentValue: "{{ $value }}", threshold: "1秒" }
- alert: CmppGatewayQueueDelayedWarning
expr: (cmpp_gateway_submit_queue_oldest_pending_age_seconds > 30) and (cmpp_gateway_submit_queue_oldest_pending_age_seconds <= 120)
for: 2m
labels: { severity: warning, service: gateway }
annotations: { summary: "Gateway 最旧 pending 达到警告阈值", description: "Gateway 最旧 pending 持续超过30秒。", currentValue: "{{ $value }}", threshold: "30秒" }
- alert: CmppGatewayQueueDelayedCritical
expr: cmpp_gateway_submit_queue_oldest_pending_age_seconds > 120
for: 2m
labels: { severity: critical, service: gateway }
annotations: { summary: "Gateway 最旧 pending 达到严重阈值", description: "Gateway 最旧 pending 持续超过120秒。", currentValue: "{{ $value }}", threshold: "120秒" }
- alert: PostgresConnectionsWarning
expr: (100 * sum(pg_stat_activity_count) / clamp_min(max(pg_settings_max_connections), 1) > 70) and (100 * sum(pg_stat_activity_count) / clamp_min(max(pg_settings_max_connections), 1) <= 85)
for: 10m
labels: { severity: warning, service: postgresql }
annotations: { summary: "PostgreSQL 连接使用率达到警告阈值", description: "PostgreSQL 连接使用率持续超过70%。", currentValue: "{{ $value }}", threshold: "70%" }
- alert: PostgresConnectionsCritical
expr: 100 * sum(pg_stat_activity_count) / clamp_min(max(pg_settings_max_connections), 1) > 85
for: 5m
labels: { severity: critical, service: postgresql }
annotations: { summary: "PostgreSQL 连接使用率达到严重阈值", description: "PostgreSQL 连接使用率持续超过85%。", currentValue: "{{ $value }}", threshold: "85%" }
- alert: RedisMemoryWarning
expr: (100 * redis_memory_used_bytes / redis_memory_max_bytes > 70) and (100 * redis_memory_used_bytes / redis_memory_max_bytes <= 85) and (redis_memory_max_bytes > 0)
for: 10m
labels: { severity: warning, service: redis }
annotations: { summary: "Redis 内存使用率达到警告阈值", description: "Redis 内存使用率持续超过70%。", currentValue: "{{ $value }}", threshold: "70%" }
- alert: RedisMemoryCritical
expr: (100 * redis_memory_used_bytes / redis_memory_max_bytes > 85) and (redis_memory_max_bytes > 0)
for: 5m
labels: { severity: critical, service: redis }
annotations: { summary: "Redis 内存使用率达到严重阈值", description: "Redis 内存使用率持续超过85%。", currentValue: "{{ $value }}", threshold: "85%" }
- alert: MinioCapacityWarning
expr: (100 * (1 - minio_cluster_capacity_usable_free_bytes / minio_cluster_capacity_usable_total_bytes) > 80) and (100 * (1 - minio_cluster_capacity_usable_free_bytes / minio_cluster_capacity_usable_total_bytes) <= 90)
for: 15m
labels: { severity: warning, service: minio }
annotations: { summary: "MinIO 容量使用率达到警告阈值", description: "MinIO 容量使用率持续超过80%。", currentValue: "{{ $value }}", threshold: "80%" }
- alert: MinioCapacityCritical
expr: 100 * (1 - minio_cluster_capacity_usable_free_bytes / minio_cluster_capacity_usable_total_bytes) > 90
for: 5m
labels: { severity: critical, service: minio }
annotations: { summary: "MinIO 容量使用率达到严重阈值", description: "MinIO 容量使用率持续超过90%。", currentValue: "{{ $value }}", threshold: "90%" }
@@ -0,0 +1,100 @@
#!/usr/bin/env bash
set -Eeuo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROMETHEUS_RETENTION_TIME="${PROMETHEUS_RETENTION_TIME:-30d}"
PROMETHEUS_RETENTION_SIZE="${PROMETHEUS_RETENTION_SIZE:-8GB}"
if [[ "$(id -u)" -ne 0 ]]; then
echo "Run as root." >&2
exit 1
fi
if ! command -v apt-get >/dev/null 2>&1; then
echo "This installer currently supports Debian/Ubuntu apt packages only." >&2
exit 1
fi
log() { printf '\n[%s] %s\n' "$(date '+%F %T')" "$*"; }
log "Installing Prometheus and Node Exporter packages"
apt-get update
DEBIAN_FRONTEND=noninteractive apt-get install -y prometheus prometheus-node-exporter curl iproute2
prometheus_bin="$(command -v prometheus)"
node_exporter_bin="$(command -v prometheus-node-exporter)"
promtool_bin="$(command -v promtool)"
backup_dir="/etc/prometheus/cmpp-backups/$(date '+%Y%m%d-%H%M%S')"
mkdir -p "$backup_dir" /etc/systemd/system/prometheus.service.d /etc/systemd/system/prometheus-node-exporter.service.d
for config_file in /etc/prometheus/prometheus.yml /etc/prometheus/cmpp-alerts.yml /etc/prometheus/cmpp-alerts-source.yml; do
if [[ -f "$config_file" ]]; then
cp --preserve=mode,timestamps "$config_file" "$backup_dir/$(basename "$config_file")"
fi
done
if [[ -f /etc/systemd/system/prometheus.service.d/cmpp-monitoring.conf ]]; then
cp --preserve=mode,timestamps /etc/systemd/system/prometheus.service.d/cmpp-monitoring.conf "$backup_dir/prometheus-service-override.conf"
fi
if [[ -f /etc/systemd/system/prometheus-node-exporter.service.d/cmpp-monitoring.conf ]]; then
cp --preserve=mode,timestamps /etc/systemd/system/prometheus-node-exporter.service.d/cmpp-monitoring.conf "$backup_dir/node-exporter-service-override.conf"
fi
log "Installing platform-owned scrape and alert configuration"
install -o root -g root -m 0644 "$SCRIPT_DIR/prometheus.yml" /etc/prometheus/prometheus.yml
install -o root -g root -m 0644 "$SCRIPT_DIR/cmpp-alerts.yml" /etc/prometheus/cmpp-alerts-source.yml
# 可配置规则由 API 管理;基础规则必须排除同名项,否则 Prometheus 会同时计算旧阈值和新阈值。
awk '
BEGIN {
split("HostCpuUsageWarning HostCpuUsageCritical HostMemoryUsageWarning HostMemoryUsageCritical HostRootDiskUsageWarning HostRootDiskUsageCritical CmppApiHttpErrorRateWarning CmppApiHttpErrorRateCritical CmppApiLatencyWarning CmppApiLatencyCritical CmppApiEventLoopLagWarning CmppApiEventLoopLagCritical CmppGatewayQueueDelayedWarning CmppGatewayQueueDelayedCritical PostgresConnectionsWarning PostgresConnectionsCritical RedisMemoryWarning RedisMemoryCritical MinioCapacityWarning MinioCapacityCritical", names, " ")
for (i in names) dropped[names[i]] = 1
}
/^ - name:/ { skip = 0 }
/^ - alert:/ { skip = ($3 in dropped) }
!skip { print }
' "$SCRIPT_DIR/cmpp-alerts.yml" > /etc/prometheus/cmpp-alerts.yml
chown root:root /etc/prometheus/cmpp-alerts.yml
chmod 0644 /etc/prometheus/cmpp-alerts.yml
# SGID ensures API原子rename生成的新规则继续继承prometheus组,否则reload会因不可读返回500。
install -d -o cmpp-api -g prometheus -m 2750 /var/lib/cmpp-platform/monitoring
if [[ ! -f /var/lib/cmpp-platform/monitoring/cmpp-managed-alerts.yml ]]; then
install -o cmpp-api -g prometheus -m 0640 "$SCRIPT_DIR/cmpp-managed-alerts.yml" /var/lib/cmpp-platform/monitoring/cmpp-managed-alerts.yml
fi
cat >/etc/systemd/system/prometheus.service.d/cmpp-monitoring.conf <<EOF
[Service]
ExecStart=
ExecStart=${prometheus_bin} --config.file=/etc/prometheus/prometheus.yml --storage.tsdb.path=/var/lib/prometheus/metrics2 --storage.tsdb.retention.time=${PROMETHEUS_RETENTION_TIME} --storage.tsdb.retention.size=${PROMETHEUS_RETENTION_SIZE} --web.listen-address=127.0.0.1:9090 --web.enable-lifecycle
EOF
cat >/etc/systemd/system/prometheus-node-exporter.service.d/cmpp-monitoring.conf <<EOF
[Service]
ExecStart=
ExecStart=${node_exporter_bin} --web.listen-address=127.0.0.1:9100 --collector.systemd --collector.systemd.unit-include='cmpp-api\\.service|cmpp-gateway\\.service|postgresql\\.service|redis(-server)?\\.service|cmpp-minio\\.service|nginx\\.service' --collector.filesystem.mount-points-exclude='^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/)'
EOF
log "Validating Prometheus configuration before restart"
"$promtool_bin" check rules /etc/prometheus/cmpp-alerts.yml
"$promtool_bin" check rules /var/lib/cmpp-platform/monitoring/cmpp-managed-alerts.yml
"$promtool_bin" check config /etc/prometheus/prometheus.yml
systemctl daemon-reload
systemctl enable prometheus prometheus-node-exporter
systemctl restart prometheus-node-exporter
systemctl restart prometheus
for _ in $(seq 1 30); do
if curl -fsS http://127.0.0.1:9090/-/ready >/dev/null; then
break
fi
sleep 1
done
curl -fsS http://127.0.0.1:9090/-/ready >/dev/null
curl -fsS http://127.0.0.1:9100/metrics >/dev/null
if ss -lnt | grep -Eq '(^|[[:space:]])(0\.0\.0\.0|\[::\]):(9090|9100)([[:space:]]|$)'; then
echo "Prometheus monitoring ports unexpectedly listen on a wildcard address." >&2
exit 1
fi
log "Prometheus monitoring is ready on loopback only"
echo "Configuration backup: $backup_dir"
echo "The CMPP API and Gateway were not restarted."
@@ -0,0 +1,92 @@
#!/usr/bin/env bash
set -Eeuo pipefail
if [[ "$(id -u)" -ne 0 ]]; then echo "Run as root." >&2; exit 1; fi
for command_name in apt-get systemctl nginx curl; do command -v "$command_name" >/dev/null || { echo "Missing command: $command_name" >&2; exit 1; }; done
log() { printf '\n[%s] %s\n' "$(date '+%F %T')" "$*"; }
backup_dir="/etc/prometheus/cmpp-backups/$(date '+%Y%m%d-%H%M%S')-service-exporters"
mkdir -p "$backup_dir"
for config_file in /etc/cmpp-platform/minio.env /etc/nginx/conf.d/cmpp-monitoring-status.conf /etc/cmpp-platform/monitoring-exporters.env; do
[[ -f "$config_file" ]] && cp --preserve=mode,timestamps "$config_file" "$backup_dir/$(basename "$config_file")"
done
log "Installing bounded service exporters"
apt-get update
DEBIAN_FRONTEND=noninteractive apt-get install -y prometheus-postgres-exporter prometheus-redis-exporter prometheus-nginx-exporter
[[ -f /etc/cmpp-platform/cmpp-platform.env ]] || { echo "Missing platform environment file." >&2; exit 1; }
set -a
# shellcheck disable=SC1091
. /etc/cmpp-platform/cmpp-platform.env
set +a
database_base="${DATABASE_URL%%\?*}"
cat >/etc/cmpp-platform/monitoring-exporters.env <<EOF
DATA_SOURCE_NAME=${database_base}?sslmode=disable
REDIS_ADDR=redis://${REDIS_HOST:-127.0.0.1}:${REDIS_PORT:-6379}
EOF
chmod 0600 /etc/cmpp-platform/monitoring-exporters.env
write_override() {
local unit="$1" executable="$2" arguments="$3"
local directory="/etc/systemd/system/${unit}.service.d"
mkdir -p "$directory"
[[ -f "$directory/cmpp-monitoring.conf" ]] && cp --preserve=mode,timestamps "$directory/cmpp-monitoring.conf" "$backup_dir/${unit}-override.conf"
cat >"$directory/cmpp-monitoring.conf" <<EOF
[Service]
EnvironmentFile=/etc/cmpp-platform/monitoring-exporters.env
ExecStart=
ExecStart=${executable} ${arguments}
EOF
}
postgres_exporter="$(command -v prometheus-postgres-exporter)"
redis_exporter="$(command -v prometheus-redis-exporter)"
nginx_exporter="$(command -v prometheus-nginx-exporter)"
write_override prometheus-postgres-exporter "$postgres_exporter" '--web.listen-address=127.0.0.1:9187'
write_override prometheus-redis-exporter "$redis_exporter" '--web.listen-address=127.0.0.1:9121'
cat >/etc/nginx/conf.d/cmpp-monitoring-status.conf <<'EOF'
server {
listen 127.0.0.1:8088;
server_name localhost;
access_log off;
location = /stub_status {
stub_status;
allow 127.0.0.1;
allow ::1;
deny all;
}
}
EOF
write_override prometheus-nginx-exporter "$nginx_exporter" '--web.listen-address=127.0.0.1:9113 --nginx.scrape-uri=http://127.0.0.1:8088/stub_status'
# MinIO exposes only operational aggregates and listens on loopback; public auth here does not expose objects or credentials.
grep -q '^MINIO_PROMETHEUS_AUTH_TYPE=' /etc/cmpp-platform/minio.env \
&& sed -i 's/^MINIO_PROMETHEUS_AUTH_TYPE=.*/MINIO_PROMETHEUS_AUTH_TYPE=public/' /etc/cmpp-platform/minio.env \
|| printf '\nMINIO_PROMETHEUS_AUTH_TYPE=public\n' >>/etc/cmpp-platform/minio.env
nginx -t
systemctl daemon-reload
systemctl enable prometheus-postgres-exporter prometheus-redis-exporter prometheus-nginx-exporter
systemctl restart prometheus-postgres-exporter prometheus-redis-exporter prometheus-nginx-exporter
systemctl restart cmpp-minio
systemctl reload nginx
wait_for_http() {
local endpoint="$1" attempt
for attempt in $(seq 1 30); do
curl -fsS "$endpoint" >/dev/null 2>&1 && return 0
sleep 1
done
echo "Monitoring endpoint did not become ready: $endpoint" >&2
return 1
}
for endpoint in 127.0.0.1:9187 127.0.0.1:9121 127.0.0.1:9113; do wait_for_http "http://${endpoint}/metrics"; done
wait_for_http http://127.0.0.1:9000/minio/v2/metrics/cluster
if ss -lnt | grep -Eq '(^|[[:space:]])(0\.0\.0\.0|\[::\]):(9187|9121|9113)([[:space:]]|$)'; then
echo "A service exporter unexpectedly listens on a wildcard address." >&2
exit 1
fi
log "Service exporters are ready on loopback only; backup: $backup_dir"
+47
View File
@@ -0,0 +1,47 @@
global:
scrape_interval: 15s
evaluation_interval: 15s
external_labels:
platform: cmpp
environment: preproduction
rule_files:
- /etc/prometheus/cmpp-alerts.yml
- /var/lib/cmpp-platform/monitoring/*.yml
scrape_configs:
- job_name: prometheus
static_configs:
- targets: [127.0.0.1:9090]
- job_name: node
static_configs:
- targets: [127.0.0.1:9100]
labels:
host: cmpp-primary
- job_name: cmpp-api
static_configs:
- targets: [127.0.0.1:9464]
- job_name: cmpp-gateway
metrics_path: /metrics
static_configs:
- targets: [127.0.0.1:8090]
- job_name: postgresql
static_configs:
- targets: [127.0.0.1:9187]
- job_name: redis
static_configs:
- targets: [127.0.0.1:9121]
- job_name: minio
metrics_path: /minio/v2/metrics/cluster
static_configs:
- targets: [127.0.0.1:9000]
- job_name: nginx
static_configs:
- targets: [127.0.0.1:9113]
+57
View File
@@ -0,0 +1,57 @@
#!/usr/bin/env bash
set -Eeuo pipefail
APP_DIR="${APP_DIR:-/opt/cmpp-platform}"
if [[ "$(id -u)" -ne 0 ]]; then echo "Run as root." >&2; exit 1; fi
for command_name in fail2ban-client nft nginx systemctl; do command -v "$command_name" >/dev/null || { echo "Missing command: $command_name" >&2; exit 1; }; done
agent_binary="$APP_DIR/dist/cmpp-security-agent"
[[ -x "$agent_binary" ]] || { echo "Missing built security agent: $agent_binary" >&2; exit 1; }
getent group cmpp-security >/dev/null || groupadd --system cmpp-security
id cmpp-api >/dev/null 2>&1 || useradd --system --home-dir /nonexistent --shell /usr/sbin/nologin cmpp-api
usermod -a -G cmpp-security cmpp-api
install -d -o root -g cmpp-security -m 0770 /run/cmpp-security-agent
install -d -o root -g cmpp-security -m 0750 /var/lib/cmpp-security-agent
install -d -o cmpp-api -g cmpp-security -m 0750 "$APP_DIR/logs/api"
[[ -d /var/lib/cmpp-platform/object-storage ]] && chown -R cmpp-api:cmpp-security /var/lib/cmpp-platform/object-storage
sed "s#@CMPP_SECURITY_AGENT_BIN@#$agent_binary#g" "$APP_DIR/deploy/security/cmpp-report-only.conf" >/etc/fail2ban/action.d/cmpp-report-only.conf
chmod 0640 /etc/fail2ban/action.d/cmpp-report-only.conf
install -m 0640 "$APP_DIR/deploy/security/cmpp-http-scan.conf" /etc/fail2ban/filter.d/cmpp-http-scan.conf
install -d -m 0750 /etc/nginx/snippets /etc/nftables.d
touch /etc/nginx/snippets/cmpp-security-deny.conf
chmod 0640 /etc/nginx/snippets/cmpp-security-deny.conf
cat >/etc/nftables.d/cmpp-security.nft <<'EOF'
table inet cmpp_security {
set blocked_ipv4 { type ipv4_addr; flags timeout; }
set blocked_ipv6 { type ipv6_addr; flags timeout; }
chain input { type filter hook input priority -10; policy accept; ip saddr @blocked_ipv4 drop; ip6 saddr @blocked_ipv6 drop; }
}
EOF
grep -q 'cmpp-security.nft' /etc/nftables.conf || printf '\ninclude "/etc/nftables.d/cmpp-security.nft"\n' >>/etc/nftables.conf
nft -c -f /etc/nftables.conf
nft list table inet cmpp_security >/dev/null 2>&1 || nft -f /etc/nftables.d/cmpp-security.nft
sed "s#@CMPP_SECURITY_AGENT_BIN@#$agent_binary#g" "$APP_DIR/deploy/security/cmpp-security-agent.service" >/etc/systemd/system/cmpp-security-agent.service
if grep -Rqs '@CMPP_SECURITY_AGENT_BIN@' /etc/systemd/system/cmpp-security-agent.service /etc/fail2ban/action.d/cmpp-report-only.conf; then
echo "Security agent executable placeholder was not rendered." >&2
exit 1
fi
install -d -m 0755 /etc/systemd/system/cmpp-api.service.d
cat >/etc/systemd/system/cmpp-api.service.d/security-boundary.conf <<EOF
[Service]
User=cmpp-api
Group=cmpp-security
NoNewPrivileges=true
PrivateTmp=true
ProtectHome=true
ProtectSystem=true
ReadWritePaths=$APP_DIR/logs/api /var/lib/cmpp-platform/object-storage
EOF
fail2ban-client -t
nginx -t
systemctl daemon-reload
systemctl enable cmpp-security-agent
echo "Security boundary installed. Restart cmpp-security-agent and cmpp-api only in the approved release window."
@@ -0,0 +1,30 @@
import { readFileSync } from 'node:fs';
import { resolve } from 'node:path';
const root = resolve(import.meta.dirname, '..', '..');
const read = (relativePath) => readFileSync(resolve(root, relativePath), 'utf8');
const installer = read('tools/security/install-security-agent.sh');
const service = read('deploy/security/cmpp-security-agent.service');
const action = read('deploy/security/cmpp-report-only.conf');
const placeholder = '@CMPP_SECURITY_AGENT_BIN@';
for (const [label, source] of [['systemd service', service], ['Fail2ban action', action]]) {
if (!source.includes(placeholder)) throw new Error(`${label} is missing the security-agent executable placeholder`);
if (source.includes('/opt/cmpp-platform/current/bin/cmpp-security-agent')) {
throw new Error(`${label} still references the removed current/bin deployment layout`);
}
}
if (!installer.includes('agent_binary="$APP_DIR/dist/cmpp-security-agent"')) {
throw new Error('installer does not bind the security agent to the built dist executable');
}
for (const target of ['cmpp-security-agent.service', 'cmpp-report-only.conf']) {
if (!installer.includes(`sed "s#${placeholder}#$agent_binary#g"`) || !installer.includes(target)) {
throw new Error(`installer does not render ${target} with the built security-agent executable`);
}
}
if (!installer.includes("grep -Rqs '@CMPP_SECURITY_AGENT_BIN@'")) {
throw new Error('installer does not fail closed when an executable placeholder remains');
}
console.log('Security deployment verified: systemd and Fail2ban use the built dist security-agent executable.');