Compare commits
9
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
c4f36fc50d | ||
|
|
482f7ac1ae | ||
|
|
79f5d3f215 | ||
|
|
2216d00d51 | ||
|
|
0cd09441da | ||
|
|
6ccc102830 | ||
|
|
1ef4380422 | ||
|
|
d30d9ea4d0 | ||
|
|
b78faa1aa2 |
@@ -25,6 +25,9 @@ OPERATION_LOG_ARCHIVE_INTERVAL_MS=86400000
|
|||||||
SMS_RECEIPT_TIMEOUT_SCAN_ENABLED=true
|
SMS_RECEIPT_TIMEOUT_SCAN_ENABLED=true
|
||||||
SMS_RECEIPT_TIMEOUT_HOURS=72
|
SMS_RECEIPT_TIMEOUT_HOURS=72
|
||||||
SMS_RECEIPT_TIMEOUT_SCAN_INTERVAL_MS=300000
|
SMS_RECEIPT_TIMEOUT_SCAN_INTERVAL_MS=300000
|
||||||
|
# System monitoring reads only fixed queries from a loopback Prometheus instance.
|
||||||
|
PROMETHEUS_URL=http://127.0.0.1:9090
|
||||||
|
PROMETHEUS_QUERY_TIMEOUT_MS=5000
|
||||||
# Local HTTP development only. Production must use HTTPS and true.
|
# Local HTTP development only. Production must use HTTPS and true.
|
||||||
SESSION_COOKIE_SECURE=false
|
SESSION_COOKIE_SECURE=false
|
||||||
MINIO_ENDPOINT=localhost:9000
|
MINIO_ENDPOINT=localhost:9000
|
||||||
|
|||||||
@@ -0,0 +1,80 @@
|
|||||||
|
CREATE TABLE "SecurityDetectionRule" (
|
||||||
|
"id" TEXT NOT NULL,
|
||||||
|
"code" TEXT NOT NULL,
|
||||||
|
"name" TEXT NOT NULL,
|
||||||
|
"sourceType" TEXT NOT NULL,
|
||||||
|
"enabled" BOOLEAN NOT NULL DEFAULT true,
|
||||||
|
"threshold" INTEGER NOT NULL,
|
||||||
|
"windowSeconds" INTEGER NOT NULL,
|
||||||
|
"cooldownSeconds" INTEGER NOT NULL,
|
||||||
|
"severity" TEXT NOT NULL,
|
||||||
|
"defaultBlockSeconds" INTEGER NOT NULL,
|
||||||
|
"maximumBlockSeconds" INTEGER NOT NULL,
|
||||||
|
"configVersion" INTEGER NOT NULL DEFAULT 1,
|
||||||
|
"effectiveVersion" INTEGER NOT NULL DEFAULT 0,
|
||||||
|
"applyStatus" TEXT NOT NULL DEFAULT 'pending',
|
||||||
|
"lastApplyError" TEXT,
|
||||||
|
"pendingConfig" JSONB,
|
||||||
|
"createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||||
|
"updatedAt" TIMESTAMP(3) NOT NULL,
|
||||||
|
CONSTRAINT "SecurityDetectionRule_pkey" PRIMARY KEY ("id")
|
||||||
|
);
|
||||||
|
CREATE TABLE "SecurityDetectionEvent" (
|
||||||
|
"id" TEXT NOT NULL, "eventKey" TEXT NOT NULL, "ruleId" TEXT NOT NULL,
|
||||||
|
"sourceIp" TEXT NOT NULL, "sourcePort" INTEGER, "accountHash" TEXT,
|
||||||
|
"path" TEXT, "protocol" TEXT, "resultCode" TEXT, "evidence" JSONB,
|
||||||
|
"occurredAt" TIMESTAMP(3) NOT NULL, "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||||
|
CONSTRAINT "SecurityDetectionEvent_pkey" PRIMARY KEY ("id")
|
||||||
|
);
|
||||||
|
CREATE TABLE "SecurityAlert" (
|
||||||
|
"id" TEXT NOT NULL, "fingerprint" TEXT NOT NULL, "ruleId" TEXT NOT NULL,
|
||||||
|
"sourceIp" TEXT NOT NULL, "severity" TEXT NOT NULL, "status" TEXT NOT NULL DEFAULT 'open',
|
||||||
|
"eventCount" INTEGER NOT NULL DEFAULT 0, "windowStartedAt" TIMESTAMP(3) NOT NULL,
|
||||||
|
"firstOccurredAt" TIMESTAMP(3) NOT NULL, "lastOccurredAt" TIMESTAMP(3) NOT NULL,
|
||||||
|
"acknowledgedAt" TIMESTAMP(3), "acknowledgedById" TEXT, "ignoredAt" TIMESTAMP(3),
|
||||||
|
"ignoredById" TEXT, "ignoreReason" TEXT, "blockId" TEXT,
|
||||||
|
"createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, "updatedAt" TIMESTAMP(3) NOT NULL,
|
||||||
|
CONSTRAINT "SecurityAlert_pkey" PRIMARY KEY ("id")
|
||||||
|
);
|
||||||
|
CREATE TABLE "SecurityBlock" (
|
||||||
|
"id" TEXT NOT NULL, "operationKey" TEXT NOT NULL, "alertId" TEXT, "sourceIp" TEXT NOT NULL,
|
||||||
|
"executor" TEXT NOT NULL, "status" TEXT NOT NULL DEFAULT 'requested', "durationSeconds" INTEGER NOT NULL,
|
||||||
|
"reason" TEXT NOT NULL, "requestedById" TEXT NOT NULL, "requestedAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||||
|
"appliedAt" TIMESTAMP(3), "expiresAt" TIMESTAMP(3), "releasedAt" TIMESTAMP(3), "releasedById" TEXT,
|
||||||
|
"executorReference" TEXT, "lastError" TEXT, "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||||
|
"updatedAt" TIMESTAMP(3) NOT NULL, CONSTRAINT "SecurityBlock_pkey" PRIMARY KEY ("id")
|
||||||
|
);
|
||||||
|
CREATE TABLE "SecurityProtectedNetwork" (
|
||||||
|
"id" TEXT NOT NULL, "network" TEXT NOT NULL, "name" TEXT NOT NULL, "reason" TEXT NOT NULL,
|
||||||
|
"enabled" BOOLEAN NOT NULL DEFAULT true, "createdById" TEXT NOT NULL,
|
||||||
|
"createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, "updatedAt" TIMESTAMP(3) NOT NULL,
|
||||||
|
CONSTRAINT "SecurityProtectedNetwork_pkey" PRIMARY KEY ("id")
|
||||||
|
);
|
||||||
|
CREATE UNIQUE INDEX "SecurityDetectionRule_code_key" ON "SecurityDetectionRule"("code");
|
||||||
|
CREATE INDEX "SecurityDetectionRule_enabled_sourceType_idx" ON "SecurityDetectionRule"("enabled", "sourceType");
|
||||||
|
CREATE UNIQUE INDEX "SecurityDetectionEvent_eventKey_key" ON "SecurityDetectionEvent"("eventKey");
|
||||||
|
CREATE INDEX "SecurityDetectionEvent_ruleId_occurredAt_idx" ON "SecurityDetectionEvent"("ruleId", "occurredAt");
|
||||||
|
CREATE INDEX "SecurityDetectionEvent_sourceIp_occurredAt_idx" ON "SecurityDetectionEvent"("sourceIp", "occurredAt");
|
||||||
|
CREATE UNIQUE INDEX "SecurityAlert_fingerprint_key" ON "SecurityAlert"("fingerprint");
|
||||||
|
CREATE INDEX "SecurityAlert_status_severity_lastOccurredAt_idx" ON "SecurityAlert"("status", "severity", "lastOccurredAt");
|
||||||
|
CREATE INDEX "SecurityAlert_sourceIp_status_lastOccurredAt_idx" ON "SecurityAlert"("sourceIp", "status", "lastOccurredAt");
|
||||||
|
CREATE INDEX "SecurityAlert_ruleId_status_lastOccurredAt_idx" ON "SecurityAlert"("ruleId", "status", "lastOccurredAt");
|
||||||
|
CREATE UNIQUE INDEX "SecurityBlock_operationKey_key" ON "SecurityBlock"("operationKey");
|
||||||
|
CREATE INDEX "SecurityBlock_status_expiresAt_idx" ON "SecurityBlock"("status", "expiresAt");
|
||||||
|
CREATE INDEX "SecurityBlock_sourceIp_status_requestedAt_idx" ON "SecurityBlock"("sourceIp", "status", "requestedAt");
|
||||||
|
CREATE INDEX "SecurityBlock_alertId_idx" ON "SecurityBlock"("alertId");
|
||||||
|
CREATE UNIQUE INDEX "SecurityProtectedNetwork_network_key" ON "SecurityProtectedNetwork"("network");
|
||||||
|
CREATE INDEX "SecurityProtectedNetwork_enabled_createdAt_idx" ON "SecurityProtectedNetwork"("enabled", "createdAt");
|
||||||
|
ALTER TABLE "SecurityDetectionEvent" ADD CONSTRAINT "SecurityDetectionEvent_ruleId_fkey" FOREIGN KEY ("ruleId") REFERENCES "SecurityDetectionRule"("id") ON DELETE RESTRICT ON UPDATE CASCADE;
|
||||||
|
ALTER TABLE "SecurityAlert" ADD CONSTRAINT "SecurityAlert_ruleId_fkey" FOREIGN KEY ("ruleId") REFERENCES "SecurityDetectionRule"("id") ON DELETE RESTRICT ON UPDATE CASCADE;
|
||||||
|
|
||||||
|
INSERT INTO "SecurityDetectionRule" ("id", "code", "name", "sourceType", "threshold", "windowSeconds", "cooldownSeconds", "severity", "defaultBlockSeconds", "maximumBlockSeconds", "configVersion", "effectiveVersion", "applyStatus", "updatedAt") VALUES
|
||||||
|
('sec_admin_login', 'admin_login_failure', '运营端登录失败', 'application', 8, 600, 900, 'medium', 3600, 604800, 1, 1, 'effective', CURRENT_TIMESTAMP),
|
||||||
|
('sec_client_login', 'client_login_failure', '客户端登录失败', 'application', 8, 600, 900, 'medium', 3600, 604800, 1, 1, 'effective', CURRENT_TIMESTAMP),
|
||||||
|
('sec_ssh_auth', 'ssh_auth_failure', 'SSH认证失败', 'fail2ban', 6, 600, 1800, 'high', 86400, 604800, 1, 1, 'effective', CURRENT_TIMESTAMP),
|
||||||
|
('sec_cmpp_auth', 'cmpp_auth_failure', 'CMPP认证失败', 'gateway', 5, 300, 900, 'high', 3600, 604800, 1, 1, 'effective', CURRENT_TIMESTAMP),
|
||||||
|
('sec_cmpp_abuse', 'cmpp_protocol_abuse', 'CMPP协议滥用', 'gateway', 20, 60, 900, 'critical', 3600, 604800, 1, 1, 'effective', CURRENT_TIMESTAMP),
|
||||||
|
('sec_http_key', 'http_invalid_api_key', 'HTTP错误密钥', 'application', 10, 300, 900, 'high', 3600, 604800, 1, 1, 'effective', CURRENT_TIMESTAMP),
|
||||||
|
('sec_http_sign', 'http_signature_failure', 'HTTP签名错误', 'application', 10, 300, 900, 'high', 3600, 604800, 1, 1, 'effective', CURRENT_TIMESTAMP),
|
||||||
|
('sec_http_replay', 'http_replay_attempt', 'HTTP重放尝试', 'application', 3, 600, 1800, 'critical', 86400, 604800, 1, 1, 'effective', CURRENT_TIMESTAMP),
|
||||||
|
('sec_http_scan', 'http_malicious_scan', 'HTTP恶意扫描', 'fail2ban', 20, 60, 900, 'high', 3600, 604800, 1, 1, 'effective', CURRENT_TIMESTAMP);
|
||||||
@@ -0,0 +1,23 @@
|
|||||||
|
CREATE TABLE "InfrastructureAlertSetting" (
|
||||||
|
"id" TEXT NOT NULL DEFAULT 'global',
|
||||||
|
"configVersion" INTEGER NOT NULL DEFAULT 1,
|
||||||
|
"effectiveVersion" INTEGER NOT NULL DEFAULT 1,
|
||||||
|
"thresholds" JSONB NOT NULL,
|
||||||
|
"effectiveThresholds" JSONB NOT NULL,
|
||||||
|
"applyStatus" TEXT NOT NULL DEFAULT 'effective',
|
||||||
|
"lastError" TEXT,
|
||||||
|
"updatedById" TEXT,
|
||||||
|
"appliedAt" TIMESTAMP(3),
|
||||||
|
"createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||||
|
"updatedAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||||
|
CONSTRAINT "InfrastructureAlertSetting_pkey" PRIMARY KEY ("id")
|
||||||
|
);
|
||||||
|
|
||||||
|
INSERT INTO "InfrastructureAlertSetting" (
|
||||||
|
"id", "thresholds", "effectiveThresholds", "appliedAt"
|
||||||
|
) VALUES (
|
||||||
|
'global',
|
||||||
|
'{"hostCpu":{"warning":80,"critical":90},"hostMemory":{"warning":85,"critical":95},"hostDisk":{"warning":80,"critical":90},"apiError":{"warning":1,"critical":5},"apiLatency":{"warning":1,"critical":3},"apiEventLoop":{"warning":0.2,"critical":1},"gatewayQueue":{"warning":30,"critical":120},"postgresConnections":{"warning":70,"critical":85},"redisMemory":{"warning":70,"critical":85},"minioCapacity":{"warning":80,"critical":90}}'::jsonb,
|
||||||
|
'{"hostCpu":{"warning":80,"critical":90},"hostMemory":{"warning":85,"critical":95},"hostDisk":{"warning":80,"critical":90},"apiError":{"warning":1,"critical":5},"apiLatency":{"warning":1,"critical":3},"apiEventLoop":{"warning":0.2,"critical":1},"gatewayQueue":{"warning":30,"critical":120},"postgresConnections":{"warning":70,"critical":85},"redisMemory":{"warning":70,"critical":85},"minioCapacity":{"warning":80,"critical":90}}'::jsonb,
|
||||||
|
CURRENT_TIMESTAMP
|
||||||
|
);
|
||||||
@@ -0,0 +1,17 @@
|
|||||||
|
CREATE TABLE "InfrastructureAlertRead" (
|
||||||
|
"id" TEXT NOT NULL,
|
||||||
|
"fingerprint" TEXT NOT NULL,
|
||||||
|
"activeAt" TIMESTAMP(3) NOT NULL,
|
||||||
|
"userId" TEXT NOT NULL,
|
||||||
|
"readAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||||
|
"createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||||
|
"updatedAt" TIMESTAMP(3) NOT NULL,
|
||||||
|
CONSTRAINT "InfrastructureAlertRead_pkey" PRIMARY KEY ("id"),
|
||||||
|
CONSTRAINT "InfrastructureAlertRead_userId_fkey" FOREIGN KEY ("userId") REFERENCES "User"("id") ON DELETE CASCADE ON UPDATE CASCADE
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE UNIQUE INDEX "InfrastructureAlertRead_fingerprint_userId_key"
|
||||||
|
ON "InfrastructureAlertRead"("fingerprint", "userId");
|
||||||
|
|
||||||
|
CREATE INDEX "InfrastructureAlertRead_userId_readAt_idx"
|
||||||
|
ON "InfrastructureAlertRead"("userId", "readAt");
|
||||||
@@ -103,6 +103,7 @@ model User {
|
|||||||
releasedPhoneFrequencyHits PhoneFrequencyHit[] @relation("PhoneFrequencyHitReleaser")
|
releasedPhoneFrequencyHits PhoneFrequencyHit[] @relation("PhoneFrequencyHitReleaser")
|
||||||
createdPhoneFrequencyWhitelistEntries PhoneFrequencyWhitelist[] @relation("PhoneFrequencyWhitelistCreator")
|
createdPhoneFrequencyWhitelistEntries PhoneFrequencyWhitelist[] @relation("PhoneFrequencyWhitelistCreator")
|
||||||
updatedPhoneFrequencyWhitelistEntries PhoneFrequencyWhitelist[] @relation("PhoneFrequencyWhitelistUpdater")
|
updatedPhoneFrequencyWhitelistEntries PhoneFrequencyWhitelist[] @relation("PhoneFrequencyWhitelistUpdater")
|
||||||
|
infrastructureAlertReads InfrastructureAlertRead[]
|
||||||
}
|
}
|
||||||
|
|
||||||
model Role {
|
model Role {
|
||||||
@@ -2321,3 +2322,143 @@ model GatewayDownstreamRecoveryStatus {
|
|||||||
@@index([state, updatedAt])
|
@@index([state, updatedAt])
|
||||||
@@index([nextRetryAt])
|
@@index([nextRetryAt])
|
||||||
}
|
}
|
||||||
|
|
||||||
|
model SecurityDetectionRule {
|
||||||
|
id String @id @default(cuid())
|
||||||
|
code String @unique
|
||||||
|
name String
|
||||||
|
sourceType String
|
||||||
|
enabled Boolean @default(true)
|
||||||
|
threshold Int
|
||||||
|
windowSeconds Int
|
||||||
|
cooldownSeconds Int
|
||||||
|
severity String
|
||||||
|
defaultBlockSeconds Int
|
||||||
|
maximumBlockSeconds Int
|
||||||
|
configVersion Int @default(1)
|
||||||
|
effectiveVersion Int @default(0)
|
||||||
|
applyStatus String @default("pending")
|
||||||
|
lastApplyError String?
|
||||||
|
pendingConfig Json?
|
||||||
|
createdAt DateTime @default(now())
|
||||||
|
updatedAt DateTime @updatedAt
|
||||||
|
|
||||||
|
events SecurityDetectionEvent[]
|
||||||
|
alerts SecurityAlert[]
|
||||||
|
|
||||||
|
@@index([enabled, sourceType])
|
||||||
|
}
|
||||||
|
|
||||||
|
model SecurityDetectionEvent {
|
||||||
|
id String @id @default(cuid())
|
||||||
|
eventKey String @unique
|
||||||
|
ruleId String
|
||||||
|
sourceIp String
|
||||||
|
sourcePort Int?
|
||||||
|
accountHash String?
|
||||||
|
path String?
|
||||||
|
protocol String?
|
||||||
|
resultCode String?
|
||||||
|
evidence Json?
|
||||||
|
occurredAt DateTime
|
||||||
|
createdAt DateTime @default(now())
|
||||||
|
|
||||||
|
rule SecurityDetectionRule @relation(fields: [ruleId], references: [id], onDelete: Restrict)
|
||||||
|
|
||||||
|
@@index([ruleId, occurredAt])
|
||||||
|
@@index([sourceIp, occurredAt])
|
||||||
|
}
|
||||||
|
|
||||||
|
model SecurityAlert {
|
||||||
|
id String @id @default(cuid())
|
||||||
|
fingerprint String @unique
|
||||||
|
ruleId String
|
||||||
|
sourceIp String
|
||||||
|
severity String
|
||||||
|
status String @default("open")
|
||||||
|
eventCount Int @default(0)
|
||||||
|
windowStartedAt DateTime
|
||||||
|
firstOccurredAt DateTime
|
||||||
|
lastOccurredAt DateTime
|
||||||
|
acknowledgedAt DateTime?
|
||||||
|
acknowledgedById String?
|
||||||
|
ignoredAt DateTime?
|
||||||
|
ignoredById String?
|
||||||
|
ignoreReason String?
|
||||||
|
blockId String?
|
||||||
|
createdAt DateTime @default(now())
|
||||||
|
updatedAt DateTime @updatedAt
|
||||||
|
|
||||||
|
rule SecurityDetectionRule @relation(fields: [ruleId], references: [id], onDelete: Restrict)
|
||||||
|
|
||||||
|
@@index([status, severity, lastOccurredAt])
|
||||||
|
@@index([sourceIp, status, lastOccurredAt])
|
||||||
|
@@index([ruleId, status, lastOccurredAt])
|
||||||
|
}
|
||||||
|
|
||||||
|
model SecurityBlock {
|
||||||
|
id String @id @default(cuid())
|
||||||
|
operationKey String @unique
|
||||||
|
alertId String?
|
||||||
|
sourceIp String
|
||||||
|
executor String
|
||||||
|
status String @default("requested")
|
||||||
|
durationSeconds Int
|
||||||
|
reason String
|
||||||
|
requestedById String
|
||||||
|
requestedAt DateTime @default(now())
|
||||||
|
appliedAt DateTime?
|
||||||
|
expiresAt DateTime?
|
||||||
|
releasedAt DateTime?
|
||||||
|
releasedById String?
|
||||||
|
executorReference String?
|
||||||
|
lastError String?
|
||||||
|
createdAt DateTime @default(now())
|
||||||
|
updatedAt DateTime @updatedAt
|
||||||
|
|
||||||
|
@@index([status, expiresAt])
|
||||||
|
@@index([sourceIp, status, requestedAt])
|
||||||
|
@@index([alertId])
|
||||||
|
}
|
||||||
|
|
||||||
|
model SecurityProtectedNetwork {
|
||||||
|
id String @id @default(cuid())
|
||||||
|
network String @unique
|
||||||
|
name String
|
||||||
|
reason String
|
||||||
|
enabled Boolean @default(true)
|
||||||
|
createdById String
|
||||||
|
createdAt DateTime @default(now())
|
||||||
|
updatedAt DateTime @updatedAt
|
||||||
|
|
||||||
|
@@index([enabled, createdAt])
|
||||||
|
}
|
||||||
|
|
||||||
|
model InfrastructureAlertSetting {
|
||||||
|
id String @id @default("global")
|
||||||
|
configVersion Int @default(1)
|
||||||
|
effectiveVersion Int @default(1)
|
||||||
|
thresholds Json
|
||||||
|
effectiveThresholds Json
|
||||||
|
applyStatus String @default("effective")
|
||||||
|
lastError String?
|
||||||
|
updatedById String?
|
||||||
|
appliedAt DateTime?
|
||||||
|
createdAt DateTime @default(now())
|
||||||
|
updatedAt DateTime @updatedAt
|
||||||
|
}
|
||||||
|
|
||||||
|
model InfrastructureAlertRead {
|
||||||
|
id String @id @default(cuid())
|
||||||
|
fingerprint String
|
||||||
|
activeAt DateTime
|
||||||
|
userId String
|
||||||
|
readAt DateTime @default(now())
|
||||||
|
createdAt DateTime @default(now())
|
||||||
|
updatedAt DateTime @updatedAt
|
||||||
|
|
||||||
|
user User @relation(fields: [userId], references: [id], onDelete: Cascade)
|
||||||
|
|
||||||
|
@@unique([fingerprint, userId])
|
||||||
|
@@index([userId, readAt])
|
||||||
|
}
|
||||||
|
|||||||
@@ -12,6 +12,7 @@ import { DictionariesModule } from './dictionaries/dictionaries.module';
|
|||||||
import { DeletionGovernanceModule } from './deletion-governance/deletion-governance.module';
|
import { DeletionGovernanceModule } from './deletion-governance/deletion-governance.module';
|
||||||
import { FilesModule } from './files/files.module';
|
import { FilesModule } from './files/files.module';
|
||||||
import { HealthController } from './health.controller';
|
import { HealthController } from './health.controller';
|
||||||
|
import { InfrastructureMonitoringModule } from './infrastructure-monitoring/infrastructure-monitoring.module';
|
||||||
import { OperationsModule } from './operations/operations.module';
|
import { OperationsModule } from './operations/operations.module';
|
||||||
import { OpenApiModule } from './open-api/open-api.module';
|
import { OpenApiModule } from './open-api/open-api.module';
|
||||||
import { PrismaModule } from './prisma/prisma.module';
|
import { PrismaModule } from './prisma/prisma.module';
|
||||||
@@ -24,6 +25,8 @@ import { SmsConfigModule } from './sms-config/sms-config.module';
|
|||||||
import { TenantsModule } from './tenants/tenants.module';
|
import { TenantsModule } from './tenants/tenants.module';
|
||||||
import { UsersModule } from './users/users.module';
|
import { UsersModule } from './users/users.module';
|
||||||
import { SignatureRetirementModule } from './signature-retirement/signature-retirement.module';
|
import { SignatureRetirementModule } from './signature-retirement/signature-retirement.module';
|
||||||
|
import { SecurityDetectionModule } from './security-detection/security-detection.module';
|
||||||
|
import { MetricsModule } from './metrics/metrics.module';
|
||||||
|
|
||||||
@Module({
|
@Module({
|
||||||
imports: [
|
imports: [
|
||||||
@@ -49,8 +52,11 @@ import { SignatureRetirementModule } from './signature-retirement/signature-reti
|
|||||||
ReportMaterialsModule,
|
ReportMaterialsModule,
|
||||||
SendChainModule,
|
SendChainModule,
|
||||||
OperationsModule,
|
OperationsModule,
|
||||||
|
InfrastructureMonitoringModule,
|
||||||
OpenApiModule,
|
OpenApiModule,
|
||||||
SignatureRetirementModule,
|
SignatureRetirementModule,
|
||||||
|
SecurityDetectionModule,
|
||||||
|
MetricsModule,
|
||||||
],
|
],
|
||||||
controllers: [HealthController],
|
controllers: [HealthController],
|
||||||
providers: [RequestContextMiddleware, SessionValidationMiddleware, ManualOperationAuditMiddleware],
|
providers: [RequestContextMiddleware, SessionValidationMiddleware, ManualOperationAuditMiddleware],
|
||||||
|
|||||||
@@ -7,6 +7,8 @@ import type { SessionRequest } from './session-validation.middleware';
|
|||||||
import { UsersService } from '../users/users.service';
|
import { UsersService } from '../users/users.service';
|
||||||
import { PrismaService } from '../prisma/prisma.service';
|
import { PrismaService } from '../prisma/prisma.service';
|
||||||
import { Prisma } from '@prisma/client';
|
import { Prisma } from '@prisma/client';
|
||||||
|
import { requestContext } from '../common/request-context';
|
||||||
|
import { SecurityDetectionService } from '../security-detection/security-detection.service';
|
||||||
|
|
||||||
type CookieResponse = {
|
type CookieResponse = {
|
||||||
cookie(name: string, value: string, options: Record<string, unknown>): void;
|
cookie(name: string, value: string, options: Record<string, unknown>): void;
|
||||||
@@ -16,7 +18,7 @@ type CookieResponse = {
|
|||||||
@ApiTags('auth')
|
@ApiTags('auth')
|
||||||
@Controller()
|
@Controller()
|
||||||
export class AuthController {
|
export class AuthController {
|
||||||
constructor(private readonly auth: AuthService, private readonly users: UsersService, private readonly sessions: SessionService, private readonly prisma: PrismaService) {}
|
constructor(private readonly auth: AuthService, private readonly users: UsersService, private readonly sessions: SessionService, private readonly prisma: PrismaService, private readonly security: SecurityDetectionService) {}
|
||||||
|
|
||||||
@Get('admin/auth/captcha')
|
@Get('admin/auth/captcha')
|
||||||
adminCaptcha() {
|
adminCaptcha() {
|
||||||
@@ -25,7 +27,14 @@ export class AuthController {
|
|||||||
|
|
||||||
@Post('admin/auth/login')
|
@Post('admin/auth/login')
|
||||||
async adminLogin(@Body() body: LoginDto, @Req() request: SessionRequest, @Res({ passthrough: true }) response: CookieResponse) {
|
async adminLogin(@Body() body: LoginDto, @Req() request: SessionRequest, @Res({ passthrough: true }) response: CookieResponse) {
|
||||||
return this.finishLogin(await this.auth.login(body, 'admin'), request, response);
|
let result: Awaited<ReturnType<AuthService['login']>>;
|
||||||
|
try {
|
||||||
|
result = await this.auth.login(body, 'admin');
|
||||||
|
} catch (error) {
|
||||||
|
await this.recordLoginFailure('admin_login_failure', body.login, request).catch(() => undefined);
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
return this.finishLogin(result, request, response);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Get('client/auth/captcha')
|
@Get('client/auth/captcha')
|
||||||
@@ -35,7 +44,14 @@ export class AuthController {
|
|||||||
|
|
||||||
@Post('client/auth/login')
|
@Post('client/auth/login')
|
||||||
async clientLogin(@Body() body: LoginDto, @Req() request: SessionRequest, @Res({ passthrough: true }) response: CookieResponse) {
|
async clientLogin(@Body() body: LoginDto, @Req() request: SessionRequest, @Res({ passthrough: true }) response: CookieResponse) {
|
||||||
return this.finishLogin(await this.auth.login(body, 'client'), request, response);
|
let result: Awaited<ReturnType<AuthService['login']>>;
|
||||||
|
try {
|
||||||
|
result = await this.auth.login(body, 'client');
|
||||||
|
} catch (error) {
|
||||||
|
await this.recordLoginFailure('client_login_failure', body.login, request).catch(() => undefined);
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
return this.finishLogin(result, request, response);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Get(['admin/auth/session', 'client/auth/session'])
|
@Get(['admin/auth/session', 'client/auth/session'])
|
||||||
@@ -121,6 +137,17 @@ export class AuthController {
|
|||||||
return publicResult;
|
return publicResult;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private recordLoginFailure(ruleCode: 'admin_login_failure' | 'client_login_failure', account: string, request: SessionRequest) {
|
||||||
|
return this.security.recordEvent({
|
||||||
|
ruleCode,
|
||||||
|
sourceIp: requestContext.getStore()?.ipAddress ?? '127.0.0.1',
|
||||||
|
account,
|
||||||
|
protocol: 'http',
|
||||||
|
path: ruleCode === 'admin_login_failure' ? '/admin/auth/login' : '/client/auth/login',
|
||||||
|
evidence: { userAgent: request.header('user-agent')?.slice(0, 256) },
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
private writeLog(request: SessionRequest, action: string, detail: Record<string, unknown>) {
|
private writeLog(request: SessionRequest, action: string, detail: Record<string, unknown>) {
|
||||||
return this.prisma.operationLog.create({
|
return this.prisma.operationLog.create({
|
||||||
data: { userId: request.sessionUserId, action, resource: 'auth_session', userAgent: request.header('user-agent'), detail: JSON.parse(JSON.stringify(detail)) as Prisma.InputJsonValue },
|
data: { userId: request.sessionUserId, action, resource: 'auth_session', userAgent: request.header('user-agent'), detail: JSON.parse(JSON.stringify(detail)) as Prisma.InputJsonValue },
|
||||||
|
|||||||
@@ -5,9 +5,10 @@ import { AuthController } from './auth.controller';
|
|||||||
import { AuthService } from './auth.service';
|
import { AuthService } from './auth.service';
|
||||||
import { RecentAuthenticationGuard } from './recent-authentication.guard';
|
import { RecentAuthenticationGuard } from './recent-authentication.guard';
|
||||||
import { SessionService } from './session.service';
|
import { SessionService } from './session.service';
|
||||||
|
import { SecurityDetectionModule } from '../security-detection/security-detection.module';
|
||||||
|
|
||||||
@Module({
|
@Module({
|
||||||
imports: [UsersModule],
|
imports: [UsersModule, SecurityDetectionModule],
|
||||||
controllers: [AuthController],
|
controllers: [AuthController],
|
||||||
providers: [
|
providers: [
|
||||||
AuthService,
|
AuthService,
|
||||||
|
|||||||
@@ -8,7 +8,10 @@ export class RequestContextMiddleware implements NestMiddleware {
|
|||||||
use(request: RequestLike, _response: unknown, next: () => void) {
|
use(request: RequestLike, _response: unknown, next: () => void) {
|
||||||
const forwarded = request.headers['x-forwarded-for'];
|
const forwarded = request.headers['x-forwarded-for'];
|
||||||
const firstForwarded = Array.isArray(forwarded) ? forwarded[0] : forwarded?.split(',')[0];
|
const firstForwarded = Array.isArray(forwarded) ? forwarded[0] : forwarded?.split(',')[0];
|
||||||
const ipAddress = (firstForwarded ?? request.socket?.remoteAddress)?.trim().replace(/^::ffff:/, '');
|
const remoteAddress = request.socket?.remoteAddress?.trim().replace(/^::ffff:/, '');
|
||||||
|
const trustedProxies = new Set((process.env.TRUSTED_PROXY_IPS ?? '127.0.0.1,::1').split(',').map((item) => item.trim()).filter(Boolean));
|
||||||
|
// 仅可信反向代理可以声明客户端地址,防止攻击者伪造 X-Forwarded-For 绕过保护名单或嫁祸他人。
|
||||||
|
const ipAddress = (remoteAddress && trustedProxies.has(remoteAddress) ? firstForwarded : remoteAddress)?.trim().replace(/^::ffff:/, '');
|
||||||
requestContext.run({ ipAddress }, next);
|
requestContext.run({ ipAddress }, next);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,21 @@
|
|||||||
|
import { BadRequestException } from '@nestjs/common';
|
||||||
|
import { DEFAULT_ALERT_THRESHOLDS, InfrastructureAlertSettingsService } from './infrastructure-alert-settings.service';
|
||||||
|
|
||||||
|
describe('InfrastructureAlertSettingsService', () => {
|
||||||
|
const service = new InfrastructureAlertSettingsService({} as never, { get: () => undefined } as never);
|
||||||
|
|
||||||
|
it('accepts the fixed threshold whitelist and renders managed rules', () => {
|
||||||
|
const validated = (service as unknown as { validate(value: unknown): unknown }).validate(DEFAULT_ALERT_THRESHOLDS);
|
||||||
|
const rules = (service as unknown as { renderRules(value: unknown): string }).renderRules(validated);
|
||||||
|
expect(rules).toContain('HostCpuUsageWarning');
|
||||||
|
expect(rules).toContain('CmppGatewayQueueDelayedCritical');
|
||||||
|
expect(rules).toContain('threshold: "120秒"');
|
||||||
|
expect(rules).toContain('redis_memory_max_bytes > 0');
|
||||||
|
expect(rules).toContain('sum(increase(cmpp_api_http_requests_total');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('rejects unknown keys and warning thresholds that are not below critical', () => {
|
||||||
|
expect(() => (service as unknown as { validate(value: unknown): unknown }).validate({ ...DEFAULT_ALERT_THRESHOLDS, promql: { warning: 1, critical: 2 } })).toThrow(BadRequestException);
|
||||||
|
expect(() => (service as unknown as { validate(value: unknown): unknown }).validate({ ...DEFAULT_ALERT_THRESHOLDS, hostCpu: { warning: 90, critical: 90 } })).toThrow(BadRequestException);
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -0,0 +1,144 @@
|
|||||||
|
import { BadRequestException, ConflictException, Injectable, Logger, ServiceUnavailableException } from '@nestjs/common';
|
||||||
|
import { ConfigService } from '@nestjs/config';
|
||||||
|
import { Prisma } from '@prisma/client';
|
||||||
|
import { execFile } from 'node:child_process';
|
||||||
|
import { mkdir, readFile, rename, rm, writeFile } from 'node:fs/promises';
|
||||||
|
import { dirname } from 'node:path';
|
||||||
|
import { promisify } from 'node:util';
|
||||||
|
import { PrismaService } from '../prisma/prisma.service';
|
||||||
|
import type { InfrastructureAlertSettings, InfrastructureAlertThresholds } from './infrastructure-monitoring.contracts';
|
||||||
|
|
||||||
|
const execFileAsync = promisify(execFile);
|
||||||
|
|
||||||
|
export const ALERT_THRESHOLD_DEFINITIONS = [
|
||||||
|
{ key: 'hostCpu', label: '主机 CPU 使用率', unit: '%', min: 1, max: 100, step: 1, warning: 80, critical: 90, expr: '100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100)', names: ['HostCpuUsageWarning', 'HostCpuUsageCritical'], service: 'host', durations: ['10m', '5m'] },
|
||||||
|
{ key: 'hostMemory', label: '主机内存使用率', unit: '%', min: 1, max: 100, step: 1, warning: 85, critical: 95, expr: '(1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100', names: ['HostMemoryUsageWarning', 'HostMemoryUsageCritical'], service: 'host', durations: ['10m', '5m'] },
|
||||||
|
{ key: 'hostDisk', label: '根磁盘使用率', unit: '%', min: 1, max: 100, step: 1, warning: 80, critical: 90, expr: '(1 - node_filesystem_avail_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"} / node_filesystem_size_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"}) * 100', names: ['HostRootDiskUsageWarning', 'HostRootDiskUsageCritical'], service: 'host', durations: ['15m', '5m'] },
|
||||||
|
{ key: 'apiError', label: 'API 5xx 错误率', unit: '%', min: 0.1, max: 100, step: 0.1, warning: 1, critical: 5, expr: '100 * sum(rate(cmpp_api_http_requests_total{status=~"5.."}[5m])) / clamp_min(sum(rate(cmpp_api_http_requests_total[5m])), 0.001)', guard: 'sum(increase(cmpp_api_http_requests_total{status=~"5.."}[5m])) >= 5', names: ['CmppApiHttpErrorRateWarning', 'CmppApiHttpErrorRateCritical'], service: 'api', durations: ['5m', '5m'] },
|
||||||
|
{ key: 'apiLatency', label: 'API P95 响应时间', unit: '秒', min: 0.1, max: 60, step: 0.1, warning: 1, critical: 3, expr: 'histogram_quantile(0.95, sum by (le) (rate(cmpp_api_http_request_duration_seconds_bucket[10m])))', names: ['CmppApiLatencyWarning', 'CmppApiLatencyCritical'], service: 'api', durations: ['10m', '5m'] },
|
||||||
|
{ key: 'apiEventLoop', label: 'API 事件循环 P99', unit: '秒', min: 0.01, max: 10, step: 0.01, warning: 0.2, critical: 1, expr: 'cmpp_api_nodejs_event_loop_lag_p99_seconds', names: ['CmppApiEventLoopLagWarning', 'CmppApiEventLoopLagCritical'], service: 'api', durations: ['10m', '5m'] },
|
||||||
|
{ key: 'gatewayQueue', label: 'Gateway 最旧 pending', unit: '秒', min: 1, max: 3600, step: 1, warning: 30, critical: 120, expr: 'cmpp_gateway_submit_queue_oldest_pending_age_seconds', names: ['CmppGatewayQueueDelayedWarning', 'CmppGatewayQueueDelayedCritical'], service: 'gateway', durations: ['2m', '2m'] },
|
||||||
|
{ key: 'postgresConnections', label: 'PostgreSQL 连接使用率', unit: '%', min: 1, max: 100, step: 1, warning: 70, critical: 85, expr: '100 * sum(pg_stat_activity_count) / clamp_min(max(pg_settings_max_connections), 1)', names: ['PostgresConnectionsWarning', 'PostgresConnectionsCritical'], service: 'postgresql', durations: ['10m', '5m'] },
|
||||||
|
{ key: 'redisMemory', label: 'Redis 内存使用率', unit: '%', min: 1, max: 100, step: 1, warning: 70, critical: 85, expr: '100 * redis_memory_used_bytes / redis_memory_max_bytes', guard: 'redis_memory_max_bytes > 0', names: ['RedisMemoryWarning', 'RedisMemoryCritical'], service: 'redis', durations: ['10m', '5m'] },
|
||||||
|
{ key: 'minioCapacity', label: 'MinIO 容量使用率', unit: '%', min: 1, max: 100, step: 1, warning: 80, critical: 90, expr: '100 * (1 - minio_cluster_capacity_usable_free_bytes / minio_cluster_capacity_usable_total_bytes)', names: ['MinioCapacityWarning', 'MinioCapacityCritical'], service: 'minio', durations: ['15m', '5m'] },
|
||||||
|
] as const;
|
||||||
|
|
||||||
|
export const DEFAULT_ALERT_THRESHOLDS: InfrastructureAlertThresholds = Object.fromEntries(
|
||||||
|
ALERT_THRESHOLD_DEFINITIONS.map((item) => [item.key, { warning: item.warning, critical: item.critical }]),
|
||||||
|
);
|
||||||
|
|
||||||
|
@Injectable()
|
||||||
|
export class InfrastructureAlertSettingsService {
|
||||||
|
private readonly logger = new Logger(InfrastructureAlertSettingsService.name);
|
||||||
|
private readonly rulesPath: string;
|
||||||
|
private readonly promtoolPath: string;
|
||||||
|
private readonly reloadUrl: string;
|
||||||
|
|
||||||
|
constructor(private readonly prisma: PrismaService, config: ConfigService) {
|
||||||
|
this.rulesPath = String(config.get('PROMETHEUS_MANAGED_RULES_PATH') ?? '/var/lib/cmpp-platform/monitoring/cmpp-managed-alerts.yml');
|
||||||
|
this.promtoolPath = String(config.get('PROMTOOL_PATH') ?? '/usr/bin/promtool');
|
||||||
|
this.reloadUrl = String(config.get('PROMETHEUS_RELOAD_URL') ?? 'http://127.0.0.1:9090/-/reload');
|
||||||
|
}
|
||||||
|
|
||||||
|
async get(): Promise<InfrastructureAlertSettings> {
|
||||||
|
const row = await this.prisma.infrastructureAlertSetting.findUnique({ where: { id: 'global' } });
|
||||||
|
const thresholds = this.asThresholds(row?.thresholds) ?? DEFAULT_ALERT_THRESHOLDS;
|
||||||
|
const effective = this.asThresholds(row?.effectiveThresholds) ?? thresholds;
|
||||||
|
return {
|
||||||
|
configVersion: row?.configVersion ?? 1,
|
||||||
|
effectiveVersion: row?.effectiveVersion ?? 1,
|
||||||
|
applyStatus: (row?.applyStatus as InfrastructureAlertSettings['applyStatus']) ?? 'effective',
|
||||||
|
lastError: row?.lastError ?? null,
|
||||||
|
appliedAt: row?.appliedAt?.toISOString() ?? null,
|
||||||
|
thresholds,
|
||||||
|
effectiveThresholds: effective,
|
||||||
|
definitions: ALERT_THRESHOLD_DEFINITIONS.map(({ key, label, unit, min, max, step }) => ({ key, label, unit, min, max, step })),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
async update(body: { configVersion?: number; thresholds?: unknown }, operatorId?: string) {
|
||||||
|
const expectedVersion = Number(body.configVersion);
|
||||||
|
if (!Number.isInteger(expectedVersion) || expectedVersion < 1) throw new BadRequestException('配置版本无效');
|
||||||
|
const thresholds = this.validate(body.thresholds);
|
||||||
|
const claimed = await this.prisma.infrastructureAlertSetting.updateMany({
|
||||||
|
where: { id: 'global', configVersion: expectedVersion },
|
||||||
|
data: { configVersion: { increment: 1 }, thresholds: thresholds as Prisma.InputJsonValue, applyStatus: 'applying', lastError: null, updatedById: operatorId },
|
||||||
|
});
|
||||||
|
// 版本条件更新是跨进程的写锁;避免两个 API 实例同时覆盖规则文件并把旧配置误标成已生效。
|
||||||
|
if (claimed.count !== 1) throw new ConflictException('告警阈值已被其他管理员修改,请刷新后重试');
|
||||||
|
const nextVersion = expectedVersion + 1;
|
||||||
|
try {
|
||||||
|
await this.applyRules(thresholds);
|
||||||
|
await this.prisma.$transaction([
|
||||||
|
this.prisma.infrastructureAlertSetting.update({ where: { id: 'global' }, data: { effectiveVersion: nextVersion, effectiveThresholds: thresholds as Prisma.InputJsonValue, applyStatus: 'effective', lastError: null, appliedAt: new Date() } }),
|
||||||
|
this.prisma.operationLog.create({ data: { userId: operatorId, action: 'monitoring.alert_thresholds_updated', resource: 'infrastructure_alert_setting', resourceId: 'global', detail: { configVersion: nextVersion, thresholds } } }),
|
||||||
|
]);
|
||||||
|
} catch (error) {
|
||||||
|
const message = error instanceof Error ? error.message.slice(0, 500) : 'unknown error';
|
||||||
|
await this.prisma.infrastructureAlertSetting.update({ where: { id: 'global' }, data: { applyStatus: 'failed', lastError: message } });
|
||||||
|
this.logger.error(`Prometheus managed rules apply failed: ${message}`);
|
||||||
|
throw new ServiceUnavailableException('阈值已保存但 Prometheus 应用失败,原生效规则已保留');
|
||||||
|
}
|
||||||
|
return this.get();
|
||||||
|
}
|
||||||
|
|
||||||
|
private validate(value: unknown): InfrastructureAlertThresholds {
|
||||||
|
if (!value || typeof value !== 'object' || Array.isArray(value)) throw new BadRequestException('告警阈值格式无效');
|
||||||
|
const input = value as Record<string, unknown>;
|
||||||
|
if (Object.keys(input).some((key) => !ALERT_THRESHOLD_DEFINITIONS.some((item) => item.key === key))) throw new BadRequestException('存在不允许配置的告警指标');
|
||||||
|
const result: InfrastructureAlertThresholds = {};
|
||||||
|
for (const definition of ALERT_THRESHOLD_DEFINITIONS) {
|
||||||
|
const pair = input[definition.key] as { warning?: unknown; critical?: unknown } | undefined;
|
||||||
|
const warning = Number(pair?.warning);
|
||||||
|
const critical = Number(pair?.critical);
|
||||||
|
if (!Number.isFinite(warning) || !Number.isFinite(critical) || warning < definition.min || critical > definition.max || warning >= critical) {
|
||||||
|
throw new BadRequestException(`${definition.label}必须满足最小值 ≤ 警告阈值 < 严重阈值 ≤ 最大值`);
|
||||||
|
}
|
||||||
|
result[definition.key] = { warning, critical };
|
||||||
|
}
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
private asThresholds(value: unknown) {
|
||||||
|
try { return this.validate(value); } catch { return null; }
|
||||||
|
}
|
||||||
|
|
||||||
|
private renderRules(thresholds: InfrastructureAlertThresholds) {
|
||||||
|
const lines = ['groups:', ' - name: cmpp-managed-thresholds', ' rules:'];
|
||||||
|
for (const definition of ALERT_THRESHOLD_DEFINITIONS) {
|
||||||
|
const pair = thresholds[definition.key];
|
||||||
|
const values = [pair.warning, pair.critical];
|
||||||
|
for (let index = 0; index < 2; index += 1) {
|
||||||
|
const isWarning = index === 0;
|
||||||
|
// 低样本量与“未配置容量上限”必须继续作为固定保护条件,避免单次错误或除零结果触发伪告警。
|
||||||
|
const guard = 'guard' in definition ? ` and (${definition.guard})` : '';
|
||||||
|
const expr = isWarning ? `(${definition.expr} > ${values[0]}) and (${definition.expr} <= ${values[1]})${guard}` : `(${definition.expr} > ${values[1]})${guard}`;
|
||||||
|
lines.push(` - alert: ${definition.names[index]}`, ` expr: ${expr}`, ` for: ${definition.durations[index]}`, ' labels:', ` severity: ${isWarning ? 'warning' : 'critical'}`, ` service: ${definition.service}`, ' annotations:', ` summary: "${definition.label}${isWarning ? '达到警告阈值' : '达到严重阈值'}"`, ` description: "${definition.label}持续超过${values[index]}${definition.unit}。"`, ' currentValue: "{{ $value }}"', ` threshold: "${values[index]}${definition.unit}"`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return `${lines.join('\n')}\n`;
|
||||||
|
}
|
||||||
|
|
||||||
|
private async applyRules(thresholds: InfrastructureAlertThresholds) {
|
||||||
|
const directory = dirname(this.rulesPath);
|
||||||
|
const temporary = `${this.rulesPath}.${process.pid}.${Date.now()}.tmp`;
|
||||||
|
await mkdir(directory, { recursive: true });
|
||||||
|
const previous = await readFile(this.rulesPath).catch(() => null);
|
||||||
|
try {
|
||||||
|
await writeFile(temporary, this.renderRules(thresholds), { mode: 0o640 });
|
||||||
|
await execFileAsync(this.promtoolPath, ['check', 'rules', temporary], { timeout: 10_000 });
|
||||||
|
await rename(temporary, this.rulesPath);
|
||||||
|
const response = await fetch(this.reloadUrl, { method: 'POST', signal: AbortSignal.timeout(5_000) });
|
||||||
|
if (!response.ok) throw new Error(`Prometheus reload HTTP ${response.status}`);
|
||||||
|
} catch (error) {
|
||||||
|
await rm(temporary, { force: true });
|
||||||
|
// 规则替换和 reload 不是一个事务,失败时必须恢复旧文件并再次 reload,避免数据库状态与实际告警漂移。
|
||||||
|
if (previous) {
|
||||||
|
await writeFile(temporary, previous, { mode: 0o640 });
|
||||||
|
await rename(temporary, this.rulesPath);
|
||||||
|
await fetch(this.reloadUrl, { method: 'POST', signal: AbortSignal.timeout(5_000) }).catch(() => undefined);
|
||||||
|
}
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,93 @@
|
|||||||
|
export type InfrastructureMonitoringRange = '1h' | '24h' | '7d';
|
||||||
|
|
||||||
|
export type InfrastructureMetricPoint = {
|
||||||
|
timestamp: string;
|
||||||
|
value: number;
|
||||||
|
};
|
||||||
|
|
||||||
|
export type InfrastructureServiceStatus = {
|
||||||
|
key: string;
|
||||||
|
name: string;
|
||||||
|
unit: string;
|
||||||
|
status: 'healthy' | 'unhealthy' | 'unknown';
|
||||||
|
};
|
||||||
|
|
||||||
|
export type InfrastructureServiceMetricGroup = {
|
||||||
|
key: string;
|
||||||
|
name: string;
|
||||||
|
available: boolean;
|
||||||
|
metrics: Array<{
|
||||||
|
key: string;
|
||||||
|
label: string;
|
||||||
|
value: number | null;
|
||||||
|
unit: 'percent' | 'seconds' | 'count' | 'per_second' | 'bytes';
|
||||||
|
}>;
|
||||||
|
};
|
||||||
|
|
||||||
|
export type InfrastructureAlert = {
|
||||||
|
fingerprint: string;
|
||||||
|
name: string;
|
||||||
|
severity: 'info' | 'warning' | 'critical';
|
||||||
|
status: string;
|
||||||
|
startedAt: string;
|
||||||
|
summary: string;
|
||||||
|
description?: string;
|
||||||
|
currentValue?: string;
|
||||||
|
threshold?: string;
|
||||||
|
service?: string;
|
||||||
|
instance?: string;
|
||||||
|
acknowledged: boolean;
|
||||||
|
acknowledgedAt?: string;
|
||||||
|
};
|
||||||
|
|
||||||
|
export type InfrastructureMonitoringOverview = {
|
||||||
|
available: boolean;
|
||||||
|
range: InfrastructureMonitoringRange;
|
||||||
|
collectedAt: string;
|
||||||
|
lastSampleAt: string | null;
|
||||||
|
error?: string;
|
||||||
|
summary: {
|
||||||
|
overallStatus: 'healthy' | 'warning' | 'critical' | 'unknown';
|
||||||
|
serviceTotal: number;
|
||||||
|
serviceHealthy: number;
|
||||||
|
warningAlerts: number;
|
||||||
|
criticalAlerts: number;
|
||||||
|
activeAlerts: number;
|
||||||
|
};
|
||||||
|
metrics: {
|
||||||
|
cpuUsagePercent: number | null;
|
||||||
|
memoryUsagePercent: number | null;
|
||||||
|
memoryTotalBytes: number | null;
|
||||||
|
memoryAvailableBytes: number | null;
|
||||||
|
diskUsagePercent: number | null;
|
||||||
|
diskTotalBytes: number | null;
|
||||||
|
diskAvailableBytes: number | null;
|
||||||
|
networkReceiveBytesPerSecond: number | null;
|
||||||
|
networkTransmitBytesPerSecond: number | null;
|
||||||
|
load1: number | null;
|
||||||
|
uptimeSeconds: number | null;
|
||||||
|
};
|
||||||
|
trends: {
|
||||||
|
cpuUsagePercent: InfrastructureMetricPoint[];
|
||||||
|
memoryUsagePercent: InfrastructureMetricPoint[];
|
||||||
|
diskUsagePercent: InfrastructureMetricPoint[];
|
||||||
|
networkReceiveBytesPerSecond: InfrastructureMetricPoint[];
|
||||||
|
networkTransmitBytesPerSecond: InfrastructureMetricPoint[];
|
||||||
|
};
|
||||||
|
services: InfrastructureServiceStatus[];
|
||||||
|
serviceMetrics: InfrastructureServiceMetricGroup[];
|
||||||
|
alerts: InfrastructureAlert[];
|
||||||
|
};
|
||||||
|
|
||||||
|
export type InfrastructureAlertThresholds = Record<string, { warning: number; critical: number }>;
|
||||||
|
|
||||||
|
export type InfrastructureAlertSettings = {
|
||||||
|
configVersion: number;
|
||||||
|
effectiveVersion: number;
|
||||||
|
applyStatus: 'effective' | 'applying' | 'failed';
|
||||||
|
lastError: string | null;
|
||||||
|
appliedAt: string | null;
|
||||||
|
thresholds: InfrastructureAlertThresholds;
|
||||||
|
effectiveThresholds: InfrastructureAlertThresholds;
|
||||||
|
definitions: Array<{ key: string; label: string; unit: string; min: number; max: number; step: number }>;
|
||||||
|
};
|
||||||
@@ -0,0 +1,34 @@
|
|||||||
|
import { Body, Controller, Get, Param, Post, Put, Query } from '@nestjs/common';
|
||||||
|
import { ApiTags } from '@nestjs/swagger';
|
||||||
|
import { CurrentSessionUserId } from '../auth/current-session-user.decorator';
|
||||||
|
import { RequireRecentAuthentication } from '../auth/require-recent-authentication.decorator';
|
||||||
|
import { InfrastructureAlertSettingsService } from './infrastructure-alert-settings.service';
|
||||||
|
import { InfrastructureMonitoringService } from './infrastructure-monitoring.service';
|
||||||
|
|
||||||
|
@ApiTags('infrastructure-monitoring')
|
||||||
|
@Controller('admin/infrastructure-monitoring')
|
||||||
|
export class InfrastructureMonitoringController {
|
||||||
|
constructor(private readonly monitoring: InfrastructureMonitoringService, private readonly settings: InfrastructureAlertSettingsService) {}
|
||||||
|
|
||||||
|
@Get('overview')
|
||||||
|
overview(@Query('range') range?: string, @CurrentSessionUserId() userId?: string) {
|
||||||
|
return this.monitoring.overview(range, userId);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Get('notification-summary')
|
||||||
|
notificationSummary(@CurrentSessionUserId() userId?: string) { return this.monitoring.notificationSummary(userId); }
|
||||||
|
|
||||||
|
@Post('alerts/:fingerprint/read')
|
||||||
|
markAlertRead(@Param('fingerprint') fingerprint: string, @Body('activeAt') activeAt: unknown, @CurrentSessionUserId() userId: string) {
|
||||||
|
return this.monitoring.markAlertRead(fingerprint, activeAt, userId);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Get('alert-thresholds')
|
||||||
|
alertThresholds() { return this.settings.get(); }
|
||||||
|
|
||||||
|
@Put('alert-thresholds')
|
||||||
|
@RequireRecentAuthentication()
|
||||||
|
updateAlertThresholds(@Body() body: { configVersion?: number; thresholds?: unknown }, @CurrentSessionUserId() operatorId?: string) {
|
||||||
|
return this.settings.update(body, operatorId);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,10 @@
|
|||||||
|
import { Module } from '@nestjs/common';
|
||||||
|
import { InfrastructureMonitoringController } from './infrastructure-monitoring.controller';
|
||||||
|
import { InfrastructureMonitoringService } from './infrastructure-monitoring.service';
|
||||||
|
import { InfrastructureAlertSettingsService } from './infrastructure-alert-settings.service';
|
||||||
|
|
||||||
|
@Module({
|
||||||
|
controllers: [InfrastructureMonitoringController],
|
||||||
|
providers: [InfrastructureMonitoringService, InfrastructureAlertSettingsService],
|
||||||
|
})
|
||||||
|
export class InfrastructureMonitoringModule {}
|
||||||
@@ -0,0 +1,137 @@
|
|||||||
|
import { BadRequestException } from '@nestjs/common';
|
||||||
|
import { ConfigService } from '@nestjs/config';
|
||||||
|
import { createHash } from 'node:crypto';
|
||||||
|
import { InfrastructureMonitoringService } from './infrastructure-monitoring.service';
|
||||||
|
|
||||||
|
function success(data: unknown) {
|
||||||
|
return {
|
||||||
|
ok: true,
|
||||||
|
status: 200,
|
||||||
|
json: async () => ({ status: 'success', data }),
|
||||||
|
} as Response;
|
||||||
|
}
|
||||||
|
|
||||||
|
describe('InfrastructureMonitoringService', () => {
|
||||||
|
const prisma = {
|
||||||
|
infrastructureAlertRead: { findMany: jest.fn().mockResolvedValue([]), create: jest.fn(), update: jest.fn(), findUniqueOrThrow: jest.fn() },
|
||||||
|
operationLog: { create: jest.fn() },
|
||||||
|
$transaction: jest.fn(),
|
||||||
|
};
|
||||||
|
|
||||||
|
afterEach(() => {
|
||||||
|
jest.restoreAllMocks();
|
||||||
|
jest.clearAllMocks();
|
||||||
|
prisma.infrastructureAlertRead.findMany.mockResolvedValue([]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('rejects ranges outside the fixed whitelist before querying Prometheus', async () => {
|
||||||
|
const fetchSpy = jest.spyOn(global, 'fetch');
|
||||||
|
const service = new InfrastructureMonitoringService(new ConfigService(), prisma as never);
|
||||||
|
|
||||||
|
await expect(service.overview('30d')).rejects.toBeInstanceOf(BadRequestException);
|
||||||
|
expect(fetchSpy).not.toHaveBeenCalled();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('rejects credential-bearing or remote plaintext Prometheus endpoints at startup', () => {
|
||||||
|
expect(() => new InfrastructureMonitoringService(new ConfigService({ PROMETHEUS_URL: 'http://user:secret@127.0.0.1:9090' }), prisma as never)).toThrow('must not contain credentials');
|
||||||
|
expect(() => new InfrastructureMonitoringService(new ConfigService({ PROMETHEUS_URL: 'http://monitor.example.com:9090' }), prisma as never)).toThrow('must use HTTPS');
|
||||||
|
expect(() => new InfrastructureMonitoringService(new ConfigService({ PROMETHEUS_URL: 'https://monitor.example.com' }), prisma as never)).not.toThrow();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('loads real Prometheus vectors, ranges, services and active alerts', async () => {
|
||||||
|
const requestedUrls: URL[] = [];
|
||||||
|
jest.spyOn(global, 'fetch').mockImplementation(async (input) => {
|
||||||
|
const url = new URL(String(input));
|
||||||
|
requestedUrls.push(url);
|
||||||
|
if (url.pathname.endsWith('/alerts')) {
|
||||||
|
return success({ alerts: [{
|
||||||
|
labels: { alertname: 'HostCpuHigh', severity: 'warning', instance: '127.0.0.1:9100' },
|
||||||
|
annotations: { summary: 'CPU持续偏高', threshold: '85%' },
|
||||||
|
state: 'firing',
|
||||||
|
activeAt: '2026-08-14T03:00:00.000Z',
|
||||||
|
value: '88.2',
|
||||||
|
}] });
|
||||||
|
}
|
||||||
|
const query = url.searchParams.get('query') ?? '';
|
||||||
|
if (url.pathname.endsWith('/query_range')) {
|
||||||
|
return success({ result: [{ metric: {}, values: [[1_765_000_000, '12.5'], [1_765_000_060, '14.5']] }] });
|
||||||
|
}
|
||||||
|
if (query.includes('node_systemd_unit_state')) {
|
||||||
|
return success({ result: [
|
||||||
|
{ metric: { name: 'cmpp-api.service' }, value: [1_765_000_060, '1'] },
|
||||||
|
{ metric: { name: 'cmpp-gateway.service' }, value: [1_765_000_060, '1'] },
|
||||||
|
{ metric: { name: 'postgresql.service' }, value: [1_765_000_060, '1'] },
|
||||||
|
{ metric: { name: 'redis-server.service' }, value: [1_765_000_060, '1'] },
|
||||||
|
{ metric: { name: 'cmpp-minio.service' }, value: [1_765_000_060, '1'] },
|
||||||
|
{ metric: { name: 'nginx.service' }, value: [1_765_000_060, '1'] },
|
||||||
|
] });
|
||||||
|
}
|
||||||
|
if (query.includes('cmpp:service_.*')) {
|
||||||
|
return success({ result: [
|
||||||
|
{ metric: { __name__: 'cmpp:service_api:requests_per_second' }, value: [1_765_000_060, '12.5'] },
|
||||||
|
{ metric: { __name__: 'cmpp:service_api:error_percent' }, value: [1_765_000_060, '0.2'] },
|
||||||
|
{ metric: { __name__: 'cmpp:service_gateway:queue_pending' }, value: [1_765_000_060, '3'] },
|
||||||
|
] });
|
||||||
|
}
|
||||||
|
if (query.includes('timestamp(node_uname_info)')) {
|
||||||
|
return success({ result: [{ metric: {}, value: [1_765_000_060, '1765000060'] }] });
|
||||||
|
}
|
||||||
|
return success({ result: [{ metric: {}, value: [1_765_000_060, '25'] }] });
|
||||||
|
});
|
||||||
|
|
||||||
|
const service = new InfrastructureMonitoringService(new ConfigService(), prisma as never);
|
||||||
|
const result = await service.overview('1h');
|
||||||
|
|
||||||
|
expect(result.available).toBe(true);
|
||||||
|
expect(result.metrics.cpuUsagePercent).toBe(25);
|
||||||
|
expect(result.trends.cpuUsagePercent).toHaveLength(2);
|
||||||
|
expect(result.summary).toMatchObject({ overallStatus: 'warning', serviceHealthy: 6, warningAlerts: 1 });
|
||||||
|
expect(result.services.find((item) => item.key === 'redis')).toMatchObject({ unit: 'redis-server.service', status: 'healthy' });
|
||||||
|
expect(result.serviceMetrics.find((item) => item.key === 'api')).toMatchObject({ available: true });
|
||||||
|
expect(result.serviceMetrics.find((item) => item.key === 'gateway')?.metrics.find((item) => item.key === 'queuePending')?.value).toBe(3);
|
||||||
|
expect(result.alerts[0]).toMatchObject({ name: 'HostCpuHigh', severity: 'warning', currentValue: '88.2' });
|
||||||
|
expect(requestedUrls.filter((url) => url.pathname.endsWith('/query_range'))).toHaveLength(5);
|
||||||
|
expect(requestedUrls.filter((url) => url.pathname.endsWith('/query_range')).every((url) => url.searchParams.get('step') === '60')).toBe(true);
|
||||||
|
expect(requestedUrls.find((url) => url.searchParams.get('query')?.includes('node_systemd_unit_state'))?.searchParams.get('query'))
|
||||||
|
.toContain('cmpp-api\\\\.service');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('returns an explicit unavailable payload without stale metrics when Prometheus fails', async () => {
|
||||||
|
jest.spyOn(global, 'fetch').mockRejectedValue(new Error('ECONNREFUSED'));
|
||||||
|
const service = new InfrastructureMonitoringService(new ConfigService(), prisma as never);
|
||||||
|
|
||||||
|
const result = await service.overview('24h');
|
||||||
|
|
||||||
|
expect(result.available).toBe(false);
|
||||||
|
expect(result.summary.overallStatus).toBe('unknown');
|
||||||
|
expect(result.metrics.cpuUsagePercent).toBeNull();
|
||||||
|
expect(result.trends.cpuUsagePercent).toEqual([]);
|
||||||
|
expect(result.serviceMetrics.every((item) => item.available === false)).toBe(true);
|
||||||
|
expect(result.error).not.toContain('ECONNREFUSED');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('excludes only the current alert occurrence after the current administrator marks it read', async () => {
|
||||||
|
const labels = { alertname: 'QaWarning', severity: 'warning', service: 'qa-preview' };
|
||||||
|
const fingerprint = createHash('sha256').update(JSON.stringify(Object.entries(labels).sort(([left], [right]) => left.localeCompare(right)))).digest('hex').slice(0, 24);
|
||||||
|
jest.spyOn(global, 'fetch').mockResolvedValue(success({ alerts: [{ labels, annotations: { summary: '演示预警' }, state: 'firing', activeAt: '2026-08-16T01:00:00.000Z' }] }));
|
||||||
|
prisma.infrastructureAlertRead.findMany.mockResolvedValue([{ fingerprint, activeAt: new Date('2026-08-16T01:00:00.000Z'), readAt: new Date('2026-08-16T01:01:00.000Z') }]);
|
||||||
|
const service = new InfrastructureMonitoringService(new ConfigService(), prisma as never);
|
||||||
|
|
||||||
|
await expect(service.notificationSummary('admin-1')).resolves.toEqual({ count: 0, criticalCount: 0 });
|
||||||
|
|
||||||
|
prisma.infrastructureAlertRead.findMany.mockResolvedValue([{ fingerprint, activeAt: new Date('2026-08-15T01:00:00.000Z'), readAt: new Date('2026-08-15T01:01:00.000Z') }]);
|
||||||
|
await expect(service.notificationSummary('admin-1')).resolves.toEqual({ count: 1, criticalCount: 0 });
|
||||||
|
});
|
||||||
|
|
||||||
|
it('upserts an idempotent per-user read record only for a currently active occurrence', async () => {
|
||||||
|
const labels = { alertname: 'QaCritical', severity: 'critical', service: 'qa-preview' };
|
||||||
|
const fingerprint = createHash('sha256').update(JSON.stringify(Object.entries(labels).sort(([left], [right]) => left.localeCompare(right)))).digest('hex').slice(0, 24);
|
||||||
|
jest.spyOn(global, 'fetch').mockResolvedValue(success({ alerts: [{ labels, annotations: { summary: '演示严重告警' }, state: 'firing', activeAt: '2026-08-16T02:00:00.000Z' }] }));
|
||||||
|
prisma.$transaction.mockResolvedValue([{ activeAt: new Date('2026-08-16T02:00:00.000Z'), readAt: new Date('2026-08-16T02:01:00.000Z') }, {}]);
|
||||||
|
const service = new InfrastructureMonitoringService(new ConfigService(), prisma as never);
|
||||||
|
|
||||||
|
await expect(service.markAlertRead(fingerprint, '2026-08-16T02:00:00.000Z', 'admin-1')).resolves.toMatchObject({ fingerprint, acknowledged: true });
|
||||||
|
expect(prisma.infrastructureAlertRead.create).toHaveBeenCalledWith(expect.objectContaining({ data: expect.objectContaining({ fingerprint, userId: 'admin-1' }) }));
|
||||||
|
await expect(service.markAlertRead(fingerprint, '2026-08-15T02:00:00.000Z', 'admin-1')).rejects.toThrow('已结束或已重新触发');
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -0,0 +1,383 @@
|
|||||||
|
import { BadRequestException, Injectable, Logger, NotFoundException, ServiceUnavailableException } from '@nestjs/common';
|
||||||
|
import { ConfigService } from '@nestjs/config';
|
||||||
|
import { Prisma } from '@prisma/client';
|
||||||
|
import { createHash } from 'node:crypto';
|
||||||
|
import { PrismaService } from '../prisma/prisma.service';
|
||||||
|
import type {
|
||||||
|
InfrastructureAlert,
|
||||||
|
InfrastructureMetricPoint,
|
||||||
|
InfrastructureMonitoringOverview,
|
||||||
|
InfrastructureMonitoringRange,
|
||||||
|
InfrastructureServiceStatus,
|
||||||
|
InfrastructureServiceMetricGroup,
|
||||||
|
} from './infrastructure-monitoring.contracts';
|
||||||
|
|
||||||
|
type PrometheusSample = [number, string];
|
||||||
|
type PrometheusSeries = {
|
||||||
|
metric: Record<string, string>;
|
||||||
|
value?: PrometheusSample;
|
||||||
|
values?: PrometheusSample[];
|
||||||
|
};
|
||||||
|
type PrometheusQueryResponse = {
|
||||||
|
status: 'success' | 'error';
|
||||||
|
data?: { result?: PrometheusSeries[] };
|
||||||
|
error?: string;
|
||||||
|
};
|
||||||
|
type PrometheusAlertResponse = {
|
||||||
|
status: 'success' | 'error';
|
||||||
|
data?: {
|
||||||
|
alerts?: Array<{
|
||||||
|
labels?: Record<string, string>;
|
||||||
|
annotations?: Record<string, string>;
|
||||||
|
state?: string;
|
||||||
|
activeAt?: string;
|
||||||
|
value?: string;
|
||||||
|
}>;
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
const RANGE_CONFIG: Record<InfrastructureMonitoringRange, { seconds: number; step: number }> = {
|
||||||
|
'1h': { seconds: 60 * 60, step: 60 },
|
||||||
|
'24h': { seconds: 24 * 60 * 60, step: 300 },
|
||||||
|
'7d': { seconds: 7 * 24 * 60 * 60, step: 1800 },
|
||||||
|
};
|
||||||
|
|
||||||
|
const QUERIES = {
|
||||||
|
cpuUsagePercent: '100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100)',
|
||||||
|
memoryUsagePercent: '(1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) * 100',
|
||||||
|
memoryTotalBytes: 'node_memory_MemTotal_bytes',
|
||||||
|
memoryAvailableBytes: 'node_memory_MemAvailable_bytes',
|
||||||
|
diskUsagePercent: '(1 - (node_filesystem_avail_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"} / node_filesystem_size_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"})) * 100',
|
||||||
|
diskTotalBytes: 'node_filesystem_size_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"}',
|
||||||
|
diskAvailableBytes: 'node_filesystem_avail_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"}',
|
||||||
|
networkReceiveBytesPerSecond: 'sum(rate(node_network_receive_bytes_total{device!~"lo"}[5m]))',
|
||||||
|
networkTransmitBytesPerSecond: 'sum(rate(node_network_transmit_bytes_total{device!~"lo"}[5m]))',
|
||||||
|
load1: 'node_load1',
|
||||||
|
uptimeSeconds: 'time() - node_boot_time_seconds',
|
||||||
|
lastSampleAt: 'max(timestamp(node_uname_info))',
|
||||||
|
// PromQL字符串本身需要两个反斜杠才能把正则的“\.”传给RE2;TypeScript字面量因此需要写四个。
|
||||||
|
services: 'max by (name) (node_systemd_unit_state{name=~"cmpp-api\\\\.service|cmpp-gateway\\\\.service|postgresql\\\\.service|redis(-server)?\\\\.service|cmpp-minio\\\\.service|nginx\\\\.service",state="active"})',
|
||||||
|
} as const;
|
||||||
|
|
||||||
|
const SERVICE_DEFINITIONS = [
|
||||||
|
{ key: 'api', name: 'API服务', units: ['cmpp-api.service'] },
|
||||||
|
{ key: 'gateway', name: 'Gateway服务', units: ['cmpp-gateway.service'] },
|
||||||
|
{ key: 'postgresql', name: 'PostgreSQL', units: ['postgresql.service'] },
|
||||||
|
{ key: 'redis', name: 'Redis', units: ['redis.service', 'redis-server.service'] },
|
||||||
|
{ key: 'minio', name: 'MinIO', units: ['cmpp-minio.service'] },
|
||||||
|
{ key: 'nginx', name: 'Nginx', units: ['nginx.service'] },
|
||||||
|
] as const;
|
||||||
|
|
||||||
|
const SERVICE_METRIC_DEFINITIONS = [
|
||||||
|
{ key: 'api', name: 'API服务', metrics: [
|
||||||
|
['requestsPerSecond', '请求速率', 'cmpp:service_api:requests_per_second', 'per_second'],
|
||||||
|
['errorPercent', '5xx错误率', 'cmpp:service_api:error_percent', 'percent'],
|
||||||
|
['latencyP95', 'P95响应', 'cmpp:service_api:latency_p95_seconds', 'seconds'],
|
||||||
|
['eventLoopP99', '事件循环P99', 'cmpp:service_api:event_loop_p99_seconds', 'seconds'],
|
||||||
|
] },
|
||||||
|
{ key: 'gateway', name: 'Gateway服务', metrics: [
|
||||||
|
['submitsPerSecond', '提交速率', 'cmpp:service_gateway:submits_per_second', 'per_second'],
|
||||||
|
['failurePercent', '提交失败率', 'cmpp:service_gateway:failure_percent', 'percent'],
|
||||||
|
['queuePending', 'Stream pending', 'cmpp:service_gateway:queue_pending', 'count'],
|
||||||
|
['queueOldestSeconds', '最旧pending', 'cmpp:service_gateway:queue_oldest_seconds', 'seconds'],
|
||||||
|
] },
|
||||||
|
{ key: 'postgresql', name: 'PostgreSQL', metrics: [
|
||||||
|
['connectionPercent', '连接使用率', 'cmpp:service_postgresql:connection_percent', 'percent'],
|
||||||
|
['deadlocks15m', '15分钟死锁', 'cmpp:service_postgresql:deadlocks_15m', 'count'],
|
||||||
|
] },
|
||||||
|
{ key: 'redis', name: 'Redis', metrics: [
|
||||||
|
['memoryPercent', '内存使用率', 'cmpp:service_redis:memory_percent', 'percent'],
|
||||||
|
['memoryUsedBytes', '已用内存', 'cmpp:service_redis:memory_used_bytes', 'bytes'],
|
||||||
|
['connectedClients', '客户端连接', 'cmpp:service_redis:connected_clients', 'count'],
|
||||||
|
['evictions5m', '5分钟淘汰', 'cmpp:service_redis:evictions_5m', 'count'],
|
||||||
|
] },
|
||||||
|
{ key: 'minio', name: 'MinIO', metrics: [
|
||||||
|
['capacityPercent', '存储容量使用率', 'cmpp:service_minio:capacity_percent', 'percent'],
|
||||||
|
['usageBytes', '对象数据量', 'cmpp:service_minio:usage_bytes', 'bytes'],
|
||||||
|
['objects', '对象数', 'cmpp:service_minio:objects', 'count'],
|
||||||
|
['drivesOffline', '离线存储盘', 'cmpp:service_minio:drives_offline', 'count'],
|
||||||
|
] },
|
||||||
|
{ key: 'nginx', name: 'Nginx', metrics: [
|
||||||
|
['connectionsActive', '活跃连接', 'cmpp:service_nginx:connections_active', 'count'],
|
||||||
|
['requestsPerSecond', '请求速率', 'cmpp:service_nginx:requests_per_second', 'per_second'],
|
||||||
|
] },
|
||||||
|
] as const;
|
||||||
|
|
||||||
|
const SERVICE_METRICS_QUERY = '{__name__=~"cmpp:service_.*"}';
|
||||||
|
|
||||||
|
function finiteNumber(value: string | number | undefined): number | null {
|
||||||
|
const parsed = Number(value);
|
||||||
|
return Number.isFinite(parsed) ? parsed : null;
|
||||||
|
}
|
||||||
|
|
||||||
|
function normalizePrometheusUrl(rawValue: unknown) {
|
||||||
|
const url = new URL(String(rawValue ?? 'http://127.0.0.1:9090'));
|
||||||
|
if (url.protocol !== 'http:' && url.protocol !== 'https:') throw new Error('PROMETHEUS_URL must use HTTP or HTTPS');
|
||||||
|
if (url.username || url.password) throw new Error('PROMETHEUS_URL must not contain credentials');
|
||||||
|
const privateIpv4 = /^(10\.|192\.168\.|172\.(1[6-9]|2\d|3[01])\.)/.test(url.hostname);
|
||||||
|
const loopback = url.hostname === '127.0.0.1' || url.hostname === 'localhost' || url.hostname === '[::1]';
|
||||||
|
// Plain HTTP is only safe on loopback or an explicit RFC1918 address; named remote endpoints must use HTTPS.
|
||||||
|
if (url.protocol === 'http:' && !loopback && !privateIpv4) throw new Error('Remote PROMETHEUS_URL must use HTTPS');
|
||||||
|
return url.toString().replace(/\/$/, '');
|
||||||
|
}
|
||||||
|
|
||||||
|
function vectorValue(response: PrometheusQueryResponse): number | null {
|
||||||
|
return finiteNumber(response.data?.result?.[0]?.value?.[1]);
|
||||||
|
}
|
||||||
|
|
||||||
|
function matrixValues(response: PrometheusQueryResponse): InfrastructureMetricPoint[] {
|
||||||
|
return (response.data?.result?.[0]?.values ?? []).flatMap(([timestamp, value]) => {
|
||||||
|
const parsed = finiteNumber(value);
|
||||||
|
return parsed === null ? [] : [{ timestamp: new Date(timestamp * 1000).toISOString(), value: parsed }];
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function emptyMetrics(): InfrastructureMonitoringOverview['metrics'] {
|
||||||
|
return {
|
||||||
|
cpuUsagePercent: null,
|
||||||
|
memoryUsagePercent: null,
|
||||||
|
memoryTotalBytes: null,
|
||||||
|
memoryAvailableBytes: null,
|
||||||
|
diskUsagePercent: null,
|
||||||
|
diskTotalBytes: null,
|
||||||
|
diskAvailableBytes: null,
|
||||||
|
networkReceiveBytesPerSecond: null,
|
||||||
|
networkTransmitBytesPerSecond: null,
|
||||||
|
load1: null,
|
||||||
|
uptimeSeconds: null,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function emptyTrends(): InfrastructureMonitoringOverview['trends'] {
|
||||||
|
return {
|
||||||
|
cpuUsagePercent: [],
|
||||||
|
memoryUsagePercent: [],
|
||||||
|
diskUsagePercent: [],
|
||||||
|
networkReceiveBytesPerSecond: [],
|
||||||
|
networkTransmitBytesPerSecond: [],
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
@Injectable()
|
||||||
|
export class InfrastructureMonitoringService {
|
||||||
|
private readonly logger = new Logger(InfrastructureMonitoringService.name);
|
||||||
|
private readonly prometheusUrl: string;
|
||||||
|
private readonly queryTimeoutMs: number;
|
||||||
|
|
||||||
|
constructor(config: ConfigService, private readonly prisma: PrismaService) {
|
||||||
|
this.prometheusUrl = normalizePrometheusUrl(config.get('PROMETHEUS_URL'));
|
||||||
|
this.queryTimeoutMs = Math.min(15_000, Math.max(1_000, Number(config.get('PROMETHEUS_QUERY_TIMEOUT_MS') ?? 5_000)));
|
||||||
|
}
|
||||||
|
|
||||||
|
async overview(rawRange?: string, userId?: string): Promise<InfrastructureMonitoringOverview> {
|
||||||
|
const range = this.parseRange(rawRange);
|
||||||
|
const collectedAt = new Date().toISOString();
|
||||||
|
try {
|
||||||
|
const [instant, trends, serviceResponse, serviceMetricResponse, alertResponse] = await Promise.all([
|
||||||
|
this.loadInstantMetrics(),
|
||||||
|
this.loadTrends(range),
|
||||||
|
this.query(QUERIES.services),
|
||||||
|
this.query(SERVICE_METRICS_QUERY),
|
||||||
|
this.getJson<PrometheusAlertResponse>('/api/v1/alerts'),
|
||||||
|
]);
|
||||||
|
const services = this.parseServices(serviceResponse);
|
||||||
|
const serviceMetrics = this.parseServiceMetrics(serviceMetricResponse);
|
||||||
|
const alerts = await this.attachReadState(this.parseAlerts(alertResponse), userId);
|
||||||
|
const warningAlerts = alerts.filter((item) => item.severity === 'warning').length;
|
||||||
|
const criticalAlerts = alerts.filter((item) => item.severity === 'critical').length;
|
||||||
|
const overallStatus = criticalAlerts > 0 ? 'critical' : warningAlerts > 0 ? 'warning' : 'healthy';
|
||||||
|
return {
|
||||||
|
available: true,
|
||||||
|
range,
|
||||||
|
collectedAt,
|
||||||
|
lastSampleAt: instant.lastSampleAt === null ? null : new Date(instant.lastSampleAt * 1000).toISOString(),
|
||||||
|
summary: {
|
||||||
|
overallStatus,
|
||||||
|
serviceTotal: services.length,
|
||||||
|
serviceHealthy: services.filter((item) => item.status === 'healthy').length,
|
||||||
|
warningAlerts,
|
||||||
|
criticalAlerts,
|
||||||
|
activeAlerts: alerts.length,
|
||||||
|
},
|
||||||
|
metrics: instant.metrics,
|
||||||
|
trends,
|
||||||
|
services,
|
||||||
|
serviceMetrics,
|
||||||
|
alerts,
|
||||||
|
};
|
||||||
|
} catch (error) {
|
||||||
|
// 页面必须整体清空陈旧指标,但服务端仍需留下不含PromQL/地址/凭据的根因摘要便于运维诊断。
|
||||||
|
this.logger.warn(`Prometheus monitoring overview unavailable: ${error instanceof Error ? error.message : 'unknown error'}`);
|
||||||
|
return this.unavailable(range, collectedAt);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async notificationSummary(userId?: string) {
|
||||||
|
try {
|
||||||
|
const alerts = await this.attachReadState(this.parseAlerts(await this.getJson<PrometheusAlertResponse>('/api/v1/alerts')), userId);
|
||||||
|
const unreadAlerts = alerts.filter((item) => !item.acknowledged);
|
||||||
|
return { count: unreadAlerts.length, criticalCount: unreadAlerts.filter((item) => item.severity === 'critical').length };
|
||||||
|
} catch (error) {
|
||||||
|
this.logger.warn(`Prometheus notification summary unavailable: ${error instanceof Error ? error.message : 'unknown error'}`);
|
||||||
|
throw new ServiceUnavailableException('Prometheus活动告警当前不可用');
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async markAlertRead(fingerprint: string, rawActiveAt: unknown, userId: string) {
|
||||||
|
if (!/^[a-f0-9]{24}$/.test(fingerprint)) throw new BadRequestException('告警指纹无效');
|
||||||
|
const activeAt = new Date(String(rawActiveAt ?? ''));
|
||||||
|
if (!Number.isFinite(activeAt.getTime())) throw new BadRequestException('告警开始时间无效');
|
||||||
|
const activeAlerts = this.parseAlerts(await this.getJson<PrometheusAlertResponse>('/api/v1/alerts'));
|
||||||
|
const current = activeAlerts.find((item) => item.fingerprint === fingerprint && Date.parse(item.startedAt) === activeAt.getTime());
|
||||||
|
if (!current) throw new NotFoundException('该次活动告警已结束或已重新触发,请刷新后重试');
|
||||||
|
const readAt = new Date();
|
||||||
|
const log = () => this.prisma.operationLog.create({
|
||||||
|
data: { userId, action: 'monitoring.alert_marked_read', resource: 'infrastructure_alert', resourceId: fingerprint, detail: { activeAt: activeAt.toISOString(), alertName: current.name, severity: current.severity } },
|
||||||
|
});
|
||||||
|
let read;
|
||||||
|
try {
|
||||||
|
[read] = await this.prisma.$transaction([
|
||||||
|
this.prisma.infrastructureAlertRead.create({ data: { fingerprint, activeAt, userId, readAt } }),
|
||||||
|
log(),
|
||||||
|
]);
|
||||||
|
} catch (error) {
|
||||||
|
if (!(error instanceof Prisma.PrismaClientKnownRequestError) || error.code !== 'P2002') throw error;
|
||||||
|
const existing = await this.prisma.infrastructureAlertRead.findUniqueOrThrow({ where: { fingerprint_userId: { fingerprint, userId } } });
|
||||||
|
// 同一次触发重复点击不更新readAt也不重复写日志;activeAt变化才代表同指纹的新触发周期。
|
||||||
|
if (existing.activeAt.getTime() === activeAt.getTime()) read = existing;
|
||||||
|
else [read] = await this.prisma.$transaction([
|
||||||
|
this.prisma.infrastructureAlertRead.update({ where: { fingerprint_userId: { fingerprint, userId } }, data: { activeAt, readAt } }),
|
||||||
|
log(),
|
||||||
|
]);
|
||||||
|
}
|
||||||
|
return { fingerprint, activeAt: read.activeAt.toISOString(), acknowledged: true, acknowledgedAt: read.readAt.toISOString() };
|
||||||
|
}
|
||||||
|
|
||||||
|
private parseRange(value?: string): InfrastructureMonitoringRange {
|
||||||
|
const range = value || '24h';
|
||||||
|
if (!(range in RANGE_CONFIG)) throw new BadRequestException('监控时间范围只支持1h、24h或7d');
|
||||||
|
return range as InfrastructureMonitoringRange;
|
||||||
|
}
|
||||||
|
|
||||||
|
private async loadInstantMetrics() {
|
||||||
|
const keys = Object.keys(emptyMetrics()) as Array<keyof InfrastructureMonitoringOverview['metrics']>;
|
||||||
|
const responses = await Promise.all([...keys.map((key) => this.query(QUERIES[key])), this.query(QUERIES.lastSampleAt)]);
|
||||||
|
const metrics = emptyMetrics();
|
||||||
|
keys.forEach((key, index) => { metrics[key] = vectorValue(responses[index]); });
|
||||||
|
return { metrics, lastSampleAt: vectorValue(responses[responses.length - 1]) };
|
||||||
|
}
|
||||||
|
|
||||||
|
private async loadTrends(range: InfrastructureMonitoringRange) {
|
||||||
|
const config = RANGE_CONFIG[range];
|
||||||
|
const end = Math.floor(Date.now() / 1000);
|
||||||
|
const start = end - config.seconds;
|
||||||
|
const keys = Object.keys(emptyTrends()) as Array<keyof InfrastructureMonitoringOverview['trends']>;
|
||||||
|
const responses = await Promise.all(keys.map((key) => this.queryRange(QUERIES[key], start, end, config.step)));
|
||||||
|
return Object.fromEntries(keys.map((key, index) => [key, matrixValues(responses[index])])) as InfrastructureMonitoringOverview['trends'];
|
||||||
|
}
|
||||||
|
|
||||||
|
private parseServices(response: PrometheusQueryResponse): InfrastructureServiceStatus[] {
|
||||||
|
const values = new Map<string, number>();
|
||||||
|
for (const item of response.data?.result ?? []) {
|
||||||
|
if (item.metric.name) values.set(item.metric.name, vectorValue({ status: 'success', data: { result: [item] } }) ?? 0);
|
||||||
|
}
|
||||||
|
return SERVICE_DEFINITIONS.map((definition) => {
|
||||||
|
const present = definition.units.filter((unit) => values.has(unit));
|
||||||
|
const status = present.length === 0 ? 'unknown' : present.some((unit) => (values.get(unit) ?? 0) >= 1) ? 'healthy' : 'unhealthy';
|
||||||
|
return { key: definition.key, name: definition.name, unit: present[0] ?? definition.units[0], status };
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
private parseAlerts(response: PrometheusAlertResponse): InfrastructureAlert[] {
|
||||||
|
return (response.data?.alerts ?? [])
|
||||||
|
.filter((item) => item.state === 'firing' || item.state === 'pending')
|
||||||
|
.map<InfrastructureAlert>((item) => {
|
||||||
|
const labels = item.labels ?? {};
|
||||||
|
const annotations = item.annotations ?? {};
|
||||||
|
const severity: InfrastructureAlert['severity'] = labels.severity === 'critical' ? 'critical' : labels.severity === 'warning' ? 'warning' : 'info';
|
||||||
|
const identity = JSON.stringify(Object.entries(labels).sort(([left], [right]) => left.localeCompare(right)));
|
||||||
|
return {
|
||||||
|
fingerprint: createHash('sha256').update(identity).digest('hex').slice(0, 24),
|
||||||
|
name: labels.alertname || '未命名告警',
|
||||||
|
severity,
|
||||||
|
status: item.state || 'unknown',
|
||||||
|
startedAt: item.activeAt || new Date().toISOString(),
|
||||||
|
summary: annotations.summary || annotations.description || labels.alertname || '监控告警',
|
||||||
|
description: annotations.description,
|
||||||
|
currentValue: annotations.currentValue || item.value,
|
||||||
|
threshold: annotations.threshold,
|
||||||
|
service: labels.service,
|
||||||
|
instance: labels.instance,
|
||||||
|
acknowledged: false,
|
||||||
|
};
|
||||||
|
})
|
||||||
|
.sort((left, right) => {
|
||||||
|
const priority: Record<InfrastructureAlert['severity'], number> = { critical: 0, warning: 1, info: 2 };
|
||||||
|
return priority[left.severity] - priority[right.severity] || Date.parse(left.startedAt) - Date.parse(right.startedAt);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
private async attachReadState(alerts: InfrastructureAlert[], userId?: string) {
|
||||||
|
if (!userId || alerts.length === 0) return alerts;
|
||||||
|
const reads = await this.prisma.infrastructureAlertRead.findMany({
|
||||||
|
where: { userId, fingerprint: { in: alerts.map((item) => item.fingerprint) } },
|
||||||
|
select: { fingerprint: true, activeAt: true, readAt: true },
|
||||||
|
});
|
||||||
|
const byFingerprint = new Map(reads.map((item) => [item.fingerprint, item]));
|
||||||
|
return alerts.map((alert) => {
|
||||||
|
const read = byFingerprint.get(alert.fingerprint);
|
||||||
|
const acknowledged = Boolean(read && read.activeAt.getTime() === Date.parse(alert.startedAt));
|
||||||
|
return { ...alert, acknowledged, acknowledgedAt: acknowledged ? read?.readAt.toISOString() : undefined };
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
private parseServiceMetrics(response: PrometheusQueryResponse): InfrastructureServiceMetricGroup[] {
|
||||||
|
const values = new Map<string, number>();
|
||||||
|
for (const item of response.data?.result ?? []) {
|
||||||
|
const metricName = item.metric.__name__;
|
||||||
|
const value = vectorValue({ status: 'success', data: { result: [item] } });
|
||||||
|
if (metricName && value !== null) values.set(metricName, value);
|
||||||
|
}
|
||||||
|
return SERVICE_METRIC_DEFINITIONS.map((group) => ({
|
||||||
|
key: group.key,
|
||||||
|
name: group.name,
|
||||||
|
available: group.metrics.some((metric) => values.has(metric[2])),
|
||||||
|
metrics: group.metrics.map(([key, label, metricName, unit]) => ({ key, label, value: values.get(metricName) ?? null, unit })),
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
|
||||||
|
private unavailable(range: InfrastructureMonitoringRange, collectedAt: string): InfrastructureMonitoringOverview {
|
||||||
|
const services = SERVICE_DEFINITIONS.map((item) => ({ key: item.key, name: item.name, unit: item.units[0], status: 'unknown' as const }));
|
||||||
|
return {
|
||||||
|
available: false,
|
||||||
|
range,
|
||||||
|
collectedAt,
|
||||||
|
lastSampleAt: null,
|
||||||
|
error: 'Prometheus监控数据当前不可用,请检查采集与服务状态',
|
||||||
|
summary: { overallStatus: 'unknown', serviceTotal: services.length, serviceHealthy: 0, warningAlerts: 0, criticalAlerts: 0, activeAlerts: 0 },
|
||||||
|
metrics: emptyMetrics(),
|
||||||
|
trends: emptyTrends(),
|
||||||
|
services,
|
||||||
|
serviceMetrics: SERVICE_METRIC_DEFINITIONS.map((group) => ({ key: group.key, name: group.name, available: false, metrics: [] })),
|
||||||
|
alerts: [],
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
private query(query: string) {
|
||||||
|
return this.getJson<PrometheusQueryResponse>('/api/v1/query', { query });
|
||||||
|
}
|
||||||
|
|
||||||
|
private queryRange(query: string, start: number, end: number, step: number) {
|
||||||
|
return this.getJson<PrometheusQueryResponse>('/api/v1/query_range', { query, start: String(start), end: String(end), step: String(step) });
|
||||||
|
}
|
||||||
|
|
||||||
|
private async getJson<T extends { status: 'success' | 'error'; error?: string }>(path: string, params: Record<string, string> = {}): Promise<T> {
|
||||||
|
const url = new URL(`${this.prometheusUrl}${path}`);
|
||||||
|
Object.entries(params).forEach(([key, value]) => url.searchParams.set(key, value));
|
||||||
|
const response = await fetch(url, { headers: { Accept: 'application/json' }, signal: AbortSignal.timeout(this.queryTimeoutMs) });
|
||||||
|
if (!response.ok) throw new Error(`Prometheus HTTP ${response.status}`);
|
||||||
|
const result = await response.json() as T;
|
||||||
|
if (result.status !== 'success') throw new Error('Prometheus query failed');
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
}
|
||||||
+22
-1
@@ -1,8 +1,10 @@
|
|||||||
import 'reflect-metadata';
|
import 'reflect-metadata';
|
||||||
import { NestFactory } from '@nestjs/core';
|
import { NestFactory } from '@nestjs/core';
|
||||||
|
import { createServer } from 'node:http';
|
||||||
import type { NestExpressApplication } from '@nestjs/platform-express';
|
import type { NestExpressApplication } from '@nestjs/platform-express';
|
||||||
import { DocumentBuilder, SwaggerModule } from '@nestjs/swagger';
|
import { DocumentBuilder, SwaggerModule } from '@nestjs/swagger';
|
||||||
import { AppModule } from './app.module';
|
import { AppModule } from './app.module';
|
||||||
|
import { MetricsService } from './metrics/metrics.service';
|
||||||
import { OpenApiModule } from './open-api/open-api.module';
|
import { OpenApiModule } from './open-api/open-api.module';
|
||||||
import { configureHttpBodyParsers } from './http-body-limits';
|
import { configureHttpBodyParsers } from './http-body-limits';
|
||||||
|
|
||||||
@@ -39,7 +41,26 @@ async function bootstrap() {
|
|||||||
SwaggerModule.setup('api/client-docs', app, clientDocument);
|
SwaggerModule.setup('api/client-docs', app, clientDocument);
|
||||||
|
|
||||||
const port = Number(process.env.API_PORT ?? 3000);
|
const port = Number(process.env.API_PORT ?? 3000);
|
||||||
await app.listen(port);
|
// 生产环境只允许 Nginx 访问管理 API;显式绑定回环,避免默认的全网卡监听绕过入口鉴权与限流。
|
||||||
|
const host = process.env.API_HOST?.trim() || '127.0.0.1';
|
||||||
|
await app.listen(port, host);
|
||||||
|
|
||||||
|
const metrics = app.get(MetricsService);
|
||||||
|
const metricsHost = process.env.API_METRICS_HOST?.trim() || '127.0.0.1';
|
||||||
|
const metricsPort = Number(process.env.API_METRICS_PORT ?? 9464);
|
||||||
|
const metricsServer = createServer((request, response) => {
|
||||||
|
if (request.method !== 'GET' || request.url !== '/metrics') {
|
||||||
|
response.writeHead(404).end();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
response.writeHead(200, { 'Content-Type': 'text/plain; version=0.0.4; charset=utf-8', 'Cache-Control': 'no-store' });
|
||||||
|
response.end(metrics.render());
|
||||||
|
});
|
||||||
|
// Metrics use a dedicated loopback listener so Nginx cannot accidentally expose them through /api/.
|
||||||
|
await new Promise<void>((resolve, reject) => {
|
||||||
|
metricsServer.once('error', reject);
|
||||||
|
metricsServer.listen(metricsPort, metricsHost, resolve);
|
||||||
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
void bootstrap();
|
void bootstrap();
|
||||||
|
|||||||
@@ -0,0 +1,24 @@
|
|||||||
|
import { CallHandler, ExecutionContext, Injectable, NestInterceptor } from '@nestjs/common';
|
||||||
|
import type { Observable } from 'rxjs';
|
||||||
|
import { finalize } from 'rxjs/operators';
|
||||||
|
import { MetricsService } from './metrics.service';
|
||||||
|
|
||||||
|
type RequestLike = { method?: string; baseUrl?: string; route?: { path?: string } };
|
||||||
|
type ResponseLike = { statusCode?: number };
|
||||||
|
|
||||||
|
@Injectable()
|
||||||
|
export class MetricsInterceptor implements NestInterceptor {
|
||||||
|
constructor(private readonly metrics: MetricsService) {}
|
||||||
|
|
||||||
|
intercept(context: ExecutionContext, next: CallHandler): Observable<unknown> {
|
||||||
|
if (context.getType() !== 'http') return next.handle();
|
||||||
|
const http = context.switchToHttp();
|
||||||
|
const request = http.getRequest<RequestLike>();
|
||||||
|
const response = http.getResponse<ResponseLike>();
|
||||||
|
const startedAt = this.metrics.beginRequest();
|
||||||
|
return next.handle().pipe(finalize(() => {
|
||||||
|
const route = `${request.baseUrl ?? ''}${request.route?.path ?? '/unmatched'}`;
|
||||||
|
this.metrics.finishRequest(startedAt, request.method ?? 'UNKNOWN', route, response.statusCode ?? 500);
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,11 @@
|
|||||||
|
import { Global, Module } from '@nestjs/common';
|
||||||
|
import { APP_INTERCEPTOR } from '@nestjs/core';
|
||||||
|
import { MetricsInterceptor } from './metrics.interceptor';
|
||||||
|
import { MetricsService } from './metrics.service';
|
||||||
|
|
||||||
|
@Global()
|
||||||
|
@Module({
|
||||||
|
providers: [MetricsService, { provide: APP_INTERCEPTOR, useClass: MetricsInterceptor }],
|
||||||
|
exports: [MetricsService],
|
||||||
|
})
|
||||||
|
export class MetricsModule {}
|
||||||
@@ -0,0 +1,16 @@
|
|||||||
|
import { MetricsService } from './metrics.service';
|
||||||
|
|
||||||
|
describe('MetricsService', () => {
|
||||||
|
it('exports bounded API process and HTTP metrics without raw identifiers', () => {
|
||||||
|
const service = new MetricsService();
|
||||||
|
const startedAt = service.beginRequest();
|
||||||
|
service.finishRequest(startedAt, 'GET', '/api/admin/tenants/:id', 200);
|
||||||
|
const output = service.render();
|
||||||
|
|
||||||
|
expect(output).toContain('cmpp_api_process_resident_memory_bytes');
|
||||||
|
expect(output).toContain('cmpp_api_http_requests_total{method="GET",route="/api/admin/tenants/:id",status="200"} 1');
|
||||||
|
expect(output).toContain('cmpp_api_http_request_duration_seconds_bucket');
|
||||||
|
expect(output).not.toContain('phone_number');
|
||||||
|
service.onModuleDestroy();
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -0,0 +1,100 @@
|
|||||||
|
import { Injectable, OnModuleDestroy } from '@nestjs/common';
|
||||||
|
import { monitorEventLoopDelay } from 'node:perf_hooks';
|
||||||
|
|
||||||
|
const HTTP_DURATION_BUCKETS = [0.05, 0.1, 0.25, 0.5, 1, 3, 10] as const;
|
||||||
|
|
||||||
|
type HttpMetric = {
|
||||||
|
count: number;
|
||||||
|
durationSum: number;
|
||||||
|
buckets: number[];
|
||||||
|
};
|
||||||
|
|
||||||
|
function escapeLabel(value: string) {
|
||||||
|
return value.replace(/\\/g, '\\\\').replace(/\n/g, '\\n').replace(/"/g, '\\"');
|
||||||
|
}
|
||||||
|
|
||||||
|
function metricLine(name: string, value: number, labels?: Record<string, string>) {
|
||||||
|
const suffix = labels
|
||||||
|
? `{${Object.entries(labels).map(([key, item]) => `${key}="${escapeLabel(item)}"`).join(',')}}`
|
||||||
|
: '';
|
||||||
|
return `${name}${suffix} ${Number.isFinite(value) ? value : 0}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Injectable()
|
||||||
|
export class MetricsService implements OnModuleDestroy {
|
||||||
|
private readonly startedAt = process.hrtime.bigint();
|
||||||
|
private readonly eventLoopDelay = monitorEventLoopDelay({ resolution: 20 });
|
||||||
|
private readonly http = new Map<string, HttpMetric>();
|
||||||
|
private inFlight = 0;
|
||||||
|
|
||||||
|
constructor() {
|
||||||
|
this.eventLoopDelay.enable();
|
||||||
|
}
|
||||||
|
|
||||||
|
beginRequest() {
|
||||||
|
this.inFlight += 1;
|
||||||
|
return process.hrtime.bigint();
|
||||||
|
}
|
||||||
|
|
||||||
|
finishRequest(startedAt: bigint, method: string, route: string, statusCode: number) {
|
||||||
|
this.inFlight = Math.max(0, this.inFlight - 1);
|
||||||
|
// Only route templates enter labels. Raw URLs, IDs, phone numbers and query strings would create unbounded time series.
|
||||||
|
const normalizedRoute = route.startsWith('/') ? route : `/${route}`;
|
||||||
|
const labels = [method.toUpperCase(), normalizedRoute, String(statusCode)];
|
||||||
|
const key = labels.join('\u0000');
|
||||||
|
const metric = this.http.get(key) ?? { count: 0, durationSum: 0, buckets: HTTP_DURATION_BUCKETS.map(() => 0) };
|
||||||
|
const durationSeconds = Number(process.hrtime.bigint() - startedAt) / 1_000_000_000;
|
||||||
|
metric.count += 1;
|
||||||
|
metric.durationSum += durationSeconds;
|
||||||
|
HTTP_DURATION_BUCKETS.forEach((bucket, index) => {
|
||||||
|
if (durationSeconds <= bucket) metric.buckets[index] += 1;
|
||||||
|
});
|
||||||
|
this.http.set(key, metric);
|
||||||
|
}
|
||||||
|
|
||||||
|
render() {
|
||||||
|
const memory = process.memoryUsage();
|
||||||
|
const uptime = Number(process.hrtime.bigint() - this.startedAt) / 1_000_000_000;
|
||||||
|
const lines = [
|
||||||
|
'# HELP cmpp_api_process_uptime_seconds API process uptime.',
|
||||||
|
'# TYPE cmpp_api_process_uptime_seconds gauge',
|
||||||
|
metricLine('cmpp_api_process_uptime_seconds', uptime),
|
||||||
|
'# HELP cmpp_api_process_resident_memory_bytes API resident memory.',
|
||||||
|
'# TYPE cmpp_api_process_resident_memory_bytes gauge',
|
||||||
|
metricLine('cmpp_api_process_resident_memory_bytes', memory.rss),
|
||||||
|
'# HELP cmpp_api_nodejs_heap_used_bytes Node.js heap currently used.',
|
||||||
|
'# TYPE cmpp_api_nodejs_heap_used_bytes gauge',
|
||||||
|
metricLine('cmpp_api_nodejs_heap_used_bytes', memory.heapUsed),
|
||||||
|
'# HELP cmpp_api_nodejs_heap_total_bytes Node.js allocated heap.',
|
||||||
|
'# TYPE cmpp_api_nodejs_heap_total_bytes gauge',
|
||||||
|
metricLine('cmpp_api_nodejs_heap_total_bytes', memory.heapTotal),
|
||||||
|
'# HELP cmpp_api_nodejs_event_loop_lag_p99_seconds Event loop delay p99 since the previous scrape.',
|
||||||
|
'# TYPE cmpp_api_nodejs_event_loop_lag_p99_seconds gauge',
|
||||||
|
metricLine('cmpp_api_nodejs_event_loop_lag_p99_seconds', this.eventLoopDelay.count ? this.eventLoopDelay.percentile(99) / 1_000_000_000 : 0),
|
||||||
|
'# HELP cmpp_api_http_requests_in_flight Current API requests in flight.',
|
||||||
|
'# TYPE cmpp_api_http_requests_in_flight gauge',
|
||||||
|
metricLine('cmpp_api_http_requests_in_flight', this.inFlight),
|
||||||
|
'# HELP cmpp_api_http_requests_total API requests grouped by bounded route templates.',
|
||||||
|
'# TYPE cmpp_api_http_requests_total counter',
|
||||||
|
'# HELP cmpp_api_http_request_duration_seconds API request duration.',
|
||||||
|
'# TYPE cmpp_api_http_request_duration_seconds histogram',
|
||||||
|
];
|
||||||
|
for (const [key, metric] of this.http) {
|
||||||
|
const [method, route, status] = key.split('\u0000');
|
||||||
|
const labels = { method, route, status };
|
||||||
|
lines.push(metricLine('cmpp_api_http_requests_total', metric.count, labels));
|
||||||
|
HTTP_DURATION_BUCKETS.forEach((bucket, index) => {
|
||||||
|
lines.push(metricLine('cmpp_api_http_request_duration_seconds_bucket', metric.buckets[index], { ...labels, le: String(bucket) }));
|
||||||
|
});
|
||||||
|
lines.push(metricLine('cmpp_api_http_request_duration_seconds_bucket', metric.count, { ...labels, le: '+Inf' }));
|
||||||
|
lines.push(metricLine('cmpp_api_http_request_duration_seconds_sum', metric.durationSum, labels));
|
||||||
|
lines.push(metricLine('cmpp_api_http_request_duration_seconds_count', metric.count, labels));
|
||||||
|
}
|
||||||
|
this.eventLoopDelay.reset();
|
||||||
|
return `${lines.join('\n')}\n`;
|
||||||
|
}
|
||||||
|
|
||||||
|
onModuleDestroy() {
|
||||||
|
this.eventLoopDelay.disable();
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -5,12 +5,13 @@ import IORedis from 'ioredis';
|
|||||||
import { PrismaService } from '../prisma/prisma.service';
|
import { PrismaService } from '../prisma/prisma.service';
|
||||||
import { decryptSecret } from './open-api.crypto';
|
import { decryptSecret } from './open-api.crypto';
|
||||||
import type { OpenApiRequestLike } from './open-api.types';
|
import type { OpenApiRequestLike } from './open-api.types';
|
||||||
|
import { SecurityDetectionService } from '../security-detection/security-detection.service';
|
||||||
|
|
||||||
@Injectable()
|
@Injectable()
|
||||||
export class OpenApiAuthGuard implements CanActivate, OnModuleDestroy {
|
export class OpenApiAuthGuard implements CanActivate, OnModuleDestroy {
|
||||||
private redis?: IORedis;
|
private redis?: IORedis;
|
||||||
|
|
||||||
constructor(private readonly prisma: PrismaService) {}
|
constructor(private readonly prisma: PrismaService, private readonly security: SecurityDetectionService) {}
|
||||||
|
|
||||||
async canActivate(context: ExecutionContext) {
|
async canActivate(context: ExecutionContext) {
|
||||||
const request = context.switchToHttp().getRequest<OpenApiRequestLike>();
|
const request = context.switchToHttp().getRequest<OpenApiRequestLike>();
|
||||||
@@ -19,9 +20,11 @@ export class OpenApiAuthGuard implements CanActivate, OnModuleDestroy {
|
|||||||
const nonce = header(request, 'x-nonce');
|
const nonce = header(request, 'x-nonce');
|
||||||
const suppliedSignature = header(request, 'x-signature')?.replace(/^sha256=/i, '');
|
const suppliedSignature = header(request, 'x-signature')?.replace(/^sha256=/i, '');
|
||||||
if (!accessKey || !timestampText || !nonce || !suppliedSignature) {
|
if (!accessKey || !timestampText || !nonce || !suppliedSignature) {
|
||||||
|
await this.recordFailure('http_signature_failure', request, undefined, 'AUTH_HEADERS_MISSING');
|
||||||
throw new UnauthorizedException({ code: 'AUTH_HEADERS_MISSING', message: '缺少HTTP接口鉴权请求头' });
|
throw new UnauthorizedException({ code: 'AUTH_HEADERS_MISSING', message: '缺少HTTP接口鉴权请求头' });
|
||||||
}
|
}
|
||||||
if (!/^[A-Za-z0-9_-]{8,128}$/.test(nonce)) {
|
if (!/^[A-Za-z0-9_-]{8,128}$/.test(nonce)) {
|
||||||
|
await this.recordFailure('http_signature_failure', request, accessKey, 'NONCE_INVALID');
|
||||||
throw new UnauthorizedException({ code: 'NONCE_INVALID', message: 'X-Nonce 格式非法' });
|
throw new UnauthorizedException({ code: 'NONCE_INVALID', message: 'X-Nonce 格式非法' });
|
||||||
}
|
}
|
||||||
const credential = await this.prisma.httpApiCredential.findUnique({
|
const credential = await this.prisma.httpApiCredential.findUnique({
|
||||||
@@ -29,6 +32,7 @@ export class OpenApiAuthGuard implements CanActivate, OnModuleDestroy {
|
|||||||
include: { application: { include: { httpConfig: true, httpIpAllowlist: true } } },
|
include: { application: { include: { httpConfig: true, httpIpAllowlist: true } } },
|
||||||
});
|
});
|
||||||
if (!credential || credential.status !== 'active' || (credential.expiresAt && credential.expiresAt <= new Date())) {
|
if (!credential || credential.status !== 'active' || (credential.expiresAt && credential.expiresAt <= new Date())) {
|
||||||
|
await this.recordFailure('http_invalid_api_key', request, accessKey, 'CREDENTIAL_INVALID');
|
||||||
throw new UnauthorizedException({ code: 'CREDENTIAL_INVALID', message: '访问凭据无效或已失效' });
|
throw new UnauthorizedException({ code: 'CREDENTIAL_INVALID', message: '访问凭据无效或已失效' });
|
||||||
}
|
}
|
||||||
const config = credential.application.httpConfig;
|
const config = credential.application.httpConfig;
|
||||||
@@ -37,6 +41,7 @@ export class OpenApiAuthGuard implements CanActivate, OnModuleDestroy {
|
|||||||
}
|
}
|
||||||
const timestamp = Number(timestampText);
|
const timestamp = Number(timestampText);
|
||||||
if (!Number.isFinite(timestamp) || Math.abs(Date.now() - timestamp * 1000) > config.timestampToleranceSeconds * 1000) {
|
if (!Number.isFinite(timestamp) || Math.abs(Date.now() - timestamp * 1000) > config.timestampToleranceSeconds * 1000) {
|
||||||
|
await this.recordFailure('http_signature_failure', request, accessKey, 'TIMESTAMP_EXPIRED');
|
||||||
throw new UnauthorizedException({ code: 'TIMESTAMP_EXPIRED', message: '请求时间戳已过期' });
|
throw new UnauthorizedException({ code: 'TIMESTAMP_EXPIRED', message: '请求时间戳已过期' });
|
||||||
}
|
}
|
||||||
const sourceIp = requestIp(request);
|
const sourceIp = requestIp(request);
|
||||||
@@ -50,11 +55,13 @@ export class OpenApiAuthGuard implements CanActivate, OnModuleDestroy {
|
|||||||
const expectedBuffer = Buffer.from(expected, 'hex');
|
const expectedBuffer = Buffer.from(expected, 'hex');
|
||||||
const suppliedBuffer = /^[0-9a-f]{64}$/i.test(suppliedSignature) ? Buffer.from(suppliedSignature, 'hex') : Buffer.alloc(0);
|
const suppliedBuffer = /^[0-9a-f]{64}$/i.test(suppliedSignature) ? Buffer.from(suppliedSignature, 'hex') : Buffer.alloc(0);
|
||||||
if (expectedBuffer.length !== suppliedBuffer.length || !timingSafeEqual(expectedBuffer, suppliedBuffer)) {
|
if (expectedBuffer.length !== suppliedBuffer.length || !timingSafeEqual(expectedBuffer, suppliedBuffer)) {
|
||||||
|
await this.recordFailure('http_signature_failure', request, accessKey, 'SIGNATURE_INVALID');
|
||||||
throw new UnauthorizedException({ code: 'SIGNATURE_INVALID', message: '请求签名校验失败' });
|
throw new UnauthorizedException({ code: 'SIGNATURE_INVALID', message: '请求签名校验失败' });
|
||||||
}
|
}
|
||||||
const redis = this.getRedis();
|
const redis = this.getRedis();
|
||||||
const nonceAccepted = await redis.set(`openapi:nonce:${credential.id}:${nonce}`, '1', 'EX', config.timestampToleranceSeconds * 2, 'NX');
|
const nonceAccepted = await redis.set(`openapi:nonce:${credential.id}:${nonce}`, '1', 'EX', config.timestampToleranceSeconds * 2, 'NX');
|
||||||
if (nonceAccepted !== 'OK') {
|
if (nonceAccepted !== 'OK') {
|
||||||
|
await this.recordFailure('http_replay_attempt', request, accessKey, 'NONCE_REPLAYED');
|
||||||
throw new UnauthorizedException({ code: 'NONCE_REPLAYED', message: 'X-Nonce 已使用' });
|
throw new UnauthorizedException({ code: 'NONCE_REPLAYED', message: 'X-Nonce 已使用' });
|
||||||
}
|
}
|
||||||
const second = Math.floor(Date.now() / 1000);
|
const second = Math.floor(Date.now() / 1000);
|
||||||
@@ -81,6 +88,13 @@ export class OpenApiAuthGuard implements CanActivate, OnModuleDestroy {
|
|||||||
this.redis ??= new IORedis(process.env.REDIS_URL ?? 'redis://127.0.0.1:6379', { maxRetriesPerRequest: 1 });
|
this.redis ??= new IORedis(process.env.REDIS_URL ?? 'redis://127.0.0.1:6379', { maxRetriesPerRequest: 1 });
|
||||||
return this.redis;
|
return this.redis;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private async recordFailure(ruleCode: 'http_invalid_api_key' | 'http_signature_failure' | 'http_replay_attempt', request: OpenApiRequestLike, account: string | undefined, resultCode: string) {
|
||||||
|
const sourceIp = requestIp(request);
|
||||||
|
if (!sourceIp) return;
|
||||||
|
// 检测记录失败不能改变原鉴权响应,避免安全辅助链路放大为业务可用性事故。
|
||||||
|
await this.security.recordEvent({ ruleCode, sourceIp, account, resultCode, protocol: 'http', path: (request.originalUrl ?? request.url ?? '').split('?')[0] }).catch(() => undefined);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
function header(request: OpenApiRequestLike, name: string) {
|
function header(request: OpenApiRequestLike, name: string) {
|
||||||
@@ -90,7 +104,9 @@ function header(request: OpenApiRequestLike, name: string) {
|
|||||||
|
|
||||||
function requestIp(request: OpenApiRequestLike) {
|
function requestIp(request: OpenApiRequestLike) {
|
||||||
const forwarded = header(request, 'x-forwarded-for')?.split(',')[0]?.trim();
|
const forwarded = header(request, 'x-forwarded-for')?.split(',')[0]?.trim();
|
||||||
return (forwarded ?? request.socket?.remoteAddress)?.replace(/^::ffff:/, '');
|
const remoteAddress = request.socket?.remoteAddress?.replace(/^::ffff:/, '');
|
||||||
|
const trustedProxies = new Set((process.env.TRUSTED_PROXY_IPS ?? '127.0.0.1,::1').split(',').map((item) => item.trim()).filter(Boolean));
|
||||||
|
return (remoteAddress && trustedProxies.has(remoteAddress) ? forwarded : remoteAddress)?.replace(/^::ffff:/, '');
|
||||||
}
|
}
|
||||||
|
|
||||||
function ipMatches(ip: string, rule: string) {
|
function ipMatches(ip: string, rule: string) {
|
||||||
|
|||||||
@@ -6,9 +6,10 @@ import { ClientOpenApiController } from './client-open-api.controller';
|
|||||||
import { OpenApiAuthGuard } from './open-api-auth.guard';
|
import { OpenApiAuthGuard } from './open-api-auth.guard';
|
||||||
import { OpenApiController } from './open-api.controller';
|
import { OpenApiController } from './open-api.controller';
|
||||||
import { OpenApiService } from './open-api.service';
|
import { OpenApiService } from './open-api.service';
|
||||||
|
import { SecurityDetectionModule } from '../security-detection/security-detection.module';
|
||||||
|
|
||||||
@Module({
|
@Module({
|
||||||
imports: [PrismaModule, forwardRef(() => SendChainModule)],
|
imports: [PrismaModule, forwardRef(() => SendChainModule), SecurityDetectionModule],
|
||||||
controllers: [OpenApiController, AdminOpenApiController, ClientOpenApiController],
|
controllers: [OpenApiController, AdminOpenApiController, ClientOpenApiController],
|
||||||
providers: [OpenApiService, OpenApiAuthGuard],
|
providers: [OpenApiService, OpenApiAuthGuard],
|
||||||
exports: [OpenApiService],
|
exports: [OpenApiService],
|
||||||
|
|||||||
@@ -0,0 +1,58 @@
|
|||||||
|
import { Injectable } from '@nestjs/common';
|
||||||
|
import { ConfigService } from '@nestjs/config';
|
||||||
|
import { createConnection } from 'node:net';
|
||||||
|
|
||||||
|
type AgentResponse = { ok: boolean; reference?: string; blocked?: boolean; active?: boolean; error?: string };
|
||||||
|
|
||||||
|
@Injectable()
|
||||||
|
export class SecurityAgentClient {
|
||||||
|
constructor(private readonly config: ConfigService) {}
|
||||||
|
|
||||||
|
block(input: { operationKey: string; sourceIp: string; executor: string; durationSeconds: number }) {
|
||||||
|
return this.call({ action: 'block', ...input });
|
||||||
|
}
|
||||||
|
|
||||||
|
unblock(input: { operationKey: string; sourceIp: string; executor: string }) {
|
||||||
|
return this.call({ action: 'unblock', ...input });
|
||||||
|
}
|
||||||
|
|
||||||
|
status(sourceIp?: string, executor?: string) {
|
||||||
|
return this.call({ action: 'status', sourceIp, executor });
|
||||||
|
}
|
||||||
|
|
||||||
|
applyRules(version: number, rules: Array<Record<string, unknown>>) {
|
||||||
|
return this.call({ action: 'apply_rules', version, rules });
|
||||||
|
}
|
||||||
|
|
||||||
|
private call(payload: Record<string, unknown>): Promise<AgentResponse> {
|
||||||
|
const socketPath = this.config.get<string>('SECURITY_AGENT_SOCKET') ?? '/run/cmpp-security-agent/agent.sock';
|
||||||
|
const timeoutMs = Number(this.config.get<string>('SECURITY_AGENT_TIMEOUT_MS') ?? 3000);
|
||||||
|
return new Promise((resolve, reject) => {
|
||||||
|
const socket = createConnection(socketPath);
|
||||||
|
let settled = false;
|
||||||
|
let response = '';
|
||||||
|
const finish = (error?: Error) => {
|
||||||
|
if (settled) return;
|
||||||
|
settled = true;
|
||||||
|
socket.destroy();
|
||||||
|
if (error) reject(error);
|
||||||
|
};
|
||||||
|
socket.setTimeout(timeoutMs, () => finish(new Error('安全执行代理响应超时')));
|
||||||
|
socket.on('error', (error) => finish(new Error(`安全执行代理不可用: ${error.message}`)));
|
||||||
|
socket.on('connect', () => socket.write(`${JSON.stringify(payload)}\n`));
|
||||||
|
socket.on('data', (chunk) => {
|
||||||
|
response += chunk.toString('utf8');
|
||||||
|
const lineEnd = response.indexOf('\n');
|
||||||
|
if (lineEnd < 0) return;
|
||||||
|
try {
|
||||||
|
const parsed = JSON.parse(response.slice(0, lineEnd)) as AgentResponse;
|
||||||
|
settled = true;
|
||||||
|
socket.end();
|
||||||
|
resolve(parsed);
|
||||||
|
} catch {
|
||||||
|
finish(new Error('安全执行代理返回了非法响应'));
|
||||||
|
}
|
||||||
|
});
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,10 @@
|
|||||||
|
export const SECURITY_RULE_CODES = [
|
||||||
|
'admin_login_failure', 'client_login_failure', 'ssh_auth_failure', 'cmpp_auth_failure',
|
||||||
|
'cmpp_protocol_abuse', 'http_invalid_api_key', 'http_signature_failure',
|
||||||
|
'http_replay_attempt', 'http_malicious_scan',
|
||||||
|
] as const;
|
||||||
|
|
||||||
|
export type SecurityRuleCode = typeof SECURITY_RULE_CODES[number];
|
||||||
|
export const SECURITY_RULE_CODE_SET = new Set<string>(SECURITY_RULE_CODES);
|
||||||
|
export const SECURITY_SEVERITIES = new Set(['low', 'medium', 'high', 'critical']);
|
||||||
|
export const SECURITY_BLOCK_DURATIONS = new Set([600, 3600, 86400, 604800]);
|
||||||
@@ -0,0 +1,22 @@
|
|||||||
|
import { Body, Controller, Get, Param, Post, Put, Query } from '@nestjs/common';
|
||||||
|
import { ApiTags } from '@nestjs/swagger';
|
||||||
|
import { CurrentSessionUserId } from '../auth/current-session-user.decorator';
|
||||||
|
import { RequireRecentAuthentication } from '../auth/require-recent-authentication.decorator';
|
||||||
|
import { SecurityDetectionService } from './security-detection.service';
|
||||||
|
|
||||||
|
@ApiTags('security-detection')
|
||||||
|
@Controller('admin/security-detection')
|
||||||
|
export class SecurityDetectionController {
|
||||||
|
constructor(private readonly security: SecurityDetectionService) {}
|
||||||
|
@Get('overview') overview(@Query('range') range?: string) { return this.security.overview(range); }
|
||||||
|
@Get('notification-summary') notificationSummary() { return this.security.notificationSummary(); }
|
||||||
|
@Get('alerts') alerts(@Query() query: Record<string, string>) { return this.security.listAlerts(query); }
|
||||||
|
@Get('rules') rules() { return this.security.listRules(); }
|
||||||
|
@Put('rules/:id') @RequireRecentAuthentication() updateRule(@Param('id') id: string, @Body() body: Record<string, unknown>, @CurrentSessionUserId() userId: string) { return this.security.updateRule(id, body, userId); }
|
||||||
|
@Post('alerts/:id/block') @RequireRecentAuthentication() block(@Param('id') id: string, @Body() body: { durationSeconds?: number; reason?: string }, @CurrentSessionUserId() userId: string) { return this.security.block(id, body, userId); }
|
||||||
|
@Post('alerts/:id/ignore') @RequireRecentAuthentication() ignore(@Param('id') id: string, @Body('reason') reason: string, @CurrentSessionUserId() userId: string) { return this.security.ignore(id, reason ?? '', userId); }
|
||||||
|
@Get('blocks') blocks() { return this.security.listBlocks(); }
|
||||||
|
@Post('blocks/:id/unblock') @RequireRecentAuthentication() unblock(@Param('id') id: string, @Body('reason') reason: string, @CurrentSessionUserId() userId: string) { return this.security.unblock(id, reason ?? '', userId); }
|
||||||
|
@Get('protected-networks') protectedNetworks() { return this.security.listProtectedNetworks(); }
|
||||||
|
@Post('protected-networks') @RequireRecentAuthentication() addProtectedNetwork(@Body() body: { network?: string; name?: string; reason?: string }, @CurrentSessionUserId() userId: string) { return this.security.addProtectedNetwork(body, userId); }
|
||||||
|
}
|
||||||
@@ -0,0 +1,8 @@
|
|||||||
|
import { Module } from '@nestjs/common';
|
||||||
|
import { SecurityAgentClient } from './security-agent.client';
|
||||||
|
import { SecurityDetectionController } from './security-detection.controller';
|
||||||
|
import { SecurityEventController } from './security-event.controller';
|
||||||
|
import { SecurityDetectionService } from './security-detection.service';
|
||||||
|
|
||||||
|
@Module({ controllers: [SecurityDetectionController, SecurityEventController], providers: [SecurityAgentClient, SecurityDetectionService], exports: [SecurityDetectionService] })
|
||||||
|
export class SecurityDetectionModule {}
|
||||||
@@ -0,0 +1,75 @@
|
|||||||
|
import { ConflictException } from '@nestjs/common';
|
||||||
|
import { SecurityDetectionService } from './security-detection.service';
|
||||||
|
|
||||||
|
function createPrisma() {
|
||||||
|
const tx = {
|
||||||
|
$executeRaw: jest.fn(),
|
||||||
|
securityDetectionEvent: { create: jest.fn(), count: jest.fn() },
|
||||||
|
securityAlert: { findFirst: jest.fn(), create: jest.fn(), update: jest.fn(), updateMany: jest.fn() },
|
||||||
|
securityBlock: { create: jest.fn(), update: jest.fn() },
|
||||||
|
};
|
||||||
|
const prisma = {
|
||||||
|
securityDetectionRule: { findUnique: jest.fn(), findMany: jest.fn() },
|
||||||
|
securityDetectionEvent: { count: jest.fn() },
|
||||||
|
securityAlert: { findUnique: jest.fn(), update: jest.fn(), count: jest.fn() },
|
||||||
|
securityBlock: { create: jest.fn(), update: jest.fn() },
|
||||||
|
securityProtectedNetwork: { findMany: jest.fn().mockResolvedValue([]) },
|
||||||
|
operationLog: { create: jest.fn() },
|
||||||
|
$transaction: jest.fn(async (value: unknown) => typeof value === 'function' ? value(tx) : Promise.all(value as Promise<unknown>[])),
|
||||||
|
};
|
||||||
|
return { prisma, tx };
|
||||||
|
}
|
||||||
|
|
||||||
|
describe('SecurityDetectionService', () => {
|
||||||
|
it('returns an independent active and critical alert summary for the global bell', async () => {
|
||||||
|
const { prisma } = createPrisma();
|
||||||
|
prisma.securityAlert.count.mockResolvedValueOnce(4).mockResolvedValueOnce(2);
|
||||||
|
const service = new SecurityDetectionService(prisma as never, {} as never);
|
||||||
|
|
||||||
|
await expect(service.notificationSummary()).resolves.toEqual({ count: 4, criticalCount: 2 });
|
||||||
|
expect(prisma.securityAlert.count).toHaveBeenNthCalledWith(1, { where: { status: { in: ['open', 'acknowledged', 'block_failed'] } } });
|
||||||
|
expect(prisma.securityAlert.count).toHaveBeenNthCalledWith(2, { where: { status: { in: ['open', 'acknowledged', 'block_failed'] }, severity: 'critical' } });
|
||||||
|
});
|
||||||
|
|
||||||
|
it('keeps a below-threshold event without creating a false alert', async () => {
|
||||||
|
const { prisma, tx } = createPrisma();
|
||||||
|
prisma.securityDetectionRule.findUnique.mockResolvedValue({ id: 'rule-1', enabled: true, threshold: 3, windowSeconds: 60, cooldownSeconds: 60, severity: 'high' });
|
||||||
|
tx.securityDetectionEvent.create.mockResolvedValue({ id: 'event-1' });
|
||||||
|
tx.securityDetectionEvent.count.mockResolvedValue(2);
|
||||||
|
const service = new SecurityDetectionService(prisma as never, {} as never);
|
||||||
|
|
||||||
|
await expect(service.recordEvent({ eventKey: 'event-key-1', ruleCode: 'http_signature_failure', sourceIp: '203.0.113.5' })).resolves.toEqual({ accepted: true, duplicate: false, alertId: null });
|
||||||
|
expect(tx.securityAlert.create).not.toHaveBeenCalled();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('refuses built-in protected addresses before calling the privileged agent', async () => {
|
||||||
|
const { prisma } = createPrisma();
|
||||||
|
prisma.securityAlert.findUnique.mockResolvedValue({
|
||||||
|
id: 'alert-1', sourceIp: '127.0.0.1', status: 'open',
|
||||||
|
rule: { code: 'ssh_auth_failure', defaultBlockSeconds: 600, maximumBlockSeconds: 604800 },
|
||||||
|
});
|
||||||
|
const agent = { block: jest.fn(), status: jest.fn() };
|
||||||
|
const service = new SecurityDetectionService(prisma as never, agent as never);
|
||||||
|
|
||||||
|
await expect(service.block('alert-1', { durationSeconds: 600, reason: '隔离测试封禁' }, 'operator-1')).rejects.toBeInstanceOf(ConflictException);
|
||||||
|
expect(agent.block).not.toHaveBeenCalled();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('maps an admin-login alert to nginx and marks blocked only after readback', async () => {
|
||||||
|
const { prisma, tx } = createPrisma();
|
||||||
|
prisma.securityAlert.findUnique.mockResolvedValue({
|
||||||
|
id: 'alert-1', sourceIp: '203.0.113.8', status: 'open',
|
||||||
|
rule: { code: 'admin_login_failure', defaultBlockSeconds: 600, maximumBlockSeconds: 604800 },
|
||||||
|
});
|
||||||
|
tx.securityAlert.updateMany.mockResolvedValue({ count: 1 });
|
||||||
|
tx.securityBlock.create.mockResolvedValue({ id: 'block-1' });
|
||||||
|
tx.securityBlock.update.mockResolvedValue({ id: 'block-1', status: 'blocked' });
|
||||||
|
tx.securityAlert.update.mockResolvedValue({ id: 'alert-1', status: 'blocked' });
|
||||||
|
const agent = { block: jest.fn().mockResolvedValue({ ok: true, reference: 'op-1' }), status: jest.fn().mockResolvedValue({ ok: true, blocked: true }) };
|
||||||
|
const service = new SecurityDetectionService(prisma as never, agent as never);
|
||||||
|
|
||||||
|
await expect(service.block('alert-1', { durationSeconds: 600, reason: '确认恶意登录扫描' }, 'operator-1')).resolves.toEqual(expect.objectContaining({ status: 'blocked' }));
|
||||||
|
expect(agent.block).toHaveBeenCalledWith(expect.objectContaining({ executor: 'nginx_real_ip', sourceIp: '203.0.113.8' }));
|
||||||
|
expect(agent.status).toHaveBeenCalledWith('203.0.113.8', 'nginx_real_ip');
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -0,0 +1,255 @@
|
|||||||
|
import { BadRequestException, ConflictException, Injectable, NotFoundException } from '@nestjs/common';
|
||||||
|
import { createHash, randomUUID } from 'node:crypto';
|
||||||
|
import { isIP } from 'node:net';
|
||||||
|
import { Prisma } from '@prisma/client';
|
||||||
|
import { PrismaService } from '../prisma/prisma.service';
|
||||||
|
import { SecurityAgentClient } from './security-agent.client';
|
||||||
|
import { SECURITY_BLOCK_DURATIONS, SECURITY_RULE_CODE_SET, SECURITY_SEVERITIES, type SecurityRuleCode } from './security-detection.constants';
|
||||||
|
|
||||||
|
export type SecurityEventInput = {
|
||||||
|
eventKey?: string; ruleCode: SecurityRuleCode; sourceIp: string; sourcePort?: number;
|
||||||
|
account?: string; path?: string; protocol?: string; resultCode?: string;
|
||||||
|
evidence?: Record<string, unknown>; occurredAt?: string | Date;
|
||||||
|
};
|
||||||
|
|
||||||
|
@Injectable()
|
||||||
|
export class SecurityDetectionService {
|
||||||
|
constructor(private readonly prisma: PrismaService, private readonly agent: SecurityAgentClient) {}
|
||||||
|
|
||||||
|
async recordEvent(input: SecurityEventInput) {
|
||||||
|
if (!SECURITY_RULE_CODE_SET.has(input.ruleCode)) throw new BadRequestException('不支持的安全检测类型');
|
||||||
|
const sourceIp = normalizeIp(input.sourceIp);
|
||||||
|
const occurredAt = input.occurredAt ? new Date(input.occurredAt) : new Date();
|
||||||
|
if (!Number.isFinite(occurredAt.getTime())) throw new BadRequestException('安全事件时间无效');
|
||||||
|
const eventKey = input.eventKey ?? createHash('sha256').update(JSON.stringify([
|
||||||
|
input.ruleCode, sourceIp, input.sourcePort, input.account, input.path, input.resultCode,
|
||||||
|
occurredAt.toISOString(), input.evidence,
|
||||||
|
])).digest('hex');
|
||||||
|
const rule = await this.prisma.securityDetectionRule.findUnique({ where: { code: input.ruleCode } });
|
||||||
|
if (!rule) throw new NotFoundException('安全检测规则不存在');
|
||||||
|
|
||||||
|
return this.prisma.$transaction(async (tx) => {
|
||||||
|
// 同一来源和规则串行聚合,避免并发计数跨过阈值时创建多个告警。
|
||||||
|
await tx.$executeRaw`SELECT pg_advisory_xact_lock(hashtext(${`${rule.id}:${sourceIp}`}))`;
|
||||||
|
try {
|
||||||
|
await tx.securityDetectionEvent.create({ data: {
|
||||||
|
eventKey, ruleId: rule.id, sourceIp, sourcePort: input.sourcePort,
|
||||||
|
accountHash: input.account ? createHash('sha256').update(input.account).digest('hex') : undefined,
|
||||||
|
path: input.path?.slice(0, 512), protocol: input.protocol?.slice(0, 32), resultCode: input.resultCode?.slice(0, 128),
|
||||||
|
evidence: sanitizeEvidence(input.evidence), occurredAt,
|
||||||
|
} });
|
||||||
|
} catch (error) {
|
||||||
|
if (isUniqueViolation(error)) return { accepted: true, duplicate: true, alertId: null };
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
if (!rule.enabled) return { accepted: true, duplicate: false, alertId: null };
|
||||||
|
|
||||||
|
const windowStartedAt = new Date(occurredAt.getTime() - rule.windowSeconds * 1000);
|
||||||
|
const storedEventCount = await tx.securityDetectionEvent.count({
|
||||||
|
where: { ruleId: rule.id, sourceIp, occurredAt: { gte: windowStartedAt, lte: occurredAt } },
|
||||||
|
});
|
||||||
|
// Fail2ban上报代表其自身窗口已经达到maxretry;应用事件则逐条在数据库窗口内计数。
|
||||||
|
const eventCount = rule.sourceType === 'fail2ban' ? Math.max(storedEventCount, rule.threshold) : storedEventCount;
|
||||||
|
if (eventCount < rule.threshold) return { accepted: true, duplicate: false, alertId: null };
|
||||||
|
|
||||||
|
const cooldownStart = new Date(occurredAt.getTime() - rule.cooldownSeconds * 1000);
|
||||||
|
const active = await tx.securityAlert.findFirst({
|
||||||
|
where: { ruleId: rule.id, sourceIp, status: { in: ['open', 'acknowledged', 'block_failed', 'blocked'] }, lastOccurredAt: { gte: cooldownStart } },
|
||||||
|
orderBy: { lastOccurredAt: 'desc' },
|
||||||
|
});
|
||||||
|
if (active) {
|
||||||
|
const updated = await tx.securityAlert.update({ where: { id: active.id }, data: { eventCount, lastOccurredAt: occurredAt } });
|
||||||
|
return { accepted: true, duplicate: false, alertId: updated.id };
|
||||||
|
}
|
||||||
|
const fingerprint = createHash('sha256').update(`${rule.id}:${sourceIp}:${occurredAt.toISOString()}`).digest('hex');
|
||||||
|
const alert = await tx.securityAlert.create({ data: {
|
||||||
|
fingerprint, ruleId: rule.id, sourceIp, severity: rule.severity, eventCount,
|
||||||
|
windowStartedAt, firstOccurredAt: occurredAt, lastOccurredAt: occurredAt,
|
||||||
|
} });
|
||||||
|
return { accepted: true, duplicate: false, alertId: alert.id };
|
||||||
|
}, { isolationLevel: Prisma.TransactionIsolationLevel.ReadCommitted });
|
||||||
|
}
|
||||||
|
|
||||||
|
async overview(range = '24h') {
|
||||||
|
if (!['1h', '24h', '7d'].includes(range)) throw new BadRequestException('仅支持1h、24h或7d安全检测范围');
|
||||||
|
const hours = range === '1h' ? 1 : range === '7d' ? 168 : 24;
|
||||||
|
const since = new Date(Date.now() - hours * 3600_000);
|
||||||
|
const activeStatuses = ['open', 'acknowledged', 'block_failed'];
|
||||||
|
const [alerts, totalEvents, activeBlocks, rules, activeAlerts, criticalAlerts, distribution, agentStatus] = await Promise.all([
|
||||||
|
this.prisma.securityAlert.findMany({ where: { lastOccurredAt: { gte: since } }, include: { rule: true }, orderBy: { lastOccurredAt: 'desc' }, take: 12 }),
|
||||||
|
this.prisma.securityDetectionEvent.count({ where: { occurredAt: { gte: since } } }),
|
||||||
|
this.prisma.securityBlock.count({ where: { status: 'blocked', expiresAt: { gt: new Date() } } }),
|
||||||
|
this.prisma.securityDetectionRule.findMany({ orderBy: { name: 'asc' } }),
|
||||||
|
this.prisma.securityAlert.count({ where: { status: { in: activeStatuses } } }),
|
||||||
|
this.prisma.securityAlert.count({ where: { status: { in: activeStatuses }, severity: 'critical' } }),
|
||||||
|
this.prisma.securityAlert.groupBy({ by: ['ruleId'], where: { lastOccurredAt: { gte: since } }, _sum: { eventCount: true } }),
|
||||||
|
this.agent.status().catch((error: Error) => ({ ok: false, active: false, error: error.message })),
|
||||||
|
]);
|
||||||
|
const ruleNames = new Map(rules.map((rule) => [rule.id, rule.name]));
|
||||||
|
return {
|
||||||
|
range, collectedAt: new Date().toISOString(), totalEvents, activeAlerts, criticalAlerts, activeBlocks,
|
||||||
|
health: { agent: agentStatus.ok && agentStatus.active ? 'healthy' : 'unavailable', agentError: agentStatus.error, rulesEffective: rules.filter((rule) => rule.applyStatus === 'effective').length, rulesTotal: rules.length },
|
||||||
|
sourceDistribution: distribution.map((item) => ({ name: ruleNames.get(item.ruleId) ?? item.ruleId, value: item._sum.eventCount ?? 0 })),
|
||||||
|
alerts,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
async notificationSummary() {
|
||||||
|
const activeStatuses = ['open', 'acknowledged', 'block_failed'];
|
||||||
|
const [count, criticalCount] = await Promise.all([
|
||||||
|
this.prisma.securityAlert.count({ where: { status: { in: activeStatuses } } }),
|
||||||
|
this.prisma.securityAlert.count({ where: { status: { in: activeStatuses }, severity: 'critical' } }),
|
||||||
|
]);
|
||||||
|
return { count, criticalCount };
|
||||||
|
}
|
||||||
|
|
||||||
|
listAlerts(query: { status?: string; ruleCode?: string; sourceIp?: string; page?: string; pageSize?: string }) {
|
||||||
|
const page = positiveInt(query.page, 1, 100000);
|
||||||
|
const pageSize = positiveInt(query.pageSize, 20, 100);
|
||||||
|
const where: Prisma.SecurityAlertWhereInput = {
|
||||||
|
...(query.status ? { status: query.status } : {}),
|
||||||
|
...(query.ruleCode ? { rule: { code: query.ruleCode } } : {}),
|
||||||
|
...(query.sourceIp ? { sourceIp: normalizeIp(query.sourceIp) } : {}),
|
||||||
|
};
|
||||||
|
return Promise.all([
|
||||||
|
this.prisma.securityAlert.findMany({ where, include: { rule: true }, orderBy: { lastOccurredAt: 'desc' }, skip: (page - 1) * pageSize, take: pageSize }),
|
||||||
|
this.prisma.securityAlert.count({ where }),
|
||||||
|
]).then(([items, total]) => ({ items, total, page, pageSize }));
|
||||||
|
}
|
||||||
|
|
||||||
|
listRules() { return this.prisma.securityDetectionRule.findMany({ orderBy: [{ sourceType: 'asc' }, { name: 'asc' }] }); }
|
||||||
|
|
||||||
|
async updateRule(id: string, input: Record<string, unknown>, operatorId: string) {
|
||||||
|
assertAllowedKeys(input, ['configVersion', 'enabled', 'threshold', 'windowSeconds', 'cooldownSeconds', 'severity', 'defaultBlockSeconds', 'maximumBlockSeconds']);
|
||||||
|
if (typeof input.enabled !== 'boolean') throw new BadRequestException('启用状态必须为布尔值');
|
||||||
|
const current = await this.prisma.securityDetectionRule.findUnique({ where: { id } });
|
||||||
|
if (!current) throw new NotFoundException('规则不存在');
|
||||||
|
if (Number(input.configVersion) !== current.configVersion) throw new ConflictException('规则已被其他管理员修改,请刷新后重试');
|
||||||
|
const threshold = boundedInt(input.threshold, 1, 100000, '触发次数');
|
||||||
|
const windowSeconds = boundedInt(input.windowSeconds, 10, 86400, '检测窗口');
|
||||||
|
const cooldownSeconds = boundedInt(input.cooldownSeconds, 0, 604800, '告警冷却');
|
||||||
|
const defaultBlockSeconds = boundedInt(input.defaultBlockSeconds, 600, 604800, '默认封禁时长');
|
||||||
|
const maximumBlockSeconds = boundedInt(input.maximumBlockSeconds, defaultBlockSeconds, 604800, '最大封禁时长');
|
||||||
|
const severity = String(input.severity ?? '');
|
||||||
|
if (!SECURITY_SEVERITIES.has(severity)) throw new BadRequestException('告警级别无效');
|
||||||
|
const version = current.configVersion + 1;
|
||||||
|
const nextConfig = { enabled: Boolean(input.enabled), threshold, windowSeconds, cooldownSeconds, severity, defaultBlockSeconds, maximumBlockSeconds };
|
||||||
|
await this.prisma.securityDetectionRule.update({ where: { id }, data: {
|
||||||
|
configVersion: version, applyStatus: 'applying', lastApplyError: null, pendingConfig: nextConfig,
|
||||||
|
} });
|
||||||
|
await this.prisma.operationLog.create({ data: { userId: operatorId, action: 'security.rule_updated', resource: 'security_detection_rule', resourceId: id, detail: { version, beforeVersion: current.configVersion } } });
|
||||||
|
try {
|
||||||
|
const response = await this.agent.applyRules(version, (await this.listRules()).map((rule) => rule.id === id
|
||||||
|
? { code: rule.code, enabled: nextConfig.enabled, threshold: nextConfig.threshold, windowSeconds: nextConfig.windowSeconds, cooldownSeconds: nextConfig.cooldownSeconds }
|
||||||
|
: { code: rule.code, enabled: rule.enabled, threshold: rule.threshold, windowSeconds: rule.windowSeconds, cooldownSeconds: rule.cooldownSeconds }));
|
||||||
|
if (!response.ok) throw new Error(response.error ?? '安全代理拒绝应用规则');
|
||||||
|
return this.prisma.securityDetectionRule.update({ where: { id }, data: { ...nextConfig, effectiveVersion: version, applyStatus: 'effective', pendingConfig: Prisma.JsonNull } });
|
||||||
|
} catch (error) {
|
||||||
|
const message = error instanceof Error ? error.message : '规则应用失败';
|
||||||
|
await this.prisma.securityDetectionRule.update({ where: { id }, data: { applyStatus: 'failed', lastApplyError: message } });
|
||||||
|
throw new ConflictException({ code: 'SECURITY_RULE_APPLY_FAILED', message });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async block(alertId: string, input: { durationSeconds?: number; reason?: string }, operatorId: string) {
|
||||||
|
assertAllowedKeys(input as Record<string, unknown>, ['durationSeconds', 'reason']);
|
||||||
|
const alert = await this.prisma.securityAlert.findUnique({ where: { id: alertId }, include: { rule: true } });
|
||||||
|
if (!alert) throw new NotFoundException('告警不存在');
|
||||||
|
if (!['open', 'acknowledged', 'block_failed'].includes(alert.status)) throw new ConflictException('该告警当前不可封禁');
|
||||||
|
const durationSeconds = Number(input.durationSeconds ?? alert.rule.defaultBlockSeconds);
|
||||||
|
if (!SECURITY_BLOCK_DURATIONS.has(durationSeconds) || durationSeconds > alert.rule.maximumBlockSeconds) throw new BadRequestException('封禁时长不在允许范围内');
|
||||||
|
const reason = String(input.reason ?? '').trim();
|
||||||
|
if (reason.length < 5 || reason.length > 500) throw new BadRequestException('封禁原因需为5至500个字符');
|
||||||
|
if (isSystemProtected(alert.sourceIp) || await this.isProtected(alert.sourceIp)) throw new ConflictException({ code: 'PROTECTED_NETWORK', message: '该地址属于系统或人工保护名单,禁止封禁' });
|
||||||
|
// 执行器由可信的规则入口固定映射,绝不接受浏览器指定,避免把Cloudflare访客IP错误交给nftables。
|
||||||
|
const executor = ['admin_login_failure', 'client_login_failure'].includes(alert.rule.code) ? 'nginx_real_ip' : 'nftables';
|
||||||
|
const operationKey = randomUUID();
|
||||||
|
const block = await this.prisma.$transaction(async (tx) => {
|
||||||
|
const claimed = await tx.securityAlert.updateMany({ where: { id: alert.id, status: { in: ['open', 'acknowledged', 'block_failed'] } }, data: { status: 'block_requested' } });
|
||||||
|
if (!claimed.count) throw new ConflictException('告警已由其他管理员处理,请刷新后重试');
|
||||||
|
return tx.securityBlock.create({ data: { operationKey, alertId, sourceIp: alert.sourceIp, executor, durationSeconds, reason, requestedById: operatorId } });
|
||||||
|
});
|
||||||
|
await this.prisma.operationLog.create({ data: { userId: operatorId, action: 'security.block_requested', resource: 'security_block', resourceId: block.id, detail: { alertId, sourceIp: alert.sourceIp, executor, durationSeconds, reason } } });
|
||||||
|
try {
|
||||||
|
const applied = await this.agent.block({ operationKey, sourceIp: alert.sourceIp, executor, durationSeconds });
|
||||||
|
if (!applied.ok) throw new Error(applied.error ?? '安全代理拒绝封禁');
|
||||||
|
const readback = await this.agent.status(alert.sourceIp, executor);
|
||||||
|
if (!readback.ok || !readback.blocked) throw new Error(readback.error ?? '执行后未读到真实封禁状态');
|
||||||
|
const expiresAt = new Date(Date.now() + durationSeconds * 1000);
|
||||||
|
const result = await this.prisma.$transaction(async (tx) => {
|
||||||
|
const updated = await tx.securityBlock.update({ where: { id: block.id }, data: { status: 'blocked', appliedAt: new Date(), expiresAt, executorReference: applied.reference } });
|
||||||
|
await tx.securityAlert.update({ where: { id: alert.id }, data: { status: 'blocked', blockId: block.id } });
|
||||||
|
return updated;
|
||||||
|
});
|
||||||
|
return result;
|
||||||
|
} catch (error) {
|
||||||
|
const message = error instanceof Error ? error.message : '封禁执行失败';
|
||||||
|
await this.prisma.$transaction([
|
||||||
|
this.prisma.securityBlock.update({ where: { id: block.id }, data: { status: 'failed', lastError: message } }),
|
||||||
|
this.prisma.securityAlert.update({ where: { id: alert.id }, data: { status: 'block_failed' } }),
|
||||||
|
]);
|
||||||
|
throw new ConflictException({ code: 'SECURITY_BLOCK_FAILED', message });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async ignore(alertId: string, reason: string, operatorId: string) {
|
||||||
|
if (reason.trim().length < 5) throw new BadRequestException('忽略原因至少5个字符');
|
||||||
|
const updated = await this.prisma.securityAlert.updateMany({ where: { id: alertId, status: { in: ['open', 'acknowledged', 'block_failed'] } }, data: { status: 'ignored', ignoredAt: new Date(), ignoredById: operatorId, ignoreReason: reason.trim() } });
|
||||||
|
if (!updated.count) throw new ConflictException('告警状态已变化,请刷新后重试');
|
||||||
|
await this.prisma.operationLog.create({ data: { userId: operatorId, action: 'security.alert_ignored', resource: 'security_alert', resourceId: alertId, detail: { reason: reason.trim() } } });
|
||||||
|
return { success: true };
|
||||||
|
}
|
||||||
|
|
||||||
|
listBlocks() { return this.prisma.securityBlock.findMany({ orderBy: { requestedAt: 'desc' }, take: 200 }); }
|
||||||
|
|
||||||
|
async unblock(blockId: string, reason: string, operatorId: string) {
|
||||||
|
if (reason.trim().length < 5) throw new BadRequestException('解封原因至少5个字符');
|
||||||
|
const block = await this.prisma.securityBlock.findUnique({ where: { id: blockId } });
|
||||||
|
if (!block) throw new NotFoundException('封禁记录不存在');
|
||||||
|
if (block.status !== 'blocked') throw new ConflictException('该记录当前不可解封');
|
||||||
|
const claimed = await this.prisma.securityBlock.updateMany({ where: { id: blockId, status: 'blocked' }, data: { status: 'unblock_requested' } });
|
||||||
|
if (!claimed.count) throw new ConflictException('封禁状态已变化,请刷新后重试');
|
||||||
|
try {
|
||||||
|
const result = await this.agent.unblock({ operationKey: randomUUID(), sourceIp: block.sourceIp, executor: block.executor });
|
||||||
|
if (!result.ok) throw new Error(result.error ?? '安全代理拒绝解封');
|
||||||
|
const readback = await this.agent.status(block.sourceIp, block.executor);
|
||||||
|
if (!readback.ok || readback.blocked) throw new Error(readback.error ?? '执行后仍读到封禁规则');
|
||||||
|
const updated = await this.prisma.securityBlock.update({ where: { id: block.id }, data: { status: 'released', releasedAt: new Date(), releasedById: operatorId } });
|
||||||
|
if (block.alertId) await this.prisma.securityAlert.updateMany({ where: { id: block.alertId, blockId: block.id }, data: { status: 'unblocked' } });
|
||||||
|
await this.prisma.operationLog.create({ data: { userId: operatorId, action: 'security.block_released', resource: 'security_block', resourceId: block.id, detail: { sourceIp: block.sourceIp, executor: block.executor, reason: reason.trim() } } });
|
||||||
|
return updated;
|
||||||
|
} catch (error) {
|
||||||
|
const message = error instanceof Error ? error.message : '解封失败';
|
||||||
|
await this.prisma.securityBlock.update({ where: { id: block.id }, data: { status: 'blocked', lastError: message } });
|
||||||
|
throw new ConflictException({ code: 'SECURITY_UNBLOCK_FAILED', message });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
listProtectedNetworks() { return this.prisma.securityProtectedNetwork.findMany({ orderBy: { createdAt: 'desc' } }); }
|
||||||
|
|
||||||
|
async addProtectedNetwork(input: { network?: string; name?: string; reason?: string }, operatorId: string) {
|
||||||
|
const network = normalizeNetwork(String(input.network ?? ''));
|
||||||
|
if (!input.name?.trim() || !input.reason?.trim()) throw new BadRequestException('名称和保护原因不能为空');
|
||||||
|
const result = await this.prisma.securityProtectedNetwork.create({ data: { network, name: input.name.trim(), reason: input.reason.trim(), createdById: operatorId } });
|
||||||
|
await this.prisma.operationLog.create({ data: { userId: operatorId, action: 'security.protected_network_created', resource: 'security_protected_network', resourceId: result.id, detail: { network } } });
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
private async isProtected(ip: string) {
|
||||||
|
const entries = await this.prisma.securityProtectedNetwork.findMany({ where: { enabled: true }, select: { network: true } });
|
||||||
|
return entries.some((entry) => networkContains(entry.network, ip));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function normalizeIp(value: string) { const normalized = value?.trim().replace(/^::ffff:/, ''); if (!isIP(normalized)) throw new BadRequestException('来源IP无效'); return normalized; }
|
||||||
|
function normalizeNetwork(value: string) { const [address, prefix] = value.trim().split('/'); const family = isIP(address); if (!family) throw new BadRequestException('保护网段无效'); if (prefix === undefined) return address; const bits = Number(prefix); const max = family === 4 ? 32 : 128; if (!Number.isInteger(bits) || bits < 0 || bits > max) throw new BadRequestException('保护网段前缀无效'); return `${address}/${bits}`; }
|
||||||
|
function networkContains(network: string, ip: string) { const [address, prefixText] = network.split('/'); if (isIP(address) !== isIP(ip)) return false; if (prefixText === undefined) return address === ip; const bits = Number(prefixText); return (addressToBigInt(address) >> BigInt((isIP(address) === 4 ? 32 : 128) - bits)) === (addressToBigInt(ip) >> BigInt((isIP(ip) === 4 ? 32 : 128) - bits)); }
|
||||||
|
function addressToBigInt(value: string) { if (isIP(value) === 4) return value.split('.').reduce((total, part) => (total << 8n) + BigInt(part), 0n); const [left, right = ''] = value.toLowerCase().split('::'); const leftParts = left ? left.split(':') : []; const rightParts = right ? right.split(':') : []; const parts = [...leftParts, ...Array(Math.max(0, 8 - leftParts.length - rightParts.length)).fill('0'), ...rightParts]; return parts.reduce((total, part) => (total << 16n) + BigInt(`0x${part || '0'}`), 0n); }
|
||||||
|
function positiveInt(value: string | undefined, fallback: number, max: number) { const parsed = Number(value ?? fallback); return Number.isInteger(parsed) && parsed > 0 ? Math.min(parsed, max) : fallback; }
|
||||||
|
function boundedInt(value: unknown, min: number, max: number, label: string) { const parsed = Number(value); if (!Number.isInteger(parsed) || parsed < min || parsed > max) throw new BadRequestException(`${label}必须在${min}至${max}之间`); return parsed; }
|
||||||
|
function sanitizeEvidence(value?: Record<string, unknown>) { if (!value) return undefined; const sanitized = JSON.parse(JSON.stringify(value, (key, item) => /password|secret|token|signature|access.?key/i.test(key) ? '[REDACTED]' : item)); return sanitized as Prisma.InputJsonValue; }
|
||||||
|
function isUniqueViolation(error: unknown) { return error instanceof Prisma.PrismaClientKnownRequestError && error.code === 'P2002'; }
|
||||||
|
function assertAllowedKeys(input: Record<string, unknown>, allowed: string[]) { const unknown = Object.keys(input).filter((key) => !allowed.includes(key)); if (unknown.length) throw new BadRequestException(`不支持的字段: ${unknown.join(', ')}`); }
|
||||||
|
function isSystemProtected(ip: string) {
|
||||||
|
const builtIns = ['0.0.0.0/8', '10.0.0.0/8', '100.64.0.0/10', '127.0.0.0/8', '169.254.0.0/16', '172.16.0.0/12', '192.168.0.0/16', '224.0.0.0/4', '::/128', '::1/128', 'fc00::/7', 'fe80::/10', ...(process.env.SECURITY_BUILTIN_PROTECTED_NETWORKS ?? '').split(',').map((item) => item.trim()).filter(Boolean)];
|
||||||
|
return builtIns.some((network) => networkContains(network, ip));
|
||||||
|
}
|
||||||
@@ -0,0 +1,20 @@
|
|||||||
|
import { UnauthorizedException } from '@nestjs/common';
|
||||||
|
import { SecurityEventController } from './security-event.controller';
|
||||||
|
|
||||||
|
describe('SecurityEventController', () => {
|
||||||
|
const security = { recordEvent: jest.fn().mockResolvedValue({ accepted: true }) };
|
||||||
|
const config = { get: jest.fn().mockReturnValue('internal-token-0123456789') };
|
||||||
|
const controller = new SecurityEventController(security as never, config as never);
|
||||||
|
|
||||||
|
beforeEach(() => jest.clearAllMocks());
|
||||||
|
|
||||||
|
it('rejects a public event injection without the internal token', () => {
|
||||||
|
expect(() => controller.record({ ruleCode: 'ssh_auth_failure', sourceIp: '203.0.113.9' }, undefined)).toThrow(UnauthorizedException);
|
||||||
|
expect(security.recordEvent).not.toHaveBeenCalled();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('accepts a fixed event from an authenticated local producer', async () => {
|
||||||
|
await expect(controller.record({ ruleCode: 'ssh_auth_failure', sourceIp: '203.0.113.9' }, 'internal-token-0123456789')).resolves.toEqual({ accepted: true });
|
||||||
|
expect(security.recordEvent).toHaveBeenCalledWith(expect.objectContaining({ ruleCode: 'ssh_auth_failure' }));
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -0,0 +1,18 @@
|
|||||||
|
import { Body, Controller, Headers, Post, UnauthorizedException } from '@nestjs/common';
|
||||||
|
import { ConfigService } from '@nestjs/config';
|
||||||
|
import { timingSafeEqual } from 'node:crypto';
|
||||||
|
import { ApiTags } from '@nestjs/swagger';
|
||||||
|
import { SecurityDetectionService, type SecurityEventInput } from './security-detection.service';
|
||||||
|
|
||||||
|
@ApiTags('gateway-security-events')
|
||||||
|
@Controller('gateway/events/security-detection')
|
||||||
|
export class SecurityEventController {
|
||||||
|
constructor(private readonly security: SecurityDetectionService, private readonly config: ConfigService) {}
|
||||||
|
@Post() record(@Body() body: SecurityEventInput, @Headers('x-security-event-token') supplied?: string) {
|
||||||
|
const expected = this.config.get<string>('SECURITY_EVENT_TOKEN');
|
||||||
|
if (!expected || !supplied || !safeEqual(expected, supplied)) throw new UnauthorizedException('安全事件来源认证失败');
|
||||||
|
return this.security.recordEvent(body);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function safeEqual(left: string, right: string) { const a = Buffer.from(left); const b = Buffer.from(right); return a.length === b.length && timingSafeEqual(a, b); }
|
||||||
@@ -11,7 +11,7 @@ describe('GatewayEventsController protocol logging', () => {
|
|||||||
const protocolLogs = {
|
const protocolLogs = {
|
||||||
record: jest.fn(),
|
record: jest.fn(),
|
||||||
};
|
};
|
||||||
const controller = new GatewayEventsController(sendChain as never, {} as never, protocolLogs as never);
|
const controller = new GatewayEventsController(sendChain as never, {} as never, protocolLogs as never, { recordEvent: jest.fn() } as never);
|
||||||
|
|
||||||
beforeEach(() => {
|
beforeEach(() => {
|
||||||
jest.clearAllMocks();
|
jest.clearAllMocks();
|
||||||
|
|||||||
@@ -18,6 +18,7 @@ import { SendChainService } from './send-chain.service';
|
|||||||
import { GatewayDownstreamConnectionEventDto } from '../sms-config/sms-config.contracts';
|
import { GatewayDownstreamConnectionEventDto } from '../sms-config/sms-config.contracts';
|
||||||
import { SmsConfigService } from '../sms-config/sms-config.service';
|
import { SmsConfigService } from '../sms-config/sms-config.service';
|
||||||
import { ProtocolLogsService, type ProtocolLogInput } from '../protocol-logs/protocol-logs.service';
|
import { ProtocolLogsService, type ProtocolLogInput } from '../protocol-logs/protocol-logs.service';
|
||||||
|
import { SecurityDetectionService } from '../security-detection/security-detection.service';
|
||||||
|
|
||||||
@ApiTags('gateway-events')
|
@ApiTags('gateway-events')
|
||||||
@Controller('gateway/events')
|
@Controller('gateway/events')
|
||||||
@@ -26,6 +27,7 @@ export class GatewayEventsController {
|
|||||||
private readonly sendChain: SendChainService,
|
private readonly sendChain: SendChainService,
|
||||||
private readonly smsConfig: SmsConfigService,
|
private readonly smsConfig: SmsConfigService,
|
||||||
private readonly protocolLogs: ProtocolLogsService,
|
private readonly protocolLogs: ProtocolLogsService,
|
||||||
|
private readonly security: SecurityDetectionService,
|
||||||
) {}
|
) {}
|
||||||
|
|
||||||
@Post('submit-result')
|
@Post('submit-result')
|
||||||
@@ -81,8 +83,13 @@ export class GatewayEventsController {
|
|||||||
}
|
}
|
||||||
|
|
||||||
@Post('inbound/authenticate')
|
@Post('inbound/authenticate')
|
||||||
authenticateInbound(@Body() body: GatewayInboundAuthDto) {
|
async authenticateInbound(@Body() body: GatewayInboundAuthDto) {
|
||||||
return this.trackGatewayEvent('connect', body, () => this.sendChain.authenticateInboundApplication(body), 'client_to_platform');
|
try {
|
||||||
|
return await this.trackGatewayEvent('connect', body, () => this.sendChain.authenticateInboundApplication(body), 'client_to_platform');
|
||||||
|
} catch (error) {
|
||||||
|
if (body.remoteIp) await this.security.recordEvent({ ruleCode: 'cmpp_auth_failure', sourceIp: body.remoteIp, account: body.account, protocol: body.version ?? 'cmpp', resultCode: error instanceof Error ? error.name : 'AUTH_FAILED' }).catch(() => undefined);
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@Post('inbound/submit')
|
@Post('inbound/submit')
|
||||||
|
|||||||
@@ -9,9 +9,10 @@ import { AdminSendChainController } from './admin-send-chain.controller';
|
|||||||
import { ClientSendChainController } from './client-send-chain.controller';
|
import { ClientSendChainController } from './client-send-chain.controller';
|
||||||
import { GatewayEventsController } from './gateway-events.controller';
|
import { GatewayEventsController } from './gateway-events.controller';
|
||||||
import { SendChainService } from './send-chain.service';
|
import { SendChainService } from './send-chain.service';
|
||||||
|
import { SecurityDetectionModule } from '../security-detection/security-detection.module';
|
||||||
|
|
||||||
@Module({
|
@Module({
|
||||||
imports: [PrismaModule, BillingModule, DictionariesModule, forwardRef(() => RiskReviewModule), SmsConfigModule, forwardRef(() => OpenApiModule)],
|
imports: [PrismaModule, BillingModule, DictionariesModule, forwardRef(() => RiskReviewModule), SmsConfigModule, forwardRef(() => OpenApiModule), SecurityDetectionModule],
|
||||||
controllers: [AdminSendChainController, ClientSendChainController, GatewayEventsController],
|
controllers: [AdminSendChainController, ClientSendChainController, GatewayEventsController],
|
||||||
providers: [SendChainService],
|
providers: [SendChainService],
|
||||||
exports: [SendChainService],
|
exports: [SendChainService],
|
||||||
|
|||||||
@@ -54,6 +54,41 @@ describe('SendDownstreamRequeueTaskService', () => {
|
|||||||
expect(mock.downstreamRequeueTaskItem.createMany).toHaveBeenCalledWith({ data: [expect.objectContaining({ deliveryId: 'd-1', applicationId: 'app-1' })] });
|
expect(mock.downstreamRequeueTaskItem.createMany).toHaveBeenCalledWith({ data: [expect.objectContaining({ deliveryId: 'd-1', applicationId: 'app-1' })] });
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('materializes client-confirmed deliveries from the signed preview range', async () => {
|
||||||
|
mock.cmppDownstreamDelivery.count.mockResolvedValue(1);
|
||||||
|
mock.cmppDownstreamDelivery.groupBy.mockResolvedValueOnce([{ status: 'delivered', _count: { _all: 1 } }]).mockResolvedValueOnce([{ applicationId: 'app-1', _count: { _all: 1 } }]);
|
||||||
|
mock.cmppDownstreamDelivery.findFirst.mockResolvedValue({ createdAt: new Date() });
|
||||||
|
const preview = await service.preview({ applicationId: 'app-1', status: 'delivered' }, 'user-1');
|
||||||
|
expect(preview).toEqual(expect.objectContaining({ matchedCount: 1, replayableCount: 1, skippedCount: 0 }));
|
||||||
|
mock.downstreamRequeueTask.findFirst.mockResolvedValue(null);
|
||||||
|
mock.cmppDownstreamDelivery.findMany.mockResolvedValue([{ id: 'd-1', applicationId: 'app-1', status: 'delivered' }]);
|
||||||
|
mock.downstreamRequeueTask.create.mockResolvedValue({ id: 'task-1' });
|
||||||
|
mock.downstreamRequeueTask.findUnique.mockResolvedValue({ id: 'task-1' });
|
||||||
|
mock.downstreamRequeueTaskItem.groupBy.mockResolvedValue([]);
|
||||||
|
await service.create({ previewToken: preview.previewToken, reason: '再次投递客户已确认记录' }, 'user-1');
|
||||||
|
expect(mock.downstreamRequeueTaskItem.createMany).toHaveBeenCalledWith({ data: [expect.objectContaining({ deliveryId: 'd-1', previousStatus: 'delivered' })] });
|
||||||
|
});
|
||||||
|
|
||||||
|
it('replays a delivery that was already client-confirmed in the task snapshot', async () => {
|
||||||
|
const requeue = jest.fn().mockResolvedValue({ status: 'awaiting_ack' });
|
||||||
|
service = new SendDownstreamRequeueTaskService(mock as never, { requeueDownstreamDelivery: requeue });
|
||||||
|
mock.cmppDownstreamDelivery.findUnique.mockResolvedValue({ id: 'd-1', applicationId: 'app-1', status: 'delivered', ackResult: 0, payload: {}, deliveryType: 'receipt', application: { status: 'active', interfaceEnabled: true } });
|
||||||
|
mock.downstreamRequeueTaskItem.findFirst.mockResolvedValue(null);
|
||||||
|
mock.cmppDownstreamConnection.count.mockResolvedValue(1);
|
||||||
|
await expect(service['processItem']('item-1', 'd-1', 'delivered')).resolves.toBe('waiting');
|
||||||
|
expect(requeue).toHaveBeenCalledWith('d-1');
|
||||||
|
expect(mock.downstreamRequeueTaskItem.update).toHaveBeenCalledWith(expect.objectContaining({ data: expect.objectContaining({ status: 'waiting_ack' }) }));
|
||||||
|
});
|
||||||
|
|
||||||
|
it('does not replay a record that became delivered after a non-delivered task snapshot', async () => {
|
||||||
|
const requeue = jest.fn();
|
||||||
|
service = new SendDownstreamRequeueTaskService(mock as never, { requeueDownstreamDelivery: requeue });
|
||||||
|
mock.cmppDownstreamDelivery.findUnique.mockResolvedValue({ id: 'd-1', applicationId: 'app-1', status: 'delivered', ackResult: 0, payload: {}, deliveryType: 'receipt', application: { status: 'active', interfaceEnabled: true } });
|
||||||
|
await expect(service['processItem']('item-1', 'd-1', 'failed')).resolves.toBe('skipped');
|
||||||
|
expect(requeue).not.toHaveBeenCalled();
|
||||||
|
expect(mock.downstreamRequeueTaskItem.update).toHaveBeenCalledWith(expect.objectContaining({ data: expect.objectContaining({ status: 'skipped', skipReason: '创建任务后已被客户确认' }) }));
|
||||||
|
});
|
||||||
|
|
||||||
it('paginates all task items with status and keyword filters', async () => {
|
it('paginates all task items with status and keyword filters', async () => {
|
||||||
mock.downstreamRequeueTask.findUnique.mockResolvedValue({ id: 'task-1' });
|
mock.downstreamRequeueTask.findUnique.mockResolvedValue({ id: 'task-1' });
|
||||||
mock.downstreamRequeueTaskItem.findMany.mockResolvedValue([{ id: 'item-1' }]);
|
mock.downstreamRequeueTaskItem.findMany.mockResolvedValue([{ id: 'item-1' }]);
|
||||||
|
|||||||
@@ -15,7 +15,7 @@ export type DownstreamRequeueFilter = {
|
|||||||
};
|
};
|
||||||
|
|
||||||
type RequeueFacade = { requeueDownstreamDelivery(id: string): Promise<unknown> };
|
type RequeueFacade = { requeueDownstreamDelivery(id: string): Promise<unknown> };
|
||||||
const REPLAYABLE_STATUSES = ['pending', 'failed', 'unconfirmed', 'rejected'];
|
const REPLAYABLE_STATUSES = ['pending', 'failed', 'unconfirmed', 'rejected', 'delivered'];
|
||||||
const ACTIVE_TASK_STATUSES = ['queued', 'running', 'paused'];
|
const ACTIVE_TASK_STATUSES = ['queued', 'running', 'paused'];
|
||||||
const PROCESSING_LEASE_MS = 2 * 60_000;
|
const PROCESSING_LEASE_MS = 2 * 60_000;
|
||||||
const SCAN_LEASE_MS = 15_000;
|
const SCAN_LEASE_MS = 15_000;
|
||||||
@@ -119,7 +119,7 @@ export class SendDownstreamRequeueTaskService {
|
|||||||
const preview = verifyPreview(data.previewToken, createdById);
|
const preview = verifyPreview(data.previewToken, createdById);
|
||||||
const filter = normalizedFilter(preview.filter);
|
const filter = normalizedFilter(preview.filter);
|
||||||
const snapshotAt = new Date(preview.snapshotAt);
|
const snapshotAt = new Date(preview.snapshotAt);
|
||||||
if (filter.status === 'delivered' || filter.status === 'awaiting_ack') throw new BadRequestException('第一版后台任务不支持已确认或等待ACK记录');
|
if (filter.status === 'awaiting_ack') throw new BadRequestException('后台任务不支持正在等待ACK的记录');
|
||||||
const activeTask = await this.prisma.downstreamRequeueTask.findFirst({ where: {
|
const activeTask = await this.prisma.downstreamRequeueTask.findFirst({ where: {
|
||||||
status: { in: ACTIVE_TASK_STATUSES },
|
status: { in: ACTIVE_TASK_STATUSES },
|
||||||
...(filter.applicationId !== 'all' ? { OR: [{ applicationId: filter.applicationId }, { applicationId: null }] } : {}),
|
...(filter.applicationId !== 'all' ? { OR: [{ applicationId: filter.applicationId }, { applicationId: null }] } : {}),
|
||||||
@@ -229,14 +229,14 @@ export class SendDownstreamRequeueTaskService {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
await this.prisma.downstreamRequeueTask.update({ where: { id: taskId }, data: { status: 'running', startedAt: task.startedAt ?? new Date() } });
|
await this.prisma.downstreamRequeueTask.update({ where: { id: taskId }, data: { status: 'running', startedAt: task.startedAt ?? new Date() } });
|
||||||
const items = await this.prisma.downstreamRequeueTaskItem.findMany({ where: { taskId, status: 'queued' }, orderBy: { createdAt: 'asc' }, take: Math.min(200, task.ratePerSecond * 3), select: { id: true, deliveryId: true, applicationId: true } });
|
const items = await this.prisma.downstreamRequeueTaskItem.findMany({ where: { taskId, status: 'queued' }, orderBy: { createdAt: 'asc' }, take: Math.min(200, task.ratePerSecond * 3), select: { id: true, deliveryId: true, applicationId: true, previousStatus: true } });
|
||||||
for (const item of items) {
|
for (const item of items) {
|
||||||
const latestTask = await this.prisma.downstreamRequeueTask.findUnique({ where: { id: taskId }, select: { status: true } });
|
const latestTask = await this.prisma.downstreamRequeueTask.findUnique({ where: { id: taskId }, select: { status: true } });
|
||||||
if (!latestTask || !['queued', 'running'].includes(latestTask.status)) break;
|
if (!latestTask || !['queued', 'running'].includes(latestTask.status)) break;
|
||||||
if (!(await this.consumeRate(item.applicationId, task.ratePerSecond))) continue;
|
if (!(await this.consumeRate(item.applicationId, task.ratePerSecond))) continue;
|
||||||
const claimed = await this.prisma.downstreamRequeueTaskItem.updateMany({ where: { id: item.id, status: 'queued' }, data: { status: 'processing', claimedAt: new Date() } });
|
const claimed = await this.prisma.downstreamRequeueTaskItem.updateMany({ where: { id: item.id, status: 'queued' }, data: { status: 'processing', claimedAt: new Date() } });
|
||||||
if (!claimed.count) continue;
|
if (!claimed.count) continue;
|
||||||
const outcome = await this.processItem(item.id, item.deliveryId);
|
const outcome = await this.processItem(item.id, item.deliveryId, item.previousStatus);
|
||||||
if (outcome === 'success') failures[item.applicationId] = 0;
|
if (outcome === 'success') failures[item.applicationId] = 0;
|
||||||
if (outcome === 'failed') failures[item.applicationId] = (failures[item.applicationId] ?? 0) + 1;
|
if (outcome === 'failed') failures[item.applicationId] = (failures[item.applicationId] ?? 0) + 1;
|
||||||
const maxFailures = Math.max(0, ...Object.values(failures));
|
const maxFailures = Math.max(0, ...Object.values(failures));
|
||||||
@@ -257,16 +257,22 @@ export class SendDownstreamRequeueTaskService {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private async processItem(itemId: string, deliveryId: string): Promise<'success' | 'failed' | 'waiting' | 'skipped'> {
|
private async processItem(itemId: string, deliveryId: string, previousStatus: string): Promise<'success' | 'failed' | 'waiting' | 'skipped'> {
|
||||||
try {
|
try {
|
||||||
const delivery = await this.prisma.cmppDownstreamDelivery.findUnique({
|
const delivery = await this.prisma.cmppDownstreamDelivery.findUnique({
|
||||||
where: { id: deliveryId },
|
where: { id: deliveryId },
|
||||||
include: { application: { select: { status: true, interfaceEnabled: true } } },
|
include: { application: { select: { status: true, interfaceEnabled: true } } },
|
||||||
});
|
});
|
||||||
if (!delivery) return this.finishItem(itemId, 'skipped', '投递记录已不存在');
|
if (!delivery) return this.finishItem(itemId, 'skipped', '投递记录已不存在');
|
||||||
|
// Only a record that was already delivered in the frozen task snapshot may be replayed as
|
||||||
|
// delivered. This preserves the operator's explicit duplicate-delivery intent while preventing
|
||||||
|
// a pending/failed record that receives a late ACK after task creation from being sent again.
|
||||||
|
if (delivery.status === 'delivered' && previousStatus !== 'delivered') {
|
||||||
|
return this.finishItem(itemId, 'skipped', '创建任务后已被客户确认');
|
||||||
|
}
|
||||||
if (!REPLAYABLE_STATUSES.includes(delivery.status)) {
|
if (!REPLAYABLE_STATUSES.includes(delivery.status)) {
|
||||||
if (delivery.status === 'awaiting_ack') { await this.prisma.downstreamRequeueTaskItem.update({ where: { id: itemId }, data: { status: 'waiting_external_ack', skipReason: null } }); return 'waiting'; }
|
if (delivery.status === 'awaiting_ack') { await this.prisma.downstreamRequeueTaskItem.update({ where: { id: itemId }, data: { status: 'waiting_external_ack', skipReason: null } }); return 'waiting'; }
|
||||||
return this.finishItem(itemId, 'skipped', delivery.status === 'delivered' ? '已被客户确认' : '执行前状态已变化');
|
return this.finishItem(itemId, 'skipped', '执行前状态已变化');
|
||||||
}
|
}
|
||||||
if (delivery.application.status !== 'active' || !delivery.application.interfaceEnabled) return this.finishItem(itemId, 'skipped', '应用或投递能力已停用');
|
if (delivery.application.status !== 'active' || !delivery.application.interfaceEnabled) return this.finishItem(itemId, 'skipped', '应用或投递能力已停用');
|
||||||
if (!delivery.payload || !['receipt', 'uplink'].includes(delivery.deliveryType)) return this.finishItem(itemId, 'skipped', '投递数据不完整');
|
if (!delivery.payload || !['receipt', 'uplink'].includes(delivery.deliveryType)) return this.finishItem(itemId, 'skipped', '投递数据不完整');
|
||||||
|
|||||||
@@ -0,0 +1,3 @@
|
|||||||
|
[Definition]
|
||||||
|
failregex = ^<HOST> .* "(?:GET|POST|HEAD) /(?:\.env|\.git|wp-admin|wp-login\.php|phpmyadmin|vendor/phpunit|actuator|cgi-bin)(?:[/? ][^\"]*)?" (?:400|403|404) .*$
|
||||||
|
ignoreregex =
|
||||||
@@ -0,0 +1,8 @@
|
|||||||
|
[Definition]
|
||||||
|
# Detection remains report-only. The fixed action is installed by the deployment
|
||||||
|
# script and can only forward Fail2ban's matched IP/jail values to the local collector.
|
||||||
|
actionstart =
|
||||||
|
actionstop =
|
||||||
|
actioncheck =
|
||||||
|
actionban = @CMPP_SECURITY_AGENT_BIN@ report <name> <ip>
|
||||||
|
actionunban =
|
||||||
@@ -0,0 +1,25 @@
|
|||||||
|
[Unit]
|
||||||
|
Description=CMPP restricted security execution agent
|
||||||
|
After=network.target fail2ban.service nftables.service
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
Type=simple
|
||||||
|
User=root
|
||||||
|
Group=cmpp-security
|
||||||
|
EnvironmentFile=/etc/cmpp-platform/cmpp-platform.env
|
||||||
|
ExecStart=@CMPP_SECURITY_AGENT_BIN@
|
||||||
|
Restart=on-failure
|
||||||
|
RestartSec=3
|
||||||
|
NoNewPrivileges=true
|
||||||
|
PrivateTmp=true
|
||||||
|
ProtectHome=true
|
||||||
|
ProtectSystem=strict
|
||||||
|
ReadWritePaths=/run/cmpp-security-agent /var/lib/cmpp-security-agent /etc/nginx/snippets /etc/fail2ban/jail.d
|
||||||
|
RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6 AF_NETLINK
|
||||||
|
CapabilityBoundingSet=CAP_NET_ADMIN CAP_DAC_OVERRIDE CAP_KILL
|
||||||
|
AmbientCapabilities=CAP_NET_ADMIN CAP_DAC_OVERRIDE
|
||||||
|
LockPersonality=true
|
||||||
|
MemoryDenyWriteExecute=true
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=multi-user.target
|
||||||
@@ -1138,3 +1138,17 @@ global,不能错误归入client。`client-signature-*`、发送页、企业认
|
|||||||
```
|
```
|
||||||
|
|
||||||
每次开始新拆分版本时,在本路线图基础上另写该版本的短实施计划,不直接把路线图当作可执行变更清单。
|
每次开始新拆分版本时,在本路线图基础上另写该版本的短实施计划,不直接把路线图当作可执行变更清单。
|
||||||
|
## 安全检测领域边界补充(2026-08-14)
|
||||||
|
|
||||||
|
- `api/src/security-detection/` 是安全事件、聚合告警、规则版本和人工封禁编排的唯一业务边界;登录、OpenAPI 和 Gateway 只上报固定类型的结构化事件,不复制聚合或封禁逻辑。
|
||||||
|
- `gateway/cmd/security-agent/` 是最小特权执行边界,不依赖 NestJS Service,不接受任意命令、路径、jail、action 或 shell 参数。该二进制与 `deploy/security/`、`tools/security/install-security-agent.sh` 作为同一发布单元评审。
|
||||||
|
- 安全代理可执行文件位置只由安装器的`agent_binary=$APP_DIR/dist/cmpp-security-agent`定义;systemd和Fail2ban模板使用同一占位符渲染,不能各自维护易漂移的绝对路径。
|
||||||
|
- 前端 `src/apps/admin/security-detection/` 通过 `src/api/admin/security-detection.api.ts` 访问稳定门面,不直接访问 Fail2ban、Nginx、nftables 或安全代理。
|
||||||
|
- NestJS进程边界由`api/src/main.ts`和生产环境`API_HOST`共同固定为回环监听,外部HTTP入口统一归Nginx模块治理;后续拆分不得让业务模块自行新增外部监听或绕过反向代理边界。
|
||||||
|
|
||||||
|
## 基础设施指标领域边界补充(2026-08-14)
|
||||||
|
|
||||||
|
- `api/src/metrics/`只负责API进程和HTTP路由模板的低基数计数/直方图,不依赖Prisma、Redis或业务Service;独立回环监听由`api/src/main.ts`组装。
|
||||||
|
- `gateway/internal/metrics/`只负责线程安全聚合和Prometheus文本暴露;上游连接池、下游Session和Submit Worker仅提供总数或有界结果,不把实体ID或凭据交给监控模块。
|
||||||
|
- `api/src/infrastructure-monitoring/`是运营端监控聚合与固定阈值应用边界:只消费代码白名单 PromQL,并通过版本化 PostgreSQL 单例、promtool 校验和原子规则热加载管理数值阈值;Exporter安装、端口隔离、固定规则模板和权限仍归`tools/monitoring/`治理。
|
||||||
|
- 活动告警已读也归该边界:Prometheus保留告警事实,Prisma仅持久化逐管理员、逐触发周期的阅读状态;全局布局只消费轻量未读汇总,不复制指纹、activeAt或用户隔离逻辑。
|
||||||
|
|||||||
@@ -1,8 +1,8 @@
|
|||||||
# 下游投递后台批量重投任务设计与实现符合性审计
|
# 下游投递后台批量重投任务设计与实现符合性审计
|
||||||
|
|
||||||
> 版本:V1.0<br>
|
> 版本:V1.1<br>
|
||||||
> 需求确认日期:2026-08-12<br>
|
> 需求确认日期:2026-08-12<br>
|
||||||
> 文档整理日期:2026-08-13<br>
|
> 文档整理日期:2026-08-13;业务口径更新:2026-08-14<br>
|
||||||
> 适用页面:运营端 → 下游投递记录<br>
|
> 适用页面:运营端 → 下游投递记录<br>
|
||||||
> 审计基线:当前工作区 `HEAD=67fee216162e638ba21004fcf87e7711facefb91`;本功能相关文件相对 HEAD 无未提交修改<br>
|
> 审计基线:当前工作区 `HEAD=67fee216162e638ba21004fcf87e7711facefb91`;本功能相关文件相对 HEAD 无未提交修改<br>
|
||||||
> 本文目的:还原 2026-08-12 已确认的设计口径,并将当前实现逐条映射到设计,不能以“已有代码”代替“符合设计”的结论。
|
> 本文目的:还原 2026-08-12 已确认的设计口径,并将当前实现逐条映射到设计,不能以“已有代码”代替“符合设计”的结论。
|
||||||
@@ -27,9 +27,9 @@
|
|||||||
|
|
||||||
1. 保留单条重投、当前页勾选批量重投,新增“按筛选条件重投”;投递记录分页支持每页 10/25/50 条。
|
1. 保留单条重投、当前页勾选批量重投,新增“按筛选条件重投”;投递记录分页支持每页 10/25/50 条。
|
||||||
2. 后台任务使用企业、应用、投递类型、状态、创建日期、关键词组成的筛选快照;页码和每页条数不属于任务范围。预检生成 `snapshotAt`,创建任务后产生的新记录不进入该任务。
|
2. 后台任务使用企业、应用、投递类型、状态、创建日期、关键词组成的筛选快照;页码和每页条数不属于任务范围。预检生成 `snapshotAt`,创建任务后产生的新记录不进入该任务。
|
||||||
3. 第一版后台任务只允许 `pending`、`failed`、`unconfirmed`、`rejected`。不得批量重投客户端已确认的 `delivered`;处于 `awaiting_ack` 的记录不得并发重投。创建前展示真实命中数、可重投数、规则跳过数、状态分布,任务原因必填。
|
3. 后台任务允许 `pending`、`failed`、`unconfirmed`、`rejected`、`delivered`。客户端已确认的 `delivered` 可按筛选快照再次投递,但必须醒目提示可能造成客户端重复处理;处于 `awaiting_ack` 的记录不得并发重投。创建前展示真实命中数、可重投数、规则跳过数、状态分布,任务原因必填。
|
||||||
4. 任务按企业应用分批执行,默认每个应用 10 条/秒。单条失败不阻断整批;连续失败达到 10 条,或 ACK 超时/拒绝达到安全阈值时,自动暂停。客户离线、已有链路等待 ACK 属于“等待”,不能记作“跳过”。
|
4. 任务按企业应用分批执行,默认每个应用 10 条/秒。单条失败不阻断整批;连续失败达到 10 条,或 ACK 超时/拒绝达到安全阈值时,自动暂停。客户离线、已有链路等待 ACK 属于“等待”,不能记作“跳过”。
|
||||||
5. “跳过”严格表示本任务没有调用 Gateway。第一版跳过原因包括:执行前状态变化、已被客户确认、已被其他任务处理、本任务已成功处理、不属于任务快照、应用或投递能力已停用、投递数据不完整、缺少原 Submit 映射。
|
5. “跳过”严格表示本任务没有调用 Gateway。跳过原因包括:执行前状态变化、创建任务后才被客户确认、已被其他任务处理、本任务已成功处理、不属于任务快照、应用或投递能力已停用、投递数据不完整、缺少原 Submit 映射。只有任务项冻结的原状态已是 `delivered` 时,才允许按已确认记录重投;其他状态在执行前收到迟到成功 ACK 时必须跳过。
|
||||||
6. 任务支持列表、详情、暂停、继续、终止。终止只影响尚未发送的任务项。任务项以 `taskId + deliveryId` 幂等;执行前原子认领并复核当前状态;API 重启后继续执行;已获得成功 ACK 的任务项不得再次发送。
|
6. 任务支持列表、详情、暂停、继续、终止。终止只影响尚未发送的任务项。任务项以 `taskId + deliveryId` 幂等;执行前原子认领并复核当前状态;API 重启后继续执行;已获得成功 ACK 的任务项不得再次发送。
|
||||||
7. 创建、暂停、继续、终止、自动暂停都必须写操作日志。任务必须使用真实 PostgreSQL、Gateway 和客户 ACK,不得使用 mock、静态数据或 localStorage。
|
7. 创建、暂停、继续、终止、自动暂停都必须写操作日志。任务必须使用真实 PostgreSQL、Gateway 和客户 ACK,不得使用 mock、静态数据或 localStorage。
|
||||||
|
|
||||||
@@ -54,7 +54,7 @@
|
|||||||
|
|
||||||
1. 用户设置筛选条件,点击“按筛选条件重投”。
|
1. 用户设置筛选条件,点击“按筛选条件重投”。
|
||||||
2. 后端在同一时点生成预检快照,返回:筛选条件、`snapshotAt`、筛选命中数、可重投数、规则跳过数、状态分布、涉及应用数、最早记录时间。
|
2. 后端在同一时点生成预检快照,返回:筛选条件、`snapshotAt`、筛选命中数、可重投数、规则跳过数、状态分布、涉及应用数、最早记录时间。
|
||||||
3. 弹窗明确告知第一版允许和禁止的状态。
|
3. 弹窗明确告知允许 `pending/failed/unconfirmed/rejected/delivered`、禁止 `awaiting_ack`,并提示已确认记录再次投递可能造成客户端重复处理。
|
||||||
4. 用户选择执行速度,填写不少于 5 个字的事故原因、工单号或处理说明。
|
4. 用户选择执行速度,填写不少于 5 个字的事故原因、工单号或处理说明。
|
||||||
5. 用户确认后,后端必须重新按预检的筛选快照和 `snapshotAt` 物化任务项,而不是使用前端传入的记录 ID 列表。
|
5. 用户确认后,后端必须重新按预检的筛选快照和 `snapshotAt` 物化任务项,而不是使用前端传入的记录 ID 列表。
|
||||||
6. 创建成功后关闭弹窗,任务出现在任务列表,状态为“排队中”。
|
6. 创建成功后关闭弹窗,任务出现在任务列表,状态为“排队中”。
|
||||||
@@ -160,8 +160,8 @@
|
|||||||
|
|
||||||
| 原因 | 判定 |
|
| 原因 | 判定 |
|
||||||
| --- | --- |
|
| --- | --- |
|
||||||
| 执行前状态已变化 | 已不属于允许重投状态,且不是等待 ACK/已确认 |
|
| 执行前状态已变化 | 已不属于允许重投状态,且不是等待 ACK |
|
||||||
| 已被客户确认 | 当前投递已是 `delivered` 且有效 ACK |
|
| 创建任务后才被客户确认 | 任务项冻结原状态不是 `delivered`,执行前收到有效成功 ACK;避免把迟到确认变成未授权重复投递 |
|
||||||
| 已被其他任务处理 | 其他任务已认领、等待 ACK 或成功 |
|
| 已被其他任务处理 | 其他任务已认领、等待 ACK 或成功 |
|
||||||
| 本任务已成功处理 | 同任务项已有成功结果,重复扫描不得再调用 |
|
| 本任务已成功处理 | 同任务项已有成功结果,重复扫描不得再调用 |
|
||||||
| 不属于任务快照 | 创建时间晚于 `snapshotAt` 或不再满足冻结范围 |
|
| 不属于任务快照 | 创建时间晚于 `snapshotAt` 或不再满足冻结范围 |
|
||||||
@@ -222,7 +222,7 @@
|
|||||||
| --- | --- | --- |
|
| --- | --- | --- |
|
||||||
| DRQ-001 | 筛选快照 | 企业、应用、类型、状态、日期、关键词均生效;分页无关;快照后新记录不进入 |
|
| DRQ-001 | 筛选快照 | 企业、应用、类型、状态、日期、关键词均生效;分页无关;快照后新记录不进入 |
|
||||||
| DRQ-002 | 预检口径 | 命中、可重投、跳过和状态分布严格基于当前筛选条件 |
|
| DRQ-002 | 预检口径 | 命中、可重投、跳过和状态分布严格基于当前筛选条件 |
|
||||||
| DRQ-003 | 状态白名单 | 只物化 `pending/failed/unconfirmed/rejected`;拒绝 `delivered/awaiting_ack` |
|
| DRQ-003 | 状态白名单 | 物化 `pending/failed/unconfirmed/rejected/delivered`;拒绝 `awaiting_ack`;仅冻结原状态为 `delivered` 的任务项可按已确认记录重投 |
|
||||||
| DRQ-004 | 每应用限速 | 多应用任务中每个应用独立达到配置速度,任意扫描重叠都不超速 |
|
| DRQ-004 | 每应用限速 | 多应用任务中每个应用独立达到配置速度,任意扫描重叠都不超速 |
|
||||||
| DRQ-005 | 客户离线 | 进入等待连接,不记失败或跳过,恢复连接后继续 |
|
| DRQ-005 | 客户离线 | 进入等待连接,不记失败或跳过,恢复连接后继续 |
|
||||||
| DRQ-006 | ACK 闭环 | 写出只进入等待;有效 ACK 成功;超时、拒绝、无效 Msg_Id 失败 |
|
| DRQ-006 | ACK 闭环 | 写出只进入等待;有效 ACK 成功;超时、拒绝、无效 Msg_Id 失败 |
|
||||||
@@ -250,13 +250,13 @@
|
|||||||
| --- | --- | --- |
|
| --- | --- | --- |
|
||||||
| 真实持久化 | 符合 | 已有任务表、任务项表和 migration,不使用前端本地状态代替任务 |
|
| 真实持久化 | 符合 | 已有任务表、任务项表和 migration,不使用前端本地状态代替任务 |
|
||||||
| 快照时间上限 | 基本符合 | 创建时按 `snapshotAt` 限制 `createdAt`,快照后记录不物化 |
|
| 快照时间上限 | 基本符合 | 创建时按 `snapshotAt` 限制 `createdAt`,快照后记录不物化 |
|
||||||
| 后台状态白名单 | 符合 | `REPLAYABLE_STATUSES` 为 `pending/failed/unconfirmed/rejected` |
|
| 后台状态白名单 | 符合 | `REPLAYABLE_STATUSES` 为 `pending/failed/unconfirmed/rejected/delivered` |
|
||||||
| 禁止后台任务处理已确认/等待 ACK 筛选 | 符合 | 创建接口明确拒绝 `delivered/awaiting_ack` |
|
| 禁止后台任务处理等待 ACK 筛选 | 符合 | 创建接口明确拒绝 `awaiting_ack`;`delivered` 按已确认重复投递风险口径放行 |
|
||||||
| 原因校验 | 符合 | 少于 5 个字拒绝 |
|
| 原因校验 | 符合 | 少于 5 个字拒绝 |
|
||||||
| 任务项幂等 | 符合 | 数据库唯一约束 `taskId + deliveryId` |
|
| 任务项幂等 | 符合 | 数据库唯一约束 `taskId + deliveryId` |
|
||||||
| 执行前任务项认领 | 基本符合 | 通过 `status=queued` 的条件更新认领为 `processing` |
|
| 执行前任务项认领 | 基本符合 | 通过 `status=queued` 的条件更新认领为 `processing` |
|
||||||
| 执行前复核 | 基本符合 | 重新查询投递、应用、payload 和其他任务状态 |
|
| 执行前复核 | 基本符合 | 重新查询投递、应用、payload 和其他任务状态 |
|
||||||
| 成功 ACK 不再发送 | 基本符合 | 已确认投递会跳过,其他任务 `success` 也会阻止调用 |
|
| 成功 ACK 不再误发送 | 基本符合 | 非 `delivered` 快照项执行前才收到成功 ACK 时跳过;其他任务 `success` 也会阻止调用;冻结原状态为 `delivered` 的项目属于运营明确授权的再次投递 |
|
||||||
| 人工控制 | 基本符合 | 已有暂停、继续、终止接口和页面按钮 |
|
| 人工控制 | 基本符合 | 已有暂停、继续、终止接口和页面按钮 |
|
||||||
| 核心操作日志 | 基本符合 | 创建、暂停、继续、终止、自动暂停均写日志 |
|
| 核心操作日志 | 基本符合 | 创建、暂停、继续、终止、自动暂停均写日志 |
|
||||||
| 投递记录分页 | 符合 | 页面支持每页 10/25/50 条并回到第一页 |
|
| 投递记录分页 | 符合 | 页面支持每页 10/25/50 条并回到第一页 |
|
||||||
@@ -279,7 +279,7 @@
|
|||||||
| P1 | 自动化覆盖远低于设计风险 | 专项仅4个测试:预检计数、参数拒绝、活动任务冲突、已确认跳过 | 未覆盖限速、多应用、离线等待、ACK阈值、重启恢复、并发扫描、完整分页及所有审计动作 |
|
| P1 | 自动化覆盖远低于设计风险 | 专项仅4个测试:预检计数、参数拒绝、活动任务冲突、已确认跳过 | 未覆盖限速、多应用、离线等待、ACK阈值、重启恢复、并发扫描、完整分页及所有审计动作 |
|
||||||
| P2 | 状态和详情表达偏内部化 | 任务列表/详情直接展示英文状态;任务详情只展示汇总和最近项 | 运营人员不易区分等待连接、等待外部ACK、任务写出等待ACK等状态 |
|
| P2 | 状态和详情表达偏内部化 | 任务列表/详情直接展示英文状态;任务详情只展示汇总和最近项 | 运营人员不易区分等待连接、等待外部ACK、任务写出等待ACK等状态 |
|
||||||
| P2 | 终止后的未处理数未进入常规汇总 | 终止把`queued`改为`unprocessed`,但任务汇总只保存成功/失败/跳过/等待 | 任务进度分子可能小于总数,页面没有单独解释未处理数量 |
|
| P2 | 终止后的未处理数未进入常规汇总 | 终止把`queued`改为`unprocessed`,但任务汇总只保存成功/失败/跳过/等待 | 任务进度分子可能小于总数,页面没有单独解释未处理数量 |
|
||||||
| P2 | 已确认跳过原因存在口径混用 | `waiting_external_ack`最终由其他链路成功后记为“跳过:已由其他投递链路完成” | 可以接受为“本任务未调用Gateway”,但详情必须明确这是外部链路成功,不应让用户误以为业务未处理 |
|
| P2 | 外部 ACK 跳过原因存在口径混用 | `waiting_external_ack`最终由其他链路成功后记为“跳过:已由其他投递链路完成” | 可以接受为“本任务未调用Gateway”,但详情必须明确这是外部链路成功,不应让用户误以为业务未处理;这与冻结原状态为 `delivered` 的主动再次投递是两种情形 |
|
||||||
|
|
||||||
### 11.3 综合结论
|
### 11.3 综合结论
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,379 @@
|
|||||||
|
# Fail2ban 安全检测与人工封禁平台设计方案
|
||||||
|
|
||||||
|
> 版本:V1.0(需求评审稿)<br>
|
||||||
|
> 日期:2026-08-14<br>
|
||||||
|
> 范围:运营端自研 UI、安全检测、阈值配置、告警处置、人工封禁与解封<br>
|
||||||
|
> 边界:第一版不自动封禁、不开放任意 Fail2ban/防火墙命令、不在线编辑正则表达式
|
||||||
|
|
||||||
|
## 1. 建设目标
|
||||||
|
|
||||||
|
在运营端“安全控制”下建设自研安全检测面板,统一接收 SSH、运营端登录、客户端登录、CMPP 入站和 HTTP API 的异常行为,按照可配置规则聚合为安全告警。平台管理员查看证据后,可人工执行固定时长封禁、解封、忽略或加入保护名单。
|
||||||
|
|
||||||
|
第一版采用“检测与执行分离”原则:
|
||||||
|
|
||||||
|
1. Fail2ban 和应用侧检测器只生成事件及告警,不自动修改防火墙。
|
||||||
|
2. 人工点击“封禁”后,后端重新校验告警、真实 IP、保护名单、执行目标和操作权限。
|
||||||
|
3. 只有受限安全代理可以执行系统级封禁;NestJS 不直接获得 root、通用 sudo 或任意 shell 能力。
|
||||||
|
4. PostgreSQL 保存配置、事件、告警、封禁事实和操作审计;页面不得使用 mock、静态数据或 localStorage 伪造状态。
|
||||||
|
|
||||||
|
## 2. 第一版范围
|
||||||
|
|
||||||
|
### 2.1 检测类型
|
||||||
|
|
||||||
|
| 规则编码 | 检测对象 | 事件来源 | 第一版默认建议值 | 默认风险 |
|
||||||
|
| --- | --- | --- | --- | --- |
|
||||||
|
| `admin_login_failure` | 运营端账号登录失败 | NestJS 结构化安全事件 | 同一 IP 10 分钟 8 次 | 中 |
|
||||||
|
| `client_login_failure` | 客户端账号登录失败 | NestJS 结构化安全事件 | 同一 IP 10 分钟 8 次 | 中 |
|
||||||
|
| `ssh_auth_failure` | SSH 12022 认证失败 | journald + Fail2ban filter | 同一 IP 10 分钟 6 次 | 高 |
|
||||||
|
| `cmpp_auth_failure` | CMPP 17890 未知账号、认证失败或不允许 IP | Gateway 结构化安全事件 | 同一 IP 5 分钟 5 次 | 高 |
|
||||||
|
| `cmpp_protocol_abuse` | 非法协议包、异常版本或高频无效连接 | Gateway 结构化安全事件 | 同一 IP 1 分钟 20 次 | 高 |
|
||||||
|
| `http_invalid_api_key` | HTTP API 错误或未知访问密钥 | NestJS HTTP API 鉴权事件 | 同一 IP 5 分钟 10 次 | 高 |
|
||||||
|
| `http_signature_failure` | HTTP API 签名缺失、格式错误或验签失败 | NestJS HTTP API 验签事件 | 同一 IP 5 分钟 10 次 | 高 |
|
||||||
|
| `http_replay_attempt` | nonce、时间戳或幂等凭证重放 | NestJS HTTP API 验签事件 | 同一 IP 10 分钟 3 次 | 严重 |
|
||||||
|
| `http_malicious_scan` | 扫描敏感路径、跨路径高频 404、明显漏洞探测 | Nginx 结构化日志 + Fail2ban filter | 同一 IP 1 分钟 20 次且涉及至少 8 个不同路径 | 高 |
|
||||||
|
|
||||||
|
表中数值只是首次安装默认值,必须保存到真实数据库并可在运营端配置。业务 400、正常参数校验失败、合法客户偶发时钟偏差、普通 404、供应商连接失败不得直接归类为攻击。
|
||||||
|
|
||||||
|
### 2.2 处置能力
|
||||||
|
|
||||||
|
- 查看检测总览、趋势、来源、风险等级、Top IP 和待处置告警。
|
||||||
|
- 查看告警详情、脱敏日志证据、规则快照和历史处置。
|
||||||
|
- 人工封禁固定时长:10 分钟、1 小时、24 小时、7 天。
|
||||||
|
- 人工解封、忽略告警、加入保护名单。
|
||||||
|
- 查看当前真实封禁、执行目标、到期时间和同步状态。
|
||||||
|
- 配置检测规则阈值、时间窗口、冷却时间、风险等级和启停状态。
|
||||||
|
- 查看 Fail2ban、事件采集器、安全代理、规则版本及封禁执行器健康状态。
|
||||||
|
|
||||||
|
### 2.3 第一版不做
|
||||||
|
|
||||||
|
- 自动封禁。
|
||||||
|
- 永久封禁或前端输入任意封禁秒数。
|
||||||
|
- 在线编辑 Fail2ban regex、日志路径、action、iptables/nftables 或 Nginx 配置文本。
|
||||||
|
- 从页面执行任意 shell、`fail2ban-client`、`systemctl` 或防火墙命令。
|
||||||
|
- 自动将外部威胁情报加入封禁。
|
||||||
|
- 删除原始告警和处置历史。
|
||||||
|
|
||||||
|
## 3. 总体架构
|
||||||
|
|
||||||
|
```text
|
||||||
|
sshd journal ── Fail2ban 检测 jail ─┐
|
||||||
|
Nginx access/error ─ Fail2ban jail ─┤
|
||||||
|
NestJS 登录/HTTP 鉴权安全事件 ──────┤
|
||||||
|
Go Gateway CMPP 安全事件 ───────────┤
|
||||||
|
↓
|
||||||
|
Security Event Collector
|
||||||
|
↓
|
||||||
|
PostgreSQL 事件、规则、告警、封禁
|
||||||
|
↓
|
||||||
|
NestJS 运营端安全 API
|
||||||
|
↓
|
||||||
|
自研运营端 UI
|
||||||
|
↓ 人工确认
|
||||||
|
Root Security Agent (Unix Socket)
|
||||||
|
↓ ↓
|
||||||
|
nftables/manual jail Nginx real-IP deny
|
||||||
|
```
|
||||||
|
|
||||||
|
### 3.1 组件职责
|
||||||
|
|
||||||
|
#### Fail2ban
|
||||||
|
|
||||||
|
- 读取 sshd 和 Nginx 等系统日志。
|
||||||
|
- 使用固定、随版本发布的 filter 识别失败行为。
|
||||||
|
- 根据已生效规则计算窗口与阈值。
|
||||||
|
- 触发“告警事件 action”,但不直接封禁。
|
||||||
|
- 不作为平台告警历史和真实封禁状态的唯一数据源。
|
||||||
|
|
||||||
|
#### 应用侧安全事件
|
||||||
|
|
||||||
|
NestJS 和 Gateway 对其掌握真实业务语义的事件直接产生结构化安全事件,避免只靠文本正则猜测:
|
||||||
|
|
||||||
|
- 登录入口、失败类型和真实请求 IP。
|
||||||
|
- HTTP API 密钥查找失败、签名失败、重放拒绝。
|
||||||
|
- CMPP 账号、AuthenticatorSource 校验、IP 白名单和协议异常。
|
||||||
|
|
||||||
|
任何安全事件不得记录明文密码、完整 API 密钥、签名密钥、AuthenticatorSource 或完整短信内容;账号、密钥标识只保存脱敏值或不可逆指纹。
|
||||||
|
|
||||||
|
#### Security Event Collector
|
||||||
|
|
||||||
|
- 通过受限 Unix Socket 或 root 写入、collector 只读的事件目录接收 Fail2ban 事件。
|
||||||
|
- 校验事件版本、规则编码、IP、时间和来源。
|
||||||
|
- 写入 PostgreSQL,并依据数据库规则做幂等聚合。
|
||||||
|
- 维护最后事件时间、丢弃数量和解析失败指标。
|
||||||
|
|
||||||
|
#### Root Security Agent
|
||||||
|
|
||||||
|
- 独立于 NestJS,以最小 root 权限运行。
|
||||||
|
- 仅监听本机 Unix Socket,不监听 TCP 公网端口。
|
||||||
|
- 只接受固定 JSON 协议:`block`、`unblock`、`status`、`apply_rule_version`。
|
||||||
|
- 对规则编码、执行器、IPv4/IPv6、时长和幂等键做白名单校验。
|
||||||
|
- 使用无 shell 参数数组或原生库执行,不拼接命令。
|
||||||
|
- 原子生成平台专属配置文件,校验后才 reload;失败保留旧版本。
|
||||||
|
|
||||||
|
#### NestJS
|
||||||
|
|
||||||
|
- 读取真实 PostgreSQL 告警和配置。
|
||||||
|
- 执行 RBAC、近期重新认证、保护名单和状态校验。
|
||||||
|
- 创建封禁操作记录并调用本地安全代理。
|
||||||
|
- 根据代理回读结果确认真实封禁状态。
|
||||||
|
- 不以 root 运行,不直接写 `/etc/fail2ban/*` 或防火墙。
|
||||||
|
|
||||||
|
## 4. Cloudflare 与执行器选择
|
||||||
|
|
||||||
|
恢复 `CF-Connecting-IP` 只能让 Nginx/应用识别访客真实 IP,并不会改变到达服务器的 TCP 源地址。对 `sms.lisglo.com` 的 Cloudflare 橙云流量,nftables 封禁访客真实 IP 无效,误封 Cloudflare 节点反而可能中断全站。
|
||||||
|
|
||||||
|
第一版按入口固定执行器:
|
||||||
|
|
||||||
|
| 入口 | 网络形态 | 检测 IP | 人工封禁执行器 |
|
||||||
|
| --- | --- | --- | --- |
|
||||||
|
| 运营端、客户端 | Cloudflare 橙云 | 经过可信 Cloudflare 网段恢复的真实 IP | Nginx real-IP deny;未来可扩展 Cloudflare API |
|
||||||
|
| `api.lisglo.com` | 灰云直连 | TCP 源 IP | nftables/manual jail |
|
||||||
|
| SSH 12022 | 公网直连 | TCP 源 IP | nftables/manual jail |
|
||||||
|
| CMPP 17890 | 公网直连 | Gateway TCP 远端 IP | nftables/manual jail |
|
||||||
|
|
||||||
|
只有当请求 TCP 来源属于定期同步的 Cloudflare 官方网段时,才信任 `CF-Connecting-IP`。客户端直连源站时提供的同名 Header 必须忽略。配置上线前必须用真实请求证明日志、事件和页面展示 IP 一致。
|
||||||
|
|
||||||
|
## 5. 可配置规则
|
||||||
|
|
||||||
|
### 5.1 可配置字段
|
||||||
|
|
||||||
|
- 启用状态 `enabled`。
|
||||||
|
- 统计窗口 `windowSeconds`。
|
||||||
|
- 触发阈值 `threshold`。
|
||||||
|
- 不同目标数量阈值 `distinctTargetThreshold`,仅恶意扫描等规则使用。
|
||||||
|
- 告警冷却时间 `cooldownSeconds`。
|
||||||
|
- 风险等级 `severity`。
|
||||||
|
- 聚合维度,只能从规则预置集合选择,例如 `ip`、`ip+accountFingerprint`。
|
||||||
|
- 默认封禁时长和允许的最大封禁时长。
|
||||||
|
- 是否允许人工封禁;只读检测规则可以关闭封禁按钮。
|
||||||
|
|
||||||
|
### 5.2 不可由页面配置的字段
|
||||||
|
|
||||||
|
- filter 正则表达式。
|
||||||
|
- 日志文件路径和 journal unit。
|
||||||
|
- shell 命令、Fail2ban action、nftables 表/链。
|
||||||
|
- Unix Socket 路径、systemd 服务名。
|
||||||
|
- Cloudflare 可信网段来源。
|
||||||
|
- 规则到执行器的映射。
|
||||||
|
|
||||||
|
这些内容属于发布资产,必须通过代码评审、自动化测试和标准部署更新。
|
||||||
|
|
||||||
|
### 5.3 配置保护
|
||||||
|
|
||||||
|
- 每类规则设置服务端最小值、最大值和允许枚举;前端约束不能替代后端校验。
|
||||||
|
- 阈值修改使用乐观锁版本号,防止多人覆盖。
|
||||||
|
- 保存后生成新规则版本,安全代理先语法校验,再原子切换并 reload。
|
||||||
|
- reload 失败时数据库状态标记 `apply_failed`,保留旧生效版本,页面明确展示“已保存但未生效”。
|
||||||
|
- 每次变更保存操作人、原因、旧值、新值、生效版本、代理回执和时间。
|
||||||
|
- 配置修改要求 `security.rule.manage` 权限和近期重新认证。
|
||||||
|
|
||||||
|
## 6. 数据模型
|
||||||
|
|
||||||
|
### 6.1 `SecurityDetectionRule`
|
||||||
|
|
||||||
|
- `id`、`code`(唯一)、`name`、`category`、`description`。
|
||||||
|
- `enabled`、`windowSeconds`、`threshold`、`distinctTargetThreshold`。
|
||||||
|
- `cooldownSeconds`、`severity`、`groupingMode`。
|
||||||
|
- `manualBlockAllowed`、`defaultBlockDurationSeconds`、`maxBlockDurationSeconds`。
|
||||||
|
- `configVersion`、`effectiveVersion`、`applyStatus`、`lastApplyError`。
|
||||||
|
- `updatedById`、`createdAt`、`updatedAt`。
|
||||||
|
|
||||||
|
### 6.2 `SecurityDetectionEvent`
|
||||||
|
|
||||||
|
- `id`、`eventKey`(唯一幂等键)、`ruleCode`、`sourceType`。
|
||||||
|
- `sourceIp`、`accountFingerprint`、`targetFingerprint`、`requestPathNormalized`。
|
||||||
|
- `occurredAt`、`receivedAt`、`evidenceSummary`、`metadata`。
|
||||||
|
- `collectorInstanceId`、`ruleVersion`。
|
||||||
|
|
||||||
|
`metadata` 使用后端安全 DTO,仅保存允许字段;禁止保存密钥和完整认证材料。
|
||||||
|
|
||||||
|
### 6.3 `SecurityAlert`
|
||||||
|
|
||||||
|
- `id`、`alertNo`、`fingerprint`、`ruleId`、`ruleSnapshot`。
|
||||||
|
- `sourceIp`、`status`、`severity`。
|
||||||
|
- `firstOccurredAt`、`lastOccurredAt`、`eventCount`、`distinctTargetCount`。
|
||||||
|
- `cooldownUntil`、`assignedToId`、`handledById`、`handledAt`、`handleReason`。
|
||||||
|
- `blockId`、`createdAt`、`updatedAt`。
|
||||||
|
|
||||||
|
同一规则、IP、聚合维度和窗口桶使用唯一 fingerprint,重复采集只增加计数,不重复创建告警。
|
||||||
|
|
||||||
|
### 6.4 `SecurityBlock`
|
||||||
|
|
||||||
|
- `id`、`operationKey`(唯一)、`alertId`、`sourceIp`。
|
||||||
|
- `executorType`、`executorTarget`、`durationSeconds`。
|
||||||
|
- `status`:`requested/applying/blocked/unblock_requested/unblocked/expired/failed`。
|
||||||
|
- `startedAt`、`expiresAt`、`verifiedAt`、`errorMessage`。
|
||||||
|
- `requestedById`、`requestReason`、`unblockedById`、`unblockReason`。
|
||||||
|
- `agentOperationId`、`createdAt`、`updatedAt`。
|
||||||
|
|
||||||
|
### 6.5 `SecurityProtectedNetwork`
|
||||||
|
|
||||||
|
- IP/CIDR、名称、类型、适用入口、启停状态、来源和备注。
|
||||||
|
- 系统内置保护项不可从页面删除,只允许通过受控发布更新。
|
||||||
|
- 人工保护项新增、修改和停用均要求重新认证和审计。
|
||||||
|
|
||||||
|
## 7. 状态机与并发控制
|
||||||
|
|
||||||
|
### 7.1 告警状态
|
||||||
|
|
||||||
|
```text
|
||||||
|
pending ──→ block_requested ──→ blocked ──→ unblocked
|
||||||
|
│ └──────────→ block_failed
|
||||||
|
├──→ ignored
|
||||||
|
├──→ whitelisted
|
||||||
|
└──→ expired
|
||||||
|
```
|
||||||
|
|
||||||
|
- `pending` 仅表示达到检测阈值,绝不代表已被防火墙封禁。
|
||||||
|
- 封禁按钮通过数据库条件更新原子认领,只有一名操作人能进入 `block_requested`。
|
||||||
|
- 代理执行成功后必须回读执行器状态,确认存在真实规则才写 `blocked`。
|
||||||
|
- 网络超时导致结果不确定时先查询 `operationKey`,不得盲目重复封禁。
|
||||||
|
- 忽略、保护名单和封禁互斥;状态变化后旧页面操作返回 409。
|
||||||
|
|
||||||
|
### 7.2 封禁到期
|
||||||
|
|
||||||
|
- 执行器负责真实到期解除;平台定时回读并同步 `expired`。
|
||||||
|
- 平台任务只做状态对账,不能仅靠数据库时间把记录标记为已解封。
|
||||||
|
- 对账发现执行器缺失、额外规则或到期未解除时生成系统告警。
|
||||||
|
|
||||||
|
## 8. 后端接口
|
||||||
|
|
||||||
|
```text
|
||||||
|
GET /api/admin/security-detection/overview
|
||||||
|
GET /api/admin/security-detection/trends
|
||||||
|
GET /api/admin/security-detection/alerts
|
||||||
|
GET /api/admin/security-detection/alerts/:id
|
||||||
|
POST /api/admin/security-detection/alerts/:id/block
|
||||||
|
POST /api/admin/security-detection/alerts/:id/ignore
|
||||||
|
POST /api/admin/security-detection/alerts/:id/protect
|
||||||
|
GET /api/admin/security-detection/blocks
|
||||||
|
POST /api/admin/security-detection/blocks/:id/unblock
|
||||||
|
GET /api/admin/security-detection/rules
|
||||||
|
PUT /api/admin/security-detection/rules/:id
|
||||||
|
GET /api/admin/security-detection/protected-networks
|
||||||
|
GET /api/admin/security-detection/health
|
||||||
|
```
|
||||||
|
|
||||||
|
封禁请求只接受固定时长枚举和原因。IP、规则、入口和执行器全部从告警及服务端映射读取,禁止前端重传或覆盖。
|
||||||
|
|
||||||
|
## 9. 权限与审计
|
||||||
|
|
||||||
|
| 权限 | 能力 |
|
||||||
|
| --- | --- |
|
||||||
|
| `security.alert.read` | 查看面板、告警和脱敏证据 |
|
||||||
|
| `security.alert.handle` | 忽略告警、分派和填写处置说明 |
|
||||||
|
| `security.block.manage` | 人工封禁与解封 |
|
||||||
|
| `security.rule.manage` | 修改阈值、启停规则和保护名单 |
|
||||||
|
|
||||||
|
封禁、解封、修改规则和保护名单必须要求近期重新认证;所有动作写入 `OperationLog`,记录资源、操作人、IP、原因、旧值、新值、代理回执和最终结果。读取完整证据也应记录访问审计。
|
||||||
|
|
||||||
|
## 10. 自研 UI 信息架构
|
||||||
|
|
||||||
|
菜单位置:`安全控制 / 安全检测`。
|
||||||
|
|
||||||
|
### 10.1 总览
|
||||||
|
|
||||||
|
- 待处置、高风险、当前真实封禁、24 小时攻击 IP、封禁失败五个指标。
|
||||||
|
- 24 小时/7 天检测事件与告警趋势。
|
||||||
|
- 按规则类型、入口和风险等级分布。
|
||||||
|
- Top 攻击 IP、Top 扫描路径、Top 被尝试账号指纹。
|
||||||
|
- Fail2ban、collector、agent、规则版本和执行器健康卡片。
|
||||||
|
|
||||||
|
### 10.2 告警中心
|
||||||
|
|
||||||
|
- 按关键词、IP、规则、风险、状态、入口和日期筛选,真实后端分页。
|
||||||
|
- 列表展示风险、IP、类型、触发数、不同目标数、首次/最近时间、状态和操作。
|
||||||
|
- 详情抽屉展示规则快照、聚合时间线、脱敏证据、关联告警和处置历史。
|
||||||
|
- 封禁确认弹窗展示执行器、影响入口、时长、保护名单结果、近期合法访问提示和必填原因。
|
||||||
|
|
||||||
|
### 10.3 规则配置
|
||||||
|
|
||||||
|
- 使用平台现有自研 Card、Table、Tag、Modal、Form、Pagination 和图表体系,不嵌入 Fail2ban 第三方面板。
|
||||||
|
- 每项配置展示当前生效值、待生效值、最后修改人和应用状态。
|
||||||
|
- 数字输入同时展示单位、允许范围和默认建议值。
|
||||||
|
- 保存前展示变更对比;应用失败不能显示成功 toast。
|
||||||
|
|
||||||
|
### 10.4 封禁与保护名单
|
||||||
|
|
||||||
|
- 独立展示真实封禁状态、执行器、到期时间、来源告警和操作人。
|
||||||
|
- 保护名单命中时封禁按钮禁用并说明原因。
|
||||||
|
- 不同执行器使用明确标签,避免把 Nginx deny 误称为防火墙封禁。
|
||||||
|
|
||||||
|
## 11. 检测准确性要求
|
||||||
|
|
||||||
|
### 11.1 HTTP API
|
||||||
|
|
||||||
|
- 错误密钥:只记录不可逆密钥指纹,不记录完整 Header 或密钥。
|
||||||
|
- 签名错误:区分缺失、格式错误、算法不支持、验签失败和时间偏差。
|
||||||
|
- 重放:只有 nonce/幂等凭证已被真实使用或时间戳明显重复时计入;正常幂等重试按既有接口语义处理。
|
||||||
|
- 恶意扫描:使用标准化路径,不保存 query 中的敏感值;规则需要“次数 + 不同路径数”双阈值,避免单个合法 404 被判攻击。
|
||||||
|
- 反向代理真实 IP 必须经过可信代理链验证,禁止直接信任客户端 Header。
|
||||||
|
|
||||||
|
### 11.2 CMPP
|
||||||
|
|
||||||
|
- 未知账号、错误 AuthenticatorSource、不允许 IP、停用企业/应用和协议异常分别分类。
|
||||||
|
- 不记录明文密码、完整 AuthenticatorSource 或平台配置密钥。
|
||||||
|
- 正常断线、心跳超时、最大连接数限制和服务重启恢复不计为恶意认证。
|
||||||
|
|
||||||
|
### 11.3 登录
|
||||||
|
|
||||||
|
- 账号锁定仍由现有账户安全逻辑负责,安全检测面板不替代账号锁定。
|
||||||
|
- 图形验证码错误、账号错误、密码错误和角色入口错误分别保存分类,但页面证据统一脱敏。
|
||||||
|
- 一个入口的登录失败不得清理另一个入口的有效会话。
|
||||||
|
|
||||||
|
## 12. 保留与隐私
|
||||||
|
|
||||||
|
- 原始检测事件默认在线保留 30 天,聚合告警、封禁记录和操作审计默认保留 180 天;最终期限在上线前由安全和运营确认。
|
||||||
|
- 清理使用小批量、可恢复任务;不得删除仍关联活动封禁、未完成处置或审计保留期内的数据。
|
||||||
|
- 页面和导出默认脱敏账号、路径参数、User-Agent 中的可识别信息。
|
||||||
|
- 第一版不提供原始日志全文导出。
|
||||||
|
|
||||||
|
## 13. 可用性与降级
|
||||||
|
|
||||||
|
- Fail2ban 不可用:页面显示检测源异常,已有告警仍可查看;相关来源不允许宣称“无攻击”。
|
||||||
|
- Collector 不可用:健康状态告警并记录事件积压;恢复后按事件键幂等补录。
|
||||||
|
- Security Agent 不可用:封禁按钮返回明确失败,不修改告警为已封禁。
|
||||||
|
- PostgreSQL 不可用:不允许执行无法审计的封禁操作。
|
||||||
|
- 规则应用失败:继续使用上一生效版本,页面显示失败版本与原因。
|
||||||
|
- Nginx 或 nftables 回读不一致:封禁状态标记异常并产生系统告警。
|
||||||
|
|
||||||
|
## 14. 部署前置条件
|
||||||
|
|
||||||
|
1. 将 `cmpp-api.service` 改为专用非 root 用户并完成文件、日志、MinIO/local storage 权限回归。
|
||||||
|
2. 安装 Fail2ban,固定版本并使用 nftables 兼容 action。
|
||||||
|
3. 为 Nginx、sshd、Gateway 和 NestJS 建立结构化、脱敏且可测试的事件格式。
|
||||||
|
4. 验证 Cloudflare 可信 IP 网段、`real_ip_header` 和源站绕过防护。
|
||||||
|
5. 建立 root security agent、Unix Socket 权限、systemd 加固和固定协议。
|
||||||
|
6. 发布前备份 PostgreSQL、运行源码、环境文件、Fail2ban/Nginx 平台生成配置和 nftables 当前规则。
|
||||||
|
7. 安装器必须将systemd `ExecStart`与Fail2ban report-only `actionban`从同一占位符渲染为实际构建产物`$APP_DIR/dist/cmpp-security-agent`,并在旧路径或未替换占位符残留时失败关闭。
|
||||||
|
|
||||||
|
## 15. 分阶段实施建议
|
||||||
|
|
||||||
|
### 阶段 A:检测与只读面板
|
||||||
|
|
||||||
|
- 数据模型、默认规则、结构化事件、collector、Fail2ban alert-only jail。
|
||||||
|
- 总览、告警列表、详情、规则只读展示和健康状态。
|
||||||
|
- 使用真实日志、真实 PostgreSQL 和真实接口验收。
|
||||||
|
|
||||||
|
### 阶段 B:阈值配置
|
||||||
|
|
||||||
|
- 规则编辑、版本、后端边界、受控配置编译、校验、原子应用和回滚。
|
||||||
|
- 配置变更对比、近期认证和审计。
|
||||||
|
|
||||||
|
### 阶段 C:人工封禁
|
||||||
|
|
||||||
|
- Security Agent、执行器、固定时长封禁、解封、保护名单、幂等和状态对账。
|
||||||
|
- Cloudflare/Nginx 与直连/nftables 分入口验收。
|
||||||
|
|
||||||
|
阶段 A、B、C 可以作为同一第一版需求连续交付,但验收必须逐阶段通过,不能为了展示按钮而跳过权限隔离和真实执行验证。
|
||||||
|
|
||||||
|
## 16. 第一版完成标准
|
||||||
|
|
||||||
|
- 九类检测全部有真实事件来源、默认规则、可配置阈值和原子测试。
|
||||||
|
- 自研 UI 通过真实 API 展示面板、告警、规则、封禁和健康数据。
|
||||||
|
- HTTP 错误密钥、签名错误、重放和恶意扫描均纳入第一版。
|
||||||
|
- NestJS 非 root,无法执行任意系统命令或编辑 Fail2ban 配置。
|
||||||
|
- 人工封禁按入口选择正确执行器,Cloudflare 场景不使用无效的访客 IP nftables 封禁。
|
||||||
|
- 保护名单、重新认证、权限、幂等、并发和操作审计全部通过。
|
||||||
|
- 只有回读执行器确认真实生效后,页面才显示“已封禁”。
|
||||||
|
- 不发送短信、不修改通道账号/密码/启停状态、企业余额或客户连接。
|
||||||
@@ -0,0 +1,175 @@
|
|||||||
|
# Fail2ban 安全检测与人工封禁第一版测试用例
|
||||||
|
|
||||||
|
> 版本:V1.0(设计评审用例)<br>
|
||||||
|
> 日期:2026-08-14<br>
|
||||||
|
> 关联设计:`docs/fail2ban-assisted-blocking-design-20260814.md`<br>
|
||||||
|
> 原则:真实后端、真实 PostgreSQL、真实日志与真实执行器;禁止 mock、静态数据和 localStorage 作为验收证据
|
||||||
|
|
||||||
|
## 1. 测试边界
|
||||||
|
|
||||||
|
- 自动化测试使用隔离的 Fail2ban/Nginx/nftables namespace 或测试节点,不得封禁测试执行机、预生产运维 IP、Cloudflare 节点或真实客户 IP。
|
||||||
|
- 预生产验收优先使用文档保留测试 IP 和短时封禁;所有人工封禁必须先确认回滚路径。
|
||||||
|
- 不发送、补发或重投短信,不修改通道账号、密码、启停状态、企业余额或客户连接。
|
||||||
|
- HTTP 密钥、签名、CMPP AuthenticatorSource、账号和日志证据必须脱敏。
|
||||||
|
|
||||||
|
## 2. 规则配置
|
||||||
|
|
||||||
|
| 编号 | 优先级 | 测试步骤 | 预期结果 |
|
||||||
|
| --- | --- | --- | --- |
|
||||||
|
| TC-F2B-RULE-001 | P0 | 查询规则列表 | 返回九类第一版规则;数据来自 PostgreSQL;展示当前配置版本和生效版本 |
|
||||||
|
| TC-F2B-RULE-002 | P0 | 修改规则阈值、窗口、冷却时间和风险等级并保存 | 写入真实数据库,生成新版本,代理校验并应用成功;回读值一致;写操作日志 |
|
||||||
|
| TC-F2B-RULE-003 | P0 | 提交小于最小值、大于最大值、零、负数、小数、非数字或非法枚举 | 后端逐项拒绝,不生成规则版本,不依赖前端校验兜底 |
|
||||||
|
| TC-F2B-RULE-004 | P0 | 两名管理员基于同一旧版本并发保存不同阈值 | 仅一方成功;另一方返回版本冲突,不覆盖新配置 |
|
||||||
|
| TC-F2B-RULE-005 | P0 | 让新配置语法校验或 reload 失败 | 数据库标记 `apply_failed`;上一生效版本继续工作;页面不得提示已生效 |
|
||||||
|
| TC-F2B-RULE-006 | P1 | 停用一条规则后持续产生匹配日志 | 停用后不创建新告警;已有告警和历史事件保留 |
|
||||||
|
| TC-F2B-RULE-007 | P1 | 重新启用规则 | 新事件按当前版本统计,不错误合并停用期间日志 |
|
||||||
|
| TC-F2B-RULE-008 | P0 | 尝试通过接口提交 regex、日志路径、shell、action、jail 名或执行器 | DTO 不接受或后端拒绝;配置文件和系统命令不受影响 |
|
||||||
|
| TC-F2B-RULE-009 | P0 | 无 `security.rule.manage` 权限修改规则 | 返回 403,不写数据库、不调用安全代理 |
|
||||||
|
| TC-F2B-RULE-010 | P0 | 超过近期认证时间后修改规则 | 要求重新认证;重新认证成功后才能保存 |
|
||||||
|
|
||||||
|
## 3. 登录与 SSH 检测
|
||||||
|
|
||||||
|
| 编号 | 优先级 | 测试步骤 | 预期结果 |
|
||||||
|
| --- | --- | --- | --- |
|
||||||
|
| TC-F2B-AUTH-001 | P0 | 同一测试 IP 在窗口内触发达到阈值的运营端密码错误 | 只生成一条聚合告警,计数准确,状态 `pending`,未自动封禁 |
|
||||||
|
| TC-F2B-AUTH-002 | P0 | 同一测试 IP 在客户端登录达到阈值 | 产生客户端规则告警,不与运营端规则错误合并 |
|
||||||
|
| TC-F2B-AUTH-003 | P1 | 失败次数低于阈值或分散在窗口外 | 保存检测事件但不产生达到阈值告警 |
|
||||||
|
| TC-F2B-AUTH-004 | P0 | 同一浏览器已有有效运营会话时,在客户端提交错误登录 | 产生正确检测事件;运营会话不被清除、广播退出或跳转 |
|
||||||
|
| TC-F2B-AUTH-005 | P0 | 检查事件、告警详情和日志 | 不含明文密码、密码散列、完整账号或验证码答案 |
|
||||||
|
| TC-F2B-SSH-001 | P0 | 从保留测试 IP 对 12022 触发达到阈值的 SSH 失败 | Fail2ban 检测 jail 产生事件和告警,但 nftables 未自动加入封禁 |
|
||||||
|
| TC-F2B-SSH-002 | P1 | SSH 登录成功、连接中断或握手超时 | 不错误计入认证失败规则 |
|
||||||
|
| TC-F2B-SSH-003 | P0 | 重启 Fail2ban/collector 后重放同一事件 | `eventKey` 幂等,不重复增加计数或创建告警 |
|
||||||
|
|
||||||
|
## 4. CMPP 检测
|
||||||
|
|
||||||
|
| 编号 | 优先级 | 测试步骤 | 预期结果 |
|
||||||
|
| --- | --- | --- | --- |
|
||||||
|
| TC-F2B-CMPP-001 | P0 | 同一测试 IP 使用未知账号达到阈值 | 生成 `cmpp_auth_failure` 告警;保存脱敏账号指纹和真实 TCP IP |
|
||||||
|
| TC-F2B-CMPP-002 | P0 | 使用错误 AuthenticatorSource 达到阈值 | 认证继续按原协议拒绝;告警分类准确;不保存完整 AuthenticatorSource |
|
||||||
|
| TC-F2B-CMPP-003 | P0 | 从不允许 IP、停用企业或停用应用发起连接 | 原认证结果不变;事件分类可区分真实拒绝原因 |
|
||||||
|
| TC-F2B-CMPP-004 | P0 | 高频发送非法包、异常版本或短连接扫描 | 达到协议滥用规则阈值后形成告警,不污染普通认证失败统计 |
|
||||||
|
| TC-F2B-CMPP-005 | P1 | 正常断线、心跳超时、最大连接数限制或服务重启恢复 | 不作为恶意认证或协议滥用告警 |
|
||||||
|
| TC-F2B-CMPP-006 | P0 | 检查事件和页面证据 | 不包含平台密码、完整认证材料、短信正文或通道凭据 |
|
||||||
|
|
||||||
|
## 5. HTTP API 检测
|
||||||
|
|
||||||
|
| 编号 | 优先级 | 测试步骤 | 预期结果 |
|
||||||
|
| --- | --- | --- | --- |
|
||||||
|
| TC-F2B-HTTP-001 | P0 | 同一 IP 使用不存在的 HTTP API 密钥达到阈值 | 生成 `http_invalid_api_key` 告警;仅保存不可逆密钥指纹 |
|
||||||
|
| TC-F2B-HTTP-002 | P0 | 使用存在但错误的签名达到阈值 | 生成 `http_signature_failure`,不与错误密钥混淆,不保存签名密钥或完整签名 Header |
|
||||||
|
| TC-F2B-HTTP-003 | P0 | 分别触发签名缺失、格式错误、算法不支持、验签失败和时间偏差 | 内部原因分类准确;页面使用安全文案;敏感细节不泄露给调用方 |
|
||||||
|
| TC-F2B-HTTP-004 | P0 | 重复使用已消费 nonce/签名请求达到阈值 | 生成 `http_replay_attempt` 严重告警;原接口仍按既有幂等/重放规则拒绝 |
|
||||||
|
| TC-F2B-HTTP-005 | P0 | 合法客户端按接口幂等协议重试同一业务请求 | 不误判为恶意重放;响应和业务幂等结果保持原语义 |
|
||||||
|
| TC-F2B-HTTP-006 | P0 | 请求常见敏感路径和漏洞路径,达到次数及不同路径双阈值 | 生成 `http_malicious_scan`;路径已标准化且 query 敏感值不保存 |
|
||||||
|
| TC-F2B-HTTP-007 | P0 | 对同一不存在业务路径重复请求,仅满足次数、不满足不同路径数 | 不触发恶意扫描告警 |
|
||||||
|
| TC-F2B-HTTP-008 | P1 | 产生普通业务 400、字段校验失败、合法 401/403 和单次 404 | 不错误计入恶意扫描或错误密钥规则 |
|
||||||
|
| TC-F2B-HTTP-009 | P0 | 从两个 IP 分别达到一半阈值 | 不跨 IP 错误聚合;每个 IP 独立计算 |
|
||||||
|
| TC-F2B-HTTP-010 | P0 | 修改 HTTP 规则阈值后继续产生事件 | 新事件使用新生效版本,告警规则快照可追溯当时阈值 |
|
||||||
|
|
||||||
|
## 6. Cloudflare 与真实 IP
|
||||||
|
|
||||||
|
| 编号 | 优先级 | 测试步骤 | 预期结果 |
|
||||||
|
| --- | --- | --- | --- |
|
||||||
|
| TC-F2B-IP-001 | P0 | 经真实 Cloudflare 回源请求运营端和客户端 | 事件 IP 等于访客真实 IP,TCP 代理 IP 保留在安全元数据中但不作为攻击者 IP |
|
||||||
|
| TC-F2B-IP-002 | P0 | 直连源站并伪造 `CF-Connecting-IP` | Header 被忽略,事件使用真实 TCP 来源 IP |
|
||||||
|
| TC-F2B-IP-003 | P0 | 人工封禁 Cloudflare 入口告警 | 使用 Nginx real-IP deny,不调用访客 IP nftables action |
|
||||||
|
| TC-F2B-IP-004 | P0 | 人工封禁灰云 API、SSH 或 CMPP 告警 | 使用 nftables/manual jail,不修改 Nginx Cloudflare deny 列表 |
|
||||||
|
| TC-F2B-IP-005 | P0 | 尝试封禁 Cloudflare 官方节点、源站自身、回环、内网、运维和健康检查 IP | 后端和安全代理双重拒绝,记录保护名单命中,不产生真实封禁 |
|
||||||
|
| TC-F2B-IP-006 | P1 | Cloudflare 官方网段更新 | 使用受控来源更新并审计;旧配置切换原子化;失败保留旧可信网段 |
|
||||||
|
|
||||||
|
## 7. 告警聚合与状态机
|
||||||
|
|
||||||
|
| 编号 | 优先级 | 测试步骤 | 预期结果 |
|
||||||
|
| --- | --- | --- | --- |
|
||||||
|
| TC-F2B-ALERT-001 | P0 | 同规则、IP、窗口内并发写入多条事件 | 唯一 fingerprint 生效,只创建一条告警,计数无丢失 |
|
||||||
|
| TC-F2B-ALERT-002 | P0 | 冷却时间内再次达到阈值 | 更新原告警计数和最近时间,不产生告警风暴 |
|
||||||
|
| TC-F2B-ALERT-003 | P1 | 冷却结束后再次达到阈值 | 按设计创建新告警或新周期,历史告警不覆盖 |
|
||||||
|
| TC-F2B-ALERT-004 | P0 | 对 `pending` 告警执行忽略 | 状态原子变为 `ignored`,保留事件和原因,封禁按钮不可再执行 |
|
||||||
|
| TC-F2B-ALERT-005 | P0 | 两名管理员同时对同一告警点击封禁和忽略 | 仅一个状态迁移成功,另一请求返回 409 |
|
||||||
|
| TC-F2B-ALERT-006 | P1 | 告警超过可处置期限 | 状态变为 `expired`;历史仍可查询;不能用旧告警封禁 |
|
||||||
|
|
||||||
|
## 8. 人工封禁与解封
|
||||||
|
|
||||||
|
| 编号 | 优先级 | 测试步骤 | 预期结果 |
|
||||||
|
| --- | --- | --- | --- |
|
||||||
|
| TC-F2B-BLOCK-001 | P0 | 有权限管理员对待处理告警选择1小时、填写原因并确认 | 创建唯一操作记录,代理执行正确执行器,回读成功后状态才为 `blocked` |
|
||||||
|
| TC-F2B-BLOCK-002 | P0 | 前端篡改 IP、jail、action、执行器或时长 | 后端忽略未定义字段或拒绝请求;实际值只来自告警和固定映射 |
|
||||||
|
| TC-F2B-BLOCK-003 | P0 | 重复点击、网络超时重试或同一 `operationKey` 重放 | 代理和数据库幂等,仅存在一条真实封禁,不延长原期限 |
|
||||||
|
| TC-F2B-BLOCK-004 | P0 | 代理返回成功但执行器回读不存在规则 | 不标记 `blocked`;状态为失败/异常并生成系统告警 |
|
||||||
|
| TC-F2B-BLOCK-005 | P0 | Security Agent 停止或 Socket 不可用 | 明确返回封禁失败;告警不伪装已封禁;记录错误和审计 |
|
||||||
|
| TC-F2B-BLOCK-006 | P0 | PostgreSQL 不可用时点击封禁 | 拒绝无法审计的操作,不调用代理 |
|
||||||
|
| TC-F2B-BLOCK-007 | P0 | 无 `security.block.manage` 权限或近期认证过期 | 返回403或要求重新认证,不产生操作记录和系统封禁 |
|
||||||
|
| TC-F2B-BLOCK-008 | P0 | 对已封禁 IP执行解封并填写原因 | 调用原执行器解除,回读确认后变为 `unblocked`,完整记录操作人和原因 |
|
||||||
|
| TC-F2B-BLOCK-009 | P0 | 封禁自然到期 | 执行器真实解除;对账任务回读后更新 `expired`,不只依赖数据库时间 |
|
||||||
|
| TC-F2B-BLOCK-010 | P0 | 到期后执行器仍存在规则或平台记录与执行器不一致 | 标记同步异常并告警,不静默显示已解封 |
|
||||||
|
| TC-F2B-BLOCK-011 | P1 | 规则配置 `manualBlockAllowed=false` | 告警可查看,封禁按钮禁用,直接调用接口同样拒绝 |
|
||||||
|
|
||||||
|
## 9. 保护名单
|
||||||
|
|
||||||
|
| 编号 | 优先级 | 测试步骤 | 预期结果 |
|
||||||
|
| --- | --- | --- | --- |
|
||||||
|
| TC-F2B-PROTECT-001 | P0 | 查询保护名单 | 返回真实数据库和系统内置项,展示类型、范围和来源 |
|
||||||
|
| TC-F2B-PROTECT-002 | P0 | 新增合法 IPv4、IPv6 或 CIDR 人工保护项 | 后端规范化、检查重叠并保存;要求权限、重新认证、原因和审计 |
|
||||||
|
| TC-F2B-PROTECT-003 | P0 | 提交非法、过宽、重复或与系统项冲突的网段 | 后端拒绝,现有保护项不变化 |
|
||||||
|
| TC-F2B-PROTECT-004 | P0 | 尝试删除系统内置保护项 | 拒绝;只能通过受控发布变更 |
|
||||||
|
| TC-F2B-PROTECT-005 | P0 | 待处理告警 IP 后续加入保护名单 | 告警更新为 `whitelisted` 或明确标记保护命中,不允许封禁 |
|
||||||
|
|
||||||
|
## 10. 自研 UI 与真实数据
|
||||||
|
|
||||||
|
| 编号 | 优先级 | 测试步骤 | 预期结果 |
|
||||||
|
| --- | --- | --- | --- |
|
||||||
|
| TC-F2B-UI-001 | P0 | 进入“安全控制 / 安全检测” | 页面使用平台自研布局、组件和中文状态,不嵌入第三方 Fail2ban UI |
|
||||||
|
| TC-F2B-UI-002 | P0 | 对照 SQL/API 检查总览五项指标、趋势、分布和 Top IP | 页面与真实后端结果一致,无 mock、静态或 localStorage 数据 |
|
||||||
|
| TC-F2B-UI-003 | P0 | 使用 IP、规则、风险、状态、入口和日期组合筛选及分页 | 查询由后端完成;总数、页码、跨页结果准确 |
|
||||||
|
| TC-F2B-UI-004 | P0 | 打开告警详情 | 显示规则快照、聚合时间线、脱敏证据和处置历史;无密钥或认证材料泄露 |
|
||||||
|
| TC-F2B-UI-005 | P0 | 打开封禁确认弹窗 | 显示执行器、影响入口、固定时长、保护检查、风险提示和必填原因 |
|
||||||
|
| TC-F2B-UI-006 | P0 | 规则保存成功、保存但应用失败、版本冲突 | 三种状态分别准确提示;失败不得显示成功 toast |
|
||||||
|
| TC-F2B-UI-007 | P1 | 1440、1280、1024、768和375宽度验收 | 指标、图表、表格、筛选和弹窗无横向溢出;关键操作可见可用 |
|
||||||
|
| TC-F2B-UI-008 | P1 | 仅键盘和读屏操作页面 | 表单有 label,错误有关联说明,状态不只依赖颜色,弹窗焦点和按钮名称准确 |
|
||||||
|
| TC-F2B-UI-009 | P0 | 后端、collector 或 agent 不可用 | 页面显示明确降级和最后成功时间,不用空数据伪装“零攻击” |
|
||||||
|
|
||||||
|
## 11. 权限、进程与系统加固
|
||||||
|
|
||||||
|
| 编号 | 优先级 | 测试步骤 | 预期结果 |
|
||||||
|
| --- | --- | --- | --- |
|
||||||
|
| TC-F2B-SEC-001 | P0 | 检查 `cmpp-api.service` 和运行进程 | NestJS 使用专用非 root 用户;有效 UID/GID 非0 |
|
||||||
|
| TC-F2B-SEC-002 | P0 | 检查 NestJS capabilities、sudoers、可写目录 | 无通用 sudo、无 `CAP_NET_ADMIN`/`CAP_SYS_ADMIN`;只能写明确业务和日志目录 |
|
||||||
|
| TC-F2B-SEC-003 | P0 | 以 NestJS 用户尝试读取/写入 `/etc/fail2ban`、运行防火墙命令或访问代理管理文件 | 全部被操作系统权限拒绝 |
|
||||||
|
| TC-F2B-SEC-004 | P0 | 检查 Security Agent Socket | 只在本机 Unix Socket,权限和用户组符合设计,无 TCP 监听 |
|
||||||
|
| TC-F2B-SEC-005 | P0 | 向 Agent 提交未知动作、非法 JSON、超长字段、shell 字符和非白名单规则 | 全部拒绝且无命令执行;记录限量安全日志 |
|
||||||
|
| TC-F2B-SEC-006 | P0 | 检查操作日志 | 规则变更、封禁、解封、忽略、保护名单、失败和回读异常均可追溯 |
|
||||||
|
| TC-F2B-SEC-007 | P1 | 查看告警详情和完整证据 | 读取动作按设计写访问审计;低权限用户只能看到脱敏数据 |
|
||||||
|
|
||||||
|
## 12. 可用性、恢复和数据保留
|
||||||
|
|
||||||
|
| 编号 | 优先级 | 测试步骤 | 预期结果 |
|
||||||
|
| --- | --- | --- | --- |
|
||||||
|
| TC-F2B-OPS-001 | P0 | 停止 Fail2ban | 健康面板显示对应来源异常;已有告警可查;不得显示“当前无攻击” |
|
||||||
|
| TC-F2B-OPS-002 | P0 | Collector 暂停后恢复并重放积压 | 事件按幂等键补录,计数准确,无重复告警 |
|
||||||
|
| TC-F2B-OPS-003 | P0 | 重启 API、Gateway、Fail2ban、collector 和 agent | 已生效规则、活动封禁和告警状态恢复一致,不自动执行新封禁 |
|
||||||
|
| TC-F2B-OPS-004 | P0 | 发布失败触发回滚 | PostgreSQL、代码、环境、Fail2ban/Nginx生成配置和防火墙规则均有可验证恢复路径 |
|
||||||
|
| TC-F2B-OPS-005 | P1 | 执行30天事件、180天告警/审计保留任务 | 只删除到期且不受保护数据;活动封禁、未完成处置和审计期数据不删除 |
|
||||||
|
| TC-F2B-OPS-006 | P1 | 大量扫描事件压测 | Collector 有界处理,API与Gateway业务不被阻塞;告警聚合避免写放大 |
|
||||||
|
| TC-F2B-OPS-007 | P0 | 对比执行器、数据库和页面 | 当前封禁集合、到期时间和执行器类型一致;差异进入异常状态和告警 |
|
||||||
|
| TC-F2B-OPS-008 | P0 | 在非默认`APP_DIR`构建并运行安全安装器,检查systemd与Fail2ban action的执行路径 | 两者均指向同一个真实可执行的`$APP_DIR/dist/cmpp-security-agent`;残留占位符、旧`current/bin`路径或不可执行目标时安装失败 |
|
||||||
|
|
||||||
|
## 13. 发布验收证据
|
||||||
|
|
||||||
|
第一版发布前至少保留:
|
||||||
|
|
||||||
|
1. Fail2ban filter 单元样例,包含命中与不命中日志。
|
||||||
|
2. 九类规则的专项自动化结果。
|
||||||
|
3. API 权限、参数边界、并发、幂等和审计测试。
|
||||||
|
4. PostgreSQL migration 状态和表/索引/唯一约束核对。
|
||||||
|
5. NestJS 非 root、capabilities、sudoers 和文件权限证据。
|
||||||
|
6. Cloudflare 真实 IP、伪造 Header 和 Nginx 执行器真实验证。
|
||||||
|
7. 直连 API/SSH/CMPP 的隔离测试 IP nftables 封禁与解封证据。
|
||||||
|
8. Fail2ban、collector、agent、API、Gateway、Nginx、PostgreSQL 和 Redis 健康证据。
|
||||||
|
9. 自研 UI 桌面端、平板端和移动端截图以及控制台日志。
|
||||||
|
10. 发布前 PostgreSQL、运行源码、环境文件、生成配置和防火墙规则恢复资产校验。
|
||||||
|
|
||||||
|
## 14. 判定规则
|
||||||
|
|
||||||
|
- 任一 P0 失败:第一版不得发布。
|
||||||
|
- 检测达到阈值但未生成告警、未达到阈值却告警、错误 IP 聚合、密钥泄露、保护地址可被封禁、页面显示封禁而执行器未生效、NestJS 仍为 root或可执行任意系统命令,均按 P0 处理。
|
||||||
|
- 只验证 Fail2ban 命令输出、只验证前端样式或只写数据库不验证真实执行器,不能作为功能通过。
|
||||||
@@ -1390,6 +1390,7 @@
|
|||||||
- 数据统计的“通道占比”默认统计北京时间当天,可选择单个历史日期重新查询;图表使用真实通道名称和该日期的通道提交量。
|
- 数据统计的“通道占比”默认统计北京时间当天,可选择单个历史日期重新查询;图表使用真实通道名称和该日期的通道提交量。
|
||||||
- 输出 500 条/秒压测报告。
|
- 输出 500 条/秒压测报告。
|
||||||
- 输出 Linux 部署方案,至少覆盖 Docker Compose 或 systemd 部署、环境变量、数据库迁移、日志目录、备份恢复、服务健康检查和回滚步骤。
|
- 输出 Linux 部署方案,至少覆盖 Docker Compose 或 systemd 部署、环境变量、数据库迁移、日志目录、备份恢复、服务健康检查和回滚步骤。
|
||||||
|
- 标准发布脚本配置Nginx压缩时必须兼容发行版已有的HTTP级`gzip on`:先排除自身生成文件检查现有有效配置,已有时复用,没有时才创建平台级配置;重复发布不得因重复指令使`nginx -t`失败。
|
||||||
|
|
||||||
## 16. 新 Codex 会话提示词
|
## 16. 新 Codex 会话提示词
|
||||||
|
|
||||||
@@ -2025,13 +2026,32 @@
|
|||||||
- 运营端“短信通道组管理”在通道组名称条件之外增加“通道”筛选。选定一个通道后,只展示成员配置中真实包含该`channelId`的未删除通道组;与通道组名称同时输入时按两个条件取交集。
|
- 运营端“短信通道组管理”在通道组名称条件之外增加“通道”筛选。选定一个通道后,只展示成员配置中真实包含该`channelId`的未删除通道组;与通道组名称同时输入时按两个条件取交集。
|
||||||
- 通道选项必须来自真实通道API,不使用静态列表、Mock或localStorage。页面首次加载时通道组与通道两个独立请求并行执行;选项同时展示通道名称和编码,已删除通道明确标记“已删除”。未加入任何通道组的真实通道仍可选择,选中后结果为零而不得隐藏该选项。
|
- 通道选项必须来自真实通道API,不使用静态列表、Mock或localStorage。页面首次加载时通道组与通道两个独立请求并行执行;选项同时展示通道名称和编码,已删除通道明确标记“已删除”。未加入任何通道组的真实通道仍可选择,选中后结果为零而不得隐藏该选项。
|
||||||
- 通道选择控件必须为通用下拉可搜索控件,支持按通道名称或编码搜索;“全部通道”表示不按通道限制,点击“重置”必须同时清空通道组名称和通道条件并回到第一页。
|
- 通道选择控件必须为通用下拉可搜索控件,支持按通道名称或编码搜索;“全部通道”表示不按通道限制,点击“重置”必须同时清空通道组名称和通道条件并回到第一页。
|
||||||
|
|
||||||
|
## Prometheus 系统监控(2026-08-14)
|
||||||
|
|
||||||
|
- 运营端“系统管理”新增“系统监控”,路由为`/admin/system-monitoring`;原“发送监控”继续负责短信通道和消息业务指标,两个页面、接口和统计口径不得混用。
|
||||||
|
- 系统监控使用Prometheus和Node Exporter作为真实指标基础设施,但全部用户界面由平台React原生实现,不嵌入Grafana、Netdata、Zabbix、Prometheus页面或第三方登录界面。
|
||||||
|
- 浏览器只请求`GET /api/admin/infrastructure-monitoring/overview?range=1h|24h|7d`。NestJS以固定PromQL模板查询Prometheus,禁止前端传任意PromQL、step、时间戳或标签选择器;9090和9100只监听本机或内网,不向公网开放。
|
||||||
|
- 第一版展示CPU、内存、根文件系统、网络收发、1分钟负载和系统运行时长,并展示API、Gateway、PostgreSQL、Redis、MinIO、Nginx六类systemd服务状态。指标缺失必须显示“暂无指标/未知”,不得用0、静态数据、Mock或localStorage冒充真实采集值。
|
||||||
|
- 支持近1小时、近24小时和近7天固定范围,步长分别为60秒、300秒和1800秒。页面可见时每30秒刷新,隐藏或卸载后停止;手动刷新保留当前范围。
|
||||||
|
- 页面展示Prometheus当前firing/pending告警,严重性使用`info/warning/critical`。告警阈值由Prometheus规则计算,前端不重复判断;第一版仅只读展示,不提供缺少审计模型的确认、备注、静默或关闭操作。
|
||||||
|
- Prometheus不可用、查询超时、响应非法时接口返回`available=false`及安全错误摘要,页面清空陈旧指标并展示监控不可用;不能继续显示上一次数据造成误判。
|
||||||
|
- 固定PromQL必须按Prometheus字符串与RE2正则两层语义正确转义,并由专项契约锁定systemd单元正则;任一查询导致整页降级时,API必须记录不含PromQL、地址或凭据的安全错误摘要,不能只向页面返回笼统不可用而没有服务端诊断证据。
|
||||||
|
- 系统监控页只保留平台通用页头中的页面名称;内容区不得再次显示大号“系统监控”标题,可保留“服务器资源、核心服务与活动告警”说明、综合状态和时间范围操作。
|
||||||
|
- 完整采集、查询、安全、响应契约、视觉规格和验收口径见`docs/prometheus-system-monitoring-design-20260814.md`。
|
||||||
|
|
||||||
|
## 全局预警通知菜单(2026-08-14)
|
||||||
|
|
||||||
|
- 运营端右上角铃铛统一作为预警入口,点击后必须分开显示“签名清退预警”和“安全检测与封禁”两个菜单项;审核待办继续使用独立审核图标和菜单,不得把审核数与预警数混合。
|
||||||
|
- 签名清退项展示今日未读且未抑制的真实消息数并跳转`/admin/signature-retirement`;安全检测项展示`open/acknowledged/block_failed`真实待处置告警总数、严重告警摘要并跳转`/admin/security-detection`。
|
||||||
|
- 铃铛角标为两个预警域数量之和;任一域接口失败时只把该域降级为0,不能影响另一域或审核待办。全局轮询使用专用轻量汇总接口,不得每30秒调用安全检测完整总览、代理回读或大列表。
|
||||||
# 下游投递后台重投任务(2026-08-12)
|
# 下游投递后台重投任务(2026-08-12)
|
||||||
|
|
||||||
## 完整设计口径与安全整改(2026-08-13)
|
## 完整设计口径与安全整改(2026-08-13)
|
||||||
|
|
||||||
1. 后台任务按企业、应用、投递类型、状态、北京时间创建日期和关键词冻结筛选快照,分页、每页条数和当前页勾选不属于范围。预检的命中数、可重投数、规则跳过数、状态分布、涉及应用和最早记录必须严格基于当前筛选条件,不得把单一状态擅自扩为全部状态。
|
1. 后台任务按企业、应用、投递类型、状态、北京时间创建日期和关键词冻结筛选快照,分页、每页条数和当前页勾选不属于范围。预检的命中数、可重投数、规则跳过数、状态分布、涉及应用和最早记录必须严格基于当前筛选条件,不得把单一状态擅自扩为全部状态。
|
||||||
2. 预检返回短期有效且绑定当前操作人、筛选条件和 `snapshotAt` 的服务端签名凭证;创建接口只接受该凭证、原因、速度和安全阈值,不再信任前端重传的范围。创建时后端按签名快照重新物化真实 PostgreSQL 任务项。
|
2. 预检返回短期有效且绑定当前操作人、筛选条件和 `snapshotAt` 的服务端签名凭证;创建接口只接受该凭证、原因、速度和安全阈值,不再信任前端重传的范围。创建时后端按签名快照重新物化真实 PostgreSQL 任务项。
|
||||||
3. 第一版后台任务仅处理 `pending/failed/unconfirmed/rejected`;`delivered/awaiting_ack` 不得进入批量任务。跳过严格表示本任务没有调用 Gateway;客户离线进入 `waiting_connection`,其他链路等待 ACK 进入 `waiting_external_ack`,本任务写出后进入 `waiting_ack`,三类等待均不计失败或跳过。
|
3. 后台任务允许处理 `pending/failed/unconfirmed/rejected/delivered`;客户端已确认的 `delivered` 必须在预检、创建弹窗和任务项原状态中明确呈现,运营确认后可以再次投递,但页面必须提示可能造成客户端重复处理。`awaiting_ack` 不得进入批量任务。若任务项创建时不是 `delivered`、执行前才收到成功 ACK,则本任务必须跳过,不能把迟到确认变成未授权重复投递。跳过严格表示本任务没有调用 Gateway;客户离线进入 `waiting_connection`,其他链路等待 ACK 进入 `waiting_external_ack`,本任务写出后进入 `waiting_ack`,三类等待均不计失败或跳过。
|
||||||
4. 限速以企业应用为维度,使用数据库原子秒级窗口在多实例、扫描重叠和执行耗时变化下保持每应用不超过配置速度。任务扫描使用数据库租约;`processing` 项使用认领租约,API 中断后过期恢复为 `queued` 并重新复核,已成功 ACK 的项目不得重放。
|
4. 限速以企业应用为维度,使用数据库原子秒级窗口在多实例、扫描重叠和执行耗时变化下保持每应用不超过配置速度。任务扫描使用数据库租约;`processing` 项使用认领租约,API 中断后过期恢复为 `queued` 并重新复核,已成功 ACK 的项目不得重放。
|
||||||
5. 连续失败按应用隔离统计。Gateway 立即失败、ACK 超时、ACK 拒绝和无法安全关联均计入阈值;只有有效 ACK 成功才清零。达到阈值前原子暂停整个任务,记录触发应用、失败数、阈值和暂停时间,人工继续后从未完成项恢复。
|
5. 连续失败按应用隔离统计。Gateway 立即失败、ACK 超时、ACK 拒绝和无法安全关联均计入阈值;只有有效 ACK 成功才清零。达到阈值前原子暂停整个任务,记录触发应用、失败数、阈值和暂停时间,人工继续后从未完成项恢复。
|
||||||
6. 任务列表支持状态筛选和真实分页,展示任务号、创建时间、企业/应用、原因、中文状态、总数、成功、失败、跳过、等待、创建人及进度。任务详情展示筛选快照、时间、安全参数、完整结果汇总和任务项分页;任务项可按中文结果、消息 ID、错误或跳过原因查询,不得仅返回最近 50 项。
|
6. 任务列表支持状态筛选和真实分页,展示任务号、创建时间、企业/应用、原因、中文状态、总数、成功、失败、跳过、等待、创建人及进度。任务详情展示筛选快照、时间、安全参数、完整结果汇总和任务项分页;任务项可按中文结果、消息 ID、错误或跳过原因查询,不得仅返回最近 50 项。
|
||||||
@@ -2040,9 +2060,9 @@
|
|||||||
|
|
||||||
1. 运营端“下游投递记录”必须同时保留单条重投、当前页勾选批量重投,并新增“按筛选条件重投”;分页支持每页 `10/25/50` 条,切换后回到第一页并重新查询真实后端。
|
1. 运营端“下游投递记录”必须同时保留单条重投、当前页勾选批量重投,并新增“按筛选条件重投”;分页支持每页 `10/25/50` 条,切换后回到第一页并重新查询真实后端。
|
||||||
2. 后台任务使用当前企业、应用、投递类型、状态、创建日期和关键词的后端筛选快照,分页不属于任务范围;任务创建时固定 `snapshotAt`,之后产生的记录不得被卷入。
|
2. 后台任务使用当前企业、应用、投递类型、状态、创建日期和关键词的后端筛选快照,分页不属于任务范围;任务创建时固定 `snapshotAt`,之后产生的记录不得被卷入。
|
||||||
3. 第一版只允许 `pending/failed/unconfirmed/rejected`,不支持批量重投客户端已确认的 `delivered`,`awaiting_ack` 不得并发重投。创建前必须真实预检命中、可重投、跳过和状态分布,原因必填。
|
3. 批量任务允许 `pending/failed/unconfirmed/rejected/delivered`,其中 `delivered` 会再次发送并可能造成客户端重复处理,创建弹窗必须醒目提示;`awaiting_ack` 不得并发重投。创建前必须真实预检命中、可重投、跳过和状态分布,原因必填。执行器仅允许重投在冻结快照中原状态已经是 `delivered` 的已确认任务项;任务建立后才变成 `delivered` 的项目必须跳过。
|
||||||
4. 任务按应用分批执行,默认每秒 10 条;单条失败不阻断整批,连续失败达到 10 条或 ACK 超时/拒绝达到安全阈值时自动暂停。客户离线、等待 ACK 属于等待状态,不得误记为跳过。
|
4. 任务按应用分批执行,默认每秒 10 条;单条失败不阻断整批,连续失败达到 10 条或 ACK 超时/拒绝达到安全阈值时自动暂停。客户离线、等待 ACK 属于等待状态,不得误记为跳过。
|
||||||
5. 跳过只表示未调用 Gateway,第一版原因包括:状态已变化、已被客户确认、已被其他任务处理、本任务已成功处理、不属于任务快照、应用或投递能力已停用、投递数据不完整、缺少原 Submit 映射。
|
5. 跳过只表示未调用 Gateway,原因包括:状态已变化、创建任务后才被客户确认、已被其他任务处理、本任务已成功处理、不属于任务快照、应用或投递能力已停用、投递数据不完整、缺少原 Submit 映射。
|
||||||
6. 任务必须支持列表、详情、暂停、继续和终止;终止只影响尚未发送的记录。任务项以 `taskId + deliveryId` 幂等,执行前原子认领并复核状态,API 重启后可继续,成功 ACK 的项目不得再次发送。
|
6. 任务必须支持列表、详情、暂停、继续和终止;终止只影响尚未发送的记录。任务项以 `taskId + deliveryId` 幂等,执行前原子认领并复核状态,API 重启后可继续,成功 ACK 的项目不得再次发送。
|
||||||
7. 所有创建、暂停、继续、终止和自动暂停均写操作日志;任务使用真实 PostgreSQL、Gateway 和客户 ACK,不得使用 mock、静态数据或 localStorage。
|
7. 所有创建、暂停、继续、终止和自动暂停均写操作日志;任务使用真实 PostgreSQL、Gateway 和客户 ACK,不得使用 mock、静态数据或 localStorage。
|
||||||
# 2026-08-13 HTTP 请求与 Gateway API 响应容量边界
|
# 2026-08-13 HTTP 请求与 Gateway API 响应容量边界
|
||||||
@@ -2059,3 +2079,25 @@
|
|||||||
2. 运营看板删除“今日签名发送统计”和“今日签名发送统计 - 含引流”两个明细模块;“今日活跃签名”指标仍使用当天真实发送聚合。今日消费金额的主数字必须与今日发送总量使用相同字号、字重和深色层级。
|
2. 运营看板删除“今日签名发送统计”和“今日签名发送统计 - 含引流”两个明细模块;“今日活跃签名”指标仍使用当天真实发送聚合。今日消费金额的主数字必须与今日发送总量使用相同字号、字重和深色层级。
|
||||||
3. 企业应用管理列表的状态、到达率、单价列在现有基础上缩窄约20%,提升大屏一次展示完整表格的概率;不得通过隐藏真实字段实现。
|
3. 企业应用管理列表的状态、到达率、单价列在现有基础上缩窄约20%,提升大屏一次展示完整表格的概率;不得通过隐藏真实字段实现。
|
||||||
4. 移动、联通、电信数据展示统一使用全局`CarrierTag`低饱和标签,包括通道与通道组、监控、报备、签名、短信审核、任务号码、发送记录及客户端发送详情等页面。筛选/表单控件的选项文案、图表图例、导出文本和业务说明仍使用纯文本,避免破坏交互、可访问性和机器可读输出;可取得运营商集合的三网通道按三个运营商标签展示,只有历史通道级字段时使用中性的“三网”标签。
|
4. 移动、联通、电信数据展示统一使用全局`CarrierTag`低饱和标签,包括通道与通道组、监控、报备、签名、短信审核、任务号码、发送记录及客户端发送详情等页面。筛选/表单控件的选项文案、图表图例、导出文本和业务说明仍使用纯文本,避免破坏交互、可访问性和机器可读输出;可取得运营商集合的三网通道按三个运营商标签展示,只有历史通道级字段时使用中性的“三网”标签。
|
||||||
|
## Fail2ban 安全检测与人工封禁(2026-08-14)
|
||||||
|
|
||||||
|
- 运营端“安全控制”新增“安全检测与封禁”自研页面,数据必须来自真实 NestJS API 与 PostgreSQL,包含总览、告警中心、九类规则配置、人工封禁记录和保护名单;不得嵌入第三方面板或使用 Mock、静态数据、localStorage 伪造检测状态。
|
||||||
|
- 第一版检测运营端登录失败、客户端登录失败、SSH 认证失败、CMPP 认证失败、CMPP 协议滥用、HTTP 错误密钥、HTTP 签名错误、HTTP 重放及 HTTP 恶意扫描。规则阈值、窗口、冷却、风险级别、启停状态与封禁时长边界保存在数据库,并使用配置版本防止并发覆盖。
|
||||||
|
- 检测只产生事件与人工告警,绝不自动封禁。人工封禁只允许 10 分钟、1 小时、24 小时或 7 天;操作要求近期重新认证、必填原因、保护名单校验、数据库原子认领和操作审计。执行器由服务端按可信入口固定映射,浏览器不得提交 jail、action、shell 参数或自选执行器。
|
||||||
|
- NestJS 必须以专用非 root 用户运行,不得获得通用 sudo、任意 shell、直接编辑 `/etc/fail2ban/*` 或防火墙的能力。独立 root security agent 只监听本机 Unix Socket、接受固定 JSON 动作、使用参数数组执行固定操作,并在真实 nftables 或 Nginx deny 回读成功后才允许数据库标记 `blocked`。
|
||||||
|
- 运营端/客户端的 Cloudflare 入口使用 Nginx real-IP deny;直连 HTTP API、SSH 与 CMPP 使用 nftables。只有可信代理 TCP 来源可以提供访客 IP;系统回环、私网、链路本地、组播和配置的运维/健康检查网段必须内置保护。
|
||||||
|
- 规则更新先保存待应用版本,由安全代理生成固定 Fail2ban 配置、执行语法校验并 reload;失败保留上一生效值并展示失败原因,不得显示为已生效。完整架构、状态机、字段、接口和安全边界以 `docs/fail2ban-assisted-blocking-design-20260814.md` 为准。
|
||||||
|
- 安全代理的systemd单元与Fail2ban report-only action必须由安装器从同一受控占位符渲染,并共同指向发布脚本实际生成的`$APP_DIR/dist/cmpp-security-agent`;安装后残留占位符、旧`current/bin`路径或不可执行目标时必须终止发布,禁止出现Agent服务可启动但Fail2ban action静默失效的分叉配置。
|
||||||
|
- NestJS API生产进程必须显式绑定`127.0.0.1`,仅由Nginx受控入口反向代理;不得依赖框架默认的全网卡监听而把3000端口直接暴露到LAN、Tailscale或公网。非生产环境如确需其他地址,只能通过明确的`API_HOST`配置覆盖。
|
||||||
|
|
||||||
|
## 服务内部指标与阈值(2026-08-14)
|
||||||
|
|
||||||
|
- API、Gateway必须以回环端点暴露低基数运行指标;PostgreSQL、Redis、Nginx使用发行版Exporter,MinIO使用原生指标。任何指标端口都不得经Nginx或安全组对公网暴露。
|
||||||
|
- 运营端展示API请求/错误/延迟/事件循环、Gateway Submit/队列/连接、PostgreSQL连接/死锁、Redis内存/连接/淘汰、Nginx连接/请求和MinIO可用性;指标缺失显示“待采集”,不以0伪装。
|
||||||
|
- 告警必须使用持续窗口和最低样本量;默认阈值、收敛关系、标签禁止项和性能预算以`docs/prometheus-system-monitoring-design-20260814.md`第9节为准。
|
||||||
|
- 指标不得包含手机号、短信正文、短信/CMPP/任务ID、密钥、完整URL或SQL原文;不得把时序指标高频写入业务PostgreSQL。
|
||||||
|
- 系统监控标题说明必须明确标注数据来自 Prometheus;“服务关键指标”位于趋势/核心服务区域之后、活动告警之前,并提供统一的“告警阈值设置”入口。安全检测页不重复渲染大号标题,说明文字必须明确标注使用 Fail2ban。
|
||||||
|
- 告警阈值仅开放固定指标的警告/严重数值,不允许前端提交 PromQL、标签、文件路径或持续时间;必须满足警告值小于严重值。配置以 PostgreSQL 保存版本、期望值、生效值和应用状态,经 `promtool check rules` 校验、同目录原子替换和 Prometheus 热加载成功后才标记生效,失败保留上一生效规则并展示原因。
|
||||||
|
- 右上角预警中心增加“系统监控告警”,通过独立轻量接口统计 Prometheus 当前 firing/pending 告警及严重数,跳转系统监控活动告警区;任一预警域失败不得清空其他域。
|
||||||
|
- 活动告警列表必须提供逐条“标记已读”。已读状态按管理员和“告警指纹 + 本次 activeAt”持久化到 PostgreSQL;仅从当前管理员的预警中心数量中扣减,不改变 Prometheus firing/pending 状态,也不减少页面活动告警总数。同标签告警恢复后再次触发时必须重新成为未读。
|
||||||
|
- 服务端只能确认 Prometheus 当前仍存在且 activeAt 一致的告警,过期、已恢复或已重新触发的请求必须拒绝;重复点击同一次告警应幂等,并写操作日志。阈值设置弹窗只保留通用 Modal 外层滚动,不得嵌套第二个独立滚动区域。
|
||||||
|
|||||||
@@ -30,6 +30,9 @@ REPO_URL=http://175.27.255.91:3000/hectorzhao/lislgosms.git
|
|||||||
BRANCH=main
|
BRANCH=main
|
||||||
PUBLIC_HTTP_PORT=12026
|
PUBLIC_HTTP_PORT=12026
|
||||||
API_PORT=3000
|
API_PORT=3000
|
||||||
|
API_HOST=127.0.0.1
|
||||||
|
API_METRICS_HOST=127.0.0.1
|
||||||
|
API_METRICS_PORT=9464
|
||||||
HTTP_API_MASTER_KEY=<至少32位随机值,用于AES-256-GCM加密HTTP访问凭据和Webhook密钥>
|
HTTP_API_MASTER_KEY=<至少32位随机值,用于AES-256-GCM加密HTTP访问凭据和Webhook密钥>
|
||||||
HTTP_API_PUBLIC_ORIGIN=https://api.lisglo.com
|
HTTP_API_PUBLIC_ORIGIN=https://api.lisglo.com
|
||||||
API_ENABLE_SEND_WORKER=true
|
API_ENABLE_SEND_WORKER=true
|
||||||
@@ -48,6 +51,8 @@ OPERATION_LOG_ARCHIVE_INTERVAL_MS=86400000
|
|||||||
SMS_RECEIPT_TIMEOUT_SCAN_ENABLED=true
|
SMS_RECEIPT_TIMEOUT_SCAN_ENABLED=true
|
||||||
SMS_RECEIPT_TIMEOUT_HOURS=72
|
SMS_RECEIPT_TIMEOUT_HOURS=72
|
||||||
SMS_RECEIPT_TIMEOUT_SCAN_INTERVAL_MS=300000
|
SMS_RECEIPT_TIMEOUT_SCAN_INTERVAL_MS=300000
|
||||||
|
PROMETHEUS_URL=http://127.0.0.1:9090
|
||||||
|
PROMETHEUS_QUERY_TIMEOUT_MS=5000
|
||||||
REPORT_DAILY_REFRESH_ENABLED=true
|
REPORT_DAILY_REFRESH_ENABLED=true
|
||||||
REPORT_REFRESH_INTERVAL_MS=3600000
|
REPORT_REFRESH_INTERVAL_MS=3600000
|
||||||
CMPP_DOWNSTREAM_ACK_TIMEOUT_SECONDS=30
|
CMPP_DOWNSTREAM_ACK_TIMEOUT_SECONDS=30
|
||||||
@@ -62,6 +67,8 @@ PROD_ADMIN_USERNAME=prod_admin
|
|||||||
PROD_ADMIN_PASSWORD='change-me'
|
PROD_ADMIN_PASSWORD='change-me'
|
||||||
```
|
```
|
||||||
|
|
||||||
|
系统监控需要在发布前单独安装,配置和规则位于`tools/monitoring/`。在Debian/Ubuntu服务器依次执行`bash tools/monitoring/install-prometheus-monitoring.sh`和`bash tools/monitoring/install-service-exporters.sh`;前者安装Prometheus/Node Exporter,后者安装PostgreSQL/Redis/Nginx Exporter并开启MinIO回环原生指标。脚本必须先备份现有配置并运行`promtool`校验;9090、9100、9187、9121、9113、9464和API/Gateway控制端口必须只监听`127.0.0.1`,不得加入Nginx公网反向代理或安全组放行。安装完成且确认全部target为up后,才可完成发布;详细口径见`docs/prometheus-system-monitoring-design-20260814.md`。
|
||||||
|
|
||||||
安全会话使用 HttpOnly Cookie,正式生产必须先为页面和管理 API 配置 HTTPS,并保持 `SESSION_COOKIE_SECURE=true`;纯 HTTP 的 `IP:12026` 不作为受支持的登录入口,即使切换期仍保留其监听,也只允许用于非登录的兼容检查并应尽快下线。`sms.lisglo.com` 只允许 Cloudflare 回源,`api.lisglo.com` 通过独立 Nginx SNI 虚拟主机只开放客户接口、客户 Swagger 和健康检查;Let’s Encrypt 使用 DNS-01 自动续期,不依赖开放 80 端口。
|
安全会话使用 HttpOnly Cookie,正式生产必须先为页面和管理 API 配置 HTTPS,并保持 `SESSION_COOKIE_SECURE=true`;纯 HTTP 的 `IP:12026` 不作为受支持的登录入口,即使切换期仍保留其监听,也只允许用于非登录的兼容检查并应尽快下线。`sms.lisglo.com` 只允许 Cloudflare 回源,`api.lisglo.com` 通过独立 Nginx SNI 虚拟主机只开放客户接口、客户 Swagger 和健康检查;Let’s Encrypt 使用 DNS-01 自动续期,不依赖开放 80 端口。
|
||||||
|
|
||||||
系统操作日志默认在线保留 180 天。API 每日以最多 20 个、每批 1000 条的小事务将过期记录搬入 `OperationLogArchive`,并用 `archiveMonth=YYYY-MM` 标记归档月份;归档记录不会自动删除。调整保留期或批量参数前,应先评估数据库、备份窗口和审计要求。归档表达到千万级或清理窗口不能满足要求时,再实施按 `createdAt` 的月度 PostgreSQL 分区,不在当前数据规模下提前改造主表分区。
|
系统操作日志默认在线保留 180 天。API 每日以最多 20 个、每批 1000 条的小事务将过期记录搬入 `OperationLogArchive`,并用 `archiveMonth=YYYY-MM` 标记归档月份;归档记录不会自动删除。调整保留期或批量参数前,应先评估数据库、备份窗口和审计要求。归档表达到千万级或清理窗口不能满足要求时,再实施按 `createdAt` 的月度 PostgreSQL 分区,不在当前数据规模下提前改造主表分区。
|
||||||
@@ -89,8 +96,26 @@ git reset --hard origin/main
|
|||||||
bash tools/deploy/production-deploy.sh
|
bash tools/deploy/production-deploy.sh
|
||||||
```
|
```
|
||||||
|
|
||||||
|
### Fail2ban 安全检测发布前置条件
|
||||||
|
|
||||||
|
本功能包含新增 PostgreSQL migration、非 root API 身份、安全代理、Fail2ban、Nginx include 和 nftables 表,不能按普通前端热发布处理。发布前除平台标准 PostgreSQL、运行源码和环境文件恢复资产外,必须额外备份 `/etc/systemd/system/cmpp-api.service*`、`/etc/systemd/system/cmpp-security-agent.service`、`/etc/fail2ban`、`/etc/nginx`、`/etc/nftables.conf`、`/etc/nftables.d` 和 `/var/lib/cmpp-security-agent`,并逐项生成、复核 SHA-256。
|
||||||
|
|
||||||
|
发布脚本会构建 `cmpp-security-agent` 并运行 `tools/security/install-security-agent.sh`。安装器创建专用 `cmpp-api` 用户和 `cmpp-security` 组、写入 systemd 加固 drop-in、安装固定 Fail2ban filter/action、校验 Nginx/Fail2ban/nftables,但不会执行任何人工封禁。环境文件至少明确:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
SECURITY_AGENT_SOCKET=/run/cmpp-security-agent/agent.sock
|
||||||
|
SECURITY_AGENT_TIMEOUT_MS=3000
|
||||||
|
SECURITY_EVENT_TOKEN=<至少32字节随机值,仅供Gateway和安全代理上报固定事件>
|
||||||
|
TRUSTED_PROXY_IPS=127.0.0.1,::1
|
||||||
|
SECURITY_BUILTIN_PROTECTED_NETWORKS=<运维出口CIDR,健康检查CIDR,源站公网IP>
|
||||||
|
```
|
||||||
|
|
||||||
|
正式 Nginx 的 `sms.lisglo.com` 运营端/客户端 server 块必须 `include /etc/nginx/snippets/cmpp-security-deny.conf;`;API 专用域名继续只开放客户接口。Cloudflare `real_ip_header` 及可信网段必须按官方来源单独维护和验证,禁止信任任意客户端 `CF-Connecting-IP` 或 `X-Forwarded-For`。发布后需证明 `cmpp-api` 进程用户不是 root、无 sudo 权限且无法写 `/etc/fail2ban`,安全代理 Socket 不监听 TCP,九类规则版本一致,Fail2ban 为 report-only,nftables/Nginx 回读与数据库状态一致。任何一项失败均不得开放人工封禁按钮。
|
||||||
|
|
||||||
部署脚本重启 Gateway、API 和 Nginx 后,会分别对 Gateway、API 健康接口执行最多 60 秒的逐秒就绪检查。Nest 初始化、活动通道恢复或生产数据量增加可能使 API 启动超过固定数秒;发布流程不得用单次固定延时把正常慢启动误判为失败。超过 60 秒仍不健康时才终止发布,并结合 systemd journal 和发布前数据库、源码、环境备份判断回滚方式。
|
部署脚本重启 Gateway、API 和 Nginx 后,会分别对 Gateway、API 健康接口执行最多 60 秒的逐秒就绪检查。Nest 初始化、活动通道恢复或生产数据量增加可能使 API 启动超过固定数秒;发布流程不得用单次固定延时把正常慢启动误判为失败。超过 60 秒仍不健康时才终止发布,并结合 systemd journal 和发布前数据库、源码、环境备份判断回滚方式。
|
||||||
|
|
||||||
|
标准发布会先清空自身管理的`/etc/nginx/conf.d/cmpp-compression.conf`,再排除该文件检查Nginx现有配置。发行版或既有虚拟主机已经启用`gzip on`时直接复用;完全未启用时才写入平台级压缩配置。不得无条件叠加第二个HTTP级`gzip on`,每次重启前必须以`nginx -t`为准。
|
||||||
|
|
||||||
## 账号和密钥
|
## 账号和密钥
|
||||||
|
|
||||||
- 生产管理员账号写入 `/root/cmpp-platform-admin.txt`。
|
- 生产管理员账号写入 `/root/cmpp-platform-admin.txt`。
|
||||||
|
|||||||
@@ -0,0 +1,251 @@
|
|||||||
|
# Prometheus 系统监控设计
|
||||||
|
|
||||||
|
> 版本:V1.0<br>
|
||||||
|
> 设计日期:2026-08-14<br>
|
||||||
|
> 适用范围:运营端 → 系统管理 → 系统监控<br>
|
||||||
|
> 实施边界:使用平台原生 React UI,不引入 Grafana、Netdata、Zabbix 或 Prometheus 自带 UI
|
||||||
|
|
||||||
|
## 1. 建设目标
|
||||||
|
|
||||||
|
运营人员需要在现有权限体系和视觉体系内查看服务器硬件资源、核心服务状态、历史趋势和活动告警。Prometheus 仅承担指标抓取、时序存储、PromQL 计算和告警规则执行;浏览器只访问 CMPP 平台 NestJS API,不直接访问 Prometheus、Node Exporter 或 Alertmanager。
|
||||||
|
|
||||||
|
第一版解决以下问题:
|
||||||
|
|
||||||
|
1. 统一查看 CPU、内存、根文件系统、网络流量、负载和运行时长。
|
||||||
|
2. 查看 `cmpp-api`、`cmpp-gateway`、PostgreSQL、Redis、MinIO、Nginx 六类核心服务状态。
|
||||||
|
3. 在近 1 小时、近 24 小时、近 7 天之间切换真实历史趋势。
|
||||||
|
4. 查看 Prometheus 当前 firing/pending 告警,不在前端伪造阈值判断。
|
||||||
|
5. Prometheus 不可用、查询超时或指标缺失时明确展示“监控不可用/暂无指标”,不得回退静态值、Mock 或 localStorage。
|
||||||
|
|
||||||
|
## 2. 非目标与边界
|
||||||
|
|
||||||
|
- 第一版不提供任意 PromQL 控制台,避免越权查询、高基数查询和资源耗尽。
|
||||||
|
- 第一版不提供告警确认、备注和关闭操作;需要审计留痕的告警处置另立需求并增加 PostgreSQL 模型。
|
||||||
|
- 第一版不采集短信正文、手机号、账号、密钥、数据库查询文本或日志原文。
|
||||||
|
- 云主机通常只能提供虚拟 CPU、内存、云盘和虚拟网卡指标;风扇、物理温度、电源、RAID 和物理硬盘 SMART 需 IPMI/Redfish 或厂商接口,未接入时不得显示伪造数据。
|
||||||
|
- Prometheus、Node Exporter 和 Alertmanager 不暴露在公网;仅 API 服务端可以访问 Prometheus。
|
||||||
|
|
||||||
|
## 3. 总体架构
|
||||||
|
|
||||||
|
```mermaid
|
||||||
|
flowchart LR
|
||||||
|
Node["Node Exporter\n主机与 systemd 指标"] --> Prom["Prometheus\n抓取、存储、PromQL、告警规则"]
|
||||||
|
API["NestJS 监控模块\n固定查询模板与响应归一化"] --> Prom
|
||||||
|
UI["运营端原生 React UI"] --> API
|
||||||
|
Prom --> Alert["Prometheus 活动告警"]
|
||||||
|
Alert --> API
|
||||||
|
```
|
||||||
|
|
||||||
|
数据流必须是 `Exporter → Prometheus → NestJS → 运营端`。前端不得直接拼接 Prometheus URL,不得保存 Prometheus Token,不得接受服务端返回的原始 PromQL。
|
||||||
|
|
||||||
|
## 4. 采集与部署设计
|
||||||
|
|
||||||
|
### 4.1 Node Exporter
|
||||||
|
|
||||||
|
Node Exporter 仅监听 `127.0.0.1:9100`,启用默认 CPU、内存、文件系统、磁盘、网络、负载和启动时间采集器,并显式启用 `systemd` 采集器。systemd 只允许采集:
|
||||||
|
|
||||||
|
- `cmpp-api.service`
|
||||||
|
- `cmpp-gateway.service`
|
||||||
|
- `postgresql.service`
|
||||||
|
- `redis.service` 或 `redis-server.service`
|
||||||
|
- `cmpp-minio.service`
|
||||||
|
- `nginx.service`
|
||||||
|
|
||||||
|
排除 `/dev`、`/proc`、`/sys`、`/run` 等伪文件系统和临时挂载,避免磁盘指标重复和高基数。
|
||||||
|
|
||||||
|
### 4.2 Prometheus
|
||||||
|
|
||||||
|
- 仅监听 `127.0.0.1:9090`。
|
||||||
|
- 默认每 15 秒抓取一次,查询超时 5 秒。
|
||||||
|
- 第一版保留 30 天或不超过 8GB 的时序数据,达到任一边界即按 Prometheus TSDB 策略清理。
|
||||||
|
- 配置文件由仓库 `tools/monitoring/` 管理;安装脚本只安装、校验和启动监控服务,不重启 CMPP Gateway 或发送 Worker。
|
||||||
|
- 生产环境变量使用 `PROMETHEUS_URL=http://127.0.0.1:9090`,API 不向响应透出该地址。
|
||||||
|
|
||||||
|
### 4.3 告警规则
|
||||||
|
|
||||||
|
规则使用持续窗口而非瞬时尖峰:
|
||||||
|
|
||||||
|
| 告警 | Warning | Critical |
|
||||||
|
|---|---:|---:|
|
||||||
|
| 主机指标失联 | — | 连续 2 分钟无数据 |
|
||||||
|
| CPU 使用率 | 连续 10 分钟 > 85% | 连续 5 分钟 > 95% |
|
||||||
|
| 内存使用率 | 连续 10 分钟 > 85% | 连续 5 分钟 > 95% |
|
||||||
|
| 根文件系统使用率 | 连续 15 分钟 > 80% | 连续 5 分钟 > 90% |
|
||||||
|
| inode 使用率 | 连续 15 分钟 > 80% | 连续 5 分钟 > 90% |
|
||||||
|
| CPU iowait | 连续 10 分钟 > 20% | 连续 10 分钟 > 35% |
|
||||||
|
| 核心 systemd 服务 | — | 非 active 2 分钟 |
|
||||||
|
|
||||||
|
告警标签至少包含 `alertname`、`severity`、`instance`、`service`;注解至少包含中文 `summary`、`description`、`currentValue`、`threshold`。敏感环境变量和凭据不得进入标签或注解。
|
||||||
|
|
||||||
|
## 5. 后端设计
|
||||||
|
|
||||||
|
### 5.1 API
|
||||||
|
|
||||||
|
```text
|
||||||
|
GET /api/admin/infrastructure-monitoring/overview?range=1h|24h|7d
|
||||||
|
```
|
||||||
|
|
||||||
|
该接口受现有运营端 Session 中间件保护,仅提供只读数据。`range` 只接受白名单;非法值返回 400。
|
||||||
|
|
||||||
|
### 5.2 固定查询
|
||||||
|
|
||||||
|
后端维护具名查询表,不接受前端 PromQL:
|
||||||
|
|
||||||
|
- CPU:非 idle CPU 秒率。
|
||||||
|
- 内存:`1 - MemAvailable / MemTotal`。
|
||||||
|
- 根文件系统:`1 - available / size`。
|
||||||
|
- 网络:排除 loopback 后的收发字节率。
|
||||||
|
- 负载:`node_load1`。
|
||||||
|
- 运行时长:当前时间减 `node_boot_time_seconds`。
|
||||||
|
- 服务:指定 unit 的 `node_systemd_unit_state{state="active"}`。
|
||||||
|
- 告警:Prometheus `/api/v1/alerts`。
|
||||||
|
|
||||||
|
瞬时查询和区间查询并行执行。区间步长固定为:1小时/60秒、24小时/300秒、7天/1800秒;后端最多返回约 340 个点/序列,禁止浏览器控制步长。
|
||||||
|
|
||||||
|
### 5.3 可用性与错误语义
|
||||||
|
|
||||||
|
- Prometheus 查询成功:`available=true`,返回真实指标、服务和告警。
|
||||||
|
- Prometheus 未配置、拒绝连接、超时、返回非成功状态或响应结构非法:`available=false`,返回采集时间和安全错误摘要,指标字段为 `null`、趋势为空数组。
|
||||||
|
- 单个指标不存在不影响其他指标,缺失项为 `null`;不得把缺失解释为 0。
|
||||||
|
- 服务状态使用 `healthy/unhealthy/unknown`,只有明确采集到 active 才是 healthy,明确采集到 0 才是 unhealthy,指标缺失是 unknown。
|
||||||
|
|
||||||
|
### 5.4 安全控制
|
||||||
|
|
||||||
|
- Prometheus URL 由服务端环境变量读取并限制为 `http://127.0.0.1` 或明确配置的内网 HTTPS 地址。
|
||||||
|
- 每次请求设置 5 秒 AbortSignal 超时。
|
||||||
|
- 不记录完整响应体;错误日志不得包含URL中的认证信息。
|
||||||
|
- 不允许前端传 query、step、start、end 或 Prometheus 标签选择器。
|
||||||
|
- 不将监控指标复制进业务 PostgreSQL,避免高频写入和业务库膨胀。
|
||||||
|
|
||||||
|
## 6. 前端信息架构
|
||||||
|
|
||||||
|
页面路由为 `/admin/system-monitoring`,保留原 `/admin/monitor` “发送监控”,两者业务含义不得混用。
|
||||||
|
|
||||||
|
### 6.1 页面结构
|
||||||
|
|
||||||
|
1. 页面标题、最近采集时间、刷新按钮、1小时/24小时/7天范围切换。
|
||||||
|
2. 横向健康概览:整体状态、服务总数、正常、警告、严重、活动告警。
|
||||||
|
3. CPU、内存、磁盘、网络四项资源区:当前值、辅助值和当前范围趋势。
|
||||||
|
4. 服务状态侧栏:六类服务、状态和最新采集时间。
|
||||||
|
5. 活动告警表:名称、级别、开始时间、持续时间、当前值、阈值。
|
||||||
|
6. 下方资源趋势:CPU、内存、磁盘和网络收发的完整趋势图。
|
||||||
|
|
||||||
|
### 6.2 视觉规格
|
||||||
|
|
||||||
|
- 复用现有 `surface`、`Button`、`Tag`、`Table`、`Chart` 和主题变量。
|
||||||
|
- 真白背景、低饱和蓝色主色、冷灰边框;正常/警告/严重使用既有语义色。
|
||||||
|
- 不使用第三方Logo、iframe、暗色监控皮肤、霓虹渐变或装饰性假指标。
|
||||||
|
- 桌面优先保持表格和趋势信息密度;1100px 以下两列,780px 以下单列。
|
||||||
|
- 图表缺失时显示“暂无真实指标”,不能绘制零线冒充数据。
|
||||||
|
|
||||||
|
### 6.3 交互与刷新
|
||||||
|
|
||||||
|
- 初次进入立即请求真实接口。
|
||||||
|
- 可见页面每 30 秒刷新;页面隐藏或 Session 锁定导致组件卸载后停止刷新。
|
||||||
|
- 切换时间范围立即重新查询并禁用重复点击;手动刷新不改变范围。
|
||||||
|
- 请求失败保留上一次成功数据会造成陈旧误导,因此本版失败时清空数据并展示不可用状态。
|
||||||
|
|
||||||
|
## 7. 响应契约摘要
|
||||||
|
|
||||||
|
```ts
|
||||||
|
type InfrastructureOverview = {
|
||||||
|
available: boolean;
|
||||||
|
range: '1h' | '24h' | '7d';
|
||||||
|
collectedAt: string;
|
||||||
|
error?: string;
|
||||||
|
summary: {
|
||||||
|
overallStatus: 'healthy' | 'warning' | 'critical' | 'unknown';
|
||||||
|
serviceTotal: number;
|
||||||
|
serviceHealthy: number;
|
||||||
|
warningAlerts: number;
|
||||||
|
criticalAlerts: number;
|
||||||
|
activeAlerts: number;
|
||||||
|
};
|
||||||
|
metrics: {
|
||||||
|
cpuUsagePercent: number | null;
|
||||||
|
memoryUsagePercent: number | null;
|
||||||
|
memoryTotalBytes: number | null;
|
||||||
|
memoryAvailableBytes: number | null;
|
||||||
|
diskUsagePercent: number | null;
|
||||||
|
diskTotalBytes: number | null;
|
||||||
|
diskAvailableBytes: number | null;
|
||||||
|
networkReceiveBytesPerSecond: number | null;
|
||||||
|
networkTransmitBytesPerSecond: number | null;
|
||||||
|
load1: number | null;
|
||||||
|
uptimeSeconds: number | null;
|
||||||
|
};
|
||||||
|
trends: Record<string, Array<{ timestamp: string; value: number }>>;
|
||||||
|
services: Array<{ key: string; name: string; unit: string; status: 'healthy' | 'unhealthy' | 'unknown' }>;
|
||||||
|
alerts: Array<{ fingerprint: string; name: string; severity: 'warning' | 'critical' | 'info'; status: string; startedAt: string; summary: string; currentValue?: string; threshold?: string }>;
|
||||||
|
};
|
||||||
|
```
|
||||||
|
|
||||||
|
## 8. 验收标准
|
||||||
|
|
||||||
|
1. 页面所有指标来自真实 Prometheus API,关闭 Prometheus 后页面明确不可用且没有静态回退。
|
||||||
|
2. 浏览器网络请求只访问 CMPP API,不访问 9090/9100。
|
||||||
|
3. 非法 range、任意 PromQL 和自定义 step 均无法进入后端查询。
|
||||||
|
4. CPU、内存、磁盘、网络当前值与 Prometheus 同时刻查询在允许的采样误差内一致。
|
||||||
|
5. 六类服务状态与 systemd 指标一致;指标缺失展示未知而非故障或正常。
|
||||||
|
6. 1小时、24小时、7天切换后时间轴和点数符合固定步长。
|
||||||
|
7. firing/pending 告警真实展示,告警恢复后不再出现在活动列表。
|
||||||
|
8. Prometheus/Exporter 只监听本机,公网不能连接 9090/9100。
|
||||||
|
9. 页面在桌面和移动宽度下无重叠、截断和横向溢出,控制台无相关错误。
|
||||||
|
10. TypeScript、API专项测试、生产构建、配置校验和 `git diff --check`通过。
|
||||||
|
|
||||||
|
## 9. 服务内部指标扩展(V1.1)
|
||||||
|
|
||||||
|
### 9.1 采集边界
|
||||||
|
|
||||||
|
- API使用独立回环端口`127.0.0.1:9464/metrics`,采集进程内存、堆内存、事件循环P99、请求量、状态码和延迟直方图。
|
||||||
|
- Gateway在已有回环控制端口`127.0.0.1:8090/metrics`暴露Go运行时、上下游连接总数、Submit成败和耗时、Redis Stream pending/lag/最旧年龄。
|
||||||
|
- PostgreSQL、Redis、Nginx使用发行版Exporter;MinIO使用原生Prometheus端点。全部Exporter只监听回环地址。
|
||||||
|
- 指标标签只允许方法、路由模板、HTTP状态、结果类别和固定服务名。禁止手机号、短信ID、CMPP Msg_Id、任务ID、通道凭据、短信正文、原始URL和SQL文本进入标签。
|
||||||
|
- 监控页只查询`cmpp:service_*`固定Recording Rules,不为每张卡片执行一条高代价PromQL。
|
||||||
|
|
||||||
|
### 9.2 默认阈值
|
||||||
|
|
||||||
|
| 领域 | Warning | Critical | 持续窗口 |
|
||||||
|
|---|---:|---:|---:|
|
||||||
|
| CPU | >80% | >90% | 10m / 5m |
|
||||||
|
| 内存 | >85% | >95% | 10m / 5m |
|
||||||
|
| 磁盘或inode | >80% | >90% | 15m / 5m |
|
||||||
|
| API 5xx | >1%且至少5次 | >5%且至少5次 | 5m |
|
||||||
|
| API P95 | >1s | >3s | 10m / 5m |
|
||||||
|
| API事件循环P99 | >200ms | >1s | 10m / 5m |
|
||||||
|
| Gateway提交队列最旧pending | >30s | >120s | 2m |
|
||||||
|
| Gateway实际上游连接 | — | 少于期朖2m | 2m |
|
||||||
|
| PostgreSQL连接使用率 | >70% | >85% | 10m / 5m |
|
||||||
|
| PostgreSQL死锁 | 15m内>0 | 15m内重复出现 | 1m |
|
||||||
|
| Redis内存/maxmemory | >70% | >85% | 10m / 5m |
|
||||||
|
| Redis淘汰或拒绝连接 | — | 5m内>0 | 1m |
|
||||||
|
| 任一核心Exporter失联 | — | >2m | 2m |
|
||||||
|
| Prometheus采集耗时/周期 | >80% | >100% | 5m |
|
||||||
|
| Prometheus规则计算失败 | — | 5m内>0 | 1m |
|
||||||
|
|
||||||
|
比例告警必须带最低样本量,不得把单次失败误报为100%错误率。短信最终回执可由运营商延迟数小时,不纳入Gateway基础设施短窗口Critical,仍由72小时业务终结机制和短信质量看板处理。
|
||||||
|
|
||||||
|
### 9.3 告警收敛与校准
|
||||||
|
|
||||||
|
- 主机或Node Exporter失联时,抑制其CPU、内存和磁盘派生告警;API/Gateway指标端点失联时,抑制对应延迟和错误率告警。
|
||||||
|
- Critical表示需立即处理;Warning表示当日检查;趋势指标未达到可操作条件时只展示,不生成告警。
|
||||||
|
- 上线后保留7至14天基线,核对业务高峰P95/P99、正常连接数和队列年龄。阈值调整必须同步修改设计、用例、规则和进度记录。
|
||||||
|
|
||||||
|
### 9.4 性能预算
|
||||||
|
|
||||||
|
- 全局采集周期保持15秒,无排障需求不降到1秒。
|
||||||
|
- API和Gateway请求路径只做内存计数、有界直方图和原子计数,不在业务请求中写PostgreSQL或Redis。
|
||||||
|
- PostgreSQL Exporter只使用发行版默认低代价查询,不采集SQL原文或扫描业务大表。
|
||||||
|
- 时序仍受30天和8GB双重上限约束;时序增长时先缩短实际保留期,不允许无界占满业务盘。
|
||||||
|
|
||||||
|
## 10. 阈值配置与全局预警入口(2026-08-14 增补)
|
||||||
|
|
||||||
|
- 可配置范围固定为主机 CPU/内存/根磁盘、API 5xx/P95/事件循环、Gateway 最旧 pending、PostgreSQL 连接、Redis 内存和 MinIO 容量十组警告/严重数值。PromQL、持续窗口、标签和规则文件路径仍由代码固定,浏览器无权提交。
|
||||||
|
- PostgreSQL 单例记录同时保存期望阈值、生效阈值、配置版本、生效版本和 `applying/effective/failed` 状态。更新使用版本条件认领,防止多个 API 实例并发覆盖;规则先经 promtool 校验,再在同一目录原子替换并调用仅回环开放的 `/-/reload`。失败恢复旧文件并保留旧生效版本。
|
||||||
|
- 安装器把可配置规则从基础规则中剥离,托管文件归 `cmpp-api:prometheus` 且权限为 0640;Prometheus 仍只监听回环。右上角铃铛只轮询轻量活动告警汇总接口,不重复加载趋势或服务指标。
|
||||||
|
|
||||||
|
## 11. 活动告警已读语义(2026-08-16 增补)
|
||||||
|
|
||||||
|
- “已读”只表示某位管理员已查看某一次 Prometheus 活动告警,不是 resolve、silence 或 acknowledge 外部告警管理器;页面活动告警总数与平台健康状态仍按 Prometheus 原始 firing/pending 计算。
|
||||||
|
- 指纹由排序后的 Prometheus labels 稳定生成,`activeAt`区分同一指纹的不同触发周期。数据库以`fingerprint + userId`唯一,upsert同时更新`activeAt/readAt`;读取时只有数据库 activeAt 与当前 Prometheus activeAt 相同才算已读。
|
||||||
|
- 标记前必须回读当前 Prometheus 告警并校验指纹和 activeAt,防止客户端伪造或把已经恢复的新周期误标已读。预警中心轻量汇总只扣减当前管理员本次已读项;数据库故障不得用 localStorage 或静态状态替代。
|
||||||
@@ -4567,6 +4567,7 @@ npm run verify:phase8
|
|||||||
| TC-SIGNATURE-RETIREMENT-027 | 在抑制管理点击“取消抑制”,填写或不填写原因 | 只出现平台自研弹窗;未填原因不能确认,填写后调用真实取消接口并刷新消息及抑制列表,不出现浏览器`prompt/confirm` |
|
| TC-SIGNATURE-RETIREMENT-027 | 在抑制管理点击“取消抑制”,填写或不填写原因 | 只出现平台自研弹窗;未填原因不能确认,填写后调用真实取消接口并刷新消息及抑制列表,不出现浏览器`prompt/confirm` |
|
||||||
| TC-REPORT-RECORD-LAYOUT-001 | 在报备记录页面查看长备注和短备注 | 备注列桌面宽度不小于320px,使用统一长文本换行样式;宽表允许内部横向滚动,备注不被其他固定列挤成窄竖列,详情仍展示全文 |
|
| TC-REPORT-RECORD-LAYOUT-001 | 在报备记录页面查看长备注和短备注 | 备注列桌面宽度不小于320px,使用统一长文本换行样式;宽表允许内部横向滚动,备注不被其他固定列挤成窄竖列,详情仍展示全文 |
|
||||||
| TC-DEPLOY-HEALTH-001 | 发布重启后模拟API初始化超过3秒但在60秒内恢复,并分别模拟API或Gateway持续60秒不可用 | 前者由部署脚本逐秒重试并正常完成,不触发误回滚;后者在60秒后明确失败并保留发布前数据库、源码和环境恢复资产,不把端口尚未就绪当作构建或migration失败 |
|
| TC-DEPLOY-HEALTH-001 | 发布重启后模拟API初始化超过3秒但在60秒内恢复,并分别模拟API或Gateway持续60秒不可用 | 前者由部署脚本逐秒重试并正常完成,不触发误回滚;后者在60秒后明确失败并保留发布前数据库、源码和环境恢复资产,不把端口尚未就绪当作构建或migration失败 |
|
||||||
|
| TC-DEPLOY-NGINX-001 | 分别在Ubuntu默认`nginx.conf`已有全局`gzip on`和完全没有gzip配置的环境执行两次标准发布 | 已有配置时平台生成文件保持为空并复用发行版配置;缺失时写入平台配置;两种环境连续执行两次`nginx -t`均通过且不存在重复gzip指令 |
|
||||||
|
|
||||||
### 2026-08-10 本地执行状态
|
### 2026-08-10 本地执行状态
|
||||||
|
|
||||||
@@ -4588,11 +4589,11 @@ npm run verify:phase8
|
|||||||
| 编号 | 场景 | 预期 |
|
| 编号 | 场景 | 预期 |
|
||||||
|---|---|---|
|
|---|---|---|
|
||||||
| TC-DOWNSTREAM-REQUEUE-TASK-001 | 当前筛选条件预检 | 后端按企业、应用、类型、状态、日期、关键词和 `snapshotAt` 返回真实命中、可重投、跳过及状态分布;分页不影响数量。 |
|
| TC-DOWNSTREAM-REQUEUE-TASK-001 | 当前筛选条件预检 | 后端按企业、应用、类型、状态、日期、关键词和 `snapshotAt` 返回真实命中、可重投、跳过及状态分布;分页不影响数量。 |
|
||||||
| TC-DOWNSTREAM-REQUEUE-TASK-002 | 创建任务 | 原因少于 5 字拒绝;仅物化 `pending/failed/unconfirmed/rejected`;`delivered/awaiting_ack` 不进入执行。 |
|
| TC-DOWNSTREAM-REQUEUE-TASK-002 | 创建任务 | 原因少于 5 字拒绝;物化 `pending/failed/unconfirmed/rejected/delivered`;客户端已确认记录在预检中计为可重投并保留 `previousStatus=delivered`,`awaiting_ack` 不进入执行。 |
|
||||||
| TC-DOWNSTREAM-REQUEUE-TASK-003 | 快照边界 | 创建任务后新增或筛选条件外记录不进入任务。 |
|
| TC-DOWNSTREAM-REQUEUE-TASK-003 | 快照边界 | 创建任务后新增或筛选条件外记录不进入任务。 |
|
||||||
| TC-DOWNSTREAM-REQUEUE-TASK-004 | 并发与幂等 | `taskId+deliveryId` 唯一;重复扫描、API 重启及并发任务不会重复调用 Gateway。 |
|
| TC-DOWNSTREAM-REQUEUE-TASK-004 | 并发与幂等 | `taskId+deliveryId` 唯一;重复扫描、API 重启及并发任务不会重复调用 Gateway。 |
|
||||||
| TC-DOWNSTREAM-REQUEUE-TASK-005 | ACK 闭环 | Gateway 写出后项目进入等待 ACK;`Result=0` 成功,拒绝/超时计失败并按阈值自动暂停。 |
|
| TC-DOWNSTREAM-REQUEUE-TASK-005 | ACK 闭环 | Gateway 写出后项目进入等待 ACK;`Result=0` 成功,拒绝/超时计失败并按阈值自动暂停。 |
|
||||||
| TC-DOWNSTREAM-REQUEUE-TASK-006 | 状态变化跳过 | 执行前状态变化、已确认或被其他操作认领时不调用 Gateway,记录明确跳过原因。 |
|
| TC-DOWNSTREAM-REQUEUE-TASK-006 | 状态变化跳过 | 创建任务时不是 `delivered`、执行前才收到成功 ACK 的记录,或已被其他操作认领时不调用 Gateway,记录明确跳过原因;快照原状态就是 `delivered` 的记录允许调用 Gateway。 |
|
||||||
| TC-DOWNSTREAM-REQUEUE-TASK-007 | 任务控制 | 待执行/执行中任务可暂停、继续、终止;终止不撤回已写出消息。 |
|
| TC-DOWNSTREAM-REQUEUE-TASK-007 | 任务控制 | 待执行/执行中任务可暂停、继续、终止;终止不撤回已写出消息。 |
|
||||||
| TC-DOWNSTREAM-REQUEUE-TASK-008 | 审计 | 创建、暂停、继续、终止和自动暂停记录操作人、筛选快照、原因和结果。 |
|
| TC-DOWNSTREAM-REQUEUE-TASK-008 | 审计 | 创建、暂停、继续、终止和自动暂停记录操作人、筛选快照、原因和结果。 |
|
||||||
| TC-DOWNSTREAM-PAGE-SIZE-001 | 分页数量 | 可选 10/25/50;切换回第一页,后端返回对应条数,总数和筛选条件保持一致。 |
|
| TC-DOWNSTREAM-PAGE-SIZE-001 | 分页数量 | 可选 10/25/50;切换回第一页,后端返回对应条数,总数和筛选条件保持一致。 |
|
||||||
@@ -4610,6 +4611,8 @@ npm run verify:phase8
|
|||||||
| TC-DOWNSTREAM-REQUEUE-TASK-015 | 列表完整分页 | 任务列表支持状态和分页,第 11 条以后可访问,中文状态、创建人、原因、进度和各结果数准确。 |
|
| TC-DOWNSTREAM-REQUEUE-TASK-015 | 列表完整分页 | 任务列表支持状态和分页,第 11 条以后可访问,中文状态、创建人、原因、进度和各结果数准确。 |
|
||||||
| TC-DOWNSTREAM-REQUEUE-TASK-016 | 完整任务项查询 | 任务项支持分页、结果及关键词查询,等待连接/外部 ACK/本任务 ACK、跳过、失败和未处理均中文展示并保留原因。 |
|
| TC-DOWNSTREAM-REQUEUE-TASK-016 | 完整任务项查询 | 任务项支持分页、结果及关键词查询,等待连接/外部 ACK/本任务 ACK、跳过、失败和未处理均中文展示并保留原因。 |
|
||||||
| TC-DOWNSTREAM-REQUEUE-TASK-017 | 终止并发边界 | 终止后未认领和等待连接项置为未处理;处理中或已写出项不撤回;执行器不再认领新项。 |
|
| TC-DOWNSTREAM-REQUEUE-TASK-017 | 终止并发边界 | 终止后未认领和等待连接项置为未处理;处理中或已写出项不撤回;执行器不再认领新项。 |
|
||||||
|
| TC-DOWNSTREAM-REQUEUE-TASK-018 | 已确认记录批量重投 | 以状态 `delivered` 预检并创建任务,核对任务项原状态后执行;页面显示重复投递风险,真实任务项进入等待 ACK/成功闭环;同一任务已成功项不重复调用 Gateway。 |
|
||||||
|
| TC-DOWNSTREAM-REQUEUE-TASK-019 | 创建弹窗与列表留白 | 桌面及窄屏打开创建弹窗和后台任务列表,输入不足5字及合法原因 | 原因使用统一多行输入组件,必填、错误、说明、字数和焦点态清晰;任务列表与卡片边缘保持设计间距,行内容不贴边、不裁切,移动端留白同步收敛。 |
|
||||||
# 2026-08-13 HTTP 与 Gateway 报文容量专项用例
|
# 2026-08-13 HTTP 与 Gateway 报文容量专项用例
|
||||||
|
|
||||||
| 用例编号 | 优先级 | 验证内容 | 预期结果 |
|
| 用例编号 | 优先级 | 验证内容 | 预期结果 |
|
||||||
@@ -4628,3 +4631,65 @@ npm run verify:phase8
|
|||||||
| TC-DASHBOARD-SIGNATURE-REMOVE-001 | 删除看板签名统计模块并统一金额样式 | 打开运营看板,检查指标卡与后续模块 | 两个“今日签名发送统计”模块均不存在;今日消费金额与今日发送总量主数字字号、字重和颜色一致;今日活跃签名仍来自真实接口 |
|
| TC-DASHBOARD-SIGNATURE-REMOVE-001 | 删除看板签名统计模块并统一金额样式 | 打开运营看板,检查指标卡与后续模块 | 两个“今日签名发送统计”模块均不存在;今日消费金额与今日发送总量主数字字号、字重和颜色一致;今日活跃签名仍来自真实接口 |
|
||||||
| TC-ENTERPRISE-APP-WIDTH-001 | 企业应用关键列缩窄 | 在桌面大屏打开企业应用管理并读取表头列宽 | 状态、到达率、单价列宽均约为原宽度80%,字段与操作均未隐藏,横向滚动需求减少 |
|
| TC-ENTERPRISE-APP-WIDTH-001 | 企业应用关键列缩窄 | 在桌面大屏打开企业应用管理并读取表头列宽 | 状态、到达率、单价列宽均约为原宽度80%,字段与操作均未隐藏,横向滚动需求减少 |
|
||||||
| TC-UI-CARRIER-TAG-002 | 全局运营商数据展示复用通用标签 | 抽查通道/通道组、监控、报备、企业签名、短信审核、批次号码、发送记录和客户端发送详情 | 移动、联通、电信均使用全局低饱和胶囊及统一色值;有运营商集合的三网通道显示三个标签,历史通道级字段显示中性“三网”;筛选选项、图表图例与导出文本保持纯文本 |
|
| TC-UI-CARRIER-TAG-002 | 全局运营商数据展示复用通用标签 | 抽查通道/通道组、监控、报备、企业签名、短信审核、批次号码、发送记录和客户端发送详情 | 移动、联通、电信均使用全局低饱和胶囊及统一色值;有运营商集合的三网通道显示三个标签,历史通道级字段显示中性“三网”;筛选选项、图表图例与导出文本保持纯文本 |
|
||||||
|
|
||||||
|
## Prometheus 系统监控专项用例(2026-08-14)
|
||||||
|
|
||||||
|
| 用例编号 | 场景 | 操作 | 预期结果 |
|
||||||
|
|---|---|---|---|
|
||||||
|
| TC-INFRA-MON-001 | 运营端权限与原生页面 | 登录运营端,打开“系统管理 → 系统监控”,检查页面和浏览器网络请求 | 页面使用平台导航、组件和样式;只请求平台`/api/admin/infrastructure-monitoring/overview`,不加载iframe,不从浏览器访问9090/9100,不出现第三方Logo或登录页 |
|
||||||
|
| TC-INFRA-MON-002 | 未登录访问 | 清除运营端Session后直接访问系统监控API和页面 | API按现有Session中间件拒绝,页面进入运营端登录流程;Prometheus数据不得绕过运营端权限公开 |
|
||||||
|
| TC-INFRA-MON-003 | 当前硬件指标真实值 | 在同一采样窗口分别请求平台监控API和Prometheus固定查询 | CPU、内存、根文件系统、网络、负载、运行时长与Prometheus结果在采样误差内一致,响应不包含Prometheus地址或PromQL |
|
||||||
|
| TC-INFRA-MON-004 | 指标缺失 | 临时禁用某个Node Exporter采集器或查询一个不存在的指标后请求页面 | 对应指标为`null`并显示“暂无真实指标”,其他指标继续展示;不得显示0或生成趋势线 |
|
||||||
|
| TC-INFRA-MON-005 | Prometheus不可用 | 停止本地测试Prometheus或将测试环境指向拒绝连接端口,请求监控API | API返回`available=false`和安全错误摘要,页面显示“监控数据不可用”并清空陈旧指标,不显示Mock或上次数据 |
|
||||||
|
| TC-INFRA-MON-006 | 查询超时 | 让测试Prometheus响应超过5秒 | 请求被AbortSignal终止,接口有限时间返回不可用状态;API进程不积累悬挂请求 |
|
||||||
|
| TC-INFRA-MON-007 | 时间范围白名单 | 分别请求`1h`、`24h`、`7d`、`30d`和注入PromQL字符串 | 前三种成功且step分别为60/300/1800秒;非法值返回400,不能进入Prometheus查询 |
|
||||||
|
| TC-INFRA-MON-008 | 趋势切换 | 页面依次选择近1小时、近24小时、近7天 | 每次只发一个新范围请求;图表时间轴、点数和当前范围同步更新,切换期间防止重复触发 |
|
||||||
|
| TC-INFRA-MON-009 | 自动与手动刷新 | 保持页面可见超过30秒,再隐藏页面并点击手动刷新 | 可见时按30秒刷新;隐藏后停止;重新可见后立即刷新;手动刷新保留范围且不会并发重复请求 |
|
||||||
|
| TC-INFRA-MON-010 | 核心服务状态 | 核对`cmpp-api`、`cmpp-gateway`、PostgreSQL、Redis、MinIO、Nginx的systemd状态与页面 | active显示正常,明确0显示异常,指标不存在显示未知;Redis兼容`redis.service`与`redis-server.service`别名 |
|
||||||
|
| TC-INFRA-MON-011 | 活动告警 | 触发一条warning和一条critical测试规则并等待Prometheus进入firing | 页面显示真实名称、严重性、开始时间、持续时间、当前值和阈值;概览计数准确,恢复后活动列表移除 |
|
||||||
|
| TC-INFRA-MON-012 | 综合状态 | 分别构造无告警、warning、critical和Prometheus不可用状态 | 综合状态依次为正常、警告、严重、未知;critical优先于warning,不按前端瞬时指标重复计算 |
|
||||||
|
| TC-INFRA-MON-013 | 并行查询与响应上限 | 检查后端请求时序,并用7天范围请求最大趋势 | 瞬时、趋势、服务、告警查询并行;每序列约不超过340点,响应不包含原始Prometheus响应体 |
|
||||||
|
| TC-INFRA-MON-014 | 监听与公网暴露 | 在服务器执行`ss -lnt`并从外部探测9090、9100 | Prometheus和Node Exporter仅监听127.0.0.1或明确内网地址;公网9090/9100不可连接,运营端仍能经平台API读取指标 |
|
||||||
|
| TC-INFRA-MON-015 | 响应式与无障碍 | 在1536×1024、1280×800和390×844打开页面,操作范围和刷新按钮 | 桌面信息层级符合设计稿;窄屏无内容重叠和页面横向溢出;按钮有可读名称,活动范围和告警严重性不只依赖颜色表达 |
|
||||||
|
| TC-INFRA-MON-016 | 配置和部署幂等 | 在测试服务器重复执行监控安装脚本和配置校验 | 不重复创建系统用户,不开放公网端口;配置通过`promtool check config/rules`,服务保持active,CMPP API/Gateway不因安装被重启 |
|
||||||
|
| TC-INFRA-MON-017 | 业务数据隔离 | 运行监控24小时并检查PostgreSQL业务库和指标标签 | 监控时序只保存在Prometheus TSDB,业务PostgreSQL无高频指标写入;标签、日志和API响应不含手机号、短信正文、账号或密钥 |
|
||||||
|
| TC-INFRA-MON-018 | systemd PromQL转义兼容 | 使用真实Prometheus执行API生成的服务状态查询,并检查自动化请求参数 | 查询文本向Prometheus传递双反斜杠转义的`\\.`正则,API返回`available=true`及真实服务状态;不得因`unknown escape sequence`把整页降级 |
|
||||||
|
| TC-INFRA-MON-019 | 页面标题去重 | 打开系统监控页并检查平台页头和内容区 | 平台通用页头保留“系统监控”,内容区不再出现重复大号标题;说明、状态、范围和刷新操作完整可用 |
|
||||||
|
| TC-INFRA-MON-020 | API内部指标 | 回环请求API metrics,再发起成功与失败的固定路由请求 | 请求量、状态码、延迟桶、堆内存和事件循环指标变化;route为路由模板,不含实体ID或查询串 |
|
||||||
|
| TC-INFRA-MON-021 | Gateway内部指标 | 请求`127.0.0.1:8090/metrics`,交叉核对连接池和Redis Stream | 上下游连接、Submit计数/耗时、worker up、pending、lag和最旧pending年龄与真实状态一致 |
|
||||||
|
| TC-INFRA-MON-022 | 服务Exporter目标 | 安装PostgreSQL、Redis、Nginx Exporter并开启MinIO原生指标 | Prometheus六个新服务target均up;数据库、Redis、MinIO、Nginx指标与各服务本地命令在采样误差内一致 |
|
||||||
|
| TC-INFRA-MON-023 | Exporter端口隔离 | 执行`ss -lnt`并从LAN/公网探测9464、9187、9121、9113、9090、9100 | 全部只监听127.0.0.1或::1,Nginx业务站点不代理metrics端点 |
|
||||||
|
| TC-INFRA-MON-024 | 阈值持续窗口 | 在隔离节点分别制造瞬时和持续的CPU/API错误/队列延迟 | 瞬时尖峰不告警;达到阈值与`for`窗口后进入pending/firing,恢复后移除 |
|
||||||
|
| TC-INFRA-MON-025 | 低流量错误率保护 | 5分钟内只产生1次API请求且返回500 | 因未达至5次错误的最低样本量,不产生5xx比例告警 |
|
||||||
|
| TC-INFRA-MON-026 | 高基数和敏感字段防护 | 检查API/Gateway/Exporter全量metrics文本及Prometheus label names/values | 不存在手机号、短信正文、message/submit/task/channel实体ID、凭据、原始URL或SQL文本 |
|
||||||
|
| TC-INFRA-MON-027 | Recording Rules查询收敛 | 刷新系统监控页并检查Prometheus请求 | 服务卡片只读取`cmpp:service_*`固定聚合,不按卡片开放任意PromQL;缺失指标显示“待采集” |
|
||||||
|
| TC-INFRA-MON-028 | 监控开销对比 | 在同等请求压力下对比开启前后API/Gateway CPU、RSS、P95和吞吐 | 无高基数增长、无业务PostgreSQL高频写入;开销超出预算时暂停发布并调整采集/桶配置 |
|
||||||
|
| TC-GLOBAL-ALERT-001 | 铃铛分域预警菜单 | 准备签名清退未读消息和安全待处置告警后点击右上角铃铛 | 弹层分开显示“签名清退预警”和“安全检测与封禁”,分别展示真实数量和摘要,角标等于两项之和 |
|
||||||
|
| TC-GLOBAL-ALERT-002 | 预警菜单跳转 | 分别点击铃铛中的两个菜单项 | 签名项跳转`/admin/signature-retirement`,安全项跳转`/admin/security-detection`,弹层关闭且对应页面读取真实后端数据 |
|
||||||
|
| TC-GLOBAL-ALERT-003 | 域间故障隔离与轻量轮询 | 分别让一个汇总接口失败并观察30秒轮询请求 | 失败域显示0且另一域数据保留;安全预警使用专用汇总接口,不调用完整overview、规则、代理状态或告警大列表 |
|
||||||
|
| TC-DEPLOY-NET-001 | API回环监听边界 | 使用标准生产环境启动API,执行`ss -lnt`并从LAN/Tailscale探测3000端口,同时经Nginx业务入口请求健康接口 | API仅监听`127.0.0.1:3000`,外部不能直连3000;Nginx入口仍正常返回真实API健康结果;部署静态门禁校验`API_HOST`默认值与启动参数一致 |
|
||||||
|
## Fail2ban 安全检测与人工封禁测试矩阵(2026-08-14)
|
||||||
|
|
||||||
|
- 本模块必须执行 `docs/fail2ban-assisted-blocking-test-cases-20260814.md` 中 TC-F2B 全量用例,专项用例是本平台功能测试的组成部分,不是可选附录。
|
||||||
|
- P0 门禁至少覆盖:九类规则真实 PostgreSQL 默认值与版本冲突、阈值边界、规则应用失败保留旧生效值、登录/HTTP/CMPP/SSH/Nginx 真实事件脱敏、事件键幂等、窗口聚合并发、可信代理 IP、Cloudflare 与直连入口执行器映射、系统和人工保护网段、近期重新认证、重复封禁原子认领、代理超时/失败、真实执行器回读、非 root NestJS 及任意命令/参数注入拒绝。
|
||||||
|
- 集成验收必须在隔离测试节点或网络 namespace 使用文档保留 IP;不得封禁预生产运维出口、Cloudflare 节点或真实客户 IP。未安装真实 Fail2ban/nftables/Nginx 资产时,只能把相关用例标记阻塞,不得用 Mock 通过代替。
|
||||||
|
- UI 验收覆盖桌面与窄屏的总览、告警、规则、封禁记录、保护名单、加载、空数据、失败和规则未生效状态;所有数字与操作结果必须能从 API、数据库、agent 与执行器证据交叉验证。
|
||||||
|
- `TC-F2B-OPS-008`:在非默认`APP_DIR`构建安全代理后执行安装器,核对systemd `ExecStart`与Fail2ban `actionban`均指向同一个真实可执行的`$APP_DIR/dist/cmpp-security-agent`;任一文件残留占位符、旧`current/bin`路径或目标不可执行时,安装/发布必须失败。
|
||||||
|
|
||||||
|
## 系统监控阈值与预警中心增量用例(2026-08-14)
|
||||||
|
|
||||||
|
| 用例ID | 场景 | 步骤 | 预期 |
|
||||||
|
| --- | --- | --- | --- |
|
||||||
|
| TC-INFRA-MON-029 | 模块顺序与技术说明 | 打开系统监控并检查标题和模块顺序 | 说明明确写明 Prometheus;服务关键指标紧邻活动告警上方,活动告警锚点可定位 |
|
||||||
|
| TC-INFRA-MON-030 | 阈值真实读取 | 打开阈值设置并核对 API 与数据库 | 十组固定指标来自 `InfrastructureAlertSetting`,不使用 Mock/localStorage,不允许编辑 PromQL |
|
||||||
|
| TC-INFRA-MON-031 | 阈值边界 | 提交警告≥严重、越界、缺项和未知指标 | API 返回 400,数据库版本和 Prometheus 规则均不改变 |
|
||||||
|
| TC-INFRA-MON-032 | 并发版本 | 两个会话以同一版本先后保存 | 仅第一个原子认领成功,后者返回 409 并提示刷新 |
|
||||||
|
| TC-INFRA-MON-033 | 规则校验与热加载 | 保存合法阈值,检查 promtool、规则文件、reload 与数据库 | 先校验再同目录原子替换,reload 成功后生效版本前进且写操作日志 |
|
||||||
|
| TC-INFRA-MON-034 | 应用失败回滚 | 令 promtool 或 reload 失败后保存 | 状态为 failed、展示原因,旧规则文件与旧生效阈值保留,不误报已生效 |
|
||||||
|
| TC-GLOBAL-ALERT-004 | 系统监控预警入口 | 准备隔离 QA Prometheus firing 告警并点击铃铛 | 第三项显示真实总数/严重数,角标计入三域总和,点击跳转系统监控活动告警区 |
|
||||||
|
| TC-SECURITY-UI-001 | Fail2ban 标识与标题规范 | 打开安全检测与封禁 | 不出现重复大号页面标题,说明明确写明使用 Fail2ban,字号遵循通用菜单标题 |
|
||||||
|
| TC-INFRA-MON-035 | 阈值弹窗单层滚动 | 在桌面和窄屏打开阈值设置,滚动到最后一组阈值 | 只有 Modal 外层内容区出现滚动条,阈值表单容器不产生第二层滚动或滚动陷阱,页头和页脚行为正常 |
|
||||||
|
| TC-INFRA-MON-036 | 活动告警逐条已读 | 使用管理员A点击一条当前活动告警的“标记已读” | PostgreSQL新增/更新管理员A与该次 activeAt 的记录;行显示“已读”,活动告警总数不变,铃铛系统监控数量减少1 |
|
||||||
|
| TC-INFRA-MON-037 | 已读用户隔离 | 管理员A标记已读后由管理员B查看同一告警 | 管理员B仍显示未读且铃铛数量不减少,管理员A的状态保持已读 |
|
||||||
|
| TC-INFRA-MON-038 | 同告警重新触发 | 标记已读后让告警恢复,再以相同标签重新触发并产生新 activeAt | 新触发记录重新显示“标记已读”,计入预警中心;旧 activeAt 不会永久屏蔽同指纹告警 |
|
||||||
|
| TC-INFRA-MON-039 | 过期与幂等 | 重复提交同一活动告警,再提交已恢复或 activeAt 不匹配的请求 | 同一次告警重复提交幂等;过期/不匹配请求返回404且不生成虚假已读记录;操作日志可追溯 |
|
||||||
|
|||||||
@@ -3573,3 +3573,117 @@ git diff --check
|
|||||||
- API、Gateway、Nginx、PostgreSQL、Redis与MinIO均active,内部API/Gateway/MinIO健康、Redis PONG;`gateway.submit.commands`消费者1、`pending=0`、`lag=0`。公网运营端、客户端和API health均HTTP 200,公网CMPP 17890 TCP连通;发布时间窗API/Gateway warning级日志为0。
|
- API、Gateway、Nginx、PostgreSQL、Redis与MinIO均active,内部API/Gateway/MinIO健康、Redis PONG;`gateway.submit.commands`消费者1、`pending=0`、`lag=0`。公网运营端、客户端和API health均HTTP 200,公网CMPP 17890 TCP连通;发布时间窗API/Gateway warning级日志为0。
|
||||||
- 首次前台SSH执行因本地等待窗口关闭而收到终止信号,包装流程按设计恢复PostgreSQL和原运行目录,运行标识回到`4c70978d`、migration回到86条、服务全部active;确认恢复资产完整后改为服务器后台日志方式重新执行并成功,未并发重复部署。
|
- 首次前台SSH执行因本地等待窗口关闭而收到终止信号,包装流程按设计恢复PostgreSQL和原运行目录,运行标识回到`4c70978d`、migration回到86条、服务全部active;确认恢复资产完整后改为服务器后台日志方式重新执行并成功,未并发重复部署。
|
||||||
- 本轮没有创建真实重投任务、调用真实Gateway重投、发送短信、修改通道配置、余额或客户连接。既有`api/tsconfig.build.tsbuildinfo`、`tsconfig.tsbuildinfo`、`outputs/`和空文件`=`继续保护,不归入业务提交。
|
- 本轮没有创建真实重投任务、调用真实Gateway重投、发送短信、修改通道配置、余额或客户连接。既有`api/tsconfig.build.tsbuildinfo`、`tsconfig.tsbuildinfo`、`outputs/`和空文件`=`继续保护,不归入业务提交。
|
||||||
|
|
||||||
|
# 2026-08-14 Prometheus 系统监控(本地未提交、未发布)
|
||||||
|
|
||||||
|
- 运营端“系统管理”新增独立“系统监控”页面,保留原“发送监控”业务职责。页面使用平台React、ECharts和通用组件原生实现,不嵌入Grafana、Prometheus或第三方iframe;支持近1小时、24小时、7天,展示CPU、内存、根磁盘、网络、负载、运行时长、六类核心systemd服务和Prometheus活动告警。
|
||||||
|
- 新增只读`GET /api/admin/infrastructure-monitoring/overview`。后端只接受`1h/24h/7d`白名单,按固定PromQL并行访问Prometheus,5秒超时;浏览器不能提交PromQL或访问9090/9100。缺失序列返回`null`,Prometheus异常返回`available=false`并清空指标和趋势,不用0、Mock、静态数据或localStorage伪装。
|
||||||
|
- 新增Debian/Ubuntu幂等安装脚本、Prometheus抓取配置和告警规则。Prometheus与Node Exporter只监听`127.0.0.1`,默认保留30天且限制8GB;脚本备份既有配置和override、运行`promtool`校验,只重启两个监控服务,不重启API、Gateway或其他业务服务。本轮未在预生产执行脚本。
|
||||||
|
- 专项Jest 1套/4项、API全量38套/467项通过,覆盖范围白名单、Prometheus HTTP响应契约解析、固定60秒step、服务别名、活动告警、不可用无陈旧数据及监控地址安全约束;全量Jest仍因仓库既有异步句柄使用`--forceExit`收尾。API正式TypeScript、前端TypeScript、Vite 8.1.5生产构建、两个Shell脚本语法及`git diff --check`通过;Vite仅保留既有约2.10MB单chunk提示。
|
||||||
|
- 浏览器优先检查现有页面:本地运营端无有效登录Session,被正常引导到带算术验证码的登录页;未绕过登录、未读取或填写验证码,因此登录后桌面与窄屏视觉验收尚未完成。真实Prometheus/Node Exporter集成和告警触发验收必须在明确授权安装的测试或预生产窗口执行,当前不以单测替代真实基础设施验收。
|
||||||
|
- 本轮代码、配置和文档保持未提交、未推送、未部署,没有发送、补发或重投短信,没有修改通道账号、密码、启停状态、企业余额、客户连接或预生产数据。既有构建缓存、`outputs/`和空文件`=`继续保护;并行会话新增的Fail2ban设计与测试文档不属于本需求,不修改、不归因。
|
||||||
|
# 2026-08-14 Fail2ban 安全检测与人工封禁第一版(本地实现完成、未发布)
|
||||||
|
|
||||||
|
- 新增真实 PostgreSQL 安全规则、事件、告警、封禁和保护网段模型及第88条 migration `20260814150000_add_security_detection`;九类默认规则覆盖运营端/客户端登录、SSH、CMPP认证与协议滥用、HTTP错误密钥/签名/重放和Nginx恶意扫描。应用事件按事件键幂等并使用PostgreSQL advisory transaction lock串行聚合;Fail2ban命中作为其自身窗口已达到阈值的聚合事件处理,不会再要求重复达到第二层阈值。
|
||||||
|
- 运营端新增“安全控制 / 安全检测与封禁”自研页面,使用真实API展示总览、告警、规则、封禁记录和保护名单;规则修改、封禁、解封、忽略和保护名单写操作要求近期重新认证。封禁时长为固定枚举,执行器由服务端按入口映射,浏览器不能传jail、action、shell或执行器参数;系统私网/回环等内置保护和人工CIDR保护在调用代理前拦截。
|
||||||
|
- 新增独立Go `cmpp-security-agent`、Unix Socket固定协议、report-only Fail2ban action/filter、Nginx real-IP deny、nftables timeout set和systemd加固。NestJS部署身份改为专用非root `cmpp-api`,安全事件入口新增独立内部令牌;agent不使用shell拼接,规则/Nginx配置校验或reload失败会恢复旧文件,只有真实执行器回读命中后数据库才标记`blocked`。
|
||||||
|
- 登录、OpenAPI鉴权和Gateway已接入结构化安全事件;错误HTTP密钥只保存不可逆账号指纹,证据字段统一过滤password/secret/token/signature/access-key。反向代理地址只在TCP来源属于`TRUSTED_PROXY_IPS`时信任`X-Forwarded-For`,避免客户端伪造来源IP。
|
||||||
|
- Prisma schema validate、API与前端TypeScript、API全量40套/472项、Fail2ban专项和Gateway控制器3套/11项、Gateway全量`go test ./...`、Vite 8.1.5生产构建、3个Shell脚本语法和`git diff --check`通过;Vite仅有既有约2.11MiB单chunk提示,全量Jest仍使用`--forceExit`收尾既有异步句柄。
|
||||||
|
- 浏览器优先接管本地路由,`/admin/security-detection`在无有效Session时正确跳转运营端登录并保留返回地址,控制台error/warn为0;未读取、重置或猜测账号,登录后页面视觉与交互验收尚未完成。真实Fail2ban、Nginx、nftables、Unix Socket和第88条migration未在本机数据库或预生产安装/执行,必须在具备恢复资产和文档保留测试IP的授权发布窗口完成,当前不以Mock替代集成验收。
|
||||||
|
- 本轮未发送、补发或重投短信,未修改通道账号、密码、启停状态、企业余额、客户连接或预生产数据;`api/tsconfig.build.tsbuildinfo`、`tsconfig.tsbuildinfo`、`outputs/`和空文件`=`继续作为受保护项排除提交。
|
||||||
|
|
||||||
|
# 2026-08-14 Prometheus 与 Fail2ban 两次提交本地部署复核(未推送、未发布)
|
||||||
|
|
||||||
|
- 从侧边会话已经落到本地 `main` 的两个提交开始复核:`b78faa1aa24a89830892b511a6cb17cc5a5fbe68`(Prometheus 系统监控)和 `d30d9ea4d0ec8a46154dfb1e39a9bba2156ec455`(Fail2ban 安全检测)。复核时本地 `HEAD=d30d9ea`,`origin/main=96e475d`,本地领先2个提交;本轮没有再次提交、推送或发布。
|
||||||
|
- migration 前已将本地真实 PostgreSQL 备份到 `C:\cmpp-platform-local\backups\cmpp-platform-before-d30d9ea-20260814-111254.dump`,637967字节,SHA-256=`523ecc82b55b5575ebe78fc4d253c5ec44932a76a51209130442f619e1971089`。Prisma generate、validate、migrate deploy/status均通过;`20260814150000_add_security_detection`真实应用一次,本地数据库由87条升级为88/88条migration,九类默认安全规则均存在且配置版本为1。
|
||||||
|
- 本地API以正式构建产物运行在3000端口,前端Vite生产预览运行在4173端口,API health与前端HTTP均为200。为避免本地Gateway连接真实供应商,本轮只编译和测试Gateway,没有启动8090;因此API日志中的活动通道恢复和供应商状态对账失败是预期的本地Gateway离线结果,未修改任何通道参数,也未触发短信提交。
|
||||||
|
- API全量40套/472项、API TypeScript正式构建、前端TypeScript与Vite 8.1.5生产构建、Gateway `go test ./... -count=1`及`go vet ./...`全部通过;Vite仅保留既有大chunk提示。Prometheus、Fail2ban、nftables和Linux systemd在Windows本机不可用,三个部署Shell及标准发布/初始化Shell均通过Bash语法检查,但不以语法检查冒充Linux真实安装验收。
|
||||||
|
- 正式服务层连接真实本地PostgreSQL和真实不可达的 `127.0.0.1:9090` 验证:监控概览返回 `available=false`,所有指标为`null`、趋势为空,非法范围`30d`返回400;安全检测写入保留测试地址`203.0.113.10`的一条 `http_invalid_api_key` 事件,证据中的访问密钥已存为`[REDACTED]`,同一`eventKey=local-qa-d30d9ea-http-invalid-api-key`第二次上报命中幂等去重且未达到告警阈值。该真实本地测试事件保留作审计证据,没有创建封禁或调用安全代理。
|
||||||
|
- 未认证HTTP请求访问监控总览、安全总览和规则列表均返回401。经用户明确授权,只临时替换本地专用 `codex_local_admin` 的密码哈希完成图形验证码登录;登录后立即恢复原密码哈希、失败计数和锁定时间,未新建账号、未变更session版本。浏览器桌面验收确认监控页切换到近1小时并刷新后仍明确显示Prometheus不可用且没有Mock/缓存数值;安全页显示24小时事件1条、安全代理不可用,并从真实数据库展示9条规则及`1/1`版本。390×844窄屏下两页核心内容和交互可用,控制台warning/error为0;安全页五个页签在窄屏中文字换行偏碎,记为非阻断视觉问题。
|
||||||
|
- 19份既有结构门禁中11份通过、8份失败;失败集中在后台重投/异常处置API、通道/发送质量哈希、报备导出、运营商集合、定时调度、全局长文本样式和签名查询等既有契约漂移,与这两个提交新增文件无交集,本轮不顺带改写其他模块契约。`git diff --check`通过。
|
||||||
|
- 本轮没有发送、补发或重投真实短信,没有修改真实通道账号、密码、启停状态、企业余额或客户连接。既有`api/tsconfig.build.tsbuildinfo`、`tsconfig.tsbuildinfo`、`outputs/`和空文件`=`继续保护;本段测试记录是本轮新增的唯一业务工作区修改。
|
||||||
|
|
||||||
|
# 2026-08-14 Prometheus 与 Fail2ban 本地服务器部署及真实集成验收
|
||||||
|
|
||||||
|
- 经用户明确授权,将本地`HEAD=d30d9ea4d0ec8a46154dfb1e39a9bba2156ec455`及当前工作区的部署修复安装到全新Ubuntu 24.04测试节点`100.93.204.60`;未把该节点当作预生产。部署前只读盘点确认4核、7.8GiB内存、根盘约98GiB且无既有平台目录/数据库/环境文件,恢复基线位于`/opt/cmpp-platform-backups/releases/20260814-135221-before-d30d9ea-localserver`,API回环修复前增量备份位于`/opt/cmpp-platform-backups/releases/20260814-144709-before-api-loopback`。
|
||||||
|
- 基础源码归档SHA-256=`c9e0b4ba5895b0d434a3335dcc6044f291779337841bb19b636bbed5d91b1b96`,首轮修复overlay=`24c5720dd0da6fab1b745d3c976cfc70d1a2f1b49da13443157a0ae89eac1bad`,API回环修复overlay=`3d6cdd45a87e12fcad50a10d18862a1527c6a36fae2b18391142c0fba61e717e`;运行标识写为`d30d9ea4d0ec8a46154dfb1e39a9bba2156ec455+localfix.3d6cdd45a87e`。生成的本地管理员凭据以0600权限保存在服务器`/home/hector/cmpp-platform-admin.txt`,未写入仓库或测试记录。
|
||||||
|
- 已安装Node 22.21.1、Go 1.26.0、PostgreSQL 16.14、Redis 7.0.15、Nginx 1.24、Fail2ban 1.0.2、Prometheus 2.45.3及Node Exporter 1.7。由于该LAN的DNS/代理返回不可路由的fake-IP,安装期间备份原DNS、hosts和APT源后改用清华Ubuntu镜像并为必要下载域名写入临时hosts固定解析;这些固定解析仅为安装绕行,网络恢复后应按恢复资产移除。首次部署因MinIO官方二进制跳转GitHub后不可达,依照部署文档临时使用真实服务器文件存储驱动`local`,没有用Mock或localStorage伪造对象存储;后续已按用户提供的MinIO二进制完成正式切换,证据见下方补充记录。
|
||||||
|
- 第88条migration真实应用,Prisma报告88 migrations且schema up to date;`SecurityDetectionRule`真实9条,`configVersion/effectiveVersion`均为1。该全新节点的通道、短信记录、下游重投任务和安全封禁记录均为0,未连接真实供应商或客户。
|
||||||
|
- 真实Linux集成检查通过:API、Gateway、安全代理、PostgreSQL、Redis、Nginx、Prometheus、Node Exporter及Fail2ban均active;API/Gateway/Prometheus健康、Redis PONG、PostgreSQL ready、Nginx语法和Fail2ban配置通过。Prometheus两个target均为`up`,配置有效且告警文件12条规则通过`promtool`;9090/9100仅监听127.0.0.1。安全代理Unix Socket为0660 root:cmpp-security,NestJS用户`cmpp-api`无sudo,systemd与Fail2ban action共同指向`/opt/cmpp-platform/dist/cmpp-security-agent`,nftables专用IPv4/IPv6 timeout set存在,Fail2ban sshd jail运行。
|
||||||
|
- 部署中发现并修复三项真实Fresh-Install缺陷:安全代理systemd与Fail2ban action旧路径漂移;Ubuntu默认gzip与发布脚本重复声明导致`nginx -t`失败;NestJS文档要求回环但代码默认监听全网卡。前两项由安装/发布静态门禁保护,第三项新增`API_HOST`并默认127.0.0.1。修复后服务器`ss`确认3000/5432/6379/8090/9090/9100均为回环,外部探测12026和17890可连、3000/9090/9100不可连,Nginx入口`/api/health`返回200。专项部署门禁、API TypeScript正式构建、Shell语法和`git diff --check`通过。
|
||||||
|
- 内置浏览器两次导航该Tailscale地址均在页面加载阶段超时;用户随后明确要求改用系统Chrome,Chrome控制扩展同样在新建页导航阶段超时,而同机PowerShell对同URL返回HTTP 200,判定为浏览器控制链路到Tailscale HTTP地址的环境阻塞。两次尝试均未到验证码或登录提交;管理员密码哈希、失败次数和锁定时间已从临时数据库备份恢复,临时备份表已删除。未绕过图形验证码,也未把登录后桌面/窄屏视觉验收伪报为通过。页面构建与真实后端/基础设施证据已通过,登录后视觉及范围切换仍需用户在本机Chrome手动打开页面,或共享已打开的具体标签页后补验。
|
||||||
|
- 本轮没有发送、补发、重投短信或创建重投任务,没有修改真实通道账号、密码、启停状态、企业余额或客户连接。工作区修复与文档尚未提交、推送;`api/tsconfig.build.tsbuildinfo`、`tsconfig.tsbuildinfo`、`outputs/`及空文件`=`继续作为受保护项,不删除、不提交、不归因。
|
||||||
|
|
||||||
|
## MinIO 正式切换补充(2026-08-14)
|
||||||
|
|
||||||
|
- 用户提供`D:\迅雷下载\minio.linux-amd64.RELEASE.2025-09-07T16-13-09Z`,本地与服务器SHA-256均为`7c5bd8512c6e966455b1d198209358b2d191c77a83ab377c4073281065fb855f`;服务器`file`确认其为静态链接Linux x86-64 ELF,运行版本为`RELEASE.2025-09-07T16-13-09Z`、Go 1.24.6。
|
||||||
|
- 切换前确认`/var/lib/cmpp-platform/object-storage`文件数为0,并将环境、MinIO环境、systemd单元和本地对象存储目录备份到`/opt/cmpp-platform-backups/releases/20260814-151145-before-minio`。安装后`cmpp-minio`与API均active,API使用`OBJECT_STORAGE_DRIVER=minio`;9000/9001只监听127.0.0.1/::1,外部探测均不可连接。
|
||||||
|
- 使用API同款MinIO Node客户端真实创建`cmpp-platform` bucket,并执行测试对象写入、读取内容比对和删除,三步均成功且测试对象已清理。API健康、Gateway、Prometheus、Node Exporter及Fail2ban继续active;未创建业务附件记录、短信、重投任务或通道连接。
|
||||||
|
|
||||||
|
# 2026-08-14 系统监控恢复、标题去重与全局预警菜单(测试服务器已部署)
|
||||||
|
|
||||||
|
- 用户在`100.93.204.60`真实页面看到系统监控整体降级。只读诊断确认Prometheus、Node Exporter、API、MinIO均active,两个采集target均为`up`,CPU、内存、磁盘、网络、负载、运行时长和systemd查询单独执行都成功;直接运行实际`InfrastructureMonitoringService`后捕获到systemd查询HTTP 400,Prometheus明确返回`unknown escape sequence '.'`。根因是TypeScript字符串只向PromQL传递单反斜杠`\.`,而Prometheus字符串层要求双反斜杠后再交给RE2。
|
||||||
|
- systemd固定PromQL改为TypeScript四反斜杠字面量,实际查询文本正确传递双反斜杠;专项测试锁定请求参数。整页降级行为继续清空陈旧数据,但新增仅含错误摘要的服务端warning,不记录PromQL、地址或凭据。部署后真实服务返回`available=true`、综合状态`healthy`、核心服务6/6、1小时CPU趋势56点,API发布后无新增监控不可用warning。
|
||||||
|
- 系统监控内容区删除重复的大号`<h1>系统监控</h1>`,保留平台通用页头、说明、状态、时间范围和刷新操作。构建产物与部署源码静态核对确认重复标题不存在。
|
||||||
|
- 右上角铃铛由签名清退直接链接改为“预警中心”弹层,分开显示“签名清退预警”和“安全检测与封禁”;审核待办继续使用独立图标和菜单。签名项读取今日未读且未抑制消息数,安全项新增轻量`GET /api/admin/security-detection/notification-summary`,只统计`open/acknowledged/block_failed`总数与严重数,不轮询完整总览、规则或安全代理。任一域失败由`Promise.allSettled`独立降级,不清空另一域。
|
||||||
|
- 本地监控/安全专项2套8项、API全量40套473项、前后端TypeScript和Vite生产构建全部通过;全量Jest仍因既有异步句柄使用`--forceExit`收尾,Vite只保留既有大chunk提示,`git diff --check`通过。
|
||||||
|
- 发布归档`outputs/cmpp-monitor-alert-fix-20260814-152652.tar.gz`及服务器副本SHA-256均为`8a00a02714ced5311b25fbc3e1cfc17f3d1a38759f1499db78c52089bc0b60b7`。发布前PostgreSQL、环境和运行源码恢复资产位于`/opt/cmpp-platform-backups/releases/20260814-152719-before-monitor-alert-fix`,三项SHA校验、源码tar目录和pg_restore清单均通过;运行标识为`d30d9ea4d0ec8a46154dfb1e39a9bba2156ec455+localfix.monitor-alert.8a00a02714ce`。
|
||||||
|
- 经用户明确要求,在该空白测试服务器真实PostgreSQL创建可清理的`qa-bell-alerts-*`预警验收数据:2条签名清退未读消息,关联一个`interfaceEnabled=false/status=disabled`的QA应用和两条QA签名;3条安全告警通过真实内部安全事件接口按规则阈值触发,使用文档保留IP`203.0.113.101-103`,其中critical 1条。铃铛真实汇总为2+3=5;封禁记录0、短信记录0,没有调用安全代理封禁、Gateway提交或供应商连接。
|
||||||
|
- Chrome中已存在登录后的测试服务器页面,但浏览器控制扩展在接管该标签页阶段持续超时,因此未伪报点击和响应式视觉验收通过。服务器真实服务、数据库、接口服务层、构建产物和静态契约均已验收;用户刷新页面即可查看,后续以用户截图继续视觉核对。
|
||||||
|
- 本轮没有发送、补发或重投短信,没有修改真实通道账号、密码、余额或客户连接。代码和文档尚未提交、推送;受保护的`*.tsbuildinfo`、`outputs/`及空文件`=`继续保留,不归因或提交。
|
||||||
|
|
||||||
|
# 2026-08-14 后台重投支持客户端已确认记录与创建/列表样式优化(本地未部署)
|
||||||
|
|
||||||
|
- 后台任务可重投状态由`pending/failed/unconfirmed/rejected`扩展为`pending/failed/unconfirmed/rejected/delivered`;状态筛选为`delivered`时,预检将客户端已确认记录计入可重投数并在真实PostgreSQL任务项保存`previousStatus=delivered`。`awaiting_ack`继续由预检计为不可重投且创建接口明确拒绝,避免确认窗口内并发写出。
|
||||||
|
- 执行器不是简单放开当前`delivered`状态:只有冻结快照原状态已经是`delivered`的任务项才允许再次调用Gateway;任务创建时为失败/待投递等状态、执行前才收到迟到成功ACK的项目会以“创建任务后已被客户确认”跳过。该判断用于保留运营明确选择已确认记录时的重复投递能力,同时防止其他任务范围意外扩大。
|
||||||
|
- 创建弹窗将原生无统一样式的`textarea`替换为平台`Textarea`,增加必填标识、少于5字错误、说明、200字计数、统一焦点态和重复投递风险提示。后台任务列表使用独立内容边框和圆角,桌面卡片边缘保留24px、行内保留20px,窄屏收敛为16px;标题区和分页同步使用设计间距。
|
||||||
|
- 已同步`docs/downstream-requeue-task-design-20260812.md` V1.1、平台需求和系统测试用例,新增已确认批量重投、迟到ACK保护及桌面/窄屏视觉用例。专项Jest 10/10、API全量41套/477项、前后端TypeScript、API正式编译、Vite 8.1.5生产构建(2549 modules)、SendChain R10结构契约和`git diff --check`通过;Vite仅保留既有约2.11MiB单chunk提示。Operations R2结构门禁仍因本轮开始前已有的`sendQuality`查询哈希漂移失败,与本次下游重投文件无交集,未为通过门禁改写其他会话业务契约。
|
||||||
|
- 本地真实PostgreSQL为88/88条migration且schema up to date。使用现有失败投递在单一数据库事务中临时改为`delivered`,真实验证状态白名单命中1条并成功物化`previousStatus=delivered`任务项1条,随后强制回滚;事务后原投递恢复`failed`且测试任务持久化数为0。该验证没有启动任务扫描、调用Gateway或发送短信。
|
||||||
|
- 本轮尚未部署测试服务器或预生产,因当前需求只授权修改且交接约束要求部署另行明确授权;没有创建真实重投任务、发送/补发/重投短信、修改通道账号、密码、启停状态、企业余额或客户连接。既有监控/预警工作区修改及`*.tsbuildinfo`、`outputs/`、空文件`=`继续保护,不归因于本次改动。
|
||||||
|
|
||||||
|
# 2026-08-14 服务内部Prometheus指标、阈值与测试机部署
|
||||||
|
|
||||||
|
- 在现有工作区上增量实施,未reset/checkout或覆盖其他会话改动。API新增独立`127.0.0.1:9464/metrics`,仅使用路由模板、HTTP方法和状态的低基数标签;Gateway回环`8090/metrics`新增Go运行时、上下游连接、Submit结果/耗时及Redis Stream pending/lag/最旧年龄。两者均只在内存计数,不写业务PostgreSQL或Redis。
|
||||||
|
- 测试机`100.93.204.60`安装Ubuntu发行版`prometheus-postgres-exporter 0.15.0`、`prometheus-redis-exporter 1.54.0`和`prometheus-nginx-exporter 1.1.0`,开启MinIO回环原生指标;Prometheus现实际采集`prometheus/node/cmpp-api/cmpp-gateway/postgresql/redis/minio/nginx`8个target,全部`up`。
|
||||||
|
- 规则扩展至61条,覆盖主机资源、API 5xx/P95/事件循环、Gateway worker/连接/队列年龄、PostgreSQL连接/死锁、Redis内存/淘汰/拒绝连接、MinIO容量/离线盘、Nginx可用性和Prometheus自监控。比例告警有最低错误样本量,Warning/Critical范围不重叠;`promtool check rules/config`通过,8个规则组计算失败计数全为0,部署后无活动告警。
|
||||||
|
- Recording Rules真实返回API P95约0.048s、Gateway pending/lag均0、PostgreSQL连接使用率3%、Redis连接10/已用约1.78MB、MinIO容量使用率约20.8%/离线盘0、Nginx活跃连接4。系统监控页新增六组“服务关键指标”卡片,只读取`cmpp:service_*`聚合;缺失指标显示破折号/待采集,不以0伪造。
|
||||||
|
- 外部真实TCP探测确认仅业务端口12026可连接;3000、8090、9000、9090、9100、9113、9121、9187、9464全部从LAN不可连接。各监控进程当时CPU合计约0.4%,Prometheus RSS约101MB、三个新Exporter RSS合计约57MB;各8个target抓取耗时0.0008至0.037s,明显低于15s周期,TSDB当时5790条series。
|
||||||
|
- 本地API专项2套5项、API全量41套/477项、API/前端TypeScript、Gateway全量`go test ./... -count=1`和`go vet ./...`通过;测试机Vite 8.0.16生产构建、API/Gateway构建和健康检查通过,仅保留既有大chunk提示。API/Gateway/Prometheus发布后warning级journal为0。
|
||||||
|
- 完整恢复资产位于`/opt/cmpp-platform-backups/releases/20260814-164653-before-service-metrics`,包含PostgreSQL、运行源码、环境/监控/Nginx/systemd配置,SHA-256和gzip校验通过。`20260814-164636-before-service-metrics`是因`pg_dump`不接受Prisma `schema` URL参数而立即停止的不完整目录,不可用于恢复;未删除以保留证据。当前运行标识为`d30d9ea4d0ec8a46154dfb1e39a9bba2156ec455+workspace.service-metrics.28575bc8a7d8.minio`。
|
||||||
|
- Edge现有测试机页面会话未登录,访问系统监控被正常引导到带图形验证码的登录页;未代填或绕过验证码,因此登录后页面视觉验收留待用户使用已有测试账号查看。本轮未发送、补发或重投短信,未修改通道账号/密码/启停、余额或客户连接;代码未提交、未推送、未发布预生产。
|
||||||
|
|
||||||
|
# 2026-08-14 跨会话工作区合并复核
|
||||||
|
|
||||||
|
- 合并复核覆盖系统监控恢复与预警菜单、服务内部Prometheus指标/Exporter/阈值、Fail2ban部署路径修复,以及后台重投支持客户端已确认记录和创建/列表样式优化。Git不存在未合并文件或冲突标记;共同修改的API模块、全局布局/样式、部署脚本、平台需求、系统用例和进度记录均已逐项核对,没有发现状态口径、路由、样式选择器或部署时间线互相覆盖。
|
||||||
|
- 测试机静态包只读核对显示当前已部署“服务关键指标”,但尚未包含“重复投递风险”和已确认重投的新表单文案,证明服务指标会话使用定向发布,没有把尚未授权部署的下游重投改动意外带入测试机;两项发布记录保持一致。
|
||||||
|
- 合并后统一回归通过:API全量41套/477项、前后端TypeScript、API正式编译、Vite 8.1.5生产构建(2549 modules)、Gateway全量`go test ./... -count=1`与`go vet ./...`、SendChain R10、生产部署和安全代理静态契约、5个Shell脚本语法、真实本地PostgreSQL 88/88 migration状态及`git diff --check`。Vite仅保留既有约2.11MiB单chunk提示;Operations R2仍因本轮开始前已有的`sendQuality`查询哈希漂移失败,与本次合并文件无交集。
|
||||||
|
- 本次合并没有发送、补发或重投短信,没有创建后台重投任务,没有修改通道账号、密码、启停状态、企业余额或客户连接;提交范围继续排除`api/tsconfig.build.tsbuildinfo`、`tsconfig.tsbuildinfo`、`outputs/`和空文件`=`。
|
||||||
|
|
||||||
|
# 2026-08-14 系统监控阈值与预警入口优化(进行中)
|
||||||
|
|
||||||
|
- 系统监控将“服务关键指标”移至活动告警正上方,标题说明明确使用 Prometheus;新增十组固定指标警告/严重阈值设置。配置真实写入 PostgreSQL,使用版本条件并发认领,promtool 校验、原子替换和回环 reload 成功后才推进生效版本,失败保留旧生效值。
|
||||||
|
- 右上角预警中心新增“系统监控告警”及数量/严重数,读取独立 Prometheus 活动告警汇总并跳转活动告警锚点;继续使用 `Promise.allSettled` 隔离签名、安全与监控域故障。安全检测页移除重复大号标题,说明明确使用 Fail2ban。
|
||||||
|
- 新增 migration `20260814173000_add_infrastructure_alert_settings`。本机 Prisma Client 生成、前后端 TypeScript 与 API 正式编译已通过;测试机恢复资产、migration、Prometheus规则校验、真实告警数据、浏览器验收和提交/部署结果待本轮后续补记。
|
||||||
|
- 首次测试机规则验收发现 Redis 未配置 `maxmemory` 时除零结果会误触发 Critical pending,立即补回 `redis_memory_max_bytes > 0` 固定保护,并补回 API 5xx 窗口至少 5 次错误的最低样本门槛;该真实验收缺陷未按通过处理。
|
||||||
|
- 第二次真实应用验证发现 API 原子 rename 后的规则文件未继承 `prometheus` 组,Prometheus reload 返回 500;数据库按设计保持旧生效版本并标记 failed。安装器将托管目录修正为 2750 SGID,确保新文件继承 `prometheus` 组,随后必须以同一真实配置重试并验证 effective。
|
||||||
|
|
||||||
|
## 发布与真实验收结果
|
||||||
|
|
||||||
|
- 功能提交 `6ccc102`、比例保护修复 `0cd0944`、规则目录权限修复 `2216d00` 已部署测试机;当前 `.deployed-commit=2216d00d511ef4daa57b44ad33cfd3e7411b913e`。API、Gateway、Prometheus、PostgreSQL、Redis、MinIO、Nginx 全部 active,API/Gateway 健康,发布后 API warning journal 无记录。
|
||||||
|
- 测试机已完成 89/89 migration。真实阈值应用从失败版本 2 重试到版本 3,数据库 `configVersion=3/effectiveVersion=3/applyStatus=effective`;托管目录为 2750、组 `prometheus`,原子生成文件为 0640、组 `prometheus`。promtool 验证基础 61 条、托管 20 条和 QA 2 条规则,Prometheus 8/8 targets up。
|
||||||
|
- 测试数据使用真实 Prometheus 规则 `/var/lib/cmpp-platform/monitoring/cmpp-qa-preview-alerts.yml` 创建 2 条隔离演示告警(1 Warning、1 Critical,均带 `qa_preview=true` 且文案说明不代表真实故障);轻量汇总真实返回 `count=2/criticalCount=1`,错误 Redis pending 已消失。
|
||||||
|
- 恢复资产分别为 `/opt/cmpp-platform-backups/releases/20260814-175325-before-6ccc102`、`/opt/cmpp-platform-backups/releases/20260814-175946-before-0cd0944`、`/opt/cmpp-platform-backups/releases/20260814-180307-before-2216d00`,均包含 PostgreSQL、源码、环境及监控/Nginx/systemd 配置并通过 SHA-256、pg_restore 与 gzip 校验。
|
||||||
|
- 本机监控专项 2 套 6 项、前后端 TypeScript、API 正式编译、Vite 生产构建和 git diff check 通过;API 全量回归两次分别在 120 秒和 300 秒到达执行时限,未取得完整通过证据,因此不记为通过。外部 Edge 当前停留测试机登录页,因没有现成登录会话且页面含验证码,本轮未代填或绕过验证码,登录后视觉验收由用户直接查看。
|
||||||
|
- 本轮没有发送、补发或重投短信,没有创建后台重投任务,没有修改通道账号、密码、启停状态、企业余额或客户连接;QA 数据仅为测试机 Prometheus 演示规则。
|
||||||
|
|
||||||
|
# 2026-08-16 系统监控弹窗滚动与活动告警已读(测试机已部署)
|
||||||
|
|
||||||
|
- 阈值设置表单移除自身 `max-height/overflow`,仅保留平台通用 Modal 内容区滚动,消除双层滚动区域。
|
||||||
|
- 新增逐管理员活动告警已读设计:真实 PostgreSQL 保存 `fingerprint + userId + activeAt + readAt`;服务端回读 Prometheus 校验当前触发周期后幂等 upsert。页面活动告警保持原始数量,铃铛轻量汇总只统计当前管理员未读,告警以相同标签重新触发但 activeAt 改变时重新计入未读。
|
||||||
|
- 新增 migration `20260816100000_add_infrastructure_alert_reads`、单条已读接口、操作日志及前端“标记已读”按钮。接口只接受当前仍活动且 `fingerprint + activeAt` 完全匹配的触发周期;重复请求保持原 `readAt` 且不重复写操作日志。
|
||||||
|
- 功能提交 `482f7ac1ae4c219e47aeaac8735c0584b7d120f2` 已部署测试机,当前 `.deployed-commit` 与该提交一致。发布前恢复资产位于 `/opt/cmpp-platform-backups/releases/20260816-150521-before-482f7ac`,包含 PostgreSQL、运行源码、环境及平台配置,SHA-256、`pg_restore --list` 和 gzip 校验通过。
|
||||||
|
- 测试机已完成 90/90 migration;API、Gateway、Prometheus、PostgreSQL、Redis、MinIO、Nginx 全部 active,Prometheus 8/8 targets up,发布时间窗 API warning 日志为 0。
|
||||||
|
- 使用测试机既有隔离 QA Prometheus 规则和真实 PostgreSQL 验证:活动告警 2 条,其中 1 条标记已读后页面仍保留 2 条,当前管理员铃铛未读数降为 1,严重未读数为 1;重复标记返回相同 `readAt`,对应操作日志只有 1 条。数据库保留 1 条 QA 已读记录,便于页面同时展示“已读”和“标记已读”状态。
|
||||||
|
- 监控专项 2 套 8 项、Prisma format/generate、前后端 TypeScript、API 正式编译、Vite 生产构建和 `git diff --check` 通过;Vite 仅保留既有大 chunk 提示。本地真实 PostgreSQL 不可用,`prisma migrate deploy` 返回 Schema engine error,因此没有伪造依赖或把本地 migration 记为通过,migration 已在测试机真实 PostgreSQL 验证。
|
||||||
|
- 最终浏览器控制会话没有可接管的现有标签页,未绕过图形验证码或另行创建登录会话,因此登录后弹窗滚动和按钮视觉点击未伪报为通过;代码样式契约、真实接口、数据库、Prometheus 与测试机部署均已验收,用户刷新测试机现有登录页面即可查看。
|
||||||
|
- 本轮没有发送、补发或重投短信,没有创建后台重投任务,没有修改通道账号、密码、启停状态、企业余额或客户连接;`api/tsconfig.build.tsbuildinfo`、`tsconfig.tsbuildinfo`、`outputs/`和空文件 `=` 继续作为受保护项排除提交。
|
||||||
|
|||||||
@@ -5,13 +5,19 @@ import (
|
|||||||
"log"
|
"log"
|
||||||
"net/http"
|
"net/http"
|
||||||
"os"
|
"os"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
"cmpp-platform/gateway/internal/control"
|
"cmpp-platform/gateway/internal/control"
|
||||||
"cmpp-platform/gateway/internal/health"
|
"cmpp-platform/gateway/internal/health"
|
||||||
"cmpp-platform/gateway/internal/inbound"
|
"cmpp-platform/gateway/internal/inbound"
|
||||||
|
platformmetrics "cmpp-platform/gateway/internal/metrics"
|
||||||
"cmpp-platform/gateway/internal/ratelimit"
|
"cmpp-platform/gateway/internal/ratelimit"
|
||||||
"cmpp-platform/gateway/internal/submitworker"
|
"cmpp-platform/gateway/internal/submitworker"
|
||||||
"cmpp-platform/gateway/internal/upstream"
|
"cmpp-platform/gateway/internal/upstream"
|
||||||
|
|
||||||
|
"github.com/redis/go-redis/v9"
|
||||||
)
|
)
|
||||||
|
|
||||||
func main() {
|
func main() {
|
||||||
@@ -25,6 +31,7 @@ func main() {
|
|||||||
}
|
}
|
||||||
apiBaseURL := os.Getenv("API_BASE_URL")
|
apiBaseURL := os.Getenv("API_BASE_URL")
|
||||||
upstreamManager := &upstream.Manager{APIBaseURL: apiBaseURL}
|
upstreamManager := &upstream.Manager{APIBaseURL: apiBaseURL}
|
||||||
|
var worker *submitworker.Worker
|
||||||
channelLimiter, err := ratelimit.New(os.Getenv("REDIS_URL"))
|
channelLimiter, err := ratelimit.New(os.Getenv("REDIS_URL"))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Fatalf("gateway channel rate limiter init failed: %v", err)
|
log.Fatalf("gateway channel rate limiter init failed: %v", err)
|
||||||
@@ -41,18 +48,19 @@ func main() {
|
|||||||
go func() {
|
go func() {
|
||||||
log.Printf("cmpp gateway inbound server listening on %s", cmppAddr)
|
log.Printf("cmpp gateway inbound server listening on %s", cmppAddr)
|
||||||
if err := (inbound.Server{
|
if err := (inbound.Server{
|
||||||
Addr: cmppAddr,
|
Addr: cmppAddr,
|
||||||
APIBaseURL: apiBaseURL,
|
APIBaseURL: apiBaseURL,
|
||||||
PresenceStore: presenceStore,
|
PresenceStore: presenceStore,
|
||||||
RecoveryStore: recoveryStore,
|
RecoveryStore: recoveryStore,
|
||||||
GatewayInstanceID: getenv("GATEWAY_INSTANCE_ID", hostname()),
|
GatewayInstanceID: getenv("GATEWAY_INSTANCE_ID", hostname()),
|
||||||
|
SecurityEventToken: os.Getenv("SECURITY_EVENT_TOKEN"),
|
||||||
}).ListenAndServe(); err != nil {
|
}).ListenAndServe(); err != nil {
|
||||||
log.Fatalf("gateway inbound server stopped: %v", err)
|
log.Fatalf("gateway inbound server stopped: %v", err)
|
||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
|
|
||||||
if os.Getenv("GATEWAY_SUBMIT_WORKER_DISABLED") != "true" {
|
if os.Getenv("GATEWAY_SUBMIT_WORKER_DISABLED") != "true" {
|
||||||
worker, err := submitworker.New(os.Getenv("REDIS_URL"), upstreamManager)
|
worker, err = submitworker.New(os.Getenv("REDIS_URL"), upstreamManager)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Printf("gateway submit worker init failed: %v", err)
|
log.Printf("gateway submit worker init failed: %v", err)
|
||||||
} else {
|
} else {
|
||||||
@@ -71,6 +79,39 @@ func main() {
|
|||||||
|
|
||||||
mux := http.NewServeMux()
|
mux := http.NewServeMux()
|
||||||
mux.Handle("/health", health.Handler())
|
mux.Handle("/health", health.Handler())
|
||||||
|
mux.Handle("/metrics", platformmetrics.Handler(func(ctx context.Context) platformmetrics.Snapshot {
|
||||||
|
desired, connected := upstreamManager.ConnectionCounts()
|
||||||
|
snapshot := platformmetrics.Snapshot{
|
||||||
|
UpstreamDesired: desired, UpstreamConnected: connected,
|
||||||
|
DownstreamConnected: inbound.ActiveConnectionCount(), SubmitWorkerUp: worker != nil,
|
||||||
|
}
|
||||||
|
if worker == nil || worker.Redis == nil {
|
||||||
|
return snapshot
|
||||||
|
}
|
||||||
|
pending, err := worker.Redis.XPending(ctx, worker.Stream, worker.Group).Result()
|
||||||
|
if err != nil {
|
||||||
|
return snapshot
|
||||||
|
}
|
||||||
|
snapshot.QueueAvailable = true
|
||||||
|
snapshot.QueuePending = pending.Count
|
||||||
|
groups, err := worker.Redis.XInfoGroups(ctx, worker.Stream).Result()
|
||||||
|
if err == nil {
|
||||||
|
for _, group := range groups {
|
||||||
|
if group.Name == worker.Group {
|
||||||
|
snapshot.QueueLag = group.Lag
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
entries, err := worker.Redis.XPendingExt(ctx, &redis.XPendingExtArgs{Stream: worker.Stream, Group: worker.Group, Start: "-", End: "+", Count: 1}).Result()
|
||||||
|
if err == nil && len(entries) > 0 {
|
||||||
|
milliseconds, parseErr := strconv.ParseInt(strings.SplitN(entries[0].ID, "-", 2)[0], 10, 64)
|
||||||
|
if parseErr == nil {
|
||||||
|
snapshot.QueueOldestAgeSeconds = max(0, time.Since(time.UnixMilli(milliseconds)).Seconds())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return snapshot
|
||||||
|
}))
|
||||||
control.Register(mux, control.Server{
|
control.Register(mux, control.Server{
|
||||||
APIBaseURL: apiBaseURL,
|
APIBaseURL: apiBaseURL,
|
||||||
Upstream: upstreamManager,
|
Upstream: upstreamManager,
|
||||||
|
|||||||
@@ -0,0 +1,395 @@
|
|||||||
|
// security-agent is the deliberately tiny privileged boundary for manual blocking.
|
||||||
|
// It accepts only a fixed JSON protocol over a Unix socket; it never invokes a shell
|
||||||
|
// and never accepts command, jail, action, path, or argument strings from NestJS.
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bufio"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"log"
|
||||||
|
"net"
|
||||||
|
"net/http"
|
||||||
|
"os"
|
||||||
|
"os/exec"
|
||||||
|
"path/filepath"
|
||||||
|
"sort"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
type request struct {
|
||||||
|
Action string `json:"action"`
|
||||||
|
OperationKey string `json:"operationKey"`
|
||||||
|
SourceIP string `json:"sourceIp"`
|
||||||
|
Executor string `json:"executor"`
|
||||||
|
DurationSeconds int `json:"durationSeconds"`
|
||||||
|
Version int `json:"version"`
|
||||||
|
Rules []rule `json:"rules"`
|
||||||
|
}
|
||||||
|
type rule struct {
|
||||||
|
Code string `json:"code"`
|
||||||
|
Enabled bool `json:"enabled"`
|
||||||
|
Threshold int `json:"threshold"`
|
||||||
|
WindowSeconds int `json:"windowSeconds"`
|
||||||
|
CooldownSeconds int `json:"cooldownSeconds"`
|
||||||
|
}
|
||||||
|
type response struct {
|
||||||
|
OK bool `json:"ok"`
|
||||||
|
Reference string `json:"reference,omitempty"`
|
||||||
|
Blocked bool `json:"blocked,omitempty"`
|
||||||
|
Active bool `json:"active,omitempty"`
|
||||||
|
Error string `json:"error,omitempty"`
|
||||||
|
}
|
||||||
|
type block struct {
|
||||||
|
OperationKey string `json:"operationKey"`
|
||||||
|
SourceIP string `json:"sourceIp"`
|
||||||
|
Executor string `json:"executor"`
|
||||||
|
ExpiresAt time.Time `json:"expiresAt"`
|
||||||
|
}
|
||||||
|
type state struct {
|
||||||
|
Blocks map[string]block `json:"blocks"`
|
||||||
|
RuleVersion int `json:"ruleVersion"`
|
||||||
|
}
|
||||||
|
|
||||||
|
var allowedDurations = map[int]bool{600: true, 3600: true, 86400: true, 604800: true}
|
||||||
|
var allowedRules = map[string]bool{"admin_login_failure": true, "client_login_failure": true, "ssh_auth_failure": true, "cmpp_auth_failure": true, "cmpp_protocol_abuse": true, "http_invalid_api_key": true, "http_signature_failure": true, "http_replay_attempt": true, "http_malicious_scan": true}
|
||||||
|
|
||||||
|
type agent struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
statePath, nginxInclude, fail2banConfig string
|
||||||
|
data state
|
||||||
|
}
|
||||||
|
|
||||||
|
func main() {
|
||||||
|
if len(os.Args) == 4 && os.Args[1] == "report" {
|
||||||
|
if err := reportEvent(os.Args[2], os.Args[3]); err != nil {
|
||||||
|
log.Fatal(err)
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
socketPath := env("SECURITY_AGENT_SOCKET", "/run/cmpp-security-agent/agent.sock")
|
||||||
|
a := &agent{statePath: env("SECURITY_AGENT_STATE", "/var/lib/cmpp-security-agent/state.json"), nginxInclude: env("SECURITY_NGINX_DENY_INCLUDE", "/etc/nginx/snippets/cmpp-security-deny.conf"), fail2banConfig: env("SECURITY_FAIL2BAN_CONFIG", "/etc/fail2ban/jail.d/cmpp-platform-generated.local"), data: state{Blocks: map[string]block{}}}
|
||||||
|
if err := a.load(); err != nil {
|
||||||
|
log.Fatalf("load state: %v", err)
|
||||||
|
}
|
||||||
|
if err := os.MkdirAll(filepath.Dir(socketPath), 0750); err != nil {
|
||||||
|
log.Fatal(err)
|
||||||
|
}
|
||||||
|
_ = os.Remove(socketPath)
|
||||||
|
listener, err := net.Listen("unix", socketPath)
|
||||||
|
if err != nil {
|
||||||
|
log.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.Chmod(socketPath, 0660); err != nil {
|
||||||
|
log.Fatal(err)
|
||||||
|
}
|
||||||
|
defer listener.Close()
|
||||||
|
log.Printf("security agent listening on %s", socketPath)
|
||||||
|
for {
|
||||||
|
connection, err := listener.Accept()
|
||||||
|
if err != nil {
|
||||||
|
log.Printf("accept: %v", err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
go a.serve(connection)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (a *agent) serve(connection net.Conn) {
|
||||||
|
defer connection.Close()
|
||||||
|
_ = connection.SetDeadline(time.Now().Add(5 * time.Second))
|
||||||
|
var req request
|
||||||
|
if err := json.NewDecoder(bufio.NewReader(connection)).Decode(&req); err != nil {
|
||||||
|
write(connection, response{Error: "invalid request"})
|
||||||
|
return
|
||||||
|
}
|
||||||
|
a.mu.Lock()
|
||||||
|
defer a.mu.Unlock()
|
||||||
|
a.prune()
|
||||||
|
var result response
|
||||||
|
switch req.Action {
|
||||||
|
case "block":
|
||||||
|
result = a.block(req)
|
||||||
|
case "unblock":
|
||||||
|
result = a.unblock(req)
|
||||||
|
case "status":
|
||||||
|
result = a.status(req)
|
||||||
|
case "apply_rules":
|
||||||
|
result = a.applyRules(req)
|
||||||
|
default:
|
||||||
|
result = response{Error: "unsupported action"}
|
||||||
|
}
|
||||||
|
write(connection, result)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (a *agent) block(req request) response {
|
||||||
|
if net.ParseIP(req.SourceIP) == nil || !allowedDurations[req.DurationSeconds] || (req.Executor != "nftables" && req.Executor != "nginx_real_ip") || len(req.OperationKey) < 16 {
|
||||||
|
return response{Error: "invalid fixed block parameters"}
|
||||||
|
}
|
||||||
|
if existing, ok := a.data.Blocks[req.OperationKey]; ok {
|
||||||
|
return response{OK: true, Reference: req.OperationKey, Blocked: existing.ExpiresAt.After(time.Now())}
|
||||||
|
}
|
||||||
|
if req.Executor == "nftables" {
|
||||||
|
if err := nftBlock(req.SourceIP, req.DurationSeconds); err != nil {
|
||||||
|
return response{Error: err.Error()}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
a.data.Blocks[req.OperationKey] = block{OperationKey: req.OperationKey, SourceIP: req.SourceIP, Executor: req.Executor, ExpiresAt: time.Now().Add(time.Duration(req.DurationSeconds) * time.Second)}
|
||||||
|
if req.Executor == "nginx_real_ip" {
|
||||||
|
if err := a.writeNginx(); err != nil {
|
||||||
|
delete(a.data.Blocks, req.OperationKey)
|
||||||
|
return response{Error: err.Error()}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := a.save(); err != nil {
|
||||||
|
return response{Error: err.Error()}
|
||||||
|
}
|
||||||
|
return a.status(req)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (a *agent) unblock(req request) response {
|
||||||
|
if net.ParseIP(req.SourceIP) == nil || (req.Executor != "nftables" && req.Executor != "nginx_real_ip") {
|
||||||
|
return response{Error: "invalid fixed unblock parameters"}
|
||||||
|
}
|
||||||
|
if req.Executor == "nftables" {
|
||||||
|
if err := nftUnblock(req.SourceIP); err != nil {
|
||||||
|
return response{Error: err.Error()}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for key, item := range a.data.Blocks {
|
||||||
|
if item.SourceIP == req.SourceIP && item.Executor == req.Executor {
|
||||||
|
delete(a.data.Blocks, key)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if req.Executor == "nginx_real_ip" {
|
||||||
|
if err := a.writeNginx(); err != nil {
|
||||||
|
return response{Error: err.Error()}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := a.save(); err != nil {
|
||||||
|
return response{Error: err.Error()}
|
||||||
|
}
|
||||||
|
return response{OK: true, Reference: req.OperationKey}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (a *agent) status(req request) response {
|
||||||
|
active := command("systemctl", "is-active", "--quiet", "fail2ban") == nil
|
||||||
|
if req.SourceIP == "" {
|
||||||
|
return response{OK: true, Active: active}
|
||||||
|
}
|
||||||
|
blocked := false
|
||||||
|
if req.Executor == "nftables" {
|
||||||
|
family := "blocked_ipv6"
|
||||||
|
if net.ParseIP(req.SourceIP).To4() != nil {
|
||||||
|
family = "blocked_ipv4"
|
||||||
|
}
|
||||||
|
output, err := exec.Command("nft", "list", "set", "inet", "cmpp_security", family).CombinedOutput()
|
||||||
|
if err != nil {
|
||||||
|
return response{Error: "nftables readback failed: " + strings.TrimSpace(string(output))}
|
||||||
|
}
|
||||||
|
blocked = strings.Contains(string(output), req.SourceIP+" timeout") || strings.Contains(string(output), req.SourceIP+" expires")
|
||||||
|
} else if req.Executor == "nginx_real_ip" {
|
||||||
|
content, err := os.ReadFile(a.nginxInclude)
|
||||||
|
if err != nil {
|
||||||
|
return response{Error: "nginx deny readback failed: " + err.Error()}
|
||||||
|
}
|
||||||
|
blocked = strings.Contains(string(content), "deny "+req.SourceIP+";")
|
||||||
|
} else {
|
||||||
|
return response{Error: "invalid executor"}
|
||||||
|
}
|
||||||
|
for _, item := range a.data.Blocks {
|
||||||
|
if blocked && item.SourceIP == req.SourceIP && item.Executor == req.Executor && item.ExpiresAt.After(time.Now()) {
|
||||||
|
return response{OK: true, Active: active, Blocked: true, Reference: item.OperationKey}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return response{OK: true, Active: active, Blocked: false}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (a *agent) applyRules(req request) response {
|
||||||
|
if req.Version <= a.data.RuleVersion {
|
||||||
|
return response{OK: true, Reference: strconv.Itoa(a.data.RuleVersion)}
|
||||||
|
}
|
||||||
|
for _, item := range req.Rules {
|
||||||
|
if !allowedRules[item.Code] || item.Threshold < 1 || item.Threshold > 100000 || item.WindowSeconds < 10 || item.WindowSeconds > 86400 || item.CooldownSeconds < 0 || item.CooldownSeconds > 604800 {
|
||||||
|
return response{Error: "invalid fixed rule configuration"}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
var ssh, scan *rule
|
||||||
|
for index := range req.Rules {
|
||||||
|
if req.Rules[index].Code == "ssh_auth_failure" {
|
||||||
|
ssh = &req.Rules[index]
|
||||||
|
}
|
||||||
|
if req.Rules[index].Code == "http_malicious_scan" {
|
||||||
|
scan = &req.Rules[index]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
content := "# Generated by cmpp-security-agent. Manual changes will be overwritten.\n"
|
||||||
|
if ssh != nil {
|
||||||
|
content += jail("sshd", *ssh)
|
||||||
|
}
|
||||||
|
if scan != nil {
|
||||||
|
content += jail("cmpp-http-scan", *scan)
|
||||||
|
}
|
||||||
|
previous, previousErr := os.ReadFile(a.fail2banConfig)
|
||||||
|
if err := atomicWrite(a.fail2banConfig, []byte(content), 0640); err != nil {
|
||||||
|
return response{Error: err.Error()}
|
||||||
|
}
|
||||||
|
if err := command("fail2ban-client", "-t"); err != nil {
|
||||||
|
restore(a.fail2banConfig, previous, previousErr, 0640)
|
||||||
|
return response{Error: "fail2ban validation failed: " + err.Error()}
|
||||||
|
}
|
||||||
|
if err := command("fail2ban-client", "reload"); err != nil {
|
||||||
|
restore(a.fail2banConfig, previous, previousErr, 0640)
|
||||||
|
return response{Error: "fail2ban reload failed: " + err.Error()}
|
||||||
|
}
|
||||||
|
a.data.RuleVersion = req.Version
|
||||||
|
if err := a.save(); err != nil {
|
||||||
|
return response{Error: err.Error()}
|
||||||
|
}
|
||||||
|
return response{OK: true, Reference: strconv.Itoa(req.Version), Active: true}
|
||||||
|
}
|
||||||
|
|
||||||
|
func jail(name string, item rule) string {
|
||||||
|
enabled := "false"
|
||||||
|
if item.Enabled {
|
||||||
|
enabled = "true"
|
||||||
|
}
|
||||||
|
bantime := item.CooldownSeconds
|
||||||
|
if bantime < item.WindowSeconds {
|
||||||
|
bantime = item.WindowSeconds
|
||||||
|
}
|
||||||
|
return fmt.Sprintf("\n[%s]\nenabled = %s\nfindtime = %d\nmaxretry = %d\nbantime = %d\naction = cmpp-report-only\n", name, enabled, item.WindowSeconds, item.Threshold, bantime)
|
||||||
|
}
|
||||||
|
func nftBlock(ip string, seconds int) error {
|
||||||
|
family := "blocked_ipv6"
|
||||||
|
if net.ParseIP(ip).To4() != nil {
|
||||||
|
family = "blocked_ipv4"
|
||||||
|
}
|
||||||
|
return command("nft", "add", "element", "inet", "cmpp_security", family, fmt.Sprintf("{ %s timeout %ds }", ip, seconds))
|
||||||
|
}
|
||||||
|
func nftUnblock(ip string) error {
|
||||||
|
family := "blocked_ipv6"
|
||||||
|
if net.ParseIP(ip).To4() != nil {
|
||||||
|
family = "blocked_ipv4"
|
||||||
|
}
|
||||||
|
err := command("nft", "delete", "element", "inet", "cmpp_security", family, fmt.Sprintf("{ %s }", ip))
|
||||||
|
if err != nil && strings.Contains(err.Error(), "No such file") {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
func (a *agent) writeNginx() error {
|
||||||
|
ips := []string{}
|
||||||
|
for _, item := range a.data.Blocks {
|
||||||
|
if item.Executor == "nginx_real_ip" && item.ExpiresAt.After(time.Now()) {
|
||||||
|
ips = append(ips, item.SourceIP)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
sort.Strings(ips)
|
||||||
|
lines := []string{"# Generated by cmpp-security-agent."}
|
||||||
|
for _, ip := range ips {
|
||||||
|
lines = append(lines, "deny "+ip+";")
|
||||||
|
}
|
||||||
|
previous, previousErr := os.ReadFile(a.nginxInclude)
|
||||||
|
if err := atomicWrite(a.nginxInclude, []byte(strings.Join(lines, "\n")+"\n"), 0640); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := command("nginx", "-t"); err != nil {
|
||||||
|
restore(a.nginxInclude, previous, previousErr, 0640)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := command("systemctl", "reload", "nginx"); err != nil {
|
||||||
|
restore(a.nginxInclude, previous, previousErr, 0640)
|
||||||
|
_ = command("systemctl", "reload", "nginx")
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
func (a *agent) prune() {
|
||||||
|
changed := false
|
||||||
|
for key, item := range a.data.Blocks {
|
||||||
|
if !item.ExpiresAt.After(time.Now()) {
|
||||||
|
delete(a.data.Blocks, key)
|
||||||
|
changed = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if changed {
|
||||||
|
_ = a.writeNginx()
|
||||||
|
_ = a.save()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
func (a *agent) load() error {
|
||||||
|
bytes, err := os.ReadFile(a.statePath)
|
||||||
|
if errors.Is(err, os.ErrNotExist) {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return json.Unmarshal(bytes, &a.data)
|
||||||
|
}
|
||||||
|
func (a *agent) save() error {
|
||||||
|
bytes, err := json.MarshalIndent(a.data, "", " ")
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return atomicWrite(a.statePath, bytes, 0600)
|
||||||
|
}
|
||||||
|
func atomicWrite(path string, bytes []byte, mode os.FileMode) error {
|
||||||
|
if err := os.MkdirAll(filepath.Dir(path), 0750); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
temporary := path + ".tmp"
|
||||||
|
if err := os.WriteFile(temporary, bytes, mode); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return os.Rename(temporary, path)
|
||||||
|
}
|
||||||
|
func restore(path string, previous []byte, previousErr error, mode os.FileMode) {
|
||||||
|
if previousErr == nil {
|
||||||
|
_ = atomicWrite(path, previous, mode)
|
||||||
|
} else if errors.Is(previousErr, os.ErrNotExist) {
|
||||||
|
_ = os.Remove(path)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
func command(name string, args ...string) error {
|
||||||
|
output, err := exec.Command(name, args...).CombinedOutput()
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("%s: %s", err, strings.TrimSpace(string(output)))
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
func write(connection net.Conn, value response) { _ = json.NewEncoder(connection).Encode(value) }
|
||||||
|
func env(name, fallback string) string {
|
||||||
|
if value := strings.TrimSpace(os.Getenv(name)); value != "" {
|
||||||
|
return value
|
||||||
|
}
|
||||||
|
return fallback
|
||||||
|
}
|
||||||
|
func reportEvent(jail, ip string) error {
|
||||||
|
ruleCode := map[string]string{"sshd": "ssh_auth_failure", "cmpp-http-scan": "http_malicious_scan"}[jail]
|
||||||
|
if ruleCode == "" || net.ParseIP(ip) == nil {
|
||||||
|
return errors.New("unsupported report event")
|
||||||
|
}
|
||||||
|
body := fmt.Sprintf(`{"ruleCode":%q,"sourceIp":%q,"protocol":%q,"resultCode":%q}`, ruleCode, ip, "fail2ban", jail)
|
||||||
|
client := &http.Client{Timeout: 3 * time.Second}
|
||||||
|
request, err := http.NewRequest(http.MethodPost, env("SECURITY_EVENT_URL", "http://127.0.0.1:3000/api/gateway/events/security-detection"), strings.NewReader(body))
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
request.Header.Set("Content-Type", "application/json")
|
||||||
|
request.Header.Set("X-Security-Event-Token", os.Getenv("SECURITY_EVENT_TOKEN"))
|
||||||
|
response, err := client.Do(request)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer response.Body.Close()
|
||||||
|
if response.StatusCode < 200 || response.StatusCode >= 300 {
|
||||||
|
return fmt.Errorf("event collector returned %s", response.Status)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
@@ -37,10 +37,12 @@ func (s Server) handleLogin(response *cmpp.Response, packet *cmpp.Packet, logger
|
|||||||
}
|
}
|
||||||
account := strings.TrimRight(req.SrcAddr, "\x00")
|
account := strings.TrimRight(req.SrcAddr, "\x00")
|
||||||
if account == "" {
|
if account == "" {
|
||||||
|
go s.reportSecurityEvent("cmpp_protocol_abuse", remoteIP(packet.Conn.Conn.RemoteAddr()), "EMPTY_SOURCE_ADDRESS", cmppVersionName(req.Version))
|
||||||
setInboundConnectResponse(response.Packer, cmpp.ErrnoConnInvalidSrcAddr, req.AuthSrc, "", req.Version)
|
setInboundConnectResponse(response.Packer, cmpp.ErrnoConnInvalidSrcAddr, req.AuthSrc, "", req.Version)
|
||||||
return false, cmpp.ConnRspStatusErrMap[cmpp.ErrnoConnInvalidSrcAddr]
|
return false, cmpp.ConnRspStatusErrMap[cmpp.ErrnoConnInvalidSrcAddr]
|
||||||
}
|
}
|
||||||
if req.Version != cmpp.V20 && req.Version != cmpp.V21 && req.Version != cmpp.V30 {
|
if req.Version != cmpp.V20 && req.Version != cmpp.V21 && req.Version != cmpp.V30 {
|
||||||
|
go s.reportSecurityEvent("cmpp_protocol_abuse", remoteIP(packet.Conn.Conn.RemoteAddr()), "UNSUPPORTED_VERSION", cmppVersionName(req.Version))
|
||||||
setInboundConnectResponse(response.Packer, cmpp.ErrnoConnVerTooHigh, req.AuthSrc, "", cmpp.V30)
|
setInboundConnectResponse(response.Packer, cmpp.ErrnoConnVerTooHigh, req.AuthSrc, "", cmpp.V30)
|
||||||
return false, cmpp.ConnRspStatusErrMap[cmpp.ErrnoConnVerTooHigh]
|
return false, cmpp.ConnRspStatusErrMap[cmpp.ErrnoConnVerTooHigh]
|
||||||
}
|
}
|
||||||
@@ -129,3 +131,18 @@ func (s Server) authenticate(remote net.Addr, account string, authSource string,
|
|||||||
err := s.post(context.Background(), "/gateway/events/inbound/authenticate", payload, &result)
|
err := s.post(context.Background(), "/gateway/events/inbound/authenticate", payload, &result)
|
||||||
return result, err
|
return result, err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (s Server) reportSecurityEvent(ruleCode, sourceIP, resultCode, protocol string) {
|
||||||
|
if net.ParseIP(sourceIP) == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
payload := map[string]any{
|
||||||
|
"ruleCode": ruleCode, "sourceIp": sourceIP, "resultCode": resultCode, "protocol": protocol,
|
||||||
|
}
|
||||||
|
var result struct {
|
||||||
|
Accepted bool `json:"accepted"`
|
||||||
|
}
|
||||||
|
if err := s.post(context.Background(), "/gateway/events/security-detection", payload, &result); err != nil {
|
||||||
|
log.Printf("security event report failed rule=%s remote=%s err=%v", ruleCode, sourceIP, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -13,6 +13,7 @@ const defaultHTTPTimeout = 10 * time.Second
|
|||||||
type Server struct {
|
type Server struct {
|
||||||
Addr string
|
Addr string
|
||||||
APIBaseURL string
|
APIBaseURL string
|
||||||
|
SecurityEventToken string
|
||||||
HTTPClient *http.Client
|
HTTPClient *http.Client
|
||||||
LogWriter io.Writer
|
LogWriter io.Writer
|
||||||
PendingFlushInterval time.Duration
|
PendingFlushInterval time.Duration
|
||||||
|
|||||||
@@ -201,6 +201,13 @@ func onlineAccounts() []string {
|
|||||||
return accounts
|
return accounts
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ActiveConnectionCount exposes only an aggregate gauge; account and remote-IP labels are intentionally excluded.
|
||||||
|
func ActiveConnectionCount() int {
|
||||||
|
downstreamRegistry.RLock()
|
||||||
|
defer downstreamRegistry.RUnlock()
|
||||||
|
return len(downstreamRegistry.byConn)
|
||||||
|
}
|
||||||
|
|
||||||
// DisconnectAccount closes every live downstream CMPP session for an
|
// DisconnectAccount closes every live downstream CMPP session for an
|
||||||
// application account. The normal connection-close callback removes registry
|
// application account. The normal connection-close callback removes registry
|
||||||
// and presence state and reports the disconnect to the API.
|
// and presence state and reports the disconnect to the API.
|
||||||
|
|||||||
@@ -28,6 +28,9 @@ func (s Server) post(ctx context.Context, path string, payload any, result any)
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
req.Header.Set("Content-Type", "application/json")
|
req.Header.Set("Content-Type", "application/json")
|
||||||
|
if strings.HasSuffix(path, "/security-detection") && s.SecurityEventToken != "" {
|
||||||
|
req.Header.Set("X-Security-Event-Token", s.SecurityEventToken)
|
||||||
|
}
|
||||||
resp, err := client.Do(req)
|
resp, err := client.Do(req)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
|
|||||||
@@ -0,0 +1,80 @@
|
|||||||
|
package metrics
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"net/http"
|
||||||
|
"runtime"
|
||||||
|
"sync/atomic"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
var startedAt = time.Now()
|
||||||
|
var submitAccepted atomic.Uint64
|
||||||
|
var submitFailed atomic.Uint64
|
||||||
|
var submitDurationNanoseconds atomic.Uint64
|
||||||
|
|
||||||
|
type Snapshot struct {
|
||||||
|
UpstreamDesired int
|
||||||
|
UpstreamConnected int
|
||||||
|
DownstreamConnected int
|
||||||
|
SubmitWorkerUp bool
|
||||||
|
QueueAvailable bool
|
||||||
|
QueuePending int64
|
||||||
|
QueueLag int64
|
||||||
|
QueueOldestAgeSeconds float64
|
||||||
|
}
|
||||||
|
|
||||||
|
type SnapshotFunc func(context.Context) Snapshot
|
||||||
|
|
||||||
|
func ObserveSubmit(accepted bool, duration time.Duration) {
|
||||||
|
if accepted {
|
||||||
|
submitAccepted.Add(1)
|
||||||
|
} else {
|
||||||
|
submitFailed.Add(1)
|
||||||
|
}
|
||||||
|
submitDurationNanoseconds.Add(uint64(max(duration, 0)))
|
||||||
|
}
|
||||||
|
|
||||||
|
func Handler(load SnapshotFunc) http.Handler {
|
||||||
|
return http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) {
|
||||||
|
if request.Method != http.MethodGet || request.URL.Path != "/metrics" {
|
||||||
|
response.WriteHeader(http.StatusNotFound)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
ctx, cancel := context.WithTimeout(request.Context(), time.Second)
|
||||||
|
defer cancel()
|
||||||
|
snapshot := Snapshot{}
|
||||||
|
if load != nil {
|
||||||
|
snapshot = load(ctx)
|
||||||
|
}
|
||||||
|
var memory runtime.MemStats
|
||||||
|
runtime.ReadMemStats(&memory)
|
||||||
|
accepted := submitAccepted.Load()
|
||||||
|
failed := submitFailed.Load()
|
||||||
|
count := accepted + failed
|
||||||
|
response.Header().Set("Content-Type", "text/plain; version=0.0.4; charset=utf-8")
|
||||||
|
response.Header().Set("Cache-Control", "no-store")
|
||||||
|
fmt.Fprintf(response, "# HELP cmpp_gateway_process_uptime_seconds Gateway process uptime.\n# TYPE cmpp_gateway_process_uptime_seconds gauge\ncmpp_gateway_process_uptime_seconds %f\n", time.Since(startedAt).Seconds())
|
||||||
|
fmt.Fprintf(response, "# HELP cmpp_gateway_go_goroutines Current goroutine count.\n# TYPE cmpp_gateway_go_goroutines gauge\ncmpp_gateway_go_goroutines %d\n", runtime.NumGoroutine())
|
||||||
|
fmt.Fprintf(response, "# HELP cmpp_gateway_go_heap_alloc_bytes Current Go heap allocation.\n# TYPE cmpp_gateway_go_heap_alloc_bytes gauge\ncmpp_gateway_go_heap_alloc_bytes %d\n", memory.HeapAlloc)
|
||||||
|
fmt.Fprintf(response, "# HELP cmpp_gateway_submit_total Upstream submit attempts by bounded result.\n# TYPE cmpp_gateway_submit_total counter\ncmpp_gateway_submit_total{result=\"accepted\"} %d\ncmpp_gateway_submit_total{result=\"failed\"} %d\n", accepted, failed)
|
||||||
|
fmt.Fprintf(response, "# HELP cmpp_gateway_submit_duration_seconds_sum Total upstream submit duration.\n# TYPE cmpp_gateway_submit_duration_seconds_sum counter\ncmpp_gateway_submit_duration_seconds_sum %f\n", float64(submitDurationNanoseconds.Load())/float64(time.Second))
|
||||||
|
fmt.Fprintf(response, "# HELP cmpp_gateway_submit_duration_seconds_count Total measured upstream submits.\n# TYPE cmpp_gateway_submit_duration_seconds_count counter\ncmpp_gateway_submit_duration_seconds_count %d\n", count)
|
||||||
|
fmt.Fprintf(response, "# HELP cmpp_gateway_upstream_connections Desired and live supplier connections.\n# TYPE cmpp_gateway_upstream_connections gauge\ncmpp_gateway_upstream_connections{state=\"desired\"} %d\ncmpp_gateway_upstream_connections{state=\"connected\"} %d\n", snapshot.UpstreamDesired, snapshot.UpstreamConnected)
|
||||||
|
fmt.Fprintf(response, "# HELP cmpp_gateway_downstream_connections Authenticated client connections.\n# TYPE cmpp_gateway_downstream_connections gauge\ncmpp_gateway_downstream_connections %d\n", snapshot.DownstreamConnected)
|
||||||
|
fmt.Fprintf(response, "# HELP cmpp_gateway_submit_worker_up Whether the submit worker was initialized.\n# TYPE cmpp_gateway_submit_worker_up gauge\ncmpp_gateway_submit_worker_up %d\n", boolNumber(snapshot.SubmitWorkerUp))
|
||||||
|
if snapshot.QueueAvailable {
|
||||||
|
fmt.Fprintf(response, "# HELP cmpp_gateway_submit_queue_pending Pending entries owned by the consumer group.\n# TYPE cmpp_gateway_submit_queue_pending gauge\ncmpp_gateway_submit_queue_pending %d\n", snapshot.QueuePending)
|
||||||
|
fmt.Fprintf(response, "# HELP cmpp_gateway_submit_queue_lag Undelivered entries for the consumer group.\n# TYPE cmpp_gateway_submit_queue_lag gauge\ncmpp_gateway_submit_queue_lag %d\n", snapshot.QueueLag)
|
||||||
|
fmt.Fprintf(response, "# HELP cmpp_gateway_submit_queue_oldest_pending_age_seconds Age of the oldest pending entry.\n# TYPE cmpp_gateway_submit_queue_oldest_pending_age_seconds gauge\ncmpp_gateway_submit_queue_oldest_pending_age_seconds %f\n", snapshot.QueueOldestAgeSeconds)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func boolNumber(value bool) int {
|
||||||
|
if value {
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
return 0
|
||||||
|
}
|
||||||
@@ -0,0 +1,29 @@
|
|||||||
|
package metrics
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestMetricsHandlerExportsOnlyAggregateGatewayState(t *testing.T) {
|
||||||
|
ObserveSubmit(true, 20*time.Millisecond)
|
||||||
|
request := httptest.NewRequest(http.MethodGet, "/metrics", nil)
|
||||||
|
response := httptest.NewRecorder()
|
||||||
|
Handler(func(context.Context) Snapshot {
|
||||||
|
return Snapshot{UpstreamDesired: 2, UpstreamConnected: 1, DownstreamConnected: 3, SubmitWorkerUp: true, QueueAvailable: true, QueuePending: 4, QueueLag: 5, QueueOldestAgeSeconds: 12}
|
||||||
|
}).ServeHTTP(response, request)
|
||||||
|
|
||||||
|
body := response.Body.String()
|
||||||
|
if response.Code != http.StatusOK || !strings.Contains(body, "cmpp_gateway_submit_queue_pending 4") || !strings.Contains(body, "cmpp_gateway_upstream_connections{state=\"connected\"} 1") {
|
||||||
|
t.Fatalf("unexpected metrics response: code=%d body=%s", response.Code, body)
|
||||||
|
}
|
||||||
|
for _, forbidden := range []string{"phone_number", "message_id", "channel_id"} {
|
||||||
|
if strings.Contains(body, forbidden) {
|
||||||
|
t.Fatalf("metrics expose forbidden label %q", forbidden)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -12,6 +12,7 @@ import (
|
|||||||
"sync"
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"cmpp-platform/gateway/internal/metrics"
|
||||||
"cmpp-platform/gateway/internal/queue"
|
"cmpp-platform/gateway/internal/queue"
|
||||||
"cmpp-platform/gateway/internal/ratelimit"
|
"cmpp-platform/gateway/internal/ratelimit"
|
||||||
"cmpp-platform/gateway/internal/upstream"
|
"cmpp-platform/gateway/internal/upstream"
|
||||||
@@ -206,6 +207,7 @@ func (w *Worker) processMessage(ctx context.Context, message redis.XMessage) err
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (w *Worker) handleCommand(ctx context.Context, command queue.SubmitCommand) error {
|
func (w *Worker) handleCommand(ctx context.Context, command queue.SubmitCommand) error {
|
||||||
|
startedAt := time.Now()
|
||||||
if w.Limiter != nil {
|
if w.Limiter != nil {
|
||||||
if _, err := w.Limiter.Wait(ctx, command.ChannelID, command.Route.RateLimitPerSecond); err != nil {
|
if _, err := w.Limiter.Wait(ctx, command.ChannelID, command.Route.RateLimitPerSecond); err != nil {
|
||||||
return err
|
return err
|
||||||
@@ -219,6 +221,8 @@ func (w *Worker) handleCommand(ctx context.Context, command queue.SubmitCommand)
|
|||||||
submit = w.Upstream.Submit
|
submit = w.Upstream.Submit
|
||||||
}
|
}
|
||||||
result, err := submit(ctx, command)
|
result, err := submit(ctx, command)
|
||||||
|
accepted := err == nil && (result.SubmitStatus == "" || result.SubmitStatus == "accepted")
|
||||||
|
metrics.ObserveSubmit(accepted, time.Since(startedAt))
|
||||||
if err != nil && (result.SubmitStatus == "" || result.SubmitStatus == "accepted") {
|
if err != nil && (result.SubmitStatus == "" || result.SubmitStatus == "accepted") {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -146,6 +146,21 @@ func (m *Manager) ensureDefaultsLocked() {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ConnectionCounts returns bounded platform totals without channel identifiers to prevent time-series cardinality growth.
|
||||||
|
func (m *Manager) ConnectionCounts() (desired int, connected int) {
|
||||||
|
m.mu.Lock()
|
||||||
|
pools := make([]*connectionPool, 0, len(m.conns))
|
||||||
|
for _, pool := range m.conns {
|
||||||
|
pools = append(pools, pool)
|
||||||
|
}
|
||||||
|
m.mu.Unlock()
|
||||||
|
for _, pool := range pools {
|
||||||
|
desired += max(pool.config.DesiredConnections, 1)
|
||||||
|
connected += pool.countActiveConnections()
|
||||||
|
}
|
||||||
|
return desired, connected
|
||||||
|
}
|
||||||
|
|
||||||
func (m *Manager) newConnectionPool(channelID string, connectionID string, config queue.UpstreamConfig) *connectionPool {
|
func (m *Manager) newConnectionPool(channelID string, connectionID string, config queue.UpstreamConfig) *connectionPool {
|
||||||
return &connectionPool{
|
return &connectionPool{
|
||||||
channelID: channelID,
|
channelID: channelID,
|
||||||
|
|||||||
+2
-1
@@ -11,7 +11,8 @@
|
|||||||
"start:local": "powershell -NoProfile -ExecutionPolicy Bypass -File tools/start-local.ps1",
|
"start:local": "powershell -NoProfile -ExecutionPolicy Bypass -File tools/start-local.ps1",
|
||||||
"start:local:minio": "powershell -NoProfile -ExecutionPolicy Bypass -File tools/start-local.ps1 -OnlyMinio",
|
"start:local:minio": "powershell -NoProfile -ExecutionPolicy Bypass -File tools/start-local.ps1 -OnlyMinio",
|
||||||
"prisma:generate": "npm --prefix api run prisma:generate",
|
"prisma:generate": "npm --prefix api run prisma:generate",
|
||||||
"security:verify": "node tools/security/verify-dependency-mitigations.mjs",
|
"security:verify": "node tools/security/verify-dependency-mitigations.mjs && node tools/security/verify-security-deployment.mjs",
|
||||||
|
"deploy:verify": "node tools/deploy/verify-production-deployment.mjs",
|
||||||
"spike:contracts": "node tools/spike/validate-gateway-queue-contract.mjs",
|
"spike:contracts": "node tools/spike/validate-gateway-queue-contract.mjs",
|
||||||
"spike:gateway": "powershell -NoProfile -ExecutionPolicy Bypass -Command \"$env:Path='C:\\Program Files\\Go\\bin;'+$env:Path; Push-Location gateway; go test ./...; Pop-Location\"",
|
"spike:gateway": "powershell -NoProfile -ExecutionPolicy Bypass -Command \"$env:Path='C:\\Program Files\\Go\\bin;'+$env:Path; Push-Location gateway; go test ./...; Pop-Location\"",
|
||||||
"spike:bullmq": "node api/src/spike/bullmq-link-spike.mjs",
|
"spike:bullmq": "node api/src/spike/bullmq-link-spike.mjs",
|
||||||
|
|||||||
@@ -0,0 +1,11 @@
|
|||||||
|
import { request, withQuery } from '../core/httpClient';
|
||||||
|
import type { InfrastructureAlertSettings, InfrastructureAlertThresholds, InfrastructureMonitoringOverview, InfrastructureMonitoringRange } from '../types';
|
||||||
|
|
||||||
|
export const adminInfrastructureMonitoringApi = {
|
||||||
|
getInfrastructureMonitoringOverview: (range: InfrastructureMonitoringRange) =>
|
||||||
|
request<InfrastructureMonitoringOverview>(withQuery('/admin/infrastructure-monitoring/overview', { range })),
|
||||||
|
getInfrastructureMonitoringNotificationSummary: () => request<{ count: number; criticalCount: number }>('/admin/infrastructure-monitoring/notification-summary'),
|
||||||
|
getInfrastructureAlertThresholds: () => request<InfrastructureAlertSettings>('/admin/infrastructure-monitoring/alert-thresholds'),
|
||||||
|
updateInfrastructureAlertThresholds: (body: { configVersion: number; thresholds: InfrastructureAlertThresholds }) => request<InfrastructureAlertSettings>('/admin/infrastructure-monitoring/alert-thresholds', { method: 'PUT', body: JSON.stringify(body) }),
|
||||||
|
markInfrastructureAlertRead: (fingerprint: string, activeAt: string) => request<{ fingerprint: string; activeAt: string; acknowledged: true; acknowledgedAt: string }>(`/admin/infrastructure-monitoring/alerts/${fingerprint}/read`, { method: 'POST', body: JSON.stringify({ activeAt }) }),
|
||||||
|
};
|
||||||
@@ -0,0 +1,16 @@
|
|||||||
|
import { request, withQuery } from '../core/httpClient';
|
||||||
|
import type { SecurityAlert, SecurityBlock, SecurityOverview, SecurityProtectedNetwork, SecurityRule } from '../types';
|
||||||
|
|
||||||
|
export const adminSecurityDetectionApi = {
|
||||||
|
getSecurityOverview: (range = '24h') => request<SecurityOverview>(withQuery('/admin/security-detection/overview', { range })),
|
||||||
|
getSecurityNotificationSummary: () => request<{ count: number; criticalCount: number }>('/admin/security-detection/notification-summary'),
|
||||||
|
listSecurityAlerts: (query: Record<string, string | number | undefined> = {}) => request<{ items: SecurityAlert[]; total: number }> (withQuery('/admin/security-detection/alerts', query)),
|
||||||
|
listSecurityRules: () => request<SecurityRule[]>('/admin/security-detection/rules'),
|
||||||
|
updateSecurityRule: (id: string, body: Partial<SecurityRule>) => request<SecurityRule>(`/admin/security-detection/rules/${id}`, { method: 'PUT', body: JSON.stringify(body) }),
|
||||||
|
blockSecurityAlert: (id: string, body: { durationSeconds: number; reason: string }) => request<SecurityBlock>(`/admin/security-detection/alerts/${id}/block`, { method: 'POST', body: JSON.stringify(body) }),
|
||||||
|
ignoreSecurityAlert: (id: string, reason: string) => request<{ success: boolean }>(`/admin/security-detection/alerts/${id}/ignore`, { method: 'POST', body: JSON.stringify({ reason }) }),
|
||||||
|
listSecurityBlocks: () => request<SecurityBlock[]>('/admin/security-detection/blocks'),
|
||||||
|
unblockSecurityBlock: (id: string, reason: string) => request<SecurityBlock>(`/admin/security-detection/blocks/${id}/unblock`, { method: 'POST', body: JSON.stringify({ reason }) }),
|
||||||
|
listProtectedNetworks: () => request<SecurityProtectedNetwork[]>('/admin/security-detection/protected-networks'),
|
||||||
|
addProtectedNetwork: (body: { network: string; name: string; reason: string }) => request<SecurityProtectedNetwork>('/admin/security-detection/protected-networks', { method: 'POST', body: JSON.stringify(body) }),
|
||||||
|
};
|
||||||
@@ -10,6 +10,8 @@ import { adminOperationsApi } from './admin/operations.api';
|
|||||||
import { adminGovernanceApi } from './admin/governance.api';
|
import { adminGovernanceApi } from './admin/governance.api';
|
||||||
import { adminFilesApi } from './admin/files.api';
|
import { adminFilesApi } from './admin/files.api';
|
||||||
import { adminSignatureRetirementApi } from './admin/signature-retirement.api';
|
import { adminSignatureRetirementApi } from './admin/signature-retirement.api';
|
||||||
|
import { adminInfrastructureMonitoringApi } from './admin/infrastructure-monitoring.api';
|
||||||
|
import { adminSecurityDetectionApi } from './admin/security-detection.api';
|
||||||
|
|
||||||
export const adminApi = {
|
export const adminApi = {
|
||||||
...adminIdentityApi,
|
...adminIdentityApi,
|
||||||
@@ -18,4 +20,6 @@ export const adminApi = {
|
|||||||
...adminGovernanceApi,
|
...adminGovernanceApi,
|
||||||
...adminFilesApi,
|
...adminFilesApi,
|
||||||
...adminSignatureRetirementApi,
|
...adminSignatureRetirementApi,
|
||||||
|
...adminInfrastructureMonitoringApi,
|
||||||
|
...adminSecurityDetectionApi,
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -4,3 +4,5 @@ export * from './channels-reports';
|
|||||||
export * from './operations';
|
export * from './operations';
|
||||||
export * from './governance';
|
export * from './governance';
|
||||||
export * from './signature-retirement';
|
export * from './signature-retirement';
|
||||||
|
export * from './infrastructure-monitoring';
|
||||||
|
export * from './security-detection';
|
||||||
|
|||||||
@@ -0,0 +1,93 @@
|
|||||||
|
export type InfrastructureMonitoringRange = '1h' | '24h' | '7d';
|
||||||
|
|
||||||
|
export type InfrastructureMetricPoint = {
|
||||||
|
timestamp: string;
|
||||||
|
value: number;
|
||||||
|
};
|
||||||
|
|
||||||
|
export type InfrastructureServiceStatus = {
|
||||||
|
key: string;
|
||||||
|
name: string;
|
||||||
|
unit: string;
|
||||||
|
status: 'healthy' | 'unhealthy' | 'unknown';
|
||||||
|
};
|
||||||
|
|
||||||
|
export type InfrastructureServiceMetricGroup = {
|
||||||
|
key: string;
|
||||||
|
name: string;
|
||||||
|
available: boolean;
|
||||||
|
metrics: Array<{
|
||||||
|
key: string;
|
||||||
|
label: string;
|
||||||
|
value: number | null;
|
||||||
|
unit: 'percent' | 'seconds' | 'count' | 'per_second' | 'bytes';
|
||||||
|
}>;
|
||||||
|
};
|
||||||
|
|
||||||
|
export type InfrastructureAlert = {
|
||||||
|
fingerprint: string;
|
||||||
|
name: string;
|
||||||
|
severity: 'info' | 'warning' | 'critical';
|
||||||
|
status: string;
|
||||||
|
startedAt: string;
|
||||||
|
summary: string;
|
||||||
|
description?: string;
|
||||||
|
currentValue?: string;
|
||||||
|
threshold?: string;
|
||||||
|
service?: string;
|
||||||
|
instance?: string;
|
||||||
|
acknowledged: boolean;
|
||||||
|
acknowledgedAt?: string;
|
||||||
|
};
|
||||||
|
|
||||||
|
export type InfrastructureMonitoringOverview = {
|
||||||
|
available: boolean;
|
||||||
|
range: InfrastructureMonitoringRange;
|
||||||
|
collectedAt: string;
|
||||||
|
lastSampleAt: string | null;
|
||||||
|
error?: string;
|
||||||
|
summary: {
|
||||||
|
overallStatus: 'healthy' | 'warning' | 'critical' | 'unknown';
|
||||||
|
serviceTotal: number;
|
||||||
|
serviceHealthy: number;
|
||||||
|
warningAlerts: number;
|
||||||
|
criticalAlerts: number;
|
||||||
|
activeAlerts: number;
|
||||||
|
};
|
||||||
|
metrics: {
|
||||||
|
cpuUsagePercent: number | null;
|
||||||
|
memoryUsagePercent: number | null;
|
||||||
|
memoryTotalBytes: number | null;
|
||||||
|
memoryAvailableBytes: number | null;
|
||||||
|
diskUsagePercent: number | null;
|
||||||
|
diskTotalBytes: number | null;
|
||||||
|
diskAvailableBytes: number | null;
|
||||||
|
networkReceiveBytesPerSecond: number | null;
|
||||||
|
networkTransmitBytesPerSecond: number | null;
|
||||||
|
load1: number | null;
|
||||||
|
uptimeSeconds: number | null;
|
||||||
|
};
|
||||||
|
trends: {
|
||||||
|
cpuUsagePercent: InfrastructureMetricPoint[];
|
||||||
|
memoryUsagePercent: InfrastructureMetricPoint[];
|
||||||
|
diskUsagePercent: InfrastructureMetricPoint[];
|
||||||
|
networkReceiveBytesPerSecond: InfrastructureMetricPoint[];
|
||||||
|
networkTransmitBytesPerSecond: InfrastructureMetricPoint[];
|
||||||
|
};
|
||||||
|
services: InfrastructureServiceStatus[];
|
||||||
|
serviceMetrics: InfrastructureServiceMetricGroup[];
|
||||||
|
alerts: InfrastructureAlert[];
|
||||||
|
};
|
||||||
|
|
||||||
|
export type InfrastructureAlertThresholds = Record<string, { warning: number; critical: number }>;
|
||||||
|
|
||||||
|
export type InfrastructureAlertSettings = {
|
||||||
|
configVersion: number;
|
||||||
|
effectiveVersion: number;
|
||||||
|
applyStatus: 'effective' | 'applying' | 'failed';
|
||||||
|
lastError: string | null;
|
||||||
|
appliedAt: string | null;
|
||||||
|
thresholds: InfrastructureAlertThresholds;
|
||||||
|
effectiveThresholds: InfrastructureAlertThresholds;
|
||||||
|
definitions: Array<{ key: string; label: string; unit: string; min: number; max: number; step: number }>;
|
||||||
|
};
|
||||||
@@ -0,0 +1,18 @@
|
|||||||
|
export type SecurityRule = {
|
||||||
|
id: string; code: string; name: string; sourceType: string; enabled: boolean;
|
||||||
|
threshold: number; windowSeconds: number; cooldownSeconds: number; severity: string;
|
||||||
|
defaultBlockSeconds: number; maximumBlockSeconds: number; configVersion: number;
|
||||||
|
effectiveVersion: number; applyStatus: string; lastApplyError?: string | null;
|
||||||
|
};
|
||||||
|
export type SecurityAlert = {
|
||||||
|
id: string; fingerprint: string; sourceIp: string; severity: string; status: string;
|
||||||
|
eventCount: number; firstOccurredAt: string; lastOccurredAt: string; rule: SecurityRule;
|
||||||
|
};
|
||||||
|
export type SecurityBlock = { id: string; sourceIp: string; executor: string; status: string; durationSeconds: number; reason: string; requestedAt: string; expiresAt?: string | null; lastError?: string | null };
|
||||||
|
export type SecurityProtectedNetwork = { id: string; network: string; name: string; reason: string; enabled: boolean; createdAt: string };
|
||||||
|
export type SecurityOverview = {
|
||||||
|
range: string; collectedAt: string; totalEvents: number; activeAlerts: number; criticalAlerts: number; activeBlocks: number;
|
||||||
|
health: { agent: string; agentError?: string; rulesEffective: number; rulesTotal: number };
|
||||||
|
sourceDistribution: Array<{ name: string; value: number }>;
|
||||||
|
alerts: SecurityAlert[];
|
||||||
|
};
|
||||||
@@ -1,7 +1,7 @@
|
|||||||
import { useCallback, useEffect, useMemo, useRef, useState } from 'react';
|
import { useCallback, useEffect, useMemo, useRef, useState } from 'react';
|
||||||
import { AlertTriangle, BarChart3, CheckCircle2, Eye, RefreshCw, Search, TimerReset } from 'lucide-react';
|
import { AlertTriangle, BarChart3, CheckCircle2, Eye, RefreshCw, Search, TimerReset } from 'lucide-react';
|
||||||
import { adminApi, type DownstreamDeliveryDashboard, type DownstreamDeliveryRecord, type DownstreamRequeuePreview, type DownstreamRequeueTask, type DownstreamRequeueTaskItem, type EnterpriseApplication, type TenantOption } from '@/api/adminApi';
|
import { adminApi, type DownstreamDeliveryDashboard, type DownstreamDeliveryRecord, type DownstreamRequeuePreview, type DownstreamRequeueTask, type DownstreamRequeueTaskItem, type EnterpriseApplication, type TenantOption } from '@/api/adminApi';
|
||||||
import { Breadcrumb, Button, DateRangeInput, Input, Modal, Pagination, Select, Tag, type DateRangeValue } from '@/components/ui';
|
import { Breadcrumb, Button, DateRangeInput, Input, Modal, Pagination, Select, Tag, Textarea, type DateRangeValue } from '@/components/ui';
|
||||||
import { formatDateTime } from '@/utils/dateTime';
|
import { formatDateTime } from '@/utils/dateTime';
|
||||||
|
|
||||||
const statusTone: Record<string, 'neutral' | 'info' | 'success' | 'warning' | 'danger'> = {
|
const statusTone: Record<string, 'neutral' | 'info' | 'success' | 'warning' | 'danger'> = {
|
||||||
@@ -677,7 +677,7 @@ export function AdminDownstreamDeliveriesPage() {
|
|||||||
/>
|
/>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<div className="surface admin-task-table-card report-task-table-card">
|
<div className="surface admin-task-table-card report-task-table-card downstream-requeue-task-card">
|
||||||
<div className="section-heading downstream-requeue-task-heading">
|
<div className="section-heading downstream-requeue-task-heading">
|
||||||
<div><h2>后台重投任务</h2><p className="muted">按筛选快照安全恢复,支持暂停、继续、终止和完整结果追踪。</p></div>
|
<div><h2>后台重投任务</h2><p className="muted">按筛选快照安全恢复,支持暂停、继续、终止和完整结果追踪。</p></div>
|
||||||
<div className="downstream-requeue-task-heading__actions">
|
<div className="downstream-requeue-task-heading__actions">
|
||||||
@@ -741,8 +741,20 @@ export function AdminDownstreamDeliveriesPage() {
|
|||||||
</div>
|
</div>
|
||||||
<div className="downstream-requeue-preview__distribution"><span>状态分布</span><div>{Object.entries(taskPreview.statusCounts).map(([key, value]) => <Tag key={key} tone={statusTone[key] ?? 'neutral'}>{statusLabel[key] ?? key} {value}</Tag>)}</div></div>
|
<div className="downstream-requeue-preview__distribution"><span>状态分布</span><div>{Object.entries(taskPreview.statusCounts).map(([key, value]) => <Tag key={key} tone={statusTone[key] ?? 'neutral'}>{statusLabel[key] ?? key} {value}</Tag>)}</div></div>
|
||||||
<Select label="执行速度" value={String(taskRate)} onChange={(event) => setTaskRate(Number(event.target.value))} options={[{ label: '平稳(每应用10条/秒)', value: '10' }, { label: '快速(每应用20条/秒)', value: '20' }, { label: '低速(每应用5条/秒)', value: '5' }]} />
|
<Select label="执行速度" value={String(taskRate)} onChange={(event) => setTaskRate(Number(event.target.value))} options={[{ label: '平稳(每应用10条/秒)', value: '10' }, { label: '快速(每应用20条/秒)', value: '20' }, { label: '低速(每应用5条/秒)', value: '5' }]} />
|
||||||
<label className="field"><span>任务原因 *</span><textarea value={taskReason} onChange={(event) => setTaskReason(event.target.value)} placeholder="请填写事故原因、工单号或处理说明(至少5个字)" rows={3} /></label>
|
<Textarea
|
||||||
<div className="downstream-requeue-warning"><AlertTriangle size={20} /><div><strong>安全边界</strong><p>仅处理待投递、失败、未确认和拒绝记录;不会批量重投客户端已确认或正在等待 ACK 的记录。客户离线时进入等待,不计失败或跳过。</p></div></div>
|
className="downstream-requeue-reason"
|
||||||
|
error={taskReason.length > 0 && taskReason.trim().length < 5 ? '任务原因至少填写 5 个字' : undefined}
|
||||||
|
hint={`请填写事故原因、工单号或处理说明 · ${taskReason.length}/200`}
|
||||||
|
id="downstream-requeue-task-reason"
|
||||||
|
label="任务原因"
|
||||||
|
maxLength={200}
|
||||||
|
onChange={(event) => setTaskReason(event.target.value)}
|
||||||
|
placeholder="例如:工单 INC-20260814,重新投递客户已确认的历史回执"
|
||||||
|
required
|
||||||
|
rows={4}
|
||||||
|
value={taskReason}
|
||||||
|
/>
|
||||||
|
<div className="downstream-requeue-warning"><AlertTriangle size={20} /><div><strong>重复投递风险</strong><p>任务支持待投递、失败、未确认、拒绝和客户端已确认记录;已确认记录会再次发送,可能导致客户端重复处理。正在等待 ACK 的记录仍不会并发重投,客户离线时进入等待。</p></div></div>
|
||||||
</div>
|
</div>
|
||||||
</Modal>
|
</Modal>
|
||||||
) : null}
|
) : null}
|
||||||
|
|||||||
@@ -0,0 +1,2 @@
|
|||||||
|
.admin-security-page{gap:20px}.security-heading{align-items:flex-end}.security-heading h1{margin:10px 0 4px;font-size:26px}.security-heading p{margin:0;color:#64748b}.security-kpis{display:grid;grid-template-columns:repeat(4,minmax(0,1fr));gap:14px}.security-kpi{display:grid;grid-template-columns:42px 1fr;grid-template-rows:auto auto;gap:3px 12px;padding:18px}.security-kpi>div{grid-row:1/3;width:42px;height:42px;border-radius:12px;display:grid;place-items:center;background:#e8f0ff;color:#2563eb}.security-kpi>div svg{width:20px}.security-kpi span{font-size:13px;color:#64748b}.security-kpi strong{font-size:24px;line-height:1.1}.security-kpi.is-danger>div{background:#fef2f2;color:#dc2626}.security-overview-grid{display:grid;grid-template-columns:minmax(0,1fr) minmax(360px,.8fr);gap:16px}.security-chart,.security-latest,.security-table-card{padding:18px}.security-chart header,.security-latest header{display:flex;justify-content:space-between}.security-chart header span,.security-latest header span{font-size:12px;color:#94a3b8}.security-latest-row{display:grid;grid-template-columns:10px 1fr auto;gap:10px;align-items:center;padding:13px 0;border-bottom:1px solid #eef2f7}.security-latest-row div{display:grid;gap:3px}.security-latest-row small,.security-latest-row time{color:#64748b;font-size:12px}.severity-dot{width:8px;height:8px;border-radius:99px;background:#3b82f6}.severity-dot.is-high{background:#f59e0b}.severity-dot.is-critical{background:#ef4444}.security-cell{display:grid;gap:3px}.security-cell span{font-size:11px;color:#94a3b8}.security-actions{display:flex;gap:6px}.security-note,.security-section-toolbar{display:flex;align-items:center;gap:9px;margin-bottom:16px;padding:12px 14px;border-radius:10px;background:#f8fafc;color:#475569;font-size:13px}.security-section-toolbar{justify-content:space-between}.security-error{display:flex;gap:8px;align-items:center;padding:12px 14px;border:1px solid #fecaca;border-radius:10px;background:#fef2f2;color:#b91c1c}.security-empty{height:265px;display:grid;place-items:center;color:#94a3b8}.security-dialog{display:grid;gap:16px}.security-target{display:grid;gap:4px;padding:14px;border-radius:10px;background:#f8fafc}.security-target span,.security-target small{color:#64748b;font-size:12px}.security-target strong{font-family:ui-monospace,monospace;font-size:18px}.security-form-grid{display:grid;grid-template-columns:1fr 1fr;gap:16px}.security-form-grid .security-error{grid-column:1/-1}.admin-security-page code{font-size:12px;color:#334155}@media(max-width:1100px){.security-kpis{grid-template-columns:repeat(2,1fr)}.security-overview-grid{grid-template-columns:1fr}}@media(max-width:640px){.security-kpis{grid-template-columns:1fr}.security-form-grid{grid-template-columns:1fr}}
|
||||||
|
.security-heading p{margin:10px 0 0}
|
||||||
@@ -0,0 +1,95 @@
|
|||||||
|
import { useCallback, useEffect, useMemo, useState, type ComponentProps } from 'react';
|
||||||
|
import type { EChartsOption } from 'echarts';
|
||||||
|
import { AlertTriangle, Ban, BellRing, RefreshCw, Settings2, ShieldCheck, ShieldOff } from 'lucide-react';
|
||||||
|
import { adminApi, type SecurityAlert, type SecurityBlock, type SecurityOverview, type SecurityProtectedNetwork, type SecurityRule } from '@/api/adminApi';
|
||||||
|
import { Breadcrumb, Button, Chart, Input, Modal as BaseModal, Select, Table, Tabs, Tag, Textarea, type TableColumn } from '@/components/ui';
|
||||||
|
import './AdminSecurityDetectionPage.css';
|
||||||
|
|
||||||
|
const statusLabels: Record<string, string> = { open: '待处理', acknowledged: '已确认', blocked: '已封禁', block_failed: '封禁失败', ignored: '已忽略', requested: '执行中', failed: '失败', released: '已解封' };
|
||||||
|
const severityLabels: Record<string, string> = { low: '低', medium: '中', high: '高', critical: '严重' };
|
||||||
|
const durationOptions = [{ value: '600', label: '10分钟' }, { value: '3600', label: '1小时' }, { value: '86400', label: '24小时' }, { value: '604800', label: '7天' }];
|
||||||
|
const formatTime = (value?: string | null) => value ? new Intl.DateTimeFormat('zh-CN', { month: '2-digit', day: '2-digit', hour: '2-digit', minute: '2-digit', hour12: false }).format(new Date(value)) : '—';
|
||||||
|
function Modal(props: Omit<ComponentProps<typeof BaseModal>, 'open'>) { return <BaseModal {...props} open />; }
|
||||||
|
|
||||||
|
export function AdminSecurityDetectionPage() {
|
||||||
|
const [overview, setOverview] = useState<SecurityOverview | null>(null);
|
||||||
|
const [alerts, setAlerts] = useState<SecurityAlert[]>([]);
|
||||||
|
const [rules, setRules] = useState<SecurityRule[]>([]);
|
||||||
|
const [blocks, setBlocks] = useState<SecurityBlock[]>([]);
|
||||||
|
const [networks, setNetworks] = useState<SecurityProtectedNetwork[]>([]);
|
||||||
|
const [loading, setLoading] = useState(true);
|
||||||
|
const [error, setError] = useState('');
|
||||||
|
const [blockAlert, setBlockAlert] = useState<SecurityAlert | null>(null);
|
||||||
|
const [editRule, setEditRule] = useState<SecurityRule | null>(null);
|
||||||
|
const [showNetwork, setShowNetwork] = useState(false);
|
||||||
|
const [reasonAction, setReasonAction] = useState<{ title: string; confirmLabel: string; run: (reason: string) => Promise<void> } | null>(null);
|
||||||
|
|
||||||
|
const load = useCallback(async () => {
|
||||||
|
setLoading(true); setError('');
|
||||||
|
try {
|
||||||
|
const [nextOverview, nextAlerts, nextRules, nextBlocks, nextNetworks] = await Promise.all([
|
||||||
|
adminApi.getSecurityOverview('24h'), adminApi.listSecurityAlerts({ pageSize: 100 }), adminApi.listSecurityRules(), adminApi.listSecurityBlocks(), adminApi.listProtectedNetworks(),
|
||||||
|
]);
|
||||||
|
setOverview(nextOverview); setAlerts(nextAlerts.items); setRules(nextRules); setBlocks(nextBlocks); setNetworks(nextNetworks);
|
||||||
|
} catch (reason) { setError(reason instanceof Error ? reason.message : '安全检测数据加载失败'); }
|
||||||
|
finally { setLoading(false); }
|
||||||
|
}, []);
|
||||||
|
useEffect(() => { void load(); }, [load]);
|
||||||
|
|
||||||
|
const alertColumns: Array<TableColumn<SecurityAlert>> = [
|
||||||
|
{ key: 'level', title: '级别', width: '80px', render: (item) => <Tag tone={item.severity === 'critical' ? 'danger' : item.severity === 'high' ? 'warning' : 'info'}>{severityLabels[item.severity] ?? item.severity}</Tag> },
|
||||||
|
{ key: 'rule', title: '检测类型', width: '180px', render: (item) => <div className="security-cell"><strong>{item.rule.name}</strong><span>{item.rule.code}</span></div> },
|
||||||
|
{ key: 'ip', title: '来源 IP', width: '150px', render: (item) => <code>{item.sourceIp}</code> },
|
||||||
|
{ key: 'count', title: '窗口命中', width: '100px', render: (item) => `${item.eventCount} 次` },
|
||||||
|
{ key: 'time', title: '最后发生', width: '140px', render: (item) => formatTime(item.lastOccurredAt) },
|
||||||
|
{ key: 'status', title: '状态', width: '96px', render: (item) => <Tag tone={item.status === 'blocked' ? 'success' : item.status === 'block_failed' ? 'danger' : 'neutral'}>{statusLabels[item.status] ?? item.status}</Tag> },
|
||||||
|
{ key: 'action', title: '人工处置', width: '190px', render: (item) => <div className="security-actions"><Button disabled={!['open', 'acknowledged', 'block_failed'].includes(item.status)} onClick={() => setBlockAlert(item)} size="sm" variant="danger">封禁</Button><Button disabled={!['open', 'acknowledged', 'block_failed'].includes(item.status)} onClick={() => void ignore(item)} size="sm" variant="ghost">忽略</Button></div> },
|
||||||
|
];
|
||||||
|
function ignore(item: SecurityAlert) { setReasonAction({ title: `忽略告警 · ${item.sourceIp}`, confirmLabel: '确认忽略', run: async (reason) => { await adminApi.ignoreSecurityAlert(item.id, reason); await load(); } }); }
|
||||||
|
|
||||||
|
const ruleColumns: Array<TableColumn<SecurityRule>> = [
|
||||||
|
{ key: 'name', title: '规则', width: '230px', render: (item) => <div className="security-cell"><strong>{item.name}</strong><span>{item.code}</span></div> },
|
||||||
|
{ key: 'source', title: '数据源', width: '110px', render: (item) => item.sourceType },
|
||||||
|
{ key: 'threshold', title: '阈值 / 窗口', width: '150px', render: (item) => `${item.threshold} 次 / ${item.windowSeconds} 秒` },
|
||||||
|
{ key: 'cooldown', title: '冷却', width: '100px', render: (item) => `${item.cooldownSeconds} 秒` },
|
||||||
|
{ key: 'version', title: '生效版本', width: '120px', render: (item) => <Tag tone={item.applyStatus === 'effective' ? 'success' : item.applyStatus === 'failed' ? 'danger' : 'warning'}>{item.effectiveVersion}/{item.configVersion}</Tag> },
|
||||||
|
{ key: 'enabled', title: '启用', width: '90px', render: (item) => <Tag tone={item.enabled ? 'success' : 'neutral'}>{item.enabled ? '启用' : '停用'}</Tag> },
|
||||||
|
{ key: 'action', title: '操作', width: '90px', render: (item) => <Button onClick={() => setEditRule(item)} size="sm" variant="ghost">配置</Button> },
|
||||||
|
];
|
||||||
|
const blockColumns: Array<TableColumn<SecurityBlock>> = [
|
||||||
|
{ key: 'ip', title: 'IP', width: '160px', render: (item) => <code>{item.sourceIp}</code> }, { key: 'executor', title: '执行器', width: '140px', render: (item) => item.executor },
|
||||||
|
{ key: 'status', title: '状态', width: '100px', render: (item) => <Tag tone={item.status === 'blocked' ? 'success' : item.status === 'failed' ? 'danger' : 'neutral'}>{statusLabels[item.status] ?? item.status}</Tag> },
|
||||||
|
{ key: 'reason', title: '原因', width: '260px', render: (item) => item.reason }, { key: 'expiry', title: '到期时间', width: '150px', render: (item) => formatTime(item.expiresAt) }, { key: 'error', title: '执行结果', width: '220px', render: (item) => item.lastError ?? '已由执行器回读确认' },
|
||||||
|
{ key: 'action', title: '操作', width: '90px', render: (item) => <Button disabled={item.status !== 'blocked'} onClick={() => void releaseBlock(item)} size="sm" variant="ghost">解封</Button> },
|
||||||
|
];
|
||||||
|
function releaseBlock(item: SecurityBlock) { setReasonAction({ title: `人工解封 · ${item.sourceIp}`, confirmLabel: '确认解封', run: async (reason) => { await adminApi.unblockSecurityBlock(item.id, reason); await load(); } }); }
|
||||||
|
const networkColumns: Array<TableColumn<SecurityProtectedNetwork>> = [
|
||||||
|
{ key: 'network', title: 'IP / 网段', width: '180px', render: (item) => <code>{item.network}</code> }, { key: 'name', title: '名称', width: '180px', render: (item) => item.name }, { key: 'reason', title: '保护原因', width: '320px', render: (item) => item.reason }, { key: 'status', title: '状态', width: '100px', render: (item) => <Tag tone={item.enabled ? 'success' : 'neutral'}>{item.enabled ? '保护中' : '已停用'}</Tag> },
|
||||||
|
];
|
||||||
|
const chartOption = useMemo<EChartsOption>(() => ({ tooltip: { trigger: 'item' }, legend: { bottom: 0, type: 'scroll' }, series: [{ type: 'pie', radius: ['52%', '76%'], center: ['50%', '43%'], label: { show: false }, data: overview?.sourceDistribution ?? [] }] }), [overview]);
|
||||||
|
|
||||||
|
const tabs = [
|
||||||
|
{ value: 'overview', label: '总览', content: <><div className="security-overview-grid"><article className="surface security-chart"><header><strong>24小时检测来源</strong><span>真实事件聚合</span></header>{overview?.sourceDistribution.length ? <Chart height={265} option={chartOption} /> : <div className="security-empty">暂无检测事件</div>}</article><article className="surface security-latest"><header><strong>最新告警</strong><span>{overview?.alerts.length ?? 0} 条</span></header>{overview?.alerts.slice(0, 6).map((item) => <div className="security-latest-row" key={item.id}><span className={`severity-dot is-${item.severity}`} /><div><strong>{item.rule.name}</strong><small>{item.sourceIp} · {item.eventCount} 次</small></div><time>{formatTime(item.lastOccurredAt)}</time></div>)}</article></div></> },
|
||||||
|
{ value: 'alerts', label: '告警中心', content: <section className="surface security-table-card"><Table columns={alertColumns} data={alerts} emptyText="暂无安全告警" rowKey="id" /></section> },
|
||||||
|
{ value: 'rules', label: '规则配置', content: <section className="surface security-table-card"><div className="security-note"><Settings2 size={17} /><span>所有阈值来自数据库;修改后必须由受限安全代理验证并应用,版本一致才标记生效。</span></div><Table columns={ruleColumns} data={rules} rowKey="id" /></section> },
|
||||||
|
{ value: 'blocks', label: '封禁记录', content: <section className="surface security-table-card"><Table columns={blockColumns} data={blocks} emptyText="暂无人工封禁记录" rowKey="id" /></section> },
|
||||||
|
{ value: 'protected', label: '保护名单', content: <section className="surface security-table-card"><div className="security-section-toolbar"><span>受保护的运维出口、内网和可信代理永远不能从面板封禁。</span><Button onClick={() => setShowNetwork(true)}>新增保护网段</Button></div><Table columns={networkColumns} data={networks} emptyText="暂无保护网段" rowKey="id" /></section> },
|
||||||
|
];
|
||||||
|
|
||||||
|
return <section className="page-stack admin-security-page"><div className="page-heading security-heading"><div><Breadcrumb items={['安全控制', '安全检测与封禁']} /><p>使用 Fail2ban 与平台业务安全事件进行检测,由运营人员复核后人工处置</p></div><Button disabled={loading} icon={<RefreshCw className={loading ? 'is-spinning' : ''} size={16} />} onClick={() => void load()} variant="ghost">刷新</Button></div>
|
||||||
|
{error ? <div className="security-error" role="alert"><AlertTriangle size={18} />{error}</div> : null}
|
||||||
|
<div className="security-kpis"><Kpi icon={<BellRing />} label="24小时检测事件" value={overview?.totalEvents ?? 0} /><Kpi icon={<AlertTriangle />} label="待处置告警" value={overview?.activeAlerts ?? 0} danger={Boolean(overview?.criticalAlerts)} /><Kpi icon={<Ban />} label="生效封禁" value={overview?.activeBlocks ?? 0} /><Kpi icon={overview?.health.agent === 'healthy' ? <ShieldCheck /> : <ShieldOff />} label="安全代理" value={overview?.health.agent === 'healthy' ? '在线' : '不可用'} danger={overview?.health.agent !== 'healthy'} /></div>
|
||||||
|
<Tabs items={tabs} />
|
||||||
|
{blockAlert ? <BlockDialog alert={blockAlert} onClose={() => setBlockAlert(null)} onSaved={async () => { setBlockAlert(null); await load(); }} /> : null}
|
||||||
|
{editRule ? <RuleDialog rule={editRule} onClose={() => setEditRule(null)} onSaved={async () => { setEditRule(null); await load(); }} /> : null}
|
||||||
|
{showNetwork ? <NetworkDialog onClose={() => setShowNetwork(false)} onSaved={async () => { setShowNetwork(false); await load(); }} /> : null}
|
||||||
|
{reasonAction ? <ActionReasonDialog action={reasonAction} onClose={() => setReasonAction(null)} /> : null}
|
||||||
|
</section>;
|
||||||
|
}
|
||||||
|
|
||||||
|
function Kpi({ icon, label, value, danger = false }: { icon: React.ReactNode; label: string; value: React.ReactNode; danger?: boolean }) { return <article className={`surface security-kpi ${danger ? 'is-danger' : ''}`}><div>{icon}</div><span>{label}</span><strong>{value}</strong></article>; }
|
||||||
|
function BlockDialog({ alert, onClose, onSaved }: { alert: SecurityAlert; onClose: () => void; onSaved: () => void }) { const [duration, setDuration] = useState(String(alert.rule.defaultBlockSeconds)); const [reason, setReason] = useState(''); const [saving, setSaving] = useState(false); const [error, setError] = useState(''); const executor = alert.rule.code.startsWith('admin_') || alert.rule.code.startsWith('client_') ? 'Nginx Real-IP deny' : 'nftables'; async function save() { setSaving(true); setError(''); try { await adminApi.blockSecurityAlert(alert.id, { durationSeconds: Number(duration), reason }); await onSaved(); } catch (reason) { setError(reason instanceof Error ? reason.message : '封禁失败'); } finally { setSaving(false); } } return <Modal title="确认人工封禁" onClose={onClose} footer={<><Button onClick={onClose} variant="ghost">取消</Button><Button disabled={saving || reason.trim().length < 5} onClick={() => void save()} variant="danger">{saving ? '执行并回读中' : '确认封禁'}</Button></>}><div className="security-dialog"><div className="security-target"><span>目标 IP</span><strong>{alert.sourceIp}</strong><small>{alert.rule.name} · 窗口命中 {alert.eventCount} 次</small></div><Input disabled label="服务端固定执行器" value={executor} /><Select label="封禁时长" options={durationOptions.filter((item) => Number(item.value) <= alert.rule.maximumBlockSeconds)} value={duration} onChange={(event) => setDuration(event.target.value)} /><Textarea label="封禁原因" minLength={5} onChange={(event) => setReason(event.target.value)} required value={reason} />{error ? <div className="security-error">{error}</div> : null}</div></Modal>; }
|
||||||
|
function RuleDialog({ rule, onClose, onSaved }: { rule: SecurityRule; onClose: () => void; onSaved: () => void }) { const [value, setValue] = useState(rule); const [saving, setSaving] = useState(false); const [error, setError] = useState(''); const number = (key: keyof SecurityRule) => (event: React.ChangeEvent<HTMLInputElement>) => setValue({ ...value, [key]: Number(event.target.value) }); async function save() { setSaving(true); try { await adminApi.updateSecurityRule(rule.id, value); await onSaved(); } catch (reason) { setError(reason instanceof Error ? reason.message : '规则保存失败'); } finally { setSaving(false); } } return <Modal title={`配置规则 · ${rule.name}`} onClose={onClose} footer={<><Button onClick={onClose} variant="ghost">取消</Button><Button disabled={saving} onClick={() => void save()}>{saving ? '验证并应用中' : '保存并应用'}</Button></>}><div className="security-form-grid"><Select label="启用状态" options={[{ value: 'true', label: '启用' }, { value: 'false', label: '停用' }]} value={String(value.enabled)} onChange={(event) => setValue({ ...value, enabled: event.target.value === 'true' })} /><Select label="告警级别" options={['low','medium','high','critical'].map((item) => ({ value: item, label: severityLabels[item] }))} value={value.severity} onChange={(event) => setValue({ ...value, severity: event.target.value })} /><Input label="触发次数" min={1} onChange={number('threshold')} type="number" value={value.threshold} /><Input label="检测窗口(秒)" min={10} onChange={number('windowSeconds')} type="number" value={value.windowSeconds} /><Input label="告警冷却(秒)" min={0} onChange={number('cooldownSeconds')} type="number" value={value.cooldownSeconds} /><Input label="默认封禁(秒)" min={600} onChange={number('defaultBlockSeconds')} type="number" value={value.defaultBlockSeconds} /><Input label="最大封禁(秒)" min={600} onChange={number('maximumBlockSeconds')} type="number" value={value.maximumBlockSeconds} />{error ? <div className="security-error">{error}</div> : null}</div></Modal>; }
|
||||||
|
function NetworkDialog({ onClose, onSaved }: { onClose: () => void; onSaved: () => void }) { const [value, setValue] = useState({ network: '', name: '', reason: '' }); const [error, setError] = useState(''); async function save() { try { await adminApi.addProtectedNetwork(value); await onSaved(); } catch (reason) { setError(reason instanceof Error ? reason.message : '新增失败'); } } return <Modal title="新增保护网段" onClose={onClose} footer={<><Button onClick={onClose} variant="ghost">取消</Button><Button disabled={!value.network || !value.name || !value.reason} onClick={() => void save()}>确认保护</Button></>}><div className="security-dialog"><Input label="IP 或 CIDR" placeholder="例如 203.0.113.10 或 10.0.0.0/8" value={value.network} onChange={(event) => setValue({ ...value, network: event.target.value })} /><Input label="名称" value={value.name} onChange={(event) => setValue({ ...value, name: event.target.value })} /><Textarea label="保护原因" value={value.reason} onChange={(event) => setValue({ ...value, reason: event.target.value })} />{error ? <div className="security-error">{error}</div> : null}</div></Modal>; }
|
||||||
|
|
||||||
|
function ActionReasonDialog({ action, onClose }: { action: { title: string; confirmLabel: string; run: (reason: string) => Promise<void> }; onClose: () => void }) { const [reason, setReason] = useState(''); const [saving, setSaving] = useState(false); const [error, setError] = useState(''); async function save() { setSaving(true); setError(''); try { await action.run(reason); onClose(); } catch (cause) { setError(cause instanceof Error ? cause.message : '操作失败'); } finally { setSaving(false); } } return <Modal title={action.title} onClose={onClose} footer={<><Button onClick={onClose} variant="ghost">取消</Button><Button disabled={saving || reason.trim().length < 5} onClick={() => void save()}>{saving ? '处理中' : action.confirmLabel}</Button></>}><div className="security-dialog"><Textarea autoFocus label="操作原因" minLength={5} onChange={(event) => setReason(event.target.value)} required value={reason} />{error ? <div className="security-error">{error}</div> : null}</div></Modal>; }
|
||||||
@@ -0,0 +1,336 @@
|
|||||||
|
.admin-system-monitoring-page {
|
||||||
|
gap: 18px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.system-monitoring-heading {
|
||||||
|
align-items: flex-end;
|
||||||
|
gap: 24px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.system-monitoring-title-row {
|
||||||
|
align-items: center;
|
||||||
|
display: flex;
|
||||||
|
gap: 12px;
|
||||||
|
margin-top: 8px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.system-monitoring-title-row p {
|
||||||
|
color: var(--color-text-muted);
|
||||||
|
font-size: 13px;
|
||||||
|
margin: 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
.system-monitoring-controls,
|
||||||
|
.system-monitoring-range {
|
||||||
|
align-items: center;
|
||||||
|
display: flex;
|
||||||
|
}
|
||||||
|
|
||||||
|
.system-monitoring-controls {
|
||||||
|
gap: 10px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.system-monitoring-range {
|
||||||
|
background: var(--color-surface-muted);
|
||||||
|
border: 1px solid var(--color-border);
|
||||||
|
border-radius: var(--radius-md);
|
||||||
|
padding: 3px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.system-monitoring-range button {
|
||||||
|
background: transparent;
|
||||||
|
border: 0;
|
||||||
|
border-radius: 4px;
|
||||||
|
color: var(--color-text-muted);
|
||||||
|
cursor: pointer;
|
||||||
|
font: inherit;
|
||||||
|
min-height: 30px;
|
||||||
|
padding: 0 12px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.system-monitoring-range button:hover {
|
||||||
|
color: var(--color-text-strong);
|
||||||
|
}
|
||||||
|
|
||||||
|
.system-monitoring-range button:focus-visible {
|
||||||
|
box-shadow: var(--focus-ring);
|
||||||
|
outline: none;
|
||||||
|
}
|
||||||
|
|
||||||
|
.system-monitoring-range button.is-active {
|
||||||
|
background: var(--color-surface);
|
||||||
|
box-shadow: var(--shadow-xs);
|
||||||
|
color: var(--color-selected);
|
||||||
|
font-weight: var(--font-weight-semibold);
|
||||||
|
}
|
||||||
|
|
||||||
|
.system-monitoring-unavailable {
|
||||||
|
align-items: flex-start;
|
||||||
|
background: var(--color-danger-soft);
|
||||||
|
border: 1px solid color-mix(in srgb, var(--color-danger) 24%, transparent);
|
||||||
|
border-radius: var(--radius-lg);
|
||||||
|
color: var(--color-danger);
|
||||||
|
display: flex;
|
||||||
|
gap: 12px;
|
||||||
|
padding: 14px 16px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.system-monitoring-unavailable div {
|
||||||
|
display: grid;
|
||||||
|
gap: 2px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.system-monitoring-unavailable span {
|
||||||
|
color: var(--color-text-muted);
|
||||||
|
font-size: 13px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.system-monitoring-health {
|
||||||
|
align-items: center;
|
||||||
|
display: grid;
|
||||||
|
gap: 0;
|
||||||
|
grid-template-columns: minmax(220px, 1.35fr) repeat(4, minmax(120px, 1fr));
|
||||||
|
padding: 18px 20px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.system-monitoring-health__mark {
|
||||||
|
align-items: center;
|
||||||
|
border-radius: 50%;
|
||||||
|
display: flex;
|
||||||
|
height: 48px;
|
||||||
|
justify-content: center;
|
||||||
|
margin-right: 13px;
|
||||||
|
width: 48px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.system-monitoring-health__mark.is-healthy { background: var(--color-success-soft); color: var(--color-success); }
|
||||||
|
.system-monitoring-health__mark.is-warning { background: var(--color-warning-soft); color: var(--color-warning); }
|
||||||
|
.system-monitoring-health__mark.is-critical { background: var(--color-danger-soft); color: var(--color-danger); }
|
||||||
|
.system-monitoring-health__mark.is-unknown { background: var(--color-surface-muted); color: var(--color-text-muted); }
|
||||||
|
|
||||||
|
.system-monitoring-health__copy {
|
||||||
|
align-items: center;
|
||||||
|
display: grid;
|
||||||
|
grid-template-columns: 48px 1fr;
|
||||||
|
grid-template-rows: repeat(3, auto);
|
||||||
|
}
|
||||||
|
|
||||||
|
.system-monitoring-health__copy .system-monitoring-health__mark { grid-row: 1 / 4; }
|
||||||
|
.system-monitoring-health__copy > span,
|
||||||
|
.system-monitoring-health__fact > span {
|
||||||
|
color: var(--color-text-muted);
|
||||||
|
font-size: 12px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.system-monitoring-health__copy > strong { color: var(--color-text-strong); font-size: 17px; }
|
||||||
|
.system-monitoring-health__copy > small,
|
||||||
|
.system-monitoring-health__fact > small { color: var(--color-text-subtle); font-size: 11px; }
|
||||||
|
|
||||||
|
.system-monitoring-health__fact {
|
||||||
|
border-left: 1px solid var(--color-border);
|
||||||
|
display: grid;
|
||||||
|
gap: 3px;
|
||||||
|
padding-left: 20px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.system-monitoring-health__fact strong {
|
||||||
|
color: var(--color-text-strong);
|
||||||
|
font-size: 18px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.system-monitoring-metrics {
|
||||||
|
display: grid;
|
||||||
|
gap: 14px;
|
||||||
|
grid-template-columns: repeat(4, minmax(0, 1fr));
|
||||||
|
}
|
||||||
|
|
||||||
|
.system-monitoring-metric {
|
||||||
|
align-items: center;
|
||||||
|
display: flex;
|
||||||
|
gap: 14px;
|
||||||
|
min-width: 0;
|
||||||
|
padding: 18px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.system-monitoring-metric__icon {
|
||||||
|
align-items: center;
|
||||||
|
border-radius: var(--radius-md);
|
||||||
|
display: flex;
|
||||||
|
flex: 0 0 40px;
|
||||||
|
height: 40px;
|
||||||
|
justify-content: center;
|
||||||
|
}
|
||||||
|
|
||||||
|
.system-monitoring-metric__icon.is-blue { background: var(--color-info-soft); color: var(--color-info); }
|
||||||
|
.system-monitoring-metric__icon.is-violet { background: #f5f3ff; color: #7c3aed; }
|
||||||
|
.system-monitoring-metric__icon.is-amber { background: var(--color-warning-soft); color: var(--color-warning); }
|
||||||
|
.system-monitoring-metric__icon.is-green { background: var(--color-success-soft); color: #0f766e; }
|
||||||
|
|
||||||
|
.system-monitoring-metric > div:last-child {
|
||||||
|
display: grid;
|
||||||
|
gap: 2px;
|
||||||
|
min-width: 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
.system-monitoring-metric span,
|
||||||
|
.system-monitoring-metric small { color: var(--color-text-muted); font-size: 12px; }
|
||||||
|
.system-monitoring-metric strong { color: var(--color-text-strong); font-size: 23px; line-height: 1.25; }
|
||||||
|
.system-monitoring-metric small { overflow: hidden; text-overflow: ellipsis; white-space: nowrap; }
|
||||||
|
|
||||||
|
.system-monitoring-main-grid {
|
||||||
|
align-items: start;
|
||||||
|
display: grid;
|
||||||
|
gap: 16px;
|
||||||
|
grid-template-columns: minmax(0, 1fr) 300px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.system-monitoring-service-metrics { padding: 18px; }
|
||||||
|
.system-monitoring-service-metrics > header {
|
||||||
|
align-items: center;
|
||||||
|
display: flex;
|
||||||
|
justify-content: space-between;
|
||||||
|
margin-bottom: 14px;
|
||||||
|
}
|
||||||
|
.system-monitoring-service-metrics > header > div { align-items: center; color: var(--color-text-strong); display: flex; gap: 8px; }
|
||||||
|
.system-monitoring-service-metrics > header > span { color: var(--color-text-muted); font-size: 12px; }
|
||||||
|
.system-monitoring-service-metric-grid { display: grid; gap: 12px; grid-template-columns: repeat(3, minmax(0, 1fr)); }
|
||||||
|
.system-monitoring-service-metric-grid article { background: var(--color-bg-subtle); border: 1px solid var(--color-border); border-radius: var(--radius-md); padding: 14px; }
|
||||||
|
.system-monitoring-service-metric-title { align-items: center; display: flex; justify-content: space-between; margin-bottom: 9px; }
|
||||||
|
.system-monitoring-service-metric-title > strong { color: var(--color-text-strong); font-size: 14px; }
|
||||||
|
.system-monitoring-service-metric-row { align-items: center; border-top: 1px solid var(--color-border); display: flex; justify-content: space-between; min-height: 34px; }
|
||||||
|
.system-monitoring-service-metric-row span { color: var(--color-text-muted); font-size: 12px; }
|
||||||
|
.system-monitoring-service-metric-row strong { color: var(--color-text); font-size: 13px; }
|
||||||
|
.system-monitoring-service-metric-empty { color: var(--color-text-subtle); font-size: 12px; line-height: 1.55; padding-top: 8px; }
|
||||||
|
|
||||||
|
.system-monitoring-chart-stack {
|
||||||
|
display: grid;
|
||||||
|
gap: 16px;
|
||||||
|
grid-template-columns: repeat(2, minmax(0, 1fr));
|
||||||
|
}
|
||||||
|
|
||||||
|
.system-monitoring-chart-card,
|
||||||
|
.system-monitoring-services,
|
||||||
|
.system-monitoring-alerts { padding: 18px; }
|
||||||
|
|
||||||
|
.system-monitoring-chart-card header,
|
||||||
|
.system-monitoring-services > header,
|
||||||
|
.system-monitoring-alerts > header {
|
||||||
|
align-items: center;
|
||||||
|
display: flex;
|
||||||
|
justify-content: space-between;
|
||||||
|
margin-bottom: 12px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.system-monitoring-chart-card header div,
|
||||||
|
.system-monitoring-services > header div,
|
||||||
|
.system-monitoring-alerts > header div {
|
||||||
|
align-items: center;
|
||||||
|
color: var(--color-text-strong);
|
||||||
|
display: flex;
|
||||||
|
gap: 8px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.system-monitoring-chart-card header > span { color: var(--color-text); font-weight: var(--font-weight-semibold); }
|
||||||
|
|
||||||
|
.system-monitoring-chart-empty {
|
||||||
|
align-items: center;
|
||||||
|
color: var(--color-text-subtle);
|
||||||
|
display: flex;
|
||||||
|
flex-direction: column;
|
||||||
|
gap: 8px;
|
||||||
|
height: 230px;
|
||||||
|
justify-content: center;
|
||||||
|
}
|
||||||
|
|
||||||
|
.system-monitoring-service-list { display: grid; }
|
||||||
|
|
||||||
|
.system-monitoring-service {
|
||||||
|
align-items: center;
|
||||||
|
border-bottom: 1px solid var(--color-border);
|
||||||
|
display: grid;
|
||||||
|
gap: 9px;
|
||||||
|
grid-template-columns: 9px 1fr auto;
|
||||||
|
padding: 13px 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
.system-monitoring-service:last-child { border-bottom: 0; }
|
||||||
|
.system-monitoring-service__dot { background: var(--color-text-subtle); border-radius: 50%; height: 8px; width: 8px; }
|
||||||
|
.system-monitoring-service__dot.is-healthy { background: var(--color-success); box-shadow: 0 0 0 3px var(--color-success-soft); }
|
||||||
|
.system-monitoring-service__dot.is-unhealthy { background: var(--color-danger); box-shadow: 0 0 0 3px var(--color-danger-soft); }
|
||||||
|
.system-monitoring-service__dot.is-unknown { background: var(--color-text-subtle); box-shadow: 0 0 0 3px var(--color-surface-muted); }
|
||||||
|
.system-monitoring-service div { display: grid; gap: 1px; min-width: 0; }
|
||||||
|
.system-monitoring-service strong { color: var(--color-text); font-size: 13px; }
|
||||||
|
.system-monitoring-service small { color: var(--color-text-subtle); font-size: 11px; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; }
|
||||||
|
.system-monitoring-service > span:last-child { color: var(--color-text-muted); font-size: 12px; }
|
||||||
|
|
||||||
|
.system-monitoring-collector-note {
|
||||||
|
align-items: flex-start;
|
||||||
|
background: var(--color-bg-subtle);
|
||||||
|
border: 1px solid var(--color-border);
|
||||||
|
border-radius: var(--radius-md);
|
||||||
|
color: var(--color-text-muted);
|
||||||
|
display: flex;
|
||||||
|
font-size: 12px;
|
||||||
|
gap: 8px;
|
||||||
|
line-height: 1.55;
|
||||||
|
margin-top: 14px;
|
||||||
|
padding: 11px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.system-monitoring-collector-note svg { flex: 0 0 auto; margin-top: 1px; }
|
||||||
|
|
||||||
|
.system-monitoring-alerts { overflow: hidden; }
|
||||||
|
.system-monitoring-alerts > header > span { align-items: center; color: var(--color-text-muted); display: flex; font-size: 12px; gap: 5px; }
|
||||||
|
.system-monitoring-alert-copy { display: grid; gap: 2px; }
|
||||||
|
.system-monitoring-alert-copy strong { color: var(--color-text-strong); }
|
||||||
|
.system-monitoring-alert-copy span { color: var(--color-text-muted); font-size: 12px; }
|
||||||
|
|
||||||
|
.is-spinning { animation: system-monitoring-spin 0.9s linear infinite; }
|
||||||
|
@keyframes system-monitoring-spin { to { transform: rotate(360deg); } }
|
||||||
|
|
||||||
|
@media (max-width: 1180px) {
|
||||||
|
.system-monitoring-health { grid-template-columns: repeat(3, minmax(0, 1fr)); row-gap: 18px; }
|
||||||
|
.system-monitoring-health__copy { grid-column: span 2; }
|
||||||
|
.system-monitoring-health__fact:nth-last-child(-n + 2) { border-left: 0; padding-left: 0; }
|
||||||
|
.system-monitoring-metrics { grid-template-columns: repeat(2, minmax(0, 1fr)); }
|
||||||
|
.system-monitoring-main-grid { grid-template-columns: minmax(0, 1fr); }
|
||||||
|
.system-monitoring-services { order: -1; }
|
||||||
|
.system-monitoring-service-list { grid-template-columns: repeat(3, minmax(0, 1fr)); }
|
||||||
|
.system-monitoring-service-metric-grid { grid-template-columns: repeat(2, minmax(0, 1fr)); }
|
||||||
|
.system-monitoring-service { border-bottom: 0; border-right: 1px solid var(--color-border); padding: 11px 12px; }
|
||||||
|
.system-monitoring-service:nth-child(3n) { border-right: 0; }
|
||||||
|
}
|
||||||
|
|
||||||
|
@media (max-width: 760px) {
|
||||||
|
.system-monitoring-heading { align-items: stretch; flex-direction: column; }
|
||||||
|
.system-monitoring-controls { align-items: stretch; flex-direction: column; }
|
||||||
|
.system-monitoring-range { display: grid; grid-template-columns: repeat(3, 1fr); }
|
||||||
|
.system-monitoring-title-row { align-items: flex-start; justify-content: space-between; }
|
||||||
|
.system-monitoring-health { grid-template-columns: repeat(2, minmax(0, 1fr)); }
|
||||||
|
.system-monitoring-health__copy { grid-column: 1 / -1; }
|
||||||
|
.system-monitoring-health__fact { border-left: 0; border-top: 1px solid var(--color-border); padding: 12px 0 0; }
|
||||||
|
.system-monitoring-metrics,
|
||||||
|
.system-monitoring-chart-stack,
|
||||||
|
.system-monitoring-service-list,
|
||||||
|
.system-monitoring-service-metric-grid { grid-template-columns: minmax(0, 1fr); }
|
||||||
|
.system-monitoring-service-metrics > header { align-items: flex-start; flex-direction: column; gap: 6px; }
|
||||||
|
.system-monitoring-service { border-bottom: 1px solid var(--color-border); border-right: 0; padding: 13px 0; }
|
||||||
|
.system-monitoring-chart-card { padding: 14px 10px; }
|
||||||
|
}
|
||||||
|
|
||||||
|
@media (prefers-reduced-motion: reduce) {
|
||||||
|
.is-spinning { animation: none; }
|
||||||
|
}
|
||||||
|
|
||||||
|
.system-monitoring-service-actions { align-items: center; display: flex; gap: 12px; }
|
||||||
|
.system-monitoring-threshold-dialog { display: grid; gap: 12px; }
|
||||||
|
.system-monitoring-threshold-note { align-items: flex-start; background: #eff6ff; border-radius: 10px; color: #475569; display: flex; font-size: 13px; gap: 9px; padding: 12px 14px; }
|
||||||
|
.system-monitoring-threshold-row { align-items: end; border-bottom: 1px solid #eef2f7; display: grid; gap: 14px; grid-template-columns: minmax(180px, 1fr) 150px 150px; padding: 12px 0; }
|
||||||
|
.system-monitoring-threshold-row > div:first-child { align-self: center; display: grid; gap: 4px; }
|
||||||
|
.system-monitoring-threshold-row small { color: #64748b; }
|
||||||
|
|
||||||
|
@media (max-width: 760px) {
|
||||||
|
.system-monitoring-service-actions { align-items: flex-start; flex-direction: column; }
|
||||||
|
.system-monitoring-threshold-row { grid-template-columns: 1fr 1fr; }
|
||||||
|
.system-monitoring-threshold-row > div:first-child { grid-column: 1 / -1; }
|
||||||
|
}
|
||||||
@@ -0,0 +1,418 @@
|
|||||||
|
import { useCallback, useEffect, useMemo, useRef, useState } from 'react';
|
||||||
|
import type { EChartsOption } from 'echarts';
|
||||||
|
import {
|
||||||
|
Activity,
|
||||||
|
AlertTriangle,
|
||||||
|
CheckCircle2,
|
||||||
|
Clock3,
|
||||||
|
Cpu,
|
||||||
|
Database,
|
||||||
|
HardDrive,
|
||||||
|
MemoryStick,
|
||||||
|
Network,
|
||||||
|
RefreshCw,
|
||||||
|
Server,
|
||||||
|
Settings2,
|
||||||
|
ShieldAlert,
|
||||||
|
} from 'lucide-react';
|
||||||
|
import {
|
||||||
|
adminApi,
|
||||||
|
type InfrastructureAlert,
|
||||||
|
type InfrastructureAlertSettings,
|
||||||
|
type InfrastructureAlertThresholds,
|
||||||
|
type InfrastructureMetricPoint,
|
||||||
|
type InfrastructureMonitoringOverview,
|
||||||
|
type InfrastructureMonitoringRange,
|
||||||
|
} from '@/api/adminApi';
|
||||||
|
import { Breadcrumb, Button, Chart, Input, Modal, Table, Tag, type TableColumn } from '@/components/ui';
|
||||||
|
import './AdminSystemMonitoringPage.css';
|
||||||
|
|
||||||
|
const RANGE_OPTIONS: Array<{ value: InfrastructureMonitoringRange; label: string }> = [
|
||||||
|
{ value: '1h', label: '近1小时' },
|
||||||
|
{ value: '24h', label: '近24小时' },
|
||||||
|
{ value: '7d', label: '近7天' },
|
||||||
|
];
|
||||||
|
|
||||||
|
const STATUS_COPY = {
|
||||||
|
healthy: { label: '运行正常', tone: 'success' as const },
|
||||||
|
warning: { label: '需要关注', tone: 'warning' as const },
|
||||||
|
critical: { label: '严重告警', tone: 'danger' as const },
|
||||||
|
unknown: { label: '状态未知', tone: 'neutral' as const },
|
||||||
|
};
|
||||||
|
|
||||||
|
function formatPercent(value: number | null) {
|
||||||
|
return value === null ? '—' : `${value.toFixed(1)}%`;
|
||||||
|
}
|
||||||
|
|
||||||
|
function formatBytes(value: number | null) {
|
||||||
|
if (value === null) return '—';
|
||||||
|
const units = ['B', 'KB', 'MB', 'GB', 'TB'];
|
||||||
|
let amount = value;
|
||||||
|
let index = 0;
|
||||||
|
while (amount >= 1024 && index < units.length - 1) {
|
||||||
|
amount /= 1024;
|
||||||
|
index += 1;
|
||||||
|
}
|
||||||
|
return `${amount.toFixed(index >= 3 ? 1 : 0)} ${units[index]}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
function formatRate(value: number | null) {
|
||||||
|
return value === null ? '—' : `${formatBytes(value)}/s`;
|
||||||
|
}
|
||||||
|
|
||||||
|
function totalNetworkRate(receive: number | null | undefined, transmit: number | null | undefined) {
|
||||||
|
if (receive === null || receive === undefined || transmit === null || transmit === undefined) return null;
|
||||||
|
return receive + transmit;
|
||||||
|
}
|
||||||
|
|
||||||
|
function formatUptime(value: number | null) {
|
||||||
|
if (value === null) return '—';
|
||||||
|
const days = Math.floor(value / 86400);
|
||||||
|
const hours = Math.floor((value % 86400) / 3600);
|
||||||
|
return days > 0 ? `${days}天 ${hours}小时` : `${hours}小时`;
|
||||||
|
}
|
||||||
|
|
||||||
|
function formatServiceMetric(value: number | null, unit: 'percent' | 'seconds' | 'count' | 'per_second' | 'bytes') {
|
||||||
|
if (value === null) return '—';
|
||||||
|
if (unit === 'percent') return `${value.toFixed(2)}%`;
|
||||||
|
if (unit === 'seconds') return value < 1 ? `${Math.round(value * 1000)} ms` : `${value.toFixed(1)} s`;
|
||||||
|
if (unit === 'per_second') return `${value.toFixed(value < 10 ? 2 : 1)}/s`;
|
||||||
|
if (unit === 'bytes') return formatBytes(value);
|
||||||
|
return Math.round(value).toLocaleString('zh-CN');
|
||||||
|
}
|
||||||
|
|
||||||
|
function formatTime(value: string | null) {
|
||||||
|
if (!value) return '暂无采样';
|
||||||
|
return new Intl.DateTimeFormat('zh-CN', {
|
||||||
|
month: '2-digit', day: '2-digit', hour: '2-digit', minute: '2-digit', second: '2-digit', hour12: false,
|
||||||
|
}).format(new Date(value));
|
||||||
|
}
|
||||||
|
|
||||||
|
function formatDuration(startedAt: string) {
|
||||||
|
const milliseconds = Date.now() - Date.parse(startedAt);
|
||||||
|
if (!Number.isFinite(milliseconds) || milliseconds < 0) return '—';
|
||||||
|
const minutes = Math.floor(milliseconds / 60_000);
|
||||||
|
if (minutes < 60) return `${Math.max(minutes, 1)}分钟`;
|
||||||
|
const hours = Math.floor(minutes / 60);
|
||||||
|
return hours < 24 ? `${hours}小时 ${minutes % 60}分钟` : `${Math.floor(hours / 24)}天 ${hours % 24}小时`;
|
||||||
|
}
|
||||||
|
|
||||||
|
function timeLabels(points: InfrastructureMetricPoint[], range: InfrastructureMonitoringRange) {
|
||||||
|
return points.map((point) => new Intl.DateTimeFormat('zh-CN', range === '7d'
|
||||||
|
? { month: '2-digit', day: '2-digit', hour: '2-digit', hour12: false }
|
||||||
|
: { hour: '2-digit', minute: '2-digit', hour12: false })
|
||||||
|
.format(new Date(point.timestamp)));
|
||||||
|
}
|
||||||
|
|
||||||
|
function makeTrendOption(params: {
|
||||||
|
range: InfrastructureMonitoringRange;
|
||||||
|
series: Array<{ name: string; points: InfrastructureMetricPoint[]; color: string }>;
|
||||||
|
suffix: string;
|
||||||
|
maximum?: number;
|
||||||
|
}): EChartsOption {
|
||||||
|
const first = params.series[0]?.points ?? [];
|
||||||
|
return {
|
||||||
|
animationDuration: 280,
|
||||||
|
color: params.series.map((item) => item.color),
|
||||||
|
grid: { left: 8, right: 16, top: 34, bottom: 4, containLabel: true },
|
||||||
|
legend: params.series.length > 1 ? { top: 0, right: 0, textStyle: { color: '#6b7280', fontSize: 12 } } : undefined,
|
||||||
|
tooltip: {
|
||||||
|
trigger: 'axis',
|
||||||
|
valueFormatter: (value) => `${Number(value).toFixed(1)}${params.suffix}`,
|
||||||
|
},
|
||||||
|
xAxis: {
|
||||||
|
type: 'category',
|
||||||
|
boundaryGap: false,
|
||||||
|
data: timeLabels(first, params.range),
|
||||||
|
axisLine: { lineStyle: { color: '#e5e7eb' } },
|
||||||
|
axisTick: { show: false },
|
||||||
|
axisLabel: { color: '#9ca3af', hideOverlap: true, margin: 12 },
|
||||||
|
},
|
||||||
|
yAxis: {
|
||||||
|
type: 'value', min: 0, max: params.maximum,
|
||||||
|
axisLabel: { color: '#9ca3af', formatter: `{value}${params.suffix}` },
|
||||||
|
splitLine: { lineStyle: { color: '#eef0f3' } },
|
||||||
|
},
|
||||||
|
series: params.series.map((item) => ({
|
||||||
|
name: item.name,
|
||||||
|
data: item.points.map((point) => point.value),
|
||||||
|
type: 'line',
|
||||||
|
smooth: true,
|
||||||
|
showSymbol: false,
|
||||||
|
lineStyle: { width: 2.5 },
|
||||||
|
areaStyle: { opacity: 0.07 },
|
||||||
|
})),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function severityTag(severity: InfrastructureAlert['severity']) {
|
||||||
|
if (severity === 'critical') return <Tag tone="danger">严重</Tag>;
|
||||||
|
if (severity === 'warning') return <Tag tone="warning">警告</Tag>;
|
||||||
|
return <Tag tone="info">提示</Tag>;
|
||||||
|
}
|
||||||
|
|
||||||
|
function makeAlertColumns(onMarkRead: (alert: InfrastructureAlert) => void, readingFingerprint: string): Array<TableColumn<InfrastructureAlert>> { return [
|
||||||
|
{ key: 'severity', title: '级别', width: '82px', render: (record) => severityTag(record.severity) },
|
||||||
|
{
|
||||||
|
key: 'alert', title: '告警', width: '280px', render: (record) => (
|
||||||
|
<div className="system-monitoring-alert-copy"><strong>{record.name}</strong><span>{record.summary}</span></div>
|
||||||
|
),
|
||||||
|
},
|
||||||
|
{ key: 'service', title: '服务 / 实例', width: '190px', render: (record) => record.service || record.instance || '主机资源' },
|
||||||
|
{ key: 'value', title: '当前值 / 阈值', width: '150px', render: (record) => `${record.currentValue || '—'} / ${record.threshold || '—'}` },
|
||||||
|
{ key: 'startedAt', title: '开始时间', width: '150px', render: (record) => formatTime(record.startedAt) },
|
||||||
|
{ key: 'duration', title: '持续时间', width: '120px', render: (record) => formatDuration(record.startedAt) },
|
||||||
|
{
|
||||||
|
key: 'actions', title: '操作', width: '112px', render: (record) => record.acknowledged
|
||||||
|
? <Tag tone="neutral">已读</Tag>
|
||||||
|
: <Button disabled={readingFingerprint === record.fingerprint} icon={<CheckCircle2 size={14} />} onClick={() => onMarkRead(record)} size="sm" variant="ghost">{readingFingerprint === record.fingerprint ? '处理中' : '标记已读'}</Button>,
|
||||||
|
},
|
||||||
|
]; }
|
||||||
|
|
||||||
|
export function AdminSystemMonitoringPage() {
|
||||||
|
const [range, setRange] = useState<InfrastructureMonitoringRange>('24h');
|
||||||
|
const [overview, setOverview] = useState<InfrastructureMonitoringOverview | null>(null);
|
||||||
|
const [loading, setLoading] = useState(true);
|
||||||
|
const [error, setError] = useState('');
|
||||||
|
const [settings, setSettings] = useState<InfrastructureAlertSettings | null>(null);
|
||||||
|
const [draftThresholds, setDraftThresholds] = useState<InfrastructureAlertThresholds>({});
|
||||||
|
const [showSettings, setShowSettings] = useState(false);
|
||||||
|
const [settingsError, setSettingsError] = useState('');
|
||||||
|
const [savingSettings, setSavingSettings] = useState(false);
|
||||||
|
const [readingFingerprint, setReadingFingerprint] = useState('');
|
||||||
|
const [readError, setReadError] = useState('');
|
||||||
|
const requestSequence = useRef(0);
|
||||||
|
const pendingRequests = useRef(0);
|
||||||
|
|
||||||
|
const loadData = useCallback(async (supersede = false) => {
|
||||||
|
if (!supersede && pendingRequests.current > 0) return;
|
||||||
|
pendingRequests.current += 1;
|
||||||
|
const sequence = ++requestSequence.current;
|
||||||
|
setLoading(true);
|
||||||
|
try {
|
||||||
|
const result = await adminApi.getInfrastructureMonitoringOverview(range);
|
||||||
|
if (sequence !== requestSequence.current) return;
|
||||||
|
setOverview(result);
|
||||||
|
setError(result.available ? '' : result.error || '监控数据当前不可用');
|
||||||
|
} catch (reason) {
|
||||||
|
if (sequence !== requestSequence.current) return;
|
||||||
|
setOverview(null);
|
||||||
|
setError(reason instanceof Error ? reason.message : '监控数据加载失败');
|
||||||
|
} finally {
|
||||||
|
if (sequence === requestSequence.current) setLoading(false);
|
||||||
|
pendingRequests.current -= 1;
|
||||||
|
}
|
||||||
|
}, [range]);
|
||||||
|
|
||||||
|
const loadSettings = useCallback(async () => {
|
||||||
|
try {
|
||||||
|
const result = await adminApi.getInfrastructureAlertThresholds();
|
||||||
|
setSettings(result);
|
||||||
|
setDraftThresholds(result.thresholds);
|
||||||
|
setSettingsError('');
|
||||||
|
} catch (reason) {
|
||||||
|
setSettingsError(reason instanceof Error ? reason.message : '告警阈值加载失败');
|
||||||
|
}
|
||||||
|
}, []);
|
||||||
|
|
||||||
|
const saveSettings = useCallback(async () => {
|
||||||
|
if (!settings) return;
|
||||||
|
setSavingSettings(true);
|
||||||
|
setSettingsError('');
|
||||||
|
try {
|
||||||
|
const result = await adminApi.updateInfrastructureAlertThresholds({ configVersion: settings.configVersion, thresholds: draftThresholds });
|
||||||
|
setSettings(result);
|
||||||
|
setDraftThresholds(result.thresholds);
|
||||||
|
setShowSettings(false);
|
||||||
|
window.dispatchEvent(new Event('cmpp-infrastructure-alert-count-refresh'));
|
||||||
|
await loadData(true);
|
||||||
|
} catch (reason) {
|
||||||
|
setSettingsError(reason instanceof Error ? reason.message : '告警阈值保存失败');
|
||||||
|
} finally {
|
||||||
|
setSavingSettings(false);
|
||||||
|
}
|
||||||
|
}, [draftThresholds, loadData, settings]);
|
||||||
|
|
||||||
|
const markAlertRead = useCallback(async (alert: InfrastructureAlert) => {
|
||||||
|
setReadingFingerprint(alert.fingerprint);
|
||||||
|
setReadError('');
|
||||||
|
try {
|
||||||
|
const result = await adminApi.markInfrastructureAlertRead(alert.fingerprint, alert.startedAt);
|
||||||
|
setOverview((current) => current ? {
|
||||||
|
...current,
|
||||||
|
alerts: current.alerts.map((item) => item.fingerprint === result.fingerprint && Date.parse(item.startedAt) === Date.parse(result.activeAt)
|
||||||
|
? { ...item, acknowledged: true, acknowledgedAt: result.acknowledgedAt }
|
||||||
|
: item),
|
||||||
|
} : current);
|
||||||
|
window.dispatchEvent(new Event('cmpp-infrastructure-alert-count-refresh'));
|
||||||
|
} catch (reason) {
|
||||||
|
setReadError(reason instanceof Error ? reason.message : '活动告警标记已读失败');
|
||||||
|
} finally {
|
||||||
|
setReadingFingerprint('');
|
||||||
|
}
|
||||||
|
}, []);
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
void loadData(true);
|
||||||
|
void loadSettings();
|
||||||
|
const intervalId = window.setInterval(() => {
|
||||||
|
if (document.visibilityState === 'visible') void loadData();
|
||||||
|
}, 30_000);
|
||||||
|
const handleVisibility = () => {
|
||||||
|
if (document.visibilityState === 'visible') void loadData();
|
||||||
|
};
|
||||||
|
document.addEventListener('visibilitychange', handleVisibility);
|
||||||
|
return () => {
|
||||||
|
requestSequence.current += 1;
|
||||||
|
window.clearInterval(intervalId);
|
||||||
|
document.removeEventListener('visibilitychange', handleVisibility);
|
||||||
|
};
|
||||||
|
}, [loadData, loadSettings]);
|
||||||
|
|
||||||
|
const status = STATUS_COPY[overview?.summary.overallStatus ?? 'unknown'];
|
||||||
|
const cpuOption = useMemo(() => makeTrendOption({
|
||||||
|
range, maximum: 100, suffix: '%', series: [{ name: 'CPU', points: overview?.trends.cpuUsagePercent ?? [], color: '#2563eb' }],
|
||||||
|
}), [overview?.trends.cpuUsagePercent, range]);
|
||||||
|
const memoryOption = useMemo(() => makeTrendOption({
|
||||||
|
range, maximum: 100, suffix: '%', series: [{ name: '内存', points: overview?.trends.memoryUsagePercent ?? [], color: '#7c3aed' }],
|
||||||
|
}), [overview?.trends.memoryUsagePercent, range]);
|
||||||
|
const diskOption = useMemo(() => makeTrendOption({
|
||||||
|
range, maximum: 100, suffix: '%', series: [{ name: '根磁盘', points: overview?.trends.diskUsagePercent ?? [], color: '#d97706' }],
|
||||||
|
}), [overview?.trends.diskUsagePercent, range]);
|
||||||
|
const networkOption = useMemo(() => makeTrendOption({
|
||||||
|
range, suffix: ' B/s', series: [
|
||||||
|
{ name: '接收', points: overview?.trends.networkReceiveBytesPerSecond ?? [], color: '#0f766e' },
|
||||||
|
{ name: '发送', points: overview?.trends.networkTransmitBytesPerSecond ?? [], color: '#2563eb' },
|
||||||
|
],
|
||||||
|
}), [overview?.trends.networkReceiveBytesPerSecond, overview?.trends.networkTransmitBytesPerSecond, range]);
|
||||||
|
|
||||||
|
const metrics = overview?.metrics;
|
||||||
|
const serviceHealthy = overview?.summary.serviceHealthy ?? 0;
|
||||||
|
const serviceTotal = overview?.summary.serviceTotal ?? 6;
|
||||||
|
const alertColumns = useMemo(() => makeAlertColumns((alert) => { void markAlertRead(alert); }, readingFingerprint), [markAlertRead, readingFingerprint]);
|
||||||
|
|
||||||
|
return (
|
||||||
|
<section className="page-stack admin-system-monitoring-page">
|
||||||
|
<div className="page-heading system-monitoring-heading">
|
||||||
|
<div>
|
||||||
|
<Breadcrumb items={['系统管理', '系统监控']} />
|
||||||
|
<div className="system-monitoring-title-row">
|
||||||
|
<p>服务器资源、核心服务与活动告警,数据由 Prometheus 采集与计算</p>
|
||||||
|
<Tag tone={status.tone}>{status.label}</Tag>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div className="system-monitoring-controls">
|
||||||
|
<div className="system-monitoring-range" aria-label="监控时间范围" role="group">
|
||||||
|
{RANGE_OPTIONS.map((option) => (
|
||||||
|
<button
|
||||||
|
aria-pressed={range === option.value}
|
||||||
|
className={range === option.value ? 'is-active' : ''}
|
||||||
|
key={option.value}
|
||||||
|
onClick={() => setRange(option.value)}
|
||||||
|
type="button"
|
||||||
|
>{option.label}</button>
|
||||||
|
))}
|
||||||
|
</div>
|
||||||
|
<Button disabled={loading} icon={<RefreshCw className={loading ? 'is-spinning' : ''} size={16} />} onClick={() => void loadData()} variant="ghost">
|
||||||
|
{loading ? '刷新中' : '刷新'}
|
||||||
|
</Button>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{error ? (
|
||||||
|
<div className="system-monitoring-unavailable" role="alert">
|
||||||
|
<ShieldAlert size={20} />
|
||||||
|
<div><strong>监控数据不可用</strong><span>{error}。页面不会展示历史缓存值。</span></div>
|
||||||
|
</div>
|
||||||
|
) : null}
|
||||||
|
|
||||||
|
<div className="system-monitoring-health surface">
|
||||||
|
<div className="system-monitoring-health__copy">
|
||||||
|
<div className={`system-monitoring-health__mark is-${overview?.summary.overallStatus ?? 'unknown'}`}>
|
||||||
|
{overview?.summary.overallStatus === 'healthy' ? <CheckCircle2 size={24} /> : <AlertTriangle size={24} />}
|
||||||
|
</div>
|
||||||
|
<span>平台基础设施</span>
|
||||||
|
<strong>{status.label}</strong>
|
||||||
|
<small>最新采样 {formatTime(overview?.lastSampleAt ?? null)}</small>
|
||||||
|
</div>
|
||||||
|
<div className="system-monitoring-health__fact"><span>核心服务</span><strong>{serviceHealthy}/{serviceTotal}</strong><small>正常运行</small></div>
|
||||||
|
<div className="system-monitoring-health__fact"><span>活动告警</span><strong>{overview?.summary.activeAlerts ?? 0}</strong><small>{overview?.summary.criticalAlerts ?? 0} 严重 · {overview?.summary.warningAlerts ?? 0} 警告</small></div>
|
||||||
|
<div className="system-monitoring-health__fact"><span>系统负载</span><strong>{metrics?.load1 === null || metrics?.load1 === undefined ? '—' : metrics.load1.toFixed(2)}</strong><small>最近1分钟</small></div>
|
||||||
|
<div className="system-monitoring-health__fact"><span>持续运行</span><strong>{formatUptime(metrics?.uptimeSeconds ?? null)}</strong><small>主机启动后</small></div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div className="system-monitoring-metrics">
|
||||||
|
<article className="surface system-monitoring-metric"><div className="system-monitoring-metric__icon is-blue"><Cpu size={19} /></div><div><span>CPU 使用率</span><strong>{formatPercent(metrics?.cpuUsagePercent ?? null)}</strong><small>5分钟平均</small></div></article>
|
||||||
|
<article className="surface system-monitoring-metric"><div className="system-monitoring-metric__icon is-violet"><MemoryStick size={19} /></div><div><span>内存使用率</span><strong>{formatPercent(metrics?.memoryUsagePercent ?? null)}</strong><small>{formatBytes(metrics?.memoryAvailableBytes ?? null)} 可用 / {formatBytes(metrics?.memoryTotalBytes ?? null)}</small></div></article>
|
||||||
|
<article className="surface system-monitoring-metric"><div className="system-monitoring-metric__icon is-amber"><HardDrive size={19} /></div><div><span>根磁盘使用率</span><strong>{formatPercent(metrics?.diskUsagePercent ?? null)}</strong><small>{formatBytes(metrics?.diskAvailableBytes ?? null)} 可用 / {formatBytes(metrics?.diskTotalBytes ?? null)}</small></div></article>
|
||||||
|
<article className="surface system-monitoring-metric"><div className="system-monitoring-metric__icon is-green"><Network size={19} /></div><div><span>网络吞吐</span><strong>{formatRate(totalNetworkRate(metrics?.networkReceiveBytesPerSecond, metrics?.networkTransmitBytesPerSecond))}</strong><small>接收 {formatRate(metrics?.networkReceiveBytesPerSecond ?? null)} · 发送 {formatRate(metrics?.networkTransmitBytesPerSecond ?? null)}</small></div></article>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div className="system-monitoring-main-grid">
|
||||||
|
<div className="system-monitoring-chart-stack">
|
||||||
|
<article className="surface system-monitoring-chart-card"><header><div><Cpu size={17} /><strong>CPU 趋势</strong></div><span>{formatPercent(metrics?.cpuUsagePercent ?? null)}</span></header>{overview?.trends.cpuUsagePercent.length ? <Chart height={230} option={cpuOption} /> : <EmptyChart />}</article>
|
||||||
|
<article className="surface system-monitoring-chart-card"><header><div><MemoryStick size={17} /><strong>内存趋势</strong></div><span>{formatPercent(metrics?.memoryUsagePercent ?? null)}</span></header>{overview?.trends.memoryUsagePercent.length ? <Chart height={230} option={memoryOption} /> : <EmptyChart />}</article>
|
||||||
|
<article className="surface system-monitoring-chart-card"><header><div><HardDrive size={17} /><strong>磁盘趋势</strong></div><span>{formatPercent(metrics?.diskUsagePercent ?? null)}</span></header>{overview?.trends.diskUsagePercent.length ? <Chart height={230} option={diskOption} /> : <EmptyChart />}</article>
|
||||||
|
<article className="surface system-monitoring-chart-card"><header><div><Activity size={17} /><strong>网络趋势</strong></div><span>{formatRate(totalNetworkRate(metrics?.networkReceiveBytesPerSecond, metrics?.networkTransmitBytesPerSecond))}</span></header>{overview?.trends.networkReceiveBytesPerSecond.length ? <Chart height={230} option={networkOption} /> : <EmptyChart />}</article>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<aside className="surface system-monitoring-services">
|
||||||
|
<header><div><Server size={18} /><strong>核心服务</strong></div><Tag tone={serviceHealthy === serviceTotal && overview?.available ? 'success' : 'neutral'}>{serviceHealthy}/{serviceTotal} 正常</Tag></header>
|
||||||
|
<div className="system-monitoring-service-list">
|
||||||
|
{(overview?.services ?? []).map((service) => (
|
||||||
|
<div className="system-monitoring-service" key={service.key}>
|
||||||
|
<span className={`system-monitoring-service__dot is-${service.status}`} />
|
||||||
|
<div><strong>{service.name}</strong><small>{service.unit}</small></div>
|
||||||
|
<span>{service.status === 'healthy' ? '正常' : service.status === 'unhealthy' ? '异常' : '未知'}</span>
|
||||||
|
</div>
|
||||||
|
))}
|
||||||
|
{!overview?.services.length ? [
|
||||||
|
['api', 'API服务'], ['gateway', 'Gateway服务'], ['postgresql', 'PostgreSQL'], ['redis', 'Redis'], ['minio', 'MinIO'], ['nginx', 'Nginx'],
|
||||||
|
].map(([key, name]) => <div className="system-monitoring-service" key={key}><span className="system-monitoring-service__dot is-unknown" /><div><strong>{name}</strong><small>等待真实采集</small></div><span>未知</span></div>) : null}
|
||||||
|
</div>
|
||||||
|
<div className="system-monitoring-collector-note"><Database size={16} /><span>指标由 Prometheus 采集,业务数据库不写入高频时序数据。</span></div>
|
||||||
|
</aside>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<section className="surface system-monitoring-service-metrics">
|
||||||
|
<header>
|
||||||
|
<div><Database size={18} /><strong>服务关键指标</strong></div>
|
||||||
|
<div className="system-monitoring-service-actions"><span>固定低基数聚合,不含手机号、短信ID或SQL文本</span><Button icon={<Settings2 size={15} />} onClick={() => setShowSettings(true)} variant="ghost">告警阈值设置</Button></div>
|
||||||
|
</header>
|
||||||
|
<div className="system-monitoring-service-metric-grid">
|
||||||
|
{(overview?.serviceMetrics ?? []).map((group) => (
|
||||||
|
<article key={group.key}>
|
||||||
|
<div className="system-monitoring-service-metric-title"><strong>{group.name}</strong><Tag tone={group.available ? 'success' : 'neutral'}>{group.available ? '已采集' : '待采集'}</Tag></div>
|
||||||
|
{group.metrics.length ? group.metrics.map((metric) => <div className="system-monitoring-service-metric-row" key={metric.key}><span>{metric.label}</span><strong>{formatServiceMetric(metric.value, metric.unit)}</strong></div>) : <div className="system-monitoring-service-metric-empty">已监控服务可用性,待原生容量指标接入</div>}
|
||||||
|
</article>
|
||||||
|
))}
|
||||||
|
</div>
|
||||||
|
</section>
|
||||||
|
|
||||||
|
<section className="surface system-monitoring-alerts" id="active-alerts">
|
||||||
|
<header><div><AlertTriangle size={18} /><strong>活动告警</strong><Tag tone={overview?.summary.activeAlerts ? 'warning' : 'success'}>{overview?.summary.activeAlerts ?? 0}</Tag></div><span><Clock3 size={14} /> 刷新于 {formatTime(overview?.collectedAt ?? null)}</span></header>
|
||||||
|
{readError ? <div className="system-monitoring-unavailable" role="alert"><AlertTriangle size={18} /><div><strong>标记已读失败</strong><span>{readError}</span></div></div> : null}
|
||||||
|
<Table columns={alertColumns} data={overview?.alerts ?? []} emptyText={overview?.available ? '当前没有活动告警' : '监控不可用,无法读取活动告警'} pagination={false} rowKey="fingerprint" />
|
||||||
|
</section>
|
||||||
|
|
||||||
|
<Modal footer={<><Button onClick={() => setShowSettings(false)} variant="ghost">取消</Button><Button disabled={savingSettings || !settings} onClick={() => void saveSettings()}>{savingSettings ? '验证并应用中' : '保存并应用'}</Button></>} onClose={() => setShowSettings(false)} open={showSettings} title="Prometheus 告警阈值设置">
|
||||||
|
<div className="system-monitoring-threshold-dialog">
|
||||||
|
<div className="system-monitoring-threshold-note"><ShieldAlert size={17} /><span>仅允许修改固定指标的数值阈值。保存时先由 promtool 校验,再原子替换规则并热加载 Prometheus。</span></div>
|
||||||
|
{settings?.definitions.map((definition) => (
|
||||||
|
<div className="system-monitoring-threshold-row" key={definition.key}>
|
||||||
|
<div><strong>{definition.label}</strong><small>单位:{definition.unit}</small></div>
|
||||||
|
<Input label="警告阈值" max={definition.max} min={definition.min} onChange={(event) => setDraftThresholds((current) => ({ ...current, [definition.key]: { ...current[definition.key], warning: Number(event.target.value) } }))} step={definition.step} type="number" value={draftThresholds[definition.key]?.warning ?? ''} />
|
||||||
|
<Input label="严重阈值" max={definition.max} min={definition.min} onChange={(event) => setDraftThresholds((current) => ({ ...current, [definition.key]: { ...current[definition.key], critical: Number(event.target.value) } }))} step={definition.step} type="number" value={draftThresholds[definition.key]?.critical ?? ''} />
|
||||||
|
</div>
|
||||||
|
))}
|
||||||
|
{settings?.applyStatus === 'failed' ? <div className="system-monitoring-unavailable"><AlertTriangle size={18} /><div><strong>上次应用失败</strong><span>{settings.lastError}</span></div></div> : null}
|
||||||
|
{settingsError ? <div className="system-monitoring-unavailable"><AlertTriangle size={18} /><div><strong>阈值配置不可用</strong><span>{settingsError}</span></div></div> : null}
|
||||||
|
</div>
|
||||||
|
</Modal>
|
||||||
|
</section>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function EmptyChart() {
|
||||||
|
return <div className="system-monitoring-chart-empty"><Activity size={22} /><span>暂无真实趋势指标</span></div>;
|
||||||
|
}
|
||||||
@@ -25,6 +25,7 @@ import {
|
|||||||
ReceiptText,
|
ReceiptText,
|
||||||
ClipboardList,
|
ClipboardList,
|
||||||
Send,
|
Send,
|
||||||
|
ServerCog,
|
||||||
ScanSearch,
|
ScanSearch,
|
||||||
Settings,
|
Settings,
|
||||||
Shield,
|
Shield,
|
||||||
@@ -47,6 +48,8 @@ export function AdminLayout() {
|
|||||||
function AdminAuthenticatedLayout({ session }: { session: LoginSession }) {
|
function AdminAuthenticatedLayout({ session }: { session: LoginSession }) {
|
||||||
const [pendingAudits, setPendingAudits] = useState(EMPTY_PENDING_AUDITS);
|
const [pendingAudits, setPendingAudits] = useState(EMPTY_PENDING_AUDITS);
|
||||||
const [retirementUnreadCount, setRetirementUnreadCount] = useState(0);
|
const [retirementUnreadCount, setRetirementUnreadCount] = useState(0);
|
||||||
|
const [securityAlertSummary, setSecurityAlertSummary] = useState({ count: 0, criticalCount: 0 });
|
||||||
|
const [infrastructureAlertSummary, setInfrastructureAlertSummary] = useState({ count: 0, criticalCount: 0 });
|
||||||
const [sessionLocked, setSessionLocked] = useState(Boolean(session.locked));
|
const [sessionLocked, setSessionLocked] = useState(Boolean(session.locked));
|
||||||
const loadPendingAuditCount = useCallback(() => {
|
const loadPendingAuditCount = useCallback(() => {
|
||||||
const currentSession = readSession('admin');
|
const currentSession = readSession('admin');
|
||||||
@@ -55,14 +58,18 @@ function AdminAuthenticatedLayout({ session }: { session: LoginSession }) {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
// This runs globally and on a timer, so it must not fan out through the full dashboard aggregation.
|
// This runs globally and on a timer, so it must not fan out through the full dashboard aggregation.
|
||||||
Promise.allSettled([adminApi.getPendingAudits(), adminApi.getSignatureRetirementUnreadCount()])
|
Promise.allSettled([adminApi.getPendingAudits(), adminApi.getSignatureRetirementUnreadCount(), adminApi.getSecurityNotificationSummary(), adminApi.getInfrastructureMonitoringNotificationSummary()])
|
||||||
.then(([audits, retirement]) => {
|
.then(([audits, retirement, security, infrastructure]) => {
|
||||||
setPendingAudits(audits.status === 'fulfilled' ? audits.value : EMPTY_PENDING_AUDITS);
|
setPendingAudits(audits.status === 'fulfilled' ? audits.value : EMPTY_PENDING_AUDITS);
|
||||||
setRetirementUnreadCount(retirement.status === 'fulfilled' ? retirement.value.count : 0);
|
setRetirementUnreadCount(retirement.status === 'fulfilled' ? retirement.value.count : 0);
|
||||||
|
setSecurityAlertSummary(security.status === 'fulfilled' ? security.value : { count: 0, criticalCount: 0 });
|
||||||
|
setInfrastructureAlertSummary(infrastructure.status === 'fulfilled' ? infrastructure.value : { count: 0, criticalCount: 0 });
|
||||||
})
|
})
|
||||||
.catch(() => {
|
.catch(() => {
|
||||||
setPendingAudits(EMPTY_PENDING_AUDITS);
|
setPendingAudits(EMPTY_PENDING_AUDITS);
|
||||||
setRetirementUnreadCount(0);
|
setRetirementUnreadCount(0);
|
||||||
|
setSecurityAlertSummary({ count: 0, criticalCount: 0 });
|
||||||
|
setInfrastructureAlertSummary({ count: 0, criticalCount: 0 });
|
||||||
});
|
});
|
||||||
}, []);
|
}, []);
|
||||||
|
|
||||||
@@ -77,11 +84,15 @@ function AdminAuthenticatedLayout({ session }: { session: LoginSession }) {
|
|||||||
window.addEventListener('focus', onFocus);
|
window.addEventListener('focus', onFocus);
|
||||||
window.addEventListener('cmpp-audit-count-refresh', onAuditRefresh);
|
window.addEventListener('cmpp-audit-count-refresh', onAuditRefresh);
|
||||||
window.addEventListener('cmpp-retirement-count-refresh', onAuditRefresh);
|
window.addEventListener('cmpp-retirement-count-refresh', onAuditRefresh);
|
||||||
|
window.addEventListener('cmpp-security-alert-count-refresh', onAuditRefresh);
|
||||||
|
window.addEventListener('cmpp-infrastructure-alert-count-refresh', onAuditRefresh);
|
||||||
return () => {
|
return () => {
|
||||||
window.clearInterval(timer);
|
window.clearInterval(timer);
|
||||||
window.removeEventListener('focus', onFocus);
|
window.removeEventListener('focus', onFocus);
|
||||||
window.removeEventListener('cmpp-audit-count-refresh', onAuditRefresh);
|
window.removeEventListener('cmpp-audit-count-refresh', onAuditRefresh);
|
||||||
window.removeEventListener('cmpp-retirement-count-refresh', onAuditRefresh);
|
window.removeEventListener('cmpp-retirement-count-refresh', onAuditRefresh);
|
||||||
|
window.removeEventListener('cmpp-security-alert-count-refresh', onAuditRefresh);
|
||||||
|
window.removeEventListener('cmpp-infrastructure-alert-count-refresh', onAuditRefresh);
|
||||||
};
|
};
|
||||||
}, [loadPendingAuditCount, session.portal, sessionLocked]);
|
}, [loadPendingAuditCount, session.portal, sessionLocked]);
|
||||||
|
|
||||||
@@ -95,7 +106,11 @@ function AdminAuthenticatedLayout({ session }: { session: LoginSession }) {
|
|||||||
userName={session.user.displayName}
|
userName={session.user.displayName}
|
||||||
userRole="平台管理员"
|
userRole="平台管理员"
|
||||||
onSessionLockedChange={setSessionLocked}
|
onSessionLockedChange={setSessionLocked}
|
||||||
retirementAlert={{ count: retirementUnreadCount, to: '/admin/signature-retirement' }}
|
alertNotifications={[
|
||||||
|
{ label: '签名清退预警', count: retirementUnreadCount, description: '今日未读且未抑制', to: '/admin/signature-retirement' },
|
||||||
|
{ label: '安全检测与封禁', count: securityAlertSummary.count, description: securityAlertSummary.criticalCount > 0 ? `${securityAlertSummary.criticalCount} 条严重告警待处置` : '待处置安全告警', to: '/admin/security-detection' },
|
||||||
|
{ label: '系统监控告警', count: infrastructureAlertSummary.count, description: infrastructureAlertSummary.criticalCount > 0 ? `${infrastructureAlertSummary.criticalCount} 条 Prometheus 严重告警` : 'Prometheus 活动告警', to: '/admin/system-monitoring#active-alerts' },
|
||||||
|
]}
|
||||||
auditNotifications={[
|
auditNotifications={[
|
||||||
{ label: '企业认证待审', count: pendingAudits.enterpriseCertifications, to: '/admin/enterprise-audit' },
|
{ label: '企业认证待审', count: pendingAudits.enterpriseCertifications, to: '/admin/enterprise-audit' },
|
||||||
{ label: '短信审核待审', count: pendingAudits.smsAudits, to: '/admin/sms-audit' },
|
{ label: '短信审核待审', count: pendingAudits.smsAudits, to: '/admin/sms-audit' },
|
||||||
@@ -187,6 +202,7 @@ function AdminAuthenticatedLayout({ session }: { session: LoginSession }) {
|
|||||||
icon: Shield,
|
icon: Shield,
|
||||||
items: [
|
items: [
|
||||||
{ label: '风控规则', to: '/admin/risk-rules', icon: Shield },
|
{ label: '风控规则', to: '/admin/risk-rules', icon: Shield },
|
||||||
|
{ label: '安全检测与封禁', to: '/admin/security-detection', icon: ShieldCheck },
|
||||||
{ label: '签名清退预警', to: '/admin/signature-retirement', icon: AlertTriangle },
|
{ label: '签名清退预警', to: '/admin/signature-retirement', icon: AlertTriangle },
|
||||||
{ label: '企业黑名单', to: '/admin/enterprise-blacklist', icon: UserX },
|
{ label: '企业黑名单', to: '/admin/enterprise-blacklist', icon: UserX },
|
||||||
{ label: '全局黑名单', to: '/admin/global-blacklist', icon: ShieldOff },
|
{ label: '全局黑名单', to: '/admin/global-blacklist', icon: ShieldOff },
|
||||||
@@ -202,6 +218,7 @@ function AdminAuthenticatedLayout({ session }: { session: LoginSession }) {
|
|||||||
{ label: '报备字段库', to: '/admin/drainage-fields', icon: Hash },
|
{ label: '报备字段库', to: '/admin/drainage-fields', icon: Hash },
|
||||||
{ label: '引流识别规则', to: '/admin/drainage-detection-rules', icon: ScanSearch },
|
{ label: '引流识别规则', to: '/admin/drainage-detection-rules', icon: ScanSearch },
|
||||||
{ label: '系统日志', to: '/admin/system-logs', icon: FileText },
|
{ label: '系统日志', to: '/admin/system-logs', icon: FileText },
|
||||||
|
{ label: '系统监控', to: '/admin/system-monitoring', icon: ServerCog },
|
||||||
],
|
],
|
||||||
},
|
},
|
||||||
]}
|
]}
|
||||||
|
|||||||
+40
-13
@@ -50,6 +50,10 @@ export type AuditNotificationItem = {
|
|||||||
to: string;
|
to: string;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
export type AlertNotificationItem = AuditNotificationItem & {
|
||||||
|
description: string;
|
||||||
|
};
|
||||||
|
|
||||||
type AppShellProps = {
|
type AppShellProps = {
|
||||||
title: string;
|
title: string;
|
||||||
subtitle: string;
|
subtitle: string;
|
||||||
@@ -60,7 +64,7 @@ type AppShellProps = {
|
|||||||
userRole: string;
|
userRole: string;
|
||||||
navSections: ShellNavSection[];
|
navSections: ShellNavSection[];
|
||||||
auditNotifications?: AuditNotificationItem[];
|
auditNotifications?: AuditNotificationItem[];
|
||||||
retirementAlert?: { count: number; to: string };
|
alertNotifications?: AlertNotificationItem[];
|
||||||
onSessionLockedChange?: (locked: boolean) => void;
|
onSessionLockedChange?: (locked: boolean) => void;
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -73,7 +77,7 @@ export function AppShell({
|
|||||||
userRole,
|
userRole,
|
||||||
navSections,
|
navSections,
|
||||||
auditNotifications = [],
|
auditNotifications = [],
|
||||||
retirementAlert,
|
alertNotifications = [],
|
||||||
onSessionLockedChange,
|
onSessionLockedChange,
|
||||||
}: AppShellProps) {
|
}: AppShellProps) {
|
||||||
const [collapsed, setCollapsed] = useState(false);
|
const [collapsed, setCollapsed] = useState(false);
|
||||||
@@ -81,6 +85,7 @@ export function AppShell({
|
|||||||
const [closedSections, setClosedSections] = useState<Record<string, boolean>>({});
|
const [closedSections, setClosedSections] = useState<Record<string, boolean>>({});
|
||||||
const [userMenuOpen, setUserMenuOpen] = useState(false);
|
const [userMenuOpen, setUserMenuOpen] = useState(false);
|
||||||
const [noticeOpen, setNoticeOpen] = useState(false);
|
const [noticeOpen, setNoticeOpen] = useState(false);
|
||||||
|
const [alertNoticeOpen, setAlertNoticeOpen] = useState(false);
|
||||||
const [passwordModalOpen, setPasswordModalOpen] = useState(false);
|
const [passwordModalOpen, setPasswordModalOpen] = useState(false);
|
||||||
const [currentPassword, setCurrentPassword] = useState('');
|
const [currentPassword, setCurrentPassword] = useState('');
|
||||||
const [newPassword, setNewPassword] = useState('');
|
const [newPassword, setNewPassword] = useState('');
|
||||||
@@ -107,6 +112,10 @@ export function AppShell({
|
|||||||
() => auditNotifications.reduce((sum, item) => sum + item.count, 0),
|
() => auditNotifications.reduce((sum, item) => sum + item.count, 0),
|
||||||
[auditNotifications],
|
[auditNotifications],
|
||||||
);
|
);
|
||||||
|
const alertTotal = useMemo(
|
||||||
|
() => alertNotifications.reduce((sum, item) => sum + item.count, 0),
|
||||||
|
[alertNotifications],
|
||||||
|
);
|
||||||
|
|
||||||
async function changeOwnPassword() {
|
async function changeOwnPassword() {
|
||||||
if (!currentPassword || newPassword.length < 6) {
|
if (!currentPassword || newPassword.length < 6) {
|
||||||
@@ -417,23 +426,41 @@ export function AppShell({
|
|||||||
<button className="icon-button topbar-help" type="button" aria-label="帮助中心">
|
<button className="icon-button topbar-help" type="button" aria-label="帮助中心">
|
||||||
<CircleHelp size={18} />
|
<CircleHelp size={18} />
|
||||||
</button>
|
</button>
|
||||||
{retirementAlert ? (
|
{alertNotifications.length ? (
|
||||||
<Link
|
<div className="notice-menu-wrap">
|
||||||
aria-label={`今日未读且未抑制签名清退预警 ${retirementAlert.count} 条`}
|
<button
|
||||||
className={['icon-button', retirementAlert.count > 0 ? 'has-dot' : ''].filter(Boolean).join(' ')}
|
aria-expanded={alertNoticeOpen}
|
||||||
title="今日未读且未抑制签名清退预警"
|
aria-haspopup="menu"
|
||||||
to={retirementAlert.to}
|
aria-label="预警通知"
|
||||||
>
|
className={['icon-button', alertTotal > 0 ? 'has-dot' : ''].filter(Boolean).join(' ')}
|
||||||
<Bell size={18} />
|
onClick={() => { setAlertNoticeOpen((open) => !open); setNoticeOpen(false); }}
|
||||||
{retirementAlert.count > 0 ? <span className="notice-count">{retirementAlert.count}</span> : null}
|
type="button"
|
||||||
</Link>
|
>
|
||||||
|
<Bell size={18} />
|
||||||
|
{alertTotal > 0 ? <span className="notice-count">{alertTotal}</span> : null}
|
||||||
|
</button>
|
||||||
|
{alertNoticeOpen ? (
|
||||||
|
<div className="notice-popover notice-popover--alerts" role="menu">
|
||||||
|
<div className="notice-popover__header">
|
||||||
|
<strong>预警中心</strong>
|
||||||
|
<span className={alertTotal === 0 ? 'is-zero' : ''}>{alertTotal} 条</span>
|
||||||
|
</div>
|
||||||
|
{alertNotifications.map((item) => (
|
||||||
|
<NavLink key={item.to} onClick={() => setAlertNoticeOpen(false)} role="menuitem" to={item.to}>
|
||||||
|
<span className="notice-popover__copy"><b>{item.label}</b><small>{item.description}</small></span>
|
||||||
|
<strong className={item.count === 0 ? 'is-zero' : ''}>{item.count}</strong>
|
||||||
|
</NavLink>
|
||||||
|
))}
|
||||||
|
</div>
|
||||||
|
) : null}
|
||||||
|
</div>
|
||||||
) : null}
|
) : null}
|
||||||
<div className="notice-menu-wrap">
|
<div className="notice-menu-wrap">
|
||||||
<button
|
<button
|
||||||
aria-expanded={noticeOpen}
|
aria-expanded={noticeOpen}
|
||||||
aria-haspopup="menu"
|
aria-haspopup="menu"
|
||||||
className={['icon-button', auditTotal > 0 ? 'has-dot' : ''].filter(Boolean).join(' ')}
|
className={['icon-button', auditTotal > 0 ? 'has-dot' : ''].filter(Boolean).join(' ')}
|
||||||
onClick={() => setNoticeOpen((open) => !open)}
|
onClick={() => { setNoticeOpen((open) => !open); setAlertNoticeOpen(false); }}
|
||||||
type="button"
|
type="button"
|
||||||
aria-label="通知"
|
aria-label="通知"
|
||||||
>
|
>
|
||||||
|
|||||||
@@ -38,6 +38,8 @@ import { AdminSignatureAuditPage } from '@/apps/admin/AdminSignatureAuditPage';
|
|||||||
import { AdminSignatureRetirementPage } from '@/apps/admin/AdminSignatureRetirementPage';
|
import { AdminSignatureRetirementPage } from '@/apps/admin/AdminSignatureRetirementPage';
|
||||||
import { AdminDrainageAuditPage } from '@/apps/admin/AdminDrainageAuditPage';
|
import { AdminDrainageAuditPage } from '@/apps/admin/AdminDrainageAuditPage';
|
||||||
import { AdminSystemLogsPage } from '@/apps/admin/AdminSystemLogsPage';
|
import { AdminSystemLogsPage } from '@/apps/admin/AdminSystemLogsPage';
|
||||||
|
import { AdminSystemMonitoringPage } from '@/apps/admin/system-monitoring/AdminSystemMonitoringPage';
|
||||||
|
import { AdminSecurityDetectionPage } from '@/apps/admin/security-detection/AdminSecurityDetectionPage';
|
||||||
import { AdminTemplateAuditPage } from '@/apps/admin/AdminTemplateAuditPage';
|
import { AdminTemplateAuditPage } from '@/apps/admin/AdminTemplateAuditPage';
|
||||||
import { AdminUsersPage } from '@/apps/admin/AdminUsersPage';
|
import { AdminUsersPage } from '@/apps/admin/AdminUsersPage';
|
||||||
import { AdminEnterpriseAuditPage } from '@/apps/admin/AdminEnterpriseAuditPage';
|
import { AdminEnterpriseAuditPage } from '@/apps/admin/AdminEnterpriseAuditPage';
|
||||||
@@ -142,6 +144,8 @@ export function AppRoutes() {
|
|||||||
<Route path="drainage-fields" element={<AdminDrainageFieldsPage />} />
|
<Route path="drainage-fields" element={<AdminDrainageFieldsPage />} />
|
||||||
<Route path="drainage-detection-rules" element={<AdminDrainageDetectionRulesPage />} />
|
<Route path="drainage-detection-rules" element={<AdminDrainageDetectionRulesPage />} />
|
||||||
<Route path="system-logs" element={<AdminSystemLogsPage />} />
|
<Route path="system-logs" element={<AdminSystemLogsPage />} />
|
||||||
|
<Route path="system-monitoring" element={<AdminSystemMonitoringPage />} />
|
||||||
|
<Route path="security-detection" element={<AdminSecurityDetectionPage />} />
|
||||||
<Route path="*" element={<PagePlaceholder />} />
|
<Route path="*" element={<PagePlaceholder />} />
|
||||||
</Route>
|
</Route>
|
||||||
</Routes>
|
</Routes>
|
||||||
|
|||||||
@@ -9424,10 +9424,14 @@
|
|||||||
.downstream-page-size { display: inline-flex; align-items: center; gap: 8px; color: var(--text-muted); font-size: 13px; }
|
.downstream-page-size { display: inline-flex; align-items: center; gap: 8px; color: var(--text-muted); font-size: 13px; }
|
||||||
.downstream-page-size select { min-width: 76px; height: 36px; border: 1px solid var(--border); border-radius: 8px; background: var(--surface); color: var(--text); padding: 0 10px; }
|
.downstream-page-size select { min-width: 76px; height: 36px; border: 1px solid var(--border); border-radius: 8px; background: var(--surface); color: var(--text); padding: 0 10px; }
|
||||||
.downstream-requeue-task-heading { gap: 20px; }
|
.downstream-requeue-task-heading { gap: 20px; }
|
||||||
|
.downstream-requeue-task-card > .downstream-requeue-task-heading { padding: var(--space-6) var(--space-6) var(--space-5); }
|
||||||
.downstream-requeue-task-heading__actions { display: flex; align-items: center; gap: 8px; }
|
.downstream-requeue-task-heading__actions { display: flex; align-items: center; gap: 8px; }
|
||||||
.downstream-requeue-task-heading__actions .ui-field { min-width: 150px; }
|
.downstream-requeue-task-heading__actions .ui-field { min-width: 150px; }
|
||||||
.downstream-requeue-task-list { display: grid; border-top: 1px solid var(--border); }
|
.downstream-requeue-task-list { display: grid; margin: 0 var(--space-6); overflow: hidden; border: 1px solid var(--border); border-radius: var(--radius-lg); background: var(--color-surface); }
|
||||||
.downstream-requeue-task-list article { display: grid; grid-template-columns: minmax(190px, .9fr) minmax(260px, 1.35fr) minmax(250px, 1fr) auto; gap: 22px; align-items: center; padding: 18px 4px; border-bottom: 1px solid var(--border); }
|
.downstream-requeue-task-list article { display: grid; grid-template-columns: minmax(190px, .9fr) minmax(260px, 1.35fr) minmax(250px, 1fr) auto; gap: 22px; align-items: center; padding: 18px var(--space-5); border-bottom: 1px solid var(--border); }
|
||||||
|
.downstream-requeue-task-list article:last-of-type { border-bottom: 0; }
|
||||||
|
.downstream-requeue-task-list > .muted { margin: 0; padding: var(--space-6); text-align: center; }
|
||||||
|
.downstream-requeue-task-card > .ui-pagination { padding: var(--space-5) var(--space-6) var(--space-6); }
|
||||||
.downstream-requeue-task-list article > div { min-width: 0; }
|
.downstream-requeue-task-list article > div { min-width: 0; }
|
||||||
.downstream-requeue-task-list__identity, .downstream-requeue-task-list__scope, .downstream-requeue-task-list__progress { display: grid; gap: 5px; }
|
.downstream-requeue-task-list__identity, .downstream-requeue-task-list__scope, .downstream-requeue-task-list__progress { display: grid; gap: 5px; }
|
||||||
.downstream-requeue-task-list__identity > strong { color: var(--text-strong); font-size: 14px; font-variant-numeric: tabular-nums; }
|
.downstream-requeue-task-list__identity > strong { color: var(--text-strong); font-size: 14px; font-variant-numeric: tabular-nums; }
|
||||||
@@ -9447,6 +9451,8 @@
|
|||||||
.downstream-requeue-preview__distribution { display: grid; gap: 8px; }
|
.downstream-requeue-preview__distribution { display: grid; gap: 8px; }
|
||||||
.downstream-requeue-preview__distribution > span { color: var(--text-muted); font-size: 13px; }
|
.downstream-requeue-preview__distribution > span { color: var(--text-muted); font-size: 13px; }
|
||||||
.downstream-requeue-preview__distribution > div { display: flex; flex-wrap: wrap; gap: 8px; }
|
.downstream-requeue-preview__distribution > div { display: flex; flex-wrap: wrap; gap: 8px; }
|
||||||
|
.downstream-requeue-reason { padding: 16px; border: 1px solid var(--border); border-radius: var(--radius-lg); background: var(--surface-subtle, #f8fafc); }
|
||||||
|
.downstream-requeue-reason .ui-textarea { min-height: 124px; background: var(--color-surface); line-height: 1.65; }
|
||||||
.downstream-requeue-warning { display: flex; align-items: flex-start; gap: 10px; padding: 14px; border: 1px solid #fed7aa; border-radius: 10px; background: var(--warning-soft, #fff7ed); color: var(--warning-text, #9a3412); }
|
.downstream-requeue-warning { display: flex; align-items: flex-start; gap: 10px; padding: 14px; border: 1px solid #fed7aa; border-radius: 10px; background: var(--warning-soft, #fff7ed); color: var(--warning-text, #9a3412); }
|
||||||
.downstream-requeue-warning > div { display: grid; gap: 4px; }
|
.downstream-requeue-warning > div { display: grid; gap: 4px; }
|
||||||
.downstream-requeue-warning p { margin: 0; color: inherit; line-height: 1.6; }
|
.downstream-requeue-warning p { margin: 0; color: inherit; line-height: 1.6; }
|
||||||
@@ -9460,4 +9466,4 @@
|
|||||||
.downstream-requeue-detail-items__head { background: var(--surface-subtle, #f8fafc); color: var(--text-muted); font-size: 12px; font-weight: 600; }
|
.downstream-requeue-detail-items__head { background: var(--surface-subtle, #f8fafc); color: var(--text-muted); font-size: 12px; font-weight: 600; }
|
||||||
.downstream-requeue-detail-items time { color: var(--text-muted); font-size: 12px; }
|
.downstream-requeue-detail-items time { color: var(--text-muted); font-size: 12px; }
|
||||||
@media (max-width: 1100px) { .downstream-requeue-task-list article { grid-template-columns: 1fr 1.4fr; } .downstream-requeue-task-list__actions { justify-content: flex-start; } .downstream-requeue-preview__summary { grid-template-columns: repeat(2, 1fr); } }
|
@media (max-width: 1100px) { .downstream-requeue-task-list article { grid-template-columns: 1fr 1.4fr; } .downstream-requeue-task-list__actions { justify-content: flex-start; } .downstream-requeue-preview__summary { grid-template-columns: repeat(2, 1fr); } }
|
||||||
@media (max-width: 780px) { .downstream-requeue-task-heading, .downstream-requeue-task-heading__actions { align-items: stretch; flex-direction: column; } .downstream-requeue-task-list article, .downstream-requeue-preview__summary, .downstream-requeue-detail-filter { grid-template-columns: 1fr; } .downstream-requeue-detail-items { border: 0; overflow: visible; gap: 10px; } .downstream-requeue-detail-items__head { display: none !important; } .downstream-requeue-detail-items > div { grid-template-columns: 1fr; gap: 6px; padding: 14px; border: 1px solid var(--border); border-radius: 10px; } }
|
@media (max-width: 780px) { .downstream-requeue-task-card > .downstream-requeue-task-heading { padding: var(--space-5) var(--space-4) var(--space-4); } .downstream-requeue-task-heading, .downstream-requeue-task-heading__actions { align-items: stretch; flex-direction: column; } .downstream-requeue-task-list { margin: 0 var(--space-4); } .downstream-requeue-task-list article, .downstream-requeue-preview__summary, .downstream-requeue-detail-filter { grid-template-columns: 1fr; } .downstream-requeue-task-card > .ui-pagination { padding: var(--space-4); } .downstream-requeue-detail-items { border: 0; overflow: visible; gap: 10px; } .downstream-requeue-detail-items__head { display: none !important; } .downstream-requeue-detail-items > div { grid-template-columns: 1fr; gap: 6px; padding: 14px; border: 1px solid var(--border); border-radius: 10px; } }
|
||||||
|
|||||||
@@ -571,6 +571,31 @@
|
|||||||
text-align: right;
|
text-align: right;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
.notice-popover--alerts {
|
||||||
|
min-width: 292px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.notice-popover--alerts a {
|
||||||
|
gap: var(--space-4);
|
||||||
|
min-height: 58px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.notice-popover__copy {
|
||||||
|
align-items: flex-start;
|
||||||
|
display: grid;
|
||||||
|
gap: 2px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.notice-popover__copy b {
|
||||||
|
color: var(--color-text-strong);
|
||||||
|
font-weight: var(--font-weight-semibold);
|
||||||
|
}
|
||||||
|
|
||||||
|
.notice-popover__copy small {
|
||||||
|
color: var(--color-text-muted);
|
||||||
|
font-size: 11px;
|
||||||
|
}
|
||||||
|
|
||||||
.page-heading__actions {
|
.page-heading__actions {
|
||||||
align-items: center;
|
align-items: center;
|
||||||
display: flex;
|
display: flex;
|
||||||
|
|||||||
@@ -6,6 +6,9 @@ REPO_URL="${REPO_URL:-http://175.27.255.91:3000/hectorzhao/lislgosms.git}"
|
|||||||
BRANCH="${BRANCH:-main}"
|
BRANCH="${BRANCH:-main}"
|
||||||
PUBLIC_HTTP_PORT="${PUBLIC_HTTP_PORT:-12026}"
|
PUBLIC_HTTP_PORT="${PUBLIC_HTTP_PORT:-12026}"
|
||||||
API_PORT="${API_PORT:-3000}"
|
API_PORT="${API_PORT:-3000}"
|
||||||
|
API_HOST="${API_HOST:-127.0.0.1}"
|
||||||
|
API_METRICS_HOST="${API_METRICS_HOST:-127.0.0.1}"
|
||||||
|
API_METRICS_PORT="${API_METRICS_PORT:-9464}"
|
||||||
API_ENABLE_SEND_WORKER="${API_ENABLE_SEND_WORKER:-true}"
|
API_ENABLE_SEND_WORKER="${API_ENABLE_SEND_WORKER:-true}"
|
||||||
API_SEND_WORKER_CONCURRENCY="${API_SEND_WORKER_CONCURRENCY:-50}"
|
API_SEND_WORKER_CONCURRENCY="${API_SEND_WORKER_CONCURRENCY:-50}"
|
||||||
GATEWAY_CONTROL_ADDR="${GATEWAY_CONTROL_ADDR:-127.0.0.1:8090}"
|
GATEWAY_CONTROL_ADDR="${GATEWAY_CONTROL_ADDR:-127.0.0.1:8090}"
|
||||||
@@ -31,6 +34,10 @@ OPERATION_LOG_ARCHIVE_INTERVAL_MS="${OPERATION_LOG_ARCHIVE_INTERVAL_MS:-86400000
|
|||||||
SMS_RECEIPT_TIMEOUT_SCAN_ENABLED="${SMS_RECEIPT_TIMEOUT_SCAN_ENABLED:-true}"
|
SMS_RECEIPT_TIMEOUT_SCAN_ENABLED="${SMS_RECEIPT_TIMEOUT_SCAN_ENABLED:-true}"
|
||||||
SMS_RECEIPT_TIMEOUT_HOURS="${SMS_RECEIPT_TIMEOUT_HOURS:-72}"
|
SMS_RECEIPT_TIMEOUT_HOURS="${SMS_RECEIPT_TIMEOUT_HOURS:-72}"
|
||||||
SMS_RECEIPT_TIMEOUT_SCAN_INTERVAL_MS="${SMS_RECEIPT_TIMEOUT_SCAN_INTERVAL_MS:-300000}"
|
SMS_RECEIPT_TIMEOUT_SCAN_INTERVAL_MS="${SMS_RECEIPT_TIMEOUT_SCAN_INTERVAL_MS:-300000}"
|
||||||
|
PROMETHEUS_URL="${PROMETHEUS_URL:-http://127.0.0.1:9090}"
|
||||||
|
PROMETHEUS_QUERY_TIMEOUT_MS="${PROMETHEUS_QUERY_TIMEOUT_MS:-5000}"
|
||||||
|
SECURITY_EVENT_TOKEN="${SECURITY_EVENT_TOKEN:-$(openssl rand -hex 32 | tr -d '\n')}"
|
||||||
|
SECURITY_BUILTIN_PROTECTED_NETWORKS="${SECURITY_BUILTIN_PROTECTED_NETWORKS:-${CMPP_PUBLIC_HOST}/32}"
|
||||||
|
|
||||||
if [[ "$(id -u)" -ne 0 ]]; then
|
if [[ "$(id -u)" -ne 0 ]]; then
|
||||||
echo "Run as root." >&2
|
echo "Run as root." >&2
|
||||||
@@ -43,7 +50,7 @@ install_packages() {
|
|||||||
log "Installing OS packages"
|
log "Installing OS packages"
|
||||||
if command -v apt-get >/dev/null 2>&1; then
|
if command -v apt-get >/dev/null 2>&1; then
|
||||||
apt-get update
|
apt-get update
|
||||||
DEBIAN_FRONTEND=noninteractive apt-get install -y ca-certificates curl gnupg git nginx redis-server postgresql postgresql-contrib build-essential tar gzip xz-utils openssl
|
DEBIAN_FRONTEND=noninteractive apt-get install -y ca-certificates curl gnupg git nginx redis-server postgresql postgresql-contrib build-essential tar gzip xz-utils openssl fail2ban nftables
|
||||||
elif command -v dnf >/dev/null 2>&1; then
|
elif command -v dnf >/dev/null 2>&1; then
|
||||||
dnf install -y ca-certificates curl git nginx redis postgresql-server postgresql-contrib gcc gcc-c++ make tar gzip xz openssl
|
dnf install -y ca-certificates curl git nginx redis postgresql-server postgresql-contrib gcc gcc-c++ make tar gzip xz openssl
|
||||||
if [[ ! -d /var/lib/pgsql/data/base ]]; then
|
if [[ ! -d /var/lib/pgsql/data/base ]]; then
|
||||||
@@ -179,6 +186,9 @@ write_env() {
|
|||||||
cat >/etc/cmpp-platform/cmpp-platform.env <<EOF
|
cat >/etc/cmpp-platform/cmpp-platform.env <<EOF
|
||||||
NODE_ENV=production
|
NODE_ENV=production
|
||||||
API_PORT=${API_PORT}
|
API_PORT=${API_PORT}
|
||||||
|
API_HOST=${API_HOST}
|
||||||
|
API_METRICS_HOST=${API_METRICS_HOST}
|
||||||
|
API_METRICS_PORT=${API_METRICS_PORT}
|
||||||
API_ENABLE_SEND_WORKER=${API_ENABLE_SEND_WORKER}
|
API_ENABLE_SEND_WORKER=${API_ENABLE_SEND_WORKER}
|
||||||
API_SEND_WORKER_CONCURRENCY=${API_SEND_WORKER_CONCURRENCY}
|
API_SEND_WORKER_CONCURRENCY=${API_SEND_WORKER_CONCURRENCY}
|
||||||
DATABASE_URL=postgresql://${DB_USER}:${DB_PASSWORD}@127.0.0.1:5432/${DB_NAME}?schema=public
|
DATABASE_URL=postgresql://${DB_USER}:${DB_PASSWORD}@127.0.0.1:5432/${DB_NAME}?schema=public
|
||||||
@@ -192,6 +202,8 @@ OPERATION_LOG_ARCHIVE_INTERVAL_MS=${OPERATION_LOG_ARCHIVE_INTERVAL_MS}
|
|||||||
SMS_RECEIPT_TIMEOUT_SCAN_ENABLED=${SMS_RECEIPT_TIMEOUT_SCAN_ENABLED}
|
SMS_RECEIPT_TIMEOUT_SCAN_ENABLED=${SMS_RECEIPT_TIMEOUT_SCAN_ENABLED}
|
||||||
SMS_RECEIPT_TIMEOUT_HOURS=${SMS_RECEIPT_TIMEOUT_HOURS}
|
SMS_RECEIPT_TIMEOUT_HOURS=${SMS_RECEIPT_TIMEOUT_HOURS}
|
||||||
SMS_RECEIPT_TIMEOUT_SCAN_INTERVAL_MS=${SMS_RECEIPT_TIMEOUT_SCAN_INTERVAL_MS}
|
SMS_RECEIPT_TIMEOUT_SCAN_INTERVAL_MS=${SMS_RECEIPT_TIMEOUT_SCAN_INTERVAL_MS}
|
||||||
|
PROMETHEUS_URL=${PROMETHEUS_URL}
|
||||||
|
PROMETHEUS_QUERY_TIMEOUT_MS=${PROMETHEUS_QUERY_TIMEOUT_MS}
|
||||||
MINIO_ENDPOINT=127.0.0.1:9000
|
MINIO_ENDPOINT=127.0.0.1:9000
|
||||||
MINIO_ACCESS_KEY=${MINIO_ROOT_USER}
|
MINIO_ACCESS_KEY=${MINIO_ROOT_USER}
|
||||||
MINIO_SECRET_KEY=${MINIO_ROOT_PASSWORD}
|
MINIO_SECRET_KEY=${MINIO_ROOT_PASSWORD}
|
||||||
@@ -204,6 +216,10 @@ GATEWAY_CMPP_ADDR=${GATEWAY_CMPP_ADDR}
|
|||||||
CMPP_PUBLIC_HOST=${CMPP_PUBLIC_HOST}
|
CMPP_PUBLIC_HOST=${CMPP_PUBLIC_HOST}
|
||||||
CMPP_PUBLIC_PORT=${CMPP_PUBLIC_PORT}
|
CMPP_PUBLIC_PORT=${CMPP_PUBLIC_PORT}
|
||||||
API_BASE_URL=http://127.0.0.1:${API_PORT}/api
|
API_BASE_URL=http://127.0.0.1:${API_PORT}/api
|
||||||
|
SECURITY_EVENT_TOKEN=${SECURITY_EVENT_TOKEN}
|
||||||
|
SECURITY_AGENT_SOCKET=/run/cmpp-security-agent/agent.sock
|
||||||
|
TRUSTED_PROXY_IPS=127.0.0.1,::1
|
||||||
|
SECURITY_BUILTIN_PROTECTED_NETWORKS=${SECURITY_BUILTIN_PROTECTED_NETWORKS}
|
||||||
EOF
|
EOF
|
||||||
chmod 600 /etc/cmpp-platform/cmpp-platform.env
|
chmod 600 /etc/cmpp-platform/cmpp-platform.env
|
||||||
cat >/etc/cmpp-platform/minio.env <<EOF
|
cat >/etc/cmpp-platform/minio.env <<EOF
|
||||||
@@ -217,6 +233,11 @@ write_services() {
|
|||||||
log "Writing systemd and nginx configuration"
|
log "Writing systemd and nginx configuration"
|
||||||
local node_bin
|
local node_bin
|
||||||
node_bin="$(command -v node)"
|
node_bin="$(command -v node)"
|
||||||
|
getent group cmpp-security >/dev/null || groupadd --system cmpp-security
|
||||||
|
id cmpp-api >/dev/null 2>&1 || useradd --system --home-dir /nonexistent --shell /usr/sbin/nologin --gid cmpp-security cmpp-api
|
||||||
|
install -d -m 0750 /etc/nginx/snippets
|
||||||
|
touch /etc/nginx/snippets/cmpp-security-deny.conf
|
||||||
|
chmod 0640 /etc/nginx/snippets/cmpp-security-deny.conf
|
||||||
cat >/etc/systemd/system/cmpp-minio.service <<'EOF'
|
cat >/etc/systemd/system/cmpp-minio.service <<'EOF'
|
||||||
[Unit]
|
[Unit]
|
||||||
Description=CMPP MinIO object storage
|
Description=CMPP MinIO object storage
|
||||||
@@ -240,6 +261,8 @@ Description=CMPP Platform API
|
|||||||
After=network.target postgresql.service redis.service cmpp-minio.service
|
After=network.target postgresql.service redis.service cmpp-minio.service
|
||||||
|
|
||||||
[Service]
|
[Service]
|
||||||
|
User=cmpp-api
|
||||||
|
Group=cmpp-security
|
||||||
WorkingDirectory=${APP_DIR}/api
|
WorkingDirectory=${APP_DIR}/api
|
||||||
EnvironmentFile=/etc/cmpp-platform/cmpp-platform.env
|
EnvironmentFile=/etc/cmpp-platform/cmpp-platform.env
|
||||||
ExecStart=${node_bin} dist/main.js
|
ExecStart=${node_bin} dist/main.js
|
||||||
@@ -282,6 +305,7 @@ server {
|
|||||||
gzip_min_length 1024;
|
gzip_min_length 1024;
|
||||||
gzip_comp_level 5;
|
gzip_comp_level 5;
|
||||||
gzip_types application/json application/javascript text/javascript text/css text/plain text/csv image/svg+xml;
|
gzip_types application/json application/javascript text/javascript text/css text/plain text/csv image/svg+xml;
|
||||||
|
include /etc/nginx/snippets/cmpp-security-deny.conf;
|
||||||
|
|
||||||
location /api/ {
|
location /api/ {
|
||||||
proxy_pass http://127.0.0.1:${API_PORT}/api/;
|
proxy_pass http://127.0.0.1:${API_PORT}/api/;
|
||||||
|
|||||||
@@ -42,6 +42,7 @@ npm --prefix api ci --include=dev
|
|||||||
|
|
||||||
echo "[deploy] Verifying dependency security mitigations"
|
echo "[deploy] Verifying dependency security mitigations"
|
||||||
npm run security:verify
|
npm run security:verify
|
||||||
|
npm run deploy:verify
|
||||||
|
|
||||||
echo "[deploy] Generating Prisma client and applying migrations"
|
echo "[deploy] Generating Prisma client and applying migrations"
|
||||||
npm --prefix api run prisma:generate
|
npm --prefix api run prisma:generate
|
||||||
@@ -52,6 +53,7 @@ rm -rf api/dist api/tsconfig.build.tsbuildinfo "$APP_DIR/dist/cmpp-gateway"
|
|||||||
npm run build
|
npm run build
|
||||||
npm --prefix api run build
|
npm --prefix api run build
|
||||||
(cd gateway && GOPROXY="${GOPROXY:-https://goproxy.cn,direct}" /usr/local/bin/go build -o "$APP_DIR/dist/cmpp-gateway" ./cmd/gateway)
|
(cd gateway && GOPROXY="${GOPROXY:-https://goproxy.cn,direct}" /usr/local/bin/go build -o "$APP_DIR/dist/cmpp-gateway" ./cmd/gateway)
|
||||||
|
(cd gateway && GOPROXY="${GOPROXY:-https://goproxy.cn,direct}" /usr/local/bin/go build -o "$APP_DIR/dist/cmpp-security-agent" ./cmd/security-agent)
|
||||||
chmod 755 "$APP_DIR/dist" "$APP_DIR/dist/assets"
|
chmod 755 "$APP_DIR/dist" "$APP_DIR/dist/assets"
|
||||||
find "$APP_DIR/dist/assets" -type d -exec chmod 755 {} +
|
find "$APP_DIR/dist/assets" -type d -exec chmod 755 {} +
|
||||||
find "$APP_DIR/dist/assets" -type f -exec chmod 644 {} +
|
find "$APP_DIR/dist/assets" -type f -exec chmod 644 {} +
|
||||||
@@ -64,14 +66,24 @@ chmod 600 "$ADMIN_CREDENTIAL_FILE" || true
|
|||||||
echo "[deploy] Ensuring runtime log directories"
|
echo "[deploy] Ensuring runtime log directories"
|
||||||
install -d -m 0755 "$APP_DIR/logs/api" "$APP_DIR/logs/gateway"
|
install -d -m 0755 "$APP_DIR/logs/api" "$APP_DIR/logs/gateway"
|
||||||
|
|
||||||
|
echo "[deploy] Installing restricted security boundary"
|
||||||
|
bash "$APP_DIR/tools/security/install-security-agent.sh"
|
||||||
|
|
||||||
echo "[deploy] Ensuring HTTP response compression"
|
echo "[deploy] Ensuring HTTP response compression"
|
||||||
cat >/etc/nginx/conf.d/cmpp-compression.conf <<'EOF'
|
compression_config=/etc/nginx/conf.d/cmpp-compression.conf
|
||||||
|
: >"$compression_config"
|
||||||
|
if grep -RqsE --exclude='cmpp-compression.conf' '^[[:space:]]*gzip[[:space:]]+on;' \
|
||||||
|
/etc/nginx/nginx.conf /etc/nginx/conf.d /etc/nginx/sites-enabled 2>/dev/null; then
|
||||||
|
echo "[deploy] Reusing existing Nginx gzip configuration"
|
||||||
|
else
|
||||||
|
cat >"$compression_config" <<'EOF'
|
||||||
gzip on;
|
gzip on;
|
||||||
gzip_vary on;
|
gzip_vary on;
|
||||||
gzip_min_length 1024;
|
gzip_min_length 1024;
|
||||||
gzip_comp_level 5;
|
gzip_comp_level 5;
|
||||||
gzip_types application/json application/javascript text/javascript text/css text/plain text/csv image/svg+xml;
|
gzip_types application/json application/javascript text/javascript text/css text/plain text/csv image/svg+xml;
|
||||||
EOF
|
EOF
|
||||||
|
fi
|
||||||
nginx -t
|
nginx -t
|
||||||
|
|
||||||
echo "[deploy] Restarting services"
|
echo "[deploy] Restarting services"
|
||||||
@@ -85,6 +97,7 @@ else
|
|||||||
systemctl enable --now cmpp-api cmpp-gateway nginx
|
systemctl enable --now cmpp-api cmpp-gateway nginx
|
||||||
fi
|
fi
|
||||||
systemctl restart cmpp-gateway
|
systemctl restart cmpp-gateway
|
||||||
|
systemctl restart cmpp-security-agent
|
||||||
systemctl restart cmpp-api
|
systemctl restart cmpp-api
|
||||||
systemctl restart nginx
|
systemctl restart nginx
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,29 @@
|
|||||||
|
import { readFileSync } from 'node:fs';
|
||||||
|
import { resolve } from 'node:path';
|
||||||
|
|
||||||
|
const deploy = readFileSync(resolve(import.meta.dirname, 'production-deploy.sh'), 'utf8');
|
||||||
|
const bootstrap = readFileSync(resolve(import.meta.dirname, 'production-bootstrap.sh'), 'utf8');
|
||||||
|
const apiMain = readFileSync(resolve(import.meta.dirname, '../../api/src/main.ts'), 'utf8');
|
||||||
|
const required = [
|
||||||
|
'compression_config=/etc/nginx/conf.d/cmpp-compression.conf',
|
||||||
|
': >"$compression_config"',
|
||||||
|
"--exclude='cmpp-compression.conf'",
|
||||||
|
"'^[[:space:]]*gzip[[:space:]]+on;'",
|
||||||
|
'Reusing existing Nginx gzip configuration',
|
||||||
|
'cat >"$compression_config"',
|
||||||
|
];
|
||||||
|
for (const marker of required) {
|
||||||
|
if (!deploy.includes(marker)) throw new Error(`production deploy is missing the idempotent Nginx compression guard: ${marker}`);
|
||||||
|
}
|
||||||
|
if (deploy.includes("cat >/etc/nginx/conf.d/cmpp-compression.conf <<'EOF'")) {
|
||||||
|
throw new Error('production deploy still writes a duplicate global gzip directive unconditionally');
|
||||||
|
}
|
||||||
|
|
||||||
|
for (const marker of ['API_HOST="${API_HOST:-127.0.0.1}"', 'API_HOST=${API_HOST}']) {
|
||||||
|
if (!bootstrap.includes(marker)) throw new Error(`production bootstrap is missing the loopback API binding: ${marker}`);
|
||||||
|
}
|
||||||
|
if (!apiMain.includes("process.env.API_HOST?.trim() || '127.0.0.1'") || !apiMain.includes('app.listen(port, host)')) {
|
||||||
|
throw new Error('NestJS API must bind to API_HOST and default to loopback');
|
||||||
|
}
|
||||||
|
|
||||||
|
console.log('Production deployment verified: Nginx compression is idempotent and NestJS defaults to loopback.');
|
||||||
@@ -0,0 +1,34 @@
|
|||||||
|
# CMPP Prometheus 监控配置
|
||||||
|
|
||||||
|
本目录提供系统监控第一版所需的真实采集配置。运营端不嵌入Prometheus或Grafana页面;NestJS仅从本机Prometheus读取固定指标,再由平台原生UI展示。
|
||||||
|
|
||||||
|
在目标Debian/Ubuntu测试服务器上,以root执行:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd /opt/cmpp-platform
|
||||||
|
bash tools/monitoring/install-prometheus-monitoring.sh
|
||||||
|
bash tools/monitoring/install-service-exporters.sh
|
||||||
|
```
|
||||||
|
|
||||||
|
第一个脚本安装Prometheus与Node Exporter,备份已有配置并校验规则;第二个脚本安装PostgreSQL、Redis和Nginx Exporter,开启MinIO回环原生指标。API指标仅监听`127.0.0.1:9464`,Gateway指标复用回环控制端口`8090`。9090、9100、9187、9121、9113和9464均不得对公网开放。两个脚本均会打印恢复资产路径。
|
||||||
|
|
||||||
|
安装后将下列配置写入`/etc/cmpp-platform/cmpp-platform.env`,再按正常发布窗口重启API:
|
||||||
|
|
||||||
|
```dotenv
|
||||||
|
PROMETHEUS_URL=http://127.0.0.1:9090
|
||||||
|
PROMETHEUS_QUERY_TIMEOUT_MS=5000
|
||||||
|
```
|
||||||
|
|
||||||
|
校验命令:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
promtool check config /etc/prometheus/prometheus.yml
|
||||||
|
promtool check rules /etc/prometheus/cmpp-alerts.yml
|
||||||
|
curl -fsS http://127.0.0.1:9090/-/ready
|
||||||
|
curl -fsS 'http://127.0.0.1:9090/api/v1/query?query=up'
|
||||||
|
ss -lnt | grep -E ':(9090|9100)'
|
||||||
|
curl -fsS http://127.0.0.1:9464/metrics
|
||||||
|
curl -fsS http://127.0.0.1:8090/metrics
|
||||||
|
```
|
||||||
|
|
||||||
|
完整架构、PromQL口径、故障语义和验收标准见`docs/prometheus-system-monitoring-design-20260814.md`。
|
||||||
@@ -0,0 +1,349 @@
|
|||||||
|
groups:
|
||||||
|
- name: cmpp-service-recording
|
||||||
|
interval: 15s
|
||||||
|
rules:
|
||||||
|
- record: cmpp:service_api:requests_per_second
|
||||||
|
expr: sum(rate(cmpp_api_http_requests_total[5m]))
|
||||||
|
- record: cmpp:service_api:error_percent
|
||||||
|
expr: 100 * sum(rate(cmpp_api_http_requests_total{status=~"5.."}[5m])) / clamp_min(sum(rate(cmpp_api_http_requests_total[5m])), 0.001)
|
||||||
|
- record: cmpp:service_api:latency_p95_seconds
|
||||||
|
expr: histogram_quantile(0.95, sum by (le) (rate(cmpp_api_http_request_duration_seconds_bucket[5m])))
|
||||||
|
- record: cmpp:service_api:event_loop_p99_seconds
|
||||||
|
expr: cmpp_api_nodejs_event_loop_lag_p99_seconds
|
||||||
|
- record: cmpp:service_gateway:submits_per_second
|
||||||
|
expr: sum(rate(cmpp_gateway_submit_total[5m]))
|
||||||
|
- record: cmpp:service_gateway:failure_percent
|
||||||
|
expr: 100 * sum(rate(cmpp_gateway_submit_total{result="failed"}[5m])) / clamp_min(sum(rate(cmpp_gateway_submit_total[5m])), 0.001)
|
||||||
|
- record: cmpp:service_gateway:queue_pending
|
||||||
|
expr: cmpp_gateway_submit_queue_pending
|
||||||
|
- record: cmpp:service_gateway:queue_lag
|
||||||
|
expr: cmpp_gateway_submit_queue_lag
|
||||||
|
- record: cmpp:service_gateway:queue_oldest_seconds
|
||||||
|
expr: cmpp_gateway_submit_queue_oldest_pending_age_seconds
|
||||||
|
- record: cmpp:service_postgresql:connection_percent
|
||||||
|
expr: 100 * sum(pg_stat_activity_count) / clamp_min(max(pg_settings_max_connections), 1)
|
||||||
|
- record: cmpp:service_postgresql:deadlocks_15m
|
||||||
|
expr: sum(increase(pg_stat_database_deadlocks[15m]))
|
||||||
|
- record: cmpp:service_redis:memory_percent
|
||||||
|
expr: (100 * redis_memory_used_bytes / redis_memory_max_bytes) and on(instance) (redis_memory_max_bytes > 0)
|
||||||
|
- record: cmpp:service_redis:memory_used_bytes
|
||||||
|
expr: redis_memory_used_bytes
|
||||||
|
- record: cmpp:service_redis:evictions_5m
|
||||||
|
expr: increase(redis_evicted_keys_total[5m])
|
||||||
|
- record: cmpp:service_redis:connected_clients
|
||||||
|
expr: redis_connected_clients
|
||||||
|
- record: cmpp:service_nginx:connections_active
|
||||||
|
expr: nginx_connections_active
|
||||||
|
- record: cmpp:service_nginx:requests_per_second
|
||||||
|
expr: rate(nginx_http_requests_total[5m])
|
||||||
|
- record: cmpp:service_minio:capacity_percent
|
||||||
|
expr: 100 * (1 - minio_cluster_capacity_usable_free_bytes / minio_cluster_capacity_usable_total_bytes)
|
||||||
|
- record: cmpp:service_minio:usage_bytes
|
||||||
|
expr: minio_cluster_usage_total_bytes
|
||||||
|
- record: cmpp:service_minio:objects
|
||||||
|
expr: minio_cluster_usage_object_total
|
||||||
|
- record: cmpp:service_minio:drives_offline
|
||||||
|
expr: minio_cluster_drive_offline_total
|
||||||
|
|
||||||
|
- name: cmpp-host-resources
|
||||||
|
rules:
|
||||||
|
- alert: NodeExporterDown
|
||||||
|
expr: up{job="node"} == 0
|
||||||
|
for: 2m
|
||||||
|
labels:
|
||||||
|
severity: critical
|
||||||
|
service: node-exporter
|
||||||
|
annotations:
|
||||||
|
summary: 主机指标采集不可用
|
||||||
|
description: Prometheus连续2分钟无法采集Node Exporter。
|
||||||
|
currentValue: "{{ $value }}"
|
||||||
|
threshold: "up = 1"
|
||||||
|
|
||||||
|
- alert: HostCpuUsageWarning
|
||||||
|
expr: (100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 80) and (100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) <= 90)
|
||||||
|
for: 10m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
service: host
|
||||||
|
annotations:
|
||||||
|
summary: CPU使用率持续偏高
|
||||||
|
description: 主机CPU使用率连续10分钟高于80%。
|
||||||
|
currentValue: "{{ printf \"%.1f\" $value }}%"
|
||||||
|
threshold: "80%"
|
||||||
|
|
||||||
|
- alert: HostCpuUsageCritical
|
||||||
|
expr: 100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 90
|
||||||
|
for: 5m
|
||||||
|
labels:
|
||||||
|
severity: critical
|
||||||
|
service: host
|
||||||
|
annotations:
|
||||||
|
summary: CPU使用率严重超限
|
||||||
|
description: 主机CPU使用率连续5分钟高于90%。
|
||||||
|
currentValue: "{{ printf \"%.1f\" $value }}%"
|
||||||
|
threshold: "90%"
|
||||||
|
|
||||||
|
- alert: HostMemoryUsageWarning
|
||||||
|
expr: ((1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100 > 85) and ((1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100 <= 95)
|
||||||
|
for: 10m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
service: host
|
||||||
|
annotations:
|
||||||
|
summary: 内存使用率持续偏高
|
||||||
|
description: 主机可用内存连续10分钟低于15%。
|
||||||
|
currentValue: "{{ printf \"%.1f\" $value }}%"
|
||||||
|
threshold: "85%"
|
||||||
|
|
||||||
|
- alert: HostMemoryUsageCritical
|
||||||
|
expr: (1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100 > 95
|
||||||
|
for: 5m
|
||||||
|
labels:
|
||||||
|
severity: critical
|
||||||
|
service: host
|
||||||
|
annotations:
|
||||||
|
summary: 内存使用率严重超限
|
||||||
|
description: 主机可用内存连续5分钟低于5%。
|
||||||
|
currentValue: "{{ printf \"%.1f\" $value }}%"
|
||||||
|
threshold: "95%"
|
||||||
|
|
||||||
|
- alert: HostRootDiskUsageWarning
|
||||||
|
expr: ((1 - node_filesystem_avail_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"} / node_filesystem_size_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"}) * 100 > 80) and ((1 - node_filesystem_avail_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"} / node_filesystem_size_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"}) * 100 <= 90)
|
||||||
|
for: 15m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
service: host
|
||||||
|
annotations:
|
||||||
|
summary: 根文件系统空间不足
|
||||||
|
description: 根文件系统使用率连续15分钟高于80%。
|
||||||
|
currentValue: "{{ printf \"%.1f\" $value }}%"
|
||||||
|
threshold: "80%"
|
||||||
|
|
||||||
|
- alert: HostRootDiskUsageCritical
|
||||||
|
expr: (1 - node_filesystem_avail_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"} / node_filesystem_size_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"}) * 100 > 90
|
||||||
|
for: 5m
|
||||||
|
labels:
|
||||||
|
severity: critical
|
||||||
|
service: host
|
||||||
|
annotations:
|
||||||
|
summary: 根文件系统空间严重不足
|
||||||
|
description: 根文件系统使用率连续5分钟高于90%。
|
||||||
|
currentValue: "{{ printf \"%.1f\" $value }}%"
|
||||||
|
threshold: "90%"
|
||||||
|
|
||||||
|
- alert: HostRootInodeUsageWarning
|
||||||
|
expr: ((1 - node_filesystem_files_free{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"} / node_filesystem_files{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"}) * 100 > 80) and ((1 - node_filesystem_files_free{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"} / node_filesystem_files{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"}) * 100 <= 90)
|
||||||
|
for: 15m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
service: host
|
||||||
|
annotations:
|
||||||
|
summary: 根文件系统inode余量偏低
|
||||||
|
description: 根文件系统inode使用率连续15分钟高于80%。
|
||||||
|
currentValue: "{{ printf \"%.1f\" $value }}%"
|
||||||
|
threshold: "80%"
|
||||||
|
|
||||||
|
- alert: HostRootInodeUsageCritical
|
||||||
|
expr: (1 - node_filesystem_files_free{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"} / node_filesystem_files{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"}) * 100 > 90
|
||||||
|
for: 5m
|
||||||
|
labels:
|
||||||
|
severity: critical
|
||||||
|
service: host
|
||||||
|
annotations:
|
||||||
|
summary: 根文件系统inode严重不足
|
||||||
|
description: 根文件系统inode使用率连续5分钟高于90%。
|
||||||
|
currentValue: "{{ printf \"%.1f\" $value }}%"
|
||||||
|
threshold: "90%"
|
||||||
|
|
||||||
|
- alert: HostCpuIowaitWarning
|
||||||
|
expr: (avg by (instance) (rate(node_cpu_seconds_total{mode="iowait"}[5m])) * 100 > 20) and (avg by (instance) (rate(node_cpu_seconds_total{mode="iowait"}[5m])) * 100 <= 35)
|
||||||
|
for: 10m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
service: host
|
||||||
|
annotations:
|
||||||
|
summary: CPU iowait持续偏高
|
||||||
|
description: 主机CPU iowait连续10分钟高于20%,请检查磁盘I/O。
|
||||||
|
currentValue: "{{ printf \"%.1f\" $value }}%"
|
||||||
|
threshold: "20%"
|
||||||
|
|
||||||
|
- alert: HostCpuIowaitCritical
|
||||||
|
expr: avg by (instance) (rate(node_cpu_seconds_total{mode="iowait"}[5m])) * 100 > 35
|
||||||
|
for: 10m
|
||||||
|
labels:
|
||||||
|
severity: critical
|
||||||
|
service: host
|
||||||
|
annotations:
|
||||||
|
summary: CPU iowait严重超限
|
||||||
|
description: 主机CPU iowait连续10分钟高于35%,磁盘I/O可能已成为瓶颈。
|
||||||
|
currentValue: "{{ printf \"%.1f\" $value }}%"
|
||||||
|
threshold: "35%"
|
||||||
|
|
||||||
|
- name: cmpp-core-services
|
||||||
|
rules:
|
||||||
|
- alert: CmppCoreServiceInactive
|
||||||
|
expr: node_systemd_unit_state{name=~"cmpp-api\\.service|cmpp-gateway\\.service|postgresql\\.service|redis(-server)?\\.service|cmpp-minio\\.service|nginx\\.service",state="active"} == 0
|
||||||
|
for: 2m
|
||||||
|
labels:
|
||||||
|
severity: critical
|
||||||
|
service: "{{ $labels.name }}"
|
||||||
|
annotations:
|
||||||
|
summary: CMPP核心服务未处于active状态
|
||||||
|
description: "systemd服务 {{ $labels.name }} 连续2分钟未处于active状态。"
|
||||||
|
currentValue: "{{ $value }}"
|
||||||
|
threshold: "active = 1"
|
||||||
|
|
||||||
|
- name: cmpp-api-runtime
|
||||||
|
rules:
|
||||||
|
- alert: CmppApiMetricsDown
|
||||||
|
expr: up{job="cmpp-api"} == 0
|
||||||
|
for: 2m
|
||||||
|
labels: { severity: critical, service: api }
|
||||||
|
annotations: { summary: "API指标采集不可用", description: "Prometheus连续2分钟无法读取API内部指标端点。", currentValue: "{{ $value }}", threshold: "up = 1" }
|
||||||
|
- alert: CmppApiHttpErrorRateWarning
|
||||||
|
expr: (sum(rate(cmpp_api_http_requests_total{status=~"5.."}[5m])) / clamp_min(sum(rate(cmpp_api_http_requests_total[5m])), 0.001) > 0.01) and (sum(rate(cmpp_api_http_requests_total{status=~"5.."}[5m])) / clamp_min(sum(rate(cmpp_api_http_requests_total[5m])), 0.001) <= 0.05) and sum(increase(cmpp_api_http_requests_total{status=~"5.."}[5m])) >= 5
|
||||||
|
for: 5m
|
||||||
|
labels: { severity: warning, service: api }
|
||||||
|
annotations: { summary: "API 5xx错误率偏高", description: "API 5xx错误率连续5分钟高于1%,且窗口内至少5次错误。", currentValue: "{{ printf \"%.2f\" $value }}", threshold: "1%" }
|
||||||
|
- alert: CmppApiHttpErrorRateCritical
|
||||||
|
expr: (sum(rate(cmpp_api_http_requests_total{status=~"5.."}[5m])) / clamp_min(sum(rate(cmpp_api_http_requests_total[5m])), 0.001) > 0.05) and sum(increase(cmpp_api_http_requests_total{status=~"5.."}[5m])) >= 5
|
||||||
|
for: 5m
|
||||||
|
labels: { severity: critical, service: api }
|
||||||
|
annotations: { summary: "API 5xx错误率严重超限", description: "API 5xx错误率连续5分钟高于5%。", currentValue: "{{ printf \"%.2f\" $value }}", threshold: "5%" }
|
||||||
|
- alert: CmppApiLatencyWarning
|
||||||
|
expr: (histogram_quantile(0.95, sum by (le) (rate(cmpp_api_http_request_duration_seconds_bucket[10m]))) > 1) and (histogram_quantile(0.95, sum by (le) (rate(cmpp_api_http_request_duration_seconds_bucket[10m]))) <= 3)
|
||||||
|
for: 10m
|
||||||
|
labels: { severity: warning, service: api }
|
||||||
|
annotations: { summary: "API P95响应偏慢", description: "API P95响应时间连续10分钟超过1秒。", currentValue: "{{ printf \"%.3f\" $value }}s", threshold: "1s" }
|
||||||
|
- alert: CmppApiLatencyCritical
|
||||||
|
expr: histogram_quantile(0.95, sum by (le) (rate(cmpp_api_http_request_duration_seconds_bucket[5m]))) > 3
|
||||||
|
for: 5m
|
||||||
|
labels: { severity: critical, service: api }
|
||||||
|
annotations: { summary: "API P95响应严重超时", description: "API P95响应时间连续5分钟超过3秒。", currentValue: "{{ printf \"%.3f\" $value }}s", threshold: "3s" }
|
||||||
|
- alert: CmppApiEventLoopLagWarning
|
||||||
|
expr: (cmpp_api_nodejs_event_loop_lag_p99_seconds > 0.2) and (cmpp_api_nodejs_event_loop_lag_p99_seconds <= 1)
|
||||||
|
for: 10m
|
||||||
|
labels: { severity: warning, service: api }
|
||||||
|
annotations: { summary: "API事件循环延迟偏高", description: "Node.js事件循环P99延迟连续10分钟超过200ms。", currentValue: "{{ printf \"%.3f\" $value }}s", threshold: "0.2s" }
|
||||||
|
- alert: CmppApiEventLoopLagCritical
|
||||||
|
expr: cmpp_api_nodejs_event_loop_lag_p99_seconds > 1
|
||||||
|
for: 5m
|
||||||
|
labels: { severity: critical, service: api }
|
||||||
|
annotations: { summary: "API事件循环严重阻塞", description: "Node.js事件循环P99延迟连续5分钟超过1秒。", currentValue: "{{ printf \"%.3f\" $value }}s", threshold: "1s" }
|
||||||
|
|
||||||
|
- name: cmpp-gateway-runtime
|
||||||
|
rules:
|
||||||
|
- alert: CmppGatewayMetricsDown
|
||||||
|
expr: up{job="cmpp-gateway"} == 0
|
||||||
|
for: 2m
|
||||||
|
labels: { severity: critical, service: gateway }
|
||||||
|
annotations: { summary: "Gateway指标采集不可用", description: "Prometheus连续2分钟无法读取Gateway指标。", currentValue: "{{ $value }}", threshold: "up = 1" }
|
||||||
|
- alert: CmppGatewaySubmitWorkerDown
|
||||||
|
expr: cmpp_gateway_submit_worker_up == 0
|
||||||
|
for: 1m
|
||||||
|
labels: { severity: critical, service: gateway }
|
||||||
|
annotations: { summary: "Gateway提交消费者未运行", description: "Gateway进程存活,但提交消费者未成功初始化。", currentValue: "{{ $value }}", threshold: "1" }
|
||||||
|
- alert: CmppGatewayUpstreamConnectionShortage
|
||||||
|
expr: cmpp_gateway_upstream_connections{state="connected"} < cmpp_gateway_upstream_connections{state="desired"}
|
||||||
|
for: 2m
|
||||||
|
labels: { severity: critical, service: gateway }
|
||||||
|
annotations: { summary: "Gateway上游连接不足", description: "实际上游CMPP连接数连续2分钟低于期望数。", currentValue: "{{ $value }}", threshold: "connected = desired" }
|
||||||
|
- alert: CmppGatewayQueueDelayedWarning
|
||||||
|
expr: (cmpp_gateway_submit_queue_oldest_pending_age_seconds > 30) and (cmpp_gateway_submit_queue_oldest_pending_age_seconds <= 120)
|
||||||
|
for: 2m
|
||||||
|
labels: { severity: warning, service: gateway }
|
||||||
|
annotations: { summary: "Gateway提交队列开始延迟", description: "Redis Stream最旧pending消息等待超过30秒。", currentValue: "{{ printf \"%.0f\" $value }}s", threshold: "30s" }
|
||||||
|
- alert: CmppGatewayQueueDelayedCritical
|
||||||
|
expr: cmpp_gateway_submit_queue_oldest_pending_age_seconds > 120
|
||||||
|
for: 2m
|
||||||
|
labels: { severity: critical, service: gateway }
|
||||||
|
annotations: { summary: "Gateway提交队列严重延迟", description: "Redis Stream最旧pending消息等待超过120秒。", currentValue: "{{ printf \"%.0f\" $value }}s", threshold: "120s" }
|
||||||
|
|
||||||
|
- name: cmpp-data-services
|
||||||
|
rules:
|
||||||
|
- alert: PostgresExporterDown
|
||||||
|
expr: up{job="postgresql"} == 0 or pg_up == 0
|
||||||
|
for: 2m
|
||||||
|
labels: { severity: critical, service: postgresql }
|
||||||
|
annotations: { summary: "PostgreSQL指标或数据库不可用", description: "PostgreSQL Exporter或其数据库连接连续2分钟不可用。", currentValue: "{{ $value }}", threshold: "up = 1" }
|
||||||
|
- alert: PostgresConnectionsWarning
|
||||||
|
expr: (sum(pg_stat_activity_count) / clamp_min(max(pg_settings_max_connections), 1) > 0.70) and (sum(pg_stat_activity_count) / clamp_min(max(pg_settings_max_connections), 1) <= 0.85)
|
||||||
|
for: 10m
|
||||||
|
labels: { severity: warning, service: postgresql }
|
||||||
|
annotations: { summary: "PostgreSQL连接使用率偏高", description: "数据库连接数连续10分钟超过上限的70%。", currentValue: "{{ printf \"%.2f\" $value }}", threshold: "70%" }
|
||||||
|
- alert: PostgresConnectionsCritical
|
||||||
|
expr: sum(pg_stat_activity_count) / clamp_min(max(pg_settings_max_connections), 1) > 0.85
|
||||||
|
for: 5m
|
||||||
|
labels: { severity: critical, service: postgresql }
|
||||||
|
annotations: { summary: "PostgreSQL连接即将耗尽", description: "数据库连接数连续5分钟超过上限的85%。", currentValue: "{{ printf \"%.2f\" $value }}", threshold: "85%" }
|
||||||
|
- alert: PostgresDeadlocksDetected
|
||||||
|
expr: sum(increase(pg_stat_database_deadlocks[15m])) > 0
|
||||||
|
for: 1m
|
||||||
|
labels: { severity: warning, service: postgresql }
|
||||||
|
annotations: { summary: "PostgreSQL发生死锁", description: "15分钟窗口内检测到数据库死锁。", currentValue: "{{ $value }}", threshold: "0" }
|
||||||
|
- alert: RedisExporterDown
|
||||||
|
expr: up{job="redis"} == 0 or redis_up == 0
|
||||||
|
for: 2m
|
||||||
|
labels: { severity: critical, service: redis }
|
||||||
|
annotations: { summary: "Redis指标或服务不可用", description: "Redis Exporter或Redis连接连续2分钟不可用。", currentValue: "{{ $value }}", threshold: "up = 1" }
|
||||||
|
- alert: RedisMemoryWarning
|
||||||
|
expr: (redis_memory_max_bytes > 0) and (redis_memory_used_bytes / redis_memory_max_bytes > 0.70) and (redis_memory_used_bytes / redis_memory_max_bytes <= 0.85)
|
||||||
|
for: 10m
|
||||||
|
labels: { severity: warning, service: redis }
|
||||||
|
annotations: { summary: "Redis内存使用率偏高", description: "Redis内存连续10分钟超过maxmemory的70%。", currentValue: "{{ printf \"%.2f\" $value }}", threshold: "70%" }
|
||||||
|
- alert: RedisMemoryCritical
|
||||||
|
expr: (redis_memory_max_bytes > 0) and (redis_memory_used_bytes / redis_memory_max_bytes > 0.85)
|
||||||
|
for: 5m
|
||||||
|
labels: { severity: critical, service: redis }
|
||||||
|
annotations: { summary: "Redis内存即将耗尽", description: "Redis内存连续5分钟超过maxmemory的85%。", currentValue: "{{ printf \"%.2f\" $value }}", threshold: "85%" }
|
||||||
|
- alert: RedisUnexpectedEvictions
|
||||||
|
expr: increase(redis_evicted_keys_total[5m]) > 0
|
||||||
|
for: 1m
|
||||||
|
labels: { severity: critical, service: redis }
|
||||||
|
annotations: { summary: "Redis发生Key淘汰", description: "Redis承载队列和运行状态,5分钟内不应出现淘汰。", currentValue: "{{ $value }}", threshold: "0" }
|
||||||
|
- alert: RedisRejectedConnections
|
||||||
|
expr: increase(redis_rejected_connections_total[5m]) > 0
|
||||||
|
for: 1m
|
||||||
|
labels: { severity: critical, service: redis }
|
||||||
|
annotations: { summary: "Redis拒绝连接", description: "5分钟内Redis出现被拒绝连接。", currentValue: "{{ $value }}", threshold: "0" }
|
||||||
|
|
||||||
|
- name: cmpp-storage-and-edge
|
||||||
|
rules:
|
||||||
|
- alert: MinioMetricsDown
|
||||||
|
expr: up{job="minio"} == 0
|
||||||
|
for: 2m
|
||||||
|
labels: { severity: critical, service: minio }
|
||||||
|
annotations: { summary: "MinIO指标采集不可用", description: "Prometheus连续2分钟无法读取MinIO原生指标。", currentValue: "{{ $value }}", threshold: "up = 1" }
|
||||||
|
- alert: MinioCapacityWarning
|
||||||
|
expr: (100 * (1 - minio_cluster_capacity_usable_free_bytes / minio_cluster_capacity_usable_total_bytes) > 80) and (100 * (1 - minio_cluster_capacity_usable_free_bytes / minio_cluster_capacity_usable_total_bytes) <= 90)
|
||||||
|
for: 15m
|
||||||
|
labels: { severity: warning, service: minio }
|
||||||
|
annotations: { summary: "MinIO存储容量偏高", description: "MinIO可用容量使用率连续15分钟超过80%。", currentValue: "{{ printf \"%.1f\" $value }}%", threshold: "80%" }
|
||||||
|
- alert: MinioCapacityCritical
|
||||||
|
expr: 100 * (1 - minio_cluster_capacity_usable_free_bytes / minio_cluster_capacity_usable_total_bytes) > 90
|
||||||
|
for: 5m
|
||||||
|
labels: { severity: critical, service: minio }
|
||||||
|
annotations: { summary: "MinIO存储容量即将耗尽", description: "MinIO可用容量使用率连续5分钟超过90%。", currentValue: "{{ printf \"%.1f\" $value }}%", threshold: "90%" }
|
||||||
|
- alert: MinioDriveOffline
|
||||||
|
expr: minio_cluster_drive_offline_total > 0
|
||||||
|
for: 1m
|
||||||
|
labels: { severity: critical, service: minio }
|
||||||
|
annotations: { summary: "MinIO存储盘离线", description: "MinIO检测到离线存储盘。", currentValue: "{{ $value }}", threshold: "0" }
|
||||||
|
- alert: NginxExporterDown
|
||||||
|
expr: up{job="nginx"} == 0 or nginx_up == 0
|
||||||
|
for: 2m
|
||||||
|
labels: { severity: critical, service: nginx }
|
||||||
|
annotations: { summary: "Nginx指标或状态页不可用", description: "Nginx Exporter或回环stub_status连续2分钟不可用。", currentValue: "{{ $value }}", threshold: "up = 1" }
|
||||||
|
|
||||||
|
- name: cmpp-monitoring-self
|
||||||
|
rules:
|
||||||
|
- alert: PrometheusScrapeSlow
|
||||||
|
expr: scrape_duration_seconds / scrape_interval_seconds > 0.8
|
||||||
|
for: 5m
|
||||||
|
labels: { severity: warning, service: prometheus }
|
||||||
|
annotations: { summary: "Prometheus采集接近超时", description: "采集耗时连续5分钟超过采集周期的80%。", currentValue: "{{ printf \"%.2f\" $value }}", threshold: "80%" }
|
||||||
|
- alert: PrometheusRuleEvaluationFailures
|
||||||
|
expr: increase(prometheus_rule_evaluation_failures_total[5m]) > 0
|
||||||
|
for: 1m
|
||||||
|
labels: { severity: critical, service: prometheus }
|
||||||
|
annotations: { summary: "Prometheus告警规则计算失败", description: "5分钟内出现告警规则计算失败。", currentValue: "{{ $value }}", threshold: "0" }
|
||||||
@@ -0,0 +1,103 @@
|
|||||||
|
groups:
|
||||||
|
- name: cmpp-managed-thresholds
|
||||||
|
rules:
|
||||||
|
- alert: HostCpuUsageWarning
|
||||||
|
expr: (100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 80) and (100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) <= 90)
|
||||||
|
for: 10m
|
||||||
|
labels: { severity: warning, service: host }
|
||||||
|
annotations: { summary: "主机 CPU 使用率达到警告阈值", description: "主机 CPU 使用率持续超过80%。", currentValue: "{{ $value }}", threshold: "80%" }
|
||||||
|
- alert: HostCpuUsageCritical
|
||||||
|
expr: 100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 90
|
||||||
|
for: 5m
|
||||||
|
labels: { severity: critical, service: host }
|
||||||
|
annotations: { summary: "主机 CPU 使用率达到严重阈值", description: "主机 CPU 使用率持续超过90%。", currentValue: "{{ $value }}", threshold: "90%" }
|
||||||
|
- alert: HostMemoryUsageWarning
|
||||||
|
expr: ((1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100 > 85) and ((1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100 <= 95)
|
||||||
|
for: 10m
|
||||||
|
labels: { severity: warning, service: host }
|
||||||
|
annotations: { summary: "主机内存使用率达到警告阈值", description: "主机内存使用率持续超过85%。", currentValue: "{{ $value }}", threshold: "85%" }
|
||||||
|
- alert: HostMemoryUsageCritical
|
||||||
|
expr: (1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100 > 95
|
||||||
|
for: 5m
|
||||||
|
labels: { severity: critical, service: host }
|
||||||
|
annotations: { summary: "主机内存使用率达到严重阈值", description: "主机内存使用率持续超过95%。", currentValue: "{{ $value }}", threshold: "95%" }
|
||||||
|
- alert: HostRootDiskUsageWarning
|
||||||
|
expr: ((1 - node_filesystem_avail_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"} / node_filesystem_size_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"}) * 100 > 80) and ((1 - node_filesystem_avail_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"} / node_filesystem_size_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"}) * 100 <= 90)
|
||||||
|
for: 15m
|
||||||
|
labels: { severity: warning, service: host }
|
||||||
|
annotations: { summary: "根磁盘使用率达到警告阈值", description: "根磁盘使用率持续超过80%。", currentValue: "{{ $value }}", threshold: "80%" }
|
||||||
|
- alert: HostRootDiskUsageCritical
|
||||||
|
expr: (1 - node_filesystem_avail_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"} / node_filesystem_size_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"}) * 100 > 90
|
||||||
|
for: 5m
|
||||||
|
labels: { severity: critical, service: host }
|
||||||
|
annotations: { summary: "根磁盘使用率达到严重阈值", description: "根磁盘使用率持续超过90%。", currentValue: "{{ $value }}", threshold: "90%" }
|
||||||
|
- alert: CmppApiHttpErrorRateWarning
|
||||||
|
expr: (100 * sum(rate(cmpp_api_http_requests_total{status=~"5.."}[5m])) / clamp_min(sum(rate(cmpp_api_http_requests_total[5m])), 0.001) > 1) and (100 * sum(rate(cmpp_api_http_requests_total{status=~"5.."}[5m])) / clamp_min(sum(rate(cmpp_api_http_requests_total[5m])), 0.001) <= 5) and (sum(increase(cmpp_api_http_requests_total{status=~"5.."}[5m])) >= 5)
|
||||||
|
for: 5m
|
||||||
|
labels: { severity: warning, service: api }
|
||||||
|
annotations: { summary: "API 5xx 错误率达到警告阈值", description: "API 5xx 错误率持续超过1%。", currentValue: "{{ $value }}", threshold: "1%" }
|
||||||
|
- alert: CmppApiHttpErrorRateCritical
|
||||||
|
expr: (100 * sum(rate(cmpp_api_http_requests_total{status=~"5.."}[5m])) / clamp_min(sum(rate(cmpp_api_http_requests_total[5m])), 0.001) > 5) and (sum(increase(cmpp_api_http_requests_total{status=~"5.."}[5m])) >= 5)
|
||||||
|
for: 5m
|
||||||
|
labels: { severity: critical, service: api }
|
||||||
|
annotations: { summary: "API 5xx 错误率达到严重阈值", description: "API 5xx 错误率持续超过5%。", currentValue: "{{ $value }}", threshold: "5%" }
|
||||||
|
- alert: CmppApiLatencyWarning
|
||||||
|
expr: (histogram_quantile(0.95, sum by (le) (rate(cmpp_api_http_request_duration_seconds_bucket[10m]))) > 1) and (histogram_quantile(0.95, sum by (le) (rate(cmpp_api_http_request_duration_seconds_bucket[10m]))) <= 3)
|
||||||
|
for: 10m
|
||||||
|
labels: { severity: warning, service: api }
|
||||||
|
annotations: { summary: "API P95 响应时间达到警告阈值", description: "API P95 响应时间持续超过1秒。", currentValue: "{{ $value }}", threshold: "1秒" }
|
||||||
|
- alert: CmppApiLatencyCritical
|
||||||
|
expr: histogram_quantile(0.95, sum by (le) (rate(cmpp_api_http_request_duration_seconds_bucket[10m]))) > 3
|
||||||
|
for: 5m
|
||||||
|
labels: { severity: critical, service: api }
|
||||||
|
annotations: { summary: "API P95 响应时间达到严重阈值", description: "API P95 响应时间持续超过3秒。", currentValue: "{{ $value }}", threshold: "3秒" }
|
||||||
|
- alert: CmppApiEventLoopLagWarning
|
||||||
|
expr: (cmpp_api_nodejs_event_loop_lag_p99_seconds > 0.2) and (cmpp_api_nodejs_event_loop_lag_p99_seconds <= 1)
|
||||||
|
for: 10m
|
||||||
|
labels: { severity: warning, service: api }
|
||||||
|
annotations: { summary: "API 事件循环 P99 达到警告阈值", description: "API 事件循环 P99 持续超过0.2秒。", currentValue: "{{ $value }}", threshold: "0.2秒" }
|
||||||
|
- alert: CmppApiEventLoopLagCritical
|
||||||
|
expr: cmpp_api_nodejs_event_loop_lag_p99_seconds > 1
|
||||||
|
for: 5m
|
||||||
|
labels: { severity: critical, service: api }
|
||||||
|
annotations: { summary: "API 事件循环 P99 达到严重阈值", description: "API 事件循环 P99 持续超过1秒。", currentValue: "{{ $value }}", threshold: "1秒" }
|
||||||
|
- alert: CmppGatewayQueueDelayedWarning
|
||||||
|
expr: (cmpp_gateway_submit_queue_oldest_pending_age_seconds > 30) and (cmpp_gateway_submit_queue_oldest_pending_age_seconds <= 120)
|
||||||
|
for: 2m
|
||||||
|
labels: { severity: warning, service: gateway }
|
||||||
|
annotations: { summary: "Gateway 最旧 pending 达到警告阈值", description: "Gateway 最旧 pending 持续超过30秒。", currentValue: "{{ $value }}", threshold: "30秒" }
|
||||||
|
- alert: CmppGatewayQueueDelayedCritical
|
||||||
|
expr: cmpp_gateway_submit_queue_oldest_pending_age_seconds > 120
|
||||||
|
for: 2m
|
||||||
|
labels: { severity: critical, service: gateway }
|
||||||
|
annotations: { summary: "Gateway 最旧 pending 达到严重阈值", description: "Gateway 最旧 pending 持续超过120秒。", currentValue: "{{ $value }}", threshold: "120秒" }
|
||||||
|
- alert: PostgresConnectionsWarning
|
||||||
|
expr: (100 * sum(pg_stat_activity_count) / clamp_min(max(pg_settings_max_connections), 1) > 70) and (100 * sum(pg_stat_activity_count) / clamp_min(max(pg_settings_max_connections), 1) <= 85)
|
||||||
|
for: 10m
|
||||||
|
labels: { severity: warning, service: postgresql }
|
||||||
|
annotations: { summary: "PostgreSQL 连接使用率达到警告阈值", description: "PostgreSQL 连接使用率持续超过70%。", currentValue: "{{ $value }}", threshold: "70%" }
|
||||||
|
- alert: PostgresConnectionsCritical
|
||||||
|
expr: 100 * sum(pg_stat_activity_count) / clamp_min(max(pg_settings_max_connections), 1) > 85
|
||||||
|
for: 5m
|
||||||
|
labels: { severity: critical, service: postgresql }
|
||||||
|
annotations: { summary: "PostgreSQL 连接使用率达到严重阈值", description: "PostgreSQL 连接使用率持续超过85%。", currentValue: "{{ $value }}", threshold: "85%" }
|
||||||
|
- alert: RedisMemoryWarning
|
||||||
|
expr: (100 * redis_memory_used_bytes / redis_memory_max_bytes > 70) and (100 * redis_memory_used_bytes / redis_memory_max_bytes <= 85) and (redis_memory_max_bytes > 0)
|
||||||
|
for: 10m
|
||||||
|
labels: { severity: warning, service: redis }
|
||||||
|
annotations: { summary: "Redis 内存使用率达到警告阈值", description: "Redis 内存使用率持续超过70%。", currentValue: "{{ $value }}", threshold: "70%" }
|
||||||
|
- alert: RedisMemoryCritical
|
||||||
|
expr: (100 * redis_memory_used_bytes / redis_memory_max_bytes > 85) and (redis_memory_max_bytes > 0)
|
||||||
|
for: 5m
|
||||||
|
labels: { severity: critical, service: redis }
|
||||||
|
annotations: { summary: "Redis 内存使用率达到严重阈值", description: "Redis 内存使用率持续超过85%。", currentValue: "{{ $value }}", threshold: "85%" }
|
||||||
|
- alert: MinioCapacityWarning
|
||||||
|
expr: (100 * (1 - minio_cluster_capacity_usable_free_bytes / minio_cluster_capacity_usable_total_bytes) > 80) and (100 * (1 - minio_cluster_capacity_usable_free_bytes / minio_cluster_capacity_usable_total_bytes) <= 90)
|
||||||
|
for: 15m
|
||||||
|
labels: { severity: warning, service: minio }
|
||||||
|
annotations: { summary: "MinIO 容量使用率达到警告阈值", description: "MinIO 容量使用率持续超过80%。", currentValue: "{{ $value }}", threshold: "80%" }
|
||||||
|
- alert: MinioCapacityCritical
|
||||||
|
expr: 100 * (1 - minio_cluster_capacity_usable_free_bytes / minio_cluster_capacity_usable_total_bytes) > 90
|
||||||
|
for: 5m
|
||||||
|
labels: { severity: critical, service: minio }
|
||||||
|
annotations: { summary: "MinIO 容量使用率达到严重阈值", description: "MinIO 容量使用率持续超过90%。", currentValue: "{{ $value }}", threshold: "90%" }
|
||||||
@@ -0,0 +1,100 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
set -Eeuo pipefail
|
||||||
|
|
||||||
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||||
|
PROMETHEUS_RETENTION_TIME="${PROMETHEUS_RETENTION_TIME:-30d}"
|
||||||
|
PROMETHEUS_RETENTION_SIZE="${PROMETHEUS_RETENTION_SIZE:-8GB}"
|
||||||
|
|
||||||
|
if [[ "$(id -u)" -ne 0 ]]; then
|
||||||
|
echo "Run as root." >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
if ! command -v apt-get >/dev/null 2>&1; then
|
||||||
|
echo "This installer currently supports Debian/Ubuntu apt packages only." >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
log() { printf '\n[%s] %s\n' "$(date '+%F %T')" "$*"; }
|
||||||
|
|
||||||
|
log "Installing Prometheus and Node Exporter packages"
|
||||||
|
apt-get update
|
||||||
|
DEBIAN_FRONTEND=noninteractive apt-get install -y prometheus prometheus-node-exporter curl iproute2
|
||||||
|
|
||||||
|
prometheus_bin="$(command -v prometheus)"
|
||||||
|
node_exporter_bin="$(command -v prometheus-node-exporter)"
|
||||||
|
promtool_bin="$(command -v promtool)"
|
||||||
|
backup_dir="/etc/prometheus/cmpp-backups/$(date '+%Y%m%d-%H%M%S')"
|
||||||
|
mkdir -p "$backup_dir" /etc/systemd/system/prometheus.service.d /etc/systemd/system/prometheus-node-exporter.service.d
|
||||||
|
|
||||||
|
for config_file in /etc/prometheus/prometheus.yml /etc/prometheus/cmpp-alerts.yml /etc/prometheus/cmpp-alerts-source.yml; do
|
||||||
|
if [[ -f "$config_file" ]]; then
|
||||||
|
cp --preserve=mode,timestamps "$config_file" "$backup_dir/$(basename "$config_file")"
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
if [[ -f /etc/systemd/system/prometheus.service.d/cmpp-monitoring.conf ]]; then
|
||||||
|
cp --preserve=mode,timestamps /etc/systemd/system/prometheus.service.d/cmpp-monitoring.conf "$backup_dir/prometheus-service-override.conf"
|
||||||
|
fi
|
||||||
|
if [[ -f /etc/systemd/system/prometheus-node-exporter.service.d/cmpp-monitoring.conf ]]; then
|
||||||
|
cp --preserve=mode,timestamps /etc/systemd/system/prometheus-node-exporter.service.d/cmpp-monitoring.conf "$backup_dir/node-exporter-service-override.conf"
|
||||||
|
fi
|
||||||
|
|
||||||
|
log "Installing platform-owned scrape and alert configuration"
|
||||||
|
install -o root -g root -m 0644 "$SCRIPT_DIR/prometheus.yml" /etc/prometheus/prometheus.yml
|
||||||
|
install -o root -g root -m 0644 "$SCRIPT_DIR/cmpp-alerts.yml" /etc/prometheus/cmpp-alerts-source.yml
|
||||||
|
# 可配置规则由 API 管理;基础规则必须排除同名项,否则 Prometheus 会同时计算旧阈值和新阈值。
|
||||||
|
awk '
|
||||||
|
BEGIN {
|
||||||
|
split("HostCpuUsageWarning HostCpuUsageCritical HostMemoryUsageWarning HostMemoryUsageCritical HostRootDiskUsageWarning HostRootDiskUsageCritical CmppApiHttpErrorRateWarning CmppApiHttpErrorRateCritical CmppApiLatencyWarning CmppApiLatencyCritical CmppApiEventLoopLagWarning CmppApiEventLoopLagCritical CmppGatewayQueueDelayedWarning CmppGatewayQueueDelayedCritical PostgresConnectionsWarning PostgresConnectionsCritical RedisMemoryWarning RedisMemoryCritical MinioCapacityWarning MinioCapacityCritical", names, " ")
|
||||||
|
for (i in names) dropped[names[i]] = 1
|
||||||
|
}
|
||||||
|
/^ - name:/ { skip = 0 }
|
||||||
|
/^ - alert:/ { skip = ($3 in dropped) }
|
||||||
|
!skip { print }
|
||||||
|
' "$SCRIPT_DIR/cmpp-alerts.yml" > /etc/prometheus/cmpp-alerts.yml
|
||||||
|
chown root:root /etc/prometheus/cmpp-alerts.yml
|
||||||
|
chmod 0644 /etc/prometheus/cmpp-alerts.yml
|
||||||
|
# SGID ensures API原子rename生成的新规则继续继承prometheus组,否则reload会因不可读返回500。
|
||||||
|
install -d -o cmpp-api -g prometheus -m 2750 /var/lib/cmpp-platform/monitoring
|
||||||
|
if [[ ! -f /var/lib/cmpp-platform/monitoring/cmpp-managed-alerts.yml ]]; then
|
||||||
|
install -o cmpp-api -g prometheus -m 0640 "$SCRIPT_DIR/cmpp-managed-alerts.yml" /var/lib/cmpp-platform/monitoring/cmpp-managed-alerts.yml
|
||||||
|
fi
|
||||||
|
|
||||||
|
cat >/etc/systemd/system/prometheus.service.d/cmpp-monitoring.conf <<EOF
|
||||||
|
[Service]
|
||||||
|
ExecStart=
|
||||||
|
ExecStart=${prometheus_bin} --config.file=/etc/prometheus/prometheus.yml --storage.tsdb.path=/var/lib/prometheus/metrics2 --storage.tsdb.retention.time=${PROMETHEUS_RETENTION_TIME} --storage.tsdb.retention.size=${PROMETHEUS_RETENTION_SIZE} --web.listen-address=127.0.0.1:9090 --web.enable-lifecycle
|
||||||
|
EOF
|
||||||
|
|
||||||
|
cat >/etc/systemd/system/prometheus-node-exporter.service.d/cmpp-monitoring.conf <<EOF
|
||||||
|
[Service]
|
||||||
|
ExecStart=
|
||||||
|
ExecStart=${node_exporter_bin} --web.listen-address=127.0.0.1:9100 --collector.systemd --collector.systemd.unit-include='cmpp-api\\.service|cmpp-gateway\\.service|postgresql\\.service|redis(-server)?\\.service|cmpp-minio\\.service|nginx\\.service' --collector.filesystem.mount-points-exclude='^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/)'
|
||||||
|
EOF
|
||||||
|
|
||||||
|
log "Validating Prometheus configuration before restart"
|
||||||
|
"$promtool_bin" check rules /etc/prometheus/cmpp-alerts.yml
|
||||||
|
"$promtool_bin" check rules /var/lib/cmpp-platform/monitoring/cmpp-managed-alerts.yml
|
||||||
|
"$promtool_bin" check config /etc/prometheus/prometheus.yml
|
||||||
|
|
||||||
|
systemctl daemon-reload
|
||||||
|
systemctl enable prometheus prometheus-node-exporter
|
||||||
|
systemctl restart prometheus-node-exporter
|
||||||
|
systemctl restart prometheus
|
||||||
|
|
||||||
|
for _ in $(seq 1 30); do
|
||||||
|
if curl -fsS http://127.0.0.1:9090/-/ready >/dev/null; then
|
||||||
|
break
|
||||||
|
fi
|
||||||
|
sleep 1
|
||||||
|
done
|
||||||
|
curl -fsS http://127.0.0.1:9090/-/ready >/dev/null
|
||||||
|
curl -fsS http://127.0.0.1:9100/metrics >/dev/null
|
||||||
|
|
||||||
|
if ss -lnt | grep -Eq '(^|[[:space:]])(0\.0\.0\.0|\[::\]):(9090|9100)([[:space:]]|$)'; then
|
||||||
|
echo "Prometheus monitoring ports unexpectedly listen on a wildcard address." >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
log "Prometheus monitoring is ready on loopback only"
|
||||||
|
echo "Configuration backup: $backup_dir"
|
||||||
|
echo "The CMPP API and Gateway were not restarted."
|
||||||
@@ -0,0 +1,92 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
set -Eeuo pipefail
|
||||||
|
|
||||||
|
if [[ "$(id -u)" -ne 0 ]]; then echo "Run as root." >&2; exit 1; fi
|
||||||
|
for command_name in apt-get systemctl nginx curl; do command -v "$command_name" >/dev/null || { echo "Missing command: $command_name" >&2; exit 1; }; done
|
||||||
|
|
||||||
|
log() { printf '\n[%s] %s\n' "$(date '+%F %T')" "$*"; }
|
||||||
|
backup_dir="/etc/prometheus/cmpp-backups/$(date '+%Y%m%d-%H%M%S')-service-exporters"
|
||||||
|
mkdir -p "$backup_dir"
|
||||||
|
|
||||||
|
for config_file in /etc/cmpp-platform/minio.env /etc/nginx/conf.d/cmpp-monitoring-status.conf /etc/cmpp-platform/monitoring-exporters.env; do
|
||||||
|
[[ -f "$config_file" ]] && cp --preserve=mode,timestamps "$config_file" "$backup_dir/$(basename "$config_file")"
|
||||||
|
done
|
||||||
|
|
||||||
|
log "Installing bounded service exporters"
|
||||||
|
apt-get update
|
||||||
|
DEBIAN_FRONTEND=noninteractive apt-get install -y prometheus-postgres-exporter prometheus-redis-exporter prometheus-nginx-exporter
|
||||||
|
|
||||||
|
[[ -f /etc/cmpp-platform/cmpp-platform.env ]] || { echo "Missing platform environment file." >&2; exit 1; }
|
||||||
|
set -a
|
||||||
|
# shellcheck disable=SC1091
|
||||||
|
. /etc/cmpp-platform/cmpp-platform.env
|
||||||
|
set +a
|
||||||
|
database_base="${DATABASE_URL%%\?*}"
|
||||||
|
cat >/etc/cmpp-platform/monitoring-exporters.env <<EOF
|
||||||
|
DATA_SOURCE_NAME=${database_base}?sslmode=disable
|
||||||
|
REDIS_ADDR=redis://${REDIS_HOST:-127.0.0.1}:${REDIS_PORT:-6379}
|
||||||
|
EOF
|
||||||
|
chmod 0600 /etc/cmpp-platform/monitoring-exporters.env
|
||||||
|
|
||||||
|
write_override() {
|
||||||
|
local unit="$1" executable="$2" arguments="$3"
|
||||||
|
local directory="/etc/systemd/system/${unit}.service.d"
|
||||||
|
mkdir -p "$directory"
|
||||||
|
[[ -f "$directory/cmpp-monitoring.conf" ]] && cp --preserve=mode,timestamps "$directory/cmpp-monitoring.conf" "$backup_dir/${unit}-override.conf"
|
||||||
|
cat >"$directory/cmpp-monitoring.conf" <<EOF
|
||||||
|
[Service]
|
||||||
|
EnvironmentFile=/etc/cmpp-platform/monitoring-exporters.env
|
||||||
|
ExecStart=
|
||||||
|
ExecStart=${executable} ${arguments}
|
||||||
|
EOF
|
||||||
|
}
|
||||||
|
|
||||||
|
postgres_exporter="$(command -v prometheus-postgres-exporter)"
|
||||||
|
redis_exporter="$(command -v prometheus-redis-exporter)"
|
||||||
|
nginx_exporter="$(command -v prometheus-nginx-exporter)"
|
||||||
|
write_override prometheus-postgres-exporter "$postgres_exporter" '--web.listen-address=127.0.0.1:9187'
|
||||||
|
write_override prometheus-redis-exporter "$redis_exporter" '--web.listen-address=127.0.0.1:9121'
|
||||||
|
|
||||||
|
cat >/etc/nginx/conf.d/cmpp-monitoring-status.conf <<'EOF'
|
||||||
|
server {
|
||||||
|
listen 127.0.0.1:8088;
|
||||||
|
server_name localhost;
|
||||||
|
access_log off;
|
||||||
|
location = /stub_status {
|
||||||
|
stub_status;
|
||||||
|
allow 127.0.0.1;
|
||||||
|
allow ::1;
|
||||||
|
deny all;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
EOF
|
||||||
|
write_override prometheus-nginx-exporter "$nginx_exporter" '--web.listen-address=127.0.0.1:9113 --nginx.scrape-uri=http://127.0.0.1:8088/stub_status'
|
||||||
|
|
||||||
|
# MinIO exposes only operational aggregates and listens on loopback; public auth here does not expose objects or credentials.
|
||||||
|
grep -q '^MINIO_PROMETHEUS_AUTH_TYPE=' /etc/cmpp-platform/minio.env \
|
||||||
|
&& sed -i 's/^MINIO_PROMETHEUS_AUTH_TYPE=.*/MINIO_PROMETHEUS_AUTH_TYPE=public/' /etc/cmpp-platform/minio.env \
|
||||||
|
|| printf '\nMINIO_PROMETHEUS_AUTH_TYPE=public\n' >>/etc/cmpp-platform/minio.env
|
||||||
|
|
||||||
|
nginx -t
|
||||||
|
systemctl daemon-reload
|
||||||
|
systemctl enable prometheus-postgres-exporter prometheus-redis-exporter prometheus-nginx-exporter
|
||||||
|
systemctl restart prometheus-postgres-exporter prometheus-redis-exporter prometheus-nginx-exporter
|
||||||
|
systemctl restart cmpp-minio
|
||||||
|
systemctl reload nginx
|
||||||
|
|
||||||
|
wait_for_http() {
|
||||||
|
local endpoint="$1" attempt
|
||||||
|
for attempt in $(seq 1 30); do
|
||||||
|
curl -fsS "$endpoint" >/dev/null 2>&1 && return 0
|
||||||
|
sleep 1
|
||||||
|
done
|
||||||
|
echo "Monitoring endpoint did not become ready: $endpoint" >&2
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
for endpoint in 127.0.0.1:9187 127.0.0.1:9121 127.0.0.1:9113; do wait_for_http "http://${endpoint}/metrics"; done
|
||||||
|
wait_for_http http://127.0.0.1:9000/minio/v2/metrics/cluster
|
||||||
|
if ss -lnt | grep -Eq '(^|[[:space:]])(0\.0\.0\.0|\[::\]):(9187|9121|9113)([[:space:]]|$)'; then
|
||||||
|
echo "A service exporter unexpectedly listens on a wildcard address." >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
log "Service exporters are ready on loopback only; backup: $backup_dir"
|
||||||
@@ -0,0 +1,47 @@
|
|||||||
|
global:
|
||||||
|
scrape_interval: 15s
|
||||||
|
evaluation_interval: 15s
|
||||||
|
external_labels:
|
||||||
|
platform: cmpp
|
||||||
|
environment: preproduction
|
||||||
|
|
||||||
|
rule_files:
|
||||||
|
- /etc/prometheus/cmpp-alerts.yml
|
||||||
|
- /var/lib/cmpp-platform/monitoring/*.yml
|
||||||
|
|
||||||
|
scrape_configs:
|
||||||
|
- job_name: prometheus
|
||||||
|
static_configs:
|
||||||
|
- targets: [127.0.0.1:9090]
|
||||||
|
|
||||||
|
- job_name: node
|
||||||
|
static_configs:
|
||||||
|
- targets: [127.0.0.1:9100]
|
||||||
|
labels:
|
||||||
|
host: cmpp-primary
|
||||||
|
|
||||||
|
- job_name: cmpp-api
|
||||||
|
static_configs:
|
||||||
|
- targets: [127.0.0.1:9464]
|
||||||
|
|
||||||
|
- job_name: cmpp-gateway
|
||||||
|
metrics_path: /metrics
|
||||||
|
static_configs:
|
||||||
|
- targets: [127.0.0.1:8090]
|
||||||
|
|
||||||
|
- job_name: postgresql
|
||||||
|
static_configs:
|
||||||
|
- targets: [127.0.0.1:9187]
|
||||||
|
|
||||||
|
- job_name: redis
|
||||||
|
static_configs:
|
||||||
|
- targets: [127.0.0.1:9121]
|
||||||
|
|
||||||
|
- job_name: minio
|
||||||
|
metrics_path: /minio/v2/metrics/cluster
|
||||||
|
static_configs:
|
||||||
|
- targets: [127.0.0.1:9000]
|
||||||
|
|
||||||
|
- job_name: nginx
|
||||||
|
static_configs:
|
||||||
|
- targets: [127.0.0.1:9113]
|
||||||
@@ -0,0 +1,57 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
set -Eeuo pipefail
|
||||||
|
|
||||||
|
APP_DIR="${APP_DIR:-/opt/cmpp-platform}"
|
||||||
|
if [[ "$(id -u)" -ne 0 ]]; then echo "Run as root." >&2; exit 1; fi
|
||||||
|
for command_name in fail2ban-client nft nginx systemctl; do command -v "$command_name" >/dev/null || { echo "Missing command: $command_name" >&2; exit 1; }; done
|
||||||
|
agent_binary="$APP_DIR/dist/cmpp-security-agent"
|
||||||
|
[[ -x "$agent_binary" ]] || { echo "Missing built security agent: $agent_binary" >&2; exit 1; }
|
||||||
|
|
||||||
|
getent group cmpp-security >/dev/null || groupadd --system cmpp-security
|
||||||
|
id cmpp-api >/dev/null 2>&1 || useradd --system --home-dir /nonexistent --shell /usr/sbin/nologin cmpp-api
|
||||||
|
usermod -a -G cmpp-security cmpp-api
|
||||||
|
install -d -o root -g cmpp-security -m 0770 /run/cmpp-security-agent
|
||||||
|
install -d -o root -g cmpp-security -m 0750 /var/lib/cmpp-security-agent
|
||||||
|
install -d -o cmpp-api -g cmpp-security -m 0750 "$APP_DIR/logs/api"
|
||||||
|
[[ -d /var/lib/cmpp-platform/object-storage ]] && chown -R cmpp-api:cmpp-security /var/lib/cmpp-platform/object-storage
|
||||||
|
|
||||||
|
sed "s#@CMPP_SECURITY_AGENT_BIN@#$agent_binary#g" "$APP_DIR/deploy/security/cmpp-report-only.conf" >/etc/fail2ban/action.d/cmpp-report-only.conf
|
||||||
|
chmod 0640 /etc/fail2ban/action.d/cmpp-report-only.conf
|
||||||
|
install -m 0640 "$APP_DIR/deploy/security/cmpp-http-scan.conf" /etc/fail2ban/filter.d/cmpp-http-scan.conf
|
||||||
|
install -d -m 0750 /etc/nginx/snippets /etc/nftables.d
|
||||||
|
touch /etc/nginx/snippets/cmpp-security-deny.conf
|
||||||
|
chmod 0640 /etc/nginx/snippets/cmpp-security-deny.conf
|
||||||
|
|
||||||
|
cat >/etc/nftables.d/cmpp-security.nft <<'EOF'
|
||||||
|
table inet cmpp_security {
|
||||||
|
set blocked_ipv4 { type ipv4_addr; flags timeout; }
|
||||||
|
set blocked_ipv6 { type ipv6_addr; flags timeout; }
|
||||||
|
chain input { type filter hook input priority -10; policy accept; ip saddr @blocked_ipv4 drop; ip6 saddr @blocked_ipv6 drop; }
|
||||||
|
}
|
||||||
|
EOF
|
||||||
|
grep -q 'cmpp-security.nft' /etc/nftables.conf || printf '\ninclude "/etc/nftables.d/cmpp-security.nft"\n' >>/etc/nftables.conf
|
||||||
|
nft -c -f /etc/nftables.conf
|
||||||
|
nft list table inet cmpp_security >/dev/null 2>&1 || nft -f /etc/nftables.d/cmpp-security.nft
|
||||||
|
|
||||||
|
sed "s#@CMPP_SECURITY_AGENT_BIN@#$agent_binary#g" "$APP_DIR/deploy/security/cmpp-security-agent.service" >/etc/systemd/system/cmpp-security-agent.service
|
||||||
|
if grep -Rqs '@CMPP_SECURITY_AGENT_BIN@' /etc/systemd/system/cmpp-security-agent.service /etc/fail2ban/action.d/cmpp-report-only.conf; then
|
||||||
|
echo "Security agent executable placeholder was not rendered." >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
install -d -m 0755 /etc/systemd/system/cmpp-api.service.d
|
||||||
|
cat >/etc/systemd/system/cmpp-api.service.d/security-boundary.conf <<EOF
|
||||||
|
[Service]
|
||||||
|
User=cmpp-api
|
||||||
|
Group=cmpp-security
|
||||||
|
NoNewPrivileges=true
|
||||||
|
PrivateTmp=true
|
||||||
|
ProtectHome=true
|
||||||
|
ProtectSystem=true
|
||||||
|
ReadWritePaths=$APP_DIR/logs/api /var/lib/cmpp-platform/object-storage
|
||||||
|
EOF
|
||||||
|
|
||||||
|
fail2ban-client -t
|
||||||
|
nginx -t
|
||||||
|
systemctl daemon-reload
|
||||||
|
systemctl enable cmpp-security-agent
|
||||||
|
echo "Security boundary installed. Restart cmpp-security-agent and cmpp-api only in the approved release window."
|
||||||
@@ -0,0 +1,30 @@
|
|||||||
|
import { readFileSync } from 'node:fs';
|
||||||
|
import { resolve } from 'node:path';
|
||||||
|
|
||||||
|
const root = resolve(import.meta.dirname, '..', '..');
|
||||||
|
const read = (relativePath) => readFileSync(resolve(root, relativePath), 'utf8');
|
||||||
|
const installer = read('tools/security/install-security-agent.sh');
|
||||||
|
const service = read('deploy/security/cmpp-security-agent.service');
|
||||||
|
const action = read('deploy/security/cmpp-report-only.conf');
|
||||||
|
const placeholder = '@CMPP_SECURITY_AGENT_BIN@';
|
||||||
|
|
||||||
|
for (const [label, source] of [['systemd service', service], ['Fail2ban action', action]]) {
|
||||||
|
if (!source.includes(placeholder)) throw new Error(`${label} is missing the security-agent executable placeholder`);
|
||||||
|
if (source.includes('/opt/cmpp-platform/current/bin/cmpp-security-agent')) {
|
||||||
|
throw new Error(`${label} still references the removed current/bin deployment layout`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!installer.includes('agent_binary="$APP_DIR/dist/cmpp-security-agent"')) {
|
||||||
|
throw new Error('installer does not bind the security agent to the built dist executable');
|
||||||
|
}
|
||||||
|
for (const target of ['cmpp-security-agent.service', 'cmpp-report-only.conf']) {
|
||||||
|
if (!installer.includes(`sed "s#${placeholder}#$agent_binary#g"`) || !installer.includes(target)) {
|
||||||
|
throw new Error(`installer does not render ${target} with the built security-agent executable`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (!installer.includes("grep -Rqs '@CMPP_SECURITY_AGENT_BIN@'")) {
|
||||||
|
throw new Error('installer does not fail closed when an executable placeholder remains');
|
||||||
|
}
|
||||||
|
|
||||||
|
console.log('Security deployment verified: systemd and Fail2ban use the built dist security-agent executable.');
|
||||||
Reference in New Issue
Block a user