feat: add monitoring alert read state

This commit is contained in:
hectorzhao
2026-08-16 15:04:26 +08:00
parent 79f5d3f215
commit 482f7ac1ae
15 changed files with 196 additions and 21 deletions
@@ -0,0 +1,17 @@
CREATE TABLE "InfrastructureAlertRead" (
"id" TEXT NOT NULL,
"fingerprint" TEXT NOT NULL,
"activeAt" TIMESTAMP(3) NOT NULL,
"userId" TEXT NOT NULL,
"readAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
"createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
"updatedAt" TIMESTAMP(3) NOT NULL,
CONSTRAINT "InfrastructureAlertRead_pkey" PRIMARY KEY ("id"),
CONSTRAINT "InfrastructureAlertRead_userId_fkey" FOREIGN KEY ("userId") REFERENCES "User"("id") ON DELETE CASCADE ON UPDATE CASCADE
);
CREATE UNIQUE INDEX "InfrastructureAlertRead_fingerprint_userId_key"
ON "InfrastructureAlertRead"("fingerprint", "userId");
CREATE INDEX "InfrastructureAlertRead_userId_readAt_idx"
ON "InfrastructureAlertRead"("userId", "readAt");
+16
View File
@@ -103,6 +103,7 @@ model User {
releasedPhoneFrequencyHits PhoneFrequencyHit[] @relation("PhoneFrequencyHitReleaser") releasedPhoneFrequencyHits PhoneFrequencyHit[] @relation("PhoneFrequencyHitReleaser")
createdPhoneFrequencyWhitelistEntries PhoneFrequencyWhitelist[] @relation("PhoneFrequencyWhitelistCreator") createdPhoneFrequencyWhitelistEntries PhoneFrequencyWhitelist[] @relation("PhoneFrequencyWhitelistCreator")
updatedPhoneFrequencyWhitelistEntries PhoneFrequencyWhitelist[] @relation("PhoneFrequencyWhitelistUpdater") updatedPhoneFrequencyWhitelistEntries PhoneFrequencyWhitelist[] @relation("PhoneFrequencyWhitelistUpdater")
infrastructureAlertReads InfrastructureAlertRead[]
} }
model Role { model Role {
@@ -2446,3 +2447,18 @@ model InfrastructureAlertSetting {
createdAt DateTime @default(now()) createdAt DateTime @default(now())
updatedAt DateTime @updatedAt updatedAt DateTime @updatedAt
} }
model InfrastructureAlertRead {
id String @id @default(cuid())
fingerprint String
activeAt DateTime
userId String
readAt DateTime @default(now())
createdAt DateTime @default(now())
updatedAt DateTime @updatedAt
user User @relation(fields: [userId], references: [id], onDelete: Cascade)
@@unique([fingerprint, userId])
@@index([userId, readAt])
}
@@ -36,6 +36,8 @@ export type InfrastructureAlert = {
threshold?: string; threshold?: string;
service?: string; service?: string;
instance?: string; instance?: string;
acknowledged: boolean;
acknowledgedAt?: string;
}; };
export type InfrastructureMonitoringOverview = { export type InfrastructureMonitoringOverview = {
@@ -1,4 +1,4 @@
import { Body, Controller, Get, Put, Query } from '@nestjs/common'; import { Body, Controller, Get, Param, Post, Put, Query } from '@nestjs/common';
import { ApiTags } from '@nestjs/swagger'; import { ApiTags } from '@nestjs/swagger';
import { CurrentSessionUserId } from '../auth/current-session-user.decorator'; import { CurrentSessionUserId } from '../auth/current-session-user.decorator';
import { RequireRecentAuthentication } from '../auth/require-recent-authentication.decorator'; import { RequireRecentAuthentication } from '../auth/require-recent-authentication.decorator';
@@ -11,12 +11,17 @@ export class InfrastructureMonitoringController {
constructor(private readonly monitoring: InfrastructureMonitoringService, private readonly settings: InfrastructureAlertSettingsService) {} constructor(private readonly monitoring: InfrastructureMonitoringService, private readonly settings: InfrastructureAlertSettingsService) {}
@Get('overview') @Get('overview')
overview(@Query('range') range?: string) { overview(@Query('range') range?: string, @CurrentSessionUserId() userId?: string) {
return this.monitoring.overview(range); return this.monitoring.overview(range, userId);
} }
@Get('notification-summary') @Get('notification-summary')
notificationSummary() { return this.monitoring.notificationSummary(); } notificationSummary(@CurrentSessionUserId() userId?: string) { return this.monitoring.notificationSummary(userId); }
@Post('alerts/:fingerprint/read')
markAlertRead(@Param('fingerprint') fingerprint: string, @Body('activeAt') activeAt: unknown, @CurrentSessionUserId() userId: string) {
return this.monitoring.markAlertRead(fingerprint, activeAt, userId);
}
@Get('alert-thresholds') @Get('alert-thresholds')
alertThresholds() { return this.settings.get(); } alertThresholds() { return this.settings.get(); }
@@ -1,5 +1,6 @@
import { BadRequestException } from '@nestjs/common'; import { BadRequestException } from '@nestjs/common';
import { ConfigService } from '@nestjs/config'; import { ConfigService } from '@nestjs/config';
import { createHash } from 'node:crypto';
import { InfrastructureMonitoringService } from './infrastructure-monitoring.service'; import { InfrastructureMonitoringService } from './infrastructure-monitoring.service';
function success(data: unknown) { function success(data: unknown) {
@@ -11,20 +12,30 @@ function success(data: unknown) {
} }
describe('InfrastructureMonitoringService', () => { describe('InfrastructureMonitoringService', () => {
afterEach(() => jest.restoreAllMocks()); const prisma = {
infrastructureAlertRead: { findMany: jest.fn().mockResolvedValue([]), create: jest.fn(), update: jest.fn(), findUniqueOrThrow: jest.fn() },
operationLog: { create: jest.fn() },
$transaction: jest.fn(),
};
afterEach(() => {
jest.restoreAllMocks();
jest.clearAllMocks();
prisma.infrastructureAlertRead.findMany.mockResolvedValue([]);
});
it('rejects ranges outside the fixed whitelist before querying Prometheus', async () => { it('rejects ranges outside the fixed whitelist before querying Prometheus', async () => {
const fetchSpy = jest.spyOn(global, 'fetch'); const fetchSpy = jest.spyOn(global, 'fetch');
const service = new InfrastructureMonitoringService(new ConfigService()); const service = new InfrastructureMonitoringService(new ConfigService(), prisma as never);
await expect(service.overview('30d')).rejects.toBeInstanceOf(BadRequestException); await expect(service.overview('30d')).rejects.toBeInstanceOf(BadRequestException);
expect(fetchSpy).not.toHaveBeenCalled(); expect(fetchSpy).not.toHaveBeenCalled();
}); });
it('rejects credential-bearing or remote plaintext Prometheus endpoints at startup', () => { it('rejects credential-bearing or remote plaintext Prometheus endpoints at startup', () => {
expect(() => new InfrastructureMonitoringService(new ConfigService({ PROMETHEUS_URL: 'http://user:secret@127.0.0.1:9090' }))).toThrow('must not contain credentials'); expect(() => new InfrastructureMonitoringService(new ConfigService({ PROMETHEUS_URL: 'http://user:secret@127.0.0.1:9090' }), prisma as never)).toThrow('must not contain credentials');
expect(() => new InfrastructureMonitoringService(new ConfigService({ PROMETHEUS_URL: 'http://monitor.example.com:9090' }))).toThrow('must use HTTPS'); expect(() => new InfrastructureMonitoringService(new ConfigService({ PROMETHEUS_URL: 'http://monitor.example.com:9090' }), prisma as never)).toThrow('must use HTTPS');
expect(() => new InfrastructureMonitoringService(new ConfigService({ PROMETHEUS_URL: 'https://monitor.example.com' }))).not.toThrow(); expect(() => new InfrastructureMonitoringService(new ConfigService({ PROMETHEUS_URL: 'https://monitor.example.com' }), prisma as never)).not.toThrow();
}); });
it('loads real Prometheus vectors, ranges, services and active alerts', async () => { it('loads real Prometheus vectors, ranges, services and active alerts', async () => {
@@ -68,7 +79,7 @@ describe('InfrastructureMonitoringService', () => {
return success({ result: [{ metric: {}, value: [1_765_000_060, '25'] }] }); return success({ result: [{ metric: {}, value: [1_765_000_060, '25'] }] });
}); });
const service = new InfrastructureMonitoringService(new ConfigService()); const service = new InfrastructureMonitoringService(new ConfigService(), prisma as never);
const result = await service.overview('1h'); const result = await service.overview('1h');
expect(result.available).toBe(true); expect(result.available).toBe(true);
@@ -87,7 +98,7 @@ describe('InfrastructureMonitoringService', () => {
it('returns an explicit unavailable payload without stale metrics when Prometheus fails', async () => { it('returns an explicit unavailable payload without stale metrics when Prometheus fails', async () => {
jest.spyOn(global, 'fetch').mockRejectedValue(new Error('ECONNREFUSED')); jest.spyOn(global, 'fetch').mockRejectedValue(new Error('ECONNREFUSED'));
const service = new InfrastructureMonitoringService(new ConfigService()); const service = new InfrastructureMonitoringService(new ConfigService(), prisma as never);
const result = await service.overview('24h'); const result = await service.overview('24h');
@@ -98,4 +109,29 @@ describe('InfrastructureMonitoringService', () => {
expect(result.serviceMetrics.every((item) => item.available === false)).toBe(true); expect(result.serviceMetrics.every((item) => item.available === false)).toBe(true);
expect(result.error).not.toContain('ECONNREFUSED'); expect(result.error).not.toContain('ECONNREFUSED');
}); });
it('excludes only the current alert occurrence after the current administrator marks it read', async () => {
const labels = { alertname: 'QaWarning', severity: 'warning', service: 'qa-preview' };
const fingerprint = createHash('sha256').update(JSON.stringify(Object.entries(labels).sort(([left], [right]) => left.localeCompare(right)))).digest('hex').slice(0, 24);
jest.spyOn(global, 'fetch').mockResolvedValue(success({ alerts: [{ labels, annotations: { summary: '演示预警' }, state: 'firing', activeAt: '2026-08-16T01:00:00.000Z' }] }));
prisma.infrastructureAlertRead.findMany.mockResolvedValue([{ fingerprint, activeAt: new Date('2026-08-16T01:00:00.000Z'), readAt: new Date('2026-08-16T01:01:00.000Z') }]);
const service = new InfrastructureMonitoringService(new ConfigService(), prisma as never);
await expect(service.notificationSummary('admin-1')).resolves.toEqual({ count: 0, criticalCount: 0 });
prisma.infrastructureAlertRead.findMany.mockResolvedValue([{ fingerprint, activeAt: new Date('2026-08-15T01:00:00.000Z'), readAt: new Date('2026-08-15T01:01:00.000Z') }]);
await expect(service.notificationSummary('admin-1')).resolves.toEqual({ count: 1, criticalCount: 0 });
});
it('upserts an idempotent per-user read record only for a currently active occurrence', async () => {
const labels = { alertname: 'QaCritical', severity: 'critical', service: 'qa-preview' };
const fingerprint = createHash('sha256').update(JSON.stringify(Object.entries(labels).sort(([left], [right]) => left.localeCompare(right)))).digest('hex').slice(0, 24);
jest.spyOn(global, 'fetch').mockResolvedValue(success({ alerts: [{ labels, annotations: { summary: '演示严重告警' }, state: 'firing', activeAt: '2026-08-16T02:00:00.000Z' }] }));
prisma.$transaction.mockResolvedValue([{ activeAt: new Date('2026-08-16T02:00:00.000Z'), readAt: new Date('2026-08-16T02:01:00.000Z') }, {}]);
const service = new InfrastructureMonitoringService(new ConfigService(), prisma as never);
await expect(service.markAlertRead(fingerprint, '2026-08-16T02:00:00.000Z', 'admin-1')).resolves.toMatchObject({ fingerprint, acknowledged: true });
expect(prisma.infrastructureAlertRead.create).toHaveBeenCalledWith(expect.objectContaining({ data: expect.objectContaining({ fingerprint, userId: 'admin-1' }) }));
await expect(service.markAlertRead(fingerprint, '2026-08-15T02:00:00.000Z', 'admin-1')).rejects.toThrow('已结束或已重新触发');
});
}); });
@@ -1,6 +1,8 @@
import { BadRequestException, Injectable, Logger, ServiceUnavailableException } from '@nestjs/common'; import { BadRequestException, Injectable, Logger, NotFoundException, ServiceUnavailableException } from '@nestjs/common';
import { ConfigService } from '@nestjs/config'; import { ConfigService } from '@nestjs/config';
import { Prisma } from '@prisma/client';
import { createHash } from 'node:crypto'; import { createHash } from 'node:crypto';
import { PrismaService } from '../prisma/prisma.service';
import type { import type {
InfrastructureAlert, InfrastructureAlert,
InfrastructureMetricPoint, InfrastructureMetricPoint,
@@ -162,12 +164,12 @@ export class InfrastructureMonitoringService {
private readonly prometheusUrl: string; private readonly prometheusUrl: string;
private readonly queryTimeoutMs: number; private readonly queryTimeoutMs: number;
constructor(config: ConfigService) { constructor(config: ConfigService, private readonly prisma: PrismaService) {
this.prometheusUrl = normalizePrometheusUrl(config.get('PROMETHEUS_URL')); this.prometheusUrl = normalizePrometheusUrl(config.get('PROMETHEUS_URL'));
this.queryTimeoutMs = Math.min(15_000, Math.max(1_000, Number(config.get('PROMETHEUS_QUERY_TIMEOUT_MS') ?? 5_000))); this.queryTimeoutMs = Math.min(15_000, Math.max(1_000, Number(config.get('PROMETHEUS_QUERY_TIMEOUT_MS') ?? 5_000)));
} }
async overview(rawRange?: string): Promise<InfrastructureMonitoringOverview> { async overview(rawRange?: string, userId?: string): Promise<InfrastructureMonitoringOverview> {
const range = this.parseRange(rawRange); const range = this.parseRange(rawRange);
const collectedAt = new Date().toISOString(); const collectedAt = new Date().toISOString();
try { try {
@@ -180,7 +182,7 @@ export class InfrastructureMonitoringService {
]); ]);
const services = this.parseServices(serviceResponse); const services = this.parseServices(serviceResponse);
const serviceMetrics = this.parseServiceMetrics(serviceMetricResponse); const serviceMetrics = this.parseServiceMetrics(serviceMetricResponse);
const alerts = this.parseAlerts(alertResponse); const alerts = await this.attachReadState(this.parseAlerts(alertResponse), userId);
const warningAlerts = alerts.filter((item) => item.severity === 'warning').length; const warningAlerts = alerts.filter((item) => item.severity === 'warning').length;
const criticalAlerts = alerts.filter((item) => item.severity === 'critical').length; const criticalAlerts = alerts.filter((item) => item.severity === 'critical').length;
const overallStatus = criticalAlerts > 0 ? 'critical' : warningAlerts > 0 ? 'warning' : 'healthy'; const overallStatus = criticalAlerts > 0 ? 'critical' : warningAlerts > 0 ? 'warning' : 'healthy';
@@ -210,16 +212,47 @@ export class InfrastructureMonitoringService {
} }
} }
async notificationSummary() { async notificationSummary(userId?: string) {
try { try {
const alerts = this.parseAlerts(await this.getJson<PrometheusAlertResponse>('/api/v1/alerts')); const alerts = await this.attachReadState(this.parseAlerts(await this.getJson<PrometheusAlertResponse>('/api/v1/alerts')), userId);
return { count: alerts.length, criticalCount: alerts.filter((item) => item.severity === 'critical').length }; const unreadAlerts = alerts.filter((item) => !item.acknowledged);
return { count: unreadAlerts.length, criticalCount: unreadAlerts.filter((item) => item.severity === 'critical').length };
} catch (error) { } catch (error) {
this.logger.warn(`Prometheus notification summary unavailable: ${error instanceof Error ? error.message : 'unknown error'}`); this.logger.warn(`Prometheus notification summary unavailable: ${error instanceof Error ? error.message : 'unknown error'}`);
throw new ServiceUnavailableException('Prometheus活动告警当前不可用'); throw new ServiceUnavailableException('Prometheus活动告警当前不可用');
} }
} }
async markAlertRead(fingerprint: string, rawActiveAt: unknown, userId: string) {
if (!/^[a-f0-9]{24}$/.test(fingerprint)) throw new BadRequestException('告警指纹无效');
const activeAt = new Date(String(rawActiveAt ?? ''));
if (!Number.isFinite(activeAt.getTime())) throw new BadRequestException('告警开始时间无效');
const activeAlerts = this.parseAlerts(await this.getJson<PrometheusAlertResponse>('/api/v1/alerts'));
const current = activeAlerts.find((item) => item.fingerprint === fingerprint && Date.parse(item.startedAt) === activeAt.getTime());
if (!current) throw new NotFoundException('该次活动告警已结束或已重新触发,请刷新后重试');
const readAt = new Date();
const log = () => this.prisma.operationLog.create({
data: { userId, action: 'monitoring.alert_marked_read', resource: 'infrastructure_alert', resourceId: fingerprint, detail: { activeAt: activeAt.toISOString(), alertName: current.name, severity: current.severity } },
});
let read;
try {
[read] = await this.prisma.$transaction([
this.prisma.infrastructureAlertRead.create({ data: { fingerprint, activeAt, userId, readAt } }),
log(),
]);
} catch (error) {
if (!(error instanceof Prisma.PrismaClientKnownRequestError) || error.code !== 'P2002') throw error;
const existing = await this.prisma.infrastructureAlertRead.findUniqueOrThrow({ where: { fingerprint_userId: { fingerprint, userId } } });
// 同一次触发重复点击不更新readAt也不重复写日志;activeAt变化才代表同指纹的新触发周期。
if (existing.activeAt.getTime() === activeAt.getTime()) read = existing;
else [read] = await this.prisma.$transaction([
this.prisma.infrastructureAlertRead.update({ where: { fingerprint_userId: { fingerprint, userId } }, data: { activeAt, readAt } }),
log(),
]);
}
return { fingerprint, activeAt: read.activeAt.toISOString(), acknowledged: true, acknowledgedAt: read.readAt.toISOString() };
}
private parseRange(value?: string): InfrastructureMonitoringRange { private parseRange(value?: string): InfrastructureMonitoringRange {
const range = value || '24h'; const range = value || '24h';
if (!(range in RANGE_CONFIG)) throw new BadRequestException('监控时间范围只支持1h、24h或7d'); if (!(range in RANGE_CONFIG)) throw new BadRequestException('监控时间范围只支持1h、24h或7d');
@@ -275,6 +308,7 @@ export class InfrastructureMonitoringService {
threshold: annotations.threshold, threshold: annotations.threshold,
service: labels.service, service: labels.service,
instance: labels.instance, instance: labels.instance,
acknowledged: false,
}; };
}) })
.sort((left, right) => { .sort((left, right) => {
@@ -283,6 +317,20 @@ export class InfrastructureMonitoringService {
}); });
} }
private async attachReadState(alerts: InfrastructureAlert[], userId?: string) {
if (!userId || alerts.length === 0) return alerts;
const reads = await this.prisma.infrastructureAlertRead.findMany({
where: { userId, fingerprint: { in: alerts.map((item) => item.fingerprint) } },
select: { fingerprint: true, activeAt: true, readAt: true },
});
const byFingerprint = new Map(reads.map((item) => [item.fingerprint, item]));
return alerts.map((alert) => {
const read = byFingerprint.get(alert.fingerprint);
const acknowledged = Boolean(read && read.activeAt.getTime() === Date.parse(alert.startedAt));
return { ...alert, acknowledged, acknowledgedAt: acknowledged ? read?.readAt.toISOString() : undefined };
});
}
private parseServiceMetrics(response: PrometheusQueryResponse): InfrastructureServiceMetricGroup[] { private parseServiceMetrics(response: PrometheusQueryResponse): InfrastructureServiceMetricGroup[] {
const values = new Map<string, number>(); const values = new Map<string, number>();
for (const item of response.data?.result ?? []) { for (const item of response.data?.result ?? []) {
+1
View File
@@ -1151,3 +1151,4 @@ global,不能错误归入client。`client-signature-*`、发送页、企业认
- `api/src/metrics/`只负责API进程和HTTP路由模板的低基数计数/直方图,不依赖Prisma、Redis或业务Service;独立回环监听由`api/src/main.ts`组装。 - `api/src/metrics/`只负责API进程和HTTP路由模板的低基数计数/直方图,不依赖Prisma、Redis或业务Service;独立回环监听由`api/src/main.ts`组装。
- `gateway/internal/metrics/`只负责线程安全聚合和Prometheus文本暴露;上游连接池、下游Session和Submit Worker仅提供总数或有界结果,不把实体ID或凭据交给监控模块。 - `gateway/internal/metrics/`只负责线程安全聚合和Prometheus文本暴露;上游连接池、下游Session和Submit Worker仅提供总数或有界结果,不把实体ID或凭据交给监控模块。
- `api/src/infrastructure-monitoring/`是运营端监控聚合与固定阈值应用边界:只消费代码白名单 PromQL,并通过版本化 PostgreSQL 单例、promtool 校验和原子规则热加载管理数值阈值;Exporter安装、端口隔离、固定规则模板和权限仍归`tools/monitoring/`治理。 - `api/src/infrastructure-monitoring/`是运营端监控聚合与固定阈值应用边界:只消费代码白名单 PromQL,并通过版本化 PostgreSQL 单例、promtool 校验和原子规则热加载管理数值阈值;Exporter安装、端口隔离、固定规则模板和权限仍归`tools/monitoring/`治理。
- 活动告警已读也归该边界:Prometheus保留告警事实,Prisma仅持久化逐管理员、逐触发周期的阅读状态;全局布局只消费轻量未读汇总,不复制指纹、activeAt或用户隔离逻辑。
@@ -2099,3 +2099,5 @@
- 系统监控标题说明必须明确标注数据来自 Prometheus;“服务关键指标”位于趋势/核心服务区域之后、活动告警之前,并提供统一的“告警阈值设置”入口。安全检测页不重复渲染大号标题,说明文字必须明确标注使用 Fail2ban。 - 系统监控标题说明必须明确标注数据来自 Prometheus;“服务关键指标”位于趋势/核心服务区域之后、活动告警之前,并提供统一的“告警阈值设置”入口。安全检测页不重复渲染大号标题,说明文字必须明确标注使用 Fail2ban。
- 告警阈值仅开放固定指标的警告/严重数值,不允许前端提交 PromQL、标签、文件路径或持续时间;必须满足警告值小于严重值。配置以 PostgreSQL 保存版本、期望值、生效值和应用状态,经 `promtool check rules` 校验、同目录原子替换和 Prometheus 热加载成功后才标记生效,失败保留上一生效规则并展示原因。 - 告警阈值仅开放固定指标的警告/严重数值,不允许前端提交 PromQL、标签、文件路径或持续时间;必须满足警告值小于严重值。配置以 PostgreSQL 保存版本、期望值、生效值和应用状态,经 `promtool check rules` 校验、同目录原子替换和 Prometheus 热加载成功后才标记生效,失败保留上一生效规则并展示原因。
- 右上角预警中心增加“系统监控告警”,通过独立轻量接口统计 Prometheus 当前 firing/pending 告警及严重数,跳转系统监控活动告警区;任一预警域失败不得清空其他域。 - 右上角预警中心增加“系统监控告警”,通过独立轻量接口统计 Prometheus 当前 firing/pending 告警及严重数,跳转系统监控活动告警区;任一预警域失败不得清空其他域。
- 活动告警列表必须提供逐条“标记已读”。已读状态按管理员和“告警指纹 + 本次 activeAt”持久化到 PostgreSQL;仅从当前管理员的预警中心数量中扣减,不改变 Prometheus firing/pending 状态,也不减少页面活动告警总数。同标签告警恢复后再次触发时必须重新成为未读。
- 服务端只能确认 Prometheus 当前仍存在且 activeAt 一致的告警,过期、已恢复或已重新触发的请求必须拒绝;重复点击同一次告警应幂等,并写操作日志。阈值设置弹窗只保留通用 Modal 外层滚动,不得嵌套第二个独立滚动区域。
@@ -243,3 +243,9 @@ type InfrastructureOverview = {
- 可配置范围固定为主机 CPU/内存/根磁盘、API 5xx/P95/事件循环、Gateway 最旧 pending、PostgreSQL 连接、Redis 内存和 MinIO 容量十组警告/严重数值。PromQL、持续窗口、标签和规则文件路径仍由代码固定,浏览器无权提交。 - 可配置范围固定为主机 CPU/内存/根磁盘、API 5xx/P95/事件循环、Gateway 最旧 pending、PostgreSQL 连接、Redis 内存和 MinIO 容量十组警告/严重数值。PromQL、持续窗口、标签和规则文件路径仍由代码固定,浏览器无权提交。
- PostgreSQL 单例记录同时保存期望阈值、生效阈值、配置版本、生效版本和 `applying/effective/failed` 状态。更新使用版本条件认领,防止多个 API 实例并发覆盖;规则先经 promtool 校验,再在同一目录原子替换并调用仅回环开放的 `/-/reload`。失败恢复旧文件并保留旧生效版本。 - PostgreSQL 单例记录同时保存期望阈值、生效阈值、配置版本、生效版本和 `applying/effective/failed` 状态。更新使用版本条件认领,防止多个 API 实例并发覆盖;规则先经 promtool 校验,再在同一目录原子替换并调用仅回环开放的 `/-/reload`。失败恢复旧文件并保留旧生效版本。
- 安装器把可配置规则从基础规则中剥离,托管文件归 `cmpp-api:prometheus` 且权限为 0640Prometheus 仍只监听回环。右上角铃铛只轮询轻量活动告警汇总接口,不重复加载趋势或服务指标。 - 安装器把可配置规则从基础规则中剥离,托管文件归 `cmpp-api:prometheus` 且权限为 0640Prometheus 仍只监听回环。右上角铃铛只轮询轻量活动告警汇总接口,不重复加载趋势或服务指标。
## 11. 活动告警已读语义(2026-08-16 增补)
- “已读”只表示某位管理员已查看某一次 Prometheus 活动告警,不是 resolve、silence 或 acknowledge 外部告警管理器;页面活动告警总数与平台健康状态仍按 Prometheus 原始 firing/pending 计算。
- 指纹由排序后的 Prometheus labels 稳定生成,`activeAt`区分同一指纹的不同触发周期。数据库以`fingerprint + userId`唯一,upsert同时更新`activeAt/readAt`;读取时只有数据库 activeAt 与当前 Prometheus activeAt 相同才算已读。
- 标记前必须回读当前 Prometheus 告警并校验指纹和 activeAt,防止客户端伪造或把已经恢复的新周期误标已读。预警中心轻量汇总只扣减当前管理员本次已读项;数据库故障不得用 localStorage 或静态状态替代。
+5
View File
@@ -4688,3 +4688,8 @@ npm run verify:phase8
| TC-INFRA-MON-034 | 应用失败回滚 | 令 promtool 或 reload 失败后保存 | 状态为 failed、展示原因,旧规则文件与旧生效阈值保留,不误报已生效 | | TC-INFRA-MON-034 | 应用失败回滚 | 令 promtool 或 reload 失败后保存 | 状态为 failed、展示原因,旧规则文件与旧生效阈值保留,不误报已生效 |
| TC-GLOBAL-ALERT-004 | 系统监控预警入口 | 准备隔离 QA Prometheus firing 告警并点击铃铛 | 第三项显示真实总数/严重数,角标计入三域总和,点击跳转系统监控活动告警区 | | TC-GLOBAL-ALERT-004 | 系统监控预警入口 | 准备隔离 QA Prometheus firing 告警并点击铃铛 | 第三项显示真实总数/严重数,角标计入三域总和,点击跳转系统监控活动告警区 |
| TC-SECURITY-UI-001 | Fail2ban 标识与标题规范 | 打开安全检测与封禁 | 不出现重复大号页面标题,说明明确写明使用 Fail2ban,字号遵循通用菜单标题 | | TC-SECURITY-UI-001 | Fail2ban 标识与标题规范 | 打开安全检测与封禁 | 不出现重复大号页面标题,说明明确写明使用 Fail2ban,字号遵循通用菜单标题 |
| TC-INFRA-MON-035 | 阈值弹窗单层滚动 | 在桌面和窄屏打开阈值设置,滚动到最后一组阈值 | 只有 Modal 外层内容区出现滚动条,阈值表单容器不产生第二层滚动或滚动陷阱,页头和页脚行为正常 |
| TC-INFRA-MON-036 | 活动告警逐条已读 | 使用管理员A点击一条当前活动告警的“标记已读” | PostgreSQL新增/更新管理员A与该次 activeAt 的记录;行显示“已读”,活动告警总数不变,铃铛系统监控数量减少1 |
| TC-INFRA-MON-037 | 已读用户隔离 | 管理员A标记已读后由管理员B查看同一告警 | 管理员B仍显示未读且铃铛数量不减少,管理员A的状态保持已读 |
| TC-INFRA-MON-038 | 同告警重新触发 | 标记已读后让告警恢复,再以相同标签重新触发并产生新 activeAt | 新触发记录重新显示“标记已读”,计入预警中心;旧 activeAt 不会永久屏蔽同指纹告警 |
| TC-INFRA-MON-039 | 过期与幂等 | 重复提交同一活动告警,再提交已恢复或 activeAt 不匹配的请求 | 同一次告警重复提交幂等;过期/不匹配请求返回404且不生成虚假已读记录;操作日志可追溯 |
+6
View File
@@ -3675,3 +3675,9 @@ git diff --check
- 恢复资产分别为 `/opt/cmpp-platform-backups/releases/20260814-175325-before-6ccc102``/opt/cmpp-platform-backups/releases/20260814-175946-before-0cd0944``/opt/cmpp-platform-backups/releases/20260814-180307-before-2216d00`,均包含 PostgreSQL、源码、环境及监控/Nginx/systemd 配置并通过 SHA-256、pg_restore 与 gzip 校验。 - 恢复资产分别为 `/opt/cmpp-platform-backups/releases/20260814-175325-before-6ccc102``/opt/cmpp-platform-backups/releases/20260814-175946-before-0cd0944``/opt/cmpp-platform-backups/releases/20260814-180307-before-2216d00`,均包含 PostgreSQL、源码、环境及监控/Nginx/systemd 配置并通过 SHA-256、pg_restore 与 gzip 校验。
- 本机监控专项 2 套 6 项、前后端 TypeScript、API 正式编译、Vite 生产构建和 git diff check 通过;API 全量回归两次分别在 120 秒和 300 秒到达执行时限,未取得完整通过证据,因此不记为通过。外部 Edge 当前停留测试机登录页,因没有现成登录会话且页面含验证码,本轮未代填或绕过验证码,登录后视觉验收由用户直接查看。 - 本机监控专项 2 套 6 项、前后端 TypeScript、API 正式编译、Vite 生产构建和 git diff check 通过;API 全量回归两次分别在 120 秒和 300 秒到达执行时限,未取得完整通过证据,因此不记为通过。外部 Edge 当前停留测试机登录页,因没有现成登录会话且页面含验证码,本轮未代填或绕过验证码,登录后视觉验收由用户直接查看。
- 本轮没有发送、补发或重投短信,没有创建后台重投任务,没有修改通道账号、密码、启停状态、企业余额或客户连接;QA 数据仅为测试机 Prometheus 演示规则。 - 本轮没有发送、补发或重投短信,没有创建后台重投任务,没有修改通道账号、密码、启停状态、企业余额或客户连接;QA 数据仅为测试机 Prometheus 演示规则。
# 2026-08-16 系统监控弹窗滚动与活动告警已读(进行中)
- 阈值设置表单移除自身 `max-height/overflow`,仅保留平台通用 Modal 内容区滚动,消除双层滚动区域。
- 新增逐管理员活动告警已读设计:真实 PostgreSQL 保存 `fingerprint + userId + activeAt + readAt`;服务端回读 Prometheus 校验当前触发周期后幂等 upsert。页面活动告警保持原始数量,铃铛轻量汇总只统计当前管理员未读,告警以相同标签重新触发但 activeAt 改变时重新计入未读。
- 计划新增 migration `20260816100000_add_infrastructure_alert_reads`、单条已读接口、操作日志及前端按钮。专项测试、真实数据库、构建、测试机恢复资产与部署证据待本轮完成后补记。
@@ -7,4 +7,5 @@ export const adminInfrastructureMonitoringApi = {
getInfrastructureMonitoringNotificationSummary: () => request<{ count: number; criticalCount: number }>('/admin/infrastructure-monitoring/notification-summary'), getInfrastructureMonitoringNotificationSummary: () => request<{ count: number; criticalCount: number }>('/admin/infrastructure-monitoring/notification-summary'),
getInfrastructureAlertThresholds: () => request<InfrastructureAlertSettings>('/admin/infrastructure-monitoring/alert-thresholds'), getInfrastructureAlertThresholds: () => request<InfrastructureAlertSettings>('/admin/infrastructure-monitoring/alert-thresholds'),
updateInfrastructureAlertThresholds: (body: { configVersion: number; thresholds: InfrastructureAlertThresholds }) => request<InfrastructureAlertSettings>('/admin/infrastructure-monitoring/alert-thresholds', { method: 'PUT', body: JSON.stringify(body) }), updateInfrastructureAlertThresholds: (body: { configVersion: number; thresholds: InfrastructureAlertThresholds }) => request<InfrastructureAlertSettings>('/admin/infrastructure-monitoring/alert-thresholds', { method: 'PUT', body: JSON.stringify(body) }),
markInfrastructureAlertRead: (fingerprint: string, activeAt: string) => request<{ fingerprint: string; activeAt: string; acknowledged: true; acknowledgedAt: string }>(`/admin/infrastructure-monitoring/alerts/${fingerprint}/read`, { method: 'POST', body: JSON.stringify({ activeAt }) }),
}; };
@@ -36,6 +36,8 @@ export type InfrastructureAlert = {
threshold?: string; threshold?: string;
service?: string; service?: string;
instance?: string; instance?: string;
acknowledged: boolean;
acknowledgedAt?: string;
}; };
export type InfrastructureMonitoringOverview = { export type InfrastructureMonitoringOverview = {
@@ -323,7 +323,7 @@
} }
.system-monitoring-service-actions { align-items: center; display: flex; gap: 12px; } .system-monitoring-service-actions { align-items: center; display: flex; gap: 12px; }
.system-monitoring-threshold-dialog { display: grid; gap: 12px; max-height: 62vh; overflow: auto; padding-right: 4px; } .system-monitoring-threshold-dialog { display: grid; gap: 12px; }
.system-monitoring-threshold-note { align-items: flex-start; background: #eff6ff; border-radius: 10px; color: #475569; display: flex; font-size: 13px; gap: 9px; padding: 12px 14px; } .system-monitoring-threshold-note { align-items: flex-start; background: #eff6ff; border-radius: 10px; color: #475569; display: flex; font-size: 13px; gap: 9px; padding: 12px 14px; }
.system-monitoring-threshold-row { align-items: end; border-bottom: 1px solid #eef2f7; display: grid; gap: 14px; grid-template-columns: minmax(180px, 1fr) 150px 150px; padding: 12px 0; } .system-monitoring-threshold-row { align-items: end; border-bottom: 1px solid #eef2f7; display: grid; gap: 14px; grid-template-columns: minmax(180px, 1fr) 150px 150px; padding: 12px 0; }
.system-monitoring-threshold-row > div:first-child { align-self: center; display: grid; gap: 4px; } .system-monitoring-threshold-row > div:first-child { align-self: center; display: grid; gap: 4px; }
@@ -151,7 +151,7 @@ function severityTag(severity: InfrastructureAlert['severity']) {
return <Tag tone="info"></Tag>; return <Tag tone="info"></Tag>;
} }
const alertColumns: Array<TableColumn<InfrastructureAlert>> = [ function makeAlertColumns(onMarkRead: (alert: InfrastructureAlert) => void, readingFingerprint: string): Array<TableColumn<InfrastructureAlert>> { return [
{ key: 'severity', title: '级别', width: '82px', render: (record) => severityTag(record.severity) }, { key: 'severity', title: '级别', width: '82px', render: (record) => severityTag(record.severity) },
{ {
key: 'alert', title: '告警', width: '280px', render: (record) => ( key: 'alert', title: '告警', width: '280px', render: (record) => (
@@ -162,7 +162,12 @@ const alertColumns: Array<TableColumn<InfrastructureAlert>> = [
{ key: 'value', title: '当前值 / 阈值', width: '150px', render: (record) => `${record.currentValue || '—'} / ${record.threshold || '—'}` }, { key: 'value', title: '当前值 / 阈值', width: '150px', render: (record) => `${record.currentValue || '—'} / ${record.threshold || '—'}` },
{ key: 'startedAt', title: '开始时间', width: '150px', render: (record) => formatTime(record.startedAt) }, { key: 'startedAt', title: '开始时间', width: '150px', render: (record) => formatTime(record.startedAt) },
{ key: 'duration', title: '持续时间', width: '120px', render: (record) => formatDuration(record.startedAt) }, { key: 'duration', title: '持续时间', width: '120px', render: (record) => formatDuration(record.startedAt) },
]; {
key: 'actions', title: '操作', width: '112px', render: (record) => record.acknowledged
? <Tag tone="neutral"></Tag>
: <Button disabled={readingFingerprint === record.fingerprint} icon={<CheckCircle2 size={14} />} onClick={() => onMarkRead(record)} size="sm" variant="ghost">{readingFingerprint === record.fingerprint ? '处理中' : '标记已读'}</Button>,
},
]; }
export function AdminSystemMonitoringPage() { export function AdminSystemMonitoringPage() {
const [range, setRange] = useState<InfrastructureMonitoringRange>('24h'); const [range, setRange] = useState<InfrastructureMonitoringRange>('24h');
@@ -174,6 +179,8 @@ export function AdminSystemMonitoringPage() {
const [showSettings, setShowSettings] = useState(false); const [showSettings, setShowSettings] = useState(false);
const [settingsError, setSettingsError] = useState(''); const [settingsError, setSettingsError] = useState('');
const [savingSettings, setSavingSettings] = useState(false); const [savingSettings, setSavingSettings] = useState(false);
const [readingFingerprint, setReadingFingerprint] = useState('');
const [readError, setReadError] = useState('');
const requestSequence = useRef(0); const requestSequence = useRef(0);
const pendingRequests = useRef(0); const pendingRequests = useRef(0);
@@ -226,6 +233,25 @@ export function AdminSystemMonitoringPage() {
} }
}, [draftThresholds, loadData, settings]); }, [draftThresholds, loadData, settings]);
const markAlertRead = useCallback(async (alert: InfrastructureAlert) => {
setReadingFingerprint(alert.fingerprint);
setReadError('');
try {
const result = await adminApi.markInfrastructureAlertRead(alert.fingerprint, alert.startedAt);
setOverview((current) => current ? {
...current,
alerts: current.alerts.map((item) => item.fingerprint === result.fingerprint && Date.parse(item.startedAt) === Date.parse(result.activeAt)
? { ...item, acknowledged: true, acknowledgedAt: result.acknowledgedAt }
: item),
} : current);
window.dispatchEvent(new Event('cmpp-infrastructure-alert-count-refresh'));
} catch (reason) {
setReadError(reason instanceof Error ? reason.message : '活动告警标记已读失败');
} finally {
setReadingFingerprint('');
}
}, []);
useEffect(() => { useEffect(() => {
void loadData(true); void loadData(true);
void loadSettings(); void loadSettings();
@@ -263,6 +289,7 @@ export function AdminSystemMonitoringPage() {
const metrics = overview?.metrics; const metrics = overview?.metrics;
const serviceHealthy = overview?.summary.serviceHealthy ?? 0; const serviceHealthy = overview?.summary.serviceHealthy ?? 0;
const serviceTotal = overview?.summary.serviceTotal ?? 6; const serviceTotal = overview?.summary.serviceTotal ?? 6;
const alertColumns = useMemo(() => makeAlertColumns((alert) => { void markAlertRead(alert); }, readingFingerprint), [markAlertRead, readingFingerprint]);
return ( return (
<section className="page-stack admin-system-monitoring-page"> <section className="page-stack admin-system-monitoring-page">
@@ -364,6 +391,7 @@ export function AdminSystemMonitoringPage() {
<section className="surface system-monitoring-alerts" id="active-alerts"> <section className="surface system-monitoring-alerts" id="active-alerts">
<header><div><AlertTriangle size={18} /><strong></strong><Tag tone={overview?.summary.activeAlerts ? 'warning' : 'success'}>{overview?.summary.activeAlerts ?? 0}</Tag></div><span><Clock3 size={14} /> {formatTime(overview?.collectedAt ?? null)}</span></header> <header><div><AlertTriangle size={18} /><strong></strong><Tag tone={overview?.summary.activeAlerts ? 'warning' : 'success'}>{overview?.summary.activeAlerts ?? 0}</Tag></div><span><Clock3 size={14} /> {formatTime(overview?.collectedAt ?? null)}</span></header>
{readError ? <div className="system-monitoring-unavailable" role="alert"><AlertTriangle size={18} /><div><strong></strong><span>{readError}</span></div></div> : null}
<Table columns={alertColumns} data={overview?.alerts ?? []} emptyText={overview?.available ? '当前没有活动告警' : '监控不可用,无法读取活动告警'} pagination={false} rowKey="fingerprint" /> <Table columns={alertColumns} data={overview?.alerts ?? []} emptyText={overview?.available ? '当前没有活动告警' : '监控不可用,无法读取活动告警'} pagination={false} rowKey="fingerprint" />
</section> </section>