feat: add monitoring alert read state

This commit is contained in:
hectorzhao
2026-08-16 15:04:26 +08:00
parent 79f5d3f215
commit 482f7ac1ae
15 changed files with 196 additions and 21 deletions
@@ -36,6 +36,8 @@ export type InfrastructureAlert = {
threshold?: string;
service?: string;
instance?: string;
acknowledged: boolean;
acknowledgedAt?: string;
};
export type InfrastructureMonitoringOverview = {
@@ -1,4 +1,4 @@
import { Body, Controller, Get, Put, Query } from '@nestjs/common';
import { Body, Controller, Get, Param, Post, Put, Query } from '@nestjs/common';
import { ApiTags } from '@nestjs/swagger';
import { CurrentSessionUserId } from '../auth/current-session-user.decorator';
import { RequireRecentAuthentication } from '../auth/require-recent-authentication.decorator';
@@ -11,12 +11,17 @@ export class InfrastructureMonitoringController {
constructor(private readonly monitoring: InfrastructureMonitoringService, private readonly settings: InfrastructureAlertSettingsService) {}
@Get('overview')
overview(@Query('range') range?: string) {
return this.monitoring.overview(range);
overview(@Query('range') range?: string, @CurrentSessionUserId() userId?: string) {
return this.monitoring.overview(range, userId);
}
@Get('notification-summary')
notificationSummary() { return this.monitoring.notificationSummary(); }
notificationSummary(@CurrentSessionUserId() userId?: string) { return this.monitoring.notificationSummary(userId); }
@Post('alerts/:fingerprint/read')
markAlertRead(@Param('fingerprint') fingerprint: string, @Body('activeAt') activeAt: unknown, @CurrentSessionUserId() userId: string) {
return this.monitoring.markAlertRead(fingerprint, activeAt, userId);
}
@Get('alert-thresholds')
alertThresholds() { return this.settings.get(); }
@@ -1,5 +1,6 @@
import { BadRequestException } from '@nestjs/common';
import { ConfigService } from '@nestjs/config';
import { createHash } from 'node:crypto';
import { InfrastructureMonitoringService } from './infrastructure-monitoring.service';
function success(data: unknown) {
@@ -11,20 +12,30 @@ function success(data: unknown) {
}
describe('InfrastructureMonitoringService', () => {
afterEach(() => jest.restoreAllMocks());
const prisma = {
infrastructureAlertRead: { findMany: jest.fn().mockResolvedValue([]), create: jest.fn(), update: jest.fn(), findUniqueOrThrow: jest.fn() },
operationLog: { create: jest.fn() },
$transaction: jest.fn(),
};
afterEach(() => {
jest.restoreAllMocks();
jest.clearAllMocks();
prisma.infrastructureAlertRead.findMany.mockResolvedValue([]);
});
it('rejects ranges outside the fixed whitelist before querying Prometheus', async () => {
const fetchSpy = jest.spyOn(global, 'fetch');
const service = new InfrastructureMonitoringService(new ConfigService());
const service = new InfrastructureMonitoringService(new ConfigService(), prisma as never);
await expect(service.overview('30d')).rejects.toBeInstanceOf(BadRequestException);
expect(fetchSpy).not.toHaveBeenCalled();
});
it('rejects credential-bearing or remote plaintext Prometheus endpoints at startup', () => {
expect(() => new InfrastructureMonitoringService(new ConfigService({ PROMETHEUS_URL: 'http://user:secret@127.0.0.1:9090' }))).toThrow('must not contain credentials');
expect(() => new InfrastructureMonitoringService(new ConfigService({ PROMETHEUS_URL: 'http://monitor.example.com:9090' }))).toThrow('must use HTTPS');
expect(() => new InfrastructureMonitoringService(new ConfigService({ PROMETHEUS_URL: 'https://monitor.example.com' }))).not.toThrow();
expect(() => new InfrastructureMonitoringService(new ConfigService({ PROMETHEUS_URL: 'http://user:secret@127.0.0.1:9090' }), prisma as never)).toThrow('must not contain credentials');
expect(() => new InfrastructureMonitoringService(new ConfigService({ PROMETHEUS_URL: 'http://monitor.example.com:9090' }), prisma as never)).toThrow('must use HTTPS');
expect(() => new InfrastructureMonitoringService(new ConfigService({ PROMETHEUS_URL: 'https://monitor.example.com' }), prisma as never)).not.toThrow();
});
it('loads real Prometheus vectors, ranges, services and active alerts', async () => {
@@ -68,7 +79,7 @@ describe('InfrastructureMonitoringService', () => {
return success({ result: [{ metric: {}, value: [1_765_000_060, '25'] }] });
});
const service = new InfrastructureMonitoringService(new ConfigService());
const service = new InfrastructureMonitoringService(new ConfigService(), prisma as never);
const result = await service.overview('1h');
expect(result.available).toBe(true);
@@ -87,7 +98,7 @@ describe('InfrastructureMonitoringService', () => {
it('returns an explicit unavailable payload without stale metrics when Prometheus fails', async () => {
jest.spyOn(global, 'fetch').mockRejectedValue(new Error('ECONNREFUSED'));
const service = new InfrastructureMonitoringService(new ConfigService());
const service = new InfrastructureMonitoringService(new ConfigService(), prisma as never);
const result = await service.overview('24h');
@@ -98,4 +109,29 @@ describe('InfrastructureMonitoringService', () => {
expect(result.serviceMetrics.every((item) => item.available === false)).toBe(true);
expect(result.error).not.toContain('ECONNREFUSED');
});
it('excludes only the current alert occurrence after the current administrator marks it read', async () => {
const labels = { alertname: 'QaWarning', severity: 'warning', service: 'qa-preview' };
const fingerprint = createHash('sha256').update(JSON.stringify(Object.entries(labels).sort(([left], [right]) => left.localeCompare(right)))).digest('hex').slice(0, 24);
jest.spyOn(global, 'fetch').mockResolvedValue(success({ alerts: [{ labels, annotations: { summary: '演示预警' }, state: 'firing', activeAt: '2026-08-16T01:00:00.000Z' }] }));
prisma.infrastructureAlertRead.findMany.mockResolvedValue([{ fingerprint, activeAt: new Date('2026-08-16T01:00:00.000Z'), readAt: new Date('2026-08-16T01:01:00.000Z') }]);
const service = new InfrastructureMonitoringService(new ConfigService(), prisma as never);
await expect(service.notificationSummary('admin-1')).resolves.toEqual({ count: 0, criticalCount: 0 });
prisma.infrastructureAlertRead.findMany.mockResolvedValue([{ fingerprint, activeAt: new Date('2026-08-15T01:00:00.000Z'), readAt: new Date('2026-08-15T01:01:00.000Z') }]);
await expect(service.notificationSummary('admin-1')).resolves.toEqual({ count: 1, criticalCount: 0 });
});
it('upserts an idempotent per-user read record only for a currently active occurrence', async () => {
const labels = { alertname: 'QaCritical', severity: 'critical', service: 'qa-preview' };
const fingerprint = createHash('sha256').update(JSON.stringify(Object.entries(labels).sort(([left], [right]) => left.localeCompare(right)))).digest('hex').slice(0, 24);
jest.spyOn(global, 'fetch').mockResolvedValue(success({ alerts: [{ labels, annotations: { summary: '演示严重告警' }, state: 'firing', activeAt: '2026-08-16T02:00:00.000Z' }] }));
prisma.$transaction.mockResolvedValue([{ activeAt: new Date('2026-08-16T02:00:00.000Z'), readAt: new Date('2026-08-16T02:01:00.000Z') }, {}]);
const service = new InfrastructureMonitoringService(new ConfigService(), prisma as never);
await expect(service.markAlertRead(fingerprint, '2026-08-16T02:00:00.000Z', 'admin-1')).resolves.toMatchObject({ fingerprint, acknowledged: true });
expect(prisma.infrastructureAlertRead.create).toHaveBeenCalledWith(expect.objectContaining({ data: expect.objectContaining({ fingerprint, userId: 'admin-1' }) }));
await expect(service.markAlertRead(fingerprint, '2026-08-15T02:00:00.000Z', 'admin-1')).rejects.toThrow('已结束或已重新触发');
});
});
@@ -1,6 +1,8 @@
import { BadRequestException, Injectable, Logger, ServiceUnavailableException } from '@nestjs/common';
import { BadRequestException, Injectable, Logger, NotFoundException, ServiceUnavailableException } from '@nestjs/common';
import { ConfigService } from '@nestjs/config';
import { Prisma } from '@prisma/client';
import { createHash } from 'node:crypto';
import { PrismaService } from '../prisma/prisma.service';
import type {
InfrastructureAlert,
InfrastructureMetricPoint,
@@ -162,12 +164,12 @@ export class InfrastructureMonitoringService {
private readonly prometheusUrl: string;
private readonly queryTimeoutMs: number;
constructor(config: ConfigService) {
constructor(config: ConfigService, private readonly prisma: PrismaService) {
this.prometheusUrl = normalizePrometheusUrl(config.get('PROMETHEUS_URL'));
this.queryTimeoutMs = Math.min(15_000, Math.max(1_000, Number(config.get('PROMETHEUS_QUERY_TIMEOUT_MS') ?? 5_000)));
}
async overview(rawRange?: string): Promise<InfrastructureMonitoringOverview> {
async overview(rawRange?: string, userId?: string): Promise<InfrastructureMonitoringOverview> {
const range = this.parseRange(rawRange);
const collectedAt = new Date().toISOString();
try {
@@ -180,7 +182,7 @@ export class InfrastructureMonitoringService {
]);
const services = this.parseServices(serviceResponse);
const serviceMetrics = this.parseServiceMetrics(serviceMetricResponse);
const alerts = this.parseAlerts(alertResponse);
const alerts = await this.attachReadState(this.parseAlerts(alertResponse), userId);
const warningAlerts = alerts.filter((item) => item.severity === 'warning').length;
const criticalAlerts = alerts.filter((item) => item.severity === 'critical').length;
const overallStatus = criticalAlerts > 0 ? 'critical' : warningAlerts > 0 ? 'warning' : 'healthy';
@@ -210,16 +212,47 @@ export class InfrastructureMonitoringService {
}
}
async notificationSummary() {
async notificationSummary(userId?: string) {
try {
const alerts = this.parseAlerts(await this.getJson<PrometheusAlertResponse>('/api/v1/alerts'));
return { count: alerts.length, criticalCount: alerts.filter((item) => item.severity === 'critical').length };
const alerts = await this.attachReadState(this.parseAlerts(await this.getJson<PrometheusAlertResponse>('/api/v1/alerts')), userId);
const unreadAlerts = alerts.filter((item) => !item.acknowledged);
return { count: unreadAlerts.length, criticalCount: unreadAlerts.filter((item) => item.severity === 'critical').length };
} catch (error) {
this.logger.warn(`Prometheus notification summary unavailable: ${error instanceof Error ? error.message : 'unknown error'}`);
throw new ServiceUnavailableException('Prometheus活动告警当前不可用');
}
}
async markAlertRead(fingerprint: string, rawActiveAt: unknown, userId: string) {
if (!/^[a-f0-9]{24}$/.test(fingerprint)) throw new BadRequestException('告警指纹无效');
const activeAt = new Date(String(rawActiveAt ?? ''));
if (!Number.isFinite(activeAt.getTime())) throw new BadRequestException('告警开始时间无效');
const activeAlerts = this.parseAlerts(await this.getJson<PrometheusAlertResponse>('/api/v1/alerts'));
const current = activeAlerts.find((item) => item.fingerprint === fingerprint && Date.parse(item.startedAt) === activeAt.getTime());
if (!current) throw new NotFoundException('该次活动告警已结束或已重新触发,请刷新后重试');
const readAt = new Date();
const log = () => this.prisma.operationLog.create({
data: { userId, action: 'monitoring.alert_marked_read', resource: 'infrastructure_alert', resourceId: fingerprint, detail: { activeAt: activeAt.toISOString(), alertName: current.name, severity: current.severity } },
});
let read;
try {
[read] = await this.prisma.$transaction([
this.prisma.infrastructureAlertRead.create({ data: { fingerprint, activeAt, userId, readAt } }),
log(),
]);
} catch (error) {
if (!(error instanceof Prisma.PrismaClientKnownRequestError) || error.code !== 'P2002') throw error;
const existing = await this.prisma.infrastructureAlertRead.findUniqueOrThrow({ where: { fingerprint_userId: { fingerprint, userId } } });
// 同一次触发重复点击不更新readAt也不重复写日志;activeAt变化才代表同指纹的新触发周期。
if (existing.activeAt.getTime() === activeAt.getTime()) read = existing;
else [read] = await this.prisma.$transaction([
this.prisma.infrastructureAlertRead.update({ where: { fingerprint_userId: { fingerprint, userId } }, data: { activeAt, readAt } }),
log(),
]);
}
return { fingerprint, activeAt: read.activeAt.toISOString(), acknowledged: true, acknowledgedAt: read.readAt.toISOString() };
}
private parseRange(value?: string): InfrastructureMonitoringRange {
const range = value || '24h';
if (!(range in RANGE_CONFIG)) throw new BadRequestException('监控时间范围只支持1h、24h或7d');
@@ -275,6 +308,7 @@ export class InfrastructureMonitoringService {
threshold: annotations.threshold,
service: labels.service,
instance: labels.instance,
acknowledged: false,
};
})
.sort((left, right) => {
@@ -283,6 +317,20 @@ export class InfrastructureMonitoringService {
});
}
private async attachReadState(alerts: InfrastructureAlert[], userId?: string) {
if (!userId || alerts.length === 0) return alerts;
const reads = await this.prisma.infrastructureAlertRead.findMany({
where: { userId, fingerprint: { in: alerts.map((item) => item.fingerprint) } },
select: { fingerprint: true, activeAt: true, readAt: true },
});
const byFingerprint = new Map(reads.map((item) => [item.fingerprint, item]));
return alerts.map((alert) => {
const read = byFingerprint.get(alert.fingerprint);
const acknowledged = Boolean(read && read.activeAt.getTime() === Date.parse(alert.startedAt));
return { ...alert, acknowledged, acknowledgedAt: acknowledged ? read?.readAt.toISOString() : undefined };
});
}
private parseServiceMetrics(response: PrometheusQueryResponse): InfrastructureServiceMetricGroup[] {
const values = new Map<string, number>();
for (const item of response.data?.result ?? []) {