feat: add monitoring alert read state
This commit is contained in:
@@ -36,6 +36,8 @@ export type InfrastructureAlert = {
|
||||
threshold?: string;
|
||||
service?: string;
|
||||
instance?: string;
|
||||
acknowledged: boolean;
|
||||
acknowledgedAt?: string;
|
||||
};
|
||||
|
||||
export type InfrastructureMonitoringOverview = {
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { Body, Controller, Get, Put, Query } from '@nestjs/common';
|
||||
import { Body, Controller, Get, Param, Post, Put, Query } from '@nestjs/common';
|
||||
import { ApiTags } from '@nestjs/swagger';
|
||||
import { CurrentSessionUserId } from '../auth/current-session-user.decorator';
|
||||
import { RequireRecentAuthentication } from '../auth/require-recent-authentication.decorator';
|
||||
@@ -11,12 +11,17 @@ export class InfrastructureMonitoringController {
|
||||
constructor(private readonly monitoring: InfrastructureMonitoringService, private readonly settings: InfrastructureAlertSettingsService) {}
|
||||
|
||||
@Get('overview')
|
||||
overview(@Query('range') range?: string) {
|
||||
return this.monitoring.overview(range);
|
||||
overview(@Query('range') range?: string, @CurrentSessionUserId() userId?: string) {
|
||||
return this.monitoring.overview(range, userId);
|
||||
}
|
||||
|
||||
@Get('notification-summary')
|
||||
notificationSummary() { return this.monitoring.notificationSummary(); }
|
||||
notificationSummary(@CurrentSessionUserId() userId?: string) { return this.monitoring.notificationSummary(userId); }
|
||||
|
||||
@Post('alerts/:fingerprint/read')
|
||||
markAlertRead(@Param('fingerprint') fingerprint: string, @Body('activeAt') activeAt: unknown, @CurrentSessionUserId() userId: string) {
|
||||
return this.monitoring.markAlertRead(fingerprint, activeAt, userId);
|
||||
}
|
||||
|
||||
@Get('alert-thresholds')
|
||||
alertThresholds() { return this.settings.get(); }
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import { BadRequestException } from '@nestjs/common';
|
||||
import { ConfigService } from '@nestjs/config';
|
||||
import { createHash } from 'node:crypto';
|
||||
import { InfrastructureMonitoringService } from './infrastructure-monitoring.service';
|
||||
|
||||
function success(data: unknown) {
|
||||
@@ -11,20 +12,30 @@ function success(data: unknown) {
|
||||
}
|
||||
|
||||
describe('InfrastructureMonitoringService', () => {
|
||||
afterEach(() => jest.restoreAllMocks());
|
||||
const prisma = {
|
||||
infrastructureAlertRead: { findMany: jest.fn().mockResolvedValue([]), create: jest.fn(), update: jest.fn(), findUniqueOrThrow: jest.fn() },
|
||||
operationLog: { create: jest.fn() },
|
||||
$transaction: jest.fn(),
|
||||
};
|
||||
|
||||
afterEach(() => {
|
||||
jest.restoreAllMocks();
|
||||
jest.clearAllMocks();
|
||||
prisma.infrastructureAlertRead.findMany.mockResolvedValue([]);
|
||||
});
|
||||
|
||||
it('rejects ranges outside the fixed whitelist before querying Prometheus', async () => {
|
||||
const fetchSpy = jest.spyOn(global, 'fetch');
|
||||
const service = new InfrastructureMonitoringService(new ConfigService());
|
||||
const service = new InfrastructureMonitoringService(new ConfigService(), prisma as never);
|
||||
|
||||
await expect(service.overview('30d')).rejects.toBeInstanceOf(BadRequestException);
|
||||
expect(fetchSpy).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('rejects credential-bearing or remote plaintext Prometheus endpoints at startup', () => {
|
||||
expect(() => new InfrastructureMonitoringService(new ConfigService({ PROMETHEUS_URL: 'http://user:secret@127.0.0.1:9090' }))).toThrow('must not contain credentials');
|
||||
expect(() => new InfrastructureMonitoringService(new ConfigService({ PROMETHEUS_URL: 'http://monitor.example.com:9090' }))).toThrow('must use HTTPS');
|
||||
expect(() => new InfrastructureMonitoringService(new ConfigService({ PROMETHEUS_URL: 'https://monitor.example.com' }))).not.toThrow();
|
||||
expect(() => new InfrastructureMonitoringService(new ConfigService({ PROMETHEUS_URL: 'http://user:secret@127.0.0.1:9090' }), prisma as never)).toThrow('must not contain credentials');
|
||||
expect(() => new InfrastructureMonitoringService(new ConfigService({ PROMETHEUS_URL: 'http://monitor.example.com:9090' }), prisma as never)).toThrow('must use HTTPS');
|
||||
expect(() => new InfrastructureMonitoringService(new ConfigService({ PROMETHEUS_URL: 'https://monitor.example.com' }), prisma as never)).not.toThrow();
|
||||
});
|
||||
|
||||
it('loads real Prometheus vectors, ranges, services and active alerts', async () => {
|
||||
@@ -68,7 +79,7 @@ describe('InfrastructureMonitoringService', () => {
|
||||
return success({ result: [{ metric: {}, value: [1_765_000_060, '25'] }] });
|
||||
});
|
||||
|
||||
const service = new InfrastructureMonitoringService(new ConfigService());
|
||||
const service = new InfrastructureMonitoringService(new ConfigService(), prisma as never);
|
||||
const result = await service.overview('1h');
|
||||
|
||||
expect(result.available).toBe(true);
|
||||
@@ -87,7 +98,7 @@ describe('InfrastructureMonitoringService', () => {
|
||||
|
||||
it('returns an explicit unavailable payload without stale metrics when Prometheus fails', async () => {
|
||||
jest.spyOn(global, 'fetch').mockRejectedValue(new Error('ECONNREFUSED'));
|
||||
const service = new InfrastructureMonitoringService(new ConfigService());
|
||||
const service = new InfrastructureMonitoringService(new ConfigService(), prisma as never);
|
||||
|
||||
const result = await service.overview('24h');
|
||||
|
||||
@@ -98,4 +109,29 @@ describe('InfrastructureMonitoringService', () => {
|
||||
expect(result.serviceMetrics.every((item) => item.available === false)).toBe(true);
|
||||
expect(result.error).not.toContain('ECONNREFUSED');
|
||||
});
|
||||
|
||||
it('excludes only the current alert occurrence after the current administrator marks it read', async () => {
|
||||
const labels = { alertname: 'QaWarning', severity: 'warning', service: 'qa-preview' };
|
||||
const fingerprint = createHash('sha256').update(JSON.stringify(Object.entries(labels).sort(([left], [right]) => left.localeCompare(right)))).digest('hex').slice(0, 24);
|
||||
jest.spyOn(global, 'fetch').mockResolvedValue(success({ alerts: [{ labels, annotations: { summary: '演示预警' }, state: 'firing', activeAt: '2026-08-16T01:00:00.000Z' }] }));
|
||||
prisma.infrastructureAlertRead.findMany.mockResolvedValue([{ fingerprint, activeAt: new Date('2026-08-16T01:00:00.000Z'), readAt: new Date('2026-08-16T01:01:00.000Z') }]);
|
||||
const service = new InfrastructureMonitoringService(new ConfigService(), prisma as never);
|
||||
|
||||
await expect(service.notificationSummary('admin-1')).resolves.toEqual({ count: 0, criticalCount: 0 });
|
||||
|
||||
prisma.infrastructureAlertRead.findMany.mockResolvedValue([{ fingerprint, activeAt: new Date('2026-08-15T01:00:00.000Z'), readAt: new Date('2026-08-15T01:01:00.000Z') }]);
|
||||
await expect(service.notificationSummary('admin-1')).resolves.toEqual({ count: 1, criticalCount: 0 });
|
||||
});
|
||||
|
||||
it('upserts an idempotent per-user read record only for a currently active occurrence', async () => {
|
||||
const labels = { alertname: 'QaCritical', severity: 'critical', service: 'qa-preview' };
|
||||
const fingerprint = createHash('sha256').update(JSON.stringify(Object.entries(labels).sort(([left], [right]) => left.localeCompare(right)))).digest('hex').slice(0, 24);
|
||||
jest.spyOn(global, 'fetch').mockResolvedValue(success({ alerts: [{ labels, annotations: { summary: '演示严重告警' }, state: 'firing', activeAt: '2026-08-16T02:00:00.000Z' }] }));
|
||||
prisma.$transaction.mockResolvedValue([{ activeAt: new Date('2026-08-16T02:00:00.000Z'), readAt: new Date('2026-08-16T02:01:00.000Z') }, {}]);
|
||||
const service = new InfrastructureMonitoringService(new ConfigService(), prisma as never);
|
||||
|
||||
await expect(service.markAlertRead(fingerprint, '2026-08-16T02:00:00.000Z', 'admin-1')).resolves.toMatchObject({ fingerprint, acknowledged: true });
|
||||
expect(prisma.infrastructureAlertRead.create).toHaveBeenCalledWith(expect.objectContaining({ data: expect.objectContaining({ fingerprint, userId: 'admin-1' }) }));
|
||||
await expect(service.markAlertRead(fingerprint, '2026-08-15T02:00:00.000Z', 'admin-1')).rejects.toThrow('已结束或已重新触发');
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
import { BadRequestException, Injectable, Logger, ServiceUnavailableException } from '@nestjs/common';
|
||||
import { BadRequestException, Injectable, Logger, NotFoundException, ServiceUnavailableException } from '@nestjs/common';
|
||||
import { ConfigService } from '@nestjs/config';
|
||||
import { Prisma } from '@prisma/client';
|
||||
import { createHash } from 'node:crypto';
|
||||
import { PrismaService } from '../prisma/prisma.service';
|
||||
import type {
|
||||
InfrastructureAlert,
|
||||
InfrastructureMetricPoint,
|
||||
@@ -162,12 +164,12 @@ export class InfrastructureMonitoringService {
|
||||
private readonly prometheusUrl: string;
|
||||
private readonly queryTimeoutMs: number;
|
||||
|
||||
constructor(config: ConfigService) {
|
||||
constructor(config: ConfigService, private readonly prisma: PrismaService) {
|
||||
this.prometheusUrl = normalizePrometheusUrl(config.get('PROMETHEUS_URL'));
|
||||
this.queryTimeoutMs = Math.min(15_000, Math.max(1_000, Number(config.get('PROMETHEUS_QUERY_TIMEOUT_MS') ?? 5_000)));
|
||||
}
|
||||
|
||||
async overview(rawRange?: string): Promise<InfrastructureMonitoringOverview> {
|
||||
async overview(rawRange?: string, userId?: string): Promise<InfrastructureMonitoringOverview> {
|
||||
const range = this.parseRange(rawRange);
|
||||
const collectedAt = new Date().toISOString();
|
||||
try {
|
||||
@@ -180,7 +182,7 @@ export class InfrastructureMonitoringService {
|
||||
]);
|
||||
const services = this.parseServices(serviceResponse);
|
||||
const serviceMetrics = this.parseServiceMetrics(serviceMetricResponse);
|
||||
const alerts = this.parseAlerts(alertResponse);
|
||||
const alerts = await this.attachReadState(this.parseAlerts(alertResponse), userId);
|
||||
const warningAlerts = alerts.filter((item) => item.severity === 'warning').length;
|
||||
const criticalAlerts = alerts.filter((item) => item.severity === 'critical').length;
|
||||
const overallStatus = criticalAlerts > 0 ? 'critical' : warningAlerts > 0 ? 'warning' : 'healthy';
|
||||
@@ -210,16 +212,47 @@ export class InfrastructureMonitoringService {
|
||||
}
|
||||
}
|
||||
|
||||
async notificationSummary() {
|
||||
async notificationSummary(userId?: string) {
|
||||
try {
|
||||
const alerts = this.parseAlerts(await this.getJson<PrometheusAlertResponse>('/api/v1/alerts'));
|
||||
return { count: alerts.length, criticalCount: alerts.filter((item) => item.severity === 'critical').length };
|
||||
const alerts = await this.attachReadState(this.parseAlerts(await this.getJson<PrometheusAlertResponse>('/api/v1/alerts')), userId);
|
||||
const unreadAlerts = alerts.filter((item) => !item.acknowledged);
|
||||
return { count: unreadAlerts.length, criticalCount: unreadAlerts.filter((item) => item.severity === 'critical').length };
|
||||
} catch (error) {
|
||||
this.logger.warn(`Prometheus notification summary unavailable: ${error instanceof Error ? error.message : 'unknown error'}`);
|
||||
throw new ServiceUnavailableException('Prometheus活动告警当前不可用');
|
||||
}
|
||||
}
|
||||
|
||||
async markAlertRead(fingerprint: string, rawActiveAt: unknown, userId: string) {
|
||||
if (!/^[a-f0-9]{24}$/.test(fingerprint)) throw new BadRequestException('告警指纹无效');
|
||||
const activeAt = new Date(String(rawActiveAt ?? ''));
|
||||
if (!Number.isFinite(activeAt.getTime())) throw new BadRequestException('告警开始时间无效');
|
||||
const activeAlerts = this.parseAlerts(await this.getJson<PrometheusAlertResponse>('/api/v1/alerts'));
|
||||
const current = activeAlerts.find((item) => item.fingerprint === fingerprint && Date.parse(item.startedAt) === activeAt.getTime());
|
||||
if (!current) throw new NotFoundException('该次活动告警已结束或已重新触发,请刷新后重试');
|
||||
const readAt = new Date();
|
||||
const log = () => this.prisma.operationLog.create({
|
||||
data: { userId, action: 'monitoring.alert_marked_read', resource: 'infrastructure_alert', resourceId: fingerprint, detail: { activeAt: activeAt.toISOString(), alertName: current.name, severity: current.severity } },
|
||||
});
|
||||
let read;
|
||||
try {
|
||||
[read] = await this.prisma.$transaction([
|
||||
this.prisma.infrastructureAlertRead.create({ data: { fingerprint, activeAt, userId, readAt } }),
|
||||
log(),
|
||||
]);
|
||||
} catch (error) {
|
||||
if (!(error instanceof Prisma.PrismaClientKnownRequestError) || error.code !== 'P2002') throw error;
|
||||
const existing = await this.prisma.infrastructureAlertRead.findUniqueOrThrow({ where: { fingerprint_userId: { fingerprint, userId } } });
|
||||
// 同一次触发重复点击不更新readAt也不重复写日志;activeAt变化才代表同指纹的新触发周期。
|
||||
if (existing.activeAt.getTime() === activeAt.getTime()) read = existing;
|
||||
else [read] = await this.prisma.$transaction([
|
||||
this.prisma.infrastructureAlertRead.update({ where: { fingerprint_userId: { fingerprint, userId } }, data: { activeAt, readAt } }),
|
||||
log(),
|
||||
]);
|
||||
}
|
||||
return { fingerprint, activeAt: read.activeAt.toISOString(), acknowledged: true, acknowledgedAt: read.readAt.toISOString() };
|
||||
}
|
||||
|
||||
private parseRange(value?: string): InfrastructureMonitoringRange {
|
||||
const range = value || '24h';
|
||||
if (!(range in RANGE_CONFIG)) throw new BadRequestException('监控时间范围只支持1h、24h或7d');
|
||||
@@ -275,6 +308,7 @@ export class InfrastructureMonitoringService {
|
||||
threshold: annotations.threshold,
|
||||
service: labels.service,
|
||||
instance: labels.instance,
|
||||
acknowledged: false,
|
||||
};
|
||||
})
|
||||
.sort((left, right) => {
|
||||
@@ -283,6 +317,20 @@ export class InfrastructureMonitoringService {
|
||||
});
|
||||
}
|
||||
|
||||
private async attachReadState(alerts: InfrastructureAlert[], userId?: string) {
|
||||
if (!userId || alerts.length === 0) return alerts;
|
||||
const reads = await this.prisma.infrastructureAlertRead.findMany({
|
||||
where: { userId, fingerprint: { in: alerts.map((item) => item.fingerprint) } },
|
||||
select: { fingerprint: true, activeAt: true, readAt: true },
|
||||
});
|
||||
const byFingerprint = new Map(reads.map((item) => [item.fingerprint, item]));
|
||||
return alerts.map((alert) => {
|
||||
const read = byFingerprint.get(alert.fingerprint);
|
||||
const acknowledged = Boolean(read && read.activeAt.getTime() === Date.parse(alert.startedAt));
|
||||
return { ...alert, acknowledged, acknowledgedAt: acknowledged ? read?.readAt.toISOString() : undefined };
|
||||
});
|
||||
}
|
||||
|
||||
private parseServiceMetrics(response: PrometheusQueryResponse): InfrastructureServiceMetricGroup[] {
|
||||
const values = new Map<string, number>();
|
||||
for (const item of response.data?.result ?? []) {
|
||||
|
||||
Reference in New Issue
Block a user