fix: correct operational statistics and form interactions
This commit is contained in:
@@ -194,6 +194,7 @@ export class ChannelReportingService {
|
||||
SELECT
|
||||
submit."channelId" AS channel_id,
|
||||
message."signatureId" AS signature_id,
|
||||
message.carrier AS carrier,
|
||||
message."drainageInfoId" AS drainage_info_id,
|
||||
submit."submitStatus" AS submit_status,
|
||||
COALESCE(submit."submittedAt", submit."createdAt") AS attempted_at,
|
||||
@@ -244,6 +245,7 @@ export class ChannelReportingService {
|
||||
SELECT
|
||||
channel_id AS "channelId",
|
||||
signature_id AS "signatureId",
|
||||
carrier,
|
||||
drainage_info_id AS "drainageInfoId",
|
||||
COUNT(*) FILTER (
|
||||
WHERE attempted_at >= ${day.startAt} AND attempted_at < ${day.endAt}
|
||||
@@ -270,7 +272,7 @@ export class ChannelReportingService {
|
||||
)::integer AS "failureCount",
|
||||
MAX(successful_at) FILTER (WHERE delivery_status = 'success') AS "lastSuccessfulSentAt"
|
||||
FROM base
|
||||
GROUP BY channel_id, signature_id, drainage_info_id
|
||||
GROUP BY channel_id, signature_id, drainage_info_id, carrier
|
||||
`);
|
||||
|
||||
return tasks.map((task) => {
|
||||
@@ -278,6 +280,7 @@ export class ChannelReportingService {
|
||||
(row) =>
|
||||
row.channelId === task.channelId &&
|
||||
row.signatureId === task.signatureId &&
|
||||
(!task.carrier || row.carrier === task.carrier) &&
|
||||
((task.reportType ?? 'signature') === 'signature' || row.drainageInfoId === task.drainageItemId),
|
||||
);
|
||||
const deliveryStats = summarizeChannelReportDelivery(taskRows);
|
||||
|
||||
@@ -2,7 +2,7 @@ import { BadRequestException, NotFoundException } from '@nestjs/common';
|
||||
import { Prisma } from '@prisma/client';
|
||||
import { randomUUID } from 'crypto';
|
||||
import { summarizeReportStatuses as summarizeCommonReportStatuses } from '../common/report-status';
|
||||
import type { CreateChannelDto, UpdateChannelDto, CreateChannelGroupDto, CreateChannelGroupItemDto, UpdateChannelGroupDto, CreateRouteRuleDto, CreateReportFieldDto, ReplaceReportFieldsDto, CreateReportMaterialDto, CreateReportTaskDto, ChangeReportTaskStatusesDto, CreateReportExportDto, CreateReceiptImportDto, UpsertConnectionStateDto, ChangeChannelStatusDto, CopyChannelDto, TestChannelDto } from './channels.contracts';
|
||||
import type { CreateChannelGroupItemDto, TestChannelDto } from './channels.contracts';
|
||||
|
||||
export function summarizeReportStatuses(statuses: string[]) {
|
||||
return summarizeCommonReportStatuses(statuses);
|
||||
@@ -141,7 +141,11 @@ export function buildChannelTestSubmitCommand({
|
||||
account: channel.account,
|
||||
passwordCipher: channel.passwordCipher,
|
||||
cmppVersion: channel.cmppVersion,
|
||||
desiredConnections: getPositiveRuntimeInteger(getConfigValue(channel.config, 'desiredConnections'), 1, 'desiredConnections'),
|
||||
desiredConnections: getPositiveRuntimeInteger(
|
||||
getConfigValue(channel.config, 'desiredConnections'),
|
||||
1,
|
||||
'desiredConnections',
|
||||
),
|
||||
windowSize: getPositiveRuntimeInteger(getConfigValue(channel.config, 'windowSize'), 16, 'windowSize'),
|
||||
heartbeatIntervalSeconds: getPositiveRuntimeInteger(
|
||||
getConfigValue(channel.config, 'heartbeatIntervalSeconds'),
|
||||
@@ -254,23 +258,22 @@ export function getRuntimeConfigInteger(
|
||||
return Number.isInteger(value) && value > 0 ? value : fallback;
|
||||
}
|
||||
|
||||
export function channelConnectionSettingsChanged(
|
||||
before: ChannelConnectionSettings,
|
||||
after: ChannelConnectionSettings,
|
||||
) {
|
||||
return before.gatewayHost !== after.gatewayHost
|
||||
|| before.gatewayPort !== after.gatewayPort
|
||||
|| before.account !== after.account
|
||||
|| before.passwordCipher !== after.passwordCipher
|
||||
|| before.cmppVersion !== after.cmppVersion
|
||||
|| getRuntimeConfigInteger(before.config, 'desiredConnections', 1)
|
||||
!== getRuntimeConfigInteger(after.config, 'desiredConnections', 1)
|
||||
|| getRuntimeConfigInteger(before.config, 'windowSize', 16)
|
||||
!== getRuntimeConfigInteger(after.config, 'windowSize', 16)
|
||||
|| getRuntimeConfigInteger(before.config, 'heartbeatIntervalSeconds', DEFAULT_HEARTBEAT_INTERVAL_SECONDS)
|
||||
!== getRuntimeConfigInteger(after.config, 'heartbeatIntervalSeconds', DEFAULT_HEARTBEAT_INTERVAL_SECONDS)
|
||||
|| getRuntimeConfigInteger(before.config, 'heartbeatMissThreshold', DEFAULT_HEARTBEAT_MISS_THRESHOLD)
|
||||
!== getRuntimeConfigInteger(after.config, 'heartbeatMissThreshold', DEFAULT_HEARTBEAT_MISS_THRESHOLD);
|
||||
export function channelConnectionSettingsChanged(before: ChannelConnectionSettings, after: ChannelConnectionSettings) {
|
||||
return (
|
||||
before.gatewayHost !== after.gatewayHost ||
|
||||
before.gatewayPort !== after.gatewayPort ||
|
||||
before.account !== after.account ||
|
||||
before.passwordCipher !== after.passwordCipher ||
|
||||
before.cmppVersion !== after.cmppVersion ||
|
||||
getRuntimeConfigInteger(before.config, 'desiredConnections', 1) !==
|
||||
getRuntimeConfigInteger(after.config, 'desiredConnections', 1) ||
|
||||
getRuntimeConfigInteger(before.config, 'windowSize', 16) !==
|
||||
getRuntimeConfigInteger(after.config, 'windowSize', 16) ||
|
||||
getRuntimeConfigInteger(before.config, 'heartbeatIntervalSeconds', DEFAULT_HEARTBEAT_INTERVAL_SECONDS) !==
|
||||
getRuntimeConfigInteger(after.config, 'heartbeatIntervalSeconds', DEFAULT_HEARTBEAT_INTERVAL_SECONDS) ||
|
||||
getRuntimeConfigInteger(before.config, 'heartbeatMissThreshold', DEFAULT_HEARTBEAT_MISS_THRESHOLD) !==
|
||||
getRuntimeConfigInteger(after.config, 'heartbeatMissThreshold', DEFAULT_HEARTBEAT_MISS_THRESHOLD)
|
||||
);
|
||||
}
|
||||
|
||||
export function channelGroupAuditSnapshot(group: {
|
||||
@@ -320,19 +323,49 @@ export function normalizeChannelRuntimeConfig(
|
||||
heartbeatIntervalSeconds?: number,
|
||||
heartbeatMissThreshold?: number,
|
||||
) {
|
||||
const existing = existingConfig && typeof existingConfig === 'object' && !Array.isArray(existingConfig)
|
||||
? existingConfig as Record<string, unknown>
|
||||
: {};
|
||||
const incoming = incomingConfig && typeof incomingConfig === 'object' && !Array.isArray(incomingConfig)
|
||||
? incomingConfig
|
||||
: {};
|
||||
const existing =
|
||||
existingConfig && typeof existingConfig === 'object' && !Array.isArray(existingConfig)
|
||||
? (existingConfig as Record<string, unknown>)
|
||||
: {};
|
||||
const incoming =
|
||||
incomingConfig && typeof incomingConfig === 'object' && !Array.isArray(incomingConfig) ? incomingConfig : {};
|
||||
const base = { ...existing, ...incoming };
|
||||
base.desiredConnections = boundedRuntimeInteger(desiredConnections ?? base.desiredConnections, 1, 8, 1, 'desiredConnections');
|
||||
base.desiredConnections = boundedRuntimeInteger(
|
||||
desiredConnections ?? base.desiredConnections,
|
||||
1,
|
||||
8,
|
||||
1,
|
||||
'desiredConnections',
|
||||
);
|
||||
base.windowSize = boundedRuntimeInteger(windowSize ?? base.windowSize, 1, 64, 16, 'windowSize');
|
||||
base.connectionWarmupSeconds = boundedRuntimeInteger(base.connectionWarmupSeconds, 0, 300, 30, 'connectionWarmupSeconds');
|
||||
base.connectionDrainTimeoutSeconds = boundedRuntimeInteger(base.connectionDrainTimeoutSeconds, 1, 600, 60, 'connectionDrainTimeoutSeconds');
|
||||
base.submitResponseTimeoutSeconds = boundedRuntimeInteger(base.submitResponseTimeoutSeconds, 1, 300, 60, 'submitResponseTimeoutSeconds');
|
||||
base.connectionFailureCooldownSeconds = boundedRuntimeInteger(base.connectionFailureCooldownSeconds, 1, 300, 30, 'connectionFailureCooldownSeconds');
|
||||
base.connectionWarmupSeconds = boundedRuntimeInteger(
|
||||
base.connectionWarmupSeconds,
|
||||
0,
|
||||
300,
|
||||
30,
|
||||
'connectionWarmupSeconds',
|
||||
);
|
||||
base.connectionDrainTimeoutSeconds = boundedRuntimeInteger(
|
||||
base.connectionDrainTimeoutSeconds,
|
||||
1,
|
||||
600,
|
||||
60,
|
||||
'connectionDrainTimeoutSeconds',
|
||||
);
|
||||
base.submitResponseTimeoutSeconds = boundedRuntimeInteger(
|
||||
base.submitResponseTimeoutSeconds,
|
||||
1,
|
||||
300,
|
||||
60,
|
||||
'submitResponseTimeoutSeconds',
|
||||
);
|
||||
base.connectionFailureCooldownSeconds = boundedRuntimeInteger(
|
||||
base.connectionFailureCooldownSeconds,
|
||||
1,
|
||||
300,
|
||||
30,
|
||||
'connectionFailureCooldownSeconds',
|
||||
);
|
||||
base.heartbeatIntervalSeconds = getPositiveRuntimeInteger(
|
||||
heartbeatIntervalSeconds ?? base.heartbeatIntervalSeconds,
|
||||
DEFAULT_HEARTBEAT_INTERVAL_SECONDS,
|
||||
@@ -423,13 +456,19 @@ export function getPositiveIntegerEnv(name: string, fallback: number) {
|
||||
}
|
||||
|
||||
export function parseReceiptContent(content: string, delimiter?: ',' | '\t') {
|
||||
const lines = content.replace(/^\uFEFF/, '').split(/\r?\n/).map((line) => line.trim()).filter(Boolean);
|
||||
const lines = content
|
||||
.replace(/^\uFEFF/, '')
|
||||
.split(/\r?\n/)
|
||||
.map((line) => line.trim())
|
||||
.filter(Boolean);
|
||||
if (lines.length === 0) {
|
||||
throw new BadRequestException('Receipt file is empty');
|
||||
}
|
||||
const separator = delimiter ?? (lines[0].includes('\t') ? '\t' : ',');
|
||||
const firstCells = splitReceiptLine(lines[0], separator);
|
||||
const hasHeader = firstCells.some((cell) => ['phone', 'mobile', 'status', 'result', '手机号', '号码', '状态', '结果'].includes(cell.toLowerCase()));
|
||||
const hasHeader = firstCells.some((cell) =>
|
||||
['phone', 'mobile', 'status', 'result', '手机号', '号码', '状态', '结果'].includes(cell.toLowerCase()),
|
||||
);
|
||||
const header = hasHeader ? firstCells : [];
|
||||
const rows = hasHeader ? lines.slice(1) : lines;
|
||||
const statusIndex = findReceiptStatusIndex(header);
|
||||
@@ -445,7 +484,7 @@ export function parseReceiptContent(content: string, delimiter?: ',' | '\t') {
|
||||
failedCount += 1;
|
||||
}
|
||||
return {
|
||||
rowNumber: (hasHeader ? index + 2 : index + 1),
|
||||
rowNumber: hasHeader ? index + 2 : index + 1,
|
||||
phone: cells[0] ?? '',
|
||||
status: normalizedStatus,
|
||||
rawStatus,
|
||||
@@ -504,10 +543,39 @@ export function findReceiptStatusIndex(header: string[]) {
|
||||
|
||||
export function normalizeReceiptStatus(value: string) {
|
||||
const normalized = value.trim().toLowerCase();
|
||||
if (['success', 'succeeded', 'approved', 'completed', 'ok', 'pass', 'passed', '通过', '成功', '已完成', '报备成功'].includes(normalized)) {
|
||||
if (
|
||||
[
|
||||
'success',
|
||||
'succeeded',
|
||||
'approved',
|
||||
'completed',
|
||||
'ok',
|
||||
'pass',
|
||||
'passed',
|
||||
'通过',
|
||||
'成功',
|
||||
'已完成',
|
||||
'报备成功',
|
||||
].includes(normalized)
|
||||
) {
|
||||
return 'success';
|
||||
}
|
||||
if (['failed', 'fail', 'rejected', 'reject', 'error', 'no', 'denied', '驳回', '失败', '不通过', '拒绝', '报备失败'].includes(normalized)) {
|
||||
if (
|
||||
[
|
||||
'failed',
|
||||
'fail',
|
||||
'rejected',
|
||||
'reject',
|
||||
'error',
|
||||
'no',
|
||||
'denied',
|
||||
'驳回',
|
||||
'失败',
|
||||
'不通过',
|
||||
'拒绝',
|
||||
'报备失败',
|
||||
].includes(normalized)
|
||||
) {
|
||||
return 'failed';
|
||||
}
|
||||
return 'failed';
|
||||
@@ -524,6 +592,7 @@ export function deriveReceiptStatus(rowCount: number, successCount: number, fail
|
||||
}
|
||||
|
||||
export type ChannelReportDeliveryRow = {
|
||||
carrier: string | null;
|
||||
channelId: string;
|
||||
signatureId: string;
|
||||
drainageInfoId: string | null;
|
||||
@@ -557,10 +626,13 @@ export function summarizeChannelReportDelivery(rows: ChannelReportDeliveryRow[])
|
||||
};
|
||||
}
|
||||
|
||||
export function sumReportDelivery(rows: ChannelReportDeliveryRow[], key: keyof Pick<
|
||||
ChannelReportDeliveryRow,
|
||||
'total' | 'acceptedCount' | 'submitFailureCount' | 'successCount' | 'unknownCount' | 'failureCount'
|
||||
>) {
|
||||
export function sumReportDelivery(
|
||||
rows: ChannelReportDeliveryRow[],
|
||||
key: keyof Pick<
|
||||
ChannelReportDeliveryRow,
|
||||
'total' | 'acceptedCount' | 'submitFailureCount' | 'successCount' | 'unknownCount' | 'failureCount'
|
||||
>,
|
||||
) {
|
||||
return rows.reduce((total, row) => total + Number(row[key] ?? 0), 0);
|
||||
}
|
||||
|
||||
@@ -580,7 +652,11 @@ export function currentShanghaiDayRange(now = new Date()) {
|
||||
return { startAt, endAt: new Date(startAt.getTime() + 24 * 60 * 60 * 1_000) };
|
||||
}
|
||||
|
||||
export function normalizeRetryTimeLimitMinutes(minutes: number | undefined, hours: number | undefined, fallbackMinutes: number) {
|
||||
export function normalizeRetryTimeLimitMinutes(
|
||||
minutes: number | undefined,
|
||||
hours: number | undefined,
|
||||
fallbackMinutes: number,
|
||||
) {
|
||||
const value = minutes ?? (hours === undefined ? fallbackMinutes : hours * 60);
|
||||
if (!Number.isInteger(value) || value <= 0 || value > 72 * 60) {
|
||||
throw new BadRequestException('retryTimeLimitMinutes must be an integer between 1 and 4320');
|
||||
@@ -588,7 +664,12 @@ export function normalizeRetryTimeLimitMinutes(minutes: number | undefined, hour
|
||||
return value;
|
||||
}
|
||||
|
||||
export function normalizeSpreadsheetSize(value: number | undefined, fallback: number, minimum: number, maximum: number) {
|
||||
export function normalizeSpreadsheetSize(
|
||||
value: number | undefined,
|
||||
fallback: number,
|
||||
minimum: number,
|
||||
maximum: number,
|
||||
) {
|
||||
if (value === undefined || !Number.isFinite(value)) return fallback;
|
||||
return Math.min(maximum, Math.max(minimum, Math.round(value)));
|
||||
}
|
||||
@@ -602,7 +683,9 @@ export function normalizeBusinessCarrier(carrier?: string | null) {
|
||||
}
|
||||
|
||||
export function normalizeChannelCarrier(carrier?: string | null) {
|
||||
const value = String(carrier ?? '').trim().toLowerCase();
|
||||
const value = String(carrier ?? '')
|
||||
.trim()
|
||||
.toLowerCase();
|
||||
if (['mobile', 'cmcc', '移动', '中国移动'].includes(value)) return 'mobile';
|
||||
if (['unicom', 'cucc', '联通', '中国联通'].includes(value)) return 'unicom';
|
||||
if (['telecom', 'ctcc', '电信', '中国电信'].includes(value)) return 'telecom';
|
||||
@@ -631,12 +714,18 @@ export function legacyCarrierFromCapabilities(carriers: string[]) {
|
||||
return 'multi';
|
||||
}
|
||||
|
||||
export function isChannelCarrierCompatible(channelCarrier: string | null | undefined, groupCarrier: string, carriers?: string[] | null) {
|
||||
export function isChannelCarrierCompatible(
|
||||
channelCarrier: string | null | undefined,
|
||||
groupCarrier: string,
|
||||
carriers?: string[] | null,
|
||||
) {
|
||||
return normalizeChannelCarriers(carriers, channelCarrier).includes(normalizeBusinessCarrier(groupCarrier));
|
||||
}
|
||||
|
||||
export function normalizeRegion(region?: string | null) {
|
||||
return String(region ?? '').replace(/省|市|自治区|壮族|回族|维吾尔/g, '').trim();
|
||||
return String(region ?? '')
|
||||
.replace(/省|市|自治区|壮族|回族|维吾尔/g, '')
|
||||
.trim();
|
||||
}
|
||||
|
||||
export function isRegionCompatible(channelRegion: string | null | undefined, itemProvince: string) {
|
||||
@@ -646,7 +735,10 @@ export function isRegionCompatible(channelRegion: string | null | undefined, ite
|
||||
export function validateGroupItems(
|
||||
groupCarrier: string,
|
||||
items: Array<Omit<CreateChannelGroupItemDto, 'groupId'>>,
|
||||
channels: Map<string, { id: string; carrier?: string | null; carriers?: string[] | null; sendRegion?: string | null }>,
|
||||
channels: Map<
|
||||
string,
|
||||
{ id: string; carrier?: string | null; carriers?: string[] | null; sendRegion?: string | null }
|
||||
>,
|
||||
) {
|
||||
const channelIds = new Set<string>();
|
||||
const provinces = new Set<string>();
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,45 @@
|
||||
import { alertHistoryRange, mergeAlertHistory } from './alert-history';
|
||||
|
||||
describe('historical alert observation cycles', () => {
|
||||
it('uses seven Shanghai calendar days and rejects invalid or excessive dates', () => {
|
||||
expect(alertHistoryRange(undefined, undefined, new Date('2026-09-09T16:30:00Z'))).toMatchObject({
|
||||
startDate: '2026-09-04',
|
||||
endDate: '2026-09-10',
|
||||
});
|
||||
for (const [from, to] of [
|
||||
['2026-02-30', '2026-03-01'],
|
||||
['2026-09-09', '2026-09-08'],
|
||||
['2026-07-01', '2026-09-09'],
|
||||
]) {
|
||||
expect(() => alertHistoryRange(from, to)).toThrow();
|
||||
}
|
||||
});
|
||||
it('retains distinct cycles, merges daily boundaries and excludes stale/nonpositive samples', () => {
|
||||
const result = new Map();
|
||||
const metric = { __name__: 'ALERTS_FOR_STATE', alertname: 'CPUHigh', instance: 'host', severity: 'warning' };
|
||||
mergeAlertHistory(
|
||||
result,
|
||||
[
|
||||
{
|
||||
metric,
|
||||
values: [
|
||||
[110, '100'],
|
||||
[120, '100'],
|
||||
[130, '0'],
|
||||
[140, 'NaN'],
|
||||
[150, '145'],
|
||||
[200, '145'],
|
||||
],
|
||||
},
|
||||
],
|
||||
110,
|
||||
200,
|
||||
);
|
||||
mergeAlertHistory(result, [{ metric, values: [[160, '145']] }], 110, 200);
|
||||
expect(result.size).toBe(2);
|
||||
expect([...result.values()].map((item) => item.lastObservedAt)).toEqual([
|
||||
new Date(120000).toISOString(),
|
||||
new Date(160000).toISOString(),
|
||||
]);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,68 @@
|
||||
import { BadRequestException } from '@nestjs/common';
|
||||
import { createHash } from 'node:crypto';
|
||||
|
||||
export function alertHistoryRange(from?: string, to?: string, now = new Date()) {
|
||||
const dateKey = (date: Date) => new Date(date.getTime() + 8 * 3600_000).toISOString().slice(0, 10);
|
||||
const endDate = to || dateKey(now);
|
||||
const startDate = from || dateKey(new Date(now.getTime() - 6 * 86400_000));
|
||||
const parse = (value: string) => {
|
||||
const result = new Date(`${value}T00:00:00+08:00`);
|
||||
if (!/^\d{4}-\d{2}-\d{2}$/.test(value) || !Number.isFinite(result.getTime()) || dateKey(result) !== value) {
|
||||
throw new BadRequestException('告警日期无效');
|
||||
}
|
||||
return result.getTime() / 1000;
|
||||
};
|
||||
const start = parse(startDate);
|
||||
const end = parse(endDate) + 86400;
|
||||
if (end <= start || end - start > 31 * 86400) throw new BadRequestException('告警日期范围须为1至31天');
|
||||
return { startDate, endDate, start, end: Math.min(end, now.getTime() / 1000) };
|
||||
}
|
||||
|
||||
export type AlertHistoryItem = {
|
||||
id: string;
|
||||
name: string;
|
||||
severity: string;
|
||||
service: string;
|
||||
instance: string;
|
||||
startedAt: string;
|
||||
firstObservedAt: string;
|
||||
lastObservedAt: string;
|
||||
};
|
||||
|
||||
// ALERTS_FOR_STATE stores activeAt as the sample value, separating repeated trigger cycles.
|
||||
// Observation boundaries are not claimed as exact recovery times.
|
||||
export function mergeAlertHistory(
|
||||
target: Map<string, AlertHistoryItem>,
|
||||
series: Array<{ metric: Record<string, string>; values?: [number, string][] }>,
|
||||
start: number,
|
||||
end: number,
|
||||
) {
|
||||
for (const { metric, values } of series) {
|
||||
const labels = Object.entries(metric)
|
||||
.filter(([key]) => key !== '__name__')
|
||||
.sort(([a], [b]) => a.localeCompare(b));
|
||||
const fingerprint = createHash('sha256').update(JSON.stringify(labels)).digest('hex');
|
||||
for (const [time, rawActiveAt] of values ?? []) {
|
||||
const activeAt = Number(rawActiveAt);
|
||||
if (time < start || time >= end || !Number.isFinite(activeAt) || activeAt <= 0 || activeAt > time) continue;
|
||||
const id = `${fingerprint}:${activeAt}`;
|
||||
const observed = new Date(time * 1000).toISOString();
|
||||
const item = target.get(id);
|
||||
if (item) {
|
||||
if (observed < item.firstObservedAt) item.firstObservedAt = observed;
|
||||
if (observed > item.lastObservedAt) item.lastObservedAt = observed;
|
||||
} else {
|
||||
target.set(id, {
|
||||
id,
|
||||
name: metric.alertname || '未命名告警',
|
||||
severity: metric.severity || 'info',
|
||||
service: metric.service || '',
|
||||
instance: metric.instance || '',
|
||||
startedAt: new Date(activeAt * 1000).toISOString(),
|
||||
firstObservedAt: observed,
|
||||
lastObservedAt: observed,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -8,7 +8,10 @@ import { InfrastructureMonitoringService } from './infrastructure-monitoring.ser
|
||||
@ApiTags('infrastructure-monitoring')
|
||||
@Controller('admin/infrastructure-monitoring')
|
||||
export class InfrastructureMonitoringController {
|
||||
constructor(private readonly monitoring: InfrastructureMonitoringService, private readonly settings: InfrastructureAlertSettingsService) {}
|
||||
constructor(
|
||||
private readonly monitoring: InfrastructureMonitoringService,
|
||||
private readonly settings: InfrastructureAlertSettingsService,
|
||||
) {}
|
||||
|
||||
@Get('overview')
|
||||
overview(@Query('range') range?: string, @CurrentSessionUserId() userId?: string) {
|
||||
@@ -16,19 +19,35 @@ export class InfrastructureMonitoringController {
|
||||
}
|
||||
|
||||
@Get('notification-summary')
|
||||
notificationSummary(@CurrentSessionUserId() userId?: string) { return this.monitoring.notificationSummary(userId); }
|
||||
notificationSummary(@CurrentSessionUserId() userId?: string) {
|
||||
return this.monitoring.notificationSummary(userId);
|
||||
}
|
||||
|
||||
@Get('alert-history')
|
||||
alertHistory(@Query('from') from?: string, @Query('to') to?: string, @Query('page') page?: string) {
|
||||
return this.monitoring.alertHistory(from, to, page);
|
||||
}
|
||||
|
||||
@Post('alerts/:fingerprint/read')
|
||||
markAlertRead(@Param('fingerprint') fingerprint: string, @Body('activeAt') activeAt: unknown, @CurrentSessionUserId() userId: string) {
|
||||
markAlertRead(
|
||||
@Param('fingerprint') fingerprint: string,
|
||||
@Body('activeAt') activeAt: unknown,
|
||||
@CurrentSessionUserId() userId: string,
|
||||
) {
|
||||
return this.monitoring.markAlertRead(fingerprint, activeAt, userId);
|
||||
}
|
||||
|
||||
@Get('alert-thresholds')
|
||||
alertThresholds() { return this.settings.get(); }
|
||||
alertThresholds() {
|
||||
return this.settings.get();
|
||||
}
|
||||
|
||||
@Put('alert-thresholds')
|
||||
@RequireRecentAuthentication()
|
||||
updateAlertThresholds(@Body() body: { configVersion?: number; thresholds?: unknown }, @CurrentSessionUserId() operatorId?: string) {
|
||||
updateAlertThresholds(
|
||||
@Body() body: { configVersion?: number; thresholds?: unknown },
|
||||
@CurrentSessionUserId() operatorId?: string,
|
||||
) {
|
||||
return this.settings.update(body, operatorId);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,9 +1,22 @@
|
||||
import { BadRequestException, Injectable, Logger, NotFoundException, ServiceUnavailableException } from '@nestjs/common';
|
||||
import {
|
||||
BadRequestException,
|
||||
Injectable,
|
||||
Logger,
|
||||
NotFoundException,
|
||||
ServiceUnavailableException,
|
||||
} from '@nestjs/common';
|
||||
import { ConfigService } from '@nestjs/config';
|
||||
import { Prisma } from '@prisma/client';
|
||||
import { createHash } from 'node:crypto';
|
||||
import { PrismaService } from '../prisma/prisma.service';
|
||||
import { compareMountpoints, FILESYSTEM_LABELS, FILESYSTEM_SELECTOR, FILESYSTEM_USAGE_PERCENT, filesystemIdentity } from './filesystem-metrics';
|
||||
import { alertHistoryRange, mergeAlertHistory, type AlertHistoryItem } from './alert-history';
|
||||
import {
|
||||
compareMountpoints,
|
||||
FILESYSTEM_LABELS,
|
||||
FILESYSTEM_SELECTOR,
|
||||
FILESYSTEM_USAGE_PERCENT,
|
||||
filesystemIdentity,
|
||||
} from './filesystem-metrics';
|
||||
import type {
|
||||
InfrastructureAlert,
|
||||
InfrastructureMetricPoint,
|
||||
@@ -57,7 +70,8 @@ const QUERIES = {
|
||||
uptimeSeconds: 'time() - node_boot_time_seconds',
|
||||
lastSampleAt: 'max(timestamp(node_uname_info))',
|
||||
// PromQL字符串本身需要两个反斜杠才能把正则的“\.”传给RE2;TypeScript字面量因此需要写四个。
|
||||
services: 'max by (name) (node_systemd_unit_state{name=~"cmpp-api\\\\.service|cmpp-gateway\\\\.service|postgresql\\\\.service|redis(-server)?\\\\.service|cmpp-minio\\\\.service|nginx\\\\.service",state="active"})',
|
||||
services:
|
||||
'max by (name) (node_systemd_unit_state{name=~"cmpp-api\\\\.service|cmpp-gateway\\\\.service|postgresql\\\\.service|redis(-server)?\\\\.service|cmpp-minio\\\\.service|nginx\\\\.service",state="active"})',
|
||||
} as const;
|
||||
|
||||
const SERVICE_DEFINITIONS = [
|
||||
@@ -70,38 +84,62 @@ const SERVICE_DEFINITIONS = [
|
||||
] as const;
|
||||
|
||||
const SERVICE_METRIC_DEFINITIONS = [
|
||||
{ key: 'api', name: 'API服务', metrics: [
|
||||
['requestsPerSecond', '请求速率', 'cmpp:service_api:requests_per_second', 'per_second'],
|
||||
['errorPercent', '5xx错误率', 'cmpp:service_api:error_percent', 'percent'],
|
||||
['latencyP95', 'P95响应', 'cmpp:service_api:latency_p95_seconds', 'seconds'],
|
||||
['eventLoopP99', '事件循环P99', 'cmpp:service_api:event_loop_p99_seconds', 'seconds'],
|
||||
] },
|
||||
{ key: 'gateway', name: 'Gateway服务', metrics: [
|
||||
['submitsPerSecond', '提交速率', 'cmpp:service_gateway:submits_per_second', 'per_second'],
|
||||
['failurePercent', '提交失败率', 'cmpp:service_gateway:failure_percent', 'percent'],
|
||||
['queuePending', 'Stream pending', 'cmpp:service_gateway:queue_pending', 'count'],
|
||||
['queueOldestSeconds', '最旧pending', 'cmpp:service_gateway:queue_oldest_seconds', 'seconds'],
|
||||
] },
|
||||
{ key: 'postgresql', name: 'PostgreSQL', metrics: [
|
||||
['connectionPercent', '连接使用率', 'cmpp:service_postgresql:connection_percent', 'percent'],
|
||||
['deadlocks15m', '15分钟死锁', 'cmpp:service_postgresql:deadlocks_15m', 'count'],
|
||||
] },
|
||||
{ key: 'redis', name: 'Redis', metrics: [
|
||||
['memoryPercent', '内存使用率', 'cmpp:service_redis:memory_percent', 'percent'],
|
||||
['memoryUsedBytes', '已用内存', 'cmpp:service_redis:memory_used_bytes', 'bytes'],
|
||||
['connectedClients', '客户端连接', 'cmpp:service_redis:connected_clients', 'count'],
|
||||
['evictions5m', '5分钟淘汰', 'cmpp:service_redis:evictions_5m', 'count'],
|
||||
] },
|
||||
{ key: 'minio', name: 'MinIO', metrics: [
|
||||
['capacityPercent', '存储容量使用率', 'cmpp:service_minio:capacity_percent', 'percent'],
|
||||
['usageBytes', '对象数据量', 'cmpp:service_minio:usage_bytes', 'bytes'],
|
||||
['objects', '对象数', 'cmpp:service_minio:objects', 'count'],
|
||||
['drivesOffline', '离线存储盘', 'cmpp:service_minio:drives_offline', 'count'],
|
||||
] },
|
||||
{ key: 'nginx', name: 'Nginx', metrics: [
|
||||
['connectionsActive', '活跃连接', 'cmpp:service_nginx:connections_active', 'count'],
|
||||
['requestsPerSecond', '请求速率', 'cmpp:service_nginx:requests_per_second', 'per_second'],
|
||||
] },
|
||||
{
|
||||
key: 'api',
|
||||
name: 'API服务',
|
||||
metrics: [
|
||||
['requestsPerSecond', '请求速率', 'cmpp:service_api:requests_per_second', 'per_second'],
|
||||
['errorPercent', '5xx错误率', 'cmpp:service_api:error_percent', 'percent'],
|
||||
['latencyP95', 'P95响应', 'cmpp:service_api:latency_p95_seconds', 'seconds'],
|
||||
['eventLoopP99', '事件循环P99', 'cmpp:service_api:event_loop_p99_seconds', 'seconds'],
|
||||
],
|
||||
},
|
||||
{
|
||||
key: 'gateway',
|
||||
name: 'Gateway服务',
|
||||
metrics: [
|
||||
['submitsPerSecond', '提交速率', 'cmpp:service_gateway:submits_per_second', 'per_second'],
|
||||
['failurePercent', '提交失败率', 'cmpp:service_gateway:failure_percent', 'percent'],
|
||||
['queuePending', 'Stream pending', 'cmpp:service_gateway:queue_pending', 'count'],
|
||||
['queueOldestSeconds', '最旧pending', 'cmpp:service_gateway:queue_oldest_seconds', 'seconds'],
|
||||
],
|
||||
},
|
||||
{
|
||||
key: 'postgresql',
|
||||
name: 'PostgreSQL',
|
||||
metrics: [
|
||||
['connectionPercent', '连接使用率', 'cmpp:service_postgresql:connection_percent', 'percent'],
|
||||
['deadlocks15m', '15分钟死锁', 'cmpp:service_postgresql:deadlocks_15m', 'count'],
|
||||
],
|
||||
},
|
||||
{
|
||||
key: 'redis',
|
||||
name: 'Redis',
|
||||
metrics: [
|
||||
['memoryPercent', '内存使用率', 'cmpp:service_redis:memory_percent', 'percent'],
|
||||
['memoryUsedBytes', '已用内存', 'cmpp:service_redis:memory_used_bytes', 'bytes'],
|
||||
['connectedClients', '客户端连接', 'cmpp:service_redis:connected_clients', 'count'],
|
||||
['evictions5m', '5分钟淘汰', 'cmpp:service_redis:evictions_5m', 'count'],
|
||||
],
|
||||
},
|
||||
{
|
||||
key: 'minio',
|
||||
name: 'MinIO',
|
||||
metrics: [
|
||||
['capacityPercent', '存储容量使用率', 'cmpp:service_minio:capacity_percent', 'percent'],
|
||||
['usageBytes', '对象数据量', 'cmpp:service_minio:usage_bytes', 'bytes'],
|
||||
['objects', '对象数', 'cmpp:service_minio:objects', 'count'],
|
||||
['drivesOffline', '离线存储盘', 'cmpp:service_minio:drives_offline', 'count'],
|
||||
],
|
||||
},
|
||||
{
|
||||
key: 'nginx',
|
||||
name: 'Nginx',
|
||||
metrics: [
|
||||
['connectionsActive', '活跃连接', 'cmpp:service_nginx:connections_active', 'count'],
|
||||
['requestsPerSecond', '请求速率', 'cmpp:service_nginx:requests_per_second', 'per_second'],
|
||||
],
|
||||
},
|
||||
] as const;
|
||||
|
||||
const SERVICE_METRICS_QUERY = '{__name__=~"cmpp:service_.*"}';
|
||||
@@ -165,7 +203,10 @@ export class InfrastructureMonitoringService {
|
||||
private readonly prometheusUrl: string;
|
||||
private readonly queryTimeoutMs: number;
|
||||
|
||||
constructor(config: ConfigService, private readonly prisma: PrismaService) {
|
||||
constructor(
|
||||
config: ConfigService,
|
||||
private readonly prisma: PrismaService,
|
||||
) {
|
||||
this.prometheusUrl = normalizePrometheusUrl(config.get('PROMETHEUS_URL'));
|
||||
this.queryTimeoutMs = Math.min(15_000, Math.max(1_000, Number(config.get('PROMETHEUS_QUERY_TIMEOUT_MS') ?? 5_000)));
|
||||
}
|
||||
@@ -202,7 +243,7 @@ export class InfrastructureMonitoringService {
|
||||
activeAlerts: alerts.length,
|
||||
},
|
||||
metrics: instant.metrics,
|
||||
trends: { ...trends.metrics, diskUsagePercent: rootDisk ? trends.disks.get(rootDisk.id) ?? [] : [] },
|
||||
trends: { ...trends.metrics, diskUsagePercent: rootDisk ? (trends.disks.get(rootDisk.id) ?? []) : [] },
|
||||
disks: instant.disks.map((disk) => ({ ...disk, trend: trends.disks.get(disk.id) ?? [] })),
|
||||
services,
|
||||
serviceMetrics,
|
||||
@@ -210,18 +251,28 @@ export class InfrastructureMonitoringService {
|
||||
};
|
||||
} catch (error) {
|
||||
// 页面必须整体清空陈旧指标,但服务端仍需留下不含PromQL/地址/凭据的根因摘要便于运维诊断。
|
||||
this.logger.warn(`Prometheus monitoring overview unavailable: ${error instanceof Error ? error.message : 'unknown error'}`);
|
||||
this.logger.warn(
|
||||
`Prometheus monitoring overview unavailable: ${error instanceof Error ? error.message : 'unknown error'}`,
|
||||
);
|
||||
return this.unavailable(range, collectedAt);
|
||||
}
|
||||
}
|
||||
|
||||
async notificationSummary(userId?: string) {
|
||||
try {
|
||||
const alerts = await this.attachReadState(this.parseAlerts(await this.getJson<PrometheusAlertResponse>('/api/v1/alerts')), userId);
|
||||
const alerts = await this.attachReadState(
|
||||
this.parseAlerts(await this.getJson<PrometheusAlertResponse>('/api/v1/alerts')),
|
||||
userId,
|
||||
);
|
||||
const unreadAlerts = alerts.filter((item) => !item.acknowledged);
|
||||
return { count: unreadAlerts.length, criticalCount: unreadAlerts.filter((item) => item.severity === 'critical').length };
|
||||
return {
|
||||
count: unreadAlerts.length,
|
||||
criticalCount: unreadAlerts.filter((item) => item.severity === 'critical').length,
|
||||
};
|
||||
} catch (error) {
|
||||
this.logger.warn(`Prometheus notification summary unavailable: ${error instanceof Error ? error.message : 'unknown error'}`);
|
||||
this.logger.warn(
|
||||
`Prometheus notification summary unavailable: ${error instanceof Error ? error.message : 'unknown error'}`,
|
||||
);
|
||||
throw new ServiceUnavailableException('Prometheus活动告警当前不可用');
|
||||
}
|
||||
}
|
||||
@@ -231,12 +282,21 @@ export class InfrastructureMonitoringService {
|
||||
const activeAt = new Date(String(rawActiveAt ?? ''));
|
||||
if (!Number.isFinite(activeAt.getTime())) throw new BadRequestException('告警开始时间无效');
|
||||
const activeAlerts = this.parseAlerts(await this.getJson<PrometheusAlertResponse>('/api/v1/alerts'));
|
||||
const current = activeAlerts.find((item) => item.fingerprint === fingerprint && Date.parse(item.startedAt) === activeAt.getTime());
|
||||
const current = activeAlerts.find(
|
||||
(item) => item.fingerprint === fingerprint && Date.parse(item.startedAt) === activeAt.getTime(),
|
||||
);
|
||||
if (!current) throw new NotFoundException('该次活动告警已结束或已重新触发,请刷新后重试');
|
||||
const readAt = new Date();
|
||||
const log = () => this.prisma.operationLog.create({
|
||||
data: { userId, action: 'monitoring.alert_marked_read', resource: 'infrastructure_alert', resourceId: fingerprint, detail: { activeAt: activeAt.toISOString(), alertName: current.name, severity: current.severity } },
|
||||
});
|
||||
const log = () =>
|
||||
this.prisma.operationLog.create({
|
||||
data: {
|
||||
userId,
|
||||
action: 'monitoring.alert_marked_read',
|
||||
resource: 'infrastructure_alert',
|
||||
resourceId: fingerprint,
|
||||
detail: { activeAt: activeAt.toISOString(), alertName: current.name, severity: current.severity },
|
||||
},
|
||||
});
|
||||
let read;
|
||||
try {
|
||||
[read] = await this.prisma.$transaction([
|
||||
@@ -245,15 +305,57 @@ export class InfrastructureMonitoringService {
|
||||
]);
|
||||
} catch (error) {
|
||||
if (!(error instanceof Prisma.PrismaClientKnownRequestError) || error.code !== 'P2002') throw error;
|
||||
const existing = await this.prisma.infrastructureAlertRead.findUniqueOrThrow({ where: { fingerprint_userId: { fingerprint, userId } } });
|
||||
const existing = await this.prisma.infrastructureAlertRead.findUniqueOrThrow({
|
||||
where: { fingerprint_userId: { fingerprint, userId } },
|
||||
});
|
||||
// 同一次触发重复点击不更新readAt也不重复写日志;activeAt变化才代表同指纹的新触发周期。
|
||||
if (existing.activeAt.getTime() === activeAt.getTime()) read = existing;
|
||||
else [read] = await this.prisma.$transaction([
|
||||
this.prisma.infrastructureAlertRead.update({ where: { fingerprint_userId: { fingerprint, userId } }, data: { activeAt, readAt } }),
|
||||
log(),
|
||||
]);
|
||||
else
|
||||
[read] = await this.prisma.$transaction([
|
||||
this.prisma.infrastructureAlertRead.update({
|
||||
where: { fingerprint_userId: { fingerprint, userId } },
|
||||
data: { activeAt, readAt },
|
||||
}),
|
||||
log(),
|
||||
]);
|
||||
}
|
||||
return { fingerprint, activeAt: read.activeAt.toISOString(), acknowledged: true, acknowledgedAt: read.readAt.toISOString() };
|
||||
return {
|
||||
fingerprint,
|
||||
activeAt: read.activeAt.toISOString(),
|
||||
acknowledged: true,
|
||||
acknowledgedAt: read.readAt.toISOString(),
|
||||
};
|
||||
}
|
||||
|
||||
async alertHistory(from?: string, to?: string, rawPage?: string) {
|
||||
const range = alertHistoryRange(from, to);
|
||||
const page = rawPage === undefined ? 1 : Number(rawPage);
|
||||
if (!Number.isSafeInteger(page) || page < 1) throw new BadRequestException('告警页码无效');
|
||||
const history = new Map<string, AlertHistoryItem>();
|
||||
try {
|
||||
// Daily raw range vectors retain short events that a coarse query_range step would miss.
|
||||
for (let start = range.start; start < range.end; start += 86400) {
|
||||
const end = Math.min(start + 86400, range.end);
|
||||
const response = await this.getJson<PrometheusQueryResponse>('/api/v1/query', {
|
||||
query: `ALERTS_FOR_STATE[${Math.ceil(end - start)}s]`,
|
||||
time: String(end),
|
||||
});
|
||||
mergeAlertHistory(history, response.data?.result ?? [], range.start, range.end);
|
||||
}
|
||||
} catch {
|
||||
throw new ServiceUnavailableException('历史告警查询失败,请稍后重试');
|
||||
}
|
||||
const items = [...history.values()].sort(
|
||||
(a, b) => b.startedAt.localeCompare(a.startedAt) || a.id.localeCompare(b.id),
|
||||
);
|
||||
return {
|
||||
items: items.slice((page - 1) * 25, page * 25),
|
||||
total: items.length,
|
||||
page,
|
||||
pageSize: 25,
|
||||
startDate: range.startDate,
|
||||
endDate: range.endDate,
|
||||
};
|
||||
}
|
||||
|
||||
private parseRange(value?: string): InfrastructureMonitoringRange {
|
||||
@@ -264,12 +366,21 @@ export class InfrastructureMonitoringService {
|
||||
|
||||
private async loadInstantMetrics() {
|
||||
const keys = Object.keys(emptyMetrics()) as Array<keyof InfrastructureMonitoringOverview['metrics']>;
|
||||
const responses = await Promise.all([...keys.map((key) => this.query(QUERIES[key])), this.query(QUERIES.lastSampleAt)]);
|
||||
const responses = await Promise.all([
|
||||
...keys.map((key) => this.query(QUERIES[key])),
|
||||
this.query(QUERIES.lastSampleAt),
|
||||
]);
|
||||
const metrics = emptyMetrics();
|
||||
keys.forEach((key, index) => { if (!key.startsWith('disk')) metrics[key] = vectorValue(responses[index]); });
|
||||
keys.forEach((key, index) => {
|
||||
if (!key.startsWith('disk')) metrics[key] = vectorValue(responses[index]);
|
||||
});
|
||||
const diskSamples = (key: keyof typeof metrics) => responses[keys.indexOf(key)].data?.result ?? [];
|
||||
const usage = new Map(diskSamples('diskUsagePercent').map((item) => [filesystemIdentity(item.metric), finiteNumber(item.value?.[1])]));
|
||||
const available = new Map(diskSamples('diskAvailableBytes').map((item) => [filesystemIdentity(item.metric), finiteNumber(item.value?.[1])]));
|
||||
const usage = new Map(
|
||||
diskSamples('diskUsagePercent').map((item) => [filesystemIdentity(item.metric), finiteNumber(item.value?.[1])]),
|
||||
);
|
||||
const available = new Map(
|
||||
diskSamples('diskAvailableBytes').map((item) => [filesystemIdentity(item.metric), finiteNumber(item.value?.[1])]),
|
||||
);
|
||||
const groups = new Map<string, PrometheusSeries[]>();
|
||||
for (const item of diskSamples('diskTotalBytes')) {
|
||||
if (!item.metric.device || !item.metric.mountpoint || (finiteNumber(item.value?.[1]) ?? 0) <= 0) continue;
|
||||
@@ -278,18 +389,32 @@ export class InfrastructureMonitoringService {
|
||||
group.push(item);
|
||||
groups.set(id, group);
|
||||
}
|
||||
const disks = [...groups].map(([id, items]) => {
|
||||
const mountpoints = [...new Set(items.map((item) => item.metric.mountpoint))].sort(compareMountpoints);
|
||||
const metric = items[0].metric;
|
||||
return {
|
||||
id, instance: metric.instance ?? '', device: metric.device, filesystem: metric.fstype ?? '',
|
||||
mountpoint: mountpoints[0], mountpoints,
|
||||
// Never sum aliases. Max/min also tolerate slight sampling differences.
|
||||
totalBytes: Math.max(...items.map((item) => finiteNumber(item.value?.[1])!)),
|
||||
availableBytes: available.get(id) ?? null, usagePercent: usage.get(id) ?? null,
|
||||
};
|
||||
})
|
||||
.sort((left, right) => left.instance.localeCompare(right.instance) || (left.mountpoint === '/' ? -1 : right.mountpoint === '/' ? 1 : left.mountpoint.localeCompare(right.mountpoint)));
|
||||
const disks = [...groups]
|
||||
.map(([id, items]) => {
|
||||
const mountpoints = [...new Set(items.map((item) => item.metric.mountpoint))].sort(compareMountpoints);
|
||||
const metric = items[0].metric;
|
||||
return {
|
||||
id,
|
||||
instance: metric.instance ?? '',
|
||||
device: metric.device,
|
||||
filesystem: metric.fstype ?? '',
|
||||
mountpoint: mountpoints[0],
|
||||
mountpoints,
|
||||
// Never sum aliases. Max/min also tolerate slight sampling differences.
|
||||
totalBytes: Math.max(...items.map((item) => finiteNumber(item.value?.[1])!)),
|
||||
availableBytes: available.get(id) ?? null,
|
||||
usagePercent: usage.get(id) ?? null,
|
||||
};
|
||||
})
|
||||
.sort(
|
||||
(left, right) =>
|
||||
left.instance.localeCompare(right.instance) ||
|
||||
(left.mountpoint === '/'
|
||||
? -1
|
||||
: right.mountpoint === '/'
|
||||
? 1
|
||||
: left.mountpoint.localeCompare(right.mountpoint)),
|
||||
);
|
||||
const rootDisk = disks.find((disk) => disk.mountpoints.includes('/'));
|
||||
metrics.diskUsagePercent = rootDisk?.usagePercent ?? null;
|
||||
metrics.diskTotalBytes = rootDisk?.totalBytes ?? null;
|
||||
@@ -304,21 +429,32 @@ export class InfrastructureMonitoringService {
|
||||
const keys = Object.keys(emptyTrends()) as Array<keyof InfrastructureMonitoringOverview['trends']>;
|
||||
const responses = await Promise.all(keys.map((key) => this.queryRange(QUERIES[key], start, end, config.step)));
|
||||
return {
|
||||
metrics: Object.fromEntries(keys.map((key, index) => [key, key === 'diskUsagePercent' ? [] : matrixValues(responses[index])])) as InfrastructureMonitoringOverview['trends'],
|
||||
disks: new Map((responses[keys.indexOf('diskUsagePercent')].data?.result ?? []).map((item) => [
|
||||
filesystemIdentity(item.metric), matrixValues({ status: 'success', data: { result: [item] } }),
|
||||
])),
|
||||
metrics: Object.fromEntries(
|
||||
keys.map((key, index) => [key, key === 'diskUsagePercent' ? [] : matrixValues(responses[index])]),
|
||||
) as InfrastructureMonitoringOverview['trends'],
|
||||
disks: new Map(
|
||||
(responses[keys.indexOf('diskUsagePercent')].data?.result ?? []).map((item) => [
|
||||
filesystemIdentity(item.metric),
|
||||
matrixValues({ status: 'success', data: { result: [item] } }),
|
||||
]),
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
private parseServices(response: PrometheusQueryResponse): InfrastructureServiceStatus[] {
|
||||
const values = new Map<string, number>();
|
||||
for (const item of response.data?.result ?? []) {
|
||||
if (item.metric.name) values.set(item.metric.name, vectorValue({ status: 'success', data: { result: [item] } }) ?? 0);
|
||||
if (item.metric.name)
|
||||
values.set(item.metric.name, vectorValue({ status: 'success', data: { result: [item] } }) ?? 0);
|
||||
}
|
||||
return SERVICE_DEFINITIONS.map((definition) => {
|
||||
const present = definition.units.filter((unit) => values.has(unit));
|
||||
const status = present.length === 0 ? 'unknown' : present.some((unit) => (values.get(unit) ?? 0) >= 1) ? 'healthy' : 'unhealthy';
|
||||
const status =
|
||||
present.length === 0
|
||||
? 'unknown'
|
||||
: present.some((unit) => (values.get(unit) ?? 0) >= 1)
|
||||
? 'healthy'
|
||||
: 'unhealthy';
|
||||
return { key: definition.key, name: definition.name, unit: present[0] ?? definition.units[0], status };
|
||||
});
|
||||
}
|
||||
@@ -329,7 +465,8 @@ export class InfrastructureMonitoringService {
|
||||
.map<InfrastructureAlert>((item) => {
|
||||
const labels = item.labels ?? {};
|
||||
const annotations = item.annotations ?? {};
|
||||
const severity: InfrastructureAlert['severity'] = labels.severity === 'critical' ? 'critical' : labels.severity === 'warning' ? 'warning' : 'info';
|
||||
const severity: InfrastructureAlert['severity'] =
|
||||
labels.severity === 'critical' ? 'critical' : labels.severity === 'warning' ? 'warning' : 'info';
|
||||
const identity = JSON.stringify(Object.entries(labels).sort(([left], [right]) => left.localeCompare(right)));
|
||||
return {
|
||||
fingerprint: createHash('sha256').update(identity).digest('hex').slice(0, 24),
|
||||
@@ -348,7 +485,9 @@ export class InfrastructureMonitoringService {
|
||||
})
|
||||
.sort((left, right) => {
|
||||
const priority: Record<InfrastructureAlert['severity'], number> = { critical: 0, warning: 1, info: 2 };
|
||||
return priority[left.severity] - priority[right.severity] || Date.parse(left.startedAt) - Date.parse(right.startedAt);
|
||||
return (
|
||||
priority[left.severity] - priority[right.severity] || Date.parse(left.startedAt) - Date.parse(right.startedAt)
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
@@ -377,24 +516,46 @@ export class InfrastructureMonitoringService {
|
||||
key: group.key,
|
||||
name: group.name,
|
||||
available: group.metrics.some((metric) => values.has(metric[2])),
|
||||
metrics: group.metrics.map(([key, label, metricName, unit]) => ({ key, label, value: values.get(metricName) ?? null, unit })),
|
||||
metrics: group.metrics.map(([key, label, metricName, unit]) => ({
|
||||
key,
|
||||
label,
|
||||
value: values.get(metricName) ?? null,
|
||||
unit,
|
||||
})),
|
||||
}));
|
||||
}
|
||||
|
||||
private unavailable(range: InfrastructureMonitoringRange, collectedAt: string): InfrastructureMonitoringOverview {
|
||||
const services = SERVICE_DEFINITIONS.map((item) => ({ key: item.key, name: item.name, unit: item.units[0], status: 'unknown' as const }));
|
||||
const services = SERVICE_DEFINITIONS.map((item) => ({
|
||||
key: item.key,
|
||||
name: item.name,
|
||||
unit: item.units[0],
|
||||
status: 'unknown' as const,
|
||||
}));
|
||||
return {
|
||||
available: false,
|
||||
range,
|
||||
collectedAt,
|
||||
lastSampleAt: null,
|
||||
error: 'Prometheus监控数据当前不可用,请检查采集与服务状态',
|
||||
summary: { overallStatus: 'unknown', serviceTotal: services.length, serviceHealthy: 0, warningAlerts: 0, criticalAlerts: 0, activeAlerts: 0 },
|
||||
summary: {
|
||||
overallStatus: 'unknown',
|
||||
serviceTotal: services.length,
|
||||
serviceHealthy: 0,
|
||||
warningAlerts: 0,
|
||||
criticalAlerts: 0,
|
||||
activeAlerts: 0,
|
||||
},
|
||||
metrics: emptyMetrics(),
|
||||
disks: [],
|
||||
trends: emptyTrends(),
|
||||
services,
|
||||
serviceMetrics: SERVICE_METRIC_DEFINITIONS.map((group) => ({ key: group.key, name: group.name, available: false, metrics: [] })),
|
||||
serviceMetrics: SERVICE_METRIC_DEFINITIONS.map((group) => ({
|
||||
key: group.key,
|
||||
name: group.name,
|
||||
available: false,
|
||||
metrics: [],
|
||||
})),
|
||||
alerts: [],
|
||||
};
|
||||
}
|
||||
@@ -404,15 +565,26 @@ export class InfrastructureMonitoringService {
|
||||
}
|
||||
|
||||
private queryRange(query: string, start: number, end: number, step: number) {
|
||||
return this.getJson<PrometheusQueryResponse>('/api/v1/query_range', { query, start: String(start), end: String(end), step: String(step) });
|
||||
return this.getJson<PrometheusQueryResponse>('/api/v1/query_range', {
|
||||
query,
|
||||
start: String(start),
|
||||
end: String(end),
|
||||
step: String(step),
|
||||
});
|
||||
}
|
||||
|
||||
private async getJson<T extends { status: 'success' | 'error'; error?: string }>(path: string, params: Record<string, string> = {}): Promise<T> {
|
||||
private async getJson<T extends { status: 'success' | 'error'; error?: string }>(
|
||||
path: string,
|
||||
params: Record<string, string> = {},
|
||||
): Promise<T> {
|
||||
const url = new URL(`${this.prometheusUrl}${path}`);
|
||||
Object.entries(params).forEach(([key, value]) => url.searchParams.set(key, value));
|
||||
const response = await fetch(url, { headers: { Accept: 'application/json' }, signal: AbortSignal.timeout(this.queryTimeoutMs) });
|
||||
const response = await fetch(url, {
|
||||
headers: { Accept: 'application/json' },
|
||||
signal: AbortSignal.timeout(this.queryTimeoutMs),
|
||||
});
|
||||
if (!response.ok) throw new Error(`Prometheus HTTP ${response.status}`);
|
||||
const result = await response.json() as T;
|
||||
const result = (await response.json()) as T;
|
||||
if (result.status !== 'success') throw new Error('Prometheus query failed');
|
||||
return result;
|
||||
}
|
||||
|
||||
@@ -634,7 +634,7 @@ describe('OperationsService', () => {
|
||||
},
|
||||
today: expect.objectContaining({
|
||||
returnedCents: 10,
|
||||
segmentCount: 20,
|
||||
segmentCount: 2,
|
||||
deliveredSegmentCount: 18,
|
||||
arrivalRate: 90,
|
||||
billedCents: 360,
|
||||
|
||||
@@ -1,16 +1,24 @@
|
||||
import { BadRequestException, NotFoundException } from '@nestjs/common';
|
||||
import { Prisma } from '@prisma/client';
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import { moneyToNumber } from '../../common/money';
|
||||
import { PrismaService } from '../../prisma/prisma.service';
|
||||
import type { MessageQuery, TraceQuery, OperationLogQuery, GatewaySubmitDeadLetterQuery, DownstreamDeliveryQuery, DownstreamDeliveryDashboardQuery, DownstreamRecoveryStatusQuery, MessageSegmentAuditQuery, SignatureQualityQuery } from '../operations.contracts';
|
||||
import { messageWhere, recognizedCarrierValues, carrierWhere, startOfShanghaiDay, endOfShanghaiDay, qualityBusinessDay, shanghaiDateKey, normalizeGroupBy, returnedTransactionWhere, createdAtRange, downstreamAlertPendingMinutes, downstreamAlertRecentFailedHours, downstreamAlertWindows, downstreamAlertWhere, stalledPendingWhere, downstreamDeliveryScopedWhere, parseDateBoundary, downstreamRecoveryStatusWhere, escapeCsvCell, formatCsvDate, formatExportTimestamp, clientApplicationView, clientReceiptView, clientMessageView, clientBatchTaskView, clientUplinkView, clientAccountView, clientRechargeView, summarizeMessageGroups, groupDownstreamByType, groupDownstreamByApplication, positiveInteger, operationLogLevelWhere, normalizeOperationLog, sanitizeGatewaySubmitException, redactGatewayCommandValue } from '../operations.helpers';
|
||||
|
||||
import {
|
||||
messageWhere,
|
||||
qualityBusinessDay,
|
||||
returnedTransactionWhere,
|
||||
downstreamAlertWindows,
|
||||
stalledPendingWhere,
|
||||
clientBatchTaskView,
|
||||
clientAccountView,
|
||||
clientRechargeView,
|
||||
summarizeMessageGroups,
|
||||
} from '../operations.helpers';
|
||||
|
||||
// R2 dashboard query domain. Method bodies are preserved byte-for-byte from the facade baseline.
|
||||
export class OperationsDashboardQueries {
|
||||
constructor(private readonly prisma: PrismaService) {}
|
||||
|
||||
async dashboard(query: { tenantId?: string }) {
|
||||
async dashboard(query: { tenantId?: string }) {
|
||||
const businessDay = qualityBusinessDay();
|
||||
const sinceToday = businessDay.startAt;
|
||||
const downstreamAlertWindow = downstreamAlertWindows();
|
||||
@@ -94,13 +102,15 @@ async dashboard(query: { tenantId?: string }) {
|
||||
orderBy: { createdAt: 'desc' },
|
||||
take: 10,
|
||||
}),
|
||||
this.prisma.$queryRaw<Array<{
|
||||
tenantId: string;
|
||||
tenantName: string;
|
||||
todaySpendCents: bigint;
|
||||
balanceCents: bigint;
|
||||
creditCents: bigint;
|
||||
}>>(Prisma.sql`
|
||||
this.prisma.$queryRaw<
|
||||
Array<{
|
||||
tenantId: string;
|
||||
tenantName: string;
|
||||
todaySpendCents: bigint;
|
||||
balanceCents: bigint;
|
||||
creditCents: bigint;
|
||||
}>
|
||||
>(Prisma.sql`
|
||||
SELECT
|
||||
tenant.id AS "tenantId",
|
||||
tenant.name AS "tenantName",
|
||||
@@ -118,12 +128,14 @@ async dashboard(query: { tenantId?: string }) {
|
||||
GROUP BY tenant.id, tenant.name, account."balanceCents", account."creditCents"
|
||||
ORDER BY "todaySpendCents" DESC, tenant.name ASC
|
||||
`),
|
||||
this.prisma.$queryRaw<Array<{
|
||||
segmentCount: bigint;
|
||||
deliveredSegmentCount: bigint;
|
||||
billedCents: bigint;
|
||||
costCents: bigint;
|
||||
}>>(Prisma.sql`
|
||||
this.prisma.$queryRaw<
|
||||
Array<{
|
||||
segmentCount: bigint;
|
||||
deliveredSegmentCount: bigint;
|
||||
billedCents: bigint;
|
||||
costCents: bigint;
|
||||
}>
|
||||
>(Prisma.sql`
|
||||
WITH segment_metrics AS (
|
||||
SELECT
|
||||
COUNT(segment.id)::bigint AS "segmentCount",
|
||||
@@ -208,11 +220,13 @@ async dashboard(query: { tenantId?: string }) {
|
||||
updatedAt: { gte: downstreamAlertWindow.recentFailedAt },
|
||||
},
|
||||
}),
|
||||
this.prisma.$queryRaw<Array<{
|
||||
hour: number;
|
||||
submittedCount: bigint;
|
||||
successCount: bigint;
|
||||
}>>(Prisma.sql`
|
||||
this.prisma.$queryRaw<
|
||||
Array<{
|
||||
hour: number;
|
||||
submittedCount: bigint;
|
||||
successCount: bigint;
|
||||
}>
|
||||
>(Prisma.sql`
|
||||
SELECT
|
||||
EXTRACT(
|
||||
HOUR FROM (message."queuedAt" AT TIME ZONE 'UTC') AT TIME ZONE 'Asia/Shanghai'
|
||||
@@ -227,11 +241,13 @@ async dashboard(query: { tenantId?: string }) {
|
||||
ORDER BY 1
|
||||
`),
|
||||
// Signature/template tables have no review timestamps, so their latest pending audit is paired with the review audit.
|
||||
this.prisma.$queryRaw<Array<{
|
||||
category: string;
|
||||
count: bigint;
|
||||
averageProcessingMs: bigint | null;
|
||||
}>>(Prisma.sql`
|
||||
this.prisma.$queryRaw<
|
||||
Array<{
|
||||
category: string;
|
||||
count: bigint;
|
||||
averageProcessingMs: bigint | null;
|
||||
}>
|
||||
>(Prisma.sql`
|
||||
WITH review_samples AS (
|
||||
SELECT
|
||||
'enterpriseCertifications'::text AS category,
|
||||
@@ -302,7 +318,8 @@ async dashboard(query: { tenantId?: string }) {
|
||||
]);
|
||||
const todayTotals = summarizeMessageGroups(todayMessageGroups);
|
||||
const todayBusinessMetrics = todayBusinessMetricsRows[0];
|
||||
const segmentCount = Number(todayBusinessMetrics?.segmentCount ?? 0);
|
||||
const supplierSegmentCount = Number(todayBusinessMetrics?.segmentCount ?? 0);
|
||||
const segmentCount = todayTotals.billingUnits;
|
||||
const deliveredSegmentCount = Number(todayBusinessMetrics?.deliveredSegmentCount ?? 0);
|
||||
const billedCents = moneyToNumber(todayBusinessMetrics?.billedCents);
|
||||
const costCents = moneyToNumber(todayBusinessMetrics?.costCents);
|
||||
@@ -334,7 +351,8 @@ async dashboard(query: { tenantId?: string }) {
|
||||
averageProcessingMs: row?.averageProcessingMs == null ? null : Number(row.averageProcessingMs),
|
||||
};
|
||||
});
|
||||
const downstreamAlertCount = downstreamStalledPendingCount + downstreamStalledAckCount + downstreamRecentFailedCount;
|
||||
const downstreamAlertCount =
|
||||
downstreamStalledPendingCount + downstreamStalledAckCount + downstreamRecentFailedCount;
|
||||
return {
|
||||
taskCount,
|
||||
messageStatus: messageGroups,
|
||||
@@ -349,7 +367,8 @@ async dashboard(query: { tenantId?: string }) {
|
||||
billingUnits: todayTotals.billingUnits,
|
||||
segmentCount,
|
||||
deliveredSegmentCount,
|
||||
arrivalRate: segmentCount > 0 ? Number(((deliveredSegmentCount / segmentCount) * 100).toFixed(1)) : 0,
|
||||
arrivalRate:
|
||||
supplierSegmentCount > 0 ? Number(((deliveredSegmentCount / supplierSegmentCount) * 100).toFixed(1)) : 0,
|
||||
billedCents,
|
||||
profitCents,
|
||||
profitRate: billedCents > 0 ? Number(((profitCents / billedCents) * 100).toFixed(1)) : 0,
|
||||
@@ -383,7 +402,7 @@ async dashboard(query: { tenantId?: string }) {
|
||||
recentRecharges,
|
||||
};
|
||||
}
|
||||
async clientDashboard(query: { tenantId?: string }) {
|
||||
async clientDashboard(query: { tenantId?: string }) {
|
||||
const tenantId = query.tenantId;
|
||||
const [dashboard, tenant, approvedCertification, signatureCount, pendingBatchTaskCount] = await Promise.all([
|
||||
this.dashboard(query),
|
||||
@@ -432,7 +451,7 @@ async clientDashboard(query: { tenantId?: string }) {
|
||||
},
|
||||
};
|
||||
}
|
||||
pendingAudits(tenantId?: string) {
|
||||
pendingAudits(tenantId?: string) {
|
||||
return Promise.all([
|
||||
this.prisma.smsTemplate.count({ where: { tenantId, auditStatus: 'pending' } }),
|
||||
this.prisma.smsSignature.count({ where: { tenantId, auditStatus: 'pending' } }),
|
||||
|
||||
@@ -0,0 +1,27 @@
|
||||
import { SendingMonitorService } from './sending-monitor.module';
|
||||
|
||||
describe('sending alert read filters', () => {
|
||||
it.each(['', 'read', 'unread'])(
|
||||
'applies identical user/state/read criteria to rows and total: %s',
|
||||
async (readStatus) => {
|
||||
const prisma = {
|
||||
$queryRawUnsafe: jest
|
||||
.fn()
|
||||
.mockResolvedValueOnce([])
|
||||
.mockResolvedValueOnce([{ total: 0 }]),
|
||||
};
|
||||
await new SendingMonitorService(prisma as never).alerts({ state: 'active', readStatus, page: '2' }, 'user-a');
|
||||
const [list, count] = prisma.$queryRawUnsafe.mock.calls;
|
||||
expect(list.slice(1)).toEqual(['user-a', 'active', readStatus, 20, 20]);
|
||||
expect(count.slice(1)).toEqual(['user-a', 'active', readStatus]);
|
||||
expect(list[0].split('FROM')[1].split('ORDER BY')[0].trim()).toBe(count[0].split('FROM')[1].trim());
|
||||
},
|
||||
);
|
||||
it('rejects unknown read states before querying', async () => {
|
||||
const prisma = { $queryRawUnsafe: jest.fn() };
|
||||
await expect(new SendingMonitorService(prisma as never).alerts({ readStatus: 'bogus' }, 'user-a')).rejects.toThrow(
|
||||
'已读状态无效',
|
||||
);
|
||||
expect(prisma.$queryRawUnsafe).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
@@ -312,17 +312,23 @@ export class SendingMonitorService {
|
||||
size = pageNumber(query.pageSize, 20, 100);
|
||||
const state = query.state ?? '';
|
||||
if (state && !['active', 'recovered', 'closed'].includes(state)) throw new BadRequestException('告警状态无效');
|
||||
const readStatus = query.readStatus ?? '';
|
||||
if (!['', 'read', 'unread'].includes(readStatus)) throw new BadRequestException('已读状态无效');
|
||||
const from = `FROM "SendingMonitorAlert" a LEFT JOIN "SendingMonitorAlertRead" r ON r."alertId"=a.id AND r."userId"=$1 WHERE ($2='' OR a.state=$2) AND ($3='' OR ($3='unread' AND r."readAt" IS NULL) OR ($3='read' AND r."readAt" IS NOT NULL))`;
|
||||
const [items, total] = await Promise.all([
|
||||
this.prisma.$queryRawUnsafe(
|
||||
`SELECT a.*,r."readAt" IS NULL unread FROM "SendingMonitorAlert" a LEFT JOIN "SendingMonitorAlertRead" r ON r."alertId"=a.id AND r."userId"=$1 WHERE ($2='' OR a.state=$2) ORDER BY a."openedAt" DESC,a.id LIMIT $3 OFFSET $4`,
|
||||
`SELECT a.*,r."readAt" IS NULL unread ${from} ORDER BY a."openedAt" DESC,a.id LIMIT $4 OFFSET $5`,
|
||||
user,
|
||||
state,
|
||||
readStatus,
|
||||
size,
|
||||
(page - 1) * size,
|
||||
),
|
||||
this.prisma.$queryRawUnsafe<Array<{ total: number }>>(
|
||||
`SELECT count(*)::int total FROM "SendingMonitorAlert" WHERE ($1='' OR state=$1)`,
|
||||
`SELECT count(*)::int total ${from}`,
|
||||
user,
|
||||
state,
|
||||
readStatus,
|
||||
),
|
||||
]);
|
||||
return { items, total: total[0].total, page, pageSize: size };
|
||||
|
||||
Reference in New Issue
Block a user