209 lines
8.1 KiB
TypeScript
209 lines
8.1 KiB
TypeScript
import { BadRequestException } from '@nestjs/common';
|
||
import { createHash } from 'node:crypto';
|
||
import type { Prisma } from '@prisma/client';
|
||
import type { PrismaService } from '../prisma/prisma.service';
|
||
|
||
export type DrainageDetectionCategory = 'url' | 'mobile' | 'landline' | string;
|
||
|
||
export type DrainageDetectionRuleSnapshot = {
|
||
id: string;
|
||
code: string;
|
||
name: string;
|
||
category: DrainageDetectionCategory;
|
||
pattern: string;
|
||
flags: string;
|
||
priority: number;
|
||
version: number;
|
||
};
|
||
|
||
export type DrainageDetectionMatch = {
|
||
ruleId: string;
|
||
ruleCode: string;
|
||
ruleName: string;
|
||
category: DrainageDetectionCategory;
|
||
text: string;
|
||
normalizedText: string;
|
||
start: number;
|
||
end: number;
|
||
};
|
||
|
||
export type DrainageDetectionResult = {
|
||
hasDrainageContent: boolean;
|
||
drainageDetection: Prisma.InputJsonValue;
|
||
drainageDetectionVersion: string;
|
||
drainageEvaluatedAt: Date;
|
||
};
|
||
|
||
type NormalizedContent = {
|
||
text: string;
|
||
sourceStarts: number[];
|
||
sourceEnds: number[];
|
||
};
|
||
|
||
const RULE_CACHE_TTL_MS = 30_000;
|
||
const MAX_PATTERN_LENGTH = 1_000;
|
||
const MAX_CONTENT_LENGTH = 20_000;
|
||
const MAX_MATCHES = 50;
|
||
|
||
let cachedRules: { expiresAt: number; rules: DrainageDetectionRuleSnapshot[] } | undefined;
|
||
|
||
export function invalidateDrainageDetectionRuleCache() {
|
||
cachedRules = undefined;
|
||
}
|
||
|
||
export function validateDrainageDetectionPattern(pattern: string, flags = 'giu') {
|
||
if (!pattern.trim()) throw new BadRequestException('识别表达式不能为空');
|
||
if (pattern.length > MAX_PATTERN_LENGTH) throw new BadRequestException(`识别表达式不能超过 ${MAX_PATTERN_LENGTH} 个字符`);
|
||
if (!/^[giu]*$/.test(flags) || new Set(flags).size !== flags.length) {
|
||
throw new BadRequestException('表达式标志仅支持 g、i、u，且不能重复');
|
||
}
|
||
// 可配置规则会运行在发送入口，禁止容易造成灾难性回溯或跨文本引用的结构。
|
||
if (/\\[1-9]/.test(pattern) || /\(\?<([=!])/.test(pattern) || /\([^)]*[+*][^)]*\)[+*{]/.test(pattern)) {
|
||
throw new BadRequestException('表达式包含不安全的回溯、后行断言或嵌套量词');
|
||
}
|
||
try {
|
||
// 强制全局匹配，避免配置遗漏 g 后只能识别首个命中。
|
||
new RegExp(pattern, flags.includes('g') ? flags : `${flags}g`);
|
||
} catch {
|
||
throw new BadRequestException('识别表达式格式不正确');
|
||
}
|
||
}
|
||
|
||
function normalizeContent(content: string, category: DrainageDetectionCategory): NormalizedContent {
|
||
let text = '';
|
||
const sourceStarts: number[] = [];
|
||
const sourceEnds: number[] = [];
|
||
let sourceIndex = 0;
|
||
for (const sourceChar of content.slice(0, MAX_CONTENT_LENGTH)) {
|
||
const sourceEnd = sourceIndex + sourceChar.length;
|
||
let normalized = sourceChar.normalize('NFKC')
|
||
.replace(/[．｡]/g, '.')
|
||
.replace(/[：﹕]/g, ':')
|
||
.replace(/[／]/g, '/')
|
||
.replace(/[（）]/g, (char) => char === '（' ? '(' : ')')
|
||
.replace(/[＋]/g, '+');
|
||
if (category === 'url') {
|
||
// 链接常被空格或中文句号拆开；句末中文句号也安全地成为正则边界。
|
||
normalized = normalized.replace(/\s+/gu, '').replace(/。/g, '.');
|
||
} else if (category === 'mobile' || category === 'landline') {
|
||
// 电话号码仅在检测副本中去除常见规避分隔符，绝不改写实际发送内容。
|
||
normalized = normalized.replace(/[\s\-‐‑‒–—―.。·,，、]/gu, '');
|
||
}
|
||
for (const char of normalized) {
|
||
text += char;
|
||
// RegExp.index 使用 UTF-16 code unit，映射数组必须采用相同计数方式，避免表情符号导致高亮偏移。
|
||
for (let codeUnit = 0; codeUnit < char.length; codeUnit += 1) {
|
||
sourceStarts.push(sourceIndex);
|
||
sourceEnds.push(sourceEnd);
|
||
}
|
||
}
|
||
sourceIndex = sourceEnd;
|
||
}
|
||
return { text, sourceStarts, sourceEnds };
|
||
}
|
||
|
||
function sourceRange(normalized: NormalizedContent, start: number, end: number) {
|
||
const safeStart = Math.max(0, Math.min(start, normalized.sourceStarts.length - 1));
|
||
const safeEnd = Math.max(safeStart, Math.min(end - 1, normalized.sourceEnds.length - 1));
|
||
return {
|
||
start: normalized.sourceStarts[safeStart] ?? 0,
|
||
end: normalized.sourceEnds[safeEnd] ?? 0,
|
||
};
|
||
}
|
||
|
||
function emailRanges(normalized: NormalizedContent) {
|
||
const ranges: Array<{ start: number; end: number }> = [];
|
||
const email = /[a-z0-9.!#$%&'*+/=?^_`{|}~-]+@[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?(?:\.[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?)+/giu;
|
||
for (const match of normalized.text.matchAll(email)) {
|
||
ranges.push({ start: match.index, end: match.index + match[0].length });
|
||
}
|
||
return ranges;
|
||
}
|
||
|
||
function overlaps(start: number, end: number, range: { start: number; end: number }) {
|
||
return start < range.end && end > range.start;
|
||
}
|
||
|
||
export function detectDrainageContentWithRules(
|
||
content: string,
|
||
rules: DrainageDetectionRuleSnapshot[],
|
||
evaluatedAt = new Date(),
|
||
): DrainageDetectionResult {
|
||
const matches: DrainageDetectionMatch[] = [];
|
||
const normalizedByCategory = new Map<string, NormalizedContent>();
|
||
const emailNormalized = normalizeContent(content, 'url');
|
||
const originalEmailRanges = emailRanges(emailNormalized).map((range) => sourceRange(emailNormalized, range.start, range.end));
|
||
for (const rule of [...rules].sort((a, b) => a.priority - b.priority || a.code.localeCompare(b.code))) {
|
||
validateDrainageDetectionPattern(rule.pattern, rule.flags);
|
||
const normalized = normalizedByCategory.get(rule.category) ?? normalizeContent(content, rule.category);
|
||
normalizedByCategory.set(rule.category, normalized);
|
||
const regex = new RegExp(rule.pattern, rule.flags.includes('g') ? rule.flags : `${rule.flags}g`);
|
||
for (const match of normalized.text.matchAll(regex)) {
|
||
const captured = match[1] || match[0];
|
||
const capturedOffset = match[0].indexOf(captured);
|
||
const normalizedStart = match.index + Math.max(0, capturedOffset);
|
||
const normalizedEnd = normalizedStart + captured.length;
|
||
const range = sourceRange(normalized, normalizedStart, normalizedEnd);
|
||
if (range.end <= range.start) continue;
|
||
// 邮箱整体不是引流信息；不仅排除其中的域名，也排除数字本地部分被电话规则误识别。
|
||
if (originalEmailRanges.some((emailRange) => overlaps(range.start, range.end, emailRange))) continue;
|
||
const candidate: DrainageDetectionMatch = {
|
||
ruleId: rule.id,
|
||
ruleCode: rule.code,
|
||
ruleName: rule.name,
|
||
category: rule.category,
|
||
text: content.slice(range.start, range.end),
|
||
normalizedText: captured,
|
||
start: range.start,
|
||
end: range.end,
|
||
};
|
||
if (!matches.some((item) => item.category === candidate.category && item.start === candidate.start && item.end === candidate.end)) {
|
||
matches.push(candidate);
|
||
}
|
||
if (matches.length >= MAX_MATCHES) break;
|
||
}
|
||
if (matches.length >= MAX_MATCHES) break;
|
||
}
|
||
matches.sort((a, b) => a.start - b.start || a.end - b.end);
|
||
const versionSource = rules
|
||
.map((rule) => `${rule.code}:${rule.version}`)
|
||
.sort()
|
||
.join('|');
|
||
const drainageDetectionVersion = createHash('sha256').update(versionSource).digest('hex').slice(0, 16);
|
||
return {
|
||
hasDrainageContent: matches.length > 0,
|
||
drainageDetection: {
|
||
matches,
|
||
categories: [...new Set(matches.map((item) => item.category))],
|
||
ruleCount: rules.length,
|
||
truncated: content.length > MAX_CONTENT_LENGTH || matches.length >= MAX_MATCHES,
|
||
} as Prisma.InputJsonValue,
|
||
drainageDetectionVersion,
|
||
drainageEvaluatedAt: evaluatedAt,
|
||
};
|
||
}
|
||
|
||
async function activeRules(prisma: PrismaService) {
|
||
if (cachedRules && cachedRules.expiresAt > Date.now()) return cachedRules.rules;
|
||
const rules = await prisma.drainageDetectionRule.findMany({
|
||
where: { status: 'active' },
|
||
orderBy: [{ priority: 'asc' }, { createdAt: 'asc' }],
|
||
select: {
|
||
id: true,
|
||
code: true,
|
||
name: true,
|
||
category: true,
|
||
pattern: true,
|
||
flags: true,
|
||
priority: true,
|
||
version: true,
|
||
},
|
||
});
|
||
cachedRules = { rules, expiresAt: Date.now() + RULE_CACHE_TTL_MS };
|
||
return rules;
|
||
}
|
||
|
||
export async function detectDrainageContent(prisma: PrismaService, content: string) {
|
||
return detectDrainageContentWithRules(content, await activeRules(prisma));
|
||
}
|