hectorzhao/lislgosms
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: 异步解析大文件WPS报备资料

Browse Source
This commit is contained in:
hectorzhao
2026-09-04 22:51:07 +08:00
parent aaf96db2d0
commit 5925cf493b
22 changed files with 720 additions and 80 deletions
Download Patch File Download Diff File Expand all files Collapse all files
api/src/report-materials/workbook-compatibility.ts
+68 -16
View File
@@ -2,7 +2,7 @@ import { BadRequestException } from '@nestjs/common';
import ExcelJS from 'exceljs';
import JSZip from 'jszip';
import { randomUUID } from 'node:crypto';
import type { EmbeddedImage, ReportWorkbookFormat } from './report-materials.contracts';
import type { EmbeddedImage, EmbeddedImageMetadata, ReportWorkbookFormat } from './report-materials.contracts';
import { normalizeImageExtension, readEmbeddedImages } from './report-materials.helpers';
const MAX_WORKBOOK_BYTES = 100 * 1024 * 1024;
@@ -72,9 +72,14 @@ function validateWorkbookValues(workbook: ExcelJS.Workbook, allowedWpsIds: Set<s
}
}
async function inspectWpsImages(zip: JSZip) {
async function inspectWpsImages(zip: JSZip, includeImageData: boolean) {
const cellImagesXml = await text(zip, 'xl/cellimages.xml');
if (!cellImagesXml) return { allowedIds: new Set<string>(), imagesBySheet: new Map<string, EmbeddedImage[]>() };
if (!cellImagesXml)
return {
allowedIds: new Set<string>(),
imagesBySheet: new Map<string, Array<EmbeddedImage | EmbeddedImageMetadata>>(),
mediaTargets: new Set<string>(),
};
const relXml = await text(zip, 'xl/_rels/cellimages.xml.rels');
const relTargets = new Map<string, string>();
for (const match of relXml.matchAll(/<Relationship\b([^>]*)\/?>(?:<\/Relationship>)?/g)) {
@@ -99,13 +104,14 @@ async function inspectWpsImages(zip: JSZip) {
sheetRelTargets.set(attrs.Id, packagePath(attrs.Target));
}
const allowedIds = new Set<string>();
const imagesBySheet = new Map<string, EmbeddedImage[]>();
const imagesBySheet = new Map<string, Array<EmbeddedImage | EmbeddedImageMetadata>>();
let totalImageBytes = 0;
for (const match of workbookXml.matchAll(/<sheet\b([^>]*)\/?>(?:<\/sheet>)?/g)) {
const attrs = attributes(match[1]);
const path = sheetRelTargets.get(attrs['r:id']);
if (!attrs.name || !path) continue;
const sheetXml = await text(zip, path);
const images: EmbeddedImage[] = [];
const images: Array<EmbeddedImage | EmbeddedImageMetadata> = [];
for (const cell of sheetXml.matchAll(/<c\b([^>]*)>([\s\S]*?)<\/c>/g)) {
const formulaText = decodeXml(cell[2].match(/<f(?:\s[^>]*)?>([\s\S]*?)<\/f>/)?.[1]?.trim() ?? '');
if (!formulaText) continue;
@@ -118,20 +124,31 @@ async function inspectWpsImages(zip: JSZip) {
if (!imageEntry) throw new BadRequestException('WPS单元格图片文件缺失');
const extension = normalizeImageExtension(target.split('.').pop() ?? 'png');
if (!['png', 'jpeg', 'gif'].includes(extension)) throw new BadRequestException('WPS单元格图片格式不受支持');
const imageBuffer = await imageEntry.async('nodebuffer');
if (imageBuffer.length > MAX_IMAGE_BYTES) throw new BadRequestException('单张WPS图片不能超过20MB');
if (!validImageSignature(extension, imageBuffer)) throw new BadRequestException('WPS单元格图片内容与格式不匹配');
images.push({ ...cellPosition, extension, buffer: imageBuffer });
const size = Number(
(imageEntry as unknown as { _data?: { uncompressedSize?: number } })._data?.uncompressedSize ?? 0,
);
if (size > MAX_IMAGE_BYTES) throw new BadRequestException('单张WPS图片不能超过20MB');
totalImageBytes += size;
if (includeImageData) {
const imageBuffer = await imageEntry.async('nodebuffer');
if (!validImageSignature(extension, imageBuffer))
throw new BadRequestException('WPS单元格图片内容与格式不匹配');
images.push({ ...cellPosition, extension, buffer: imageBuffer });
} else {
images.push({ ...cellPosition, extension, size, target });
}
allowedIds.add(formula[1]);
}
imagesBySheet.set(attrs.name, images);
}
const totalImageBytes = [...imagesBySheet.values()].flat().reduce((sum, image) => sum + image.buffer.length, 0);
if (totalImageBytes > MAX_TOTAL_IMAGE_BYTES) throw new BadRequestException('WPS图片总量不能超过300MB');
return { allowedIds, imagesBySheet };
return { allowedIds, imagesBySheet, mediaTargets: new Set(imageTargets.values()) };
}
export async function loadCompatibleWorkbook(buffer: Buffer) {
export async function loadCompatibleWorkbook(
buffer: Buffer,
options: { includeImageData?: boolean; onProgress?: (progress: number, stage: string) => void | Promise<void> } = {},
) {
if (buffer.length > MAX_WORKBOOK_BYTES) throw new BadRequestException('导入文件不能超过100MB');
let zip: JSZip;
try {
@@ -139,6 +156,7 @@ export async function loadCompatibleWorkbook(buffer: Buffer) {
} catch {
throw new BadRequestException('仅支持有效的 XLSX 文件');
}
await options.onProgress?.(35, '已读取工作簿压缩包');
const expandedBytes = Object.values(zip.files).reduce(
(sum, entry) =>
sum + Number((entry as unknown as { _data?: { uncompressedSize?: number } })._data?.uncompressedSize ?? 0),
@@ -148,17 +166,32 @@ export async function loadCompatibleWorkbook(buffer: Buffer) {
if (expandedBytes > MAX_EXPANDED_BYTES) throw new BadRequestException('工作簿解压后超过500MB安全限制');
if (Object.keys(zip.files).some((name) => /(^|\/)(vbaProject|externalLinks|embeddings|activeX)(\/|\.)/i.test(name)))
throw new BadRequestException('工作簿包含不允许的外部对象或宏');
const wps = await inspectWpsImages(zip);
const wps = await inspectWpsImages(zip, options.includeImageData !== false);
await options.onProgress?.(55, '已定位WPS单元格图片');
let excelJsInput = buffer;
if (options.includeImageData === false && wps.mediaTargets.size) {
for (const target of wps.mediaTargets) zip.remove(target);
zip.remove('xl/cellimages.xml');
zip.remove('xl/_rels/cellimages.xml.rels');
excelJsInput = await zip.generateAsync({
type: 'nodebuffer',
compression: 'DEFLATE',
compressionOptions: { level: 1 },
});
await options.onProgress?.(70, '已剥离解析阶段无需加载的WPS图片数据');
}
const workbook = new ExcelJS.Workbook();
await workbook.xlsx.load(buffer as never);
await workbook.xlsx.load(excelJsInput as never);
await options.onProgress?.(80, '已解析工作表内容');
validateWorkbookValues(workbook, wps.allowedIds);
await options.onProgress?.(90, '已完成工作簿安全校验');
return { workbook, wpsImagesBySheet: wps.imagesBySheet };
}
export function compatibleImages(
workbook: ExcelJS.Workbook,
worksheet: ExcelJS.Worksheet,
wpsImagesBySheet: Map<string, EmbeddedImage[]>,
wpsImagesBySheet: Map<string, Array<EmbeddedImage | EmbeddedImageMetadata>>,
) {
const merged = new Map<string, EmbeddedImage>();
const wpsImages = wpsImagesBySheet.get(worksheet.name) ?? [];
@@ -169,10 +202,29 @@ export function compatibleImages(
if (!validImageSignature(extension, image.buffer)) throw new BadRequestException('工作簿图片内容与格式不匹配');
merged.set(`${image.row}:${image.column}`, image);
}
for (const image of wpsImages) merged.set(`${image.row}:${image.column}`, image);
for (const image of wpsImages) {
if (!('buffer' in image)) throw new BadRequestException('图片元数据不能用于提交导入');
merged.set(`${image.row}:${image.column}`, image);
}
return [...merged.values()];
}
export function compatibleImagePositions(
workbook: ExcelJS.Workbook,
worksheet: ExcelJS.Worksheet,
wpsImagesBySheet: Map<string, Array<EmbeddedImage | EmbeddedImageMetadata>>,
) {
const wpsImages = wpsImagesBySheet.get(worksheet.name) ?? [];
const positions = new Map<string, { row: number; column: number }>();
for (const image of readEmbeddedImages(workbook, worksheet)) {
if (wpsImages.length && image.buffer.length <= 128) continue;
positions.set(`${image.row}:${image.column}`, { row: image.row, column: image.column });
}
for (const image of wpsImages)
positions.set(`${image.row}:${image.column}`, { row: image.row, column: image.column });
return [...positions.values()];
}
export async function convertWorkbookOutput(content: Buffer, outputFormat: ReportWorkbookFormat = 'excel_drawing') {
if (outputFormat === 'excel_drawing') return content;
if (outputFormat !== 'wps_cell_image') throw new BadRequestException('不支持的报备文件格式');