feat: 异步解析大文件WPS报备资料

This commit is contained in:
hectorzhao
2026-09-04 22:51:07 +08:00
parent aaf96db2d0
commit 5925cf493b
22 changed files with 720 additions and 80 deletions
+1
View File
@@ -11,6 +11,7 @@
"test:watch": "jest --watch",
"start": "node dist/main.js",
"start:dev": "ts-node src/main.ts",
"start:report-material-worker": "node dist/report-material-analysis-worker.js",
"prisma:generate": "prisma generate",
"prisma:migrate:dev": "prisma migrate dev",
"prisma:migrate:deploy": "prisma migrate deploy"
@@ -0,0 +1,9 @@
ALTER TABLE "ReportMaterialImportBatch"
ADD COLUMN "progress" INTEGER NOT NULL DEFAULT 100,
ADD COLUMN "progressStage" TEXT,
ADD COLUMN "errorMessage" TEXT,
ADD COLUMN "startedAt" TIMESTAMP(3),
ADD COLUMN "heartbeatAt" TIMESTAMP(3);
CREATE INDEX "ReportMaterialImportBatch_status_heartbeatAt_idx"
ON "ReportMaterialImportBatch"("status", "heartbeatAt");
+6
View File
@@ -1395,6 +1395,11 @@ model ReportMaterialImportBatch {
fileName String
reportType String
status String @default("analyzed")
progress Int @default(100)
progressStage String?
errorMessage String?
startedAt DateTime?
heartbeatAt DateTime?
sheetName String
headerRowCount Int @default(1)
dataStartRow Int @default(2)
@@ -1413,6 +1418,7 @@ model ReportMaterialImportBatch {
@@index([tenantId, createdAt])
@@index([status, createdAt])
@@index([status, heartbeatAt])
}
model ReportMaterialImportItem {
+18
View File
@@ -0,0 +1,18 @@
import { assertUploadSize } from './files.service';
describe('FilesService report-material upload limits', () => {
const workbook = {
originalname: '行业报备.xlsx',
mimetype: 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
};
it('allows report-material imports up to 100MB without raising the generic file limit', () => {
expect(() =>
assertUploadSize('report_material_import', { ...workbook, size: 40 * 1024 * 1024 }),
).not.toThrow();
expect(() => assertUploadSize('other', { ...workbook, size: 40 * 1024 * 1024 })).toThrow('10MB');
expect(() =>
assertUploadSize('report_material_import', { ...workbook, size: 101 * 1024 * 1024 }),
).toThrow('100MB');
});
});
+31 -10
View File
@@ -40,10 +40,12 @@ export class FilesService {
) {}
list(tenantId?: string) {
return this.prisma.fileObject.findMany({
where: tenantId ? { tenantId } : undefined,
orderBy: { createdAt: 'desc' },
}).then((items) => items.map(serializeFileObject));
return this.prisma.fileObject
.findMany({
where: tenantId ? { tenantId } : undefined,
orderBy: { createdAt: 'desc' },
})
.then((items) => items.map(serializeFileObject));
}
create(data: CreateFileObjectDto) {
@@ -71,7 +73,7 @@ export class FilesService {
}
async upload(data: UploadFileDto, file: { originalname: string; mimetype: string; size: number; buffer: Buffer }) {
assertUploadSize(file);
assertUploadSize(data.purpose, file);
const fileName = normalizeMultipartFileName(file.originalname);
const safeName = fileName.replace(/[^\w.\-\u4e00-\u9fa5]/g, '_');
const objectKey = `${data.prefix ?? data.purpose}/${Date.now()}-${randomUUID()}-${safeName}`;
@@ -87,7 +89,11 @@ export class FilesService {
});
}
async uploadForClient(userId: string | undefined, data: UploadFileDto, file: { originalname: string; mimetype: string; size: number; buffer: Buffer }) {
async uploadForClient(
userId: string | undefined,
data: UploadFileDto,
file: { originalname: string; mimetype: string; size: number; buffer: Buffer },
) {
const tenantId = await this.resolveClientTenantId(userId);
const prefixRule = CLIENT_UPLOAD_RULES[data.purpose];
if (!prefixRule || !data.prefix || !prefixRule.test(data.prefix)) {
@@ -135,18 +141,33 @@ export class FilesService {
const IMAGE_UPLOAD_MAX_BYTES = 2 * 1024 * 1024;
const FILE_UPLOAD_MAX_BYTES = 10 * 1024 * 1024;
const REPORT_MATERIAL_IMPORT_MAX_BYTES = 100 * 1024 * 1024;
const IMAGE_FILE_EXTENSION = /\.(?:avif|bmp|gif|heic|heif|jpe?g|png|svg|webp)$/i;
function assertUploadSize(file: { originalname: string; mimetype: string; size: number }) {
export function assertUploadSize(purpose: string, file: { originalname: string; mimetype: string; size: number }) {
const image = file.mimetype.toLowerCase().startsWith('image/') || IMAGE_FILE_EXTENSION.test(file.originalname);
const limit = image ? IMAGE_UPLOAD_MAX_BYTES : FILE_UPLOAD_MAX_BYTES;
const reportMaterialImport = purpose === 'report_material_import';
const limit = reportMaterialImport
? REPORT_MATERIAL_IMPORT_MAX_BYTES
: image
? IMAGE_UPLOAD_MAX_BYTES
: FILE_UPLOAD_MAX_BYTES;
if (file.size > limit) {
throw new BadRequestException(image ? '图片大小不能超过 2MB' : '文件大小不能超过 10MB');
throw new BadRequestException(
reportMaterialImport
? '报备资料文件大小不能超过 100MB'
: image
? '图片大小不能超过 2MB'
: '文件大小不能超过 10MB',
);
}
}
function normalizeMultipartFileName(value: string) {
if (![...value].some((character) => character.charCodeAt(0) > 0x7f) || [...value].some((character) => character.charCodeAt(0) > 0xff)) {
if (
![...value].some((character) => character.charCodeAt(0) > 0x7f) ||
[...value].some((character) => character.charCodeAt(0) > 0xff)
) {
return value;
}
@@ -0,0 +1,13 @@
import { Module } from '@nestjs/common';
import { ConfigModule } from '@nestjs/config';
import { PrismaModule } from './prisma/prisma.module';
import { ReportMaterialsModule } from './report-materials/report-materials.module';
@Module({
imports: [
ConfigModule.forRoot({ isGlobal: true, envFilePath: ['.env.local', '.env'] }),
PrismaModule,
ReportMaterialsModule,
],
})
export class ReportMaterialAnalysisWorkerModule {}
@@ -0,0 +1,31 @@
import 'reflect-metadata';
import { NestFactory } from '@nestjs/core';
import { ReportMaterialAnalysisWorkerModule } from './report-material-analysis-worker.module';
import { ReportMaterialsService } from './report-materials/report-materials.service';
const POLL_INTERVAL_MS = 1000;
async function bootstrap() {
if ((process.env.CMPP_PROCESS_ROLE?.trim() || 'report-material-worker') !== 'report-material-worker') {
throw new Error('report-material-analysis-worker requires CMPP_PROCESS_ROLE=report-material-worker');
}
const app = await NestFactory.createApplicationContext(ReportMaterialAnalysisWorkerModule, {
logger: ['log', 'warn', 'error'],
});
app.enableShutdownHooks();
const reports = app.get(ReportMaterialsService);
await reports.recoverStaleAnalysisJobs();
let stopping = false;
const stop = () => {
stopping = true;
};
process.once('SIGTERM', stop);
process.once('SIGINT', stop);
while (!stopping) {
const worked = await reports.processNextAnalysisJob();
if (!worked) await new Promise((resolve) => setTimeout(resolve, POLL_INTERVAL_MS));
}
await app.close();
}
void bootstrap();
+209 -38
View File
@@ -1,4 +1,4 @@
import { BadRequestException } from '@nestjs/common';
import { BadRequestException, NotFoundException } from '@nestjs/common';
import { Prisma } from '@prisma/client';
import { extname } from 'node:path';
import { FilesService } from '../files/files.service';
@@ -13,7 +13,7 @@ import {
clamp,
cellText,
} from './report-materials.helpers';
import { compatibleImages, loadCompatibleWorkbook } from './workbook-compatibility';
import { compatibleImagePositions, loadCompatibleWorkbook } from './workbook-compatibility';
/** R4 report-materials domain service composed behind ReportMaterialsService. */
export class ReportImportParserService {
@@ -63,14 +63,123 @@ export class ReportImportParserService {
file: { originalname: string; mimetype: string; size: number; buffer: Buffer },
options: AnalyzeImportOptions,
) {
if (!options.tenantId) throw new BadRequestException('tenantId is required');
if (!options.applicationId) throw new BadRequestException('applicationId is required');
await this.smsConfig.getApplication(options.applicationId, options.tenantId);
if (!['signature', 'drainage'].includes(options.reportType))
throw new BadRequestException('reportType must be signature or drainage');
if (extname(file.originalname).toLowerCase() !== '.xlsx' || file.buffer[0] !== 0x50 || file.buffer[1] !== 0x4b)
throw new BadRequestException('仅支持有效的 XLSX 文件');
const { workbook, wpsImagesBySheet } = await loadCompatibleWorkbook(file.buffer);
await this.validateImport(file, options);
const analysis = await this.parseAnalysis(file.buffer, options);
const sourceFile = await this.files.upload(
{ tenantId: options.tenantId, purpose: 'report_material_import', prefix: 'report-material-imports' },
file,
);
const batch = await this.createAnalyzedBatch(sourceFile, options, analysis);
await this.logAnalyzed(batch.id, sourceFile.fileName, options, analysis.sheetName, analysis.rows.length);
return { ...batch, sourceFile, ...analysis };
}
async enqueueImport(
file: { originalname: string; mimetype: string; size: number; buffer: Buffer },
options: AnalyzeImportOptions,
) {
await this.validateImport(file, options);
const sourceFile = await this.files.upload(
{ tenantId: options.tenantId, purpose: 'report_material_import', prefix: 'report-material-imports' },
file,
);
const batch = await this.prisma.reportMaterialImportBatch.create({
data: {
tenantId: options.tenantId,
applicationId: options.applicationId,
profileId: options.profileId,
fileObjectId: sourceFile.id,
fileName: sourceFile.fileName,
reportType: options.reportType,
status: 'queued',
progress: 10,
progressStage: '文件已上传,等待解析',
sheetName: options.sheetName ?? '',
headerRowCount: clamp(options.headerRowCount, 1, 5),
dataStartRow: Math.max(options.dataStartRow, clamp(options.headerRowCount, 1, 5) + 1),
mapping: [] as Prisma.InputJsonValue,
result: { operatorId: options.operatorId } as Prisma.InputJsonValue,
},
});
return this.analysisStatus(batch);
}
async getAnalysisStatus(batchId: string, tenantId: string) {
if (!tenantId) throw new BadRequestException('tenantId is required');
const batch = await this.prisma.reportMaterialImportBatch.findFirst({ where: { id: batchId, tenantId } });
if (!batch) throw new NotFoundException('导入解析任务不存在');
return this.analysisStatus(batch);
}
async processQueuedBatch(batchId: string) {
const batch = await this.prisma.reportMaterialImportBatch.findUnique({ where: { id: batchId } });
if (!batch || batch.status !== 'analyzing') return false;
try {
await this.progress(batchId, 25, '正在读取工作簿');
const { content } = await this.files.getDownload(batch.fileObjectId);
const resultData = (batch.result && typeof batch.result === 'object' ? batch.result : {}) as {
operatorId?: string;
};
const options: AnalyzeImportOptions = {
tenantId: batch.tenantId,
applicationId: batch.applicationId ?? '',
reportType: batch.reportType as 'signature' | 'drainage',
sheetName: batch.sheetName || undefined,
headerRowCount: batch.headerRowCount,
dataStartRow: batch.dataStartRow,
profileId: batch.profileId ?? undefined,
operatorId: resultData.operatorId,
};
await this.progress(batchId, 30, '正在解析表格与图片位置');
const analysis = await this.parseAnalysis(content, options, async (progress, stage) => {
await this.progress(batchId, progress, stage);
});
const updated = await this.prisma.reportMaterialImportBatch.update({
where: { id: batchId },
data: {
status: 'analyzed',
progress: 100,
progressStage: '解析完成',
errorMessage: null,
heartbeatAt: new Date(),
sheetName: analysis.sheetName,
mapping: analysis.suggestedMappings as Prisma.InputJsonValue,
preview: {
sheets: analysis.sheets,
columns: analysis.columns,
rows: analysis.rows,
imageCount: analysis.imageCount,
} as Prisma.InputJsonValue,
result: Prisma.JsonNull,
rowCount: analysis.rowCount,
},
});
await this.logAnalyzed(batchId, batch.fileName, options, analysis.sheetName, analysis.rows.length);
return this.analysisStatus(updated);
} catch (error) {
const message = error instanceof Error ? error.message : '文件解析失败';
await this.prisma.reportMaterialImportBatch.update({
where: { id: batchId },
data: {
status: 'failed',
progressStage: '解析失败',
errorMessage: message.slice(0, 1000),
heartbeatAt: new Date(),
},
});
return false;
}
}
private async parseAnalysis(
buffer: Buffer,
options: AnalyzeImportOptions,
onProgress?: (progress: number, stage: string) => void | Promise<void>,
) {
const { workbook, wpsImagesBySheet } = await loadCompatibleWorkbook(buffer, {
includeImageData: false,
onProgress,
});
const profile = options.profileId
? await this.prisma.reportMaterialImportProfile.findUnique({
where: { id: options.profileId },
@@ -82,7 +191,7 @@ export class ReportImportParserService {
if (!worksheet) throw new BadRequestException('工作簿没有可读取的工作表');
const headerRowCount = clamp(options.headerRowCount, 1, 5);
const dataStartRow = Math.max(options.dataStartRow, headerRowCount + 1);
const images = compatibleImages(workbook, worksheet, wpsImagesBySheet);
const images = compatibleImagePositions(workbook, worksheet, wpsImagesBySheet);
const columnCount = Math.min(worksheet.columnCount, 200);
const columns = Array.from({ length: columnCount }, (_, offset) => {
const sourceColumnIndex = offset + 1;
@@ -110,10 +219,6 @@ export class ReportImportParserService {
if (Object.values(values).some(Boolean) || imageColumns.length)
previewRows.push({ rowNumber, values, imageColumns });
}
const sourceFile = await this.files.upload(
{ tenantId: options.tenantId, purpose: 'report_material_import', prefix: 'report-material-imports' },
file,
);
const profileMappings = profile?.columns.map((column) => ({
sourceHeader: column.sourceHeader,
sourceHeaderPath: column.sourceHeaderPath ?? undefined,
@@ -128,7 +233,24 @@ export class ReportImportParserService {
const suggestedMappings = profileMappings?.length
? remapProfileColumns(profileMappings, columns)
: suggestMappings(columns, options.reportType);
const batch = await this.prisma.reportMaterialImportBatch.create({
await onProgress?.(95, '已生成字段映射与预览');
return {
sheetName: worksheet.name,
sheets: workbook.worksheets.map((sheet) => sheet.name),
columns,
rows: previewRows,
imageCount: images.length,
suggestedMappings,
rowCount: Math.max(0, worksheet.rowCount - dataStartRow + 1),
};
}
private createAnalyzedBatch(
sourceFile: { id: string; fileName: string },
options: AnalyzeImportOptions,
analysis: Awaited<ReturnType<ReportImportParserService['parseAnalysis']>>,
) {
return this.prisma.reportMaterialImportBatch.create({
data: {
tenantId: options.tenantId,
applicationId: options.applicationId,
@@ -136,42 +258,91 @@ export class ReportImportParserService {
fileObjectId: sourceFile.id,
fileName: sourceFile.fileName,
reportType: options.reportType,
sheetName: worksheet.name,
headerRowCount,
dataStartRow,
mapping: suggestedMappings as Prisma.InputJsonValue,
sheetName: analysis.sheetName,
headerRowCount: clamp(options.headerRowCount, 1, 5),
dataStartRow: Math.max(options.dataStartRow, clamp(options.headerRowCount, 1, 5) + 1),
mapping: analysis.suggestedMappings as Prisma.InputJsonValue,
preview: {
sheets: workbook.worksheets.map((sheet) => sheet.name),
columns,
rows: previewRows,
imageCount: images.length,
sheets: analysis.sheets,
columns: analysis.columns,
rows: analysis.rows,
imageCount: analysis.imageCount,
} as Prisma.InputJsonValue,
rowCount: Math.max(0, worksheet.rowCount - dataStartRow + 1),
rowCount: analysis.rowCount,
},
});
}
private async validateImport(
file: { originalname: string; size: number; buffer: Buffer },
options: AnalyzeImportOptions,
) {
if (!options.tenantId) throw new BadRequestException('tenantId is required');
if (!options.applicationId) throw new BadRequestException('applicationId is required');
await this.smsConfig.getApplication(options.applicationId, options.tenantId);
if (!['signature', 'drainage'].includes(options.reportType))
throw new BadRequestException('reportType must be signature or drainage');
if (extname(file.originalname).toLowerCase() !== '.xlsx' || file.buffer[0] !== 0x50 || file.buffer[1] !== 0x4b)
throw new BadRequestException('仅支持有效的 XLSX 文件');
}
private progress(batchId: string, progress: number, progressStage: string) {
return this.prisma.reportMaterialImportBatch.update({
where: { id: batchId },
data: { progress, progressStage, heartbeatAt: new Date() },
});
}
private analysisStatus(batch: {
id: string;
status: string;
progress: number;
progressStage: string | null;
errorMessage: string | null;
sheetName: string;
preview: Prisma.JsonValue | null;
mapping: Prisma.JsonValue;
}) {
const preview =
batch.preview && typeof batch.preview === 'object' && !Array.isArray(batch.preview)
? (batch.preview as Record<string, unknown>)
: {};
return {
id: batch.id,
status: batch.status,
progress: batch.progress,
progressStage: batch.progressStage,
errorMessage: batch.errorMessage,
sheetName: batch.sheetName || undefined,
sheets: preview.sheets ?? [],
columns: preview.columns ?? [],
rows: preview.rows ?? [],
imageCount: preview.imageCount ?? 0,
suggestedMappings: batch.mapping ?? [],
};
}
private async logAnalyzed(
batchId: string,
fileName: string,
options: AnalyzeImportOptions,
sheetName: string,
previewCount: number,
) {
await this.prisma.operationLog.create({
data: {
tenantId: options.tenantId,
userId: options.operatorId,
action: 'report_material.import_analyzed',
resource: 'report_material_import',
resourceId: batch.id,
resourceId: batchId,
detail: {
fileName: sourceFile.fileName,
filters: { applicationId: options.applicationId, reportType: options.reportType, sheetName: worksheet.name },
successCount: previewRows.length,
fileName,
filters: { applicationId: options.applicationId, reportType: options.reportType, sheetName },
successCount: previewCount,
failedCount: 0,
} as Prisma.InputJsonValue,
},
});
return {
...batch,
sourceFile,
sheets: workbook.worksheets.map((sheet) => sheet.name),
columns,
rows: previewRows,
imageCount: images.length,
suggestedMappings,
};
}
}
@@ -105,5 +105,6 @@ export type AnalyzeImportOptions = {
};
export type EmbeddedImage = { row: number; column: number; extension: string; buffer: Buffer };
export type EmbeddedImageMetadata = { row: number; column: number; extension: string; size: number; target: string };
export type ReportWorkbookFormat = 'excel_drawing' | 'wps_cell_image';
@@ -3,6 +3,7 @@ import {
Body,
Controller,
Get,
HttpCode,
Param,
Post,
Put,
@@ -93,6 +94,7 @@ export class ReportMaterialsController {
}
@Post('imports/analyze')
@HttpCode(202)
@UseInterceptors(
FileInterceptor('file', { limits: { fileSize: 100 * 1024 * 1024, files: 1, fields: 12, parts: 13 } }),
)
@@ -102,7 +104,7 @@ export class ReportMaterialsController {
@CurrentSessionUserId() operatorId?: string,
) {
if (!file) throw new BadRequestException('请选择 XLSX 文件');
return this.service.analyzeImport(file, {
return this.service.enqueueImport(file, {
tenantId: body.tenantId,
applicationId: body.applicationId,
reportType: body.reportType as 'signature' | 'drainage',
@@ -114,6 +116,11 @@ export class ReportMaterialsController {
});
}
@Get('imports/:id/analysis-status')
getImportAnalysisStatus(@Param('id') id: string, @Query('tenantId') tenantId: string) {
return this.service.getAnalysisStatus(id, tenantId);
}
@Put('imports/:id/commit')
@RequireRecentAuthentication()
commitImport(@Param('id') id: string, @Body() body: ImportCommitDto, @CurrentSessionUserId() operatorId?: string) {
@@ -4,6 +4,75 @@ import { ReportMaterialsService } from './report-materials.service';
import { mappedCorePatchValue, signatureIdentityReportValues } from './report-materials.helpers';
describe('ReportMaterialsService', () => {
it('stores a valid workbook as a durable queued analysis job before parsing it', async () => {
const prisma = {
reportMaterialImportBatch: {
create: jest.fn().mockImplementation(({ data }: { data: Record<string, unknown> }) =>
Promise.resolve({ id: 'job-1', preview: null, errorMessage: null, ...data }),
),
},
};
const files = { upload: jest.fn().mockResolvedValue({ id: 'file-1', fileName: '行业报备.xlsx' }) };
const service = new ReportMaterialsService(
prisma as never,
files as never,
{ getApplication: jest.fn().mockResolvedValue({ id: 'app-1' }) } as never,
);
const result = await service.enqueueImport(
{ originalname: '行业报备.xlsx', mimetype: 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', size: 2, buffer: Buffer.from('PK') },
{ tenantId: 'tenant-1', applicationId: 'app-1', reportType: 'signature', headerRowCount: 1, dataStartRow: 2 },
);
expect(files.upload).toHaveBeenCalledWith(
expect.objectContaining({ purpose: 'report_material_import', tenantId: 'tenant-1' }),
expect.any(Object),
);
expect(prisma.reportMaterialImportBatch.create).toHaveBeenCalledWith({
data: expect.objectContaining({ status: 'queued', progress: 10, applicationId: 'app-1' }),
});
expect(result).toMatchObject({ id: 'job-1', status: 'queued', progress: 10 });
});
it('only returns analysis progress inside the requested tenant', async () => {
const findFirst = jest.fn().mockResolvedValue(null);
const service = new ReportMaterialsService(
{ reportMaterialImportBatch: { findFirst } } as never,
{} as never,
{} as never,
);
await expect(service.getAnalysisStatus('job-1', 'tenant-2')).rejects.toThrow('导入解析任务不存在');
expect(findFirst).toHaveBeenCalledWith({ where: { id: 'job-1', tenantId: 'tenant-2' } });
});
it('claims only a queued analysis job before handing it to the worker parser', async () => {
const prisma = {
reportMaterialImportBatch: {
findFirst: jest.fn().mockResolvedValue({ id: 'job-1' }),
updateMany: jest.fn().mockResolvedValue({ count: 1 }),
findUnique: jest.fn().mockResolvedValue(null),
},
};
const service = new ReportMaterialsService(prisma as never, {} as never, {} as never);
await expect(service.processNextAnalysisJob()).resolves.toBe(true);
expect(prisma.reportMaterialImportBatch.updateMany).toHaveBeenCalledWith({
where: { id: 'job-1', status: 'queued' },
data: expect.objectContaining({ status: 'analyzing', progress: 15 }),
});
});
it('requeues stale analyzing jobs so a restarted worker can resume them', async () => {
const updateMany = jest.fn().mockResolvedValue({ count: 2 });
const service = new ReportMaterialsService(
{ reportMaterialImportBatch: { updateMany } } as never,
{} as never,
{} as never,
);
await expect(service.recoverStaleAnalysisJobs()).resolves.toEqual({ count: 2 });
expect(updateMany).toHaveBeenCalledWith({
where: { status: 'analyzing', OR: [{ heartbeatAt: null }, { heartbeatAt: { lt: expect.any(Date) } }] },
data: { status: 'queued', progress: 10, progressStage: '等待重新解析', errorMessage: null },
});
});
it('requires an enterprise application before parsing an import workbook', async () => {
const service = new ReportMaterialsService({} as never, { upload: jest.fn() } as never, {} as never);
await expect(
@@ -34,7 +34,11 @@ export class ReportMaterialsService {
private readonly channelExport: ReportChannelExportService;
private readonly batchGeneration: ReportBatchGenerationService;
constructor(prisma: PrismaService, files: FilesService, smsConfig: SmsConfigService) {
constructor(
private readonly prisma: PrismaService,
files: FilesService,
smsConfig: SmsConfigService,
) {
this.pendingQuery = new ReportPendingQueryService(prisma, files, smsConfig);
this.officialExport = new ReportOfficialExportService(prisma, files, smsConfig, this.pendingQuery);
this.importParser = new ReportImportParserService(prisma, files, smsConfig);
@@ -82,6 +86,48 @@ export class ReportMaterialsService {
return this.importParser.analyzeImport(file, options);
}
async enqueueImport(
file: { originalname: string; mimetype: string; size: number; buffer: Buffer },
options: AnalyzeImportOptions,
) {
return this.importParser.enqueueImport(file, options);
}
async getAnalysisStatus(batchId: string, tenantId: string) {
return this.importParser.getAnalysisStatus(batchId, tenantId);
}
async recoverStaleAnalysisJobs() {
const staleBefore = new Date(Date.now() - 5 * 60 * 1000);
return this.prisma.reportMaterialImportBatch.updateMany({
where: { status: 'analyzing', OR: [{ heartbeatAt: null }, { heartbeatAt: { lt: staleBefore } }] },
data: { status: 'queued', progress: 10, progressStage: '等待重新解析', errorMessage: null },
});
}
async processNextAnalysisJob() {
const candidate = await this.prisma.reportMaterialImportBatch.findFirst({
where: { status: 'queued' },
orderBy: { createdAt: 'asc' },
select: { id: true },
});
if (!candidate) return false;
const claimed = await this.prisma.reportMaterialImportBatch.updateMany({
where: { id: candidate.id, status: 'queued' },
data: {
status: 'analyzing',
progress: 15,
progressStage: '后台解析已开始',
startedAt: new Date(),
heartbeatAt: new Date(),
errorMessage: null,
},
});
if (!claimed.count) return true;
await this.importParser.processQueuedBatch(candidate.id);
return true;
}
async commitImport(batchId: string, data: ImportCommitDto) {
return this.importReview.commitImport(batchId, data);
}
@@ -1,6 +1,11 @@
import ExcelJS from 'exceljs';
import JSZip from 'jszip';
import { compatibleImages, convertWorkbookOutput, loadCompatibleWorkbook } from './workbook-compatibility';
import {
compatibleImagePositions,
compatibleImages,
convertWorkbookOutput,
loadCompatibleWorkbook,
} from './workbook-compatibility';
const PNG = Buffer.from(
'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAusB9Y9ZrYQAAAAASUVORK5CYII=',
@@ -39,4 +44,15 @@ describe('WPS workbook compatibility', () => {
const content = Buffer.from(await workbook.xlsx.writeBuffer());
await expect(loadCompatibleWorkbook(content)).rejects.toThrow('不允许的公式');
});
it('reads WPS image positions without inflating image buffers during analysis', async () => {
const converted = await convertWorkbookOutput(await standardWorkbook(), 'wps_cell_image');
const loaded = await loadCompatibleWorkbook(converted, { includeImageData: false });
const wpsImage = loaded.wpsImagesBySheet.get('签名报备')?.[0];
expect(wpsImage).toMatchObject({ row: 2, column: 1, extension: 'png', size: PNG.length });
expect(wpsImage).not.toHaveProperty('buffer');
expect(
compatibleImagePositions(loaded.workbook, loaded.workbook.getWorksheet('签名报备')!, loaded.wpsImagesBySheet),
).toEqual([{ row: 2, column: 1 }]);
});
});
@@ -2,7 +2,7 @@ import { BadRequestException } from '@nestjs/common';
import ExcelJS from 'exceljs';
import JSZip from 'jszip';
import { randomUUID } from 'node:crypto';
import type { EmbeddedImage, ReportWorkbookFormat } from './report-materials.contracts';
import type { EmbeddedImage, EmbeddedImageMetadata, ReportWorkbookFormat } from './report-materials.contracts';
import { normalizeImageExtension, readEmbeddedImages } from './report-materials.helpers';
const MAX_WORKBOOK_BYTES = 100 * 1024 * 1024;
@@ -72,9 +72,14 @@ function validateWorkbookValues(workbook: ExcelJS.Workbook, allowedWpsIds: Set<s
}
}
async function inspectWpsImages(zip: JSZip) {
async function inspectWpsImages(zip: JSZip, includeImageData: boolean) {
const cellImagesXml = await text(zip, 'xl/cellimages.xml');
if (!cellImagesXml) return { allowedIds: new Set<string>(), imagesBySheet: new Map<string, EmbeddedImage[]>() };
if (!cellImagesXml)
return {
allowedIds: new Set<string>(),
imagesBySheet: new Map<string, Array<EmbeddedImage | EmbeddedImageMetadata>>(),
mediaTargets: new Set<string>(),
};
const relXml = await text(zip, 'xl/_rels/cellimages.xml.rels');
const relTargets = new Map<string, string>();
for (const match of relXml.matchAll(/<Relationship\b([^>]*)\/?>(?:<\/Relationship>)?/g)) {
@@ -99,13 +104,14 @@ async function inspectWpsImages(zip: JSZip) {
sheetRelTargets.set(attrs.Id, packagePath(attrs.Target));
}
const allowedIds = new Set<string>();
const imagesBySheet = new Map<string, EmbeddedImage[]>();
const imagesBySheet = new Map<string, Array<EmbeddedImage | EmbeddedImageMetadata>>();
let totalImageBytes = 0;
for (const match of workbookXml.matchAll(/<sheet\b([^>]*)\/?>(?:<\/sheet>)?/g)) {
const attrs = attributes(match[1]);
const path = sheetRelTargets.get(attrs['r:id']);
if (!attrs.name || !path) continue;
const sheetXml = await text(zip, path);
const images: EmbeddedImage[] = [];
const images: Array<EmbeddedImage | EmbeddedImageMetadata> = [];
for (const cell of sheetXml.matchAll(/<c\b([^>]*)>([\s\S]*?)<\/c>/g)) {
const formulaText = decodeXml(cell[2].match(/<f(?:\s[^>]*)?>([\s\S]*?)<\/f>/)?.[1]?.trim() ?? '');
if (!formulaText) continue;
@@ -118,20 +124,31 @@ async function inspectWpsImages(zip: JSZip) {
if (!imageEntry) throw new BadRequestException('WPS单元格图片文件缺失');
const extension = normalizeImageExtension(target.split('.').pop() ?? 'png');
if (!['png', 'jpeg', 'gif'].includes(extension)) throw new BadRequestException('WPS单元格图片格式不受支持');
const imageBuffer = await imageEntry.async('nodebuffer');
if (imageBuffer.length > MAX_IMAGE_BYTES) throw new BadRequestException('单张WPS图片不能超过20MB');
if (!validImageSignature(extension, imageBuffer)) throw new BadRequestException('WPS单元格图片内容与格式不匹配');
images.push({ ...cellPosition, extension, buffer: imageBuffer });
const size = Number(
(imageEntry as unknown as { _data?: { uncompressedSize?: number } })._data?.uncompressedSize ?? 0,
);
if (size > MAX_IMAGE_BYTES) throw new BadRequestException('单张WPS图片不能超过20MB');
totalImageBytes += size;
if (includeImageData) {
const imageBuffer = await imageEntry.async('nodebuffer');
if (!validImageSignature(extension, imageBuffer))
throw new BadRequestException('WPS单元格图片内容与格式不匹配');
images.push({ ...cellPosition, extension, buffer: imageBuffer });
} else {
images.push({ ...cellPosition, extension, size, target });
}
allowedIds.add(formula[1]);
}
imagesBySheet.set(attrs.name, images);
}
const totalImageBytes = [...imagesBySheet.values()].flat().reduce((sum, image) => sum + image.buffer.length, 0);
if (totalImageBytes > MAX_TOTAL_IMAGE_BYTES) throw new BadRequestException('WPS图片总量不能超过300MB');
return { allowedIds, imagesBySheet };
return { allowedIds, imagesBySheet, mediaTargets: new Set(imageTargets.values()) };
}
export async function loadCompatibleWorkbook(buffer: Buffer) {
export async function loadCompatibleWorkbook(
buffer: Buffer,
options: { includeImageData?: boolean; onProgress?: (progress: number, stage: string) => void | Promise<void> } = {},
) {
if (buffer.length > MAX_WORKBOOK_BYTES) throw new BadRequestException('导入文件不能超过100MB');
let zip: JSZip;
try {
@@ -139,6 +156,7 @@ export async function loadCompatibleWorkbook(buffer: Buffer) {
} catch {
throw new BadRequestException('仅支持有效的 XLSX 文件');
}
await options.onProgress?.(35, '已读取工作簿压缩包');
const expandedBytes = Object.values(zip.files).reduce(
(sum, entry) =>
sum + Number((entry as unknown as { _data?: { uncompressedSize?: number } })._data?.uncompressedSize ?? 0),
@@ -148,17 +166,32 @@ export async function loadCompatibleWorkbook(buffer: Buffer) {
if (expandedBytes > MAX_EXPANDED_BYTES) throw new BadRequestException('工作簿解压后超过500MB安全限制');
if (Object.keys(zip.files).some((name) => /(^|\/)(vbaProject|externalLinks|embeddings|activeX)(\/|\.)/i.test(name)))
throw new BadRequestException('工作簿包含不允许的外部对象或宏');
const wps = await inspectWpsImages(zip);
const wps = await inspectWpsImages(zip, options.includeImageData !== false);
await options.onProgress?.(55, '已定位WPS单元格图片');
let excelJsInput = buffer;
if (options.includeImageData === false && wps.mediaTargets.size) {
for (const target of wps.mediaTargets) zip.remove(target);
zip.remove('xl/cellimages.xml');
zip.remove('xl/_rels/cellimages.xml.rels');
excelJsInput = await zip.generateAsync({
type: 'nodebuffer',
compression: 'DEFLATE',
compressionOptions: { level: 1 },
});
await options.onProgress?.(70, '已剥离解析阶段无需加载的WPS图片数据');
}
const workbook = new ExcelJS.Workbook();
await workbook.xlsx.load(buffer as never);
await workbook.xlsx.load(excelJsInput as never);
await options.onProgress?.(80, '已解析工作表内容');
validateWorkbookValues(workbook, wps.allowedIds);
await options.onProgress?.(90, '已完成工作簿安全校验');
return { workbook, wpsImagesBySheet: wps.imagesBySheet };
}
export function compatibleImages(
workbook: ExcelJS.Workbook,
worksheet: ExcelJS.Worksheet,
wpsImagesBySheet: Map<string, EmbeddedImage[]>,
wpsImagesBySheet: Map<string, Array<EmbeddedImage | EmbeddedImageMetadata>>,
) {
const merged = new Map<string, EmbeddedImage>();
const wpsImages = wpsImagesBySheet.get(worksheet.name) ?? [];
@@ -169,10 +202,29 @@ export function compatibleImages(
if (!validImageSignature(extension, image.buffer)) throw new BadRequestException('工作簿图片内容与格式不匹配');
merged.set(`${image.row}:${image.column}`, image);
}
for (const image of wpsImages) merged.set(`${image.row}:${image.column}`, image);
for (const image of wpsImages) {
if (!('buffer' in image)) throw new BadRequestException('图片元数据不能用于提交导入');
merged.set(`${image.row}:${image.column}`, image);
}
return [...merged.values()];
}
export function compatibleImagePositions(
workbook: ExcelJS.Workbook,
worksheet: ExcelJS.Worksheet,
wpsImagesBySheet: Map<string, Array<EmbeddedImage | EmbeddedImageMetadata>>,
) {
const wpsImages = wpsImagesBySheet.get(worksheet.name) ?? [];
const positions = new Map<string, { row: number; column: number }>();
for (const image of readEmbeddedImages(workbook, worksheet)) {
if (wpsImages.length && image.buffer.length <= 128) continue;
positions.set(`${image.row}:${image.column}`, { row: image.row, column: image.column });
}
for (const image of wpsImages)
positions.set(`${image.row}:${image.column}`, { row: image.row, column: image.column });
return [...positions.values()];
}
export async function convertWorkbookOutput(content: Buffer, outputFormat: ReportWorkbookFormat = 'excel_drawing') {
if (outputFormat === 'excel_drawing') return content;
if (outputFormat !== 'wps_cell_image') throw new BadRequestException('不支持的报备文件格式');
+2
View File
@@ -5066,6 +5066,8 @@ npm run verify:phase8
| TC-REPORT-WPS-001 | 上传`行业报备.xlsx`等使用`_xlfn.DISPIMG``xl/cellimages.xml`及关系文件的WPS XLSX | 仅精确白名单内且图片关系完整的DISPIMG公式可解析;图片按真实单元格映射进入预览与审核,透明Drawing占位图不计入;普通公式、宏、外部对象、缺失关系和非法图片格式均明确拒绝 |
| TC-REPORT-WPS-002 | 分别上传10MB以上且不超过100MB、超过100MB、解压后超过500MB的报备XLSX | 第一类可进入解析;后两类分别在上传层或解包层明确拒绝;Nginx私有站点允许100MB业务文件及multipart开销,不扩大公网单条API边界 |
| TC-REPORT-WPS-003 | 在批次和单条报备导出选择“系统 Excel 文件”或“WPS 单元格图片文件” | 默认保持ExcelJS Drawing格式;WPS选项生成DISPIMG与cellimages关系且图片按对应单元格可回读;两种格式均来自同一真实资料快照,不改变报备状态或短信链路 |
| TC-REPORT-WPS-004 | 上传40MB以上且不超过100MB的WPS报备文件,并持续观察解析弹窗与服务重启 | 上传接口在文件写入对象存储并创建PostgreSQL任务后返回202;独立解析Worker单并发处理,页面轮询真实进度;API/浏览器不再等待完整解析,Worker重启后超时租约任务可重新排队且不重复提交导入 |
| TC-REPORT-WPS-005 | 分析包含大量DISPIMG图片的WPS文件,再提交映射进入审核 | 分析阶段只读取图片关系、位置、格式和声明大小,不展开全部WPS图片二进制;提交导入阶段重新读取并校验真实图片签名和20MB单图/300MB总量限制,公式安全校验不放宽 |
| TC-REPORT-IMPORT-APPLICATION-001 | 未选择企业应用、选择其他企业的应用、选择当前企业应用后解析导入 | 前两种前后端均阻止导入;合法应用可解析并把applicationId写入导入批次,后续补资料只作用于该企业应用范围 |
| TC-REPORT-FIELD-MODAL-001 | 打开签名或引流字段配置弹窗,在左侧字段池数量不同的情况下连续添加字段,并检查通用字段与移除按钮 | 左侧每个字段卡片始终固定为68px,不因可选字段数量或添加操作被Grid拉伸;右侧默认包含同资料类型的通用字段;移除按钮为通用宽度、无红色填充,保存仍调用真实通道字段接口 |
| TC-REPORT-MATERIAL-DETAIL-001 | 查看包含当前图片字段、未删除历史字段及旧图片引用的报备资料 | 当前字段展示名称、代码、导出名及图片预览;未删除历史字段继续展示且同时显示字段名称和代码;图片可内联查看并保留下载入口,缺失内容显示明确占位 |
+21
View File
@@ -4478,3 +4478,24 @@ git diff --check
- 测试环境发布前为`bb435fb0ac1e7812fcdb59950a4b13757899ab39`。发布包大小2766696字节,本地和服务器SHA-256均为`2d3c9a022ba1b6494a51e2ee0f3eb04eabb2513af25ba3631dae5ea70ea68783`;恢复点为`/opt/cmpp-platform-backups/import-review-fix-20260904T122649Z`,包含PostgreSQL custom dump、原运行目录、系统配置及发布包,四项SHA、`pg_restore --list`和两份tar可读性通过。95项migration齐全且无待执行项,没有执行迁移;最终`.deployed-commit=41962e7a6e6cfd34b4313c8bf52d9345d37d8e10`,旧运行目录为`/opt/cmpp-platform.previous-import-review-20260904T122649Z`
- 使用自行生成的标准Excel Drawing工作簿`签名导入全流程测试-含图片.xlsx`走真实HTTP API:解析得到6列、1张图片,导入批次`cmtmxvjuh000xluleh38c4sjy`审核结果`approved / approvedCount=1 / failedCount=0`;生成签名`cmtmxvjyh0014lule5o2t16m9``approved`,四项系统身份字段、图片`FileObject`及6个通道图片材料均在PostgreSQL存在。报备批次`cmtmxvk8c0043lules3upx5f2`完成并生成6个真实通道文件,下载ZIP中的首个XLSX包含`xl/media`图片。批次生成后签名`pendingReport=false`,没有发送、补发、重投或重新入队短信。
- API验收使用本轮临时平台管理员,登录、近期认证、导入、审核、生成和下载均经过真实HTTP会话;结束时调用登出并删除临时账号,查询剩余数为0。浏览器已读取测试环境登录页、验证码控件和页面标题,控制台无warning/error;登录后页面交互需按浏览器安全规则由用户确认验证码后继续,不将当前登录页检查冒充已完成登录后验收。
## 2026-09-04 WPS报备兼容与导入审核修复(预生产发布完成)
- 用户明确授权发布预生产。发布前重新核验本地与`origin/main`均为`aaf96db2d018cfcea79b6cdbf553cee9cb982fa2`,预生产原标记为`dada0d978bb05d7046469c1ec77371ddb2fb03bc`;累计范围为9个提交、44个文件,包含运营看板、报备字段配置、WPS单元格图片导入导出、100MiB上传限制、字段卡片固定高度及导入审核系统身份字段补齐。没有新增migration,既有未跟踪补救方案文档未进入发布归档。
- 发布前数据盘UUID`ef4ee3bb-a19b-4aeb-b00c-aa2b995611c2`、PostgreSQL/Redis/MinIO绑定挂载、存储保护脚本、systemd drop-in和固定备份入口全部通过;95项migration齐全,相关服务健康,三条Redis Stream均`pending=0 / lag=0`,供应商连接为`desired=9 / connected=9`
- 首次恢复点`/opt/cmpp-platform-backups/preprod-wps-import-20260904T211224Z-before-aaf96db``pg_dump`不接受Prisma连接串的`schema`查询参数而在数据库导出阶段安全停止;当时尚未构建、迁移、切换、重启或修改Nginx。改用去除查询参数的数据库连接后重新建立有效恢复点`/opt/cmpp-platform-backups/preprod-wps-import-20260904T211255Z-before-aaf96db`,约613MiB,包含PostgreSQL custom dump、Redis RDB、原运行目录、环境/systemd/Nginx/Fail2ban/nftables及数据盘保护配置、原标记和发布基线。
- 有效恢复点的8项最终SHA-256全部通过,`pg_restore --list`包含825项,运行目录tar包含52641项、系统配置tar包含370项;精确Git归档大小2768402字节,SHA-256为`d984c79d0deef1a83ec418b4806875eca210343bb76e03f174887614ba6517c1`,服务器回读一致。候选版本通过依赖安全、部署契约、Prisma生成与迁移状态、前端/API/Gateway/Security Agent生产构建以及`cmpp-api`用户运行文件可读/可执行门禁。
- 最终`.deployed-commit=aaf96db2d018cfcea79b6cdbf553cee9cb982fa2`,上一运行目录保留为`/opt/cmpp-platform.previous-preprod-dada0d9-20260904T211255Z`。95项migration仍齐全且无待执行项,本轮未执行数据库迁移。Nginx仅将`sms.lisglo.com`和兼容端口12026的两个`client_max_body_size 50m`精确调整为`110m`API专用域名保持`10m``nginx -t`通过后reload,没有修改数据盘、fstab、UUID、存储保护脚本或systemd存储drop-in。
- API、Send Worker、Submit Outbox、Gateway Callback、Protocol Log Worker、Gateway、Security Agent、Nginx、PostgreSQL、Redis、MinIO、Node Exporter和Prometheus均为`active`,相关应用服务`NRestarts=0 / Result=success`API、Callback、Gateway、PostgreSQL和Redis健康通过。`12026``sms.lisglo.com/api/health``api.lisglo.com/api/health`均为HTTP 200,主JS`index-CFz3CyND.js`及CSS`index-C3z6fQED.css`的公网下载SHA-256与服务器产物逐项一致。
- 发布后三条Stream的`last-delivered-id / entries-read`与发布前一致:commands为`1788517020869-2 / 84015`results为`1788527156466-0 / 45876`protocol logs为`1788527156467-0 / 32227`;全部保持`pending=0 / lag=0`。本轮未发送、补发、重投、重新入队或手工ACK短信,也没有修改余额、通道、客户或签名业务资料。
- 一条富泷供应商通道在重启后首次鉴权失败,连接短暂为`8/9`,系统按既有计划于21:20:25自动重试并恢复`9/9`,未修改通道配置或凭据。真实下游客户端受旧心跳租约影响的重复连接被连接数门禁拒绝,并记录3次既有`close of closed channel`连接协程panic;Gateway进程未退出、下游心跳持续、systemd error级journal为0,该已知重启恢复缺陷不由本次报备功能引入,继续保留专项治理。
- 浏览器控制在读取预生产运营登录页时连续两次超时,因此本轮没有取得可复核的DOM、截图或控制台证据,也未输入账号、密码或验证码;只把真实HTTP、资源哈希和服务端证据记为通过,不将其冒充登录后浏览器验收。
## 2026-09-04 大文件WPS报备异步解析与真实进度(本地验证完成)
- 40MB以上WPS文件同步解析会让单个HTTP请求同时承担上传、两轮ZIP/图片展开和ExcelJS解析,预生产证据已出现约126秒后由入口断开的499;单纯展示前端动画进度不能延长代理读超时。本轮将接口改为文件入MinIO并创建PostgreSQL任务后返回202,由独立`cmpp-report-material-worker`单并发解析,页面轮询租户隔离的真实任务状态。
- 新增`queued/analyzing/analyzed/failed`状态、0–100进度、阶段、错误、开始和心跳时间;Worker启动时将心跳超过5分钟的解析任务恢复为排队状态,领取时使用状态条件更新防止重复占用。该变更新增1项Prisma migration和1个systemd服务,发布时必须同时迁移数据库、安装并验证Worker,不涉及短信队列、通道、余额或客户配置。
- 报备导入使用独立100MiB上限,普通文件仍为10MiB、图片仍为2MiB。文件类型、扩展名和ZIP签名在入库前校验;工作簿公式、外部对象、解压体积、图片格式及图片数量关系在后台解析或提交阶段明确失败,不静默吞错。
- 分析阶段读取WPS`cellimages.xml`关系、单元格位置、扩展名和ZIP声明大小,不展开图片Buffer,并在交给ExcelJS前从临时解析副本剥离WPS媒体;提交导入时仍重新下载原文件并完整读取、签名校验和上传图片,没有放宽DISPIMG白名单或20MiB单图/300MiB图片总量限制。
- 真实样本`行业报备.xlsx`仍得到工作表“行业”、17行、13列和43张图片。相同进程环境下原完整图片模式约5522ms/RSS 259MiB,新分析模式约1899ms/RSS 227MiB,耗时下降约66%,结果计数一致;样本只读,未覆盖或写回。
- 定向API 3套25项、API全量54套616项、前端全量13文件64项通过;前后端TypeScript、API/Vite生产构建、依赖与安全门禁、部署契约、结构质量、入口包体积和`git diff --check`通过。测试环境部署及真实HTTP/MinIO/PostgreSQL/浏览器验收将在本提交推送后执行,预生产不在本轮范围内。
+23
View File
@@ -179,6 +179,10 @@ export const adminChannelsReportsApi = {
return requestForm<
Record<string, unknown> & {
id: string;
status: 'queued' | 'analyzing' | 'analyzed' | 'failed';
progress: number;
progressStage?: string;
errorMessage?: string;
columns: Array<{
sourceColumnIndex: number;
columnLetter: string;
@@ -191,6 +195,25 @@ export const adminChannelsReportsApi = {
}
>('/admin/report-materials/imports/analyze', form);
},
getReportMaterialImportAnalysisStatus: (id: string, tenantId: string, signal?: AbortSignal) =>
request<
Record<string, unknown> & {
id: string;
status: 'queued' | 'analyzing' | 'analyzed' | 'failed';
progress: number;
progressStage?: string;
errorMessage?: string;
columns: Array<{
sourceColumnIndex: number;
columnLetter: string;
sourceHeader: string;
sourceHeaderPath: string;
imageCount: number;
}>;
rows: Array<Record<string, unknown>>;
suggestedMappings: ReportImportMapping[];
}
>(withQuery(`/admin/report-materials/imports/${id}/analysis-status`, { tenantId }), { signal }),
commitReportMaterialImport: (
id: string,
body: { mappings: ReportImportMapping[]; profile?: Omit<ReportImportProfile, 'id'> & { id?: string } },
@@ -6,6 +6,7 @@ import { ReportMaterialImportModal } from './ReportMaterialImportModal';
const { adminApi } = vi.hoisted(() => ({
adminApi: {
analyzeReportMaterialImport: vi.fn(),
getReportMaterialImportAnalysisStatus: vi.fn(),
listDrainageFields: vi.fn(),
listEnterpriseApplicationOptions: vi.fn(),
listReportImportProfiles: vi.fn(),
@@ -27,6 +28,15 @@ describe('ReportMaterialImportModal mapping profile action', () => {
adminApi.listReportImportProfiles.mockResolvedValue([]);
adminApi.analyzeReportMaterialImport.mockResolvedValue({
id: 'analysis-1',
status: 'queued',
progress: 10,
progressStage: '文件已上传,等待解析',
});
adminApi.getReportMaterialImportAnalysisStatus.mockResolvedValue({
id: 'analysis-1',
status: 'analyzed',
progress: 100,
progressStage: '解析完成',
columns: [
{ sourceColumnIndex: 0, columnLetter: 'A', sourceHeader: '签名', sourceHeaderPath: '签名', imageCount: 0 },
],
@@ -59,5 +69,5 @@ describe('ReportMaterialImportModal mapping profile action', () => {
expect(screen.getByRole('button', { name: '本次将保存/更新映射方案' })).toHaveAttribute('aria-pressed', 'true'),
);
expect(screen.getByLabelText('映射方案名称')).toBeVisible();
});
}, 10_000);
});
+85 -8
View File
@@ -25,6 +25,7 @@ type AnalyzeResult = {
rows: Array<{ rowNumber: number; values: Record<string, string>; imageColumns: number[] }>;
suggestedMappings: ReportImportMapping[];
};
type AnalysisProgress = { id: string; status: string; progress: number; progressStage?: string; errorMessage?: string };
const transforms = [
{ label: '保持原值', value: '' },
@@ -60,6 +61,7 @@ export function ReportMaterialImportModal({ onClose, onCompleted }: { onClose: (
const [headerRowCount, setHeaderRowCount] = useState(1);
const [dataStartRow, setDataStartRow] = useState(2);
const [analysis, setAnalysis] = useState<AnalyzeResult>();
const [analysisJob, setAnalysisJob] = useState<AnalysisProgress>();
const [mappings, setMappings] = useState<ReportImportMapping[]>([]);
const [profileName, setProfileName] = useState('');
const [saveProfile, setSaveProfile] = useState(false);
@@ -87,10 +89,52 @@ export function ReportMaterialImportModal({ onClose, onCompleted }: { onClose: (
.catch(() => setProfiles([]));
}, [reportType]);
useEffect(() => {
if (!analysisJob?.id || !tenantId || analysisJob.status === 'analyzed' || analysisJob.status === 'failed') return;
const controller = new AbortController();
let timer: ReturnType<typeof setTimeout> | undefined;
const poll = async () => {
try {
const result = await adminApi.getReportMaterialImportAnalysisStatus(
analysisJob.id,
tenantId,
controller.signal,
);
if (controller.signal.aborted) return;
setError('');
setAnalysisJob(result);
if (result.status === 'analyzed') {
const completed = result as AnalyzeResult & AnalysisProgress;
setAnalysis(completed);
setMappings(completed.suggestedMappings ?? []);
const selectedProfile = profiles.find((item) => item.id === profileId);
if (selectedProfile) setProfileName(selectedProfile.name);
return;
}
if (result.status === 'failed') {
setError(result.errorMessage || '文件解析失败');
return;
}
timer = setTimeout(() => void poll(), 1200);
} catch (failure) {
if (!controller.signal.aborted) {
setError(failure instanceof Error ? `${failure.message},正在重试` : '解析进度查询失败,正在重试');
timer = setTimeout(() => void poll(), 2000);
}
}
};
void poll();
return () => {
controller.abort();
if (timer) clearTimeout(timer);
};
}, [analysisJob?.id, analysisJob?.status, profileId, profiles, tenantId]);
function changeReportType(next: ReportType) {
setReportType(next);
setProfileId('');
setAnalysis(undefined);
setAnalysisJob(undefined);
setMappings([]);
}
@@ -108,18 +152,15 @@ export function ReportMaterialImportModal({ onClose, onCompleted }: { onClose: (
setBusy(true);
setError('');
try {
const result = (await adminApi.analyzeReportMaterialImport(file, {
const result = await adminApi.analyzeReportMaterialImport(file, {
tenantId,
applicationId,
reportType,
headerRowCount,
dataStartRow,
profileId: profileId || undefined,
})) as AnalyzeResult;
setAnalysis(result);
setMappings(result.suggestedMappings ?? []);
const selectedProfile = profiles.find((item) => item.id === profileId);
if (selectedProfile) setProfileName(selectedProfile.name);
});
setAnalysisJob(result);
} catch (failure) {
setError(failure instanceof Error ? failure.message : '文件解析失败');
} finally {
@@ -216,8 +257,21 @@ export function ReportMaterialImportModal({ onClose, onCompleted }: { onClose: (
{busy ? '提交中...' : '提交导入审核'}
</Button>
) : (
<Button disabled={busy || !file || !tenantId || !applicationId} onClick={() => void analyze()}>
{busy ? '解析中...' : '解析文件并配置映射'}
<Button
disabled={
busy ||
Boolean(analysisJob && !['failed', 'analyzed'].includes(analysisJob.status)) ||
!file ||
!tenantId ||
!applicationId
}
onClick={() => void analyze()}
>
{busy
? '上传中...'
: analysisJob && !['failed', 'analyzed'].includes(analysisJob.status)
? '后台解析中...'
: '解析文件并配置映射'}
</Button>
)}
</>
@@ -306,10 +360,33 @@ export function ReportMaterialImportModal({ onClose, onCompleted }: { onClose: (
onChange={(event) => {
setFile(event.target.files?.[0]);
setAnalysis(undefined);
setAnalysisJob(undefined);
setError('');
}}
type="file"
/>
</label>
{analysisJob && analysisJob.status !== 'analyzed' ? (
<div className="report-import-analysis-progress" aria-live="polite">
<div>
<strong>{analysisJob.progressStage || '正在处理'}</strong>
<span>{Math.max(0, Math.min(100, analysisJob.progress))}%</span>
</div>
<div
className="batch-progress__track"
role="progressbar"
aria-valuemin={0}
aria-valuemax={100}
aria-valuenow={analysisJob.progress}
>
<span
className="batch-progress__bar batch-progress__bar--sending"
style={{ width: `${Math.max(0, Math.min(100, analysisJob.progress))}%` }}
/>
</div>
<small></small>
</div>
) : null}
{analysis ? (
<div className="report-import-mapping">
<div className="channel-field-section-head">
+3
View File
@@ -6099,6 +6099,9 @@
.report-import-file span { display: flex; align-items: center; gap: 9px; }
.report-import-file input { max-width: 310px; }
.report-import-analysis-progress { display: grid; gap: 8px; padding: 12px; border: 1px solid var(--border); border-radius: 10px; background: var(--surface-muted); }
.report-import-analysis-progress > div:first-child { display: flex; justify-content: space-between; gap: 12px; }
.report-import-analysis-progress small { color: var(--text-muted); }
.report-import-mapping { display: grid; gap: 12px; }
.report-import-mapping-table { overflow-x: auto; border: 1px solid var(--border); border-radius: 10px; }
.report-import-mapping-head,
+22 -1
View File
@@ -206,7 +206,7 @@ SQL
write_env() {
log "Writing production environment"
mkdir -p /etc/cmpp-platform "$APP_DIR" "$APP_DIR/logs/api" "$APP_DIR/logs/send-worker" "$APP_DIR/logs/gateway" "$APP_DIR/backups" "$OBJECT_STORAGE_LOCAL_ROOT"
mkdir -p /etc/cmpp-platform "$APP_DIR" "$APP_DIR/logs/api" "$APP_DIR/logs/send-worker" "$APP_DIR/logs/report-material-worker" "$APP_DIR/logs/gateway" "$APP_DIR/backups" "$OBJECT_STORAGE_LOCAL_ROOT"
cat >/etc/cmpp-platform/cmpp-platform.env <<EOF
NODE_ENV=production
API_PORT=${API_PORT}
@@ -332,6 +332,27 @@ RestartSec=5
StandardOutput=append:${APP_DIR}/logs/send-worker/stdout.log
StandardError=append:${APP_DIR}/logs/send-worker/stderr.log
[Install]
WantedBy=multi-user.target
EOF
cat >/etc/systemd/system/cmpp-report-material-worker.service <<EOF
[Unit]
Description=CMPP report material workbook analysis worker
After=network.target postgresql.service cmpp-minio.service
[Service]
User=cmpp-api
Group=cmpp-security
WorkingDirectory=${APP_DIR}/api
EnvironmentFile=/etc/cmpp-platform/cmpp-platform.env
Environment=CMPP_PROCESS_ROLE=report-material-worker
ExecStart=${node_bin} dist/report-material-analysis-worker.js
Restart=always
RestartSec=5
StandardOutput=append:${APP_DIR}/logs/report-material-worker/stdout.log
StandardError=append:${APP_DIR}/logs/report-material-worker/stderr.log
[Install]
WantedBy=multi-user.target
EOF
+25 -3
View File
@@ -129,7 +129,7 @@ PROD_ADMIN_CREDENTIAL_FILE="$ADMIN_CREDENTIAL_FILE" node tools/deploy/ensure-pro
chmod 600 "$ADMIN_CREDENTIAL_FILE" || true
echo "[deploy] Ensuring runtime log directories"
install -d -m 0755 "$APP_DIR/logs/api" "$APP_DIR/logs/send-worker" "$APP_DIR/logs/submit-outbox" "$APP_DIR/logs/gateway-callback" "$APP_DIR/logs/protocol-log-worker" "$APP_DIR/logs/gateway"
install -d -m 0755 "$APP_DIR/logs/api" "$APP_DIR/logs/send-worker" "$APP_DIR/logs/report-material-worker" "$APP_DIR/logs/submit-outbox" "$APP_DIR/logs/gateway-callback" "$APP_DIR/logs/protocol-log-worker" "$APP_DIR/logs/gateway"
echo "[deploy] Installing split API and send-worker services"
node_bin="$(command -v node)"
@@ -155,6 +155,26 @@ RestartSec=5
StandardOutput=append:$APP_DIR/logs/send-worker/stdout.log
StandardError=append:$APP_DIR/logs/send-worker/stderr.log
[Install]
WantedBy=multi-user.target
EOF
cat >/etc/systemd/system/cmpp-report-material-worker.service <<EOF
[Unit]
Description=CMPP report material workbook analysis worker
After=network.target postgresql.service cmpp-minio.service
[Service]
User=cmpp-api
Group=cmpp-security
WorkingDirectory=$APP_DIR/api
EnvironmentFile=$ENV_FILE
Environment=CMPP_PROCESS_ROLE=report-material-worker
ExecStart=$node_bin dist/report-material-analysis-worker.js
Restart=always
RestartSec=5
StandardOutput=append:$APP_DIR/logs/report-material-worker/stdout.log
StandardError=append:$APP_DIR/logs/report-material-worker/stderr.log
[Install]
WantedBy=multi-user.target
EOF
@@ -243,11 +263,11 @@ echo "[deploy] Restarting services"
systemctl daemon-reload
if [[ "${OBJECT_STORAGE_DRIVER:-minio}" == "local" ]]; then
systemctl disable --now cmpp-minio 2>/dev/null || true
systemctl enable --now cmpp-api cmpp-send-worker cmpp-gateway nginx
systemctl enable --now cmpp-api cmpp-send-worker cmpp-report-material-worker cmpp-gateway nginx
else
systemctl enable --now cmpp-minio
systemctl restart cmpp-minio
systemctl enable --now cmpp-api cmpp-send-worker cmpp-gateway nginx
systemctl enable --now cmpp-api cmpp-send-worker cmpp-report-material-worker cmpp-gateway nginx
fi
systemctl restart cmpp-security-agent
if [[ "${SEND_SUBMIT_OUTBOX_SEPARATE_PROCESS_ENABLED:-false}" == "true" ]]; then
@@ -273,6 +293,7 @@ fi
systemctl restart cmpp-gateway
systemctl restart cmpp-api
systemctl restart cmpp-send-worker
systemctl restart cmpp-report-material-worker
systemctl restart nginx
echo "[deploy] Health checks"
@@ -292,6 +313,7 @@ wait_for_http() {
}
wait_for_http "API" "http://127.0.0.1:${API_PORT:-3000}/api/health"
wait_for_http "Send worker metrics" "http://127.0.0.1:${API_WORKER_METRICS_PORT:-9465}/metrics"
systemctl is-active --quiet cmpp-report-material-worker
if [[ "${SEND_SUBMIT_OUTBOX_SEPARATE_PROCESS_ENABLED:-false}" == "true" ]]; then
wait_for_http "Submit Outbox metrics" "http://127.0.0.1:${API_OUTBOX_METRICS_PORT:-9467}/metrics"
fi