diff --git a/api/package.json b/api/package.json index 2205473..dbd0d23 100644 --- a/api/package.json +++ b/api/package.json @@ -11,6 +11,7 @@ "test:watch": "jest --watch", "start": "node dist/main.js", "start:dev": "ts-node src/main.ts", + "start:report-material-worker": "node dist/report-material-analysis-worker.js", "prisma:generate": "prisma generate", "prisma:migrate:dev": "prisma migrate dev", "prisma:migrate:deploy": "prisma migrate deploy" diff --git a/api/prisma/migrations/20260904090000_report_material_async_analysis/migration.sql b/api/prisma/migrations/20260904090000_report_material_async_analysis/migration.sql new file mode 100644 index 0000000..c05ce65 --- /dev/null +++ b/api/prisma/migrations/20260904090000_report_material_async_analysis/migration.sql @@ -0,0 +1,9 @@ +ALTER TABLE "ReportMaterialImportBatch" + ADD COLUMN "progress" INTEGER NOT NULL DEFAULT 100, + ADD COLUMN "progressStage" TEXT, + ADD COLUMN "errorMessage" TEXT, + ADD COLUMN "startedAt" TIMESTAMP(3), + ADD COLUMN "heartbeatAt" TIMESTAMP(3); + +CREATE INDEX "ReportMaterialImportBatch_status_heartbeatAt_idx" + ON "ReportMaterialImportBatch"("status", "heartbeatAt"); diff --git a/api/prisma/schema.prisma b/api/prisma/schema.prisma index 58a44e5..2fe8558 100644 --- a/api/prisma/schema.prisma +++ b/api/prisma/schema.prisma @@ -1395,6 +1395,11 @@ model ReportMaterialImportBatch { fileName String reportType String status String @default("analyzed") + progress Int @default(100) + progressStage String? + errorMessage String? + startedAt DateTime? + heartbeatAt DateTime? sheetName String headerRowCount Int @default(1) dataStartRow Int @default(2) @@ -1413,6 +1418,7 @@ model ReportMaterialImportBatch { @@index([tenantId, createdAt]) @@index([status, createdAt]) + @@index([status, heartbeatAt]) } model ReportMaterialImportItem { diff --git a/api/src/files/files-upload-limits.spec.ts b/api/src/files/files-upload-limits.spec.ts new file mode 100644 index 0000000..04eb1fb --- /dev/null +++ b/api/src/files/files-upload-limits.spec.ts @@ -0,0 +1,18 @@ +import { assertUploadSize } from './files.service'; + +describe('FilesService report-material upload limits', () => { + const workbook = { + originalname: '行业报备.xlsx', + mimetype: 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', + }; + + it('allows report-material imports up to 100MB without raising the generic file limit', () => { + expect(() => + assertUploadSize('report_material_import', { ...workbook, size: 40 * 1024 * 1024 }), + ).not.toThrow(); + expect(() => assertUploadSize('other', { ...workbook, size: 40 * 1024 * 1024 })).toThrow('10MB'); + expect(() => + assertUploadSize('report_material_import', { ...workbook, size: 101 * 1024 * 1024 }), + ).toThrow('100MB'); + }); +}); diff --git a/api/src/files/files.service.ts b/api/src/files/files.service.ts index 882f0b5..c215896 100644 --- a/api/src/files/files.service.ts +++ b/api/src/files/files.service.ts @@ -40,10 +40,12 @@ export class FilesService { ) {} list(tenantId?: string) { - return this.prisma.fileObject.findMany({ - where: tenantId ? { tenantId } : undefined, - orderBy: { createdAt: 'desc' }, - }).then((items) => items.map(serializeFileObject)); + return this.prisma.fileObject + .findMany({ + where: tenantId ? { tenantId } : undefined, + orderBy: { createdAt: 'desc' }, + }) + .then((items) => items.map(serializeFileObject)); } create(data: CreateFileObjectDto) { @@ -71,7 +73,7 @@ export class FilesService { } async upload(data: UploadFileDto, file: { originalname: string; mimetype: string; size: number; buffer: Buffer }) { - assertUploadSize(file); + assertUploadSize(data.purpose, file); const fileName = normalizeMultipartFileName(file.originalname); const safeName = fileName.replace(/[^\w.\-\u4e00-\u9fa5]/g, '_'); const objectKey = `${data.prefix ?? data.purpose}/${Date.now()}-${randomUUID()}-${safeName}`; @@ -87,7 +89,11 @@ export class FilesService { }); } - async uploadForClient(userId: string | undefined, data: UploadFileDto, file: { originalname: string; mimetype: string; size: number; buffer: Buffer }) { + async uploadForClient( + userId: string | undefined, + data: UploadFileDto, + file: { originalname: string; mimetype: string; size: number; buffer: Buffer }, + ) { const tenantId = await this.resolveClientTenantId(userId); const prefixRule = CLIENT_UPLOAD_RULES[data.purpose]; if (!prefixRule || !data.prefix || !prefixRule.test(data.prefix)) { @@ -135,18 +141,33 @@ export class FilesService { const IMAGE_UPLOAD_MAX_BYTES = 2 * 1024 * 1024; const FILE_UPLOAD_MAX_BYTES = 10 * 1024 * 1024; +const REPORT_MATERIAL_IMPORT_MAX_BYTES = 100 * 1024 * 1024; const IMAGE_FILE_EXTENSION = /\.(?:avif|bmp|gif|heic|heif|jpe?g|png|svg|webp)$/i; -function assertUploadSize(file: { originalname: string; mimetype: string; size: number }) { +export function assertUploadSize(purpose: string, file: { originalname: string; mimetype: string; size: number }) { const image = file.mimetype.toLowerCase().startsWith('image/') || IMAGE_FILE_EXTENSION.test(file.originalname); - const limit = image ? IMAGE_UPLOAD_MAX_BYTES : FILE_UPLOAD_MAX_BYTES; + const reportMaterialImport = purpose === 'report_material_import'; + const limit = reportMaterialImport + ? REPORT_MATERIAL_IMPORT_MAX_BYTES + : image + ? IMAGE_UPLOAD_MAX_BYTES + : FILE_UPLOAD_MAX_BYTES; if (file.size > limit) { - throw new BadRequestException(image ? '图片大小不能超过 2MB' : '文件大小不能超过 10MB'); + throw new BadRequestException( + reportMaterialImport + ? '报备资料文件大小不能超过 100MB' + : image + ? '图片大小不能超过 2MB' + : '文件大小不能超过 10MB', + ); } } function normalizeMultipartFileName(value: string) { - if (![...value].some((character) => character.charCodeAt(0) > 0x7f) || [...value].some((character) => character.charCodeAt(0) > 0xff)) { + if ( + ![...value].some((character) => character.charCodeAt(0) > 0x7f) || + [...value].some((character) => character.charCodeAt(0) > 0xff) + ) { return value; } diff --git a/api/src/report-material-analysis-worker.module.ts b/api/src/report-material-analysis-worker.module.ts new file mode 100644 index 0000000..e397412 --- /dev/null +++ b/api/src/report-material-analysis-worker.module.ts @@ -0,0 +1,13 @@ +import { Module } from '@nestjs/common'; +import { ConfigModule } from '@nestjs/config'; +import { PrismaModule } from './prisma/prisma.module'; +import { ReportMaterialsModule } from './report-materials/report-materials.module'; + +@Module({ + imports: [ + ConfigModule.forRoot({ isGlobal: true, envFilePath: ['.env.local', '.env'] }), + PrismaModule, + ReportMaterialsModule, + ], +}) +export class ReportMaterialAnalysisWorkerModule {} diff --git a/api/src/report-material-analysis-worker.ts b/api/src/report-material-analysis-worker.ts new file mode 100644 index 0000000..dfef746 --- /dev/null +++ b/api/src/report-material-analysis-worker.ts @@ -0,0 +1,31 @@ +import 'reflect-metadata'; +import { NestFactory } from '@nestjs/core'; +import { ReportMaterialAnalysisWorkerModule } from './report-material-analysis-worker.module'; +import { ReportMaterialsService } from './report-materials/report-materials.service'; + +const POLL_INTERVAL_MS = 1000; + +async function bootstrap() { + if ((process.env.CMPP_PROCESS_ROLE?.trim() || 'report-material-worker') !== 'report-material-worker') { + throw new Error('report-material-analysis-worker requires CMPP_PROCESS_ROLE=report-material-worker'); + } + const app = await NestFactory.createApplicationContext(ReportMaterialAnalysisWorkerModule, { + logger: ['log', 'warn', 'error'], + }); + app.enableShutdownHooks(); + const reports = app.get(ReportMaterialsService); + await reports.recoverStaleAnalysisJobs(); + let stopping = false; + const stop = () => { + stopping = true; + }; + process.once('SIGTERM', stop); + process.once('SIGINT', stop); + while (!stopping) { + const worked = await reports.processNextAnalysisJob(); + if (!worked) await new Promise((resolve) => setTimeout(resolve, POLL_INTERVAL_MS)); + } + await app.close(); +} + +void bootstrap(); diff --git a/api/src/report-materials/import-parser.service.ts b/api/src/report-materials/import-parser.service.ts index 2fcb734..61fca3b 100644 --- a/api/src/report-materials/import-parser.service.ts +++ b/api/src/report-materials/import-parser.service.ts @@ -1,4 +1,4 @@ -import { BadRequestException } from '@nestjs/common'; +import { BadRequestException, NotFoundException } from '@nestjs/common'; import { Prisma } from '@prisma/client'; import { extname } from 'node:path'; import { FilesService } from '../files/files.service'; @@ -13,7 +13,7 @@ import { clamp, cellText, } from './report-materials.helpers'; -import { compatibleImages, loadCompatibleWorkbook } from './workbook-compatibility'; +import { compatibleImagePositions, loadCompatibleWorkbook } from './workbook-compatibility'; /** R4 report-materials domain service composed behind ReportMaterialsService. */ export class ReportImportParserService { @@ -63,14 +63,123 @@ export class ReportImportParserService { file: { originalname: string; mimetype: string; size: number; buffer: Buffer }, options: AnalyzeImportOptions, ) { - if (!options.tenantId) throw new BadRequestException('tenantId is required'); - if (!options.applicationId) throw new BadRequestException('applicationId is required'); - await this.smsConfig.getApplication(options.applicationId, options.tenantId); - if (!['signature', 'drainage'].includes(options.reportType)) - throw new BadRequestException('reportType must be signature or drainage'); - if (extname(file.originalname).toLowerCase() !== '.xlsx' || file.buffer[0] !== 0x50 || file.buffer[1] !== 0x4b) - throw new BadRequestException('仅支持有效的 XLSX 文件'); - const { workbook, wpsImagesBySheet } = await loadCompatibleWorkbook(file.buffer); + await this.validateImport(file, options); + const analysis = await this.parseAnalysis(file.buffer, options); + const sourceFile = await this.files.upload( + { tenantId: options.tenantId, purpose: 'report_material_import', prefix: 'report-material-imports' }, + file, + ); + const batch = await this.createAnalyzedBatch(sourceFile, options, analysis); + await this.logAnalyzed(batch.id, sourceFile.fileName, options, analysis.sheetName, analysis.rows.length); + return { ...batch, sourceFile, ...analysis }; + } + + async enqueueImport( + file: { originalname: string; mimetype: string; size: number; buffer: Buffer }, + options: AnalyzeImportOptions, + ) { + await this.validateImport(file, options); + const sourceFile = await this.files.upload( + { tenantId: options.tenantId, purpose: 'report_material_import', prefix: 'report-material-imports' }, + file, + ); + const batch = await this.prisma.reportMaterialImportBatch.create({ + data: { + tenantId: options.tenantId, + applicationId: options.applicationId, + profileId: options.profileId, + fileObjectId: sourceFile.id, + fileName: sourceFile.fileName, + reportType: options.reportType, + status: 'queued', + progress: 10, + progressStage: '文件已上传，等待解析', + sheetName: options.sheetName ?? '', + headerRowCount: clamp(options.headerRowCount, 1, 5), + dataStartRow: Math.max(options.dataStartRow, clamp(options.headerRowCount, 1, 5) + 1), + mapping: [] as Prisma.InputJsonValue, + result: { operatorId: options.operatorId } as Prisma.InputJsonValue, + }, + }); + return this.analysisStatus(batch); + } + + async getAnalysisStatus(batchId: string, tenantId: string) { + if (!tenantId) throw new BadRequestException('tenantId is required'); + const batch = await this.prisma.reportMaterialImportBatch.findFirst({ where: { id: batchId, tenantId } }); + if (!batch) throw new NotFoundException('导入解析任务不存在'); + return this.analysisStatus(batch); + } + + async processQueuedBatch(batchId: string) { + const batch = await this.prisma.reportMaterialImportBatch.findUnique({ where: { id: batchId } }); + if (!batch || batch.status !== 'analyzing') return false; + try { + await this.progress(batchId, 25, '正在读取工作簿'); + const { content } = await this.files.getDownload(batch.fileObjectId); + const resultData = (batch.result && typeof batch.result === 'object' ? batch.result : {}) as { + operatorId?: string; + }; + const options: AnalyzeImportOptions = { + tenantId: batch.tenantId, + applicationId: batch.applicationId ?? '', + reportType: batch.reportType as 'signature' | 'drainage', + sheetName: batch.sheetName || undefined, + headerRowCount: batch.headerRowCount, + dataStartRow: batch.dataStartRow, + profileId: batch.profileId ?? undefined, + operatorId: resultData.operatorId, + }; + await this.progress(batchId, 30, '正在解析表格与图片位置'); + const analysis = await this.parseAnalysis(content, options, async (progress, stage) => { + await this.progress(batchId, progress, stage); + }); + const updated = await this.prisma.reportMaterialImportBatch.update({ + where: { id: batchId }, + data: { + status: 'analyzed', + progress: 100, + progressStage: '解析完成', + errorMessage: null, + heartbeatAt: new Date(), + sheetName: analysis.sheetName, + mapping: analysis.suggestedMappings as Prisma.InputJsonValue, + preview: { + sheets: analysis.sheets, + columns: analysis.columns, + rows: analysis.rows, + imageCount: analysis.imageCount, + } as Prisma.InputJsonValue, + result: Prisma.JsonNull, + rowCount: analysis.rowCount, + }, + }); + await this.logAnalyzed(batchId, batch.fileName, options, analysis.sheetName, analysis.rows.length); + return this.analysisStatus(updated); + } catch (error) { + const message = error instanceof Error ? error.message : '文件解析失败'; + await this.prisma.reportMaterialImportBatch.update({ + where: { id: batchId }, + data: { + status: 'failed', + progressStage: '解析失败', + errorMessage: message.slice(0, 1000), + heartbeatAt: new Date(), + }, + }); + return false; + } + } + + private async parseAnalysis( + buffer: Buffer, + options: AnalyzeImportOptions, + onProgress?: (progress: number, stage: string) => void | Promise, + ) { + const { workbook, wpsImagesBySheet } = await loadCompatibleWorkbook(buffer, { + includeImageData: false, + onProgress, + }); const profile = options.profileId ? await this.prisma.reportMaterialImportProfile.findUnique({ where: { id: options.profileId }, @@ -82,7 +191,7 @@ export class ReportImportParserService { if (!worksheet) throw new BadRequestException('工作簿没有可读取的工作表'); const headerRowCount = clamp(options.headerRowCount, 1, 5); const dataStartRow = Math.max(options.dataStartRow, headerRowCount + 1); - const images = compatibleImages(workbook, worksheet, wpsImagesBySheet); + const images = compatibleImagePositions(workbook, worksheet, wpsImagesBySheet); const columnCount = Math.min(worksheet.columnCount, 200); const columns = Array.from({ length: columnCount }, (_, offset) => { const sourceColumnIndex = offset + 1; @@ -110,10 +219,6 @@ export class ReportImportParserService { if (Object.values(values).some(Boolean) || imageColumns.length) previewRows.push({ rowNumber, values, imageColumns }); } - const sourceFile = await this.files.upload( - { tenantId: options.tenantId, purpose: 'report_material_import', prefix: 'report-material-imports' }, - file, - ); const profileMappings = profile?.columns.map((column) => ({ sourceHeader: column.sourceHeader, sourceHeaderPath: column.sourceHeaderPath ?? undefined, @@ -128,7 +233,24 @@ export class ReportImportParserService { const suggestedMappings = profileMappings?.length ? remapProfileColumns(profileMappings, columns) : suggestMappings(columns, options.reportType); - const batch = await this.prisma.reportMaterialImportBatch.create({ + await onProgress?.(95, '已生成字段映射与预览'); + return { + sheetName: worksheet.name, + sheets: workbook.worksheets.map((sheet) => sheet.name), + columns, + rows: previewRows, + imageCount: images.length, + suggestedMappings, + rowCount: Math.max(0, worksheet.rowCount - dataStartRow + 1), + }; + } + + private createAnalyzedBatch( + sourceFile: { id: string; fileName: string }, + options: AnalyzeImportOptions, + analysis: Awaited>, + ) { + return this.prisma.reportMaterialImportBatch.create({ data: { tenantId: options.tenantId, applicationId: options.applicationId, @@ -136,42 +258,91 @@ export class ReportImportParserService { fileObjectId: sourceFile.id, fileName: sourceFile.fileName, reportType: options.reportType, - sheetName: worksheet.name, - headerRowCount, - dataStartRow, - mapping: suggestedMappings as Prisma.InputJsonValue, + sheetName: analysis.sheetName, + headerRowCount: clamp(options.headerRowCount, 1, 5), + dataStartRow: Math.max(options.dataStartRow, clamp(options.headerRowCount, 1, 5) + 1), + mapping: analysis.suggestedMappings as Prisma.InputJsonValue, preview: { - sheets: workbook.worksheets.map((sheet) => sheet.name), - columns, - rows: previewRows, - imageCount: images.length, + sheets: analysis.sheets, + columns: analysis.columns, + rows: analysis.rows, + imageCount: analysis.imageCount, } as Prisma.InputJsonValue, - rowCount: Math.max(0, worksheet.rowCount - dataStartRow + 1), + rowCount: analysis.rowCount, }, }); + } + + private async validateImport( + file: { originalname: string; size: number; buffer: Buffer }, + options: AnalyzeImportOptions, + ) { + if (!options.tenantId) throw new BadRequestException('tenantId is required'); + if (!options.applicationId) throw new BadRequestException('applicationId is required'); + await this.smsConfig.getApplication(options.applicationId, options.tenantId); + if (!['signature', 'drainage'].includes(options.reportType)) + throw new BadRequestException('reportType must be signature or drainage'); + if (extname(file.originalname).toLowerCase() !== '.xlsx' || file.buffer[0] !== 0x50 || file.buffer[1] !== 0x4b) + throw new BadRequestException('仅支持有效的 XLSX 文件'); + } + + private progress(batchId: string, progress: number, progressStage: string) { + return this.prisma.reportMaterialImportBatch.update({ + where: { id: batchId }, + data: { progress, progressStage, heartbeatAt: new Date() }, + }); + } + + private analysisStatus(batch: { + id: string; + status: string; + progress: number; + progressStage: string | null; + errorMessage: string | null; + sheetName: string; + preview: Prisma.JsonValue | null; + mapping: Prisma.JsonValue; + }) { + const preview = + batch.preview && typeof batch.preview === 'object' && !Array.isArray(batch.preview) + ? (batch.preview as Record) + : {}; + return { + id: batch.id, + status: batch.status, + progress: batch.progress, + progressStage: batch.progressStage, + errorMessage: batch.errorMessage, + sheetName: batch.sheetName || undefined, + sheets: preview.sheets ?? [], + columns: preview.columns ?? [], + rows: preview.rows ?? [], + imageCount: preview.imageCount ?? 0, + suggestedMappings: batch.mapping ?? [], + }; + } + + private async logAnalyzed( + batchId: string, + fileName: string, + options: AnalyzeImportOptions, + sheetName: string, + previewCount: number, + ) { await this.prisma.operationLog.create({ data: { tenantId: options.tenantId, userId: options.operatorId, action: 'report_material.import_analyzed', resource: 'report_material_import', - resourceId: batch.id, + resourceId: batchId, detail: { - fileName: sourceFile.fileName, - filters: { applicationId: options.applicationId, reportType: options.reportType, sheetName: worksheet.name }, - successCount: previewRows.length, + fileName, + filters: { applicationId: options.applicationId, reportType: options.reportType, sheetName }, + successCount: previewCount, failedCount: 0, } as Prisma.InputJsonValue, }, }); - return { - ...batch, - sourceFile, - sheets: workbook.worksheets.map((sheet) => sheet.name), - columns, - rows: previewRows, - imageCount: images.length, - suggestedMappings, - }; } } diff --git a/api/src/report-materials/report-materials.contracts.ts b/api/src/report-materials/report-materials.contracts.ts index 329f9b1..eedf293 100644 --- a/api/src/report-materials/report-materials.contracts.ts +++ b/api/src/report-materials/report-materials.contracts.ts @@ -105,5 +105,6 @@ export type AnalyzeImportOptions = { }; export type EmbeddedImage = { row: number; column: number; extension: string; buffer: Buffer }; +export type EmbeddedImageMetadata = { row: number; column: number; extension: string; size: number; target: string }; export type ReportWorkbookFormat = 'excel_drawing' | 'wps_cell_image'; diff --git a/api/src/report-materials/report-materials.controller.ts b/api/src/report-materials/report-materials.controller.ts index 517f4ae..881fa29 100644 --- a/api/src/report-materials/report-materials.controller.ts +++ b/api/src/report-materials/report-materials.controller.ts @@ -3,6 +3,7 @@ import { Body, Controller, Get, + HttpCode, Param, Post, Put, @@ -93,6 +94,7 @@ export class ReportMaterialsController { } @Post('imports/analyze') + @HttpCode(202) @UseInterceptors( FileInterceptor('file', { limits: { fileSize: 100 * 1024 * 1024, files: 1, fields: 12, parts: 13 } }), ) @@ -102,7 +104,7 @@ export class ReportMaterialsController { @CurrentSessionUserId() operatorId?: string, ) { if (!file) throw new BadRequestException('请选择 XLSX 文件'); - return this.service.analyzeImport(file, { + return this.service.enqueueImport(file, { tenantId: body.tenantId, applicationId: body.applicationId, reportType: body.reportType as 'signature' | 'drainage', @@ -114,6 +116,11 @@ export class ReportMaterialsController { }); } + @Get('imports/:id/analysis-status') + getImportAnalysisStatus(@Param('id') id: string, @Query('tenantId') tenantId: string) { + return this.service.getAnalysisStatus(id, tenantId); + } + @Put('imports/:id/commit') @RequireRecentAuthentication() commitImport(@Param('id') id: string, @Body() body: ImportCommitDto, @CurrentSessionUserId() operatorId?: string) { diff --git a/api/src/report-materials/report-materials.service.spec.ts b/api/src/report-materials/report-materials.service.spec.ts index f4446cb..5ac7aa7 100644 --- a/api/src/report-materials/report-materials.service.spec.ts +++ b/api/src/report-materials/report-materials.service.spec.ts @@ -4,6 +4,75 @@ import { ReportMaterialsService } from './report-materials.service'; import { mappedCorePatchValue, signatureIdentityReportValues } from './report-materials.helpers'; describe('ReportMaterialsService', () => { + it('stores a valid workbook as a durable queued analysis job before parsing it', async () => { + const prisma = { + reportMaterialImportBatch: { + create: jest.fn().mockImplementation(({ data }: { data: Record }) => + Promise.resolve({ id: 'job-1', preview: null, errorMessage: null, ...data }), + ), + }, + }; + const files = { upload: jest.fn().mockResolvedValue({ id: 'file-1', fileName: '行业报备.xlsx' }) }; + const service = new ReportMaterialsService( + prisma as never, + files as never, + { getApplication: jest.fn().mockResolvedValue({ id: 'app-1' }) } as never, + ); + const result = await service.enqueueImport( + { originalname: '行业报备.xlsx', mimetype: 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', size: 2, buffer: Buffer.from('PK') }, + { tenantId: 'tenant-1', applicationId: 'app-1', reportType: 'signature', headerRowCount: 1, dataStartRow: 2 }, + ); + expect(files.upload).toHaveBeenCalledWith( + expect.objectContaining({ purpose: 'report_material_import', tenantId: 'tenant-1' }), + expect.any(Object), + ); + expect(prisma.reportMaterialImportBatch.create).toHaveBeenCalledWith({ + data: expect.objectContaining({ status: 'queued', progress: 10, applicationId: 'app-1' }), + }); + expect(result).toMatchObject({ id: 'job-1', status: 'queued', progress: 10 }); + }); + + it('only returns analysis progress inside the requested tenant', async () => { + const findFirst = jest.fn().mockResolvedValue(null); + const service = new ReportMaterialsService( + { reportMaterialImportBatch: { findFirst } } as never, + {} as never, + {} as never, + ); + await expect(service.getAnalysisStatus('job-1', 'tenant-2')).rejects.toThrow('导入解析任务不存在'); + expect(findFirst).toHaveBeenCalledWith({ where: { id: 'job-1', tenantId: 'tenant-2' } }); + }); + + it('claims only a queued analysis job before handing it to the worker parser', async () => { + const prisma = { + reportMaterialImportBatch: { + findFirst: jest.fn().mockResolvedValue({ id: 'job-1' }), + updateMany: jest.fn().mockResolvedValue({ count: 1 }), + findUnique: jest.fn().mockResolvedValue(null), + }, + }; + const service = new ReportMaterialsService(prisma as never, {} as never, {} as never); + await expect(service.processNextAnalysisJob()).resolves.toBe(true); + expect(prisma.reportMaterialImportBatch.updateMany).toHaveBeenCalledWith({ + where: { id: 'job-1', status: 'queued' }, + data: expect.objectContaining({ status: 'analyzing', progress: 15 }), + }); + }); + + it('requeues stale analyzing jobs so a restarted worker can resume them', async () => { + const updateMany = jest.fn().mockResolvedValue({ count: 2 }); + const service = new ReportMaterialsService( + { reportMaterialImportBatch: { updateMany } } as never, + {} as never, + {} as never, + ); + await expect(service.recoverStaleAnalysisJobs()).resolves.toEqual({ count: 2 }); + expect(updateMany).toHaveBeenCalledWith({ + where: { status: 'analyzing', OR: [{ heartbeatAt: null }, { heartbeatAt: { lt: expect.any(Date) } }] }, + data: { status: 'queued', progress: 10, progressStage: '等待重新解析', errorMessage: null }, + }); + }); + it('requires an enterprise application before parsing an import workbook', async () => { const service = new ReportMaterialsService({} as never, { upload: jest.fn() } as never, {} as never); await expect( diff --git a/api/src/report-materials/report-materials.service.ts b/api/src/report-materials/report-materials.service.ts index e286f68..22a683c 100644 --- a/api/src/report-materials/report-materials.service.ts +++ b/api/src/report-materials/report-materials.service.ts @@ -34,7 +34,11 @@ export class ReportMaterialsService { private readonly channelExport: ReportChannelExportService; private readonly batchGeneration: ReportBatchGenerationService; - constructor(prisma: PrismaService, files: FilesService, smsConfig: SmsConfigService) { + constructor( + private readonly prisma: PrismaService, + files: FilesService, + smsConfig: SmsConfigService, + ) { this.pendingQuery = new ReportPendingQueryService(prisma, files, smsConfig); this.officialExport = new ReportOfficialExportService(prisma, files, smsConfig, this.pendingQuery); this.importParser = new ReportImportParserService(prisma, files, smsConfig); @@ -82,6 +86,48 @@ export class ReportMaterialsService { return this.importParser.analyzeImport(file, options); } + async enqueueImport( + file: { originalname: string; mimetype: string; size: number; buffer: Buffer }, + options: AnalyzeImportOptions, + ) { + return this.importParser.enqueueImport(file, options); + } + + async getAnalysisStatus(batchId: string, tenantId: string) { + return this.importParser.getAnalysisStatus(batchId, tenantId); + } + + async recoverStaleAnalysisJobs() { + const staleBefore = new Date(Date.now() - 5 * 60 * 1000); + return this.prisma.reportMaterialImportBatch.updateMany({ + where: { status: 'analyzing', OR: [{ heartbeatAt: null }, { heartbeatAt: { lt: staleBefore } }] }, + data: { status: 'queued', progress: 10, progressStage: '等待重新解析', errorMessage: null }, + }); + } + + async processNextAnalysisJob() { + const candidate = await this.prisma.reportMaterialImportBatch.findFirst({ + where: { status: 'queued' }, + orderBy: { createdAt: 'asc' }, + select: { id: true }, + }); + if (!candidate) return false; + const claimed = await this.prisma.reportMaterialImportBatch.updateMany({ + where: { id: candidate.id, status: 'queued' }, + data: { + status: 'analyzing', + progress: 15, + progressStage: '后台解析已开始', + startedAt: new Date(), + heartbeatAt: new Date(), + errorMessage: null, + }, + }); + if (!claimed.count) return true; + await this.importParser.processQueuedBatch(candidate.id); + return true; + } + async commitImport(batchId: string, data: ImportCommitDto) { return this.importReview.commitImport(batchId, data); } diff --git a/api/src/report-materials/workbook-compatibility.spec.ts b/api/src/report-materials/workbook-compatibility.spec.ts index 206ff9d..dddc47c 100644 --- a/api/src/report-materials/workbook-compatibility.spec.ts +++ b/api/src/report-materials/workbook-compatibility.spec.ts @@ -1,6 +1,11 @@ import ExcelJS from 'exceljs'; import JSZip from 'jszip'; -import { compatibleImages, convertWorkbookOutput, loadCompatibleWorkbook } from './workbook-compatibility'; +import { + compatibleImagePositions, + compatibleImages, + convertWorkbookOutput, + loadCompatibleWorkbook, +} from './workbook-compatibility'; const PNG = Buffer.from( 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAusB9Y9ZrYQAAAAASUVORK5CYII=', @@ -39,4 +44,15 @@ describe('WPS workbook compatibility', () => { const content = Buffer.from(await workbook.xlsx.writeBuffer()); await expect(loadCompatibleWorkbook(content)).rejects.toThrow('不允许的公式'); }); + + it('reads WPS image positions without inflating image buffers during analysis', async () => { + const converted = await convertWorkbookOutput(await standardWorkbook(), 'wps_cell_image'); + const loaded = await loadCompatibleWorkbook(converted, { includeImageData: false }); + const wpsImage = loaded.wpsImagesBySheet.get('签名报备')?.[0]; + expect(wpsImage).toMatchObject({ row: 2, column: 1, extension: 'png', size: PNG.length }); + expect(wpsImage).not.toHaveProperty('buffer'); + expect( + compatibleImagePositions(loaded.workbook, loaded.workbook.getWorksheet('签名报备')!, loaded.wpsImagesBySheet), + ).toEqual([{ row: 2, column: 1 }]); + }); }); diff --git a/api/src/report-materials/workbook-compatibility.ts b/api/src/report-materials/workbook-compatibility.ts index 7938063..8bf7467 100644 --- a/api/src/report-materials/workbook-compatibility.ts +++ b/api/src/report-materials/workbook-compatibility.ts @@ -2,7 +2,7 @@ import { BadRequestException } from '@nestjs/common'; import ExcelJS from 'exceljs'; import JSZip from 'jszip'; import { randomUUID } from 'node:crypto'; -import type { EmbeddedImage, ReportWorkbookFormat } from './report-materials.contracts'; +import type { EmbeddedImage, EmbeddedImageMetadata, ReportWorkbookFormat } from './report-materials.contracts'; import { normalizeImageExtension, readEmbeddedImages } from './report-materials.helpers'; const MAX_WORKBOOK_BYTES = 100 * 1024 * 1024; @@ -72,9 +72,14 @@ function validateWorkbookValues(workbook: ExcelJS.Workbook, allowedWpsIds: Set(), imagesBySheet: new Map() }; + if (!cellImagesXml) + return { + allowedIds: new Set(), + imagesBySheet: new Map>(), + mediaTargets: new Set(), + }; const relXml = await text(zip, 'xl/_rels/cellimages.xml.rels'); const relTargets = new Map(); for (const match of relXml.matchAll(/]*)\/?>(?:<\/Relationship>)?/g)) { @@ -99,13 +104,14 @@ async function inspectWpsImages(zip: JSZip) { sheetRelTargets.set(attrs.Id, packagePath(attrs.Target)); } const allowedIds = new Set(); - const imagesBySheet = new Map(); + const imagesBySheet = new Map>(); + let totalImageBytes = 0; for (const match of workbookXml.matchAll(/]*)\/?>(?:<\/sheet>)?/g)) { const attrs = attributes(match[1]); const path = sheetRelTargets.get(attrs['r:id']); if (!attrs.name || !path) continue; const sheetXml = await text(zip, path); - const images: EmbeddedImage[] = []; + const images: Array = []; for (const cell of sheetXml.matchAll(/]*)>([\s\S]*?)<\/c>/g)) { const formulaText = decodeXml(cell[2].match(/]*)?>([\s\S]*?)<\/f>/)?.[1]?.trim() ?? ''); if (!formulaText) continue; @@ -118,20 +124,31 @@ async function inspectWpsImages(zip: JSZip) { if (!imageEntry) throw new BadRequestException('WPS单元格图片文件缺失'); const extension = normalizeImageExtension(target.split('.').pop() ?? 'png'); if (!['png', 'jpeg', 'gif'].includes(extension)) throw new BadRequestException('WPS单元格图片格式不受支持'); - const imageBuffer = await imageEntry.async('nodebuffer'); - if (imageBuffer.length > MAX_IMAGE_BYTES) throw new BadRequestException('单张WPS图片不能超过20MB'); - if (!validImageSignature(extension, imageBuffer)) throw new BadRequestException('WPS单元格图片内容与格式不匹配'); - images.push({ ...cellPosition, extension, buffer: imageBuffer }); + const size = Number( + (imageEntry as unknown as { _data?: { uncompressedSize?: number } })._data?.uncompressedSize ?? 0, + ); + if (size > MAX_IMAGE_BYTES) throw new BadRequestException('单张WPS图片不能超过20MB'); + totalImageBytes += size; + if (includeImageData) { + const imageBuffer = await imageEntry.async('nodebuffer'); + if (!validImageSignature(extension, imageBuffer)) + throw new BadRequestException('WPS单元格图片内容与格式不匹配'); + images.push({ ...cellPosition, extension, buffer: imageBuffer }); + } else { + images.push({ ...cellPosition, extension, size, target }); + } allowedIds.add(formula[1]); } imagesBySheet.set(attrs.name, images); } - const totalImageBytes = [...imagesBySheet.values()].flat().reduce((sum, image) => sum + image.buffer.length, 0); if (totalImageBytes > MAX_TOTAL_IMAGE_BYTES) throw new BadRequestException('WPS图片总量不能超过300MB'); - return { allowedIds, imagesBySheet }; + return { allowedIds, imagesBySheet, mediaTargets: new Set(imageTargets.values()) }; } -export async function loadCompatibleWorkbook(buffer: Buffer) { +export async function loadCompatibleWorkbook( + buffer: Buffer, + options: { includeImageData?: boolean; onProgress?: (progress: number, stage: string) => void | Promise } = {}, +) { if (buffer.length > MAX_WORKBOOK_BYTES) throw new BadRequestException('导入文件不能超过100MB'); let zip: JSZip; try { @@ -139,6 +156,7 @@ export async function loadCompatibleWorkbook(buffer: Buffer) { } catch { throw new BadRequestException('仅支持有效的 XLSX 文件'); } + await options.onProgress?.(35, '已读取工作簿压缩包'); const expandedBytes = Object.values(zip.files).reduce( (sum, entry) => sum + Number((entry as unknown as { _data?: { uncompressedSize?: number } })._data?.uncompressedSize ?? 0), @@ -148,17 +166,32 @@ export async function loadCompatibleWorkbook(buffer: Buffer) { if (expandedBytes > MAX_EXPANDED_BYTES) throw new BadRequestException('工作簿解压后超过500MB安全限制'); if (Object.keys(zip.files).some((name) => /(^|\/)(vbaProject|externalLinks|embeddings|activeX)(\/|\.)/i.test(name))) throw new BadRequestException('工作簿包含不允许的外部对象或宏'); - const wps = await inspectWpsImages(zip); + const wps = await inspectWpsImages(zip, options.includeImageData !== false); + await options.onProgress?.(55, '已定位WPS单元格图片'); + let excelJsInput = buffer; + if (options.includeImageData === false && wps.mediaTargets.size) { + for (const target of wps.mediaTargets) zip.remove(target); + zip.remove('xl/cellimages.xml'); + zip.remove('xl/_rels/cellimages.xml.rels'); + excelJsInput = await zip.generateAsync({ + type: 'nodebuffer', + compression: 'DEFLATE', + compressionOptions: { level: 1 }, + }); + await options.onProgress?.(70, '已剥离解析阶段无需加载的WPS图片数据'); + } const workbook = new ExcelJS.Workbook(); - await workbook.xlsx.load(buffer as never); + await workbook.xlsx.load(excelJsInput as never); + await options.onProgress?.(80, '已解析工作表内容'); validateWorkbookValues(workbook, wps.allowedIds); + await options.onProgress?.(90, '已完成工作簿安全校验'); return { workbook, wpsImagesBySheet: wps.imagesBySheet }; } export function compatibleImages( workbook: ExcelJS.Workbook, worksheet: ExcelJS.Worksheet, - wpsImagesBySheet: Map, + wpsImagesBySheet: Map>, ) { const merged = new Map(); const wpsImages = wpsImagesBySheet.get(worksheet.name) ?? []; @@ -169,10 +202,29 @@ export function compatibleImages( if (!validImageSignature(extension, image.buffer)) throw new BadRequestException('工作簿图片内容与格式不匹配'); merged.set(`${image.row}:${image.column}`, image); } - for (const image of wpsImages) merged.set(`${image.row}:${image.column}`, image); + for (const image of wpsImages) { + if (!('buffer' in image)) throw new BadRequestException('图片元数据不能用于提交导入'); + merged.set(`${image.row}:${image.column}`, image); + } return [...merged.values()]; } +export function compatibleImagePositions( + workbook: ExcelJS.Workbook, + worksheet: ExcelJS.Worksheet, + wpsImagesBySheet: Map>, +) { + const wpsImages = wpsImagesBySheet.get(worksheet.name) ?? []; + const positions = new Map(); + for (const image of readEmbeddedImages(workbook, worksheet)) { + if (wpsImages.length && image.buffer.length <= 128) continue; + positions.set(`${image.row}:${image.column}`, { row: image.row, column: image.column }); + } + for (const image of wpsImages) + positions.set(`${image.row}:${image.column}`, { row: image.row, column: image.column }); + return [...positions.values()]; +} + export async function convertWorkbookOutput(content: Buffer, outputFormat: ReportWorkbookFormat = 'excel_drawing') { if (outputFormat === 'excel_drawing') return content; if (outputFormat !== 'wps_cell_image') throw new BadRequestException('不支持的报备文件格式'); diff --git a/docs/system-functional-test-cases.md b/docs/system-functional-test-cases.md index 0d04dfa..67b7820 100644 --- a/docs/system-functional-test-cases.md +++ b/docs/system-functional-test-cases.md @@ -5066,6 +5066,8 @@ npm run verify:phase8 | TC-REPORT-WPS-001 | 上传`行业报备.xlsx`等使用`_xlfn.DISPIMG`、`xl/cellimages.xml`及关系文件的WPS XLSX | 仅精确白名单内且图片关系完整的DISPIMG公式可解析；图片按真实单元格映射进入预览与审核，透明Drawing占位图不计入；普通公式、宏、外部对象、缺失关系和非法图片格式均明确拒绝 | | TC-REPORT-WPS-002 | 分别上传10MB以上且不超过100MB、超过100MB、解压后超过500MB的报备XLSX | 第一类可进入解析；后两类分别在上传层或解包层明确拒绝；Nginx私有站点允许100MB业务文件及multipart开销，不扩大公网单条API边界 | | TC-REPORT-WPS-003 | 在批次和单条报备导出选择“系统 Excel 文件”或“WPS 单元格图片文件” | 默认保持ExcelJS Drawing格式；WPS选项生成DISPIMG与cellimages关系且图片按对应单元格可回读；两种格式均来自同一真实资料快照，不改变报备状态或短信链路 | +| TC-REPORT-WPS-004 | 上传40MB以上且不超过100MB的WPS报备文件，并持续观察解析弹窗与服务重启 | 上传接口在文件写入对象存储并创建PostgreSQL任务后返回202；独立解析Worker单并发处理，页面轮询真实进度；API/浏览器不再等待完整解析，Worker重启后超时租约任务可重新排队且不重复提交导入 | +| TC-REPORT-WPS-005 | 分析包含大量DISPIMG图片的WPS文件，再提交映射进入审核 | 分析阶段只读取图片关系、位置、格式和声明大小，不展开全部WPS图片二进制；提交导入阶段重新读取并校验真实图片签名和20MB单图/300MB总量限制，公式安全校验不放宽 | | TC-REPORT-IMPORT-APPLICATION-001 | 未选择企业应用、选择其他企业的应用、选择当前企业应用后解析导入 | 前两种前后端均阻止导入；合法应用可解析并把applicationId写入导入批次，后续补资料只作用于该企业应用范围 | | TC-REPORT-FIELD-MODAL-001 | 打开签名或引流字段配置弹窗，在左侧字段池数量不同的情况下连续添加字段，并检查通用字段与移除按钮 | 左侧每个字段卡片始终固定为68px，不因可选字段数量或添加操作被Grid拉伸；右侧默认包含同资料类型的通用字段；移除按钮为通用宽度、无红色填充，保存仍调用真实通道字段接口 | | TC-REPORT-MATERIAL-DETAIL-001 | 查看包含当前图片字段、未删除历史字段及旧图片引用的报备资料 | 当前字段展示名称、代码、导出名及图片预览；未删除历史字段继续展示且同时显示字段名称和代码；图片可内联查看并保留下载入口，缺失内容显示明确占位 | diff --git a/docs/testing-progress.md b/docs/testing-progress.md index 5211e84..e9b0234 100644 --- a/docs/testing-progress.md +++ b/docs/testing-progress.md @@ -4478,3 +4478,24 @@ git diff --check - 测试环境发布前为`bb435fb0ac1e7812fcdb59950a4b13757899ab39`。发布包大小2766696字节，本地和服务器SHA-256均为`2d3c9a022ba1b6494a51e2ee0f3eb04eabb2513af25ba3631dae5ea70ea68783`；恢复点为`/opt/cmpp-platform-backups/import-review-fix-20260904T122649Z`，包含PostgreSQL custom dump、原运行目录、系统配置及发布包，四项SHA、`pg_restore --list`和两份tar可读性通过。95项migration齐全且无待执行项，没有执行迁移；最终`.deployed-commit=41962e7a6e6cfd34b4313c8bf52d9345d37d8e10`，旧运行目录为`/opt/cmpp-platform.previous-import-review-20260904T122649Z`。 - 使用自行生成的标准Excel Drawing工作簿`签名导入全流程测试-含图片.xlsx`走真实HTTP API：解析得到6列、1张图片，导入批次`cmtmxvjuh000xluleh38c4sjy`审核结果`approved / approvedCount=1 / failedCount=0`；生成签名`cmtmxvjyh0014lule5o2t16m9`为`approved`，四项系统身份字段、图片`FileObject`及6个通道图片材料均在PostgreSQL存在。报备批次`cmtmxvk8c0043lules3upx5f2`完成并生成6个真实通道文件，下载ZIP中的首个XLSX包含`xl/media`图片。批次生成后签名`pendingReport=false`，没有发送、补发、重投或重新入队短信。 - API验收使用本轮临时平台管理员，登录、近期认证、导入、审核、生成和下载均经过真实HTTP会话；结束时调用登出并删除临时账号，查询剩余数为0。浏览器已读取测试环境登录页、验证码控件和页面标题，控制台无warning/error；登录后页面交互需按浏览器安全规则由用户确认验证码后继续，不将当前登录页检查冒充已完成登录后验收。 + +## 2026-09-04 WPS报备兼容与导入审核修复（预生产发布完成） + +- 用户明确授权发布预生产。发布前重新核验本地与`origin/main`均为`aaf96db2d018cfcea79b6cdbf553cee9cb982fa2`，预生产原标记为`dada0d978bb05d7046469c1ec77371ddb2fb03bc`；累计范围为9个提交、44个文件，包含运营看板、报备字段配置、WPS单元格图片导入导出、100MiB上传限制、字段卡片固定高度及导入审核系统身份字段补齐。没有新增migration，既有未跟踪补救方案文档未进入发布归档。 +- 发布前数据盘UUID`ef4ee3bb-a19b-4aeb-b00c-aa2b995611c2`、PostgreSQL/Redis/MinIO绑定挂载、存储保护脚本、systemd drop-in和固定备份入口全部通过；95项migration齐全，相关服务健康，三条Redis Stream均`pending=0 / lag=0`，供应商连接为`desired=9 / connected=9`。 +- 首次恢复点`/opt/cmpp-platform-backups/preprod-wps-import-20260904T211224Z-before-aaf96db`因`pg_dump`不接受Prisma连接串的`schema`查询参数而在数据库导出阶段安全停止；当时尚未构建、迁移、切换、重启或修改Nginx。改用去除查询参数的数据库连接后重新建立有效恢复点`/opt/cmpp-platform-backups/preprod-wps-import-20260904T211255Z-before-aaf96db`，约613MiB，包含PostgreSQL custom dump、Redis RDB、原运行目录、环境/systemd/Nginx/Fail2ban/nftables及数据盘保护配置、原标记和发布基线。 +- 有效恢复点的8项最终SHA-256全部通过，`pg_restore --list`包含825项，运行目录tar包含52641项、系统配置tar包含370项；精确Git归档大小2768402字节，SHA-256为`d984c79d0deef1a83ec418b4806875eca210343bb76e03f174887614ba6517c1`，服务器回读一致。候选版本通过依赖安全、部署契约、Prisma生成与迁移状态、前端/API/Gateway/Security Agent生产构建以及`cmpp-api`用户运行文件可读/可执行门禁。 +- 最终`.deployed-commit=aaf96db2d018cfcea79b6cdbf553cee9cb982fa2`，上一运行目录保留为`/opt/cmpp-platform.previous-preprod-dada0d9-20260904T211255Z`。95项migration仍齐全且无待执行项，本轮未执行数据库迁移。Nginx仅将`sms.lisglo.com`和兼容端口12026的两个`client_max_body_size 50m`精确调整为`110m`，API专用域名保持`10m`；`nginx -t`通过后reload，没有修改数据盘、fstab、UUID、存储保护脚本或systemd存储drop-in。 +- API、Send Worker、Submit Outbox、Gateway Callback、Protocol Log Worker、Gateway、Security Agent、Nginx、PostgreSQL、Redis、MinIO、Node Exporter和Prometheus均为`active`，相关应用服务`NRestarts=0 / Result=success`；API、Callback、Gateway、PostgreSQL和Redis健康通过。`12026`、`sms.lisglo.com/api/health`和`api.lisglo.com/api/health`均为HTTP 200，主JS`index-CFz3CyND.js`及CSS`index-C3z6fQED.css`的公网下载SHA-256与服务器产物逐项一致。 +- 发布后三条Stream的`last-delivered-id / entries-read`与发布前一致：commands为`1788517020869-2 / 84015`，results为`1788527156466-0 / 45876`，protocol logs为`1788527156467-0 / 32227`；全部保持`pending=0 / lag=0`。本轮未发送、补发、重投、重新入队或手工ACK短信，也没有修改余额、通道、客户或签名业务资料。 +- 一条富泷供应商通道在重启后首次鉴权失败，连接短暂为`8/9`，系统按既有计划于21:20:25自动重试并恢复`9/9`，未修改通道配置或凭据。真实下游客户端受旧心跳租约影响的重复连接被连接数门禁拒绝，并记录3次既有`close of closed channel`连接协程panic；Gateway进程未退出、下游心跳持续、systemd error级journal为0，该已知重启恢复缺陷不由本次报备功能引入，继续保留专项治理。 +- 浏览器控制在读取预生产运营登录页时连续两次超时，因此本轮没有取得可复核的DOM、截图或控制台证据，也未输入账号、密码或验证码；只把真实HTTP、资源哈希和服务端证据记为通过，不将其冒充登录后浏览器验收。 + +## 2026-09-04 大文件WPS报备异步解析与真实进度（本地验证完成） + +- 40MB以上WPS文件同步解析会让单个HTTP请求同时承担上传、两轮ZIP/图片展开和ExcelJS解析，预生产证据已出现约126秒后由入口断开的499；单纯展示前端动画进度不能延长代理读超时。本轮将接口改为文件入MinIO并创建PostgreSQL任务后返回202，由独立`cmpp-report-material-worker`单并发解析，页面轮询租户隔离的真实任务状态。 +- 新增`queued/analyzing/analyzed/failed`状态、0–100进度、阶段、错误、开始和心跳时间；Worker启动时将心跳超过5分钟的解析任务恢复为排队状态，领取时使用状态条件更新防止重复占用。该变更新增1项Prisma migration和1个systemd服务，发布时必须同时迁移数据库、安装并验证Worker，不涉及短信队列、通道、余额或客户配置。 +- 报备导入使用独立100MiB上限，普通文件仍为10MiB、图片仍为2MiB。文件类型、扩展名和ZIP签名在入库前校验；工作簿公式、外部对象、解压体积、图片格式及图片数量关系在后台解析或提交阶段明确失败，不静默吞错。 +- 分析阶段读取WPS`cellimages.xml`关系、单元格位置、扩展名和ZIP声明大小，不展开图片Buffer，并在交给ExcelJS前从临时解析副本剥离WPS媒体；提交导入时仍重新下载原文件并完整读取、签名校验和上传图片，没有放宽DISPIMG白名单或20MiB单图/300MiB图片总量限制。 +- 真实样本`行业报备.xlsx`仍得到工作表“行业”、17行、13列和43张图片。相同进程环境下原完整图片模式约5522ms/RSS 259MiB，新分析模式约1899ms/RSS 227MiB，耗时下降约66%，结果计数一致；样本只读，未覆盖或写回。 +- 定向API 3套25项、API全量54套616项、前端全量13文件64项通过；前后端TypeScript、API/Vite生产构建、依赖与安全门禁、部署契约、结构质量、入口包体积和`git diff --check`通过。测试环境部署及真实HTTP/MinIO/PostgreSQL/浏览器验收将在本提交推送后执行，预生产不在本轮范围内。 diff --git a/src/api/admin/channels-reports.api.ts b/src/api/admin/channels-reports.api.ts index b7b4ce0..7a68d11 100644 --- a/src/api/admin/channels-reports.api.ts +++ b/src/api/admin/channels-reports.api.ts @@ -179,6 +179,10 @@ export const adminChannelsReportsApi = { return requestForm< Record & { id: string; + status: 'queued' | 'analyzing' | 'analyzed' | 'failed'; + progress: number; + progressStage?: string; + errorMessage?: string; columns: Array<{ sourceColumnIndex: number; columnLetter: string; @@ -191,6 +195,25 @@ export const adminChannelsReportsApi = { } >('/admin/report-materials/imports/analyze', form); }, + getReportMaterialImportAnalysisStatus: (id: string, tenantId: string, signal?: AbortSignal) => + request< + Record & { + id: string; + status: 'queued' | 'analyzing' | 'analyzed' | 'failed'; + progress: number; + progressStage?: string; + errorMessage?: string; + columns: Array<{ + sourceColumnIndex: number; + columnLetter: string; + sourceHeader: string; + sourceHeaderPath: string; + imageCount: number; + }>; + rows: Array>; + suggestedMappings: ReportImportMapping[]; + } + >(withQuery(`/admin/report-materials/imports/${id}/analysis-status`, { tenantId }), { signal }), commitReportMaterialImport: ( id: string, body: { mappings: ReportImportMapping[]; profile?: Omit & { id?: string } }, diff --git a/src/apps/admin/ReportMaterialImportModal.test.tsx b/src/apps/admin/ReportMaterialImportModal.test.tsx index ea22d8e..2d38cb2 100644 --- a/src/apps/admin/ReportMaterialImportModal.test.tsx +++ b/src/apps/admin/ReportMaterialImportModal.test.tsx @@ -6,6 +6,7 @@ import { ReportMaterialImportModal } from './ReportMaterialImportModal'; const { adminApi } = vi.hoisted(() => ({ adminApi: { analyzeReportMaterialImport: vi.fn(), + getReportMaterialImportAnalysisStatus: vi.fn(), listDrainageFields: vi.fn(), listEnterpriseApplicationOptions: vi.fn(), listReportImportProfiles: vi.fn(), @@ -27,6 +28,15 @@ describe('ReportMaterialImportModal mapping profile action', () => { adminApi.listReportImportProfiles.mockResolvedValue([]); adminApi.analyzeReportMaterialImport.mockResolvedValue({ id: 'analysis-1', + status: 'queued', + progress: 10, + progressStage: '文件已上传，等待解析', + }); + adminApi.getReportMaterialImportAnalysisStatus.mockResolvedValue({ + id: 'analysis-1', + status: 'analyzed', + progress: 100, + progressStage: '解析完成', columns: [ { sourceColumnIndex: 0, columnLetter: 'A', sourceHeader: '签名', sourceHeaderPath: '签名', imageCount: 0 }, ], @@ -59,5 +69,5 @@ describe('ReportMaterialImportModal mapping profile action', () => { expect(screen.getByRole('button', { name: '本次将保存/更新映射方案' })).toHaveAttribute('aria-pressed', 'true'), ); expect(screen.getByLabelText('映射方案名称')).toBeVisible(); - }); + }, 10_000); }); diff --git a/src/apps/admin/ReportMaterialImportModal.tsx b/src/apps/admin/ReportMaterialImportModal.tsx index 999cd3a..82d5f4f 100644 --- a/src/apps/admin/ReportMaterialImportModal.tsx +++ b/src/apps/admin/ReportMaterialImportModal.tsx @@ -25,6 +25,7 @@ type AnalyzeResult = { rows: Array<{ rowNumber: number; values: Record; imageColumns: number[] }>; suggestedMappings: ReportImportMapping[]; }; +type AnalysisProgress = { id: string; status: string; progress: number; progressStage?: string; errorMessage?: string }; const transforms = [ { label: '保持原值', value: '' }, @@ -60,6 +61,7 @@ export function ReportMaterialImportModal({ onClose, onCompleted }: { onClose: ( const [headerRowCount, setHeaderRowCount] = useState(1); const [dataStartRow, setDataStartRow] = useState(2); const [analysis, setAnalysis] = useState(); + const [analysisJob, setAnalysisJob] = useState(); const [mappings, setMappings] = useState([]); const [profileName, setProfileName] = useState(''); const [saveProfile, setSaveProfile] = useState(false); @@ -87,10 +89,52 @@ export function ReportMaterialImportModal({ onClose, onCompleted }: { onClose: ( .catch(() => setProfiles([])); }, [reportType]); + useEffect(() => { + if (!analysisJob?.id || !tenantId || analysisJob.status === 'analyzed' || analysisJob.status === 'failed') return; + const controller = new AbortController(); + let timer: ReturnType | undefined; + const poll = async () => { + try { + const result = await adminApi.getReportMaterialImportAnalysisStatus( + analysisJob.id, + tenantId, + controller.signal, + ); + if (controller.signal.aborted) return; + setError(''); + setAnalysisJob(result); + if (result.status === 'analyzed') { + const completed = result as AnalyzeResult & AnalysisProgress; + setAnalysis(completed); + setMappings(completed.suggestedMappings ?? []); + const selectedProfile = profiles.find((item) => item.id === profileId); + if (selectedProfile) setProfileName(selectedProfile.name); + return; + } + if (result.status === 'failed') { + setError(result.errorMessage || '文件解析失败'); + return; + } + timer = setTimeout(() => void poll(), 1200); + } catch (failure) { + if (!controller.signal.aborted) { + setError(failure instanceof Error ? `${failure.message}，正在重试` : '解析进度查询失败，正在重试'); + timer = setTimeout(() => void poll(), 2000); + } + } + }; + void poll(); + return () => { + controller.abort(); + if (timer) clearTimeout(timer); + }; + }, [analysisJob?.id, analysisJob?.status, profileId, profiles, tenantId]); + function changeReportType(next: ReportType) { setReportType(next); setProfileId(''); setAnalysis(undefined); + setAnalysisJob(undefined); setMappings([]); } @@ -108,18 +152,15 @@ export function ReportMaterialImportModal({ onClose, onCompleted }: { onClose: ( setBusy(true); setError(''); try { - const result = (await adminApi.analyzeReportMaterialImport(file, { + const result = await adminApi.analyzeReportMaterialImport(file, { tenantId, applicationId, reportType, headerRowCount, dataStartRow, profileId: profileId || undefined, - })) as AnalyzeResult; - setAnalysis(result); - setMappings(result.suggestedMappings ?? []); - const selectedProfile = profiles.find((item) => item.id === profileId); - if (selectedProfile) setProfileName(selectedProfile.name); + }); + setAnalysisJob(result); } catch (failure) { setError(failure instanceof Error ? failure.message : '文件解析失败'); } finally { @@ -216,8 +257,21 @@ export function ReportMaterialImportModal({ onClose, onCompleted }: { onClose: ( {busy ? '提交中...' : '提交导入审核'} ) : ( - )} @@ -306,10 +360,33 @@ export function ReportMaterialImportModal({ onClose, onCompleted }: { onClose: ( onChange={(event) => { setFile(event.target.files?.[0]); setAnalysis(undefined); + setAnalysisJob(undefined); + setError(''); }} type="file" /> + {analysisJob && analysisJob.status !== 'analyzed' ? ( +
+
+ {analysisJob.progressStage || '正在处理'} + {Math.max(0, Math.min(100, analysisJob.progress))}% +
+
+ +
+ 文件上传完成后可由后台持续解析，页面通过真实任务状态更新进度。 +
+ ) : null} {analysis ? (
diff --git a/src/styles/global.css b/src/styles/global.css index a0bf2bb..412eeb7 100644 --- a/src/styles/global.css +++ b/src/styles/global.css @@ -6099,6 +6099,9 @@ .report-import-file span { display: flex; align-items: center; gap: 9px; } .report-import-file input { max-width: 310px; } +.report-import-analysis-progress { display: grid; gap: 8px; padding: 12px; border: 1px solid var(--border); border-radius: 10px; background: var(--surface-muted); } +.report-import-analysis-progress > div:first-child { display: flex; justify-content: space-between; gap: 12px; } +.report-import-analysis-progress small { color: var(--text-muted); } .report-import-mapping { display: grid; gap: 12px; } .report-import-mapping-table { overflow-x: auto; border: 1px solid var(--border); border-radius: 10px; } .report-import-mapping-head, diff --git a/tools/deploy/production-bootstrap.sh b/tools/deploy/production-bootstrap.sh index 28cdb74..b3dad41 100644 --- a/tools/deploy/production-bootstrap.sh +++ b/tools/deploy/production-bootstrap.sh @@ -206,7 +206,7 @@ SQL write_env() { log "Writing production environment" - mkdir -p /etc/cmpp-platform "$APP_DIR" "$APP_DIR/logs/api" "$APP_DIR/logs/send-worker" "$APP_DIR/logs/gateway" "$APP_DIR/backups" "$OBJECT_STORAGE_LOCAL_ROOT" + mkdir -p /etc/cmpp-platform "$APP_DIR" "$APP_DIR/logs/api" "$APP_DIR/logs/send-worker" "$APP_DIR/logs/report-material-worker" "$APP_DIR/logs/gateway" "$APP_DIR/backups" "$OBJECT_STORAGE_LOCAL_ROOT" cat >/etc/cmpp-platform/cmpp-platform.env </etc/systemd/system/cmpp-report-material-worker.service </etc/systemd/system/cmpp-report-material-worker.service </dev/null || true - systemctl enable --now cmpp-api cmpp-send-worker cmpp-gateway nginx + systemctl enable --now cmpp-api cmpp-send-worker cmpp-report-material-worker cmpp-gateway nginx else systemctl enable --now cmpp-minio systemctl restart cmpp-minio - systemctl enable --now cmpp-api cmpp-send-worker cmpp-gateway nginx + systemctl enable --now cmpp-api cmpp-send-worker cmpp-report-material-worker cmpp-gateway nginx fi systemctl restart cmpp-security-agent if [[ "${SEND_SUBMIT_OUTBOX_SEPARATE_PROCESS_ENABLED:-false}" == "true" ]]; then @@ -273,6 +293,7 @@ fi systemctl restart cmpp-gateway systemctl restart cmpp-api systemctl restart cmpp-send-worker +systemctl restart cmpp-report-material-worker systemctl restart nginx echo "[deploy] Health checks" @@ -292,6 +313,7 @@ wait_for_http() { } wait_for_http "API" "http://127.0.0.1:${API_PORT:-3000}/api/health" wait_for_http "Send worker metrics" "http://127.0.0.1:${API_WORKER_METRICS_PORT:-9465}/metrics" +systemctl is-active --quiet cmpp-report-material-worker if [[ "${SEND_SUBMIT_OUTBOX_SEPARATE_PROCESS_ENABLED:-false}" == "true" ]]; then wait_for_http "Submit Outbox metrics" "http://127.0.0.1:${API_OUTBOX_METRICS_PORT:-9467}/metrics" fi