feat: 异步解析大文件WPS报备资料

This commit is contained in:
hectorzhao
2026-09-04 22:51:07 +08:00
parent aaf96db2d0
commit 5925cf493b
22 changed files with 720 additions and 80 deletions
+1
View File
@@ -11,6 +11,7 @@
"test:watch": "jest --watch", "test:watch": "jest --watch",
"start": "node dist/main.js", "start": "node dist/main.js",
"start:dev": "ts-node src/main.ts", "start:dev": "ts-node src/main.ts",
"start:report-material-worker": "node dist/report-material-analysis-worker.js",
"prisma:generate": "prisma generate", "prisma:generate": "prisma generate",
"prisma:migrate:dev": "prisma migrate dev", "prisma:migrate:dev": "prisma migrate dev",
"prisma:migrate:deploy": "prisma migrate deploy" "prisma:migrate:deploy": "prisma migrate deploy"
@@ -0,0 +1,9 @@
ALTER TABLE "ReportMaterialImportBatch"
ADD COLUMN "progress" INTEGER NOT NULL DEFAULT 100,
ADD COLUMN "progressStage" TEXT,
ADD COLUMN "errorMessage" TEXT,
ADD COLUMN "startedAt" TIMESTAMP(3),
ADD COLUMN "heartbeatAt" TIMESTAMP(3);
CREATE INDEX "ReportMaterialImportBatch_status_heartbeatAt_idx"
ON "ReportMaterialImportBatch"("status", "heartbeatAt");
+6
View File
@@ -1395,6 +1395,11 @@ model ReportMaterialImportBatch {
fileName String fileName String
reportType String reportType String
status String @default("analyzed") status String @default("analyzed")
progress Int @default(100)
progressStage String?
errorMessage String?
startedAt DateTime?
heartbeatAt DateTime?
sheetName String sheetName String
headerRowCount Int @default(1) headerRowCount Int @default(1)
dataStartRow Int @default(2) dataStartRow Int @default(2)
@@ -1413,6 +1418,7 @@ model ReportMaterialImportBatch {
@@index([tenantId, createdAt]) @@index([tenantId, createdAt])
@@index([status, createdAt]) @@index([status, createdAt])
@@index([status, heartbeatAt])
} }
model ReportMaterialImportItem { model ReportMaterialImportItem {
+18
View File
@@ -0,0 +1,18 @@
import { assertUploadSize } from './files.service';
describe('FilesService report-material upload limits', () => {
const workbook = {
originalname: '行业报备.xlsx',
mimetype: 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
};
it('allows report-material imports up to 100MB without raising the generic file limit', () => {
expect(() =>
assertUploadSize('report_material_import', { ...workbook, size: 40 * 1024 * 1024 }),
).not.toThrow();
expect(() => assertUploadSize('other', { ...workbook, size: 40 * 1024 * 1024 })).toThrow('10MB');
expect(() =>
assertUploadSize('report_material_import', { ...workbook, size: 101 * 1024 * 1024 }),
).toThrow('100MB');
});
});
+29 -8
View File
@@ -40,10 +40,12 @@ export class FilesService {
) {} ) {}
list(tenantId?: string) { list(tenantId?: string) {
return this.prisma.fileObject.findMany({ return this.prisma.fileObject
.findMany({
where: tenantId ? { tenantId } : undefined, where: tenantId ? { tenantId } : undefined,
orderBy: { createdAt: 'desc' }, orderBy: { createdAt: 'desc' },
}).then((items) => items.map(serializeFileObject)); })
.then((items) => items.map(serializeFileObject));
} }
create(data: CreateFileObjectDto) { create(data: CreateFileObjectDto) {
@@ -71,7 +73,7 @@ export class FilesService {
} }
async upload(data: UploadFileDto, file: { originalname: string; mimetype: string; size: number; buffer: Buffer }) { async upload(data: UploadFileDto, file: { originalname: string; mimetype: string; size: number; buffer: Buffer }) {
assertUploadSize(file); assertUploadSize(data.purpose, file);
const fileName = normalizeMultipartFileName(file.originalname); const fileName = normalizeMultipartFileName(file.originalname);
const safeName = fileName.replace(/[^\w.\-\u4e00-\u9fa5]/g, '_'); const safeName = fileName.replace(/[^\w.\-\u4e00-\u9fa5]/g, '_');
const objectKey = `${data.prefix ?? data.purpose}/${Date.now()}-${randomUUID()}-${safeName}`; const objectKey = `${data.prefix ?? data.purpose}/${Date.now()}-${randomUUID()}-${safeName}`;
@@ -87,7 +89,11 @@ export class FilesService {
}); });
} }
async uploadForClient(userId: string | undefined, data: UploadFileDto, file: { originalname: string; mimetype: string; size: number; buffer: Buffer }) { async uploadForClient(
userId: string | undefined,
data: UploadFileDto,
file: { originalname: string; mimetype: string; size: number; buffer: Buffer },
) {
const tenantId = await this.resolveClientTenantId(userId); const tenantId = await this.resolveClientTenantId(userId);
const prefixRule = CLIENT_UPLOAD_RULES[data.purpose]; const prefixRule = CLIENT_UPLOAD_RULES[data.purpose];
if (!prefixRule || !data.prefix || !prefixRule.test(data.prefix)) { if (!prefixRule || !data.prefix || !prefixRule.test(data.prefix)) {
@@ -135,18 +141,33 @@ export class FilesService {
const IMAGE_UPLOAD_MAX_BYTES = 2 * 1024 * 1024; const IMAGE_UPLOAD_MAX_BYTES = 2 * 1024 * 1024;
const FILE_UPLOAD_MAX_BYTES = 10 * 1024 * 1024; const FILE_UPLOAD_MAX_BYTES = 10 * 1024 * 1024;
const REPORT_MATERIAL_IMPORT_MAX_BYTES = 100 * 1024 * 1024;
const IMAGE_FILE_EXTENSION = /\.(?:avif|bmp|gif|heic|heif|jpe?g|png|svg|webp)$/i; const IMAGE_FILE_EXTENSION = /\.(?:avif|bmp|gif|heic|heif|jpe?g|png|svg|webp)$/i;
function assertUploadSize(file: { originalname: string; mimetype: string; size: number }) { export function assertUploadSize(purpose: string, file: { originalname: string; mimetype: string; size: number }) {
const image = file.mimetype.toLowerCase().startsWith('image/') || IMAGE_FILE_EXTENSION.test(file.originalname); const image = file.mimetype.toLowerCase().startsWith('image/') || IMAGE_FILE_EXTENSION.test(file.originalname);
const limit = image ? IMAGE_UPLOAD_MAX_BYTES : FILE_UPLOAD_MAX_BYTES; const reportMaterialImport = purpose === 'report_material_import';
const limit = reportMaterialImport
? REPORT_MATERIAL_IMPORT_MAX_BYTES
: image
? IMAGE_UPLOAD_MAX_BYTES
: FILE_UPLOAD_MAX_BYTES;
if (file.size > limit) { if (file.size > limit) {
throw new BadRequestException(image ? '图片大小不能超过 2MB' : '文件大小不能超过 10MB'); throw new BadRequestException(
reportMaterialImport
? '报备资料文件大小不能超过 100MB'
: image
? '图片大小不能超过 2MB'
: '文件大小不能超过 10MB',
);
} }
} }
function normalizeMultipartFileName(value: string) { function normalizeMultipartFileName(value: string) {
if (![...value].some((character) => character.charCodeAt(0) > 0x7f) || [...value].some((character) => character.charCodeAt(0) > 0xff)) { if (
![...value].some((character) => character.charCodeAt(0) > 0x7f) ||
[...value].some((character) => character.charCodeAt(0) > 0xff)
) {
return value; return value;
} }
@@ -0,0 +1,13 @@
import { Module } from '@nestjs/common';
import { ConfigModule } from '@nestjs/config';
import { PrismaModule } from './prisma/prisma.module';
import { ReportMaterialsModule } from './report-materials/report-materials.module';
@Module({
imports: [
ConfigModule.forRoot({ isGlobal: true, envFilePath: ['.env.local', '.env'] }),
PrismaModule,
ReportMaterialsModule,
],
})
export class ReportMaterialAnalysisWorkerModule {}
@@ -0,0 +1,31 @@
import 'reflect-metadata';
import { NestFactory } from '@nestjs/core';
import { ReportMaterialAnalysisWorkerModule } from './report-material-analysis-worker.module';
import { ReportMaterialsService } from './report-materials/report-materials.service';
const POLL_INTERVAL_MS = 1000;
async function bootstrap() {
if ((process.env.CMPP_PROCESS_ROLE?.trim() || 'report-material-worker') !== 'report-material-worker') {
throw new Error('report-material-analysis-worker requires CMPP_PROCESS_ROLE=report-material-worker');
}
const app = await NestFactory.createApplicationContext(ReportMaterialAnalysisWorkerModule, {
logger: ['log', 'warn', 'error'],
});
app.enableShutdownHooks();
const reports = app.get(ReportMaterialsService);
await reports.recoverStaleAnalysisJobs();
let stopping = false;
const stop = () => {
stopping = true;
};
process.once('SIGTERM', stop);
process.once('SIGINT', stop);
while (!stopping) {
const worked = await reports.processNextAnalysisJob();
if (!worked) await new Promise((resolve) => setTimeout(resolve, POLL_INTERVAL_MS));
}
await app.close();
}
void bootstrap();
+209 -38
View File
@@ -1,4 +1,4 @@
import { BadRequestException } from '@nestjs/common'; import { BadRequestException, NotFoundException } from '@nestjs/common';
import { Prisma } from '@prisma/client'; import { Prisma } from '@prisma/client';
import { extname } from 'node:path'; import { extname } from 'node:path';
import { FilesService } from '../files/files.service'; import { FilesService } from '../files/files.service';
@@ -13,7 +13,7 @@ import {
clamp, clamp,
cellText, cellText,
} from './report-materials.helpers'; } from './report-materials.helpers';
import { compatibleImages, loadCompatibleWorkbook } from './workbook-compatibility'; import { compatibleImagePositions, loadCompatibleWorkbook } from './workbook-compatibility';
/** R4 report-materials domain service composed behind ReportMaterialsService. */ /** R4 report-materials domain service composed behind ReportMaterialsService. */
export class ReportImportParserService { export class ReportImportParserService {
@@ -63,14 +63,123 @@ export class ReportImportParserService {
file: { originalname: string; mimetype: string; size: number; buffer: Buffer }, file: { originalname: string; mimetype: string; size: number; buffer: Buffer },
options: AnalyzeImportOptions, options: AnalyzeImportOptions,
) { ) {
if (!options.tenantId) throw new BadRequestException('tenantId is required'); await this.validateImport(file, options);
if (!options.applicationId) throw new BadRequestException('applicationId is required'); const analysis = await this.parseAnalysis(file.buffer, options);
await this.smsConfig.getApplication(options.applicationId, options.tenantId); const sourceFile = await this.files.upload(
if (!['signature', 'drainage'].includes(options.reportType)) { tenantId: options.tenantId, purpose: 'report_material_import', prefix: 'report-material-imports' },
throw new BadRequestException('reportType must be signature or drainage'); file,
if (extname(file.originalname).toLowerCase() !== '.xlsx' || file.buffer[0] !== 0x50 || file.buffer[1] !== 0x4b) );
throw new BadRequestException('仅支持有效的 XLSX 文件'); const batch = await this.createAnalyzedBatch(sourceFile, options, analysis);
const { workbook, wpsImagesBySheet } = await loadCompatibleWorkbook(file.buffer); await this.logAnalyzed(batch.id, sourceFile.fileName, options, analysis.sheetName, analysis.rows.length);
return { ...batch, sourceFile, ...analysis };
}
async enqueueImport(
file: { originalname: string; mimetype: string; size: number; buffer: Buffer },
options: AnalyzeImportOptions,
) {
await this.validateImport(file, options);
const sourceFile = await this.files.upload(
{ tenantId: options.tenantId, purpose: 'report_material_import', prefix: 'report-material-imports' },
file,
);
const batch = await this.prisma.reportMaterialImportBatch.create({
data: {
tenantId: options.tenantId,
applicationId: options.applicationId,
profileId: options.profileId,
fileObjectId: sourceFile.id,
fileName: sourceFile.fileName,
reportType: options.reportType,
status: 'queued',
progress: 10,
progressStage: '文件已上传,等待解析',
sheetName: options.sheetName ?? '',
headerRowCount: clamp(options.headerRowCount, 1, 5),
dataStartRow: Math.max(options.dataStartRow, clamp(options.headerRowCount, 1, 5) + 1),
mapping: [] as Prisma.InputJsonValue,
result: { operatorId: options.operatorId } as Prisma.InputJsonValue,
},
});
return this.analysisStatus(batch);
}
async getAnalysisStatus(batchId: string, tenantId: string) {
if (!tenantId) throw new BadRequestException('tenantId is required');
const batch = await this.prisma.reportMaterialImportBatch.findFirst({ where: { id: batchId, tenantId } });
if (!batch) throw new NotFoundException('导入解析任务不存在');
return this.analysisStatus(batch);
}
async processQueuedBatch(batchId: string) {
const batch = await this.prisma.reportMaterialImportBatch.findUnique({ where: { id: batchId } });
if (!batch || batch.status !== 'analyzing') return false;
try {
await this.progress(batchId, 25, '正在读取工作簿');
const { content } = await this.files.getDownload(batch.fileObjectId);
const resultData = (batch.result && typeof batch.result === 'object' ? batch.result : {}) as {
operatorId?: string;
};
const options: AnalyzeImportOptions = {
tenantId: batch.tenantId,
applicationId: batch.applicationId ?? '',
reportType: batch.reportType as 'signature' | 'drainage',
sheetName: batch.sheetName || undefined,
headerRowCount: batch.headerRowCount,
dataStartRow: batch.dataStartRow,
profileId: batch.profileId ?? undefined,
operatorId: resultData.operatorId,
};
await this.progress(batchId, 30, '正在解析表格与图片位置');
const analysis = await this.parseAnalysis(content, options, async (progress, stage) => {
await this.progress(batchId, progress, stage);
});
const updated = await this.prisma.reportMaterialImportBatch.update({
where: { id: batchId },
data: {
status: 'analyzed',
progress: 100,
progressStage: '解析完成',
errorMessage: null,
heartbeatAt: new Date(),
sheetName: analysis.sheetName,
mapping: analysis.suggestedMappings as Prisma.InputJsonValue,
preview: {
sheets: analysis.sheets,
columns: analysis.columns,
rows: analysis.rows,
imageCount: analysis.imageCount,
} as Prisma.InputJsonValue,
result: Prisma.JsonNull,
rowCount: analysis.rowCount,
},
});
await this.logAnalyzed(batchId, batch.fileName, options, analysis.sheetName, analysis.rows.length);
return this.analysisStatus(updated);
} catch (error) {
const message = error instanceof Error ? error.message : '文件解析失败';
await this.prisma.reportMaterialImportBatch.update({
where: { id: batchId },
data: {
status: 'failed',
progressStage: '解析失败',
errorMessage: message.slice(0, 1000),
heartbeatAt: new Date(),
},
});
return false;
}
}
private async parseAnalysis(
buffer: Buffer,
options: AnalyzeImportOptions,
onProgress?: (progress: number, stage: string) => void | Promise<void>,
) {
const { workbook, wpsImagesBySheet } = await loadCompatibleWorkbook(buffer, {
includeImageData: false,
onProgress,
});
const profile = options.profileId const profile = options.profileId
? await this.prisma.reportMaterialImportProfile.findUnique({ ? await this.prisma.reportMaterialImportProfile.findUnique({
where: { id: options.profileId }, where: { id: options.profileId },
@@ -82,7 +191,7 @@ export class ReportImportParserService {
if (!worksheet) throw new BadRequestException('工作簿没有可读取的工作表'); if (!worksheet) throw new BadRequestException('工作簿没有可读取的工作表');
const headerRowCount = clamp(options.headerRowCount, 1, 5); const headerRowCount = clamp(options.headerRowCount, 1, 5);
const dataStartRow = Math.max(options.dataStartRow, headerRowCount + 1); const dataStartRow = Math.max(options.dataStartRow, headerRowCount + 1);
const images = compatibleImages(workbook, worksheet, wpsImagesBySheet); const images = compatibleImagePositions(workbook, worksheet, wpsImagesBySheet);
const columnCount = Math.min(worksheet.columnCount, 200); const columnCount = Math.min(worksheet.columnCount, 200);
const columns = Array.from({ length: columnCount }, (_, offset) => { const columns = Array.from({ length: columnCount }, (_, offset) => {
const sourceColumnIndex = offset + 1; const sourceColumnIndex = offset + 1;
@@ -110,10 +219,6 @@ export class ReportImportParserService {
if (Object.values(values).some(Boolean) || imageColumns.length) if (Object.values(values).some(Boolean) || imageColumns.length)
previewRows.push({ rowNumber, values, imageColumns }); previewRows.push({ rowNumber, values, imageColumns });
} }
const sourceFile = await this.files.upload(
{ tenantId: options.tenantId, purpose: 'report_material_import', prefix: 'report-material-imports' },
file,
);
const profileMappings = profile?.columns.map((column) => ({ const profileMappings = profile?.columns.map((column) => ({
sourceHeader: column.sourceHeader, sourceHeader: column.sourceHeader,
sourceHeaderPath: column.sourceHeaderPath ?? undefined, sourceHeaderPath: column.sourceHeaderPath ?? undefined,
@@ -128,7 +233,24 @@ export class ReportImportParserService {
const suggestedMappings = profileMappings?.length const suggestedMappings = profileMappings?.length
? remapProfileColumns(profileMappings, columns) ? remapProfileColumns(profileMappings, columns)
: suggestMappings(columns, options.reportType); : suggestMappings(columns, options.reportType);
const batch = await this.prisma.reportMaterialImportBatch.create({ await onProgress?.(95, '已生成字段映射与预览');
return {
sheetName: worksheet.name,
sheets: workbook.worksheets.map((sheet) => sheet.name),
columns,
rows: previewRows,
imageCount: images.length,
suggestedMappings,
rowCount: Math.max(0, worksheet.rowCount - dataStartRow + 1),
};
}
private createAnalyzedBatch(
sourceFile: { id: string; fileName: string },
options: AnalyzeImportOptions,
analysis: Awaited<ReturnType<ReportImportParserService['parseAnalysis']>>,
) {
return this.prisma.reportMaterialImportBatch.create({
data: { data: {
tenantId: options.tenantId, tenantId: options.tenantId,
applicationId: options.applicationId, applicationId: options.applicationId,
@@ -136,42 +258,91 @@ export class ReportImportParserService {
fileObjectId: sourceFile.id, fileObjectId: sourceFile.id,
fileName: sourceFile.fileName, fileName: sourceFile.fileName,
reportType: options.reportType, reportType: options.reportType,
sheetName: worksheet.name, sheetName: analysis.sheetName,
headerRowCount, headerRowCount: clamp(options.headerRowCount, 1, 5),
dataStartRow, dataStartRow: Math.max(options.dataStartRow, clamp(options.headerRowCount, 1, 5) + 1),
mapping: suggestedMappings as Prisma.InputJsonValue, mapping: analysis.suggestedMappings as Prisma.InputJsonValue,
preview: { preview: {
sheets: workbook.worksheets.map((sheet) => sheet.name), sheets: analysis.sheets,
columns, columns: analysis.columns,
rows: previewRows, rows: analysis.rows,
imageCount: images.length, imageCount: analysis.imageCount,
} as Prisma.InputJsonValue, } as Prisma.InputJsonValue,
rowCount: Math.max(0, worksheet.rowCount - dataStartRow + 1), rowCount: analysis.rowCount,
}, },
}); });
}
private async validateImport(
file: { originalname: string; size: number; buffer: Buffer },
options: AnalyzeImportOptions,
) {
if (!options.tenantId) throw new BadRequestException('tenantId is required');
if (!options.applicationId) throw new BadRequestException('applicationId is required');
await this.smsConfig.getApplication(options.applicationId, options.tenantId);
if (!['signature', 'drainage'].includes(options.reportType))
throw new BadRequestException('reportType must be signature or drainage');
if (extname(file.originalname).toLowerCase() !== '.xlsx' || file.buffer[0] !== 0x50 || file.buffer[1] !== 0x4b)
throw new BadRequestException('仅支持有效的 XLSX 文件');
}
private progress(batchId: string, progress: number, progressStage: string) {
return this.prisma.reportMaterialImportBatch.update({
where: { id: batchId },
data: { progress, progressStage, heartbeatAt: new Date() },
});
}
private analysisStatus(batch: {
id: string;
status: string;
progress: number;
progressStage: string | null;
errorMessage: string | null;
sheetName: string;
preview: Prisma.JsonValue | null;
mapping: Prisma.JsonValue;
}) {
const preview =
batch.preview && typeof batch.preview === 'object' && !Array.isArray(batch.preview)
? (batch.preview as Record<string, unknown>)
: {};
return {
id: batch.id,
status: batch.status,
progress: batch.progress,
progressStage: batch.progressStage,
errorMessage: batch.errorMessage,
sheetName: batch.sheetName || undefined,
sheets: preview.sheets ?? [],
columns: preview.columns ?? [],
rows: preview.rows ?? [],
imageCount: preview.imageCount ?? 0,
suggestedMappings: batch.mapping ?? [],
};
}
private async logAnalyzed(
batchId: string,
fileName: string,
options: AnalyzeImportOptions,
sheetName: string,
previewCount: number,
) {
await this.prisma.operationLog.create({ await this.prisma.operationLog.create({
data: { data: {
tenantId: options.tenantId, tenantId: options.tenantId,
userId: options.operatorId, userId: options.operatorId,
action: 'report_material.import_analyzed', action: 'report_material.import_analyzed',
resource: 'report_material_import', resource: 'report_material_import',
resourceId: batch.id, resourceId: batchId,
detail: { detail: {
fileName: sourceFile.fileName, fileName,
filters: { applicationId: options.applicationId, reportType: options.reportType, sheetName: worksheet.name }, filters: { applicationId: options.applicationId, reportType: options.reportType, sheetName },
successCount: previewRows.length, successCount: previewCount,
failedCount: 0, failedCount: 0,
} as Prisma.InputJsonValue, } as Prisma.InputJsonValue,
}, },
}); });
return {
...batch,
sourceFile,
sheets: workbook.worksheets.map((sheet) => sheet.name),
columns,
rows: previewRows,
imageCount: images.length,
suggestedMappings,
};
} }
} }
@@ -105,5 +105,6 @@ export type AnalyzeImportOptions = {
}; };
export type EmbeddedImage = { row: number; column: number; extension: string; buffer: Buffer }; export type EmbeddedImage = { row: number; column: number; extension: string; buffer: Buffer };
export type EmbeddedImageMetadata = { row: number; column: number; extension: string; size: number; target: string };
export type ReportWorkbookFormat = 'excel_drawing' | 'wps_cell_image'; export type ReportWorkbookFormat = 'excel_drawing' | 'wps_cell_image';
@@ -3,6 +3,7 @@ import {
Body, Body,
Controller, Controller,
Get, Get,
HttpCode,
Param, Param,
Post, Post,
Put, Put,
@@ -93,6 +94,7 @@ export class ReportMaterialsController {
} }
@Post('imports/analyze') @Post('imports/analyze')
@HttpCode(202)
@UseInterceptors( @UseInterceptors(
FileInterceptor('file', { limits: { fileSize: 100 * 1024 * 1024, files: 1, fields: 12, parts: 13 } }), FileInterceptor('file', { limits: { fileSize: 100 * 1024 * 1024, files: 1, fields: 12, parts: 13 } }),
) )
@@ -102,7 +104,7 @@ export class ReportMaterialsController {
@CurrentSessionUserId() operatorId?: string, @CurrentSessionUserId() operatorId?: string,
) { ) {
if (!file) throw new BadRequestException('请选择 XLSX 文件'); if (!file) throw new BadRequestException('请选择 XLSX 文件');
return this.service.analyzeImport(file, { return this.service.enqueueImport(file, {
tenantId: body.tenantId, tenantId: body.tenantId,
applicationId: body.applicationId, applicationId: body.applicationId,
reportType: body.reportType as 'signature' | 'drainage', reportType: body.reportType as 'signature' | 'drainage',
@@ -114,6 +116,11 @@ export class ReportMaterialsController {
}); });
} }
@Get('imports/:id/analysis-status')
getImportAnalysisStatus(@Param('id') id: string, @Query('tenantId') tenantId: string) {
return this.service.getAnalysisStatus(id, tenantId);
}
@Put('imports/:id/commit') @Put('imports/:id/commit')
@RequireRecentAuthentication() @RequireRecentAuthentication()
commitImport(@Param('id') id: string, @Body() body: ImportCommitDto, @CurrentSessionUserId() operatorId?: string) { commitImport(@Param('id') id: string, @Body() body: ImportCommitDto, @CurrentSessionUserId() operatorId?: string) {
@@ -4,6 +4,75 @@ import { ReportMaterialsService } from './report-materials.service';
import { mappedCorePatchValue, signatureIdentityReportValues } from './report-materials.helpers'; import { mappedCorePatchValue, signatureIdentityReportValues } from './report-materials.helpers';
describe('ReportMaterialsService', () => { describe('ReportMaterialsService', () => {
it('stores a valid workbook as a durable queued analysis job before parsing it', async () => {
const prisma = {
reportMaterialImportBatch: {
create: jest.fn().mockImplementation(({ data }: { data: Record<string, unknown> }) =>
Promise.resolve({ id: 'job-1', preview: null, errorMessage: null, ...data }),
),
},
};
const files = { upload: jest.fn().mockResolvedValue({ id: 'file-1', fileName: '行业报备.xlsx' }) };
const service = new ReportMaterialsService(
prisma as never,
files as never,
{ getApplication: jest.fn().mockResolvedValue({ id: 'app-1' }) } as never,
);
const result = await service.enqueueImport(
{ originalname: '行业报备.xlsx', mimetype: 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', size: 2, buffer: Buffer.from('PK') },
{ tenantId: 'tenant-1', applicationId: 'app-1', reportType: 'signature', headerRowCount: 1, dataStartRow: 2 },
);
expect(files.upload).toHaveBeenCalledWith(
expect.objectContaining({ purpose: 'report_material_import', tenantId: 'tenant-1' }),
expect.any(Object),
);
expect(prisma.reportMaterialImportBatch.create).toHaveBeenCalledWith({
data: expect.objectContaining({ status: 'queued', progress: 10, applicationId: 'app-1' }),
});
expect(result).toMatchObject({ id: 'job-1', status: 'queued', progress: 10 });
});
it('only returns analysis progress inside the requested tenant', async () => {
const findFirst = jest.fn().mockResolvedValue(null);
const service = new ReportMaterialsService(
{ reportMaterialImportBatch: { findFirst } } as never,
{} as never,
{} as never,
);
await expect(service.getAnalysisStatus('job-1', 'tenant-2')).rejects.toThrow('导入解析任务不存在');
expect(findFirst).toHaveBeenCalledWith({ where: { id: 'job-1', tenantId: 'tenant-2' } });
});
it('claims only a queued analysis job before handing it to the worker parser', async () => {
const prisma = {
reportMaterialImportBatch: {
findFirst: jest.fn().mockResolvedValue({ id: 'job-1' }),
updateMany: jest.fn().mockResolvedValue({ count: 1 }),
findUnique: jest.fn().mockResolvedValue(null),
},
};
const service = new ReportMaterialsService(prisma as never, {} as never, {} as never);
await expect(service.processNextAnalysisJob()).resolves.toBe(true);
expect(prisma.reportMaterialImportBatch.updateMany).toHaveBeenCalledWith({
where: { id: 'job-1', status: 'queued' },
data: expect.objectContaining({ status: 'analyzing', progress: 15 }),
});
});
it('requeues stale analyzing jobs so a restarted worker can resume them', async () => {
const updateMany = jest.fn().mockResolvedValue({ count: 2 });
const service = new ReportMaterialsService(
{ reportMaterialImportBatch: { updateMany } } as never,
{} as never,
{} as never,
);
await expect(service.recoverStaleAnalysisJobs()).resolves.toEqual({ count: 2 });
expect(updateMany).toHaveBeenCalledWith({
where: { status: 'analyzing', OR: [{ heartbeatAt: null }, { heartbeatAt: { lt: expect.any(Date) } }] },
data: { status: 'queued', progress: 10, progressStage: '等待重新解析', errorMessage: null },
});
});
it('requires an enterprise application before parsing an import workbook', async () => { it('requires an enterprise application before parsing an import workbook', async () => {
const service = new ReportMaterialsService({} as never, { upload: jest.fn() } as never, {} as never); const service = new ReportMaterialsService({} as never, { upload: jest.fn() } as never, {} as never);
await expect( await expect(
@@ -34,7 +34,11 @@ export class ReportMaterialsService {
private readonly channelExport: ReportChannelExportService; private readonly channelExport: ReportChannelExportService;
private readonly batchGeneration: ReportBatchGenerationService; private readonly batchGeneration: ReportBatchGenerationService;
constructor(prisma: PrismaService, files: FilesService, smsConfig: SmsConfigService) { constructor(
private readonly prisma: PrismaService,
files: FilesService,
smsConfig: SmsConfigService,
) {
this.pendingQuery = new ReportPendingQueryService(prisma, files, smsConfig); this.pendingQuery = new ReportPendingQueryService(prisma, files, smsConfig);
this.officialExport = new ReportOfficialExportService(prisma, files, smsConfig, this.pendingQuery); this.officialExport = new ReportOfficialExportService(prisma, files, smsConfig, this.pendingQuery);
this.importParser = new ReportImportParserService(prisma, files, smsConfig); this.importParser = new ReportImportParserService(prisma, files, smsConfig);
@@ -82,6 +86,48 @@ export class ReportMaterialsService {
return this.importParser.analyzeImport(file, options); return this.importParser.analyzeImport(file, options);
} }
async enqueueImport(
file: { originalname: string; mimetype: string; size: number; buffer: Buffer },
options: AnalyzeImportOptions,
) {
return this.importParser.enqueueImport(file, options);
}
async getAnalysisStatus(batchId: string, tenantId: string) {
return this.importParser.getAnalysisStatus(batchId, tenantId);
}
async recoverStaleAnalysisJobs() {
const staleBefore = new Date(Date.now() - 5 * 60 * 1000);
return this.prisma.reportMaterialImportBatch.updateMany({
where: { status: 'analyzing', OR: [{ heartbeatAt: null }, { heartbeatAt: { lt: staleBefore } }] },
data: { status: 'queued', progress: 10, progressStage: '等待重新解析', errorMessage: null },
});
}
async processNextAnalysisJob() {
const candidate = await this.prisma.reportMaterialImportBatch.findFirst({
where: { status: 'queued' },
orderBy: { createdAt: 'asc' },
select: { id: true },
});
if (!candidate) return false;
const claimed = await this.prisma.reportMaterialImportBatch.updateMany({
where: { id: candidate.id, status: 'queued' },
data: {
status: 'analyzing',
progress: 15,
progressStage: '后台解析已开始',
startedAt: new Date(),
heartbeatAt: new Date(),
errorMessage: null,
},
});
if (!claimed.count) return true;
await this.importParser.processQueuedBatch(candidate.id);
return true;
}
async commitImport(batchId: string, data: ImportCommitDto) { async commitImport(batchId: string, data: ImportCommitDto) {
return this.importReview.commitImport(batchId, data); return this.importReview.commitImport(batchId, data);
} }
@@ -1,6 +1,11 @@
import ExcelJS from 'exceljs'; import ExcelJS from 'exceljs';
import JSZip from 'jszip'; import JSZip from 'jszip';
import { compatibleImages, convertWorkbookOutput, loadCompatibleWorkbook } from './workbook-compatibility'; import {
compatibleImagePositions,
compatibleImages,
convertWorkbookOutput,
loadCompatibleWorkbook,
} from './workbook-compatibility';
const PNG = Buffer.from( const PNG = Buffer.from(
'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAusB9Y9ZrYQAAAAASUVORK5CYII=', 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAusB9Y9ZrYQAAAAASUVORK5CYII=',
@@ -39,4 +44,15 @@ describe('WPS workbook compatibility', () => {
const content = Buffer.from(await workbook.xlsx.writeBuffer()); const content = Buffer.from(await workbook.xlsx.writeBuffer());
await expect(loadCompatibleWorkbook(content)).rejects.toThrow('不允许的公式'); await expect(loadCompatibleWorkbook(content)).rejects.toThrow('不允许的公式');
}); });
it('reads WPS image positions without inflating image buffers during analysis', async () => {
const converted = await convertWorkbookOutput(await standardWorkbook(), 'wps_cell_image');
const loaded = await loadCompatibleWorkbook(converted, { includeImageData: false });
const wpsImage = loaded.wpsImagesBySheet.get('签名报备')?.[0];
expect(wpsImage).toMatchObject({ row: 2, column: 1, extension: 'png', size: PNG.length });
expect(wpsImage).not.toHaveProperty('buffer');
expect(
compatibleImagePositions(loaded.workbook, loaded.workbook.getWorksheet('签名报备')!, loaded.wpsImagesBySheet),
).toEqual([{ row: 2, column: 1 }]);
});
}); });
@@ -2,7 +2,7 @@ import { BadRequestException } from '@nestjs/common';
import ExcelJS from 'exceljs'; import ExcelJS from 'exceljs';
import JSZip from 'jszip'; import JSZip from 'jszip';
import { randomUUID } from 'node:crypto'; import { randomUUID } from 'node:crypto';
import type { EmbeddedImage, ReportWorkbookFormat } from './report-materials.contracts'; import type { EmbeddedImage, EmbeddedImageMetadata, ReportWorkbookFormat } from './report-materials.contracts';
import { normalizeImageExtension, readEmbeddedImages } from './report-materials.helpers'; import { normalizeImageExtension, readEmbeddedImages } from './report-materials.helpers';
const MAX_WORKBOOK_BYTES = 100 * 1024 * 1024; const MAX_WORKBOOK_BYTES = 100 * 1024 * 1024;
@@ -72,9 +72,14 @@ function validateWorkbookValues(workbook: ExcelJS.Workbook, allowedWpsIds: Set<s
} }
} }
async function inspectWpsImages(zip: JSZip) { async function inspectWpsImages(zip: JSZip, includeImageData: boolean) {
const cellImagesXml = await text(zip, 'xl/cellimages.xml'); const cellImagesXml = await text(zip, 'xl/cellimages.xml');
if (!cellImagesXml) return { allowedIds: new Set<string>(), imagesBySheet: new Map<string, EmbeddedImage[]>() }; if (!cellImagesXml)
return {
allowedIds: new Set<string>(),
imagesBySheet: new Map<string, Array<EmbeddedImage | EmbeddedImageMetadata>>(),
mediaTargets: new Set<string>(),
};
const relXml = await text(zip, 'xl/_rels/cellimages.xml.rels'); const relXml = await text(zip, 'xl/_rels/cellimages.xml.rels');
const relTargets = new Map<string, string>(); const relTargets = new Map<string, string>();
for (const match of relXml.matchAll(/<Relationship\b([^>]*)\/?>(?:<\/Relationship>)?/g)) { for (const match of relXml.matchAll(/<Relationship\b([^>]*)\/?>(?:<\/Relationship>)?/g)) {
@@ -99,13 +104,14 @@ async function inspectWpsImages(zip: JSZip) {
sheetRelTargets.set(attrs.Id, packagePath(attrs.Target)); sheetRelTargets.set(attrs.Id, packagePath(attrs.Target));
} }
const allowedIds = new Set<string>(); const allowedIds = new Set<string>();
const imagesBySheet = new Map<string, EmbeddedImage[]>(); const imagesBySheet = new Map<string, Array<EmbeddedImage | EmbeddedImageMetadata>>();
let totalImageBytes = 0;
for (const match of workbookXml.matchAll(/<sheet\b([^>]*)\/?>(?:<\/sheet>)?/g)) { for (const match of workbookXml.matchAll(/<sheet\b([^>]*)\/?>(?:<\/sheet>)?/g)) {
const attrs = attributes(match[1]); const attrs = attributes(match[1]);
const path = sheetRelTargets.get(attrs['r:id']); const path = sheetRelTargets.get(attrs['r:id']);
if (!attrs.name || !path) continue; if (!attrs.name || !path) continue;
const sheetXml = await text(zip, path); const sheetXml = await text(zip, path);
const images: EmbeddedImage[] = []; const images: Array<EmbeddedImage | EmbeddedImageMetadata> = [];
for (const cell of sheetXml.matchAll(/<c\b([^>]*)>([\s\S]*?)<\/c>/g)) { for (const cell of sheetXml.matchAll(/<c\b([^>]*)>([\s\S]*?)<\/c>/g)) {
const formulaText = decodeXml(cell[2].match(/<f(?:\s[^>]*)?>([\s\S]*?)<\/f>/)?.[1]?.trim() ?? ''); const formulaText = decodeXml(cell[2].match(/<f(?:\s[^>]*)?>([\s\S]*?)<\/f>/)?.[1]?.trim() ?? '');
if (!formulaText) continue; if (!formulaText) continue;
@@ -118,20 +124,31 @@ async function inspectWpsImages(zip: JSZip) {
if (!imageEntry) throw new BadRequestException('WPS单元格图片文件缺失'); if (!imageEntry) throw new BadRequestException('WPS单元格图片文件缺失');
const extension = normalizeImageExtension(target.split('.').pop() ?? 'png'); const extension = normalizeImageExtension(target.split('.').pop() ?? 'png');
if (!['png', 'jpeg', 'gif'].includes(extension)) throw new BadRequestException('WPS单元格图片格式不受支持'); if (!['png', 'jpeg', 'gif'].includes(extension)) throw new BadRequestException('WPS单元格图片格式不受支持');
const size = Number(
(imageEntry as unknown as { _data?: { uncompressedSize?: number } })._data?.uncompressedSize ?? 0,
);
if (size > MAX_IMAGE_BYTES) throw new BadRequestException('单张WPS图片不能超过20MB');
totalImageBytes += size;
if (includeImageData) {
const imageBuffer = await imageEntry.async('nodebuffer'); const imageBuffer = await imageEntry.async('nodebuffer');
if (imageBuffer.length > MAX_IMAGE_BYTES) throw new BadRequestException('单张WPS图片不能超过20MB'); if (!validImageSignature(extension, imageBuffer))
if (!validImageSignature(extension, imageBuffer)) throw new BadRequestException('WPS单元格图片内容与格式不匹配'); throw new BadRequestException('WPS单元格图片内容与格式不匹配');
images.push({ ...cellPosition, extension, buffer: imageBuffer }); images.push({ ...cellPosition, extension, buffer: imageBuffer });
} else {
images.push({ ...cellPosition, extension, size, target });
}
allowedIds.add(formula[1]); allowedIds.add(formula[1]);
} }
imagesBySheet.set(attrs.name, images); imagesBySheet.set(attrs.name, images);
} }
const totalImageBytes = [...imagesBySheet.values()].flat().reduce((sum, image) => sum + image.buffer.length, 0);
if (totalImageBytes > MAX_TOTAL_IMAGE_BYTES) throw new BadRequestException('WPS图片总量不能超过300MB'); if (totalImageBytes > MAX_TOTAL_IMAGE_BYTES) throw new BadRequestException('WPS图片总量不能超过300MB');
return { allowedIds, imagesBySheet }; return { allowedIds, imagesBySheet, mediaTargets: new Set(imageTargets.values()) };
} }
export async function loadCompatibleWorkbook(buffer: Buffer) { export async function loadCompatibleWorkbook(
buffer: Buffer,
options: { includeImageData?: boolean; onProgress?: (progress: number, stage: string) => void | Promise<void> } = {},
) {
if (buffer.length > MAX_WORKBOOK_BYTES) throw new BadRequestException('导入文件不能超过100MB'); if (buffer.length > MAX_WORKBOOK_BYTES) throw new BadRequestException('导入文件不能超过100MB');
let zip: JSZip; let zip: JSZip;
try { try {
@@ -139,6 +156,7 @@ export async function loadCompatibleWorkbook(buffer: Buffer) {
} catch { } catch {
throw new BadRequestException('仅支持有效的 XLSX 文件'); throw new BadRequestException('仅支持有效的 XLSX 文件');
} }
await options.onProgress?.(35, '已读取工作簿压缩包');
const expandedBytes = Object.values(zip.files).reduce( const expandedBytes = Object.values(zip.files).reduce(
(sum, entry) => (sum, entry) =>
sum + Number((entry as unknown as { _data?: { uncompressedSize?: number } })._data?.uncompressedSize ?? 0), sum + Number((entry as unknown as { _data?: { uncompressedSize?: number } })._data?.uncompressedSize ?? 0),
@@ -148,17 +166,32 @@ export async function loadCompatibleWorkbook(buffer: Buffer) {
if (expandedBytes > MAX_EXPANDED_BYTES) throw new BadRequestException('工作簿解压后超过500MB安全限制'); if (expandedBytes > MAX_EXPANDED_BYTES) throw new BadRequestException('工作簿解压后超过500MB安全限制');
if (Object.keys(zip.files).some((name) => /(^|\/)(vbaProject|externalLinks|embeddings|activeX)(\/|\.)/i.test(name))) if (Object.keys(zip.files).some((name) => /(^|\/)(vbaProject|externalLinks|embeddings|activeX)(\/|\.)/i.test(name)))
throw new BadRequestException('工作簿包含不允许的外部对象或宏'); throw new BadRequestException('工作簿包含不允许的外部对象或宏');
const wps = await inspectWpsImages(zip); const wps = await inspectWpsImages(zip, options.includeImageData !== false);
await options.onProgress?.(55, '已定位WPS单元格图片');
let excelJsInput = buffer;
if (options.includeImageData === false && wps.mediaTargets.size) {
for (const target of wps.mediaTargets) zip.remove(target);
zip.remove('xl/cellimages.xml');
zip.remove('xl/_rels/cellimages.xml.rels');
excelJsInput = await zip.generateAsync({
type: 'nodebuffer',
compression: 'DEFLATE',
compressionOptions: { level: 1 },
});
await options.onProgress?.(70, '已剥离解析阶段无需加载的WPS图片数据');
}
const workbook = new ExcelJS.Workbook(); const workbook = new ExcelJS.Workbook();
await workbook.xlsx.load(buffer as never); await workbook.xlsx.load(excelJsInput as never);
await options.onProgress?.(80, '已解析工作表内容');
validateWorkbookValues(workbook, wps.allowedIds); validateWorkbookValues(workbook, wps.allowedIds);
await options.onProgress?.(90, '已完成工作簿安全校验');
return { workbook, wpsImagesBySheet: wps.imagesBySheet }; return { workbook, wpsImagesBySheet: wps.imagesBySheet };
} }
export function compatibleImages( export function compatibleImages(
workbook: ExcelJS.Workbook, workbook: ExcelJS.Workbook,
worksheet: ExcelJS.Worksheet, worksheet: ExcelJS.Worksheet,
wpsImagesBySheet: Map<string, EmbeddedImage[]>, wpsImagesBySheet: Map<string, Array<EmbeddedImage | EmbeddedImageMetadata>>,
) { ) {
const merged = new Map<string, EmbeddedImage>(); const merged = new Map<string, EmbeddedImage>();
const wpsImages = wpsImagesBySheet.get(worksheet.name) ?? []; const wpsImages = wpsImagesBySheet.get(worksheet.name) ?? [];
@@ -169,10 +202,29 @@ export function compatibleImages(
if (!validImageSignature(extension, image.buffer)) throw new BadRequestException('工作簿图片内容与格式不匹配'); if (!validImageSignature(extension, image.buffer)) throw new BadRequestException('工作簿图片内容与格式不匹配');
merged.set(`${image.row}:${image.column}`, image); merged.set(`${image.row}:${image.column}`, image);
} }
for (const image of wpsImages) merged.set(`${image.row}:${image.column}`, image); for (const image of wpsImages) {
if (!('buffer' in image)) throw new BadRequestException('图片元数据不能用于提交导入');
merged.set(`${image.row}:${image.column}`, image);
}
return [...merged.values()]; return [...merged.values()];
} }
export function compatibleImagePositions(
workbook: ExcelJS.Workbook,
worksheet: ExcelJS.Worksheet,
wpsImagesBySheet: Map<string, Array<EmbeddedImage | EmbeddedImageMetadata>>,
) {
const wpsImages = wpsImagesBySheet.get(worksheet.name) ?? [];
const positions = new Map<string, { row: number; column: number }>();
for (const image of readEmbeddedImages(workbook, worksheet)) {
if (wpsImages.length && image.buffer.length <= 128) continue;
positions.set(`${image.row}:${image.column}`, { row: image.row, column: image.column });
}
for (const image of wpsImages)
positions.set(`${image.row}:${image.column}`, { row: image.row, column: image.column });
return [...positions.values()];
}
export async function convertWorkbookOutput(content: Buffer, outputFormat: ReportWorkbookFormat = 'excel_drawing') { export async function convertWorkbookOutput(content: Buffer, outputFormat: ReportWorkbookFormat = 'excel_drawing') {
if (outputFormat === 'excel_drawing') return content; if (outputFormat === 'excel_drawing') return content;
if (outputFormat !== 'wps_cell_image') throw new BadRequestException('不支持的报备文件格式'); if (outputFormat !== 'wps_cell_image') throw new BadRequestException('不支持的报备文件格式');
+2
View File
@@ -5066,6 +5066,8 @@ npm run verify:phase8
| TC-REPORT-WPS-001 | 上传`行业报备.xlsx`等使用`_xlfn.DISPIMG``xl/cellimages.xml`及关系文件的WPS XLSX | 仅精确白名单内且图片关系完整的DISPIMG公式可解析;图片按真实单元格映射进入预览与审核,透明Drawing占位图不计入;普通公式、宏、外部对象、缺失关系和非法图片格式均明确拒绝 | | TC-REPORT-WPS-001 | 上传`行业报备.xlsx`等使用`_xlfn.DISPIMG``xl/cellimages.xml`及关系文件的WPS XLSX | 仅精确白名单内且图片关系完整的DISPIMG公式可解析;图片按真实单元格映射进入预览与审核,透明Drawing占位图不计入;普通公式、宏、外部对象、缺失关系和非法图片格式均明确拒绝 |
| TC-REPORT-WPS-002 | 分别上传10MB以上且不超过100MB、超过100MB、解压后超过500MB的报备XLSX | 第一类可进入解析;后两类分别在上传层或解包层明确拒绝;Nginx私有站点允许100MB业务文件及multipart开销,不扩大公网单条API边界 | | TC-REPORT-WPS-002 | 分别上传10MB以上且不超过100MB、超过100MB、解压后超过500MB的报备XLSX | 第一类可进入解析;后两类分别在上传层或解包层明确拒绝;Nginx私有站点允许100MB业务文件及multipart开销,不扩大公网单条API边界 |
| TC-REPORT-WPS-003 | 在批次和单条报备导出选择“系统 Excel 文件”或“WPS 单元格图片文件” | 默认保持ExcelJS Drawing格式;WPS选项生成DISPIMG与cellimages关系且图片按对应单元格可回读;两种格式均来自同一真实资料快照,不改变报备状态或短信链路 | | TC-REPORT-WPS-003 | 在批次和单条报备导出选择“系统 Excel 文件”或“WPS 单元格图片文件” | 默认保持ExcelJS Drawing格式;WPS选项生成DISPIMG与cellimages关系且图片按对应单元格可回读;两种格式均来自同一真实资料快照,不改变报备状态或短信链路 |
| TC-REPORT-WPS-004 | 上传40MB以上且不超过100MB的WPS报备文件,并持续观察解析弹窗与服务重启 | 上传接口在文件写入对象存储并创建PostgreSQL任务后返回202;独立解析Worker单并发处理,页面轮询真实进度;API/浏览器不再等待完整解析,Worker重启后超时租约任务可重新排队且不重复提交导入 |
| TC-REPORT-WPS-005 | 分析包含大量DISPIMG图片的WPS文件,再提交映射进入审核 | 分析阶段只读取图片关系、位置、格式和声明大小,不展开全部WPS图片二进制;提交导入阶段重新读取并校验真实图片签名和20MB单图/300MB总量限制,公式安全校验不放宽 |
| TC-REPORT-IMPORT-APPLICATION-001 | 未选择企业应用、选择其他企业的应用、选择当前企业应用后解析导入 | 前两种前后端均阻止导入;合法应用可解析并把applicationId写入导入批次,后续补资料只作用于该企业应用范围 | | TC-REPORT-IMPORT-APPLICATION-001 | 未选择企业应用、选择其他企业的应用、选择当前企业应用后解析导入 | 前两种前后端均阻止导入;合法应用可解析并把applicationId写入导入批次,后续补资料只作用于该企业应用范围 |
| TC-REPORT-FIELD-MODAL-001 | 打开签名或引流字段配置弹窗,在左侧字段池数量不同的情况下连续添加字段,并检查通用字段与移除按钮 | 左侧每个字段卡片始终固定为68px,不因可选字段数量或添加操作被Grid拉伸;右侧默认包含同资料类型的通用字段;移除按钮为通用宽度、无红色填充,保存仍调用真实通道字段接口 | | TC-REPORT-FIELD-MODAL-001 | 打开签名或引流字段配置弹窗,在左侧字段池数量不同的情况下连续添加字段,并检查通用字段与移除按钮 | 左侧每个字段卡片始终固定为68px,不因可选字段数量或添加操作被Grid拉伸;右侧默认包含同资料类型的通用字段;移除按钮为通用宽度、无红色填充,保存仍调用真实通道字段接口 |
| TC-REPORT-MATERIAL-DETAIL-001 | 查看包含当前图片字段、未删除历史字段及旧图片引用的报备资料 | 当前字段展示名称、代码、导出名及图片预览;未删除历史字段继续展示且同时显示字段名称和代码;图片可内联查看并保留下载入口,缺失内容显示明确占位 | | TC-REPORT-MATERIAL-DETAIL-001 | 查看包含当前图片字段、未删除历史字段及旧图片引用的报备资料 | 当前字段展示名称、代码、导出名及图片预览;未删除历史字段继续展示且同时显示字段名称和代码;图片可内联查看并保留下载入口,缺失内容显示明确占位 |
+21
View File
@@ -4478,3 +4478,24 @@ git diff --check
- 测试环境发布前为`bb435fb0ac1e7812fcdb59950a4b13757899ab39`。发布包大小2766696字节,本地和服务器SHA-256均为`2d3c9a022ba1b6494a51e2ee0f3eb04eabb2513af25ba3631dae5ea70ea68783`;恢复点为`/opt/cmpp-platform-backups/import-review-fix-20260904T122649Z`,包含PostgreSQL custom dump、原运行目录、系统配置及发布包,四项SHA、`pg_restore --list`和两份tar可读性通过。95项migration齐全且无待执行项,没有执行迁移;最终`.deployed-commit=41962e7a6e6cfd34b4313c8bf52d9345d37d8e10`,旧运行目录为`/opt/cmpp-platform.previous-import-review-20260904T122649Z` - 测试环境发布前为`bb435fb0ac1e7812fcdb59950a4b13757899ab39`。发布包大小2766696字节,本地和服务器SHA-256均为`2d3c9a022ba1b6494a51e2ee0f3eb04eabb2513af25ba3631dae5ea70ea68783`;恢复点为`/opt/cmpp-platform-backups/import-review-fix-20260904T122649Z`,包含PostgreSQL custom dump、原运行目录、系统配置及发布包,四项SHA、`pg_restore --list`和两份tar可读性通过。95项migration齐全且无待执行项,没有执行迁移;最终`.deployed-commit=41962e7a6e6cfd34b4313c8bf52d9345d37d8e10`,旧运行目录为`/opt/cmpp-platform.previous-import-review-20260904T122649Z`
- 使用自行生成的标准Excel Drawing工作簿`签名导入全流程测试-含图片.xlsx`走真实HTTP API:解析得到6列、1张图片,导入批次`cmtmxvjuh000xluleh38c4sjy`审核结果`approved / approvedCount=1 / failedCount=0`;生成签名`cmtmxvjyh0014lule5o2t16m9``approved`,四项系统身份字段、图片`FileObject`及6个通道图片材料均在PostgreSQL存在。报备批次`cmtmxvk8c0043lules3upx5f2`完成并生成6个真实通道文件,下载ZIP中的首个XLSX包含`xl/media`图片。批次生成后签名`pendingReport=false`,没有发送、补发、重投或重新入队短信。 - 使用自行生成的标准Excel Drawing工作簿`签名导入全流程测试-含图片.xlsx`走真实HTTP API:解析得到6列、1张图片,导入批次`cmtmxvjuh000xluleh38c4sjy`审核结果`approved / approvedCount=1 / failedCount=0`;生成签名`cmtmxvjyh0014lule5o2t16m9``approved`,四项系统身份字段、图片`FileObject`及6个通道图片材料均在PostgreSQL存在。报备批次`cmtmxvk8c0043lules3upx5f2`完成并生成6个真实通道文件,下载ZIP中的首个XLSX包含`xl/media`图片。批次生成后签名`pendingReport=false`,没有发送、补发、重投或重新入队短信。
- API验收使用本轮临时平台管理员,登录、近期认证、导入、审核、生成和下载均经过真实HTTP会话;结束时调用登出并删除临时账号,查询剩余数为0。浏览器已读取测试环境登录页、验证码控件和页面标题,控制台无warning/error;登录后页面交互需按浏览器安全规则由用户确认验证码后继续,不将当前登录页检查冒充已完成登录后验收。 - API验收使用本轮临时平台管理员,登录、近期认证、导入、审核、生成和下载均经过真实HTTP会话;结束时调用登出并删除临时账号,查询剩余数为0。浏览器已读取测试环境登录页、验证码控件和页面标题,控制台无warning/error;登录后页面交互需按浏览器安全规则由用户确认验证码后继续,不将当前登录页检查冒充已完成登录后验收。
## 2026-09-04 WPS报备兼容与导入审核修复(预生产发布完成)
- 用户明确授权发布预生产。发布前重新核验本地与`origin/main`均为`aaf96db2d018cfcea79b6cdbf553cee9cb982fa2`,预生产原标记为`dada0d978bb05d7046469c1ec77371ddb2fb03bc`;累计范围为9个提交、44个文件,包含运营看板、报备字段配置、WPS单元格图片导入导出、100MiB上传限制、字段卡片固定高度及导入审核系统身份字段补齐。没有新增migration,既有未跟踪补救方案文档未进入发布归档。
- 发布前数据盘UUID`ef4ee3bb-a19b-4aeb-b00c-aa2b995611c2`、PostgreSQL/Redis/MinIO绑定挂载、存储保护脚本、systemd drop-in和固定备份入口全部通过;95项migration齐全,相关服务健康,三条Redis Stream均`pending=0 / lag=0`,供应商连接为`desired=9 / connected=9`
- 首次恢复点`/opt/cmpp-platform-backups/preprod-wps-import-20260904T211224Z-before-aaf96db``pg_dump`不接受Prisma连接串的`schema`查询参数而在数据库导出阶段安全停止;当时尚未构建、迁移、切换、重启或修改Nginx。改用去除查询参数的数据库连接后重新建立有效恢复点`/opt/cmpp-platform-backups/preprod-wps-import-20260904T211255Z-before-aaf96db`,约613MiB,包含PostgreSQL custom dump、Redis RDB、原运行目录、环境/systemd/Nginx/Fail2ban/nftables及数据盘保护配置、原标记和发布基线。
- 有效恢复点的8项最终SHA-256全部通过,`pg_restore --list`包含825项,运行目录tar包含52641项、系统配置tar包含370项;精确Git归档大小2768402字节,SHA-256为`d984c79d0deef1a83ec418b4806875eca210343bb76e03f174887614ba6517c1`,服务器回读一致。候选版本通过依赖安全、部署契约、Prisma生成与迁移状态、前端/API/Gateway/Security Agent生产构建以及`cmpp-api`用户运行文件可读/可执行门禁。
- 最终`.deployed-commit=aaf96db2d018cfcea79b6cdbf553cee9cb982fa2`,上一运行目录保留为`/opt/cmpp-platform.previous-preprod-dada0d9-20260904T211255Z`。95项migration仍齐全且无待执行项,本轮未执行数据库迁移。Nginx仅将`sms.lisglo.com`和兼容端口12026的两个`client_max_body_size 50m`精确调整为`110m`API专用域名保持`10m``nginx -t`通过后reload,没有修改数据盘、fstab、UUID、存储保护脚本或systemd存储drop-in。
- API、Send Worker、Submit Outbox、Gateway Callback、Protocol Log Worker、Gateway、Security Agent、Nginx、PostgreSQL、Redis、MinIO、Node Exporter和Prometheus均为`active`,相关应用服务`NRestarts=0 / Result=success`API、Callback、Gateway、PostgreSQL和Redis健康通过。`12026``sms.lisglo.com/api/health``api.lisglo.com/api/health`均为HTTP 200,主JS`index-CFz3CyND.js`及CSS`index-C3z6fQED.css`的公网下载SHA-256与服务器产物逐项一致。
- 发布后三条Stream的`last-delivered-id / entries-read`与发布前一致:commands为`1788517020869-2 / 84015`results为`1788527156466-0 / 45876`protocol logs为`1788527156467-0 / 32227`;全部保持`pending=0 / lag=0`。本轮未发送、补发、重投、重新入队或手工ACK短信,也没有修改余额、通道、客户或签名业务资料。
- 一条富泷供应商通道在重启后首次鉴权失败,连接短暂为`8/9`,系统按既有计划于21:20:25自动重试并恢复`9/9`,未修改通道配置或凭据。真实下游客户端受旧心跳租约影响的重复连接被连接数门禁拒绝,并记录3次既有`close of closed channel`连接协程panic;Gateway进程未退出、下游心跳持续、systemd error级journal为0,该已知重启恢复缺陷不由本次报备功能引入,继续保留专项治理。
- 浏览器控制在读取预生产运营登录页时连续两次超时,因此本轮没有取得可复核的DOM、截图或控制台证据,也未输入账号、密码或验证码;只把真实HTTP、资源哈希和服务端证据记为通过,不将其冒充登录后浏览器验收。
## 2026-09-04 大文件WPS报备异步解析与真实进度(本地验证完成)
- 40MB以上WPS文件同步解析会让单个HTTP请求同时承担上传、两轮ZIP/图片展开和ExcelJS解析,预生产证据已出现约126秒后由入口断开的499;单纯展示前端动画进度不能延长代理读超时。本轮将接口改为文件入MinIO并创建PostgreSQL任务后返回202,由独立`cmpp-report-material-worker`单并发解析,页面轮询租户隔离的真实任务状态。
- 新增`queued/analyzing/analyzed/failed`状态、0–100进度、阶段、错误、开始和心跳时间;Worker启动时将心跳超过5分钟的解析任务恢复为排队状态,领取时使用状态条件更新防止重复占用。该变更新增1项Prisma migration和1个systemd服务,发布时必须同时迁移数据库、安装并验证Worker,不涉及短信队列、通道、余额或客户配置。
- 报备导入使用独立100MiB上限,普通文件仍为10MiB、图片仍为2MiB。文件类型、扩展名和ZIP签名在入库前校验;工作簿公式、外部对象、解压体积、图片格式及图片数量关系在后台解析或提交阶段明确失败,不静默吞错。
- 分析阶段读取WPS`cellimages.xml`关系、单元格位置、扩展名和ZIP声明大小,不展开图片Buffer,并在交给ExcelJS前从临时解析副本剥离WPS媒体;提交导入时仍重新下载原文件并完整读取、签名校验和上传图片,没有放宽DISPIMG白名单或20MiB单图/300MiB图片总量限制。
- 真实样本`行业报备.xlsx`仍得到工作表“行业”、17行、13列和43张图片。相同进程环境下原完整图片模式约5522ms/RSS 259MiB,新分析模式约1899ms/RSS 227MiB,耗时下降约66%,结果计数一致;样本只读,未覆盖或写回。
- 定向API 3套25项、API全量54套616项、前端全量13文件64项通过;前后端TypeScript、API/Vite生产构建、依赖与安全门禁、部署契约、结构质量、入口包体积和`git diff --check`通过。测试环境部署及真实HTTP/MinIO/PostgreSQL/浏览器验收将在本提交推送后执行,预生产不在本轮范围内。
+23
View File
@@ -179,6 +179,10 @@ export const adminChannelsReportsApi = {
return requestForm< return requestForm<
Record<string, unknown> & { Record<string, unknown> & {
id: string; id: string;
status: 'queued' | 'analyzing' | 'analyzed' | 'failed';
progress: number;
progressStage?: string;
errorMessage?: string;
columns: Array<{ columns: Array<{
sourceColumnIndex: number; sourceColumnIndex: number;
columnLetter: string; columnLetter: string;
@@ -191,6 +195,25 @@ export const adminChannelsReportsApi = {
} }
>('/admin/report-materials/imports/analyze', form); >('/admin/report-materials/imports/analyze', form);
}, },
getReportMaterialImportAnalysisStatus: (id: string, tenantId: string, signal?: AbortSignal) =>
request<
Record<string, unknown> & {
id: string;
status: 'queued' | 'analyzing' | 'analyzed' | 'failed';
progress: number;
progressStage?: string;
errorMessage?: string;
columns: Array<{
sourceColumnIndex: number;
columnLetter: string;
sourceHeader: string;
sourceHeaderPath: string;
imageCount: number;
}>;
rows: Array<Record<string, unknown>>;
suggestedMappings: ReportImportMapping[];
}
>(withQuery(`/admin/report-materials/imports/${id}/analysis-status`, { tenantId }), { signal }),
commitReportMaterialImport: ( commitReportMaterialImport: (
id: string, id: string,
body: { mappings: ReportImportMapping[]; profile?: Omit<ReportImportProfile, 'id'> & { id?: string } }, body: { mappings: ReportImportMapping[]; profile?: Omit<ReportImportProfile, 'id'> & { id?: string } },
@@ -6,6 +6,7 @@ import { ReportMaterialImportModal } from './ReportMaterialImportModal';
const { adminApi } = vi.hoisted(() => ({ const { adminApi } = vi.hoisted(() => ({
adminApi: { adminApi: {
analyzeReportMaterialImport: vi.fn(), analyzeReportMaterialImport: vi.fn(),
getReportMaterialImportAnalysisStatus: vi.fn(),
listDrainageFields: vi.fn(), listDrainageFields: vi.fn(),
listEnterpriseApplicationOptions: vi.fn(), listEnterpriseApplicationOptions: vi.fn(),
listReportImportProfiles: vi.fn(), listReportImportProfiles: vi.fn(),
@@ -27,6 +28,15 @@ describe('ReportMaterialImportModal mapping profile action', () => {
adminApi.listReportImportProfiles.mockResolvedValue([]); adminApi.listReportImportProfiles.mockResolvedValue([]);
adminApi.analyzeReportMaterialImport.mockResolvedValue({ adminApi.analyzeReportMaterialImport.mockResolvedValue({
id: 'analysis-1', id: 'analysis-1',
status: 'queued',
progress: 10,
progressStage: '文件已上传,等待解析',
});
adminApi.getReportMaterialImportAnalysisStatus.mockResolvedValue({
id: 'analysis-1',
status: 'analyzed',
progress: 100,
progressStage: '解析完成',
columns: [ columns: [
{ sourceColumnIndex: 0, columnLetter: 'A', sourceHeader: '签名', sourceHeaderPath: '签名', imageCount: 0 }, { sourceColumnIndex: 0, columnLetter: 'A', sourceHeader: '签名', sourceHeaderPath: '签名', imageCount: 0 },
], ],
@@ -59,5 +69,5 @@ describe('ReportMaterialImportModal mapping profile action', () => {
expect(screen.getByRole('button', { name: '本次将保存/更新映射方案' })).toHaveAttribute('aria-pressed', 'true'), expect(screen.getByRole('button', { name: '本次将保存/更新映射方案' })).toHaveAttribute('aria-pressed', 'true'),
); );
expect(screen.getByLabelText('映射方案名称')).toBeVisible(); expect(screen.getByLabelText('映射方案名称')).toBeVisible();
}); }, 10_000);
}); });
+85 -8
View File
@@ -25,6 +25,7 @@ type AnalyzeResult = {
rows: Array<{ rowNumber: number; values: Record<string, string>; imageColumns: number[] }>; rows: Array<{ rowNumber: number; values: Record<string, string>; imageColumns: number[] }>;
suggestedMappings: ReportImportMapping[]; suggestedMappings: ReportImportMapping[];
}; };
type AnalysisProgress = { id: string; status: string; progress: number; progressStage?: string; errorMessage?: string };
const transforms = [ const transforms = [
{ label: '保持原值', value: '' }, { label: '保持原值', value: '' },
@@ -60,6 +61,7 @@ export function ReportMaterialImportModal({ onClose, onCompleted }: { onClose: (
const [headerRowCount, setHeaderRowCount] = useState(1); const [headerRowCount, setHeaderRowCount] = useState(1);
const [dataStartRow, setDataStartRow] = useState(2); const [dataStartRow, setDataStartRow] = useState(2);
const [analysis, setAnalysis] = useState<AnalyzeResult>(); const [analysis, setAnalysis] = useState<AnalyzeResult>();
const [analysisJob, setAnalysisJob] = useState<AnalysisProgress>();
const [mappings, setMappings] = useState<ReportImportMapping[]>([]); const [mappings, setMappings] = useState<ReportImportMapping[]>([]);
const [profileName, setProfileName] = useState(''); const [profileName, setProfileName] = useState('');
const [saveProfile, setSaveProfile] = useState(false); const [saveProfile, setSaveProfile] = useState(false);
@@ -87,10 +89,52 @@ export function ReportMaterialImportModal({ onClose, onCompleted }: { onClose: (
.catch(() => setProfiles([])); .catch(() => setProfiles([]));
}, [reportType]); }, [reportType]);
useEffect(() => {
if (!analysisJob?.id || !tenantId || analysisJob.status === 'analyzed' || analysisJob.status === 'failed') return;
const controller = new AbortController();
let timer: ReturnType<typeof setTimeout> | undefined;
const poll = async () => {
try {
const result = await adminApi.getReportMaterialImportAnalysisStatus(
analysisJob.id,
tenantId,
controller.signal,
);
if (controller.signal.aborted) return;
setError('');
setAnalysisJob(result);
if (result.status === 'analyzed') {
const completed = result as AnalyzeResult & AnalysisProgress;
setAnalysis(completed);
setMappings(completed.suggestedMappings ?? []);
const selectedProfile = profiles.find((item) => item.id === profileId);
if (selectedProfile) setProfileName(selectedProfile.name);
return;
}
if (result.status === 'failed') {
setError(result.errorMessage || '文件解析失败');
return;
}
timer = setTimeout(() => void poll(), 1200);
} catch (failure) {
if (!controller.signal.aborted) {
setError(failure instanceof Error ? `${failure.message},正在重试` : '解析进度查询失败,正在重试');
timer = setTimeout(() => void poll(), 2000);
}
}
};
void poll();
return () => {
controller.abort();
if (timer) clearTimeout(timer);
};
}, [analysisJob?.id, analysisJob?.status, profileId, profiles, tenantId]);
function changeReportType(next: ReportType) { function changeReportType(next: ReportType) {
setReportType(next); setReportType(next);
setProfileId(''); setProfileId('');
setAnalysis(undefined); setAnalysis(undefined);
setAnalysisJob(undefined);
setMappings([]); setMappings([]);
} }
@@ -108,18 +152,15 @@ export function ReportMaterialImportModal({ onClose, onCompleted }: { onClose: (
setBusy(true); setBusy(true);
setError(''); setError('');
try { try {
const result = (await adminApi.analyzeReportMaterialImport(file, { const result = await adminApi.analyzeReportMaterialImport(file, {
tenantId, tenantId,
applicationId, applicationId,
reportType, reportType,
headerRowCount, headerRowCount,
dataStartRow, dataStartRow,
profileId: profileId || undefined, profileId: profileId || undefined,
})) as AnalyzeResult; });
setAnalysis(result); setAnalysisJob(result);
setMappings(result.suggestedMappings ?? []);
const selectedProfile = profiles.find((item) => item.id === profileId);
if (selectedProfile) setProfileName(selectedProfile.name);
} catch (failure) { } catch (failure) {
setError(failure instanceof Error ? failure.message : '文件解析失败'); setError(failure instanceof Error ? failure.message : '文件解析失败');
} finally { } finally {
@@ -216,8 +257,21 @@ export function ReportMaterialImportModal({ onClose, onCompleted }: { onClose: (
{busy ? '提交中...' : '提交导入审核'} {busy ? '提交中...' : '提交导入审核'}
</Button> </Button>
) : ( ) : (
<Button disabled={busy || !file || !tenantId || !applicationId} onClick={() => void analyze()}> <Button
{busy ? '解析中...' : '解析文件并配置映射'} disabled={
busy ||
Boolean(analysisJob && !['failed', 'analyzed'].includes(analysisJob.status)) ||
!file ||
!tenantId ||
!applicationId
}
onClick={() => void analyze()}
>
{busy
? '上传中...'
: analysisJob && !['failed', 'analyzed'].includes(analysisJob.status)
? '后台解析中...'
: '解析文件并配置映射'}
</Button> </Button>
)} )}
</> </>
@@ -306,10 +360,33 @@ export function ReportMaterialImportModal({ onClose, onCompleted }: { onClose: (
onChange={(event) => { onChange={(event) => {
setFile(event.target.files?.[0]); setFile(event.target.files?.[0]);
setAnalysis(undefined); setAnalysis(undefined);
setAnalysisJob(undefined);
setError('');
}} }}
type="file" type="file"
/> />
</label> </label>
{analysisJob && analysisJob.status !== 'analyzed' ? (
<div className="report-import-analysis-progress" aria-live="polite">
<div>
<strong>{analysisJob.progressStage || '正在处理'}</strong>
<span>{Math.max(0, Math.min(100, analysisJob.progress))}%</span>
</div>
<div
className="batch-progress__track"
role="progressbar"
aria-valuemin={0}
aria-valuemax={100}
aria-valuenow={analysisJob.progress}
>
<span
className="batch-progress__bar batch-progress__bar--sending"
style={{ width: `${Math.max(0, Math.min(100, analysisJob.progress))}%` }}
/>
</div>
<small></small>
</div>
) : null}
{analysis ? ( {analysis ? (
<div className="report-import-mapping"> <div className="report-import-mapping">
<div className="channel-field-section-head"> <div className="channel-field-section-head">
+3
View File
@@ -6099,6 +6099,9 @@
.report-import-file span { display: flex; align-items: center; gap: 9px; } .report-import-file span { display: flex; align-items: center; gap: 9px; }
.report-import-file input { max-width: 310px; } .report-import-file input { max-width: 310px; }
.report-import-analysis-progress { display: grid; gap: 8px; padding: 12px; border: 1px solid var(--border); border-radius: 10px; background: var(--surface-muted); }
.report-import-analysis-progress > div:first-child { display: flex; justify-content: space-between; gap: 12px; }
.report-import-analysis-progress small { color: var(--text-muted); }
.report-import-mapping { display: grid; gap: 12px; } .report-import-mapping { display: grid; gap: 12px; }
.report-import-mapping-table { overflow-x: auto; border: 1px solid var(--border); border-radius: 10px; } .report-import-mapping-table { overflow-x: auto; border: 1px solid var(--border); border-radius: 10px; }
.report-import-mapping-head, .report-import-mapping-head,
+22 -1
View File
@@ -206,7 +206,7 @@ SQL
write_env() { write_env() {
log "Writing production environment" log "Writing production environment"
mkdir -p /etc/cmpp-platform "$APP_DIR" "$APP_DIR/logs/api" "$APP_DIR/logs/send-worker" "$APP_DIR/logs/gateway" "$APP_DIR/backups" "$OBJECT_STORAGE_LOCAL_ROOT" mkdir -p /etc/cmpp-platform "$APP_DIR" "$APP_DIR/logs/api" "$APP_DIR/logs/send-worker" "$APP_DIR/logs/report-material-worker" "$APP_DIR/logs/gateway" "$APP_DIR/backups" "$OBJECT_STORAGE_LOCAL_ROOT"
cat >/etc/cmpp-platform/cmpp-platform.env <<EOF cat >/etc/cmpp-platform/cmpp-platform.env <<EOF
NODE_ENV=production NODE_ENV=production
API_PORT=${API_PORT} API_PORT=${API_PORT}
@@ -332,6 +332,27 @@ RestartSec=5
StandardOutput=append:${APP_DIR}/logs/send-worker/stdout.log StandardOutput=append:${APP_DIR}/logs/send-worker/stdout.log
StandardError=append:${APP_DIR}/logs/send-worker/stderr.log StandardError=append:${APP_DIR}/logs/send-worker/stderr.log
[Install]
WantedBy=multi-user.target
EOF
cat >/etc/systemd/system/cmpp-report-material-worker.service <<EOF
[Unit]
Description=CMPP report material workbook analysis worker
After=network.target postgresql.service cmpp-minio.service
[Service]
User=cmpp-api
Group=cmpp-security
WorkingDirectory=${APP_DIR}/api
EnvironmentFile=/etc/cmpp-platform/cmpp-platform.env
Environment=CMPP_PROCESS_ROLE=report-material-worker
ExecStart=${node_bin} dist/report-material-analysis-worker.js
Restart=always
RestartSec=5
StandardOutput=append:${APP_DIR}/logs/report-material-worker/stdout.log
StandardError=append:${APP_DIR}/logs/report-material-worker/stderr.log
[Install] [Install]
WantedBy=multi-user.target WantedBy=multi-user.target
EOF EOF
+25 -3
View File
@@ -129,7 +129,7 @@ PROD_ADMIN_CREDENTIAL_FILE="$ADMIN_CREDENTIAL_FILE" node tools/deploy/ensure-pro
chmod 600 "$ADMIN_CREDENTIAL_FILE" || true chmod 600 "$ADMIN_CREDENTIAL_FILE" || true
echo "[deploy] Ensuring runtime log directories" echo "[deploy] Ensuring runtime log directories"
install -d -m 0755 "$APP_DIR/logs/api" "$APP_DIR/logs/send-worker" "$APP_DIR/logs/submit-outbox" "$APP_DIR/logs/gateway-callback" "$APP_DIR/logs/protocol-log-worker" "$APP_DIR/logs/gateway" install -d -m 0755 "$APP_DIR/logs/api" "$APP_DIR/logs/send-worker" "$APP_DIR/logs/report-material-worker" "$APP_DIR/logs/submit-outbox" "$APP_DIR/logs/gateway-callback" "$APP_DIR/logs/protocol-log-worker" "$APP_DIR/logs/gateway"
echo "[deploy] Installing split API and send-worker services" echo "[deploy] Installing split API and send-worker services"
node_bin="$(command -v node)" node_bin="$(command -v node)"
@@ -155,6 +155,26 @@ RestartSec=5
StandardOutput=append:$APP_DIR/logs/send-worker/stdout.log StandardOutput=append:$APP_DIR/logs/send-worker/stdout.log
StandardError=append:$APP_DIR/logs/send-worker/stderr.log StandardError=append:$APP_DIR/logs/send-worker/stderr.log
[Install]
WantedBy=multi-user.target
EOF
cat >/etc/systemd/system/cmpp-report-material-worker.service <<EOF
[Unit]
Description=CMPP report material workbook analysis worker
After=network.target postgresql.service cmpp-minio.service
[Service]
User=cmpp-api
Group=cmpp-security
WorkingDirectory=$APP_DIR/api
EnvironmentFile=$ENV_FILE
Environment=CMPP_PROCESS_ROLE=report-material-worker
ExecStart=$node_bin dist/report-material-analysis-worker.js
Restart=always
RestartSec=5
StandardOutput=append:$APP_DIR/logs/report-material-worker/stdout.log
StandardError=append:$APP_DIR/logs/report-material-worker/stderr.log
[Install] [Install]
WantedBy=multi-user.target WantedBy=multi-user.target
EOF EOF
@@ -243,11 +263,11 @@ echo "[deploy] Restarting services"
systemctl daemon-reload systemctl daemon-reload
if [[ "${OBJECT_STORAGE_DRIVER:-minio}" == "local" ]]; then if [[ "${OBJECT_STORAGE_DRIVER:-minio}" == "local" ]]; then
systemctl disable --now cmpp-minio 2>/dev/null || true systemctl disable --now cmpp-minio 2>/dev/null || true
systemctl enable --now cmpp-api cmpp-send-worker cmpp-gateway nginx systemctl enable --now cmpp-api cmpp-send-worker cmpp-report-material-worker cmpp-gateway nginx
else else
systemctl enable --now cmpp-minio systemctl enable --now cmpp-minio
systemctl restart cmpp-minio systemctl restart cmpp-minio
systemctl enable --now cmpp-api cmpp-send-worker cmpp-gateway nginx systemctl enable --now cmpp-api cmpp-send-worker cmpp-report-material-worker cmpp-gateway nginx
fi fi
systemctl restart cmpp-security-agent systemctl restart cmpp-security-agent
if [[ "${SEND_SUBMIT_OUTBOX_SEPARATE_PROCESS_ENABLED:-false}" == "true" ]]; then if [[ "${SEND_SUBMIT_OUTBOX_SEPARATE_PROCESS_ENABLED:-false}" == "true" ]]; then
@@ -273,6 +293,7 @@ fi
systemctl restart cmpp-gateway systemctl restart cmpp-gateway
systemctl restart cmpp-api systemctl restart cmpp-api
systemctl restart cmpp-send-worker systemctl restart cmpp-send-worker
systemctl restart cmpp-report-material-worker
systemctl restart nginx systemctl restart nginx
echo "[deploy] Health checks" echo "[deploy] Health checks"
@@ -292,6 +313,7 @@ wait_for_http() {
} }
wait_for_http "API" "http://127.0.0.1:${API_PORT:-3000}/api/health" wait_for_http "API" "http://127.0.0.1:${API_PORT:-3000}/api/health"
wait_for_http "Send worker metrics" "http://127.0.0.1:${API_WORKER_METRICS_PORT:-9465}/metrics" wait_for_http "Send worker metrics" "http://127.0.0.1:${API_WORKER_METRICS_PORT:-9465}/metrics"
systemctl is-active --quiet cmpp-report-material-worker
if [[ "${SEND_SUBMIT_OUTBOX_SEPARATE_PROCESS_ENABLED:-false}" == "true" ]]; then if [[ "${SEND_SUBMIT_OUTBOX_SEPARATE_PROCESS_ENABLED:-false}" == "true" ]]; then
wait_for_http "Submit Outbox metrics" "http://127.0.0.1:${API_OUTBOX_METRICS_PORT:-9467}/metrics" wait_for_http "Submit Outbox metrics" "http://127.0.0.1:${API_OUTBOX_METRICS_PORT:-9467}/metrics"
fi fi