diff --git a/api/src/org/labkey/api/pipeline/AppPipelineJobNotificationProvider.java b/api/src/org/labkey/api/pipeline/AppPipelineJobNotificationProvider.java index e6b305b0c36..da8e549c16f 100644 --- a/api/src/org/labkey/api/pipeline/AppPipelineJobNotificationProvider.java +++ b/api/src/org/labkey/api/pipeline/AppPipelineJobNotificationProvider.java @@ -182,7 +182,7 @@ private String getJobSuccessMsg(PipelineJob job, @NotNull ImportType importType, .append(!crossTypeImport ? type + " " : "") .append(importType.name()) .append(" from ") - .append(queryImportPipelineJob.getImportContextBuilder().getPrimaryFile().getName()); + .append(queryImportPipelineJob.getImportSourceDescription()); if (job.getContainer().hasProductFolders()) { @@ -224,7 +224,7 @@ private String getJobErrorMsg(PipelineJob job, String rawErrorMsg) return "Failed to import " + type + " from " + - queryImportPipelineJob.getImportContextBuilder().getPrimaryFile().getName() + + queryImportPipelineJob.getImportSourceDescription() + "\n" + rawErrorMsg;// resolveErrorMessage on client } diff --git a/api/src/org/labkey/api/query/AbstractQueryImportAction.java b/api/src/org/labkey/api/query/AbstractQueryImportAction.java index d78c859fe93..800cb031f8c 100644 --- a/api/src/org/labkey/api/query/AbstractQueryImportAction.java +++ b/api/src/org/labkey/api/query/AbstractQueryImportAction.java @@ -45,6 +45,7 @@ import org.labkey.api.module.Module; import org.labkey.api.module.ModuleLoader; import org.labkey.api.pipeline.PipeRoot; +import org.labkey.api.pipeline.PipelineJob; import org.labkey.api.pipeline.PipelineService; import org.labkey.api.reader.ColumnDescriptor; import org.labkey.api.reader.DataLoader; @@ -71,6 +72,7 @@ import org.labkey.api.webdav.WebdavService; import org.labkey.api.workflow.WorkflowService; import org.labkey.vfs.FileLike; +import org.springframework.util.MultiValueMap; import org.springframework.validation.BindException; import org.springframework.web.multipart.MultipartFile; import org.springframework.web.multipart.MultipartHttpServletRequest; @@ -435,7 +437,7 @@ public final ApiResponse _execute(FORM form, BindException errors) throws Except if (errors.hasErrors()) throw errors; - FileLike dataFile = null; + List dataFiles = new ArrayList<>(); boolean hasPostData = false; FileStream file = null; String originalName = null; @@ -547,17 +549,19 @@ else if (getViewContext().getRequest() instanceof MultipartHttpServletRequest) { Map files = getFileMap(); MultipartFile multipartfile = null==files ? null : files.get("file"); - if (null != multipartfile && multipartfile.getSize() > 0) + List uploadedFiles = getUploadedFiles(); + if (!uploadedFiles.isEmpty()) { - transactionDetails.put(TransactionAuditProvider.TransactionDetail.ImportFileName, multipartfile.getOriginalFilename()); hasPostData = true; - originalName = multipartfile.getOriginalFilename(); - // can't read the multipart file twice so create temp file (12800) - dataFile = FileUtil.createTempFileLike("~upload", multipartfile.getOriginalFilename()); PipeRoot root = PipelineService.get().findPipelineRoot(getContainer()); + originalName = uploadedFiles.getFirst().getOriginalFilename(); + transactionDetails.put(TransactionAuditProvider.TransactionDetail.ImportFileName, originalName); + + FileLike dataFileDir = null; + if (null != root && (Boolean.parseBoolean(saveToPipeline) || _useAsync)) { - FileLike dataFileDir = root.getRootFileLike().resolveChild("QueryImportFiles"); + dataFileDir = root.getRootFileLike().resolveChild("QueryImportFiles"); if (dataFileDir.isFile()) { dataFileDir = FileUtil.findUniqueFileName("QueryImportFiles", root.getRootFileLike()); @@ -571,14 +575,24 @@ else if (!dataFileDir.exists()) throw new RuntimeException("Error attempting to create directory " + dataFileDir + " for uploaded query import file " + multipartfile.getOriginalFilename()); } + } + + for (MultipartFile uploadedFile : uploadedFiles) + { + // can't read the multipart file twice so create temp file (12800) + FileLike uploaded = FileUtil.createTempFileLike("~upload", uploadedFile.getOriginalFilename()); - dataFile = FileUtil.findUniqueFileName(multipartfile.getOriginalFilename(), dataFileDir); + if (dataFileDir != null) + uploaded = FileUtil.findUniqueFileName(uploadedFile.getOriginalFilename(), dataFileDir); + uploadedFile.transferTo(uploaded.toNioPathForWrite()); + + dataFiles.add(uploaded); } - multipartfile.transferTo(dataFile.toNioPathForWrite()); + if (_useAsync) { - if (!isBackgroundImportSupported(dataFile.getName())) + if (!isBackgroundImportSupported(dataFiles.getFirst().getName())) throw new RuntimeException("Importing in background currently is not supported for this table"); ViewBackgroundInfo info = new ViewBackgroundInfo(getContainer(), getUser(), new ActionURL()); @@ -586,34 +600,10 @@ else if (!dataFileDir.exists()) UserSchema schema = getTargetSchema(); if (schema != null) { - String schemaName = schema.getSchemaName(); - String queryName = getPipelineTargetQueryName(); - - QueryImportPipelineJob.QueryImportAsyncContextBuilder importContextBuilder = new QueryImportPipelineJob.QueryImportAsyncContextBuilder(); - importContextBuilder - .setPrimaryFile(dataFile) - .setHasColumnHeaders(_hasColumnHeaders) - .setFileContentType(multipartfile.getContentType()) - .setSchemaName(schemaName) - .setQueryName(queryName) - .setRenamedColumns(getRenamedColumns()) - .setLineageImportAliases(getLineageImportAliases()) - .setInsertOption(_insertOption) - .setAuditBehaviorType(behaviorType) - .setAuditUserComment(_auditUserComment) - .setOptionParamsMap(getOptionParamsMap()) - .setLookupResolutionType(getLookupResolutionType()) - .setAllowLineageColumns(allowLineageColumns()) - .setJobDescription(getQueryImportDescription()) - .setJobNotificationProvider(getQueryImportJobNotificationProviderName()); - if (WorkflowService.get() != null) - { - Map workflowParams = WorkflowService.get().getConfigParameters(getViewContext().getRequest()); - importContextBuilder.setWorkflowParams(workflowParams); - } - - importContextBuilder.setTransactionDetails(transactionDetails); - QueryImportPipelineJob job = new QueryImportPipelineJob(getQueryImportProviderName(), info, root, importContextBuilder); + QueryImportPipelineJob.QueryImportAsyncContextBuilder importContextBuilder = getBackgroundImportContext(dataFiles.getFirst(), multipartfile, behaviorType, transactionDetails); + + PipelineJob job = getImportPipelineJob(dataFiles, info, root, importContextBuilder); + PipelineService.get().queueJob(job, getQueryImportJobNotificationProviderName()); if (_target != null) @@ -626,8 +616,9 @@ else if (!dataFileDir.exists()) } - loader = DataLoader.get().createLoader(dataFile.toNioPathForRead().toFile(), multipartfile.getContentType(), _hasColumnHeaders, null, null); - file = new FileAttachmentFile(dataFile, multipartfile.getOriginalFilename()); + Pair loadedFile = prepareDataFileLoader(dataFiles.getFirst(), uploadedFiles.getFirst()); + loader = loadedFile.first; + file = loadedFile.second; } } @@ -636,25 +627,11 @@ else if (!dataFileDir.exists()) if (errors.hasErrors()) throw errors; - BatchValidationException ve = new BatchValidationException(); - //di = wrap(di, ve); - //importData(di, ve); - - configureLoader(loader, _target, getRenamedColumns(), allowLineageColumns(), getLineageImportAliases(), getOptionParamsMap()); - TransactionAuditProvider.TransactionAuditEvent auditEvent = null; if (isCrossTypeImport || (behaviorType != null && behaviorType != AuditBehaviorType.NONE)) auditEvent = createTransactionAuditEvent(getContainer(), _insertOption.auditAction, transactionDetails); - int rowCount = importData(loader, file, originalName, ve, behaviorType, auditEvent, _auditUserComment); - - if (ve.hasErrors()) - { - addImportValidationErrorMetric(_insertOption, _target, (form instanceof QueryForm qf) ? qf.getQueryName() : null); - throw ve; - } - - JSONObject response = createSuccessResponse(rowCount); + JSONObject response = handleImportData(loader, dataFiles, file, originalName, behaviorType, auditEvent, (form instanceof QueryForm qf ? qf.getQueryName() : null)); if (auditEvent != null) { response.put("transactionAuditId", auditEvent.getRowId()); @@ -675,10 +652,95 @@ else if (!dataFileDir.exists()) loader.close(); if (null != file) file.closeInputStream(); - if (null != dataFile && !Boolean.parseBoolean(saveToPipeline) && !_useAsync) - dataFile.delete(); + if (!dataFiles.isEmpty() && !Boolean.parseBoolean(saveToPipeline) && !_useAsync) + { + for (FileLike tmpFile : dataFiles) + tmpFile.delete(); + } + + } + + } + + protected Pair prepareDataFileLoader(FileLike dataFile, MultipartFile multipartFile) throws IOException + { + DataLoader loader = DataLoader.get().createLoader(dataFile.toNioPathForRead().toFile(), multipartFile.getContentType(), _hasColumnHeaders, null, null); + FileStream file = new FileAttachmentFile(dataFile, multipartFile.getOriginalFilename()); + return new Pair<>(loader, file); + } + + protected JSONObject handleImportData(DataLoader loader, List dataFiles, FileStream file, String originalName, AuditBehaviorType behaviorType, + TransactionAuditProvider.TransactionAuditEvent auditEvent, String queryName) throws BatchValidationException, IOException + { + BatchValidationException ve = new BatchValidationException(); + //di = wrap(di, ve); + //importData(di, ve); + + configureLoader(loader, _target, getRenamedColumns(), allowLineageColumns(), getLineageImportAliases(), getOptionParamsMap()); + + int rowCount = importData(loader, file, originalName, ve, behaviorType, auditEvent, _auditUserComment); + + if (ve.hasErrors()) + { + addImportValidationErrorMetric(_insertOption, _target, queryName); + throw ve; + } + + return createSuccessResponse(rowCount); + } + + private @NotNull List getUploadedFiles() + { + MultiValueMap fileMap = PageFlowUtil.getMultiFileMap(getViewContext().getRequest()); + List files = new ArrayList<>(); + if (fileMap != null && fileMap.get("file") != null) + { + for (MultipartFile mf : fileMap.get("file")) + { + if (mf != null && mf.getSize() > 0) + files.add(mf); + } + } + return files; + } + + protected PipelineJob getImportPipelineJob(List dataFiles, ViewBackgroundInfo info, PipeRoot root, QueryImportPipelineJob.QueryImportAsyncContextBuilder importContextBuilder) + { + return new QueryImportPipelineJob(getQueryImportProviderName(), info, root, importContextBuilder); + } + + protected QueryImportPipelineJob.QueryImportAsyncContextBuilder getBackgroundImportContext(FileLike dataFile, @Nullable MultipartFile multipartfile, AuditBehaviorType behaviorType, Map transactionDetails) throws IOException, ValidationException + { + UserSchema schema = getTargetSchema(); + String schemaName = schema.getSchemaName(); + String queryName = getPipelineTargetQueryName(); + + QueryImportPipelineJob.QueryImportAsyncContextBuilder importContextBuilder = new QueryImportPipelineJob.QueryImportAsyncContextBuilder(); + importContextBuilder + .setPrimaryFile(dataFile) + .setHasColumnHeaders(_hasColumnHeaders) + .setFileContentType(multipartfile != null ? multipartfile.getContentType() : null) + .setSchemaName(schemaName) + .setQueryName(queryName) + .setRenamedColumns(getRenamedColumns()) + .setLineageImportAliases(getLineageImportAliases()) + .setInsertOption(_insertOption) + .setAuditBehaviorType(behaviorType) + .setAuditUserComment(_auditUserComment) + .setOptionParamsMap(getOptionParamsMap()) + .setLookupResolutionType(getLookupResolutionType()) + .setAllowLineageColumns(allowLineageColumns()) + .setJobDescription(getQueryImportDescription()) + .setJobNotificationProvider(getQueryImportJobNotificationProviderName()); + if (WorkflowService.get() != null) + { + Map workflowParams = WorkflowService.get().getConfigParameters(getViewContext().getRequest()); + importContextBuilder.setWorkflowParams(workflowParams); } + importContextBuilder.setTransactionDetails(transactionDetails); + + return importContextBuilder; } public static void configureLoader(DataLoader loader, @Nullable TableInfo target, @Nullable Map renamedColumns, boolean allowLineageColumns, @Nullable Set lineageAliasNames, @Nullable Map optionParamsMap) throws IOException diff --git a/api/src/org/labkey/api/query/QueryImportPipelineJob.java b/api/src/org/labkey/api/query/QueryImportPipelineJob.java index 91672dd36dd..026614e872e 100644 --- a/api/src/org/labkey/api/query/QueryImportPipelineJob.java +++ b/api/src/org/labkey/api/query/QueryImportPipelineJob.java @@ -441,4 +441,10 @@ public Map getAdditionalJobResponseInfo() return _additionalJobResponseInfo; } + public String getImportSourceDescription() + { + FileLike primaryFile = _importContextBuilder == null ? null : _importContextBuilder.getPrimaryFile(); + return primaryFile != null ? primaryFile.getName() : ""; + } + } diff --git a/api/src/org/labkey/api/reader/FastaDataLoader.java b/api/src/org/labkey/api/reader/FastaDataLoader.java index ddd58b6e1c6..c6272d45ebb 100644 --- a/api/src/org/labkey/api/reader/FastaDataLoader.java +++ b/api/src/org/labkey/api/reader/FastaDataLoader.java @@ -42,7 +42,7 @@ */ public class FastaDataLoader extends DataLoader { - public static FileType FILE_TYPE = new FileType(Arrays.asList(".fna", ".fasta"), ".fna"); + public static FileType FILE_TYPE = new FileType(Arrays.asList(".fna", ".fasta", ".fa"), ".fna"); static { FILE_TYPE.setExtensionsMutuallyExclusive(false); }