From b8f064a171d30bbb5ae4b755c47133b6e965453a Mon Sep 17 00:00:00 2001 From: XingY Date: Tue, 30 Jun 2026 14:13:11 -0700 Subject: [PATCH 01/10] Multi genbank/fasta import --- .../labkey/api/reader/FastaDataLoader.java | 428 +++++++++--------- 1 file changed, 214 insertions(+), 214 deletions(-) diff --git a/api/src/org/labkey/api/reader/FastaDataLoader.java b/api/src/org/labkey/api/reader/FastaDataLoader.java index ddd58b6e1c6..8a2ebbc9674 100644 --- a/api/src/org/labkey/api/reader/FastaDataLoader.java +++ b/api/src/org/labkey/api/reader/FastaDataLoader.java @@ -1,214 +1,214 @@ -/* - * Copyright (c) 2012-2026 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.labkey.api.reader; - -import org.apache.commons.io.IOUtils; -import org.jetbrains.annotations.NotNull; -import org.labkey.api.data.Container; -import org.labkey.api.iterator.CloseableIterator; -import org.labkey.api.util.FileType; -import org.labkey.api.util.FileUtil; -import org.labkey.vfs.FileLike; -import org.labkey.vfs.FileSystemLike; - -import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; - -/** - * User: klum - * Date: Mar 14, 2012 - */ -public class FastaDataLoader extends DataLoader -{ - public static FileType FILE_TYPE = new FileType(Arrays.asList(".fna", ".fasta"), ".fna"); - static { - FILE_TYPE.setExtensionsMutuallyExclusive(false); - } - - public static class Factory extends AbstractDataLoaderFactory - { - @NotNull @Override - public DataLoader createLoader(File file, boolean hasColumnHeaders, Container mvIndicatorContainer) throws IOException - { - FastaDataLoader result = new FastaDataLoader(file, hasColumnHeaders, mvIndicatorContainer); - result.setCharacterFilter(new FastaLoader.UpperAndLowercaseCharacterFilter()); - return result; - } - - @NotNull @Override - public DataLoader createLoader(InputStream is, boolean hasColumnHeaders, Container mvIndicatorContainer) throws IOException - { - File f = FileUtil.createTempFile("import", ".fasta"); - f.deleteOnExit(); - IOUtils.copy(is, new FileOutputStream(f)); - - FastaDataLoader result = new FastaDataLoader(f, hasColumnHeaders, mvIndicatorContainer); - result.setCharacterFilter(new FastaLoader.UpperAndLowercaseCharacterFilter()); - return result; - } - - @NotNull @Override - public FileType getFileType() { return FILE_TYPE; } - } - - GenericFastaLoader _loader; - - public FastaDataLoader(File inputFile, Boolean hasColumnHeaders) throws IOException - { - this(inputFile, hasColumnHeaders, null); - } - - public FastaDataLoader(File inputFile, boolean hasColumnHeaders, Container mvIndicatorContainer) throws IOException - { - super(mvIndicatorContainer); - setSource(inputFile); - setScrollable(true); - setHasColumnHeaders(hasColumnHeaders); - - _loader = new GenericFastaLoader(FileSystemLike.wrapFile(inputFile)); - } - - public void setCharacterFilter(FastaLoader.CharacterFilter characterFilter) - { - _loader.setCharacterFilter(characterFilter); - } - - @Override - public String[][] getFirstNLines(int n) throws IOException - { - try (GenericFastaLoader.GenericIterator it = _loader.iterator()) - { - List lineFields = new ArrayList<>(n); - int count = 0; - - while (it.hasNext() && count < n) - { - Map row = it.next(); - - // add the internally generated column header - Set strings = row.keySet(); - if (count == 0) - lineFields.add(strings.toArray(new String[0])); - - Collection values = row.values(); - lineFields.add(values.toArray(new String[0])); - - count++; - } - - if (lineFields.isEmpty()) - return new String[0][]; - - return lineFields.toArray(new String[count][]); - } - catch (IllegalArgumentException e) - { - throw new IOException(e.getMessage() == null ? e.toString() : e.getMessage(), e); - } - } - - @Override - protected CloseableIterator> _iterator(boolean includeRowHash) - { - return _loader.iterator(); - } - - @Override - public void close() - { - _loader.iterator().close(); - } - - - private static class GenericFastaLoader extends FastaLoader> - { - public GenericFastaLoader(FileLike fastaFile) - { - super(fastaFile, new FastaIteratorElementFactory<>() - { - @Override - public Map createNext(String header, byte[] body) - { - Map row = new LinkedHashMap<>(); - - parseHeader(header, row); - row.put("sequence", new String(body)); - - return row; - } - - /** - * Attempt to parse out any information from the header, some fasta formats may - * contain key/value pairs in the header that we can use to create field/value pairs. - * Otherwise the default is to return the entire header. - */ - private void parseHeader(String header, Map rowMap) - { - if (header != null) - { - StringBuilder sb = new StringBuilder(); - String delim = ""; - - for (String part : header.split("\\s")) - { - String[] field = part.split("="); - - if (field.length == 2) - { - if (field[0].startsWith("[") && field[1].endsWith("]") || - field[0].startsWith("(") && field[1].endsWith(")")) - rowMap.put(field[0].substring(1), field[1].substring(0, field[1].length()-1)); - else - rowMap.put(field[0], field[1]); - } - else - { - sb.append(delim).append(part); - delim = " "; - } - } - - if (!sb.isEmpty()) - rowMap.put("header", sb.toString()); - } - } - }); - } - - @Override - public @NotNull GenericIterator iterator() - { - return new GenericIterator(); - } - - public class GenericIterator extends FastaIterator implements CloseableIterator> - { - private GenericIterator() - { - super(); - } - } - } -} +/* + * Copyright (c) 2012-2026 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.labkey.api.reader; + +import org.apache.commons.io.IOUtils; +import org.jetbrains.annotations.NotNull; +import org.labkey.api.data.Container; +import org.labkey.api.iterator.CloseableIterator; +import org.labkey.api.util.FileType; +import org.labkey.api.util.FileUtil; +import org.labkey.vfs.FileLike; +import org.labkey.vfs.FileSystemLike; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * User: klum + * Date: Mar 14, 2012 + */ +public class FastaDataLoader extends DataLoader +{ + public static FileType FILE_TYPE = new FileType(Arrays.asList(".fna", ".fasta", ".fa"), ".fna"); + static { + FILE_TYPE.setExtensionsMutuallyExclusive(false); + } + + public static class Factory extends AbstractDataLoaderFactory + { + @NotNull @Override + public DataLoader createLoader(File file, boolean hasColumnHeaders, Container mvIndicatorContainer) throws IOException + { + FastaDataLoader result = new FastaDataLoader(file, hasColumnHeaders, mvIndicatorContainer); + result.setCharacterFilter(new FastaLoader.UpperAndLowercaseCharacterFilter()); + return result; + } + + @NotNull @Override + public DataLoader createLoader(InputStream is, boolean hasColumnHeaders, Container mvIndicatorContainer) throws IOException + { + File f = FileUtil.createTempFile("import", ".fasta"); + f.deleteOnExit(); + IOUtils.copy(is, new FileOutputStream(f)); + + FastaDataLoader result = new FastaDataLoader(f, hasColumnHeaders, mvIndicatorContainer); + result.setCharacterFilter(new FastaLoader.UpperAndLowercaseCharacterFilter()); + return result; + } + + @NotNull @Override + public FileType getFileType() { return FILE_TYPE; } + } + + GenericFastaLoader _loader; + + public FastaDataLoader(File inputFile, Boolean hasColumnHeaders) throws IOException + { + this(inputFile, hasColumnHeaders, null); + } + + public FastaDataLoader(File inputFile, boolean hasColumnHeaders, Container mvIndicatorContainer) throws IOException + { + super(mvIndicatorContainer); + setSource(inputFile); + setScrollable(true); + setHasColumnHeaders(hasColumnHeaders); + + _loader = new GenericFastaLoader(FileSystemLike.wrapFile(inputFile)); + } + + public void setCharacterFilter(FastaLoader.CharacterFilter characterFilter) + { + _loader.setCharacterFilter(characterFilter); + } + + @Override + public String[][] getFirstNLines(int n) throws IOException + { + try (GenericFastaLoader.GenericIterator it = _loader.iterator()) + { + List lineFields = new ArrayList<>(n); + int count = 0; + + while (it.hasNext() && count < n) + { + Map row = it.next(); + + // add the internally generated column header + Set strings = row.keySet(); + if (count == 0) + lineFields.add(strings.toArray(new String[0])); + + Collection values = row.values(); + lineFields.add(values.toArray(new String[0])); + + count++; + } + + if (lineFields.isEmpty()) + return new String[0][]; + + return lineFields.toArray(new String[count][]); + } + catch (IllegalArgumentException e) + { + throw new IOException(e.getMessage() == null ? e.toString() : e.getMessage(), e); + } + } + + @Override + protected CloseableIterator> _iterator(boolean includeRowHash) + { + return _loader.iterator(); + } + + @Override + public void close() + { + _loader.iterator().close(); + } + + + private static class GenericFastaLoader extends FastaLoader> + { + public GenericFastaLoader(FileLike fastaFile) + { + super(fastaFile, new FastaIteratorElementFactory<>() + { + @Override + public Map createNext(String header, byte[] body) + { + Map row = new LinkedHashMap<>(); + + parseHeader(header, row); + row.put("sequence", new String(body)); + + return row; + } + + /** + * Attempt to parse out any information from the header, some fasta formats may + * contain key/value pairs in the header that we can use to create field/value pairs. + * Otherwise the default is to return the entire header. + */ + private void parseHeader(String header, Map rowMap) + { + if (header != null) + { + StringBuilder sb = new StringBuilder(); + String delim = ""; + + for (String part : header.split("\\s")) + { + String[] field = part.split("="); + + if (field.length == 2) + { + if (field[0].startsWith("[") && field[1].endsWith("]") || + field[0].startsWith("(") && field[1].endsWith(")")) + rowMap.put(field[0].substring(1), field[1].substring(0, field[1].length()-1)); + else + rowMap.put(field[0], field[1]); + } + else + { + sb.append(delim).append(part); + delim = " "; + } + } + + if (!sb.isEmpty()) + rowMap.put("header", sb.toString()); + } + } + }); + } + + @Override + public @NotNull GenericIterator iterator() + { + return new GenericIterator(); + } + + public class GenericIterator extends FastaIterator implements CloseableIterator> + { + private GenericIterator() + { + super(); + } + } + } +} From b64eb38f0607081827bac1473208866c9cee0ff4 Mon Sep 17 00:00:00 2001 From: Marty Pradere Date: Sun, 28 Jun 2026 22:18:50 -0600 Subject: [PATCH 02/10] Fix LovCombo dropping string selections under native RegExp.escape (#7796) ## Rationale Newer browsers (Chrome, Firefox 140+) ship a native `RegExp.escape` that throws a `TypeError` on non-String input. `Ext.ux.form.LovCombo.setValue()` passed `JSON.stringify(value)` to `RegExp.escape` to avoid that crash, but `JSON.stringify()` quotes string values, so they no longer matched the raw, unquoted output of `getCheckedValue()` and every selection in a string-valued combo (e.g. the WNPRC EHR Time of Day multi-select) was silently dropped. Using `String()` yields a valid String for `RegExp.escape` without quoting string values, fixing both the numeric-`valueField` crash and the string-`valueField` regression. See https://github.com/LabKey/internal-issues/issues/1266. ## Related Pull Requests - https://github.com/LabKey/wnprc-modules/pull/982 - https://github.com/LabKey/premiumModules/pull/638 ## Changes - `core/webapp/Ext.ux.form.LovCombo.js`: in `setValue()`, pass `String(...)` rather than `JSON.stringify(...)` to `RegExp.escape`. --- core/webapp/Ext.ux.form.LovCombo.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/core/webapp/Ext.ux.form.LovCombo.js b/core/webapp/Ext.ux.form.LovCombo.js index 88bd0f0011c..5bf01396cb2 100644 --- a/core/webapp/Ext.ux.form.LovCombo.js +++ b/core/webapp/Ext.ux.form.LovCombo.js @@ -293,8 +293,10 @@ Ext.ux.form.LovCombo = Ext.extend(Ext.form.ComboBox, { this.store.suspendEvents(true); this.store.clearFilter(); this.store.each(function (r) { + // GH Issue 1266: String() (not JSON.stringify) so native RegExp.escape doesn't throw on a + // numeric valueField, while string values stay unquoted to match getCheckedValue() output. var checked = !(!v.match( - '(^|' + this.separator + ')' + RegExp.escape(JSON.stringify(r.get(this.valueField))) + '(^|' + this.separator + ')' + RegExp.escape(String(r.get(this.valueField))) + '(' + this.separator + '|$)')); r.set(this.checkField, checked); }, this); From fc068216adf6314799bd7d9b464492fd8c02bd60 Mon Sep 17 00:00:00 2001 From: Adam Rauch Date: Mon, 29 Jun 2026 14:06:38 -0700 Subject: [PATCH 03/10] Fix field error handling in BeanViewForm (#7799) --- api/src/org/labkey/api/data/BeanViewForm.java | 10 ++++++++++ api/src/org/labkey/api/data/TableViewForm.java | 9 +++++++-- api/src/org/labkey/api/query/UserSchemaAction.java | 3 +-- .../labkey/study/pipeline/FileAnalysisDatasetTask.java | 3 +-- 4 files changed, 19 insertions(+), 6 deletions(-) diff --git a/api/src/org/labkey/api/data/BeanViewForm.java b/api/src/org/labkey/api/data/BeanViewForm.java index a5a0bc33c97..2212ac5b0ee 100644 --- a/api/src/org/labkey/api/data/BeanViewForm.java +++ b/api/src/org/labkey/api/data/BeanViewForm.java @@ -20,7 +20,10 @@ import org.apache.commons.beanutils.ConvertUtils; import org.apache.commons.beanutils.DynaBean; import org.apache.commons.beanutils.DynaClass; +import org.labkey.api.action.BaseViewAction.BeanUtilsPropertyBindingResult; import org.labkey.api.action.HasBindParameters; +import org.labkey.api.action.NullSafeBindException; +import org.springframework.validation.BindException; import java.util.HashMap; import java.util.Map; @@ -86,6 +89,13 @@ public void setBean(K bean) setTypedValues(factory.toMap(bean, null), false); } + @Override + public BindException createErrors() + { + // Teaches Spring to resolve field names via string lookups instead of getters. e.g., errors.rejectValue(). + return new NullSafeBindException(new BeanUtilsPropertyBindingResult(this, "form")); + } + @Override public Map getValuesToBind() { diff --git a/api/src/org/labkey/api/data/TableViewForm.java b/api/src/org/labkey/api/data/TableViewForm.java index 971235ee5de..aa14b6ffadf 100644 --- a/api/src/org/labkey/api/data/TableViewForm.java +++ b/api/src/org/labkey/api/data/TableViewForm.java @@ -30,7 +30,6 @@ import org.labkey.api.action.NullSafeBindException; import org.labkey.api.action.SpringActionController; import org.labkey.api.collections.CaseInsensitiveHashMap; -import org.labkey.api.ontology.Quantity; import org.labkey.api.query.FieldKey; import org.labkey.api.security.permissions.DeletePermission; import org.labkey.api.security.permissions.InsertPermission; @@ -890,10 +889,16 @@ else if (orig.getName().startsWith(ARRAY_MARKER) && orig.getValue()!=null) setValueToBind(pv.getName(), pv.getValue()); } - BindException errors = new NullSafeBindException(this, "form"); + BindException errors = createErrors(); validateBind(errors); return errors; } + + // Construct and return a BindException that's appropriate for this form + public BindException createErrors() + { + return new NullSafeBindException(this, "form"); + } } diff --git a/api/src/org/labkey/api/query/UserSchemaAction.java b/api/src/org/labkey/api/query/UserSchemaAction.java index 4eff280dbfd..44f510f866c 100644 --- a/api/src/org/labkey/api/query/UserSchemaAction.java +++ b/api/src/org/labkey/api/query/UserSchemaAction.java @@ -17,7 +17,6 @@ import org.jetbrains.annotations.Nullable; import org.labkey.api.action.FormViewAction; -import org.labkey.api.action.NullSafeBindException; import org.labkey.api.action.SpringActionController; import org.labkey.api.attachments.SpringAttachmentFile; import org.labkey.api.audit.TransactionAuditProvider; @@ -79,7 +78,7 @@ public BindException bindParameters(PropertyValues m) throws Exception QueryUpdateForm command = new QueryUpdateForm(_table, getViewContext(), null); if (command.isBulkUpdate()) command.setValidateRequired(false); - BindException errors = new NullSafeBindException(new BeanUtilsPropertyBindingResult(command, "form")); + BindException errors = command.createErrors(); command.validateBind(errors); return errors; } diff --git a/study/src/org/labkey/study/pipeline/FileAnalysisDatasetTask.java b/study/src/org/labkey/study/pipeline/FileAnalysisDatasetTask.java index ffe83ad0968..86987f45eae 100644 --- a/study/src/org/labkey/study/pipeline/FileAnalysisDatasetTask.java +++ b/study/src/org/labkey/study/pipeline/FileAnalysisDatasetTask.java @@ -17,7 +17,6 @@ import org.apache.commons.lang3.math.NumberUtils; import org.jetbrains.annotations.NotNull; -import org.labkey.api.action.BaseViewAction; import org.labkey.api.action.NullSafeBindException; import org.labkey.api.admin.PipelineJobLoggerGetter; import org.labkey.api.assay.transform.DataTransformService; @@ -140,7 +139,7 @@ else if (params.containsKey(DataTransformService.ORIGINAL_SOURCE_PATH)) for (String error : readerErrors) _ctx.getLogger().error(error); - BindException errors = new NullSafeBindException(new BaseViewAction.BeanUtilsPropertyBindingResult(this, "pipeline")); + BindException errors = new NullSafeBindException(this, "pipeline"); boolean allowDomainUpdates = true; if (_ctx.getProperties().containsKey(StudyImportContext.ALLOW_DOMAIN_UPDATES)) allowDomainUpdates = Boolean.parseBoolean(_ctx.getProperties().get(StudyImportContext.ALLOW_DOMAIN_UPDATES)); From a292db3996487e013ee6e4b44c5b85b1681ebccc Mon Sep 17 00:00:00 2001 From: XingY Date: Mon, 6 Jul 2026 12:08:31 -0700 Subject: [PATCH 04/10] feature verification changes --- .../pipeline/AppPipelineJobNotificationProvider.java | 4 ++-- .../org/labkey/api/query/QueryImportPipelineJob.java | 11 +++++++++++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/api/src/org/labkey/api/pipeline/AppPipelineJobNotificationProvider.java b/api/src/org/labkey/api/pipeline/AppPipelineJobNotificationProvider.java index e6b305b0c36..da8e549c16f 100644 --- a/api/src/org/labkey/api/pipeline/AppPipelineJobNotificationProvider.java +++ b/api/src/org/labkey/api/pipeline/AppPipelineJobNotificationProvider.java @@ -182,7 +182,7 @@ private String getJobSuccessMsg(PipelineJob job, @NotNull ImportType importType, .append(!crossTypeImport ? type + " " : "") .append(importType.name()) .append(" from ") - .append(queryImportPipelineJob.getImportContextBuilder().getPrimaryFile().getName()); + .append(queryImportPipelineJob.getImportSourceDescription()); if (job.getContainer().hasProductFolders()) { @@ -224,7 +224,7 @@ private String getJobErrorMsg(PipelineJob job, String rawErrorMsg) return "Failed to import " + type + " from " + - queryImportPipelineJob.getImportContextBuilder().getPrimaryFile().getName() + + queryImportPipelineJob.getImportSourceDescription() + "\n" + rawErrorMsg;// resolveErrorMessage on client } diff --git a/api/src/org/labkey/api/query/QueryImportPipelineJob.java b/api/src/org/labkey/api/query/QueryImportPipelineJob.java index 91672dd36dd..3a3746f3a98 100644 --- a/api/src/org/labkey/api/query/QueryImportPipelineJob.java +++ b/api/src/org/labkey/api/query/QueryImportPipelineJob.java @@ -441,4 +441,15 @@ public Map getAdditionalJobResponseInfo() return _additionalJobResponseInfo; } + /** + * Human-readable description of the import source, used in job success/error notification messages + * (e.g. "imported ... from <this>"). Defaults to the primary file name; subclasses that import multiple + * files may override (e.g. "3 files"). + */ + public String getImportSourceDescription() + { + FileLike primaryFile = _importContextBuilder == null ? null : _importContextBuilder.getPrimaryFile(); + return primaryFile != null ? primaryFile.getName() : ""; + } + } From 0ec5bb5237477a46d85de0db91bf38fa4c62ffce Mon Sep 17 00:00:00 2001 From: XingY Date: Tue, 7 Jul 2026 14:00:00 -0700 Subject: [PATCH 05/10] crlf --- .../api/query/QueryImportPipelineJob.java | 5 - .../labkey/api/reader/FastaDataLoader.java | 428 +++++++++--------- 2 files changed, 214 insertions(+), 219 deletions(-) diff --git a/api/src/org/labkey/api/query/QueryImportPipelineJob.java b/api/src/org/labkey/api/query/QueryImportPipelineJob.java index 3a3746f3a98..026614e872e 100644 --- a/api/src/org/labkey/api/query/QueryImportPipelineJob.java +++ b/api/src/org/labkey/api/query/QueryImportPipelineJob.java @@ -441,11 +441,6 @@ public Map getAdditionalJobResponseInfo() return _additionalJobResponseInfo; } - /** - * Human-readable description of the import source, used in job success/error notification messages - * (e.g. "imported ... from <this>"). Defaults to the primary file name; subclasses that import multiple - * files may override (e.g. "3 files"). - */ public String getImportSourceDescription() { FileLike primaryFile = _importContextBuilder == null ? null : _importContextBuilder.getPrimaryFile(); diff --git a/api/src/org/labkey/api/reader/FastaDataLoader.java b/api/src/org/labkey/api/reader/FastaDataLoader.java index 8a2ebbc9674..c6272d45ebb 100644 --- a/api/src/org/labkey/api/reader/FastaDataLoader.java +++ b/api/src/org/labkey/api/reader/FastaDataLoader.java @@ -1,214 +1,214 @@ -/* - * Copyright (c) 2012-2026 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.labkey.api.reader; - -import org.apache.commons.io.IOUtils; -import org.jetbrains.annotations.NotNull; -import org.labkey.api.data.Container; -import org.labkey.api.iterator.CloseableIterator; -import org.labkey.api.util.FileType; -import org.labkey.api.util.FileUtil; -import org.labkey.vfs.FileLike; -import org.labkey.vfs.FileSystemLike; - -import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; - -/** - * User: klum - * Date: Mar 14, 2012 - */ -public class FastaDataLoader extends DataLoader -{ - public static FileType FILE_TYPE = new FileType(Arrays.asList(".fna", ".fasta", ".fa"), ".fna"); - static { - FILE_TYPE.setExtensionsMutuallyExclusive(false); - } - - public static class Factory extends AbstractDataLoaderFactory - { - @NotNull @Override - public DataLoader createLoader(File file, boolean hasColumnHeaders, Container mvIndicatorContainer) throws IOException - { - FastaDataLoader result = new FastaDataLoader(file, hasColumnHeaders, mvIndicatorContainer); - result.setCharacterFilter(new FastaLoader.UpperAndLowercaseCharacterFilter()); - return result; - } - - @NotNull @Override - public DataLoader createLoader(InputStream is, boolean hasColumnHeaders, Container mvIndicatorContainer) throws IOException - { - File f = FileUtil.createTempFile("import", ".fasta"); - f.deleteOnExit(); - IOUtils.copy(is, new FileOutputStream(f)); - - FastaDataLoader result = new FastaDataLoader(f, hasColumnHeaders, mvIndicatorContainer); - result.setCharacterFilter(new FastaLoader.UpperAndLowercaseCharacterFilter()); - return result; - } - - @NotNull @Override - public FileType getFileType() { return FILE_TYPE; } - } - - GenericFastaLoader _loader; - - public FastaDataLoader(File inputFile, Boolean hasColumnHeaders) throws IOException - { - this(inputFile, hasColumnHeaders, null); - } - - public FastaDataLoader(File inputFile, boolean hasColumnHeaders, Container mvIndicatorContainer) throws IOException - { - super(mvIndicatorContainer); - setSource(inputFile); - setScrollable(true); - setHasColumnHeaders(hasColumnHeaders); - - _loader = new GenericFastaLoader(FileSystemLike.wrapFile(inputFile)); - } - - public void setCharacterFilter(FastaLoader.CharacterFilter characterFilter) - { - _loader.setCharacterFilter(characterFilter); - } - - @Override - public String[][] getFirstNLines(int n) throws IOException - { - try (GenericFastaLoader.GenericIterator it = _loader.iterator()) - { - List lineFields = new ArrayList<>(n); - int count = 0; - - while (it.hasNext() && count < n) - { - Map row = it.next(); - - // add the internally generated column header - Set strings = row.keySet(); - if (count == 0) - lineFields.add(strings.toArray(new String[0])); - - Collection values = row.values(); - lineFields.add(values.toArray(new String[0])); - - count++; - } - - if (lineFields.isEmpty()) - return new String[0][]; - - return lineFields.toArray(new String[count][]); - } - catch (IllegalArgumentException e) - { - throw new IOException(e.getMessage() == null ? e.toString() : e.getMessage(), e); - } - } - - @Override - protected CloseableIterator> _iterator(boolean includeRowHash) - { - return _loader.iterator(); - } - - @Override - public void close() - { - _loader.iterator().close(); - } - - - private static class GenericFastaLoader extends FastaLoader> - { - public GenericFastaLoader(FileLike fastaFile) - { - super(fastaFile, new FastaIteratorElementFactory<>() - { - @Override - public Map createNext(String header, byte[] body) - { - Map row = new LinkedHashMap<>(); - - parseHeader(header, row); - row.put("sequence", new String(body)); - - return row; - } - - /** - * Attempt to parse out any information from the header, some fasta formats may - * contain key/value pairs in the header that we can use to create field/value pairs. - * Otherwise the default is to return the entire header. - */ - private void parseHeader(String header, Map rowMap) - { - if (header != null) - { - StringBuilder sb = new StringBuilder(); - String delim = ""; - - for (String part : header.split("\\s")) - { - String[] field = part.split("="); - - if (field.length == 2) - { - if (field[0].startsWith("[") && field[1].endsWith("]") || - field[0].startsWith("(") && field[1].endsWith(")")) - rowMap.put(field[0].substring(1), field[1].substring(0, field[1].length()-1)); - else - rowMap.put(field[0], field[1]); - } - else - { - sb.append(delim).append(part); - delim = " "; - } - } - - if (!sb.isEmpty()) - rowMap.put("header", sb.toString()); - } - } - }); - } - - @Override - public @NotNull GenericIterator iterator() - { - return new GenericIterator(); - } - - public class GenericIterator extends FastaIterator implements CloseableIterator> - { - private GenericIterator() - { - super(); - } - } - } -} +/* + * Copyright (c) 2012-2026 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.labkey.api.reader; + +import org.apache.commons.io.IOUtils; +import org.jetbrains.annotations.NotNull; +import org.labkey.api.data.Container; +import org.labkey.api.iterator.CloseableIterator; +import org.labkey.api.util.FileType; +import org.labkey.api.util.FileUtil; +import org.labkey.vfs.FileLike; +import org.labkey.vfs.FileSystemLike; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * User: klum + * Date: Mar 14, 2012 + */ +public class FastaDataLoader extends DataLoader +{ + public static FileType FILE_TYPE = new FileType(Arrays.asList(".fna", ".fasta", ".fa"), ".fna"); + static { + FILE_TYPE.setExtensionsMutuallyExclusive(false); + } + + public static class Factory extends AbstractDataLoaderFactory + { + @NotNull @Override + public DataLoader createLoader(File file, boolean hasColumnHeaders, Container mvIndicatorContainer) throws IOException + { + FastaDataLoader result = new FastaDataLoader(file, hasColumnHeaders, mvIndicatorContainer); + result.setCharacterFilter(new FastaLoader.UpperAndLowercaseCharacterFilter()); + return result; + } + + @NotNull @Override + public DataLoader createLoader(InputStream is, boolean hasColumnHeaders, Container mvIndicatorContainer) throws IOException + { + File f = FileUtil.createTempFile("import", ".fasta"); + f.deleteOnExit(); + IOUtils.copy(is, new FileOutputStream(f)); + + FastaDataLoader result = new FastaDataLoader(f, hasColumnHeaders, mvIndicatorContainer); + result.setCharacterFilter(new FastaLoader.UpperAndLowercaseCharacterFilter()); + return result; + } + + @NotNull @Override + public FileType getFileType() { return FILE_TYPE; } + } + + GenericFastaLoader _loader; + + public FastaDataLoader(File inputFile, Boolean hasColumnHeaders) throws IOException + { + this(inputFile, hasColumnHeaders, null); + } + + public FastaDataLoader(File inputFile, boolean hasColumnHeaders, Container mvIndicatorContainer) throws IOException + { + super(mvIndicatorContainer); + setSource(inputFile); + setScrollable(true); + setHasColumnHeaders(hasColumnHeaders); + + _loader = new GenericFastaLoader(FileSystemLike.wrapFile(inputFile)); + } + + public void setCharacterFilter(FastaLoader.CharacterFilter characterFilter) + { + _loader.setCharacterFilter(characterFilter); + } + + @Override + public String[][] getFirstNLines(int n) throws IOException + { + try (GenericFastaLoader.GenericIterator it = _loader.iterator()) + { + List lineFields = new ArrayList<>(n); + int count = 0; + + while (it.hasNext() && count < n) + { + Map row = it.next(); + + // add the internally generated column header + Set strings = row.keySet(); + if (count == 0) + lineFields.add(strings.toArray(new String[0])); + + Collection values = row.values(); + lineFields.add(values.toArray(new String[0])); + + count++; + } + + if (lineFields.isEmpty()) + return new String[0][]; + + return lineFields.toArray(new String[count][]); + } + catch (IllegalArgumentException e) + { + throw new IOException(e.getMessage() == null ? e.toString() : e.getMessage(), e); + } + } + + @Override + protected CloseableIterator> _iterator(boolean includeRowHash) + { + return _loader.iterator(); + } + + @Override + public void close() + { + _loader.iterator().close(); + } + + + private static class GenericFastaLoader extends FastaLoader> + { + public GenericFastaLoader(FileLike fastaFile) + { + super(fastaFile, new FastaIteratorElementFactory<>() + { + @Override + public Map createNext(String header, byte[] body) + { + Map row = new LinkedHashMap<>(); + + parseHeader(header, row); + row.put("sequence", new String(body)); + + return row; + } + + /** + * Attempt to parse out any information from the header, some fasta formats may + * contain key/value pairs in the header that we can use to create field/value pairs. + * Otherwise the default is to return the entire header. + */ + private void parseHeader(String header, Map rowMap) + { + if (header != null) + { + StringBuilder sb = new StringBuilder(); + String delim = ""; + + for (String part : header.split("\\s")) + { + String[] field = part.split("="); + + if (field.length == 2) + { + if (field[0].startsWith("[") && field[1].endsWith("]") || + field[0].startsWith("(") && field[1].endsWith(")")) + rowMap.put(field[0].substring(1), field[1].substring(0, field[1].length()-1)); + else + rowMap.put(field[0], field[1]); + } + else + { + sb.append(delim).append(part); + delim = " "; + } + } + + if (!sb.isEmpty()) + rowMap.put("header", sb.toString()); + } + } + }); + } + + @Override + public @NotNull GenericIterator iterator() + { + return new GenericIterator(); + } + + public class GenericIterator extends FastaIterator implements CloseableIterator> + { + private GenericIterator() + { + super(); + } + } + } +} From eb1ee61721271080dba88950c28b6295d440059a Mon Sep 17 00:00:00 2001 From: XingY Date: Tue, 7 Jul 2026 16:42:45 -0700 Subject: [PATCH 06/10] refactor import --- .../api/query/AbstractQueryImportAction.java | 41 ++++++++++++++----- 1 file changed, 31 insertions(+), 10 deletions(-) diff --git a/api/src/org/labkey/api/query/AbstractQueryImportAction.java b/api/src/org/labkey/api/query/AbstractQueryImportAction.java index d78c859fe93..09b5ecde19b 100644 --- a/api/src/org/labkey/api/query/AbstractQueryImportAction.java +++ b/api/src/org/labkey/api/query/AbstractQueryImportAction.java @@ -16,6 +16,7 @@ package org.labkey.api.query; import jakarta.servlet.ServletException; +import jakarta.servlet.http.HttpServletRequest; import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.Logger; import org.jetbrains.annotations.NotNull; @@ -325,27 +326,47 @@ protected String getParam(Params p) protected Map getOptionParamsMap() { if (_optionParamsMap == null) - { - _optionParamsMap = new HashMap<>(); - _optionParamsMap.put(Params.importIdentity, Boolean.valueOf(getParam(Params.importIdentity))); - _optionParamsMap.put(Params.crossTypeImport, Boolean.valueOf(getParam(Params.crossTypeImport))); - _optionParamsMap.put(Params.allowCreateStorage, Boolean.valueOf(getParam(Params.allowCreateStorage))); - _optionParamsMap.put(Params.useTransactionAuditCache, Boolean.valueOf(getParam(Params.useTransactionAuditCache))); - } + _optionParamsMap = parseOptionParamsMap(getViewContext().getRequest()); return _optionParamsMap; } + /** + * Builds the standard import option flags ({@link Params#importIdentity}, {@link Params#crossTypeImport}, + * {@link Params#allowCreateStorage}, {@link Params#useTransactionAuditCache}) from request parameters. Exposed as a + * static so actions that do not go through the {@code final} single-file {@code _execute()} flow (e.g. multi-file + * importers) can produce the same map. + */ + public static Map parseOptionParamsMap(HttpServletRequest request) + { + Map map = new HashMap<>(); + map.put(Params.importIdentity, Boolean.valueOf(request.getParameter(Params.importIdentity.name()))); + map.put(Params.crossTypeImport, Boolean.valueOf(request.getParameter(Params.crossTypeImport.name()))); + map.put(Params.allowCreateStorage, Boolean.valueOf(request.getParameter(Params.allowCreateStorage.name()))); + map.put(Params.useTransactionAuditCache, Boolean.valueOf(request.getParameter(Params.useTransactionAuditCache.name()))); + return map; + } + protected Set getTransactionImportParams(String insertOption, boolean useAsync) + { + return parseTransactionImportParams(getViewContext().getRequest(), insertOption, useAsync); + } + + /** + * Builds the {@link TransactionAuditProvider.TransactionDetail#ImportOptions} set from request parameters. Exposed + * as a static so actions that do not go through the {@code final} single-file {@code _execute()} flow (e.g. + * multi-file importers) can record the same import options on their transaction audit event. + */ + public static Set parseTransactionImportParams(HttpServletRequest request, String insertOption, boolean useAsync) { Set importParams = new TreeSet<>(); importParams.add(insertOption); if (useAsync) importParams.add("backgroundImport"); - if (Boolean.valueOf(getParam(Params.crossTypeImport))) + if (Boolean.valueOf(request.getParameter(Params.crossTypeImport.name()))) importParams.add(Params.crossTypeImport.name()); - if (Boolean.valueOf(getParam(Params.useTransactionAuditCache))) + if (Boolean.valueOf(request.getParameter(Params.useTransactionAuditCache.name()))) importParams.add(Params.useTransactionAuditCache.name()); - if (Boolean.valueOf(getParam(Params.allowCreateStorage))) + if (Boolean.valueOf(request.getParameter(Params.allowCreateStorage.name()))) importParams.add(Params.allowCreateStorage.name()); return importParams; } From dd14fb6e89d034c399c90c21c3b7633363b98f77 Mon Sep 17 00:00:00 2001 From: XingY Date: Tue, 7 Jul 2026 19:34:50 -0700 Subject: [PATCH 07/10] refactor to extend query import action --- .../api/query/AbstractQueryImportAction.java | 174 ++++++++++++------ 1 file changed, 118 insertions(+), 56 deletions(-) diff --git a/api/src/org/labkey/api/query/AbstractQueryImportAction.java b/api/src/org/labkey/api/query/AbstractQueryImportAction.java index 09b5ecde19b..e12606eb132 100644 --- a/api/src/org/labkey/api/query/AbstractQueryImportAction.java +++ b/api/src/org/labkey/api/query/AbstractQueryImportAction.java @@ -46,6 +46,7 @@ import org.labkey.api.module.Module; import org.labkey.api.module.ModuleLoader; import org.labkey.api.pipeline.PipeRoot; +import org.labkey.api.pipeline.PipelineJob; import org.labkey.api.pipeline.PipelineService; import org.labkey.api.reader.ColumnDescriptor; import org.labkey.api.reader.DataLoader; @@ -72,6 +73,7 @@ import org.labkey.api.webdav.WebdavService; import org.labkey.api.workflow.WorkflowService; import org.labkey.vfs.FileLike; +import org.springframework.util.MultiValueMap; import org.springframework.validation.BindException; import org.springframework.web.multipart.MultipartFile; import org.springframework.web.multipart.MultipartHttpServletRequest; @@ -456,7 +458,7 @@ public final ApiResponse _execute(FORM form, BindException errors) throws Except if (errors.hasErrors()) throw errors; - FileLike dataFile = null; + List dataFiles = new ArrayList<>(); boolean hasPostData = false; FileStream file = null; String originalName = null; @@ -568,17 +570,19 @@ else if (getViewContext().getRequest() instanceof MultipartHttpServletRequest) { Map files = getFileMap(); MultipartFile multipartfile = null==files ? null : files.get("file"); - if (null != multipartfile && multipartfile.getSize() > 0) + List uploadedFiles = getUploadedFiles(); + if (!uploadedFiles.isEmpty()) { transactionDetails.put(TransactionAuditProvider.TransactionDetail.ImportFileName, multipartfile.getOriginalFilename()); hasPostData = true; - originalName = multipartfile.getOriginalFilename(); - // can't read the multipart file twice so create temp file (12800) - dataFile = FileUtil.createTempFileLike("~upload", multipartfile.getOriginalFilename()); PipeRoot root = PipelineService.get().findPipelineRoot(getContainer()); + originalName = uploadedFiles.getFirst().getOriginalFilename(); + + FileLike dataFileDir = null; + if (null != root && (Boolean.parseBoolean(saveToPipeline) || _useAsync)) { - FileLike dataFileDir = root.getRootFileLike().resolveChild("QueryImportFiles"); + dataFileDir = root.getRootFileLike().resolveChild("QueryImportFiles"); if (dataFileDir.isFile()) { dataFileDir = FileUtil.findUniqueFileName("QueryImportFiles", root.getRootFileLike()); @@ -592,14 +596,24 @@ else if (!dataFileDir.exists()) throw new RuntimeException("Error attempting to create directory " + dataFileDir + " for uploaded query import file " + multipartfile.getOriginalFilename()); } + } + + for (MultipartFile uploadedFile : uploadedFiles) + { + // can't read the multipart file twice so create temp file (12800) + FileLike uploaded = FileUtil.createTempFileLike("~upload", uploadedFile.getOriginalFilename()); + + if (dataFileDir != null) + uploaded = FileUtil.findUniqueFileName(uploadedFile.getOriginalFilename(), dataFileDir); - dataFile = FileUtil.findUniqueFileName(multipartfile.getOriginalFilename(), dataFileDir); + uploadedFile.transferTo(uploaded.toNioPathForWrite()); + dataFiles.add(uploaded); } - multipartfile.transferTo(dataFile.toNioPathForWrite()); + if (_useAsync) { - if (!isBackgroundImportSupported(dataFile.getName())) + if (!isBackgroundImportSupported(dataFiles.getFirst().getName())) throw new RuntimeException("Importing in background currently is not supported for this table"); ViewBackgroundInfo info = new ViewBackgroundInfo(getContainer(), getUser(), new ActionURL()); @@ -607,34 +621,10 @@ else if (!dataFileDir.exists()) UserSchema schema = getTargetSchema(); if (schema != null) { - String schemaName = schema.getSchemaName(); - String queryName = getPipelineTargetQueryName(); - - QueryImportPipelineJob.QueryImportAsyncContextBuilder importContextBuilder = new QueryImportPipelineJob.QueryImportAsyncContextBuilder(); - importContextBuilder - .setPrimaryFile(dataFile) - .setHasColumnHeaders(_hasColumnHeaders) - .setFileContentType(multipartfile.getContentType()) - .setSchemaName(schemaName) - .setQueryName(queryName) - .setRenamedColumns(getRenamedColumns()) - .setLineageImportAliases(getLineageImportAliases()) - .setInsertOption(_insertOption) - .setAuditBehaviorType(behaviorType) - .setAuditUserComment(_auditUserComment) - .setOptionParamsMap(getOptionParamsMap()) - .setLookupResolutionType(getLookupResolutionType()) - .setAllowLineageColumns(allowLineageColumns()) - .setJobDescription(getQueryImportDescription()) - .setJobNotificationProvider(getQueryImportJobNotificationProviderName()); - if (WorkflowService.get() != null) - { - Map workflowParams = WorkflowService.get().getConfigParameters(getViewContext().getRequest()); - importContextBuilder.setWorkflowParams(workflowParams); - } - - importContextBuilder.setTransactionDetails(transactionDetails); - QueryImportPipelineJob job = new QueryImportPipelineJob(getQueryImportProviderName(), info, root, importContextBuilder); + QueryImportPipelineJob.QueryImportAsyncContextBuilder importContextBuilder = getBackgroundImportContext(dataFiles.getFirst(), multipartfile, behaviorType, transactionDetails); + + PipelineJob job = getImportPipelineJob(dataFiles, info, root, importContextBuilder); + PipelineService.get().queueJob(job, getQueryImportJobNotificationProviderName()); if (_target != null) @@ -647,8 +637,9 @@ else if (!dataFileDir.exists()) } - loader = DataLoader.get().createLoader(dataFile.toNioPathForRead().toFile(), multipartfile.getContentType(), _hasColumnHeaders, null, null); - file = new FileAttachmentFile(dataFile, multipartfile.getOriginalFilename()); + Pair loadedFile = prepareDataFileLoader(dataFiles.getFirst(), uploadedFiles.getFirst()); + loader = loadedFile.first; + file = loadedFile.second; } } @@ -657,25 +648,11 @@ else if (!dataFileDir.exists()) if (errors.hasErrors()) throw errors; - BatchValidationException ve = new BatchValidationException(); - //di = wrap(di, ve); - //importData(di, ve); - - configureLoader(loader, _target, getRenamedColumns(), allowLineageColumns(), getLineageImportAliases(), getOptionParamsMap()); - TransactionAuditProvider.TransactionAuditEvent auditEvent = null; if (isCrossTypeImport || (behaviorType != null && behaviorType != AuditBehaviorType.NONE)) auditEvent = createTransactionAuditEvent(getContainer(), _insertOption.auditAction, transactionDetails); - int rowCount = importData(loader, file, originalName, ve, behaviorType, auditEvent, _auditUserComment); - - if (ve.hasErrors()) - { - addImportValidationErrorMetric(_insertOption, _target, (form instanceof QueryForm qf) ? qf.getQueryName() : null); - throw ve; - } - - JSONObject response = createSuccessResponse(rowCount); + JSONObject response = handleImportData(loader, dataFiles, file, originalName, behaviorType, auditEvent, (form instanceof QueryForm qf ? qf.getQueryName() : null)); if (auditEvent != null) { response.put("transactionAuditId", auditEvent.getRowId()); @@ -696,12 +673,97 @@ else if (!dataFileDir.exists()) loader.close(); if (null != file) file.closeInputStream(); - if (null != dataFile && !Boolean.parseBoolean(saveToPipeline) && !_useAsync) - dataFile.delete(); + if (dataFiles.isEmpty() && !Boolean.parseBoolean(saveToPipeline) && !_useAsync) + { + for (FileLike tmpFile : dataFiles) + tmpFile.delete(); + } + } } + protected Pair prepareDataFileLoader(FileLike dataFile, MultipartFile multipartFile) throws IOException + { + DataLoader loader = DataLoader.get().createLoader(dataFile.toNioPathForRead().toFile(), multipartFile.getContentType(), _hasColumnHeaders, null, null); + FileStream file = new FileAttachmentFile(dataFile, multipartFile.getOriginalFilename()); + return new Pair<>(loader, file); + } + + protected JSONObject handleImportData(DataLoader loader, List dataFiles, FileStream file, String originalName, AuditBehaviorType behaviorType, + TransactionAuditProvider.TransactionAuditEvent auditEvent, String queryName) throws BatchValidationException, IOException + { + BatchValidationException ve = new BatchValidationException(); + //di = wrap(di, ve); + //importData(di, ve); + + configureLoader(loader, _target, getRenamedColumns(), allowLineageColumns(), getLineageImportAliases(), getOptionParamsMap()); + + int rowCount = importData(loader, file, originalName, ve, behaviorType, auditEvent, _auditUserComment); + + if (ve.hasErrors()) + { + addImportValidationErrorMetric(_insertOption, _target, queryName); + throw ve; + } + + return createSuccessResponse(rowCount); + } + + private @NotNull List getUploadedFiles() + { + MultiValueMap fileMap = PageFlowUtil.getMultiFileMap(getViewContext().getRequest()); + List files = new ArrayList<>(); + if (fileMap != null && fileMap.get("file") != null) + { + for (MultipartFile mf : fileMap.get("file")) + { + if (mf != null && mf.getSize() > 0) + files.add(mf); + } + } + return files; + } + + protected PipelineJob getImportPipelineJob(List dataFiles, ViewBackgroundInfo info, PipeRoot root, QueryImportPipelineJob.QueryImportAsyncContextBuilder importContextBuilder) + { + return new QueryImportPipelineJob(getQueryImportProviderName(), info, root, importContextBuilder); + } + + protected QueryImportPipelineJob.QueryImportAsyncContextBuilder getBackgroundImportContext(FileLike dataFile, @Nullable MultipartFile multipartfile, AuditBehaviorType behaviorType, Map transactionDetails) throws IOException, ValidationException + { + UserSchema schema = getTargetSchema(); + String schemaName = schema.getSchemaName(); + String queryName = getPipelineTargetQueryName(); + + QueryImportPipelineJob.QueryImportAsyncContextBuilder importContextBuilder = new QueryImportPipelineJob.QueryImportAsyncContextBuilder(); + importContextBuilder + .setPrimaryFile(dataFile) + .setHasColumnHeaders(_hasColumnHeaders) + .setFileContentType(multipartfile != null ? multipartfile.getContentType() : null) + .setSchemaName(schemaName) + .setQueryName(queryName) + .setRenamedColumns(getRenamedColumns()) + .setLineageImportAliases(getLineageImportAliases()) + .setInsertOption(_insertOption) + .setAuditBehaviorType(behaviorType) + .setAuditUserComment(_auditUserComment) + .setOptionParamsMap(getOptionParamsMap()) + .setLookupResolutionType(getLookupResolutionType()) + .setAllowLineageColumns(allowLineageColumns()) + .setJobDescription(getQueryImportDescription()) + .setJobNotificationProvider(getQueryImportJobNotificationProviderName()); + if (WorkflowService.get() != null) + { + Map workflowParams = WorkflowService.get().getConfigParameters(getViewContext().getRequest()); + importContextBuilder.setWorkflowParams(workflowParams); + } + + importContextBuilder.setTransactionDetails(transactionDetails); + + return importContextBuilder; + } + public static void configureLoader(DataLoader loader, @Nullable TableInfo target, @Nullable Map renamedColumns, boolean allowLineageColumns, @Nullable Set lineageAliasNames, @Nullable Map optionParamsMap) throws IOException { // Issue 53804: When updating or adding samples across different sample types, the strings 'Yes' and 'No' are converted to their boolean values From ed995761a57f52625a4c547b97e6c71f46dea222 Mon Sep 17 00:00:00 2001 From: XingY Date: Tue, 7 Jul 2026 19:54:19 -0700 Subject: [PATCH 08/10] clean --- .../api/query/AbstractQueryImportAction.java | 40 +++++-------------- 1 file changed, 10 insertions(+), 30 deletions(-) diff --git a/api/src/org/labkey/api/query/AbstractQueryImportAction.java b/api/src/org/labkey/api/query/AbstractQueryImportAction.java index e12606eb132..53d39926de5 100644 --- a/api/src/org/labkey/api/query/AbstractQueryImportAction.java +++ b/api/src/org/labkey/api/query/AbstractQueryImportAction.java @@ -328,47 +328,27 @@ protected String getParam(Params p) protected Map getOptionParamsMap() { if (_optionParamsMap == null) - _optionParamsMap = parseOptionParamsMap(getViewContext().getRequest()); + { + _optionParamsMap = new HashMap<>(); + _optionParamsMap.put(Params.importIdentity, Boolean.valueOf(getParam(Params.importIdentity))); + _optionParamsMap.put(Params.crossTypeImport, Boolean.valueOf(getParam(Params.crossTypeImport))); + _optionParamsMap.put(Params.allowCreateStorage, Boolean.valueOf(getParam(Params.allowCreateStorage))); + _optionParamsMap.put(Params.useTransactionAuditCache, Boolean.valueOf(getParam(Params.useTransactionAuditCache))); + } return _optionParamsMap; } - /** - * Builds the standard import option flags ({@link Params#importIdentity}, {@link Params#crossTypeImport}, - * {@link Params#allowCreateStorage}, {@link Params#useTransactionAuditCache}) from request parameters. Exposed as a - * static so actions that do not go through the {@code final} single-file {@code _execute()} flow (e.g. multi-file - * importers) can produce the same map. - */ - public static Map parseOptionParamsMap(HttpServletRequest request) - { - Map map = new HashMap<>(); - map.put(Params.importIdentity, Boolean.valueOf(request.getParameter(Params.importIdentity.name()))); - map.put(Params.crossTypeImport, Boolean.valueOf(request.getParameter(Params.crossTypeImport.name()))); - map.put(Params.allowCreateStorage, Boolean.valueOf(request.getParameter(Params.allowCreateStorage.name()))); - map.put(Params.useTransactionAuditCache, Boolean.valueOf(request.getParameter(Params.useTransactionAuditCache.name()))); - return map; - } - protected Set getTransactionImportParams(String insertOption, boolean useAsync) - { - return parseTransactionImportParams(getViewContext().getRequest(), insertOption, useAsync); - } - - /** - * Builds the {@link TransactionAuditProvider.TransactionDetail#ImportOptions} set from request parameters. Exposed - * as a static so actions that do not go through the {@code final} single-file {@code _execute()} flow (e.g. - * multi-file importers) can record the same import options on their transaction audit event. - */ - public static Set parseTransactionImportParams(HttpServletRequest request, String insertOption, boolean useAsync) { Set importParams = new TreeSet<>(); importParams.add(insertOption); if (useAsync) importParams.add("backgroundImport"); - if (Boolean.valueOf(request.getParameter(Params.crossTypeImport.name()))) + if (Boolean.valueOf(getParam(Params.crossTypeImport))) importParams.add(Params.crossTypeImport.name()); - if (Boolean.valueOf(request.getParameter(Params.useTransactionAuditCache.name()))) + if (Boolean.valueOf(getParam(Params.useTransactionAuditCache))) importParams.add(Params.useTransactionAuditCache.name()); - if (Boolean.valueOf(request.getParameter(Params.allowCreateStorage.name()))) + if (Boolean.valueOf(getParam(Params.allowCreateStorage))) importParams.add(Params.allowCreateStorage.name()); return importParams; } From 5c7ec90ef80f96e0f9f17074976b4f4b7e27a77a Mon Sep 17 00:00:00 2001 From: XingY Date: Tue, 7 Jul 2026 20:03:39 -0700 Subject: [PATCH 09/10] clean --- api/src/org/labkey/api/query/AbstractQueryImportAction.java | 1 - 1 file changed, 1 deletion(-) diff --git a/api/src/org/labkey/api/query/AbstractQueryImportAction.java b/api/src/org/labkey/api/query/AbstractQueryImportAction.java index 53d39926de5..bb09d45c691 100644 --- a/api/src/org/labkey/api/query/AbstractQueryImportAction.java +++ b/api/src/org/labkey/api/query/AbstractQueryImportAction.java @@ -16,7 +16,6 @@ package org.labkey.api.query; import jakarta.servlet.ServletException; -import jakarta.servlet.http.HttpServletRequest; import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.Logger; import org.jetbrains.annotations.NotNull; From d8d43865c86f5f434f30c5a6e8dd0507c2ead8ac Mon Sep 17 00:00:00 2001 From: XingY Date: Tue, 7 Jul 2026 20:16:04 -0700 Subject: [PATCH 10/10] CC review --- api/src/org/labkey/api/query/AbstractQueryImportAction.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/src/org/labkey/api/query/AbstractQueryImportAction.java b/api/src/org/labkey/api/query/AbstractQueryImportAction.java index bb09d45c691..800cb031f8c 100644 --- a/api/src/org/labkey/api/query/AbstractQueryImportAction.java +++ b/api/src/org/labkey/api/query/AbstractQueryImportAction.java @@ -552,10 +552,10 @@ else if (getViewContext().getRequest() instanceof MultipartHttpServletRequest) List uploadedFiles = getUploadedFiles(); if (!uploadedFiles.isEmpty()) { - transactionDetails.put(TransactionAuditProvider.TransactionDetail.ImportFileName, multipartfile.getOriginalFilename()); hasPostData = true; PipeRoot root = PipelineService.get().findPipelineRoot(getContainer()); originalName = uploadedFiles.getFirst().getOriginalFilename(); + transactionDetails.put(TransactionAuditProvider.TransactionDetail.ImportFileName, originalName); FileLike dataFileDir = null; @@ -652,7 +652,7 @@ else if (!dataFileDir.exists()) loader.close(); if (null != file) file.closeInputStream(); - if (dataFiles.isEmpty() && !Boolean.parseBoolean(saveToPipeline) && !_useAsync) + if (!dataFiles.isEmpty() && !Boolean.parseBoolean(saveToPipeline) && !_useAsync) { for (FileLike tmpFile : dataFiles) tmpFile.delete();