From 3cf91e7fa46cd835b49051382b6cf7ffaf86767c Mon Sep 17 00:00:00 2001 From: XingY Date: Thu, 2 Apr 2026 22:37:59 -0700 Subject: [PATCH 1/5] GitHub Issue 915: Bulk edit doesn't completely remove attachments for sources --- .../dataiterator/AttachmentDataIterator.java | 49 +- .../ExistingRecordDataIterator.java | 7 +- .../api/query/DefaultQueryUpdateService.java | 1880 ++++++++--------- 3 files changed, 990 insertions(+), 946 deletions(-) diff --git a/api/src/org/labkey/api/dataiterator/AttachmentDataIterator.java b/api/src/org/labkey/api/dataiterator/AttachmentDataIterator.java index 05266bc05fc..e7c4ba2efbe 100644 --- a/api/src/org/labkey/api/dataiterator/AttachmentDataIterator.java +++ b/api/src/org/labkey/api/dataiterator/AttachmentDataIterator.java @@ -42,6 +42,9 @@ import java.io.IOException; import java.io.InputStream; import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; public class AttachmentDataIterator extends WrapperDataIterator { @@ -49,6 +52,7 @@ public class AttachmentDataIterator extends WrapperDataIterator final BatchValidationException errors; final int entityIdIndex; final ArrayList<_AttachmentUploadHelper> attachmentColumns; + final Map attachmentColumnsAliases; final QueryUpdateService.InsertOption insertOption; final User user; final Container container; @@ -60,6 +64,7 @@ public class AttachmentDataIterator extends WrapperDataIterator @Nullable VirtualFile attachmentDir, int entityIdIndex, ArrayList<_AttachmentUploadHelper> attachmentColumns, + Map attachmentColumnsAliases, QueryUpdateService.InsertOption insertOption, Container container, AttachmentParentFactory parentFactory) @@ -69,6 +74,7 @@ public class AttachmentDataIterator extends WrapperDataIterator this.errors = errors; this.entityIdIndex = entityIdIndex; this.attachmentColumns = attachmentColumns; + this.attachmentColumnsAliases = attachmentColumnsAliases; this.insertOption = insertOption; this.user = user; this.container = container; @@ -83,19 +89,41 @@ public boolean next() throws BatchValidationException return false; ArrayList attachmentFiles = null; + List oldAttachments = new ArrayList<>(); + try { + Map existing = getExistingRecord(); + for (_AttachmentUploadHelper p : attachmentColumns) { Object attachmentValue = get(p.index); + String oldAttachmentValue = null; + if (insertOption.allowUpdate && existing != null) + { + // GitHub Issue 915: Bulk edit doesn't completely remove attachments for sources + Object oldValue = existing.get(p.domainProperty.getName()); + if (oldValue == null) + oldValue = existing.get(attachmentColumnsAliases.get(p.domainProperty.getName())); + if (oldValue != null) + oldAttachmentValue = oldValue.toString(); + } + if (null == attachmentValue) + { + if (oldAttachmentValue != null) + oldAttachments.add(oldAttachmentValue); continue; + } String filename; AttachmentFile attachmentFile; if (attachmentValue instanceof String str) { + if (str.equals(oldAttachmentValue)) + continue; + if (null == attachmentDir) { errors.addRowError(propertyValidationException(p.domainProperty, attachmentValue)); @@ -141,11 +169,18 @@ else if (attachmentValue instanceof File file) attachmentFiles.add(attachmentFile); } + if ((null == attachmentFiles || attachmentFiles.isEmpty()) && oldAttachments.isEmpty()) + return ret; + + String entityId = String.valueOf(get(entityIdIndex)); + var attachmentParent = getAttachmentParent(entityId, container); + if (null != attachmentFiles && !attachmentFiles.isEmpty()) - { - String entityId = String.valueOf(get(entityIdIndex)); - AttachmentService.get().addAttachments(getAttachmentParent(entityId, container), attachmentFiles, user); - } + AttachmentService.get().addAttachments(attachmentParent, attachmentFiles, user); + + if (!oldAttachments.isEmpty()) + AttachmentService.get().deleteAttachments(attachmentParent, oldAttachments, user); + return ret; } catch (AttachmentService.DuplicateFilenameException | AttachmentService.FileTooLargeException e) @@ -212,6 +247,7 @@ public static DataIteratorBuilder getAttachmentDataIteratorBuilder(TableInfo ti, // find attachment columns int entityIdIndex = 0; final ArrayList<_AttachmentUploadHelper> attachmentColumns = new ArrayList<>(); + final Map attachmentColumnsAliases = new HashMap<>(); for (int c = 1; c <= it.getColumnCount(); c++) { @@ -229,6 +265,7 @@ public static DataIteratorBuilder getAttachmentDataIteratorBuilder(TableInfo ti, continue; attachmentColumns.add(new _AttachmentUploadHelper(c,domainProperty)); + attachmentColumnsAliases.put(domainProperty.getName(), col.getAlias().getId()); } catch (IndexOutOfBoundsException ignored) // Until issue is resolved between StatementDataIterator.getColumnCount() and SimpleTranslator.getColumnCount() { @@ -236,7 +273,9 @@ public static DataIteratorBuilder getAttachmentDataIteratorBuilder(TableInfo ti, } if (!attachmentColumns.isEmpty()) - return new AttachmentDataIterator(it, context.getErrors(), user, attachmentDir, entityIdIndex, attachmentColumns, context.getInsertOption(), container, parentFactory ); + { + return new AttachmentDataIterator(it, context.getErrors(), user, attachmentDir, entityIdIndex, attachmentColumns, attachmentColumnsAliases, context.getInsertOption(), container, parentFactory); + } return it; }; diff --git a/api/src/org/labkey/api/dataiterator/ExistingRecordDataIterator.java b/api/src/org/labkey/api/dataiterator/ExistingRecordDataIterator.java index c39acf459a1..c314e6f0362 100644 --- a/api/src/org/labkey/api/dataiterator/ExistingRecordDataIterator.java +++ b/api/src/org/labkey/api/dataiterator/ExistingRecordDataIterator.java @@ -21,6 +21,7 @@ import org.labkey.api.module.Module; import org.labkey.api.module.ModuleLoader; import org.labkey.api.query.BatchValidationException; +import org.labkey.api.query.DefaultQueryUpdateService; import org.labkey.api.query.InvalidKeyException; import org.labkey.api.query.QueryUpdateService; import org.labkey.api.query.QueryUpdateServiceException; @@ -238,10 +239,14 @@ public static DataIteratorBuilder createBuilder(DataIteratorBuilder dib, TableIn QueryUpdateService.InsertOption option = context.getInsertOption(); if (option.allowUpdate) { + boolean hasAttachmentProperties = false; + QueryUpdateService qus = target.getUpdateService(); + if (qus instanceof DefaultQueryUpdateService dQus) + hasAttachmentProperties = dQus.hasAttachmentProperties(); // if true, we need to fetch existing records to properly handle old attachment delete AuditBehaviorType auditType = AuditBehaviorType.NONE; if (target.supportsAuditTracking()) auditType = target.getEffectiveAuditBehavior((AuditBehaviorType) context.getConfigParameter(DetailedAuditLogDataIterator.AuditConfigs.AuditBehavior)); - boolean detailed = auditType == DETAILED; + boolean detailed = auditType == DETAILED || hasAttachmentProperties; if (useGetRows) return new ExistingDataIteratorsGetRows(new CachingDataIterator(di), target, keys, sharedKeys, context, detailed); else diff --git a/api/src/org/labkey/api/query/DefaultQueryUpdateService.java b/api/src/org/labkey/api/query/DefaultQueryUpdateService.java index 3fd1c9c97a5..0b177237ddf 100644 --- a/api/src/org/labkey/api/query/DefaultQueryUpdateService.java +++ b/api/src/org/labkey/api/query/DefaultQueryUpdateService.java @@ -1,940 +1,940 @@ -/* - * Copyright (c) 2009-2019 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.labkey.api.query; - -import org.apache.commons.beanutils.ConversionException; -import org.apache.commons.lang3.StringUtils; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; -import org.labkey.api.attachments.AttachmentFile; -import org.labkey.api.collections.ArrayListMap; -import org.labkey.api.collections.CaseInsensitiveHashMap; -import org.labkey.api.collections.CaseInsensitiveMapWrapper; -import org.labkey.api.data.ColumnInfo; -import org.labkey.api.data.Container; -import org.labkey.api.data.ConvertHelper; -import org.labkey.api.data.ExpDataFileConverter; -import org.labkey.api.data.JdbcType; -import org.labkey.api.data.MvUtil; -import org.labkey.api.data.Parameter; -import org.labkey.api.data.SQLFragment; -import org.labkey.api.data.SimpleFilter; -import org.labkey.api.data.Table; -import org.labkey.api.data.TableInfo; -import org.labkey.api.data.TableSelector; -import org.labkey.api.data.UpdateableTableInfo; -import org.labkey.api.data.validator.ColumnValidator; -import org.labkey.api.data.validator.ColumnValidators; -import org.labkey.api.dataiterator.DataIteratorBuilder; -import org.labkey.api.dataiterator.DataIteratorContext; -import org.labkey.api.dataiterator.DataIteratorUtil; -import org.labkey.api.dataiterator.MapDataIterator; -import org.labkey.api.exp.OntologyManager; -import org.labkey.api.exp.OntologyObject; -import org.labkey.api.exp.PropertyColumn; -import org.labkey.api.exp.PropertyDescriptor; -import org.labkey.api.exp.PropertyType; -import org.labkey.api.exp.api.ExperimentService; -import org.labkey.api.exp.property.Domain; -import org.labkey.api.exp.property.DomainProperty; -import org.labkey.api.exp.property.ValidatorContext; -import org.labkey.api.reader.ColumnDescriptor; -import org.labkey.api.reader.DataLoader; -import org.labkey.api.security.User; -import org.labkey.api.security.permissions.DeletePermission; -import org.labkey.api.security.permissions.InsertPermission; -import org.labkey.api.security.permissions.Permission; -import org.labkey.api.security.permissions.UpdatePermission; -import org.labkey.api.util.CachingSupplier; -import org.labkey.api.util.Pair; -import org.labkey.api.view.UnauthorizedException; -import org.labkey.vfs.FileLike; -import org.springframework.web.multipart.MultipartFile; - -import java.io.IOException; -import java.nio.file.Path; -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Set; -import java.util.function.Supplier; - -/** - * QueryUpdateService implementation that supports Query TableInfos that are backed by both a hard table and a Domain. - * To update the Domain, a DomainUpdateHelper is required, otherwise the DefaultQueryUpdateService will only update the - * hard table columns. - */ -public class DefaultQueryUpdateService extends AbstractQueryUpdateService -{ - private final TableInfo _dbTable; - private DomainUpdateHelper _helper = null; - /** - * Map from DbTable column names to QueryTable column names, if they have been aliased - */ - protected Map _columnMapping = Collections.emptyMap(); - /** - * Hold onto the ColumnInfos, so we don't have to regenerate them for every row we process - */ - private final Supplier> _tableMapSupplier = new CachingSupplier<>(() -> DataIteratorUtil.createTableMap(getQueryTable(), true)); - private final ValidatorContext _validatorContext; - private final FileColumnValueMapper _fileColumnValueMapping = new FileColumnValueMapper(); - - public DefaultQueryUpdateService(@NotNull TableInfo queryTable, TableInfo dbTable) - { - super(queryTable); - _dbTable = dbTable; - - if (queryTable.getUserSchema() == null) - throw new RuntimeValidationException("User schema not defined for " + queryTable.getName()); - - _validatorContext = new ValidatorContext(queryTable.getUserSchema().getContainer(), queryTable.getUserSchema().getUser()); - } - - public DefaultQueryUpdateService(TableInfo queryTable, TableInfo dbTable, DomainUpdateHelper helper) - { - this(queryTable, dbTable); - _helper = helper; - } - - /** - * @param columnMapping Map from DbTable column names to QueryTable column names, if they have been aliased - */ - public DefaultQueryUpdateService(TableInfo queryTable, TableInfo dbTable, Map columnMapping) - { - this(queryTable, dbTable); - _columnMapping = columnMapping; - } - - protected TableInfo getDbTable() - { - return _dbTable; - } - - protected Domain getDomain() - { - return _helper == null ? null : _helper.getDomain(); - } - - protected ColumnInfo getObjectUriColumn() - { - return _helper == null ? null : _helper.getObjectUriColumn(); - } - - protected String createObjectURI() - { - return _helper == null ? null : _helper.createObjectURI(); - } - - protected Iterable getPropertyColumns() - { - return _helper == null ? Collections.emptyList() : _helper.getPropertyColumns(); - } - - protected Map getColumnMapping() - { - return _columnMapping; - } - - /** - * Returns the container that the domain is defined - */ - protected Container getDomainContainer(Container c) - { - return _helper == null ? c : _helper.getDomainContainer(c); - } - - /** - * Returns the container to insert/update values into - */ - protected Container getDomainObjContainer(Container c) - { - return _helper == null ? c : _helper.getDomainObjContainer(c); - } - - protected Set getAutoPopulatedColumns() - { - return Table.AUTOPOPULATED_COLUMN_NAMES; - } - - public interface DomainUpdateHelper - { - Domain getDomain(); - - ColumnInfo getObjectUriColumn(); - - String createObjectURI(); - - // Could probably be just Iterable or be removed and just get all PropertyDescriptors in the Domain. - Iterable getPropertyColumns(); - - Container getDomainContainer(Container c); - - Container getDomainObjContainer(Container c); - } - - public class ImportHelper implements OntologyManager.ImportHelper - { - ImportHelper() - { - } - - @Override - public String beforeImportObject(Map map) - { - ColumnInfo objectUriCol = getObjectUriColumn(); - - // Get existing Lsid - String lsid = (String) map.get(objectUriCol.getName()); - if (lsid != null) - return lsid; - - // Generate a new Lsid - lsid = createObjectURI(); - map.put(objectUriCol.getName(), lsid); - return lsid; - } - - @Override - public void afterBatchInsert(int currentRow) - { - } - - @Override - public void updateStatistics(int currentRow) - { - } - } - - @Override - protected Map getRow(User user, Container container, Map keys) - throws InvalidKeyException, QueryUpdateServiceException, SQLException - { - aliasColumns(_columnMapping, keys); - Map row = _select(container, getKeys(keys, container)); - - //PostgreSQL includes a column named _row for the row index, but since this is selecting by - //primary key, it will always be 1, which is not only unnecessary, but confusing, so strip it - if (null != row) - { - if (row instanceof ArrayListMap) - ((ArrayListMap) row).getFindMap().remove("_row"); - else - row.remove("_row"); - } - - return row; - } - - protected Map _select(Container container, Object[] keys) throws ConversionException - { - TableInfo table = getDbTable(); - Object[] typedParameters = convertToTypedValues(keys, table.getPkColumns()); - - Map row = new TableSelector(table).getMap(typedParameters); - - ColumnInfo objectUriCol = getObjectUriColumn(); - Domain domain = getDomain(); - if (objectUriCol != null && domain != null && !domain.getProperties().isEmpty() && row != null) - { - String lsid = (String) row.get(objectUriCol.getName()); - if (lsid != null) - { - Map propertyValues = OntologyManager.getProperties(getDomainObjContainer(container), lsid); - if (!propertyValues.isEmpty()) - { - // convert PropertyURI->value map into "Property name"->value map - Map propertyMap = domain.createImportMap(false); - for (Map.Entry entry : propertyValues.entrySet()) - { - String propertyURI = entry.getKey(); - DomainProperty dp = propertyMap.get(propertyURI); - PropertyDescriptor pd = dp != null ? dp.getPropertyDescriptor() : null; - if (pd != null) - row.put(pd.getName(), entry.getValue()); - } - } - } - // Issue 46985: Be tolerant of a row not having an LSID value (as the row may have been - // inserted before the table was made extensible), but make sure that we got an LSID field - // when fetching the row - else if (!row.containsKey(objectUriCol.getName())) - { - throw new IllegalStateException("LSID value not returned when querying table - " + table.getName()); - } - } - - return row; - } - - - private Object[] convertToTypedValues(Object[] keys, List cols) - { - Object[] typedParameters = new Object[keys.length]; - int t = 0; - for (int i = 0; i < keys.length; i++) - { - if (i >= cols.size() || keys[i] instanceof Parameter.TypedValue) - { - typedParameters[t++] = keys[i]; - continue; - } - Object v = keys[i]; - JdbcType type = cols.get(i).getJdbcType(); - if (v instanceof String) - v = type.convert(v); - Parameter.TypedValue tv = new Parameter.TypedValue(v, type); - typedParameters[t++] = tv; - } - return typedParameters; - } - - - @Override - protected Map insertRow(User user, Container container, Map row) - throws DuplicateKeyException, ValidationException, QueryUpdateServiceException, SQLException - { - aliasColumns(_columnMapping, row); - convertTypes(user, container, row); - setSpecialColumns(container, row, user, InsertPermission.class); - validateInsertRow(row); - return _insert(user, container, row); - } - - protected Map _insert(User user, Container c, Map row) - throws SQLException, ValidationException - { - assert (getQueryTable().supportsInsertOption(InsertOption.INSERT)); - - try - { - ColumnInfo objectUriCol = getObjectUriColumn(); - Domain domain = getDomain(); - if (objectUriCol != null && domain != null && !domain.getProperties().isEmpty()) - { - // convert "Property name"->value map into PropertyURI->value map - List pds = new ArrayList<>(); - Map values = new CaseInsensitiveMapWrapper<>(new HashMap<>()); - for (PropertyColumn pc : getPropertyColumns()) - { - PropertyDescriptor pd = pc.getPropertyDescriptor(); - pds.add(pd); - Object value = getPropertyValue(row, pd); - values.put(pd.getPropertyURI(), value); - } - - LsidCollector collector = new LsidCollector(); - OntologyManager.insertTabDelimited(getDomainObjContainer(c), user, null, new ImportHelper(), pds, MapDataIterator.of(Collections.singletonList(values)).getDataIterator(new DataIteratorContext()), true, collector); - String lsid = collector.getLsid(); - - // Add the new lsid to the row map. - row.put(objectUriCol.getName(), lsid); - } - - return Table.insert(user, getDbTable(), row); - } - catch (RuntimeValidationException e) - { - throw e.getValidationException(); - } - catch (BatchValidationException e) - { - throw e.getLastRowError(); - } - } - - @Override - protected Map updateRow(User user, Container container, Map row, @NotNull Map oldRow, @Nullable Map configParameters) - throws InvalidKeyException, ValidationException, QueryUpdateServiceException, SQLException - { - return updateRow(user, container, row, oldRow, false, false); - } - - protected Map updateRow(User user, Container container, Map row, @NotNull Map oldRow, boolean allowOwner, boolean retainCreation) - throws InvalidKeyException, ValidationException, QueryUpdateServiceException, SQLException - { - Map rowStripped = new CaseInsensitiveHashMap<>(row.size()); - - // Flip the key/value pairs around for easy lookup - Map queryToDb = new CaseInsensitiveHashMap<>(); - for (Map.Entry entry : _columnMapping.entrySet()) - { - queryToDb.put(entry.getValue(), entry.getKey()); - } - - setSpecialColumns(container, row, user, UpdatePermission.class); - - Map tableAliasesMap = _tableMapSupplier.get(); - Map> colFrequency = new HashMap<>(); - - //resolve passed in row including columns in the table and other properties (vocabulary properties) not in the Domain/table - for (Map.Entry entry: row.entrySet()) - { - if (!rowStripped.containsKey(entry.getKey())) - { - ColumnInfo col = getQueryTable().getColumn(entry.getKey()); - - if (null == col) - { - col = tableAliasesMap.get(entry.getKey()); - } - - if (null != col) - { - final String name = col.getName(); - - // Skip readonly and wrapped columns. The wrapped column is usually a pk column and can't be updated. - if (col.isReadOnly() || col.isCalculated()) - continue; - - //when updating a row, we should strip the following fields, as they are - //automagically maintained by the table layer, and should not be allowed - //to change once the record exists. - //unfortunately, the Table.update() method doesn't strip these, so we'll - //do that here. - // Owner, CreatedBy, Created, EntityId - if ((!retainCreation && (name.equalsIgnoreCase("CreatedBy") || name.equalsIgnoreCase("Created"))) - || (!allowOwner && name.equalsIgnoreCase("Owner")) - || name.equalsIgnoreCase("EntityId")) - continue; - - // Throw error if more than one row properties having different values match up to the same column. - if (!colFrequency.containsKey(col)) - { - colFrequency.put(col, Pair.of(entry.getKey(),entry.getValue())); - } - else - { - if (!Objects.equals(colFrequency.get(col).second, entry.getValue())) - { - throw new ValidationException("Property key - " + colFrequency.get(col).first + " and " + entry.getKey() + " matched for the same column."); - } - } - - // We want a map using the DbTable column names as keys, so figure out the right name to use - String dbName = queryToDb.getOrDefault(name, name); - rowStripped.put(dbName, entry.getValue()); - } - } - } - - convertTypes(user, container, rowStripped); - validateUpdateRow(rowStripped); - - if (row.get("container") != null) - { - Container rowContainer = UserSchema.translateRowSuppliedContainer(row.get("container"), container, user, getQueryTable(), UpdatePermission.class, null); - if (rowContainer == null) - { - throw new ValidationException("Unknown container: " + row.get("container")); - } - else - { - Container oldContainer = UserSchema.translateRowSuppliedContainer(new CaseInsensitiveHashMap<>(oldRow).get("container"), container, user, getQueryTable(), UpdatePermission.class, null); - if (null != oldContainer && !rowContainer.equals(oldContainer)) - throw new UnauthorizedException("The row is from the wrong container."); - } - } - - Map updatedRow = _update(user, container, rowStripped, oldRow, oldRow == null ? getKeys(row, container) : getKeys(oldRow, container)); - - //when passing a map for the row, the Table layer returns the map of fields it updated, which excludes - //the primary key columns as well as those marked read-only. So we can't simply return the map returned - //from Table.update(). Instead, we need to copy values from updatedRow into row and return that. - row.putAll(updatedRow); - return row; - } - - protected void validateValue(ColumnInfo column, Object value, Object providedValue) throws ValidationException - { - DomainProperty dp = getDomain() == null ? null : getDomain().getPropertyByName(column.getColumnName()); - List validators = ColumnValidators.create(column, dp); - for (ColumnValidator v : validators) - { - String msg = v.validate(-1, value, _validatorContext, providedValue); - if (msg != null) - throw new ValidationException(msg, column.getName()); - } - } - - protected void validateInsertRow(Map row) throws ValidationException - { - for (ColumnInfo col : getQueryTable().getColumns()) - { - Object value = row.get(col.getColumnName()); - - // Check required values aren't null or empty - if (null == value || value instanceof String s && s.isEmpty()) - { - if (!col.isAutoIncrement() && col.isRequired() && - !getAutoPopulatedColumns().contains(col.getName()) && - col.getJdbcDefaultValue() == null) - { - throw new ValidationException("A value is required for field '" + col.getName() + "'", col.getName()); - } - } - else - { - validateValue(col, value, null); - } - } - } - - protected void validateUpdateRow(Map row) throws ValidationException - { - for (ColumnInfo col : getQueryTable().getColumns()) - { - // Only validate incoming values - if (row.containsKey(col.getColumnName())) - { - Object value = row.get(col.getColumnName()); - validateValue(col, value, null); - } - } - } - - protected Map _update(User user, Container c, Map row, Map oldRow, Object[] keys) - throws SQLException, ValidationException - { - assert(getQueryTable().supportsInsertOption(InsertOption.UPDATE)); - - try - { - ColumnInfo objectUriCol = getObjectUriColumn(); - Domain domain = getDomain(); - - // The lsid may be null for the row until a property has been inserted - String lsid = null; - if (objectUriCol != null) - lsid = (String) oldRow.get(objectUriCol.getName()); - - List tableProperties = new ArrayList<>(); - if (objectUriCol != null && domain != null && !domain.getProperties().isEmpty()) - { - // convert "Property name"->value map into PropertyURI->value map - Map newValues = new CaseInsensitiveMapWrapper<>(new HashMap<>()); - - for (PropertyColumn pc : getPropertyColumns()) - { - PropertyDescriptor pd = pc.getPropertyDescriptor(); - tableProperties.add(pd); - - // clear out the old value if it exists and is contained in the new row (it may be incoming as null) - if (lsid != null && (hasProperty(row, pd) && hasProperty(oldRow, pd))) - OntologyManager.deleteProperty(lsid, pd.getPropertyURI(), getDomainObjContainer(c), getDomainContainer(c)); - - Object value = getPropertyValue(row, pd); - if (value != null) - newValues.put(pd.getPropertyURI(), value); - } - - // Note: copy lsid into newValues map so it will be found by the ImportHelper.beforeImportObject() - newValues.put(objectUriCol.getName(), lsid); - - LsidCollector collector = new LsidCollector(); - OntologyManager.insertTabDelimited(getDomainObjContainer(c), user, null, new ImportHelper(), tableProperties, MapDataIterator.of(Collections.singletonList(newValues)).getDataIterator(new DataIteratorContext()), true, collector); - - // Update the lsid in the row: the lsid may have not existed in the row before the update. - lsid = collector.getLsid(); - row.put(objectUriCol.getName(), lsid); - } - - // Get lsid value if it hasn't been set. - // This should only happen if the QueryUpdateService doesn't have a DomainUpdateHelper (DataClass and SampleType) - if (lsid == null && getQueryTable() instanceof UpdateableTableInfo updateableTableInfo) - { - String objectUriColName = updateableTableInfo.getObjectURIColumnName(); - if (objectUriColName != null) - lsid = (String) row.getOrDefault(objectUriColName, oldRow.get(objectUriColName)); - } - - // handle vocabulary properties - if (lsid != null) - { - for (Map.Entry rowEntry : row.entrySet()) - { - String colName = rowEntry.getKey(); - Object value = rowEntry.getValue(); - - ColumnInfo col = getQueryTable().getColumn(colName); - if (col instanceof PropertyColumn propCol) - { - PropertyDescriptor pd = propCol.getPropertyDescriptor(); - if (pd.isVocabulary() && !tableProperties.contains(pd)) - { - OntologyManager.updateObjectProperty(user, c, pd, lsid, value, null, false); - } - } - } - } - } - catch (BatchValidationException e) - { - throw e.getLastRowError(); - } - - checkDuplicateUpdate(keys); - - return Table.update(user, getDbTable(), row, keys); // Cache-invalidation handled in caller (TreatmentManager.saveAssaySpecimen()) - } - - private static class LsidCollector implements OntologyManager.RowCallback - { - private String _lsid; - - @Override - public void rowProcessed(Map row, String lsid) - { - if (_lsid != null) - { - throw new IllegalStateException("Only expected a single LSID"); - } - _lsid = lsid; - } - - public String getLsid() - { - if (_lsid == null) - { - throw new IllegalStateException("No LSID returned"); - } - return _lsid; - } - } - - // Get value from row map where the keys are column names. - private Object getPropertyValue(Map row, PropertyDescriptor pd) - { - if (row.containsKey(pd.getName())) - return row.get(pd.getName()); - - if (row.containsKey(pd.getLabel())) - return row.get(pd.getLabel()); - - for (String alias : pd.getImportAliasSet()) - { - if (row.containsKey(alias)) - return row.get(alias); - } - - return null; - } - - // Checks a value exists in the row map (value may be null) - private boolean hasProperty(Map row, PropertyDescriptor pd) - { - if (row.containsKey(pd.getName())) - return true; - - if (row.containsKey(pd.getLabel())) - return true; - - for (String alias : pd.getImportAliasSet()) - { - if (row.containsKey(alias)) - return true; - } - - return false; - } - - @Override - protected Map deleteRow(User user, Container container, Map oldRowMap) throws QueryUpdateServiceException, SQLException, InvalidKeyException - { - if (oldRowMap == null) - return null; - - aliasColumns(_columnMapping, oldRowMap); - - if (container != null && getDbTable().getColumn("container") != null) - { - // UNDONE: 9077: check container permission on each row before delete - Container rowContainer = UserSchema.translateRowSuppliedContainer(new CaseInsensitiveHashMap<>(oldRowMap).get("container"), container, user, getQueryTable(), DeletePermission.class, null); - if (null != rowContainer && !container.equals(rowContainer)) - { - //Issue 15301: allow workbooks records to be deleted/updated from the parent container - if (container.allowRowMutationForContainer(rowContainer)) - container = rowContainer; - else - throw new UnauthorizedException("The row is from the container: " + rowContainer.getId() + " which does not allow deletes from the container: " + container.getPath()); - } - } - - _delete(container, oldRowMap); - return oldRowMap; - } - - protected void _delete(Container c, Map row) throws InvalidKeyException - { - ColumnInfo objectUriCol = getObjectUriColumn(); - if (objectUriCol != null) - { - String lsid = (String)row.get(objectUriCol.getName()); - if (lsid != null) - { - OntologyObject oo = OntologyManager.getOntologyObject(c, lsid); - if (oo != null) - OntologyManager.deleteProperties(c, oo.getObjectId()); - } - } - Table.delete(getDbTable(), getKeys(row, c)); - } - - // classes should override this method if they need to do more work than delete all the rows from the table - // this implementation will delete all rows from the table for the given container as well as delete - // any properties associated with the table - @Override - protected int truncateRows(User user, Container container) throws QueryUpdateServiceException, SQLException - { - // get rid of the properties for this table - if (null != getObjectUriColumn()) - { - SQLFragment lsids = new SQLFragment() - .append("SELECT t.").append(getObjectUriColumn().getColumnName()) - .append(" FROM ").append(getDbTable(), "t") - .append(" WHERE t.").append(getObjectUriColumn().getColumnName()).append(" IS NOT NULL"); - if (null != getDbTable().getColumn("container")) - { - lsids.append(" AND t.Container = ?"); - lsids.add(container.getId()); - } - - OntologyManager.deleteOntologyObjects(ExperimentService.get().getSchema(), lsids, container); - } - - // delete all the rows in this table, scoping to the container if the column - // is available - if (null != getDbTable().getColumn("container")) - return Table.delete(getDbTable(), SimpleFilter.createContainerFilter(container)); - - return Table.delete(getDbTable()); - } - - protected Object[] getKeys(Map map, Container container) throws InvalidKeyException - { - //build an array of pk values based on the table info - TableInfo table = getDbTable(); - List pks = table.getPkColumns(); - Object[] pkVals = new Object[pks.size()]; - - if (map == null || map.isEmpty()) - return pkVals; - - for (int idx = 0; idx < pks.size(); ++idx) - { - ColumnInfo pk = pks.get(idx); - Object pkValue = map.get(pk.getName()); - // Check the type and coerce if needed - if (pkValue != null && !pk.getJavaObjectClass().isInstance(pkValue)) - { - try - { - pkValue = pk.convert(pkValue); - } - catch (ConversionException ignored) { /* Maybe the database can do the conversion */ } - } - pkVals[idx] = pkValue; - if (null == pkVals[idx] && pk.getColumnName().equalsIgnoreCase("Container")) - { - pkVals[idx] = container; - } - if(null == pkVals[idx]) - { - throw new InvalidKeyException("Value for key field '" + pk.getName() + "' was null or not supplied!", map); - } - } - return pkVals; - } - - private Map _missingValues = null; - private Container _missingValuesContainer; - - protected boolean validMissingValue(Container c, String mv) - { - if (null == c) - return false; - if (null == _missingValues || !c.getId().equals(_missingValuesContainer.getId())) - { - _missingValues = MvUtil.getIndicatorsAndLabels(c); - _missingValuesContainer = c; - } - return _missingValues.containsKey(mv); - } - - protected TableInfo getTableInfoForConversion() - { - return getDbTable(); - } - - final protected void convertTypes(User user, Container c, Map row) throws ValidationException - { - convertTypes(user, c, row, getTableInfoForConversion(), null); - } - - // TODO Path->FileObject - // why is coerceTypes() in AbstractQueryUpdateService and convertTypes() in DefaultQueryUpdateService? - protected void convertTypes(User user, Container c, Map row, TableInfo t, @Nullable Path fileLinkDirPath) throws ValidationException - { - for (ColumnInfo col : t.getColumns()) - { - if (col.isMvIndicatorColumn()) - continue; - boolean isColumnPresent = row.containsKey(col.getName()) || col.isMvEnabled() && row.containsKey(col.getMvColumnName().getName()); - if (!isColumnPresent) - continue; - - Object value = row.get(col.getName()); - - /* NOTE: see MissingValueConvertColumn.convert() these methods should have similar behavior. - * If you update this code, check that code as well. */ - if (col.isMvEnabled()) - { - if (value instanceof String s && StringUtils.isEmpty(s)) - value = null; - - Object mvObj = row.get(col.getMvColumnName().getName()); - String mv = Objects.toString(mvObj, null); - if (StringUtils.isEmpty(mv)) - mv = null; - - if (null != mv) - { - if (!validMissingValue(c, mv)) - throw new ValidationException("Value is not a valid missing value indicator: " + mv); - } - else if (null != value) - { - String s = Objects.toString(value, null); - if (validMissingValue(c, s)) - { - mv = s; - value = null; - } - } - row.put(col.getMvColumnName().getName(), mv); - } - - value = convertColumnValue(col, value, user, c, fileLinkDirPath); - row.put(col.getName(), value); - } - } - - protected Object convertColumnValue(ColumnInfo col, Object value, User user, Container c, @Nullable Path fileLinkDirPath) throws ValidationException - { - // Issue 13951: PSQLException from org.labkey.api.query.DefaultQueryUpdateService._update() - // improve handling of conversion errors - try - { - if (PropertyType.FILE_LINK == col.getPropertyType()) - { - if ((value instanceof MultipartFile || value instanceof AttachmentFile)) - { - FileLike fl = (FileLike)_fileColumnValueMapping.saveFileColumnValue(user, c, fileLinkDirPath, col.getName(), value); - value = fl.toNioPathForRead().toString(); - } - return ExpDataFileConverter.convert(value); - } - return col.getConvertFn().convert(value); - } - catch (ConvertHelper.FileConversionException e) - { - throw new ValidationException(e.getMessage()); - } - catch (ConversionException e) - { - String type = ColumnInfo.getFriendlyTypeName(col.getJdbcType().getJavaClass()); - throw new ValidationException("Unable to convert value '" + value.toString() + "' to " + type, col.getName()); - } - catch (QueryUpdateServiceException e) - { - throw new ValidationException("Save file link failed: " + col.getName()); - } - } - - /** - * Override this method to alter the row before insert or update. - * For example, you can automatically adjust certain column values based on context. - * @param container The current container - * @param row The row data - * @param user The current user - * @param clazz A permission class to test - */ - protected void setSpecialColumns(Container container, Map row, User user, Class clazz) - { - if (null != container) - { - //Issue 15301: allow workbooks records to be deleted/updated from the parent container - if (row.get("container") != null) - { - Container rowContainer = UserSchema.translateRowSuppliedContainer(row.get("container"), container, user, getQueryTable(), clazz, null); - if (rowContainer != null && container.allowRowMutationForContainer(rowContainer)) - { - row.put("container", rowContainer.getId()); //normalize to container ID - return; //accept the row-provided value - } - } - row.put("container", container.getId()); - } - } - - protected boolean hasAttachmentProperties() - { - Domain domain = getDomain(); - if (null != domain) - { - for (DomainProperty dp : domain.getProperties()) - if (null != dp && isAttachmentProperty(dp)) - return true; - } - return false; - } - - protected boolean isAttachmentProperty(@NotNull DomainProperty dp) - { - PropertyDescriptor pd = dp.getPropertyDescriptor(); - return PropertyType.ATTACHMENT.equals(pd.getPropertyType()); - } - - protected boolean isAttachmentProperty(String name) - { - DomainProperty dp = getDomain().getPropertyByName(name); - if (dp != null) - return isAttachmentProperty(dp); - return false; - } - - protected void configureCrossFolderImport(DataIteratorBuilder rows, DataIteratorContext context) throws IOException - { - if (!context.getInsertOption().updateOnly && context.isCrossFolderImport() && rows instanceof DataLoader dataLoader) - { - boolean hasContainerField = false; - for (ColumnDescriptor columnDescriptor : dataLoader.getColumns()) - { - String fieldName = columnDescriptor.getColumnName(); - if (fieldName.equalsIgnoreCase("Container") || fieldName.equalsIgnoreCase("Folder")) - { - hasContainerField = true; - break; - } - } - if (!hasContainerField) - context.setCrossFolderImport(false); - } - } -} +/* + * Copyright (c) 2009-2019 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.labkey.api.query; + +import org.apache.commons.beanutils.ConversionException; +import org.apache.commons.lang3.StringUtils; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.labkey.api.attachments.AttachmentFile; +import org.labkey.api.collections.ArrayListMap; +import org.labkey.api.collections.CaseInsensitiveHashMap; +import org.labkey.api.collections.CaseInsensitiveMapWrapper; +import org.labkey.api.data.ColumnInfo; +import org.labkey.api.data.Container; +import org.labkey.api.data.ConvertHelper; +import org.labkey.api.data.ExpDataFileConverter; +import org.labkey.api.data.JdbcType; +import org.labkey.api.data.MvUtil; +import org.labkey.api.data.Parameter; +import org.labkey.api.data.SQLFragment; +import org.labkey.api.data.SimpleFilter; +import org.labkey.api.data.Table; +import org.labkey.api.data.TableInfo; +import org.labkey.api.data.TableSelector; +import org.labkey.api.data.UpdateableTableInfo; +import org.labkey.api.data.validator.ColumnValidator; +import org.labkey.api.data.validator.ColumnValidators; +import org.labkey.api.dataiterator.DataIteratorBuilder; +import org.labkey.api.dataiterator.DataIteratorContext; +import org.labkey.api.dataiterator.DataIteratorUtil; +import org.labkey.api.dataiterator.MapDataIterator; +import org.labkey.api.exp.OntologyManager; +import org.labkey.api.exp.OntologyObject; +import org.labkey.api.exp.PropertyColumn; +import org.labkey.api.exp.PropertyDescriptor; +import org.labkey.api.exp.PropertyType; +import org.labkey.api.exp.api.ExperimentService; +import org.labkey.api.exp.property.Domain; +import org.labkey.api.exp.property.DomainProperty; +import org.labkey.api.exp.property.ValidatorContext; +import org.labkey.api.reader.ColumnDescriptor; +import org.labkey.api.reader.DataLoader; +import org.labkey.api.security.User; +import org.labkey.api.security.permissions.DeletePermission; +import org.labkey.api.security.permissions.InsertPermission; +import org.labkey.api.security.permissions.Permission; +import org.labkey.api.security.permissions.UpdatePermission; +import org.labkey.api.util.CachingSupplier; +import org.labkey.api.util.Pair; +import org.labkey.api.view.UnauthorizedException; +import org.labkey.vfs.FileLike; +import org.springframework.web.multipart.MultipartFile; + +import java.io.IOException; +import java.nio.file.Path; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.function.Supplier; + +/** + * QueryUpdateService implementation that supports Query TableInfos that are backed by both a hard table and a Domain. + * To update the Domain, a DomainUpdateHelper is required, otherwise the DefaultQueryUpdateService will only update the + * hard table columns. + */ +public class DefaultQueryUpdateService extends AbstractQueryUpdateService +{ + private final TableInfo _dbTable; + private DomainUpdateHelper _helper = null; + /** + * Map from DbTable column names to QueryTable column names, if they have been aliased + */ + protected Map _columnMapping = Collections.emptyMap(); + /** + * Hold onto the ColumnInfos, so we don't have to regenerate them for every row we process + */ + private final Supplier> _tableMapSupplier = new CachingSupplier<>(() -> DataIteratorUtil.createTableMap(getQueryTable(), true)); + private final ValidatorContext _validatorContext; + private final FileColumnValueMapper _fileColumnValueMapping = new FileColumnValueMapper(); + + public DefaultQueryUpdateService(@NotNull TableInfo queryTable, TableInfo dbTable) + { + super(queryTable); + _dbTable = dbTable; + + if (queryTable.getUserSchema() == null) + throw new RuntimeValidationException("User schema not defined for " + queryTable.getName()); + + _validatorContext = new ValidatorContext(queryTable.getUserSchema().getContainer(), queryTable.getUserSchema().getUser()); + } + + public DefaultQueryUpdateService(TableInfo queryTable, TableInfo dbTable, DomainUpdateHelper helper) + { + this(queryTable, dbTable); + _helper = helper; + } + + /** + * @param columnMapping Map from DbTable column names to QueryTable column names, if they have been aliased + */ + public DefaultQueryUpdateService(TableInfo queryTable, TableInfo dbTable, Map columnMapping) + { + this(queryTable, dbTable); + _columnMapping = columnMapping; + } + + protected TableInfo getDbTable() + { + return _dbTable; + } + + protected Domain getDomain() + { + return _helper == null ? null : _helper.getDomain(); + } + + protected ColumnInfo getObjectUriColumn() + { + return _helper == null ? null : _helper.getObjectUriColumn(); + } + + protected String createObjectURI() + { + return _helper == null ? null : _helper.createObjectURI(); + } + + protected Iterable getPropertyColumns() + { + return _helper == null ? Collections.emptyList() : _helper.getPropertyColumns(); + } + + protected Map getColumnMapping() + { + return _columnMapping; + } + + /** + * Returns the container that the domain is defined + */ + protected Container getDomainContainer(Container c) + { + return _helper == null ? c : _helper.getDomainContainer(c); + } + + /** + * Returns the container to insert/update values into + */ + protected Container getDomainObjContainer(Container c) + { + return _helper == null ? c : _helper.getDomainObjContainer(c); + } + + protected Set getAutoPopulatedColumns() + { + return Table.AUTOPOPULATED_COLUMN_NAMES; + } + + public interface DomainUpdateHelper + { + Domain getDomain(); + + ColumnInfo getObjectUriColumn(); + + String createObjectURI(); + + // Could probably be just Iterable or be removed and just get all PropertyDescriptors in the Domain. + Iterable getPropertyColumns(); + + Container getDomainContainer(Container c); + + Container getDomainObjContainer(Container c); + } + + public class ImportHelper implements OntologyManager.ImportHelper + { + ImportHelper() + { + } + + @Override + public String beforeImportObject(Map map) + { + ColumnInfo objectUriCol = getObjectUriColumn(); + + // Get existing Lsid + String lsid = (String) map.get(objectUriCol.getName()); + if (lsid != null) + return lsid; + + // Generate a new Lsid + lsid = createObjectURI(); + map.put(objectUriCol.getName(), lsid); + return lsid; + } + + @Override + public void afterBatchInsert(int currentRow) + { + } + + @Override + public void updateStatistics(int currentRow) + { + } + } + + @Override + protected Map getRow(User user, Container container, Map keys) + throws InvalidKeyException, QueryUpdateServiceException, SQLException + { + aliasColumns(_columnMapping, keys); + Map row = _select(container, getKeys(keys, container)); + + //PostgreSQL includes a column named _row for the row index, but since this is selecting by + //primary key, it will always be 1, which is not only unnecessary, but confusing, so strip it + if (null != row) + { + if (row instanceof ArrayListMap) + ((ArrayListMap) row).getFindMap().remove("_row"); + else + row.remove("_row"); + } + + return row; + } + + protected Map _select(Container container, Object[] keys) throws ConversionException + { + TableInfo table = getDbTable(); + Object[] typedParameters = convertToTypedValues(keys, table.getPkColumns()); + + Map row = new TableSelector(table).getMap(typedParameters); + + ColumnInfo objectUriCol = getObjectUriColumn(); + Domain domain = getDomain(); + if (objectUriCol != null && domain != null && !domain.getProperties().isEmpty() && row != null) + { + String lsid = (String) row.get(objectUriCol.getName()); + if (lsid != null) + { + Map propertyValues = OntologyManager.getProperties(getDomainObjContainer(container), lsid); + if (!propertyValues.isEmpty()) + { + // convert PropertyURI->value map into "Property name"->value map + Map propertyMap = domain.createImportMap(false); + for (Map.Entry entry : propertyValues.entrySet()) + { + String propertyURI = entry.getKey(); + DomainProperty dp = propertyMap.get(propertyURI); + PropertyDescriptor pd = dp != null ? dp.getPropertyDescriptor() : null; + if (pd != null) + row.put(pd.getName(), entry.getValue()); + } + } + } + // Issue 46985: Be tolerant of a row not having an LSID value (as the row may have been + // inserted before the table was made extensible), but make sure that we got an LSID field + // when fetching the row + else if (!row.containsKey(objectUriCol.getName())) + { + throw new IllegalStateException("LSID value not returned when querying table - " + table.getName()); + } + } + + return row; + } + + + private Object[] convertToTypedValues(Object[] keys, List cols) + { + Object[] typedParameters = new Object[keys.length]; + int t = 0; + for (int i = 0; i < keys.length; i++) + { + if (i >= cols.size() || keys[i] instanceof Parameter.TypedValue) + { + typedParameters[t++] = keys[i]; + continue; + } + Object v = keys[i]; + JdbcType type = cols.get(i).getJdbcType(); + if (v instanceof String) + v = type.convert(v); + Parameter.TypedValue tv = new Parameter.TypedValue(v, type); + typedParameters[t++] = tv; + } + return typedParameters; + } + + + @Override + protected Map insertRow(User user, Container container, Map row) + throws DuplicateKeyException, ValidationException, QueryUpdateServiceException, SQLException + { + aliasColumns(_columnMapping, row); + convertTypes(user, container, row); + setSpecialColumns(container, row, user, InsertPermission.class); + validateInsertRow(row); + return _insert(user, container, row); + } + + protected Map _insert(User user, Container c, Map row) + throws SQLException, ValidationException + { + assert (getQueryTable().supportsInsertOption(InsertOption.INSERT)); + + try + { + ColumnInfo objectUriCol = getObjectUriColumn(); + Domain domain = getDomain(); + if (objectUriCol != null && domain != null && !domain.getProperties().isEmpty()) + { + // convert "Property name"->value map into PropertyURI->value map + List pds = new ArrayList<>(); + Map values = new CaseInsensitiveMapWrapper<>(new HashMap<>()); + for (PropertyColumn pc : getPropertyColumns()) + { + PropertyDescriptor pd = pc.getPropertyDescriptor(); + pds.add(pd); + Object value = getPropertyValue(row, pd); + values.put(pd.getPropertyURI(), value); + } + + LsidCollector collector = new LsidCollector(); + OntologyManager.insertTabDelimited(getDomainObjContainer(c), user, null, new ImportHelper(), pds, MapDataIterator.of(Collections.singletonList(values)).getDataIterator(new DataIteratorContext()), true, collector); + String lsid = collector.getLsid(); + + // Add the new lsid to the row map. + row.put(objectUriCol.getName(), lsid); + } + + return Table.insert(user, getDbTable(), row); + } + catch (RuntimeValidationException e) + { + throw e.getValidationException(); + } + catch (BatchValidationException e) + { + throw e.getLastRowError(); + } + } + + @Override + protected Map updateRow(User user, Container container, Map row, @NotNull Map oldRow, @Nullable Map configParameters) + throws InvalidKeyException, ValidationException, QueryUpdateServiceException, SQLException + { + return updateRow(user, container, row, oldRow, false, false); + } + + protected Map updateRow(User user, Container container, Map row, @NotNull Map oldRow, boolean allowOwner, boolean retainCreation) + throws InvalidKeyException, ValidationException, QueryUpdateServiceException, SQLException + { + Map rowStripped = new CaseInsensitiveHashMap<>(row.size()); + + // Flip the key/value pairs around for easy lookup + Map queryToDb = new CaseInsensitiveHashMap<>(); + for (Map.Entry entry : _columnMapping.entrySet()) + { + queryToDb.put(entry.getValue(), entry.getKey()); + } + + setSpecialColumns(container, row, user, UpdatePermission.class); + + Map tableAliasesMap = _tableMapSupplier.get(); + Map> colFrequency = new HashMap<>(); + + //resolve passed in row including columns in the table and other properties (vocabulary properties) not in the Domain/table + for (Map.Entry entry: row.entrySet()) + { + if (!rowStripped.containsKey(entry.getKey())) + { + ColumnInfo col = getQueryTable().getColumn(entry.getKey()); + + if (null == col) + { + col = tableAliasesMap.get(entry.getKey()); + } + + if (null != col) + { + final String name = col.getName(); + + // Skip readonly and wrapped columns. The wrapped column is usually a pk column and can't be updated. + if (col.isReadOnly() || col.isCalculated()) + continue; + + //when updating a row, we should strip the following fields, as they are + //automagically maintained by the table layer, and should not be allowed + //to change once the record exists. + //unfortunately, the Table.update() method doesn't strip these, so we'll + //do that here. + // Owner, CreatedBy, Created, EntityId + if ((!retainCreation && (name.equalsIgnoreCase("CreatedBy") || name.equalsIgnoreCase("Created"))) + || (!allowOwner && name.equalsIgnoreCase("Owner")) + || name.equalsIgnoreCase("EntityId")) + continue; + + // Throw error if more than one row properties having different values match up to the same column. + if (!colFrequency.containsKey(col)) + { + colFrequency.put(col, Pair.of(entry.getKey(),entry.getValue())); + } + else + { + if (!Objects.equals(colFrequency.get(col).second, entry.getValue())) + { + throw new ValidationException("Property key - " + colFrequency.get(col).first + " and " + entry.getKey() + " matched for the same column."); + } + } + + // We want a map using the DbTable column names as keys, so figure out the right name to use + String dbName = queryToDb.getOrDefault(name, name); + rowStripped.put(dbName, entry.getValue()); + } + } + } + + convertTypes(user, container, rowStripped); + validateUpdateRow(rowStripped); + + if (row.get("container") != null) + { + Container rowContainer = UserSchema.translateRowSuppliedContainer(row.get("container"), container, user, getQueryTable(), UpdatePermission.class, null); + if (rowContainer == null) + { + throw new ValidationException("Unknown container: " + row.get("container")); + } + else + { + Container oldContainer = UserSchema.translateRowSuppliedContainer(new CaseInsensitiveHashMap<>(oldRow).get("container"), container, user, getQueryTable(), UpdatePermission.class, null); + if (null != oldContainer && !rowContainer.equals(oldContainer)) + throw new UnauthorizedException("The row is from the wrong container."); + } + } + + Map updatedRow = _update(user, container, rowStripped, oldRow, oldRow == null ? getKeys(row, container) : getKeys(oldRow, container)); + + //when passing a map for the row, the Table layer returns the map of fields it updated, which excludes + //the primary key columns as well as those marked read-only. So we can't simply return the map returned + //from Table.update(). Instead, we need to copy values from updatedRow into row and return that. + row.putAll(updatedRow); + return row; + } + + protected void validateValue(ColumnInfo column, Object value, Object providedValue) throws ValidationException + { + DomainProperty dp = getDomain() == null ? null : getDomain().getPropertyByName(column.getColumnName()); + List validators = ColumnValidators.create(column, dp); + for (ColumnValidator v : validators) + { + String msg = v.validate(-1, value, _validatorContext, providedValue); + if (msg != null) + throw new ValidationException(msg, column.getName()); + } + } + + protected void validateInsertRow(Map row) throws ValidationException + { + for (ColumnInfo col : getQueryTable().getColumns()) + { + Object value = row.get(col.getColumnName()); + + // Check required values aren't null or empty + if (null == value || value instanceof String s && s.isEmpty()) + { + if (!col.isAutoIncrement() && col.isRequired() && + !getAutoPopulatedColumns().contains(col.getName()) && + col.getJdbcDefaultValue() == null) + { + throw new ValidationException("A value is required for field '" + col.getName() + "'", col.getName()); + } + } + else + { + validateValue(col, value, null); + } + } + } + + protected void validateUpdateRow(Map row) throws ValidationException + { + for (ColumnInfo col : getQueryTable().getColumns()) + { + // Only validate incoming values + if (row.containsKey(col.getColumnName())) + { + Object value = row.get(col.getColumnName()); + validateValue(col, value, null); + } + } + } + + protected Map _update(User user, Container c, Map row, Map oldRow, Object[] keys) + throws SQLException, ValidationException + { + assert(getQueryTable().supportsInsertOption(InsertOption.UPDATE)); + + try + { + ColumnInfo objectUriCol = getObjectUriColumn(); + Domain domain = getDomain(); + + // The lsid may be null for the row until a property has been inserted + String lsid = null; + if (objectUriCol != null) + lsid = (String) oldRow.get(objectUriCol.getName()); + + List tableProperties = new ArrayList<>(); + if (objectUriCol != null && domain != null && !domain.getProperties().isEmpty()) + { + // convert "Property name"->value map into PropertyURI->value map + Map newValues = new CaseInsensitiveMapWrapper<>(new HashMap<>()); + + for (PropertyColumn pc : getPropertyColumns()) + { + PropertyDescriptor pd = pc.getPropertyDescriptor(); + tableProperties.add(pd); + + // clear out the old value if it exists and is contained in the new row (it may be incoming as null) + if (lsid != null && (hasProperty(row, pd) && hasProperty(oldRow, pd))) + OntologyManager.deleteProperty(lsid, pd.getPropertyURI(), getDomainObjContainer(c), getDomainContainer(c)); + + Object value = getPropertyValue(row, pd); + if (value != null) + newValues.put(pd.getPropertyURI(), value); + } + + // Note: copy lsid into newValues map so it will be found by the ImportHelper.beforeImportObject() + newValues.put(objectUriCol.getName(), lsid); + + LsidCollector collector = new LsidCollector(); + OntologyManager.insertTabDelimited(getDomainObjContainer(c), user, null, new ImportHelper(), tableProperties, MapDataIterator.of(Collections.singletonList(newValues)).getDataIterator(new DataIteratorContext()), true, collector); + + // Update the lsid in the row: the lsid may have not existed in the row before the update. + lsid = collector.getLsid(); + row.put(objectUriCol.getName(), lsid); + } + + // Get lsid value if it hasn't been set. + // This should only happen if the QueryUpdateService doesn't have a DomainUpdateHelper (DataClass and SampleType) + if (lsid == null && getQueryTable() instanceof UpdateableTableInfo updateableTableInfo) + { + String objectUriColName = updateableTableInfo.getObjectURIColumnName(); + if (objectUriColName != null) + lsid = (String) row.getOrDefault(objectUriColName, oldRow.get(objectUriColName)); + } + + // handle vocabulary properties + if (lsid != null) + { + for (Map.Entry rowEntry : row.entrySet()) + { + String colName = rowEntry.getKey(); + Object value = rowEntry.getValue(); + + ColumnInfo col = getQueryTable().getColumn(colName); + if (col instanceof PropertyColumn propCol) + { + PropertyDescriptor pd = propCol.getPropertyDescriptor(); + if (pd.isVocabulary() && !tableProperties.contains(pd)) + { + OntologyManager.updateObjectProperty(user, c, pd, lsid, value, null, false); + } + } + } + } + } + catch (BatchValidationException e) + { + throw e.getLastRowError(); + } + + checkDuplicateUpdate(keys); + + return Table.update(user, getDbTable(), row, keys); // Cache-invalidation handled in caller (TreatmentManager.saveAssaySpecimen()) + } + + private static class LsidCollector implements OntologyManager.RowCallback + { + private String _lsid; + + @Override + public void rowProcessed(Map row, String lsid) + { + if (_lsid != null) + { + throw new IllegalStateException("Only expected a single LSID"); + } + _lsid = lsid; + } + + public String getLsid() + { + if (_lsid == null) + { + throw new IllegalStateException("No LSID returned"); + } + return _lsid; + } + } + + // Get value from row map where the keys are column names. + private Object getPropertyValue(Map row, PropertyDescriptor pd) + { + if (row.containsKey(pd.getName())) + return row.get(pd.getName()); + + if (row.containsKey(pd.getLabel())) + return row.get(pd.getLabel()); + + for (String alias : pd.getImportAliasSet()) + { + if (row.containsKey(alias)) + return row.get(alias); + } + + return null; + } + + // Checks a value exists in the row map (value may be null) + private boolean hasProperty(Map row, PropertyDescriptor pd) + { + if (row.containsKey(pd.getName())) + return true; + + if (row.containsKey(pd.getLabel())) + return true; + + for (String alias : pd.getImportAliasSet()) + { + if (row.containsKey(alias)) + return true; + } + + return false; + } + + @Override + protected Map deleteRow(User user, Container container, Map oldRowMap) throws QueryUpdateServiceException, SQLException, InvalidKeyException + { + if (oldRowMap == null) + return null; + + aliasColumns(_columnMapping, oldRowMap); + + if (container != null && getDbTable().getColumn("container") != null) + { + // UNDONE: 9077: check container permission on each row before delete + Container rowContainer = UserSchema.translateRowSuppliedContainer(new CaseInsensitiveHashMap<>(oldRowMap).get("container"), container, user, getQueryTable(), DeletePermission.class, null); + if (null != rowContainer && !container.equals(rowContainer)) + { + //Issue 15301: allow workbooks records to be deleted/updated from the parent container + if (container.allowRowMutationForContainer(rowContainer)) + container = rowContainer; + else + throw new UnauthorizedException("The row is from the container: " + rowContainer.getId() + " which does not allow deletes from the container: " + container.getPath()); + } + } + + _delete(container, oldRowMap); + return oldRowMap; + } + + protected void _delete(Container c, Map row) throws InvalidKeyException + { + ColumnInfo objectUriCol = getObjectUriColumn(); + if (objectUriCol != null) + { + String lsid = (String)row.get(objectUriCol.getName()); + if (lsid != null) + { + OntologyObject oo = OntologyManager.getOntologyObject(c, lsid); + if (oo != null) + OntologyManager.deleteProperties(c, oo.getObjectId()); + } + } + Table.delete(getDbTable(), getKeys(row, c)); + } + + // classes should override this method if they need to do more work than delete all the rows from the table + // this implementation will delete all rows from the table for the given container as well as delete + // any properties associated with the table + @Override + protected int truncateRows(User user, Container container) throws QueryUpdateServiceException, SQLException + { + // get rid of the properties for this table + if (null != getObjectUriColumn()) + { + SQLFragment lsids = new SQLFragment() + .append("SELECT t.").append(getObjectUriColumn().getColumnName()) + .append(" FROM ").append(getDbTable(), "t") + .append(" WHERE t.").append(getObjectUriColumn().getColumnName()).append(" IS NOT NULL"); + if (null != getDbTable().getColumn("container")) + { + lsids.append(" AND t.Container = ?"); + lsids.add(container.getId()); + } + + OntologyManager.deleteOntologyObjects(ExperimentService.get().getSchema(), lsids, container); + } + + // delete all the rows in this table, scoping to the container if the column + // is available + if (null != getDbTable().getColumn("container")) + return Table.delete(getDbTable(), SimpleFilter.createContainerFilter(container)); + + return Table.delete(getDbTable()); + } + + protected Object[] getKeys(Map map, Container container) throws InvalidKeyException + { + //build an array of pk values based on the table info + TableInfo table = getDbTable(); + List pks = table.getPkColumns(); + Object[] pkVals = new Object[pks.size()]; + + if (map == null || map.isEmpty()) + return pkVals; + + for (int idx = 0; idx < pks.size(); ++idx) + { + ColumnInfo pk = pks.get(idx); + Object pkValue = map.get(pk.getName()); + // Check the type and coerce if needed + if (pkValue != null && !pk.getJavaObjectClass().isInstance(pkValue)) + { + try + { + pkValue = pk.convert(pkValue); + } + catch (ConversionException ignored) { /* Maybe the database can do the conversion */ } + } + pkVals[idx] = pkValue; + if (null == pkVals[idx] && pk.getColumnName().equalsIgnoreCase("Container")) + { + pkVals[idx] = container; + } + if(null == pkVals[idx]) + { + throw new InvalidKeyException("Value for key field '" + pk.getName() + "' was null or not supplied!", map); + } + } + return pkVals; + } + + private Map _missingValues = null; + private Container _missingValuesContainer; + + protected boolean validMissingValue(Container c, String mv) + { + if (null == c) + return false; + if (null == _missingValues || !c.getId().equals(_missingValuesContainer.getId())) + { + _missingValues = MvUtil.getIndicatorsAndLabels(c); + _missingValuesContainer = c; + } + return _missingValues.containsKey(mv); + } + + protected TableInfo getTableInfoForConversion() + { + return getDbTable(); + } + + final protected void convertTypes(User user, Container c, Map row) throws ValidationException + { + convertTypes(user, c, row, getTableInfoForConversion(), null); + } + + // TODO Path->FileObject + // why is coerceTypes() in AbstractQueryUpdateService and convertTypes() in DefaultQueryUpdateService? + protected void convertTypes(User user, Container c, Map row, TableInfo t, @Nullable Path fileLinkDirPath) throws ValidationException + { + for (ColumnInfo col : t.getColumns()) + { + if (col.isMvIndicatorColumn()) + continue; + boolean isColumnPresent = row.containsKey(col.getName()) || col.isMvEnabled() && row.containsKey(col.getMvColumnName().getName()); + if (!isColumnPresent) + continue; + + Object value = row.get(col.getName()); + + /* NOTE: see MissingValueConvertColumn.convert() these methods should have similar behavior. + * If you update this code, check that code as well. */ + if (col.isMvEnabled()) + { + if (value instanceof String s && StringUtils.isEmpty(s)) + value = null; + + Object mvObj = row.get(col.getMvColumnName().getName()); + String mv = Objects.toString(mvObj, null); + if (StringUtils.isEmpty(mv)) + mv = null; + + if (null != mv) + { + if (!validMissingValue(c, mv)) + throw new ValidationException("Value is not a valid missing value indicator: " + mv); + } + else if (null != value) + { + String s = Objects.toString(value, null); + if (validMissingValue(c, s)) + { + mv = s; + value = null; + } + } + row.put(col.getMvColumnName().getName(), mv); + } + + value = convertColumnValue(col, value, user, c, fileLinkDirPath); + row.put(col.getName(), value); + } + } + + protected Object convertColumnValue(ColumnInfo col, Object value, User user, Container c, @Nullable Path fileLinkDirPath) throws ValidationException + { + // Issue 13951: PSQLException from org.labkey.api.query.DefaultQueryUpdateService._update() + // improve handling of conversion errors + try + { + if (PropertyType.FILE_LINK == col.getPropertyType()) + { + if ((value instanceof MultipartFile || value instanceof AttachmentFile)) + { + FileLike fl = (FileLike)_fileColumnValueMapping.saveFileColumnValue(user, c, fileLinkDirPath, col.getName(), value); + value = fl.toNioPathForRead().toString(); + } + return ExpDataFileConverter.convert(value); + } + return col.getConvertFn().convert(value); + } + catch (ConvertHelper.FileConversionException e) + { + throw new ValidationException(e.getMessage()); + } + catch (ConversionException e) + { + String type = ColumnInfo.getFriendlyTypeName(col.getJdbcType().getJavaClass()); + throw new ValidationException("Unable to convert value '" + value.toString() + "' to " + type, col.getName()); + } + catch (QueryUpdateServiceException e) + { + throw new ValidationException("Save file link failed: " + col.getName()); + } + } + + /** + * Override this method to alter the row before insert or update. + * For example, you can automatically adjust certain column values based on context. + * @param container The current container + * @param row The row data + * @param user The current user + * @param clazz A permission class to test + */ + protected void setSpecialColumns(Container container, Map row, User user, Class clazz) + { + if (null != container) + { + //Issue 15301: allow workbooks records to be deleted/updated from the parent container + if (row.get("container") != null) + { + Container rowContainer = UserSchema.translateRowSuppliedContainer(row.get("container"), container, user, getQueryTable(), clazz, null); + if (rowContainer != null && container.allowRowMutationForContainer(rowContainer)) + { + row.put("container", rowContainer.getId()); //normalize to container ID + return; //accept the row-provided value + } + } + row.put("container", container.getId()); + } + } + + public boolean hasAttachmentProperties() + { + Domain domain = getDomain(); + if (null != domain) + { + for (DomainProperty dp : domain.getProperties()) + if (null != dp && isAttachmentProperty(dp)) + return true; + } + return false; + } + + protected boolean isAttachmentProperty(@NotNull DomainProperty dp) + { + PropertyDescriptor pd = dp.getPropertyDescriptor(); + return PropertyType.ATTACHMENT.equals(pd.getPropertyType()); + } + + protected boolean isAttachmentProperty(String name) + { + DomainProperty dp = getDomain().getPropertyByName(name); + if (dp != null) + return isAttachmentProperty(dp); + return false; + } + + protected void configureCrossFolderImport(DataIteratorBuilder rows, DataIteratorContext context) throws IOException + { + if (!context.getInsertOption().updateOnly && context.isCrossFolderImport() && rows instanceof DataLoader dataLoader) + { + boolean hasContainerField = false; + for (ColumnDescriptor columnDescriptor : dataLoader.getColumns()) + { + String fieldName = columnDescriptor.getColumnName(); + if (fieldName.equalsIgnoreCase("Container") || fieldName.equalsIgnoreCase("Folder")) + { + hasContainerField = true; + break; + } + } + if (!hasContainerField) + context.setCrossFolderImport(false); + } + } +} From e75f427f9e28865e595a5732de89b900c7076f40 Mon Sep 17 00:00:00 2001 From: XingY Date: Thu, 2 Apr 2026 22:39:06 -0700 Subject: [PATCH 2/5] CRLF --- .../api/query/DefaultQueryUpdateService.java | 1880 ++++++++--------- 1 file changed, 940 insertions(+), 940 deletions(-) diff --git a/api/src/org/labkey/api/query/DefaultQueryUpdateService.java b/api/src/org/labkey/api/query/DefaultQueryUpdateService.java index 0b177237ddf..19e38c431ce 100644 --- a/api/src/org/labkey/api/query/DefaultQueryUpdateService.java +++ b/api/src/org/labkey/api/query/DefaultQueryUpdateService.java @@ -1,940 +1,940 @@ -/* - * Copyright (c) 2009-2019 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.labkey.api.query; - -import org.apache.commons.beanutils.ConversionException; -import org.apache.commons.lang3.StringUtils; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; -import org.labkey.api.attachments.AttachmentFile; -import org.labkey.api.collections.ArrayListMap; -import org.labkey.api.collections.CaseInsensitiveHashMap; -import org.labkey.api.collections.CaseInsensitiveMapWrapper; -import org.labkey.api.data.ColumnInfo; -import org.labkey.api.data.Container; -import org.labkey.api.data.ConvertHelper; -import org.labkey.api.data.ExpDataFileConverter; -import org.labkey.api.data.JdbcType; -import org.labkey.api.data.MvUtil; -import org.labkey.api.data.Parameter; -import org.labkey.api.data.SQLFragment; -import org.labkey.api.data.SimpleFilter; -import org.labkey.api.data.Table; -import org.labkey.api.data.TableInfo; -import org.labkey.api.data.TableSelector; -import org.labkey.api.data.UpdateableTableInfo; -import org.labkey.api.data.validator.ColumnValidator; -import org.labkey.api.data.validator.ColumnValidators; -import org.labkey.api.dataiterator.DataIteratorBuilder; -import org.labkey.api.dataiterator.DataIteratorContext; -import org.labkey.api.dataiterator.DataIteratorUtil; -import org.labkey.api.dataiterator.MapDataIterator; -import org.labkey.api.exp.OntologyManager; -import org.labkey.api.exp.OntologyObject; -import org.labkey.api.exp.PropertyColumn; -import org.labkey.api.exp.PropertyDescriptor; -import org.labkey.api.exp.PropertyType; -import org.labkey.api.exp.api.ExperimentService; -import org.labkey.api.exp.property.Domain; -import org.labkey.api.exp.property.DomainProperty; -import org.labkey.api.exp.property.ValidatorContext; -import org.labkey.api.reader.ColumnDescriptor; -import org.labkey.api.reader.DataLoader; -import org.labkey.api.security.User; -import org.labkey.api.security.permissions.DeletePermission; -import org.labkey.api.security.permissions.InsertPermission; -import org.labkey.api.security.permissions.Permission; -import org.labkey.api.security.permissions.UpdatePermission; -import org.labkey.api.util.CachingSupplier; -import org.labkey.api.util.Pair; -import org.labkey.api.view.UnauthorizedException; -import org.labkey.vfs.FileLike; -import org.springframework.web.multipart.MultipartFile; - -import java.io.IOException; -import java.nio.file.Path; -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Set; -import java.util.function.Supplier; - -/** - * QueryUpdateService implementation that supports Query TableInfos that are backed by both a hard table and a Domain. - * To update the Domain, a DomainUpdateHelper is required, otherwise the DefaultQueryUpdateService will only update the - * hard table columns. - */ -public class DefaultQueryUpdateService extends AbstractQueryUpdateService -{ - private final TableInfo _dbTable; - private DomainUpdateHelper _helper = null; - /** - * Map from DbTable column names to QueryTable column names, if they have been aliased - */ - protected Map _columnMapping = Collections.emptyMap(); - /** - * Hold onto the ColumnInfos, so we don't have to regenerate them for every row we process - */ - private final Supplier> _tableMapSupplier = new CachingSupplier<>(() -> DataIteratorUtil.createTableMap(getQueryTable(), true)); - private final ValidatorContext _validatorContext; - private final FileColumnValueMapper _fileColumnValueMapping = new FileColumnValueMapper(); - - public DefaultQueryUpdateService(@NotNull TableInfo queryTable, TableInfo dbTable) - { - super(queryTable); - _dbTable = dbTable; - - if (queryTable.getUserSchema() == null) - throw new RuntimeValidationException("User schema not defined for " + queryTable.getName()); - - _validatorContext = new ValidatorContext(queryTable.getUserSchema().getContainer(), queryTable.getUserSchema().getUser()); - } - - public DefaultQueryUpdateService(TableInfo queryTable, TableInfo dbTable, DomainUpdateHelper helper) - { - this(queryTable, dbTable); - _helper = helper; - } - - /** - * @param columnMapping Map from DbTable column names to QueryTable column names, if they have been aliased - */ - public DefaultQueryUpdateService(TableInfo queryTable, TableInfo dbTable, Map columnMapping) - { - this(queryTable, dbTable); - _columnMapping = columnMapping; - } - - protected TableInfo getDbTable() - { - return _dbTable; - } - - protected Domain getDomain() - { - return _helper == null ? null : _helper.getDomain(); - } - - protected ColumnInfo getObjectUriColumn() - { - return _helper == null ? null : _helper.getObjectUriColumn(); - } - - protected String createObjectURI() - { - return _helper == null ? null : _helper.createObjectURI(); - } - - protected Iterable getPropertyColumns() - { - return _helper == null ? Collections.emptyList() : _helper.getPropertyColumns(); - } - - protected Map getColumnMapping() - { - return _columnMapping; - } - - /** - * Returns the container that the domain is defined - */ - protected Container getDomainContainer(Container c) - { - return _helper == null ? c : _helper.getDomainContainer(c); - } - - /** - * Returns the container to insert/update values into - */ - protected Container getDomainObjContainer(Container c) - { - return _helper == null ? c : _helper.getDomainObjContainer(c); - } - - protected Set getAutoPopulatedColumns() - { - return Table.AUTOPOPULATED_COLUMN_NAMES; - } - - public interface DomainUpdateHelper - { - Domain getDomain(); - - ColumnInfo getObjectUriColumn(); - - String createObjectURI(); - - // Could probably be just Iterable or be removed and just get all PropertyDescriptors in the Domain. - Iterable getPropertyColumns(); - - Container getDomainContainer(Container c); - - Container getDomainObjContainer(Container c); - } - - public class ImportHelper implements OntologyManager.ImportHelper - { - ImportHelper() - { - } - - @Override - public String beforeImportObject(Map map) - { - ColumnInfo objectUriCol = getObjectUriColumn(); - - // Get existing Lsid - String lsid = (String) map.get(objectUriCol.getName()); - if (lsid != null) - return lsid; - - // Generate a new Lsid - lsid = createObjectURI(); - map.put(objectUriCol.getName(), lsid); - return lsid; - } - - @Override - public void afterBatchInsert(int currentRow) - { - } - - @Override - public void updateStatistics(int currentRow) - { - } - } - - @Override - protected Map getRow(User user, Container container, Map keys) - throws InvalidKeyException, QueryUpdateServiceException, SQLException - { - aliasColumns(_columnMapping, keys); - Map row = _select(container, getKeys(keys, container)); - - //PostgreSQL includes a column named _row for the row index, but since this is selecting by - //primary key, it will always be 1, which is not only unnecessary, but confusing, so strip it - if (null != row) - { - if (row instanceof ArrayListMap) - ((ArrayListMap) row).getFindMap().remove("_row"); - else - row.remove("_row"); - } - - return row; - } - - protected Map _select(Container container, Object[] keys) throws ConversionException - { - TableInfo table = getDbTable(); - Object[] typedParameters = convertToTypedValues(keys, table.getPkColumns()); - - Map row = new TableSelector(table).getMap(typedParameters); - - ColumnInfo objectUriCol = getObjectUriColumn(); - Domain domain = getDomain(); - if (objectUriCol != null && domain != null && !domain.getProperties().isEmpty() && row != null) - { - String lsid = (String) row.get(objectUriCol.getName()); - if (lsid != null) - { - Map propertyValues = OntologyManager.getProperties(getDomainObjContainer(container), lsid); - if (!propertyValues.isEmpty()) - { - // convert PropertyURI->value map into "Property name"->value map - Map propertyMap = domain.createImportMap(false); - for (Map.Entry entry : propertyValues.entrySet()) - { - String propertyURI = entry.getKey(); - DomainProperty dp = propertyMap.get(propertyURI); - PropertyDescriptor pd = dp != null ? dp.getPropertyDescriptor() : null; - if (pd != null) - row.put(pd.getName(), entry.getValue()); - } - } - } - // Issue 46985: Be tolerant of a row not having an LSID value (as the row may have been - // inserted before the table was made extensible), but make sure that we got an LSID field - // when fetching the row - else if (!row.containsKey(objectUriCol.getName())) - { - throw new IllegalStateException("LSID value not returned when querying table - " + table.getName()); - } - } - - return row; - } - - - private Object[] convertToTypedValues(Object[] keys, List cols) - { - Object[] typedParameters = new Object[keys.length]; - int t = 0; - for (int i = 0; i < keys.length; i++) - { - if (i >= cols.size() || keys[i] instanceof Parameter.TypedValue) - { - typedParameters[t++] = keys[i]; - continue; - } - Object v = keys[i]; - JdbcType type = cols.get(i).getJdbcType(); - if (v instanceof String) - v = type.convert(v); - Parameter.TypedValue tv = new Parameter.TypedValue(v, type); - typedParameters[t++] = tv; - } - return typedParameters; - } - - - @Override - protected Map insertRow(User user, Container container, Map row) - throws DuplicateKeyException, ValidationException, QueryUpdateServiceException, SQLException - { - aliasColumns(_columnMapping, row); - convertTypes(user, container, row); - setSpecialColumns(container, row, user, InsertPermission.class); - validateInsertRow(row); - return _insert(user, container, row); - } - - protected Map _insert(User user, Container c, Map row) - throws SQLException, ValidationException - { - assert (getQueryTable().supportsInsertOption(InsertOption.INSERT)); - - try - { - ColumnInfo objectUriCol = getObjectUriColumn(); - Domain domain = getDomain(); - if (objectUriCol != null && domain != null && !domain.getProperties().isEmpty()) - { - // convert "Property name"->value map into PropertyURI->value map - List pds = new ArrayList<>(); - Map values = new CaseInsensitiveMapWrapper<>(new HashMap<>()); - for (PropertyColumn pc : getPropertyColumns()) - { - PropertyDescriptor pd = pc.getPropertyDescriptor(); - pds.add(pd); - Object value = getPropertyValue(row, pd); - values.put(pd.getPropertyURI(), value); - } - - LsidCollector collector = new LsidCollector(); - OntologyManager.insertTabDelimited(getDomainObjContainer(c), user, null, new ImportHelper(), pds, MapDataIterator.of(Collections.singletonList(values)).getDataIterator(new DataIteratorContext()), true, collector); - String lsid = collector.getLsid(); - - // Add the new lsid to the row map. - row.put(objectUriCol.getName(), lsid); - } - - return Table.insert(user, getDbTable(), row); - } - catch (RuntimeValidationException e) - { - throw e.getValidationException(); - } - catch (BatchValidationException e) - { - throw e.getLastRowError(); - } - } - - @Override - protected Map updateRow(User user, Container container, Map row, @NotNull Map oldRow, @Nullable Map configParameters) - throws InvalidKeyException, ValidationException, QueryUpdateServiceException, SQLException - { - return updateRow(user, container, row, oldRow, false, false); - } - - protected Map updateRow(User user, Container container, Map row, @NotNull Map oldRow, boolean allowOwner, boolean retainCreation) - throws InvalidKeyException, ValidationException, QueryUpdateServiceException, SQLException - { - Map rowStripped = new CaseInsensitiveHashMap<>(row.size()); - - // Flip the key/value pairs around for easy lookup - Map queryToDb = new CaseInsensitiveHashMap<>(); - for (Map.Entry entry : _columnMapping.entrySet()) - { - queryToDb.put(entry.getValue(), entry.getKey()); - } - - setSpecialColumns(container, row, user, UpdatePermission.class); - - Map tableAliasesMap = _tableMapSupplier.get(); - Map> colFrequency = new HashMap<>(); - - //resolve passed in row including columns in the table and other properties (vocabulary properties) not in the Domain/table - for (Map.Entry entry: row.entrySet()) - { - if (!rowStripped.containsKey(entry.getKey())) - { - ColumnInfo col = getQueryTable().getColumn(entry.getKey()); - - if (null == col) - { - col = tableAliasesMap.get(entry.getKey()); - } - - if (null != col) - { - final String name = col.getName(); - - // Skip readonly and wrapped columns. The wrapped column is usually a pk column and can't be updated. - if (col.isReadOnly() || col.isCalculated()) - continue; - - //when updating a row, we should strip the following fields, as they are - //automagically maintained by the table layer, and should not be allowed - //to change once the record exists. - //unfortunately, the Table.update() method doesn't strip these, so we'll - //do that here. - // Owner, CreatedBy, Created, EntityId - if ((!retainCreation && (name.equalsIgnoreCase("CreatedBy") || name.equalsIgnoreCase("Created"))) - || (!allowOwner && name.equalsIgnoreCase("Owner")) - || name.equalsIgnoreCase("EntityId")) - continue; - - // Throw error if more than one row properties having different values match up to the same column. - if (!colFrequency.containsKey(col)) - { - colFrequency.put(col, Pair.of(entry.getKey(),entry.getValue())); - } - else - { - if (!Objects.equals(colFrequency.get(col).second, entry.getValue())) - { - throw new ValidationException("Property key - " + colFrequency.get(col).first + " and " + entry.getKey() + " matched for the same column."); - } - } - - // We want a map using the DbTable column names as keys, so figure out the right name to use - String dbName = queryToDb.getOrDefault(name, name); - rowStripped.put(dbName, entry.getValue()); - } - } - } - - convertTypes(user, container, rowStripped); - validateUpdateRow(rowStripped); - - if (row.get("container") != null) - { - Container rowContainer = UserSchema.translateRowSuppliedContainer(row.get("container"), container, user, getQueryTable(), UpdatePermission.class, null); - if (rowContainer == null) - { - throw new ValidationException("Unknown container: " + row.get("container")); - } - else - { - Container oldContainer = UserSchema.translateRowSuppliedContainer(new CaseInsensitiveHashMap<>(oldRow).get("container"), container, user, getQueryTable(), UpdatePermission.class, null); - if (null != oldContainer && !rowContainer.equals(oldContainer)) - throw new UnauthorizedException("The row is from the wrong container."); - } - } - - Map updatedRow = _update(user, container, rowStripped, oldRow, oldRow == null ? getKeys(row, container) : getKeys(oldRow, container)); - - //when passing a map for the row, the Table layer returns the map of fields it updated, which excludes - //the primary key columns as well as those marked read-only. So we can't simply return the map returned - //from Table.update(). Instead, we need to copy values from updatedRow into row and return that. - row.putAll(updatedRow); - return row; - } - - protected void validateValue(ColumnInfo column, Object value, Object providedValue) throws ValidationException - { - DomainProperty dp = getDomain() == null ? null : getDomain().getPropertyByName(column.getColumnName()); - List validators = ColumnValidators.create(column, dp); - for (ColumnValidator v : validators) - { - String msg = v.validate(-1, value, _validatorContext, providedValue); - if (msg != null) - throw new ValidationException(msg, column.getName()); - } - } - - protected void validateInsertRow(Map row) throws ValidationException - { - for (ColumnInfo col : getQueryTable().getColumns()) - { - Object value = row.get(col.getColumnName()); - - // Check required values aren't null or empty - if (null == value || value instanceof String s && s.isEmpty()) - { - if (!col.isAutoIncrement() && col.isRequired() && - !getAutoPopulatedColumns().contains(col.getName()) && - col.getJdbcDefaultValue() == null) - { - throw new ValidationException("A value is required for field '" + col.getName() + "'", col.getName()); - } - } - else - { - validateValue(col, value, null); - } - } - } - - protected void validateUpdateRow(Map row) throws ValidationException - { - for (ColumnInfo col : getQueryTable().getColumns()) - { - // Only validate incoming values - if (row.containsKey(col.getColumnName())) - { - Object value = row.get(col.getColumnName()); - validateValue(col, value, null); - } - } - } - - protected Map _update(User user, Container c, Map row, Map oldRow, Object[] keys) - throws SQLException, ValidationException - { - assert(getQueryTable().supportsInsertOption(InsertOption.UPDATE)); - - try - { - ColumnInfo objectUriCol = getObjectUriColumn(); - Domain domain = getDomain(); - - // The lsid may be null for the row until a property has been inserted - String lsid = null; - if (objectUriCol != null) - lsid = (String) oldRow.get(objectUriCol.getName()); - - List tableProperties = new ArrayList<>(); - if (objectUriCol != null && domain != null && !domain.getProperties().isEmpty()) - { - // convert "Property name"->value map into PropertyURI->value map - Map newValues = new CaseInsensitiveMapWrapper<>(new HashMap<>()); - - for (PropertyColumn pc : getPropertyColumns()) - { - PropertyDescriptor pd = pc.getPropertyDescriptor(); - tableProperties.add(pd); - - // clear out the old value if it exists and is contained in the new row (it may be incoming as null) - if (lsid != null && (hasProperty(row, pd) && hasProperty(oldRow, pd))) - OntologyManager.deleteProperty(lsid, pd.getPropertyURI(), getDomainObjContainer(c), getDomainContainer(c)); - - Object value = getPropertyValue(row, pd); - if (value != null) - newValues.put(pd.getPropertyURI(), value); - } - - // Note: copy lsid into newValues map so it will be found by the ImportHelper.beforeImportObject() - newValues.put(objectUriCol.getName(), lsid); - - LsidCollector collector = new LsidCollector(); - OntologyManager.insertTabDelimited(getDomainObjContainer(c), user, null, new ImportHelper(), tableProperties, MapDataIterator.of(Collections.singletonList(newValues)).getDataIterator(new DataIteratorContext()), true, collector); - - // Update the lsid in the row: the lsid may have not existed in the row before the update. - lsid = collector.getLsid(); - row.put(objectUriCol.getName(), lsid); - } - - // Get lsid value if it hasn't been set. - // This should only happen if the QueryUpdateService doesn't have a DomainUpdateHelper (DataClass and SampleType) - if (lsid == null && getQueryTable() instanceof UpdateableTableInfo updateableTableInfo) - { - String objectUriColName = updateableTableInfo.getObjectURIColumnName(); - if (objectUriColName != null) - lsid = (String) row.getOrDefault(objectUriColName, oldRow.get(objectUriColName)); - } - - // handle vocabulary properties - if (lsid != null) - { - for (Map.Entry rowEntry : row.entrySet()) - { - String colName = rowEntry.getKey(); - Object value = rowEntry.getValue(); - - ColumnInfo col = getQueryTable().getColumn(colName); - if (col instanceof PropertyColumn propCol) - { - PropertyDescriptor pd = propCol.getPropertyDescriptor(); - if (pd.isVocabulary() && !tableProperties.contains(pd)) - { - OntologyManager.updateObjectProperty(user, c, pd, lsid, value, null, false); - } - } - } - } - } - catch (BatchValidationException e) - { - throw e.getLastRowError(); - } - - checkDuplicateUpdate(keys); - - return Table.update(user, getDbTable(), row, keys); // Cache-invalidation handled in caller (TreatmentManager.saveAssaySpecimen()) - } - - private static class LsidCollector implements OntologyManager.RowCallback - { - private String _lsid; - - @Override - public void rowProcessed(Map row, String lsid) - { - if (_lsid != null) - { - throw new IllegalStateException("Only expected a single LSID"); - } - _lsid = lsid; - } - - public String getLsid() - { - if (_lsid == null) - { - throw new IllegalStateException("No LSID returned"); - } - return _lsid; - } - } - - // Get value from row map where the keys are column names. - private Object getPropertyValue(Map row, PropertyDescriptor pd) - { - if (row.containsKey(pd.getName())) - return row.get(pd.getName()); - - if (row.containsKey(pd.getLabel())) - return row.get(pd.getLabel()); - - for (String alias : pd.getImportAliasSet()) - { - if (row.containsKey(alias)) - return row.get(alias); - } - - return null; - } - - // Checks a value exists in the row map (value may be null) - private boolean hasProperty(Map row, PropertyDescriptor pd) - { - if (row.containsKey(pd.getName())) - return true; - - if (row.containsKey(pd.getLabel())) - return true; - - for (String alias : pd.getImportAliasSet()) - { - if (row.containsKey(alias)) - return true; - } - - return false; - } - - @Override - protected Map deleteRow(User user, Container container, Map oldRowMap) throws QueryUpdateServiceException, SQLException, InvalidKeyException - { - if (oldRowMap == null) - return null; - - aliasColumns(_columnMapping, oldRowMap); - - if (container != null && getDbTable().getColumn("container") != null) - { - // UNDONE: 9077: check container permission on each row before delete - Container rowContainer = UserSchema.translateRowSuppliedContainer(new CaseInsensitiveHashMap<>(oldRowMap).get("container"), container, user, getQueryTable(), DeletePermission.class, null); - if (null != rowContainer && !container.equals(rowContainer)) - { - //Issue 15301: allow workbooks records to be deleted/updated from the parent container - if (container.allowRowMutationForContainer(rowContainer)) - container = rowContainer; - else - throw new UnauthorizedException("The row is from the container: " + rowContainer.getId() + " which does not allow deletes from the container: " + container.getPath()); - } - } - - _delete(container, oldRowMap); - return oldRowMap; - } - - protected void _delete(Container c, Map row) throws InvalidKeyException - { - ColumnInfo objectUriCol = getObjectUriColumn(); - if (objectUriCol != null) - { - String lsid = (String)row.get(objectUriCol.getName()); - if (lsid != null) - { - OntologyObject oo = OntologyManager.getOntologyObject(c, lsid); - if (oo != null) - OntologyManager.deleteProperties(c, oo.getObjectId()); - } - } - Table.delete(getDbTable(), getKeys(row, c)); - } - - // classes should override this method if they need to do more work than delete all the rows from the table - // this implementation will delete all rows from the table for the given container as well as delete - // any properties associated with the table - @Override - protected int truncateRows(User user, Container container) throws QueryUpdateServiceException, SQLException - { - // get rid of the properties for this table - if (null != getObjectUriColumn()) - { - SQLFragment lsids = new SQLFragment() - .append("SELECT t.").append(getObjectUriColumn().getColumnName()) - .append(" FROM ").append(getDbTable(), "t") - .append(" WHERE t.").append(getObjectUriColumn().getColumnName()).append(" IS NOT NULL"); - if (null != getDbTable().getColumn("container")) - { - lsids.append(" AND t.Container = ?"); - lsids.add(container.getId()); - } - - OntologyManager.deleteOntologyObjects(ExperimentService.get().getSchema(), lsids, container); - } - - // delete all the rows in this table, scoping to the container if the column - // is available - if (null != getDbTable().getColumn("container")) - return Table.delete(getDbTable(), SimpleFilter.createContainerFilter(container)); - - return Table.delete(getDbTable()); - } - - protected Object[] getKeys(Map map, Container container) throws InvalidKeyException - { - //build an array of pk values based on the table info - TableInfo table = getDbTable(); - List pks = table.getPkColumns(); - Object[] pkVals = new Object[pks.size()]; - - if (map == null || map.isEmpty()) - return pkVals; - - for (int idx = 0; idx < pks.size(); ++idx) - { - ColumnInfo pk = pks.get(idx); - Object pkValue = map.get(pk.getName()); - // Check the type and coerce if needed - if (pkValue != null && !pk.getJavaObjectClass().isInstance(pkValue)) - { - try - { - pkValue = pk.convert(pkValue); - } - catch (ConversionException ignored) { /* Maybe the database can do the conversion */ } - } - pkVals[idx] = pkValue; - if (null == pkVals[idx] && pk.getColumnName().equalsIgnoreCase("Container")) - { - pkVals[idx] = container; - } - if(null == pkVals[idx]) - { - throw new InvalidKeyException("Value for key field '" + pk.getName() + "' was null or not supplied!", map); - } - } - return pkVals; - } - - private Map _missingValues = null; - private Container _missingValuesContainer; - - protected boolean validMissingValue(Container c, String mv) - { - if (null == c) - return false; - if (null == _missingValues || !c.getId().equals(_missingValuesContainer.getId())) - { - _missingValues = MvUtil.getIndicatorsAndLabels(c); - _missingValuesContainer = c; - } - return _missingValues.containsKey(mv); - } - - protected TableInfo getTableInfoForConversion() - { - return getDbTable(); - } - - final protected void convertTypes(User user, Container c, Map row) throws ValidationException - { - convertTypes(user, c, row, getTableInfoForConversion(), null); - } - - // TODO Path->FileObject - // why is coerceTypes() in AbstractQueryUpdateService and convertTypes() in DefaultQueryUpdateService? - protected void convertTypes(User user, Container c, Map row, TableInfo t, @Nullable Path fileLinkDirPath) throws ValidationException - { - for (ColumnInfo col : t.getColumns()) - { - if (col.isMvIndicatorColumn()) - continue; - boolean isColumnPresent = row.containsKey(col.getName()) || col.isMvEnabled() && row.containsKey(col.getMvColumnName().getName()); - if (!isColumnPresent) - continue; - - Object value = row.get(col.getName()); - - /* NOTE: see MissingValueConvertColumn.convert() these methods should have similar behavior. - * If you update this code, check that code as well. */ - if (col.isMvEnabled()) - { - if (value instanceof String s && StringUtils.isEmpty(s)) - value = null; - - Object mvObj = row.get(col.getMvColumnName().getName()); - String mv = Objects.toString(mvObj, null); - if (StringUtils.isEmpty(mv)) - mv = null; - - if (null != mv) - { - if (!validMissingValue(c, mv)) - throw new ValidationException("Value is not a valid missing value indicator: " + mv); - } - else if (null != value) - { - String s = Objects.toString(value, null); - if (validMissingValue(c, s)) - { - mv = s; - value = null; - } - } - row.put(col.getMvColumnName().getName(), mv); - } - - value = convertColumnValue(col, value, user, c, fileLinkDirPath); - row.put(col.getName(), value); - } - } - - protected Object convertColumnValue(ColumnInfo col, Object value, User user, Container c, @Nullable Path fileLinkDirPath) throws ValidationException - { - // Issue 13951: PSQLException from org.labkey.api.query.DefaultQueryUpdateService._update() - // improve handling of conversion errors - try - { - if (PropertyType.FILE_LINK == col.getPropertyType()) - { - if ((value instanceof MultipartFile || value instanceof AttachmentFile)) - { - FileLike fl = (FileLike)_fileColumnValueMapping.saveFileColumnValue(user, c, fileLinkDirPath, col.getName(), value); - value = fl.toNioPathForRead().toString(); - } - return ExpDataFileConverter.convert(value); - } - return col.getConvertFn().convert(value); - } - catch (ConvertHelper.FileConversionException e) - { - throw new ValidationException(e.getMessage()); - } - catch (ConversionException e) - { - String type = ColumnInfo.getFriendlyTypeName(col.getJdbcType().getJavaClass()); - throw new ValidationException("Unable to convert value '" + value.toString() + "' to " + type, col.getName()); - } - catch (QueryUpdateServiceException e) - { - throw new ValidationException("Save file link failed: " + col.getName()); - } - } - - /** - * Override this method to alter the row before insert or update. - * For example, you can automatically adjust certain column values based on context. - * @param container The current container - * @param row The row data - * @param user The current user - * @param clazz A permission class to test - */ - protected void setSpecialColumns(Container container, Map row, User user, Class clazz) - { - if (null != container) - { - //Issue 15301: allow workbooks records to be deleted/updated from the parent container - if (row.get("container") != null) - { - Container rowContainer = UserSchema.translateRowSuppliedContainer(row.get("container"), container, user, getQueryTable(), clazz, null); - if (rowContainer != null && container.allowRowMutationForContainer(rowContainer)) - { - row.put("container", rowContainer.getId()); //normalize to container ID - return; //accept the row-provided value - } - } - row.put("container", container.getId()); - } - } - - public boolean hasAttachmentProperties() - { - Domain domain = getDomain(); - if (null != domain) - { - for (DomainProperty dp : domain.getProperties()) - if (null != dp && isAttachmentProperty(dp)) - return true; - } - return false; - } - - protected boolean isAttachmentProperty(@NotNull DomainProperty dp) - { - PropertyDescriptor pd = dp.getPropertyDescriptor(); - return PropertyType.ATTACHMENT.equals(pd.getPropertyType()); - } - - protected boolean isAttachmentProperty(String name) - { - DomainProperty dp = getDomain().getPropertyByName(name); - if (dp != null) - return isAttachmentProperty(dp); - return false; - } - - protected void configureCrossFolderImport(DataIteratorBuilder rows, DataIteratorContext context) throws IOException - { - if (!context.getInsertOption().updateOnly && context.isCrossFolderImport() && rows instanceof DataLoader dataLoader) - { - boolean hasContainerField = false; - for (ColumnDescriptor columnDescriptor : dataLoader.getColumns()) - { - String fieldName = columnDescriptor.getColumnName(); - if (fieldName.equalsIgnoreCase("Container") || fieldName.equalsIgnoreCase("Folder")) - { - hasContainerField = true; - break; - } - } - if (!hasContainerField) - context.setCrossFolderImport(false); - } - } -} +/* + * Copyright (c) 2009-2019 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.labkey.api.query; + +import org.apache.commons.beanutils.ConversionException; +import org.apache.commons.lang3.StringUtils; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.labkey.api.attachments.AttachmentFile; +import org.labkey.api.collections.ArrayListMap; +import org.labkey.api.collections.CaseInsensitiveHashMap; +import org.labkey.api.collections.CaseInsensitiveMapWrapper; +import org.labkey.api.data.ColumnInfo; +import org.labkey.api.data.Container; +import org.labkey.api.data.ConvertHelper; +import org.labkey.api.data.ExpDataFileConverter; +import org.labkey.api.data.JdbcType; +import org.labkey.api.data.MvUtil; +import org.labkey.api.data.Parameter; +import org.labkey.api.data.SQLFragment; +import org.labkey.api.data.SimpleFilter; +import org.labkey.api.data.Table; +import org.labkey.api.data.TableInfo; +import org.labkey.api.data.TableSelector; +import org.labkey.api.data.UpdateableTableInfo; +import org.labkey.api.data.validator.ColumnValidator; +import org.labkey.api.data.validator.ColumnValidators; +import org.labkey.api.dataiterator.DataIteratorBuilder; +import org.labkey.api.dataiterator.DataIteratorContext; +import org.labkey.api.dataiterator.DataIteratorUtil; +import org.labkey.api.dataiterator.MapDataIterator; +import org.labkey.api.exp.OntologyManager; +import org.labkey.api.exp.OntologyObject; +import org.labkey.api.exp.PropertyColumn; +import org.labkey.api.exp.PropertyDescriptor; +import org.labkey.api.exp.PropertyType; +import org.labkey.api.exp.api.ExperimentService; +import org.labkey.api.exp.property.Domain; +import org.labkey.api.exp.property.DomainProperty; +import org.labkey.api.exp.property.ValidatorContext; +import org.labkey.api.reader.ColumnDescriptor; +import org.labkey.api.reader.DataLoader; +import org.labkey.api.security.User; +import org.labkey.api.security.permissions.DeletePermission; +import org.labkey.api.security.permissions.InsertPermission; +import org.labkey.api.security.permissions.Permission; +import org.labkey.api.security.permissions.UpdatePermission; +import org.labkey.api.util.CachingSupplier; +import org.labkey.api.util.Pair; +import org.labkey.api.view.UnauthorizedException; +import org.labkey.vfs.FileLike; +import org.springframework.web.multipart.MultipartFile; + +import java.io.IOException; +import java.nio.file.Path; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.function.Supplier; + +/** + * QueryUpdateService implementation that supports Query TableInfos that are backed by both a hard table and a Domain. + * To update the Domain, a DomainUpdateHelper is required, otherwise the DefaultQueryUpdateService will only update the + * hard table columns. + */ +public class DefaultQueryUpdateService extends AbstractQueryUpdateService +{ + private final TableInfo _dbTable; + private DomainUpdateHelper _helper = null; + /** + * Map from DbTable column names to QueryTable column names, if they have been aliased + */ + protected Map _columnMapping = Collections.emptyMap(); + /** + * Hold onto the ColumnInfos, so we don't have to regenerate them for every row we process + */ + private final Supplier> _tableMapSupplier = new CachingSupplier<>(() -> DataIteratorUtil.createTableMap(getQueryTable(), true)); + private final ValidatorContext _validatorContext; + private final FileColumnValueMapper _fileColumnValueMapping = new FileColumnValueMapper(); + + public DefaultQueryUpdateService(@NotNull TableInfo queryTable, TableInfo dbTable) + { + super(queryTable); + _dbTable = dbTable; + + if (queryTable.getUserSchema() == null) + throw new RuntimeValidationException("User schema not defined for " + queryTable.getName()); + + _validatorContext = new ValidatorContext(queryTable.getUserSchema().getContainer(), queryTable.getUserSchema().getUser()); + } + + public DefaultQueryUpdateService(TableInfo queryTable, TableInfo dbTable, DomainUpdateHelper helper) + { + this(queryTable, dbTable); + _helper = helper; + } + + /** + * @param columnMapping Map from DbTable column names to QueryTable column names, if they have been aliased + */ + public DefaultQueryUpdateService(TableInfo queryTable, TableInfo dbTable, Map columnMapping) + { + this(queryTable, dbTable); + _columnMapping = columnMapping; + } + + protected TableInfo getDbTable() + { + return _dbTable; + } + + protected Domain getDomain() + { + return _helper == null ? null : _helper.getDomain(); + } + + protected ColumnInfo getObjectUriColumn() + { + return _helper == null ? null : _helper.getObjectUriColumn(); + } + + protected String createObjectURI() + { + return _helper == null ? null : _helper.createObjectURI(); + } + + protected Iterable getPropertyColumns() + { + return _helper == null ? Collections.emptyList() : _helper.getPropertyColumns(); + } + + protected Map getColumnMapping() + { + return _columnMapping; + } + + /** + * Returns the container that the domain is defined + */ + protected Container getDomainContainer(Container c) + { + return _helper == null ? c : _helper.getDomainContainer(c); + } + + /** + * Returns the container to insert/update values into + */ + protected Container getDomainObjContainer(Container c) + { + return _helper == null ? c : _helper.getDomainObjContainer(c); + } + + protected Set getAutoPopulatedColumns() + { + return Table.AUTOPOPULATED_COLUMN_NAMES; + } + + public interface DomainUpdateHelper + { + Domain getDomain(); + + ColumnInfo getObjectUriColumn(); + + String createObjectURI(); + + // Could probably be just Iterable or be removed and just get all PropertyDescriptors in the Domain. + Iterable getPropertyColumns(); + + Container getDomainContainer(Container c); + + Container getDomainObjContainer(Container c); + } + + public class ImportHelper implements OntologyManager.ImportHelper + { + ImportHelper() + { + } + + @Override + public String beforeImportObject(Map map) + { + ColumnInfo objectUriCol = getObjectUriColumn(); + + // Get existing Lsid + String lsid = (String) map.get(objectUriCol.getName()); + if (lsid != null) + return lsid; + + // Generate a new Lsid + lsid = createObjectURI(); + map.put(objectUriCol.getName(), lsid); + return lsid; + } + + @Override + public void afterBatchInsert(int currentRow) + { + } + + @Override + public void updateStatistics(int currentRow) + { + } + } + + @Override + protected Map getRow(User user, Container container, Map keys) + throws InvalidKeyException, QueryUpdateServiceException, SQLException + { + aliasColumns(_columnMapping, keys); + Map row = _select(container, getKeys(keys, container)); + + //PostgreSQL includes a column named _row for the row index, but since this is selecting by + //primary key, it will always be 1, which is not only unnecessary, but confusing, so strip it + if (null != row) + { + if (row instanceof ArrayListMap) + ((ArrayListMap) row).getFindMap().remove("_row"); + else + row.remove("_row"); + } + + return row; + } + + protected Map _select(Container container, Object[] keys) throws ConversionException + { + TableInfo table = getDbTable(); + Object[] typedParameters = convertToTypedValues(keys, table.getPkColumns()); + + Map row = new TableSelector(table).getMap(typedParameters); + + ColumnInfo objectUriCol = getObjectUriColumn(); + Domain domain = getDomain(); + if (objectUriCol != null && domain != null && !domain.getProperties().isEmpty() && row != null) + { + String lsid = (String) row.get(objectUriCol.getName()); + if (lsid != null) + { + Map propertyValues = OntologyManager.getProperties(getDomainObjContainer(container), lsid); + if (!propertyValues.isEmpty()) + { + // convert PropertyURI->value map into "Property name"->value map + Map propertyMap = domain.createImportMap(false); + for (Map.Entry entry : propertyValues.entrySet()) + { + String propertyURI = entry.getKey(); + DomainProperty dp = propertyMap.get(propertyURI); + PropertyDescriptor pd = dp != null ? dp.getPropertyDescriptor() : null; + if (pd != null) + row.put(pd.getName(), entry.getValue()); + } + } + } + // Issue 46985: Be tolerant of a row not having an LSID value (as the row may have been + // inserted before the table was made extensible), but make sure that we got an LSID field + // when fetching the row + else if (!row.containsKey(objectUriCol.getName())) + { + throw new IllegalStateException("LSID value not returned when querying table - " + table.getName()); + } + } + + return row; + } + + + private Object[] convertToTypedValues(Object[] keys, List cols) + { + Object[] typedParameters = new Object[keys.length]; + int t = 0; + for (int i = 0; i < keys.length; i++) + { + if (i >= cols.size() || keys[i] instanceof Parameter.TypedValue) + { + typedParameters[t++] = keys[i]; + continue; + } + Object v = keys[i]; + JdbcType type = cols.get(i).getJdbcType(); + if (v instanceof String) + v = type.convert(v); + Parameter.TypedValue tv = new Parameter.TypedValue(v, type); + typedParameters[t++] = tv; + } + return typedParameters; + } + + + @Override + protected Map insertRow(User user, Container container, Map row) + throws DuplicateKeyException, ValidationException, QueryUpdateServiceException, SQLException + { + aliasColumns(_columnMapping, row); + convertTypes(user, container, row); + setSpecialColumns(container, row, user, InsertPermission.class); + validateInsertRow(row); + return _insert(user, container, row); + } + + protected Map _insert(User user, Container c, Map row) + throws SQLException, ValidationException + { + assert (getQueryTable().supportsInsertOption(InsertOption.INSERT)); + + try + { + ColumnInfo objectUriCol = getObjectUriColumn(); + Domain domain = getDomain(); + if (objectUriCol != null && domain != null && !domain.getProperties().isEmpty()) + { + // convert "Property name"->value map into PropertyURI->value map + List pds = new ArrayList<>(); + Map values = new CaseInsensitiveMapWrapper<>(new HashMap<>()); + for (PropertyColumn pc : getPropertyColumns()) + { + PropertyDescriptor pd = pc.getPropertyDescriptor(); + pds.add(pd); + Object value = getPropertyValue(row, pd); + values.put(pd.getPropertyURI(), value); + } + + LsidCollector collector = new LsidCollector(); + OntologyManager.insertTabDelimited(getDomainObjContainer(c), user, null, new ImportHelper(), pds, MapDataIterator.of(Collections.singletonList(values)).getDataIterator(new DataIteratorContext()), true, collector); + String lsid = collector.getLsid(); + + // Add the new lsid to the row map. + row.put(objectUriCol.getName(), lsid); + } + + return Table.insert(user, getDbTable(), row); + } + catch (RuntimeValidationException e) + { + throw e.getValidationException(); + } + catch (BatchValidationException e) + { + throw e.getLastRowError(); + } + } + + @Override + protected Map updateRow(User user, Container container, Map row, @NotNull Map oldRow, @Nullable Map configParameters) + throws InvalidKeyException, ValidationException, QueryUpdateServiceException, SQLException + { + return updateRow(user, container, row, oldRow, false, false); + } + + protected Map updateRow(User user, Container container, Map row, @NotNull Map oldRow, boolean allowOwner, boolean retainCreation) + throws InvalidKeyException, ValidationException, QueryUpdateServiceException, SQLException + { + Map rowStripped = new CaseInsensitiveHashMap<>(row.size()); + + // Flip the key/value pairs around for easy lookup + Map queryToDb = new CaseInsensitiveHashMap<>(); + for (Map.Entry entry : _columnMapping.entrySet()) + { + queryToDb.put(entry.getValue(), entry.getKey()); + } + + setSpecialColumns(container, row, user, UpdatePermission.class); + + Map tableAliasesMap = _tableMapSupplier.get(); + Map> colFrequency = new HashMap<>(); + + //resolve passed in row including columns in the table and other properties (vocabulary properties) not in the Domain/table + for (Map.Entry entry: row.entrySet()) + { + if (!rowStripped.containsKey(entry.getKey())) + { + ColumnInfo col = getQueryTable().getColumn(entry.getKey()); + + if (null == col) + { + col = tableAliasesMap.get(entry.getKey()); + } + + if (null != col) + { + final String name = col.getName(); + + // Skip readonly and wrapped columns. The wrapped column is usually a pk column and can't be updated. + if (col.isReadOnly() || col.isCalculated()) + continue; + + //when updating a row, we should strip the following fields, as they are + //automagically maintained by the table layer, and should not be allowed + //to change once the record exists. + //unfortunately, the Table.update() method doesn't strip these, so we'll + //do that here. + // Owner, CreatedBy, Created, EntityId + if ((!retainCreation && (name.equalsIgnoreCase("CreatedBy") || name.equalsIgnoreCase("Created"))) + || (!allowOwner && name.equalsIgnoreCase("Owner")) + || name.equalsIgnoreCase("EntityId")) + continue; + + // Throw error if more than one row properties having different values match up to the same column. + if (!colFrequency.containsKey(col)) + { + colFrequency.put(col, Pair.of(entry.getKey(),entry.getValue())); + } + else + { + if (!Objects.equals(colFrequency.get(col).second, entry.getValue())) + { + throw new ValidationException("Property key - " + colFrequency.get(col).first + " and " + entry.getKey() + " matched for the same column."); + } + } + + // We want a map using the DbTable column names as keys, so figure out the right name to use + String dbName = queryToDb.getOrDefault(name, name); + rowStripped.put(dbName, entry.getValue()); + } + } + } + + convertTypes(user, container, rowStripped); + validateUpdateRow(rowStripped); + + if (row.get("container") != null) + { + Container rowContainer = UserSchema.translateRowSuppliedContainer(row.get("container"), container, user, getQueryTable(), UpdatePermission.class, null); + if (rowContainer == null) + { + throw new ValidationException("Unknown container: " + row.get("container")); + } + else + { + Container oldContainer = UserSchema.translateRowSuppliedContainer(new CaseInsensitiveHashMap<>(oldRow).get("container"), container, user, getQueryTable(), UpdatePermission.class, null); + if (null != oldContainer && !rowContainer.equals(oldContainer)) + throw new UnauthorizedException("The row is from the wrong container."); + } + } + + Map updatedRow = _update(user, container, rowStripped, oldRow, oldRow == null ? getKeys(row, container) : getKeys(oldRow, container)); + + //when passing a map for the row, the Table layer returns the map of fields it updated, which excludes + //the primary key columns as well as those marked read-only. So we can't simply return the map returned + //from Table.update(). Instead, we need to copy values from updatedRow into row and return that. + row.putAll(updatedRow); + return row; + } + + protected void validateValue(ColumnInfo column, Object value, Object providedValue) throws ValidationException + { + DomainProperty dp = getDomain() == null ? null : getDomain().getPropertyByName(column.getColumnName()); + List validators = ColumnValidators.create(column, dp); + for (ColumnValidator v : validators) + { + String msg = v.validate(-1, value, _validatorContext, providedValue); + if (msg != null) + throw new ValidationException(msg, column.getName()); + } + } + + protected void validateInsertRow(Map row) throws ValidationException + { + for (ColumnInfo col : getQueryTable().getColumns()) + { + Object value = row.get(col.getColumnName()); + + // Check required values aren't null or empty + if (null == value || value instanceof String s && s.isEmpty()) + { + if (!col.isAutoIncrement() && col.isRequired() && + !getAutoPopulatedColumns().contains(col.getName()) && + col.getJdbcDefaultValue() == null) + { + throw new ValidationException("A value is required for field '" + col.getName() + "'", col.getName()); + } + } + else + { + validateValue(col, value, null); + } + } + } + + protected void validateUpdateRow(Map row) throws ValidationException + { + for (ColumnInfo col : getQueryTable().getColumns()) + { + // Only validate incoming values + if (row.containsKey(col.getColumnName())) + { + Object value = row.get(col.getColumnName()); + validateValue(col, value, null); + } + } + } + + protected Map _update(User user, Container c, Map row, Map oldRow, Object[] keys) + throws SQLException, ValidationException + { + assert(getQueryTable().supportsInsertOption(InsertOption.UPDATE)); + + try + { + ColumnInfo objectUriCol = getObjectUriColumn(); + Domain domain = getDomain(); + + // The lsid may be null for the row until a property has been inserted + String lsid = null; + if (objectUriCol != null) + lsid = (String) oldRow.get(objectUriCol.getName()); + + List tableProperties = new ArrayList<>(); + if (objectUriCol != null && domain != null && !domain.getProperties().isEmpty()) + { + // convert "Property name"->value map into PropertyURI->value map + Map newValues = new CaseInsensitiveMapWrapper<>(new HashMap<>()); + + for (PropertyColumn pc : getPropertyColumns()) + { + PropertyDescriptor pd = pc.getPropertyDescriptor(); + tableProperties.add(pd); + + // clear out the old value if it exists and is contained in the new row (it may be incoming as null) + if (lsid != null && (hasProperty(row, pd) && hasProperty(oldRow, pd))) + OntologyManager.deleteProperty(lsid, pd.getPropertyURI(), getDomainObjContainer(c), getDomainContainer(c)); + + Object value = getPropertyValue(row, pd); + if (value != null) + newValues.put(pd.getPropertyURI(), value); + } + + // Note: copy lsid into newValues map so it will be found by the ImportHelper.beforeImportObject() + newValues.put(objectUriCol.getName(), lsid); + + LsidCollector collector = new LsidCollector(); + OntologyManager.insertTabDelimited(getDomainObjContainer(c), user, null, new ImportHelper(), tableProperties, MapDataIterator.of(Collections.singletonList(newValues)).getDataIterator(new DataIteratorContext()), true, collector); + + // Update the lsid in the row: the lsid may have not existed in the row before the update. + lsid = collector.getLsid(); + row.put(objectUriCol.getName(), lsid); + } + + // Get lsid value if it hasn't been set. + // This should only happen if the QueryUpdateService doesn't have a DomainUpdateHelper (DataClass and SampleType) + if (lsid == null && getQueryTable() instanceof UpdateableTableInfo updateableTableInfo) + { + String objectUriColName = updateableTableInfo.getObjectURIColumnName(); + if (objectUriColName != null) + lsid = (String) row.getOrDefault(objectUriColName, oldRow.get(objectUriColName)); + } + + // handle vocabulary properties + if (lsid != null) + { + for (Map.Entry rowEntry : row.entrySet()) + { + String colName = rowEntry.getKey(); + Object value = rowEntry.getValue(); + + ColumnInfo col = getQueryTable().getColumn(colName); + if (col instanceof PropertyColumn propCol) + { + PropertyDescriptor pd = propCol.getPropertyDescriptor(); + if (pd.isVocabulary() && !tableProperties.contains(pd)) + { + OntologyManager.updateObjectProperty(user, c, pd, lsid, value, null, false); + } + } + } + } + } + catch (BatchValidationException e) + { + throw e.getLastRowError(); + } + + checkDuplicateUpdate(keys); + + return Table.update(user, getDbTable(), row, keys); // Cache-invalidation handled in caller (TreatmentManager.saveAssaySpecimen()) + } + + private static class LsidCollector implements OntologyManager.RowCallback + { + private String _lsid; + + @Override + public void rowProcessed(Map row, String lsid) + { + if (_lsid != null) + { + throw new IllegalStateException("Only expected a single LSID"); + } + _lsid = lsid; + } + + public String getLsid() + { + if (_lsid == null) + { + throw new IllegalStateException("No LSID returned"); + } + return _lsid; + } + } + + // Get value from row map where the keys are column names. + private Object getPropertyValue(Map row, PropertyDescriptor pd) + { + if (row.containsKey(pd.getName())) + return row.get(pd.getName()); + + if (row.containsKey(pd.getLabel())) + return row.get(pd.getLabel()); + + for (String alias : pd.getImportAliasSet()) + { + if (row.containsKey(alias)) + return row.get(alias); + } + + return null; + } + + // Checks a value exists in the row map (value may be null) + private boolean hasProperty(Map row, PropertyDescriptor pd) + { + if (row.containsKey(pd.getName())) + return true; + + if (row.containsKey(pd.getLabel())) + return true; + + for (String alias : pd.getImportAliasSet()) + { + if (row.containsKey(alias)) + return true; + } + + return false; + } + + @Override + protected Map deleteRow(User user, Container container, Map oldRowMap) throws QueryUpdateServiceException, SQLException, InvalidKeyException + { + if (oldRowMap == null) + return null; + + aliasColumns(_columnMapping, oldRowMap); + + if (container != null && getDbTable().getColumn("container") != null) + { + // UNDONE: 9077: check container permission on each row before delete + Container rowContainer = UserSchema.translateRowSuppliedContainer(new CaseInsensitiveHashMap<>(oldRowMap).get("container"), container, user, getQueryTable(), DeletePermission.class, null); + if (null != rowContainer && !container.equals(rowContainer)) + { + //Issue 15301: allow workbooks records to be deleted/updated from the parent container + if (container.allowRowMutationForContainer(rowContainer)) + container = rowContainer; + else + throw new UnauthorizedException("The row is from the container: " + rowContainer.getId() + " which does not allow deletes from the container: " + container.getPath()); + } + } + + _delete(container, oldRowMap); + return oldRowMap; + } + + protected void _delete(Container c, Map row) throws InvalidKeyException + { + ColumnInfo objectUriCol = getObjectUriColumn(); + if (objectUriCol != null) + { + String lsid = (String)row.get(objectUriCol.getName()); + if (lsid != null) + { + OntologyObject oo = OntologyManager.getOntologyObject(c, lsid); + if (oo != null) + OntologyManager.deleteProperties(c, oo.getObjectId()); + } + } + Table.delete(getDbTable(), getKeys(row, c)); + } + + // classes should override this method if they need to do more work than delete all the rows from the table + // this implementation will delete all rows from the table for the given container as well as delete + // any properties associated with the table + @Override + protected int truncateRows(User user, Container container) throws QueryUpdateServiceException, SQLException + { + // get rid of the properties for this table + if (null != getObjectUriColumn()) + { + SQLFragment lsids = new SQLFragment() + .append("SELECT t.").append(getObjectUriColumn().getColumnName()) + .append(" FROM ").append(getDbTable(), "t") + .append(" WHERE t.").append(getObjectUriColumn().getColumnName()).append(" IS NOT NULL"); + if (null != getDbTable().getColumn("container")) + { + lsids.append(" AND t.Container = ?"); + lsids.add(container.getId()); + } + + OntologyManager.deleteOntologyObjects(ExperimentService.get().getSchema(), lsids, container); + } + + // delete all the rows in this table, scoping to the container if the column + // is available + if (null != getDbTable().getColumn("container")) + return Table.delete(getDbTable(), SimpleFilter.createContainerFilter(container)); + + return Table.delete(getDbTable()); + } + + protected Object[] getKeys(Map map, Container container) throws InvalidKeyException + { + //build an array of pk values based on the table info + TableInfo table = getDbTable(); + List pks = table.getPkColumns(); + Object[] pkVals = new Object[pks.size()]; + + if (map == null || map.isEmpty()) + return pkVals; + + for (int idx = 0; idx < pks.size(); ++idx) + { + ColumnInfo pk = pks.get(idx); + Object pkValue = map.get(pk.getName()); + // Check the type and coerce if needed + if (pkValue != null && !pk.getJavaObjectClass().isInstance(pkValue)) + { + try + { + pkValue = pk.convert(pkValue); + } + catch (ConversionException ignored) { /* Maybe the database can do the conversion */ } + } + pkVals[idx] = pkValue; + if (null == pkVals[idx] && pk.getColumnName().equalsIgnoreCase("Container")) + { + pkVals[idx] = container; + } + if(null == pkVals[idx]) + { + throw new InvalidKeyException("Value for key field '" + pk.getName() + "' was null or not supplied!", map); + } + } + return pkVals; + } + + private Map _missingValues = null; + private Container _missingValuesContainer; + + protected boolean validMissingValue(Container c, String mv) + { + if (null == c) + return false; + if (null == _missingValues || !c.getId().equals(_missingValuesContainer.getId())) + { + _missingValues = MvUtil.getIndicatorsAndLabels(c); + _missingValuesContainer = c; + } + return _missingValues.containsKey(mv); + } + + protected TableInfo getTableInfoForConversion() + { + return getDbTable(); + } + + final protected void convertTypes(User user, Container c, Map row) throws ValidationException + { + convertTypes(user, c, row, getTableInfoForConversion(), null); + } + + // TODO Path->FileObject + // why is coerceTypes() in AbstractQueryUpdateService and convertTypes() in DefaultQueryUpdateService? + protected void convertTypes(User user, Container c, Map row, TableInfo t, @Nullable Path fileLinkDirPath) throws ValidationException + { + for (ColumnInfo col : t.getColumns()) + { + if (col.isMvIndicatorColumn()) + continue; + boolean isColumnPresent = row.containsKey(col.getName()) || col.isMvEnabled() && row.containsKey(col.getMvColumnName().getName()); + if (!isColumnPresent) + continue; + + Object value = row.get(col.getName()); + + /* NOTE: see MissingValueConvertColumn.convert() these methods should have similar behavior. + * If you update this code, check that code as well. */ + if (col.isMvEnabled()) + { + if (value instanceof String s && StringUtils.isEmpty(s)) + value = null; + + Object mvObj = row.get(col.getMvColumnName().getName()); + String mv = Objects.toString(mvObj, null); + if (StringUtils.isEmpty(mv)) + mv = null; + + if (null != mv) + { + if (!validMissingValue(c, mv)) + throw new ValidationException("Value is not a valid missing value indicator: " + mv); + } + else if (null != value) + { + String s = Objects.toString(value, null); + if (validMissingValue(c, s)) + { + mv = s; + value = null; + } + } + row.put(col.getMvColumnName().getName(), mv); + } + + value = convertColumnValue(col, value, user, c, fileLinkDirPath); + row.put(col.getName(), value); + } + } + + protected Object convertColumnValue(ColumnInfo col, Object value, User user, Container c, @Nullable Path fileLinkDirPath) throws ValidationException + { + // Issue 13951: PSQLException from org.labkey.api.query.DefaultQueryUpdateService._update() + // improve handling of conversion errors + try + { + if (PropertyType.FILE_LINK == col.getPropertyType()) + { + if ((value instanceof MultipartFile || value instanceof AttachmentFile)) + { + FileLike fl = (FileLike)_fileColumnValueMapping.saveFileColumnValue(user, c, fileLinkDirPath, col.getName(), value); + value = fl.toNioPathForRead().toString(); + } + return ExpDataFileConverter.convert(value); + } + return col.getConvertFn().convert(value); + } + catch (ConvertHelper.FileConversionException e) + { + throw new ValidationException(e.getMessage()); + } + catch (ConversionException e) + { + String type = ColumnInfo.getFriendlyTypeName(col.getJdbcType().getJavaClass()); + throw new ValidationException("Unable to convert value '" + value.toString() + "' to " + type, col.getName()); + } + catch (QueryUpdateServiceException e) + { + throw new ValidationException("Save file link failed: " + col.getName()); + } + } + + /** + * Override this method to alter the row before insert or update. + * For example, you can automatically adjust certain column values based on context. + * @param container The current container + * @param row The row data + * @param user The current user + * @param clazz A permission class to test + */ + protected void setSpecialColumns(Container container, Map row, User user, Class clazz) + { + if (null != container) + { + //Issue 15301: allow workbooks records to be deleted/updated from the parent container + if (row.get("container") != null) + { + Container rowContainer = UserSchema.translateRowSuppliedContainer(row.get("container"), container, user, getQueryTable(), clazz, null); + if (rowContainer != null && container.allowRowMutationForContainer(rowContainer)) + { + row.put("container", rowContainer.getId()); //normalize to container ID + return; //accept the row-provided value + } + } + row.put("container", container.getId()); + } + } + + public boolean hasAttachmentProperties() + { + Domain domain = getDomain(); + if (null != domain) + { + for (DomainProperty dp : domain.getProperties()) + if (null != dp && isAttachmentProperty(dp)) + return true; + } + return false; + } + + protected boolean isAttachmentProperty(@NotNull DomainProperty dp) + { + PropertyDescriptor pd = dp.getPropertyDescriptor(); + return PropertyType.ATTACHMENT.equals(pd.getPropertyType()); + } + + protected boolean isAttachmentProperty(String name) + { + DomainProperty dp = getDomain().getPropertyByName(name); + if (dp != null) + return isAttachmentProperty(dp); + return false; + } + + protected void configureCrossFolderImport(DataIteratorBuilder rows, DataIteratorContext context) throws IOException + { + if (!context.getInsertOption().updateOnly && context.isCrossFolderImport() && rows instanceof DataLoader dataLoader) + { + boolean hasContainerField = false; + for (ColumnDescriptor columnDescriptor : dataLoader.getColumns()) + { + String fieldName = columnDescriptor.getColumnName(); + if (fieldName.equalsIgnoreCase("Container") || fieldName.equalsIgnoreCase("Folder")) + { + hasContainerField = true; + break; + } + } + if (!hasContainerField) + context.setCrossFolderImport(false); + } + } +} From d86537c155c5a4ec04d35e6c516a80398d9438c0 Mon Sep 17 00:00:00 2001 From: XingY Date: Thu, 2 Apr 2026 23:14:20 -0700 Subject: [PATCH 3/5] fix --- .../dataiterator/AttachmentDataIterator.java | 23 +++++++++++++++---- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/api/src/org/labkey/api/dataiterator/AttachmentDataIterator.java b/api/src/org/labkey/api/dataiterator/AttachmentDataIterator.java index e7c4ba2efbe..dc8b8ed8754 100644 --- a/api/src/org/labkey/api/dataiterator/AttachmentDataIterator.java +++ b/api/src/org/labkey/api/dataiterator/AttachmentDataIterator.java @@ -15,6 +15,7 @@ */ package org.labkey.api.dataiterator; +import io.micrometer.common.util.StringUtils; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import org.labkey.api.attachments.AttachmentFile; @@ -111,7 +112,8 @@ public boolean next() throws BatchValidationException if (null == attachmentValue) { - if (oldAttachmentValue != null) + // Remove existing attachment + if (!StringUtils.isEmpty(oldAttachmentValue)) oldAttachments.add(oldAttachmentValue); continue; } @@ -121,9 +123,20 @@ public boolean next() throws BatchValidationException if (attachmentValue instanceof String str) { - if (str.equals(oldAttachmentValue)) + if (StringUtils.isEmpty(str)) + { + // Remove existing attachment + if (!StringUtils.isEmpty(oldAttachmentValue)) + oldAttachments.add(oldAttachmentValue); + continue; + } + + if (str.equals(oldAttachmentValue)) // no change continue; + if (!StringUtils.isEmpty(oldAttachmentValue)) // replace old attachment with new attachment, so mark old attachment for deletion + oldAttachments.add(oldAttachmentValue); + if (null == attachmentDir) { errors.addRowError(propertyValidationException(p.domainProperty, attachmentValue)); @@ -175,12 +188,12 @@ else if (attachmentValue instanceof File file) String entityId = String.valueOf(get(entityIdIndex)); var attachmentParent = getAttachmentParent(entityId, container); - if (null != attachmentFiles && !attachmentFiles.isEmpty()) - AttachmentService.get().addAttachments(attachmentParent, attachmentFiles, user); - if (!oldAttachments.isEmpty()) AttachmentService.get().deleteAttachments(attachmentParent, oldAttachments, user); + if (null != attachmentFiles && !attachmentFiles.isEmpty()) + AttachmentService.get().addAttachments(attachmentParent, attachmentFiles, user); + return ret; } catch (AttachmentService.DuplicateFilenameException | AttachmentService.FileTooLargeException e) From 0150443a65e0488be074f82e8870e4b3bd4a9596 Mon Sep 17 00:00:00 2001 From: XingY Date: Fri, 3 Apr 2026 09:28:41 -0700 Subject: [PATCH 4/5] fix import --- api/src/org/labkey/api/dataiterator/AttachmentDataIterator.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/src/org/labkey/api/dataiterator/AttachmentDataIterator.java b/api/src/org/labkey/api/dataiterator/AttachmentDataIterator.java index dc8b8ed8754..e4e00ae8a1f 100644 --- a/api/src/org/labkey/api/dataiterator/AttachmentDataIterator.java +++ b/api/src/org/labkey/api/dataiterator/AttachmentDataIterator.java @@ -15,7 +15,7 @@ */ package org.labkey.api.dataiterator; -import io.micrometer.common.util.StringUtils; +import org.apache.commons.lang3.StringUtils; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import org.labkey.api.attachments.AttachmentFile; From ddc907a699188debc5b46b124f58b5a764afa4d1 Mon Sep 17 00:00:00 2001 From: XingY Date: Sun, 5 Apr 2026 20:58:39 -0700 Subject: [PATCH 5/5] Selenium test for GitHub Issue 915: Bulk edit doesn't completely remove attachments for sources --- .../org/labkey/api/dataiterator/AttachmentDataIterator.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/api/src/org/labkey/api/dataiterator/AttachmentDataIterator.java b/api/src/org/labkey/api/dataiterator/AttachmentDataIterator.java index e4e00ae8a1f..e636cf4e974 100644 --- a/api/src/org/labkey/api/dataiterator/AttachmentDataIterator.java +++ b/api/src/org/labkey/api/dataiterator/AttachmentDataIterator.java @@ -156,11 +156,15 @@ else if (attachmentValue instanceof AttachmentFile file) { attachmentFile = file; filename = attachmentFile.getFilename(); + if (!StringUtils.isEmpty(oldAttachmentValue)) + oldAttachments.add(oldAttachmentValue); } else if (attachmentValue instanceof File file) { attachmentFile = new FileAttachmentFile(file); filename = attachmentFile.getFilename(); + if (!StringUtils.isEmpty(oldAttachmentValue)) + oldAttachments.add(oldAttachmentValue); } else {