[bot] Merge 26.3 to 26.4

api/src/org/labkey/api/exp/list/ListService.java

@@ -51,6 +51,7 @@ static void setInstance(ListService ls)
     boolean hasLists(Container container, boolean includeProjectAndShared);
     ListDefinition createList(Container container, String name, ListDefinition.KeyType keyType);
     ListDefinition createList(Container container, String name, ListDefinition.KeyType keyType, @Nullable TemplateInfo templateInfo, @Nullable ListDefinition.Category category);
+    void deleteLists(Container container, User user, @Nullable String auditUserComment);
     @Nullable ListDefinition getList(Container container, int listId);
     @Nullable ListDefinition getList(Container container, String name);
     @Nullable ListDefinition getList(Container container, String name, boolean includeProjectAndShared);

api/src/org/labkey/api/exp/property/AbstractDomainKind.java

@@ -206,7 +206,7 @@ protected boolean getBlanksSql(Domain domain, DomainProperty prop, SQLFragment b
         else
         {
             // Issue 29047
-            blankRowsSQL.appendIdentifier(columnId).append(" IS NOT NULL");
+            blankRowsSQL.appendIdentifier(columnId).append(" IS NULL");
         }
         if (prop.isMvEnabled())
         {

api/src/org/labkey/api/query/QueryService.java

@@ -16,18 +16,38 @@
 
 package org.labkey.api.query;
 
+import jakarta.servlet.http.HttpSession;
 import org.apache.commons.collections4.SetValuedMap;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
 import org.json.JSONObject;
 import org.labkey.api.audit.AuditHandler;
 import org.labkey.api.audit.DetailedAuditTypeEvent;
+import org.labkey.api.data.ColumnHeaderType;
+import org.labkey.api.data.ColumnInfo;
+import org.labkey.api.data.CompareType;
+import org.labkey.api.data.Container;
+import org.labkey.api.data.ContainerFilter;
+import org.labkey.api.data.DbSchema;
+import org.labkey.api.data.DisplayColumn;
+import org.labkey.api.data.Filter;
+import org.labkey.api.data.JdbcType;
+import org.labkey.api.data.MethodInfo;
+import org.labkey.api.data.MutableColumnInfo;
+import org.labkey.api.data.ParameterDescription;
+import org.labkey.api.data.ParameterDescriptionImpl;
+import org.labkey.api.data.QueryLogging;
+import org.labkey.api.data.Results;
+import org.labkey.api.data.SQLFragment;
+import org.labkey.api.data.Sort;
+import org.labkey.api.data.SqlSelector;
+import org.labkey.api.data.TableInfo;
+import org.labkey.api.data.TableSelector;
+import org.labkey.api.data.dialect.SqlDialect;
 import org.labkey.api.gwt.client.model.GWTPropertyDescriptor;
+import org.labkey.api.module.Module;
 import org.labkey.api.pipeline.PipelineJob;
 import org.labkey.api.query.column.ColumnInfoTransformer;
-import org.labkey.api.data.*;
-import org.labkey.api.data.dialect.SqlDialect;
-import org.labkey.api.module.Module;
 import org.labkey.api.query.column.ConceptURIColumnInfoTransformer;
 import org.labkey.api.query.snapshot.QuerySnapshotDefinition;
 import org.labkey.api.security.User;
@@ -41,7 +61,6 @@
 import org.labkey.data.xml.TableType;
 import org.springframework.web.servlet.mvc.Controller;
 
-import jakarta.servlet.http.HttpSession;
 import java.io.IOException;
 import java.sql.ResultSet;
 import java.sql.SQLException;
@@ -653,6 +672,13 @@ default MutableColumnInfo applyColumnTransformer(MutableColumnInfo col)
     @Nullable
     ContainerFilter getContainerFilterForLookups(Container container, User user);
 
+    /**
+     * Provides the configured ContainerFilter to utilize when requesting data that is being read
+     * within a folder context. Equivalent to the client side function in @labkey/components
+     */
+    @Nullable
+    ContainerFilter getContainerFilterForFolder(Container container, User user);
+
 
     interface SelectBuilder
     {

api/src/org/labkey/api/study/publish/StudyPublishService.java

@@ -156,7 +156,7 @@ ActionURL publishData(User user, Container sourceContainer, @Nullable Container
 
     String checkForLockedLinks(Dataset def, @Nullable List<Long> rowIds);
 
-    void addRecallAuditEvent(Container sourceContainer, User user, Dataset def, int rowCount, @Nullable Collection<Pair<String,Long>> datasetRowLsidAndSourceRowIds);
+    void addRecallAuditEvent(Container sourceContainer, User user, Dataset def, int rowCount, @Nullable Collection<Long> rowIds);
 
     /**
      * Adds columns to an assay data table, providing a link to any datasets that have

api/src/org/labkey/api/util/XmlBeansUtil.java

@@ -132,6 +132,7 @@ public static void addComment(XmlTokenSource doc, String comment)
     public static final SAXParserFactory SAX_PARSER_FACTORY_ALLOWING_DOCTYPE;
     public static final XMLInputFactory XML_INPUT_FACTORY;
     public static final DocumentBuilderFactory DOCUMENT_BUILDER_FACTORY;
+    public static final DocumentBuilderFactory DOCUMENT_BUILDER_FACTORY_ALLOWING_DOCTYPE;
 
     static
     {
@@ -145,16 +146,9 @@ public static void addComment(XmlTokenSource doc, String comment)
             SAX_PARSER_FACTORY = saxParserFactory(false);
             SAX_PARSER_FACTORY_ALLOWING_DOCTYPE = saxParserFactory(true);
 
-            //noinspection XMLInputFactory
-            DOCUMENT_BUILDER_FACTORY = DocumentBuilderFactory.newInstance();
-            DOCUMENT_BUILDER_FACTORY.setNamespaceAware(true);
-            DOCUMENT_BUILDER_FACTORY.setFeature("http://apache.org/xml/features/disallow-doctype-decl", true);
-            DOCUMENT_BUILDER_FACTORY.setFeature("http://xml.org/sax/features/external-general-entities", false);
-            DOCUMENT_BUILDER_FACTORY.setFeature("http://xml.org/sax/features/external-parameter-entities", false);
-            DOCUMENT_BUILDER_FACTORY.setFeature("http://apache.org/xml/features/nonvalidating/load-external-dtd", false);
-            DOCUMENT_BUILDER_FACTORY.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true);
-            DOCUMENT_BUILDER_FACTORY.setXIncludeAware(false);
-            DOCUMENT_BUILDER_FACTORY.setExpandEntityReferences(false);
+            DOCUMENT_BUILDER_FACTORY = documentBuilderFactory(false);
+            // Use the ALLOWING_DOCTYPE variant when parsing XML that contains a <!DOCTYPE> declaration (e.g. NCBI's eSummary responses)
+            DOCUMENT_BUILDER_FACTORY_ALLOWING_DOCTYPE = documentBuilderFactory(true);
         }
         catch (ParserConfigurationException | SAXException e)
         {
@@ -181,4 +175,26 @@ private static SAXParserFactory saxParserFactory(boolean allowDocType) throws SA
         result.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true);
         return result;
     }
+
+    private static DocumentBuilderFactory documentBuilderFactory(boolean allowDocType) throws ParserConfigurationException
+    {
+        //noinspection XMLInputFactory
+        DocumentBuilderFactory result = DocumentBuilderFactory.newInstance();
+        result.setNamespaceAware(true);
+
+        // Disable features that could lead to XXE or other vulnerabilities.
+        // When allowDocType is true the DOCTYPE declaration is permitted. External entity
+        // resolution remains disabled, so XXE protection is still in effect.
+        if (!allowDocType)
+        {
+            result.setFeature("http://apache.org/xml/features/disallow-doctype-decl", true);
+        }
+        result.setFeature("http://xml.org/sax/features/external-general-entities", false);
+        result.setFeature("http://xml.org/sax/features/external-parameter-entities", false);
+        result.setFeature("http://apache.org/xml/features/nonvalidating/load-external-dtd", false);
+        result.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true);
+        result.setXIncludeAware(false);
+        result.setExpandEntityReferences(false);
+        return result;
+    }
 }

core/src/org/labkey/core/admin/sql/ScriptReorderer.java

@@ -118,6 +118,7 @@ public String getReorderedScript(boolean isHtml)
             patterns.add(new SqlPattern("DROP INDEX (IF EXISTS )?\\w+ ON " + TABLE_NAME_REGEX + STATEMENT_ENDING_REGEX, Type.Table, Operation.Other));
 
             patterns.add(new SqlPattern("(CREATE|ALTER) PROCEDURE .+?" + STATEMENT_ENDING_REGEX, Type.NonTable, Operation.Other));
+            patterns.add(new SqlPattern("(CREATE|ALTER) TRIGGER .+?" + "END GO\\s*$", Type.NonTable, Operation.Other));
             patterns.add(new SqlPattern("ALTER TABLE " + TABLE_NAME_REGEX + " CHECK CONSTRAINT " + CONSTRAINT_NAME_REGEX + STATEMENT_ENDING_REGEX, Type.Table, Operation.Other));
         }
         else

core/src/org/labkey/core/admin/sql/SqlScriptController.java

@@ -1311,10 +1311,10 @@ protected ActionURL getSaveScriptActionURL(SqlScript script, String newContents,
         - `EXEC core.fn_dropifexists 'MyTable', 'MySchema', 'COLUMN', 'MyColumn` is the same as `ALTER TABLE TableName DROP COLUMN IF EXISTS ColumnName`
 
         Please do the following:
-        - Consolidate all iterative changes (column additions, PK changes, and renames) into the initial CREATE TABLE statements.
+        - Consolidate all iterative changes (column additions & renames, PK changes, and FK changes) into the initial CREATE TABLE statements.
         - Remove unnecessary DROP TABLE statements and core.fn_dropifexists calls, for example, those that come before a table has been created.
         - Remove all intermediate DROP and ALTER statements that are superseded by later logic.
-        - Remove CREATE TABLE and ALTER TABLE statements followed by DROP TABLE or and core.fn_dropifexists 'TABLE' call on that same table.
+        - Remove CREATE TABLE and ALTER TABLE statements followed by DROP TABLE or a core.fn_dropifexists 'TABLE' call on that same table.
 
         Include a summary of the changes you made at the end.
         """;

experiment/src/org/labkey/experiment/api/ExperimentServiceImpl.java

@@ -5497,18 +5497,9 @@ public void deleteAllExpObjInContainer(Container c, User user) throws Experiment
             {
                 deleteExperimentRunsByRowIds(c, user, runId);
             }
-            ListService ls = ListService.get();
-            if (ls != null)
-            {
-                for (ListDefinition list : ListService.get().getLists(c, null, false).values())
-                {
-                    // Temporary fix for Issue 21400: **Deleting workbook deletes lists defined in parent container
-                    if (list.getContainer().equals(c))
-                    {
-                        list.delete(user);
-                    }
-                }
-            }
+
+            // Delete lists (because for some reason lists are under the purview of experiment...)
+            ListService.get().deleteLists(c, user, null);
 
             // Delete DataClasses and their exp.Data members
             // Need to delete DataClass before SampleTypes since they may be referenced by the DataClass

issues/src/org/labkey/issue/model/IssueManager.java

@@ -948,6 +948,11 @@ private static class IndexGroup implements Consumer<SearchService.TaskIndexingQu
         public void accept(SearchService.TaskIndexingQueue a)
         {
             User user = new LimitedUser(UserManager.getGuestUser(), ReaderRole.class);
+            if (IssuesListDefService.get().getRestrictedIssueProvider() != null)
+            {
+                // Pass in an admin user to allow all restricted issues to be indexed by the crawler
+                user = User.getAdminServiceUser();
+            }
             indexIssues(a, user, _ids);
         }
     }
@@ -968,16 +973,9 @@ public static void indexIssues(SearchService.TaskIndexingQueue queue, User user,
 
         for (Integer id : ids)
         {
-            try
-            {
-                IssueObject issue = IssueManager.getIssue(container, user, id);
-                if (issue != null)
-                    queueIssue(queue, id, issue.getProperties(), issue.getCommentObjects());
-            }
-            catch (UnauthorizedException e)
-            {
-                // Issue 51607 ignore restricted issue failures
-            }
+            IssueObject issue = IssueManager.getIssue(container, user, id, false);
+            if (issue != null)
+                queueIssue(queue, id, issue.getProperties(), issue.getCommentObjects());
         }
     }
 
@@ -1015,6 +1013,8 @@ public WebdavResource resolve(@NotNull String resourceIdentifier)
             public HttpView getCustomSearchResult(User user, @NotNull String resourceIdentifier)
             {
                 int issueId;
+                boolean isRestricted = false;       // controls rendering for a restricted issue
+
                 try
                 {
                     issueId = Integer.parseInt(resourceIdentifier);
@@ -1024,23 +1024,34 @@ public HttpView getCustomSearchResult(User user, @NotNull String resourceIdentif
                     return null;
                 }
 
-                final IssueObject issue = getIssue(null, user, issueId, false);
+                IssueObject issue = getIssue(null, user, issueId, false);
                 if (null == issue)
-                    return null;
+                {
+                    if (IssuesListDefService.get().getRestrictedIssueProvider() != null)
+                    {
+                        // allow users to see the summary of a restricted issue, but there will be limited
+                        // information that is rendered.
+                        issue = getIssue(null, User.getAdminServiceUser(), issueId, false);
+                        isRestricted = true;
+                    }
+
+                    if (issue == null)
+                        return null;
+                }
                 Container c = issue.lookupContainer();
                 if (null == c || !c.hasPermission(user, ReadPermission.class))
                     return null;
 
-                return new IssueSummaryView(issue);
+                return new IssueSummaryView(issue, isRestricted);
             }
         };
     }
 
     public static class IssueSummaryView extends JspView
     {
-        IssueSummaryView(IssueObject issue)
+        IssueSummaryView(IssueObject issue, boolean isRestricted)
         {
-            super("/org/labkey/issue/view/searchSummary.jsp", issue);
+            super("/org/labkey/issue/view/searchSummary.jsp", new Pair<>(issue, isRestricted));
         }
     }
 

issues/src/org/labkey/issue/view/searchSummary.jsp

@@ -23,12 +23,15 @@
 <%@ page import="java.util.regex.Matcher" %>
 <%@ page import="java.util.regex.Pattern" %>
 <%@ page import="org.apache.commons.lang3.Strings" %>
+<%@ page import="org.labkey.api.util.Pair" %>
 <%@ taglib prefix="labkey" uri="http://www.labkey.org/taglib" %>
 <%@ page extends="org.labkey.api.jsp.JspBase" %>
 <%
-    JspView<IssueObject> me = HttpView.currentView();
-    final IssueObject issue = me.getModelBean();
+    JspView<Pair<IssueObject, Boolean>> me = HttpView.currentView();
+    final Pair<IssueObject, Boolean> rec = me.getModelBean();
     final User user = getUser();
+    final IssueObject issue = rec.first;
+    final Boolean isRestricted = rec.second;
     final boolean isClosed = Strings.CI.equals(issue.getStatus(),"closed");
     final boolean isOpen = Strings.CI.equals(issue.getStatus(),"open");
 %>
@@ -47,29 +50,36 @@
     <%
         StringBuilder html = new StringBuilder();
         boolean hasTextComment = false;
-        for (IssueObject.CommentObject comment : issue.getCommentObjects())
+        if (isRestricted)
         {
-            String s = comment.getHtmlComment().toString();
-            String pattern1 = "<div class=\"labkey-wiki\">";
-            String pattern2 = "</div>";
-            String regexString = Pattern.quote(pattern1) + "(?s)(.*?)" + Pattern.quote(pattern2);
-            Pattern p = Pattern.compile(regexString);
-            Matcher matcher = p.matcher(s);
-            while (matcher.find())
+            html.append("<div class=\"labkey-error\">Restricted Issue: You do not have access. Contact your administrator for access.</div>");
+        }
+        else
+        {
+            for (IssueObject.CommentObject comment : issue.getCommentObjects())
             {
-                String commentContentText = matcher.group(1);
-                if (!StringUtils.isEmpty(commentContentText))
+                String s = comment.getHtmlComment().toString();
+                String pattern1 = "<div class=\"labkey-wiki\">";
+                String pattern2 = "</div>";
+                String regexString = Pattern.quote(pattern1) + "(?s)(.*?)" + Pattern.quote(pattern2);
+                Pattern p = Pattern.compile(regexString);
+                Matcher matcher = p.matcher(s);
+                while (matcher.find())
                 {
-                    hasTextComment = true;
-                    html.append(commentContentText);
-                    html.append("<br>");
-                    if (html.length() > 500)
-                        break;
+                    String commentContentText = matcher.group(1);
+                    if (!StringUtils.isEmpty(commentContentText))
+                    {
+                        hasTextComment = true;
+                        html.append(commentContentText);
+                        html.append("<br>");
+                        if (html.length() > 500)
+                            break;
+                    }
                 }
             }
         }
 
-    if (hasTextComment) { %>
+    if (hasTextComment && !isRestricted) { %>
         <label style="text-decoration: underline">Comments</label>
     <% } %>
     <div style="max-height:4em; overflow-y:hidden; word-wrap:break-word; white-space: normal; display: -webkit-box; -webkit-line-clamp: 3; -webkit-box-orient: vertical;"><%= unsafe(html.toString()) %></div>

list/src/org/labkey/list/model/ListDefinitionImpl.java

@@ -565,37 +565,9 @@ public void delete(User user) throws DomainNotFoundException
     @Override
     public void delete(User user, @Nullable String auditUserComment) throws DomainNotFoundException
     {
-        TableInfo table = getTable(user);
-        QueryUpdateService qus = null;
-
-        if (null != table)
-            qus = table.getUpdateService();
-
-        // In certain cases we may create a list that is not viable (i.e., one in which a table was never created because
-        // the metadata wasn't valid). Still allow deleting the list
-        try (DbScope.Transaction transaction = (table != null) ? table.getSchema().getScope().ensureTransaction() :
-             ExperimentService.get().ensureTransaction())
-        {
-            // remove related full-text search docs and attachments
-            ListManager.get().deleteIndexedList(this);
-            if (qus instanceof ListQueryUpdateService listQus)
-                listQus.deleteRelatedListData(null);
-
-            // then delete the list itself
-            ListManager.get().deleteListDef(getContainer(), getListId());
-            Domain domain = getDomainOrThrow();
-            domain.delete(user, auditUserComment);
-
-            ListManager.get().addAuditEvent(this, user, String.format("The list %s was deleted", _def.getName()));
-
-            transaction.commit();
-        }
-
-        SchemaKey schemaPath = SchemaKey.fromParts(ListQuerySchema.NAME);
-        QueryService.get().fireQueryDeleted(user, getContainer(), null, schemaPath, Collections.singleton(getName()));
+        ListManager.get().deleteList(user, this, auditUserComment);
     }
 
-
     @Override
     public int insertListItems(User user, Container container, List<ListItem> listItems)
     {