Merge remote-tracking branch 'origin/develop' into fb_requireComments

LabKey · Jan 19, 2024 · b805a65 · b805a65
2 parents d93b299 + 4cadccd
commit b805a65
Show file tree

Hide file tree

Showing 17 changed files with 69 additions and 19 deletions.
diff --git a/api/src/org/labkey/api/data/JsonTest.java b/api/src/org/labkey/api/data/JsonTest.java
@@ -101,12 +101,14 @@ public void jsonOrgViaJackson() throws IOException
         obj.put("str", "hello");
         obj.put("arr", new JSONArray(Arrays.asList("one", null, 3, new JSONObject(Collections.singletonMap("four", 4)))));
         obj.put("nul", (Object)null);
+        obj.put("key", "</tricky\tvalue\\");
 //        obj.put("d", d);  //TODO: new JSONObject serializes date-times as ISO
 
         // Verify serializing JSONObject via Jackson is equivalent
         String jacksonToString = mapper.writeValueAsString(obj);
         String jsonOrgToString = obj.toString();
         assertEquals(jsonOrgToString, jacksonToString);
+        assertTrue(jsonOrgToString.contains("<\\/"));
 
         // Verify deserializing JSONObject via Jackson is equivalent
         // NOTE: In both cases, the date value is deserialized as a string because JSON sucks

diff --git a/api/src/org/labkey/api/exp/api/ExperimentService.java b/api/src/org/labkey/api/exp/api/ExperimentService.java
@@ -143,7 +143,8 @@ enum DataTypeForExclusion
         SampleType,
         DataClass,
         AssayDesign,
-        StorageLocation
+        StorageLocation,
+        DashboardSampleType
     }
 
     @Nullable
@@ -962,7 +963,7 @@ List<? extends ExpProtocol> getExpProtocolsWithParameterValue(
 
     Set<String> getDataTypeContainerExclusions(@NotNull DataTypeForExclusion dataType, @NotNull Integer dataTypeRowId);
 
-    void ensureContainerDataTypeExclusions(@NotNull DataTypeForExclusion dataType, @Nullable Collection<Integer> excludedDataTypeRowIds, @NotNull String excludedContainerId, User user);
+    void ensureContainerDataTypeExclusions(@NotNull DataTypeForExclusion dataType, @Nullable DataTypeForExclusion relatedDataType, @Nullable Collection<Integer> excludedDataTypeRowIds, @NotNull String excludedContainerId, User user);
 
     void ensureDataTypeContainerExclusions(@NotNull DataTypeForExclusion dataType, @Nullable Collection<String> excludedContainerIds, @NotNull Integer dataTypeId, User user);
 

diff --git a/api/src/org/labkey/api/exp/api/SampleTypeDomainKind.java b/api/src/org/labkey/api/exp/api/SampleTypeDomainKind.java
@@ -537,6 +537,7 @@ public Domain createDomain(GWTDomain domain, @Nullable SampleTypeDomainKindPrope
         String category = null;
         Map<String, String> aliases = null;
         List<String> excludedContainerIds = null;
+        List<String> excludedDashboardContainerIds = null;
 
         if (arguments != null)
         {
@@ -557,12 +558,13 @@ public Domain createDomain(GWTDomain domain, @Nullable SampleTypeDomainKindPrope
             category = StringUtils.trimToNull(arguments.getCategory());
             aliases = arguments.getImportAliases();
             excludedContainerIds = arguments.getExcludedContainerIds();
+            excludedDashboardContainerIds = arguments.getExcludedDashboardContainerIds();
         }
         ExpSampleType st;
         try
         {
             st = SampleTypeService.get().createSampleType(container, user, name, description, properties, indices, idCol1, idCol2, idCol3, parentCol, nameExpression, aliquotNameExpression,
-                    templateInfo, aliases, labelColor, metricUnit, autoLinkTargetContainer, autoLinkCategory, category, domain.getDisabledSystemFields(), excludedContainerIds);
+                    templateInfo, aliases, labelColor, metricUnit, autoLinkTargetContainer, autoLinkCategory, category, domain.getDisabledSystemFields(), excludedContainerIds, excludedDashboardContainerIds);
         }
         catch (SQLException e)
         {

diff --git a/api/src/org/labkey/api/exp/api/SampleTypeDomainKindProperties.java b/api/src/org/labkey/api/exp/api/SampleTypeDomainKindProperties.java
@@ -65,6 +65,7 @@ public SampleTypeDomainKindProperties(ExpSampleType st)
     private Integer parentCol;
     private String category;
     private List<String> excludedContainerIds;
+    private List<String> excludedDashboardContainerIds;
 
     //Ignored on import/save, use Domain.name & Domain.description instead
     private String name;
@@ -236,4 +237,14 @@ public void setExcludedContainerIds(List<String> excludedContainerIds)
     {
         this.excludedContainerIds = excludedContainerIds;
     }
+
+    public List<String> getExcludedDashboardContainerIds()
+    {
+        return excludedDashboardContainerIds;
+    }
+
+    public void setExcludedDashboardContainerIds(List<String> excludedDashboardContainerIds)
+    {
+        this.excludedDashboardContainerIds = excludedDashboardContainerIds;
+    }
 }
diff --git a/api/src/org/labkey/api/exp/api/SampleTypeService.java b/api/src/org/labkey/api/exp/api/SampleTypeService.java
@@ -122,7 +122,8 @@ ExpSampleType createSampleType(Container container, User user, String name, Stri
     @NotNull
     ExpSampleType createSampleType(Container c, User u, String name, String description, List<GWTPropertyDescriptor> properties, List<GWTIndex> indices, int idCol1, int idCol2, int idCol3, int parentCol,
                                               String nameExpression, String aliquotNameExpression, @Nullable TemplateInfo templateInfo, @Nullable Map<String, String> importAliases, @Nullable String labelColor, @Nullable String metricUnit,
-                                              @Nullable Container autoLinkTargetContainer, @Nullable String autoLinkCategory, @Nullable String category, @Nullable List<String> disabledSystemField, @Nullable List<String> excludedContainerIds)
+                                              @Nullable Container autoLinkTargetContainer, @Nullable String autoLinkCategory, @Nullable String category, @Nullable List<String> disabledSystemField,
+                                              @Nullable List<String> excludedContainerIds, @Nullable List<String> excludedDashboardContainerIds)
             throws ExperimentException, SQLException;
 
     @NotNull

diff --git a/api/src/org/labkey/api/notification/notificationpanel.jsp b/api/src/org/labkey/api/notification/notificationpanel.jsp
@@ -43,7 +43,7 @@
     %>
     <a id="<%=h(linkId)%>" href="#">
         <i class="fa fa-inbox labkey-notification-inbox"></i>
-        <span id=<%=q(notificationCountId)%>>&nbsp;</span>
+        <span id="<%=h(notificationCountId)%>">&nbsp;</span>
     </a>
 </li>
 

diff --git a/api/src/org/labkey/api/reports/report/r/RReport.java b/api/src/org/labkey/api/reports/report/r/RReport.java
@@ -219,6 +219,7 @@ public static synchronized String getDefaultRPath()
         return DEFAULT_APP_PATH;
     }
 
+
     public static String toR(String s)
     {
         if (s == null)
@@ -259,6 +260,7 @@ public static String toR(String s)
         return r.toString();
     }
 
+
     // static for access by RserveScriptEngine with no backing report
     public static void appendParamList(StringBuilder labkey, Map<String, Object> inputParameters)
     {

diff --git a/core/src/org/labkey/core/CoreModule.java b/core/src/org/labkey/core/CoreModule.java
@@ -15,6 +15,7 @@
  */
 package org.labkey.core;
 
+import com.fasterxml.jackson.core.io.CharTypes;
 import com.google.common.collect.Sets;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.logging.log4j.LogManager;
@@ -317,6 +318,18 @@ public class CoreModule extends SpringModule implements SearchService.DocumentPr
 
         // Register dialect extra early, since we need to initialize the data sources before calling DefaultModule.initialize()
         SqlDialectRegistry.register(new PostgreSqlDialectFactory());
+
+        try
+        {
+            var field = CharTypes.class.getDeclaredField("sOutputEscapes128");
+            field.setAccessible(true);
+            ((int[])field.get(null))['/'] = '/';
+            field.setAccessible(false);
+        }
+        catch (NoSuchFieldException|IllegalArgumentException|IllegalAccessException x)
+        {
+            // pass
+        }
     }
 
     private CoreWarningProvider _warningProvider;

diff --git a/core/src/org/labkey/core/admin/AdminController.java b/core/src/org/labkey/core/admin/AdminController.java
@@ -11120,6 +11120,10 @@ public Object execute(SimpleApiJsonForm form, BindException errors) throws Excep
             if (!_log.isWarnEnabled())
                 return ret;
 
+            var userAgent = getViewContext().getRequest().getHeader("User-Agent");
+            if (PageFlowUtil.isRobotUserAgent(userAgent) && !_log.isDebugEnabled())
+                return ret;
+
             // NOTE User will always be "guest".  Seems like a bad design to force the server to accept guest w/o CSRF here.
             var jsonObj = form.getJsonObject();
             if (null != jsonObj)
@@ -11131,9 +11135,8 @@ public Object execute(SimpleApiJsonForm form, BindException errors) throws Excep
                     if (urlString != null)
                     {
                         String path = new URLHelper(urlString).deleteParameters().getPath();
-                        if (null == reports.put(path, Boolean.TRUE))
+                        if (null == reports.put(path, Boolean.TRUE) || _log.isDebugEnabled())
                         {
-                            var userAgent = getViewContext().getRequest().getHeader("User-Agent");
                             if (isNotBlank(userAgent))
                                 jsonObj.put("user-agent", userAgent);
                             var jsonStr = jsonObj.toString(2);

diff --git a/core/src/org/labkey/core/analytics/AnalyticsServiceImpl.java b/core/src/org/labkey/core/analytics/AnalyticsServiceImpl.java
@@ -205,7 +205,7 @@ private String getSanitizedUrl(ViewContext context)
     private static final String GA4_TRACKING_SCRIPT_TEMPLATE =
             """
                     <!-- Global site tag (gtag.js) - Google Analytics -->
-                    <script async src="${GA4_JS:htmlEncode}"></script>
+                    <script async src="${GA4_JS:htmlEncode}" nonce="${SCRIPT_NONCE:htmlEncode}"></script>
                     <script nonce="${SCRIPT_NONCE:htmlEncode}">
                       window.dataLayer = window.dataLayer || [];
                       function gtag(){dataLayer.push(arguments);}

diff --git a/core/src/org/labkey/core/analytics/analyticsSettings.jsp b/core/src/org/labkey/core/analytics/analyticsSettings.jsp
@@ -70,10 +70,10 @@
             <td style="padding-left: 1em;">
                 <strong><label for="customScript">Custom JavaScript Analytics</label></strong>
                 <p>
-                    Add <label for="ff_trackingScript">custom analytics script</label> to the <code>&lt;head&gt;</code> of every page.  Include required <code>&lt;script&gt;</code> tags.
+                    Add <label for="ff_trackingScript">custom analytics script</label> to the <code>&lt;head&gt;</code> of every page. Include required <code>&lt;script&gt;</code> tags. If the server enforces a Content Security Policy, script blocks may need a nonce to function: <code>&lt;script nonce="\${SCRIPT_NONCE:htmlEncode}"&gt;</code>.
                 </p>
                 <p>
-                    <strong>NOTE:</strong> You can mess up your site if you make a mistake here.  You may want to bookmark this page to aid in making corrections, just in case.
+                    <strong>NOTE:</strong> You can mess up your site if you make a mistake here. You may want to bookmark this page to aid in making corrections, just in case.
                 </p>
                 <textarea <%=unsafe(hasAdminOpsPerms?"":"disabled=\"disabled\"")%> style="width:100%; height:15em;" id="ff_trackingScript" name="ff_trackingScript"><%=h(settingsForm.ff_trackingScript)%></textarea>
             </td>

diff --git a/core/src/org/labkey/core/user/securityAccess.jsp b/core/src/org/labkey/core/user/securityAccess.jsp
@@ -79,7 +79,7 @@
     However, if this account were re-enabled, it would have the following permissions.</div>
 <% } %>
 
-<table id=<%=q(accessRegion.getDomId())%> lk-region-name=<%=q(accessRegion.getName())%> class="labkey-data-region-legacy labkey-show-borders">
+<table id="<%=h(accessRegion.getDomId())%>" lk-region-name="<%=h(accessRegion.getName())%>" class="labkey-data-region-legacy labkey-show-borders">
     <colgroup><col><col><col></colgroup>
     <tr id="dataregion_column_header_row_access">
         <th>&nbsp;</th>

diff --git a/experiment/src/org/labkey/experiment/api/ExperimentServiceImpl.java b/experiment/src/org/labkey/experiment/api/ExperimentServiceImpl.java
@@ -8643,12 +8643,13 @@ private Set<Integer> _getContainerDataTypeExclusions(DataTypeForExclusion dataTy
     }
 
     @Override
-    public void ensureContainerDataTypeExclusions(@NotNull DataTypeForExclusion dataType, @Nullable Collection<Integer> excludedDataTypeRowIds, @NotNull String excludedContainerId, User user)
+    public void ensureContainerDataTypeExclusions(@NotNull DataTypeForExclusion dataType, @Nullable DataTypeForExclusion relatedDataType, @Nullable Collection<Integer> excludedDataTypeRowIds, @NotNull String excludedContainerId, User user)
     {
         if (excludedDataTypeRowIds == null)
             return;
 
         Set<Integer> previousExclusions = _getContainerDataTypeExclusions(dataType, excludedContainerId);
+        Set<Integer> relatedExclusions = relatedDataType != null ? _getContainerDataTypeExclusions(relatedDataType, excludedContainerId) : null;
         Set<Integer> updatedExclusions = new HashSet<>(excludedDataTypeRowIds);
 
         Set<Integer> toAdd = new HashSet<>(updatedExclusions);
@@ -8660,7 +8661,15 @@ public void ensureContainerDataTypeExclusions(@NotNull DataTypeForExclusion data
         if (!toAdd.isEmpty())
         {
             for (Integer add : toAdd)
+            {
                 addDataTypeExclusion(add, dataType, excludedContainerId, user);
+
+                // Prevent "double exclusion" for related exclusion types (i.e. if a sample type is excluded from the
+                // project, then we can delete any "Dashboard Sample Type" exclusions for that same sample type).
+                // Note that "double exclusions" won't cause any harm, they just aren't necessary and can be cleaned up here.
+                if (relatedExclusions != null && relatedExclusions.contains(add))
+                    removeDataTypeExclusion(Collections.singleton(add), relatedDataType, excludedContainerId);
+            }
         }
 
         if (!toRemove.isEmpty())

diff --git a/experiment/src/org/labkey/experiment/api/SampleTypeServiceImpl.java b/experiment/src/org/labkey/experiment/api/SampleTypeServiceImpl.java
@@ -610,6 +610,7 @@ public void deleteSampleType(int rowId, Container c, User user, @Nullable String
             addSampleTypeDeletedAuditEvent(user, c, source, transaction.getAuditId(), auditUserComment);
 
             ExperimentService.get().removeDataTypeExclusion(Collections.singleton(rowId), ExperimentService.DataTypeForExclusion.SampleType);
+            ExperimentService.get().removeDataTypeExclusion(Collections.singleton(rowId), ExperimentService.DataTypeForExclusion.DashboardSampleType);
 
             transaction.addCommitTask(() -> clearMaterialSourceCache(c), DbScope.CommitTaskOption.IMMEDIATE, POSTCOMMIT, POSTROLLBACK);
             transaction.commit();
@@ -691,14 +692,15 @@ public ExpSampleTypeImpl createSampleType(Container c, User u, String name, Stri
     public ExpSampleTypeImpl createSampleType(Container c, User u, String name, String description, List<GWTPropertyDescriptor> properties, List<GWTIndex> indices, int idCol1, int idCol2, int idCol3, int parentCol,
                                               String nameExpression, String aliquotNameExpression, @Nullable TemplateInfo templateInfo, @Nullable Map<String, String> importAliases, @Nullable String labelColor, @Nullable String metricUnit) throws ExperimentException
     {
-        return createSampleType(c, u, name, description, properties, indices, idCol1, idCol2, idCol3, parentCol, nameExpression, aliquotNameExpression, templateInfo, importAliases, labelColor, metricUnit, null, null, null, null, null);
+        return createSampleType(c, u, name, description, properties, indices, idCol1, idCol2, idCol3, parentCol, nameExpression, aliquotNameExpression, templateInfo, importAliases, labelColor, metricUnit, null, null, null, null, null, null);
     }
 
     @NotNull
     @Override
     public ExpSampleTypeImpl createSampleType(Container c, User u, String name, String description, List<GWTPropertyDescriptor> properties, List<GWTIndex> indices, int idCol1, int idCol2, int idCol3, int parentCol,
                                               String nameExpression, String aliquotNameExpression, @Nullable TemplateInfo templateInfo, @Nullable Map<String, String> importAliases, @Nullable String labelColor, @Nullable String metricUnit,
-                                              @Nullable Container autoLinkTargetContainer, @Nullable String autoLinkCategory, @Nullable String category, @Nullable List<String> disabledSystemField, @Nullable List<String> excludedContainerIds)
+                                              @Nullable Container autoLinkTargetContainer, @Nullable String autoLinkCategory, @Nullable String category, @Nullable List<String> disabledSystemField,
+                                              @Nullable List<String> excludedContainerIds, @Nullable List<String> excludedDashboardContainerIds)
         throws ExperimentException
     {
         if (name == null)
@@ -877,6 +879,8 @@ public ExpSampleTypeImpl createSampleType(Container c, User u, String name, Stri
                     DefaultValueService.get().setDefaultValues(domain.getContainer(), defaultValues);
                     if (excludedContainerIds != null && !excludedContainerIds.isEmpty())
                         ExperimentService.get().ensureDataTypeContainerExclusions(ExperimentService.DataTypeForExclusion.SampleType, excludedContainerIds, st.getRowId(), u);
+                    if (excludedDashboardContainerIds != null && !excludedDashboardContainerIds.isEmpty())
+                        ExperimentService.get().ensureDataTypeContainerExclusions(ExperimentService.DataTypeForExclusion.DashboardSampleType, excludedDashboardContainerIds, st.getRowId(), u);
                     transaction.addCommitTask(() -> clearMaterialSourceCache(c), DbScope.CommitTaskOption.IMMEDIATE, POSTCOMMIT, POSTROLLBACK);
                     return st;
                 }
@@ -1046,6 +1050,8 @@ public ValidationException updateSampleType(GWTDomain<? extends GWTPropertyDescr
 
             if (options != null && options.getExcludedContainerIds() != null)
                 ExperimentService.get().ensureDataTypeContainerExclusions(ExperimentService.DataTypeForExclusion.SampleType, options.getExcludedContainerIds(), st.getRowId(), user);
+            if (options != null && options.getExcludedDashboardContainerIds() != null)
+                ExperimentService.get().ensureDataTypeContainerExclusions(ExperimentService.DataTypeForExclusion.DashboardSampleType, options.getExcludedDashboardContainerIds(), st.getRowId(), user);
 
             if (!errors.hasErrors())
             {

diff --git a/mothership/test/src/org/labkey/test/tests/mothership/MothershipReportTest.java b/mothership/test/src/org/labkey/test/tests/mothership/MothershipReportTest.java
@@ -99,7 +99,7 @@ public void testTopLevelItems() throws Exception
 
     private String getDeployedDistributionName()
     {
-        File distFile = new File(TestFileUtils.getDefaultDeployDir(), "labkeyWebapp/WEB-INF/classes/distribution");
+        File distFile = new File(TestFileUtils.getDefaultWebAppRoot(), "WEB-INF/classes/distribution");
         if (distFile.exists())
         {
             // Deployed from distribution

diff --git a/survey/src/org/labkey/survey/view/customizeSurveysWebPart.jsp b/survey/src/org/labkey/survey/view/customizeSurveysWebPart.jsp
@@ -40,7 +40,7 @@
 %>
 
 This webpart displays a list of survey instances created by the end user. Select which survey design this webpart should use:<br><br>
-<div id=<%=q(divId)%>></div>
+<div id="<%=h(divId)%>"></div>
 
 <script type="text/javascript" nonce="<%=getScriptNonce()%>">
     Ext4.onReady(function(){

diff --git a/survey/src/org/labkey/survey/view/surveyWizard.jsp b/survey/src/org/labkey/survey/view/surveyWizard.jsp
@@ -76,9 +76,9 @@
     else
     {
 %>
-<div id=<%=q(headerRenderId)%>></div>
-<div id=<%=q(formRenderId)%>></div>
-<div id=<%=q(footerRenderId)%>></div>
+<div id="<%=h(headerRenderId)%>"></div>
+<div id="<%=h(formRenderId)%>"></div>
+<div id="<%=h(footerRenderId)%>"></div>
 <script type="text/javascript" nonce="<%=getScriptNonce()%>">
 
     Ext4.onReady(function(){