Add the usage sharing options to share all Evidence and all query string parameters

pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/data/EvidenceKeyFilterShareUsage.java

@@ -29,6 +29,7 @@
 
 import java.util.HashSet;
 import java.util.List;
+import java.util.Locale;
 import java.util.Set;
 
 import static fiftyone.pipeline.core.Constants.*;
@@ -47,9 +48,12 @@
  *     {@link Constants#FIFTYONE_COOKIE_PREFIX} or has been configured to be
  *     shared using the constructor parameter.
  * 5) Not a header, cookie, session value or query string parameter.
+ * When all Evidence is shared, everything is included. When all query string
+ * parameters are shared, every query string parameter is included.
  * The prefix of a key is everything before its first '.', and the field
  * everything after it. An email address, the salt for hashed email
- * identifiers, the query string and the path are never included.
+ * identifiers, the query string, the path and the Cookie header are never
+ * included, whatever the configuration.
  */
 public class EvidenceKeyFilterShareUsage implements EvidenceKeyFilter {
 
@@ -65,6 +69,17 @@ public class EvidenceKeyFilterShareUsage implements EvidenceKeyFilter {
      */
     private final String sessionCookieName;
 
+    /**
+     * If true then every key is included, apart from those that are never
+     * shared.
+     */
+    private final boolean shareAllEvidence;
+
+    /**
+     * If true then every query string parameter is included.
+     */
+    private final boolean shareAllQueryStringParameters;
+
     /**
      * The content of HTTP headers in this array will not be included in the
      * request information sent to 51degrees. Any header names added here are
@@ -113,21 +128,55 @@ public EvidenceKeyFilterShareUsage(
         List<String> includedQueryStringParams,
         boolean includeSession,
         String sessionCookieName) {
+        this(
+            blockedHttpHeaders,
+            includedQueryStringParams,
+            includeSession,
+            sessionCookieName,
+            false,
+            false);
+    }
+
+    /**
+     * Constructor.
+     * @param blockedHttpHeaders a list of the names of the HTTP headers that
+     *                           share usage should not send to 51Degrees
+     * @param includedQueryStringParams a list of the names of query string
+     *                                  parameters that share usage should send
+     *                                  to 51Degrees
+     * @param includeSession if true then the session cookie will be included in
+     *                       the filter
+     * @param sessionCookieName the name of the cookie that contains the session
+     *                          id
+     * @param shareAllEvidence if true then all Evidence is included, apart from
+     *                         the keys that are never shared
+     * @param shareAllQueryStringParameters if true then all query string
+     *                                      parameters are included
+     */
+    public EvidenceKeyFilterShareUsage(
+        List<String> blockedHttpHeaders,
+        List<String> includedQueryStringParams,
+        boolean includeSession,
+        String sessionCookieName,
+        boolean shareAllEvidence,
+        boolean shareAllQueryStringParameters) {
+        this.shareAllEvidence = shareAllEvidence;
+        this.shareAllQueryStringParameters = shareAllQueryStringParameters;
         // The Cookie header carries every cookie, including the ones the
         // cookie rule leaves out.
         this.blockedHttpHeaders.add("cookies");
         this.blockedHttpHeaders.add("cookie");
         this.includeSession = includeSession;
         this.sessionCookieName = sessionCookieName == null ?
-            null : sessionCookieName.toLowerCase();
+            null : sessionCookieName.toLowerCase(Locale.ROOT);
         for (String header : blockedHttpHeaders) {
-            String lowerHeader = header.toLowerCase();
+            String lowerHeader = header.toLowerCase(Locale.ROOT);
             if (!this.blockedHttpHeaders.contains(lowerHeader)) {
                 this.blockedHttpHeaders.add(lowerHeader);
             }
         }
         for (String parameter : includedQueryStringParams) {
-            String lowerParameter = parameter.toLowerCase();
+            String lowerParameter = parameter.toLowerCase(Locale.ROOT);
             if (!this.includedQueryStringParams.contains(lowerParameter)) {
                 this.includedQueryStringParams.add(lowerParameter);
             }
@@ -136,14 +185,18 @@ public EvidenceKeyFilterShareUsage(
 
     @Override
     public boolean include(String key) {
-        String lowerKey = key.toLowerCase();
+        String lowerKey = key.toLowerCase(Locale.ROOT);
         for (String neverShared : NEVER_SHARED_EVIDENCE_KEYS) {
             if (lowerKey.equals(neverShared) ||
                 lowerKey.endsWith(EVIDENCE_SEPERATOR + neverShared)) {
                 return false;
             }
         }
 
+        if (shareAllEvidence) {
+            return true;
+        }
+
         boolean result;
         int separator = lowerKey.indexOf(EVIDENCE_SEPERATOR);
         if (separator > 0) {
@@ -169,8 +222,9 @@ public boolean include(String key) {
                 case EVIDENCE_QUERY_PREFIX:
                     // Only include query string parameters that start with the
                     // 51Degrees prefix or have been specified in the
-                    // constructor.
-                    result = field.startsWith(Constants.FIFTYONE_COOKIE_PREFIX) ||
+                    // constructor, unless all of them are shared.
+                    result = shareAllQueryStringParameters ||
+                        field.startsWith(Constants.FIFTYONE_COOKIE_PREFIX) ||
                         includedQueryStringParams.contains(field);
                     break;
                 default:

pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/data/EvidenceKeyFilterShareUsageTracker.java

@@ -57,6 +57,38 @@ public EvidenceKeyFilterShareUsageTracker(
             sessionCookieName);
     }
 
+    /**
+     * Constructor.
+     * @param blockedHttpHeaders a list of the names of the HTTP headers that
+     *                           share usage should not send to 51Degrees
+     * @param includedQueryStringParams a list of the names of query string
+     *                                  parameters that share usage should send
+     *                                  to 51Degrees
+     * @param includeSession if true then the session cookie will be included in
+     *                       the filter
+     * @param sessionCookieName the name of the cookie that contains the session
+     *                          id
+     * @param shareAllEvidence if true then all Evidence is included, apart from
+     *                         the keys that are never shared
+     * @param shareAllQueryStringParameters if true then all query string
+     *                                      parameters are included
+     */
+    public EvidenceKeyFilterShareUsageTracker(
+        List<String> blockedHttpHeaders,
+        List<String> includedQueryStringParams,
+        boolean includeSession,
+        String sessionCookieName,
+        boolean shareAllEvidence,
+        boolean shareAllQueryStringParameters) {
+        super(
+            blockedHttpHeaders,
+            includedQueryStringParams,
+            includeSession,
+            sessionCookieName,
+            shareAllEvidence,
+            shareAllQueryStringParameters);
+    }
+
     @Override
     public boolean include(String key) {
         if(key.equalsIgnoreCase(EVIDENCE_SESSIONID) ||

pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBase.java

@@ -396,6 +396,79 @@ protected ShareUsageBase(
         List<Map.Entry<String, String>> ignoreDataEvidenceFilter,
         String sessionCookieName,
         Tracker tracker) {
+        this(
+            logger,
+            sharePercentage,
+            minimumEntriesPerMessage,
+            maximumQueueSize,
+            addTimeout,
+            takeTimeout,
+            repeatEvidenceIntervalMinutes,
+            trackSession,
+            shareUsageUrl,
+            blockedHttpHeaders,
+            includedQueryStringParameters,
+            ignoreDataEvidenceFilter,
+            sessionCookieName,
+            tracker,
+            false,
+            false);
+    }
+
+    /**
+     * Constructor
+     * @param logger the logger to use
+     * @param sharePercentage the approximate proportion of requests to share.
+     *                        1 = 100%, 0.5 = 50%, etc.
+     * @param minimumEntriesPerMessage the minimum number of request entries per
+     *                                 message sent to 51Degrees
+     * @param maximumQueueSize the maximum number of items to hold in the queue
+     *                         at one time. This must be larger than minimum
+     *                         entries
+     * @param addTimeout the timeout in milliseconds to allow when attempting to
+     *                   add an item to the queue. If this timeout is exceeded
+     *                   then usage sharing will be disabled
+     * @param takeTimeout the timeout in milliseconds to allow when attempting
+     *                    to take an item to the queue
+     * @param repeatEvidenceIntervalMinutes the interval (in minutes) which is
+     *                                      used to decide if repeat evidence is
+     *                                      old enough to consider a new session
+     * @param trackSession set if the tracker should consider sessions in share
+     *                     usage
+     * @param shareUsageUrl the URL to send data to
+     * @param blockedHttpHeaders a list of the names of the HTTP headers that
+     *                           share usage should not send to 51Degrees
+     * @param includedQueryStringParameters a list of the names of query string
+     *                                      parameters that share usage should
+     *                                      send to 51Degrees
+     * @param ignoreDataEvidenceFilter the filter used to determine if an item
+     *                                 of evidence should be ignored or not
+     * @param sessionCookieName the name of the cookie that contains the session
+     *                          id
+     * @param tracker the {@link Tracker} to use to determine if a given
+     *                {@link FlowData} instance should be shared or not
+     * @param shareAllEvidence if true then all Evidence is shared, apart from
+     *                         the keys that are never shared
+     * @param shareAllQueryStringParameters if true then all query string
+     *                                      parameters are shared
+     */
+    protected ShareUsageBase(
+        Logger logger,
+        double sharePercentage,
+        int minimumEntriesPerMessage,
+        int maximumQueueSize,
+        int addTimeout,
+        int takeTimeout,
+        int repeatEvidenceIntervalMinutes,
+        boolean trackSession,
+        String shareUsageUrl,
+        List<String> blockedHttpHeaders,
+        List<String> includedQueryStringParameters,
+        List<Map.Entry<String, String>> ignoreDataEvidenceFilter,
+        String sessionCookieName,
+        Tracker tracker,
+        boolean shareAllEvidence,
+        boolean shareAllQueryStringParameters) {
         super(logger, null);
 
         if (minimumEntriesPerMessage > maximumQueueSize) {
@@ -430,7 +503,12 @@ protected ShareUsageBase(
         includedQueryStringParameters.add(Constants.EVIDENCE_SEQUENCE_SUFIX);
 
         evidenceKeyFilterExclSession = new EvidenceKeyFilterShareUsage(
-            blockedHttpHeaders, includedQueryStringParameters, false, sessionCookieName);
+            blockedHttpHeaders,
+            includedQueryStringParameters,
+            false,
+            sessionCookieName,
+            shareAllEvidence,
+            shareAllQueryStringParameters);
 
         this.ignoreDataEvidenceFilter = ignoreDataEvidenceFilter;
 
@@ -444,7 +522,12 @@ protected ShareUsageBase(
                 cacheConfiguration,
                     intervalMillis,
                 new EvidenceKeyFilterShareUsageTracker(
-                    blockedHttpHeaders, includedQueryStringParameters, trackSession, sessionCookieName));
+                    blockedHttpHeaders,
+                    includedQueryStringParameters,
+                    trackSession,
+                    sessionCookieName,
+                    shareAllEvidence,
+                    shareAllQueryStringParameters));
         }
 
         properties = new ArrayList<>();
@@ -711,7 +794,9 @@ private ShareUsageData getDataFromEvidence(Map<String, Object> found) {
                 continue;
             }
 
-            switch (entry.getKey()) {
+            // The keys are compared without regard to case, and the constants
+            // are lower case.
+            switch (entry.getKey().toLowerCase(Locale.ROOT)) {
                 case fiftyone.pipeline.core.Constants.EVIDENCE_CLIENTIP_KEY:
                     // The client IP is dealt with separately for backwards
                     // compatibility purposes.

pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBuilder.java

@@ -129,7 +129,9 @@ public ShareUsageElement build() throws IOException {
             includedQueryStringParameters,
             ignoreDataEvidenceFilter,
             sessionCookieName,
-            null);
+            null,
+            shareAllEvidence,
+            shareAllQueryStringParameters);
         if (Objects.nonNull(this.httpClient)) {
             sue.httpClient = httpClient;
         } else if (Objects.nonNull(dataUploader)){

pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBuilderBase.java

@@ -59,6 +59,8 @@ public abstract class ShareUsageBuilderBase<T extends ShareUsageBase> {
     protected final List<String> includedQueryStringParameters = new ArrayList<>();
     protected final List<Map.Entry<String, String>> ignoreDataEvidenceFilter = new ArrayList<>();
     protected boolean trackSession;
+    protected boolean shareAllEvidence;
+    protected boolean shareAllQueryStringParameters;
 
     /**
      * Constructor
@@ -143,6 +145,56 @@ public ShareUsageBuilderBase<T> setIncludedQueryStringParameter(
         return this;
     }
 
+    /**
+     * If set to true, every query string and HTTP form parameter is shared,
+     * rather than only those that start with 51D_ or are named in the
+     * included parameters. The whole query string and the parameters that are
+     * never shared are still not shared.
+     * <p>
+     * Default value is false
+     * @param shareAll true to share all query string parameters
+     * @return this builder
+     */
+    @DefaultValue(booleanValue = false)
+    public ShareUsageBuilderBase<T> setShareAllQueryStringParameters(
+        boolean shareAll) {
+        this.shareAllQueryStringParameters = shareAll;
+        return this;
+    }
+
+    /**
+     * Whether every query string parameter is shared.
+     * @return true if all query string parameters are shared
+     */
+    public boolean isShareAllQueryStringParameters() {
+        return shareAllQueryStringParameters;
+    }
+
+    /**
+     * If set to true, all Evidence is shared, which overrides the blocked
+     * HTTP headers and the rules for cookies, session values and query string
+     * parameters. The keys that are never shared (an email address, the salt
+     * for hashed email identifiers, the whole query string, the path and the
+     * Cookie header) are still not shared.
+     * <p>
+     * Default value is false
+     * @param shareAll true to share all Evidence
+     * @return this builder
+     */
+    @DefaultValue(booleanValue = false)
+    public ShareUsageBuilderBase<T> setShareAllEvidence(boolean shareAll) {
+        this.shareAllEvidence = shareAll;
+        return this;
+    }
+
+    /**
+     * Whether all Evidence is shared.
+     * @return true if all Evidence is shared
+     */
+    public boolean isShareAllEvidence() {
+        return shareAllEvidence;
+    }
+
     /**
      * By default, all HTTP headers (excluding a few such as 'cookies', if they
      * don't start with 51D_) are shared. Individual headers can be excluded from sharing by