Accept no Evidence for usage sharing, and give it the whole web request

pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineDefault.java

@@ -63,6 +63,15 @@
 public class CloudRequestEngineDefault
     extends AspectEngineBase<CloudRequestData, AspectPropertyMetaData>
     implements CloudRequestEngine {
+    /**
+     * The Evidence key of the whole web request, which the web integration
+     * adds for usage sharing to read. It holds every header and cookie of
+     * the request, so it is never sent to the cloud service. The usage
+     * sharing package defines the same key, and this package does not
+     * depend on that one.
+     */
+    private static final String HTTP_REQUEST_KEY = "server.http-request";
+
     private HttpClient httpClient;
 
     private String endPoint;
@@ -266,7 +275,9 @@ private byte[] getContent(FlowData data) throws UnsupportedEncodingException {
     }
 
     Map<String, Object> getFormData(FlowData flowData) {
-        Map<String, Object> evidence = flowData.getEvidence().asKeyMap();
+        Map<String, Object> evidence = new HashMap<>(
+            flowData.getEvidence().asKeyMap());
+        evidence.keySet().removeIf(HTTP_REQUEST_KEY::equalsIgnoreCase);
         Map<String, Object> formData = new HashMap<>();
 
         // Add evidence in reverse alphabetical order, excluding special keys.

pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineTests.java

@@ -464,6 +464,36 @@ public void getFormData(String name, Map<String, Object> evidence, String expect
         }
     }
 
+    /**
+     * The web integration adds the whole web request as server.http-request
+     * for usage sharing to read. It holds every header and cookie, so it is
+     * never sent to the cloud service.
+     */
+    @Test
+    public void getFormData_LeavesOutTheWebRequest() throws Exception {
+        configureMockedClient();
+        CloudRequestEngineDefault engine = (CloudRequestEngineDefault) new CloudRequestEngineBuilder(loggerFactory, httpClient)
+                .setResourceKey("resourcekey")
+                .build();
+
+        Map<String, Object> request = new HashMap<>();
+        request.put("header.Cookie", "session=secret");
+        try (Pipeline pipeline = new PipelineBuilder(loggerFactory)
+                .addFlowElement(engine)
+                .build();
+             FlowData data = pipeline.createFlowData()) {
+            data.addEvidence("header.User-Agent", "iPhone");
+            data.addEvidence("server.client-ip", "1.2.3.4");
+            data.addEvidence("server.http-request", request);
+
+            Map<String, Object> result = engine.getFormData(data);
+            assertEquals("iPhone", result.get("User-Agent"));
+            assertEquals("1.2.3.4", result.get("client-ip"));
+            assertFalse(result.containsKey("http-request"),
+                "The web request was sent to the cloud service.");
+        }
+    }
+
     /**
      * An evidence key whose name itself holds a dot, such as
      * {@code query.id.usage}, must reach the cloud service under its whole

pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/data/EvidenceKeyFilterShareUsage.java

@@ -34,7 +34,9 @@
 import static fiftyone.pipeline.core.Constants.*;
 
 /**
- * This filter is used by the {@link ShareUsageElement}.
+ * This filter is used by the {@link ShareUsageElement} to decide which of the
+ * values {@link ShareUsageEvidence#find} finds are shared. It is not the
+ * element's accepted Evidence, which is empty.
  * It will include anything that is:
  * 1) An HTTP header that is not blocked by the constructor parameter, other
  *     than Cookie.
@@ -48,10 +50,6 @@
  * The prefix of a key is everything before its first '.', and the field
  * everything after it. An email address, the salt for hashed email
  * identifiers, the query string and the path are never included.
- *
- * As this filter is generally inclusive, it will often cause far more
- * evidence to be passed into a pipeline than the engine-specific
- * filters, which tend to be based on a white list
  */
 public class EvidenceKeyFilterShareUsage implements EvidenceKeyFilter {
 
@@ -85,13 +83,15 @@ public class EvidenceKeyFilterShareUsage implements EvidenceKeyFilter {
      * Evidence keys that are never shared, matched without regard to case as
      * the whole key or after a '.'. 'id.email' and 'id.salt' are a raw email
      * address and the salt used to make hashed email identifiers.
+     * 'header.cookie' is the Cookie header, which carries every cookie.
      * 'request-query' is the whole query string, which would undo the rules
      * for its parameters. 'request-path' would make each page a visitor opens
      * look like a new session to the tracker.
      */
     private static final String[] NEVER_SHARED_EVIDENCE_KEYS = {
         "id.email",
         "id.salt",
+        "header.cookie",
         "request-query",
         "request-path"
     };

pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/data/ShareUsageEvidence.java

@@ -0,0 +1,78 @@
+/* *********************************************************************
+ * This Original Work is copyright of 51 Degrees Mobile Experts Limited.
+ * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House,
+ * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU.
+ *
+ * This Original Work is licensed under the European Union Public Licence
+ * (EUPL) v.1.2 and is subject to its terms as set out below.
+ *
+ * If a copy of the EUPL was not distributed with this file, You can obtain
+ * one at https://opensource.org/licenses/EUPL-1.2.
+ *
+ * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be
+ * amended by the European Commission) shall be deemed incompatible for
+ * the purposes of the Work and the provisions of the compatibility
+ * clause in Article 5 of the EUPL shall not apply.
+ *
+ * If using the Work as, or as part of, a network application, by
+ * including the attribution notice(s) required under Article 5 of the EUPL
+ * in the end user terms of the application under an appropriate heading,
+ * such notice(s) shall fulfill the requirements of that article.
+ * ********************************************************************* */
+
+package fiftyone.pipeline.engines.fiftyone.data;
+
+import fiftyone.pipeline.core.data.Evidence;
+
+import java.util.Map;
+import java.util.TreeMap;
+
+/**
+ * The values usage sharing finds for a request. The usage sharing element
+ * accepts no Evidence, so that cache keys, the ETag and the Vary header never
+ * depend on it, and finds what to share here instead. The sharing rules of
+ * {@link EvidenceKeyFilterShareUsage} then decide which of these are shared.
+ * @see <a href="https://github.com/51Degrees/specifications/blob/main/pipeline-specification/pipeline-elements/usage-sharing-element.md#accepted-evidence">Specification</a>
+ */
+public final class ShareUsageEvidence {
+
+    /**
+     * The Evidence key under which the web integration adds the web request,
+     * as a read-only map keyed the way Evidence is keyed, such as
+     * header.user-agent or cookie.51D_Profile. No element accepts it.
+     */
+    public static final String HTTP_REQUEST_KEY = "server.http-request";
+
+    private ShareUsageEvidence() {
+    }
+
+    /**
+     * Every Evidence value, and every value of the web request when the
+     * Evidence holds it, keyed without regard to case. A key in both keeps
+     * the Evidence value. The web request itself is not one of the values.
+     * @param evidence the Evidence of the request
+     * @return the values found
+     */
+    public static Map<String, Object> find(Evidence evidence) {
+        Map<String, Object> values = evidence.asKeyMap();
+        Map<String, Object> found =
+            new TreeMap<>(String.CASE_INSENSITIVE_ORDER);
+        Object request = null;
+        for (Map.Entry<String, Object> entry : values.entrySet()) {
+            if (HTTP_REQUEST_KEY.equalsIgnoreCase(entry.getKey())) {
+                request = entry.getValue();
+            } else if (entry.getValue() != null) {
+                found.put(entry.getKey(), entry.getValue());
+            }
+        }
+        if (request instanceof Map) {
+            for (Map.Entry<?, ?> entry : ((Map<?, ?>) request).entrySet()) {
+                if (entry.getKey() instanceof String &&
+                    entry.getValue() != null) {
+                    found.putIfAbsent((String) entry.getKey(), entry.getValue());
+                }
+            }
+        }
+        return found;
+    }
+}

pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/SequenceElement.java

@@ -26,6 +26,7 @@
 import fiftyone.pipeline.core.flowelements.FlowElement;
 import fiftyone.pipeline.core.flowelements.FlowElementBase;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
 import java.util.UUID;
@@ -38,6 +39,18 @@
 public class SequenceElement 
     extends FlowElementBase<ElementData, ElementPropertyMetaData> {
 
+    /**
+     * The session id and sequence of a callback, which the specification
+     * lists as this element's accepted Evidence. Without them the web
+     * integration would not add them, and every callback would start a new
+     * session.
+     */
+    private final EvidenceKeyFilter acceptedEvidence =
+        new EvidenceKeyFilterWhitelist(Arrays.asList(
+            Constants.EVIDENCE_SESSIONID,
+            Constants.EVIDENCE_SEQUENCE),
+            String.CASE_INSENSITIVE_ORDER);
+
     /**
      * Construct a new instance of the {@link FlowElement}.
      * @param logger logger instance to use for logging
@@ -59,9 +72,25 @@ protected void processInternal(FlowData data) throws Exception {
         if (evidence.containsKey(Constants.EVIDENCE_SEQUENCE) == false) {
             data.addEvidence(Constants.EVIDENCE_SEQUENCE, 1);
         } else {
-            Object sequence = evidence.get(Constants.EVIDENCE_SEQUENCE);
-            int seq = (int) sequence;
-            data.addEvidence(Constants.EVIDENCE_SEQUENCE, seq + 1);
+            data.addEvidence(
+                Constants.EVIDENCE_SEQUENCE,
+                getSequence(evidence.get(Constants.EVIDENCE_SEQUENCE)) + 1);
+        }
+    }
+
+    /**
+     * The sequence in Evidence, which is a number when an element set it and
+     * text when it came from the query string of a callback. Anything that is
+     * not a number counts as no sequence.
+     */
+    private static int getSequence(Object sequence) {
+        if (sequence instanceof Number) {
+            return ((Number) sequence).intValue();
+        }
+        try {
+            return Integer.parseInt(String.valueOf(sequence).trim());
+        } catch (NumberFormatException e) {
+            return 0;
         }
     }
 
@@ -72,7 +101,7 @@ public String getElementDataKey() {
 
     @Override
     public EvidenceKeyFilter getEvidenceKeyFilter() {
-        return new EvidenceKeyFilterWhitelist(new ArrayList<String>());
+        return acceptedEvidence;
     }
 
     @Override

pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBase.java

@@ -30,6 +30,7 @@
 import fiftyone.pipeline.engines.configuration.CacheConfiguration;
 import fiftyone.pipeline.engines.fiftyone.data.EvidenceKeyFilterShareUsage;
 import fiftyone.pipeline.engines.fiftyone.data.EvidenceKeyFilterShareUsageTracker;
+import fiftyone.pipeline.engines.fiftyone.data.ShareUsageEvidence;
 import fiftyone.pipeline.engines.fiftyone.exceptions.HttpException;
 import fiftyone.pipeline.engines.fiftyone.trackers.ShareUsageTracker;
 import fiftyone.pipeline.engines.trackers.Tracker;
@@ -106,12 +107,15 @@ public abstract class ShareUsageBase
     protected final String shareUsageUrl;
 
     /**
-     * Evidence key filter including the session id.
+     * The Evidence this element accepts, which is none, so that cache keys,
+     * the ETag and the Vary header never depend on it. It finds what to share
+     * with {@link ShareUsageEvidence#find} instead.
      */
-    private final EvidenceKeyFilter evidenceKeyFilter;
+    private final EvidenceKeyFilter acceptedEvidence =
+        new EvidenceKeyFilterWhitelist(new ArrayList<String>());
 
     /**
-     * Evidence key filter excluding the session id
+     * Which of the values found are shared, leaving out the session cookie.
      */
     private final EvidenceKeyFilter evidenceKeyFilterExclSession;
 
@@ -425,8 +429,6 @@ protected ShareUsageBase(
         includedQueryStringParameters.add(Constants.EVIDENCE_SESSIONID_SUFFIX);
         includedQueryStringParameters.add(Constants.EVIDENCE_SEQUENCE_SUFIX);
 
-        evidenceKeyFilter = new EvidenceKeyFilterShareUsage(
-            blockedHttpHeaders, includedQueryStringParameters, true, sessionCookieName);
         evidenceKeyFilterExclSession = new EvidenceKeyFilterShareUsage(
             blockedHttpHeaders, includedQueryStringParameters, false, sessionCookieName);
 
@@ -518,15 +520,16 @@ public void addPipeline(Pipeline pipeline) {
     @Override
     protected void processInternal(FlowData flowData){
         boolean ignoreData = false;
-        Map<String, Object> evidence = flowData.getEvidence().asKeyMap();
+        Map<String, Object> found =
+            ShareUsageEvidence.find(flowData.getEvidence());
 
         logger.debug("processInternal()");
 
         if (ignoreDataEvidenceFilter != null) {
             for (Map.Entry<String, String> entry : ignoreDataEvidenceFilter) {
                 String key = entry.getKey();
-                if (evidence.containsKey(key)) {
-                    if (evidence.get(key).toString().equals(entry.getValue())) {
+                if (found.containsKey(key)) {
+                    if (found.get(key).toString().equals(entry.getValue())) {
                         ignoreData = true;
                         break;
                     }
@@ -535,13 +538,13 @@ protected void processInternal(FlowData flowData){
         }
 
         if (isCanceled() == false && ignoreData == false) {
-            processData(flowData);
+            processData(flowData, found);
         }
     }
 
     @Override
     public EvidenceKeyFilter getEvidenceKeyFilter() {
-        return evidenceKeyFilter;
+        return acceptedEvidence;
     }
 
     @Override
@@ -653,8 +656,9 @@ private boolean isLocalHost(String address) throws UnknownHostException {
     /**
      * Process the supplied request data
      * @param data the {@link FlowData} instance that provides the evidence
+     * @param found the values {@link ShareUsageEvidence#find} found for it
      */
-    private void processData(FlowData data) {
+    private void processData(FlowData data, Map<String, Object> found) {
         logger.debug("processData()");
         if (random.nextDouble() <= sharePercentage) {
             // Check if the tracker will allow sharing of this data
@@ -663,7 +667,7 @@ private void processData(FlowData data) {
                 // it to the collection.
                 try {
                     if (evidenceCollection.offer(
-                            getDataFromEvidence(data.getEvidence()),
+                            getDataFromEvidence(found),
                             addTimeout,
                             TimeUnit.MILLISECONDS)) {
                         logger.debug("Queued {} entries", evidenceCollection.size());
@@ -689,18 +693,19 @@ private void processData(FlowData data) {
     }
 
     /**
-     * Extract the desired data from the evidence.
+     * Extract the desired data from the values found.
      * In order to avoid problems with the evidence data being disposed before
      * it is sent, the data placed into a new object rather than being a
      * reference to the existing evidence instance.
      *
-     * @param evidence an {@link Evidence} instance that contains the data to be extracted
-     * @return a {@link ShareUsageData} instance populated with data from the evidence
+     * @param found the values {@link ShareUsageEvidence#find} found
+     * @return a {@link ShareUsageData} instance populated with the values
+     * the sharing rules allow
      */
-    private ShareUsageData getDataFromEvidence(Evidence evidence) {
+    private ShareUsageData getDataFromEvidence(Map<String, Object> found) {
         ShareUsageData shareUsageData = new ShareUsageData();
 
-        for (Map.Entry<String, Object> entry : evidence.asKeyMap().entrySet()) {
+        for (Map.Entry<String, Object> entry : found.entrySet()) {
             // Check if we can send this piece of evidence
             if (evidenceKeyFilterExclSession.include(entry.getKey()) == false) {
                 continue;