Accept no evidence for usage sharing, and share what it finds in the flow data and the web request

pipeline/fiftyone_pipeline_cloudrequestengine/examples/tests/test_cloud_mixed_web_offline.py

@@ -302,3 +302,14 @@ def test_licence_key_gets_an_engine_only_for_what_it_asks_for(cloud):
 def test_licence_key_never_reaches_the_page(cloud):
     cloud(EVERY_SECTION)
     assert "a-licence-key" not in page_for(license_key="a-licence-key")
+
+
+def test_the_web_request_is_not_sent_to_the_cloud(cloud):
+    # The web integration adds the request itself as server.http-request for
+    # the usage sharing element, and it is never evidence another element
+    # reads.
+    stub = cloud(EVERY_SECTION)
+    page_for(resource_key="a-resource-key")
+
+    url, data = stub.requests[-1]
+    assert not any("http-request" in key for key in data), data

pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/evidence.py

@@ -20,6 +20,11 @@
 # such notice(s) shall fulfill the requirements of that article.
 # *********************************************************************
 
+# The key the web integration gives the web request itself, which the usage
+# sharing element reads. No element uses it as evidence, so filter() always
+# sets it aside, whatever the elements' filters accept.
+HTTP_REQUEST_KEY = "server.http-request"
+
 
 class Evidence:
     def __init__(self, flowdata):
@@ -32,6 +37,7 @@ def __init__(self, flowdata):
         self.evidence = {}
         self.evidence_filtered = False
         self.flowdata = flowdata
+        self.unaccepted = {}
 
     def add(self, key, value):
         """!
@@ -89,22 +95,37 @@ def get_all(self):
 
         return self.evidence
 
+    def get_unaccepted(self):
+        """!
+        The evidence filter() set aside, being the keys no FlowElement in
+        the pipeline accepts and the web request itself. The usage sharing
+        element accepts no evidence, and shares what it finds here and in
+        get_all().
+
+        @rtype: dict
+        @return: The evidence set aside, empty until filter() runs
+        """
+
+        return self.unaccepted
+
     def filter(self) -> None:
         """!
         Filter evidence using `filter_evidence_key` and
-        `filter_evidence` methods of each FlowElement in the pipeline
+        `filter_evidence` methods of each FlowElement in the pipeline. The
+        keys that do not pass, and the web request under HTTP_REQUEST_KEY,
+        move to get_unaccepted().
         """
 
         if self.evidence_filtered:
             return
 
         # Loop over a copy of the keys, as keys are removed on the way.
         for key in list(self.evidence):
-            if not any([
+            if str(key).lower() == HTTP_REQUEST_KEY or not any([
                 flow_element.filter_evidence_key(key)
                 for flow_element
                 in self.flowdata.pipeline.flow_elements
             ]):
-                del self.evidence[key]
+                self.unaccepted[key] = self.evidence.pop(key)
 
         self.evidence_filtered = True

pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/web.py

@@ -20,13 +20,30 @@
 # such notice(s) shall fulfill the requirements of that article.
 # *********************************************************************
 
-from flask import request
+from types import MappingProxyType
+
+from flask import has_request_context, request, session
+
+from .evidence import HTTP_REQUEST_KEY
+
+def request_line_path(request):
+
+    """!
+    The path of the request line as the server received it, when the server
+    says what that was, and otherwise the path Flask decoded.
+    """
+
+    raw = request.environ.get("RAW_URI") or request.environ.get("REQUEST_URI")
+    if raw:
+        return raw.split("?", 1)[0].split("#", 1)[0]
+    return request.path
 
 def webevidence(request):
 
     """!
-    Get evidence from a web request (gets headers, cookies, query parameters
-    and the fields of a posted form)
+    Get evidence from a web request (gets headers, cookies, session values,
+    query parameters and the fields of a posted form, and the request itself
+    as server.http-request)
 
     @type request: Request 
     @param request: A Request object
@@ -59,6 +76,14 @@ def webevidence(request):
 
             webevidence["query." + field] = value
 
+    # Values stored in the session. They are read without marking the session
+    # as accessed, which would add a Vary: Cookie header to the response, and
+    # only plain values are taken.
+    if has_request_context():
+        for name, value in dict.items(session._get_current_object()):
+            if isinstance(value, (str, int, float, bool)):
+                webevidence["session." + name] = value
+
     webevidence["server.client-ip"] =  request.remote_addr
 
     webevidence["server.host-ip"] =  request.host
@@ -68,6 +93,17 @@ def webevidence(request):
     else:
         webevidence["header.protocol"] = "http"
 
+    # The request itself, for the usage sharing element, which accepts no
+    # evidence and shares what it finds. A read-only view of the parts above
+    # and the method and path of the request line, keyed as evidence is. It
+    # leaves out the Cookie header, which cookies already cover. Evidence
+    # keeps it from every other element.
+    view = {key: value for key, value in webevidence.items()
+            if key not in ("header.cookie", "server.host-ip")}
+    view["server.request-method"] = request.method
+    view["server.request-path"] = request_line_path(request)
+    webevidence[HTTP_REQUEST_KEY] = MappingProxyType(view)
+
     return webevidence
 
 def set_response_header(flowData, response):

pipeline/fiftyone_pipeline_core/tests/test_evidence_unaccepted.py

@@ -0,0 +1,110 @@
+# *********************************************************************
+# This Original Work is copyright of 51 Degrees Mobile Experts Limited.
+# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House,
+# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU.
+#
+# This Original Work is licensed under the European Union Public Licence
+# (EUPL) v.1.2 and is subject to its terms as set out below.
+#
+# If a copy of the EUPL was not distributed with this file, You can obtain
+# one at https://opensource.org/licenses/EUPL-1.2.
+#
+# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be
+# amended by the European Commission) shall be deemed incompatible for
+# the purposes of the Work and the provisions of the compatibility
+# clause in Article 5 of the EUPL shall not apply.
+#
+# If using the Work as, or as part of, a network application, by
+# including the attribution notice(s) required under Article 5 of the EUPL
+# in the end user terms of the application under an appropriate heading,
+# such notice(s) shall fulfill the requirements of that article.
+# *********************************************************************
+
+"""The evidence a pipeline's elements do not accept, and the web request the
+web integration adds as server.http-request."""
+
+import unittest
+
+from flask import Flask, request, session
+
+from fiftyone_pipeline_core.pipelinebuilder import PipelineBuilder
+from fiftyone_pipeline_core.web import webevidence
+
+from .classes.exampleflowelement1 import ExampleFlowElement1
+
+# No JavaScript or set header elements, so the only filter is that of
+# ExampleFlowElement1, which accepts header.user-agent.
+NO_DEFAULT_ELEMENTS = {
+    "add_javascript_builder": False,
+    "use_setheader_properties": False,
+}
+
+
+class UnacceptedEvidenceTests(unittest.TestCase):
+
+    def test_evidence_no_element_accepts_is_set_aside(self):
+        pipeline = PipelineBuilder(NO_DEFAULT_ELEMENTS) \
+            .add(ExampleFlowElement1()).build()
+        flowdata = pipeline.create_flowdata()
+        flowdata.evidence.add("header.user-agent", "a browser")
+        flowdata.evidence.add("header.x-test", "set aside")
+        flowdata.process()
+
+        self.assertEqual(
+            {"header.user-agent": "a browser"}, flowdata.evidence.get_all())
+        self.assertEqual(
+            {"header.x-test": "set aside"},
+            flowdata.evidence.get_unaccepted())
+
+    def test_the_web_request_is_added_and_set_aside(self):
+        """No element reads server.http-request as evidence, even an element
+        that accepts every key, as the JavaScript builder does."""
+
+        pipeline = PipelineBuilder().add(ExampleFlowElement1()).build()
+        app = Flask(__name__)
+        with app.test_request_context(
+                "/?campaign=a", headers={"User-Agent": "a browser",
+                                         "X-Test": "a header"}):
+            evidence = webevidence(request)
+        flowdata = pipeline.create_flowdata()
+        flowdata.evidence.add_from_dict(evidence)
+        flowdata.process()
+
+        self.assertNotIn("server.http-request", flowdata.evidence.get_all())
+        view = flowdata.evidence.get_unaccepted()["server.http-request"]
+        self.assertEqual("a header", view["header.x-test"])
+        self.assertEqual("a browser", view["header.user-agent"])
+        self.assertEqual("a", view["query.campaign"])
+        # A read-only view of the request, not evidence an element can
+        # change.
+        with self.assertRaises(TypeError):
+            view["header.x-test"] = "changed"
+
+    def test_the_web_request_has_every_part_the_specification_names(self):
+        app = Flask(__name__)
+        app.secret_key = "a secret"
+        with app.test_request_context(
+                "/page?campaign=a",
+                headers={"User-Agent": "a browser",
+                         "Cookie": "sessionid=cookievalue"},
+                environ_base={"REMOTE_ADDR": "203.0.113.5"}):
+            session["51D_one"] = "first"
+            session["theme"] = "dark"
+            view = webevidence(request)["server.http-request"]
+
+        self.assertEqual("a browser", view["header.user-agent"])
+        self.assertEqual("cookievalue", view["cookie.sessionid"])
+        self.assertEqual("a", view["query.campaign"])
+        self.assertEqual("first", view["session.51D_one"])
+        self.assertEqual("dark", view["session.theme"])
+        self.assertEqual("203.0.113.5", view["server.client-ip"])
+        self.assertEqual("http", view["header.protocol"])
+        self.assertEqual("GET", view["server.request-method"])
+        self.assertEqual("/page", view["server.request-path"])
+        # The Cookie header carries every cookie, and cookies are their own
+        # part of the view.
+        self.assertNotIn("header.cookie", view)
+
+
+if __name__ == "__main__":
+    unittest.main()

pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage.py

@@ -29,12 +29,19 @@
 from importlib.metadata import version
 
 import requests
+from fiftyone_pipeline_core.basiclist_evidence_keyfilter import \
+    BasicListEvidenceKeyFilter
 from fiftyone_pipeline_engines.engine import Engine
 
 from .share_usage_evidencekeyfilter import ShareUsageEvidenceKeyFilter, \
     ShareUsageTrackerEvidenceKeyFilter
 from .share_usage_tracker import ShareUsageTracker
 
+# The key the web integration gives the web request itself. Written out
+# rather than imported, so the element still loads beside an older
+# fiftyone_pipeline_core.
+HTTP_REQUEST_KEY = "server.http-request"
+
 # The maximum length of a piece of evidence's value which can be
 # added to the usage data being sent.
 SHARE_USAGE_MAX_EVIDENCE_LENGTH = 512
@@ -123,11 +130,13 @@ def __init__(
         included = None if query_whitelist is None else \
             list(query_whitelist) + ["session-id", "sequence"]
 
-        # When sessions are tracked, the pipeline keeps the session cookie
-        # for the tracker. The data sent leaves it out unless all evidence
+        # The element accepts no evidence, so that caching, the ETag and the
+        # Vary header never depend on it. It shares what it finds instead,
+        # and these filters decide which of that is shared and which tells
+        # a repeat apart. When sessions are tracked the tracker reads the
+        # session cookie, which the data sent leaves out unless all evidence
         # is shared.
-        self.evidence_key_filter = ShareUsageEvidenceKeyFilter(
-            cookie, included, header_blacklist, track_session, share_all)
+        self.evidence_key_filter = BasicListEvidenceKeyFilter([])
         self.data_evidence_key_filter = ShareUsageEvidenceKeyFilter(
             cookie, included, header_blacklist, False, share_all)
         self.tracker_evidence_key_filter = \
@@ -214,14 +223,40 @@ def get_evidence_key_filter(self):
 
         """!
 
-        The share useage element comes with its own evidence
-        key filter that uses the whitelists and blacklists to
-        determine which evidence to share    
-
+        The usage sharing element accepts no evidence. See found_evidence
+        for what it shares.
         """
 
         return self.evidence_key_filter
 
+    def found_evidence(self, flow_data):
+
+        """!
+        The evidence the element finds to share. These are the values of the
+        web request, when the web integration added it as
+        server.http-request, and every evidence value in the flow data,
+        whether an element accepted it or not. A key found in both places
+        takes the evidence value. The web request itself is never one of
+        them.
+
+        @type flow_data: FlowData
+        @param flow_data: The flow data being processed
+        @rtype: dict
+        @return: The evidence found, before the sharing rules apply
+        """
+
+        evidence = flow_data.evidence
+        # An older fiftyone_pipeline_core sets nothing aside.
+        unaccepted = getattr(evidence, "get_unaccepted", dict)()
+        web_request = unaccepted.get(HTTP_REQUEST_KEY) or \
+            evidence.get(HTTP_REQUEST_KEY) or {}
+        found = {}
+        for source in (web_request, unaccepted, evidence.get_all()):
+            for key, value in source.items():
+                if str(key).lower() != HTTP_REQUEST_KEY:
+                    found[key] = value
+        return found
+
     def share_send_usage(self):
         """!
         Internal method to send the share usage bundle to the 51Degrees servers
@@ -319,8 +354,11 @@ def process_internal(self, flow_data):
         """
 
         if random.uniform(0, 1) <= self.share_percentage:
-            cachekey = self.tracker_evidence_key_filter.filter_evidence(flow_data.evidence.get_all())
-            cachekey = json.dumps(cachekey)
+            # A hash of the values that would be shared, as the element
+            # accepts no evidence of its own.
+            cachekey = self.tracker_evidence_key_filter.filter_evidence(
+                self.found_evidence(flow_data))
+            cachekey = json.dumps(cachekey, sort_keys=True, default=str)
             share = self.tracker.track(cachekey) 
             if share:
                 self.tracker.set_cache_value(cachekey)
@@ -336,7 +374,7 @@ def get_data_from_evidence(self, flow_data):
         """
         data = ShareUsageData()
 
-        for key, value in flow_data.evidence.get_all().items():
+        for key, value in self.found_evidence(flow_data).items():
             if key == 'server.client-ip':
                 # The client IP is dealt with separately for backwards
                 # compatibility purposes.

pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage_evidencekeyfilter.py

@@ -26,16 +26,15 @@
 # this are the ones 51Degrees sets, and are shared.
 FIFTYONE_PREFIX = "51d_"
 
-# The Cookie header carries every cookie, including the ones the cookie rule
-# leaves out, so it is not shared unless all evidence is.
-BLOCKED_HEADERS = ("cookie",)
-
 # Evidence that is never shared, whatever the settings, matched as the whole
 # key or after a '.'. 'id.email' and 'id.salt' are a raw email address and the
 # salt used to make hashed email identifiers. 'request-query' is the whole query
 # string, which would undo the rules for its parameters. 'request-path' would
 # make each page a visitor opens look like a new session to the tracker.
-NEVER_SHARED_KEYS = ("id.email", "id.salt", "request-query", "request-path")
+# 'header.cookie' is the Cookie header, which carries every cookie, and the
+# cookies that are shared are shared as their own 'cookie.' evidence.
+NEVER_SHARED_KEYS = (
+    "id.email", "id.salt", "request-query", "request-path", "header.cookie")
 
 class ShareUsageEvidenceKeyFilter(BasicListEvidenceKeyFilter):
     """
@@ -110,8 +109,7 @@ def filter_evidence_key(self, key):
         elif prefix == "header":
             # Add the header to the list if the header name does not
             # appear in the list of blocked headers
-            return field not in self.header_blacklist and \
-                field not in BLOCKED_HEADERS
+            return field not in self.header_blacklist
         elif prefix == "cookie":
             # Only add cookies that start with the 51Degrees cookie
             # prefix