From 89775867c66630b9ebd26614653c5046e750e218 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Sat, 7 Nov 2026 21:56:06 +0000 Subject: [PATCH 1/8] Test that usage sharing accepts no Evidence and reads the web request The Pipeline with usage sharing should accept only its other elements' Evidence, the web integration should add the web request as server.http-request, usage sharing should share a header only that request holds under its rules, the tracker should key on what would be shared, a revalidation's ETag should be the one it sends, and the cloud request engine should not send the web request. --- .../flowelements/CloudRequestEngineTests.java | 30 +++ .../ShareUsageAcceptedEvidenceTests.java | 74 +++++++ .../flowelements/ShareUsageElementTests.java | 76 +++++++ .../trackers/ShareUsageTrackerTests.java | 44 ++++ .../web/UsageSharingEvidenceTests.java | 199 ++++++++++++++++++ 5 files changed, 423 insertions(+) create mode 100644 pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageAcceptedEvidenceTests.java create mode 100644 pipeline/pipeline.web.packages/pipeline.web/src/test/java/fiftyone/pipeline/web/UsageSharingEvidenceTests.java diff --git a/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineTests.java b/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineTests.java index 01b82a180..80ebde301 100644 --- a/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineTests.java +++ b/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineTests.java @@ -464,6 +464,36 @@ public void getFormData(String name, Map evidence, String expect } } + /** + * The web integration adds the whole web request as server.http-request + * for usage sharing to read. It holds every header and cookie, so it is + * never sent to the cloud service. + */ + @Test + public void getFormData_LeavesOutTheWebRequest() throws Exception { + configureMockedClient(); + CloudRequestEngineDefault engine = (CloudRequestEngineDefault) new CloudRequestEngineBuilder(loggerFactory, httpClient) + .setResourceKey("resourcekey") + .build(); + + Map request = new HashMap<>(); + request.put("header.Cookie", "session=secret"); + try (Pipeline pipeline = new PipelineBuilder(loggerFactory) + .addFlowElement(engine) + .build(); + FlowData data = pipeline.createFlowData()) { + data.addEvidence("header.User-Agent", "iPhone"); + data.addEvidence("server.client-ip", "1.2.3.4"); + data.addEvidence("server.http-request", request); + + Map result = engine.getFormData(data); + assertEquals("iPhone", result.get("User-Agent")); + assertEquals("1.2.3.4", result.get("client-ip")); + assertFalse(result.containsKey("http-request"), + "The web request was sent to the cloud service."); + } + } + /** * An evidence key whose name itself holds a dot, such as * {@code query.id.usage}, must reach the cloud service under its whole diff --git a/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageAcceptedEvidenceTests.java b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageAcceptedEvidenceTests.java new file mode 100644 index 000000000..6fae6d5ae --- /dev/null +++ b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageAcceptedEvidenceTests.java @@ -0,0 +1,74 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.engines.fiftyone.flowelements; + +import fiftyone.pipeline.core.data.EvidenceKeyFilter; +import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.core.flowelements.PipelineBuilder; +import fiftyone.pipeline.engines.testhelpers.flowelements.EmptyEngine; +import fiftyone.pipeline.engines.testhelpers.flowelements.EmptyEngineBuilder; +import org.junit.jupiter.api.Test; +import org.slf4j.LoggerFactory; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * The usage sharing element accepts no Evidence, so the Pipeline's accepted + * Evidence, from which cache keys, the ETag and the Vary header are made, is + * that of its other elements alone. + */ +public class ShareUsageAcceptedEvidenceTests { + + @Test + public void Pipeline_WithShareUsage_AcceptsOnlyTheOtherElementsEvidence() + throws Exception { + ShareUsageElement shareUsage = new ShareUsageBuilder( + LoggerFactory.getILoggerFactory()) + .setSharePercentage(0) + .build(); + EmptyEngine engine = new EmptyEngineBuilder( + LoggerFactory.getILoggerFactory()) + .build(); + try (Pipeline pipeline = new PipelineBuilder( + LoggerFactory.getILoggerFactory()) + .addFlowElement(engine) + .addFlowElement(shareUsage) + .setAutoCloseElements(true) + .build()) { + EvidenceKeyFilter accepted = pipeline.getEvidenceKeyFilter(); + + assertTrue(accepted.include("test.value"), + "The other element's Evidence is not accepted."); + assertFalse(accepted.include("header.if-none-match"), + "If-None-Match is accepted, so a revalidation's ETag differs " + + "from the one it sends."); + assertFalse(accepted.include("header.x-test"), + "A header no element uses is accepted."); + assertFalse(accepted.include("server.http-request"), + "The web request is accepted."); + assertFalse(shareUsage.getEvidenceKeyFilter().include("header.user-agent"), + "The usage sharing element accepts Evidence."); + } + } +} diff --git a/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageElementTests.java b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageElementTests.java index 260118fe6..b2520614c 100644 --- a/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageElementTests.java +++ b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageElementTests.java @@ -487,6 +487,82 @@ public void ShareUsageElement_RestrictedHeaders() throws Exception { } } + /** + * The element accepts no Evidence, so a header no other element accepts + * reaches it only through the web request the web integration adds as + * server.http-request. It shares that header under the key it would have + * as Evidence, keeps the Evidence value for a key in both places, and + * still leaves out the cookie header, a blocked header and the web + * request itself. + */ + @Test + public void ShareUsageElement_WebRequest_SharesWhatNoOtherElementAccepts() + throws Exception { + createShareUsage( + 1, + 1, + 1, + new ArrayList<>(Collections.singletonList("x-blocked")), + new ArrayList<>(), + new ArrayList<>()); + + Map request = new HashMap<>(); + request.put("header.User-Agent", "request-ua"); + request.put("header.X-Test", "shared"); + request.put("header.Cookie", "a=1; 51D_Profile=123456"); + request.put("header.X-Blocked", "blocked"); + request.put("cookie.51D_Profile", "123456"); + request.put("cookie.other", "not-shared"); + request.put("query.secret", "not-shared"); + request.put(EVIDENCE_CLIENTIP_KEY, "1.2.3.4"); + + try (FlowData data = pipeline.createFlowData()) { + data.addEvidence( + EVIDENCE_HTTPHEADER_PREFIX + EVIDENCE_SEPERATOR + "user-agent", + "evidence-ua"); + data.addEvidence("server.http-request", + Collections.unmodifiableMap(request)); + shareUsageElement.process(data); + assertNotNull(shareUsageElement.getSendDataFuture()); + shareUsageElement.getSendDataFuture().get(); + + Document doc = docBuilderFactory.newDocumentBuilder().parse( + new ByteArrayInputStream(connector.getBaos().toByteArray())); + + assertEquals("shared", xpath.evaluate( + "//header[@Name='X-Test']", doc), + "A header only the web request holds was not shared."); + NodeList userAgents = (NodeList) xpath.evaluate( + "//header[translate(@Name,'USERAGNT','useragnt')='user-agent']", + doc, XPathConstants.NODESET); + assertEquals(1, userAgents.getLength(), + "The User-Agent in both places was not shared once."); + assertEquals("evidence-ua", userAgents.item(0).getTextContent(), + "The User-Agent in both places did not keep the Evidence value."); + assertEquals(0, ((NodeList) xpath.evaluate( + "//header[@Name='Cookie' or @Name='cookie']", doc, + XPathConstants.NODESET)).getLength(), + "The cookie header was shared."); + assertEquals(0, ((NodeList) xpath.evaluate( + "//header[@Name='X-Blocked']", doc, + XPathConstants.NODESET)).getLength(), + "A blocked header was shared."); + assertEquals("123456", xpath.evaluate( + "//cookie[@Name='51D_Profile']", doc), + "A 51D_ cookie from the web request was not shared."); + assertEquals(0, ((NodeList) xpath.evaluate( + "//cookie[@Name='other'] | //query[@Name='secret']", doc, + XPathConstants.NODESET)).getLength(), + "A cookie or query string parameter the rules leave out was shared."); + assertEquals("1.2.3.4", xpath.evaluate("//ClientIP", doc), + "The client IP from the web request was not shared."); + assertEquals(0, ((NodeList) xpath.evaluate( + "//*[@Name='http-request']", doc, + XPathConstants.NODESET)).getLength(), + "The web request itself was shared."); + } + } + @Test public void ShareUsageElement_LowPercentage() throws Exception { // Arrange diff --git a/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/trackers/ShareUsageTrackerTests.java b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/trackers/ShareUsageTrackerTests.java index 32550b140..a60177c83 100644 --- a/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/trackers/ShareUsageTrackerTests.java +++ b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/trackers/ShareUsageTrackerTests.java @@ -184,6 +184,50 @@ public DataKey answer(InvocationOnMock invocationOnMock) throws Throwable { assertEquals(1, trackedEvents); } + /** + * The tracker's key is of the values usage sharing would share, which + * take in those of the web request. Two requests that differ only in the + * session cookie the web request holds are both tracked. + */ + @Test + public void ShareUsageTracker_WebRequest_SessionCookieTracked() { + assertEquals(2, trackWebRequests( + "cookie." + DEFAULT_SESSION_COOKIE_NAME, "session-0", "session-1")); + } + + /** + * Two requests whose web requests differ only in a value usage sharing + * leaves out, here a cookie other than the session or 51Degrees ones, + * have the same key, so only the first is tracked. + */ + @Test + public void ShareUsageTracker_WebRequest_UnsharedValueIgnored() { + assertEquals(1, trackWebRequests( + "cookie.other", "value-0", "value-1")); + } + + private int trackWebRequests(String key, String... values) { + int trackedEvents = 0; + for (String value : values) { + Map request = new HashMap<>(); + request.put(EVIDENCE_HTTPHEADER_PREFIX + EVIDENCE_SEPERATOR + + "user-agent", "iPhone"); + request.put(key, value); + final Map evidenceData = new HashMap<>(); + evidenceData.put("server.http-request", request); + FlowData data = MockFlowData.createFromEvidence(evidenceData, false); + // A real FlowData makes its key from its Evidence and a filter. + when(data.generateKey(any(EvidenceKeyFilter.class))).then( + invocation -> generateKey( + (EvidenceKeyFilter) invocation.getArgument(0), + evidenceData)); + if (shareUsageTracker.track(data)) { + trackedEvents++; + } + } + return trackedEvents; + } + public DataKey generateKey( EvidenceKeyFilter filter, Map evidence) { diff --git a/pipeline/pipeline.web.packages/pipeline.web/src/test/java/fiftyone/pipeline/web/UsageSharingEvidenceTests.java b/pipeline/pipeline.web.packages/pipeline.web/src/test/java/fiftyone/pipeline/web/UsageSharingEvidenceTests.java new file mode 100644 index 000000000..a2f0aad3b --- /dev/null +++ b/pipeline/pipeline.web.packages/pipeline.web/src/test/java/fiftyone/pipeline/web/UsageSharingEvidenceTests.java @@ -0,0 +1,199 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.web; + +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.core.flowelements.PipelineBuilder; +import fiftyone.pipeline.engines.fiftyone.flowelements.SequenceElementBuilder; +import fiftyone.pipeline.engines.fiftyone.flowelements.ShareUsageBuilder; +import fiftyone.pipeline.jsonbuilder.flowelements.JsonBuilderElementBuilder; +import fiftyone.pipeline.web.services.ClientsidePropertyServiceCore; +import fiftyone.pipeline.web.services.FlowDataProviderCore; +import fiftyone.pipeline.web.services.WebRequestEvidenceServiceCore; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.mockito.ArgumentCaptor; +import org.slf4j.LoggerFactory; + +import javax.servlet.http.Cookie; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import java.io.PrintWriter; +import java.io.StringWriter; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** + * The web integration gives usage sharing the whole web request, and the + * usage sharing element accepts no Evidence, so the client-side endpoints' + * ETag does not change with headers no element uses, If-None-Match among + * them. + */ +public class UsageSharingEvidenceTests { + + private static final String USER_AGENT = + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " + + "(KHTML, like Gecko) Chrome/98.0.4758.102 Safari/537.36"; + + private Pipeline pipeline; + + private final WebRequestEvidenceServiceCore evidenceService = + new WebRequestEvidenceServiceCore.Default(); + + @Before + public void init() throws Exception { + pipeline = new PipelineBuilder(LoggerFactory.getILoggerFactory()) + .addFlowElement(new SequenceElementBuilder( + LoggerFactory.getILoggerFactory()).build()) + .addFlowElement(new JsonBuilderElementBuilder( + LoggerFactory.getILoggerFactory()).build()) + .addFlowElement(new ShareUsageBuilder( + LoggerFactory.getILoggerFactory()) + .setSharePercentage(0) + .build()) + .setAutoCloseElements(true) + .build(); + } + + @After + public void cleanUp() throws Exception { + pipeline.close(); + } + + private static HttpServletRequest request(Map headers) { + HttpServletRequest request = mock(HttpServletRequest.class); + when(request.getMethod()).thenReturn("GET"); + when(request.getHeaderNames()).thenReturn( + Collections.enumeration(headers.keySet())); + for (Map.Entry header : headers.entrySet()) { + when(request.getHeader(header.getKey())) + .thenReturn(header.getValue()); + } + when(request.getCookies()).thenReturn(new Cookie[]{ + new Cookie("51D_Profile", "123456")}); + when(request.getParameterNames()).thenReturn( + Collections.enumeration(Collections.singletonList("q"))); + when(request.getParameter("q")).thenReturn("search"); + when(request.getRemoteAddr()).thenReturn("1.2.3.4"); + when(request.getScheme()).thenReturn("https"); + return request; + } + + private static Map headers() { + Map headers = new LinkedHashMap<>(); + headers.put("User-Agent", USER_AGENT); + headers.put("X-Test", "shared"); + headers.put("Cookie", "51D_Profile=123456"); + return headers; + } + + /** + * The web request is added as server.http-request on every request, + * although no element accepts it, keyed the way Evidence is keyed. A + * header no element accepts is not added as Evidence of its own. + */ + @Test + public void addEvidenceFromRequest_AddsTheWebRequest() throws Exception { + try (FlowData flowData = pipeline.createFlowData()) { + evidenceService.addEvidenceFromRequest( + flowData, request(headers())); + + Map evidence = flowData.getEvidence().asKeyMap(); + assertTrue("The web request was not added.", + evidence.get("server.http-request") instanceof Map); + Map webRequest = (Map) evidence.get("server.http-request"); + assertEquals("shared", webRequest.get("header.X-Test")); + assertEquals(USER_AGENT, webRequest.get("header.User-Agent")); + assertEquals("123456", webRequest.get("cookie.51D_Profile")); + assertEquals("search", webRequest.get("query.q")); + assertEquals("1.2.3.4", webRequest.get("server.client-ip")); + assertFalse("A header no element accepts was added as Evidence.", + evidence.containsKey("header.X-Test")); + } + } + + /** + * A browser revalidates with the ETag it was given in If-None-Match. + * That header changes no result, so the ETag of the revalidation is the + * one it sends, and it gets a 304. + */ + @Test + public void Revalidation_SameETag_Gets304() throws Exception { + Map revalidationHeaders = headers(); + HttpServletRequest first = request(headers()); + try (FlowData firstData = pipeline.createFlowData(); + FlowData revalidationData = pipeline.createFlowData()) { + evidenceService.addEvidenceFromRequest(firstData, first); + firstData.process(); + + FlowDataProviderCore provider = mock(FlowDataProviderCore.class); + when(provider.getFlowData(first)).thenReturn(firstData); + ClientsidePropertyServiceCore service = + new ClientsidePropertyServiceCore.Default(provider, pipeline); + + HttpServletResponse firstResponse = response(); + service.serveJson(first, firstResponse); + ArgumentCaptor etag = ArgumentCaptor.forClass(String.class); + verify(firstResponse).setHeader(eq("ETag"), etag.capture()); + + revalidationHeaders.put("If-None-Match", etag.getValue()); + HttpServletRequest revalidation = request(revalidationHeaders); + evidenceService.addEvidenceFromRequest( + revalidationData, revalidation); + revalidationData.process(); + when(provider.getFlowData(revalidation)) + .thenReturn(revalidationData); + + assertEquals("The revalidation's ETag differs from the one it sends.", + etag.getValue(), + Integer.toString(revalidationData.generateKey( + pipeline.getEvidenceKeyFilter()).hashCode())); + + HttpServletResponse revalidationResponse = response(); + service.serveJson(revalidation, revalidationResponse); + verify(revalidationResponse).setStatus(304); + verify(revalidationResponse, never()).setStatus(200); + verify(revalidationResponse, never()).setContentType(anyString()); + } + } + + private static HttpServletResponse response() throws Exception { + HttpServletResponse response = mock(HttpServletResponse.class); + when(response.getWriter()).thenReturn( + new PrintWriter(new StringWriter())); + return response; + } +} From a7c8bbcdc2b9c19546c8e01eb110248d6ceff05d Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Sat, 7 Nov 2026 22:01:53 +0000 Subject: [PATCH 2/8] Accept no Evidence for usage sharing, and give it the whole web request The usage sharing element's filter accepted every header except Cookie, and the Pipeline's filter is the union of its elements' filters, so it accepted every header too. The client-side ETag is a hash of the Evidence in that filter, so a revalidation carrying If-None-Match hashed differently and never got a 304, and the Vary header named every header. - The element accepts no Evidence. EvidenceKeyFilterShareUsage stays as the rules for which of the values it finds are shared. - ShareUsageEvidence.find gives it every Evidence value and every value of the web request when the Evidence holds server.http-request, keeping the Evidence value for a key in both. The tracker keys on the same values, so the session tracking hash is of what would be shared. - The web integration records every value it offers, accepted or not, in a read-only WebRequestEvidence map, added as server.http-request on every request. It is a map rather than the HttpServletRequest because the usage sharing element does not depend on the servlet API. - The cloud request engine sent every Evidence value whose prefix is not header, cookie or query, so it would have sent the web request. It leaves that key out. --- .../CloudRequestEngineDefault.java | 13 ++- .../data/EvidenceKeyFilterShareUsage.java | 8 +- .../fiftyone/data/ShareUsageEvidence.java | 78 ++++++++++++++ .../fiftyone/flowelements/ShareUsageBase.java | 39 ++++--- .../fiftyone/trackers/ShareUsageTracker.java | 56 ++++++++++ .../web/services/WebRequestEvidence.java | 101 ++++++++++++++++++ .../WebRequestEvidenceServiceCore.java | 10 +- 7 files changed, 280 insertions(+), 25 deletions(-) create mode 100644 pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/data/ShareUsageEvidence.java create mode 100644 pipeline/pipeline.web.packages/pipeline.web/src/main/java/fiftyone/pipeline/web/services/WebRequestEvidence.java diff --git a/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineDefault.java b/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineDefault.java index f3e00e7e7..12ccc17f2 100644 --- a/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineDefault.java +++ b/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineDefault.java @@ -63,6 +63,15 @@ public class CloudRequestEngineDefault extends AspectEngineBase implements CloudRequestEngine { + /** + * The Evidence key of the whole web request, which the web integration + * adds for usage sharing to read. It holds every header and cookie of + * the request, so it is never sent to the cloud service. The usage + * sharing package defines the same key, and this package does not + * depend on that one. + */ + private static final String HTTP_REQUEST_KEY = "server.http-request"; + private HttpClient httpClient; private String endPoint; @@ -266,7 +275,9 @@ private byte[] getContent(FlowData data) throws UnsupportedEncodingException { } Map getFormData(FlowData flowData) { - Map evidence = flowData.getEvidence().asKeyMap(); + Map evidence = new HashMap<>( + flowData.getEvidence().asKeyMap()); + evidence.keySet().removeIf(HTTP_REQUEST_KEY::equalsIgnoreCase); Map formData = new HashMap<>(); // Add evidence in reverse alphabetical order, excluding special keys. diff --git a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/data/EvidenceKeyFilterShareUsage.java b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/data/EvidenceKeyFilterShareUsage.java index 996694614..7aaa496a9 100644 --- a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/data/EvidenceKeyFilterShareUsage.java +++ b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/data/EvidenceKeyFilterShareUsage.java @@ -34,7 +34,9 @@ import static fiftyone.pipeline.core.Constants.*; /** - * This filter is used by the {@link ShareUsageElement}. + * This filter is used by the {@link ShareUsageElement} to decide which of the + * values {@link ShareUsageEvidence#find} finds are shared. It is not the + * element's accepted Evidence, which is empty. * It will include anything that is: * 1) An HTTP header that is not blocked by the constructor parameter, other * than Cookie. @@ -48,10 +50,6 @@ * The prefix of a key is everything before its first '.', and the field * everything after it. An email address, the salt for hashed email * identifiers, the query string and the path are never included. - * - * As this filter is generally inclusive, it will often cause far more - * evidence to be passed into a pipeline than the engine-specific - * filters, which tend to be based on a white list */ public class EvidenceKeyFilterShareUsage implements EvidenceKeyFilter { diff --git a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/data/ShareUsageEvidence.java b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/data/ShareUsageEvidence.java new file mode 100644 index 000000000..a48438286 --- /dev/null +++ b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/data/ShareUsageEvidence.java @@ -0,0 +1,78 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.engines.fiftyone.data; + +import fiftyone.pipeline.core.data.Evidence; + +import java.util.Map; +import java.util.TreeMap; + +/** + * The values usage sharing finds for a request. The usage sharing element + * accepts no Evidence, so that cache keys, the ETag and the Vary header never + * depend on it, and finds what to share here instead. The sharing rules of + * {@link EvidenceKeyFilterShareUsage} then decide which of these are shared. + * @see Specification + */ +public final class ShareUsageEvidence { + + /** + * The Evidence key under which the web integration adds the web request, + * as a read-only map keyed the way Evidence is keyed, such as + * header.user-agent or cookie.51D_Profile. No element accepts it. + */ + public static final String HTTP_REQUEST_KEY = "server.http-request"; + + private ShareUsageEvidence() { + } + + /** + * Every Evidence value, and every value of the web request when the + * Evidence holds it, keyed without regard to case. A key in both keeps + * the Evidence value. The web request itself is not one of the values. + * @param evidence the Evidence of the request + * @return the values found + */ + public static Map find(Evidence evidence) { + Map values = evidence.asKeyMap(); + Map found = + new TreeMap<>(String.CASE_INSENSITIVE_ORDER); + Object request = null; + for (Map.Entry entry : values.entrySet()) { + if (HTTP_REQUEST_KEY.equalsIgnoreCase(entry.getKey())) { + request = entry.getValue(); + } else if (entry.getValue() != null) { + found.put(entry.getKey(), entry.getValue()); + } + } + if (request instanceof Map) { + for (Map.Entry entry : ((Map) request).entrySet()) { + if (entry.getKey() instanceof String && + entry.getValue() != null) { + found.putIfAbsent((String) entry.getKey(), entry.getValue()); + } + } + } + return found; + } +} diff --git a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBase.java b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBase.java index ddb8809ba..96b240773 100644 --- a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBase.java +++ b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBase.java @@ -30,6 +30,7 @@ import fiftyone.pipeline.engines.configuration.CacheConfiguration; import fiftyone.pipeline.engines.fiftyone.data.EvidenceKeyFilterShareUsage; import fiftyone.pipeline.engines.fiftyone.data.EvidenceKeyFilterShareUsageTracker; +import fiftyone.pipeline.engines.fiftyone.data.ShareUsageEvidence; import fiftyone.pipeline.engines.fiftyone.exceptions.HttpException; import fiftyone.pipeline.engines.fiftyone.trackers.ShareUsageTracker; import fiftyone.pipeline.engines.trackers.Tracker; @@ -106,12 +107,15 @@ public abstract class ShareUsageBase protected final String shareUsageUrl; /** - * Evidence key filter including the session id. + * The Evidence this element accepts, which is none, so that cache keys, + * the ETag and the Vary header never depend on it. It finds what to share + * with {@link ShareUsageEvidence#find} instead. */ - private final EvidenceKeyFilter evidenceKeyFilter; + private final EvidenceKeyFilter acceptedEvidence = + new EvidenceKeyFilterWhitelist(new ArrayList()); /** - * Evidence key filter excluding the session id + * Which of the values found are shared, leaving out the session cookie. */ private final EvidenceKeyFilter evidenceKeyFilterExclSession; @@ -425,8 +429,6 @@ protected ShareUsageBase( includedQueryStringParameters.add(Constants.EVIDENCE_SESSIONID_SUFFIX); includedQueryStringParameters.add(Constants.EVIDENCE_SEQUENCE_SUFIX); - evidenceKeyFilter = new EvidenceKeyFilterShareUsage( - blockedHttpHeaders, includedQueryStringParameters, true, sessionCookieName); evidenceKeyFilterExclSession = new EvidenceKeyFilterShareUsage( blockedHttpHeaders, includedQueryStringParameters, false, sessionCookieName); @@ -518,15 +520,16 @@ public void addPipeline(Pipeline pipeline) { @Override protected void processInternal(FlowData flowData){ boolean ignoreData = false; - Map evidence = flowData.getEvidence().asKeyMap(); + Map found = + ShareUsageEvidence.find(flowData.getEvidence()); logger.debug("processInternal()"); if (ignoreDataEvidenceFilter != null) { for (Map.Entry entry : ignoreDataEvidenceFilter) { String key = entry.getKey(); - if (evidence.containsKey(key)) { - if (evidence.get(key).toString().equals(entry.getValue())) { + if (found.containsKey(key)) { + if (found.get(key).toString().equals(entry.getValue())) { ignoreData = true; break; } @@ -535,13 +538,13 @@ protected void processInternal(FlowData flowData){ } if (isCanceled() == false && ignoreData == false) { - processData(flowData); + processData(flowData, found); } } @Override public EvidenceKeyFilter getEvidenceKeyFilter() { - return evidenceKeyFilter; + return acceptedEvidence; } @Override @@ -653,8 +656,9 @@ private boolean isLocalHost(String address) throws UnknownHostException { /** * Process the supplied request data * @param data the {@link FlowData} instance that provides the evidence + * @param found the values {@link ShareUsageEvidence#find} found for it */ - private void processData(FlowData data) { + private void processData(FlowData data, Map found) { logger.debug("processData()"); if (random.nextDouble() <= sharePercentage) { // Check if the tracker will allow sharing of this data @@ -663,7 +667,7 @@ private void processData(FlowData data) { // it to the collection. try { if (evidenceCollection.offer( - getDataFromEvidence(data.getEvidence()), + getDataFromEvidence(found), addTimeout, TimeUnit.MILLISECONDS)) { logger.debug("Queued {} entries", evidenceCollection.size()); @@ -689,18 +693,19 @@ private void processData(FlowData data) { } /** - * Extract the desired data from the evidence. + * Extract the desired data from the values found. * In order to avoid problems with the evidence data being disposed before * it is sent, the data placed into a new object rather than being a * reference to the existing evidence instance. * - * @param evidence an {@link Evidence} instance that contains the data to be extracted - * @return a {@link ShareUsageData} instance populated with data from the evidence + * @param found the values {@link ShareUsageEvidence#find} found + * @return a {@link ShareUsageData} instance populated with the values + * the sharing rules allow */ - private ShareUsageData getDataFromEvidence(Evidence evidence) { + private ShareUsageData getDataFromEvidence(Map found) { ShareUsageData shareUsageData = new ShareUsageData(); - for (Map.Entry entry : evidence.asKeyMap().entrySet()) { + for (Map.Entry entry : found.entrySet()) { // Check if we can send this piece of evidence if (evidenceKeyFilterExclSession.include(entry.getKey()) == false) { continue; diff --git a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/trackers/ShareUsageTracker.java b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/trackers/ShareUsageTracker.java index 33514287d..c6082300e 100644 --- a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/trackers/ShareUsageTracker.java +++ b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/trackers/ShareUsageTracker.java @@ -22,12 +22,19 @@ package fiftyone.pipeline.engines.fiftyone.trackers; +import fiftyone.caching.PutCache; +import fiftyone.pipeline.core.data.DataKey; +import fiftyone.pipeline.core.data.DataKeyBuilder; +import fiftyone.pipeline.core.data.DataKeyBuilderDefault; import fiftyone.pipeline.core.data.EvidenceKeyFilter; import fiftyone.pipeline.core.data.FlowData; import fiftyone.pipeline.engines.configuration.CacheConfiguration; +import fiftyone.pipeline.engines.fiftyone.data.ShareUsageEvidence; import fiftyone.pipeline.engines.trackers.TrackerBase; +import java.io.IOException; import java.util.Date; +import java.util.Map; /** * A tracker used by share usage to attempt to avoid repeatedly sending data @@ -46,6 +53,13 @@ public class ShareUsageTracker extends TrackerBase { */ private final long interval; + /** + * The times keyed by the values usage sharing would share. The cache of + * the base class is keyed by the Evidence alone, which no longer holds + * the web request's values. + */ + private final PutCache sharedValuesCache; + /** * Construct a new instance. * @param configuration the cache configuration to use when building the @@ -61,6 +75,10 @@ public ShareUsageTracker( super(configuration); this.interval = intervalMillis; this.filter = filter; + // The base class has checked that the builder makes a PutCache. + this.sharedValuesCache = (PutCache) + configuration.getCacheBuilder().build( + null, configuration.getSize()); } @Override @@ -79,4 +97,42 @@ protected boolean match(FlowData data, Date value) { protected EvidenceKeyFilter getFilter() { return filter; } + + @Override + public Date get(FlowData data) { + return sharedValuesCache.get(getSharedValuesKey(data)); + } + + @Override + public void put(FlowData data, Date value) { + sharedValuesCache.put(getSharedValuesKey(data), value); + } + + @Override + public void close() throws IOException { + try { + sharedValuesCache.close(); + } finally { + super.close(); + } + } + + /** + * The key of the values {@link ShareUsageEvidence#find} finds that the + * filter includes, which take in the web request's values, as the + * session tracking hash is of the values that would be shared. + */ + private DataKey getSharedValuesKey(FlowData data) { + DataKeyBuilder builder = new DataKeyBuilderDefault(); + for (Map.Entry entry : + ShareUsageEvidence.find(data.getEvidence()).entrySet()) { + if (filter.include(entry.getKey())) { + builder.add( + filter.order(entry.getKey()), + entry.getKey(), + entry.getValue()); + } + } + return builder.build(); + } } diff --git a/pipeline/pipeline.web.packages/pipeline.web/src/main/java/fiftyone/pipeline/web/services/WebRequestEvidence.java b/pipeline/pipeline.web.packages/pipeline.web/src/main/java/fiftyone/pipeline/web/services/WebRequestEvidence.java new file mode 100644 index 000000000..6206d41e5 --- /dev/null +++ b/pipeline/pipeline.web.packages/pipeline.web/src/main/java/fiftyone/pipeline/web/services/WebRequestEvidence.java @@ -0,0 +1,101 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.web.services; + +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.engines.fiftyone.data.ShareUsageEvidence; + +import java.util.AbstractMap; +import java.util.Collections; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; + +/** + * A read-only view of every value the web integration offered as Evidence + * for a request, accepted or not, keyed the way Evidence is keyed, such as + * header.user-agent. It is added to Evidence as + * {@link ShareUsageEvidence#HTTP_REQUEST_KEY} for the usage sharing element, + * which accepts no Evidence and reads the request here instead. + *

+ * It is a map rather than the HttpServletRequest because the usage sharing + * element does not depend on the servlet API, and the values are copied so + * that they stay the same once the request is recycled. + */ +public class WebRequestEvidence extends AbstractMap { + + private final Map values = + new TreeMap<>(String.CASE_INSENSITIVE_ORDER); + + private final Set> entries = + Collections.unmodifiableMap(values).entrySet(); + + WebRequestEvidence() { + } + + /** + * The view in the Evidence of the {@link FlowData}, which is added to it + * the first time. + */ + static WebRequestEvidence of(FlowData flowData) { + Object existing = flowData.getEvidence().asKeyMap() + .get(ShareUsageEvidence.HTTP_REQUEST_KEY); + if (existing instanceof WebRequestEvidence) { + return (WebRequestEvidence) existing; + } + WebRequestEvidence created = new WebRequestEvidence(); + flowData.addEvidence(ShareUsageEvidence.HTTP_REQUEST_KEY, created); + return created; + } + + void record(String key, Object value) { + if (value != null) { + values.put(key, value); + } + } + + @Override + public Set> entrySet() { + return entries; + } + + @Override + public Object get(Object key) { + return key instanceof String ? values.get(key) : null; + } + + @Override + public boolean containsKey(Object key) { + return key instanceof String && values.containsKey(key); + } + + /** + * Says how many values the view holds rather than listing them, so that + * code which prints every Evidence value, such as a debug log or an + * example page, does not print every header and cookie of the request. + */ + @Override + public String toString() { + return "web request (" + values.size() + " values)"; + } +} diff --git a/pipeline/pipeline.web.packages/pipeline.web/src/main/java/fiftyone/pipeline/web/services/WebRequestEvidenceServiceCore.java b/pipeline/pipeline.web.packages/pipeline.web/src/main/java/fiftyone/pipeline/web/services/WebRequestEvidenceServiceCore.java index a12e080f5..0218705a2 100644 --- a/pipeline/pipeline.web.packages/pipeline.web/src/main/java/fiftyone/pipeline/web/services/WebRequestEvidenceServiceCore.java +++ b/pipeline/pipeline.web.packages/pipeline.web/src/main/java/fiftyone/pipeline/web/services/WebRequestEvidenceServiceCore.java @@ -37,7 +37,9 @@ * Service used to populate the {@link Evidence} from a * {@link HttpServletRequest} ready for it to be processed by the Pipeline. * The Pipeline's {@link EvidenceKeyFilter} is used to determine what should be - * collected from the request. + * collected from the request. Every value offered, accepted or not, is also + * added in one {@link WebRequestEvidence} for the usage sharing element, + * which accepts no Evidence. * @see Specification */ public interface WebRequestEvidenceServiceCore { @@ -61,6 +63,8 @@ class Default implements WebRequestEvidenceServiceCore { public void addEvidenceFromRequest( FlowData flowData, HttpServletRequest request) { + // Added on every request, whether or not any element accepts it. + WebRequestEvidence.of(flowData); Enumeration headerNames = request.getHeaderNames(); if (headerNames != null) { @@ -120,7 +124,8 @@ public void addEvidenceFromRequest( /** * Check if the {@link FlowData} requires this item of evidence, and add - * it if it does. + * it if it does. It is recorded in the {@link WebRequestEvidence} + * either way. * @param flowData to check and add to * @param key the key to check and add * @param value the value to add if the check passes @@ -129,6 +134,7 @@ protected void checkAndAdd(FlowData flowData, String key, Object value) { if (flowData.getEvidenceKeyFilter().include(key)) { flowData.addEvidence(key, value); } + WebRequestEvidence.of(flowData).record(key, value); } } } From a645d1f41138a4daad64f2ecfddd2a40b055416d Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Sat, 7 Nov 2026 22:06:29 +0000 Subject: [PATCH 3/8] Test that a callback's session id and sequence still reach the sequence element With usage sharing accepting no Evidence, the session id and sequence of a client-side callback reach Evidence only if the sequence element accepts them, as its specification lists. The revalidation test now sends them, as a callback does, and the usage sharing element's session cookie test expects the element to accept nothing. --- .../flowelements/ShareUsageElementTests.java | 10 +++--- .../web/UsageSharingEvidenceTests.java | 33 ++++++++++++++++++- 2 files changed, 38 insertions(+), 5 deletions(-) diff --git a/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageElementTests.java b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageElementTests.java index b2520614c..6e90d9369 100644 --- a/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageElementTests.java +++ b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageElementTests.java @@ -1248,9 +1248,11 @@ public void ShareUsageElement_RepeatWithNewSequence_NotShared() throws Exception } /** - * The session cookie the element names is kept in evidence for the - * tracker, whatever the case of its name, and never sent. With sessions - * tracked, the same evidence from a new session is shared again. + * The session cookie the element names is read by the tracker, whatever + * the case of its name, and never sent. With sessions tracked, the same + * evidence from a new session is shared again. The element accepts no + * Evidence, so the cookie reaches the tracker from what it finds rather + * than through its accepted Evidence. */ @Test public void ShareUsageElement_TrackSession_SessionCookie() throws Exception { @@ -1280,7 +1282,7 @@ public void ShareUsageElement_TrackSession_SessionCookie() throws Exception { shareUsageElement.close(); // Assert - assertTrue(shareUsageElement.getEvidenceKeyFilter().include(sessionCookie)); + assertFalse(shareUsageElement.getEvidenceKeyFilter().include(sessionCookie)); byte[] output = connector.getBaos().toByteArray(); assertFalse(new String(output, StandardCharsets.UTF_8).contains( "session cookie"), "the session cookie is never sent"); diff --git a/pipeline/pipeline.web.packages/pipeline.web/src/test/java/fiftyone/pipeline/web/UsageSharingEvidenceTests.java b/pipeline/pipeline.web.packages/pipeline.web/src/test/java/fiftyone/pipeline/web/UsageSharingEvidenceTests.java index a2f0aad3b..426d99594 100644 --- a/pipeline/pipeline.web.packages/pipeline.web/src/test/java/fiftyone/pipeline/web/UsageSharingEvidenceTests.java +++ b/pipeline/pipeline.web.packages/pipeline.web/src/test/java/fiftyone/pipeline/web/UsageSharingEvidenceTests.java @@ -42,6 +42,7 @@ import javax.servlet.http.HttpServletResponse; import java.io.PrintWriter; import java.io.StringWriter; +import java.util.Arrays; import java.util.Collections; import java.util.LinkedHashMap; import java.util.Map; @@ -68,6 +69,9 @@ public class UsageSharingEvidenceTests { "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " + "(KHTML, like Gecko) Chrome/98.0.4758.102 Safari/537.36"; + private static final String SESSION_ID = + "1f2e3d4c-5b6a-4987-8a1b-2c3d4e5f6a7b"; + private Pipeline pipeline; private final WebRequestEvidenceServiceCore evidenceService = @@ -104,9 +108,14 @@ private static HttpServletRequest request(Map headers) { } when(request.getCookies()).thenReturn(new Cookie[]{ new Cookie("51D_Profile", "123456")}); + // A callback from the client-side script carries its session id and + // sequence in the query string. when(request.getParameterNames()).thenReturn( - Collections.enumeration(Collections.singletonList("q"))); + Collections.enumeration( + Arrays.asList("q", "session-id", "sequence"))); when(request.getParameter("q")).thenReturn("search"); + when(request.getParameter("session-id")).thenReturn(SESSION_ID); + when(request.getParameter("sequence")).thenReturn("2"); when(request.getRemoteAddr()).thenReturn("1.2.3.4"); when(request.getScheme()).thenReturn("https"); return request; @@ -145,6 +154,28 @@ public void addEvidenceFromRequest_AddsTheWebRequest() throws Exception { } } + /** + * The sequence element accepts the session id and sequence of a + * callback, so they reach Evidence although usage sharing accepts + * nothing, and the sequence counts on from the callback's. + */ + @Test + public void Callback_SessionAndSequence_CarriedOn() throws Exception { + try (FlowData flowData = pipeline.createFlowData()) { + evidenceService.addEvidenceFromRequest( + flowData, request(headers())); + flowData.process(); + + Map evidence = flowData.getEvidence().asKeyMap(); + assertEquals("The callback's session id was not kept.", + SESSION_ID, evidence.get("query.session-id")); + assertEquals("The sequence did not count on from the callback's.", + 3, evidence.get("query.sequence")); + assertTrue("Processing recorded errors: " + flowData.getErrors(), + flowData.getErrors().isEmpty()); + } + } + /** * A browser revalidates with the ETag it was given in If-None-Match. * That header changes no result, so the ETag of the revalidation is the From 7e0f8eaa4296b339293fbe4aa567ed05be7f3c16 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Sat, 7 Nov 2026 22:06:30 +0000 Subject: [PATCH 4/8] Accept the session id and sequence in the sequence element, and read a sequence sent as text The sequence element accepted no Evidence and relied on usage sharing accepting query.session-id and query.sequence, so without it every callback started a new session. It now accepts both, as the specification lists, and reads a sequence that arrives from the query string as text, which the cast to int failed on. --- .../flowelements/SequenceElement.java | 37 +++++++++++++++++-- 1 file changed, 33 insertions(+), 4 deletions(-) diff --git a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/SequenceElement.java b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/SequenceElement.java index 39ca21a6f..5542e99f9 100644 --- a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/SequenceElement.java +++ b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/SequenceElement.java @@ -26,6 +26,7 @@ import fiftyone.pipeline.core.flowelements.FlowElement; import fiftyone.pipeline.core.flowelements.FlowElementBase; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.UUID; @@ -38,6 +39,18 @@ public class SequenceElement extends FlowElementBase { + /** + * The session id and sequence of a callback, which the specification + * lists as this element's accepted Evidence. Without them the web + * integration would not add them, and every callback would start a new + * session. + */ + private final EvidenceKeyFilter acceptedEvidence = + new EvidenceKeyFilterWhitelist(Arrays.asList( + Constants.EVIDENCE_SESSIONID, + Constants.EVIDENCE_SEQUENCE), + String.CASE_INSENSITIVE_ORDER); + /** * Construct a new instance of the {@link FlowElement}. * @param logger logger instance to use for logging @@ -59,9 +72,25 @@ protected void processInternal(FlowData data) throws Exception { if (evidence.containsKey(Constants.EVIDENCE_SEQUENCE) == false) { data.addEvidence(Constants.EVIDENCE_SEQUENCE, 1); } else { - Object sequence = evidence.get(Constants.EVIDENCE_SEQUENCE); - int seq = (int) sequence; - data.addEvidence(Constants.EVIDENCE_SEQUENCE, seq + 1); + data.addEvidence( + Constants.EVIDENCE_SEQUENCE, + getSequence(evidence.get(Constants.EVIDENCE_SEQUENCE)) + 1); + } + } + + /** + * The sequence in Evidence, which is a number when an element set it and + * text when it came from the query string of a callback. Anything that is + * not a number counts as no sequence. + */ + private static int getSequence(Object sequence) { + if (sequence instanceof Number) { + return ((Number) sequence).intValue(); + } + try { + return Integer.parseInt(String.valueOf(sequence).trim()); + } catch (NumberFormatException e) { + return 0; } } @@ -72,7 +101,7 @@ public String getElementDataKey() { @Override public EvidenceKeyFilter getEvidenceKeyFilter() { - return new EvidenceKeyFilterWhitelist(new ArrayList()); + return acceptedEvidence; } @Override From 2b1e52ad234d1c7b398541c7ad9d8c8300d82154 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Sat, 7 Nov 2026 23:16:52 +0000 Subject: [PATCH 5/8] Test that a web pipeline accepts only what its elements use A pipeline of the sequence, set headers, JSON builder, JavaScript builder and usage sharing elements should accept neither If-None-Match nor a header no element uses, and still accept the keys its elements read. --- .../web/UsageSharingEvidenceTests.java | 42 +++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/pipeline/pipeline.web.packages/pipeline.web/src/test/java/fiftyone/pipeline/web/UsageSharingEvidenceTests.java b/pipeline/pipeline.web.packages/pipeline.web/src/test/java/fiftyone/pipeline/web/UsageSharingEvidenceTests.java index 426d99594..b7fa2ac6b 100644 --- a/pipeline/pipeline.web.packages/pipeline.web/src/test/java/fiftyone/pipeline/web/UsageSharingEvidenceTests.java +++ b/pipeline/pipeline.web.packages/pipeline.web/src/test/java/fiftyone/pipeline/web/UsageSharingEvidenceTests.java @@ -22,11 +22,14 @@ package fiftyone.pipeline.web; +import fiftyone.pipeline.core.data.EvidenceKeyFilter; import fiftyone.pipeline.core.data.FlowData; import fiftyone.pipeline.core.flowelements.Pipeline; import fiftyone.pipeline.core.flowelements.PipelineBuilder; import fiftyone.pipeline.engines.fiftyone.flowelements.SequenceElementBuilder; +import fiftyone.pipeline.engines.fiftyone.flowelements.SetHeadersElementBuilder; import fiftyone.pipeline.engines.fiftyone.flowelements.ShareUsageBuilder; +import fiftyone.pipeline.javascriptbuilder.flowelements.JavaScriptBuilderElementBuilder; import fiftyone.pipeline.jsonbuilder.flowelements.JsonBuilderElementBuilder; import fiftyone.pipeline.web.services.ClientsidePropertyServiceCore; import fiftyone.pipeline.web.services.FlowDataProviderCore; @@ -154,6 +157,45 @@ public void addEvidenceFromRequest_AddsTheWebRequest() throws Exception { } } + /** + * Every element a web pipeline of this repository holds accepts only the + * Evidence that can change its results, so the Pipeline accepts neither + * If-None-Match nor a header no element uses, while it still accepts the + * keys its elements read. + */ + @Test + public void WebPipeline_AcceptsOnlyWhatItsElementsUse() throws Exception { + try (Pipeline webPipeline = new PipelineBuilder( + LoggerFactory.getILoggerFactory()) + .addFlowElement(new SequenceElementBuilder( + LoggerFactory.getILoggerFactory()).build()) + .addFlowElement(new SetHeadersElementBuilder( + LoggerFactory.getILoggerFactory()).build()) + .addFlowElement(new JsonBuilderElementBuilder( + LoggerFactory.getILoggerFactory()).build()) + .addFlowElement(new JavaScriptBuilderElementBuilder( + LoggerFactory.getILoggerFactory()).build()) + .addFlowElement(new ShareUsageBuilder( + LoggerFactory.getILoggerFactory()) + .setSharePercentage(0) + .build()) + .setAutoCloseElements(true) + .build()) { + EvidenceKeyFilter accepted = webPipeline.getEvidenceKeyFilter(); + + assertFalse("If-None-Match is accepted.", + accepted.include("header.if-none-match")); + assertFalse("A header no element uses is accepted.", + accepted.include("header.x-test")); + assertFalse("The web request is accepted.", + accepted.include("server.http-request")); + assertTrue("The JavaScript builder's host is not accepted.", + accepted.include("header.host")); + assertTrue("The sequence element's session id is not accepted.", + accepted.include("query.session-id")); + } + } + /** * The sequence element accepts the session id and sequence of a * callback, so they reach Evidence although usage sharing accepts From 60b55503e8166da5ab67dc0cd1839e5d06a57d6d Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Sat, 7 Nov 2026 23:31:47 +0000 Subject: [PATCH 6/8] Give the web request tracker tests an interval no runner reaches They track two requests that should count as one repeat, so the class's 50 millisecond interval let a slow runner track both. --- .../fiftyone/trackers/ShareUsageTrackerTests.java | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/trackers/ShareUsageTrackerTests.java b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/trackers/ShareUsageTrackerTests.java index a60177c83..dff24cbae 100644 --- a/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/trackers/ShareUsageTrackerTests.java +++ b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/trackers/ShareUsageTrackerTests.java @@ -207,6 +207,13 @@ public void ShareUsageTracker_WebRequest_UnsharedValueIgnored() { } private int trackWebRequests(String key, String... values) { + // An interval no run of these tests comes near, so a slow runner + // cannot make the second request count as new. + long noExpiryIntervalMillis = 60 * 60 * 1000; + ShareUsageTracker tracker = new ShareUsageTracker( + new CacheConfiguration(new LruPutCache.Builder(), 1000), + noExpiryIntervalMillis, + evidenceKeyFilter); int trackedEvents = 0; for (String value : values) { Map request = new HashMap<>(); @@ -221,7 +228,7 @@ private int trackWebRequests(String key, String... values) { invocation -> generateKey( (EvidenceKeyFilter) invocation.getArgument(0), evidenceData)); - if (shareUsageTracker.track(data)) { + if (tracker.track(data)) { trackedEvents++; } } From 5fc60ab49a42b4433a76c91dc7d7d860a3ea8c81 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Mon, 9 Nov 2026 14:39:44 +0000 Subject: [PATCH 7/8] Test that the web request holds the method and path, and the Cookie header is never shared after a prefix --- .../EvidenceKeyFilterShareUsageTests.java | 16 ++++++++++++++ .../flowelements/ShareUsageElementTests.java | 9 ++++++++ .../web/UsageSharingEvidenceTests.java | 22 +++++++++++++++++++ 3 files changed, 47 insertions(+) diff --git a/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/data/EvidenceKeyFilterShareUsageTests.java b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/data/EvidenceKeyFilterShareUsageTests.java index 6e6e17147..c0c97c288 100644 --- a/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/data/EvidenceKeyFilterShareUsageTests.java +++ b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/data/EvidenceKeyFilterShareUsageTests.java @@ -159,6 +159,22 @@ public void neverShared() { } } + /** + * The Cookie header carries every cookie, so it is never shared, whether + * bare or after a prefix, and without regard to case. + */ + @Test + public void cookieHeaderNeverShared() { + EvidenceKeyFilterShareUsage filter = defaults(); + for (String key : Arrays.asList( + "header.cookie", "HEADER.COOKIE", "x.header.cookie", + "server.header.cookie")) { + assertFalse(filter.include(key), key); + } + assertTrue(filter.include("header.cookies-allowed")); + assertTrue(filter.include("server.request-method")); + } + /** * A key that only ends with the same letters as a key never shared is * treated as an ordinary key. diff --git a/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageElementTests.java b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageElementTests.java index 6e90d9369..5a7a62563 100644 --- a/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageElementTests.java +++ b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageElementTests.java @@ -515,6 +515,8 @@ public void ShareUsageElement_WebRequest_SharesWhatNoOtherElementAccepts() request.put("cookie.other", "not-shared"); request.put("query.secret", "not-shared"); request.put(EVIDENCE_CLIENTIP_KEY, "1.2.3.4"); + request.put("server.request-method", "GET"); + request.put("server.request-path", "/a%2Fb/c"); try (FlowData data = pipeline.createFlowData()) { data.addEvidence( @@ -556,6 +558,13 @@ public void ShareUsageElement_WebRequest_SharesWhatNoOtherElementAccepts() "A cookie or query string parameter the rules leave out was shared."); assertEquals("1.2.3.4", xpath.evaluate("//ClientIP", doc), "The client IP from the web request was not shared."); + assertEquals("GET", xpath.evaluate( + "//server[@Name='request-method']", doc), + "The method of the request line was not shared."); + assertEquals(0, ((NodeList) xpath.evaluate( + "//*[@Name='request-path']", doc, + XPathConstants.NODESET)).getLength(), + "The path of the request line was shared."); assertEquals(0, ((NodeList) xpath.evaluate( "//*[@Name='http-request']", doc, XPathConstants.NODESET)).getLength(), diff --git a/pipeline/pipeline.web.packages/pipeline.web/src/test/java/fiftyone/pipeline/web/UsageSharingEvidenceTests.java b/pipeline/pipeline.web.packages/pipeline.web/src/test/java/fiftyone/pipeline/web/UsageSharingEvidenceTests.java index b7fa2ac6b..b06d5e8b2 100644 --- a/pipeline/pipeline.web.packages/pipeline.web/src/test/java/fiftyone/pipeline/web/UsageSharingEvidenceTests.java +++ b/pipeline/pipeline.web.packages/pipeline.web/src/test/java/fiftyone/pipeline/web/UsageSharingEvidenceTests.java @@ -103,6 +103,7 @@ public void cleanUp() throws Exception { private static HttpServletRequest request(Map headers) { HttpServletRequest request = mock(HttpServletRequest.class); when(request.getMethod()).thenReturn("GET"); + when(request.getRequestURI()).thenReturn("/a%2Fb/c"); when(request.getHeaderNames()).thenReturn( Collections.enumeration(headers.keySet())); for (Map.Entry header : headers.entrySet()) { @@ -157,6 +158,27 @@ public void addEvidenceFromRequest_AddsTheWebRequest() throws Exception { } } + /** + * The web request holds the method and the path of the request line as + * sent, with no decoding, beside the other parts the specification names. + */ + @Test + public void addEvidenceFromRequest_WebRequestHoldsMethodAndPath() + throws Exception { + try (FlowData flowData = pipeline.createFlowData()) { + evidenceService.addEvidenceFromRequest( + flowData, request(headers())); + + Map webRequest = (Map) flowData.getEvidence() + .asKeyMap().get("server.http-request"); + assertEquals("The method is missing.", + "GET", webRequest.get("server.request-method")); + assertEquals("The path is missing or was decoded.", + "/a%2Fb/c", webRequest.get("server.request-path")); + assertEquals("https", webRequest.get("header.protocol")); + } + } + /** * Every element a web pipeline of this repository holds accepts only the * Evidence that can change its results, so the Pipeline accepts neither From ec35e71cf5ac0e675a73cee1a84827ea4c912260 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Mon, 9 Nov 2026 14:41:25 +0000 Subject: [PATCH 8/8] Add the request line's method and path to the web request, and never share the Cookie header after a prefix --- .../fiftyone/data/EvidenceKeyFilterShareUsage.java | 2 ++ .../web/services/WebRequestEvidenceServiceCore.java | 13 +++++++++++++ 2 files changed, 15 insertions(+) diff --git a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/data/EvidenceKeyFilterShareUsage.java b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/data/EvidenceKeyFilterShareUsage.java index 7aaa496a9..07a21856e 100644 --- a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/data/EvidenceKeyFilterShareUsage.java +++ b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/data/EvidenceKeyFilterShareUsage.java @@ -83,6 +83,7 @@ public class EvidenceKeyFilterShareUsage implements EvidenceKeyFilter { * Evidence keys that are never shared, matched without regard to case as * the whole key or after a '.'. 'id.email' and 'id.salt' are a raw email * address and the salt used to make hashed email identifiers. + * 'header.cookie' is the Cookie header, which carries every cookie. * 'request-query' is the whole query string, which would undo the rules * for its parameters. 'request-path' would make each page a visitor opens * look like a new session to the tracker. @@ -90,6 +91,7 @@ public class EvidenceKeyFilterShareUsage implements EvidenceKeyFilter { private static final String[] NEVER_SHARED_EVIDENCE_KEYS = { "id.email", "id.salt", + "header.cookie", "request-query", "request-path" }; diff --git a/pipeline/pipeline.web.packages/pipeline.web/src/main/java/fiftyone/pipeline/web/services/WebRequestEvidenceServiceCore.java b/pipeline/pipeline.web.packages/pipeline.web/src/main/java/fiftyone/pipeline/web/services/WebRequestEvidenceServiceCore.java index 0218705a2..03f853e17 100644 --- a/pipeline/pipeline.web.packages/pipeline.web/src/main/java/fiftyone/pipeline/web/services/WebRequestEvidenceServiceCore.java +++ b/pipeline/pipeline.web.packages/pipeline.web/src/main/java/fiftyone/pipeline/web/services/WebRequestEvidenceServiceCore.java @@ -59,6 +59,12 @@ public interface WebRequestEvidenceServiceCore { */ class Default implements WebRequestEvidenceServiceCore { + private static final String EVIDENCE_REQUEST_METHOD = + EVIDENCE_SERVER_PREFIX + EVIDENCE_SEPERATOR + "request-method"; + + private static final String EVIDENCE_REQUEST_PATH = + EVIDENCE_SERVER_PREFIX + EVIDENCE_SEPERATOR + "request-path"; + @Override public void addEvidenceFromRequest( FlowData flowData, @@ -120,6 +126,13 @@ public void addEvidenceFromRequest( checkAndAdd(flowData, EVIDENCE_CLIENTIP_KEY, request.getRemoteAddr()); checkAndAdd(flowData, EVIDENCE_PROTOCOL, request.getScheme()); + + // The method and the path of the request line, as sent. + checkAndAdd(flowData, EVIDENCE_REQUEST_METHOD, request.getMethod()); + checkAndAdd( + flowData, + EVIDENCE_REQUEST_PATH, + request.getRequestURI()); } /**