From d989ba0412dce71bb0c838389aa914fb5e6748df Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Mon, 9 Nov 2026 20:38:44 +0000 Subject: [PATCH 1/6] Add LicenseKeyMask, which shows a license key by its last 4 characters A license key in a log entry or an error message is shown only by its last 4 characters, with every character before them replaced by '*', so a reader can tell two keys apart but cannot use either. A key of 4 characters or fewer is replaced in full, because its last 4 characters would be all of it. maskIn masks each key in a text, as written and as encoded in a URL, and gives a throwable to log in place of one whose message, causes or suppressed throwables hold a key, keeping the class names and stack traces. --- .../pipeline/util/LicenseKeyMask.java | 210 ++++++++++++++++++ .../pipeline/util/LicenseKeyMaskTests.java | 156 +++++++++++++ 2 files changed, 366 insertions(+) create mode 100644 pipeline/pipeline.core/src/main/java/fiftyone/pipeline/util/LicenseKeyMask.java create mode 100644 pipeline/pipeline.core/src/test/java/fiftyone/pipeline/util/LicenseKeyMaskTests.java diff --git a/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/util/LicenseKeyMask.java b/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/util/LicenseKeyMask.java new file mode 100644 index 000000000..d5d82f864 --- /dev/null +++ b/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/util/LicenseKeyMask.java @@ -0,0 +1,210 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.util; + +import java.io.UnsupportedEncodingException; +import java.net.URLEncoder; +import java.util.ArrayList; +import java.util.Collections; +import java.util.IdentityHashMap; +import java.util.List; +import java.util.Map; + +/** + * Masks license keys in text written to a log entry or an error message. A + * masked key shows only its last 4 characters, with every character before + * them replaced by '*', so a reader can tell two keys apart but cannot use + * either. A key of 4 characters or fewer is replaced in full, because its + * last 4 characters would be the whole key. + * @see Specification + */ +public final class LicenseKeyMask { + + /** + * How many characters at the end of a key stay visible. + */ + public static final int VISIBLE_CHARACTERS = 4; + + /** + * The character that replaces each hidden character of a key. + */ + public static final char MASK_CHARACTER = '*'; + + private LicenseKeyMask() { + } + + /** + * The key with every character before its last 4 replaced. + * @param key a license key, or null + * @return the masked key, or null if the key is null + */ + public static String mask(String key) { + if (key == null) { + return null; + } + int hidden = key.length() > VISIBLE_CHARACTERS ? + key.length() - VISIBLE_CHARACTERS : + key.length(); + StringBuilder masked = new StringBuilder(key.length()); + for (int i = 0; i < hidden; i++) { + masked.append(MASK_CHARACTER); + } + return masked.append(key, hidden, key.length()).toString(); + } + + /** + * The text with every occurrence of each key masked, both as the key is + * written and as it is encoded in a URL. + * @param text the text, such as a URL or an error message, or null + * @param keys the license keys, or null + * @return the masked text, or null if the text is null + */ + public static String maskIn(String text, Iterable keys) { + if (text == null || keys == null) { + return text; + } + String result = text; + for (String key : longestFirst(keys)) { + result = result.replace(key, mask(key)); + String encoded = urlEncode(key); + if (encoded.equals(key) == false) { + result = result.replace(encoded, mask(encoded)); + } + } + return result; + } + + /** + * The throwable to log or report in place of the one given, so that no + * message in it, its causes or its suppressed throwables holds a key. + * When none does, the throwable given is returned. Otherwise a copy is + * returned whose messages are masked, each starting with the name of the + * class it stands for, and whose stack traces are the originals. + * @param throwable the throwable, or null + * @param keys the license keys, or null + * @return a throwable that holds no key + */ + public static Throwable maskIn(Throwable throwable, Iterable keys) { + if (throwable == null || + keys == null || + holdsKey(throwable, keys, new IdentityHashMap()) + == false) { + return throwable; + } + return copy(throwable, keys, new IdentityHashMap()); + } + + private static boolean holdsKey( + Throwable throwable, + Iterable keys, + Map seen) { + if (throwable == null || seen.put(throwable, Boolean.TRUE) != null) { + return false; + } + String message = throwable.getMessage(); + if (message != null && message.equals(maskIn(message, keys)) == false) { + return true; + } + if (holdsKey(throwable.getCause(), keys, seen)) { + return true; + } + for (Throwable suppressed : throwable.getSuppressed()) { + if (holdsKey(suppressed, keys, seen)) { + return true; + } + } + return false; + } + + private static Throwable copy( + Throwable throwable, + Iterable keys, + Map copies) { + Throwable existing = copies.get(throwable); + if (existing != null) { + return existing; + } + String message = maskIn(throwable.getMessage(), keys); + MaskedThrowable masked = new MaskedThrowable( + throwable.getClass().getName() + + (message == null ? "" : ": " + message)); + copies.put(throwable, masked); + masked.setStackTrace(throwable.getStackTrace()); + Throwable cause = throwable.getCause(); + if (cause != null) { + Throwable maskedCause = copy(cause, keys, copies); + if (maskedCause != masked) { + masked.initCause(maskedCause); + } + } + for (Throwable suppressed : throwable.getSuppressed()) { + Throwable maskedSuppressed = copy(suppressed, keys, copies); + if (maskedSuppressed != masked) { + masked.addSuppressed(maskedSuppressed); + } + } + return masked; + } + + /** + * The keys that are not null or empty, longest first, so that a key + * that holds another is masked whole. + */ + private static List longestFirst(Iterable keys) { + List result = new ArrayList<>(); + for (String key : keys) { + if (key != null && key.isEmpty() == false) { + result.add(key); + } + } + Collections.sort(result, (a, b) -> b.length() - a.length()); + return result; + } + + private static String urlEncode(String key) { + try { + return URLEncoder.encode(key, "UTF-8"); + } catch (UnsupportedEncodingException e) { + // Every Java platform supports UTF-8. + throw new IllegalStateException(e); + } + } + + /** + * Stands for a throwable whose message held a license key. Its message + * starts with the name of the class it stands for, and it prints as + * that message alone. + */ + private static final class MaskedThrowable extends Exception { + private static final long serialVersionUID = 1L; + + MaskedThrowable(String message) { + super(message); + } + + @Override + public String toString() { + return getMessage(); + } + } +} diff --git a/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/util/LicenseKeyMaskTests.java b/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/util/LicenseKeyMaskTests.java new file mode 100644 index 000000000..7664d4984 --- /dev/null +++ b/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/util/LicenseKeyMaskTests.java @@ -0,0 +1,156 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.util; + +import org.junit.Test; + +import java.io.IOException; +import java.io.PrintWriter; +import java.io.StringWriter; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; + +/** + * A license key in a log entry or an error message shows only its last 4 + * characters, with every character before them replaced. + */ +public class LicenseKeyMaskTests { + + private static final String KEY = "TESTKEYNOTREALABCDEFGHIJ1234"; + private static final String MASKED = "************************1234"; + private static final String OTHER_KEY = "OTHERTESTKEYNOTREALXYZ5678"; + private static final String OTHER_MASKED = "**********************5678"; + + @Test + public void mask_ShowsTheLast4Characters() { + assertEquals(MASKED, LicenseKeyMask.mask(KEY)); + assertEquals(KEY.length(), LicenseKeyMask.mask(KEY).length()); + } + + /** + * A key of 4 characters or fewer would be shown whole, so all of it is + * replaced. + */ + @Test + public void mask_ShortKeyIsReplacedInFull() { + assertEquals("****", LicenseKeyMask.mask("1234")); + assertEquals("**", LicenseKeyMask.mask("12")); + assertEquals("*2345", LicenseKeyMask.mask("12345")); + assertEquals("", LicenseKeyMask.mask("")); + assertNull(LicenseKeyMask.mask(null)); + } + + @Test + public void maskIn_MasksEveryKeyInAUrl() { + String url = "https://distributor.example/api/v2/download?" + + "LicenseKeys=" + KEY + "|" + OTHER_KEY + "&Download=True"; + + String masked = LicenseKeyMask.maskIn( + url, + Arrays.asList(KEY, OTHER_KEY)); + + assertEquals( + "https://distributor.example/api/v2/download?" + + "LicenseKeys=" + MASKED + "|" + OTHER_MASKED + + "&Download=True", + masked); + } + + /** + * A key that a URL holds encoded is masked in its encoded form. + */ + @Test + public void maskIn_MasksAKeyEncodedInAUrl() { + String key = "KEY+WITH/CHARS=ENCODED9876"; + String url = "https://example.test/?license=" + + "KEY%2BWITH%2FCHARS%3DENCODED9876"; + + String masked = LicenseKeyMask.maskIn(url, Collections.singletonList(key)); + + assertFalse(masked, masked.contains("KEY%2BWITH")); + assertTrue(masked, masked.endsWith("9876")); + } + + @Test + public void maskIn_LeavesTextWithoutAKeyAlone() { + String text = "No key here."; + assertEquals(text, LicenseKeyMask.maskIn(text, Arrays.asList(KEY))); + assertNull(LicenseKeyMask.maskIn((String) null, Arrays.asList(KEY))); + assertEquals(text, LicenseKeyMask.maskIn(text, null)); + assertEquals( + text, + LicenseKeyMask.maskIn(text, Arrays.asList(null, ""))); + } + + /** + * The message of an exception an HTTP library raises often holds the + * URL it requested, and so the key. The throwable to log holds the key + * in none of its messages, its causes' or its suppressed throwables', + * and keeps the name of each class it stands for. + */ + @Test + public void maskIn_MasksAThrowableAndItsCauses() { + IOException cause = new IOException( + "Server returned HTTP response code: 403 for URL: " + + "https://distributor.example/?LicenseKeys=" + KEY); + RuntimeException thrown = new RuntimeException( + "Download failed for " + KEY, cause); + thrown.addSuppressed(new IllegalStateException(OTHER_KEY)); + List keys = Arrays.asList(KEY, OTHER_KEY); + + Throwable masked = LicenseKeyMask.maskIn(thrown, keys); + String printed = printed(masked); + + assertFalse(printed, printed.contains(KEY)); + assertFalse(printed, printed.contains(OTHER_KEY)); + assertTrue(printed, printed.contains(MASKED)); + assertTrue(printed, printed.contains(OTHER_MASKED)); + assertTrue(printed, printed.contains("java.lang.RuntimeException")); + assertTrue(printed, printed.contains("java.io.IOException")); + assertTrue(printed, + printed.contains("java.lang.IllegalStateException")); + assertEquals( + thrown.getStackTrace().length, + masked.getStackTrace().length); + } + + @Test + public void maskIn_ThrowableWithoutAKeyIsReturnedAsItIs() { + IOException thrown = new IOException("Connection refused"); + assertSame(thrown, LicenseKeyMask.maskIn(thrown, Arrays.asList(KEY))); + assertNull(LicenseKeyMask.maskIn((Throwable) null, Arrays.asList(KEY))); + } + + private static String printed(Throwable throwable) { + StringWriter text = new StringWriter(); + throwable.printStackTrace(new PrintWriter(text)); + return text.toString(); + } +} From 382fa6c3dfc3022ccd4b35c94306b11ee77938b0 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Mon, 9 Nov 2026 20:41:03 +0000 Subject: [PATCH 2/6] Test that a license key reaches no log entry or error message in full The data update service is tested for a 403, a 429, another failing status, a 304, no response and an exception that names the URL, with two keys in the update URL. Everything it passes to the logger, at every level and with each exception's stack trace, holds each key only by its last 4 characters. The cloud request engine is tested with an error from the cloud that names the license key, and the 51Did client with a 401 body and a 400 errors list that name the licence, and with a key of 4 characters, which is replaced in full. --- .../flowelements/CloudRequestEngineTests.java | 62 ++++++ .../fiftyone/pipeline/did/DidClientTests.java | 68 ++++++ .../DataUpdateServiceLicenseKeyTests.java | 194 ++++++++++++++++++ 3 files changed, 324 insertions(+) create mode 100644 pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/DataUpdateServiceLicenseKeyTests.java diff --git a/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineTests.java b/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineTests.java index 80ebde301..1259fd677 100644 --- a/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineTests.java +++ b/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineTests.java @@ -40,6 +40,8 @@ import org.junit.jupiter.params.provider.MethodSource; import org.mockito.ArgumentCaptor; import org.mockito.ArgumentMatchers; +import org.slf4j.ILoggerFactory; +import org.slf4j.Logger; import java.io.IOException; import java.net.MalformedURLException; @@ -101,6 +103,66 @@ public void Process() throws Exception { } } + /** + * The cloud answers a refused license key with an error that names it. + * The exception the engine raises, and every log entry, show the key + * only by its last 4 characters. + */ + @Test + public void Process_LicenseKeyInCloudError_Masked() throws Exception { + final String licenseKey = "TESTKEYNOTREALABCDEFGHIJ1234"; + final String masked = "************************1234"; + jsonResponse = "{\"errors\":[\"Invalid license key(s) supplied: " + + licenseKey + "\"]}"; + configureMockedClient(); + Logger logger = mock(Logger.class); + when(logger.isDebugEnabled()).thenReturn(true); + when(logger.isInfoEnabled()).thenReturn(true); + when(logger.isWarnEnabled()).thenReturn(true); + when(logger.isErrorEnabled()).thenReturn(true); + ILoggerFactory capturingLoggerFactory = mock(ILoggerFactory.class); + when(capturingLoggerFactory.getLogger(anyString())).thenReturn(logger); + + CloudRequestEngine engine = new CloudRequestEngineBuilder( + capturingLoggerFactory, httpClient) + .setResourceKey("resource_key") + .setLicenseKey(licenseKey) + .build(); + + try (Pipeline pipeline = new PipelineBuilder(capturingLoggerFactory) + .setSuppressProcessException(true) + .addFlowElement(engine) + .build(); + FlowData data = pipeline.createFlowData()) { + data.addEvidence("query.User-Agent", "iPhone"); + data.process(); + + assertFalse(data.getErrors().isEmpty()); + StringBuilder reported = new StringBuilder(); + for (FlowError error : data.getErrors()) { + reported.append(printed(error.getThrowable())); + } + for (org.mockito.invocation.Invocation invocation : + mockingDetails(logger).getInvocations()) { + for (Object argument : invocation.getArguments()) { + reported.append(argument instanceof Throwable ? + printed((Throwable) argument) : + String.valueOf(argument)); + } + } + assertFalse(reported.toString().contains(licenseKey), + reported.toString()); + assertTrue(reported.toString().contains(masked), + reported.toString()); + } + } + + private static String printed(Throwable throwable) { + java.io.StringWriter text = new java.io.StringWriter(); + throwable.printStackTrace(new java.io.PrintWriter(text)); + return text.toString(); + } + /** * Test errors thrown by cloud request engine are added to flow.errors when SuppressProcessExceptions == true */ diff --git a/pipeline/pipeline.did/src/test/java/fiftyone/pipeline/did/DidClientTests.java b/pipeline/pipeline.did/src/test/java/fiftyone/pipeline/did/DidClientTests.java index 71969a6ac..cea3ec788 100644 --- a/pipeline/pipeline.did/src/test/java/fiftyone/pipeline/did/DidClientTests.java +++ b/pipeline/pipeline.did/src/test/java/fiftyone/pipeline/did/DidClientTests.java @@ -67,6 +67,9 @@ public class DidClientTests { private static final String ENDPOINT = "https://example.test/api/v4/"; + private static final String TEST_LICENCE = "TESTKEYNOTREALABCDEFGHIJ1234"; + private static final String TEST_LICENCE_MASKED = + "************************1234"; private static final Instant WEEK1 = Instant.parse("2026-08-03T00:00:00Z"); private static final Instant WEEK2 = WEEK1.plus(Duration.ofDays(7)); private static final Instant WEEK3 = WEEK2.plus(Duration.ofDays(7)); @@ -1273,6 +1276,71 @@ public void redeem_TransportFailureFailsWithIoException() { client.redeem(validDid, "sealed", "abc")); } + /** + * The service answers a refused licence key with a body that names it. + * The error message shows the key only by its last 4 characters, with + * every character before them replaced, as every 51Degrees package + * writes a licence key in a message. + */ + @Test + public void redeem_RefusedLicenceIsMaskedInTheMessage() { + transport.queue(401, + "Invalid license key(s) supplied: " + TEST_LICENCE); + + DidHttpException error = failure(DidHttpException.class, + licensedClient().redeem(validDid, "sealed", "abc")); + + assertEquals(401, error.getStatusCode()); + assertFalse(error.getMessage(), + error.getMessage().contains(TEST_LICENCE)); + assertTrue(error.getMessage(), + error.getMessage().contains(TEST_LICENCE_MASKED)); + } + + @Test + public void redeem_LicenceInErrorsIsMaskedInTheMessage() { + transport.queue(400, "{\"errors\":[\"The license key " + + TEST_LICENCE + " does not include this product.\"]}"); + + IllegalArgumentException error = failure( + IllegalArgumentException.class, + licensedClient().redeem(validDid, "sealed", "abc")); + + assertFalse(error.getMessage(), + error.getMessage().contains(TEST_LICENCE)); + assertTrue(error.getMessage(), + error.getMessage().contains(TEST_LICENCE_MASKED)); + } + + /** + * A key of 4 characters or fewer is replaced in full, because its last + * 4 characters would be all of it. + */ + @Test + public void redeem_ShortLicenceIsMaskedInFull() { + DidClient shortLicence = DidClient.builder("resource") + .licenceKey("1234") + .endpoint(ENDPOINT) + .transport(transport) + .build(); + transport.queue(401, "Invalid license key(s) supplied: 1234"); + + DidHttpException error = failure(DidHttpException.class, + shortLicence.redeem(validDid, "sealed", "abc")); + + assertFalse(error.getMessage(), error.getMessage().contains("1234")); + assertTrue(error.getMessage(), error.getMessage().endsWith("****")); + } + + private DidClient licensedClient() { + return DidClient.builder("resource") + .licenceKey(TEST_LICENCE) + .endpoint(ENDPOINT) + .transport(transport) + .clock(clock) + .build(); + } + @Test public void redeem_WithoutLicenceKeyOmitsTheField() throws Exception { DidClient noLicence = DidClient.builder("resource") diff --git a/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/DataUpdateServiceLicenseKeyTests.java b/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/DataUpdateServiceLicenseKeyTests.java new file mode 100644 index 000000000..ca5594a1f --- /dev/null +++ b/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/DataUpdateServiceLicenseKeyTests.java @@ -0,0 +1,194 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.engines.services; + +import fiftyone.common.wrappers.io.FileWrapperFactory; +import fiftyone.pipeline.engines.configuration.DataFileConfiguration; +import fiftyone.pipeline.engines.configuration.DataFileConfigurationDefault; +import fiftyone.pipeline.engines.data.AspectData; +import fiftyone.pipeline.engines.data.AspectEngineDataFileDefault; +import fiftyone.pipeline.engines.data.AspectPropertyMetaData; +import fiftyone.pipeline.engines.flowelements.OnPremiseAspectEngine; +import fiftyone.pipeline.engines.services.update.FutureFactory; +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mockito; +import org.mockito.invocation.Invocation; +import org.slf4j.Logger; + +import java.io.IOException; +import java.io.PrintWriter; +import java.io.StringWriter; +import java.net.HttpURLConnection; +import java.net.URL; +import java.util.Arrays; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * The data update URL carries the license keys in its query. Every log + * entry the data update service writes about a request, at any level and + * with any exception it logs, shows each key only by its last 4 + * characters. + */ +public class DataUpdateServiceLicenseKeyTests { + + private static final String KEY = "TESTKEYNOTREALABCDEFGHIJ1234"; + private static final String MASKED = "************************1234"; + private static final String OTHER_KEY = "OTHERTESTKEYNOTREALXYZ5678"; + private static final String OTHER_MASKED = "**********************5678"; + private static final String UPDATE_URL = + "https://distributor.example/api/v2/download"; + + private Logger logger; + private HttpClient httpClient; + private HttpURLConnection connection; + private DataUpdateServiceDefault service; + private OnPremiseAspectEngine engine; + + @Before + @SuppressWarnings("unchecked") + public void init() throws IOException { + logger = mock(Logger.class); + when(logger.isDebugEnabled()).thenReturn(true); + when(logger.isInfoEnabled()).thenReturn(true); + when(logger.isWarnEnabled()).thenReturn(true); + when(logger.isErrorEnabled()).thenReturn(true); + connection = mock(HttpURLConnection.class); + httpClient = mock(HttpClient.class); + when(httpClient.connect(any(URL.class))).thenReturn(connection); + service = new DataUpdateServiceDefault( + logger, + httpClient, + mock(FileWrapperFactory.class), + mock(FutureFactory.class)); + + // A data file updated in memory, whose URL carries the keys as the + // 51Degrees URL formatter puts them. + final DataFileConfiguration config = new DataFileConfigurationDefault(); + config.setDataUpdateUrl(UPDATE_URL); + config.setDataUpdateLicenseKeys(Arrays.asList(KEY, OTHER_KEY)); + config.setUrlFormatter(dataFile -> config.getDataUpdateUrl() + + "?LicenseKeys=" + KEY + "|" + OTHER_KEY + + "&Download=True&Type=HashV41"); + config.setVerifyModifiedSince(false); + config.setFileSystemWatcherEnabled(false); + config.setAutomaticUpdatesEnabled(false); + AspectEngineDataFileDefault file = new AspectEngineDataFileDefault(); + file.setConfiguration(config); + + engine = mock(OnPremiseAspectEngine.class); + when(engine.getDataFileMetaData()).thenReturn(file); + } + + @Test + public void Forbidden_LogsMaskedKeys() throws IOException { + when(connection.getResponseCode()) + .thenReturn(HttpURLConnection.HTTP_FORBIDDEN); + assertLogsMaskedKeys(); + } + + @Test + public void TooManyRequests_LogsMaskedKeys() throws IOException { + when(connection.getResponseCode()).thenReturn(429); + assertLogsMaskedKeys(); + } + + @Test + public void OtherStatus_LogsMaskedKeys() throws IOException { + when(connection.getResponseCode()) + .thenReturn(HttpURLConnection.HTTP_INTERNAL_ERROR); + assertLogsMaskedKeys(); + } + + @Test + public void NotModified_LogsMaskedKeys() throws IOException { + when(connection.getResponseCode()) + .thenReturn(HttpURLConnection.HTTP_NOT_MODIFIED); + assertLogsMaskedKeys(); + } + + @Test + public void NoResponse_LogsMaskedKeys() throws IOException { + when(httpClient.connect(any(URL.class))).thenReturn(null); + assertLogsMaskedKeys(); + } + + /** + * The exception an HTTP library raises often names the URL it + * requested, and the exception is logged with its stack trace. + */ + @Test + public void RequestException_LogsMaskedKeys() throws IOException { + when(connection.getResponseCode()).thenThrow(new IOException( + "Server returned HTTP response code: 403 for URL: " + + UPDATE_URL + "?LicenseKeys=" + KEY + "|" + OTHER_KEY)); + assertLogsMaskedKeys(); + } + + private void assertLogsMaskedKeys() { + service.checkForUpdate(engine); + + String logged = logged(); + assertFalse(logged, logged.contains(KEY)); + assertFalse(logged, logged.contains(OTHER_KEY)); + assertTrue(logged, logged.contains(MASKED)); + assertTrue(logged, logged.contains(OTHER_MASKED)); + } + + /** + * Everything passed to the logger, at every level, with the stack trace + * of each throwable as it would be printed. + */ + private String logged() { + StringBuilder text = new StringBuilder(); + for (Invocation invocation : + Mockito.mockingDetails(logger).getInvocations()) { + text.append(invocation.getMethod().getName()).append(':'); + for (Object argument : invocation.getArguments()) { + append(text, argument); + } + text.append('\n'); + } + return text.toString(); + } + + private static void append(StringBuilder text, Object argument) { + if (argument instanceof Object[]) { + for (Object item : (Object[]) argument) { + append(text, item); + } + } else if (argument instanceof Throwable) { + StringWriter trace = new StringWriter(); + ((Throwable) argument).printStackTrace(new PrintWriter(trace)); + text.append(' ').append(trace); + } else { + text.append(' ').append(argument); + } + } +} From cb0b80e52e68862311d53406a361abac01cadf3d Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Mon, 9 Nov 2026 20:42:39 +0000 Subject: [PATCH 3/6] Mask license keys in data update logs, cloud errors and 51Did redeem errors The data update service logged the formatted update URL, whose query carries the license keys, at debug and error level, and logged the exception an HTTP library raised, whose message often names that URL. Every message now names the URL with each configured key masked by LicenseKeyMask, and the exception is logged as a copy whose messages are masked. The cloud answers a refused license key with an error that names it, and the cloud request engine put that text in the exception it raised, which can reach the log and the JSON builder's errors list. Its messages now mask the license key, and each key in it when it holds several separated by '|'. The 51Did client puts the start of the body in the message of a redeem error, and the errors list of a 400 answer in the message of the IllegalArgumentException it raises. Both now mask the licence key the redeem request sent. The 51Did package depends on nothing in the Pipeline, so it masks with a private method that gives the same result as LicenseKeyMask. The exception's body is kept as the service sent it. --- .../CloudRequestEngineDefault.java | 17 ++++++ .../java/fiftyone/pipeline/did/DidClient.java | 58 +++++++++++++++++-- .../services/DataUpdateServiceDefault.java | 22 ++++--- 3 files changed, 84 insertions(+), 13 deletions(-) diff --git a/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineDefault.java b/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineDefault.java index 12ccc17f2..985b5429c 100644 --- a/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineDefault.java +++ b/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineDefault.java @@ -36,6 +36,7 @@ import fiftyone.pipeline.engines.flowelements.AspectEngineBase; import fiftyone.pipeline.engines.services.HttpClient; import fiftyone.pipeline.exceptions.AggregateException; +import fiftyone.pipeline.util.LicenseKeyMask; import org.json.JSONArray; import org.json.JSONObject; import org.slf4j.Logger; @@ -425,6 +426,19 @@ private static URL toUrl(String spec) throws MalformedURLException { } } + /** + * The license key sent with each request, and each key in it when it + * holds several separated by '|', to mask in messages. + */ + private List getLicenseKeys() { + List keys = new ArrayList<>(); + if (licenseKey != null && licenseKey.isEmpty() == false) { + keys.add(licenseKey); + keys.addAll(Arrays.asList(licenseKey.split(Pattern.quote("|")))); + } + return keys; + } + private void getCloudProperties() throws CloudRequestException, AggregateException, PropertyNotLoadedException { try { String jsonResult; @@ -544,6 +558,9 @@ else if (messages.size() == 0 && code != 200) messages.add(message); } + // The cloud's errors can name the license key it was sent. + messages.replaceAll(m -> LicenseKeyMask.maskIn(m, getLicenseKeys())); + Map> headers = null; if (messages.size() > 0){ headers = connection.getHeaderFields(); diff --git a/pipeline/pipeline.did/src/main/java/fiftyone/pipeline/did/DidClient.java b/pipeline/pipeline.did/src/main/java/fiftyone/pipeline/did/DidClient.java index f0d2c4010..9afb29af1 100644 --- a/pipeline/pipeline.did/src/main/java/fiftyone/pipeline/did/DidClient.java +++ b/pipeline/pipeline.did/src/main/java/fiftyone/pipeline/did/DidClient.java @@ -940,7 +940,11 @@ public CompletableFuture redeem( } } - /** The typed result a redeem answer carries. */ + /** + * The typed result a redeem answer carries. The request sent the + * licence key, and the answer can name it, so the messages of the + * errors show the key masked. + */ private RedeemResult readRedeem(HttpTransport.Response response) { int status = response.getStatusCode(); JSONObject json = asObject(response.getBody()); @@ -949,21 +953,22 @@ private RedeemResult readRedeem(HttpTransport.Response response) { case 503: if (json == null) { throw new CompletionException( - httpError("Redemption", response)); + httpError("Redemption", response, licenceKey)); } return RedeemResult.parse(status, json, response.getBody()); case 400: if (json != null && json.has("errors")) { - throw new IllegalArgumentException(errorsText(json)); + throw new IllegalArgumentException( + maskLicence(errorsText(json), licenceKey)); } throw new CompletionException( - httpError("Redemption", response)); + httpError("Redemption", response, licenceKey)); case 404: throw new CompletionException(new DidNotSupportedException( endpoint, response.getBody())); default: throw new CompletionException( - httpError("Redemption", response)); + httpError("Redemption", response, licenceKey)); } } @@ -1045,13 +1050,54 @@ private static void ensureReadsAs51Did(String fodId) { private static DidHttpException httpError( String what, HttpTransport.Response response) { + return httpError(what, response, null); + } + + /** + * An error whose message summarises the body, with the licence key the + * request sent masked in it. + */ + private static DidHttpException httpError( + String what, HttpTransport.Response response, String licence) { String body = response.getBody(); - String summary = body.length() > 200 ? body.substring(0, 200) : body; + String masked = maskLicence(body, licence); + String summary = masked.length() > 200 ? + masked.substring(0, 200) : + masked; return new DidHttpException( what + " answered HTTP " + response.getStatusCode() + ": " + summary, response.getStatusCode(), body); } + /** + * The text with the licence key shown only by its last 4 characters, + * every character before them replaced by '*', or replaced in full when + * it has 4 characters or fewer. This is how every 51Degrees package + * writes a licence key in a message, and it matches + * fiftyone.pipeline.util.LicenseKeyMask, which this package cannot use + * because it depends on nothing in the Pipeline. + */ + private static String maskLicence(String text, String licence) { + if (text == null || licence == null || licence.isEmpty()) { + return text; + } + String result = text.replace(licence, maskKey(licence)); + String encoded = encode(licence); + if (encoded.equals(licence) == false) { + result = result.replace(encoded, maskKey(encoded)); + } + return result; + } + + private static String maskKey(String key) { + int hidden = key.length() > 4 ? key.length() - 4 : key.length(); + StringBuilder masked = new StringBuilder(key.length()); + for (int i = 0; i < hidden; i++) { + masked.append('*'); + } + return masked.append(key, hidden, key.length()).toString(); + } + /** The body as a JSON object, or null when it is not one. */ private static JSONObject asObject(String body) { try { diff --git a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/services/DataUpdateServiceDefault.java b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/services/DataUpdateServiceDefault.java index 007b21211..fcf00f6b2 100644 --- a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/services/DataUpdateServiceDefault.java +++ b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/services/DataUpdateServiceDefault.java @@ -36,6 +36,7 @@ import fiftyone.pipeline.engines.services.update.FutureFactory; import fiftyone.pipeline.engines.services.update.FutureFactoryDefault; import fiftyone.pipeline.util.Check; +import fiftyone.pipeline.util.LicenseKeyMask; import jakarta.xml.bind.DatatypeConverter; import org.apache.commons.lang3.ObjectUtils; import org.slf4j.Logger; @@ -615,13 +616,18 @@ private AutoUpdateStatus downloadFile( FileWrapper tempFile) { String url = dataFile.getFormattedUrl(); - logger.debug("downloadFile from {}", url); + // The URL carries the license keys, so every message names it with + // each key masked. + List licenseKeys = + dataFile.getConfiguration().getDataUpdateLicenseKeys(); + String loggedUrl = LicenseKeyMask.maskIn(url, licenseKeys); + logger.debug("downloadFile from {}", loggedUrl); try { HttpURLConnection connection = httpClient.connect(new URI(url.trim()).toURL()); if (connection == null) { logger.error("No response from data update service at '{}' for engine '{}'", - dataFile.getFormattedUrl(), Objects.nonNull(dataFile.getEngine()) ? + loggedUrl, Objects.nonNull(dataFile.getEngine()) ? dataFile.getEngine().getClass().getSimpleName() : "No Engine"); return AutoUpdateStatus.AUTO_UPDATE_HTTPS_ERR; } @@ -645,19 +651,19 @@ private AutoUpdateStatus downloadFile( // in some versions of the HttpStatusCode enum. case (429): logger.error("Too many requests to '{}' for {}", - dataFile.getFormattedUrl(), getIdForLogging(dataFile)); + loggedUrl, getIdForLogging(dataFile)); return AutoUpdateStatus.AUTO_UPDATE_ERR_429_TOO_MANY_ATTEMPTS; case HttpURLConnection.HTTP_NOT_MODIFIED: logger.debug("No data update available from '{}' for {}", - dataFile.getFormattedUrl(), getIdForLogging(dataFile)); + loggedUrl, getIdForLogging(dataFile)); return AutoUpdateStatus.AUTO_UPDATE_NOT_NEEDED; case HttpURLConnection.HTTP_FORBIDDEN: logger.error("Access denied to data update service at '{}' for {}", - dataFile.getFormattedUrl(), getIdForLogging(dataFile)); + loggedUrl, getIdForLogging(dataFile)); return AutoUpdateStatus.AUTO_UPDATE_ERR_403_FORBIDDEN; default: logger.error("HTTP status code '{}' from data update service at '{}' for {}", - connection.getResponseCode(), dataFile.getFormattedUrl(), + connection.getResponseCode(), loggedUrl, getIdForLogging(dataFile)); return AutoUpdateStatus.AUTO_UPDATE_HTTPS_ERR; } @@ -665,7 +671,9 @@ private AutoUpdateStatus downloadFile( connection.disconnect(); } } catch (Exception e) { - logger.error("Error while processing data file download", e); + // An HTTP library's exception often names the URL it requested. + logger.error("Error while processing data file download", + LicenseKeyMask.maskIn(e, licenseKeys)); return AutoUpdateStatus.AUTO_UPDATE_HTTPS_ERR; } } From 6318f93089610eb0a8e66462dc9c8d0f5eada4e1 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Tue, 10 Nov 2026 00:17:59 +0000 Subject: [PATCH 4/6] Test the masked form of an encoded key and a key typed into the update URL maskIn_EncodedKeyShowsThePlainLast4 expects a key whose last characters are percent-encoded in a URL to show the encoded form of its plain last 4 characters. maskIn_MasksAKeyEncodedInLowerCase expects a key encoded with lower case hexadecimal digits to be masked too. KeyTypedIntoTheUrl_LogsMaskedKey gives the data update service a URL whose LicenseKeys parameter holds a key that is not among the data update license keys, and expects the key masked in every log entry. All three are written to fail on the previous commit. --- .../pipeline/util/LicenseKeyMaskTests.java | 35 +++++++++++++++++++ .../DataUpdateServiceLicenseKeyTests.java | 29 +++++++++++++++ 2 files changed, 64 insertions(+) diff --git a/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/util/LicenseKeyMaskTests.java b/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/util/LicenseKeyMaskTests.java index 7664d4984..312e2c08e 100644 --- a/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/util/LicenseKeyMaskTests.java +++ b/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/util/LicenseKeyMaskTests.java @@ -98,6 +98,41 @@ public void maskIn_MasksAKeyEncodedInAUrl() { assertTrue(masked, masked.endsWith("9876")); } + /** + * A key whose last characters are encoded in a URL shows the same last + * 4 characters as its plain form, encoded, rather than the last 4 + * characters of its encoded form. + */ + @Test + public void maskIn_EncodedKeyShowsThePlainLast4() { + String key = "ABCDEFGHIJKLMNOPQRST12+="; + String url = "https://example.test/?license=" + + "ABCDEFGHIJKLMNOPQRST12%2B%3D"; + + String masked = LicenseKeyMask.maskIn(url, Collections.singletonList(key)); + + assertEquals( + "https://example.test/?license=********************12%2B%3D", + masked); + } + + /** + * A key encoded with lower case hexadecimal digits, as some libraries + * write them, is masked as well. + */ + @Test + public void maskIn_MasksAKeyEncodedInLowerCase() { + String key = "ABCDEFGHIJKLMNOPQRST12+="; + String url = "https://example.test/?license=" + + "ABCDEFGHIJKLMNOPQRST12%2b%3d"; + + String masked = LicenseKeyMask.maskIn(url, Collections.singletonList(key)); + + assertEquals( + "https://example.test/?license=********************12%2b%3d", + masked); + } + @Test public void maskIn_LeavesTextWithoutAKeyAlone() { String text = "No key here."; diff --git a/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/DataUpdateServiceLicenseKeyTests.java b/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/DataUpdateServiceLicenseKeyTests.java index ca5594a1f..1ea82ece7 100644 --- a/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/DataUpdateServiceLicenseKeyTests.java +++ b/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/DataUpdateServiceLicenseKeyTests.java @@ -151,6 +151,35 @@ public void RequestException_LogsMaskedKeys() throws IOException { assertLogsMaskedKeys(); } + /** + * A key typed straight into the data update URL, rather than given as + * a data update license key, is masked too. + */ + @Test + @SuppressWarnings("unchecked") + public void KeyTypedIntoTheUrl_LogsMaskedKey() throws IOException { + DataFileConfiguration config = new DataFileConfigurationDefault(); + config.setDataUpdateUrl(UPDATE_URL + "?LicenseKeys=" + KEY + + "&Download=True&Type=HashV41"); + config.setVerifyModifiedSince(false); + config.setFileSystemWatcherEnabled(false); + config.setAutomaticUpdatesEnabled(false); + AspectEngineDataFileDefault file = new AspectEngineDataFileDefault(); + file.setConfiguration(config); + OnPremiseAspectEngine typed = + mock(OnPremiseAspectEngine.class); + when(typed.getDataFileMetaData()).thenReturn(file); + when(connection.getResponseCode()) + .thenReturn(HttpURLConnection.HTTP_FORBIDDEN); + + service.checkForUpdate(typed); + + String logged = logged(); + assertFalse(logged, logged.contains(KEY)); + assertTrue(logged, logged.contains(MASKED)); + } + private void assertLogsMaskedKeys() { service.checkForUpdate(engine); From 4261571fc31931e1678fcd34430b99e7a49d2be5 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Tue, 10 Nov 2026 00:19:59 +0000 Subject: [PATCH 5/6] Mask an encoded key by its plain form, and keys typed into the update URL LicenseKeyMask.maskIn now replaces a URL-encoded key with the encoded form of its masked plain form, so it shows the same last 4 characters as the plain key, and also masks the key encoded with lower case hexadecimal digits. Its Javadoc now says that every occurrence of a key's text is masked, so a key of only a few characters masks those characters elsewhere too, as the specification asks for keys of 4 characters or fewer. The specification link now points at rc, where the rule is. The data update service now also masks any key in the LicenseKeys parameter of the data update URL, such as one typed straight into the DataUpdateUrl setting rather than given as a data update license key. DidHttpException.getBody now says the body is not masked and can hold the license key sent, so a caller that logs it must mask it first. --- .../pipeline/util/LicenseKeyMask.java | 32 ++++++++++-- .../pipeline/did/DidHttpException.java | 8 ++- .../services/DataUpdateServiceDefault.java | 49 ++++++++++++++++++- 3 files changed, 82 insertions(+), 7 deletions(-) diff --git a/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/util/LicenseKeyMask.java b/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/util/LicenseKeyMask.java index d5d82f864..6f323b5ba 100644 --- a/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/util/LicenseKeyMask.java +++ b/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/util/LicenseKeyMask.java @@ -36,7 +36,7 @@ * them replaced by '*', so a reader can tell two keys apart but cannot use * either. A key of 4 characters or fewer is replaced in full, because its * last 4 characters would be the whole key. - * @see Specification + * @see Specification */ public final class LicenseKeyMask { @@ -74,7 +74,11 @@ public static String mask(String key) { /** * The text with every occurrence of each key masked, both as the key is - * written and as it is encoded in a URL. + * written and as it is encoded in a URL, with upper or lower case + * hexadecimal digits. An encoded key is replaced by the encoded form of + * its masked plain form, so it shows the same last 4 characters. Every + * occurrence of a key's text is masked, so a key of only a few + * characters also masks those characters wherever else they appear. * @param text the text, such as a URL or an error message, or null * @param keys the license keys, or null * @return the masked text, or null if the text is null @@ -85,10 +89,15 @@ public static String maskIn(String text, Iterable keys) { } String result = text; for (String key : longestFirst(keys)) { - result = result.replace(key, mask(key)); + String masked = mask(key); + result = result.replace(key, masked); String encoded = urlEncode(key); if (encoded.equals(key) == false) { - result = result.replace(encoded, mask(encoded)); + String maskedEncoded = urlEncode(masked); + result = result.replace(encoded, maskedEncoded); + result = result.replace( + lowerCaseEscapes(encoded), + lowerCaseEscapes(maskedEncoded)); } } return result; @@ -181,6 +190,21 @@ private static List longestFirst(Iterable keys) { return result; } + /** + * The text with the two hexadecimal digits after each '%' in lower + * case. + */ + private static String lowerCaseEscapes(String text) { + StringBuilder result = new StringBuilder(text); + for (int i = text.indexOf('%'); + i >= 0 && i + 2 < text.length(); + i = text.indexOf('%', i + 1)) { + result.setCharAt(i + 1, Character.toLowerCase(text.charAt(i + 1))); + result.setCharAt(i + 2, Character.toLowerCase(text.charAt(i + 2))); + } + return result.toString(); + } + private static String urlEncode(String key) { try { return URLEncoder.encode(key, "UTF-8"); diff --git a/pipeline/pipeline.did/src/main/java/fiftyone/pipeline/did/DidHttpException.java b/pipeline/pipeline.did/src/main/java/fiftyone/pipeline/did/DidHttpException.java index 49d6d41cb..5a1747925 100644 --- a/pipeline/pipeline.did/src/main/java/fiftyone/pipeline/did/DidHttpException.java +++ b/pipeline/pipeline.did/src/main/java/fiftyone/pipeline/did/DidHttpException.java @@ -54,7 +54,13 @@ public int getStatusCode() { return statusCode; } - /** @return the body the service answered with, never null */ + /** + * The body the service answered with, as it was sent. It is not masked, + * and the answer to a request that sent a license key can hold that + * key, so a caller that logs the body or puts it in a message must + * mask any license key in it first. + * @return the body, never null + */ public String getBody() { return body; } diff --git a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/services/DataUpdateServiceDefault.java b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/services/DataUpdateServiceDefault.java index fcf00f6b2..344475372 100644 --- a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/services/DataUpdateServiceDefault.java +++ b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/services/DataUpdateServiceDefault.java @@ -45,6 +45,7 @@ import java.io.*; import java.net.HttpURLConnection; import java.net.URI; +import java.net.URLDecoder; import java.nio.file.*; import java.security.DigestInputStream; import java.security.MessageDigest; @@ -65,6 +66,12 @@ */ public class DataUpdateServiceDefault implements DataUpdateService { + /** + * The query parameter of a data update URL that holds the license keys, + * as the 51Degrees URL formatter writes it. + */ + private static final String LICENSE_KEYS_PARAMETER = "LicenseKeys"; + final List configurations = new ArrayList<>(); private final Logger logger; private final boolean closeFutureFactory; @@ -611,6 +618,45 @@ private long applyIntervalRandomisationMillis( * @param tempFile the temp file to write the data to * @return the {@link AutoUpdateStatus} value indicating the result */ + /** + * The license keys to mask in messages that name the data update URL, + * being the data update license keys and any key in the URL's + * LicenseKeys parameter, such as one typed straight into the data + * update URL. + * @param dataFile the data file being updated + * @param url the formatted data update URL + * @return the keys to mask + */ + private static List licenseKeysToMask( + AspectEngineDataFile dataFile, + String url) { + List keys = new ArrayList<>(); + List configured = + dataFile.getConfiguration().getDataUpdateLicenseKeys(); + if (configured != null) { + keys.addAll(configured); + } + int query = url == null ? -1 : url.indexOf('?'); + if (query < 0) { + return keys; + } + for (String parameter : url.substring(query + 1).split("&")) { + int equals = parameter.indexOf('='); + if (equals > 0 && parameter.substring(0, equals) + .equalsIgnoreCase(LICENSE_KEYS_PARAMETER)) { + String value = parameter.substring(equals + 1); + try { + value = URLDecoder.decode(value, "UTF-8"); + } catch (UnsupportedEncodingException | + IllegalArgumentException e) { + // Masked as it is written. + } + keys.addAll(Arrays.asList(value.split("\\|"))); + } + } + return keys; + } + private AutoUpdateStatus downloadFile( AspectEngineDataFile dataFile, FileWrapper tempFile) { @@ -618,8 +664,7 @@ private AutoUpdateStatus downloadFile( String url = dataFile.getFormattedUrl(); // The URL carries the license keys, so every message names it with // each key masked. - List licenseKeys = - dataFile.getConfiguration().getDataUpdateLicenseKeys(); + List licenseKeys = licenseKeysToMask(dataFile, url); String loggedUrl = LicenseKeyMask.maskIn(url, licenseKeys); logger.debug("downloadFile from {}", loggedUrl); From e3672e5ce995c9d56fa3c22184348343fadbfcb1 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Tue, 10 Nov 2026 00:30:23 +0000 Subject: [PATCH 6/6] Put the Javadoc of downloadFile back above it licenseKeysToMask was added between downloadFile and its Javadoc, so the Javadoc described the wrong method. It is now above downloadFile again. --- .../services/DataUpdateServiceDefault.java | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/services/DataUpdateServiceDefault.java b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/services/DataUpdateServiceDefault.java index 344475372..e3bfd7894 100644 --- a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/services/DataUpdateServiceDefault.java +++ b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/services/DataUpdateServiceDefault.java @@ -609,15 +609,6 @@ private long applyIntervalRandomisationMillis( return intervalMillis + randomSeconds * 1000; } - /** - * Get the most recent data file available from the configured update URL. - * If the data currently used by the engine is the newest available then - * nothing will be downloaded. - * - * @param dataFile the data file to use - * @param tempFile the temp file to write the data to - * @return the {@link AutoUpdateStatus} value indicating the result - */ /** * The license keys to mask in messages that name the data update URL, * being the data update license keys and any key in the URL's @@ -657,6 +648,15 @@ private static List licenseKeysToMask( return keys; } + /** + * Get the most recent data file available from the configured update URL. + * If the data currently used by the engine is the newest available then + * nothing will be downloaded. + * + * @param dataFile the data file to use + * @param tempFile the temp file to write the data to + * @return the {@link AutoUpdateStatus} value indicating the result + */ private AutoUpdateStatus downloadFile( AspectEngineDataFile dataFile, FileWrapper tempFile) {