diff --git a/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineDefault.java b/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineDefault.java index 12ccc17f2..985b5429c 100644 --- a/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineDefault.java +++ b/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineDefault.java @@ -36,6 +36,7 @@ import fiftyone.pipeline.engines.flowelements.AspectEngineBase; import fiftyone.pipeline.engines.services.HttpClient; import fiftyone.pipeline.exceptions.AggregateException; +import fiftyone.pipeline.util.LicenseKeyMask; import org.json.JSONArray; import org.json.JSONObject; import org.slf4j.Logger; @@ -425,6 +426,19 @@ private static URL toUrl(String spec) throws MalformedURLException { } } + /** + * The license key sent with each request, and each key in it when it + * holds several separated by '|', to mask in messages. + */ + private List getLicenseKeys() { + List keys = new ArrayList<>(); + if (licenseKey != null && licenseKey.isEmpty() == false) { + keys.add(licenseKey); + keys.addAll(Arrays.asList(licenseKey.split(Pattern.quote("|")))); + } + return keys; + } + private void getCloudProperties() throws CloudRequestException, AggregateException, PropertyNotLoadedException { try { String jsonResult; @@ -544,6 +558,9 @@ else if (messages.size() == 0 && code != 200) messages.add(message); } + // The cloud's errors can name the license key it was sent. + messages.replaceAll(m -> LicenseKeyMask.maskIn(m, getLicenseKeys())); + Map> headers = null; if (messages.size() > 0){ headers = connection.getHeaderFields(); diff --git a/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineTests.java b/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineTests.java index 80ebde301..1259fd677 100644 --- a/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineTests.java +++ b/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineTests.java @@ -40,6 +40,8 @@ import org.junit.jupiter.params.provider.MethodSource; import org.mockito.ArgumentCaptor; import org.mockito.ArgumentMatchers; +import org.slf4j.ILoggerFactory; +import org.slf4j.Logger; import java.io.IOException; import java.net.MalformedURLException; @@ -101,6 +103,66 @@ public void Process() throws Exception { } } + /** + * The cloud answers a refused license key with an error that names it. + * The exception the engine raises, and every log entry, show the key + * only by its last 4 characters. + */ + @Test + public void Process_LicenseKeyInCloudError_Masked() throws Exception { + final String licenseKey = "TESTKEYNOTREALABCDEFGHIJ1234"; + final String masked = "************************1234"; + jsonResponse = "{\"errors\":[\"Invalid license key(s) supplied: " + + licenseKey + "\"]}"; + configureMockedClient(); + Logger logger = mock(Logger.class); + when(logger.isDebugEnabled()).thenReturn(true); + when(logger.isInfoEnabled()).thenReturn(true); + when(logger.isWarnEnabled()).thenReturn(true); + when(logger.isErrorEnabled()).thenReturn(true); + ILoggerFactory capturingLoggerFactory = mock(ILoggerFactory.class); + when(capturingLoggerFactory.getLogger(anyString())).thenReturn(logger); + + CloudRequestEngine engine = new CloudRequestEngineBuilder( + capturingLoggerFactory, httpClient) + .setResourceKey("resource_key") + .setLicenseKey(licenseKey) + .build(); + + try (Pipeline pipeline = new PipelineBuilder(capturingLoggerFactory) + .setSuppressProcessException(true) + .addFlowElement(engine) + .build(); + FlowData data = pipeline.createFlowData()) { + data.addEvidence("query.User-Agent", "iPhone"); + data.process(); + + assertFalse(data.getErrors().isEmpty()); + StringBuilder reported = new StringBuilder(); + for (FlowError error : data.getErrors()) { + reported.append(printed(error.getThrowable())); + } + for (org.mockito.invocation.Invocation invocation : + mockingDetails(logger).getInvocations()) { + for (Object argument : invocation.getArguments()) { + reported.append(argument instanceof Throwable ? + printed((Throwable) argument) : + String.valueOf(argument)); + } + } + assertFalse(reported.toString().contains(licenseKey), + reported.toString()); + assertTrue(reported.toString().contains(masked), + reported.toString()); + } + } + + private static String printed(Throwable throwable) { + java.io.StringWriter text = new java.io.StringWriter(); + throwable.printStackTrace(new java.io.PrintWriter(text)); + return text.toString(); + } + /** * Test errors thrown by cloud request engine are added to flow.errors when SuppressProcessExceptions == true */ diff --git a/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/util/LicenseKeyMask.java b/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/util/LicenseKeyMask.java new file mode 100644 index 000000000..6f323b5ba --- /dev/null +++ b/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/util/LicenseKeyMask.java @@ -0,0 +1,234 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.util; + +import java.io.UnsupportedEncodingException; +import java.net.URLEncoder; +import java.util.ArrayList; +import java.util.Collections; +import java.util.IdentityHashMap; +import java.util.List; +import java.util.Map; + +/** + * Masks license keys in text written to a log entry or an error message. A + * masked key shows only its last 4 characters, with every character before + * them replaced by '*', so a reader can tell two keys apart but cannot use + * either. A key of 4 characters or fewer is replaced in full, because its + * last 4 characters would be the whole key. + * @see Specification + */ +public final class LicenseKeyMask { + + /** + * How many characters at the end of a key stay visible. + */ + public static final int VISIBLE_CHARACTERS = 4; + + /** + * The character that replaces each hidden character of a key. + */ + public static final char MASK_CHARACTER = '*'; + + private LicenseKeyMask() { + } + + /** + * The key with every character before its last 4 replaced. + * @param key a license key, or null + * @return the masked key, or null if the key is null + */ + public static String mask(String key) { + if (key == null) { + return null; + } + int hidden = key.length() > VISIBLE_CHARACTERS ? + key.length() - VISIBLE_CHARACTERS : + key.length(); + StringBuilder masked = new StringBuilder(key.length()); + for (int i = 0; i < hidden; i++) { + masked.append(MASK_CHARACTER); + } + return masked.append(key, hidden, key.length()).toString(); + } + + /** + * The text with every occurrence of each key masked, both as the key is + * written and as it is encoded in a URL, with upper or lower case + * hexadecimal digits. An encoded key is replaced by the encoded form of + * its masked plain form, so it shows the same last 4 characters. Every + * occurrence of a key's text is masked, so a key of only a few + * characters also masks those characters wherever else they appear. + * @param text the text, such as a URL or an error message, or null + * @param keys the license keys, or null + * @return the masked text, or null if the text is null + */ + public static String maskIn(String text, Iterable keys) { + if (text == null || keys == null) { + return text; + } + String result = text; + for (String key : longestFirst(keys)) { + String masked = mask(key); + result = result.replace(key, masked); + String encoded = urlEncode(key); + if (encoded.equals(key) == false) { + String maskedEncoded = urlEncode(masked); + result = result.replace(encoded, maskedEncoded); + result = result.replace( + lowerCaseEscapes(encoded), + lowerCaseEscapes(maskedEncoded)); + } + } + return result; + } + + /** + * The throwable to log or report in place of the one given, so that no + * message in it, its causes or its suppressed throwables holds a key. + * When none does, the throwable given is returned. Otherwise a copy is + * returned whose messages are masked, each starting with the name of the + * class it stands for, and whose stack traces are the originals. + * @param throwable the throwable, or null + * @param keys the license keys, or null + * @return a throwable that holds no key + */ + public static Throwable maskIn(Throwable throwable, Iterable keys) { + if (throwable == null || + keys == null || + holdsKey(throwable, keys, new IdentityHashMap()) + == false) { + return throwable; + } + return copy(throwable, keys, new IdentityHashMap()); + } + + private static boolean holdsKey( + Throwable throwable, + Iterable keys, + Map seen) { + if (throwable == null || seen.put(throwable, Boolean.TRUE) != null) { + return false; + } + String message = throwable.getMessage(); + if (message != null && message.equals(maskIn(message, keys)) == false) { + return true; + } + if (holdsKey(throwable.getCause(), keys, seen)) { + return true; + } + for (Throwable suppressed : throwable.getSuppressed()) { + if (holdsKey(suppressed, keys, seen)) { + return true; + } + } + return false; + } + + private static Throwable copy( + Throwable throwable, + Iterable keys, + Map copies) { + Throwable existing = copies.get(throwable); + if (existing != null) { + return existing; + } + String message = maskIn(throwable.getMessage(), keys); + MaskedThrowable masked = new MaskedThrowable( + throwable.getClass().getName() + + (message == null ? "" : ": " + message)); + copies.put(throwable, masked); + masked.setStackTrace(throwable.getStackTrace()); + Throwable cause = throwable.getCause(); + if (cause != null) { + Throwable maskedCause = copy(cause, keys, copies); + if (maskedCause != masked) { + masked.initCause(maskedCause); + } + } + for (Throwable suppressed : throwable.getSuppressed()) { + Throwable maskedSuppressed = copy(suppressed, keys, copies); + if (maskedSuppressed != masked) { + masked.addSuppressed(maskedSuppressed); + } + } + return masked; + } + + /** + * The keys that are not null or empty, longest first, so that a key + * that holds another is masked whole. + */ + private static List longestFirst(Iterable keys) { + List result = new ArrayList<>(); + for (String key : keys) { + if (key != null && key.isEmpty() == false) { + result.add(key); + } + } + Collections.sort(result, (a, b) -> b.length() - a.length()); + return result; + } + + /** + * The text with the two hexadecimal digits after each '%' in lower + * case. + */ + private static String lowerCaseEscapes(String text) { + StringBuilder result = new StringBuilder(text); + for (int i = text.indexOf('%'); + i >= 0 && i + 2 < text.length(); + i = text.indexOf('%', i + 1)) { + result.setCharAt(i + 1, Character.toLowerCase(text.charAt(i + 1))); + result.setCharAt(i + 2, Character.toLowerCase(text.charAt(i + 2))); + } + return result.toString(); + } + + private static String urlEncode(String key) { + try { + return URLEncoder.encode(key, "UTF-8"); + } catch (UnsupportedEncodingException e) { + // Every Java platform supports UTF-8. + throw new IllegalStateException(e); + } + } + + /** + * Stands for a throwable whose message held a license key. Its message + * starts with the name of the class it stands for, and it prints as + * that message alone. + */ + private static final class MaskedThrowable extends Exception { + private static final long serialVersionUID = 1L; + + MaskedThrowable(String message) { + super(message); + } + + @Override + public String toString() { + return getMessage(); + } + } +} diff --git a/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/util/LicenseKeyMaskTests.java b/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/util/LicenseKeyMaskTests.java new file mode 100644 index 000000000..312e2c08e --- /dev/null +++ b/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/util/LicenseKeyMaskTests.java @@ -0,0 +1,191 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.util; + +import org.junit.Test; + +import java.io.IOException; +import java.io.PrintWriter; +import java.io.StringWriter; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; + +/** + * A license key in a log entry or an error message shows only its last 4 + * characters, with every character before them replaced. + */ +public class LicenseKeyMaskTests { + + private static final String KEY = "TESTKEYNOTREALABCDEFGHIJ1234"; + private static final String MASKED = "************************1234"; + private static final String OTHER_KEY = "OTHERTESTKEYNOTREALXYZ5678"; + private static final String OTHER_MASKED = "**********************5678"; + + @Test + public void mask_ShowsTheLast4Characters() { + assertEquals(MASKED, LicenseKeyMask.mask(KEY)); + assertEquals(KEY.length(), LicenseKeyMask.mask(KEY).length()); + } + + /** + * A key of 4 characters or fewer would be shown whole, so all of it is + * replaced. + */ + @Test + public void mask_ShortKeyIsReplacedInFull() { + assertEquals("****", LicenseKeyMask.mask("1234")); + assertEquals("**", LicenseKeyMask.mask("12")); + assertEquals("*2345", LicenseKeyMask.mask("12345")); + assertEquals("", LicenseKeyMask.mask("")); + assertNull(LicenseKeyMask.mask(null)); + } + + @Test + public void maskIn_MasksEveryKeyInAUrl() { + String url = "https://distributor.example/api/v2/download?" + + "LicenseKeys=" + KEY + "|" + OTHER_KEY + "&Download=True"; + + String masked = LicenseKeyMask.maskIn( + url, + Arrays.asList(KEY, OTHER_KEY)); + + assertEquals( + "https://distributor.example/api/v2/download?" + + "LicenseKeys=" + MASKED + "|" + OTHER_MASKED + + "&Download=True", + masked); + } + + /** + * A key that a URL holds encoded is masked in its encoded form. + */ + @Test + public void maskIn_MasksAKeyEncodedInAUrl() { + String key = "KEY+WITH/CHARS=ENCODED9876"; + String url = "https://example.test/?license=" + + "KEY%2BWITH%2FCHARS%3DENCODED9876"; + + String masked = LicenseKeyMask.maskIn(url, Collections.singletonList(key)); + + assertFalse(masked, masked.contains("KEY%2BWITH")); + assertTrue(masked, masked.endsWith("9876")); + } + + /** + * A key whose last characters are encoded in a URL shows the same last + * 4 characters as its plain form, encoded, rather than the last 4 + * characters of its encoded form. + */ + @Test + public void maskIn_EncodedKeyShowsThePlainLast4() { + String key = "ABCDEFGHIJKLMNOPQRST12+="; + String url = "https://example.test/?license=" + + "ABCDEFGHIJKLMNOPQRST12%2B%3D"; + + String masked = LicenseKeyMask.maskIn(url, Collections.singletonList(key)); + + assertEquals( + "https://example.test/?license=********************12%2B%3D", + masked); + } + + /** + * A key encoded with lower case hexadecimal digits, as some libraries + * write them, is masked as well. + */ + @Test + public void maskIn_MasksAKeyEncodedInLowerCase() { + String key = "ABCDEFGHIJKLMNOPQRST12+="; + String url = "https://example.test/?license=" + + "ABCDEFGHIJKLMNOPQRST12%2b%3d"; + + String masked = LicenseKeyMask.maskIn(url, Collections.singletonList(key)); + + assertEquals( + "https://example.test/?license=********************12%2b%3d", + masked); + } + + @Test + public void maskIn_LeavesTextWithoutAKeyAlone() { + String text = "No key here."; + assertEquals(text, LicenseKeyMask.maskIn(text, Arrays.asList(KEY))); + assertNull(LicenseKeyMask.maskIn((String) null, Arrays.asList(KEY))); + assertEquals(text, LicenseKeyMask.maskIn(text, null)); + assertEquals( + text, + LicenseKeyMask.maskIn(text, Arrays.asList(null, ""))); + } + + /** + * The message of an exception an HTTP library raises often holds the + * URL it requested, and so the key. The throwable to log holds the key + * in none of its messages, its causes' or its suppressed throwables', + * and keeps the name of each class it stands for. + */ + @Test + public void maskIn_MasksAThrowableAndItsCauses() { + IOException cause = new IOException( + "Server returned HTTP response code: 403 for URL: " + + "https://distributor.example/?LicenseKeys=" + KEY); + RuntimeException thrown = new RuntimeException( + "Download failed for " + KEY, cause); + thrown.addSuppressed(new IllegalStateException(OTHER_KEY)); + List keys = Arrays.asList(KEY, OTHER_KEY); + + Throwable masked = LicenseKeyMask.maskIn(thrown, keys); + String printed = printed(masked); + + assertFalse(printed, printed.contains(KEY)); + assertFalse(printed, printed.contains(OTHER_KEY)); + assertTrue(printed, printed.contains(MASKED)); + assertTrue(printed, printed.contains(OTHER_MASKED)); + assertTrue(printed, printed.contains("java.lang.RuntimeException")); + assertTrue(printed, printed.contains("java.io.IOException")); + assertTrue(printed, + printed.contains("java.lang.IllegalStateException")); + assertEquals( + thrown.getStackTrace().length, + masked.getStackTrace().length); + } + + @Test + public void maskIn_ThrowableWithoutAKeyIsReturnedAsItIs() { + IOException thrown = new IOException("Connection refused"); + assertSame(thrown, LicenseKeyMask.maskIn(thrown, Arrays.asList(KEY))); + assertNull(LicenseKeyMask.maskIn((Throwable) null, Arrays.asList(KEY))); + } + + private static String printed(Throwable throwable) { + StringWriter text = new StringWriter(); + throwable.printStackTrace(new PrintWriter(text)); + return text.toString(); + } +} diff --git a/pipeline/pipeline.did/src/main/java/fiftyone/pipeline/did/DidClient.java b/pipeline/pipeline.did/src/main/java/fiftyone/pipeline/did/DidClient.java index f0d2c4010..9afb29af1 100644 --- a/pipeline/pipeline.did/src/main/java/fiftyone/pipeline/did/DidClient.java +++ b/pipeline/pipeline.did/src/main/java/fiftyone/pipeline/did/DidClient.java @@ -940,7 +940,11 @@ public CompletableFuture redeem( } } - /** The typed result a redeem answer carries. */ + /** + * The typed result a redeem answer carries. The request sent the + * licence key, and the answer can name it, so the messages of the + * errors show the key masked. + */ private RedeemResult readRedeem(HttpTransport.Response response) { int status = response.getStatusCode(); JSONObject json = asObject(response.getBody()); @@ -949,21 +953,22 @@ private RedeemResult readRedeem(HttpTransport.Response response) { case 503: if (json == null) { throw new CompletionException( - httpError("Redemption", response)); + httpError("Redemption", response, licenceKey)); } return RedeemResult.parse(status, json, response.getBody()); case 400: if (json != null && json.has("errors")) { - throw new IllegalArgumentException(errorsText(json)); + throw new IllegalArgumentException( + maskLicence(errorsText(json), licenceKey)); } throw new CompletionException( - httpError("Redemption", response)); + httpError("Redemption", response, licenceKey)); case 404: throw new CompletionException(new DidNotSupportedException( endpoint, response.getBody())); default: throw new CompletionException( - httpError("Redemption", response)); + httpError("Redemption", response, licenceKey)); } } @@ -1045,13 +1050,54 @@ private static void ensureReadsAs51Did(String fodId) { private static DidHttpException httpError( String what, HttpTransport.Response response) { + return httpError(what, response, null); + } + + /** + * An error whose message summarises the body, with the licence key the + * request sent masked in it. + */ + private static DidHttpException httpError( + String what, HttpTransport.Response response, String licence) { String body = response.getBody(); - String summary = body.length() > 200 ? body.substring(0, 200) : body; + String masked = maskLicence(body, licence); + String summary = masked.length() > 200 ? + masked.substring(0, 200) : + masked; return new DidHttpException( what + " answered HTTP " + response.getStatusCode() + ": " + summary, response.getStatusCode(), body); } + /** + * The text with the licence key shown only by its last 4 characters, + * every character before them replaced by '*', or replaced in full when + * it has 4 characters or fewer. This is how every 51Degrees package + * writes a licence key in a message, and it matches + * fiftyone.pipeline.util.LicenseKeyMask, which this package cannot use + * because it depends on nothing in the Pipeline. + */ + private static String maskLicence(String text, String licence) { + if (text == null || licence == null || licence.isEmpty()) { + return text; + } + String result = text.replace(licence, maskKey(licence)); + String encoded = encode(licence); + if (encoded.equals(licence) == false) { + result = result.replace(encoded, maskKey(encoded)); + } + return result; + } + + private static String maskKey(String key) { + int hidden = key.length() > 4 ? key.length() - 4 : key.length(); + StringBuilder masked = new StringBuilder(key.length()); + for (int i = 0; i < hidden; i++) { + masked.append('*'); + } + return masked.append(key, hidden, key.length()).toString(); + } + /** The body as a JSON object, or null when it is not one. */ private static JSONObject asObject(String body) { try { diff --git a/pipeline/pipeline.did/src/main/java/fiftyone/pipeline/did/DidHttpException.java b/pipeline/pipeline.did/src/main/java/fiftyone/pipeline/did/DidHttpException.java index 49d6d41cb..5a1747925 100644 --- a/pipeline/pipeline.did/src/main/java/fiftyone/pipeline/did/DidHttpException.java +++ b/pipeline/pipeline.did/src/main/java/fiftyone/pipeline/did/DidHttpException.java @@ -54,7 +54,13 @@ public int getStatusCode() { return statusCode; } - /** @return the body the service answered with, never null */ + /** + * The body the service answered with, as it was sent. It is not masked, + * and the answer to a request that sent a license key can hold that + * key, so a caller that logs the body or puts it in a message must + * mask any license key in it first. + * @return the body, never null + */ public String getBody() { return body; } diff --git a/pipeline/pipeline.did/src/test/java/fiftyone/pipeline/did/DidClientTests.java b/pipeline/pipeline.did/src/test/java/fiftyone/pipeline/did/DidClientTests.java index 71969a6ac..cea3ec788 100644 --- a/pipeline/pipeline.did/src/test/java/fiftyone/pipeline/did/DidClientTests.java +++ b/pipeline/pipeline.did/src/test/java/fiftyone/pipeline/did/DidClientTests.java @@ -67,6 +67,9 @@ public class DidClientTests { private static final String ENDPOINT = "https://example.test/api/v4/"; + private static final String TEST_LICENCE = "TESTKEYNOTREALABCDEFGHIJ1234"; + private static final String TEST_LICENCE_MASKED = + "************************1234"; private static final Instant WEEK1 = Instant.parse("2026-08-03T00:00:00Z"); private static final Instant WEEK2 = WEEK1.plus(Duration.ofDays(7)); private static final Instant WEEK3 = WEEK2.plus(Duration.ofDays(7)); @@ -1273,6 +1276,71 @@ public void redeem_TransportFailureFailsWithIoException() { client.redeem(validDid, "sealed", "abc")); } + /** + * The service answers a refused licence key with a body that names it. + * The error message shows the key only by its last 4 characters, with + * every character before them replaced, as every 51Degrees package + * writes a licence key in a message. + */ + @Test + public void redeem_RefusedLicenceIsMaskedInTheMessage() { + transport.queue(401, + "Invalid license key(s) supplied: " + TEST_LICENCE); + + DidHttpException error = failure(DidHttpException.class, + licensedClient().redeem(validDid, "sealed", "abc")); + + assertEquals(401, error.getStatusCode()); + assertFalse(error.getMessage(), + error.getMessage().contains(TEST_LICENCE)); + assertTrue(error.getMessage(), + error.getMessage().contains(TEST_LICENCE_MASKED)); + } + + @Test + public void redeem_LicenceInErrorsIsMaskedInTheMessage() { + transport.queue(400, "{\"errors\":[\"The license key " + + TEST_LICENCE + " does not include this product.\"]}"); + + IllegalArgumentException error = failure( + IllegalArgumentException.class, + licensedClient().redeem(validDid, "sealed", "abc")); + + assertFalse(error.getMessage(), + error.getMessage().contains(TEST_LICENCE)); + assertTrue(error.getMessage(), + error.getMessage().contains(TEST_LICENCE_MASKED)); + } + + /** + * A key of 4 characters or fewer is replaced in full, because its last + * 4 characters would be all of it. + */ + @Test + public void redeem_ShortLicenceIsMaskedInFull() { + DidClient shortLicence = DidClient.builder("resource") + .licenceKey("1234") + .endpoint(ENDPOINT) + .transport(transport) + .build(); + transport.queue(401, "Invalid license key(s) supplied: 1234"); + + DidHttpException error = failure(DidHttpException.class, + shortLicence.redeem(validDid, "sealed", "abc")); + + assertFalse(error.getMessage(), error.getMessage().contains("1234")); + assertTrue(error.getMessage(), error.getMessage().endsWith("****")); + } + + private DidClient licensedClient() { + return DidClient.builder("resource") + .licenceKey(TEST_LICENCE) + .endpoint(ENDPOINT) + .transport(transport) + .clock(clock) + .build(); + } + @Test public void redeem_WithoutLicenceKeyOmitsTheField() throws Exception { DidClient noLicence = DidClient.builder("resource") diff --git a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/services/DataUpdateServiceDefault.java b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/services/DataUpdateServiceDefault.java index 007b21211..e3bfd7894 100644 --- a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/services/DataUpdateServiceDefault.java +++ b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/services/DataUpdateServiceDefault.java @@ -36,6 +36,7 @@ import fiftyone.pipeline.engines.services.update.FutureFactory; import fiftyone.pipeline.engines.services.update.FutureFactoryDefault; import fiftyone.pipeline.util.Check; +import fiftyone.pipeline.util.LicenseKeyMask; import jakarta.xml.bind.DatatypeConverter; import org.apache.commons.lang3.ObjectUtils; import org.slf4j.Logger; @@ -44,6 +45,7 @@ import java.io.*; import java.net.HttpURLConnection; import java.net.URI; +import java.net.URLDecoder; import java.nio.file.*; import java.security.DigestInputStream; import java.security.MessageDigest; @@ -64,6 +66,12 @@ */ public class DataUpdateServiceDefault implements DataUpdateService { + /** + * The query parameter of a data update URL that holds the license keys, + * as the 51Degrees URL formatter writes it. + */ + private static final String LICENSE_KEYS_PARAMETER = "LicenseKeys"; + final List configurations = new ArrayList<>(); private final Logger logger; private final boolean closeFutureFactory; @@ -601,6 +609,45 @@ private long applyIntervalRandomisationMillis( return intervalMillis + randomSeconds * 1000; } + /** + * The license keys to mask in messages that name the data update URL, + * being the data update license keys and any key in the URL's + * LicenseKeys parameter, such as one typed straight into the data + * update URL. + * @param dataFile the data file being updated + * @param url the formatted data update URL + * @return the keys to mask + */ + private static List licenseKeysToMask( + AspectEngineDataFile dataFile, + String url) { + List keys = new ArrayList<>(); + List configured = + dataFile.getConfiguration().getDataUpdateLicenseKeys(); + if (configured != null) { + keys.addAll(configured); + } + int query = url == null ? -1 : url.indexOf('?'); + if (query < 0) { + return keys; + } + for (String parameter : url.substring(query + 1).split("&")) { + int equals = parameter.indexOf('='); + if (equals > 0 && parameter.substring(0, equals) + .equalsIgnoreCase(LICENSE_KEYS_PARAMETER)) { + String value = parameter.substring(equals + 1); + try { + value = URLDecoder.decode(value, "UTF-8"); + } catch (UnsupportedEncodingException | + IllegalArgumentException e) { + // Masked as it is written. + } + keys.addAll(Arrays.asList(value.split("\\|"))); + } + } + return keys; + } + /** * Get the most recent data file available from the configured update URL. * If the data currently used by the engine is the newest available then @@ -615,13 +662,17 @@ private AutoUpdateStatus downloadFile( FileWrapper tempFile) { String url = dataFile.getFormattedUrl(); - logger.debug("downloadFile from {}", url); + // The URL carries the license keys, so every message names it with + // each key masked. + List licenseKeys = licenseKeysToMask(dataFile, url); + String loggedUrl = LicenseKeyMask.maskIn(url, licenseKeys); + logger.debug("downloadFile from {}", loggedUrl); try { HttpURLConnection connection = httpClient.connect(new URI(url.trim()).toURL()); if (connection == null) { logger.error("No response from data update service at '{}' for engine '{}'", - dataFile.getFormattedUrl(), Objects.nonNull(dataFile.getEngine()) ? + loggedUrl, Objects.nonNull(dataFile.getEngine()) ? dataFile.getEngine().getClass().getSimpleName() : "No Engine"); return AutoUpdateStatus.AUTO_UPDATE_HTTPS_ERR; } @@ -645,19 +696,19 @@ private AutoUpdateStatus downloadFile( // in some versions of the HttpStatusCode enum. case (429): logger.error("Too many requests to '{}' for {}", - dataFile.getFormattedUrl(), getIdForLogging(dataFile)); + loggedUrl, getIdForLogging(dataFile)); return AutoUpdateStatus.AUTO_UPDATE_ERR_429_TOO_MANY_ATTEMPTS; case HttpURLConnection.HTTP_NOT_MODIFIED: logger.debug("No data update available from '{}' for {}", - dataFile.getFormattedUrl(), getIdForLogging(dataFile)); + loggedUrl, getIdForLogging(dataFile)); return AutoUpdateStatus.AUTO_UPDATE_NOT_NEEDED; case HttpURLConnection.HTTP_FORBIDDEN: logger.error("Access denied to data update service at '{}' for {}", - dataFile.getFormattedUrl(), getIdForLogging(dataFile)); + loggedUrl, getIdForLogging(dataFile)); return AutoUpdateStatus.AUTO_UPDATE_ERR_403_FORBIDDEN; default: logger.error("HTTP status code '{}' from data update service at '{}' for {}", - connection.getResponseCode(), dataFile.getFormattedUrl(), + connection.getResponseCode(), loggedUrl, getIdForLogging(dataFile)); return AutoUpdateStatus.AUTO_UPDATE_HTTPS_ERR; } @@ -665,7 +716,9 @@ private AutoUpdateStatus downloadFile( connection.disconnect(); } } catch (Exception e) { - logger.error("Error while processing data file download", e); + // An HTTP library's exception often names the URL it requested. + logger.error("Error while processing data file download", + LicenseKeyMask.maskIn(e, licenseKeys)); return AutoUpdateStatus.AUTO_UPDATE_HTTPS_ERR; } } diff --git a/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/DataUpdateServiceLicenseKeyTests.java b/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/DataUpdateServiceLicenseKeyTests.java new file mode 100644 index 000000000..1ea82ece7 --- /dev/null +++ b/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/DataUpdateServiceLicenseKeyTests.java @@ -0,0 +1,223 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.engines.services; + +import fiftyone.common.wrappers.io.FileWrapperFactory; +import fiftyone.pipeline.engines.configuration.DataFileConfiguration; +import fiftyone.pipeline.engines.configuration.DataFileConfigurationDefault; +import fiftyone.pipeline.engines.data.AspectData; +import fiftyone.pipeline.engines.data.AspectEngineDataFileDefault; +import fiftyone.pipeline.engines.data.AspectPropertyMetaData; +import fiftyone.pipeline.engines.flowelements.OnPremiseAspectEngine; +import fiftyone.pipeline.engines.services.update.FutureFactory; +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mockito; +import org.mockito.invocation.Invocation; +import org.slf4j.Logger; + +import java.io.IOException; +import java.io.PrintWriter; +import java.io.StringWriter; +import java.net.HttpURLConnection; +import java.net.URL; +import java.util.Arrays; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * The data update URL carries the license keys in its query. Every log + * entry the data update service writes about a request, at any level and + * with any exception it logs, shows each key only by its last 4 + * characters. + */ +public class DataUpdateServiceLicenseKeyTests { + + private static final String KEY = "TESTKEYNOTREALABCDEFGHIJ1234"; + private static final String MASKED = "************************1234"; + private static final String OTHER_KEY = "OTHERTESTKEYNOTREALXYZ5678"; + private static final String OTHER_MASKED = "**********************5678"; + private static final String UPDATE_URL = + "https://distributor.example/api/v2/download"; + + private Logger logger; + private HttpClient httpClient; + private HttpURLConnection connection; + private DataUpdateServiceDefault service; + private OnPremiseAspectEngine engine; + + @Before + @SuppressWarnings("unchecked") + public void init() throws IOException { + logger = mock(Logger.class); + when(logger.isDebugEnabled()).thenReturn(true); + when(logger.isInfoEnabled()).thenReturn(true); + when(logger.isWarnEnabled()).thenReturn(true); + when(logger.isErrorEnabled()).thenReturn(true); + connection = mock(HttpURLConnection.class); + httpClient = mock(HttpClient.class); + when(httpClient.connect(any(URL.class))).thenReturn(connection); + service = new DataUpdateServiceDefault( + logger, + httpClient, + mock(FileWrapperFactory.class), + mock(FutureFactory.class)); + + // A data file updated in memory, whose URL carries the keys as the + // 51Degrees URL formatter puts them. + final DataFileConfiguration config = new DataFileConfigurationDefault(); + config.setDataUpdateUrl(UPDATE_URL); + config.setDataUpdateLicenseKeys(Arrays.asList(KEY, OTHER_KEY)); + config.setUrlFormatter(dataFile -> config.getDataUpdateUrl() + + "?LicenseKeys=" + KEY + "|" + OTHER_KEY + + "&Download=True&Type=HashV41"); + config.setVerifyModifiedSince(false); + config.setFileSystemWatcherEnabled(false); + config.setAutomaticUpdatesEnabled(false); + AspectEngineDataFileDefault file = new AspectEngineDataFileDefault(); + file.setConfiguration(config); + + engine = mock(OnPremiseAspectEngine.class); + when(engine.getDataFileMetaData()).thenReturn(file); + } + + @Test + public void Forbidden_LogsMaskedKeys() throws IOException { + when(connection.getResponseCode()) + .thenReturn(HttpURLConnection.HTTP_FORBIDDEN); + assertLogsMaskedKeys(); + } + + @Test + public void TooManyRequests_LogsMaskedKeys() throws IOException { + when(connection.getResponseCode()).thenReturn(429); + assertLogsMaskedKeys(); + } + + @Test + public void OtherStatus_LogsMaskedKeys() throws IOException { + when(connection.getResponseCode()) + .thenReturn(HttpURLConnection.HTTP_INTERNAL_ERROR); + assertLogsMaskedKeys(); + } + + @Test + public void NotModified_LogsMaskedKeys() throws IOException { + when(connection.getResponseCode()) + .thenReturn(HttpURLConnection.HTTP_NOT_MODIFIED); + assertLogsMaskedKeys(); + } + + @Test + public void NoResponse_LogsMaskedKeys() throws IOException { + when(httpClient.connect(any(URL.class))).thenReturn(null); + assertLogsMaskedKeys(); + } + + /** + * The exception an HTTP library raises often names the URL it + * requested, and the exception is logged with its stack trace. + */ + @Test + public void RequestException_LogsMaskedKeys() throws IOException { + when(connection.getResponseCode()).thenThrow(new IOException( + "Server returned HTTP response code: 403 for URL: " + + UPDATE_URL + "?LicenseKeys=" + KEY + "|" + OTHER_KEY)); + assertLogsMaskedKeys(); + } + + /** + * A key typed straight into the data update URL, rather than given as + * a data update license key, is masked too. + */ + @Test + @SuppressWarnings("unchecked") + public void KeyTypedIntoTheUrl_LogsMaskedKey() throws IOException { + DataFileConfiguration config = new DataFileConfigurationDefault(); + config.setDataUpdateUrl(UPDATE_URL + "?LicenseKeys=" + KEY + + "&Download=True&Type=HashV41"); + config.setVerifyModifiedSince(false); + config.setFileSystemWatcherEnabled(false); + config.setAutomaticUpdatesEnabled(false); + AspectEngineDataFileDefault file = new AspectEngineDataFileDefault(); + file.setConfiguration(config); + OnPremiseAspectEngine typed = + mock(OnPremiseAspectEngine.class); + when(typed.getDataFileMetaData()).thenReturn(file); + when(connection.getResponseCode()) + .thenReturn(HttpURLConnection.HTTP_FORBIDDEN); + + service.checkForUpdate(typed); + + String logged = logged(); + assertFalse(logged, logged.contains(KEY)); + assertTrue(logged, logged.contains(MASKED)); + } + + private void assertLogsMaskedKeys() { + service.checkForUpdate(engine); + + String logged = logged(); + assertFalse(logged, logged.contains(KEY)); + assertFalse(logged, logged.contains(OTHER_KEY)); + assertTrue(logged, logged.contains(MASKED)); + assertTrue(logged, logged.contains(OTHER_MASKED)); + } + + /** + * Everything passed to the logger, at every level, with the stack trace + * of each throwable as it would be printed. + */ + private String logged() { + StringBuilder text = new StringBuilder(); + for (Invocation invocation : + Mockito.mockingDetails(logger).getInvocations()) { + text.append(invocation.getMethod().getName()).append(':'); + for (Object argument : invocation.getArguments()) { + append(text, argument); + } + text.append('\n'); + } + return text.toString(); + } + + private static void append(StringBuilder text, Object argument) { + if (argument instanceof Object[]) { + for (Object item : (Object[]) argument) { + append(text, item); + } + } else if (argument instanceof Throwable) { + StringWriter trace = new StringWriter(); + ((Throwable) argument).printStackTrace(new PrintWriter(trace)); + text.append(' ').append(trace); + } else { + text.append(' ').append(argument); + } + } +}