From f8496c665518b00a01129e4c7eeb6e24ecbd4196 Mon Sep 17 00:00:00 2001 From: abhinav-phi Date: Sat, 29 Aug 2026 23:09:04 +0530 Subject: [PATCH 1/4] Make the WARC digest algorithm configurable WARCRecordFormat hard coded the algorithm for the WARC-Payload-Digest and WARC-Block-Digest fields to SHA-1 with no way for an operator to choose another algorithm. Add the configuration key warc.digest.algorithm which accepts sha1 (the default, kept for compatibility with CDX indexes and other downstream tooling) and sha256, and thread it through WARCHdfsBolt so that response, request, resource and metadata records all use it. The static getDigestSha1 helpers are kept and deprecated in favour of the new getDigest instance methods. https://github.com/apache/stormcrawler/issues/2106 --- docs/src/main/asciidoc/configuration.adoc | 1 + external/warc/README.md | 8 + .../warc/MetadataRecordFormat.java | 8 +- .../stormcrawler/warc/WARCHdfsBolt.java | 11 +- .../stormcrawler/warc/WARCRecordFormat.java | 97 +++++++- .../warc/WARCRequestRecordFormat.java | 6 +- .../warc/WARCDigestAlgorithmTest.java | 210 ++++++++++++++++++ .../stormcrawler/warc/WARCHdfsBoltTest.java | 31 +++ 8 files changed, 362 insertions(+), 10 deletions(-) create mode 100644 external/warc/src/test/java/org/apache/stormcrawler/warc/WARCDigestAlgorithmTest.java diff --git a/docs/src/main/asciidoc/configuration.adoc b/docs/src/main/asciidoc/configuration.adoc index 047a86ea4..51b4302a8 100644 --- a/docs/src/main/asciidoc/configuration.adoc +++ b/docs/src/main/asciidoc/configuration.adoc @@ -737,6 +737,7 @@ See the link:https://github.com/apache/stormcrawler/tree/main/external/warc[warc | key | default value | description | warc.metadata.keys | - | Metadata keys to include as WARC metadata records (optional list). +| warc.digest.algorithm | sha1 | Algorithm for the WARC-Payload-Digest and WARC-Block-Digest headers: `sha1` or `sha256`. SHA-1 is the convention across the WARC ecosystem; downstream tooling such as CDX indexers may expect `sha1` digests. |=== NOTE: For complete WARC records, set `http.store.headers` to `true`. The OkHttp protocol (`org.apache.stormcrawler.protocol.okhttp.HttpProtocol`) is recommended for WARC generation as it provides verbatim HTTP headers. diff --git a/external/warc/README.md b/external/warc/README.md index 8f22aa117..511b2cf38 100644 --- a/external/warc/README.md +++ b/external/warc/README.md @@ -161,6 +161,14 @@ A note on the recording of HTTP requests and responses with StormCrawler and the You can specify in the configuration which metadata key/values to store as WARC metadata using `warc.metadata.keys`. +The algorithm used to compute the `WARC-Payload-Digest` and `WARC-Block-Digest` fields is configurable with `warc.digest.algorithm`. It accepts `sha1` (the default) and `sha256`: + +``` + warc.digest.algorithm: sha256 +``` + +The value is matched case-insensitively and an optional hyphen is ignored, i.e. `SHA-256` is also accepted. SHA-1 is the convention across the WARC ecosystem and downstream tooling such as CDX indexers may expect `sha1` digests, so change the default only if your downstream tooling supports the alternative algorithm. Invalid values make the bolt fail when it is prepared. + ## Consuming WARC files Web archives harvested in the [WARC format](https://iipc.github.io/warc-specifications/specifications/warc-format/warc-1.1/) can be used as input for StormCrawler – instead of fetching content from remote servers, the WARCSpout reads WARC files and emits the archive web page captures as tuples into the topology. diff --git a/external/warc/src/main/java/org/apache/stormcrawler/warc/MetadataRecordFormat.java b/external/warc/src/main/java/org/apache/stormcrawler/warc/MetadataRecordFormat.java index 75b79924e..979337529 100644 --- a/external/warc/src/main/java/org/apache/stormcrawler/warc/MetadataRecordFormat.java +++ b/external/warc/src/main/java/org/apache/stormcrawler/warc/MetadataRecordFormat.java @@ -52,7 +52,11 @@ public class MetadataRecordFormat extends WARCRecordFormat { private final List metadataKeys; public MetadataRecordFormat(List metadataKeys) { - super(""); + this(metadataKeys, WARCRecordFormat.DIGEST_ALGORITHM_SHA1); + } + + public MetadataRecordFormat(List metadataKeys, String digestAlgorithm) { + super("", digestAlgorithm); // the keys are fixed configuration: validate them once here instead of // for every record final List validKeys = new ArrayList<>(metadataKeys.size()); @@ -118,7 +122,7 @@ public byte[] format(Tuple tuple) { int contentLength = metadata_representation.length; buffer.append("Content-Length: ").append(Integer.toString(contentLength)).append(CRLF); - String blockDigest = getDigestSha1(metadata_representation); + String blockDigest = getDigest(metadata_representation); String captureTime = getCaptureTime(metadata); buffer.append("WARC-Date: ").append(captureTime).append(CRLF); diff --git a/external/warc/src/main/java/org/apache/stormcrawler/warc/WARCHdfsBolt.java b/external/warc/src/main/java/org/apache/stormcrawler/warc/WARCHdfsBolt.java index 48dcf32d9..8847d0911 100644 --- a/external/warc/src/main/java/org/apache/stormcrawler/warc/WARCHdfsBolt.java +++ b/external/warc/src/main/java/org/apache/stormcrawler/warc/WARCHdfsBolt.java @@ -74,14 +74,19 @@ public void doPrepare( throws IOException { super.doPrepare(conf, topologyContext, collector); protocolMDprefix = ConfUtils.getString(conf, ProtocolResponse.PROTOCOL_MD_PREFIX_PARAM, ""); - withRecordFormat(new WARCRecordFormat(protocolMDprefix)); + String digestAlgorithm = + ConfUtils.getString( + conf, + WARCRecordFormat.DIGEST_ALGORITHM_PARAM, + WARCRecordFormat.DIGEST_ALGORITHM_SHA1); + withRecordFormat(new WARCRecordFormat(protocolMDprefix, digestAlgorithm)); if (withRequestRecords) { - addRecordFormat(new WARCRequestRecordFormat(protocolMDprefix), 0); + addRecordFormat(new WARCRequestRecordFormat(protocolMDprefix, digestAlgorithm), 0); } // detect if a list of keys was specified to be stored in the metadata List metadataToWrite = ConfUtils.loadListFromConf(METADATA_KEYS_STORE, conf); if (!metadataToWrite.isEmpty()) { - addRecordFormat(new MetadataRecordFormat(metadataToWrite), 1); + addRecordFormat(new MetadataRecordFormat(metadataToWrite, digestAlgorithm), 1); } } diff --git a/external/warc/src/main/java/org/apache/stormcrawler/warc/WARCRecordFormat.java b/external/warc/src/main/java/org/apache/stormcrawler/warc/WARCRecordFormat.java index ad8f2d46a..14e4881b7 100644 --- a/external/warc/src/main/java/org/apache/stormcrawler/warc/WARCRecordFormat.java +++ b/external/warc/src/main/java/org/apache/stormcrawler/warc/WARCRecordFormat.java @@ -129,19 +129,108 @@ static String sanitizeWarcFieldValue(String value) { Pattern.compile("(?i)(?:Content-(?:Encoding|Length)|Transfer-Encoding)"); protected static final String X_HIDE_HEADER = "X-Crawler-"; + /** + * Configuration key setting the algorithm used to compute the WARC-Payload-Digest and + * WARC-Block-Digest fields. Supported values are {@value #DIGEST_ALGORITHM_SHA1} (the default) + * and {@value #DIGEST_ALGORITHM_SHA256}. + * + *

Note: SHA-1 is the convention across the WARC ecosystem and downstream tooling (CDX + * indexes, revisit record handling) may expect it. Change the default deliberately, not + * casually. + */ + public static final String DIGEST_ALGORITHM_PARAM = "warc.digest.algorithm"; + + public static final String DIGEST_ALGORITHM_SHA1 = "sha1"; + + public static final String DIGEST_ALGORITHM_SHA256 = "sha256"; + private static final Base32 base32 = new Base32(); - private static final String digestNoContent = getDigestSha1(new byte[0]); protected final String protocolMDprefix; + /** JCA name of the message digest algorithm, e.g. "SHA-1". */ + private final String digestJCAName; + + /** Algorithm prefix of the WARC digest fields, e.g. "sha1:". */ + private final String digestPrefix; + + private final String digestNoContent; + public WARCRecordFormat(String protocolMDprefix) { + this(protocolMDprefix, DIGEST_ALGORITHM_SHA1); + } + + public WARCRecordFormat(String protocolMDprefix, String digestAlgorithm) { this.protocolMDprefix = protocolMDprefix; + this.digestJCAName = getDigestJCAName(digestAlgorithm); + this.digestPrefix = digestJCAName.toLowerCase(Locale.ROOT).replace("-", "") + ":"; + this.digestNoContent = getDigest(new byte[0]); } + /** + * Resolve the configured digest algorithm to the JCA name of the message digest. The value is + * matched case-insensitively and an optional hyphen is ignored, i.e. "sha256", + * "SHA-256" etc. are all accepted. + * + * @throws IllegalArgumentException if the value is not a supported algorithm + */ + private static String getDigestJCAName(String digestAlgorithm) { + if (digestAlgorithm == null) { + return "SHA-1"; + } + return switch (digestAlgorithm.trim().toLowerCase(Locale.ROOT).replace("-", "")) { + case DIGEST_ALGORITHM_SHA1 -> "SHA-1"; + case DIGEST_ALGORITHM_SHA256 -> "SHA-256"; + default -> throw new IllegalArgumentException( + "Unsupported value [" + + digestAlgorithm + + "] for " + + DIGEST_ALGORITHM_PARAM + + ", supported algorithms: " + + DIGEST_ALGORITHM_SHA1 + + ", " + + DIGEST_ALGORITHM_SHA256); + }; + } + + /** + * Compute the digest of the given bytes with the configured algorithm. + * + * @return digest in the form "<algorithm>:<base32>", e.g. + * "sha1:..." + */ + public String getDigest(byte[] bytes) { + MessageDigest md = DigestUtils.getDigest(digestJCAName); + return digestPrefix + base32.encodeAsString(md.digest(bytes)); + } + + /** + * Compute the digest of the concatenation of the two given byte arrays with the configured + * algorithm. + * + * @return digest in the form "<algorithm>:<base32>", e.g. + * "sha1:..." + */ + public String getDigest(byte[] bytes1, byte[] bytes2) { + MessageDigest md = DigestUtils.getDigest(digestJCAName); + md.update(bytes1); + return digestPrefix + base32.encodeAsString(md.digest(bytes2)); + } + + /** + * @deprecated use {@link #getDigest(byte[])} instead; the algorithm is set by {@link + * #DIGEST_ALGORITHM_PARAM} and no longer fixed to SHA-1 + */ + @Deprecated public static String getDigestSha1(byte[] bytes) { return "sha1:" + base32.encodeAsString(DigestUtils.sha1(bytes)); } + /** + * @deprecated use {@link #getDigest(byte[], byte[])} instead; the algorithm is set by {@link + * #DIGEST_ALGORITHM_PARAM} and no longer fixed to SHA-1 + */ + @Deprecated public static String getDigestSha1(byte[] bytes1, byte[] bytes2) { MessageDigest sha1 = DigestUtils.getSha1Digest(); sha1.update(bytes1); @@ -435,14 +524,14 @@ public byte[] format(Tuple tuple) { String blockDigest = digestNoContent; if (content != null) { contentLength = content.length; - payloadDigest = getDigestSha1(content); + payloadDigest = getDigest(content); if (WARCTypeValue.equals(WARC_TYPE_RESPONSE)) { - blockDigest = getDigestSha1(httpheaders, content); + blockDigest = getDigest(httpheaders, content); } else { blockDigest = payloadDigest; } } else if (WARCTypeValue.equals(WARC_TYPE_RESPONSE)) { - blockDigest = getDigestSha1(httpheaders); + blockDigest = getDigest(httpheaders); } // add the length of the http header diff --git a/external/warc/src/main/java/org/apache/stormcrawler/warc/WARCRequestRecordFormat.java b/external/warc/src/main/java/org/apache/stormcrawler/warc/WARCRequestRecordFormat.java index da35d1af0..21b42fb10 100644 --- a/external/warc/src/main/java/org/apache/stormcrawler/warc/WARCRequestRecordFormat.java +++ b/external/warc/src/main/java/org/apache/stormcrawler/warc/WARCRequestRecordFormat.java @@ -45,6 +45,10 @@ public WARCRequestRecordFormat(String protocolMDprefix) { super(protocolMDprefix); } + public WARCRequestRecordFormat(String protocolMDprefix, String digestAlgorithm) { + super(protocolMDprefix, digestAlgorithm); + } + @Override public byte[] format(Tuple tuple) { @@ -82,7 +86,7 @@ public byte[] format(Tuple tuple) { int contentLength = httpheaders.length; buffer.append("Content-Length: ").append(Integer.toString(contentLength)).append(CRLF); - String blockDigest = getDigestSha1(httpheaders); + String blockDigest = getDigest(httpheaders); String captureTime = getCaptureTime(metadata); buffer.append("WARC-Date: ").append(captureTime).append(CRLF); diff --git a/external/warc/src/test/java/org/apache/stormcrawler/warc/WARCDigestAlgorithmTest.java b/external/warc/src/test/java/org/apache/stormcrawler/warc/WARCDigestAlgorithmTest.java new file mode 100644 index 000000000..214a53fce --- /dev/null +++ b/external/warc/src/test/java/org/apache/stormcrawler/warc/WARCDigestAlgorithmTest.java @@ -0,0 +1,210 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.stormcrawler.warc; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.List; +import org.apache.commons.codec.binary.Base32; +import org.apache.storm.tuple.Tuple; +import org.apache.stormcrawler.Metadata; +import org.apache.stormcrawler.protocol.ProtocolResponse; +import org.junit.jupiter.api.Test; + +/** + * Tests that the algorithm used for the WARC-Payload-Digest and WARC-Block-Digest fields is + * configurable with {@link WARCRecordFormat#DIGEST_ALGORITHM_PARAM} and that SHA-1 remains the + * default. + */ +class WARCDigestAlgorithmTest { + + private static final String URL = "https://www.example.org/"; + + private static final byte[] CONTENT = "abcdef".getBytes(StandardCharsets.UTF_8); + + private static final String SHA1_ABCDEF = "sha1:D6FMCDZDYW23YELHXWUEXAZ6LQCXU56S"; + + private static final String SHA256_ABCDEF = + "sha256:X32X5R7VHJWUBPVWICTYBJRZZA54FGWIVGAW6H6GYXDNZWJ4I4QQ===="; + + private static final String SHA256_EMPTY = + "sha256:4OYMIQUY7QOBJGX36TEJS35ZEQT24QPEMSNZGTFESWMRW6CSXBKQ===="; + + /** Compute the expected digest independently of the code under test. */ + private static String expectedDigest(String jcaAlgorithm, String prefix, byte[]... byteArrays) { + try { + MessageDigest md = MessageDigest.getInstance(jcaAlgorithm); + for (byte[] bytes : byteArrays) { + md.update(bytes); + } + return prefix + new Base32().encodeAsString(md.digest()); + } catch (NoSuchAlgorithmException e) { + throw new IllegalStateException(e); + } + } + + /** + * The bytes covered by the WARC-Block-Digest: everything between the end of the WARC header + * and the final CRLF CRLF. + */ + private static byte[] recordBlock(String warcString) { + int start = warcString.indexOf("\r\n\r\n") + 4; + return warcString + .substring(start, warcString.length() - 4) + .getBytes(StandardCharsets.UTF_8); + } + + private static Tuple tupleWithContent(Metadata metadata) { + Tuple tuple = mock(Tuple.class); + when(tuple.getBinaryByField("content")).thenReturn(CONTENT); + when(tuple.getStringByField("url")).thenReturn(URL); + when(tuple.getValueByField("metadata")).thenReturn(metadata); + return tuple; + } + + @Test + void testDigestDefaultsToSha1() { + assertEquals( + SHA1_ABCDEF, + new WARCRecordFormat("").getDigest(CONTENT), + "digest algorithm must default to SHA-1"); + // a null algorithm must be treated as the default + assertEquals(SHA1_ABCDEF, new WARCRecordFormat("", null).getDigest(CONTENT)); + } + + @Test + void testGetDigestSha256() { + WARCRecordFormat format = + new WARCRecordFormat("", WARCRecordFormat.DIGEST_ALGORITHM_SHA256); + assertEquals(SHA256_ABCDEF, format.getDigest(CONTENT), "Wrong sha256 digest"); + assertEquals(SHA256_EMPTY, format.getDigest(new byte[0]), "Wrong sha256 digest"); + } + + @Test + void testGetDigestSha256TwoByteArrays() { + WARCRecordFormat format = + new WARCRecordFormat("", WARCRecordFormat.DIGEST_ALGORITHM_SHA256); + byte[] content1 = "abc".getBytes(StandardCharsets.UTF_8); + byte[] content2 = "def".getBytes(StandardCharsets.UTF_8); + assertEquals( + SHA256_ABCDEF, + format.getDigest(content1, content2), + "Wrong sha256 digest over concatenated byte arrays"); + } + + @Test + void testDigestAlgorithmValueVariants() { + // the value is matched case-insensitively, an optional hyphen is ignored and + // surrounding whitespace is trimmed + assertEquals(SHA256_ABCDEF, new WARCRecordFormat("", "SHA256").getDigest(CONTENT)); + assertEquals(SHA256_ABCDEF, new WARCRecordFormat("", "SHA-256").getDigest(CONTENT)); + assertEquals(SHA256_ABCDEF, new WARCRecordFormat("", " sha256 ").getDigest(CONTENT)); + assertEquals(SHA1_ABCDEF, new WARCRecordFormat("", "SHA-1").getDigest(CONTENT)); + } + + @Test + void testUnsupportedDigestAlgorithm() { + assertThrows(IllegalArgumentException.class, () -> new WARCRecordFormat("", "md5")); + assertThrows(IllegalArgumentException.class, () -> new WARCRecordFormat("", "sha512")); + assertThrows( + IllegalArgumentException.class, + () -> new MetadataRecordFormat(List.of("source"), "md5")); + assertThrows(IllegalArgumentException.class, () -> new WARCRequestRecordFormat("", "md5")); + } + + @Test + void testResponseRecordDigestsSha256() { + Metadata metadata = new Metadata(); + metadata.addValue( + "protocol." + ProtocolResponse.RESPONSE_HEADERS_KEY, + "HTTP/1.1 200 OK\r\nContent-Type: text/html\r\n"); + Tuple tuple = tupleWithContent(metadata); + WARCRecordFormat format = + new WARCRecordFormat("protocol.", WARCRecordFormat.DIGEST_ALGORITHM_SHA256); + String warcString = new String(format.format(tuple), StandardCharsets.UTF_8); + + assertTrue( + warcString.contains("\r\nWARC-Payload-Digest: " + SHA256_ABCDEF + "\r\n"), + "WARC response record: payload digest must be SHA-256"); + String expectedBlockDigest = expectedDigest("SHA-256", "sha256:", recordBlock(warcString)); + assertTrue( + warcString.contains("\r\nWARC-Block-Digest: " + expectedBlockDigest + "\r\n"), + "WARC response record: block digest must be SHA-256 over HTTP headers and payload"); + } + + @Test + void testResourceRecordDigestsSha256() { + // no verbatim HTTP headers stored -> resource record, block digest equals payload digest + Metadata metadata = new Metadata(); + Tuple tuple = tupleWithContent(metadata); + WARCRecordFormat format = new WARCRecordFormat("", WARCRecordFormat.DIGEST_ALGORITHM_SHA256); + String warcString = new String(format.format(tuple), StandardCharsets.UTF_8); + assertTrue(warcString.contains("\r\nWARC-Type: resource\r\n")); + assertTrue( + warcString.contains("\r\nWARC-Payload-Digest: " + SHA256_ABCDEF + "\r\n"), + "WARC resource record: payload digest must be SHA-256"); + assertTrue( + warcString.contains("\r\nWARC-Block-Digest: " + SHA256_ABCDEF + "\r\n"), + "WARC resource record: block digest must be SHA-256"); + } + + @Test + void testRequestRecordBlockDigestSha256() { + Metadata metadata = new Metadata(); + metadata.addValue( + "protocol." + ProtocolResponse.REQUEST_HEADERS_KEY, + "GET / HTTP/2\r\nUser-Agent: mybot\r\nConnection: Keep-Alive\r\n\r\n"); + Tuple tuple = mock(Tuple.class); + when(tuple.getStringByField("url")).thenReturn(URL); + when(tuple.getValueByField("metadata")).thenReturn(metadata); + WARCRequestRecordFormat format = + new WARCRequestRecordFormat("protocol.", WARCRecordFormat.DIGEST_ALGORITHM_SHA256); + String warcString = new String(format.format(tuple), StandardCharsets.UTF_8); + + String expectedBlockDigest = expectedDigest("SHA-256", "sha256:", recordBlock(warcString)); + assertTrue( + warcString.contains("\r\nWARC-Block-Digest: " + expectedBlockDigest + "\r\n"), + "WARC request record: block digest must be SHA-256 over the request headers"); + } + + @Test + void testMetadataRecordBlockDigestSha256() { + Metadata metadata = new Metadata(); + metadata.addValue("source", "a source"); + Tuple tuple = mock(Tuple.class); + when(tuple.getStringByField("url")).thenReturn(URL); + when(tuple.getValueByField("metadata")).thenReturn(metadata); + MetadataRecordFormat format = + new MetadataRecordFormat(List.of("source"), WARCRecordFormat.DIGEST_ALGORITHM_SHA256); + String warcString = new String(format.format(tuple), StandardCharsets.UTF_8); + + // the payload of the metadata record are the metadata fields themselves + byte[] payload = "source: a source\r\n".getBytes(StandardCharsets.UTF_8); + String expectedBlockDigest = expectedDigest("SHA-256", "sha256:", payload); + assertTrue( + warcString.contains("\r\nWARC-Block-Digest: " + expectedBlockDigest + "\r\n"), + "WARC metadata record: block digest must be SHA-256 over the metadata payload"); + } +} diff --git a/external/warc/src/test/java/org/apache/stormcrawler/warc/WARCHdfsBoltTest.java b/external/warc/src/test/java/org/apache/stormcrawler/warc/WARCHdfsBoltTest.java index 662355122..548ea7944 100644 --- a/external/warc/src/test/java/org/apache/stormcrawler/warc/WARCHdfsBoltTest.java +++ b/external/warc/src/test/java/org/apache/stormcrawler/warc/WARCHdfsBoltTest.java @@ -135,6 +135,37 @@ void testHttp2() throws IOException { "WARC response record is expected to include WARC header \"WARC-IP-Address\""); } + @Test + void testDigestAlgorithmConfig() throws IOException { + // instantiate a second bolt with warc.digest.algorithm: sha256 + HdfsBolt sha256Bolt = makeBolt(); + sha256Bolt.withConfigKey("warc"); + Map sha256Conf = new HashMap<>(conf); + sha256Conf.put( + WARCRecordFormat.DIGEST_ALGORITHM_PARAM, + WARCRecordFormat.DIGEST_ALGORITHM_SHA256); + sha256Bolt.prepare( + sha256Conf, TestUtil.getMockedTopologyContext(), new OutputCollector(output)); + try { + sha256Bolt.execute(getPage()); + } finally { + sha256Bolt.cleanup(); + } + List records = readWARCs(warcDir).collect(Collectors.toList()); + // expected 3 records (warcinfo, request, response) + assertEquals(3, records.size()); + for (WarcRecord record : records) { + String payloadDigest = record.headers().first("WARC-Payload-Digest").orElse(""); + String blockDigest = record.headers().first("WARC-Block-Digest").orElse(""); + assertTrue( + payloadDigest.isEmpty() || payloadDigest.startsWith("sha256:"), + "WARC-Payload-Digest must use the configured algorithm sha256"); + assertTrue( + blockDigest.isEmpty() || blockDigest.startsWith("sha256:"), + "WARC-Block-Digest must use the configured algorithm sha256"); + } + } + private static Stream readWARCs(Path warcDir) { try { return Files.walk(warcDir) From 09d373a33b11e3702dbe657c1be9eadb2cd19a86 Mon Sep 17 00:00:00 2001 From: abhinav-phi Date: Tue, 1 Sep 2026 20:44:38 +0530 Subject: [PATCH 2/4] Omit Base32 padding in WARC digest values The WARC digest fields define the digest value as a token, which does not allow the character "=" used for Base32 padding (ISO 28500 WARC 1.1). Strip the trailing padding so that e.g. SHA-256 digests conform; SHA-1 digests are 32 characters without padding and are unaffected. Suggested in review of #2110. --- docs/src/main/asciidoc/configuration.adoc | 2 +- external/warc/README.md | 2 ++ .../stormcrawler/warc/WARCRecordFormat.java | 15 ++++++++++-- .../warc/WARCDigestAlgorithmTest.java | 23 ++++++++++++++++--- 4 files changed, 36 insertions(+), 6 deletions(-) diff --git a/docs/src/main/asciidoc/configuration.adoc b/docs/src/main/asciidoc/configuration.adoc index 51b4302a8..d5996a6fc 100644 --- a/docs/src/main/asciidoc/configuration.adoc +++ b/docs/src/main/asciidoc/configuration.adoc @@ -737,7 +737,7 @@ See the link:https://github.com/apache/stormcrawler/tree/main/external/warc[warc | key | default value | description | warc.metadata.keys | - | Metadata keys to include as WARC metadata records (optional list). -| warc.digest.algorithm | sha1 | Algorithm for the WARC-Payload-Digest and WARC-Block-Digest headers: `sha1` or `sha256`. SHA-1 is the convention across the WARC ecosystem; downstream tooling such as CDX indexers may expect `sha1` digests. +| warc.digest.algorithm | sha1 | Algorithm for the WARC-Payload-Digest and WARC-Block-Digest headers: `sha1` or `sha256`. SHA-1 is the convention across the WARC ecosystem; downstream tooling such as CDX indexers may expect `sha1` digests. Digest values are Base32-encoded without padding. |=== NOTE: For complete WARC records, set `http.store.headers` to `true`. The OkHttp protocol (`org.apache.stormcrawler.protocol.okhttp.HttpProtocol`) is recommended for WARC generation as it provides verbatim HTTP headers. diff --git a/external/warc/README.md b/external/warc/README.md index 511b2cf38..3aa7e1545 100644 --- a/external/warc/README.md +++ b/external/warc/README.md @@ -169,6 +169,8 @@ The algorithm used to compute the `WARC-Payload-Digest` and `WARC-Block-Digest` The value is matched case-insensitively and an optional hyphen is ignored, i.e. `SHA-256` is also accepted. SHA-1 is the convention across the WARC ecosystem and downstream tooling such as CDX indexers may expect `sha1` digests, so change the default only if your downstream tooling supports the alternative algorithm. Invalid values make the bolt fail when it is prepared. +The digest values are written as Base32 without the trailing `=` padding, as required by the grammar of the WARC digest fields (the digest value is a token, which does not allow the padding character). + ## Consuming WARC files Web archives harvested in the [WARC format](https://iipc.github.io/warc-specifications/specifications/warc-format/warc-1.1/) can be used as input for StormCrawler – instead of fetching content from remote servers, the WARCSpout reads WARC files and emits the archive web page captures as tuples into the topology. diff --git a/external/warc/src/main/java/org/apache/stormcrawler/warc/WARCRecordFormat.java b/external/warc/src/main/java/org/apache/stormcrawler/warc/WARCRecordFormat.java index 14e4881b7..69ecc9b4d 100644 --- a/external/warc/src/main/java/org/apache/stormcrawler/warc/WARCRecordFormat.java +++ b/external/warc/src/main/java/org/apache/stormcrawler/warc/WARCRecordFormat.java @@ -201,7 +201,7 @@ private static String getDigestJCAName(String digestAlgorithm) { */ public String getDigest(byte[] bytes) { MessageDigest md = DigestUtils.getDigest(digestJCAName); - return digestPrefix + base32.encodeAsString(md.digest(bytes)); + return digestPrefix + base32Unpadded(md.digest(bytes)); } /** @@ -214,7 +214,18 @@ public String getDigest(byte[] bytes) { public String getDigest(byte[] bytes1, byte[] bytes2) { MessageDigest md = DigestUtils.getDigest(digestJCAName); md.update(bytes1); - return digestPrefix + base32.encodeAsString(md.digest(bytes2)); + return digestPrefix + base32Unpadded(md.digest(bytes2)); + } + + /** + * Base32-encode a digest value without the trailing "=" padding characters: the + * WARC digest fields define the digest value as a token, which does not allow the padding + * character (cf. ISO 28500 WARC 1.1, WARC-Block-Digest / WARC-Payload-Digest). SHA-1 digests + * are unaffected (32 characters without padding), while e.g. SHA-256 digests would end in + * "====". + */ + private static String base32Unpadded(byte[] digest) { + return StringUtils.stripEnd(base32.encodeAsString(digest), "="); } /** diff --git a/external/warc/src/test/java/org/apache/stormcrawler/warc/WARCDigestAlgorithmTest.java b/external/warc/src/test/java/org/apache/stormcrawler/warc/WARCDigestAlgorithmTest.java index 214a53fce..50226dc20 100644 --- a/external/warc/src/test/java/org/apache/stormcrawler/warc/WARCDigestAlgorithmTest.java +++ b/external/warc/src/test/java/org/apache/stormcrawler/warc/WARCDigestAlgorithmTest.java @@ -18,6 +18,7 @@ package org.apache.stormcrawler.warc; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.mock; @@ -28,6 +29,7 @@ import java.security.NoSuchAlgorithmException; import java.util.List; import org.apache.commons.codec.binary.Base32; +import org.apache.commons.lang3.StringUtils; import org.apache.storm.tuple.Tuple; import org.apache.stormcrawler.Metadata; import org.apache.stormcrawler.protocol.ProtocolResponse; @@ -46,11 +48,15 @@ class WARCDigestAlgorithmTest { private static final String SHA1_ABCDEF = "sha1:D6FMCDZDYW23YELHXWUEXAZ6LQCXU56S"; + /* + * The Base32 padding is omitted: the WARC digest fields define the digest value as a token, + * which does not allow the padding character "=" (cf. ISO 28500 WARC 1.1). + */ private static final String SHA256_ABCDEF = - "sha256:X32X5R7VHJWUBPVWICTYBJRZZA54FGWIVGAW6H6GYXDNZWJ4I4QQ===="; + "sha256:X32X5R7VHJWUBPVWICTYBJRZZA54FGWIVGAW6H6GYXDNZWJ4I4QQ"; private static final String SHA256_EMPTY = - "sha256:4OYMIQUY7QOBJGX36TEJS35ZEQT24QPEMSNZGTFESWMRW6CSXBKQ===="; + "sha256:4OYMIQUY7QOBJGX36TEJS35ZEQT24QPEMSNZGTFESWMRW6CSXBKQ"; /** Compute the expected digest independently of the code under test. */ private static String expectedDigest(String jcaAlgorithm, String prefix, byte[]... byteArrays) { @@ -59,7 +65,8 @@ private static String expectedDigest(String jcaAlgorithm, String prefix, byte[]. for (byte[] bytes : byteArrays) { md.update(bytes); } - return prefix + new Base32().encodeAsString(md.digest()); + return prefix + + StringUtils.stripEnd(new Base32().encodeAsString(md.digest()), "="); } catch (NoSuchAlgorithmException e) { throw new IllegalStateException(e); } @@ -102,6 +109,16 @@ void testGetDigestSha256() { assertEquals(SHA256_EMPTY, format.getDigest(new byte[0]), "Wrong sha256 digest"); } + @Test + void testDigestValueContainsNoBase32Padding() { + // the digest value is a token per the WARC 1.1 grammar and must not contain "=" + String sha256 = new WARCRecordFormat("", WARCRecordFormat.DIGEST_ALGORITHM_SHA256) + .getDigest(CONTENT); + assertFalse(sha256.contains("="), "digest value must not contain Base32 padding"); + String sha1 = new WARCRecordFormat("").getDigest(CONTENT); + assertFalse(sha1.contains("="), "digest value must not contain Base32 padding"); + } + @Test void testGetDigestSha256TwoByteArrays() { WARCRecordFormat format = From 2a6aec0ad41f76ace4a73360986cb3eb0d7a6744 Mon Sep 17 00:00:00 2001 From: abhinav-phi Date: Tue, 1 Sep 2026 22:00:02 +0530 Subject: [PATCH 3/4] Apply google-java-format The validate-code-format check requires google-java-format (AOSP style); run git-code-format:format-code on the touched files. No functional changes. --- .../stormcrawler/warc/WARCRecordFormat.java | 27 ++++++++++--------- .../warc/WARCDigestAlgorithmTest.java | 18 +++++++------ .../stormcrawler/warc/WARCHdfsBoltTest.java | 3 +-- 3 files changed, 25 insertions(+), 23 deletions(-) diff --git a/external/warc/src/main/java/org/apache/stormcrawler/warc/WARCRecordFormat.java b/external/warc/src/main/java/org/apache/stormcrawler/warc/WARCRecordFormat.java index 69ecc9b4d..c54fae17d 100644 --- a/external/warc/src/main/java/org/apache/stormcrawler/warc/WARCRecordFormat.java +++ b/external/warc/src/main/java/org/apache/stormcrawler/warc/WARCRecordFormat.java @@ -181,15 +181,16 @@ private static String getDigestJCAName(String digestAlgorithm) { return switch (digestAlgorithm.trim().toLowerCase(Locale.ROOT).replace("-", "")) { case DIGEST_ALGORITHM_SHA1 -> "SHA-1"; case DIGEST_ALGORITHM_SHA256 -> "SHA-256"; - default -> throw new IllegalArgumentException( - "Unsupported value [" - + digestAlgorithm - + "] for " - + DIGEST_ALGORITHM_PARAM - + ", supported algorithms: " - + DIGEST_ALGORITHM_SHA1 - + ", " - + DIGEST_ALGORITHM_SHA256); + default -> + throw new IllegalArgumentException( + "Unsupported value [" + + digestAlgorithm + + "] for " + + DIGEST_ALGORITHM_PARAM + + ", supported algorithms: " + + DIGEST_ALGORITHM_SHA1 + + ", " + + DIGEST_ALGORITHM_SHA256); }; } @@ -218,10 +219,10 @@ public String getDigest(byte[] bytes1, byte[] bytes2) { } /** - * Base32-encode a digest value without the trailing "=" padding characters: the - * WARC digest fields define the digest value as a token, which does not allow the padding - * character (cf. ISO 28500 WARC 1.1, WARC-Block-Digest / WARC-Payload-Digest). SHA-1 digests - * are unaffected (32 characters without padding), while e.g. SHA-256 digests would end in + * Base32-encode a digest value without the trailing "=" padding characters: the WARC + * digest fields define the digest value as a token, which does not allow the padding character + * (cf. ISO 28500 WARC 1.1, WARC-Block-Digest / WARC-Payload-Digest). SHA-1 digests are + * unaffected (32 characters without padding), while e.g. SHA-256 digests would end in * "====". */ private static String base32Unpadded(byte[] digest) { diff --git a/external/warc/src/test/java/org/apache/stormcrawler/warc/WARCDigestAlgorithmTest.java b/external/warc/src/test/java/org/apache/stormcrawler/warc/WARCDigestAlgorithmTest.java index 50226dc20..cd1d7d2b4 100644 --- a/external/warc/src/test/java/org/apache/stormcrawler/warc/WARCDigestAlgorithmTest.java +++ b/external/warc/src/test/java/org/apache/stormcrawler/warc/WARCDigestAlgorithmTest.java @@ -65,16 +65,15 @@ private static String expectedDigest(String jcaAlgorithm, String prefix, byte[]. for (byte[] bytes : byteArrays) { md.update(bytes); } - return prefix - + StringUtils.stripEnd(new Base32().encodeAsString(md.digest()), "="); + return prefix + StringUtils.stripEnd(new Base32().encodeAsString(md.digest()), "="); } catch (NoSuchAlgorithmException e) { throw new IllegalStateException(e); } } /** - * The bytes covered by the WARC-Block-Digest: everything between the end of the WARC header - * and the final CRLF CRLF. + * The bytes covered by the WARC-Block-Digest: everything between the end of the WARC header and + * the final CRLF CRLF. */ private static byte[] recordBlock(String warcString) { int start = warcString.indexOf("\r\n\r\n") + 4; @@ -112,8 +111,9 @@ void testGetDigestSha256() { @Test void testDigestValueContainsNoBase32Padding() { // the digest value is a token per the WARC 1.1 grammar and must not contain "=" - String sha256 = new WARCRecordFormat("", WARCRecordFormat.DIGEST_ALGORITHM_SHA256) - .getDigest(CONTENT); + String sha256 = + new WARCRecordFormat("", WARCRecordFormat.DIGEST_ALGORITHM_SHA256) + .getDigest(CONTENT); assertFalse(sha256.contains("="), "digest value must not contain Base32 padding"); String sha1 = new WARCRecordFormat("").getDigest(CONTENT); assertFalse(sha1.contains("="), "digest value must not contain Base32 padding"); @@ -176,7 +176,8 @@ void testResourceRecordDigestsSha256() { // no verbatim HTTP headers stored -> resource record, block digest equals payload digest Metadata metadata = new Metadata(); Tuple tuple = tupleWithContent(metadata); - WARCRecordFormat format = new WARCRecordFormat("", WARCRecordFormat.DIGEST_ALGORITHM_SHA256); + WARCRecordFormat format = + new WARCRecordFormat("", WARCRecordFormat.DIGEST_ALGORITHM_SHA256); String warcString = new String(format.format(tuple), StandardCharsets.UTF_8); assertTrue(warcString.contains("\r\nWARC-Type: resource\r\n")); assertTrue( @@ -214,7 +215,8 @@ void testMetadataRecordBlockDigestSha256() { when(tuple.getStringByField("url")).thenReturn(URL); when(tuple.getValueByField("metadata")).thenReturn(metadata); MetadataRecordFormat format = - new MetadataRecordFormat(List.of("source"), WARCRecordFormat.DIGEST_ALGORITHM_SHA256); + new MetadataRecordFormat( + List.of("source"), WARCRecordFormat.DIGEST_ALGORITHM_SHA256); String warcString = new String(format.format(tuple), StandardCharsets.UTF_8); // the payload of the metadata record are the metadata fields themselves diff --git a/external/warc/src/test/java/org/apache/stormcrawler/warc/WARCHdfsBoltTest.java b/external/warc/src/test/java/org/apache/stormcrawler/warc/WARCHdfsBoltTest.java index 548ea7944..2bc8340f2 100644 --- a/external/warc/src/test/java/org/apache/stormcrawler/warc/WARCHdfsBoltTest.java +++ b/external/warc/src/test/java/org/apache/stormcrawler/warc/WARCHdfsBoltTest.java @@ -142,8 +142,7 @@ void testDigestAlgorithmConfig() throws IOException { sha256Bolt.withConfigKey("warc"); Map sha256Conf = new HashMap<>(conf); sha256Conf.put( - WARCRecordFormat.DIGEST_ALGORITHM_PARAM, - WARCRecordFormat.DIGEST_ALGORITHM_SHA256); + WARCRecordFormat.DIGEST_ALGORITHM_PARAM, WARCRecordFormat.DIGEST_ALGORITHM_SHA256); sha256Bolt.prepare( sha256Conf, TestUtil.getMockedTopologyContext(), new OutputCollector(output)); try { From 727ec1df70a9c901e4788520de49176939f16407 Mon Sep 17 00:00:00 2001 From: abhinav-phi Date: Wed, 2 Sep 2026 23:40:26 +0530 Subject: [PATCH 4/4] Address review feedback on the configurable digest algorithm - remove the deprecated static getDigestSha1 methods: the next release is a major one (4.0.0), so the deprecated API can go straight away - validate the inputs of the getDigest methods (requireNonNull) - treat a null or blank warc.digest.algorithm value as the default (SHA-1), like an unset one - add javadoc to the constructors of the record formats and document the parameters, including the null/blank behaviour Suggested in review of #2110 by rzo1. --- .../warc/MetadataRecordFormat.java | 12 +++++ .../stormcrawler/warc/WARCRecordFormat.java | 51 +++++++++++-------- .../warc/WARCRequestRecordFormat.java | 11 ++++ .../warc/WARCDigestAlgorithmTest.java | 20 ++++++++ .../warc/WARCRecordFormatTest.java | 10 ++-- 5 files changed, 79 insertions(+), 25 deletions(-) diff --git a/external/warc/src/main/java/org/apache/stormcrawler/warc/MetadataRecordFormat.java b/external/warc/src/main/java/org/apache/stormcrawler/warc/MetadataRecordFormat.java index 979337529..b4c10620e 100644 --- a/external/warc/src/main/java/org/apache/stormcrawler/warc/MetadataRecordFormat.java +++ b/external/warc/src/main/java/org/apache/stormcrawler/warc/MetadataRecordFormat.java @@ -51,10 +51,22 @@ public class MetadataRecordFormat extends WARCRecordFormat { private final List metadataKeys; + /** + * Creates a metadata record format computing the digests with the default algorithm (SHA-1). + */ public MetadataRecordFormat(List metadataKeys) { this(metadataKeys, WARCRecordFormat.DIGEST_ALGORITHM_SHA1); } + /** + * Creates a metadata record format computing the WARC-Block-Digest field with the given + * algorithm. + * + * @param metadataKeys metadata keys to include as fields of the metadata record + * @param digestAlgorithm algorithm for the digest fields; see {@link + * WARCRecordFormat#WARCRecordFormat(String, String)} + * @throws IllegalArgumentException if the value is not a supported algorithm + */ public MetadataRecordFormat(List metadataKeys, String digestAlgorithm) { super("", digestAlgorithm); // the keys are fixed configuration: validate them once here instead of diff --git a/external/warc/src/main/java/org/apache/stormcrawler/warc/WARCRecordFormat.java b/external/warc/src/main/java/org/apache/stormcrawler/warc/WARCRecordFormat.java index c54fae17d..473c0f137 100644 --- a/external/warc/src/main/java/org/apache/stormcrawler/warc/WARCRecordFormat.java +++ b/external/warc/src/main/java/org/apache/stormcrawler/warc/WARCRecordFormat.java @@ -33,6 +33,7 @@ import java.util.Locale; import java.util.Map; import java.util.Map.Entry; +import java.util.Objects; import java.util.Optional; import java.util.UUID; import java.util.regex.Pattern; @@ -156,10 +157,28 @@ static String sanitizeWarcFieldValue(String value) { private final String digestNoContent; + /** + * Creates a record format computing the digests with the default algorithm (SHA-1). + * + * @param protocolMDprefix prefix of the metadata keys holding the protocol response, as set by + * {@code protocol.md.prefix}; may be empty + */ public WARCRecordFormat(String protocolMDprefix) { this(protocolMDprefix, DIGEST_ALGORITHM_SHA1); } + /** + * Creates a record format computing the WARC-Payload-Digest and WARC-Block-Digest fields with + * the given algorithm. + * + * @param protocolMDprefix prefix of the metadata keys holding the protocol response, as set by + * {@code protocol.md.prefix}; may be empty + * @param digestAlgorithm algorithm for the digest fields, {@value #DIGEST_ALGORITHM_SHA1} (the + * default) or {@value #DIGEST_ALGORITHM_SHA256}; matched case-insensitively with an + * optional hyphen, surrounding whitespace is trimmed. A {@code null} or blank value selects + * the default SHA-1. + * @throws IllegalArgumentException if the value is not a supported algorithm + */ public WARCRecordFormat(String protocolMDprefix, String digestAlgorithm) { this.protocolMDprefix = protocolMDprefix; this.digestJCAName = getDigestJCAName(digestAlgorithm); @@ -172,10 +191,12 @@ public WARCRecordFormat(String protocolMDprefix, String digestAlgorithm) { * matched case-insensitively and an optional hyphen is ignored, i.e. "sha256", * "SHA-256" etc. are all accepted. * + * @param digestAlgorithm algorithm to resolve; {@code null} or blank selects the default SHA-1 + * @return the JCA name of the message digest, e.g. "SHA-1" * @throws IllegalArgumentException if the value is not a supported algorithm */ private static String getDigestJCAName(String digestAlgorithm) { - if (digestAlgorithm == null) { + if (StringUtils.isBlank(digestAlgorithm)) { return "SHA-1"; } return switch (digestAlgorithm.trim().toLowerCase(Locale.ROOT).replace("-", "")) { @@ -197,10 +218,13 @@ private static String getDigestJCAName(String digestAlgorithm) { /** * Compute the digest of the given bytes with the configured algorithm. * + * @param bytes bytes to digest, must not be null * @return digest in the form "<algorithm>:<base32>", e.g. * "sha1:..." + * @throws NullPointerException if the input is null */ public String getDigest(byte[] bytes) { + Objects.requireNonNull(bytes, "bytes to digest must not be null"); MessageDigest md = DigestUtils.getDigest(digestJCAName); return digestPrefix + base32Unpadded(md.digest(bytes)); } @@ -209,10 +233,15 @@ public String getDigest(byte[] bytes) { * Compute the digest of the concatenation of the two given byte arrays with the configured * algorithm. * + * @param bytes1 first bytes to digest, must not be null + * @param bytes2 second bytes to digest, must not be null * @return digest in the form "<algorithm>:<base32>", e.g. * "sha1:..." + * @throws NullPointerException if one of the inputs is null */ public String getDigest(byte[] bytes1, byte[] bytes2) { + Objects.requireNonNull(bytes1, "first bytes to digest must not be null"); + Objects.requireNonNull(bytes2, "second bytes to digest must not be null"); MessageDigest md = DigestUtils.getDigest(digestJCAName); md.update(bytes1); return digestPrefix + base32Unpadded(md.digest(bytes2)); @@ -229,26 +258,6 @@ private static String base32Unpadded(byte[] digest) { return StringUtils.stripEnd(base32.encodeAsString(digest), "="); } - /** - * @deprecated use {@link #getDigest(byte[])} instead; the algorithm is set by {@link - * #DIGEST_ALGORITHM_PARAM} and no longer fixed to SHA-1 - */ - @Deprecated - public static String getDigestSha1(byte[] bytes) { - return "sha1:" + base32.encodeAsString(DigestUtils.sha1(bytes)); - } - - /** - * @deprecated use {@link #getDigest(byte[], byte[])} instead; the algorithm is set by {@link - * #DIGEST_ALGORITHM_PARAM} and no longer fixed to SHA-1 - */ - @Deprecated - public static String getDigestSha1(byte[] bytes1, byte[] bytes2) { - MessageDigest sha1 = DigestUtils.getSha1Digest(); - sha1.update(bytes1); - return "sha1:" + base32.encodeAsString(sha1.digest(bytes2)); - } - /** Generates a WARC info entry which can be stored at the beginning of each WARC file. */ public static byte[] generateWARCInfo(Map fields) { StringBuilder buffer = new StringBuilder(); diff --git a/external/warc/src/main/java/org/apache/stormcrawler/warc/WARCRequestRecordFormat.java b/external/warc/src/main/java/org/apache/stormcrawler/warc/WARCRequestRecordFormat.java index 21b42fb10..b1ea5a833 100644 --- a/external/warc/src/main/java/org/apache/stormcrawler/warc/WARCRequestRecordFormat.java +++ b/external/warc/src/main/java/org/apache/stormcrawler/warc/WARCRequestRecordFormat.java @@ -41,10 +41,21 @@ public class WARCRequestRecordFormat extends WARCRecordFormat { protected static final Pattern REQUEST_LINE_PATTERN = Pattern.compile("^\\S+ \\S+ HTTP/1\\.[01]$"); + /** Creates a request record format computing the digests with the default algorithm (SHA-1). */ public WARCRequestRecordFormat(String protocolMDprefix) { super(protocolMDprefix); } + /** + * Creates a request record format computing the WARC-Block-Digest field with the given + * algorithm. + * + * @param protocolMDprefix prefix of the metadata keys holding the protocol response, as set by + * {@code protocol.md.prefix}; may be empty + * @param digestAlgorithm algorithm for the digest fields; see {@link + * WARCRecordFormat#WARCRecordFormat(String, String)} + * @throws IllegalArgumentException if the value is not a supported algorithm + */ public WARCRequestRecordFormat(String protocolMDprefix, String digestAlgorithm) { super(protocolMDprefix, digestAlgorithm); } diff --git a/external/warc/src/test/java/org/apache/stormcrawler/warc/WARCDigestAlgorithmTest.java b/external/warc/src/test/java/org/apache/stormcrawler/warc/WARCDigestAlgorithmTest.java index cd1d7d2b4..79ea6b1ac 100644 --- a/external/warc/src/test/java/org/apache/stormcrawler/warc/WARCDigestAlgorithmTest.java +++ b/external/warc/src/test/java/org/apache/stormcrawler/warc/WARCDigestAlgorithmTest.java @@ -98,6 +98,26 @@ void testDigestDefaultsToSha1() { "digest algorithm must default to SHA-1"); // a null algorithm must be treated as the default assertEquals(SHA1_ABCDEF, new WARCRecordFormat("", null).getDigest(CONTENT)); + // a blank algorithm must be treated as the default as well + assertEquals(SHA1_ABCDEF, new WARCRecordFormat("", "").getDigest(CONTENT)); + assertEquals(SHA1_ABCDEF, new WARCRecordFormat("", " ").getDigest(CONTENT)); + } + + @Test + void testGetDigestRejectsNullBytes() { + WARCRecordFormat format = new WARCRecordFormat(""); + assertThrows( + NullPointerException.class, + () -> format.getDigest(null), + "getDigest(byte[]) must reject null input"); + assertThrows( + NullPointerException.class, + () -> format.getDigest(CONTENT, null), + "getDigest(byte[], byte[]) must reject null input"); + assertThrows( + NullPointerException.class, + () -> format.getDigest(null, CONTENT), + "getDigest(byte[], byte[]) must reject null input"); } @Test diff --git a/external/warc/src/test/java/org/apache/stormcrawler/warc/WARCRecordFormatTest.java b/external/warc/src/test/java/org/apache/stormcrawler/warc/WARCRecordFormatTest.java index d834d8ffd..224deb74e 100644 --- a/external/warc/src/test/java/org/apache/stormcrawler/warc/WARCRecordFormatTest.java +++ b/external/warc/src/test/java/org/apache/stormcrawler/warc/WARCRecordFormatTest.java @@ -81,14 +81,14 @@ class WARCRecordFormatTest { void testGetDigestSha1() { byte[] content = {'a', 'b', 'c', 'd', 'e', 'f'}; String sha1str = "sha1:D6FMCDZDYW23YELHXWUEXAZ6LQCXU56S"; - assertEquals(sha1str, WARCRecordFormat.getDigestSha1(content), "Wrong sha1 digest"); + assertEquals(sha1str, new WARCRecordFormat("").getDigest(content), "Wrong sha1 digest"); } @Test void testGetDigestSha1Empty() { byte[] content = {}; String sha1str = "sha1:3I42H3S6NNFQ2MSVX7XZKYAYSCX5QBYJ"; - assertEquals(sha1str, WARCRecordFormat.getDigestSha1(content), "Wrong sha1 digest"); + assertEquals(sha1str, new WARCRecordFormat("").getDigest(content), "Wrong sha1 digest"); } @Test @@ -97,7 +97,9 @@ void testGetDigestSha1TwoByteArrays() { byte[] content2 = {'d', 'e', 'f'}; String sha1str = "sha1:D6FMCDZDYW23YELHXWUEXAZ6LQCXU56S"; assertEquals( - sha1str, WARCRecordFormat.getDigestSha1(content1, content2), "Wrong sha1 digest"); + sha1str, + new WARCRecordFormat("").getDigest(content1, content2), + "Wrong sha1 digest"); } @Test @@ -106,7 +108,7 @@ void testGetDigestSha1RobotsTxt() { String robotsTxt = "User-agent: *\r\nDisallow:"; byte[] content = robotsTxt.getBytes(StandardCharsets.UTF_8); String sha1str = "sha1:DHBVNHAJABWFHIYUHNCKYYIB3OBPFX3Y"; - assertEquals(sha1str, WARCRecordFormat.getDigestSha1(content), "Wrong sha1 digest"); + assertEquals(sha1str, new WARCRecordFormat("").getDigest(content), "Wrong sha1 digest"); } @Test