diff --git a/docs/src/main/asciidoc/configuration.adoc b/docs/src/main/asciidoc/configuration.adoc index 047a86ea4..d5996a6fc 100644 --- a/docs/src/main/asciidoc/configuration.adoc +++ b/docs/src/main/asciidoc/configuration.adoc @@ -737,6 +737,7 @@ See the link:https://github.com/apache/stormcrawler/tree/main/external/warc[warc | key | default value | description | warc.metadata.keys | - | Metadata keys to include as WARC metadata records (optional list). +| warc.digest.algorithm | sha1 | Algorithm for the WARC-Payload-Digest and WARC-Block-Digest headers: `sha1` or `sha256`. SHA-1 is the convention across the WARC ecosystem; downstream tooling such as CDX indexers may expect `sha1` digests. Digest values are Base32-encoded without padding. |=== NOTE: For complete WARC records, set `http.store.headers` to `true`. The OkHttp protocol (`org.apache.stormcrawler.protocol.okhttp.HttpProtocol`) is recommended for WARC generation as it provides verbatim HTTP headers. diff --git a/external/warc/README.md b/external/warc/README.md index 8f22aa117..3aa7e1545 100644 --- a/external/warc/README.md +++ b/external/warc/README.md @@ -161,6 +161,16 @@ A note on the recording of HTTP requests and responses with StormCrawler and the You can specify in the configuration which metadata key/values to store as WARC metadata using `warc.metadata.keys`. +The algorithm used to compute the `WARC-Payload-Digest` and `WARC-Block-Digest` fields is configurable with `warc.digest.algorithm`. It accepts `sha1` (the default) and `sha256`: + +``` + warc.digest.algorithm: sha256 +``` + +The value is matched case-insensitively and an optional hyphen is ignored, i.e. `SHA-256` is also accepted. SHA-1 is the convention across the WARC ecosystem and downstream tooling such as CDX indexers may expect `sha1` digests, so change the default only if your downstream tooling supports the alternative algorithm. Invalid values make the bolt fail when it is prepared. + +The digest values are written as Base32 without the trailing `=` padding, as required by the grammar of the WARC digest fields (the digest value is a token, which does not allow the padding character). + ## Consuming WARC files Web archives harvested in the [WARC format](https://iipc.github.io/warc-specifications/specifications/warc-format/warc-1.1/) can be used as input for StormCrawler – instead of fetching content from remote servers, the WARCSpout reads WARC files and emits the archive web page captures as tuples into the topology. diff --git a/external/warc/src/main/java/org/apache/stormcrawler/warc/MetadataRecordFormat.java b/external/warc/src/main/java/org/apache/stormcrawler/warc/MetadataRecordFormat.java index 75b79924e..b4c10620e 100644 --- a/external/warc/src/main/java/org/apache/stormcrawler/warc/MetadataRecordFormat.java +++ b/external/warc/src/main/java/org/apache/stormcrawler/warc/MetadataRecordFormat.java @@ -51,8 +51,24 @@ public class MetadataRecordFormat extends WARCRecordFormat { private final List metadataKeys; + /** + * Creates a metadata record format computing the digests with the default algorithm (SHA-1). + */ public MetadataRecordFormat(List metadataKeys) { - super(""); + this(metadataKeys, WARCRecordFormat.DIGEST_ALGORITHM_SHA1); + } + + /** + * Creates a metadata record format computing the WARC-Block-Digest field with the given + * algorithm. + * + * @param metadataKeys metadata keys to include as fields of the metadata record + * @param digestAlgorithm algorithm for the digest fields; see {@link + * WARCRecordFormat#WARCRecordFormat(String, String)} + * @throws IllegalArgumentException if the value is not a supported algorithm + */ + public MetadataRecordFormat(List metadataKeys, String digestAlgorithm) { + super("", digestAlgorithm); // the keys are fixed configuration: validate them once here instead of // for every record final List validKeys = new ArrayList<>(metadataKeys.size()); @@ -118,7 +134,7 @@ public byte[] format(Tuple tuple) { int contentLength = metadata_representation.length; buffer.append("Content-Length: ").append(Integer.toString(contentLength)).append(CRLF); - String blockDigest = getDigestSha1(metadata_representation); + String blockDigest = getDigest(metadata_representation); String captureTime = getCaptureTime(metadata); buffer.append("WARC-Date: ").append(captureTime).append(CRLF); diff --git a/external/warc/src/main/java/org/apache/stormcrawler/warc/WARCHdfsBolt.java b/external/warc/src/main/java/org/apache/stormcrawler/warc/WARCHdfsBolt.java index 48dcf32d9..8847d0911 100644 --- a/external/warc/src/main/java/org/apache/stormcrawler/warc/WARCHdfsBolt.java +++ b/external/warc/src/main/java/org/apache/stormcrawler/warc/WARCHdfsBolt.java @@ -74,14 +74,19 @@ public void doPrepare( throws IOException { super.doPrepare(conf, topologyContext, collector); protocolMDprefix = ConfUtils.getString(conf, ProtocolResponse.PROTOCOL_MD_PREFIX_PARAM, ""); - withRecordFormat(new WARCRecordFormat(protocolMDprefix)); + String digestAlgorithm = + ConfUtils.getString( + conf, + WARCRecordFormat.DIGEST_ALGORITHM_PARAM, + WARCRecordFormat.DIGEST_ALGORITHM_SHA1); + withRecordFormat(new WARCRecordFormat(protocolMDprefix, digestAlgorithm)); if (withRequestRecords) { - addRecordFormat(new WARCRequestRecordFormat(protocolMDprefix), 0); + addRecordFormat(new WARCRequestRecordFormat(protocolMDprefix, digestAlgorithm), 0); } // detect if a list of keys was specified to be stored in the metadata List metadataToWrite = ConfUtils.loadListFromConf(METADATA_KEYS_STORE, conf); if (!metadataToWrite.isEmpty()) { - addRecordFormat(new MetadataRecordFormat(metadataToWrite), 1); + addRecordFormat(new MetadataRecordFormat(metadataToWrite, digestAlgorithm), 1); } } diff --git a/external/warc/src/main/java/org/apache/stormcrawler/warc/WARCRecordFormat.java b/external/warc/src/main/java/org/apache/stormcrawler/warc/WARCRecordFormat.java index ad8f2d46a..473c0f137 100644 --- a/external/warc/src/main/java/org/apache/stormcrawler/warc/WARCRecordFormat.java +++ b/external/warc/src/main/java/org/apache/stormcrawler/warc/WARCRecordFormat.java @@ -33,6 +33,7 @@ import java.util.Locale; import java.util.Map; import java.util.Map.Entry; +import java.util.Objects; import java.util.Optional; import java.util.UUID; import java.util.regex.Pattern; @@ -129,23 +130,132 @@ static String sanitizeWarcFieldValue(String value) { Pattern.compile("(?i)(?:Content-(?:Encoding|Length)|Transfer-Encoding)"); protected static final String X_HIDE_HEADER = "X-Crawler-"; + /** + * Configuration key setting the algorithm used to compute the WARC-Payload-Digest and + * WARC-Block-Digest fields. Supported values are {@value #DIGEST_ALGORITHM_SHA1} (the default) + * and {@value #DIGEST_ALGORITHM_SHA256}. + * + *

Note: SHA-1 is the convention across the WARC ecosystem and downstream tooling (CDX + * indexes, revisit record handling) may expect it. Change the default deliberately, not + * casually. + */ + public static final String DIGEST_ALGORITHM_PARAM = "warc.digest.algorithm"; + + public static final String DIGEST_ALGORITHM_SHA1 = "sha1"; + + public static final String DIGEST_ALGORITHM_SHA256 = "sha256"; + private static final Base32 base32 = new Base32(); - private static final String digestNoContent = getDigestSha1(new byte[0]); protected final String protocolMDprefix; + /** JCA name of the message digest algorithm, e.g. "SHA-1". */ + private final String digestJCAName; + + /** Algorithm prefix of the WARC digest fields, e.g. "sha1:". */ + private final String digestPrefix; + + private final String digestNoContent; + + /** + * Creates a record format computing the digests with the default algorithm (SHA-1). + * + * @param protocolMDprefix prefix of the metadata keys holding the protocol response, as set by + * {@code protocol.md.prefix}; may be empty + */ public WARCRecordFormat(String protocolMDprefix) { + this(protocolMDprefix, DIGEST_ALGORITHM_SHA1); + } + + /** + * Creates a record format computing the WARC-Payload-Digest and WARC-Block-Digest fields with + * the given algorithm. + * + * @param protocolMDprefix prefix of the metadata keys holding the protocol response, as set by + * {@code protocol.md.prefix}; may be empty + * @param digestAlgorithm algorithm for the digest fields, {@value #DIGEST_ALGORITHM_SHA1} (the + * default) or {@value #DIGEST_ALGORITHM_SHA256}; matched case-insensitively with an + * optional hyphen, surrounding whitespace is trimmed. A {@code null} or blank value selects + * the default SHA-1. + * @throws IllegalArgumentException if the value is not a supported algorithm + */ + public WARCRecordFormat(String protocolMDprefix, String digestAlgorithm) { this.protocolMDprefix = protocolMDprefix; + this.digestJCAName = getDigestJCAName(digestAlgorithm); + this.digestPrefix = digestJCAName.toLowerCase(Locale.ROOT).replace("-", "") + ":"; + this.digestNoContent = getDigest(new byte[0]); } - public static String getDigestSha1(byte[] bytes) { - return "sha1:" + base32.encodeAsString(DigestUtils.sha1(bytes)); + /** + * Resolve the configured digest algorithm to the JCA name of the message digest. The value is + * matched case-insensitively and an optional hyphen is ignored, i.e. "sha256", + * "SHA-256" etc. are all accepted. + * + * @param digestAlgorithm algorithm to resolve; {@code null} or blank selects the default SHA-1 + * @return the JCA name of the message digest, e.g. "SHA-1" + * @throws IllegalArgumentException if the value is not a supported algorithm + */ + private static String getDigestJCAName(String digestAlgorithm) { + if (StringUtils.isBlank(digestAlgorithm)) { + return "SHA-1"; + } + return switch (digestAlgorithm.trim().toLowerCase(Locale.ROOT).replace("-", "")) { + case DIGEST_ALGORITHM_SHA1 -> "SHA-1"; + case DIGEST_ALGORITHM_SHA256 -> "SHA-256"; + default -> + throw new IllegalArgumentException( + "Unsupported value [" + + digestAlgorithm + + "] for " + + DIGEST_ALGORITHM_PARAM + + ", supported algorithms: " + + DIGEST_ALGORITHM_SHA1 + + ", " + + DIGEST_ALGORITHM_SHA256); + }; + } + + /** + * Compute the digest of the given bytes with the configured algorithm. + * + * @param bytes bytes to digest, must not be null + * @return digest in the form "<algorithm>:<base32>", e.g. + * "sha1:..." + * @throws NullPointerException if the input is null + */ + public String getDigest(byte[] bytes) { + Objects.requireNonNull(bytes, "bytes to digest must not be null"); + MessageDigest md = DigestUtils.getDigest(digestJCAName); + return digestPrefix + base32Unpadded(md.digest(bytes)); + } + + /** + * Compute the digest of the concatenation of the two given byte arrays with the configured + * algorithm. + * + * @param bytes1 first bytes to digest, must not be null + * @param bytes2 second bytes to digest, must not be null + * @return digest in the form "<algorithm>:<base32>", e.g. + * "sha1:..." + * @throws NullPointerException if one of the inputs is null + */ + public String getDigest(byte[] bytes1, byte[] bytes2) { + Objects.requireNonNull(bytes1, "first bytes to digest must not be null"); + Objects.requireNonNull(bytes2, "second bytes to digest must not be null"); + MessageDigest md = DigestUtils.getDigest(digestJCAName); + md.update(bytes1); + return digestPrefix + base32Unpadded(md.digest(bytes2)); } - public static String getDigestSha1(byte[] bytes1, byte[] bytes2) { - MessageDigest sha1 = DigestUtils.getSha1Digest(); - sha1.update(bytes1); - return "sha1:" + base32.encodeAsString(sha1.digest(bytes2)); + /** + * Base32-encode a digest value without the trailing "=" padding characters: the WARC + * digest fields define the digest value as a token, which does not allow the padding character + * (cf. ISO 28500 WARC 1.1, WARC-Block-Digest / WARC-Payload-Digest). SHA-1 digests are + * unaffected (32 characters without padding), while e.g. SHA-256 digests would end in + * "====". + */ + private static String base32Unpadded(byte[] digest) { + return StringUtils.stripEnd(base32.encodeAsString(digest), "="); } /** Generates a WARC info entry which can be stored at the beginning of each WARC file. */ @@ -435,14 +545,14 @@ public byte[] format(Tuple tuple) { String blockDigest = digestNoContent; if (content != null) { contentLength = content.length; - payloadDigest = getDigestSha1(content); + payloadDigest = getDigest(content); if (WARCTypeValue.equals(WARC_TYPE_RESPONSE)) { - blockDigest = getDigestSha1(httpheaders, content); + blockDigest = getDigest(httpheaders, content); } else { blockDigest = payloadDigest; } } else if (WARCTypeValue.equals(WARC_TYPE_RESPONSE)) { - blockDigest = getDigestSha1(httpheaders); + blockDigest = getDigest(httpheaders); } // add the length of the http header diff --git a/external/warc/src/main/java/org/apache/stormcrawler/warc/WARCRequestRecordFormat.java b/external/warc/src/main/java/org/apache/stormcrawler/warc/WARCRequestRecordFormat.java index da35d1af0..b1ea5a833 100644 --- a/external/warc/src/main/java/org/apache/stormcrawler/warc/WARCRequestRecordFormat.java +++ b/external/warc/src/main/java/org/apache/stormcrawler/warc/WARCRequestRecordFormat.java @@ -41,10 +41,25 @@ public class WARCRequestRecordFormat extends WARCRecordFormat { protected static final Pattern REQUEST_LINE_PATTERN = Pattern.compile("^\\S+ \\S+ HTTP/1\\.[01]$"); + /** Creates a request record format computing the digests with the default algorithm (SHA-1). */ public WARCRequestRecordFormat(String protocolMDprefix) { super(protocolMDprefix); } + /** + * Creates a request record format computing the WARC-Block-Digest field with the given + * algorithm. + * + * @param protocolMDprefix prefix of the metadata keys holding the protocol response, as set by + * {@code protocol.md.prefix}; may be empty + * @param digestAlgorithm algorithm for the digest fields; see {@link + * WARCRecordFormat#WARCRecordFormat(String, String)} + * @throws IllegalArgumentException if the value is not a supported algorithm + */ + public WARCRequestRecordFormat(String protocolMDprefix, String digestAlgorithm) { + super(protocolMDprefix, digestAlgorithm); + } + @Override public byte[] format(Tuple tuple) { @@ -82,7 +97,7 @@ public byte[] format(Tuple tuple) { int contentLength = httpheaders.length; buffer.append("Content-Length: ").append(Integer.toString(contentLength)).append(CRLF); - String blockDigest = getDigestSha1(httpheaders); + String blockDigest = getDigest(httpheaders); String captureTime = getCaptureTime(metadata); buffer.append("WARC-Date: ").append(captureTime).append(CRLF); diff --git a/external/warc/src/test/java/org/apache/stormcrawler/warc/WARCDigestAlgorithmTest.java b/external/warc/src/test/java/org/apache/stormcrawler/warc/WARCDigestAlgorithmTest.java new file mode 100644 index 000000000..79ea6b1ac --- /dev/null +++ b/external/warc/src/test/java/org/apache/stormcrawler/warc/WARCDigestAlgorithmTest.java @@ -0,0 +1,249 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.stormcrawler.warc; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.List; +import org.apache.commons.codec.binary.Base32; +import org.apache.commons.lang3.StringUtils; +import org.apache.storm.tuple.Tuple; +import org.apache.stormcrawler.Metadata; +import org.apache.stormcrawler.protocol.ProtocolResponse; +import org.junit.jupiter.api.Test; + +/** + * Tests that the algorithm used for the WARC-Payload-Digest and WARC-Block-Digest fields is + * configurable with {@link WARCRecordFormat#DIGEST_ALGORITHM_PARAM} and that SHA-1 remains the + * default. + */ +class WARCDigestAlgorithmTest { + + private static final String URL = "https://www.example.org/"; + + private static final byte[] CONTENT = "abcdef".getBytes(StandardCharsets.UTF_8); + + private static final String SHA1_ABCDEF = "sha1:D6FMCDZDYW23YELHXWUEXAZ6LQCXU56S"; + + /* + * The Base32 padding is omitted: the WARC digest fields define the digest value as a token, + * which does not allow the padding character "=" (cf. ISO 28500 WARC 1.1). + */ + private static final String SHA256_ABCDEF = + "sha256:X32X5R7VHJWUBPVWICTYBJRZZA54FGWIVGAW6H6GYXDNZWJ4I4QQ"; + + private static final String SHA256_EMPTY = + "sha256:4OYMIQUY7QOBJGX36TEJS35ZEQT24QPEMSNZGTFESWMRW6CSXBKQ"; + + /** Compute the expected digest independently of the code under test. */ + private static String expectedDigest(String jcaAlgorithm, String prefix, byte[]... byteArrays) { + try { + MessageDigest md = MessageDigest.getInstance(jcaAlgorithm); + for (byte[] bytes : byteArrays) { + md.update(bytes); + } + return prefix + StringUtils.stripEnd(new Base32().encodeAsString(md.digest()), "="); + } catch (NoSuchAlgorithmException e) { + throw new IllegalStateException(e); + } + } + + /** + * The bytes covered by the WARC-Block-Digest: everything between the end of the WARC header and + * the final CRLF CRLF. + */ + private static byte[] recordBlock(String warcString) { + int start = warcString.indexOf("\r\n\r\n") + 4; + return warcString + .substring(start, warcString.length() - 4) + .getBytes(StandardCharsets.UTF_8); + } + + private static Tuple tupleWithContent(Metadata metadata) { + Tuple tuple = mock(Tuple.class); + when(tuple.getBinaryByField("content")).thenReturn(CONTENT); + when(tuple.getStringByField("url")).thenReturn(URL); + when(tuple.getValueByField("metadata")).thenReturn(metadata); + return tuple; + } + + @Test + void testDigestDefaultsToSha1() { + assertEquals( + SHA1_ABCDEF, + new WARCRecordFormat("").getDigest(CONTENT), + "digest algorithm must default to SHA-1"); + // a null algorithm must be treated as the default + assertEquals(SHA1_ABCDEF, new WARCRecordFormat("", null).getDigest(CONTENT)); + // a blank algorithm must be treated as the default as well + assertEquals(SHA1_ABCDEF, new WARCRecordFormat("", "").getDigest(CONTENT)); + assertEquals(SHA1_ABCDEF, new WARCRecordFormat("", " ").getDigest(CONTENT)); + } + + @Test + void testGetDigestRejectsNullBytes() { + WARCRecordFormat format = new WARCRecordFormat(""); + assertThrows( + NullPointerException.class, + () -> format.getDigest(null), + "getDigest(byte[]) must reject null input"); + assertThrows( + NullPointerException.class, + () -> format.getDigest(CONTENT, null), + "getDigest(byte[], byte[]) must reject null input"); + assertThrows( + NullPointerException.class, + () -> format.getDigest(null, CONTENT), + "getDigest(byte[], byte[]) must reject null input"); + } + + @Test + void testGetDigestSha256() { + WARCRecordFormat format = + new WARCRecordFormat("", WARCRecordFormat.DIGEST_ALGORITHM_SHA256); + assertEquals(SHA256_ABCDEF, format.getDigest(CONTENT), "Wrong sha256 digest"); + assertEquals(SHA256_EMPTY, format.getDigest(new byte[0]), "Wrong sha256 digest"); + } + + @Test + void testDigestValueContainsNoBase32Padding() { + // the digest value is a token per the WARC 1.1 grammar and must not contain "=" + String sha256 = + new WARCRecordFormat("", WARCRecordFormat.DIGEST_ALGORITHM_SHA256) + .getDigest(CONTENT); + assertFalse(sha256.contains("="), "digest value must not contain Base32 padding"); + String sha1 = new WARCRecordFormat("").getDigest(CONTENT); + assertFalse(sha1.contains("="), "digest value must not contain Base32 padding"); + } + + @Test + void testGetDigestSha256TwoByteArrays() { + WARCRecordFormat format = + new WARCRecordFormat("", WARCRecordFormat.DIGEST_ALGORITHM_SHA256); + byte[] content1 = "abc".getBytes(StandardCharsets.UTF_8); + byte[] content2 = "def".getBytes(StandardCharsets.UTF_8); + assertEquals( + SHA256_ABCDEF, + format.getDigest(content1, content2), + "Wrong sha256 digest over concatenated byte arrays"); + } + + @Test + void testDigestAlgorithmValueVariants() { + // the value is matched case-insensitively, an optional hyphen is ignored and + // surrounding whitespace is trimmed + assertEquals(SHA256_ABCDEF, new WARCRecordFormat("", "SHA256").getDigest(CONTENT)); + assertEquals(SHA256_ABCDEF, new WARCRecordFormat("", "SHA-256").getDigest(CONTENT)); + assertEquals(SHA256_ABCDEF, new WARCRecordFormat("", " sha256 ").getDigest(CONTENT)); + assertEquals(SHA1_ABCDEF, new WARCRecordFormat("", "SHA-1").getDigest(CONTENT)); + } + + @Test + void testUnsupportedDigestAlgorithm() { + assertThrows(IllegalArgumentException.class, () -> new WARCRecordFormat("", "md5")); + assertThrows(IllegalArgumentException.class, () -> new WARCRecordFormat("", "sha512")); + assertThrows( + IllegalArgumentException.class, + () -> new MetadataRecordFormat(List.of("source"), "md5")); + assertThrows(IllegalArgumentException.class, () -> new WARCRequestRecordFormat("", "md5")); + } + + @Test + void testResponseRecordDigestsSha256() { + Metadata metadata = new Metadata(); + metadata.addValue( + "protocol." + ProtocolResponse.RESPONSE_HEADERS_KEY, + "HTTP/1.1 200 OK\r\nContent-Type: text/html\r\n"); + Tuple tuple = tupleWithContent(metadata); + WARCRecordFormat format = + new WARCRecordFormat("protocol.", WARCRecordFormat.DIGEST_ALGORITHM_SHA256); + String warcString = new String(format.format(tuple), StandardCharsets.UTF_8); + + assertTrue( + warcString.contains("\r\nWARC-Payload-Digest: " + SHA256_ABCDEF + "\r\n"), + "WARC response record: payload digest must be SHA-256"); + String expectedBlockDigest = expectedDigest("SHA-256", "sha256:", recordBlock(warcString)); + assertTrue( + warcString.contains("\r\nWARC-Block-Digest: " + expectedBlockDigest + "\r\n"), + "WARC response record: block digest must be SHA-256 over HTTP headers and payload"); + } + + @Test + void testResourceRecordDigestsSha256() { + // no verbatim HTTP headers stored -> resource record, block digest equals payload digest + Metadata metadata = new Metadata(); + Tuple tuple = tupleWithContent(metadata); + WARCRecordFormat format = + new WARCRecordFormat("", WARCRecordFormat.DIGEST_ALGORITHM_SHA256); + String warcString = new String(format.format(tuple), StandardCharsets.UTF_8); + assertTrue(warcString.contains("\r\nWARC-Type: resource\r\n")); + assertTrue( + warcString.contains("\r\nWARC-Payload-Digest: " + SHA256_ABCDEF + "\r\n"), + "WARC resource record: payload digest must be SHA-256"); + assertTrue( + warcString.contains("\r\nWARC-Block-Digest: " + SHA256_ABCDEF + "\r\n"), + "WARC resource record: block digest must be SHA-256"); + } + + @Test + void testRequestRecordBlockDigestSha256() { + Metadata metadata = new Metadata(); + metadata.addValue( + "protocol." + ProtocolResponse.REQUEST_HEADERS_KEY, + "GET / HTTP/2\r\nUser-Agent: mybot\r\nConnection: Keep-Alive\r\n\r\n"); + Tuple tuple = mock(Tuple.class); + when(tuple.getStringByField("url")).thenReturn(URL); + when(tuple.getValueByField("metadata")).thenReturn(metadata); + WARCRequestRecordFormat format = + new WARCRequestRecordFormat("protocol.", WARCRecordFormat.DIGEST_ALGORITHM_SHA256); + String warcString = new String(format.format(tuple), StandardCharsets.UTF_8); + + String expectedBlockDigest = expectedDigest("SHA-256", "sha256:", recordBlock(warcString)); + assertTrue( + warcString.contains("\r\nWARC-Block-Digest: " + expectedBlockDigest + "\r\n"), + "WARC request record: block digest must be SHA-256 over the request headers"); + } + + @Test + void testMetadataRecordBlockDigestSha256() { + Metadata metadata = new Metadata(); + metadata.addValue("source", "a source"); + Tuple tuple = mock(Tuple.class); + when(tuple.getStringByField("url")).thenReturn(URL); + when(tuple.getValueByField("metadata")).thenReturn(metadata); + MetadataRecordFormat format = + new MetadataRecordFormat( + List.of("source"), WARCRecordFormat.DIGEST_ALGORITHM_SHA256); + String warcString = new String(format.format(tuple), StandardCharsets.UTF_8); + + // the payload of the metadata record are the metadata fields themselves + byte[] payload = "source: a source\r\n".getBytes(StandardCharsets.UTF_8); + String expectedBlockDigest = expectedDigest("SHA-256", "sha256:", payload); + assertTrue( + warcString.contains("\r\nWARC-Block-Digest: " + expectedBlockDigest + "\r\n"), + "WARC metadata record: block digest must be SHA-256 over the metadata payload"); + } +} diff --git a/external/warc/src/test/java/org/apache/stormcrawler/warc/WARCHdfsBoltTest.java b/external/warc/src/test/java/org/apache/stormcrawler/warc/WARCHdfsBoltTest.java index 662355122..2bc8340f2 100644 --- a/external/warc/src/test/java/org/apache/stormcrawler/warc/WARCHdfsBoltTest.java +++ b/external/warc/src/test/java/org/apache/stormcrawler/warc/WARCHdfsBoltTest.java @@ -135,6 +135,36 @@ void testHttp2() throws IOException { "WARC response record is expected to include WARC header \"WARC-IP-Address\""); } + @Test + void testDigestAlgorithmConfig() throws IOException { + // instantiate a second bolt with warc.digest.algorithm: sha256 + HdfsBolt sha256Bolt = makeBolt(); + sha256Bolt.withConfigKey("warc"); + Map sha256Conf = new HashMap<>(conf); + sha256Conf.put( + WARCRecordFormat.DIGEST_ALGORITHM_PARAM, WARCRecordFormat.DIGEST_ALGORITHM_SHA256); + sha256Bolt.prepare( + sha256Conf, TestUtil.getMockedTopologyContext(), new OutputCollector(output)); + try { + sha256Bolt.execute(getPage()); + } finally { + sha256Bolt.cleanup(); + } + List records = readWARCs(warcDir).collect(Collectors.toList()); + // expected 3 records (warcinfo, request, response) + assertEquals(3, records.size()); + for (WarcRecord record : records) { + String payloadDigest = record.headers().first("WARC-Payload-Digest").orElse(""); + String blockDigest = record.headers().first("WARC-Block-Digest").orElse(""); + assertTrue( + payloadDigest.isEmpty() || payloadDigest.startsWith("sha256:"), + "WARC-Payload-Digest must use the configured algorithm sha256"); + assertTrue( + blockDigest.isEmpty() || blockDigest.startsWith("sha256:"), + "WARC-Block-Digest must use the configured algorithm sha256"); + } + } + private static Stream readWARCs(Path warcDir) { try { return Files.walk(warcDir) diff --git a/external/warc/src/test/java/org/apache/stormcrawler/warc/WARCRecordFormatTest.java b/external/warc/src/test/java/org/apache/stormcrawler/warc/WARCRecordFormatTest.java index d834d8ffd..224deb74e 100644 --- a/external/warc/src/test/java/org/apache/stormcrawler/warc/WARCRecordFormatTest.java +++ b/external/warc/src/test/java/org/apache/stormcrawler/warc/WARCRecordFormatTest.java @@ -81,14 +81,14 @@ class WARCRecordFormatTest { void testGetDigestSha1() { byte[] content = {'a', 'b', 'c', 'd', 'e', 'f'}; String sha1str = "sha1:D6FMCDZDYW23YELHXWUEXAZ6LQCXU56S"; - assertEquals(sha1str, WARCRecordFormat.getDigestSha1(content), "Wrong sha1 digest"); + assertEquals(sha1str, new WARCRecordFormat("").getDigest(content), "Wrong sha1 digest"); } @Test void testGetDigestSha1Empty() { byte[] content = {}; String sha1str = "sha1:3I42H3S6NNFQ2MSVX7XZKYAYSCX5QBYJ"; - assertEquals(sha1str, WARCRecordFormat.getDigestSha1(content), "Wrong sha1 digest"); + assertEquals(sha1str, new WARCRecordFormat("").getDigest(content), "Wrong sha1 digest"); } @Test @@ -97,7 +97,9 @@ void testGetDigestSha1TwoByteArrays() { byte[] content2 = {'d', 'e', 'f'}; String sha1str = "sha1:D6FMCDZDYW23YELHXWUEXAZ6LQCXU56S"; assertEquals( - sha1str, WARCRecordFormat.getDigestSha1(content1, content2), "Wrong sha1 digest"); + sha1str, + new WARCRecordFormat("").getDigest(content1, content2), + "Wrong sha1 digest"); } @Test @@ -106,7 +108,7 @@ void testGetDigestSha1RobotsTxt() { String robotsTxt = "User-agent: *\r\nDisallow:"; byte[] content = robotsTxt.getBytes(StandardCharsets.UTF_8); String sha1str = "sha1:DHBVNHAJABWFHIYUHNCKYYIB3OBPFX3Y"; - assertEquals(sha1str, WARCRecordFormat.getDigestSha1(content), "Wrong sha1 digest"); + assertEquals(sha1str, new WARCRecordFormat("").getDigest(content), "Wrong sha1 digest"); } @Test