From 607184885ff0fea7c93cee046b22504f651fdb83 Mon Sep 17 00:00:00 2001 From: Harsh Srivastava <76532840+HarshDevelops@users.noreply.github.com> Date: Thu, 3 Sep 2026 17:39:40 +0530 Subject: [PATCH] #108 Resolve HTTP header aliases at lookup sites --- .../stormcrawler/bolt/FeedParserBolt.java | 4 +- .../apache/stormcrawler/bolt/FetcherBolt.java | 4 +- .../stormcrawler/bolt/JSoupParserBolt.java | 4 +- .../stormcrawler/bolt/SimpleFetcherBolt.java | 5 +- .../stormcrawler/bolt/SiteMapParserBolt.java | 3 +- .../protocol/HttpRobotRulesParser.java | 9 +- .../protocol/okhttp/HttpProtocol.java | 7 +- .../util/CharsetIdentification.java | 3 +- .../stormcrawler/util/HttpHeaderResolver.java | 91 ++++++++++++++++++ .../util/HttpHeaderResolverTest.java | 94 +++++++++++++++++++ .../apache/stormcrawler/tika/ParserBolt.java | 9 +- .../stormcrawler/warc/WARCRecordFormat.java | 5 +- 12 files changed, 225 insertions(+), 13 deletions(-) create mode 100644 core/src/main/java/org/apache/stormcrawler/util/HttpHeaderResolver.java create mode 100644 core/src/test/java/org/apache/stormcrawler/util/HttpHeaderResolverTest.java diff --git a/core/src/main/java/org/apache/stormcrawler/bolt/FeedParserBolt.java b/core/src/main/java/org/apache/stormcrawler/bolt/FeedParserBolt.java index 1591c14d9..20311c466 100644 --- a/core/src/main/java/org/apache/stormcrawler/bolt/FeedParserBolt.java +++ b/core/src/main/java/org/apache/stormcrawler/bolt/FeedParserBolt.java @@ -52,6 +52,7 @@ import org.apache.stormcrawler.persistence.Status; import org.apache.stormcrawler.protocol.ProtocolResponse; import org.apache.stormcrawler.util.ConfUtils; +import org.apache.stormcrawler.util.HttpHeaderResolver; import org.apache.stormcrawler.util.URLUtil; import org.slf4j.LoggerFactory; import org.xml.sax.InputSource; @@ -86,7 +87,8 @@ public void execute(Tuple tuple) { // won't work when servers return text/xml // TODO: use Tika instead? String ct = - metadata.getFirstValue(HttpHeaders.CONTENT_TYPE, protocolMetadataPrefix); + HttpHeaderResolver.getFirstValue( + metadata, HttpHeaders.CONTENT_TYPE, protocolMetadataPrefix); if (ct != null && ct.contains("rss+xml")) { isfeed = true; } else { diff --git a/core/src/main/java/org/apache/stormcrawler/bolt/FetcherBolt.java b/core/src/main/java/org/apache/stormcrawler/bolt/FetcherBolt.java index ac141a4ad..ef06aa924 100644 --- a/core/src/main/java/org/apache/stormcrawler/bolt/FetcherBolt.java +++ b/core/src/main/java/org/apache/stormcrawler/bolt/FetcherBolt.java @@ -67,6 +67,7 @@ import org.apache.stormcrawler.protocol.ProtocolResponse; import org.apache.stormcrawler.protocol.RobotRules; import org.apache.stormcrawler.util.ConfUtils; +import org.apache.stormcrawler.util.HttpHeaderResolver; import org.apache.stormcrawler.util.URLUtil; import org.slf4j.LoggerFactory; @@ -843,7 +844,8 @@ public void run() { // find the URL it redirects to String redirection = - response.getMetadata().getFirstValue(HttpHeaders.LOCATION); + HttpHeaderResolver.getFirstValue( + response.getMetadata(), HttpHeaders.LOCATION); // stores the URL it redirects to // used for debugging mainly - do not resolve the target diff --git a/core/src/main/java/org/apache/stormcrawler/bolt/JSoupParserBolt.java b/core/src/main/java/org/apache/stormcrawler/bolt/JSoupParserBolt.java index 1b84a4652..3434ba868 100644 --- a/core/src/main/java/org/apache/stormcrawler/bolt/JSoupParserBolt.java +++ b/core/src/main/java/org/apache/stormcrawler/bolt/JSoupParserBolt.java @@ -59,6 +59,7 @@ import org.apache.stormcrawler.protocol.ProtocolResponse; import org.apache.stormcrawler.util.CharsetIdentification; import org.apache.stormcrawler.util.ConfUtils; +import org.apache.stormcrawler.util.HttpHeaderResolver; import org.apache.stormcrawler.util.RefreshTag; import org.apache.stormcrawler.util.RobotsTags; import org.apache.stormcrawler.util.URLUtil; @@ -227,7 +228,8 @@ public void execute(Tuple tuple) { boolean isPlainText = false; String mimeType = - metadata.getFirstValue(HttpHeaders.CONTENT_TYPE, this.protocolMetadataPrefix); + HttpHeaderResolver.getFirstValue( + metadata, HttpHeaders.CONTENT_TYPE, this.protocolMetadataPrefix); if (detectMimeType) { try { diff --git a/core/src/main/java/org/apache/stormcrawler/bolt/SimpleFetcherBolt.java b/core/src/main/java/org/apache/stormcrawler/bolt/SimpleFetcherBolt.java index 6c2dae6e2..35ee7f6a1 100644 --- a/core/src/main/java/org/apache/stormcrawler/bolt/SimpleFetcherBolt.java +++ b/core/src/main/java/org/apache/stormcrawler/bolt/SimpleFetcherBolt.java @@ -57,6 +57,7 @@ import org.apache.stormcrawler.protocol.ProtocolResponse; import org.apache.stormcrawler.protocol.RobotRules; import org.apache.stormcrawler.util.ConfUtils; +import org.apache.stormcrawler.util.HttpHeaderResolver; import org.apache.stormcrawler.util.URLUtil; import org.slf4j.LoggerFactory; @@ -546,7 +547,9 @@ public void execute(Tuple input) { } else if (status.equals(Status.REDIRECTION)) { // find the URL it redirects to - String redirection = response.getMetadata().getFirstValue(HttpHeaders.LOCATION); + String redirection = + HttpHeaderResolver.getFirstValue( + response.getMetadata(), HttpHeaders.LOCATION); // stores the URL it redirects to // used for debugging mainly - do not resolve the target diff --git a/core/src/main/java/org/apache/stormcrawler/bolt/SiteMapParserBolt.java b/core/src/main/java/org/apache/stormcrawler/bolt/SiteMapParserBolt.java index 66bea2afe..0b5c838fc 100644 --- a/core/src/main/java/org/apache/stormcrawler/bolt/SiteMapParserBolt.java +++ b/core/src/main/java/org/apache/stormcrawler/bolt/SiteMapParserBolt.java @@ -61,6 +61,7 @@ import org.apache.stormcrawler.persistence.DefaultScheduler; import org.apache.stormcrawler.persistence.Status; import org.apache.stormcrawler.util.ConfUtils; +import org.apache.stormcrawler.util.HttpHeaderResolver; import org.apache.stormcrawler.util.URLUtil; import org.slf4j.LoggerFactory; @@ -99,7 +100,7 @@ public void execute(Tuple tuple) { byte[] content = tuple.getBinaryByField("content"); String url = tuple.getStringByField("url"); - String ct = metadata.getFirstValue(HttpHeaders.CONTENT_TYPE); + String ct = HttpHeaderResolver.getFirstValue(metadata, HttpHeaders.CONTENT_TYPE); LOG.debug("Processing {}", url); diff --git a/core/src/main/java/org/apache/stormcrawler/protocol/HttpRobotRulesParser.java b/core/src/main/java/org/apache/stormcrawler/protocol/HttpRobotRulesParser.java index a24afe62b..eba166a06 100644 --- a/core/src/main/java/org/apache/stormcrawler/protocol/HttpRobotRulesParser.java +++ b/core/src/main/java/org/apache/stormcrawler/protocol/HttpRobotRulesParser.java @@ -32,6 +32,7 @@ import org.apache.storm.Config; import org.apache.stormcrawler.Metadata; import org.apache.stormcrawler.util.ConfUtils; +import org.apache.stormcrawler.util.HttpHeaderResolver; import org.apache.stormcrawler.util.URLUtil; /** @@ -247,7 +248,9 @@ public BaseRobotRules getRobotRulesSet(Protocol http, URL url) { while ((code == 301 || code == 302 || code == 303 || code == 307 || code == 308) && numRedirects < MAX_NUM_REDIRECTS) { numRedirects++; - String redirection = response.getMetadata().getFirstValue(HttpHeaders.LOCATION); + String redirection = + HttpHeaderResolver.getFirstValue( + response.getMetadata(), HttpHeaders.LOCATION); LOG.debug("Redirected from {} to {}", redir, redirection); if (StringUtils.isNotBlank(redirection)) { URL target = URLUtil.resolveUrl(redir, redirection); @@ -312,7 +315,9 @@ public BaseRobotRules getRobotRulesSet(Protocol http, URL url) { // Parsing found rules according to RFC 9309 if (code == 200) { // Only if the status code 200 is returned, the rules are parsed - String ct = response.getMetadata().getFirstValue(HttpHeaders.CONTENT_TYPE); + String ct = + HttpHeaderResolver.getFirstValue( + response.getMetadata(), HttpHeaders.CONTENT_TYPE); robotRules = parseRules(url.toString(), response.getContent(), ct, agentNames); } else if (code == 403 && !allowForbidden) { // If the fetch of the robots.txt file is forbidden, then forbid also the fetch diff --git a/core/src/main/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocol.java b/core/src/main/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocol.java index 90cb4d742..44ba80051 100644 --- a/core/src/main/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocol.java +++ b/core/src/main/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocol.java @@ -81,6 +81,7 @@ import org.apache.stormcrawler.proxy.SCProxy; import org.apache.stormcrawler.util.ConfUtils; import org.apache.stormcrawler.util.CookieConverter; +import org.apache.stormcrawler.util.HttpHeaderResolver; import org.apache.stormcrawler.util.URLUtil; import org.jetbrains.annotations.NotNull; import org.slf4j.LoggerFactory; @@ -418,13 +419,15 @@ public ProtocolResponse getProtocolOutput(String url, final Metadata metadata) if (metadata != null) { addHeadersToRequest(rb, metadata); - final String lastModified = metadata.getFirstValue(HttpHeaders.LAST_MODIFIED); + final String lastModified = + HttpHeaderResolver.getFirstValue(metadata, HttpHeaders.LAST_MODIFIED); if (StringUtils.isNotBlank(lastModified)) { rb.header(HttpHeaders.IF_MODIFIED_SINCE, formatHttpDate(lastModified)); } final String ifNoneMatch = - metadata.getFirstValue(HttpHeaders.ETAG, protocolMetadataPrefix); + HttpHeaderResolver.getFirstValue( + metadata, HttpHeaders.ETAG, protocolMetadataPrefix); if (StringUtils.isNotBlank(ifNoneMatch)) { rb.header(HttpHeaders.IF_NONE_MATCH, ifNoneMatch); } diff --git a/core/src/main/java/org/apache/stormcrawler/util/CharsetIdentification.java b/core/src/main/java/org/apache/stormcrawler/util/CharsetIdentification.java index c9ebf21d5..8de90e74f 100644 --- a/core/src/main/java/org/apache/stormcrawler/util/CharsetIdentification.java +++ b/core/src/main/java/org/apache/stormcrawler/util/CharsetIdentification.java @@ -128,7 +128,8 @@ public static String getCharset( /** Returns the charset declared by the server if any. */ private static String getCharsetFromHTTP(Metadata metadata) { - return getCharsetFromContentType(metadata.getFirstValue(HttpHeaders.CONTENT_TYPE)); + return getCharsetFromContentType( + HttpHeaderResolver.getFirstValue(metadata, HttpHeaders.CONTENT_TYPE)); } /** Detects any BOMs and returns the corresponding charset. */ diff --git a/core/src/main/java/org/apache/stormcrawler/util/HttpHeaderResolver.java b/core/src/main/java/org/apache/stormcrawler/util/HttpHeaderResolver.java new file mode 100644 index 000000000..0861a5932 --- /dev/null +++ b/core/src/main/java/org/apache/stormcrawler/util/HttpHeaderResolver.java @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.stormcrawler.util; + +import java.util.Locale; +import org.apache.stormcrawler.Metadata; + +/** Resolves separator variations of an expected HTTP response header. */ +public final class HttpHeaderResolver { + + private HttpHeaderResolver() {} + + /** + * Returns the first value for an expected HTTP header. + * + *

The exact, case-insensitive metadata key is checked first. Only when it is absent are + * separator variations considered, for example {@code ContentType} or {@code content_type} for + * {@code Content-Type}. This lookup is non-destructive and intentionally does not use fuzzy + * matching, so unrelated extension headers such as {@code X-Location} are not treated as + * {@code Location}. + * + * @param metadata metadata containing HTTP response headers + * @param headerName expected HTTP header name + * @return the first value, or {@code null} if the header or an unambiguous alias is absent + */ + public static String getFirstValue(Metadata metadata, String headerName) { + return getFirstValue(metadata, headerName, null); + } + + /** + * Returns the first value for an expected HTTP header stored with a metadata prefix. + * + * @param metadata metadata containing HTTP response headers + * @param headerName expected HTTP header name + * @param prefix optional metadata key prefix + * @return the first value, or {@code null} if the header or an unambiguous alias is absent + */ + public static String getFirstValue(Metadata metadata, String headerName, String prefix) { + if (metadata == null || headerName == null || headerName.isEmpty()) { + return null; + } + + String normalizedPrefix = prefix == null ? "" : prefix.toLowerCase(Locale.ROOT); + String exactKey = normalizedPrefix + headerName; + if (metadata.containsKey(exactKey)) { + return metadata.getFirstValue(exactKey); + } + + String normalizedHeaderName = normalizeSeparators(headerName); + String matchingKey = null; + for (String key : metadata.keySet(normalizedPrefix)) { + String candidate = key.substring(normalizedPrefix.length()); + if (!normalizeSeparators(candidate).equals(normalizedHeaderName)) { + continue; + } + if (matchingKey != null) { + // Multiple aliases are ambiguous. Do not let map iteration order choose a value. + return null; + } + matchingKey = key; + } + return matchingKey == null ? null : metadata.getFirstValue(matchingKey); + } + + private static String normalizeSeparators(String value) { + StringBuilder normalized = new StringBuilder(value.length()); + for (int i = 0; i < value.length(); i++) { + char c = value.charAt(i); + if (c == '-' || c == '_') { + continue; + } + normalized.append(Character.toLowerCase(c)); + } + return normalized.toString(); + } +} diff --git a/core/src/test/java/org/apache/stormcrawler/util/HttpHeaderResolverTest.java b/core/src/test/java/org/apache/stormcrawler/util/HttpHeaderResolverTest.java new file mode 100644 index 000000000..78835a83f --- /dev/null +++ b/core/src/test/java/org/apache/stormcrawler/util/HttpHeaderResolverTest.java @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.stormcrawler.util; + +import org.apache.http.HttpHeaders; +import org.apache.stormcrawler.Metadata; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +class HttpHeaderResolverTest { + + @Test + void testExactHeaderTakesPrecedence() { + Metadata metadata = new Metadata(); + metadata.setValue("Location", "https://example.com/exact"); + metadata.setValue("Location_", "https://example.com/alias"); + + Assertions.assertEquals( + "https://example.com/exact", + HttpHeaderResolver.getFirstValue(metadata, HttpHeaders.LOCATION)); + } + + @Test + void testSeparatorAliasesAreResolved() { + Metadata metadata = new Metadata(); + metadata.setValue("ContentType", "text/html"); + + Assertions.assertEquals( + "text/html", HttpHeaderResolver.getFirstValue(metadata, HttpHeaders.CONTENT_TYPE)); + + metadata = new Metadata(); + metadata.setValue("content_type", "application/json"); + + Assertions.assertEquals( + "application/json", + HttpHeaderResolver.getFirstValue(metadata, HttpHeaders.CONTENT_TYPE)); + } + + @Test + void testExtensionHeaderIsNotTreatedAsAlias() { + Metadata metadata = new Metadata(); + metadata.setValue("X-Location", "cached"); + + Assertions.assertNull( + HttpHeaderResolver.getFirstValue(metadata, HttpHeaders.LOCATION)); + Assertions.assertEquals("cached", metadata.getFirstValue("X-Location")); + } + + @Test + void testMisspellingIsNotFuzzyMatched() { + Metadata metadata = new Metadata(); + metadata.setValue("ConTnTtYpe", "text/html"); + + Assertions.assertNull( + HttpHeaderResolver.getFirstValue(metadata, HttpHeaders.CONTENT_TYPE)); + Assertions.assertEquals("text/html", metadata.getFirstValue("ConTnTtYpe")); + } + + @Test + void testAmbiguousAliasesAreIgnored() { + Metadata metadata = new Metadata(); + metadata.setValue("ContentType", "text/html"); + metadata.setValue("Content_Type", "application/json"); + + Assertions.assertNull( + HttpHeaderResolver.getFirstValue(metadata, HttpHeaders.CONTENT_TYPE)); + } + + @Test + void testPrefixedHeaderAliasIsResolved() { + Metadata metadata = new Metadata(); + metadata.setValue("http.ContentType", "text/html"); + + Assertions.assertEquals( + "text/html", + HttpHeaderResolver.getFirstValue( + metadata, HttpHeaders.CONTENT_TYPE, "http.")); + } +} diff --git a/external/tika/src/main/java/org/apache/stormcrawler/tika/ParserBolt.java b/external/tika/src/main/java/org/apache/stormcrawler/tika/ParserBolt.java index 3d814138e..b27096485 100644 --- a/external/tika/src/main/java/org/apache/stormcrawler/tika/ParserBolt.java +++ b/external/tika/src/main/java/org/apache/stormcrawler/tika/ParserBolt.java @@ -52,6 +52,7 @@ import org.apache.stormcrawler.persistence.Status; import org.apache.stormcrawler.protocol.ProtocolResponse; import org.apache.stormcrawler.util.ConfUtils; +import org.apache.stormcrawler.util.HttpHeaderResolver; import org.apache.stormcrawler.util.InitialisationUtil; import org.apache.stormcrawler.util.MetadataTransfer; import org.apache.stormcrawler.util.URLUtil; @@ -166,7 +167,9 @@ public void execute(Tuple tuple) { String mimeType = metadata.getFirstValue("parse.Content-Type"); // otherwise rely on what could have been obtained from HTTP if (mimeType == null) { - mimeType = metadata.getFirstValue(HttpHeaders.CONTENT_TYPE, this.protocolMDprefix); + mimeType = + HttpHeaderResolver.getFirstValue( + metadata, HttpHeaders.CONTENT_TYPE, this.protocolMDprefix); } if (mimeType != null) { for (Pattern mt : mimeTypeWhiteList) { @@ -198,7 +201,9 @@ public void execute(Tuple tuple) { org.apache.tika.metadata.Metadata md = new org.apache.tika.metadata.Metadata(); // provide the mime-type as a clue for guessing - String httpCT = metadata.getFirstValue(HttpHeaders.CONTENT_TYPE, this.protocolMDprefix); + String httpCT = + HttpHeaderResolver.getFirstValue( + metadata, HttpHeaders.CONTENT_TYPE, this.protocolMDprefix); if (StringUtils.isNotBlank(httpCT)) { // pass content type from server as a clue md.set(org.apache.tika.metadata.Metadata.CONTENT_TYPE, httpCT); diff --git a/external/warc/src/main/java/org/apache/stormcrawler/warc/WARCRecordFormat.java b/external/warc/src/main/java/org/apache/stormcrawler/warc/WARCRecordFormat.java index ad8f2d46a..1ed4dd1e2 100644 --- a/external/warc/src/main/java/org/apache/stormcrawler/warc/WARCRecordFormat.java +++ b/external/warc/src/main/java/org/apache/stormcrawler/warc/WARCRecordFormat.java @@ -44,6 +44,7 @@ import org.apache.storm.tuple.Tuple; import org.apache.stormcrawler.Metadata; import org.apache.stormcrawler.protocol.ProtocolResponse; +import org.apache.stormcrawler.util.HttpHeaderResolver; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -477,7 +478,9 @@ public byte[] format(Tuple tuple) { buffer.append("Content-Type: application/http; msgtype=response").append(CRLF); } else { // for resources just use the content type provided by the server if any - String ct = metadata.getFirstValue(HttpHeaders.CONTENT_TYPE, this.protocolMDprefix); + String ct = + HttpHeaderResolver.getFirstValue( + metadata, HttpHeaders.CONTENT_TYPE, this.protocolMDprefix); if (StringUtils.isBlank(ct)) { ct = "application/octet-stream"; }