apache · HarshDevelops · Sep 3, 2026
diff --git a/core/src/main/java/org/apache/stormcrawler/bolt/FeedParserBolt.java b/core/src/main/java/org/apache/stormcrawler/bolt/FeedParserBolt.java
@@ -52,6 +52,7 @@
 import org.apache.stormcrawler.persistence.Status;
 import org.apache.stormcrawler.protocol.ProtocolResponse;
 import org.apache.stormcrawler.util.ConfUtils;
+import org.apache.stormcrawler.util.HttpHeaderResolver;
 import org.apache.stormcrawler.util.URLUtil;
 import org.slf4j.LoggerFactory;
 import org.xml.sax.InputSource;
@@ -86,7 +87,8 @@ public void execute(Tuple tuple) {
                 // won't work when servers return text/xml
                 // TODO: use Tika instead?
                 String ct =
-                        metadata.getFirstValue(HttpHeaders.CONTENT_TYPE, protocolMetadataPrefix);
+                        HttpHeaderResolver.getFirstValue(
+                                metadata, HttpHeaders.CONTENT_TYPE, protocolMetadataPrefix);
                 if (ct != null && ct.contains("rss+xml")) {
                     isfeed = true;
                 } else {

diff --git a/core/src/main/java/org/apache/stormcrawler/bolt/FetcherBolt.java b/core/src/main/java/org/apache/stormcrawler/bolt/FetcherBolt.java
@@ -67,6 +67,7 @@
 import org.apache.stormcrawler.protocol.ProtocolResponse;
 import org.apache.stormcrawler.protocol.RobotRules;
 import org.apache.stormcrawler.util.ConfUtils;
+import org.apache.stormcrawler.util.HttpHeaderResolver;
 import org.apache.stormcrawler.util.URLUtil;
 import org.slf4j.LoggerFactory;
 
@@ -843,7 +844,8 @@ public void run() {
 
                         // find the URL it redirects to
                         String redirection =
-                                response.getMetadata().getFirstValue(HttpHeaders.LOCATION);
+                                HttpHeaderResolver.getFirstValue(
+                                        response.getMetadata(), HttpHeaders.LOCATION);
 
                         // stores the URL it redirects to
                         // used for debugging mainly - do not resolve the target

diff --git a/core/src/main/java/org/apache/stormcrawler/bolt/JSoupParserBolt.java b/core/src/main/java/org/apache/stormcrawler/bolt/JSoupParserBolt.java
@@ -59,6 +59,7 @@
 import org.apache.stormcrawler.protocol.ProtocolResponse;
 import org.apache.stormcrawler.util.CharsetIdentification;
 import org.apache.stormcrawler.util.ConfUtils;
+import org.apache.stormcrawler.util.HttpHeaderResolver;
 import org.apache.stormcrawler.util.RefreshTag;
 import org.apache.stormcrawler.util.RobotsTags;
 import org.apache.stormcrawler.util.URLUtil;
@@ -227,7 +228,8 @@ public void execute(Tuple tuple) {
         boolean isPlainText = false;
 
         String mimeType =
-                metadata.getFirstValue(HttpHeaders.CONTENT_TYPE, this.protocolMetadataPrefix);
+                HttpHeaderResolver.getFirstValue(
+                        metadata, HttpHeaders.CONTENT_TYPE, this.protocolMetadataPrefix);
 
         if (detectMimeType) {
             try {

diff --git a/core/src/main/java/org/apache/stormcrawler/bolt/SimpleFetcherBolt.java b/core/src/main/java/org/apache/stormcrawler/bolt/SimpleFetcherBolt.java
@@ -57,6 +57,7 @@
 import org.apache.stormcrawler.protocol.ProtocolResponse;
 import org.apache.stormcrawler.protocol.RobotRules;
 import org.apache.stormcrawler.util.ConfUtils;
+import org.apache.stormcrawler.util.HttpHeaderResolver;
 import org.apache.stormcrawler.util.URLUtil;
 import org.slf4j.LoggerFactory;
 
@@ -546,7 +547,9 @@ public void execute(Tuple input) {
             } else if (status.equals(Status.REDIRECTION)) {
 
                 // find the URL it redirects to
-                String redirection = response.getMetadata().getFirstValue(HttpHeaders.LOCATION);
+                String redirection =
+                        HttpHeaderResolver.getFirstValue(
+                                response.getMetadata(), HttpHeaders.LOCATION);
 
                 // stores the URL it redirects to
                 // used for debugging mainly - do not resolve the target

diff --git a/core/src/main/java/org/apache/stormcrawler/bolt/SiteMapParserBolt.java b/core/src/main/java/org/apache/stormcrawler/bolt/SiteMapParserBolt.java
@@ -61,6 +61,7 @@
 import org.apache.stormcrawler.persistence.DefaultScheduler;
 import org.apache.stormcrawler.persistence.Status;
 import org.apache.stormcrawler.util.ConfUtils;
+import org.apache.stormcrawler.util.HttpHeaderResolver;
 import org.apache.stormcrawler.util.URLUtil;
 import org.slf4j.LoggerFactory;
 
@@ -99,7 +100,7 @@ public void execute(Tuple tuple) {
         byte[] content = tuple.getBinaryByField("content");
         String url = tuple.getStringByField("url");
 
-        String ct = metadata.getFirstValue(HttpHeaders.CONTENT_TYPE);
+        String ct = HttpHeaderResolver.getFirstValue(metadata, HttpHeaders.CONTENT_TYPE);
 
         LOG.debug("Processing {}", url);
 

diff --git a/core/src/main/java/org/apache/stormcrawler/protocol/HttpRobotRulesParser.java b/core/src/main/java/org/apache/stormcrawler/protocol/HttpRobotRulesParser.java
@@ -32,6 +32,7 @@
 import org.apache.storm.Config;
 import org.apache.stormcrawler.Metadata;
 import org.apache.stormcrawler.util.ConfUtils;
+import org.apache.stormcrawler.util.HttpHeaderResolver;
 import org.apache.stormcrawler.util.URLUtil;
 
 /**
@@ -247,7 +248,9 @@ public BaseRobotRules getRobotRulesSet(Protocol http, URL url) {
             while ((code == 301 || code == 302 || code == 303 || code == 307 || code == 308)
                     && numRedirects < MAX_NUM_REDIRECTS) {
                 numRedirects++;
-                String redirection = response.getMetadata().getFirstValue(HttpHeaders.LOCATION);
+                String redirection =
+                        HttpHeaderResolver.getFirstValue(
+                                response.getMetadata(), HttpHeaders.LOCATION);
                 LOG.debug("Redirected from {} to {}", redir, redirection);
                 if (StringUtils.isNotBlank(redirection)) {
                     URL target = URLUtil.resolveUrl(redir, redirection);
@@ -312,7 +315,9 @@ public BaseRobotRules getRobotRulesSet(Protocol http, URL url) {
             // Parsing found rules according to RFC 9309
             if (code == 200) {
                 // Only if the status code 200 is returned, the rules are parsed
-                String ct = response.getMetadata().getFirstValue(HttpHeaders.CONTENT_TYPE);
+                String ct =
+                        HttpHeaderResolver.getFirstValue(
+                                response.getMetadata(), HttpHeaders.CONTENT_TYPE);
                 robotRules = parseRules(url.toString(), response.getContent(), ct, agentNames);
             } else if (code == 403 && !allowForbidden) {
                 // If the fetch of the robots.txt file is forbidden, then forbid also the fetch

diff --git a/core/src/main/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocol.java b/core/src/main/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocol.java
@@ -81,6 +81,7 @@
 import org.apache.stormcrawler.proxy.SCProxy;
 import org.apache.stormcrawler.util.ConfUtils;
 import org.apache.stormcrawler.util.CookieConverter;
+import org.apache.stormcrawler.util.HttpHeaderResolver;
 import org.apache.stormcrawler.util.URLUtil;
 import org.jetbrains.annotations.NotNull;
 import org.slf4j.LoggerFactory;
@@ -418,13 +419,15 @@ public ProtocolResponse getProtocolOutput(String url, final Metadata metadata)
         if (metadata != null) {
             addHeadersToRequest(rb, metadata);
 
-            final String lastModified = metadata.getFirstValue(HttpHeaders.LAST_MODIFIED);
+            final String lastModified =
+                    HttpHeaderResolver.getFirstValue(metadata, HttpHeaders.LAST_MODIFIED);
             if (StringUtils.isNotBlank(lastModified)) {
                 rb.header(HttpHeaders.IF_MODIFIED_SINCE, formatHttpDate(lastModified));
             }
 
             final String ifNoneMatch =
-                    metadata.getFirstValue(HttpHeaders.ETAG, protocolMetadataPrefix);
+                    HttpHeaderResolver.getFirstValue(
+                            metadata, HttpHeaders.ETAG, protocolMetadataPrefix);
             if (StringUtils.isNotBlank(ifNoneMatch)) {
                 rb.header(HttpHeaders.IF_NONE_MATCH, ifNoneMatch);
             }

diff --git a/core/src/main/java/org/apache/stormcrawler/util/CharsetIdentification.java b/core/src/main/java/org/apache/stormcrawler/util/CharsetIdentification.java
@@ -128,7 +128,8 @@ public static String getCharset(
 
     /** Returns the charset declared by the server if any. */
     private static String getCharsetFromHTTP(Metadata metadata) {
-        return getCharsetFromContentType(metadata.getFirstValue(HttpHeaders.CONTENT_TYPE));
+        return getCharsetFromContentType(
+                HttpHeaderResolver.getFirstValue(metadata, HttpHeaders.CONTENT_TYPE));
     }
 
     /** Detects any BOMs and returns the corresponding charset. */

diff --git a/core/src/main/java/org/apache/stormcrawler/util/HttpHeaderResolver.java b/core/src/main/java/org/apache/stormcrawler/util/HttpHeaderResolver.java
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.stormcrawler.util;
+
+import java.util.Locale;
+import org.apache.stormcrawler.Metadata;
+
+/** Resolves separator variations of an expected HTTP response header. */
+public final class HttpHeaderResolver {
+
+    private HttpHeaderResolver() {}
+
+    /**
+     * Returns the first value for an expected HTTP header.
+     *
+     * <p>The exact, case-insensitive metadata key is checked first. Only when it is absent are
+     * separator variations considered, for example {@code ContentType} or {@code content_type} for
+     * {@code Content-Type}. This lookup is non-destructive and intentionally does not use fuzzy
+     * matching, so unrelated extension headers such as {@code X-Location} are not treated as
+     * {@code Location}.
+     *
+     * @param metadata metadata containing HTTP response headers
+     * @param headerName expected HTTP header name
+     * @return the first value, or {@code null} if the header or an unambiguous alias is absent
+     */
+    public static String getFirstValue(Metadata metadata, String headerName) {
+        return getFirstValue(metadata, headerName, null);
+    }
+
+    /**
+     * Returns the first value for an expected HTTP header stored with a metadata prefix.
+     *
+     * @param metadata metadata containing HTTP response headers
+     * @param headerName expected HTTP header name
+     * @param prefix optional metadata key prefix
+     * @return the first value, or {@code null} if the header or an unambiguous alias is absent
+     */
+    public static String getFirstValue(Metadata metadata, String headerName, String prefix) {
+        if (metadata == null || headerName == null || headerName.isEmpty()) {
+            return null;
+        }
+
+        String normalizedPrefix = prefix == null ? "" : prefix.toLowerCase(Locale.ROOT);
+        String exactKey = normalizedPrefix + headerName;
+        if (metadata.containsKey(exactKey)) {
+            return metadata.getFirstValue(exactKey);
+        }
+
+        String normalizedHeaderName = normalizeSeparators(headerName);
+        String matchingKey = null;
+        for (String key : metadata.keySet(normalizedPrefix)) {
+            String candidate = key.substring(normalizedPrefix.length());
+            if (!normalizeSeparators(candidate).equals(normalizedHeaderName)) {
+                continue;
+            }
+            if (matchingKey != null) {
+                // Multiple aliases are ambiguous. Do not let map iteration order choose a value.
+                return null;
+            }
+            matchingKey = key;
+        }
+        return matchingKey == null ? null : metadata.getFirstValue(matchingKey);
+    }
+
+    private static String normalizeSeparators(String value) {
+        StringBuilder normalized = new StringBuilder(value.length());
+        for (int i = 0; i < value.length(); i++) {
+            char c = value.charAt(i);
+            if (c == '-' || c == '_') {
+                continue;
+            }
+            normalized.append(Character.toLowerCase(c));
+        }
+        return normalized.toString();
+    }
+}
diff --git a/core/src/test/java/org/apache/stormcrawler/util/HttpHeaderResolverTest.java b/core/src/test/java/org/apache/stormcrawler/util/HttpHeaderResolverTest.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.stormcrawler.util;
+
+import org.apache.http.HttpHeaders;
+import org.apache.stormcrawler.Metadata;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+class HttpHeaderResolverTest {
+
+    @Test
+    void testExactHeaderTakesPrecedence() {
+        Metadata metadata = new Metadata();
+        metadata.setValue("Location", "https://example.com/exact");
+        metadata.setValue("Location_", "https://example.com/alias");
+
+        Assertions.assertEquals(
+                "https://example.com/exact",
+                HttpHeaderResolver.getFirstValue(metadata, HttpHeaders.LOCATION));
+    }
+
+    @Test
+    void testSeparatorAliasesAreResolved() {
+        Metadata metadata = new Metadata();
+        metadata.setValue("ContentType", "text/html");
+
+        Assertions.assertEquals(
+                "text/html", HttpHeaderResolver.getFirstValue(metadata, HttpHeaders.CONTENT_TYPE));
+
+        metadata = new Metadata();
+        metadata.setValue("content_type", "application/json");
+
+        Assertions.assertEquals(
+                "application/json",
+                HttpHeaderResolver.getFirstValue(metadata, HttpHeaders.CONTENT_TYPE));
+    }
+
+    @Test
+    void testExtensionHeaderIsNotTreatedAsAlias() {
+        Metadata metadata = new Metadata();
+        metadata.setValue("X-Location", "cached");
+
+        Assertions.assertNull(
+                HttpHeaderResolver.getFirstValue(metadata, HttpHeaders.LOCATION));
+        Assertions.assertEquals("cached", metadata.getFirstValue("X-Location"));
+    }
+
+    @Test
+    void testMisspellingIsNotFuzzyMatched() {
+        Metadata metadata = new Metadata();
+        metadata.setValue("ConTnTtYpe", "text/html");
+
+        Assertions.assertNull(
+                HttpHeaderResolver.getFirstValue(metadata, HttpHeaders.CONTENT_TYPE));
+        Assertions.assertEquals("text/html", metadata.getFirstValue("ConTnTtYpe"));
+    }
+
+    @Test
+    void testAmbiguousAliasesAreIgnored() {
+        Metadata metadata = new Metadata();
+        metadata.setValue("ContentType", "text/html");
+        metadata.setValue("Content_Type", "application/json");
+
+        Assertions.assertNull(
+                HttpHeaderResolver.getFirstValue(metadata, HttpHeaders.CONTENT_TYPE));
+    }
+
+    @Test
+    void testPrefixedHeaderAliasIsResolved() {
+        Metadata metadata = new Metadata();
+        metadata.setValue("http.ContentType", "text/html");
+
+        Assertions.assertEquals(
+                "text/html",
+                HttpHeaderResolver.getFirstValue(
+                        metadata, HttpHeaders.CONTENT_TYPE, "http."));
+    }
+}
diff --git a/external/tika/src/main/java/org/apache/stormcrawler/tika/ParserBolt.java b/external/tika/src/main/java/org/apache/stormcrawler/tika/ParserBolt.java
@@ -52,6 +52,7 @@
 import org.apache.stormcrawler.persistence.Status;
 import org.apache.stormcrawler.protocol.ProtocolResponse;
 import org.apache.stormcrawler.util.ConfUtils;
+import org.apache.stormcrawler.util.HttpHeaderResolver;
 import org.apache.stormcrawler.util.InitialisationUtil;
 import org.apache.stormcrawler.util.MetadataTransfer;
 import org.apache.stormcrawler.util.URLUtil;
@@ -166,7 +167,9 @@ public void execute(Tuple tuple) {
             String mimeType = metadata.getFirstValue("parse.Content-Type");
             // otherwise rely on what could have been obtained from HTTP
             if (mimeType == null) {
-                mimeType = metadata.getFirstValue(HttpHeaders.CONTENT_TYPE, this.protocolMDprefix);
+                mimeType =
+                        HttpHeaderResolver.getFirstValue(
+                                metadata, HttpHeaders.CONTENT_TYPE, this.protocolMDprefix);
             }
             if (mimeType != null) {
                 for (Pattern mt : mimeTypeWhiteList) {
@@ -198,7 +201,9 @@ public void execute(Tuple tuple) {
         org.apache.tika.metadata.Metadata md = new org.apache.tika.metadata.Metadata();
 
         // provide the mime-type as a clue for guessing
-        String httpCT = metadata.getFirstValue(HttpHeaders.CONTENT_TYPE, this.protocolMDprefix);
+        String httpCT =
+                HttpHeaderResolver.getFirstValue(
+                        metadata, HttpHeaders.CONTENT_TYPE, this.protocolMDprefix);
         if (StringUtils.isNotBlank(httpCT)) {
             // pass content type from server as a clue
             md.set(org.apache.tika.metadata.Metadata.CONTENT_TYPE, httpCT);

diff --git a/external/warc/src/main/java/org/apache/stormcrawler/warc/WARCRecordFormat.java b/external/warc/src/main/java/org/apache/stormcrawler/warc/WARCRecordFormat.java
@@ -44,6 +44,7 @@
 import org.apache.storm.tuple.Tuple;
 import org.apache.stormcrawler.Metadata;
 import org.apache.stormcrawler.protocol.ProtocolResponse;
+import org.apache.stormcrawler.util.HttpHeaderResolver;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -477,7 +478,9 @@ public byte[] format(Tuple tuple) {
             buffer.append("Content-Type: application/http; msgtype=response").append(CRLF);
         } else {
             // for resources just use the content type provided by the server if any
-            String ct = metadata.getFirstValue(HttpHeaders.CONTENT_TYPE, this.protocolMDprefix);
+            String ct =
+                    HttpHeaderResolver.getFirstValue(
+                            metadata, HttpHeaders.CONTENT_TYPE, this.protocolMDprefix);
             if (StringUtils.isBlank(ct)) {
                 ct = "application/octet-stream";
             }