From df4670521658eb86a658212e2d563253bd2753cd Mon Sep 17 00:00:00 2001 From: abhinav-phi Date: Sun, 6 Sep 2026 04:12:45 +0530 Subject: [PATCH] Canonicalise hosts for politeness queues and the robots.txt cache (#2082) okhttp percent-decodes and lowercases the host when it parses the URL, but FetchItem.create, SimpleFetcherBolt.getPolitenessKey and HttpRobotRulesParser.getCacheKey keyed on the raw host string, so http://exampl%65.org/ and http://example.org/ were one origin at connect time and two queue ids and two robots.txt cache entries in the bolt. One server could end up with several delay clocks running in parallel and its robots.txt fetched once per spelling. URLUtil.getCanonicalHost percent-decodes the host, strips a trailing dot and lowercases it, and all three call sites now use it. --- .../apache/stormcrawler/bolt/FetcherBolt.java | 2 +- .../stormcrawler/bolt/SimpleFetcherBolt.java | 2 +- .../protocol/HttpRobotRulesParser.java | 7 +- .../org/apache/stormcrawler/util/URLUtil.java | 26 +++++++ .../protocol/HostAliasCacheKeyTest.java | 67 +++++++++++++++++++ 5 files changed, 101 insertions(+), 3 deletions(-) create mode 100644 core/src/test/java/org/apache/stormcrawler/protocol/HostAliasCacheKeyTest.java diff --git a/core/src/main/java/org/apache/stormcrawler/bolt/FetcherBolt.java b/core/src/main/java/org/apache/stormcrawler/bolt/FetcherBolt.java index ac141a4ad..c52688a1e 100644 --- a/core/src/main/java/org/apache/stormcrawler/bolt/FetcherBolt.java +++ b/core/src/main/java/org/apache/stormcrawler/bolt/FetcherBolt.java @@ -186,7 +186,7 @@ public static FetchItem create(URL u, String url, Tuple t, String queueMode) { key = u.getHost(); } } else { - key = u.getHost(); + key = URLUtil.getCanonicalHost(u); } if (key == null) { diff --git a/core/src/main/java/org/apache/stormcrawler/bolt/SimpleFetcherBolt.java b/core/src/main/java/org/apache/stormcrawler/bolt/SimpleFetcherBolt.java index 6c2dae6e2..be3d9590e 100644 --- a/core/src/main/java/org/apache/stormcrawler/bolt/SimpleFetcherBolt.java +++ b/core/src/main/java/org/apache/stormcrawler/bolt/SimpleFetcherBolt.java @@ -628,7 +628,7 @@ private String getPolitenessKey(URL u) { key = u.getHost(); } } else { - key = u.getHost(); + key = URLUtil.getCanonicalHost(u); if (key == null) { LOG.warn("Unknown host for url: {}, using URL string as key", u.toExternalForm()); key = u.toExternalForm(); diff --git a/core/src/main/java/org/apache/stormcrawler/protocol/HttpRobotRulesParser.java b/core/src/main/java/org/apache/stormcrawler/protocol/HttpRobotRulesParser.java index a24afe62b..a34b6c4b2 100644 --- a/core/src/main/java/org/apache/stormcrawler/protocol/HttpRobotRulesParser.java +++ b/core/src/main/java/org/apache/stormcrawler/protocol/HttpRobotRulesParser.java @@ -103,7 +103,12 @@ private static void logForwardedRequestHeaders(Config conf) { /** Compose unique key to store and access robot rules in cache for given URL. */ protected static String getCacheKey(URL url) { String protocol = url.getProtocol().toLowerCase(Locale.ROOT); - String host = url.getHost().toLowerCase(Locale.ROOT); + // canonicalise the host so aliases of one server (percent-escaping, + // case, trailing dot) share one cache entry and one robots.txt fetch + String host = URLUtil.getCanonicalHost(url); + if (host == null) { + host = ""; + } int port = url.getPort(); if (port == -1) { diff --git a/core/src/main/java/org/apache/stormcrawler/util/URLUtil.java b/core/src/main/java/org/apache/stormcrawler/util/URLUtil.java index e55e7cb1a..3d4e28405 100644 --- a/core/src/main/java/org/apache/stormcrawler/util/URLUtil.java +++ b/core/src/main/java/org/apache/stormcrawler/util/URLUtil.java @@ -22,6 +22,7 @@ import java.net.URI; import java.net.URISyntaxException; import java.net.URL; +import java.net.URLDecoder; import java.nio.charset.StandardCharsets; import java.util.Locale; import java.util.regex.Matcher; @@ -253,6 +254,31 @@ public static String getHost(String url) { } } + /** + * Returns the host in the form the HTTP client will connect to it: percent-escapes decoded, + * lowercased and without a trailing dot. Host strings which only differ in escaping or case + * reach the same server, so politeness queues and robots.txt caches must key on the same + * value, otherwise one server is fetched under several queue ids and its robots.txt is + * downloaded once per spelling. + * + * @param url The url to check. + * @return String The canonical host for the url, or null if the url is not well formed or has + * no host. + */ + public static String getCanonicalHost(URL url) { + String host = url.getHost(); + if (host == null) { + return null; + } + // okhttp percent-decodes the host when it parses the URL; do the same + // so keys derived from the URL string agree with what it connects to + String decoded = URLDecoder.decode(host, StandardCharsets.UTF_8); + if (decoded.endsWith(".")) { + decoded = decoded.substring(0, decoded.length() - 1); + } + return decoded.toLowerCase(Locale.ROOT); + } + /** * Returns the page for the url. The page consists of the protocol, host, and path, but does not * include the query string. The host is lowercased but the path is not. diff --git a/core/src/test/java/org/apache/stormcrawler/protocol/HostAliasCacheKeyTest.java b/core/src/test/java/org/apache/stormcrawler/protocol/HostAliasCacheKeyTest.java new file mode 100644 index 000000000..6cf793587 --- /dev/null +++ b/core/src/test/java/org/apache/stormcrawler/protocol/HostAliasCacheKeyTest.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.stormcrawler.protocol; + +import java.net.URL; +import okhttp3.HttpUrl; +import org.apache.stormcrawler.util.URLUtil; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +/** + * Two URLs whose host strings differ only by percent-escaping or by a trailing dot are sent to the + * same server by okhttp, so they must share one robots.txt cache entry and one politeness queue. + */ +class HostAliasCacheKeyTest { + + @Test + void okhttpCollapsesHostAliases() { + // what the client actually connects to; okhttp percent-decodes and + // lowercases the host but keeps a trailing dot (as the JDK does) + Assertions.assertEquals( + "example.org", HttpUrl.parse("http://%65xample.org/a").host(), "percent-escaped"); + Assertions.assertEquals( + "example.org", HttpUrl.parse("http://exampl%65.org/a").host(), "percent-escaped"); + Assertions.assertEquals( + "example.org", HttpUrl.parse("http://EXAMPLE.org/a").host(), "upper case"); + Assertions.assertEquals( + "example.org.", HttpUrl.parse("http://example.org./a").host(), "trailing dot"); + } + + @Test + void canonicalHostMatchesWhatOkHttpConnectsTo() throws Exception { + Assertions.assertEquals( + HttpUrl.parse("http://exampl%65.org/a").host(), + URLUtil.getCanonicalHost(new URL("http://exampl%65.org/a"))); + Assertions.assertEquals( + "example.org", URLUtil.getCanonicalHost(new URL("http://example.org./a"))); + Assertions.assertEquals( + "example.org", URLUtil.getCanonicalHost(new URL("http://EXAMPLE.org/a"))); + } + + @Test + void robotsCacheKeyIsTheSameForHostAliases() throws Exception { + String canonical = HttpRobotRulesParser.getCacheKey(new URL("http://example.org/a")); + Assertions.assertEquals( + canonical, HttpRobotRulesParser.getCacheKey(new URL("http://exampl%65.org/a"))); + Assertions.assertEquals( + canonical, HttpRobotRulesParser.getCacheKey(new URL("http://example.org./a"))); + Assertions.assertEquals( + canonical, HttpRobotRulesParser.getCacheKey(new URL("http://EXAMPLE.org/a"))); + } +}