Canonicalise hosts for politeness queues and the robots.txt cache

core/src/main/java/org/apache/stormcrawler/bolt/FetcherBolt.java

@@ -186,7 +186,7 @@ public static FetchItem create(URL u, String url, Tuple t, String queueMode) {
                     key = u.getHost();
                 }
             } else {
-                key = u.getHost();
+                key = URLUtil.getCanonicalHost(u);
             }
 
             if (key == null) {

core/src/main/java/org/apache/stormcrawler/bolt/SimpleFetcherBolt.java

@@ -628,7 +628,7 @@ private String getPolitenessKey(URL u) {
                 key = u.getHost();
             }
         } else {
-            key = u.getHost();
+            key = URLUtil.getCanonicalHost(u);
             if (key == null) {
                 LOG.warn("Unknown host for url: {}, using URL string as key", u.toExternalForm());
                 key = u.toExternalForm();

core/src/main/java/org/apache/stormcrawler/protocol/HttpRobotRulesParser.java

@@ -103,7 +103,12 @@ private static void logForwardedRequestHeaders(Config conf) {
     /** Compose unique key to store and access robot rules in cache for given URL. */
     protected static String getCacheKey(URL url) {
         String protocol = url.getProtocol().toLowerCase(Locale.ROOT);
-        String host = url.getHost().toLowerCase(Locale.ROOT);
+        // canonicalise the host so aliases of one server (percent-escaping,
+        // case, trailing dot) share one cache entry and one robots.txt fetch
+        String host = URLUtil.getCanonicalHost(url);
+        if (host == null) {
+            host = "";
+        }
 
         int port = url.getPort();
         if (port == -1) {

core/src/main/java/org/apache/stormcrawler/util/URLUtil.java

@@ -22,6 +22,7 @@
 import java.net.URI;
 import java.net.URISyntaxException;
 import java.net.URL;
+import java.net.URLDecoder;
 import java.nio.charset.StandardCharsets;
 import java.util.Locale;
 import java.util.regex.Matcher;
@@ -253,6 +254,31 @@ public static String getHost(String url) {
         }
     }
 
+    /**
+     * Returns the host in the form the HTTP client will connect to it: percent-escapes decoded,
+     * lowercased and without a trailing dot. Host strings which only differ in escaping or case
+     * reach the same server, so politeness queues and robots.txt caches must key on the same
+     * value, otherwise one server is fetched under several queue ids and its robots.txt is
+     * downloaded once per spelling.
+     *
+     * @param url The url to check.
+     * @return String The canonical host for the url, or null if the url is not well formed or has
+     *     no host.
+     */
+    public static String getCanonicalHost(URL url) {
+        String host = url.getHost();
+        if (host == null) {
+            return null;
+        }
+        // okhttp percent-decodes the host when it parses the URL; do the same
+        // so keys derived from the URL string agree with what it connects to
+        String decoded = URLDecoder.decode(host, StandardCharsets.UTF_8);
+        if (decoded.endsWith(".")) {
+            decoded = decoded.substring(0, decoded.length() - 1);
+        }
+        return decoded.toLowerCase(Locale.ROOT);
+    }
+
     /**
      * Returns the page for the url. The page consists of the protocol, host, and path, but does not
      * include the query string. The host is lowercased but the path is not.

core/src/test/java/org/apache/stormcrawler/protocol/HostAliasCacheKeyTest.java

@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.stormcrawler.protocol;
+
+import java.net.URL;
+import okhttp3.HttpUrl;
+import org.apache.stormcrawler.util.URLUtil;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+/**
+ * Two URLs whose host strings differ only by percent-escaping or by a trailing dot are sent to the
+ * same server by okhttp, so they must share one robots.txt cache entry and one politeness queue.
+ */
+class HostAliasCacheKeyTest {
+
+    @Test
+    void okhttpCollapsesHostAliases() {
+        // what the client actually connects to; okhttp percent-decodes and
+        // lowercases the host but keeps a trailing dot (as the JDK does)
+        Assertions.assertEquals(
+                "example.org", HttpUrl.parse("http://%65xample.org/a").host(), "percent-escaped");
+        Assertions.assertEquals(
+                "example.org", HttpUrl.parse("http://exampl%65.org/a").host(), "percent-escaped");
+        Assertions.assertEquals(
+                "example.org", HttpUrl.parse("http://EXAMPLE.org/a").host(), "upper case");
+        Assertions.assertEquals(
+                "example.org.", HttpUrl.parse("http://example.org./a").host(), "trailing dot");
+    }
+
+    @Test
+    void canonicalHostMatchesWhatOkHttpConnectsTo() throws Exception {
+        Assertions.assertEquals(
+                HttpUrl.parse("http://exampl%65.org/a").host(),
+                URLUtil.getCanonicalHost(new URL("http://exampl%65.org/a")));
+        Assertions.assertEquals(
+                "example.org", URLUtil.getCanonicalHost(new URL("http://example.org./a")));
+        Assertions.assertEquals(
+                "example.org", URLUtil.getCanonicalHost(new URL("http://EXAMPLE.org/a")));
+    }
+
+    @Test
+    void robotsCacheKeyIsTheSameForHostAliases() throws Exception {
+        String canonical = HttpRobotRulesParser.getCacheKey(new URL("http://example.org/a"));
+        Assertions.assertEquals(
+                canonical, HttpRobotRulesParser.getCacheKey(new URL("http://exampl%65.org/a")));
+        Assertions.assertEquals(
+                canonical, HttpRobotRulesParser.getCacheKey(new URL("http://example.org./a")));
+        Assertions.assertEquals(
+                canonical, HttpRobotRulesParser.getCacheKey(new URL("http://EXAMPLE.org/a")));
+    }
+}