diff --git a/core/src/main/java/org/apache/stormcrawler/bolt/FetcherBolt.java b/core/src/main/java/org/apache/stormcrawler/bolt/FetcherBolt.java index ac141a4ad..c52688a1e 100644 --- a/core/src/main/java/org/apache/stormcrawler/bolt/FetcherBolt.java +++ b/core/src/main/java/org/apache/stormcrawler/bolt/FetcherBolt.java @@ -186,7 +186,7 @@ public static FetchItem create(URL u, String url, Tuple t, String queueMode) { key = u.getHost(); } } else { - key = u.getHost(); + key = URLUtil.getCanonicalHost(u); } if (key == null) { diff --git a/core/src/main/java/org/apache/stormcrawler/bolt/SimpleFetcherBolt.java b/core/src/main/java/org/apache/stormcrawler/bolt/SimpleFetcherBolt.java index 6c2dae6e2..be3d9590e 100644 --- a/core/src/main/java/org/apache/stormcrawler/bolt/SimpleFetcherBolt.java +++ b/core/src/main/java/org/apache/stormcrawler/bolt/SimpleFetcherBolt.java @@ -628,7 +628,7 @@ private String getPolitenessKey(URL u) { key = u.getHost(); } } else { - key = u.getHost(); + key = URLUtil.getCanonicalHost(u); if (key == null) { LOG.warn("Unknown host for url: {}, using URL string as key", u.toExternalForm()); key = u.toExternalForm(); diff --git a/core/src/main/java/org/apache/stormcrawler/protocol/HttpRobotRulesParser.java b/core/src/main/java/org/apache/stormcrawler/protocol/HttpRobotRulesParser.java index a24afe62b..a34b6c4b2 100644 --- a/core/src/main/java/org/apache/stormcrawler/protocol/HttpRobotRulesParser.java +++ b/core/src/main/java/org/apache/stormcrawler/protocol/HttpRobotRulesParser.java @@ -103,7 +103,12 @@ private static void logForwardedRequestHeaders(Config conf) { /** Compose unique key to store and access robot rules in cache for given URL. */ protected static String getCacheKey(URL url) { String protocol = url.getProtocol().toLowerCase(Locale.ROOT); - String host = url.getHost().toLowerCase(Locale.ROOT); + // canonicalise the host so aliases of one server (percent-escaping, + // case, trailing dot) share one cache entry and one robots.txt fetch + String host = URLUtil.getCanonicalHost(url); + if (host == null) { + host = ""; + } int port = url.getPort(); if (port == -1) { diff --git a/core/src/main/java/org/apache/stormcrawler/util/URLUtil.java b/core/src/main/java/org/apache/stormcrawler/util/URLUtil.java index e55e7cb1a..3d4e28405 100644 --- a/core/src/main/java/org/apache/stormcrawler/util/URLUtil.java +++ b/core/src/main/java/org/apache/stormcrawler/util/URLUtil.java @@ -22,6 +22,7 @@ import java.net.URI; import java.net.URISyntaxException; import java.net.URL; +import java.net.URLDecoder; import java.nio.charset.StandardCharsets; import java.util.Locale; import java.util.regex.Matcher; @@ -253,6 +254,31 @@ public static String getHost(String url) { } } + /** + * Returns the host in the form the HTTP client will connect to it: percent-escapes decoded, + * lowercased and without a trailing dot. Host strings which only differ in escaping or case + * reach the same server, so politeness queues and robots.txt caches must key on the same + * value, otherwise one server is fetched under several queue ids and its robots.txt is + * downloaded once per spelling. + * + * @param url The url to check. + * @return String The canonical host for the url, or null if the url is not well formed or has + * no host. + */ + public static String getCanonicalHost(URL url) { + String host = url.getHost(); + if (host == null) { + return null; + } + // okhttp percent-decodes the host when it parses the URL; do the same + // so keys derived from the URL string agree with what it connects to + String decoded = URLDecoder.decode(host, StandardCharsets.UTF_8); + if (decoded.endsWith(".")) { + decoded = decoded.substring(0, decoded.length() - 1); + } + return decoded.toLowerCase(Locale.ROOT); + } + /** * Returns the page for the url. The page consists of the protocol, host, and path, but does not * include the query string. The host is lowercased but the path is not. diff --git a/core/src/test/java/org/apache/stormcrawler/protocol/HostAliasCacheKeyTest.java b/core/src/test/java/org/apache/stormcrawler/protocol/HostAliasCacheKeyTest.java new file mode 100644 index 000000000..6cf793587 --- /dev/null +++ b/core/src/test/java/org/apache/stormcrawler/protocol/HostAliasCacheKeyTest.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.stormcrawler.protocol; + +import java.net.URL; +import okhttp3.HttpUrl; +import org.apache.stormcrawler.util.URLUtil; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +/** + * Two URLs whose host strings differ only by percent-escaping or by a trailing dot are sent to the + * same server by okhttp, so they must share one robots.txt cache entry and one politeness queue. + */ +class HostAliasCacheKeyTest { + + @Test + void okhttpCollapsesHostAliases() { + // what the client actually connects to; okhttp percent-decodes and + // lowercases the host but keeps a trailing dot (as the JDK does) + Assertions.assertEquals( + "example.org", HttpUrl.parse("http://%65xample.org/a").host(), "percent-escaped"); + Assertions.assertEquals( + "example.org", HttpUrl.parse("http://exampl%65.org/a").host(), "percent-escaped"); + Assertions.assertEquals( + "example.org", HttpUrl.parse("http://EXAMPLE.org/a").host(), "upper case"); + Assertions.assertEquals( + "example.org.", HttpUrl.parse("http://example.org./a").host(), "trailing dot"); + } + + @Test + void canonicalHostMatchesWhatOkHttpConnectsTo() throws Exception { + Assertions.assertEquals( + HttpUrl.parse("http://exampl%65.org/a").host(), + URLUtil.getCanonicalHost(new URL("http://exampl%65.org/a"))); + Assertions.assertEquals( + "example.org", URLUtil.getCanonicalHost(new URL("http://example.org./a"))); + Assertions.assertEquals( + "example.org", URLUtil.getCanonicalHost(new URL("http://EXAMPLE.org/a"))); + } + + @Test + void robotsCacheKeyIsTheSameForHostAliases() throws Exception { + String canonical = HttpRobotRulesParser.getCacheKey(new URL("http://example.org/a")); + Assertions.assertEquals( + canonical, HttpRobotRulesParser.getCacheKey(new URL("http://exampl%65.org/a"))); + Assertions.assertEquals( + canonical, HttpRobotRulesParser.getCacheKey(new URL("http://example.org./a"))); + Assertions.assertEquals( + canonical, HttpRobotRulesParser.getCacheKey(new URL("http://EXAMPLE.org/a"))); + } +}