Robots.txt fetch no longer replaces the configured content limit

core/src/main/java/org/apache/stormcrawler/protocol/HttpRobotRulesParser.java

@@ -64,9 +64,16 @@ public void setConf(Config conf) {
         super.setConf(conf);
         allowForbidden = ConfUtils.getBoolean(conf, "http.robots.403.allow", true);
         fetchRobotsMd = new Metadata();
-        /* http.content.limit for fetching the robots.txt */
+        /*
+         * http.content.limit for fetching the robots.txt. The default of -1
+         * means "same as http.content.limit": writing the key into the fetch
+         * metadata would override the global limit of the protocol with "no
+         * limit", so it is only set when a robots specific limit is configured.
+         */
         int robotsTxtContentLimit = ConfUtils.getInt(conf, "http.robots.content.limit", -1);
-        fetchRobotsMd.addValue("http.content.limit", Integer.toString(robotsTxtContentLimit));
+        if (robotsTxtContentLimit != -1) {
+            fetchRobotsMd.addValue("http.content.limit", Integer.toString(robotsTxtContentLimit));
+        }
         allow5xx = ConfUtils.getBoolean(conf, "http.robots.5xx.allow", false);
         allowCrossOriginRedirects =
                 ConfUtils.getBoolean(conf, "http.robots.redirect.crossorigin.allow", false);

core/src/main/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocol.java

@@ -442,7 +442,15 @@ public ProtocolResponse getProtocolOutput(String url, final Metadata metadata)
             final String pageMaxContentStr = metadata.getFirstValue("http.content.limit");
             if (StringUtils.isNotBlank(pageMaxContentStr)) {
                 try {
-                    pageMaxContent = Integer.parseInt(pageMaxContentStr);
+                    int metadataLimit = Integer.parseInt(pageMaxContentStr);
+                    /*
+                     * per-URL metadata can tighten the limit but not remove it:
+                     * a value of -1 means "no limit" and would turn the finite
+                     * global limit of the operator into an unbounded read
+                     */
+                    if (metadataLimit != -1 || globalMaxContent == -1) {
+                        pageMaxContent = metadataLimit;
+                    }
                 } catch (NumberFormatException e) {
                     LOG.warn("Invalid http.content.limit in metadata: {}", pageMaxContentStr);
                 }

core/src/main/resources/crawler-default.yaml

@@ -201,8 +201,11 @@ config:
   # http.content.limit when fetching the robots.txt
   # (the robots.txt RFC draft requires to fetch and parse at least 500 kiB,
   #  see https://datatracker.ietf.org/doc/html/draft-rep-wg-topic-00#section-2.5)
+  # A value of -1 means "same as http.content.limit": the robots.txt fetch
+  # then uses whatever limit is configured for pages, including "no limit"
+  # when http.content.limit is -1 too.
   # http.robots.content.limit: 524288  # 512 kiB
-  http.robots.content.limit: -1  # default same as http.content.limit
+  http.robots.content.limit: 524288
 
   # Implementation of RobotRulesParser used by the HTTP protocol implementations
   # to fetch and parse robots.txt. Override to plug in custom robots.txt

core/src/test/java/org/apache/stormcrawler/protocol/RobotsContentLimitTest.java

@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.stormcrawler.protocol;
+
+import org.apache.storm.Config;
+import org.apache.stormcrawler.Metadata;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+/** Checks the content limit used for the robots.txt fetch. */
+class RobotsContentLimitTest {
+
+    /** Protocol stub recording the metadata it is called with. */
+    private static class RecordingProtocol implements Protocol {
+
+        Metadata seen;
+
+        @Override
+        public void configure(Config conf) {}
+
+        @Override
+        public ProtocolResponse getProtocolOutput(String url, Metadata metadata) {
+            seen = metadata;
+            return new ProtocolResponse(new byte[0], 200, new Metadata());
+        }
+
+        @Override
+        public crawlercommons.robots.BaseRobotRules getRobotRules(String url) {
+            return null;
+        }
+
+        @Override
+        public void cleanup() {}
+    }
+
+    @Test
+    void robotsFetchKeepsTheGlobalContentLimit() {
+        Config conf = new Config();
+        conf.put("http.agent.name", "this_is_only_a_test");
+        // operator sets a finite limit and does not touch http.robots.content.limit
+        conf.put("http.content.limit", 65536);
+
+        RecordingProtocol protocol = new RecordingProtocol();
+        HttpRobotRulesParser parser = new HttpRobotRulesParser();
+        parser.setConf(conf);
+        parser.getRobotRulesSet(protocol, "http://limit.example.org/");
+
+        String limit = protocol.seen.getFirstValue("http.content.limit");
+        Assertions.assertNull(
+                limit,
+                "the robots.txt fetch should not override the global content limit, but saw: "
+                        + limit);
+    }
+
+    @Test
+    void robotsSpecificLimitIsApplied() {
+        Config conf = new Config();
+        conf.put("http.agent.name", "this_is_only_a_test");
+        conf.put("http.content.limit", 65536);
+        conf.put("http.robots.content.limit", 524288);
+
+        RecordingProtocol protocol = new RecordingProtocol();
+        HttpRobotRulesParser parser = new HttpRobotRulesParser();
+        parser.setConf(conf);
+        parser.getRobotRulesSet(protocol, "http://limit.example.org/");
+
+        Assertions.assertEquals("524288", protocol.seen.getFirstValue("http.content.limit"));
+    }
+}