From 4f75e71d93ca15fcbacbb158a0e549c1edda4a20 Mon Sep 17 00:00:00 2001 From: abhinav-phi Date: Sun, 6 Sep 2026 04:07:22 +0530 Subject: [PATCH] Robots.txt fetch no longer removes the configured content limit (#2087) HttpRobotRulesParser wrote http.robots.content.limit (-1 by default) into the fetch metadata as http.content.limit, which HttpProtocol reads unconditionally: a robots.txt fetch ran with no limit at all, even when the operator had configured a finite http.content.limit, despite the shipped comment saying the default was 'same as http.content.limit'. - a http.robots.content.limit of -1 now means 'inherit the global limit' and the key is only written into the fetch metadata when a robots specific limit is configured - the shipped default of http.robots.content.limit is 524288 (512 kiB), matching the minimum the robots.txt RFC draft asks crawlers to parse - HttpProtocol ignores a metadata http.content.limit of -1 when the global limit is finite, so per-URL metadata can tighten the limit but not remove it --- .../protocol/HttpRobotRulesParser.java | 11 ++- .../protocol/okhttp/HttpProtocol.java | 10 ++- core/src/main/resources/crawler-default.yaml | 5 +- .../protocol/RobotsContentLimitTest.java | 84 +++++++++++++++++++ 4 files changed, 106 insertions(+), 4 deletions(-) create mode 100644 core/src/test/java/org/apache/stormcrawler/protocol/RobotsContentLimitTest.java diff --git a/core/src/main/java/org/apache/stormcrawler/protocol/HttpRobotRulesParser.java b/core/src/main/java/org/apache/stormcrawler/protocol/HttpRobotRulesParser.java index a24afe62b..58fb842d1 100644 --- a/core/src/main/java/org/apache/stormcrawler/protocol/HttpRobotRulesParser.java +++ b/core/src/main/java/org/apache/stormcrawler/protocol/HttpRobotRulesParser.java @@ -64,9 +64,16 @@ public void setConf(Config conf) { super.setConf(conf); allowForbidden = ConfUtils.getBoolean(conf, "http.robots.403.allow", true); fetchRobotsMd = new Metadata(); - /* http.content.limit for fetching the robots.txt */ + /* + * http.content.limit for fetching the robots.txt. The default of -1 + * means "same as http.content.limit": writing the key into the fetch + * metadata would override the global limit of the protocol with "no + * limit", so it is only set when a robots specific limit is configured. + */ int robotsTxtContentLimit = ConfUtils.getInt(conf, "http.robots.content.limit", -1); - fetchRobotsMd.addValue("http.content.limit", Integer.toString(robotsTxtContentLimit)); + if (robotsTxtContentLimit != -1) { + fetchRobotsMd.addValue("http.content.limit", Integer.toString(robotsTxtContentLimit)); + } allow5xx = ConfUtils.getBoolean(conf, "http.robots.5xx.allow", false); allowCrossOriginRedirects = ConfUtils.getBoolean(conf, "http.robots.redirect.crossorigin.allow", false); diff --git a/core/src/main/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocol.java b/core/src/main/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocol.java index 90cb4d742..1e5936f95 100644 --- a/core/src/main/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocol.java +++ b/core/src/main/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocol.java @@ -442,7 +442,15 @@ public ProtocolResponse getProtocolOutput(String url, final Metadata metadata) final String pageMaxContentStr = metadata.getFirstValue("http.content.limit"); if (StringUtils.isNotBlank(pageMaxContentStr)) { try { - pageMaxContent = Integer.parseInt(pageMaxContentStr); + int metadataLimit = Integer.parseInt(pageMaxContentStr); + /* + * per-URL metadata can tighten the limit but not remove it: + * a value of -1 means "no limit" and would turn the finite + * global limit of the operator into an unbounded read + */ + if (metadataLimit != -1 || globalMaxContent == -1) { + pageMaxContent = metadataLimit; + } } catch (NumberFormatException e) { LOG.warn("Invalid http.content.limit in metadata: {}", pageMaxContentStr); } diff --git a/core/src/main/resources/crawler-default.yaml b/core/src/main/resources/crawler-default.yaml index 6945b2c4c..49bd48da6 100644 --- a/core/src/main/resources/crawler-default.yaml +++ b/core/src/main/resources/crawler-default.yaml @@ -201,8 +201,11 @@ config: # http.content.limit when fetching the robots.txt # (the robots.txt RFC draft requires to fetch and parse at least 500 kiB, # see https://datatracker.ietf.org/doc/html/draft-rep-wg-topic-00#section-2.5) + # A value of -1 means "same as http.content.limit": the robots.txt fetch + # then uses whatever limit is configured for pages, including "no limit" + # when http.content.limit is -1 too. # http.robots.content.limit: 524288 # 512 kiB - http.robots.content.limit: -1 # default same as http.content.limit + http.robots.content.limit: 524288 # Implementation of RobotRulesParser used by the HTTP protocol implementations # to fetch and parse robots.txt. Override to plug in custom robots.txt diff --git a/core/src/test/java/org/apache/stormcrawler/protocol/RobotsContentLimitTest.java b/core/src/test/java/org/apache/stormcrawler/protocol/RobotsContentLimitTest.java new file mode 100644 index 000000000..048a8da4a --- /dev/null +++ b/core/src/test/java/org/apache/stormcrawler/protocol/RobotsContentLimitTest.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.stormcrawler.protocol; + +import org.apache.storm.Config; +import org.apache.stormcrawler.Metadata; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +/** Checks the content limit used for the robots.txt fetch. */ +class RobotsContentLimitTest { + + /** Protocol stub recording the metadata it is called with. */ + private static class RecordingProtocol implements Protocol { + + Metadata seen; + + @Override + public void configure(Config conf) {} + + @Override + public ProtocolResponse getProtocolOutput(String url, Metadata metadata) { + seen = metadata; + return new ProtocolResponse(new byte[0], 200, new Metadata()); + } + + @Override + public crawlercommons.robots.BaseRobotRules getRobotRules(String url) { + return null; + } + + @Override + public void cleanup() {} + } + + @Test + void robotsFetchKeepsTheGlobalContentLimit() { + Config conf = new Config(); + conf.put("http.agent.name", "this_is_only_a_test"); + // operator sets a finite limit and does not touch http.robots.content.limit + conf.put("http.content.limit", 65536); + + RecordingProtocol protocol = new RecordingProtocol(); + HttpRobotRulesParser parser = new HttpRobotRulesParser(); + parser.setConf(conf); + parser.getRobotRulesSet(protocol, "http://limit.example.org/"); + + String limit = protocol.seen.getFirstValue("http.content.limit"); + Assertions.assertNull( + limit, + "the robots.txt fetch should not override the global content limit, but saw: " + + limit); + } + + @Test + void robotsSpecificLimitIsApplied() { + Config conf = new Config(); + conf.put("http.agent.name", "this_is_only_a_test"); + conf.put("http.content.limit", 65536); + conf.put("http.robots.content.limit", 524288); + + RecordingProtocol protocol = new RecordingProtocol(); + HttpRobotRulesParser parser = new HttpRobotRulesParser(); + parser.setConf(conf); + parser.getRobotRulesSet(protocol, "http://limit.example.org/"); + + Assertions.assertEquals("524288", protocol.seen.getFirstValue("http.content.limit")); + } +}