File scheme off by default; reads confined to file.protocol.root

core/src/main/java/org/apache/stormcrawler/protocol/file/FileProtocol.java

@@ -18,6 +18,9 @@
 package org.apache.stormcrawler.protocol.file;
 
 import crawlercommons.robots.BaseRobotRules;
+import java.io.File;
+import java.io.IOException;
+import org.apache.commons.lang3.StringUtils;
 import org.apache.storm.Config;
 import org.apache.stormcrawler.Metadata;
 import org.apache.stormcrawler.protocol.Protocol;
@@ -27,11 +30,37 @@
 
 public class FileProtocol implements Protocol {
 
+    /**
+     * Directory reads are confined to when set. Null or empty means the file scheme serves nothing:
+     * a URL decides which path the worker opens, so reads must not be possible until an operator
+     * has chosen a root on purpose.
+     */
+    public static final String ROOT_KEY = "file.protocol.root";
+
     private String encoding;
 
+    private File root;
+
     @Override
     public void configure(Config conf) {
         encoding = ConfUtils.getString(conf, "file.encoding", "UTF-8");
+        String rootPath = ConfUtils.getString(conf, ROOT_KEY, null);
+        if (StringUtils.isNotBlank(rootPath)) {
+            root = new File(rootPath);
+            try {
+                root = root.getCanonicalFile();
+            } catch (IOException e) {
+                throw new RuntimeException("Cannot resolve " + ROOT_KEY + ": " + rootPath, e);
+            }
+            if (!root.isDirectory()) {
+                throw new RuntimeException(ROOT_KEY + " is not a directory: " + rootPath);
+            }
+        }
+    }
+
+    /** Returns the configured confinement root, or null when reads are disabled entirely. */
+    File getRoot() {
+        return root;
     }
 
     @Override

core/src/main/java/org/apache/stormcrawler/protocol/file/FileResponse.java

@@ -61,6 +61,23 @@ public FileResponse(String u, Metadata md, FileProtocol fileProtocol) throws IOE
 
         File file = new File(URLDecoder.decode(path, fileProtocol.getEncoding()));
 
+        /*
+         * A URL decides which path the worker opens: without a configured root
+         * nothing is served, with one the resolved path must stay below it.
+         */
+        File root = fileProtocol.getRoot();
+        if (root == null) {
+            LOG.warn("Refusing to read {} because {} is not configured", url, FileProtocol.ROOT_KEY);
+            statusCode = HttpStatus.SC_FORBIDDEN;
+            return;
+        }
+
+        if (!file.getCanonicalFile().toPath().startsWith(root.toPath())) {
+            LOG.warn("Refusing to read {} because it is outside {}", url, root);
+            statusCode = HttpStatus.SC_FORBIDDEN;
+            return;
+        }
+
         if (!file.exists()) {
             statusCode = HttpStatus.SC_NOT_FOUND;
             return;

core/src/main/resources/crawler-default.yaml

@@ -215,10 +215,16 @@ config:
   robots.cache.spec: "maximumSize=10000,expireAfterWrite=6h"
   robots.error.cache.spec: "maximumSize=10000,expireAfterWrite=1h"
 
-  protocols: "http,https,file"
+  # The file scheme must be enabled deliberately: a fetched page can put a
+  # file:// URL into the frontier, and FileProtocol reads whatever path the
+  # worker user can read unless file.protocol.root confines it. When enabled,
+  # set file.protocol.root to the directory reads are confined to; without it
+  # the file scheme serves nothing.
+  protocols: "http,https"
   http.protocol.implementation: "org.apache.stormcrawler.protocol.okhttp.HttpProtocol"
   https.protocol.implementation: "org.apache.stormcrawler.protocol.okhttp.HttpProtocol"
   file.protocol.implementation: "org.apache.stormcrawler.protocol.file.FileProtocol"
+  # file.protocol.root: "/data/corpus"
 
   # number of instances for each protocol implementation
   protocol.instances.num: 1

core/src/test/java/org/apache/stormcrawler/protocol/file/FileProtocolDefaultsTest.java

@@ -0,0 +1,131 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.stormcrawler.protocol.file;
+
+import java.io.File;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import org.apache.storm.Config;
+import org.apache.storm.utils.Utils;
+import org.apache.stormcrawler.Metadata;
+import org.apache.stormcrawler.protocol.ProtocolResponse;
+import org.apache.stormcrawler.util.ConfUtils;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+class FileProtocolDefaultsTest {
+
+    /** The file scheme must not be part of the shipped default protocols list. */
+    @Test
+    void fileSchemeIsNotEnabledByDefault() {
+        Config conf = new Config();
+        Map<String, Object> defaults = Utils.findAndReadConfigFile("crawler-default.yaml", false);
+        conf.putAll(ConfUtils.extractConfigElement(defaults));
+        String protocols = ConfUtils.getString(conf, "protocols", "http,https");
+        List<String> schemes = Arrays.asList(protocols.split(" *, *"));
+        Assertions.assertFalse(
+                schemes.contains("file"),
+                "crawler-default.yaml enables the file scheme by default: " + protocols);
+    }
+
+    /**
+     * Without a configured root, FileProtocol must refuse to serve anything.
+     */
+    @Test
+    void fileProtocolServesNothingWithoutARoot(@TempDir Path tmp) throws Exception {
+        Path inside = tmp.resolve("inside.txt");
+        Files.write(inside, "content".getBytes(StandardCharsets.UTF_8));
+
+        Config conf = new Config();
+        FileProtocol protocol = new FileProtocol();
+        protocol.configure(conf);
+
+        String url = inside.toUri().toURL().toString();
+        ProtocolResponse response = protocol.getProtocolOutput(url, new Metadata());
+        Assertions.assertEquals(
+                403, response.getStatusCode(), "FileProtocol served a file with no root set");
+    }
+
+    /**
+     * With a root directory configured, FileProtocol must refuse to read a file outside that root.
+     */
+    @Test
+    void fileProtocolConfinesReadsToConfiguredRoot(@TempDir Path tmp) throws Exception {
+        Path base = tmp.toRealPath();
+        Path root = base.resolve("root");
+        Files.createDirectories(root);
+        Path inside = root.resolve("inside.txt");
+        Files.write(inside, "for the crawler".getBytes(StandardCharsets.UTF_8));
+        Path outside = base.resolve("outside.txt");
+        Files.write(outside, "not for the crawler".getBytes(StandardCharsets.UTF_8));
+
+        Config conf = new Config();
+        conf.put(FileProtocol.ROOT_KEY, root.toString());
+
+        FileProtocol protocol = new FileProtocol();
+        protocol.configure(conf);
+
+        String insideUrl = inside.toUri().toURL().toString();
+        ProtocolResponse response = protocol.getProtocolOutput(insideUrl, new Metadata());
+        Assertions.assertEquals(
+                200, response.getStatusCode(), "FileProtocol refused a file inside the root");
+
+        String outsideUrl = outside.toUri().toURL().toString();
+        response = protocol.getProtocolOutput(outsideUrl, new Metadata());
+        Assertions.assertNotEquals(
+                200,
+                response.getStatusCode(),
+                "FileProtocol read a file outside the configured root: " + outsideUrl);
+    }
+
+    /** A symlink or parent-directory escape must not leave the root either. */
+    @Test
+    void fileProtocolConfinesCanonicalisedPaths(@TempDir Path tmp) throws Exception {
+        Path base = tmp.toRealPath();
+        Path root = base.resolve("root");
+        Files.createDirectories(root);
+        Path secret = base.resolve("secret.txt");
+        Files.write(secret, "not for the crawler".getBytes(StandardCharsets.UTF_8));
+
+        File link = root.resolve("link.txt").toFile();
+        try {
+            Files.createSymbolicLink(
+                    link.toPath().toAbsolutePath(), secret.toAbsolutePath());
+        } catch (UnsupportedOperationException | java.io.IOException e) {
+            // filesystem without symlink support; nothing to test here
+            return;
+        }
+
+        Config conf = new Config();
+        conf.put(FileProtocol.ROOT_KEY, root.toString());
+        FileProtocol protocol = new FileProtocol();
+        protocol.configure(conf);
+
+        String url = link.toURI().toURL().toString();
+        ProtocolResponse response = protocol.getProtocolOutput(url, new Metadata());
+        Assertions.assertNotEquals(
+                200,
+                response.getStatusCode(),
+                "FileProtocol followed a symlink out of the configured root: " + url);
+    }
+}