diff --git a/core/src/main/java/org/apache/stormcrawler/protocol/file/FileProtocol.java b/core/src/main/java/org/apache/stormcrawler/protocol/file/FileProtocol.java index 42ad0309e..18b3dc53b 100644 --- a/core/src/main/java/org/apache/stormcrawler/protocol/file/FileProtocol.java +++ b/core/src/main/java/org/apache/stormcrawler/protocol/file/FileProtocol.java @@ -18,6 +18,9 @@ package org.apache.stormcrawler.protocol.file; import crawlercommons.robots.BaseRobotRules; +import java.io.File; +import java.io.IOException; +import org.apache.commons.lang3.StringUtils; import org.apache.storm.Config; import org.apache.stormcrawler.Metadata; import org.apache.stormcrawler.protocol.Protocol; @@ -27,11 +30,37 @@ public class FileProtocol implements Protocol { + /** + * Directory reads are confined to when set. Null or empty means the file scheme serves nothing: + * a URL decides which path the worker opens, so reads must not be possible until an operator + * has chosen a root on purpose. + */ + public static final String ROOT_KEY = "file.protocol.root"; + private String encoding; + private File root; + @Override public void configure(Config conf) { encoding = ConfUtils.getString(conf, "file.encoding", "UTF-8"); + String rootPath = ConfUtils.getString(conf, ROOT_KEY, null); + if (StringUtils.isNotBlank(rootPath)) { + root = new File(rootPath); + try { + root = root.getCanonicalFile(); + } catch (IOException e) { + throw new RuntimeException("Cannot resolve " + ROOT_KEY + ": " + rootPath, e); + } + if (!root.isDirectory()) { + throw new RuntimeException(ROOT_KEY + " is not a directory: " + rootPath); + } + } + } + + /** Returns the configured confinement root, or null when reads are disabled entirely. */ + File getRoot() { + return root; } @Override diff --git a/core/src/main/java/org/apache/stormcrawler/protocol/file/FileResponse.java b/core/src/main/java/org/apache/stormcrawler/protocol/file/FileResponse.java index 47504a649..86f85e3c5 100644 --- a/core/src/main/java/org/apache/stormcrawler/protocol/file/FileResponse.java +++ b/core/src/main/java/org/apache/stormcrawler/protocol/file/FileResponse.java @@ -61,6 +61,23 @@ public FileResponse(String u, Metadata md, FileProtocol fileProtocol) throws IOE File file = new File(URLDecoder.decode(path, fileProtocol.getEncoding())); + /* + * A URL decides which path the worker opens: without a configured root + * nothing is served, with one the resolved path must stay below it. + */ + File root = fileProtocol.getRoot(); + if (root == null) { + LOG.warn("Refusing to read {} because {} is not configured", url, FileProtocol.ROOT_KEY); + statusCode = HttpStatus.SC_FORBIDDEN; + return; + } + + if (!file.getCanonicalFile().toPath().startsWith(root.toPath())) { + LOG.warn("Refusing to read {} because it is outside {}", url, root); + statusCode = HttpStatus.SC_FORBIDDEN; + return; + } + if (!file.exists()) { statusCode = HttpStatus.SC_NOT_FOUND; return; diff --git a/core/src/main/resources/crawler-default.yaml b/core/src/main/resources/crawler-default.yaml index 6945b2c4c..aac1c73dc 100644 --- a/core/src/main/resources/crawler-default.yaml +++ b/core/src/main/resources/crawler-default.yaml @@ -215,10 +215,16 @@ config: robots.cache.spec: "maximumSize=10000,expireAfterWrite=6h" robots.error.cache.spec: "maximumSize=10000,expireAfterWrite=1h" - protocols: "http,https,file" + # The file scheme must be enabled deliberately: a fetched page can put a + # file:// URL into the frontier, and FileProtocol reads whatever path the + # worker user can read unless file.protocol.root confines it. When enabled, + # set file.protocol.root to the directory reads are confined to; without it + # the file scheme serves nothing. + protocols: "http,https" http.protocol.implementation: "org.apache.stormcrawler.protocol.okhttp.HttpProtocol" https.protocol.implementation: "org.apache.stormcrawler.protocol.okhttp.HttpProtocol" file.protocol.implementation: "org.apache.stormcrawler.protocol.file.FileProtocol" + # file.protocol.root: "/data/corpus" # number of instances for each protocol implementation protocol.instances.num: 1 diff --git a/core/src/test/java/org/apache/stormcrawler/protocol/file/FileProtocolDefaultsTest.java b/core/src/test/java/org/apache/stormcrawler/protocol/file/FileProtocolDefaultsTest.java new file mode 100644 index 000000000..93dd33843 --- /dev/null +++ b/core/src/test/java/org/apache/stormcrawler/protocol/file/FileProtocolDefaultsTest.java @@ -0,0 +1,131 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.stormcrawler.protocol.file; + +import java.io.File; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import org.apache.storm.Config; +import org.apache.storm.utils.Utils; +import org.apache.stormcrawler.Metadata; +import org.apache.stormcrawler.protocol.ProtocolResponse; +import org.apache.stormcrawler.util.ConfUtils; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +class FileProtocolDefaultsTest { + + /** The file scheme must not be part of the shipped default protocols list. */ + @Test + void fileSchemeIsNotEnabledByDefault() { + Config conf = new Config(); + Map defaults = Utils.findAndReadConfigFile("crawler-default.yaml", false); + conf.putAll(ConfUtils.extractConfigElement(defaults)); + String protocols = ConfUtils.getString(conf, "protocols", "http,https"); + List schemes = Arrays.asList(protocols.split(" *, *")); + Assertions.assertFalse( + schemes.contains("file"), + "crawler-default.yaml enables the file scheme by default: " + protocols); + } + + /** + * Without a configured root, FileProtocol must refuse to serve anything. + */ + @Test + void fileProtocolServesNothingWithoutARoot(@TempDir Path tmp) throws Exception { + Path inside = tmp.resolve("inside.txt"); + Files.write(inside, "content".getBytes(StandardCharsets.UTF_8)); + + Config conf = new Config(); + FileProtocol protocol = new FileProtocol(); + protocol.configure(conf); + + String url = inside.toUri().toURL().toString(); + ProtocolResponse response = protocol.getProtocolOutput(url, new Metadata()); + Assertions.assertEquals( + 403, response.getStatusCode(), "FileProtocol served a file with no root set"); + } + + /** + * With a root directory configured, FileProtocol must refuse to read a file outside that root. + */ + @Test + void fileProtocolConfinesReadsToConfiguredRoot(@TempDir Path tmp) throws Exception { + Path base = tmp.toRealPath(); + Path root = base.resolve("root"); + Files.createDirectories(root); + Path inside = root.resolve("inside.txt"); + Files.write(inside, "for the crawler".getBytes(StandardCharsets.UTF_8)); + Path outside = base.resolve("outside.txt"); + Files.write(outside, "not for the crawler".getBytes(StandardCharsets.UTF_8)); + + Config conf = new Config(); + conf.put(FileProtocol.ROOT_KEY, root.toString()); + + FileProtocol protocol = new FileProtocol(); + protocol.configure(conf); + + String insideUrl = inside.toUri().toURL().toString(); + ProtocolResponse response = protocol.getProtocolOutput(insideUrl, new Metadata()); + Assertions.assertEquals( + 200, response.getStatusCode(), "FileProtocol refused a file inside the root"); + + String outsideUrl = outside.toUri().toURL().toString(); + response = protocol.getProtocolOutput(outsideUrl, new Metadata()); + Assertions.assertNotEquals( + 200, + response.getStatusCode(), + "FileProtocol read a file outside the configured root: " + outsideUrl); + } + + /** A symlink or parent-directory escape must not leave the root either. */ + @Test + void fileProtocolConfinesCanonicalisedPaths(@TempDir Path tmp) throws Exception { + Path base = tmp.toRealPath(); + Path root = base.resolve("root"); + Files.createDirectories(root); + Path secret = base.resolve("secret.txt"); + Files.write(secret, "not for the crawler".getBytes(StandardCharsets.UTF_8)); + + File link = root.resolve("link.txt").toFile(); + try { + Files.createSymbolicLink( + link.toPath().toAbsolutePath(), secret.toAbsolutePath()); + } catch (UnsupportedOperationException | java.io.IOException e) { + // filesystem without symlink support; nothing to test here + return; + } + + Config conf = new Config(); + conf.put(FileProtocol.ROOT_KEY, root.toString()); + FileProtocol protocol = new FileProtocol(); + protocol.configure(conf); + + String url = link.toURI().toURL().toString(); + ProtocolResponse response = protocol.getProtocolOutput(url, new Metadata()); + Assertions.assertNotEquals( + 200, + response.getStatusCode(), + "FileProtocol followed a symlink out of the configured root: " + url); + } +}