diff --git a/core/src/main/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocol.java b/core/src/main/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocol.java index 90cb4d742..d5d0917cd 100644 --- a/core/src/main/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocol.java +++ b/core/src/main/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocol.java @@ -54,6 +54,7 @@ import okhttp3.Gzip; import okhttp3.Handshake; import okhttp3.Headers; +import okhttp3.HttpUrl; import okhttp3.Interceptor; import okhttp3.MediaType; import okhttp3.OkHttpClient; @@ -74,6 +75,7 @@ import org.apache.storm.Config; import org.apache.stormcrawler.Constants; import org.apache.stormcrawler.Metadata; +import org.apache.stormcrawler.filtering.URLFilters; import org.apache.stormcrawler.protocol.AbstractHttpProtocol; import org.apache.stormcrawler.protocol.IPFilterRules; import org.apache.stormcrawler.protocol.ProtocolResponse; @@ -89,6 +91,13 @@ public class HttpProtocol extends AbstractHttpProtocol { private static final org.slf4j.Logger LOG = LoggerFactory.getLogger(HttpProtocol.class); + /** + * Maximum number of redirect hops followed when {@code http.allow.redirects} is enabled. A + * chain which does not end within this many hops returns its last redirect response, which the + * caller handles like it does when redirect following is off. + */ + private static final int MAX_REDIRECT_HOPS = 5; + private final MediaType json = MediaType.parse("application/json; charset=utf-8"); private OkHttpClient client; @@ -100,6 +109,12 @@ public class HttpProtocol extends AbstractHttpProtocol { /** Accept partially fetched content as trimmed content */ private boolean partialContentAsTrimmed = false; + /** Redirect targets run through the URL filters before a hop is taken. */ + private URLFilters urlFilters = URLFilters.emptyURLFilters; + + /** Whether redirect responses are followed, from {@code http.allow.redirects}. */ + private boolean followRedirects = false; + private final List customRequestHeaders = new LinkedList<>(); // track the time spent for each URL in DNS resolution @@ -158,12 +173,22 @@ public void configure(Config conf) { this.partialContentAsTrimmed = ConfUtils.getBoolean(conf, "http.content.partial.as.trimmed", false); + /* + * Redirects are followed in getProtocolOutput and not by the client: + * every target has to pass through the URL filters first, so that the + * scheme exclusions, host confinement and depth rules configured for + * the crawl also apply to the hops a fetched page steers the fetcher + * to. The client must therefore never follow them on its own. + */ + final boolean allowRedirects = ConfUtils.getBoolean(conf, "http.allow.redirects", false); + this.followRedirects = allowRedirects; + urlFilters = URLFilters.fromConf(conf); builder = new OkHttpClient.Builder() .retryOnConnectionFailure( ConfUtils.getBoolean( conf, "http.retry.on.connection.failure", true)) - .followRedirects(ConfUtils.getBoolean(conf, "http.allow.redirects", false)) + .followRedirects(false) .connectTimeout(timeout, TimeUnit.MILLISECONDS) .writeTimeout(timeout, TimeUnit.MILLISECONDS) .readTimeout(timeout, TimeUnit.MILLISECONDS); @@ -466,9 +491,85 @@ public ProtocolResponse getProtocolOutput(String url, final Metadata metadata) final Request request = rb.build(); - final Call call = localClient.newCall(request); + /* + * Follow redirect responses manually: every target runs through the + * URL filters before the hop is taken, and a target which is rejected + * ends the chain - the redirect response is then returned as is and + * the caller handles it like it does when http.allow.redirects is + * off. The final URL is recorded in the response metadata so that + * callers can tell that the content is not from the URL they asked + * for. + */ + Response lastResponse = null; + Call call = null; + Request currentRequest = request; + String currentUrl = url; + + try { + for (int hops = 0; hops <= MAX_REDIRECT_HOPS; hops++) { + if (lastResponse != null) { + // release the connection before issuing the next request + lastResponse.close(); + lastResponse = null; + } + call = client.newCall(currentRequest); + lastResponse = call.execute(); + + if (hops == MAX_REDIRECT_HOPS) { + LOG.warn("More than {} redirect hops for {}", MAX_REDIRECT_HOPS, url); + break; + } + if (!followRedirects || !isRedirect(lastResponse)) { + break; + } + + final String location = lastResponse.header(HttpHeaders.LOCATION); + if (StringUtils.isBlank(location)) { + LOG.debug( + "Got redirect response {} for {} without location", + lastResponse.code(), + url); + break; + } + + final HttpUrl target = currentRequest.url().resolve(location); + if (target == null) { + LOG.warn( + "Redirect target {} could not be resolved against {}", + location, + currentUrl); + break; + } - try (Response response = call.execute()) { + final Metadata sourceMetadata = metadata != null ? metadata : new Metadata(); + final String filtered = + urlFilters.filter( + currentRequest.url().url(), sourceMetadata, target.toString()); + if (filtered == null) { + LOG.info("Redirect target {} rejected by the URL filters", target); + break; + } + + final HttpUrl accepted = HttpUrl.parse(filtered); + if (accepted == null) { + LOG.warn("Filtered redirect target {} is not a URL", filtered); + break; + } + + final Request.Builder followBuilder = currentRequest.newBuilder().url(accepted); + // for 303, and for 301 or 302 after a POST, the request is + // repeated as a GET, like the client would do + final int code = lastResponse.code(); + if (code == 303 + || ((code == 301 || code == 302) && !"GET".equals(currentRequest.method()))) { + followBuilder.method("GET", null); + } + currentRequest = followBuilder.build(); + currentUrl = filtered; + } + + final Response response = lastResponse; + final Call executedCall = call; final Metadata responsemetadata = new Metadata(); final Headers headers = response.headers(); @@ -487,6 +588,10 @@ public ProtocolResponse getProtocolOutput(String url, final Metadata metadata) responsemetadata.addValue(key.toLowerCase(Locale.ROOT), value); } + if (!currentUrl.equals(url)) { + responsemetadata.setValue("_redirTo", currentUrl); + } + // the Set-Cookie header does not say which host sent it: record the url of this // response so that the cookies can be scoped to it when they are sent back. The // key is dropped first so that a server sending a header of that name can not @@ -501,8 +606,8 @@ public ProtocolResponse getProtocolOutput(String url, final Metadata metadata) new MutableObject<>(TrimmedContentReason.NOT_TRIMMED); final byte[] bytes = toByteArray(response.body(), pageMaxContent, trimmed); if (trimmed.get() != TrimmedContentReason.NOT_TRIMMED) { - if (!call.isCanceled()) { - call.cancel(); + if (!executedCall.isCanceled()) { + executedCall.cancel(); } responsemetadata.setValue(ProtocolResponse.TRIMMED_RESPONSE_KEY, "true"); responsemetadata.setValue( @@ -511,15 +616,25 @@ public ProtocolResponse getProtocolOutput(String url, final Metadata metadata) LOG.warn("HTTP content trimmed to {} (reason: {})", bytes.length, trimmed.get()); } - final Long dnsResolution = DNStimes.remove(call.toString()); + final Long dnsResolution = DNStimes.remove(executedCall.toString()); if (dnsResolution != null) { responsemetadata.setValue("metrics.dns.resolution.msec", dnsResolution.toString()); } return new ProtocolResponse(bytes, response.code(), responsemetadata); + } finally { + if (lastResponse != null) { + lastResponse.close(); + } } } + /** Checks whether the response is a redirect whose Location must be resolved. */ + private static boolean isRedirect(Response response) { + final int code = response.code(); + return code == 301 || code == 302 || code == 303 || code == 307 || code == 308; + } + private byte[] toByteArray( final ResponseBody responseBody, int maxContent, diff --git a/core/src/test/java/org/apache/stormcrawler/protocol/OkHttpFollowRedirectsTest.java b/core/src/test/java/org/apache/stormcrawler/protocol/OkHttpFollowRedirectsTest.java new file mode 100644 index 000000000..8c2532fa8 --- /dev/null +++ b/core/src/test/java/org/apache/stormcrawler/protocol/OkHttpFollowRedirectsTest.java @@ -0,0 +1,150 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.stormcrawler.protocol; + +import jakarta.servlet.http.HttpServletResponse; +import java.io.IOException; +import java.io.OutputStream; +import java.nio.charset.StandardCharsets; +import org.apache.storm.Config; +import org.apache.stormcrawler.Metadata; +import org.apache.stormcrawler.protocol.okhttp.HttpProtocol; +import org.eclipse.jetty.server.Handler; +import org.eclipse.jetty.server.Request; +import org.eclipse.jetty.server.handler.AbstractHandler; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +/** + * When http.allow.redirects is enabled, every hop of the redirect chain runs through the URL + * filters, and the final URL is recorded in the response metadata. + */ +class OkHttpFollowRedirectsTest extends AbstractProtocolTest { + + /** Redirects /start to /elsewhere, serves plain text for anything else. */ + @Override + protected Handler[] getHandlers() { + return new Handler[] { + new AbstractHandler() { + @Override + public void handle( + String target, + Request baseRequest, + jakarta.servlet.http.HttpServletRequest request, + HttpServletResponse response) + throws IOException { + baseRequest.setHandled(true); + if (target.equals("/start")) { + response.setStatus(HttpServletResponse.SC_MOVED_TEMPORARILY); + response.setHeader( + "Location", "http://127.0.0.1:" + HTTP_PORT + "/elsewhere"); + response.setContentLength(0); + response.getOutputStream().close(); + return; + } + response.setStatus(HttpServletResponse.SC_OK); + response.setContentType("text/plain"); + final byte[] content = ("body of " + target).getBytes(StandardCharsets.UTF_8); + response.setContentLength(content.length); + try (OutputStream out = response.getOutputStream()) { + out.write(content); + } + } + } + }; + } + + private static HttpProtocol protocol(Config conf) { + HttpProtocol protocol = new HttpProtocol(); + protocol.configure(conf); + return protocol; + } + + private static Config config() { + Config conf = new Config(); + conf.put("http.agent.name", "this_is_only_a_test"); + conf.put("http.allow.redirects", true); + return conf; + } + + @Test + void redirectTargetIsFollowedAndRecorded() throws Exception { + // no urlfilters.config.file: the chain is empty, every target passes + HttpProtocol protocol = protocol(config()); + ProtocolResponse response = + protocol.getProtocolOutput( + "http://127.0.0.1:" + HTTP_PORT + "/start", new Metadata()); + protocol.cleanup(); + Assertions.assertEquals(200, response.getStatusCode()); + Assertions.assertEquals( + "body of /elsewhere", + new String(response.getContent(), StandardCharsets.UTF_8), + "the redirect is followed"); + Assertions.assertEquals( + "http://127.0.0.1:" + HTTP_PORT + "/elsewhere", + response.getMetadata().getFirstValue("_redirTo"), + "the final URL must be recorded in the response metadata"); + } + + @Test + void rejectedRedirectTargetIsNotFetched() throws Exception { + // a chain which rejects everything must stop the hop from being taken + Config conf = config(); + conf.put("urlfilters.config.file", "urlfilters-reject-all.json"); + HttpProtocol protocol = protocol(conf); + ProtocolResponse response = + protocol.getProtocolOutput( + "http://127.0.0.1:" + HTTP_PORT + "/start", new Metadata()); + protocol.cleanup(); + Assertions.assertEquals( + 302, response.getStatusCode(), "the redirect response itself is returned"); + Assertions.assertEquals( + "http://127.0.0.1:" + HTTP_PORT + "/elsewhere", + response.getMetadata().getFirstValue("location"), + "the Location header tells the caller where the chain stopped"); + } + + /** Stands in for an exclusion rule which rejects every target. */ + public static class RejectAllURLFilter extends org.apache.stormcrawler.filtering.URLFilter { + @Override + public String filter( + java.net.URL sourceUrl, + Metadata sourceMetadata, + @org.jetbrains.annotations.NotNull String urlToFilter) { + return null; + } + } + + @Test + void redirectsAreNotFollowedWhenDisabled() throws Exception { + Config conf = config(); + conf.put("http.allow.redirects", false); + HttpProtocol protocol = protocol(conf); + ProtocolResponse response = + protocol.getProtocolOutput( + "http://127.0.0.1:" + HTTP_PORT + "/start", new Metadata()); + protocol.cleanup(); + Assertions.assertEquals( + 302, + response.getStatusCode(), + "the redirect response itself is returned, nothing is followed"); + Assertions.assertNull( + response.getMetadata().getFirstValue("_redirTo"), + "no redirect was followed, so no final URL is recorded"); + } +} diff --git a/core/src/test/resources/urlfilters-reject-all.json b/core/src/test/resources/urlfilters-reject-all.json new file mode 100644 index 000000000..c8169f4bb --- /dev/null +++ b/core/src/test/resources/urlfilters-reject-all.json @@ -0,0 +1,8 @@ +{ + "org.apache.stormcrawler.filtering.URLFilters": [ + { + "class": "org.apache.stormcrawler.protocol.OkHttpFollowRedirectsTest$RejectAllURLFilter", + "name": "RejectAllURLFilter" + } + ] +} diff --git a/docs/src/main/asciidoc/configuration.adoc b/docs/src/main/asciidoc/configuration.adoc index 047a86ea4..5489fcfbf 100644 --- a/docs/src/main/asciidoc/configuration.adoc +++ b/docs/src/main/asciidoc/configuration.adoc @@ -232,7 +232,7 @@ implementation. | partition.url.mode | byHost | Defines how URLs are partitioned: byHost, byDomain, or byIP. | protocols | http,https,file | Supported protocols. | redirections.allowed | true | If true, emit redirect target URLs as "outlinks" to the status stream. If false, do not follow redirects. See also `http.allow.redirects`. -| http.allow.redirects | false | (OkHttp only) Follow HTTP redirects immediately in the HTTP protocol client. Note: if followed immediately, redirect target URLs are not emitted to the status stream, are not filtered, not deduplicated, and not checked against robots.txt. +| http.allow.redirects | false | (OkHttp only) Follow HTTP redirects immediately in the HTTP protocol client. Note: if followed immediately, redirect target URLs are not emitted to the status stream, are not deduplicated and not checked against robots.txt, but each target runs through the URL filter chain before the hop is taken (up to 5 hops; the final URL is recorded in the response metadata as `_redirTo`). | sitemap.discovery | false | Enable automatic sitemap discovery. | urlbuffer.class | org.apache.stormcrawler.persistence.urlbuffer.SimpleURLBuffer | URL buffer implementation used by spouts. |=== diff --git a/docs/src/main/asciidoc/internals.adoc b/docs/src/main/asciidoc/internals.adoc index 7d2044db3..4a389c02f 100644 --- a/docs/src/main/asciidoc/internals.adoc +++ b/docs/src/main/asciidoc/internals.adoc @@ -385,7 +385,7 @@ When handling HTTP redirects, StormCrawler offers three modes: 2. **Redirects disabled** (`redirections.allowed: false`): Redirect target URLs are not sent to the status stream. Redirects are effectively ignored. -3. **Immediate follow** (`http.allow.redirects: true`): Redirects are followed immediately in the HTTP client and the target URLs are not emitted to the status stream. This is the default behavior for the browser-based Playwright protocol, but it is also supported by the OkHttp protocol. Note that with immediate follows, redirect targets bypass URL filtering, deduplication, and `robots.txt` checks. +3. **Immediate follow** (`http.allow.redirects: true`): Redirects are followed immediately in the HTTP client and the target URLs are not emitted to the status stream. This is the default behavior for the browser-based Playwright protocol, but it is also supported by the OkHttp protocol. With immediate follows on OkHttp, every redirect target runs through the URL filter chain before the hop is taken and a rejected target ends the chain, so the filter rules configured for the crawl still apply to the hops. The chain is limited to 5 hops; a chain which does not end within that returns its last redirect response, which the caller handles like it does when immediate follows are off. The final URL is recorded in the response metadata under `_redirTo`. Deduplication and `robots.txt` checks do not apply to the followed hops. ==== Network Protocols