diff --git a/archetype/src/main/resources/archetype-resources/crawler-conf.yaml b/archetype/src/main/resources/archetype-resources/crawler-conf.yaml index bc57e3714..6db628282 100644 --- a/archetype/src/main/resources/archetype-resources/crawler-conf.yaml +++ b/archetype/src/main/resources/archetype-resources/crawler-conf.yaml @@ -72,6 +72,12 @@ config: http.protocol.implementation: "org.apache.stormcrawler.protocol.okhttp.HttpProtocol" https.protocol.implementation: "org.apache.stormcrawler.protocol.okhttp.HttpProtocol" + # Accept any TLS certificate, e.g. self-signed ones? The servers are not + # authenticated then; credentials (basic auth, credential headers, cookies) + # are withheld over such connections unless + # http.credentials.allow.insecure is set to true. + http.trust.everything: false + # The maximum number of bytes for returned HTTP response bodies. # The fetched page will be trimmed to 65KB in this case # Set -1 to disable the limit. diff --git a/core/src/main/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocol.java b/core/src/main/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocol.java index 90cb4d742..f4cc662de 100644 --- a/core/src/main/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocol.java +++ b/core/src/main/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocol.java @@ -34,12 +34,11 @@ import java.util.Map; import java.util.Objects; import java.util.Optional; +import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; -import javax.net.ssl.HostnameVerifier; import javax.net.ssl.SSLContext; -import javax.net.ssl.SSLSession; import javax.net.ssl.SSLSocketFactory; import javax.net.ssl.TrustManager; import javax.net.ssl.X509TrustManager; @@ -108,6 +107,23 @@ public class HttpProtocol extends AbstractHttpProtocol { // makes sure that a missing cookie origin is reported once and not for every url private final AtomicBoolean missingCookieOriginLogged = new AtomicBoolean(); + // makes sure that withheld cookies are reported once and not for every url + private final AtomicBoolean withheldCookiesLogged = new AtomicBoolean(); + + // whether credentials (basic auth, credential headers, cookies) may be sent: + // false when the servers are not authenticated and + // http.credentials.allow.insecure is not enabled + private boolean sendCredentials = true; + + /** Header names carrying credentials, see {@link #isCredentialHeader(String)}. */ + private static final Set CREDENTIAL_HEADERS = + Set.of( + HttpHeaders.AUTHORIZATION.toLowerCase(Locale.ROOT), + HttpHeaders.PROXY_AUTHORIZATION.toLowerCase(Locale.ROOT), + // the cookie header is not a constant in HttpHeaders + "cookie", + "x-api-key"); + private OkHttpClient.Builder builder; private static final TrustManager[] trustAllCerts = @@ -130,11 +146,12 @@ public java.security.cert.X509Certificate[] getAcceptedIssuers() { } }; - private static final SSLContext trustAllSslContext; + // package-private so that the OkHttpTrustEverythingTest can check the protocol + static final SSLContext trustAllSslContext; static { try { - trustAllSslContext = SSLContext.getInstance("SSL"); + trustAllSslContext = SSLContext.getInstance("TLS"); trustAllSslContext.init(null, trustAllCerts, new java.security.SecureRandom()); } catch (Exception e) { throw new RuntimeException(e); @@ -158,6 +175,37 @@ public void configure(Config conf) { this.partialContentAsTrimmed = ConfUtils.getBoolean(conf, "http.content.partial.as.trimmed", false); + /* + * certificate trust and hostname verification are separate decisions: + * accepting any certificate does not imply accepting any name + */ + final boolean trustEverything = ConfUtils.getBoolean(conf, "http.trust.everything", false); + final boolean verifyHostnames = ConfUtils.getBoolean(conf, "http.verify.hostnames", true); + // credentials are withheld over connections whose server certificate + // was not validated, unless explicitly opted in + final boolean insecureCredentialsAllowed = + ConfUtils.getBoolean(conf, "http.credentials.allow.insecure", false); + this.sendCredentials = !trustEverything || insecureCredentialsAllowed; + + if (trustEverything) { + LOG.warn( + "http.trust.everything is enabled: TLS certificate chains are accepted without " + + "validation, the identity of the servers is not authenticated. Anybody " + + "able to answer for the host name receives everything sent to them."); + } + if (!verifyHostnames) { + LOG.warn( + "http.verify.hostnames is disabled: the certificates are not checked against " + + "the host name either, the identity of the servers is not authenticated."); + } + if (trustEverything && !sendCredentials) { + LOG.warn( + "Credentials configured with http.basicauth.*, credential headers in " + + "http.custom.headers and cookies are withheld because the servers " + + "are not authenticated. Set http.credentials.allow.insecure to true " + + "to send them anyway."); + } + builder = new OkHttpClient.Builder() .retryOnConnectionFailure( @@ -223,15 +271,33 @@ public void configure(Config conf) { // use a basic auth? if (StringUtils.isNotBlank(basicAuthUser)) { final String basicAuthPass = ConfUtils.getString(conf, "http.basicauth.password", ""); - final String encoding = - Base64.getEncoder() - .encodeToString( - (basicAuthUser + ":" + basicAuthPass) - .getBytes(StandardCharsets.UTF_8)); - customRequestHeaders.add(new KeyValue(HttpHeaders.AUTHORIZATION, "Basic " + encoding)); + if (sendCredentials) { + final String encoding = + Base64.getEncoder() + .encodeToString( + (basicAuthUser + ":" + basicAuthPass) + .getBytes(StandardCharsets.UTF_8)); + customRequestHeaders.add( + new KeyValue(HttpHeaders.AUTHORIZATION, "Basic " + encoding)); + } else { + LOG.warn( + "Basic authentication configured with http.basicauth.user is withheld " + + "because the servers are not authenticated (http.trust.everything). " + + "Set http.credentials.allow.insecure to true to send it anyway."); + } } - customHeaders.forEach(customRequestHeaders::add); + for (KeyValue customHeader : customHeaders) { + if (!sendCredentials && isCredentialHeader(customHeader.getKey())) { + LOG.warn( + "Custom header {} is withheld because the servers are not authenticated " + + "(http.trust.everything). Set http.credentials.allow.insecure to " + + "true to send it anyway.", + customHeader.getKey()); + continue; + } + customRequestHeaders.add(customHeader); + } // optionally block connections to forbidden IP address ranges // (e.g. localhost/loopback, private/site-local addresses), see @@ -245,15 +311,11 @@ public void configure(Config conf) { builder.addNetworkInterceptor(new HTTPHeadersInterceptor()); } - if (ConfUtils.getBoolean(conf, "http.trust.everything", true)) { + if (trustEverything) { builder.sslSocketFactory(trustAllSslSocketFactory, (X509TrustManager) trustAllCerts[0]); - builder.hostnameVerifier( - new HostnameVerifier() { - @Override - public boolean verify(String hostname, SSLSession session) { - return true; - } - }); + } + if (!verifyHostnames) { + builder.hostnameVerifier((hostname, session) -> true); } builder.eventListenerFactory( @@ -285,6 +347,15 @@ public EventListener create(Call call) { } private void addCookiesToRequest(Builder rb, String url, Metadata md) { + if (!sendCredentials) { + if (withheldCookiesLogged.compareAndSet(false, true)) { + LOG.warn( + "Cookies are withheld because the servers are not authenticated " + + "(http.trust.everything). Set http.credentials.allow.insecure to " + + "true to send them anyway."); + } + return; + } final String[] cookieStrings = md.getValues(RESPONSE_COOKIES_HEADER, protocolMetadataPrefix); if (cookieStrings == null || cookieStrings.length == 0) { @@ -337,12 +408,33 @@ private URL getCookieOrigin(Metadata md, String url) { } } + /** + * Returns true when the header carries credentials which must not be disclosed to servers that + * were not authenticated. Covers the standard credential headers plus any header commonly used + * for API keys; cookie headers are handled separately in {@link #addCookiesToRequest}. + */ + private static boolean isCredentialHeader(String name) { + if (name == null) { + return false; + } + final String normalised = name.trim().toLowerCase(Locale.ROOT); + return CREDENTIAL_HEADERS.contains(normalised); + } + protected void addHeadersToRequest(Builder rb, Metadata md) { final String[] headerStrings = md.getValues(SET_HEADER_BY_REQUEST, protocolMetadataPrefix); if (headerStrings != null && headerStrings.length > 0) { for (String hs : headerStrings) { KeyValue h = KeyValue.build(hs); + if (!sendCredentials && isCredentialHeader(h.getKey())) { + LOG.warn( + "Header {} set by request is withheld because the servers are not " + + "authenticated (http.trust.everything). Set " + + "http.credentials.allow.insecure to true to send it anyway.", + h.getKey()); + continue; + } rb.addHeader(h.getKey(), h.getValue()); } } diff --git a/core/src/main/resources/crawler-default.yaml b/core/src/main/resources/crawler-default.yaml index 6945b2c4c..feb7f5752 100644 --- a/core/src/main/resources/crawler-default.yaml +++ b/core/src/main/resources/crawler-default.yaml @@ -149,6 +149,30 @@ config: # Follow redirect HTTP responses: http.allow.redirects: false + # Accept any TLS certificate, including self-signed, expired or otherwise + # invalid ones (okhttp protocol only)? The certificate chains are accepted + # without validation, i.e. the servers are not authenticated: anyone able to + # answer for the host name receives everything sent to them. Needed for + # crawling hosts with unvalidatable certificates, e.g. self-signed ones, but + # also disables protection against man-in-the-middle attacks. Credentials + # (basic auth, credential headers, cookies) are withheld over such + # connections unless http.credentials.allow.insecure is set to true. + http.trust.everything: false + + # Check that the certificate presented by the server matches the host name + # contacted (okhttp protocol only)? Disabling this is independent of + # http.trust.everything: a valid certificate for a different name is then + # accepted. Hostname verification does not need to be disabled for hosts + # with self-signed certificates when http.trust.everything is enabled. + http.verify.hostnames: true + + # Send credentials (basic auth, credential headers, cookies) even over + # connections whose server certificate was not validated, i.e. when + # http.trust.everything is true (okhttp protocol only)? Without it, + # credentials are withheld so that they are not disclosed to servers which + # were never authenticated. + http.credentials.allow.insecure: false + # IP address filtering (okhttp protocol only). Optionally limit or block # connections to IP address ranges once the host name has been resolved. This # prevents information leakage to a public index when a DNS entry points to a diff --git a/core/src/test/java/org/apache/stormcrawler/protocol/okhttp/OkHttpTrustEverythingTest.java b/core/src/test/java/org/apache/stormcrawler/protocol/okhttp/OkHttpTrustEverythingTest.java new file mode 100644 index 000000000..3ff51de76 --- /dev/null +++ b/core/src/test/java/org/apache/stormcrawler/protocol/okhttp/OkHttpTrustEverythingTest.java @@ -0,0 +1,266 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.stormcrawler.protocol.okhttp; + +import static com.github.tomakehurst.wiremock.client.WireMock.any; +import static com.github.tomakehurst.wiremock.client.WireMock.anyUrl; +import static com.github.tomakehurst.wiremock.client.WireMock.equalTo; +import static com.github.tomakehurst.wiremock.client.WireMock.getRequestedFor; +import static com.github.tomakehurst.wiremock.client.WireMock.ok; +import static com.github.tomakehurst.wiremock.client.WireMock.urlPathEqualTo; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import com.github.tomakehurst.wiremock.WireMockServer; +import com.github.tomakehurst.wiremock.common.ConsoleNotifier; +import com.github.tomakehurst.wiremock.core.WireMockConfiguration; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.util.Base64; +import java.util.List; +import javax.net.ssl.SSLHandshakeException; +import javax.net.ssl.SSLPeerUnverifiedException; +import org.apache.storm.Config; +import org.apache.stormcrawler.Metadata; +import org.apache.stormcrawler.protocol.ProtocolResponse; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; + +/** + * Regression tests for https://github.com/apache/stormcrawler/issues/2079: the trust-all + * configuration of the okhttp protocol must be an explicit choice, must not disable hostname + * verification and must not disclose credentials to servers which are not authenticated. + */ +class OkHttpTrustEverythingTest { + + private static final String KEYSTORE_PASSWORD = "changeit"; + + /** Certificate issued for localhost: valid for the host the tests connect to. */ + private static final String LOCALHOST_KEYSTORE = "/ssl/localhost.p12"; + + /** Certificate issued for another host name: trusted under trust-all, wrong name. */ + private static final String OTHERHOST_KEYSTORE = "/ssl/otherhost.p12"; + + private WireMockServer server; + + @AfterEach + void stopServer() { + if (server != null) { + server.stop(); + server = null; + } + } + + @Test + void trustAllContextUsesTls() { + assertEquals( + "TLS", + HttpProtocol.trustAllSslContext.getProtocol(), + "the trust-all SSLContext should be a TLS context"); + } + + @Test + void selfSignedCertificateRejectedByDefault() throws Exception { + // http.trust.everything defaults to false: an unvalidatable certificate + // must not be accepted + startServer(LOCALHOST_KEYSTORE); + final HttpProtocol protocol = protocol(config()); + assertThrows( + SSLHandshakeException.class, + () -> fetch(protocol, "/default"), + "self-signed certificates must be rejected by default"); + } + + @Test + void trustAllFetchesServerWithSelfSignedCertificate() throws Exception { + final Config conf = config(); + conf.put("http.trust.everything", true); + startServer(LOCALHOST_KEYSTORE); + final ProtocolResponse response = fetch(protocol(conf), "/trustall"); + assertEquals(200, response.getStatusCode(), "the self-signed certificate is accepted"); + } + + @Test + void hostnameIsStillVerified() throws Exception { + // the certificate is issued for another host name: trusting any + // certificate must not imply accepting any name + final Config conf = config(); + conf.put("http.trust.everything", true); + startServer(OTHERHOST_KEYSTORE); + assertThrows( + SSLPeerUnverifiedException.class, + () -> fetch(protocol(conf), "/hostname"), + "the hostname verifier should not accept any name unconditionally"); + } + + @Test + void hostnameVerificationCanBeDisabledSeparately() throws Exception { + final Config conf = config(); + conf.put("http.trust.everything", true); + conf.put("http.verify.hostnames", false); + startServer(OTHERHOST_KEYSTORE); + final ProtocolResponse response = fetch(protocol(conf), "/nohostnamecheck"); + assertEquals(200, response.getStatusCode(), "the name mismatch is accepted as configured"); + } + + @Test + void basicAuthIsWithheldFromUnauthenticatedServers() throws Exception { + final Config conf = config(); + conf.put("http.trust.everything", true); + conf.put("http.basicauth.user", "user"); + conf.put("http.basicauth.password", "secret"); + startServer(LOCALHOST_KEYSTORE); + fetch(protocol(conf), "/basicauth"); + server.verify( + 1, getRequestedFor(urlPathEqualTo("/basicauth")).withoutHeader("Authorization")); + } + + @Test + void basicAuthIsSentWhenExplicitlyAllowed() throws Exception { + final Config conf = config(); + conf.put("http.trust.everything", true); + conf.put("http.credentials.allow.insecure", true); + conf.put("http.basicauth.user", "user"); + conf.put("http.basicauth.password", "secret"); + startServer(LOCALHOST_KEYSTORE); + fetch(protocol(conf), "/basicauth"); + final String expected = + "Basic " + + Base64.getEncoder() + .encodeToString("user:secret".getBytes(StandardCharsets.UTF_8)); + server.verify( + 1, + getRequestedFor(urlPathEqualTo("/basicauth")) + .withHeader("Authorization", equalTo(expected))); + } + + @Test + void credentialCustomHeadersAreWithheldFromUnauthenticatedServers() throws Exception { + final Config conf = config(); + conf.put("http.trust.everything", true); + conf.put("http.custom.headers", List.of("X-Api-Key=s3cret", "X-Trace=public")); + startServer(LOCALHOST_KEYSTORE); + fetch(protocol(conf), "/customheaders"); + server.verify( + 1, getRequestedFor(urlPathEqualTo("/customheaders")).withoutHeader("X-Api-Key")); + server.verify( + 1, + getRequestedFor(urlPathEqualTo("/customheaders")) + .withHeader("X-Trace", equalTo("public"))); + } + + @Test + void credentialCustomHeadersAreSentWhenExplicitlyAllowed() throws Exception { + final Config conf = config(); + conf.put("http.trust.everything", true); + conf.put("http.credentials.allow.insecure", true); + conf.put("http.custom.headers", List.of("X-Api-Key=s3cret", "X-Trace=public")); + startServer(LOCALHOST_KEYSTORE); + fetch(protocol(conf), "/customheaders"); + server.verify( + 1, + getRequestedFor(urlPathEqualTo("/customheaders")) + .withHeader("X-Api-Key", equalTo("s3cret"))); + } + + @Test + void cookiesAreWithheldFromUnauthenticatedServers() throws Exception { + final Config conf = config(); + conf.put("http.trust.everything", true); + conf.put("http.use.cookies", true); + startServer(LOCALHOST_KEYSTORE); + fetch(protocol(conf), "/cookies", metadata()); + server.verify(1, getRequestedFor(urlPathEqualTo("/cookies")).withoutHeader("Cookie")); + } + + @Test + void cookiesAreSentWhenExplicitlyAllowed() throws Exception { + final Config conf = config(); + conf.put("http.trust.everything", true); + conf.put("http.credentials.allow.insecure", true); + conf.put("http.use.cookies", true); + startServer(LOCALHOST_KEYSTORE); + fetch(protocol(conf), "/cookies", metadata()); + server.verify( + 1, + getRequestedFor(urlPathEqualTo("/cookies")).withHeader("Cookie", equalTo("sid=x"))); + } + + /** Metadata as an outlink would inherit it, with a cookie scoped to the server. */ + private Metadata metadata() { + final Metadata md = new Metadata(); + md.setValue("protocol.set-cookie", "sid=x; Path=/"); + md.setValue("protocol.set-cookie-origin", "https://localhost:" + server.httpsPort() + "/"); + return md; + } + + private Config config() { + final Config conf = new Config(); + conf.put("http.agent.name", "test"); + conf.put("http.agent.version", "1.0"); + conf.put("http.agent.description", "test"); + conf.put("http.agent.url", "http://test.example.com"); + conf.put("http.agent.email", "test@example.com"); + conf.put("protocol.md.prefix", "protocol."); + return conf; + } + + private HttpProtocol protocol(Config conf) { + final HttpProtocol protocol = new HttpProtocol(); + protocol.configure(conf); + return protocol; + } + + private ProtocolResponse fetch(HttpProtocol protocol, String path) throws Exception { + return fetch(protocol, path, new Metadata()); + } + + private ProtocolResponse fetch(HttpProtocol protocol, String path, Metadata md) + throws Exception { + return protocol.getProtocolOutput("https://localhost:" + server.httpsPort() + path, md); + } + + /** + * Starts an HTTPS server on a random port presenting the certificate of the keystore. The + * keystore is copied to a temporary file as WireMock reads it from the file system. + */ + private void startServer(String keystoreResource) throws Exception { + final Path keystoreFile = Files.createTempFile("wiremock-keystore", ".p12"); + keystoreFile.toFile().deleteOnExit(); + try (InputStream in = + OkHttpTrustEverythingTest.class.getResourceAsStream(keystoreResource)) { + Files.copy(in, keystoreFile, StandardCopyOption.REPLACE_EXISTING); + } + + server = + new WireMockServer( + WireMockConfiguration.options() + .dynamicPort() + .dynamicHttpsPort() + .keystorePath(keystoreFile.toAbsolutePath().toString()) + .keystorePassword(KEYSTORE_PASSWORD) + .keyManagerPassword(KEYSTORE_PASSWORD) + .keystoreType("PKCS12") + .notifier(new ConsoleNotifier(false))); + server.start(); + server.stubFor(any(anyUrl()).willReturn(ok("Success!"))); + } +} diff --git a/core/src/test/resources/ssl/localhost.p12 b/core/src/test/resources/ssl/localhost.p12 new file mode 100644 index 000000000..222897c69 Binary files /dev/null and b/core/src/test/resources/ssl/localhost.p12 differ diff --git a/core/src/test/resources/ssl/otherhost.p12 b/core/src/test/resources/ssl/otherhost.p12 new file mode 100644 index 000000000..d9543037e Binary files /dev/null and b/core/src/test/resources/ssl/otherhost.p12 differ diff --git a/docs/src/main/asciidoc/configuration.adoc b/docs/src/main/asciidoc/configuration.adoc index 047a86ea4..619bea3ff 100644 --- a/docs/src/main/asciidoc/configuration.adoc +++ b/docs/src/main/asciidoc/configuration.adoc @@ -260,7 +260,9 @@ header. | http.content.partial.as.trimmed | false | Accepts partially fetched content in OKHTTP. | http.filter.ipaddress.include | - | (OkHttp only) Comma-separated list (or YAML list) of allowed IP ranges. If empty, all addresses are allowed unless excluded. See <>. | http.filter.ipaddress.exclude | - | (OkHttp only) Comma-separated list (or YAML list) of blocked IP ranges. See <>. -| http.trust.everything | true | If true, trust all SSL/TLS connections. +| http.trust.everything | false | (OkHttp only) If true, accept any TLS certificate, including self-signed, expired or otherwise invalid ones. The servers are then not authenticated: anyone able to answer for the host name receives everything sent to them. Credentials (basic auth, credential headers, cookies) are withheld over such connections unless `http.credentials.allow.insecure` is set to true. +| http.verify.hostnames | true | (OkHttp only) If true, check that the certificate presented by the server matches the host name contacted. Independent of `http.trust.everything`: a valid certificate for a different name is accepted when this is false. +| http.credentials.allow.insecure | false | (OkHttp only) If true, send credentials (basic auth, credential headers, cookies) even over connections whose server certificate was not validated, i.e. when `http.trust.everything` is true. | topology.message.timeout.secs | -1 | OKHTTP message timeout. |=== diff --git a/docs/src/main/asciidoc/extending.adoc b/docs/src/main/asciidoc/extending.adoc index 59ece5f50..548196eac 100644 --- a/docs/src/main/asciidoc/extending.adoc +++ b/docs/src/main/asciidoc/extending.adoc @@ -392,7 +392,7 @@ Keep the pool size (idle + active) under 1000. For large crawls, increase `proto ==== SSL/TLS -By default, StormCrawler's OkHttp protocol trusts all certificates (`http.trust.everything: true`). For production crawls over untrusted networks, consider setting this to `false` and configuring a proper trust store. +By default, StormCrawler's OkHttp protocol validates the certificate chains of HTTPS servers and checks that the certificate matches the host name contacted (`http.trust.everything: false`, `http.verify.hostnames: true`). For crawls of hosts whose certificates cannot be validated, e.g. self-signed ones, set `http.trust.everything` to `true`: the servers are then not authenticated, so credentials (basic auth, credential headers in `http.custom.headers`, cookies) are withheld over such connections unless `http.credentials.allow.insecure` is set to `true`. Hostname checking is a separate decision and can be disabled with `http.verify.hostnames: false`. ==== Authentication @@ -404,7 +404,7 @@ http.basicauth.user: "username" http.basicauth.password: "password" ---- -These credentials are sent as an `Authorization` header with every request. For per-site authentication, use metadata-driven headers instead (see <> in the Internals section). +These credentials are sent as an `Authorization` header with every request to servers authenticated by their TLS certificate; they are withheld when `http.trust.everything` is enabled, unless `http.credentials.allow.insecure` is set to `true`. For per-site authentication, use metadata-driven headers instead (see <> in the Internals section). ==== Proxy Authentication diff --git a/external/opensearch/archetype/src/main/resources/archetype-resources/crawler-conf.yaml b/external/opensearch/archetype/src/main/resources/archetype-resources/crawler-conf.yaml index f62103faf..f2d9860f5 100644 --- a/external/opensearch/archetype/src/main/resources/archetype-resources/crawler-conf.yaml +++ b/external/opensearch/archetype/src/main/resources/archetype-resources/crawler-conf.yaml @@ -72,6 +72,12 @@ config: http.protocol.implementation: "org.apache.stormcrawler.protocol.okhttp.HttpProtocol" https.protocol.implementation: "org.apache.stormcrawler.protocol.okhttp.HttpProtocol" + # Accept any TLS certificate, e.g. self-signed ones? The servers are not + # authenticated then; credentials (basic auth, credential headers, cookies) + # are withheld over such connections unless + # http.credentials.allow.insecure is set to true. + http.trust.everything: false + # The maximum number of bytes for returned HTTP response bodies. # The fetched page will be trimmed to 65KB in this case # Set -1 to disable the limit. diff --git a/external/solr/archetype/src/main/resources/archetype-resources/crawler-conf.yaml b/external/solr/archetype/src/main/resources/archetype-resources/crawler-conf.yaml index f62103faf..f2d9860f5 100644 --- a/external/solr/archetype/src/main/resources/archetype-resources/crawler-conf.yaml +++ b/external/solr/archetype/src/main/resources/archetype-resources/crawler-conf.yaml @@ -72,6 +72,12 @@ config: http.protocol.implementation: "org.apache.stormcrawler.protocol.okhttp.HttpProtocol" https.protocol.implementation: "org.apache.stormcrawler.protocol.okhttp.HttpProtocol" + # Accept any TLS certificate, e.g. self-signed ones? The servers are not + # authenticated then; credentials (basic auth, credential headers, cookies) + # are withheld over such connections unless + # http.credentials.allow.insecure is set to true. + http.trust.everything: false + # The maximum number of bytes for returned HTTP response bodies. # The fetched page will be trimmed to 65KB in this case # Set -1 to disable the limit.