From 10c7ba3e8820f706e04ed620e544eb5bfb1e32e9 Mon Sep 17 00:00:00 2001 From: abhinav-phi Date: Sun, 6 Sep 2026 04:45:19 +0530 Subject: [PATCH 1/2] OKHttp: authenticated TLS by default, credentials withheld from unauthenticated servers (#2079) http.trust.everything shipped enabled with a static SSLContext('SSL'), a no-op trust manager and a hostname verifier that accepted any name: anybody able to answer for a host name received the Authorization header built from http.basicauth.*, credential headers from http.custom.headers and the replayed cookies. - http.trust.everything now defaults to false: certificate chains are validated and the servers are authenticated, as a normal TLS client would do. Self-signed intranet hosts need the operator to opt in - the trust-all context is built from 'TLS' instead of 'SSL' - hostname verification is a separate decision (http.verify.hostnames, default true) and is no longer disabled as a side effect of trusting any certificate - credentials are withheld over connections whose server certificate was not validated, unless http.credentials.allow.insecure is set: basic auth, credential headers (authorization, proxy-authorization, cookie, x-api-key) in http.custom.headers, headers set by request and cookies are all covered - every insecure state logs a WARN naming the keys, and the keys are surfaced in crawler-default.yaml and the archetype crawler-conf.yaml files --- .../archetype-resources/crawler-conf.yaml | 6 + .../protocol/okhttp/HttpProtocol.java | 130 ++++++-- core/src/main/resources/crawler-default.yaml | 24 ++ .../okhttp/OkHttpTrustEverythingTest.java | 289 ++++++++++++++++++ core/src/test/resources/ssl/localhost.p12 | Bin 0 -> 2606 bytes core/src/test/resources/ssl/otherhost.p12 | Bin 0 -> 2622 bytes docs/src/main/asciidoc/configuration.adoc | 4 +- docs/src/main/asciidoc/extending.adoc | 4 +- .../archetype-resources/crawler-conf.yaml | 6 + .../archetype-resources/crawler-conf.yaml | 6 + 10 files changed, 447 insertions(+), 22 deletions(-) create mode 100644 core/src/test/java/org/apache/stormcrawler/protocol/okhttp/OkHttpTrustEverythingTest.java create mode 100644 core/src/test/resources/ssl/localhost.p12 create mode 100644 core/src/test/resources/ssl/otherhost.p12 diff --git a/archetype/src/main/resources/archetype-resources/crawler-conf.yaml b/archetype/src/main/resources/archetype-resources/crawler-conf.yaml index bc57e3714..6db628282 100644 --- a/archetype/src/main/resources/archetype-resources/crawler-conf.yaml +++ b/archetype/src/main/resources/archetype-resources/crawler-conf.yaml @@ -72,6 +72,12 @@ config: http.protocol.implementation: "org.apache.stormcrawler.protocol.okhttp.HttpProtocol" https.protocol.implementation: "org.apache.stormcrawler.protocol.okhttp.HttpProtocol" + # Accept any TLS certificate, e.g. self-signed ones? The servers are not + # authenticated then; credentials (basic auth, credential headers, cookies) + # are withheld over such connections unless + # http.credentials.allow.insecure is set to true. + http.trust.everything: false + # The maximum number of bytes for returned HTTP response bodies. # The fetched page will be trimmed to 65KB in this case # Set -1 to disable the limit. diff --git a/core/src/main/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocol.java b/core/src/main/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocol.java index 90cb4d742..f4cc662de 100644 --- a/core/src/main/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocol.java +++ b/core/src/main/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocol.java @@ -34,12 +34,11 @@ import java.util.Map; import java.util.Objects; import java.util.Optional; +import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; -import javax.net.ssl.HostnameVerifier; import javax.net.ssl.SSLContext; -import javax.net.ssl.SSLSession; import javax.net.ssl.SSLSocketFactory; import javax.net.ssl.TrustManager; import javax.net.ssl.X509TrustManager; @@ -108,6 +107,23 @@ public class HttpProtocol extends AbstractHttpProtocol { // makes sure that a missing cookie origin is reported once and not for every url private final AtomicBoolean missingCookieOriginLogged = new AtomicBoolean(); + // makes sure that withheld cookies are reported once and not for every url + private final AtomicBoolean withheldCookiesLogged = new AtomicBoolean(); + + // whether credentials (basic auth, credential headers, cookies) may be sent: + // false when the servers are not authenticated and + // http.credentials.allow.insecure is not enabled + private boolean sendCredentials = true; + + /** Header names carrying credentials, see {@link #isCredentialHeader(String)}. */ + private static final Set CREDENTIAL_HEADERS = + Set.of( + HttpHeaders.AUTHORIZATION.toLowerCase(Locale.ROOT), + HttpHeaders.PROXY_AUTHORIZATION.toLowerCase(Locale.ROOT), + // the cookie header is not a constant in HttpHeaders + "cookie", + "x-api-key"); + private OkHttpClient.Builder builder; private static final TrustManager[] trustAllCerts = @@ -130,11 +146,12 @@ public java.security.cert.X509Certificate[] getAcceptedIssuers() { } }; - private static final SSLContext trustAllSslContext; + // package-private so that the OkHttpTrustEverythingTest can check the protocol + static final SSLContext trustAllSslContext; static { try { - trustAllSslContext = SSLContext.getInstance("SSL"); + trustAllSslContext = SSLContext.getInstance("TLS"); trustAllSslContext.init(null, trustAllCerts, new java.security.SecureRandom()); } catch (Exception e) { throw new RuntimeException(e); @@ -158,6 +175,37 @@ public void configure(Config conf) { this.partialContentAsTrimmed = ConfUtils.getBoolean(conf, "http.content.partial.as.trimmed", false); + /* + * certificate trust and hostname verification are separate decisions: + * accepting any certificate does not imply accepting any name + */ + final boolean trustEverything = ConfUtils.getBoolean(conf, "http.trust.everything", false); + final boolean verifyHostnames = ConfUtils.getBoolean(conf, "http.verify.hostnames", true); + // credentials are withheld over connections whose server certificate + // was not validated, unless explicitly opted in + final boolean insecureCredentialsAllowed = + ConfUtils.getBoolean(conf, "http.credentials.allow.insecure", false); + this.sendCredentials = !trustEverything || insecureCredentialsAllowed; + + if (trustEverything) { + LOG.warn( + "http.trust.everything is enabled: TLS certificate chains are accepted without " + + "validation, the identity of the servers is not authenticated. Anybody " + + "able to answer for the host name receives everything sent to them."); + } + if (!verifyHostnames) { + LOG.warn( + "http.verify.hostnames is disabled: the certificates are not checked against " + + "the host name either, the identity of the servers is not authenticated."); + } + if (trustEverything && !sendCredentials) { + LOG.warn( + "Credentials configured with http.basicauth.*, credential headers in " + + "http.custom.headers and cookies are withheld because the servers " + + "are not authenticated. Set http.credentials.allow.insecure to true " + + "to send them anyway."); + } + builder = new OkHttpClient.Builder() .retryOnConnectionFailure( @@ -223,15 +271,33 @@ public void configure(Config conf) { // use a basic auth? if (StringUtils.isNotBlank(basicAuthUser)) { final String basicAuthPass = ConfUtils.getString(conf, "http.basicauth.password", ""); - final String encoding = - Base64.getEncoder() - .encodeToString( - (basicAuthUser + ":" + basicAuthPass) - .getBytes(StandardCharsets.UTF_8)); - customRequestHeaders.add(new KeyValue(HttpHeaders.AUTHORIZATION, "Basic " + encoding)); + if (sendCredentials) { + final String encoding = + Base64.getEncoder() + .encodeToString( + (basicAuthUser + ":" + basicAuthPass) + .getBytes(StandardCharsets.UTF_8)); + customRequestHeaders.add( + new KeyValue(HttpHeaders.AUTHORIZATION, "Basic " + encoding)); + } else { + LOG.warn( + "Basic authentication configured with http.basicauth.user is withheld " + + "because the servers are not authenticated (http.trust.everything). " + + "Set http.credentials.allow.insecure to true to send it anyway."); + } } - customHeaders.forEach(customRequestHeaders::add); + for (KeyValue customHeader : customHeaders) { + if (!sendCredentials && isCredentialHeader(customHeader.getKey())) { + LOG.warn( + "Custom header {} is withheld because the servers are not authenticated " + + "(http.trust.everything). Set http.credentials.allow.insecure to " + + "true to send it anyway.", + customHeader.getKey()); + continue; + } + customRequestHeaders.add(customHeader); + } // optionally block connections to forbidden IP address ranges // (e.g. localhost/loopback, private/site-local addresses), see @@ -245,15 +311,11 @@ public void configure(Config conf) { builder.addNetworkInterceptor(new HTTPHeadersInterceptor()); } - if (ConfUtils.getBoolean(conf, "http.trust.everything", true)) { + if (trustEverything) { builder.sslSocketFactory(trustAllSslSocketFactory, (X509TrustManager) trustAllCerts[0]); - builder.hostnameVerifier( - new HostnameVerifier() { - @Override - public boolean verify(String hostname, SSLSession session) { - return true; - } - }); + } + if (!verifyHostnames) { + builder.hostnameVerifier((hostname, session) -> true); } builder.eventListenerFactory( @@ -285,6 +347,15 @@ public EventListener create(Call call) { } private void addCookiesToRequest(Builder rb, String url, Metadata md) { + if (!sendCredentials) { + if (withheldCookiesLogged.compareAndSet(false, true)) { + LOG.warn( + "Cookies are withheld because the servers are not authenticated " + + "(http.trust.everything). Set http.credentials.allow.insecure to " + + "true to send them anyway."); + } + return; + } final String[] cookieStrings = md.getValues(RESPONSE_COOKIES_HEADER, protocolMetadataPrefix); if (cookieStrings == null || cookieStrings.length == 0) { @@ -337,12 +408,33 @@ private URL getCookieOrigin(Metadata md, String url) { } } + /** + * Returns true when the header carries credentials which must not be disclosed to servers that + * were not authenticated. Covers the standard credential headers plus any header commonly used + * for API keys; cookie headers are handled separately in {@link #addCookiesToRequest}. + */ + private static boolean isCredentialHeader(String name) { + if (name == null) { + return false; + } + final String normalised = name.trim().toLowerCase(Locale.ROOT); + return CREDENTIAL_HEADERS.contains(normalised); + } + protected void addHeadersToRequest(Builder rb, Metadata md) { final String[] headerStrings = md.getValues(SET_HEADER_BY_REQUEST, protocolMetadataPrefix); if (headerStrings != null && headerStrings.length > 0) { for (String hs : headerStrings) { KeyValue h = KeyValue.build(hs); + if (!sendCredentials && isCredentialHeader(h.getKey())) { + LOG.warn( + "Header {} set by request is withheld because the servers are not " + + "authenticated (http.trust.everything). Set " + + "http.credentials.allow.insecure to true to send it anyway.", + h.getKey()); + continue; + } rb.addHeader(h.getKey(), h.getValue()); } } diff --git a/core/src/main/resources/crawler-default.yaml b/core/src/main/resources/crawler-default.yaml index 6945b2c4c..feb7f5752 100644 --- a/core/src/main/resources/crawler-default.yaml +++ b/core/src/main/resources/crawler-default.yaml @@ -149,6 +149,30 @@ config: # Follow redirect HTTP responses: http.allow.redirects: false + # Accept any TLS certificate, including self-signed, expired or otherwise + # invalid ones (okhttp protocol only)? The certificate chains are accepted + # without validation, i.e. the servers are not authenticated: anyone able to + # answer for the host name receives everything sent to them. Needed for + # crawling hosts with unvalidatable certificates, e.g. self-signed ones, but + # also disables protection against man-in-the-middle attacks. Credentials + # (basic auth, credential headers, cookies) are withheld over such + # connections unless http.credentials.allow.insecure is set to true. + http.trust.everything: false + + # Check that the certificate presented by the server matches the host name + # contacted (okhttp protocol only)? Disabling this is independent of + # http.trust.everything: a valid certificate for a different name is then + # accepted. Hostname verification does not need to be disabled for hosts + # with self-signed certificates when http.trust.everything is enabled. + http.verify.hostnames: true + + # Send credentials (basic auth, credential headers, cookies) even over + # connections whose server certificate was not validated, i.e. when + # http.trust.everything is true (okhttp protocol only)? Without it, + # credentials are withheld so that they are not disclosed to servers which + # were never authenticated. + http.credentials.allow.insecure: false + # IP address filtering (okhttp protocol only). Optionally limit or block # connections to IP address ranges once the host name has been resolved. This # prevents information leakage to a public index when a DNS entry points to a diff --git a/core/src/test/java/org/apache/stormcrawler/protocol/okhttp/OkHttpTrustEverythingTest.java b/core/src/test/java/org/apache/stormcrawler/protocol/okhttp/OkHttpTrustEverythingTest.java new file mode 100644 index 000000000..82d1c136c --- /dev/null +++ b/core/src/test/java/org/apache/stormcrawler/protocol/okhttp/OkHttpTrustEverythingTest.java @@ -0,0 +1,289 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.stormcrawler.protocol.okhttp; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import com.sun.net.httpserver.HttpExchange; +import com.sun.net.httpserver.HttpHandler; +import com.sun.net.httpserver.HttpsConfigurator; +import com.sun.net.httpserver.HttpsServer; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.InetSocketAddress; +import java.nio.charset.StandardCharsets; +import java.security.KeyStore; +import java.util.Base64; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import javax.net.ssl.KeyManagerFactory; +import javax.net.ssl.SSLContext; +import org.apache.storm.Config; +import org.apache.stormcrawler.Metadata; +import org.apache.stormcrawler.protocol.ProtocolResponse; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; + +/** + * Regression tests for https://github.com/apache/stormcrawler/issues/2079: the trust-all + * configuration of the okhttp protocol must be an explicit choice, must not disable hostname + * verification and must not disclose credentials to servers which are not authenticated. + */ +class OkHttpTrustEverythingTest { + + private static final char[] KEYSTORE_PASSWORD = "changeit".toCharArray(); + + /** Certificate issued for localhost: valid for the host the tests connect to. */ + private static final String LOCALHOST_KEYSTORE = "/ssl/localhost.p12"; + + /** Certificate issued for another host name: trusted under trust-all, wrong name. */ + private static final String OTHERHOST_KEYSTORE = "/ssl/otherhost.p12"; + + private HttpsServer server; + + private RecordingHandler handler; + + @AfterEach + void stopServer() { + if (server != null) { + server.stop(0); + server = null; + handler = null; + } + } + + @Test + void trustAllContextUsesTls() { + assertEquals( + "TLS", + HttpProtocol.trustAllSslContext.getProtocol(), + "the trust-all SSLContext should be a TLS context"); + } + + @Test + void selfSignedCertificateRejectedByDefault() throws Exception { + // http.trust.everything defaults to false: an unvalidatable certificate + // must not be accepted. Depending on the platform the handshake fails or + // the connection is dropped while it is retried. + startServer(LOCALHOST_KEYSTORE); + final HttpProtocol protocol = protocol(config()); + assertThrows( + Exception.class, + () -> fetch(protocol, "/default"), + "self-signed certificates must be rejected by default"); + } + + @Test + void trustAllFetchesServerWithSelfSignedCertificate() throws Exception { + final Config conf = config(); + conf.put("http.trust.everything", true); + startServer(LOCALHOST_KEYSTORE); + final ProtocolResponse response = fetch(protocol(conf), "/trustall"); + assertEquals(200, response.getStatusCode(), "the self-signed certificate is accepted"); + } + + @Test + void hostnameIsStillVerified() throws Exception { + // the certificate is issued for another host name: trusting any + // certificate must not imply accepting any name. Depending on the + // platform the failed verification surfaces as an SSL exception or the + // connection is dropped while it is retried. + final Config conf = config(); + conf.put("http.trust.everything", true); + startServer(OTHERHOST_KEYSTORE); + assertThrows( + Exception.class, + () -> fetch(protocol(conf), "/hostname"), + "the hostname verifier should not accept any name unconditionally"); + } + + @Test + void hostnameVerificationCanBeDisabledSeparately() throws Exception { + final Config conf = config(); + conf.put("http.trust.everything", true); + conf.put("http.verify.hostnames", false); + startServer(OTHERHOST_KEYSTORE); + final ProtocolResponse response = fetch(protocol(conf), "/nohostnamecheck"); + assertEquals(200, response.getStatusCode(), "the name mismatch is accepted as configured"); + } + + @Test + void basicAuthIsWithheldFromUnauthenticatedServers() throws Exception { + final Config conf = config(); + conf.put("http.trust.everything", true); + conf.put("http.basicauth.user", "user"); + conf.put("http.basicauth.password", "secret"); + startServer(LOCALHOST_KEYSTORE); + final ProtocolResponse response = fetch(protocol(conf), "/basicauth"); + assertEquals(200, response.getStatusCode(), "the connection must succeed"); + assertNull( + handler.lastHeaders.get("authorization"), + "credentials must not be sent to unauthenticated servers"); + } + + @Test + void basicAuthIsSentWhenExplicitlyAllowed() throws Exception { + final Config conf = config(); + conf.put("http.trust.everything", true); + conf.put("http.credentials.allow.insecure", true); + conf.put("http.basicauth.user", "user"); + conf.put("http.basicauth.password", "secret"); + startServer(LOCALHOST_KEYSTORE); + fetch(protocol(conf), "/basicauth"); + final String expected = + "Basic " + + Base64.getEncoder() + .encodeToString("user:secret".getBytes(StandardCharsets.UTF_8)); + assertEquals( + expected, + handler.lastHeaders.get("authorization"), + "the opt-in sends credentials"); + } + + @Test + void credentialCustomHeadersAreWithheldFromUnauthenticatedServers() throws Exception { + final Config conf = config(); + conf.put("http.trust.everything", true); + conf.put("http.custom.headers", List.of("X-Api-Key=s3cret", "X-Trace=public")); + startServer(LOCALHOST_KEYSTORE); + final ProtocolResponse response = fetch(protocol(conf), "/customheaders"); + assertEquals(200, response.getStatusCode(), "the connection must succeed"); + assertNull( + handler.lastHeaders.get("x-api-key"), "credential headers must be withheld"); + assertEquals("public", handler.lastHeaders.get("x-trace"), "other headers are sent"); + } + + @Test + void credentialCustomHeadersAreSentWhenExplicitlyAllowed() throws Exception { + final Config conf = config(); + conf.put("http.trust.everything", true); + conf.put("http.credentials.allow.insecure", true); + conf.put("http.custom.headers", List.of("X-Api-Key=s3cret", "X-Trace=public")); + startServer(LOCALHOST_KEYSTORE); + fetch(protocol(conf), "/customheaders"); + assertEquals("s3cret", handler.lastHeaders.get("x-api-key"), "the opt-in sends credentials"); + assertEquals("public", handler.lastHeaders.get("x-trace")); + } + + @Test + void cookiesAreWithheldFromUnauthenticatedServers() throws Exception { + final Config conf = config(); + conf.put("http.trust.everything", true); + conf.put("http.use.cookies", true); + startServer(LOCALHOST_KEYSTORE); + final ProtocolResponse response = fetch(protocol(conf), "/cookies", metadata()); + assertEquals(200, response.getStatusCode(), "the connection must succeed"); + assertNull( + handler.lastHeaders.get("cookie"), + "cookies must not be sent to unauthenticated servers"); + } + + @Test + void cookiesAreSentWhenExplicitlyAllowed() throws Exception { + final Config conf = config(); + conf.put("http.trust.everything", true); + conf.put("http.credentials.allow.insecure", true); + conf.put("http.use.cookies", true); + startServer(LOCALHOST_KEYSTORE); + fetch(protocol(conf), "/cookies", metadata()); + assertEquals("sid=x", handler.lastHeaders.get("cookie"), "the opt-in sends cookies"); + } + + /** Metadata as an outlink would inherit it, with a cookie scoped to the server. */ + private Metadata metadata() { + final Metadata md = new Metadata(); + md.setValue("protocol.set-cookie", "sid=x; Path=/"); + md.setValue( + "protocol.set-cookie-origin", + "https://localhost:" + server.getAddress().getPort() + "/"); + return md; + } + + private Config config() { + final Config conf = new Config(); + conf.put("http.agent.name", "test"); + conf.put("http.agent.version", "1.0"); + conf.put("http.agent.description", "test"); + conf.put("http.agent.url", "http://test.example.com"); + conf.put("http.agent.email", "test@example.com"); + conf.put("protocol.md.prefix", "protocol."); + return conf; + } + + private HttpProtocol protocol(Config conf) { + final HttpProtocol protocol = new HttpProtocol(); + protocol.configure(conf); + return protocol; + } + + private ProtocolResponse fetch(HttpProtocol protocol, String path) throws Exception { + return fetch(protocol, path, new Metadata()); + } + + private ProtocolResponse fetch(HttpProtocol protocol, String path, Metadata md) + throws Exception { + return protocol.getProtocolOutput( + "https://localhost:" + server.getAddress().getPort() + path, md); + } + + /** Starts an HTTPS server on a random port presenting the certificate of the keystore. */ + private void startServer(String keystoreResource) throws Exception { + final KeyStore keyStore = KeyStore.getInstance("PKCS12"); + try (InputStream in = OkHttpTrustEverythingTest.class.getResourceAsStream(keystoreResource)) { + keyStore.load(in, KEYSTORE_PASSWORD); + } + final KeyManagerFactory keyManagerFactory = + KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm()); + keyManagerFactory.init(keyStore, KEYSTORE_PASSWORD); + final SSLContext sslContext = SSLContext.getInstance("TLS"); + sslContext.init(keyManagerFactory.getKeyManagers(), null, null); + + server = HttpsServer.create(new InetSocketAddress(java.net.InetAddress.getByName("127.0.0.1"), 0), 0); + server.setHttpsConfigurator(new HttpsConfigurator(sslContext)); + handler = new RecordingHandler(); + server.createContext("/", handler); + server.start(); + } + + /** Records the request headers of the last request received. */ + static class RecordingHandler implements HttpHandler { + + final Map lastHeaders = new ConcurrentHashMap<>(); + + @Override + public void handle(HttpExchange exchange) throws IOException { + for (Map.Entry> header : + exchange.getRequestHeaders().entrySet()) { + lastHeaders.put( + header.getKey().toLowerCase(Locale.ROOT), + String.join(",", header.getValue())); + } + final byte[] body = "Success!".getBytes(StandardCharsets.UTF_8); + exchange.getResponseHeaders().set("Content-Type", "text/html"); + exchange.sendResponseHeaders(200, body.length); + try (OutputStream out = exchange.getResponseBody()) { + out.write(body); + } + } + } +} diff --git a/core/src/test/resources/ssl/localhost.p12 b/core/src/test/resources/ssl/localhost.p12 new file mode 100644 index 0000000000000000000000000000000000000000..222897c6900140ce7e01bf4005095cedd48611ae GIT binary patch literal 2606 zcma)8S5%V?5=}?~gx*1E0i=kOj~?--Mi7y%BB6?av`|GvKp+tVgepbpB25&LCZP0= zgx)(+LXlntCE~JY_xSHV?LN%8GxyHS+uSn%jz*jU3;}TPVM@3-UI%|f52gZV;oxmh z9K7j_t^;tekN+jX@}M}_>oYpzEK(?G|2;)V1E$Er!PL*lzkvIHCSddc8bJPC(gByC zY=Sa-7jtZdd1jK1McYUYpwFj)aTFAyOcY=QKu1aauZ7 zaIFo$!zohuWUwN4un;B(#Xl)V{O z`}>B3>i*c)Z{9b2H@I#)P@P6KEt&6-;NG=Yn_HK3A-8Y?&zWrS*2^&uhB#VIjDuS^ z7+Q3cjXdQp9<(kV#Z6nRXw=)+quiS7ctm1vMNd1H6<)d6rWRt@E8bLB7(P}A;hP-%pFiv|A8U<{x6kY+y(JI~O}cIhT+(vSq%O{}>gTrD)qxHL6) zqczm3Q|0k-n*$tA>k=zVl=_9&WS!(i^&va0Hq%;AOi%9U&-h1HiH2#~9d~;lX2BL7 z2^ynyF-_=({Qh)?&}(t?-#k9+j>Z>MD`IL+luA<_vP}Rr`^v~9XIWP7%;DttJFge3 z{Pj=Y>lQ3T%MicVEv4E(df-%#Ttl|Mn~{?91wCXF^`(w%!;`hge+n$us#VIGIT=RaEZP>S?yn2(oFM+YTHzeDC)$LUo03XPh`& zaY=4)pPI5oTqrS_tU=N=Y@0JEKQ*Q@5NaLuE!Sg7oRknpVv88YwueHFsYlX{Xa|c8 zZJ(WZ(?)oe{A@lyP;%F;XZ#4NKRMyPAZa9^D}2?`u5Z2?pjznDdN;(CNv1PRxUe&6 zMtrog@)T8e*|Bo+)8QB3&Qna&*o4TDrk)oMs&_m*U*5(aD^T{bn?{jxa$_0VL`}|4 za~62&Jy%MXKvw>uL&%Y&?Nq#S6GSG3eiI+e26@p>ffvOT_wXTsFfCi1KEWTa1(-P( zXxFu34)vbN0#aeSokG@rjhBf>{=wz1gO4W_UVf&-Tg_iD7f`iGs#_-PpgLZmt~o^R zxbzqDj^W%CK9nkL?-#5rcnPzx*Z3%oLAEn>3JmxXsRdQHH=8!1m0t9jLu4XxA4XZq%lJ?@kK$6e1^nX-asi*XPpSMYEUoF1^Rc zYnRv;<$6?}pYPWkmn1JQ{YV-%?(=ex20sYwYd_G})&tGvF(9q@ObGCN4tE7=+Q-i< z1fZjSr||_3L0x?|jg00Qys3SaCf~0lxTG$!w}1A2e=+U6?pw~ott|htrdP3zbK(1l zyH{e5H@)C}#w`IV3ZmxuH4KbBu^T%@7l1Is!Pi+{Ep$k9m(FaeXlqOn_D4%n_)JHd z|BDm#BpGc$;E!Fx*<{%GSU?z%2gnX&dxlPD*#nTzf3~p8ut8aFIlDO^WR#FdMfuBs zoGemF2EakVfA&yOX5k=v01mQ60S2AH%HIn3f0^}8F)F^JXT<#-Zd%I4gMxtNQTqD- zn6>Zv{6;^i>o;zMYb|u^`;8DB#B%^Glgrk4SIux0lB)-J377j$!hA=s=#1)OrXZ~^;?{~-8%ILq?}RF-J4?LM($2@ za*PMl6(dt3)c2?1m*aFll)9 zh3p|_3Z#i#c>PQoLKnko8=6GCeXdqVjf=&gq;JMIyVQlCw$;4GMPBeLTf^jE?#ebr z@uh$*?dj9dcR=#wjoR(_U-~~uX(E12Bbi;(rX9$(=A^uL~7wCZbow1+% zV`>3Obu$ex5>5~ZHJLHzmt71U)_yOEY1eKQf`-vVKGkG}q?HNVT^FQ5Up>j1VM?u* z;uU#rIBFXxB&J|$I<4;GRQB_fdbYCR+6TmIbfk52ek$~~?-04LvB8#Yi<&<_a)Wpj z8B)|7aCF@X`^4CL)4({#J<1Fva>}Pml&%l95Tfp|6Jrg)8doPKy@`*%^H+Ycwnjog zP}KGKCtnm}9W6Afi0bQ@=`il%3X2Pa3Z>);#o~$Uwp7Oy#@uG8F(HZI^JWx{L9-;K zQ6#t{vw^vb+T4u~R;pt8ah`r>Q+W>tOqZ-(J1kWwSHpi6v5^q-bz-zmmY2>7-}YM| z-w&35NN%`XRCascr&R7IGS_=EagIqVOY?c0=rW;K)n#3AVyg>*O8dqM#7o~RqLnG6 zObm`mL3U=rtQ(&eJ$_M(4iZsSF|5A14qgcl$05J3>o3N;5#CHHovY$;LohunO3oNrstEL!(N9@^(ZHa)Yt8$<~?&-)c#i;G)J@10JQZ zUKz0qcIo(@?)(`W7Xlo^I6CADIM0rHnm+p)A?lPqE?lj8N;6QiZh!1r+Ag8(oKb`4 z8MG}}V-e55h`88cVLs%-%D0U3b)&kscbhJm1nI{plmOWf8EX-8Vxx?gSRXP+}%?Ty}8 m*~F70i{q%=p=^{KRku5fZ)|nqjy&p*J^D>ELrnj9^8Wy?KDQ$P literal 0 HcmV?d00001 diff --git a/core/src/test/resources/ssl/otherhost.p12 b/core/src/test/resources/ssl/otherhost.p12 new file mode 100644 index 0000000000000000000000000000000000000000..d9543037ea52ac1c7c235cffe715d193c7b9207d GIT binary patch literal 2622 zcma)8XE+;-7LG_n5Uci9BOyg1wD_o1pH)@6D6vJOMvYJ@NvzapscNmF_GkCA)hL2m zEgeRwS~Y6V()zgWbMMpd-e33rIOja)J?DL&^Xq*Mio&D@q@zbspbHF8*%*VE?=U(D zT`mRs5k!HGoWVmV3b^*aM&Lpa1^oOBes&h=49x##u`tmAb17i0Gf*An@rwb5p=?nq zf43|sB#29bKV}}&4zq?5(@sL0`Mhm5x2ZrNoE=CfhGJo0{FfpS3I?Er8K6!v26XQ9 zAUatPd$|jmt73c*FlU%bh@Bb!7_qlyBS^!fS{R`; zYp?9wec7i!U^|$oR_WD}{78at_g0JI+)AImnRL)K1Jg#G{1>L&C(^IfMdzzG2 z?@!h$fsDkr7I~Tmli#Lh4><546NmS3Qc6AX7n0?A$mV*rIZ}xRH)r3COk}RZ_=j**`ESmSE$W?rTYE{vu06hNIr{Z%&xE9l^yS10j zDOKIv>SQIJ3okM1(yS;)fXr?Pqe}bl$sOB#xIT#8<)q#OKkP3hPVSW3Bo%*Xfap@rd!>R?nPz?Ry`&M9jKsF1T9Fm)VV=`V@a_zQbLW&AyG|!hvQ|RBhdY_7ajj(iGee$ z!WTwg9^APk6gTqy)>~ESVf;=>N6K7is{hjA;>C0&`j)(@Vh7nn19UN@A*)-zC7~M- zWdqXov9yt{7aW$9LoSv|&>GXVO-}Ho`%nOZwq2I*P6t|!F9^P)sqF?Ioz|XJ`P7WQ z>(>7Re0-}V4o*YmHekXe`MU(ZJvy-^W?)Tu`-jX1&2lV5@`_x+NGva69tT%592<>k z{F0m8RMmWJZ8$4jo?>T!lX;Al)>wNWHnlyW6G#|odQ&ax$PP%`1X2n58pC&u60Y|y z>XoZ50%ND*t`tj5>#+Xy=kK=pdG?nBWqDb_Cfu3=HaDNx4?H;D<#IG`F?-1AfiO5I z^|BK1&#%bhnUf!Q@A1!r2zH%pT@80e3UHd?xnpto~$yIW-8G0~WOE^;~jiv$VO z+>uk1V$_xHjeAaX&y4XPFFnQDrTbLFR)eAjJ|x|Y(V*lnkr7U~9rhzNlrqLkA0Y89*b5vPjoAZ}TS(Mg|IjW-3x zo{|X;2ZP1!Zy6a>uikj~MhzE{f?hQ|aE(Xo51qF4;XQH{IAThjJg0W}>uL_y&&k>H z$=>m-M#hS~1AAXoiU?s@+0XS}n{z4inaSvqlhfBc-1gz=cwLed`DqRZa7c4ETe8(! zm9O7on&b*EnXAE-cXQppJ~VIL5p4#Kv-k8r3BM2#ZC&fp?^J<*YOl57uo@X=p;M(< zMsqZ;(>gtI4-~+^LBq4HQ(baIJ5S`Mcirr?cMMv`Y9e>|$u#jKn|adY?kE2d+iB7> z$o=q0ArSg^l=v^ZgmR&|L^uJyfIt8a;10NRmVE&MfWZG;;YM?TI4!+yyNjVOsi<63 zQbl1f$`}la0^<0k17XOefPg3p{Rxl`a8|zmJ)!$wWkpKT*eK@ZXE*r{L$(2*3||Bd*K?HDN<=D0krVIJk|yZVm=_M~Bj$%;pDQ#)pK-I&XUPUx;M2$+KxtZJ${#w0#Z0r<~tck;O=SY~hym<4|_w zOin8rvgZ_Sa0r>_gW7I(*l3+{`cu4xPmfdRd0{aNnQQp6j++sNtKRCJtW;u7Vf`&) zc93N2cWX)fWuv9+(#qxhqC5ymYV0$PCmG<<7y%K4d^9a}T{}D`T5hNd9Mc4tFV;V< zv2g5trP++Vx$UP+MMSu81kOI4dHsDRzqW()l@-D5!#;7n|NWq@PP=Mz<0EGOcmAZ2 z2T~Tm7ums{Knzdkl<%=&LmZ#lH!?Pc#fNoPjWlQ?{Zz5)5?!l8rOq^r!8Se=;f%j? zYSr*?sYKWE_-J(QC)L-vq~d*PdsXk&li^0I;#^uGd?$m-=d+N-{r3;U$_;^mD?tgNiVDhpqP`%aqlswLLtcyjTf z*Mh7AZkH&A=*i4xPizg=LHin5;;Mi4{@HLxXt;yIrN z+A{s-#aSs4DpYfk=s}&cwafT3{#iuzV})7S`?nXqRiv~(_`$od9akA}G`igHC-Nd| z{2;Fi?~huO`i&`MDyUW`Bw0o&depS${n%kcS|WV)ucmZ&5C|o%)d2oEiRNH}F0h$- zW!TmsE+Oz(vqbi?r60)~lUj<~I(w#c6KiQl0Rd zC9w0sFtF%6z21%%Rmf3gAl``GWO_kYOc49+5#LLDCKje2C=(PK1^e|=0qMX1kQhU; z*d?xdFv}GM%|+}-t#^eX5?4S-lVp7CMO`S>W}gQkMxKl{?d{w9t3P>_&q_t Ie_Fsl055K*s{jB1 literal 0 HcmV?d00001 diff --git a/docs/src/main/asciidoc/configuration.adoc b/docs/src/main/asciidoc/configuration.adoc index 047a86ea4..619bea3ff 100644 --- a/docs/src/main/asciidoc/configuration.adoc +++ b/docs/src/main/asciidoc/configuration.adoc @@ -260,7 +260,9 @@ header. | http.content.partial.as.trimmed | false | Accepts partially fetched content in OKHTTP. | http.filter.ipaddress.include | - | (OkHttp only) Comma-separated list (or YAML list) of allowed IP ranges. If empty, all addresses are allowed unless excluded. See <>. | http.filter.ipaddress.exclude | - | (OkHttp only) Comma-separated list (or YAML list) of blocked IP ranges. See <>. -| http.trust.everything | true | If true, trust all SSL/TLS connections. +| http.trust.everything | false | (OkHttp only) If true, accept any TLS certificate, including self-signed, expired or otherwise invalid ones. The servers are then not authenticated: anyone able to answer for the host name receives everything sent to them. Credentials (basic auth, credential headers, cookies) are withheld over such connections unless `http.credentials.allow.insecure` is set to true. +| http.verify.hostnames | true | (OkHttp only) If true, check that the certificate presented by the server matches the host name contacted. Independent of `http.trust.everything`: a valid certificate for a different name is accepted when this is false. +| http.credentials.allow.insecure | false | (OkHttp only) If true, send credentials (basic auth, credential headers, cookies) even over connections whose server certificate was not validated, i.e. when `http.trust.everything` is true. | topology.message.timeout.secs | -1 | OKHTTP message timeout. |=== diff --git a/docs/src/main/asciidoc/extending.adoc b/docs/src/main/asciidoc/extending.adoc index 59ece5f50..548196eac 100644 --- a/docs/src/main/asciidoc/extending.adoc +++ b/docs/src/main/asciidoc/extending.adoc @@ -392,7 +392,7 @@ Keep the pool size (idle + active) under 1000. For large crawls, increase `proto ==== SSL/TLS -By default, StormCrawler's OkHttp protocol trusts all certificates (`http.trust.everything: true`). For production crawls over untrusted networks, consider setting this to `false` and configuring a proper trust store. +By default, StormCrawler's OkHttp protocol validates the certificate chains of HTTPS servers and checks that the certificate matches the host name contacted (`http.trust.everything: false`, `http.verify.hostnames: true`). For crawls of hosts whose certificates cannot be validated, e.g. self-signed ones, set `http.trust.everything` to `true`: the servers are then not authenticated, so credentials (basic auth, credential headers in `http.custom.headers`, cookies) are withheld over such connections unless `http.credentials.allow.insecure` is set to `true`. Hostname checking is a separate decision and can be disabled with `http.verify.hostnames: false`. ==== Authentication @@ -404,7 +404,7 @@ http.basicauth.user: "username" http.basicauth.password: "password" ---- -These credentials are sent as an `Authorization` header with every request. For per-site authentication, use metadata-driven headers instead (see <> in the Internals section). +These credentials are sent as an `Authorization` header with every request to servers authenticated by their TLS certificate; they are withheld when `http.trust.everything` is enabled, unless `http.credentials.allow.insecure` is set to `true`. For per-site authentication, use metadata-driven headers instead (see <> in the Internals section). ==== Proxy Authentication diff --git a/external/opensearch/archetype/src/main/resources/archetype-resources/crawler-conf.yaml b/external/opensearch/archetype/src/main/resources/archetype-resources/crawler-conf.yaml index f62103faf..f2d9860f5 100644 --- a/external/opensearch/archetype/src/main/resources/archetype-resources/crawler-conf.yaml +++ b/external/opensearch/archetype/src/main/resources/archetype-resources/crawler-conf.yaml @@ -72,6 +72,12 @@ config: http.protocol.implementation: "org.apache.stormcrawler.protocol.okhttp.HttpProtocol" https.protocol.implementation: "org.apache.stormcrawler.protocol.okhttp.HttpProtocol" + # Accept any TLS certificate, e.g. self-signed ones? The servers are not + # authenticated then; credentials (basic auth, credential headers, cookies) + # are withheld over such connections unless + # http.credentials.allow.insecure is set to true. + http.trust.everything: false + # The maximum number of bytes for returned HTTP response bodies. # The fetched page will be trimmed to 65KB in this case # Set -1 to disable the limit. diff --git a/external/solr/archetype/src/main/resources/archetype-resources/crawler-conf.yaml b/external/solr/archetype/src/main/resources/archetype-resources/crawler-conf.yaml index f62103faf..f2d9860f5 100644 --- a/external/solr/archetype/src/main/resources/archetype-resources/crawler-conf.yaml +++ b/external/solr/archetype/src/main/resources/archetype-resources/crawler-conf.yaml @@ -72,6 +72,12 @@ config: http.protocol.implementation: "org.apache.stormcrawler.protocol.okhttp.HttpProtocol" https.protocol.implementation: "org.apache.stormcrawler.protocol.okhttp.HttpProtocol" + # Accept any TLS certificate, e.g. self-signed ones? The servers are not + # authenticated then; credentials (basic auth, credential headers, cookies) + # are withheld over such connections unless + # http.credentials.allow.insecure is set to true. + http.trust.everything: false + # The maximum number of bytes for returned HTTP response bodies. # The fetched page will be trimmed to 65KB in this case # Set -1 to disable the limit. From 89b6519190cba11179340a91a42078927cf752c2 Mon Sep 17 00:00:00 2001 From: abhinav-phi Date: Sun, 6 Sep 2026 13:15:22 +0530 Subject: [PATCH 2/2] OkHttpTrustEverythingTest: serve TLS with WireMock The JDK's com.sun.net.httpserver classes are rejected by the forbiddenapis check (jdk-non-portable). WireMock is already a test dependency and serves HTTPS from a PKCS12 keystore; the request journal replaces the hand-written header recorder. The custom headers use the Name=value syntax that http.custom.headers parses and the code is google-java-format clean. --- .../okhttp/OkHttpTrustEverythingTest.java | 155 ++++++++---------- 1 file changed, 66 insertions(+), 89 deletions(-) diff --git a/core/src/test/java/org/apache/stormcrawler/protocol/okhttp/OkHttpTrustEverythingTest.java b/core/src/test/java/org/apache/stormcrawler/protocol/okhttp/OkHttpTrustEverythingTest.java index 82d1c136c..3ff51de76 100644 --- a/core/src/test/java/org/apache/stormcrawler/protocol/okhttp/OkHttpTrustEverythingTest.java +++ b/core/src/test/java/org/apache/stormcrawler/protocol/okhttp/OkHttpTrustEverythingTest.java @@ -17,27 +17,27 @@ package org.apache.stormcrawler.protocol.okhttp; +import static com.github.tomakehurst.wiremock.client.WireMock.any; +import static com.github.tomakehurst.wiremock.client.WireMock.anyUrl; +import static com.github.tomakehurst.wiremock.client.WireMock.equalTo; +import static com.github.tomakehurst.wiremock.client.WireMock.getRequestedFor; +import static com.github.tomakehurst.wiremock.client.WireMock.ok; +import static com.github.tomakehurst.wiremock.client.WireMock.urlPathEqualTo; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; -import com.sun.net.httpserver.HttpExchange; -import com.sun.net.httpserver.HttpHandler; -import com.sun.net.httpserver.HttpsConfigurator; -import com.sun.net.httpserver.HttpsServer; -import java.io.IOException; +import com.github.tomakehurst.wiremock.WireMockServer; +import com.github.tomakehurst.wiremock.common.ConsoleNotifier; +import com.github.tomakehurst.wiremock.core.WireMockConfiguration; import java.io.InputStream; -import java.io.OutputStream; -import java.net.InetSocketAddress; import java.nio.charset.StandardCharsets; -import java.security.KeyStore; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; import java.util.Base64; import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; -import javax.net.ssl.KeyManagerFactory; -import javax.net.ssl.SSLContext; +import javax.net.ssl.SSLHandshakeException; +import javax.net.ssl.SSLPeerUnverifiedException; import org.apache.storm.Config; import org.apache.stormcrawler.Metadata; import org.apache.stormcrawler.protocol.ProtocolResponse; @@ -51,7 +51,7 @@ */ class OkHttpTrustEverythingTest { - private static final char[] KEYSTORE_PASSWORD = "changeit".toCharArray(); + private static final String KEYSTORE_PASSWORD = "changeit"; /** Certificate issued for localhost: valid for the host the tests connect to. */ private static final String LOCALHOST_KEYSTORE = "/ssl/localhost.p12"; @@ -59,16 +59,13 @@ class OkHttpTrustEverythingTest { /** Certificate issued for another host name: trusted under trust-all, wrong name. */ private static final String OTHERHOST_KEYSTORE = "/ssl/otherhost.p12"; - private HttpsServer server; - - private RecordingHandler handler; + private WireMockServer server; @AfterEach void stopServer() { if (server != null) { - server.stop(0); + server.stop(); server = null; - handler = null; } } @@ -83,12 +80,11 @@ void trustAllContextUsesTls() { @Test void selfSignedCertificateRejectedByDefault() throws Exception { // http.trust.everything defaults to false: an unvalidatable certificate - // must not be accepted. Depending on the platform the handshake fails or - // the connection is dropped while it is retried. + // must not be accepted startServer(LOCALHOST_KEYSTORE); final HttpProtocol protocol = protocol(config()); assertThrows( - Exception.class, + SSLHandshakeException.class, () -> fetch(protocol, "/default"), "self-signed certificates must be rejected by default"); } @@ -105,14 +101,12 @@ void trustAllFetchesServerWithSelfSignedCertificate() throws Exception { @Test void hostnameIsStillVerified() throws Exception { // the certificate is issued for another host name: trusting any - // certificate must not imply accepting any name. Depending on the - // platform the failed verification surfaces as an SSL exception or the - // connection is dropped while it is retried. + // certificate must not imply accepting any name final Config conf = config(); conf.put("http.trust.everything", true); startServer(OTHERHOST_KEYSTORE); assertThrows( - Exception.class, + SSLPeerUnverifiedException.class, () -> fetch(protocol(conf), "/hostname"), "the hostname verifier should not accept any name unconditionally"); } @@ -134,11 +128,9 @@ void basicAuthIsWithheldFromUnauthenticatedServers() throws Exception { conf.put("http.basicauth.user", "user"); conf.put("http.basicauth.password", "secret"); startServer(LOCALHOST_KEYSTORE); - final ProtocolResponse response = fetch(protocol(conf), "/basicauth"); - assertEquals(200, response.getStatusCode(), "the connection must succeed"); - assertNull( - handler.lastHeaders.get("authorization"), - "credentials must not be sent to unauthenticated servers"); + fetch(protocol(conf), "/basicauth"); + server.verify( + 1, getRequestedFor(urlPathEqualTo("/basicauth")).withoutHeader("Authorization")); } @Test @@ -154,10 +146,10 @@ void basicAuthIsSentWhenExplicitlyAllowed() throws Exception { "Basic " + Base64.getEncoder() .encodeToString("user:secret".getBytes(StandardCharsets.UTF_8)); - assertEquals( - expected, - handler.lastHeaders.get("authorization"), - "the opt-in sends credentials"); + server.verify( + 1, + getRequestedFor(urlPathEqualTo("/basicauth")) + .withHeader("Authorization", equalTo(expected))); } @Test @@ -166,11 +158,13 @@ void credentialCustomHeadersAreWithheldFromUnauthenticatedServers() throws Excep conf.put("http.trust.everything", true); conf.put("http.custom.headers", List.of("X-Api-Key=s3cret", "X-Trace=public")); startServer(LOCALHOST_KEYSTORE); - final ProtocolResponse response = fetch(protocol(conf), "/customheaders"); - assertEquals(200, response.getStatusCode(), "the connection must succeed"); - assertNull( - handler.lastHeaders.get("x-api-key"), "credential headers must be withheld"); - assertEquals("public", handler.lastHeaders.get("x-trace"), "other headers are sent"); + fetch(protocol(conf), "/customheaders"); + server.verify( + 1, getRequestedFor(urlPathEqualTo("/customheaders")).withoutHeader("X-Api-Key")); + server.verify( + 1, + getRequestedFor(urlPathEqualTo("/customheaders")) + .withHeader("X-Trace", equalTo("public"))); } @Test @@ -181,8 +175,10 @@ void credentialCustomHeadersAreSentWhenExplicitlyAllowed() throws Exception { conf.put("http.custom.headers", List.of("X-Api-Key=s3cret", "X-Trace=public")); startServer(LOCALHOST_KEYSTORE); fetch(protocol(conf), "/customheaders"); - assertEquals("s3cret", handler.lastHeaders.get("x-api-key"), "the opt-in sends credentials"); - assertEquals("public", handler.lastHeaders.get("x-trace")); + server.verify( + 1, + getRequestedFor(urlPathEqualTo("/customheaders")) + .withHeader("X-Api-Key", equalTo("s3cret"))); } @Test @@ -191,11 +187,8 @@ void cookiesAreWithheldFromUnauthenticatedServers() throws Exception { conf.put("http.trust.everything", true); conf.put("http.use.cookies", true); startServer(LOCALHOST_KEYSTORE); - final ProtocolResponse response = fetch(protocol(conf), "/cookies", metadata()); - assertEquals(200, response.getStatusCode(), "the connection must succeed"); - assertNull( - handler.lastHeaders.get("cookie"), - "cookies must not be sent to unauthenticated servers"); + fetch(protocol(conf), "/cookies", metadata()); + server.verify(1, getRequestedFor(urlPathEqualTo("/cookies")).withoutHeader("Cookie")); } @Test @@ -206,16 +199,16 @@ void cookiesAreSentWhenExplicitlyAllowed() throws Exception { conf.put("http.use.cookies", true); startServer(LOCALHOST_KEYSTORE); fetch(protocol(conf), "/cookies", metadata()); - assertEquals("sid=x", handler.lastHeaders.get("cookie"), "the opt-in sends cookies"); + server.verify( + 1, + getRequestedFor(urlPathEqualTo("/cookies")).withHeader("Cookie", equalTo("sid=x"))); } /** Metadata as an outlink would inherit it, with a cookie scoped to the server. */ private Metadata metadata() { final Metadata md = new Metadata(); md.setValue("protocol.set-cookie", "sid=x; Path=/"); - md.setValue( - "protocol.set-cookie-origin", - "https://localhost:" + server.getAddress().getPort() + "/"); + md.setValue("protocol.set-cookie-origin", "https://localhost:" + server.httpsPort() + "/"); return md; } @@ -242,48 +235,32 @@ private ProtocolResponse fetch(HttpProtocol protocol, String path) throws Except private ProtocolResponse fetch(HttpProtocol protocol, String path, Metadata md) throws Exception { - return protocol.getProtocolOutput( - "https://localhost:" + server.getAddress().getPort() + path, md); + return protocol.getProtocolOutput("https://localhost:" + server.httpsPort() + path, md); } - /** Starts an HTTPS server on a random port presenting the certificate of the keystore. */ + /** + * Starts an HTTPS server on a random port presenting the certificate of the keystore. The + * keystore is copied to a temporary file as WireMock reads it from the file system. + */ private void startServer(String keystoreResource) throws Exception { - final KeyStore keyStore = KeyStore.getInstance("PKCS12"); - try (InputStream in = OkHttpTrustEverythingTest.class.getResourceAsStream(keystoreResource)) { - keyStore.load(in, KEYSTORE_PASSWORD); + final Path keystoreFile = Files.createTempFile("wiremock-keystore", ".p12"); + keystoreFile.toFile().deleteOnExit(); + try (InputStream in = + OkHttpTrustEverythingTest.class.getResourceAsStream(keystoreResource)) { + Files.copy(in, keystoreFile, StandardCopyOption.REPLACE_EXISTING); } - final KeyManagerFactory keyManagerFactory = - KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm()); - keyManagerFactory.init(keyStore, KEYSTORE_PASSWORD); - final SSLContext sslContext = SSLContext.getInstance("TLS"); - sslContext.init(keyManagerFactory.getKeyManagers(), null, null); - server = HttpsServer.create(new InetSocketAddress(java.net.InetAddress.getByName("127.0.0.1"), 0), 0); - server.setHttpsConfigurator(new HttpsConfigurator(sslContext)); - handler = new RecordingHandler(); - server.createContext("/", handler); + server = + new WireMockServer( + WireMockConfiguration.options() + .dynamicPort() + .dynamicHttpsPort() + .keystorePath(keystoreFile.toAbsolutePath().toString()) + .keystorePassword(KEYSTORE_PASSWORD) + .keyManagerPassword(KEYSTORE_PASSWORD) + .keystoreType("PKCS12") + .notifier(new ConsoleNotifier(false))); server.start(); - } - - /** Records the request headers of the last request received. */ - static class RecordingHandler implements HttpHandler { - - final Map lastHeaders = new ConcurrentHashMap<>(); - - @Override - public void handle(HttpExchange exchange) throws IOException { - for (Map.Entry> header : - exchange.getRequestHeaders().entrySet()) { - lastHeaders.put( - header.getKey().toLowerCase(Locale.ROOT), - String.join(",", header.getValue())); - } - final byte[] body = "Success!".getBytes(StandardCharsets.UTF_8); - exchange.getResponseHeaders().set("Content-Type", "text/html"); - exchange.sendResponseHeaders(200, body.length); - try (OutputStream out = exchange.getResponseBody()) { - out.write(body); - } - } + server.stubFor(any(anyUrl()).willReturn(ok("Success!"))); } }